1// WebAssemblyInstrAtomics.td-WebAssembly Atomic codegen support-*- tablegen -*- 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8/// 9/// \file 10/// WebAssembly Atomic operand code-gen constructs. 11/// 12//===----------------------------------------------------------------------===// 13 14let UseNamedOperandTable = 1 in 15multiclass ATOMIC_I<dag oops_r, dag iops_r, dag oops_s, dag iops_s, 16 list<dag> pattern_r, string asmstr_r, 17 string asmstr_s, bits<32> atomic_op, 18 bit is64 = false> { 19 defm "" : I<oops_r, iops_r, oops_s, iops_s, pattern_r, asmstr_r, asmstr_s, 20 !or(0xfe00, !and(0xff, atomic_op)), is64>, 21 Requires<[HasAtomics]>; 22} 23 24multiclass ATOMIC_NRI<dag oops, dag iops, list<dag> pattern, string asmstr = "", 25 bits<32> atomic_op = -1> { 26 defm "" : NRI<oops, iops, pattern, asmstr, 27 !or(0xfe00, !and(0xff, atomic_op))>, 28 Requires<[HasAtomics]>; 29} 30 31//===----------------------------------------------------------------------===// 32// Atomic wait / notify 33//===----------------------------------------------------------------------===// 34 35let hasSideEffects = 1 in { 36defm MEMORY_ATOMIC_NOTIFY_A32 : 37 ATOMIC_I<(outs I32:$dst), 38 (ins P2Align:$p2align, offset32_op:$off, I32:$addr, I32:$count), 39 (outs), (ins P2Align:$p2align, offset32_op:$off), [], 40 "memory.atomic.notify \t$dst, ${off}(${addr})${p2align}, $count", 41 "memory.atomic.notify \t${off}${p2align}", 0x00, false>; 42defm MEMORY_ATOMIC_NOTIFY_A64 : 43 ATOMIC_I<(outs I32:$dst), 44 (ins P2Align:$p2align, offset64_op:$off, I64:$addr, I32:$count), 45 (outs), (ins P2Align:$p2align, offset64_op:$off), [], 46 "memory.atomic.notify \t$dst, ${off}(${addr})${p2align}, $count", 47 "memory.atomic.notify \t${off}${p2align}", 0x00, true>; 48let mayLoad = 1 in { 49defm MEMORY_ATOMIC_WAIT32_A32 : 50 ATOMIC_I<(outs I32:$dst), 51 (ins P2Align:$p2align, offset32_op:$off, I32:$addr, I32:$exp, 52 I64:$timeout), 53 (outs), (ins P2Align:$p2align, offset32_op:$off), [], 54 "memory.atomic.wait32 \t$dst, ${off}(${addr})${p2align}, $exp, $timeout", 55 "memory.atomic.wait32 \t${off}${p2align}", 0x01, false>; 56defm MEMORY_ATOMIC_WAIT32_A64 : 57 ATOMIC_I<(outs I32:$dst), 58 (ins P2Align:$p2align, offset64_op:$off, I64:$addr, I32:$exp, 59 I64:$timeout), 60 (outs), (ins P2Align:$p2align, offset64_op:$off), [], 61 "memory.atomic.wait32 \t$dst, ${off}(${addr})${p2align}, $exp, $timeout", 62 "memory.atomic.wait32 \t${off}${p2align}", 0x01, true>; 63defm MEMORY_ATOMIC_WAIT64_A32 : 64 ATOMIC_I<(outs I32:$dst), 65 (ins P2Align:$p2align, offset32_op:$off, I32:$addr, I64:$exp, 66 I64:$timeout), 67 (outs), (ins P2Align:$p2align, offset32_op:$off), [], 68 "memory.atomic.wait64 \t$dst, ${off}(${addr})${p2align}, $exp, $timeout", 69 "memory.atomic.wait64 \t${off}${p2align}", 0x02, false>; 70defm MEMORY_ATOMIC_WAIT64_A64 : 71 ATOMIC_I<(outs I32:$dst), 72 (ins P2Align:$p2align, offset64_op:$off, I64:$addr, I64:$exp, 73 I64:$timeout), 74 (outs), (ins P2Align:$p2align, offset64_op:$off), [], 75 "memory.atomic.wait64 \t$dst, ${off}(${addr})${p2align}, $exp, $timeout", 76 "memory.atomic.wait64 \t${off}${p2align}", 0x02, true>; 77} // mayLoad = 1 78} // hasSideEffects = 1 79 80def NotifyPat_A32 : 81 Pat<(i32 (int_wasm_memory_atomic_notify (AddrOps32 offset32_op:$offset, I32:$addr), I32:$count)), 82 (MEMORY_ATOMIC_NOTIFY_A32 0, $offset, $addr, $count)>, 83 Requires<[HasAddr32, HasAtomics]>; 84def NotifyPat_A64 : 85 Pat<(i32 (int_wasm_memory_atomic_notify (AddrOps64 offset64_op:$offset, I64:$addr), I32:$count)), 86 (MEMORY_ATOMIC_NOTIFY_A64 0, $offset, $addr, $count)>, 87 Requires<[HasAddr64, HasAtomics]>; 88 89 90multiclass WaitPat<ValueType ty, Intrinsic kind, string inst> { 91 def WaitPat_A32 : 92 Pat<(i32 (kind (AddrOps32 offset32_op:$offset, I32:$addr), ty:$exp, I64:$timeout)), 93 (!cast<NI>(inst#_A32) 0, $offset, $addr, $exp, $timeout)>, 94 Requires<[HasAddr32, HasAtomics]>; 95 def WaitPat_A64 : 96 Pat<(i32 (kind (AddrOps64 offset64_op:$offset, I64:$addr), ty:$exp, I64:$timeout)), 97 (!cast<NI>(inst#_A64) 0, $offset, $addr, $exp, $timeout)>, 98 Requires<[HasAddr64, HasAtomics]>; 99} 100 101defm : WaitPat<i32, int_wasm_memory_atomic_wait32, "MEMORY_ATOMIC_WAIT32">; 102defm : WaitPat<i64, int_wasm_memory_atomic_wait64, "MEMORY_ATOMIC_WAIT64">; 103 104//===----------------------------------------------------------------------===// 105// Atomic fences 106//===----------------------------------------------------------------------===// 107 108// A compiler fence instruction that prevents reordering of instructions. 109let Defs = [ARGUMENTS] in { 110let isPseudo = 1, hasSideEffects = 1 in 111defm COMPILER_FENCE : ATOMIC_NRI<(outs), (ins), [], "compiler_fence">; 112let hasSideEffects = 1 in 113defm ATOMIC_FENCE : ATOMIC_NRI<(outs), (ins i8imm:$flags), [], "atomic.fence", 114 0x03>; 115} // Defs = [ARGUMENTS] 116 117//===----------------------------------------------------------------------===// 118// Atomic loads 119//===----------------------------------------------------------------------===// 120 121multiclass AtomicLoad<WebAssemblyRegClass rc, string name, int atomic_op> { 122 defm "" : WebAssemblyLoad<rc, name, !or(0xfe00, !and(0xff, atomic_op)), 123 [HasAtomics]>; 124} 125 126defm ATOMIC_LOAD_I32 : AtomicLoad<I32, "i32.atomic.load", 0x10>; 127defm ATOMIC_LOAD_I64 : AtomicLoad<I64, "i64.atomic.load", 0x11>; 128 129// Select loads 130defm : LoadPat<i32, atomic_load_32, "ATOMIC_LOAD_I32">; 131defm : LoadPat<i64, atomic_load_64, "ATOMIC_LOAD_I64">; 132 133// Extending loads. Note that there are only zero-extending atomic loads, no 134// sign-extending loads. 135defm ATOMIC_LOAD8_U_I32 : AtomicLoad<I32, "i32.atomic.load8_u", 0x12>; 136defm ATOMIC_LOAD16_U_I32 : AtomicLoad<I32, "i32.atomic.load16_u", 0x13>; 137defm ATOMIC_LOAD8_U_I64 : AtomicLoad<I64, "i64.atomic.load8_u", 0x14>; 138defm ATOMIC_LOAD16_U_I64 : AtomicLoad<I64, "i64.atomic.load16_u", 0x15>; 139defm ATOMIC_LOAD32_U_I64 : AtomicLoad<I64, "i64.atomic.load32_u", 0x16>; 140 141// Fragments for extending loads. These are different from regular loads because 142// the SDNodes are derived from AtomicSDNode rather than LoadSDNode and 143// therefore don't have the extension type field. So instead of matching that, 144// we match the patterns that the type legalizer expands them to. 145 146// Unlike regular loads, extension to i64 is handled differently than i32. 147// i64 (zext (i8 (atomic_load_8))) gets legalized to 148// i64 (and (i64 (anyext (i32 (atomic_load_8)))), 255) 149// Extension to i32 is elided by SelectionDAG as our atomic loads are 150// zero-extending. 151def zext_aload_8_64 : 152 PatFrag<(ops node:$addr), 153 (i64 (zext (i32 (atomic_load_8 node:$addr))))>; 154def zext_aload_16_64 : 155 PatFrag<(ops node:$addr), 156 (i64 (zext (i32 (atomic_load_16 node:$addr))))>; 157def zext_aload_32_64 : 158 PatFrag<(ops node:$addr), 159 (i64 (zext (i32 (atomic_load_32 node:$addr))))>; 160 161// We don't have single sext atomic load instructions. So for sext loads, we 162// match bare subword loads (for 32-bit results) and anyext loads (for 64-bit 163// results) and select a zext load; the next instruction will be sext_inreg 164// which is selected by itself. 165def sext_aload_8_64 : 166 PatFrag<(ops node:$addr), (anyext (i32 (atomic_load_8 node:$addr)))>; 167def sext_aload_16_64 : 168 PatFrag<(ops node:$addr), (anyext (i32 (atomic_load_16 node:$addr)))>; 169 170// Select zero-extending loads 171defm : LoadPat<i64, zext_aload_8_64, "ATOMIC_LOAD8_U_I64">; 172defm : LoadPat<i64, zext_aload_16_64, "ATOMIC_LOAD16_U_I64">; 173defm : LoadPat<i64, zext_aload_32_64, "ATOMIC_LOAD32_U_I64">; 174 175// Select sign-extending loads 176defm : LoadPat<i32, atomic_load_8, "ATOMIC_LOAD8_U_I32">; 177defm : LoadPat<i32, atomic_load_16, "ATOMIC_LOAD16_U_I32">; 178defm : LoadPat<i64, sext_aload_8_64, "ATOMIC_LOAD8_U_I64">; 179defm : LoadPat<i64, sext_aload_16_64, "ATOMIC_LOAD16_U_I64">; 180// 32->64 sext load gets selected as i32.atomic.load, i64.extend_i32_s 181 182 183//===----------------------------------------------------------------------===// 184// Atomic stores 185//===----------------------------------------------------------------------===// 186 187multiclass AtomicStore<WebAssemblyRegClass rc, string name, int atomic_op> { 188 defm "" : WebAssemblyStore<rc, name, !or(0xfe00, !and(0xff, atomic_op)), 189 [HasAtomics]>; 190} 191 192defm ATOMIC_STORE_I32 : AtomicStore<I32, "i32.atomic.store", 0x17>; 193defm ATOMIC_STORE_I64 : AtomicStore<I64, "i64.atomic.store", 0x18>; 194 195// We need an 'atomic' version of store patterns because store and atomic_store 196// nodes have different operand orders: 197// store: (store $val, $ptr) 198// atomic_store: (store $ptr, $val) 199 200multiclass AStorePat<ValueType ty, PatFrag kind, string inst> { 201 def : Pat<(kind (AddrOps32 offset32_op:$offset, I32:$addr), ty:$val), 202 (!cast<NI>(inst#_A32) 0, $offset, $addr, $val)>, 203 Requires<[HasAddr32, HasAtomics]>; 204 def : Pat<(kind (AddrOps64 offset64_op:$offset, I64:$addr), ty:$val), 205 (!cast<NI>(inst#_A64) 0, $offset, $addr, $val)>, 206 Requires<[HasAddr64, HasAtomics]>; 207} 208defm : AStorePat<i32, atomic_store_32, "ATOMIC_STORE_I32">; 209defm : AStorePat<i64, atomic_store_64, "ATOMIC_STORE_I64">; 210 211// Truncating stores. 212defm ATOMIC_STORE8_I32 : AtomicStore<I32, "i32.atomic.store8", 0x19>; 213defm ATOMIC_STORE16_I32 : AtomicStore<I32, "i32.atomic.store16", 0x1a>; 214defm ATOMIC_STORE8_I64 : AtomicStore<I64, "i64.atomic.store8", 0x1b>; 215defm ATOMIC_STORE16_I64 : AtomicStore<I64, "i64.atomic.store16", 0x1c>; 216defm ATOMIC_STORE32_I64 : AtomicStore<I64, "i64.atomic.store32", 0x1d>; 217 218// Fragments for truncating stores. 219 220// We don't have single truncating atomic store instructions. For 32-bit 221// instructions, we just need to match bare atomic stores. On the other hand, 222// truncating stores from i64 values are once truncated to i32 first. 223class trunc_astore_64<PatFrag kind> : 224 PatFrag<(ops node:$addr, node:$val), 225 (kind node:$addr, (i32 (trunc (i64 node:$val))))>; 226def trunc_astore_8_64 : trunc_astore_64<atomic_store_8>; 227def trunc_astore_16_64 : trunc_astore_64<atomic_store_16>; 228def trunc_astore_32_64 : trunc_astore_64<atomic_store_32>; 229 230// Truncating stores with no constant offset 231defm : AStorePat<i32, atomic_store_8, "ATOMIC_STORE8_I32">; 232defm : AStorePat<i32, atomic_store_16, "ATOMIC_STORE16_I32">; 233defm : AStorePat<i64, trunc_astore_8_64, "ATOMIC_STORE8_I64">; 234defm : AStorePat<i64, trunc_astore_16_64, "ATOMIC_STORE16_I64">; 235defm : AStorePat<i64, trunc_astore_32_64, "ATOMIC_STORE32_I64">; 236 237//===----------------------------------------------------------------------===// 238// Atomic binary read-modify-writes 239//===----------------------------------------------------------------------===// 240 241multiclass WebAssemblyBinRMW<WebAssemblyRegClass rc, string name, 242 int atomic_op> { 243 defm "_A32" : 244 ATOMIC_I<(outs rc:$dst), 245 (ins P2Align:$p2align, offset32_op:$off, I32:$addr, rc:$val), 246 (outs), (ins P2Align:$p2align, offset32_op:$off), [], 247 !strconcat(name, "\t$dst, ${off}(${addr})${p2align}, $val"), 248 !strconcat(name, "\t${off}${p2align}"), atomic_op, false>; 249 defm "_A64" : 250 ATOMIC_I<(outs rc:$dst), 251 (ins P2Align:$p2align, offset64_op:$off, I64:$addr, rc:$val), 252 (outs), (ins P2Align:$p2align, offset64_op:$off), [], 253 !strconcat(name, "\t$dst, ${off}(${addr})${p2align}, $val"), 254 !strconcat(name, "\t${off}${p2align}"), atomic_op, true>; 255} 256 257defm ATOMIC_RMW_ADD_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.add", 0x1e>; 258defm ATOMIC_RMW_ADD_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.add", 0x1f>; 259defm ATOMIC_RMW8_U_ADD_I32 : 260 WebAssemblyBinRMW<I32, "i32.atomic.rmw8.add_u", 0x20>; 261defm ATOMIC_RMW16_U_ADD_I32 : 262 WebAssemblyBinRMW<I32, "i32.atomic.rmw16.add_u", 0x21>; 263defm ATOMIC_RMW8_U_ADD_I64 : 264 WebAssemblyBinRMW<I64, "i64.atomic.rmw8.add_u", 0x22>; 265defm ATOMIC_RMW16_U_ADD_I64 : 266 WebAssemblyBinRMW<I64, "i64.atomic.rmw16.add_u", 0x23>; 267defm ATOMIC_RMW32_U_ADD_I64 : 268 WebAssemblyBinRMW<I64, "i64.atomic.rmw32.add_u", 0x24>; 269 270defm ATOMIC_RMW_SUB_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.sub", 0x25>; 271defm ATOMIC_RMW_SUB_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.sub", 0x26>; 272defm ATOMIC_RMW8_U_SUB_I32 : 273 WebAssemblyBinRMW<I32, "i32.atomic.rmw8.sub_u", 0x27>; 274defm ATOMIC_RMW16_U_SUB_I32 : 275 WebAssemblyBinRMW<I32, "i32.atomic.rmw16.sub_u", 0x28>; 276defm ATOMIC_RMW8_U_SUB_I64 : 277 WebAssemblyBinRMW<I64, "i64.atomic.rmw8.sub_u", 0x29>; 278defm ATOMIC_RMW16_U_SUB_I64 : 279 WebAssemblyBinRMW<I64, "i64.atomic.rmw16.sub_u", 0x2a>; 280defm ATOMIC_RMW32_U_SUB_I64 : 281 WebAssemblyBinRMW<I64, "i64.atomic.rmw32.sub_u", 0x2b>; 282 283defm ATOMIC_RMW_AND_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.and", 0x2c>; 284defm ATOMIC_RMW_AND_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.and", 0x2d>; 285defm ATOMIC_RMW8_U_AND_I32 : 286 WebAssemblyBinRMW<I32, "i32.atomic.rmw8.and_u", 0x2e>; 287defm ATOMIC_RMW16_U_AND_I32 : 288 WebAssemblyBinRMW<I32, "i32.atomic.rmw16.and_u", 0x2f>; 289defm ATOMIC_RMW8_U_AND_I64 : 290 WebAssemblyBinRMW<I64, "i64.atomic.rmw8.and_u", 0x30>; 291defm ATOMIC_RMW16_U_AND_I64 : 292 WebAssemblyBinRMW<I64, "i64.atomic.rmw16.and_u", 0x31>; 293defm ATOMIC_RMW32_U_AND_I64 : 294 WebAssemblyBinRMW<I64, "i64.atomic.rmw32.and_u", 0x32>; 295 296defm ATOMIC_RMW_OR_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.or", 0x33>; 297defm ATOMIC_RMW_OR_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.or", 0x34>; 298defm ATOMIC_RMW8_U_OR_I32 : 299 WebAssemblyBinRMW<I32, "i32.atomic.rmw8.or_u", 0x35>; 300defm ATOMIC_RMW16_U_OR_I32 : 301 WebAssemblyBinRMW<I32, "i32.atomic.rmw16.or_u", 0x36>; 302defm ATOMIC_RMW8_U_OR_I64 : 303 WebAssemblyBinRMW<I64, "i64.atomic.rmw8.or_u", 0x37>; 304defm ATOMIC_RMW16_U_OR_I64 : 305 WebAssemblyBinRMW<I64, "i64.atomic.rmw16.or_u", 0x38>; 306defm ATOMIC_RMW32_U_OR_I64 : 307 WebAssemblyBinRMW<I64, "i64.atomic.rmw32.or_u", 0x39>; 308 309defm ATOMIC_RMW_XOR_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.xor", 0x3a>; 310defm ATOMIC_RMW_XOR_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.xor", 0x3b>; 311defm ATOMIC_RMW8_U_XOR_I32 : 312 WebAssemblyBinRMW<I32, "i32.atomic.rmw8.xor_u", 0x3c>; 313defm ATOMIC_RMW16_U_XOR_I32 : 314 WebAssemblyBinRMW<I32, "i32.atomic.rmw16.xor_u", 0x3d>; 315defm ATOMIC_RMW8_U_XOR_I64 : 316 WebAssemblyBinRMW<I64, "i64.atomic.rmw8.xor_u", 0x3e>; 317defm ATOMIC_RMW16_U_XOR_I64 : 318 WebAssemblyBinRMW<I64, "i64.atomic.rmw16.xor_u", 0x3f>; 319defm ATOMIC_RMW32_U_XOR_I64 : 320 WebAssemblyBinRMW<I64, "i64.atomic.rmw32.xor_u", 0x40>; 321 322defm ATOMIC_RMW_XCHG_I32 : 323 WebAssemblyBinRMW<I32, "i32.atomic.rmw.xchg", 0x41>; 324defm ATOMIC_RMW_XCHG_I64 : 325 WebAssemblyBinRMW<I64, "i64.atomic.rmw.xchg", 0x42>; 326defm ATOMIC_RMW8_U_XCHG_I32 : 327 WebAssemblyBinRMW<I32, "i32.atomic.rmw8.xchg_u", 0x43>; 328defm ATOMIC_RMW16_U_XCHG_I32 : 329 WebAssemblyBinRMW<I32, "i32.atomic.rmw16.xchg_u", 0x44>; 330defm ATOMIC_RMW8_U_XCHG_I64 : 331 WebAssemblyBinRMW<I64, "i64.atomic.rmw8.xchg_u", 0x45>; 332defm ATOMIC_RMW16_U_XCHG_I64 : 333 WebAssemblyBinRMW<I64, "i64.atomic.rmw16.xchg_u", 0x46>; 334defm ATOMIC_RMW32_U_XCHG_I64 : 335 WebAssemblyBinRMW<I64, "i64.atomic.rmw32.xchg_u", 0x47>; 336 337multiclass BinRMWPat<ValueType ty, PatFrag kind, string inst> { 338 def : Pat<(ty (kind (AddrOps32 offset32_op:$offset, I32:$addr), ty:$val)), 339 (!cast<NI>(inst#_A32) 0, $offset, $addr, $val)>, 340 Requires<[HasAddr32, HasAtomics]>; 341 def : Pat<(ty (kind (AddrOps64 offset64_op:$offset, I64:$addr), ty:$val)), 342 (!cast<NI>(inst#_A64) 0, $offset, $addr, $val)>, 343 Requires<[HasAddr64, HasAtomics]>; 344} 345 346// Patterns for various addressing modes. 347multiclass BinRMWPattern<PatFrag rmw_32, PatFrag rmw_64, string inst_32, 348 string inst_64> { 349 defm : BinRMWPat<i32, rmw_32, inst_32>; 350 defm : BinRMWPat<i64, rmw_64, inst_64>; 351} 352 353defm : BinRMWPattern<atomic_load_add_32, atomic_load_add_64, 354 "ATOMIC_RMW_ADD_I32", "ATOMIC_RMW_ADD_I64">; 355defm : BinRMWPattern<atomic_load_sub_32, atomic_load_sub_64, 356 "ATOMIC_RMW_SUB_I32", "ATOMIC_RMW_SUB_I64">; 357defm : BinRMWPattern<atomic_load_and_32, atomic_load_and_64, 358 "ATOMIC_RMW_AND_I32", "ATOMIC_RMW_AND_I64">; 359defm : BinRMWPattern<atomic_load_or_32, atomic_load_or_64, 360 "ATOMIC_RMW_OR_I32", "ATOMIC_RMW_OR_I64">; 361defm : BinRMWPattern<atomic_load_xor_32, atomic_load_xor_64, 362 "ATOMIC_RMW_XOR_I32", "ATOMIC_RMW_XOR_I64">; 363defm : BinRMWPattern<atomic_swap_32, atomic_swap_64, 364 "ATOMIC_RMW_XCHG_I32", "ATOMIC_RMW_XCHG_I64">; 365 366// Truncating & zero-extending binary RMW patterns. 367// These are combined patterns of truncating store patterns and zero-extending 368// load patterns above. 369class zext_bin_rmw_8_32<PatFrag kind> : 370 PatFrag<(ops node:$addr, node:$val), (i32 (kind node:$addr, node:$val))>; 371class zext_bin_rmw_16_32<PatFrag kind> : zext_bin_rmw_8_32<kind>; 372class zext_bin_rmw_8_64<PatFrag kind> : 373 PatFrag<(ops node:$addr, node:$val), 374 (zext (i32 (kind node:$addr, (i32 (trunc (i64 node:$val))))))>; 375class zext_bin_rmw_16_64<PatFrag kind> : zext_bin_rmw_8_64<kind>; 376class zext_bin_rmw_32_64<PatFrag kind> : zext_bin_rmw_8_64<kind>; 377 378// Truncating & sign-extending binary RMW patterns. 379// These are combined patterns of truncating store patterns and sign-extending 380// load patterns above. We match subword RMWs (for 32-bit) and anyext RMWs (for 381// 64-bit) and select a zext RMW; the next instruction will be sext_inreg which 382// is selected by itself. 383class sext_bin_rmw_8_32<PatFrag kind> : 384 PatFrag<(ops node:$addr, node:$val), (kind node:$addr, node:$val)>; 385class sext_bin_rmw_16_32<PatFrag kind> : sext_bin_rmw_8_32<kind>; 386class sext_bin_rmw_8_64<PatFrag kind> : 387 PatFrag<(ops node:$addr, node:$val), 388 (anyext (i32 (kind node:$addr, (i32 (trunc (i64 node:$val))))))>; 389class sext_bin_rmw_16_64<PatFrag kind> : sext_bin_rmw_8_64<kind>; 390// 32->64 sext RMW gets selected as i32.atomic.rmw.***, i64.extend_i32_s 391 392// Patterns for various addressing modes for truncating-extending binary RMWs. 393multiclass BinRMWTruncExtPattern< 394 PatFrag rmw_8, PatFrag rmw_16, PatFrag rmw_32, 395 string inst8_32, string inst16_32, string inst8_64, string inst16_64, string inst32_64> { 396 // Truncating-extending binary RMWs 397 defm : BinRMWPat<i32, zext_bin_rmw_8_32<rmw_8>, inst8_32>; 398 defm : BinRMWPat<i32, zext_bin_rmw_16_32<rmw_16>, inst16_32>; 399 defm : BinRMWPat<i64, zext_bin_rmw_8_64<rmw_8>, inst8_64>; 400 defm : BinRMWPat<i64, zext_bin_rmw_16_64<rmw_16>, inst16_64>; 401 defm : BinRMWPat<i64, zext_bin_rmw_32_64<rmw_32>, inst32_64>; 402 403 defm : BinRMWPat<i32, sext_bin_rmw_8_32<rmw_8>, inst8_32>; 404 defm : BinRMWPat<i32, sext_bin_rmw_16_32<rmw_16>, inst16_32>; 405 defm : BinRMWPat<i64, sext_bin_rmw_8_64<rmw_8>, inst8_64>; 406 defm : BinRMWPat<i64, sext_bin_rmw_16_64<rmw_16>, inst16_64>; 407} 408 409defm : BinRMWTruncExtPattern< 410 atomic_load_add_8, atomic_load_add_16, atomic_load_add_32, 411 "ATOMIC_RMW8_U_ADD_I32", "ATOMIC_RMW16_U_ADD_I32", 412 "ATOMIC_RMW8_U_ADD_I64", "ATOMIC_RMW16_U_ADD_I64", "ATOMIC_RMW32_U_ADD_I64">; 413defm : BinRMWTruncExtPattern< 414 atomic_load_sub_8, atomic_load_sub_16, atomic_load_sub_32, 415 "ATOMIC_RMW8_U_SUB_I32", "ATOMIC_RMW16_U_SUB_I32", 416 "ATOMIC_RMW8_U_SUB_I64", "ATOMIC_RMW16_U_SUB_I64", "ATOMIC_RMW32_U_SUB_I64">; 417defm : BinRMWTruncExtPattern< 418 atomic_load_and_8, atomic_load_and_16, atomic_load_and_32, 419 "ATOMIC_RMW8_U_AND_I32", "ATOMIC_RMW16_U_AND_I32", 420 "ATOMIC_RMW8_U_AND_I64", "ATOMIC_RMW16_U_AND_I64", "ATOMIC_RMW32_U_AND_I64">; 421defm : BinRMWTruncExtPattern< 422 atomic_load_or_8, atomic_load_or_16, atomic_load_or_32, 423 "ATOMIC_RMW8_U_OR_I32", "ATOMIC_RMW16_U_OR_I32", 424 "ATOMIC_RMW8_U_OR_I64", "ATOMIC_RMW16_U_OR_I64", "ATOMIC_RMW32_U_OR_I64">; 425defm : BinRMWTruncExtPattern< 426 atomic_load_xor_8, atomic_load_xor_16, atomic_load_xor_32, 427 "ATOMIC_RMW8_U_XOR_I32", "ATOMIC_RMW16_U_XOR_I32", 428 "ATOMIC_RMW8_U_XOR_I64", "ATOMIC_RMW16_U_XOR_I64", "ATOMIC_RMW32_U_XOR_I64">; 429defm : BinRMWTruncExtPattern< 430 atomic_swap_8, atomic_swap_16, atomic_swap_32, 431 "ATOMIC_RMW8_U_XCHG_I32", "ATOMIC_RMW16_U_XCHG_I32", 432 "ATOMIC_RMW8_U_XCHG_I64", "ATOMIC_RMW16_U_XCHG_I64", 433 "ATOMIC_RMW32_U_XCHG_I64">; 434 435//===----------------------------------------------------------------------===// 436// Atomic ternary read-modify-writes 437//===----------------------------------------------------------------------===// 438 439// TODO LLVM IR's cmpxchg instruction returns a pair of {loaded value, success 440// flag}. When we use the success flag or both values, we can't make use of i64 441// truncate/extend versions of instructions for now, which is suboptimal. 442// Consider adding a pass after instruction selection that optimizes this case 443// if it is frequent. 444 445multiclass WebAssemblyTerRMW<WebAssemblyRegClass rc, string name, 446 int atomic_op> { 447 defm "_A32" : 448 ATOMIC_I<(outs rc:$dst), 449 (ins P2Align:$p2align, offset32_op:$off, I32:$addr, rc:$exp, 450 rc:$new_), 451 (outs), (ins P2Align:$p2align, offset32_op:$off), [], 452 !strconcat(name, "\t$dst, ${off}(${addr})${p2align}, $exp, $new_"), 453 !strconcat(name, "\t${off}${p2align}"), atomic_op, false>; 454 defm "_A64" : 455 ATOMIC_I<(outs rc:$dst), 456 (ins P2Align:$p2align, offset64_op:$off, I64:$addr, rc:$exp, 457 rc:$new_), 458 (outs), (ins P2Align:$p2align, offset64_op:$off), [], 459 !strconcat(name, "\t$dst, ${off}(${addr})${p2align}, $exp, $new_"), 460 !strconcat(name, "\t${off}${p2align}"), atomic_op, true>; 461} 462 463defm ATOMIC_RMW_CMPXCHG_I32 : 464 WebAssemblyTerRMW<I32, "i32.atomic.rmw.cmpxchg", 0x48>; 465defm ATOMIC_RMW_CMPXCHG_I64 : 466 WebAssemblyTerRMW<I64, "i64.atomic.rmw.cmpxchg", 0x49>; 467defm ATOMIC_RMW8_U_CMPXCHG_I32 : 468 WebAssemblyTerRMW<I32, "i32.atomic.rmw8.cmpxchg_u", 0x4a>; 469defm ATOMIC_RMW16_U_CMPXCHG_I32 : 470 WebAssemblyTerRMW<I32, "i32.atomic.rmw16.cmpxchg_u", 0x4b>; 471defm ATOMIC_RMW8_U_CMPXCHG_I64 : 472 WebAssemblyTerRMW<I64, "i64.atomic.rmw8.cmpxchg_u", 0x4c>; 473defm ATOMIC_RMW16_U_CMPXCHG_I64 : 474 WebAssemblyTerRMW<I64, "i64.atomic.rmw16.cmpxchg_u", 0x4d>; 475defm ATOMIC_RMW32_U_CMPXCHG_I64 : 476 WebAssemblyTerRMW<I64, "i64.atomic.rmw32.cmpxchg_u", 0x4e>; 477 478multiclass TerRMWPat<ValueType ty, PatFrag kind, string inst> { 479 def : Pat<(ty (kind (AddrOps32 offset32_op:$offset, I32:$addr), ty:$exp, ty:$new)), 480 (!cast<NI>(inst#_A32) 0, $offset, $addr, $exp, $new)>, 481 Requires<[HasAddr32, HasAtomics]>; 482 def : Pat<(ty (kind (AddrOps64 offset64_op:$offset, I64:$addr), ty:$exp, ty:$new)), 483 (!cast<NI>(inst#_A64) 0, $offset, $addr, $exp, $new)>, 484 Requires<[HasAddr64, HasAtomics]>; 485} 486 487defm : TerRMWPat<i32, atomic_cmp_swap_32, "ATOMIC_RMW_CMPXCHG_I32">; 488defm : TerRMWPat<i64, atomic_cmp_swap_64, "ATOMIC_RMW_CMPXCHG_I64">; 489 490// Truncating & zero-extending ternary RMW patterns. 491// DAG legalization & optimization before instruction selection may introduce 492// additional nodes such as anyext or assertzext depending on operand types. 493class zext_ter_rmw_8_32<PatFrag kind> : 494 PatFrag<(ops node:$addr, node:$exp, node:$new), 495 (i32 (kind node:$addr, node:$exp, node:$new))>; 496class zext_ter_rmw_16_32<PatFrag kind> : zext_ter_rmw_8_32<kind>; 497class zext_ter_rmw_8_64<PatFrag kind> : 498 PatFrag<(ops node:$addr, node:$exp, node:$new), 499 (zext (i32 (assertzext (i32 (kind node:$addr, 500 (i32 (trunc (i64 node:$exp))), 501 (i32 (trunc (i64 node:$new))))))))>; 502class zext_ter_rmw_16_64<PatFrag kind> : zext_ter_rmw_8_64<kind>; 503class zext_ter_rmw_32_64<PatFrag kind> : 504 PatFrag<(ops node:$addr, node:$exp, node:$new), 505 (zext (i32 (kind node:$addr, 506 (i32 (trunc (i64 node:$exp))), 507 (i32 (trunc (i64 node:$new))))))>; 508 509// Truncating & sign-extending ternary RMW patterns. 510// We match subword RMWs (for 32-bit) and anyext RMWs (for 64-bit) and select a 511// zext RMW; the next instruction will be sext_inreg which is selected by 512// itself. 513class sext_ter_rmw_8_32<PatFrag kind> : 514 PatFrag<(ops node:$addr, node:$exp, node:$new), 515 (kind node:$addr, node:$exp, node:$new)>; 516class sext_ter_rmw_16_32<PatFrag kind> : sext_ter_rmw_8_32<kind>; 517class sext_ter_rmw_8_64<PatFrag kind> : 518 PatFrag<(ops node:$addr, node:$exp, node:$new), 519 (anyext (i32 (assertzext (i32 520 (kind node:$addr, 521 (i32 (trunc (i64 node:$exp))), 522 (i32 (trunc (i64 node:$new))))))))>; 523class sext_ter_rmw_16_64<PatFrag kind> : sext_ter_rmw_8_64<kind>; 524// 32->64 sext RMW gets selected as i32.atomic.rmw.***, i64.extend_i32_s 525 526defm : TerRMWPat<i32, zext_ter_rmw_8_32<atomic_cmp_swap_8>, "ATOMIC_RMW8_U_CMPXCHG_I32">; 527defm : TerRMWPat<i32, zext_ter_rmw_16_32<atomic_cmp_swap_16>, "ATOMIC_RMW16_U_CMPXCHG_I32">; 528defm : TerRMWPat<i64, zext_ter_rmw_8_64<atomic_cmp_swap_8>, "ATOMIC_RMW8_U_CMPXCHG_I64">; 529defm : TerRMWPat<i64, zext_ter_rmw_16_64<atomic_cmp_swap_16>, "ATOMIC_RMW16_U_CMPXCHG_I64">; 530defm : TerRMWPat<i64, zext_ter_rmw_32_64<atomic_cmp_swap_32>, "ATOMIC_RMW32_U_CMPXCHG_I64">; 531 532defm : TerRMWPat<i32, sext_ter_rmw_8_32<atomic_cmp_swap_8>, "ATOMIC_RMW8_U_CMPXCHG_I32">; 533defm : TerRMWPat<i32, sext_ter_rmw_16_32<atomic_cmp_swap_16>, "ATOMIC_RMW16_U_CMPXCHG_I32">; 534defm : TerRMWPat<i64, sext_ter_rmw_8_64<atomic_cmp_swap_8>, "ATOMIC_RMW8_U_CMPXCHG_I64">; 535defm : TerRMWPat<i64, sext_ter_rmw_16_64<atomic_cmp_swap_16>, "ATOMIC_RMW16_U_CMPXCHG_I64">; 536