1//==--- riscv_vector.td - RISC-V V-ext Builtin function list --------------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8// 9// This file defines the builtins for RISC-V V-extension. See: 10// 11// https://github.com/riscv/rvv-intrinsic-doc 12// 13//===----------------------------------------------------------------------===// 14 15include "riscv_vector_common.td" 16 17defvar TypeList = ["c","s","i","l","x","f","d","y"]; 18defvar EEWList = [["8", "(Log2EEW:3)"], 19 ["16", "(Log2EEW:4)"], 20 ["32", "(Log2EEW:5)"], 21 ["64", "(Log2EEW:6)"]]; 22 23class IsFloat<string type> { 24 bit val = !or(!eq(type, "x"), !eq(type, "f"), !eq(type, "d"), !eq(type, "y")); 25} 26 27let SupportOverloading = false, 28 MaskedPolicyScheme = NonePolicy in { 29 class RVVVLEMaskBuiltin : RVVOutBuiltin<"m", "mPCUe", "c"> { 30 let Name = "vlm_v"; 31 let IRName = "vlm"; 32 let HasMasked = false; 33 } 34} 35 36let SupportOverloading = false, 37 UnMaskedPolicyScheme = HasPassthruOperand in { 38 multiclass RVVVLEBuiltin<list<string> types> { 39 let Name = NAME # "_v", 40 IRName = "vle", 41 MaskedIRName ="vle_mask" in { 42 foreach type = types in { 43 def : RVVOutBuiltin<"v", "vPCe", type>; 44 if !not(IsFloat<type>.val) then { 45 def : RVVOutBuiltin<"Uv", "UvPCUe", type>; 46 } 47 } 48 } 49 } 50} 51 52multiclass RVVVLEFFBuiltin<list<string> types> { 53 let Name = NAME # "_v", 54 IRName = "vleff", 55 MaskedIRName = "vleff_mask", 56 SupportOverloading = false, 57 UnMaskedPolicyScheme = HasPassthruOperand, 58 ManualCodegen = [{ 59 { 60 if (IsMasked) { 61 // Move mask to right before vl. 62 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); 63 if ((PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) 64 Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); 65 Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); 66 IntrinsicTypes = {ResultType, Ops[4]->getType()}; 67 } else { 68 if (PolicyAttrs & RVV_VTA) 69 Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); 70 IntrinsicTypes = {ResultType, Ops[3]->getType()}; 71 } 72 Value *NewVL = Ops[2]; 73 Ops.erase(Ops.begin() + 2); 74 llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); 75 llvm::Value *LoadValue = Builder.CreateCall(F, Ops, ""); 76 llvm::Value *V = Builder.CreateExtractValue(LoadValue, {0}); 77 // Store new_vl. 78 clang::CharUnits Align; 79 if (IsMasked) 80 Align = CGM.getNaturalPointeeTypeAlignment(E->getArg(E->getNumArgs()-2)->getType()); 81 else 82 Align = CGM.getNaturalPointeeTypeAlignment(E->getArg(1)->getType()); 83 llvm::Value *Val = Builder.CreateExtractValue(LoadValue, {1}); 84 Builder.CreateStore(Val, Address(NewVL, Val->getType(), Align)); 85 return V; 86 } 87 }] in { 88 foreach type = types in { 89 def : RVVBuiltin<"v", "vPCePz", type>; 90 // Skip floating types for unsigned versions. 91 if !not(IsFloat<type>.val) then { 92 def : RVVBuiltin<"Uv", "UvPCUePz", type>; 93 } 94 } 95 } 96} 97 98multiclass RVVVLSEBuiltin<list<string> types> { 99 let Name = NAME # "_v", 100 IRName = "vlse", 101 MaskedIRName ="vlse_mask", 102 SupportOverloading = false, 103 UnMaskedPolicyScheme = HasPassthruOperand in { 104 foreach type = types in { 105 def : RVVOutBuiltin<"v", "vPCet", type>; 106 if !not(IsFloat<type>.val) then { 107 def : RVVOutBuiltin<"Uv", "UvPCUet", type>; 108 } 109 } 110 } 111} 112 113multiclass RVVIndexedLoad<string op> { 114 let UnMaskedPolicyScheme = HasPassthruOperand in { 115 foreach type = TypeList in { 116 foreach eew_list = EEWList[0-2] in { 117 defvar eew = eew_list[0]; 118 defvar eew_type = eew_list[1]; 119 let Name = op # eew # "_v", IRName = op, MaskedIRName = op # "_mask", 120 RequiredFeatures = !if(!eq(type, "x"), ["Zvfhmin"], 121 !if(!eq(type, "y"), ["Zvfbfmin"], 122 []<string>)) in { 123 def: RVVOutOp1Builtin<"v", "vPCe" # eew_type # "Uv", type>; 124 if !not(IsFloat<type>.val) then { 125 def: RVVOutOp1Builtin<"Uv", "UvPCUe" # eew_type # "Uv", type>; 126 } 127 } 128 } 129 defvar eew64 = "64"; 130 defvar eew64_type = "(Log2EEW:6)"; 131 let Name = op # eew64 # "_v", IRName = op, MaskedIRName = op # "_mask", 132 RequiredFeatures = !if(!eq(type, "x"), ["Zvfhmin", "RV64"], 133 !if(!eq(type, "y"), ["Zvfbfmin", "RV64"], 134 ["RV64"])) in { 135 def: RVVOutOp1Builtin<"v", "vPCe" # eew64_type # "Uv", type>; 136 if !not(IsFloat<type>.val) then { 137 def: RVVOutOp1Builtin<"Uv", "UvPCUe" # eew64_type # "Uv", type>; 138 } 139 } 140 } 141 } 142} 143 144let HasMaskedOffOperand = false, 145 MaskedPolicyScheme = NonePolicy, 146 ManualCodegen = [{ 147 if (IsMasked) { 148 // Builtin: (mask, ptr, value, vl). Intrinsic: (value, ptr, mask, vl) 149 std::swap(Ops[0], Ops[2]); 150 } else { 151 // Builtin: (ptr, value, vl). Intrinsic: (value, ptr, vl) 152 std::swap(Ops[0], Ops[1]); 153 } 154 if (IsMasked) 155 IntrinsicTypes = {Ops[0]->getType(), Ops[3]->getType()}; 156 else 157 IntrinsicTypes = {Ops[0]->getType(), Ops[2]->getType()}; 158 }] in { 159 class RVVVSEMaskBuiltin : RVVBuiltin<"m", "0PUem", "c"> { 160 let Name = "vsm_v"; 161 let IRName = "vsm"; 162 let HasMasked = false; 163 } 164 multiclass RVVVSEBuiltin<list<string> types> { 165 let Name = NAME # "_v", 166 IRName = "vse", 167 MaskedIRName = "vse_mask" in { 168 foreach type = types in { 169 def : RVVBuiltin<"v", "0Pev", type>; 170 if !not(IsFloat<type>.val) then { 171 def : RVVBuiltin<"Uv", "0PUeUv", type>; 172 } 173 } 174 } 175 } 176} 177 178multiclass RVVVSSEBuiltin<list<string> types> { 179 let Name = NAME # "_v", 180 IRName = "vsse", 181 MaskedIRName = "vsse_mask", 182 HasMaskedOffOperand = false, 183 MaskedPolicyScheme = NonePolicy, 184 ManualCodegen = [{ 185 if (IsMasked) { 186 // Builtin: (mask, ptr, stride, value, vl). Intrinsic: (value, ptr, stride, mask, vl) 187 std::swap(Ops[0], Ops[3]); 188 } else { 189 // Builtin: (ptr, stride, value, vl). Intrinsic: (value, ptr, stride, vl) 190 std::rotate(Ops.begin(), Ops.begin() + 2, Ops.begin() + 3); 191 } 192 if (IsMasked) 193 IntrinsicTypes = {Ops[0]->getType(), Ops[4]->getType()}; 194 else 195 IntrinsicTypes = {Ops[0]->getType(), Ops[3]->getType()}; 196 }] in { 197 foreach type = types in { 198 def : RVVBuiltin<"v", "0Petv", type>; 199 if !not(IsFloat<type>.val) then { 200 def : RVVBuiltin<"Uv", "0PUetUv", type>; 201 } 202 } 203 } 204} 205 206multiclass RVVIndexedStore<string op> { 207 let HasMaskedOffOperand = false, 208 MaskedPolicyScheme = NonePolicy, 209 ManualCodegen = [{ 210 if (IsMasked) { 211 // Builtin: (mask, ptr, index, value, vl). Intrinsic: (value, ptr, index, mask, vl) 212 std::swap(Ops[0], Ops[3]); 213 } else { 214 // Builtin: (ptr, index, value, vl). Intrinsic: (value, ptr, index, vl) 215 std::rotate(Ops.begin(), Ops.begin() + 2, Ops.begin() + 3); 216 } 217 if (IsMasked) 218 IntrinsicTypes = {Ops[0]->getType(), Ops[2]->getType(), Ops[4]->getType()}; 219 else 220 IntrinsicTypes = {Ops[0]->getType(), Ops[2]->getType(), Ops[3]->getType()}; 221 }] in { 222 foreach type = TypeList in { 223 foreach eew_list = EEWList[0-2] in { 224 defvar eew = eew_list[0]; 225 defvar eew_type = eew_list[1]; 226 let Name = op # eew # "_v", IRName = op, MaskedIRName = op # "_mask", 227 RequiredFeatures = !if(!eq(type, "x"), ["Zvfhmin"], 228 !if(!eq(type, "y"), ["Zvfbfmin"], 229 []<string>)) in { 230 def : RVVBuiltin<"v", "0Pe" # eew_type # "Uvv", type>; 231 if !not(IsFloat<type>.val) then { 232 def : RVVBuiltin<"Uv", "0PUe" # eew_type # "UvUv", type>; 233 } 234 } 235 } 236 defvar eew64 = "64"; 237 defvar eew64_type = "(Log2EEW:6)"; 238 let Name = op # eew64 # "_v", IRName = op, MaskedIRName = op # "_mask", 239 RequiredFeatures = !if(!eq(type, "x"), ["Zvfhmin", "RV64"], 240 !if(!eq(type, "y"), ["Zvfbfmin", "RV64"], 241 ["RV64"])) in { 242 def : RVVBuiltin<"v", "0Pe" # eew64_type # "Uvv", type>; 243 if !not(IsFloat<type>.val) then { 244 def : RVVBuiltin<"Uv", "0PUe" # eew64_type # "UvUv", type>; 245 } 246 } 247 } 248 } 249} 250 251defvar NFList = [2, 3, 4, 5, 6, 7, 8]; 252/* 253A segment load builtin has different variants. 254 255Therefore a segment unit-stride load builtin can have 4 variants, 2561. When unmasked and the policies are all specified as agnostic: 257(Address0, ..., Address{NF - 1}, Ptr, VL) 2582. When masked and the policies are all specified as agnostic: 259(Address0, ..., Address{NF - 1}, Mask, Ptr, VL) 2603. When unmasked and one of the policies is specified as undisturbed: 261(Address0, ..., Address{NF - 1}, Maskedoff0, ..., Maskedoff{NF - 1}, 262 Ptr, VL) 2634. When masked and one of the policies is specified as undisturbed: 264(Address0, ..., Address{NF - 1}, Mask, Maskedoff0, ..., Maskedoff{NF - 1}, 265 Ptr, VL) 266 267Other variants of segment load builtin share the same structure, but they 268have their own extra parameter. 269 270The segment unit-stride fault-only-first load builtin has a 'NewVL' 271operand after the 'Ptr' operand. 2721. When unmasked and the policies are all specified as agnostic: 273(Address0, ..., Address{NF - 1}, Ptr, NewVL, VL) 2742. When masked and the policies are all specified as agnostic: 275(Address0, ..., Address{NF - 1}, Mask, Ptr, NewVL, VL) 2763. When unmasked and one of the policies is specified as undisturbed: 277(Address0, ..., Address{NF - 1}, Maskedoff0, ..., Maskedoff{NF - 1}, 278 Ptr, NewVL, VL) 2794. When masked and one of the policies is specified as undisturbed: 280(Address0, ..., Address{NF - 1}, Mask, Maskedoff0, ..., Maskedoff{NF - 1}, 281 Ptr, NewVL, VL) 282 283The segment strided load builtin has a 'Stride' operand after the 'Ptr' 284operand. 2851. When unmasked and the policies are all specified as agnostic: 286(Address0, ..., Address{NF - 1}, Ptr, Stride, VL) 2872. When masked and the policies are all specified as agnostic: 288(Address0, ..., Address{NF - 1}, Mask, Ptr, Stride, VL) 2893. When unmasked and one of the policies is specified as undisturbed: 290(Address0, ..., Address{NF - 1}, Maskedoff0, ..., Maskedoff{NF - 1}, 291 Ptr, Stride, VL) 2924. When masked and one of the policies is specified as undisturbed: 293(Address0, ..., Address{NF - 1}, Mask, Maskedoff0, ..., Maskedoff{NF - 1}, 294 Ptr, Stride, VL) 295 296The segment indexed load builtin has a 'Idx' operand after the 'Ptr' operand. 2971. When unmasked and the policies are all specified as agnostic: 298(Address0, ..., Address{NF - 1}, Ptr, Idx, VL) 2992. When masked and the policies are all specified as agnostic: 300(Address0, ..., Address{NF - 1}, Mask, Ptr, Idx, VL) 3013. When unmasked and one of the policies is specified as undisturbed: 302(Address0, ..., Address{NF - 1}, Maskedoff0, ..., Maskedoff{NF - 1}, 303 Ptr, Idx, VL) 3044. When masked and one of the policies is specified as undisturbed: 305(Address0, ..., Address{NF - 1}, Mask, Maskedoff0, ..., Maskedoff{NF - 1}, 306 Ptr, Idx, VL) 307 308Segment load intrinsics has different variants similar to their builtins. 309 310Segment unit-stride load intrinsic, 311 Masked: (Vector0, ..., Vector{NF - 1}, Ptr, Mask, VL, Policy) 312 Unmasked: (Vector0, ..., Vector{NF - 1}, Ptr, VL) 313Segment unit-stride fault-only-first load intrinsic, 314 Masked: (Vector0, ..., Vector{NF - 1}, Ptr, Mask, VL, Policy) 315 Unmasked: (Vector0, ..., Vector{NF - 1}, Ptr, VL) 316Segment strided load intrinsic, 317 Masked: (Vector0, ..., Vector{NF - 1}, Ptr, Stride, Mask, VL, Policy) 318 Unmasked: (Vector0, ..., Vector{NF - 1}, Ptr, Stride, VL) 319Segment indexed load intrinsic, 320 Masked: (Vector0, ..., Vector{NF - 1}, Ptr, Index, Mask, VL, Policy) 321 Unmasked: (Vector0, ..., Vector{NF - 1}, Ptr, Index, VL) 322 323The Vector(s) is poison when the policy behavior allows us to not care 324about any masked-off elements. 325*/ 326 327class PVString<int nf, bit signed> { 328 string S = 329 !cond(!eq(nf, 2): !if(signed, "PvPv", "PUvPUv"), 330 !eq(nf, 3): !if(signed, "PvPvPv", "PUvPUvPUv"), 331 !eq(nf, 4): !if(signed, "PvPvPvPv", "PUvPUvPUvPUv"), 332 !eq(nf, 5): !if(signed, "PvPvPvPvPv", "PUvPUvPUvPUvPUv"), 333 !eq(nf, 6): !if(signed, "PvPvPvPvPvPv", "PUvPUvPUvPUvPUvPUv"), 334 !eq(nf, 7): !if(signed, "PvPvPvPvPvPvPv", "PUvPUvPUvPUvPUvPUvPUv"), 335 !eq(nf, 8): !if(signed, "PvPvPvPvPvPvPvPv", "PUvPUvPUvPUvPUvPUvPUvPUv")); 336} 337 338class VString<int nf, bit signed> { 339 string S = !cond(!eq(nf, 2): !if(signed, "vv", "UvUv"), 340 !eq(nf, 3): !if(signed, "vvv", "UvUvUv"), 341 !eq(nf, 4): !if(signed, "vvvv", "UvUvUvUv"), 342 !eq(nf, 5): !if(signed, "vvvvv", "UvUvUvUvUv"), 343 !eq(nf, 6): !if(signed, "vvvvvv", "UvUvUvUvUvUv"), 344 !eq(nf, 7): !if(signed, "vvvvvvv", "UvUvUvUvUvUvUv"), 345 !eq(nf, 8): !if(signed, "vvvvvvvv", "UvUvUvUvUvUvUvUv")); 346} 347 348 349class FixedVString<int fixed_lmul, int num, string vec> { 350 string V = "(LFixedLog2LMUL:" # fixed_lmul # ")" # vec; 351 string S = !interleave(!listsplat(V, num), ""); 352} 353 354multiclass RVVNonTupleVCreateBuiltin<int dst_lmul, list<int> src_lmul_list> { 355 defvar dst_v = FixedVString<dst_lmul, 1, "v">.V; 356 defvar dst_uv = FixedVString<dst_lmul, 1, "Uv">.V; 357 foreach src_lmul = src_lmul_list in { 358 defvar num = !shl(1, !sub(dst_lmul, src_lmul)); 359 360 defvar src_v = FixedVString<src_lmul, num, "v">.V; 361 defvar src_s = FixedVString<src_lmul, num, "v">.S; 362 def vcreate # src_v # dst_v : RVVBuiltin<src_v # dst_v, 363 dst_v # src_s, 364 "csilxfd">; 365 let RequiredFeatures = ["Zvfbfmin"] in 366 def vcreate_bf16 # src_v # dst_v : RVVBuiltin<src_v # dst_v, 367 dst_v # src_s, 368 "y", dst_v>; 369 370 defvar src_uv = FixedVString<src_lmul, num, "Uv">.V; 371 defvar src_us = FixedVString<src_lmul, num, "Uv">.S; 372 def vcreate_u # src_uv # dst_uv : RVVBuiltin<src_uv # dst_uv, 373 dst_uv # src_us, 374 "csil">; 375 } 376} 377 378multiclass RVVPseudoUnaryBuiltin<string IR, string type_range> { 379 let Name = NAME, 380 IRName = IR, 381 MaskedIRName = IR # "_mask", 382 UnMaskedPolicyScheme = HasPassthruOperand, 383 ManualCodegen = [{ 384 { 385 if (IsMasked) { 386 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); 387 if ((PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) 388 Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); 389 } else { 390 if (PolicyAttrs & RVV_VTA) 391 Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); 392 } 393 auto ElemTy = cast<llvm::VectorType>(ResultType)->getElementType(); 394 Ops.insert(Ops.begin() + 2, llvm::Constant::getNullValue(ElemTy)); 395 396 if (IsMasked) { 397 Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); 398 // maskedoff, op1, op2, mask, vl, policy 399 IntrinsicTypes = {ResultType, ElemTy, Ops[4]->getType()}; 400 } else { 401 // passthru, op1, op2, vl 402 IntrinsicTypes = {ResultType, ElemTy, Ops[3]->getType()}; 403 } 404 break; 405 } 406 }] in { 407 def : RVVBuiltin<"v", "vv", type_range>; 408 } 409} 410 411multiclass RVVPseudoVNotBuiltin<string IR, string type_range> { 412 let Name = NAME, 413 IRName = IR, 414 MaskedIRName = IR # "_mask", 415 UnMaskedPolicyScheme = HasPassthruOperand, 416 ManualCodegen = [{ 417 { 418 if (IsMasked) { 419 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); 420 if ((PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) 421 Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); 422 } else { 423 if (PolicyAttrs & RVV_VTA) 424 Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); 425 } 426 auto ElemTy = cast<llvm::VectorType>(ResultType)->getElementType(); 427 Ops.insert(Ops.begin() + 2, 428 llvm::Constant::getAllOnesValue(ElemTy)); 429 if (IsMasked) { 430 Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); 431 // maskedoff, op1, po2, mask, vl, policy 432 IntrinsicTypes = {ResultType, 433 ElemTy, 434 Ops[4]->getType()}; 435 } else { 436 // passthru, op1, op2, vl 437 IntrinsicTypes = {ResultType, 438 ElemTy, 439 Ops[3]->getType()}; 440 } 441 break; 442 } 443 }] in { 444 def : RVVBuiltin<"v", "vv", type_range>; 445 def : RVVBuiltin<"Uv", "UvUv", type_range>; 446 } 447} 448 449multiclass RVVPseudoMaskBuiltin<string IR, string type_range> { 450 let Name = NAME, 451 IRName = IR, 452 HasMasked = false, 453 ManualCodegen = [{ 454 { 455 // op1, vl 456 IntrinsicTypes = {ResultType, 457 Ops[1]->getType()}; 458 Ops.insert(Ops.begin() + 1, Ops[0]); 459 break; 460 } 461 }] in { 462 def : RVVBuiltin<"m", "mm", type_range>; 463 } 464} 465 466multiclass RVVPseudoVFUnaryBuiltin<string IR, string type_range> { 467 let Name = NAME, 468 IRName = IR, 469 MaskedIRName = IR # "_mask", 470 UnMaskedPolicyScheme = HasPassthruOperand, 471 ManualCodegen = [{ 472 { 473 if (IsMasked) { 474 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); 475 if ((PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) 476 Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); 477 Ops.insert(Ops.begin() + 2, Ops[1]); 478 Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); 479 // maskedoff, op1, op2, mask, vl 480 IntrinsicTypes = {ResultType, 481 Ops[2]->getType(), 482 Ops.back()->getType()}; 483 } else { 484 if (PolicyAttrs & RVV_VTA) 485 Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); 486 // op1, po2, vl 487 IntrinsicTypes = {ResultType, 488 Ops[1]->getType(), Ops[2]->getType()}; 489 Ops.insert(Ops.begin() + 2, Ops[1]); 490 break; 491 } 492 break; 493 } 494 }] in { 495 def : RVVBuiltin<"v", "vv", type_range>; 496 } 497} 498 499multiclass RVVPseudoVWCVTBuiltin<string IR, string MName, string type_range, 500 list<list<string>> suffixes_prototypes> { 501 let Name = NAME, 502 OverloadedName = MName, 503 IRName = IR, 504 MaskedIRName = IR # "_mask", 505 UnMaskedPolicyScheme = HasPassthruOperand, 506 ManualCodegen = [{ 507 { 508 if (IsMasked) { 509 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); 510 if ((PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) 511 Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); 512 } else { 513 if (PolicyAttrs & RVV_VTA) 514 Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); 515 } 516 auto ElemTy = cast<llvm::VectorType>(ResultType)->getElementType(); 517 Ops.insert(Ops.begin() + 2, llvm::Constant::getNullValue(ElemTy)); 518 if (IsMasked) { 519 Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); 520 // maskedoff, op1, op2, mask, vl, policy 521 IntrinsicTypes = {ResultType, 522 Ops[1]->getType(), 523 ElemTy, 524 Ops[4]->getType()}; 525 } else { 526 // passtru, op1, op2, vl 527 IntrinsicTypes = {ResultType, 528 Ops[1]->getType(), 529 ElemTy, 530 Ops[3]->getType()}; 531 } 532 break; 533 } 534 }] in { 535 foreach s_p = suffixes_prototypes in { 536 def : RVVBuiltin<s_p[0], s_p[1], type_range>; 537 } 538 } 539} 540 541multiclass RVVPseudoVNCVTBuiltin<string IR, string MName, string type_range, 542 list<list<string>> suffixes_prototypes> { 543 let Name = NAME, 544 OverloadedName = MName, 545 IRName = IR, 546 MaskedIRName = IR # "_mask", 547 UnMaskedPolicyScheme = HasPassthruOperand, 548 ManualCodegen = [{ 549 { 550 if (IsMasked) { 551 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); 552 if ((PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) 553 Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); 554 } else { 555 if (PolicyAttrs & RVV_VTA) 556 Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); 557 } 558 Ops.insert(Ops.begin() + 2, llvm::Constant::getNullValue(Ops.back()->getType())); 559 if (IsMasked) { 560 Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); 561 // maskedoff, op1, xlen, mask, vl 562 IntrinsicTypes = {ResultType, 563 Ops[1]->getType(), 564 Ops[4]->getType(), 565 Ops[4]->getType()}; 566 } else { 567 // passthru, op1, xlen, vl 568 IntrinsicTypes = {ResultType, 569 Ops[1]->getType(), 570 Ops[3]->getType(), 571 Ops[3]->getType()}; 572 } 573 break; 574 } 575 }] in { 576 foreach s_p = suffixes_prototypes in { 577 def : RVVBuiltin<s_p[0], s_p[1], type_range>; 578 } 579 } 580} 581 582let HeaderCode = 583[{ 584#define __riscv_vlenb() __builtin_rvv_vlenb() 585}] in 586def vlenb_macro: RVVHeader; 587 588let HasBuiltinAlias = false, HasVL = false, HasMasked = false, 589 UnMaskedPolicyScheme = NonePolicy, MaskedPolicyScheme = NonePolicy, 590 Log2LMUL = [0], IRName = "", 591 ManualCodegen = [{ 592 { 593 LLVMContext &Context = CGM.getLLVMContext(); 594 llvm::MDBuilder MDHelper(Context); 595 596 llvm::Metadata *Ops[] = {llvm::MDString::get(Context, "vlenb")}; 597 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops); 598 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName); 599 llvm::Function *F = 600 CGM.getIntrinsic(llvm::Intrinsic::read_register, {SizeTy}); 601 return Builder.CreateCall(F, Metadata); 602 } 603 }] in 604{ 605 def vlenb : RVVBuiltin<"", "u", "i">; 606} 607 608// 6. Configuration-Setting Instructions 609// 6.1. vsetvli/vsetvl instructions 610 611// vsetvl/vsetvlmax are a macro because they require constant integers in SEW 612// and LMUL. 613let HeaderCode = 614[{ 615#define __riscv_vsetvl_e8mf4(avl) __builtin_rvv_vsetvli((size_t)(avl), 0, 6) 616#define __riscv_vsetvl_e8mf2(avl) __builtin_rvv_vsetvli((size_t)(avl), 0, 7) 617#define __riscv_vsetvl_e8m1(avl) __builtin_rvv_vsetvli((size_t)(avl), 0, 0) 618#define __riscv_vsetvl_e8m2(avl) __builtin_rvv_vsetvli((size_t)(avl), 0, 1) 619#define __riscv_vsetvl_e8m4(avl) __builtin_rvv_vsetvli((size_t)(avl), 0, 2) 620#define __riscv_vsetvl_e8m8(avl) __builtin_rvv_vsetvli((size_t)(avl), 0, 3) 621 622#define __riscv_vsetvl_e16mf2(avl) __builtin_rvv_vsetvli((size_t)(avl), 1, 7) 623#define __riscv_vsetvl_e16m1(avl) __builtin_rvv_vsetvli((size_t)(avl), 1, 0) 624#define __riscv_vsetvl_e16m2(avl) __builtin_rvv_vsetvli((size_t)(avl), 1, 1) 625#define __riscv_vsetvl_e16m4(avl) __builtin_rvv_vsetvli((size_t)(avl), 1, 2) 626#define __riscv_vsetvl_e16m8(avl) __builtin_rvv_vsetvli((size_t)(avl), 1, 3) 627 628#define __riscv_vsetvl_e32m1(avl) __builtin_rvv_vsetvli((size_t)(avl), 2, 0) 629#define __riscv_vsetvl_e32m2(avl) __builtin_rvv_vsetvli((size_t)(avl), 2, 1) 630#define __riscv_vsetvl_e32m4(avl) __builtin_rvv_vsetvli((size_t)(avl), 2, 2) 631#define __riscv_vsetvl_e32m8(avl) __builtin_rvv_vsetvli((size_t)(avl), 2, 3) 632 633#if __riscv_v_elen >= 64 634#define __riscv_vsetvl_e8mf8(avl) __builtin_rvv_vsetvli((size_t)(avl), 0, 5) 635#define __riscv_vsetvl_e16mf4(avl) __builtin_rvv_vsetvli((size_t)(avl), 1, 6) 636#define __riscv_vsetvl_e32mf2(avl) __builtin_rvv_vsetvli((size_t)(avl), 2, 7) 637 638#define __riscv_vsetvl_e64m1(avl) __builtin_rvv_vsetvli((size_t)(avl), 3, 0) 639#define __riscv_vsetvl_e64m2(avl) __builtin_rvv_vsetvli((size_t)(avl), 3, 1) 640#define __riscv_vsetvl_e64m4(avl) __builtin_rvv_vsetvli((size_t)(avl), 3, 2) 641#define __riscv_vsetvl_e64m8(avl) __builtin_rvv_vsetvli((size_t)(avl), 3, 3) 642#endif 643 644#define __riscv_vsetvlmax_e8mf4() __builtin_rvv_vsetvlimax(0, 6) 645#define __riscv_vsetvlmax_e8mf2() __builtin_rvv_vsetvlimax(0, 7) 646#define __riscv_vsetvlmax_e8m1() __builtin_rvv_vsetvlimax(0, 0) 647#define __riscv_vsetvlmax_e8m2() __builtin_rvv_vsetvlimax(0, 1) 648#define __riscv_vsetvlmax_e8m4() __builtin_rvv_vsetvlimax(0, 2) 649#define __riscv_vsetvlmax_e8m8() __builtin_rvv_vsetvlimax(0, 3) 650 651#define __riscv_vsetvlmax_e16mf2() __builtin_rvv_vsetvlimax(1, 7) 652#define __riscv_vsetvlmax_e16m1() __builtin_rvv_vsetvlimax(1, 0) 653#define __riscv_vsetvlmax_e16m2() __builtin_rvv_vsetvlimax(1, 1) 654#define __riscv_vsetvlmax_e16m4() __builtin_rvv_vsetvlimax(1, 2) 655#define __riscv_vsetvlmax_e16m8() __builtin_rvv_vsetvlimax(1, 3) 656 657#define __riscv_vsetvlmax_e32m1() __builtin_rvv_vsetvlimax(2, 0) 658#define __riscv_vsetvlmax_e32m2() __builtin_rvv_vsetvlimax(2, 1) 659#define __riscv_vsetvlmax_e32m4() __builtin_rvv_vsetvlimax(2, 2) 660#define __riscv_vsetvlmax_e32m8() __builtin_rvv_vsetvlimax(2, 3) 661 662#if __riscv_v_elen >= 64 663#define __riscv_vsetvlmax_e8mf8() __builtin_rvv_vsetvlimax(0, 5) 664#define __riscv_vsetvlmax_e16mf4() __builtin_rvv_vsetvlimax(1, 6) 665#define __riscv_vsetvlmax_e32mf2() __builtin_rvv_vsetvlimax(2, 7) 666 667#define __riscv_vsetvlmax_e64m1() __builtin_rvv_vsetvlimax(3, 0) 668#define __riscv_vsetvlmax_e64m2() __builtin_rvv_vsetvlimax(3, 1) 669#define __riscv_vsetvlmax_e64m4() __builtin_rvv_vsetvlimax(3, 2) 670#define __riscv_vsetvlmax_e64m8() __builtin_rvv_vsetvlimax(3, 3) 671#endif 672 673}] in 674def vsetvl_macro: RVVHeader; 675 676let HasBuiltinAlias = false, 677 HasVL = false, 678 HasMasked = false, 679 MaskedPolicyScheme = NonePolicy, 680 Log2LMUL = [0], 681 ManualCodegen = [{IntrinsicTypes = {ResultType};}] in // Set XLEN type 682{ 683 def vsetvli : RVVBuiltin<"", "zzKzKz", "i">; 684 def vsetvlimax : RVVBuiltin<"", "zKzKz", "i">; 685} 686 687// 7. Vector Loads and Stores 688// 7.4. Vector Unit-Stride Instructions 689def vlm: RVVVLEMaskBuiltin; 690defm vle8: RVVVLEBuiltin<["c"]>; 691defm vle16: RVVVLEBuiltin<["s"]>; 692let Name = "vle16_v", RequiredFeatures = ["Zvfhmin"] in 693 defm vle16_h: RVVVLEBuiltin<["x"]>; 694let Name = "vle16_v", RequiredFeatures = ["Zvfbfmin"] in 695 defm vle16_bf16 : RVVVLEBuiltin<["y"]>; 696defm vle32: RVVVLEBuiltin<["i","f"]>; 697defm vle64: RVVVLEBuiltin<["l","d"]>; 698 699def vsm : RVVVSEMaskBuiltin; 700defm vse8 : RVVVSEBuiltin<["c"]>; 701defm vse16: RVVVSEBuiltin<["s"]>; 702let Name = "vse16_v", RequiredFeatures = ["Zvfhmin"] in 703 defm vse16_h: RVVVSEBuiltin<["x"]>; 704let Name = "vse16_v", RequiredFeatures = ["Zvfbfmin"] in 705 defm vse16_bf16: RVVVSEBuiltin<["y"]>; 706defm vse32: RVVVSEBuiltin<["i","f"]>; 707defm vse64: RVVVSEBuiltin<["l","d"]>; 708 709// 7.5. Vector Strided Instructions 710defm vlse8: RVVVLSEBuiltin<["c"]>; 711defm vlse16: RVVVLSEBuiltin<["s"]>; 712let Name = "vlse16_v", RequiredFeatures = ["Zvfhmin"] in 713 defm vlse16_h: RVVVLSEBuiltin<["x"]>; 714let Name = "vlse16_v", RequiredFeatures = ["Zvfbfmin"] in 715 defm vlse16_bf16: RVVVLSEBuiltin<["y"]>; 716defm vlse32: RVVVLSEBuiltin<["i","f"]>; 717defm vlse64: RVVVLSEBuiltin<["l","d"]>; 718 719defm vsse8 : RVVVSSEBuiltin<["c"]>; 720defm vsse16: RVVVSSEBuiltin<["s"]>; 721let Name = "vsse16_v", RequiredFeatures = ["Zvfhmin"] in 722 defm vsse16_h: RVVVSSEBuiltin<["x"]>; 723let Name = "vsse16_v", RequiredFeatures = ["Zvfbfmin"] in 724 defm vsse16_bf: RVVVSSEBuiltin<["y"]>; 725defm vsse32: RVVVSSEBuiltin<["i","f"]>; 726defm vsse64: RVVVSSEBuiltin<["l","d"]>; 727 728// 7.6. Vector Indexed Instructions 729defm : RVVIndexedLoad<"vluxei">; 730defm : RVVIndexedLoad<"vloxei">; 731 732defm : RVVIndexedStore<"vsuxei">; 733defm : RVVIndexedStore<"vsoxei">; 734 735// 7.7. Unit-stride Fault-Only-First Loads 736defm vle8ff: RVVVLEFFBuiltin<["c"]>; 737defm vle16ff: RVVVLEFFBuiltin<["s"]>; 738let Name = "vle16ff_v", RequiredFeatures = ["Zvfhmin"] in 739 defm vle16ff: RVVVLEFFBuiltin<["x"]>; 740let Name = "vle16ff_v", RequiredFeatures = ["Zvfbfmin"] in 741 defm vle16ff: RVVVLEFFBuiltin<["y"]>; 742defm vle32ff: RVVVLEFFBuiltin<["i", "f"]>; 743defm vle64ff: RVVVLEFFBuiltin<["l", "d"]>; 744 745multiclass RVVUnitStridedSegLoadTuple<string op> { 746 foreach type = TypeList in { 747 defvar eew = !cond(!eq(type, "c") : "8", 748 !eq(type, "s") : "16", 749 !eq(type, "i") : "32", 750 !eq(type, "l") : "64", 751 !eq(type, "x") : "16", 752 !eq(type, "f") : "32", 753 !eq(type, "d") : "64", 754 !eq(type, "y") : "16"); 755 foreach nf = NFList in { 756 let Name = op # nf # "e" # eew # "_v", 757 IRName = op # nf, 758 MaskedIRName = op # nf # "_mask", 759 NF = nf, 760 RequiredFeatures = !if(!eq(type, "x"), ["Zvfhmin"], 761 !if(!eq(type, "y"), ["Zvfbfmin"], 762 []<string>)), 763 ManualCodegen = [{ 764 { 765 llvm::Type *ElementVectorType = cast<StructType>(ResultType)->elements()[0]; 766 IntrinsicTypes = {ElementVectorType, Ops.back()->getType()}; 767 SmallVector<llvm::Value*, 12> Operands; 768 769 bool NoPassthru = 770 (IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) | 771 (!IsMasked && (PolicyAttrs & RVV_VTA)); 772 unsigned Offset = IsMasked ? NoPassthru ? 1 : 2 : NoPassthru ? 0 : 1; 773 774 if (NoPassthru) { // Push poison into passthru 775 Operands.append(NF, llvm::PoisonValue::get(ElementVectorType)); 776 } else { // Push intrinsics operands into passthru 777 llvm::Value *PassthruOperand = IsMasked ? Ops[1] : Ops[0]; 778 for (unsigned I = 0; I < NF; ++I) 779 Operands.push_back(Builder.CreateExtractValue(PassthruOperand, {I})); 780 } 781 782 Operands.push_back(Ops[Offset]); // Ptr 783 if (IsMasked) 784 Operands.push_back(Ops[0]); 785 Operands.push_back(Ops[Offset + 1]); // VL 786 if (IsMasked) 787 Operands.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); 788 789 llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); 790 791 llvm::Value *LoadValue = Builder.CreateCall(F, Operands, ""); 792 if (ReturnValue.isNull()) 793 return LoadValue; 794 else 795 return Builder.CreateStore(LoadValue, ReturnValue.getValue()); 796 } 797 }] in { 798 defvar T = "(Tuple:" # nf # ")"; 799 def : RVVBuiltin<T # "v", T # "vPCe", type>; 800 if !not(IsFloat<type>.val) then { 801 def : RVVBuiltin<T # "Uv", T # "UvPCUe", type>; 802 } 803 } 804 } 805 } 806} 807 808multiclass RVVUnitStridedSegStoreTuple<string op> { 809 foreach type = TypeList in { 810 defvar eew = !cond(!eq(type, "c") : "8", 811 !eq(type, "s") : "16", 812 !eq(type, "i") : "32", 813 !eq(type, "l") : "64", 814 !eq(type, "x") : "16", 815 !eq(type, "f") : "32", 816 !eq(type, "d") : "64", 817 !eq(type, "y") : "16"); 818 foreach nf = NFList in { 819 let Name = op # nf # "e" # eew # "_v", 820 IRName = op # nf, 821 MaskedIRName = op # nf # "_mask", 822 NF = nf, 823 HasMaskedOffOperand = false, 824 RequiredFeatures = !if(!eq(type, "x"), ["Zvfhmin"], 825 !if(!eq(type, "y"), ["Zvfbfmin"], 826 []<string>)), 827 ManualCodegen = [{ 828 { 829 // Masked 830 // Builtin: (mask, ptr, v_tuple, vl) 831 // Intrinsic: (val0, val1, ..., ptr, mask, vl) 832 // Unmasked 833 // Builtin: (ptr, v_tuple, vl) 834 // Intrinsic: (val0, val1, ..., ptr, vl) 835 unsigned Offset = IsMasked ? 1 : 0; 836 llvm::Value *VTupleOperand = Ops[Offset + 1]; 837 838 SmallVector<llvm::Value*, 12> Operands; 839 for (unsigned I = 0; I < NF; ++I) { 840 llvm::Value *V = Builder.CreateExtractValue(VTupleOperand, {I}); 841 Operands.push_back(V); 842 } 843 Operands.push_back(Ops[Offset]); // Ptr 844 if (IsMasked) 845 Operands.push_back(Ops[0]); 846 Operands.push_back(Ops[Offset + 2]); // VL 847 848 IntrinsicTypes = {Operands[0]->getType(), Operands.back()->getType()}; 849 llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); 850 return Builder.CreateCall(F, Operands, ""); 851 } 852 }] in { 853 defvar T = "(Tuple:" # nf # ")"; 854 def : RVVBuiltin<T # "v", "0Pe" # T # "v", type>; 855 if !not(IsFloat<type>.val) then { 856 def : RVVBuiltin<T # "Uv", "0PUe" # T # "Uv", type>; 857 } 858 } 859 } 860 } 861} 862 863multiclass RVVUnitStridedSegLoadFFTuple<string op> { 864 foreach type = TypeList in { 865 defvar eew = !cond(!eq(type, "c") : "8", 866 !eq(type, "s") : "16", 867 !eq(type, "i") : "32", 868 !eq(type, "l") : "64", 869 !eq(type, "x") : "16", 870 !eq(type, "f") : "32", 871 !eq(type, "d") : "64", 872 !eq(type, "y") : "16"); 873 foreach nf = NFList in { 874 let Name = op # nf # "e" # eew # "ff_v", 875 IRName = op # nf # "ff", 876 MaskedIRName = op # nf # "ff_mask", 877 NF = nf, 878 RequiredFeatures = !if(!eq(type, "x"), ["Zvfhmin"], 879 !if(!eq(type, "y"), ["Zvfbfmin"], 880 []<string>)), 881 ManualCodegen = [{ 882 { 883 llvm::Type *ElementVectorType = cast<StructType>(ResultType)->elements()[0]; 884 IntrinsicTypes = {ElementVectorType, Ops.back()->getType()}; 885 SmallVector<llvm::Value*, 12> Operands; 886 887 bool NoPassthru = 888 (IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) | 889 (!IsMasked && (PolicyAttrs & RVV_VTA)); 890 unsigned Offset = IsMasked ? NoPassthru ? 1 : 2 : NoPassthru ? 0 : 1; 891 892 if (NoPassthru) { // Push poison into passthru 893 Operands.append(NF, llvm::PoisonValue::get(ElementVectorType)); 894 } else { // Push intrinsics operands into passthru 895 llvm::Value *PassthruOperand = IsMasked ? Ops[1] : Ops[0]; 896 for (unsigned I = 0; I < NF; ++I) 897 Operands.push_back(Builder.CreateExtractValue(PassthruOperand, {I})); 898 } 899 900 Operands.push_back(Ops[Offset]); // Ptr 901 if (IsMasked) 902 Operands.push_back(Ops[0]); 903 Operands.push_back(Ops[Offset + 2]); // vl 904 if (IsMasked) 905 Operands.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); 906 907 llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); 908 909 llvm::Value *LoadValue = Builder.CreateCall(F, Operands, ""); 910 // Get alignment from the new vl operand 911 clang::CharUnits Align = 912 CGM.getNaturalPointeeTypeAlignment(E->getArg(Offset + 1)->getType()); 913 914 llvm::Value *ReturnTuple = llvm::PoisonValue::get(ResultType); 915 for (unsigned I = 0; I < NF; ++I) { 916 llvm::Value *V = Builder.CreateExtractValue(LoadValue, {I}); 917 ReturnTuple = Builder.CreateInsertValue(ReturnTuple, V, {I}); 918 } 919 920 // Store new_vl 921 llvm::Value *V = Builder.CreateExtractValue(LoadValue, {NF}); 922 Builder.CreateStore(V, Address(Ops[Offset + 1], V->getType(), Align)); 923 924 if (ReturnValue.isNull()) 925 return ReturnTuple; 926 else 927 return Builder.CreateStore(ReturnTuple, ReturnValue.getValue()); 928 } 929 }] in { 930 defvar T = "(Tuple:" # nf # ")"; 931 def : RVVBuiltin<T # "v", T # "vPCePz", type>; 932 if !not(IsFloat<type>.val) then { 933 def : RVVBuiltin<T # "Uv", T # "UvPCUePz", type>; 934 } 935 } 936 } 937 } 938} 939 940multiclass RVVStridedSegLoadTuple<string op> { 941 foreach type = TypeList in { 942 defvar eew = !cond(!eq(type, "c") : "8", 943 !eq(type, "s") : "16", 944 !eq(type, "i") : "32", 945 !eq(type, "l") : "64", 946 !eq(type, "x") : "16", 947 !eq(type, "f") : "32", 948 !eq(type, "d") : "64", 949 !eq(type, "y") : "16"); 950 foreach nf = NFList in { 951 let Name = op # nf # "e" # eew # "_v", 952 IRName = op # nf, 953 MaskedIRName = op # nf # "_mask", 954 NF = nf, 955 RequiredFeatures = !if(!eq(type, "x"), ["Zvfhmin"], 956 !if(!eq(type, "y"), ["Zvfbfmin"], 957 []<string>)), 958 ManualCodegen = [{ 959 { 960 llvm::Type *ElementVectorType = cast<StructType>(ResultType)->elements()[0]; 961 IntrinsicTypes = {ElementVectorType, Ops.back()->getType()}; 962 SmallVector<llvm::Value*, 12> Operands; 963 964 bool NoPassthru = 965 (IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) | 966 (!IsMasked && (PolicyAttrs & RVV_VTA)); 967 unsigned Offset = IsMasked ? NoPassthru ? 1 : 2 : NoPassthru ? 0 : 1; 968 969 if (NoPassthru) { // Push poison into passthru 970 Operands.append(NF, llvm::PoisonValue::get(ElementVectorType)); 971 } else { // Push intrinsics operands into passthru 972 llvm::Value *PassthruOperand = IsMasked ? Ops[1] : Ops[0]; 973 for (unsigned I = 0; I < NF; ++I) 974 Operands.push_back(Builder.CreateExtractValue(PassthruOperand, {I})); 975 } 976 977 Operands.push_back(Ops[Offset]); // Ptr 978 Operands.push_back(Ops[Offset + 1]); // Stride 979 if (IsMasked) 980 Operands.push_back(Ops[0]); 981 Operands.push_back(Ops[Offset + 2]); // VL 982 if (IsMasked) 983 Operands.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); 984 985 llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); 986 llvm::Value *LoadValue = Builder.CreateCall(F, Operands, ""); 987 988 if (ReturnValue.isNull()) 989 return LoadValue; 990 else 991 return Builder.CreateStore(LoadValue, ReturnValue.getValue()); 992 } 993 }] in { 994 defvar T = "(Tuple:" # nf # ")"; 995 def : RVVBuiltin<T # "v", T # "vPCet", type>; 996 if !not(IsFloat<type>.val) then { 997 def : RVVBuiltin<T # "Uv", T # "UvPCUet", type>; 998 } 999 } 1000 } 1001 } 1002} 1003 1004multiclass RVVStridedSegStoreTuple<string op> { 1005 foreach type = TypeList in { 1006 defvar eew = !cond(!eq(type, "c") : "8", 1007 !eq(type, "s") : "16", 1008 !eq(type, "i") : "32", 1009 !eq(type, "l") : "64", 1010 !eq(type, "x") : "16", 1011 !eq(type, "f") : "32", 1012 !eq(type, "d") : "64", 1013 !eq(type, "y") : "16"); 1014 foreach nf = NFList in { 1015 let Name = op # nf # "e" # eew # "_v", 1016 IRName = op # nf, 1017 MaskedIRName = op # nf # "_mask", 1018 NF = nf, 1019 HasMaskedOffOperand = false, 1020 MaskedPolicyScheme = NonePolicy, 1021 RequiredFeatures = !if(!eq(type, "x"), ["Zvfhmin"], 1022 !if(!eq(type, "y"), ["Zvfbfmin"], 1023 []<string>)), 1024 ManualCodegen = [{ 1025 { 1026 // Masked 1027 // Builtin: (mask, ptr, stride, v_tuple, vl) 1028 // Intrinsic: (val0, val1, ..., ptr, stride, mask, vl) 1029 // Unmasked 1030 // Builtin: (ptr, stride, v_tuple, vl) 1031 // Intrinsic: (val0, val1, ..., ptr, stride, vl) 1032 unsigned Offset = IsMasked ? 1 : 0; 1033 llvm::Value *VTupleOperand = Ops[Offset + 2]; 1034 1035 SmallVector<llvm::Value*, 12> Operands; 1036 for (unsigned I = 0; I < NF; ++I) { 1037 llvm::Value *V = Builder.CreateExtractValue(VTupleOperand, {I}); 1038 Operands.push_back(V); 1039 } 1040 Operands.push_back(Ops[Offset]); // Ptr 1041 Operands.push_back(Ops[Offset + 1]); // Stride 1042 if (IsMasked) 1043 Operands.push_back(Ops[0]); 1044 Operands.push_back(Ops[Offset + 3]); // VL 1045 1046 IntrinsicTypes = {Operands[0]->getType(), Operands.back()->getType()}; 1047 llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); 1048 return Builder.CreateCall(F, Operands, ""); 1049 } 1050 }] in { 1051 defvar T = "(Tuple:" # nf # ")"; 1052 def : RVVBuiltin<T # "v", "0Pet" # T # "v", type>; 1053 if !not(IsFloat<type>.val) then { 1054 def : RVVBuiltin<T # "Uv", "0PUet" # T # "Uv", type>; 1055 } 1056 } 1057 } 1058 } 1059} 1060 1061multiclass RVVIndexedSegLoadTuple<string op> { 1062 foreach type = TypeList in { 1063 foreach eew_info = EEWList in { 1064 defvar eew = eew_info[0]; 1065 defvar eew_type = eew_info[1]; 1066 foreach nf = NFList in { 1067 let Name = op # nf # "ei" # eew # "_v", 1068 IRName = op # nf, 1069 MaskedIRName = op # nf # "_mask", 1070 NF = nf, 1071 RequiredFeatures = !if(!eq(type, "x"), ["Zvfhmin"], 1072 !if(!eq(type, "y"), ["Zvfbfmin"], 1073 []<string>)), 1074 ManualCodegen = [{ 1075 { 1076 llvm::Type *ElementVectorType = cast<StructType>(ResultType)->elements()[0]; 1077 IntrinsicTypes = {ElementVectorType, Ops.back()->getType()}; 1078 SmallVector<llvm::Value*, 12> Operands; 1079 1080 bool NoPassthru = 1081 (IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) | 1082 (!IsMasked && (PolicyAttrs & RVV_VTA)); 1083 unsigned Offset = IsMasked ? NoPassthru ? 1 : 2 : NoPassthru ? 0 : 1; 1084 1085 if (NoPassthru) { // Push poison into passthru 1086 Operands.append(NF, llvm::PoisonValue::get(ElementVectorType)); 1087 } else { // Push intrinsics operands into passthru 1088 llvm::Value *PassthruOperand = IsMasked ? Ops[1] : Ops[0]; 1089 for (unsigned I = 0; I < NF; ++I) 1090 Operands.push_back(Builder.CreateExtractValue(PassthruOperand, {I})); 1091 } 1092 1093 Operands.push_back(Ops[Offset]); // Ptr 1094 Operands.push_back(Ops[Offset + 1]); // Idx 1095 if (IsMasked) 1096 Operands.push_back(Ops[0]); 1097 Operands.push_back(Ops[Offset + 2]); // VL 1098 if (IsMasked) 1099 Operands.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); 1100 1101 IntrinsicTypes = {ElementVectorType, Ops[Offset + 1]->getType(), 1102 Ops.back()->getType()}; 1103 llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); 1104 llvm::Value *LoadValue = Builder.CreateCall(F, Operands, ""); 1105 1106 if (ReturnValue.isNull()) 1107 return LoadValue; 1108 else 1109 return Builder.CreateStore(LoadValue, ReturnValue.getValue()); 1110 } 1111 }] in { 1112 defvar T = "(Tuple:" # nf # ")"; 1113 def : RVVBuiltin<T # "v", T # "vPCe" # eew_type # "Uv", type>; 1114 if !not(IsFloat<type>.val) then { 1115 def : RVVBuiltin<T # "Uv", T # "UvPCUe" # eew_type # "Uv", type>; 1116 } 1117 } 1118 } 1119 } 1120 } 1121} 1122 1123multiclass RVVIndexedSegStoreTuple<string op> { 1124 foreach type = TypeList in { 1125 foreach eew_info = EEWList in { 1126 defvar eew = eew_info[0]; 1127 defvar eew_type = eew_info[1]; 1128 foreach nf = NFList in { 1129 let Name = op # nf # "ei" # eew # "_v", 1130 IRName = op # nf, 1131 MaskedIRName = op # nf # "_mask", 1132 NF = nf, 1133 HasMaskedOffOperand = false, 1134 MaskedPolicyScheme = NonePolicy, 1135 RequiredFeatures = !if(!eq(type, "x"), ["Zvfhmin"], 1136 !if(!eq(type, "y"), ["Zvfbfmin"], 1137 []<string>)), 1138 ManualCodegen = [{ 1139 { 1140 // Masked 1141 // Builtin: (mask, ptr, index, v_tuple, vl) 1142 // Intrinsic: (val0, val1, ..., ptr, index, mask, vl) 1143 // Unmasked 1144 // Builtin: (ptr, index, v_tuple, vl) 1145 // Intrinsic: (val0, val1, ..., ptr, index, vl) 1146 unsigned Offset = IsMasked ? 1 : 0; 1147 llvm::Value *VTupleOperand = Ops[Offset + 2]; 1148 1149 SmallVector<llvm::Value*, 12> Operands; 1150 for (unsigned I = 0; I < NF; ++I) { 1151 llvm::Value *V = Builder.CreateExtractValue(VTupleOperand, {I}); 1152 Operands.push_back(V); 1153 } 1154 Operands.push_back(Ops[Offset]); // Ptr 1155 Operands.push_back(Ops[Offset + 1]); // Idx 1156 if (IsMasked) 1157 Operands.push_back(Ops[0]); 1158 Operands.push_back(Ops[Offset + 3]); // VL 1159 1160 IntrinsicTypes = {Operands[0]->getType(), Ops[Offset + 1]->getType(), 1161 Operands.back()->getType()}; 1162 llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); 1163 return Builder.CreateCall(F, Operands, ""); 1164 } 1165 }] in { 1166 defvar T = "(Tuple:" # nf # ")"; 1167 def : RVVBuiltin<T # "v", "0Pe" # eew_type # "Uv" # T # "v", type>; 1168 if !not(IsFloat<type>.val) then { 1169 def : RVVBuiltin<T # "Uv", "0PUe" # eew_type # "Uv" # T # "Uv", type>; 1170 } 1171 } 1172 } 1173 } 1174 } 1175} 1176 1177// 7.8 Vector Load/Store Segment Instructions 1178let UnMaskedPolicyScheme = HasPassthruOperand, 1179 IsTuple = true in { 1180 defm : RVVUnitStridedSegLoadTuple<"vlseg">; 1181 defm : RVVUnitStridedSegLoadFFTuple<"vlseg">; 1182 defm : RVVStridedSegLoadTuple<"vlsseg">; 1183 defm : RVVIndexedSegLoadTuple<"vluxseg">; 1184 defm : RVVIndexedSegLoadTuple<"vloxseg">; 1185} 1186 1187let UnMaskedPolicyScheme = NonePolicy, 1188 MaskedPolicyScheme = NonePolicy, 1189 IsTuple = true in { 1190defm : RVVUnitStridedSegStoreTuple<"vsseg">; 1191defm : RVVStridedSegStoreTuple<"vssseg">; 1192defm : RVVIndexedSegStoreTuple<"vsuxseg">; 1193defm : RVVIndexedSegStoreTuple<"vsoxseg">; 1194} 1195 1196// 11. Vector Integer Arithmetic Instructions 1197// 11.1. Vector Single-Width Integer Add and Subtract 1198let UnMaskedPolicyScheme = HasPassthruOperand in { 1199defm vadd : RVVIntBinBuiltinSet; 1200defm vsub : RVVIntBinBuiltinSet; 1201defm vrsub : RVVOutOp1BuiltinSet<"vrsub", "csil", 1202 [["vx", "v", "vve"], 1203 ["vx", "Uv", "UvUvUe"]]>; 1204} 1205defm vneg_v : RVVPseudoUnaryBuiltin<"vrsub", "csil">; 1206 1207// 11.2. Vector Widening Integer Add/Subtract 1208// Widening unsigned integer add/subtract, 2*SEW = SEW +/- SEW 1209let UnMaskedPolicyScheme = HasPassthruOperand in { 1210defm vwaddu : RVVUnsignedWidenBinBuiltinSet; 1211defm vwsubu : RVVUnsignedWidenBinBuiltinSet; 1212// Widening signed integer add/subtract, 2*SEW = SEW +/- SEW 1213defm vwadd : RVVSignedWidenBinBuiltinSet; 1214defm vwsub : RVVSignedWidenBinBuiltinSet; 1215// Widening unsigned integer add/subtract, 2*SEW = 2*SEW +/- SEW 1216defm vwaddu : RVVUnsignedWidenOp0BinBuiltinSet; 1217defm vwsubu : RVVUnsignedWidenOp0BinBuiltinSet; 1218// Widening signed integer add/subtract, 2*SEW = 2*SEW +/- SEW 1219defm vwadd : RVVSignedWidenOp0BinBuiltinSet; 1220defm vwsub : RVVSignedWidenOp0BinBuiltinSet; 1221} 1222defm vwcvtu_x_x_v : RVVPseudoVWCVTBuiltin<"vwaddu", "vwcvtu_x", "csi", 1223 [["Uw", "UwUv"]]>; 1224defm vwcvt_x_x_v : RVVPseudoVWCVTBuiltin<"vwadd", "vwcvt_x", "csi", 1225 [["w", "wv"]]>; 1226 1227// 11.3. Vector Integer Extension 1228let UnMaskedPolicyScheme = HasPassthruOperand in { 1229let Log2LMUL = [-3, -2, -1, 0, 1, 2] in { 1230 def vsext_vf2 : RVVIntExt<"vsext", "w", "wv", "csi">; 1231 def vzext_vf2 : RVVIntExt<"vzext", "Uw", "UwUv", "csi">; 1232} 1233let Log2LMUL = [-3, -2, -1, 0, 1] in { 1234 def vsext_vf4 : RVVIntExt<"vsext", "q", "qv", "cs">; 1235 def vzext_vf4 : RVVIntExt<"vzext", "Uq", "UqUv", "cs">; 1236} 1237let Log2LMUL = [-3, -2, -1, 0] in { 1238 def vsext_vf8 : RVVIntExt<"vsext", "o", "ov", "c">; 1239 def vzext_vf8 : RVVIntExt<"vzext", "Uo", "UoUv", "c">; 1240} 1241} 1242 1243// 11.4. Vector Integer Add-with-Carry / Subtract-with-Borrow Instructions 1244let HasMasked = false, MaskedPolicyScheme = NonePolicy in { 1245 let UnMaskedPolicyScheme = HasPassthruOperand in { 1246 defm vadc : RVVCarryinBuiltinSet; 1247 defm vsbc : RVVCarryinBuiltinSet; 1248 } 1249 defm vmadc : RVVCarryOutInBuiltinSet<"vmadc_carry_in">; 1250 defm vmadc : RVVIntMaskOutBuiltinSet; 1251 defm vmsbc : RVVCarryOutInBuiltinSet<"vmsbc_borrow_in">; 1252 defm vmsbc : RVVIntMaskOutBuiltinSet; 1253} 1254 1255// 11.5. Vector Bitwise Logical Instructions 1256let UnMaskedPolicyScheme = HasPassthruOperand in { 1257defm vand : RVVIntBinBuiltinSet; 1258defm vxor : RVVIntBinBuiltinSet; 1259defm vor : RVVIntBinBuiltinSet; 1260} 1261defm vnot_v : RVVPseudoVNotBuiltin<"vxor", "csil">; 1262 1263// 11.6. Vector Single-Width Shift Instructions 1264let UnMaskedPolicyScheme = HasPassthruOperand in { 1265defm vsll : RVVShiftBuiltinSet; 1266defm vsrl : RVVUnsignedShiftBuiltinSet; 1267defm vsra : RVVSignedShiftBuiltinSet; 1268 1269// 11.7. Vector Narrowing Integer Right Shift Instructions 1270defm vnsrl : RVVUnsignedNShiftBuiltinSet; 1271defm vnsra : RVVSignedNShiftBuiltinSet; 1272} 1273defm vncvt_x_x_w : RVVPseudoVNCVTBuiltin<"vnsrl", "vncvt_x", "csi", 1274 [["v", "vw"], 1275 ["Uv", "UvUw"]]>; 1276 1277// 11.8. Vector Integer Compare Instructions 1278let MaskedPolicyScheme = HasPassthruOperand, 1279 HasTailPolicy = false in { 1280defm vmseq : RVVIntMaskOutBuiltinSet; 1281defm vmsne : RVVIntMaskOutBuiltinSet; 1282defm vmsltu : RVVUnsignedMaskOutBuiltinSet; 1283defm vmslt : RVVSignedMaskOutBuiltinSet; 1284defm vmsleu : RVVUnsignedMaskOutBuiltinSet; 1285defm vmsle : RVVSignedMaskOutBuiltinSet; 1286defm vmsgtu : RVVUnsignedMaskOutBuiltinSet; 1287defm vmsgt : RVVSignedMaskOutBuiltinSet; 1288defm vmsgeu : RVVUnsignedMaskOutBuiltinSet; 1289defm vmsge : RVVSignedMaskOutBuiltinSet; 1290} 1291 1292// 11.9. Vector Integer Min/Max Instructions 1293let UnMaskedPolicyScheme = HasPassthruOperand in { 1294defm vminu : RVVUnsignedBinBuiltinSet; 1295defm vmin : RVVSignedBinBuiltinSet; 1296defm vmaxu : RVVUnsignedBinBuiltinSet; 1297defm vmax : RVVSignedBinBuiltinSet; 1298 1299// 11.10. Vector Single-Width Integer Multiply Instructions 1300defm vmul : RVVIntBinBuiltinSet; 1301defm vmulh : RVVSignedBinBuiltinSet; 1302defm vmulhu : RVVUnsignedBinBuiltinSet; 1303defm vmulhsu : RVVOutOp1BuiltinSet<"vmulhsu", "csil", 1304 [["vv", "v", "vvUv"], 1305 ["vx", "v", "vvUe"]]>; 1306 1307// 11.11. Vector Integer Divide Instructions 1308defm vdivu : RVVUnsignedBinBuiltinSet; 1309defm vdiv : RVVSignedBinBuiltinSet; 1310defm vremu : RVVUnsignedBinBuiltinSet; 1311defm vrem : RVVSignedBinBuiltinSet; 1312} 1313 1314// 11.12. Vector Widening Integer Multiply Instructions 1315let Log2LMUL = [-3, -2, -1, 0, 1, 2], UnMaskedPolicyScheme = HasPassthruOperand in { 1316defm vwmul : RVVOutOp0Op1BuiltinSet<"vwmul", "csi", 1317 [["vv", "w", "wvv"], 1318 ["vx", "w", "wve"]]>; 1319defm vwmulu : RVVOutOp0Op1BuiltinSet<"vwmulu", "csi", 1320 [["vv", "Uw", "UwUvUv"], 1321 ["vx", "Uw", "UwUvUe"]]>; 1322defm vwmulsu : RVVOutOp0Op1BuiltinSet<"vwmulsu", "csi", 1323 [["vv", "w", "wvUv"], 1324 ["vx", "w", "wvUe"]]>; 1325} 1326 1327// 11.13. Vector Single-Width Integer Multiply-Add Instructions 1328let UnMaskedPolicyScheme = HasPolicyOperand in { 1329defm vmacc : RVVIntTerBuiltinSet; 1330defm vnmsac : RVVIntTerBuiltinSet; 1331defm vmadd : RVVIntTerBuiltinSet; 1332defm vnmsub : RVVIntTerBuiltinSet; 1333 1334// 11.14. Vector Widening Integer Multiply-Add Instructions 1335let HasMaskedOffOperand = false, 1336 Log2LMUL = [-3, -2, -1, 0, 1, 2] in { 1337defm vwmaccu : RVVOutOp1Op2BuiltinSet<"vwmaccu", "csi", 1338 [["vv", "Uw", "UwUwUvUv"], 1339 ["vx", "Uw", "UwUwUeUv"]]>; 1340defm vwmacc : RVVOutOp1Op2BuiltinSet<"vwmacc", "csi", 1341 [["vv", "w", "wwvv"], 1342 ["vx", "w", "wwev"]]>; 1343defm vwmaccsu : RVVOutOp1Op2BuiltinSet<"vwmaccsu", "csi", 1344 [["vv", "w", "wwvUv"], 1345 ["vx", "w", "wweUv"]]>; 1346defm vwmaccus : RVVOutOp1Op2BuiltinSet<"vwmaccus", "csi", 1347 [["vx", "w", "wwUev"]]>; 1348} 1349} 1350 1351// 11.15. Vector Integer Merge Instructions 1352// C/C++ Operand: (mask, op1, op2, vl), Intrinsic: (passthru, op1, op2, mask, vl) 1353let HasMasked = false, 1354 UnMaskedPolicyScheme = HasPassthruOperand, 1355 MaskedPolicyScheme = NonePolicy, 1356 ManualCodegen = [{ 1357 // insert poison passthru 1358 if (PolicyAttrs & RVV_VTA) 1359 Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); 1360 IntrinsicTypes = {ResultType, Ops[2]->getType(), Ops.back()->getType()}; 1361 }] in { 1362 defm vmerge : RVVOutOp1BuiltinSet<"vmerge", "csil", 1363 [["vvm", "v", "vvvm"], 1364 ["vxm", "v", "vvem"], 1365 ["vvm", "Uv", "UvUvUvm"], 1366 ["vxm", "Uv", "UvUvUem"]]>; 1367} 1368 1369// 11.16. Vector Integer Move Instructions 1370let HasMasked = false, 1371 UnMaskedPolicyScheme = HasPassthruOperand, 1372 MaskedPolicyScheme = NonePolicy, 1373 OverloadedName = "vmv_v" in { 1374 defm vmv_v : RVVOutBuiltinSet<"vmv_v_v", "csil", 1375 [["v", "Uv", "UvUv"]]>; 1376 defm vmv_v : RVVOutBuiltinSet<"vmv_v_v", "csilfd", 1377 [["v", "v", "vv"]]>; 1378 let RequiredFeatures = ["Zvfhmin"] in 1379 defm vmv_v : RVVOutBuiltinSet<"vmv_v_v", "x", 1380 [["v", "v", "vv"]]>; 1381 let SupportOverloading = false in 1382 defm vmv_v : RVVOutBuiltinSet<"vmv_v_x", "csil", 1383 [["x", "v", "ve"], 1384 ["x", "Uv", "UvUe"]]>; 1385} 1386 1387// 12. Vector Fixed-Point Arithmetic Instructions 1388let HeaderCode = 1389[{ 1390enum __RISCV_VXRM { 1391 __RISCV_VXRM_RNU = 0, 1392 __RISCV_VXRM_RNE = 1, 1393 __RISCV_VXRM_RDN = 2, 1394 __RISCV_VXRM_ROD = 3, 1395}; 1396}] in 1397def vxrm_enum : RVVHeader; 1398 1399// 12.1. Vector Single-Width Saturating Add and Subtract 1400let UnMaskedPolicyScheme = HasPassthruOperand in { 1401defm vsaddu : RVVUnsignedBinBuiltinSet; 1402defm vsadd : RVVSignedBinBuiltinSet; 1403defm vssubu : RVVUnsignedBinBuiltinSet; 1404defm vssub : RVVSignedBinBuiltinSet; 1405 1406let ManualCodegen = [{ 1407 { 1408 // LLVM intrinsic 1409 // Unmasked: (passthru, op0, op1, round_mode, vl) 1410 // Masked: (passthru, vector_in, vector_in/scalar_in, mask, vxrm, vl, policy) 1411 1412 SmallVector<llvm::Value*, 7> Operands; 1413 bool HasMaskedOff = !( 1414 (IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) || 1415 (!IsMasked && PolicyAttrs & RVV_VTA)); 1416 unsigned Offset = IsMasked ? 1417 (HasMaskedOff ? 2 : 1) : (HasMaskedOff ? 1 : 0); 1418 1419 if (!HasMaskedOff) 1420 Operands.push_back(llvm::PoisonValue::get(ResultType)); 1421 else 1422 Operands.push_back(Ops[IsMasked ? 1 : 0]); 1423 1424 Operands.push_back(Ops[Offset]); // op0 1425 Operands.push_back(Ops[Offset + 1]); // op1 1426 1427 if (IsMasked) 1428 Operands.push_back(Ops[0]); // mask 1429 1430 Operands.push_back(Ops[Offset + 2]); // vxrm 1431 Operands.push_back(Ops[Offset + 3]); // vl 1432 1433 if (IsMasked) 1434 Operands.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); 1435 1436 IntrinsicTypes = {ResultType, Ops[Offset + 1]->getType(), Ops.back()->getType()}; 1437 llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); 1438 return Builder.CreateCall(F, Operands, ""); 1439 } 1440}] in { 1441 // 12.2. Vector Single-Width Averaging Add and Subtract 1442 defm vaaddu : RVVUnsignedBinBuiltinSetRoundingMode; 1443 defm vaadd : RVVSignedBinBuiltinSetRoundingMode; 1444 defm vasubu : RVVUnsignedBinBuiltinSetRoundingMode; 1445 defm vasub : RVVSignedBinBuiltinSetRoundingMode; 1446 1447 // 12.3. Vector Single-Width Fractional Multiply with Rounding and Saturation 1448 defm vsmul : RVVSignedBinBuiltinSetRoundingMode; 1449 1450 // 12.4. Vector Single-Width Scaling Shift Instructions 1451 defm vssrl : RVVUnsignedShiftBuiltinSetRoundingMode; 1452 defm vssra : RVVSignedShiftBuiltinSetRoundingMode; 1453} 1454 1455let ManualCodegen = [{ 1456 { 1457 // LLVM intrinsic 1458 // Unmasked: (passthru, op0, op1, round_mode, vl) 1459 // Masked: (passthru, vector_in, vector_in/scalar_in, mask, vxrm, vl, policy) 1460 1461 SmallVector<llvm::Value*, 7> Operands; 1462 bool HasMaskedOff = !( 1463 (IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) || 1464 (!IsMasked && PolicyAttrs & RVV_VTA)); 1465 unsigned Offset = IsMasked ? 1466 (HasMaskedOff ? 2 : 1) : (HasMaskedOff ? 1 : 0); 1467 1468 if (!HasMaskedOff) 1469 Operands.push_back(llvm::PoisonValue::get(ResultType)); 1470 else 1471 Operands.push_back(Ops[IsMasked ? 1 : 0]); 1472 1473 Operands.push_back(Ops[Offset]); // op0 1474 Operands.push_back(Ops[Offset + 1]); // op1 1475 1476 if (IsMasked) 1477 Operands.push_back(Ops[0]); // mask 1478 1479 Operands.push_back(Ops[Offset + 2]); // vxrm 1480 Operands.push_back(Ops[Offset + 3]); // vl 1481 1482 if (IsMasked) 1483 Operands.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); 1484 1485 IntrinsicTypes = {ResultType, Ops[Offset]->getType(), Ops[Offset + 1]->getType(), 1486 Ops.back()->getType()}; 1487 llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); 1488 return Builder.CreateCall(F, Operands, ""); 1489 } 1490}] in { 1491 // 12.5. Vector Narrowing Fixed-Point Clip Instructions 1492 defm vnclipu : RVVUnsignedNShiftBuiltinSetRoundingMode; 1493 defm vnclip : RVVSignedNShiftBuiltinSetRoundingMode; 1494} 1495} 1496 1497// 13. Vector Floating-Point Instructions 1498let HeaderCode = 1499[{ 1500enum __RISCV_FRM { 1501 __RISCV_FRM_RNE = 0, 1502 __RISCV_FRM_RTZ = 1, 1503 __RISCV_FRM_RDN = 2, 1504 __RISCV_FRM_RUP = 3, 1505 __RISCV_FRM_RMM = 4, 1506}; 1507}] in def frm_enum : RVVHeader; 1508 1509let UnMaskedPolicyScheme = HasPassthruOperand in { 1510let ManualCodegen = [{ 1511 { 1512 // LLVM intrinsic 1513 // Unmasked: (passthru, op0, op1, round_mode, vl) 1514 // Masked: (passthru, vector_in, vector_in/scalar_in, mask, frm, vl, policy) 1515 1516 SmallVector<llvm::Value*, 7> Operands; 1517 bool HasMaskedOff = !( 1518 (IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) || 1519 (!IsMasked && PolicyAttrs & RVV_VTA)); 1520 bool HasRoundModeOp = IsMasked ? 1521 (HasMaskedOff ? Ops.size() == 6 : Ops.size() == 5) : 1522 (HasMaskedOff ? Ops.size() == 5 : Ops.size() == 4); 1523 1524 unsigned Offset = IsMasked ? 1525 (HasMaskedOff ? 2 : 1) : (HasMaskedOff ? 1 : 0); 1526 1527 if (!HasMaskedOff) 1528 Operands.push_back(llvm::PoisonValue::get(ResultType)); 1529 else 1530 Operands.push_back(Ops[IsMasked ? 1 : 0]); 1531 1532 Operands.push_back(Ops[Offset]); // op0 1533 Operands.push_back(Ops[Offset + 1]); // op1 1534 1535 if (IsMasked) 1536 Operands.push_back(Ops[0]); // mask 1537 1538 if (HasRoundModeOp) { 1539 Operands.push_back(Ops[Offset + 2]); // frm 1540 Operands.push_back(Ops[Offset + 3]); // vl 1541 } else { 1542 Operands.push_back(ConstantInt::get(Ops[Offset + 2]->getType(), 7)); // frm 1543 Operands.push_back(Ops[Offset + 2]); // vl 1544 } 1545 1546 if (IsMasked) 1547 Operands.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); 1548 1549 IntrinsicTypes = {ResultType, Ops[Offset + 1]->getType(), 1550 Operands.back()->getType()}; 1551 llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); 1552 return Builder.CreateCall(F, Operands, ""); 1553 } 1554}] in { 1555 let HasFRMRoundModeOp = true in { 1556 // 13.2. Vector Single-Width Floating-Point Add/Subtract Instructions 1557 defm vfadd : RVVFloatingBinBuiltinSetRoundingMode; 1558 defm vfsub : RVVFloatingBinBuiltinSetRoundingMode; 1559 defm vfrsub : RVVFloatingBinVFBuiltinSetRoundingMode; 1560 1561 // 13.3. Vector Widening Floating-Point Add/Subtract Instructions 1562 // Widening FP add/subtract, 2*SEW = 2*SEW +/- SEW 1563 defm vfwadd : RVVFloatingWidenOp0BinBuiltinSetRoundingMode; 1564 defm vfwsub : RVVFloatingWidenOp0BinBuiltinSetRoundingMode; 1565 1566 // 13.4. Vector Single-Width Floating-Point Multiply/Divide Instructions 1567 defm vfmul : RVVFloatingBinBuiltinSetRoundingMode; 1568 defm vfdiv : RVVFloatingBinBuiltinSetRoundingMode; 1569 defm vfrdiv : RVVFloatingBinVFBuiltinSetRoundingMode; 1570 } 1571 // 13.2. Vector Single-Width Floating-Point Add/Subtract Instructions 1572 defm vfadd : RVVFloatingBinBuiltinSet; 1573 defm vfsub : RVVFloatingBinBuiltinSet; 1574 defm vfrsub : RVVFloatingBinVFBuiltinSet; 1575 1576 // 13.3. Vector Widening Floating-Point Add/Subtract Instructions 1577 // Widening FP add/subtract, 2*SEW = 2*SEW +/- SEW 1578 defm vfwadd : RVVFloatingWidenOp0BinBuiltinSet; 1579 defm vfwsub : RVVFloatingWidenOp0BinBuiltinSet; 1580 1581 // 13.4. Vector Single-Width Floating-Point Multiply/Divide Instructions 1582 defm vfmul : RVVFloatingBinBuiltinSet; 1583 defm vfdiv : RVVFloatingBinBuiltinSet; 1584 defm vfrdiv : RVVFloatingBinVFBuiltinSet; 1585} 1586 1587let ManualCodegen = [{ 1588 { 1589 // LLVM intrinsic 1590 // Unmasked: (passthru, op0, op1, round_mode, vl) 1591 // Masked: (passthru, vector_in, vector_in/scalar_in, mask, frm, vl, policy) 1592 1593 SmallVector<llvm::Value*, 7> Operands; 1594 bool HasMaskedOff = !( 1595 (IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) || 1596 (!IsMasked && PolicyAttrs & RVV_VTA)); 1597 bool HasRoundModeOp = IsMasked ? 1598 (HasMaskedOff ? Ops.size() == 6 : Ops.size() == 5) : 1599 (HasMaskedOff ? Ops.size() == 5 : Ops.size() == 4); 1600 1601 unsigned Offset = IsMasked ? 1602 (HasMaskedOff ? 2 : 1) : (HasMaskedOff ? 1 : 0); 1603 1604 if (!HasMaskedOff) 1605 Operands.push_back(llvm::PoisonValue::get(ResultType)); 1606 else 1607 Operands.push_back(Ops[IsMasked ? 1 : 0]); 1608 1609 Operands.push_back(Ops[Offset]); // op0 1610 Operands.push_back(Ops[Offset + 1]); // op1 1611 1612 if (IsMasked) 1613 Operands.push_back(Ops[0]); // mask 1614 1615 if (HasRoundModeOp) { 1616 Operands.push_back(Ops[Offset + 2]); // frm 1617 Operands.push_back(Ops[Offset + 3]); // vl 1618 } else { 1619 Operands.push_back(ConstantInt::get(Ops[Offset + 2]->getType(), 7)); // frm 1620 Operands.push_back(Ops[Offset + 2]); // vl 1621 } 1622 1623 if (IsMasked) 1624 Operands.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); 1625 1626 IntrinsicTypes = {ResultType, Ops[Offset]->getType(), Ops[Offset + 1]->getType(), 1627 Ops.back()->getType()}; 1628 llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); 1629 return Builder.CreateCall(F, Operands, ""); 1630 } 1631}] in { 1632 let HasFRMRoundModeOp = true in { 1633 // 13.3. Vector Widening Floating-Point Add/Subtract Instructions 1634 // Widening FP add/subtract, 2*SEW = SEW +/- SEW 1635 defm vfwadd : RVVFloatingWidenBinBuiltinSetRoundingMode; 1636 defm vfwsub : RVVFloatingWidenBinBuiltinSetRoundingMode; 1637 1638 // 13.5. Vector Widening Floating-Point Multiply 1639 let Log2LMUL = [-2, -1, 0, 1, 2] in { 1640 defm vfwmul : RVVOutOp0Op1BuiltinSet<"vfwmul", "xf", 1641 [["vv", "w", "wvvu"], 1642 ["vf", "w", "wveu"]]>; 1643 } 1644 } 1645 // 13.3. Vector Widening Floating-Point Add/Subtract Instructions 1646 // Widening FP add/subtract, 2*SEW = SEW +/- SEW 1647 defm vfwadd : RVVFloatingWidenBinBuiltinSet; 1648 defm vfwsub : RVVFloatingWidenBinBuiltinSet; 1649 1650 // 13.5. Vector Widening Floating-Point Multiply 1651 let Log2LMUL = [-2, -1, 0, 1, 2] in { 1652 defm vfwmul : RVVOutOp0Op1BuiltinSet<"vfwmul", "xf", 1653 [["vv", "w", "wvv"], 1654 ["vf", "w", "wve"]]>; 1655 } 1656} 1657} 1658 1659 1660let UnMaskedPolicyScheme = HasPolicyOperand in { 1661let ManualCodegen = [{ 1662 { 1663 // LLVM intrinsic 1664 // Unmasked: (passthru, op0, op1, round_mode, vl) 1665 // Masked: (passthru, vector_in, vector_in/scalar_in, mask, frm, vl, policy) 1666 1667 SmallVector<llvm::Value*, 7> Operands; 1668 bool HasRoundModeOp = IsMasked ? Ops.size() == 6 : Ops.size() == 5; 1669 1670 unsigned Offset = IsMasked ? 2 : 1; 1671 1672 Operands.push_back(Ops[IsMasked ? 1 : 0]); // passthrough 1673 1674 Operands.push_back(Ops[Offset]); // op0 1675 Operands.push_back(Ops[Offset + 1]); // op1 1676 1677 if (IsMasked) 1678 Operands.push_back(Ops[0]); // mask 1679 1680 if (HasRoundModeOp) { 1681 Operands.push_back(Ops[Offset + 2]); // frm 1682 Operands.push_back(Ops[Offset + 3]); // vl 1683 } else { 1684 Operands.push_back(ConstantInt::get(Ops[Offset + 2]->getType(), 7)); // frm 1685 Operands.push_back(Ops[Offset + 2]); // vl 1686 } 1687 1688 Operands.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); 1689 1690 IntrinsicTypes = {ResultType, Ops[Offset]->getType(), 1691 Operands.back()->getType()}; 1692 1693 llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); 1694 1695 return Builder.CreateCall(F, Operands, ""); 1696 } 1697}] in { 1698 let HasFRMRoundModeOp = 1 in { 1699 // 13.6. Vector Single-Width Floating-Point Fused Multiply-Add Instructions 1700 defm vfmacc : RVVFloatingTerBuiltinSetRoundingMode; 1701 defm vfnmacc : RVVFloatingTerBuiltinSetRoundingMode; 1702 defm vfmsac : RVVFloatingTerBuiltinSetRoundingMode; 1703 defm vfnmsac : RVVFloatingTerBuiltinSetRoundingMode; 1704 defm vfmadd : RVVFloatingTerBuiltinSetRoundingMode; 1705 defm vfnmadd : RVVFloatingTerBuiltinSetRoundingMode; 1706 defm vfmsub : RVVFloatingTerBuiltinSetRoundingMode; 1707 defm vfnmsub : RVVFloatingTerBuiltinSetRoundingMode; 1708 } 1709 // 13.6. Vector Single-Width Floating-Point Fused Multiply-Add Instructions 1710 defm vfmacc : RVVFloatingTerBuiltinSet; 1711 defm vfnmacc : RVVFloatingTerBuiltinSet; 1712 defm vfmsac : RVVFloatingTerBuiltinSet; 1713 defm vfnmsac : RVVFloatingTerBuiltinSet; 1714 defm vfmadd : RVVFloatingTerBuiltinSet; 1715 defm vfnmadd : RVVFloatingTerBuiltinSet; 1716 defm vfmsub : RVVFloatingTerBuiltinSet; 1717 defm vfnmsub : RVVFloatingTerBuiltinSet; 1718} 1719 1720let ManualCodegen = [{ 1721 { 1722 // LLVM intrinsic 1723 // Unmasked: (passthru, op0, op1, round_mode, vl) 1724 // Masked: (passthru, vector_in, vector_in/scalar_in, mask, frm, vl, policy) 1725 1726 SmallVector<llvm::Value*, 7> Operands; 1727 bool HasRoundModeOp = IsMasked ? Ops.size() == 6 : Ops.size() == 5; 1728 1729 unsigned Offset = IsMasked ? 2 : 1; 1730 1731 Operands.push_back(Ops[IsMasked ? 1 : 0]); // passthrough 1732 1733 Operands.push_back(Ops[Offset]); // op0 1734 Operands.push_back(Ops[Offset + 1]); // op1 1735 1736 if (IsMasked) 1737 Operands.push_back(Ops[0]); // mask 1738 1739 if (HasRoundModeOp) { 1740 Operands.push_back(Ops[Offset + 2]); // frm 1741 Operands.push_back(Ops[Offset + 3]); // vl 1742 } else { 1743 Operands.push_back(ConstantInt::get(Ops[Offset + 2]->getType(), 7)); // frm 1744 Operands.push_back(Ops[Offset + 2]); // vl 1745 } 1746 1747 Operands.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); 1748 1749 IntrinsicTypes = {ResultType, Ops[Offset]->getType(), Ops[Offset + 1]->getType(), 1750 Operands.back()->getType()}; 1751 1752 llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); 1753 1754 return Builder.CreateCall(F, Operands, ""); 1755 } 1756}] in { 1757 let HasFRMRoundModeOp = 1 in { 1758 // 13.7. Vector Widening Floating-Point Fused Multiply-Add Instructions 1759 defm vfwmacc : RVVFloatingWidenTerBuiltinSetRoundingMode; 1760 defm vfwnmacc : RVVFloatingWidenTerBuiltinSetRoundingMode; 1761 defm vfwmsac : RVVFloatingWidenTerBuiltinSetRoundingMode; 1762 defm vfwnmsac : RVVFloatingWidenTerBuiltinSetRoundingMode; 1763 1764 // Vector BF16 widening multiply-accumulate 1765 let Log2LMUL = [-2, -1, 0, 1, 2], 1766 RequiredFeatures = ["Zvfbfwma"], 1767 HasMaskedOffOperand = false in 1768 defm vfwmaccbf16 : RVVOutOp1Op2BuiltinSet<"vfwmaccbf16", "y", 1769 [["vv", "Fw", "FwFwvvu"], 1770 ["vf", "Fw", "FwFwevu"]]>; 1771 } 1772 // 13.7. Vector Widening Floating-Point Fused Multiply-Add Instructions 1773 defm vfwmacc : RVVFloatingWidenTerBuiltinSet; 1774 defm vfwnmacc : RVVFloatingWidenTerBuiltinSet; 1775 defm vfwmsac : RVVFloatingWidenTerBuiltinSet; 1776 defm vfwnmsac : RVVFloatingWidenTerBuiltinSet; 1777 1778 // Vector BF16 widening multiply-accumulate 1779 let Log2LMUL = [-2, -1, 0, 1, 2], 1780 RequiredFeatures = ["Zvfbfwma"], 1781 HasMaskedOffOperand = false in 1782 defm vfwmaccbf16 : RVVOutOp1Op2BuiltinSet<"vfwmaccbf16", "y", 1783 [["vv", "Fw", "FwFwvv"], 1784 ["vf", "Fw", "FwFwev"]]>; 1785} 1786 1787} 1788 1789let UnMaskedPolicyScheme = HasPassthruOperand in { 1790let ManualCodegen = [{ 1791 { 1792 // LLVM intrinsic 1793 // Unmasked: (passthru, op0, round_mode, vl) 1794 // Masked: (passthru, op0, mask, frm, vl, policy) 1795 1796 SmallVector<llvm::Value*, 7> Operands; 1797 bool HasMaskedOff = !( 1798 (IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) || 1799 (!IsMasked && PolicyAttrs & RVV_VTA)); 1800 bool HasRoundModeOp = IsMasked ? 1801 (HasMaskedOff ? Ops.size() == 5 : Ops.size() == 4) : 1802 (HasMaskedOff ? Ops.size() == 4 : Ops.size() == 3); 1803 1804 unsigned Offset = IsMasked ? 1805 (HasMaskedOff ? 2 : 1) : (HasMaskedOff ? 1 : 0); 1806 1807 if (!HasMaskedOff) 1808 Operands.push_back(llvm::PoisonValue::get(ResultType)); 1809 else 1810 Operands.push_back(Ops[IsMasked ? 1 : 0]); 1811 1812 Operands.push_back(Ops[Offset]); // op0 1813 1814 if (IsMasked) 1815 Operands.push_back(Ops[0]); // mask 1816 1817 if (HasRoundModeOp) { 1818 Operands.push_back(Ops[Offset + 1]); // frm 1819 Operands.push_back(Ops[Offset + 2]); // vl 1820 } else { 1821 Operands.push_back(ConstantInt::get(Ops[Offset + 1]->getType(), 7)); // frm 1822 Operands.push_back(Ops[Offset + 1]); // vl 1823 } 1824 1825 if (IsMasked) 1826 Operands.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); 1827 1828 IntrinsicTypes = {ResultType, Operands.back()->getType()}; 1829 llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); 1830 return Builder.CreateCall(F, Operands, ""); 1831 } 1832}] in { 1833 let HasFRMRoundModeOp = 1 in { 1834 // 13.8. Vector Floating-Point Square-Root Instruction 1835 defm vfsqrt : RVVOutBuiltinSet<"vfsqrt", "xfd", [["v", "v", "vvu"]]>; 1836 1837 // 13.10. Vector Floating-Point Reciprocal Estimate Instruction 1838 defm vfrec7 : RVVOutBuiltinSet<"vfrec7", "xfd", [["v", "v", "vvu"]]>; 1839 } 1840 // 13.8. Vector Floating-Point Square-Root Instruction 1841 defm vfsqrt : RVVOutBuiltinSet<"vfsqrt", "xfd", [["v", "v", "vv"]]>; 1842 1843 // 13.10. Vector Floating-Point Reciprocal Estimate Instruction 1844 defm vfrec7 : RVVOutBuiltinSet<"vfrec7", "xfd", [["v", "v", "vv"]]>; 1845} 1846 1847// 13.9. Vector Floating-Point Reciprocal Square-Root Estimate Instruction 1848def vfrsqrt7 : RVVFloatingUnaryVVBuiltin; 1849 1850// 13.11. Vector Floating-Point MIN/MAX Instructions 1851defm vfmin : RVVFloatingBinBuiltinSet; 1852defm vfmax : RVVFloatingBinBuiltinSet; 1853 1854// 13.12. Vector Floating-Point Sign-Injection Instructions 1855defm vfsgnj : RVVFloatingBinBuiltinSet; 1856defm vfsgnjn : RVVFloatingBinBuiltinSet; 1857defm vfsgnjx : RVVFloatingBinBuiltinSet; 1858} 1859defm vfneg_v : RVVPseudoVFUnaryBuiltin<"vfsgnjn", "xfd">; 1860defm vfabs_v : RVVPseudoVFUnaryBuiltin<"vfsgnjx", "xfd">; 1861 1862// 13.13. Vector Floating-Point Compare Instructions 1863let MaskedPolicyScheme = HasPassthruOperand, 1864 HasTailPolicy = false in { 1865defm vmfeq : RVVFloatingMaskOutBuiltinSet; 1866defm vmfne : RVVFloatingMaskOutBuiltinSet; 1867defm vmflt : RVVFloatingMaskOutBuiltinSet; 1868defm vmfle : RVVFloatingMaskOutBuiltinSet; 1869defm vmfgt : RVVFloatingMaskOutBuiltinSet; 1870defm vmfge : RVVFloatingMaskOutBuiltinSet; 1871} 1872 1873// 13.14. Vector Floating-Point Classify Instruction 1874let Name = "vfclass_v", UnMaskedPolicyScheme = HasPassthruOperand in 1875 def vfclass : RVVOp0Builtin<"Uv", "Uvv", "xfd">; 1876 1877// 13.15. Vector Floating-Point Merge Instruction 1878// C/C++ Operand: (mask, op1, op2, vl), Builtin: (op1, op2, mask, vl) 1879let HasMasked = false, 1880 UnMaskedPolicyScheme = HasPassthruOperand, 1881 MaskedPolicyScheme = NonePolicy, 1882 ManualCodegen = [{ 1883 // insert poison passthru 1884 if (PolicyAttrs & RVV_VTA) 1885 Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); 1886 IntrinsicTypes = {ResultType, Ops[2]->getType(), Ops.back()->getType()}; 1887 }] in { 1888 defm vmerge : RVVOutOp1BuiltinSet<"vmerge", "fd", 1889 [["vvm", "v", "vvvm"]]>; 1890 let RequiredFeatures = ["Zvfhmin"] in 1891 defm vmerge : RVVOutOp1BuiltinSet<"vmerge", "x", 1892 [["vvm", "v", "vvvm"]]>; 1893 defm vfmerge : RVVOutOp1BuiltinSet<"vfmerge", "xfd", 1894 [["vfm", "v", "vvem"]]>; 1895} 1896 1897// 13.16. Vector Floating-Point Move Instruction 1898let HasMasked = false, 1899 UnMaskedPolicyScheme = HasPassthruOperand, 1900 SupportOverloading = false, 1901 MaskedPolicyScheme = NonePolicy, 1902 OverloadedName = "vfmv_v" in 1903 defm vfmv_v : RVVOutBuiltinSet<"vfmv_v_f", "xfd", 1904 [["f", "v", "ve"]]>; 1905 1906// 13.17. Single-Width Floating-Point/Integer Type-Convert Instructions 1907let UnMaskedPolicyScheme = HasPassthruOperand in { 1908def vfcvt_rtz_xu_f_v : RVVConvToUnsignedBuiltin<"vfcvt_rtz_xu">; 1909def vfcvt_rtz_x_f_v : RVVConvToSignedBuiltin<"vfcvt_rtz_x">; 1910 1911// 13.18. Widening Floating-Point/Integer Type-Convert Instructions 1912let Log2LMUL = [-3, -2, -1, 0, 1, 2] in { 1913 def vfwcvt_rtz_xu_f_v : RVVConvToWidenUnsignedBuiltin<"vfwcvt_rtz_xu">; 1914 def vfwcvt_rtz_x_f_v : RVVConvToWidenSignedBuiltin<"vfwcvt_rtz_x">; 1915 def vfwcvt_f_xu_v : RVVConvBuiltin<"Fw", "FwUv", "csi", "vfwcvt_f">; 1916 def vfwcvt_f_x_v : RVVConvBuiltin<"Fw", "Fwv", "csi", "vfwcvt_f">; 1917 def vfwcvt_f_f_v : RVVConvBuiltin<"w", "wv", "f", "vfwcvt_f">; 1918 let RequiredFeatures = ["Zvfhmin"] in 1919 def vfwcvt_f_f_v_fp16 : RVVConvBuiltin<"w", "wv", "x", "vfwcvt_f"> { 1920 let Name = "vfwcvt_f_f_v"; 1921 let IRName = "vfwcvt_f_f_v"; 1922 let MaskedIRName = "vfwcvt_f_f_v_mask"; 1923 } 1924} 1925 1926// 13.19. Narrowing Floating-Point/Integer Type-Convert Instructions 1927let Log2LMUL = [-3, -2, -1, 0, 1, 2] in { 1928 def vfncvt_rtz_xu_f_w : RVVConvToNarrowingUnsignedBuiltin<"vfncvt_rtz_xu">; 1929 def vfncvt_rtz_x_f_w : RVVConvToNarrowingSignedBuiltin<"vfncvt_rtz_x">; 1930 def vfncvt_rod_f_f_w : RVVConvBuiltin<"v", "vw", "xf", "vfncvt_rod_f">; 1931} 1932 1933// Zvfbfmin - Vector convert BF16 to FP32 1934let Log2LMUL = [-2, -1, 0, 1, 2] in 1935def vfwcvtbf16_f_f_v : RVVConvBuiltin<"Fw", "Fwv", "y", "vfwcvtbf16_f">; 1936 1937let ManualCodegen = [{ 1938 { 1939 // LLVM intrinsic 1940 // Unmasked: (passthru, op0, frm, vl) 1941 // Masked: (passthru, op0, mask, frm, vl, policy) 1942 SmallVector<llvm::Value*, 7> Operands; 1943 bool HasMaskedOff = !( 1944 (IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) || 1945 (!IsMasked && PolicyAttrs & RVV_VTA)); 1946 bool HasRoundModeOp = IsMasked ? 1947 (HasMaskedOff ? Ops.size() == 5 : Ops.size() == 4) : 1948 (HasMaskedOff ? Ops.size() == 4 : Ops.size() == 3); 1949 1950 unsigned Offset = IsMasked ? 1951 (HasMaskedOff ? 2 : 1) : (HasMaskedOff ? 1 : 0); 1952 1953 if (!HasMaskedOff) 1954 Operands.push_back(llvm::PoisonValue::get(ResultType)); 1955 else 1956 Operands.push_back(Ops[IsMasked ? 1 : 0]); 1957 1958 Operands.push_back(Ops[Offset]); // op0 1959 1960 if (IsMasked) 1961 Operands.push_back(Ops[0]); // mask 1962 1963 if (HasRoundModeOp) { 1964 Operands.push_back(Ops[Offset + 1]); // frm 1965 Operands.push_back(Ops[Offset + 2]); // vl 1966 } else { 1967 Operands.push_back(ConstantInt::get(Ops[Offset + 1]->getType(), 7)); // frm 1968 Operands.push_back(Ops[Offset + 1]); // vl 1969 } 1970 1971 if (IsMasked) 1972 Operands.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); 1973 1974 IntrinsicTypes = {ResultType, Ops[Offset]->getType(), 1975 Operands.back()->getType()}; 1976 llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); 1977 return Builder.CreateCall(F, Operands, ""); 1978 } 1979}] in { 1980 let HasFRMRoundModeOp = 1 in { 1981 // 14.17. Single-Width Floating-Point/Integer Type-Convert Instructions 1982 let OverloadedName = "vfcvt_x" in 1983 defm : 1984 RVVConvBuiltinSet<"vfcvt_x_f_v", "xfd", [["Iv", "Ivvu"]]>; 1985 let OverloadedName = "vfcvt_xu" in 1986 defm : 1987 RVVConvBuiltinSet<"vfcvt_xu_f_v", "xfd", [["Uv", "Uvvu"]]>; 1988 let OverloadedName = "vfcvt_f" in { 1989 defm : 1990 RVVConvBuiltinSet<"vfcvt_f_x_v", "sil", [["Fv", "Fvvu"]]>; 1991 defm : 1992 RVVConvBuiltinSet<"vfcvt_f_xu_v", "sil", [["Fv", "FvUvu"]]>; 1993 } 1994 1995 // 13.18. Widening Floating-Point/Integer Type-Convert Instructions 1996 let Log2LMUL = [-3, -2, -1, 0, 1, 2] in { 1997 let OverloadedName = "vfwcvt_x" in 1998 defm : 1999 RVVConvBuiltinSet<"vfwcvt_x_f_v", "xf", [["Iw", "Iwvu"]]>; 2000 let OverloadedName = "vfwcvt_xu" in 2001 defm : 2002 RVVConvBuiltinSet<"vfwcvt_xu_f_v", "xf", [["Uw", "Uwvu"]]>; 2003 } 2004 // 13.19. Narrowing Floating-Point/Integer Type-Convert Instructions 2005 let Log2LMUL = [-3, -2, -1, 0, 1, 2] in { 2006 let OverloadedName = "vfncvt_x" in 2007 defm : 2008 RVVConvBuiltinSet<"vfncvt_x_f_w", "csi", [["Iv", "IvFwu"]]>; 2009 let OverloadedName = "vfncvt_xu" in 2010 defm : 2011 RVVConvBuiltinSet<"vfncvt_xu_f_w", "csi", [["Uv", "UvFwu"]]>; 2012 let OverloadedName = "vfncvt_f" in { 2013 defm : 2014 RVVConvBuiltinSet<"vfncvt_f_x_w", "csi", [["Fv", "Fvwu"]]>; 2015 defm : 2016 RVVConvBuiltinSet<"vfncvt_f_xu_w", "csi", [["Fv", "FvUwu"]]>; 2017 } 2018 let OverloadedName = "vfncvt_f" in { 2019 defm : RVVConvBuiltinSet<"vfncvt_f_f_w", "f", [["v", "vwu"]]>; 2020 let RequiredFeatures = ["Zvfhmin"] in 2021 defm : RVVConvBuiltinSet<"vfncvt_f_f_w", "x", [["v", "vwu"]]>; 2022 } 2023 } 2024 2025 // Zvfbfmin - Vector convert FP32 to BF16 2026 let Log2LMUL = [-2, -1, 0, 1, 2], 2027 OverloadedName = "vfncvtbf16_f" in 2028 defm : RVVConvBuiltinSet<"vfncvtbf16_f_f_w", "y", [["v", "vFwu"]]>; 2029 } 2030 2031 // 13.17. Single-Width Floating-Point/Integer Type-Convert Instructions 2032 let OverloadedName = "vfcvt_x" in 2033 defm : 2034 RVVConvBuiltinSet<"vfcvt_x_f_v", "xfd", [["Iv", "Ivv"]]>; 2035 let OverloadedName = "vfcvt_xu" in 2036 defm : 2037 RVVConvBuiltinSet<"vfcvt_xu_f_v", "xfd", [["Uv", "Uvv"]]>; 2038 let OverloadedName = "vfcvt_f" in { 2039 defm : 2040 RVVConvBuiltinSet<"vfcvt_f_x_v", "sil", [["Fv", "Fvv"]]>; 2041 defm : 2042 RVVConvBuiltinSet<"vfcvt_f_xu_v", "sil", [["Fv", "FvUv"]]>; 2043 } 2044 2045 // 13.18. Widening Floating-Point/Integer Type-Convert Instructions 2046 let Log2LMUL = [-3, -2, -1, 0, 1, 2] in { 2047 let OverloadedName = "vfwcvt_x" in 2048 defm : 2049 RVVConvBuiltinSet<"vfwcvt_x_f_v", "xf", [["Iw", "Iwv"]]>; 2050 let OverloadedName = "vfwcvt_xu" in 2051 defm : 2052 RVVConvBuiltinSet<"vfwcvt_xu_f_v", "xf", [["Uw", "Uwv"]]>; 2053 } 2054 // 13.19. Narrowing Floating-Point/Integer Type-Convert Instructions 2055 let Log2LMUL = [-3, -2, -1, 0, 1, 2] in { 2056 let OverloadedName = "vfncvt_x" in 2057 defm : 2058 RVVConvBuiltinSet<"vfncvt_x_f_w", "csi", [["Iv", "IvFw"]]>; 2059 let OverloadedName = "vfncvt_xu" in 2060 defm : 2061 RVVConvBuiltinSet<"vfncvt_xu_f_w", "csi", [["Uv", "UvFw"]]>; 2062 let OverloadedName = "vfncvt_f" in { 2063 defm : 2064 RVVConvBuiltinSet<"vfncvt_f_x_w", "csi", [["Fv", "Fvw"]]>; 2065 defm : 2066 RVVConvBuiltinSet<"vfncvt_f_xu_w", "csi", [["Fv", "FvUw"]]>; 2067 } 2068 let OverloadedName = "vfncvt_f" in { 2069 defm : RVVConvBuiltinSet<"vfncvt_f_f_w", "f", [["v", "vw"]]>; 2070 let RequiredFeatures = ["Zvfhmin"] in 2071 defm : RVVConvBuiltinSet<"vfncvt_f_f_w", "x", [["v", "vw"]]>; 2072 } 2073 } 2074 2075 // Zvfbfmin - Vector convert FP32 to BF16 2076 let Log2LMUL = [-2, -1, 0, 1, 2], 2077 OverloadedName = "vfncvtbf16_f" in 2078 defm : RVVConvBuiltinSet<"vfncvtbf16_f_f_w", "y", [["v", "vFw"]]>; 2079} 2080} 2081 2082// 14. Vector Reduction Operations 2083// 14.1. Vector Single-Width Integer Reduction Instructions 2084let UnMaskedPolicyScheme = HasPassthruOperand, 2085 MaskedPolicyScheme = HasPassthruOperand, 2086 HasMaskPolicy = false in { 2087defm vredsum : RVVIntReductionBuiltinSet; 2088defm vredmaxu : RVVUnsignedReductionBuiltin; 2089defm vredmax : RVVSignedReductionBuiltin; 2090defm vredminu : RVVUnsignedReductionBuiltin; 2091defm vredmin : RVVSignedReductionBuiltin; 2092defm vredand : RVVIntReductionBuiltinSet; 2093defm vredor : RVVIntReductionBuiltinSet; 2094defm vredxor : RVVIntReductionBuiltinSet; 2095 2096// 14.2. Vector Widening Integer Reduction Instructions 2097// Vector Widening Integer Reduction Operations 2098let HasMaskedOffOperand = true in { 2099 defm vwredsum : RVVOutOp0BuiltinSet<"vwredsum", "csi", 2100 [["vs", "vSw", "SwvSw"]]>; 2101 defm vwredsumu : RVVOutOp0BuiltinSet<"vwredsumu", "csi", 2102 [["vs", "UvUSw", "USwUvUSw"]]>; 2103} 2104 2105// 14.3. Vector Single-Width Floating-Point Reduction Instructions 2106defm vfredmax : RVVFloatingReductionBuiltin; 2107defm vfredmin : RVVFloatingReductionBuiltin; 2108let ManualCodegen = [{ 2109 { 2110 // LLVM intrinsic 2111 // Unmasked: (passthru, op0, op1, round_mode, vl) 2112 // Masked: (passthru, vector_in, vector_in/scalar_in, mask, frm, vl, policy) 2113 2114 SmallVector<llvm::Value*, 7> Operands; 2115 bool HasMaskedOff = !( 2116 (IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) || 2117 (!IsMasked && PolicyAttrs & RVV_VTA)); 2118 bool HasRoundModeOp = IsMasked ? 2119 (HasMaskedOff ? Ops.size() == 6 : Ops.size() == 5) : 2120 (HasMaskedOff ? Ops.size() == 5 : Ops.size() == 4); 2121 2122 unsigned Offset = IsMasked ? 2123 (HasMaskedOff ? 2 : 1) : (HasMaskedOff ? 1 : 0); 2124 2125 if (!HasMaskedOff) 2126 Operands.push_back(llvm::PoisonValue::get(ResultType)); 2127 else 2128 Operands.push_back(Ops[IsMasked ? 1 : 0]); 2129 2130 Operands.push_back(Ops[Offset]); // op0 2131 Operands.push_back(Ops[Offset + 1]); // op1 2132 2133 if (IsMasked) 2134 Operands.push_back(Ops[0]); // mask 2135 2136 if (HasRoundModeOp) { 2137 Operands.push_back(Ops[Offset + 2]); // frm 2138 Operands.push_back(Ops[Offset + 3]); // vl 2139 } else { 2140 Operands.push_back(ConstantInt::get(Ops[Offset + 2]->getType(), 7)); // frm 2141 Operands.push_back(Ops[Offset + 2]); // vl 2142 } 2143 2144 IntrinsicTypes = {ResultType, Ops[Offset]->getType(), 2145 Ops.back()->getType()}; 2146 llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); 2147 return Builder.CreateCall(F, Operands, ""); 2148 } 2149}] in { 2150 let HasFRMRoundModeOp = 1 in { 2151 // 14.3. Vector Single-Width Floating-Point Reduction Instructions 2152 defm vfredusum : RVVFloatingReductionBuiltinRoundingMode; 2153 defm vfredosum : RVVFloatingReductionBuiltinRoundingMode; 2154 2155 // 14.4. Vector Widening Floating-Point Reduction Instructions 2156 defm vfwredusum : RVVFloatingWidenReductionBuiltinRoundingMode; 2157 defm vfwredosum : RVVFloatingWidenReductionBuiltinRoundingMode; 2158 } 2159 // 14.3. Vector Single-Width Floating-Point Reduction Instructions 2160 defm vfredusum : RVVFloatingReductionBuiltin; 2161 defm vfredosum : RVVFloatingReductionBuiltin; 2162 2163 // 14.4. Vector Widening Floating-Point Reduction Instructions 2164 defm vfwredusum : RVVFloatingWidenReductionBuiltin; 2165 defm vfwredosum : RVVFloatingWidenReductionBuiltin; 2166} 2167} 2168 2169// 15. Vector Mask Instructions 2170// 15.1. Vector Mask-Register Logical Instructions 2171def vmand : RVVMaskBinBuiltin; 2172def vmnand : RVVMaskBinBuiltin; 2173def vmandn : RVVMaskBinBuiltin; 2174def vmxor : RVVMaskBinBuiltin; 2175def vmor : RVVMaskBinBuiltin; 2176def vmnor : RVVMaskBinBuiltin; 2177def vmorn : RVVMaskBinBuiltin; 2178def vmxnor : RVVMaskBinBuiltin; 2179// pseudoinstructions 2180def vmclr : RVVMaskNullaryBuiltin; 2181def vmset : RVVMaskNullaryBuiltin; 2182defm vmmv_m : RVVPseudoMaskBuiltin<"vmand", "c">; 2183defm vmnot_m : RVVPseudoMaskBuiltin<"vmnand", "c">; 2184 2185let MaskedPolicyScheme = NonePolicy in { 2186// 15.2. Vector count population in mask vcpop.m 2187def vcpop : RVVMaskOp0Builtin<"um">; 2188 2189// 15.3. vfirst find-first-set mask bit 2190def vfirst : RVVMaskOp0Builtin<"lm">; 2191} 2192 2193let MaskedPolicyScheme = HasPassthruOperand, 2194 HasTailPolicy = false in { 2195// 15.4. vmsbf.m set-before-first mask bit 2196def vmsbf : RVVMaskUnaryBuiltin; 2197 2198// 15.5. vmsif.m set-including-first mask bit 2199def vmsif : RVVMaskUnaryBuiltin; 2200 2201// 15.6. vmsof.m set-only-first mask bit 2202def vmsof : RVVMaskUnaryBuiltin; 2203} 2204 2205let UnMaskedPolicyScheme = HasPassthruOperand, SupportOverloading = false in { 2206 // 15.8. Vector Iota Instruction 2207 defm viota : RVVOutBuiltinSet<"viota", "csil", [["m", "Uv", "Uvm"]]>; 2208 2209 // 15.9. Vector Element Index Instruction 2210 defm vid : RVVOutBuiltinSet<"vid", "csil", [["v", "v", "v"], 2211 ["v", "Uv", "Uv"]]>; 2212} 2213 2214// 16. Vector Permutation Instructions 2215// 16.1. Integer Scalar Move Instructions 2216let HasMasked = false, MaskedPolicyScheme = NonePolicy in { 2217 let HasVL = false, OverloadedName = "vmv_x" in 2218 defm vmv_x : RVVOp0BuiltinSet<"vmv_x_s", "csil", 2219 [["s", "ve", "ev"], 2220 ["s", "UvUe", "UeUv"]]>; 2221 let OverloadedName = "vmv_s", 2222 UnMaskedPolicyScheme = HasPassthruOperand, 2223 SupportOverloading = false in 2224 defm vmv_s : RVVOutBuiltinSet<"vmv_s_x", "csil", 2225 [["x", "v", "ve"], 2226 ["x", "Uv", "UvUe"]]>; 2227} 2228 2229// 16.2. Floating-Point Scalar Move Instructions 2230let HasMasked = false, MaskedPolicyScheme = NonePolicy in { 2231 let HasVL = false, OverloadedName = "vfmv_f" in 2232 defm vfmv_f : RVVOp0BuiltinSet<"vfmv_f_s", "xfd", 2233 [["s", "ve", "ev"]]>; 2234 let OverloadedName = "vfmv_s", 2235 UnMaskedPolicyScheme = HasPassthruOperand, 2236 SupportOverloading = false in 2237 defm vfmv_s : RVVOutBuiltinSet<"vfmv_s_f", "xfd", 2238 [["f", "v", "ve"], 2239 ["x", "Uv", "UvUe"]]>; 2240} 2241 2242// 16.3. Vector Slide Instructions 2243// 16.3.1. Vector Slideup Instructions 2244defm vslideup : RVVSlideUpBuiltinSet; 2245// 16.3.2. Vector Slidedown Instructions 2246defm vslidedown : RVVSlideDownBuiltinSet; 2247 2248// 16.3.3. Vector Slide1up Instructions 2249let UnMaskedPolicyScheme = HasPassthruOperand in { 2250defm vslide1up : RVVSlideOneBuiltinSet; 2251defm vfslide1up : RVVFloatingBinVFBuiltinSet; 2252 2253// 16.3.4. Vector Slide1down Instruction 2254defm vslide1down : RVVSlideOneBuiltinSet; 2255defm vfslide1down : RVVFloatingBinVFBuiltinSet; 2256 2257// 16.4. Vector Register Gather Instructions 2258// signed and floating type 2259defm vrgather : RVVOutBuiltinSet<"vrgather_vv", "csilxfd", 2260 [["vv", "v", "vvUv"]]>; 2261defm vrgather : RVVOutBuiltinSet<"vrgather_vx", "csilxfd", 2262 [["vx", "v", "vvz"]]>; 2263defm vrgatherei16 : RVVOutBuiltinSet<"vrgatherei16_vv", "csilxfd", 2264 [["vv", "v", "vv(Log2EEW:4)Uv"]]>; 2265// unsigned type 2266defm vrgather : RVVOutBuiltinSet<"vrgather_vv", "csil", 2267 [["vv", "Uv", "UvUvUv"]]>; 2268defm vrgather : RVVOutBuiltinSet<"vrgather_vx", "csil", 2269 [["vx", "Uv", "UvUvz"]]>; 2270defm vrgatherei16 : RVVOutBuiltinSet<"vrgatherei16_vv", "csil", 2271 [["vv", "Uv", "UvUv(Log2EEW:4)Uv"]]>; 2272} 2273 2274// 16.5. Vector Compress Instruction 2275let HasMasked = false, 2276 UnMaskedPolicyScheme = HasPassthruOperand, 2277 MaskedPolicyScheme = NonePolicy, 2278 ManualCodegen = [{ 2279 // insert poison passthru 2280 if (PolicyAttrs & RVV_VTA) 2281 Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); 2282 IntrinsicTypes = {ResultType, Ops.back()->getType()}; 2283 }] in { 2284 // signed and floating type 2285 defm vcompress : RVVOutBuiltinSet<"vcompress", "csilxfd", 2286 [["vm", "v", "vvm"]]>; 2287 // unsigned type 2288 defm vcompress : RVVOutBuiltinSet<"vcompress", "csil", 2289 [["vm", "Uv", "UvUvm"]]>; 2290} 2291 2292// Miscellaneous 2293let HasMasked = false, HasVL = false, IRName = "" in { 2294 let Name = "vreinterpret_v", MaskedPolicyScheme = NonePolicy, 2295 ManualCodegen = [{ 2296 if (ResultType->isIntOrIntVectorTy(1) || 2297 Ops[0]->getType()->isIntOrIntVectorTy(1)) { 2298 assert(isa<ScalableVectorType>(ResultType) && 2299 isa<ScalableVectorType>(Ops[0]->getType())); 2300 2301 LLVMContext &Context = CGM.getLLVMContext(); 2302 ScalableVectorType *Boolean64Ty = 2303 ScalableVectorType::get(llvm::Type::getInt1Ty(Context), 64); 2304 2305 if (ResultType->isIntOrIntVectorTy(1)) { 2306 // Casting from m1 vector integer -> vector boolean 2307 // Ex: <vscale x 8 x i8> 2308 // --(bitcast)--------> <vscale x 64 x i1> 2309 // --(vector_extract)-> <vscale x 8 x i1> 2310 llvm::Value *BitCast = Builder.CreateBitCast(Ops[0], Boolean64Ty); 2311 return Builder.CreateExtractVector(ResultType, BitCast, 2312 ConstantInt::get(Int64Ty, 0)); 2313 } else { 2314 // Casting from vector boolean -> m1 vector integer 2315 // Ex: <vscale x 1 x i1> 2316 // --(vector_insert)-> <vscale x 64 x i1> 2317 // --(bitcast)-------> <vscale x 8 x i8> 2318 llvm::Value *Boolean64Val = 2319 Builder.CreateInsertVector(Boolean64Ty, 2320 llvm::PoisonValue::get(Boolean64Ty), 2321 Ops[0], 2322 ConstantInt::get(Int64Ty, 0)); 2323 return Builder.CreateBitCast(Boolean64Val, ResultType); 2324 } 2325 } 2326 return Builder.CreateBitCast(Ops[0], ResultType); 2327 }] in { 2328 // Reinterpret between different type under the same SEW and LMUL 2329 def vreinterpret_i_u : RVVBuiltin<"Uvv", "vUv", "csil", "v">; 2330 def vreinterpret_i_f : RVVBuiltin<"Fvv", "vFv", "il", "v">; 2331 def vreinterpret_u_i : RVVBuiltin<"vUv", "Uvv", "csil", "Uv">; 2332 def vreinterpret_u_f : RVVBuiltin<"FvUv", "UvFv", "il", "Uv">; 2333 def vreinterpret_f_i : RVVBuiltin<"vFv", "Fvv", "il", "Fv">; 2334 def vreinterpret_f_u : RVVBuiltin<"UvFv", "FvUv", "il", "Fv">; 2335 let RequiredFeatures = ["Zvfhmin"] in { 2336 def vreinterpret_i_h : RVVBuiltin<"Fvv", "vFv", "s", "v">; 2337 def vreinterpret_u_h : RVVBuiltin<"FvUv", "UvFv", "s", "Uv">; 2338 def vreinterpret_h_i : RVVBuiltin<"vFv", "Fvv", "s", "Fv">; 2339 def vreinterpret_h_u : RVVBuiltin<"UvFv", "FvUv", "s", "Fv">; 2340 } 2341 let RequiredFeatures = ["Zvfbfmin"] in { 2342 def vreinterpret_i_bf16 : RVVBuiltin<"vIv", "Ivv", "y", "Iv">; 2343 def vreinterpret_u_bf16 : RVVBuiltin<"vUv", "Uvv", "y", "Uv">; 2344 def vreinterpret_bf16_i : RVVBuiltin<"Ivv", "vIv", "y", "v">; 2345 def vreinterpret_bf16_u : RVVBuiltin<"Uvv", "vUv", "y", "v">; 2346 } 2347 2348 // Reinterpret between different SEW under the same LMUL 2349 foreach dst_sew = ["(FixedSEW:8)", "(FixedSEW:16)", "(FixedSEW:32)", 2350 "(FixedSEW:64)"] in { 2351 def vreinterpret_i_ # dst_sew : RVVBuiltin<"v" # dst_sew # "v", 2352 dst_sew # "vv", "csil", dst_sew # "v">; 2353 def vreinterpret_u_ # dst_sew : RVVBuiltin<"Uv" # dst_sew # "Uv", 2354 dst_sew # "UvUv", "csil", dst_sew # "Uv">; 2355 } 2356 2357 // Existing users of FixedSEW - the reinterpretation between different SEW 2358 // and same LMUL has the implicit assumption that if FixedSEW is set to the 2359 // given element width, then the type will be identified as invalid, thus 2360 // skipping definition of reinterpret of SEW=8 to SEW=8. However this blocks 2361 // our usage here of defining all possible combinations of a fixed SEW to 2362 // any boolean. So we need to separately define SEW=8 here. 2363 // Reinterpret from LMUL=1 integer type to vector boolean type 2364 def vreintrepret_m1_b8_signed : 2365 RVVBuiltin<"Svm", 2366 "mSv", 2367 "c", "m">; 2368 def vreintrepret_m1_b8_usigned : 2369 RVVBuiltin<"USvm", 2370 "mUSv", 2371 "c", "m">; 2372 2373 // Reinterpret from vector boolean type to LMUL=1 integer type 2374 def vreintrepret_b8_m1_signed : 2375 RVVBuiltin<"mSv", 2376 "Svm", 2377 "c", "Sv">; 2378 def vreintrepret_b8_m1_usigned : 2379 RVVBuiltin<"mUSv", 2380 "USvm", 2381 "c", "USv">; 2382 2383 foreach dst_sew = ["16", "32", "64"] in { 2384 // Reinterpret from LMUL=1 integer type to vector boolean type 2385 def vreinterpret_m1_b # dst_sew # _signed: 2386 RVVBuiltin<"(FixedSEW:" # dst_sew # ")Svm", 2387 "m(FixedSEW:" # dst_sew # ")Sv", 2388 "c", "m">; 2389 def vreinterpret_m1_b # dst_sew # _unsigned: 2390 RVVBuiltin<"(FixedSEW:" # dst_sew # ")USvm", 2391 "m(FixedSEW:" # dst_sew # ")USv", 2392 "c", "m">; 2393 // Reinterpret from vector boolean type to LMUL=1 integer type 2394 def vreinterpret_b # dst_sew # _m1_signed: 2395 RVVBuiltin<"m(FixedSEW:" # dst_sew # ")Sv", 2396 "(FixedSEW:" # dst_sew # ")Svm", 2397 "c", "(FixedSEW:" # dst_sew # ")Sv">; 2398 def vreinterpret_b # dst_sew # _m1_unsigned: 2399 RVVBuiltin<"m(FixedSEW:" # dst_sew # ")USv", 2400 "(FixedSEW:" # dst_sew # ")USvm", 2401 "c", "(FixedSEW:" # dst_sew # ")USv">; 2402 } 2403 } 2404 2405 let Name = "vundefined", SupportOverloading = false, 2406 MaskedPolicyScheme = NonePolicy, 2407 ManualCodegen = [{ 2408 return llvm::PoisonValue::get(ResultType); 2409 }] in { 2410 def vundefined : RVVBuiltin<"v", "v", "csilxfd">; 2411 let RequiredFeatures = ["Zvfbfmin"] in 2412 def vundefined_bf16 : RVVBuiltin<"v", "v", "y">; 2413 def vundefined_u : RVVBuiltin<"Uv", "Uv", "csil">; 2414 2415 foreach nf = NFList in { 2416 let NF = nf in { 2417 defvar T = "(Tuple:" # nf # ")"; 2418 def : RVVBuiltin<T # "v", T # "v", "csilxfd">; 2419 let RequiredFeatures = ["Zvfbfmin"] in 2420 def : RVVBuiltin<T # "v", T # "v", "y">; 2421 def : RVVBuiltin<T # "Uv", T # "Uv", "csil">; 2422 } 2423 } 2424 2425 } 2426 2427 // LMUL truncation 2428 // C/C++ Operand: VecTy, IR Operand: VecTy, Index 2429 let Name = "vlmul_trunc_v", OverloadedName = "vlmul_trunc", 2430 MaskedPolicyScheme = NonePolicy, 2431 ManualCodegen = [{ { 2432 return Builder.CreateExtractVector(ResultType, Ops[0], 2433 ConstantInt::get(Int64Ty, 0)); 2434 } }] in { 2435 foreach dst_lmul = ["(SFixedLog2LMUL:-3)", "(SFixedLog2LMUL:-2)", "(SFixedLog2LMUL:-1)", 2436 "(SFixedLog2LMUL:0)", "(SFixedLog2LMUL:1)", "(SFixedLog2LMUL:2)"] in { 2437 def vlmul_trunc # dst_lmul : RVVBuiltin<"v" # dst_lmul # "v", 2438 dst_lmul # "vv", "csilxfd", dst_lmul # "v">; 2439 let RequiredFeatures = ["Zvfbfmin"] in 2440 def vlmul_trunc_bf16 # dst_lmul : RVVBuiltin<"v" # dst_lmul # "v", 2441 dst_lmul # "vv", "y", dst_lmul # "v">; 2442 def vlmul_trunc_u # dst_lmul : RVVBuiltin<"Uv" # dst_lmul # "Uv", 2443 dst_lmul # "UvUv", "csil", dst_lmul # "Uv">; 2444 } 2445 } 2446 2447 // LMUL extension 2448 // C/C++ Operand: SubVecTy, IR Operand: VecTy, SubVecTy, Index 2449 let Name = "vlmul_ext_v", OverloadedName = "vlmul_ext", 2450 MaskedPolicyScheme = NonePolicy, 2451 ManualCodegen = [{ 2452 return Builder.CreateInsertVector(ResultType, 2453 llvm::PoisonValue::get(ResultType), 2454 Ops[0], ConstantInt::get(Int64Ty, 0)); 2455 }] in { 2456 foreach dst_lmul = ["(LFixedLog2LMUL:-2)", "(LFixedLog2LMUL:-1)", "(LFixedLog2LMUL:-0)", 2457 "(LFixedLog2LMUL:1)", "(LFixedLog2LMUL:2)", "(LFixedLog2LMUL:3)"] in { 2458 def vlmul_ext # dst_lmul : RVVBuiltin<"v" # dst_lmul # "v", 2459 dst_lmul # "vv", "csilxfd", dst_lmul # "v">; 2460 let RequiredFeatures = ["Zvfbfmin"] in 2461 def vlmul_ext_bf16 # dst_lmul : RVVBuiltin<"v" # dst_lmul # "v", 2462 dst_lmul # "vv", "y", dst_lmul # "v">; 2463 def vlmul_ext_u # dst_lmul : RVVBuiltin<"Uv" # dst_lmul # "Uv", 2464 dst_lmul # "UvUv", "csil", dst_lmul # "Uv">; 2465 } 2466 } 2467 2468 let Name = "vget_v", MaskedPolicyScheme = NonePolicy, 2469 ManualCodegen = [{ 2470 { 2471 if (isa<StructType>(Ops[0]->getType())) // For tuple type 2472 // Extract value from index (operand 1) of vtuple (operand 0) 2473 return Builder.CreateExtractValue( 2474 Ops[0], 2475 {(unsigned)cast<ConstantInt>(Ops[1])->getZExtValue()}); 2476 auto *VecTy = cast<ScalableVectorType>(ResultType); 2477 auto *OpVecTy = cast<ScalableVectorType>(Ops[0]->getType()); 2478 // Mask to only valid indices. 2479 unsigned MaxIndex = OpVecTy->getMinNumElements() / VecTy->getMinNumElements(); 2480 assert(isPowerOf2_32(MaxIndex)); 2481 Ops[1] = Builder.CreateZExt(Ops[1], Builder.getInt64Ty()); 2482 Ops[1] = Builder.CreateAnd(Ops[1], MaxIndex - 1); 2483 Ops[1] = Builder.CreateMul(Ops[1], 2484 ConstantInt::get(Ops[1]->getType(), 2485 VecTy->getMinNumElements())); 2486 return Builder.CreateExtractVector(ResultType, Ops[0], Ops[1]); 2487 } 2488 }] in { 2489 foreach dst_lmul = ["(SFixedLog2LMUL:0)", "(SFixedLog2LMUL:1)", "(SFixedLog2LMUL:2)"] in { 2490 def : RVVBuiltin<"v" # dst_lmul # "v", dst_lmul # "vvKz", "csilxfdy", dst_lmul # "v">; 2491 def : RVVBuiltin<"Uv" # dst_lmul # "Uv", dst_lmul # "UvUvKz", "csil", dst_lmul # "Uv">; 2492 } 2493 foreach nf = NFList in { 2494 defvar T = "(Tuple:" # nf # ")"; 2495 def : RVVBuiltin<T # "vv", "v" # T # "vKz", "csilxfdy", "v">; 2496 def : RVVBuiltin<T # "UvUv", "Uv" # T # "UvKz", "csil", "Uv">; 2497 } 2498 } 2499 2500 let Name = "vset_v", MaskedPolicyScheme = NonePolicy, 2501 ManualCodegen = [{ 2502 { 2503 if (isa<StructType>(ResultType)) // For tuple type 2504 // Insert value (operand 2) into index (operand 1) of vtuple (operand 0) 2505 return Builder.CreateInsertValue( 2506 Ops[0], Ops[2], 2507 {(unsigned)cast<ConstantInt>(Ops[1])->getZExtValue()}); 2508 auto *ResVecTy = cast<ScalableVectorType>(ResultType); 2509 auto *VecTy = cast<ScalableVectorType>(Ops[2]->getType()); 2510 // Mask to only valid indices. 2511 unsigned MaxIndex = ResVecTy->getMinNumElements() / VecTy->getMinNumElements(); 2512 assert(isPowerOf2_32(MaxIndex)); 2513 Ops[1] = Builder.CreateZExt(Ops[1], Builder.getInt64Ty()); 2514 Ops[1] = Builder.CreateAnd(Ops[1], MaxIndex - 1); 2515 Ops[1] = Builder.CreateMul(Ops[1], 2516 ConstantInt::get(Ops[1]->getType(), 2517 VecTy->getMinNumElements())); 2518 return Builder.CreateInsertVector(ResultType, Ops[0], Ops[2], Ops[1]); 2519 } 2520 }] in { 2521 foreach dst_lmul = ["(LFixedLog2LMUL:1)", "(LFixedLog2LMUL:2)", "(LFixedLog2LMUL:3)"] in { 2522 def : RVVBuiltin<"v" # dst_lmul # "v", dst_lmul # "v" # dst_lmul # "vKzv", "csilxfd">; 2523 let RequiredFeatures = ["Zvfbfmin"] in 2524 def : RVVBuiltin<"v" # dst_lmul # "v", dst_lmul # "v" # dst_lmul # "vKzv", "y">; 2525 def : RVVBuiltin<"Uv" # dst_lmul # "Uv", dst_lmul # "Uv" # dst_lmul #"UvKzUv", "csil">; 2526 } 2527 foreach nf = NFList in { 2528 defvar T = "(Tuple:" # nf # ")"; 2529 def : RVVBuiltin<"v" # T # "v", T # "v" # T # "vKzv", "csilxfd">; 2530 let RequiredFeatures = ["Zvfbfmin"] in 2531 def : RVVBuiltin<"v" # T # "v", T # "v" # T # "vKzv", "y">; 2532 def : RVVBuiltin<"Uv" # T # "Uv", T # "Uv" # T # "UvKzUv", "csil">; 2533 } 2534 } 2535 2536 let Name = "vcreate_v", 2537 UnMaskedPolicyScheme = NonePolicy, 2538 MaskedPolicyScheme = NonePolicy, 2539 SupportOverloading = false, 2540 ManualCodegen = [{ 2541 { 2542 if (isa<StructType>(ResultType)) { 2543 unsigned NF = cast<StructType>(ResultType)->getNumElements(); 2544 llvm::Value *ReturnTuple = llvm::PoisonValue::get(ResultType); 2545 for (unsigned I = 0; I < NF; ++I) { 2546 ReturnTuple = Builder.CreateInsertValue(ReturnTuple, Ops[I], {I}); 2547 } 2548 return ReturnTuple; 2549 } 2550 llvm::Value *ReturnVector = llvm::PoisonValue::get(ResultType); 2551 auto *VecTy = cast<ScalableVectorType>(Ops[0]->getType()); 2552 for (unsigned I = 0, N = Ops.size(); I < N; ++I) { 2553 llvm::Value *Idx = 2554 ConstantInt::get(Builder.getInt64Ty(), 2555 VecTy->getMinNumElements() * I); 2556 ReturnVector = 2557 Builder.CreateInsertVector(ResultType, ReturnVector, Ops[I], Idx); 2558 } 2559 return ReturnVector; 2560 } 2561 }] in { 2562 2563 // Since the vcreate_v uses LFixedLog2LMUL, setting the Log2LMUL to [-3] can 2564 // avoid creating the intrinsics which contain the same name and prototype. 2565 let Log2LMUL = [-3] in { 2566 defm : RVVNonTupleVCreateBuiltin<1, [0]>; 2567 defm : RVVNonTupleVCreateBuiltin<2, [0, 1]>; 2568 defm : RVVNonTupleVCreateBuiltin<3, [0, 1, 2]>; 2569 } 2570 2571 foreach nf = NFList in { 2572 let NF = nf in { 2573 defvar T = "(Tuple:" # nf # ")"; 2574 defvar V = VString<nf, /*signed=*/true>.S; 2575 defvar UV = VString<nf, /*signed=*/false>.S; 2576 def : RVVBuiltin<T # "v", T # "v" # V, "csilxfdy">; 2577 let RequiredFeatures = ["Zvfbfmin"] in 2578 def : RVVBuiltin<T # "v", T # "v" # V, "y">; 2579 def : RVVBuiltin<T # "Uv", T # "Uv" # UV, "csil">; 2580 } 2581 } 2582 } 2583} 2584 2585multiclass RVVOutBuiltinSetZvbb { 2586 let OverloadedName = NAME in 2587 defm "" : RVVOutBuiltinSet<NAME, "csil", [["v", "v", "vv"], 2588 ["v", "Uv", "UvUv"]]>; 2589} 2590 2591multiclass RVVOutBuiltinSetZvk<bit HasVV = 1, bit HasVS = 1> { 2592 // vaesz only has 'vs' and vgmul only has 'vv' and they do not have ambiguous 2593 // prototypes like other zvkned instructions (e.g. vaesdf), so we don't 2594 // need to encode the operand mnemonics into its intrinsic function name. 2595 if HasVV then { 2596 defvar name = NAME # !if(!eq(NAME, "vgmul"), "", "_vv"); 2597 let OverloadedName = name in 2598 defm "" : RVVOutBuiltinSet<NAME # "_vv", "i", 2599 [["vv", "Uv", "UvUvUv"]]>; 2600 } 2601 2602 if HasVS then { 2603 foreach vs2_lmul = ["(SEFixedLog2LMUL:-1)", "(SEFixedLog2LMUL:0)", 2604 "(SEFixedLog2LMUL:1)", "(SEFixedLog2LMUL:2)"] in { 2605 defvar name = NAME # !if(!eq(NAME, "vaesz"), "", "_vs"); 2606 let OverloadedName = name, IRName = NAME # "_vs", Name = NAME # "_vs", 2607 IntrinsicTypes = [-1, 1] in 2608 def NAME # vs2_lmul 2609 : RVVBuiltin<vs2_lmul # "UvUv", "UvUv" # vs2_lmul # "Uv", "i">; 2610 } 2611 } 2612} 2613 2614multiclass RVVOutOp2BuiltinSetVVZvk<string type_range = "i"> 2615 : RVVOutOp2BuiltinSet<NAME, type_range, [["vv", "Uv", "UvUvUvUv"]]>; 2616 2617multiclass RVVOutOp2BuiltinSetVIZvk<string type_range = "i"> 2618 : RVVOutOp2BuiltinSet<NAME, type_range, [["vi", "Uv", "UvUvUvKz"]]>; 2619 2620multiclass RVVSignedWidenBinBuiltinSetVwsll 2621 : RVVWidenBuiltinSet<NAME, "csi", 2622 [["vv", "Uw", "UwUvUv"], 2623 ["vx", "Uw", "UwUvz"]]>; 2624 2625let UnMaskedPolicyScheme = HasPassthruOperand in { 2626 // zvkb 2627 let RequiredFeatures = ["Zvkb", "Experimental"] in { 2628 defm vandn : RVVUnsignedBinBuiltinSet; 2629 defm vbrev8 : RVVOutBuiltinSetZvbb; 2630 defm vrev8 : RVVOutBuiltinSetZvbb; 2631 defm vrol : RVVUnsignedShiftBuiltinSet; 2632 defm vror : RVVUnsignedShiftBuiltinSet; 2633 } 2634 2635 // zvbb 2636 let RequiredFeatures = ["Zvbb", "Experimental"] in { 2637 defm vbrev : RVVOutBuiltinSetZvbb; 2638 defm vclz : RVVOutBuiltinSetZvbb; 2639 defm vctz : RVVOutBuiltinSetZvbb; 2640 let IRName = "vcpopv", MaskedIRName = "vcpopv_mask" in 2641 defm vcpop : RVVOutBuiltinSetZvbb; 2642 let OverloadedName = "vwsll" in 2643 defm vwsll : RVVSignedWidenBinBuiltinSetVwsll; 2644 } 2645 2646 // zvbc 2647 let RequiredFeatures = ["Zvbc", "Experimental"] in { 2648 defm vclmul : RVVInt64BinBuiltinSet; 2649 defm vclmulh : RVVInt64BinBuiltinSet; 2650 } 2651} 2652 2653let UnMaskedPolicyScheme = HasPolicyOperand, HasMasked = false in { 2654 // zvkg 2655 let RequiredFeatures = ["Zvkg", "Experimental"] in { 2656 defm vghsh : RVVOutOp2BuiltinSetVVZvk; 2657 defm vgmul : RVVOutBuiltinSetZvk<HasVV=1, HasVS=0>; 2658 } 2659 2660 // zvkned 2661 let RequiredFeatures = ["Zvkned", "Experimental"] in { 2662 defm vaesdf : RVVOutBuiltinSetZvk; 2663 defm vaesdm : RVVOutBuiltinSetZvk; 2664 defm vaesef : RVVOutBuiltinSetZvk; 2665 defm vaesem : RVVOutBuiltinSetZvk; 2666 let UnMaskedPolicyScheme = HasPassthruOperand in 2667 defm vaeskf1 : RVVOutOp1BuiltinSet<"vaeskf1", "i", [["vi", "Uv", "UvUvKz"]]>; 2668 defm vaeskf2 : RVVOutOp2BuiltinSetVIZvk; 2669 defm vaesz : RVVOutBuiltinSetZvk<HasVV=0>; 2670 } 2671 2672 // zvknha 2673 let RequiredFeatures = ["Zvknha", "Experimental"] in { 2674 defm vsha2ch : RVVOutOp2BuiltinSetVVZvk<"i">; 2675 defm vsha2cl : RVVOutOp2BuiltinSetVVZvk<"i">; 2676 defm vsha2ms : RVVOutOp2BuiltinSetVVZvk<"i">; 2677 } 2678 2679 // zvknhb 2680 let RequiredFeatures = ["Zvknhb", "Experimental"] in { 2681 defm vsha2ch : RVVOutOp2BuiltinSetVVZvk<"il">; 2682 defm vsha2cl : RVVOutOp2BuiltinSetVVZvk<"il">; 2683 defm vsha2ms : RVVOutOp2BuiltinSetVVZvk<"il">; 2684 } 2685 2686 // zvksed 2687 let RequiredFeatures = ["Zvksed", "Experimental"] in { 2688 let UnMaskedPolicyScheme = HasPassthruOperand in 2689 defm vsm4k : RVVOutOp1BuiltinSet<"vsm4k", "i", [["vi", "Uv", "UvUvKz"]]>; 2690 defm vsm4r : RVVOutBuiltinSetZvk; 2691 } 2692 2693 // zvksh 2694 let RequiredFeatures = ["Zvksh", "Experimental"] in { 2695 defm vsm3c : RVVOutOp2BuiltinSetVIZvk; 2696 let UnMaskedPolicyScheme = HasPassthruOperand in 2697 defm vsm3me : RVVOutOp1BuiltinSet<"vsm3me", "i", [["vv", "Uv", "UvUvUv"]]>; 2698 } 2699} 2700