1 //===-- R600ISelLowering.cpp - R600 DAG Lowering Implementation -----------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 /// \file 10 /// Custom DAG lowering for R600 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "R600ISelLowering.h" 15 #include "AMDGPUFrameLowering.h" 16 #include "AMDGPUSubtarget.h" 17 #include "R600Defines.h" 18 #include "R600FrameLowering.h" 19 #include "R600InstrInfo.h" 20 #include "R600MachineFunctionInfo.h" 21 #include "MCTargetDesc/AMDGPUMCTargetDesc.h" 22 #include "Utils/AMDGPUBaseInfo.h" 23 #include "llvm/ADT/APFloat.h" 24 #include "llvm/ADT/APInt.h" 25 #include "llvm/ADT/ArrayRef.h" 26 #include "llvm/ADT/DenseMap.h" 27 #include "llvm/ADT/SmallVector.h" 28 #include "llvm/CodeGen/CallingConvLower.h" 29 #include "llvm/CodeGen/DAGCombine.h" 30 #include "llvm/CodeGen/ISDOpcodes.h" 31 #include "llvm/CodeGen/MachineBasicBlock.h" 32 #include "llvm/CodeGen/MachineFunction.h" 33 #include "llvm/CodeGen/MachineInstr.h" 34 #include "llvm/CodeGen/MachineInstrBuilder.h" 35 #include "llvm/CodeGen/MachineMemOperand.h" 36 #include "llvm/CodeGen/MachineRegisterInfo.h" 37 #include "llvm/CodeGen/SelectionDAG.h" 38 #include "llvm/IR/Constants.h" 39 #include "llvm/IR/DerivedTypes.h" 40 #include "llvm/IR/IntrinsicsR600.h" 41 #include "llvm/Support/Casting.h" 42 #include "llvm/Support/Compiler.h" 43 #include "llvm/Support/ErrorHandling.h" 44 #include "llvm/Support/MachineValueType.h" 45 #include "llvm/Support/MathExtras.h" 46 #include <cassert> 47 #include <cstdint> 48 #include <iterator> 49 #include <utility> 50 #include <vector> 51 52 using namespace llvm; 53 54 #include "R600GenCallingConv.inc" 55 56 R600TargetLowering::R600TargetLowering(const TargetMachine &TM, 57 const R600Subtarget &STI) 58 : AMDGPUTargetLowering(TM, STI), Subtarget(&STI), Gen(STI.getGeneration()) { 59 addRegisterClass(MVT::f32, &R600::R600_Reg32RegClass); 60 addRegisterClass(MVT::i32, &R600::R600_Reg32RegClass); 61 addRegisterClass(MVT::v2f32, &R600::R600_Reg64RegClass); 62 addRegisterClass(MVT::v2i32, &R600::R600_Reg64RegClass); 63 addRegisterClass(MVT::v4f32, &R600::R600_Reg128RegClass); 64 addRegisterClass(MVT::v4i32, &R600::R600_Reg128RegClass); 65 66 setBooleanContents(ZeroOrNegativeOneBooleanContent); 67 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent); 68 69 computeRegisterProperties(Subtarget->getRegisterInfo()); 70 71 // Legalize loads and stores to the private address space. 72 setOperationAction(ISD::LOAD, MVT::i32, Custom); 73 setOperationAction(ISD::LOAD, MVT::v2i32, Custom); 74 setOperationAction(ISD::LOAD, MVT::v4i32, Custom); 75 76 // EXTLOAD should be the same as ZEXTLOAD. It is legal for some address 77 // spaces, so it is custom lowered to handle those where it isn't. 78 for (MVT VT : MVT::integer_valuetypes()) { 79 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Promote); 80 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i8, Custom); 81 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i16, Custom); 82 83 setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::i1, Promote); 84 setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::i8, Custom); 85 setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::i16, Custom); 86 87 setLoadExtAction(ISD::EXTLOAD, VT, MVT::i1, Promote); 88 setLoadExtAction(ISD::EXTLOAD, VT, MVT::i8, Custom); 89 setLoadExtAction(ISD::EXTLOAD, VT, MVT::i16, Custom); 90 } 91 92 // Workaround for LegalizeDAG asserting on expansion of i1 vector loads. 93 setLoadExtAction(ISD::EXTLOAD, MVT::v2i32, MVT::v2i1, Expand); 94 setLoadExtAction(ISD::SEXTLOAD, MVT::v2i32, MVT::v2i1, Expand); 95 setLoadExtAction(ISD::ZEXTLOAD, MVT::v2i32, MVT::v2i1, Expand); 96 97 setLoadExtAction(ISD::EXTLOAD, MVT::v4i32, MVT::v4i1, Expand); 98 setLoadExtAction(ISD::SEXTLOAD, MVT::v4i32, MVT::v4i1, Expand); 99 setLoadExtAction(ISD::ZEXTLOAD, MVT::v4i32, MVT::v4i1, Expand); 100 101 setOperationAction(ISD::STORE, MVT::i8, Custom); 102 setOperationAction(ISD::STORE, MVT::i32, Custom); 103 setOperationAction(ISD::STORE, MVT::v2i32, Custom); 104 setOperationAction(ISD::STORE, MVT::v4i32, Custom); 105 106 setTruncStoreAction(MVT::i32, MVT::i8, Custom); 107 setTruncStoreAction(MVT::i32, MVT::i16, Custom); 108 // We need to include these since trunc STORES to PRIVATE need 109 // special handling to accommodate RMW 110 setTruncStoreAction(MVT::v2i32, MVT::v2i16, Custom); 111 setTruncStoreAction(MVT::v4i32, MVT::v4i16, Custom); 112 setTruncStoreAction(MVT::v8i32, MVT::v8i16, Custom); 113 setTruncStoreAction(MVT::v16i32, MVT::v16i16, Custom); 114 setTruncStoreAction(MVT::v32i32, MVT::v32i16, Custom); 115 setTruncStoreAction(MVT::v2i32, MVT::v2i8, Custom); 116 setTruncStoreAction(MVT::v4i32, MVT::v4i8, Custom); 117 setTruncStoreAction(MVT::v8i32, MVT::v8i8, Custom); 118 setTruncStoreAction(MVT::v16i32, MVT::v16i8, Custom); 119 setTruncStoreAction(MVT::v32i32, MVT::v32i8, Custom); 120 121 // Workaround for LegalizeDAG asserting on expansion of i1 vector stores. 122 setTruncStoreAction(MVT::v2i32, MVT::v2i1, Expand); 123 setTruncStoreAction(MVT::v4i32, MVT::v4i1, Expand); 124 125 // Set condition code actions 126 setCondCodeAction(ISD::SETO, MVT::f32, Expand); 127 setCondCodeAction(ISD::SETUO, MVT::f32, Expand); 128 setCondCodeAction(ISD::SETLT, MVT::f32, Expand); 129 setCondCodeAction(ISD::SETLE, MVT::f32, Expand); 130 setCondCodeAction(ISD::SETOLT, MVT::f32, Expand); 131 setCondCodeAction(ISD::SETOLE, MVT::f32, Expand); 132 setCondCodeAction(ISD::SETONE, MVT::f32, Expand); 133 setCondCodeAction(ISD::SETUEQ, MVT::f32, Expand); 134 setCondCodeAction(ISD::SETUGE, MVT::f32, Expand); 135 setCondCodeAction(ISD::SETUGT, MVT::f32, Expand); 136 setCondCodeAction(ISD::SETULT, MVT::f32, Expand); 137 setCondCodeAction(ISD::SETULE, MVT::f32, Expand); 138 139 setCondCodeAction(ISD::SETLE, MVT::i32, Expand); 140 setCondCodeAction(ISD::SETLT, MVT::i32, Expand); 141 setCondCodeAction(ISD::SETULE, MVT::i32, Expand); 142 setCondCodeAction(ISD::SETULT, MVT::i32, Expand); 143 144 setOperationAction(ISD::FCOS, MVT::f32, Custom); 145 setOperationAction(ISD::FSIN, MVT::f32, Custom); 146 147 setOperationAction(ISD::SETCC, MVT::v4i32, Expand); 148 setOperationAction(ISD::SETCC, MVT::v2i32, Expand); 149 150 setOperationAction(ISD::BR_CC, MVT::i32, Expand); 151 setOperationAction(ISD::BR_CC, MVT::f32, Expand); 152 setOperationAction(ISD::BRCOND, MVT::Other, Custom); 153 154 setOperationAction(ISD::FSUB, MVT::f32, Expand); 155 156 setOperationAction(ISD::FCEIL, MVT::f64, Custom); 157 setOperationAction(ISD::FTRUNC, MVT::f64, Custom); 158 setOperationAction(ISD::FRINT, MVT::f64, Custom); 159 setOperationAction(ISD::FFLOOR, MVT::f64, Custom); 160 161 setOperationAction(ISD::SELECT_CC, MVT::f32, Custom); 162 setOperationAction(ISD::SELECT_CC, MVT::i32, Custom); 163 164 setOperationAction(ISD::SETCC, MVT::i32, Expand); 165 setOperationAction(ISD::SETCC, MVT::f32, Expand); 166 setOperationAction(ISD::FP_TO_UINT, MVT::i1, Custom); 167 setOperationAction(ISD::FP_TO_SINT, MVT::i1, Custom); 168 setOperationAction(ISD::FP_TO_SINT, MVT::i64, Custom); 169 setOperationAction(ISD::FP_TO_UINT, MVT::i64, Custom); 170 171 setOperationAction(ISD::SELECT, MVT::i32, Expand); 172 setOperationAction(ISD::SELECT, MVT::f32, Expand); 173 setOperationAction(ISD::SELECT, MVT::v2i32, Expand); 174 setOperationAction(ISD::SELECT, MVT::v4i32, Expand); 175 176 // ADD, SUB overflow. 177 // TODO: turn these into Legal? 178 if (Subtarget->hasCARRY()) 179 setOperationAction(ISD::UADDO, MVT::i32, Custom); 180 181 if (Subtarget->hasBORROW()) 182 setOperationAction(ISD::USUBO, MVT::i32, Custom); 183 184 // Expand sign extension of vectors 185 if (!Subtarget->hasBFE()) 186 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand); 187 188 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i1, Expand); 189 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v4i1, Expand); 190 191 if (!Subtarget->hasBFE()) 192 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8, Expand); 193 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i8, Expand); 194 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v4i8, Expand); 195 196 if (!Subtarget->hasBFE()) 197 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16, Expand); 198 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i16, Expand); 199 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v4i16, Expand); 200 201 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i32, Legal); 202 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i32, Expand); 203 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v4i32, Expand); 204 205 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::Other, Expand); 206 207 setOperationAction(ISD::FrameIndex, MVT::i32, Custom); 208 209 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i32, Custom); 210 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2f32, Custom); 211 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4i32, Custom); 212 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Custom); 213 214 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2i32, Custom); 215 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2f32, Custom); 216 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i32, Custom); 217 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f32, Custom); 218 219 // We don't have 64-bit shifts. Thus we need either SHX i64 or SHX_PARTS i32 220 // to be Legal/Custom in order to avoid library calls. 221 setOperationAction(ISD::SHL_PARTS, MVT::i32, Custom); 222 setOperationAction(ISD::SRL_PARTS, MVT::i32, Custom); 223 setOperationAction(ISD::SRA_PARTS, MVT::i32, Custom); 224 225 if (!Subtarget->hasFMA()) { 226 setOperationAction(ISD::FMA, MVT::f32, Expand); 227 setOperationAction(ISD::FMA, MVT::f64, Expand); 228 } 229 230 // FIXME: May need no denormals check 231 setOperationAction(ISD::FMAD, MVT::f32, Legal); 232 233 if (!Subtarget->hasBFI()) { 234 // fcopysign can be done in a single instruction with BFI. 235 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Expand); 236 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand); 237 } 238 239 if (!Subtarget->hasBCNT(32)) 240 setOperationAction(ISD::CTPOP, MVT::i32, Expand); 241 242 if (!Subtarget->hasBCNT(64)) 243 setOperationAction(ISD::CTPOP, MVT::i64, Expand); 244 245 if (Subtarget->hasFFBH()) 246 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i32, Custom); 247 248 if (Subtarget->hasFFBL()) 249 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i32, Custom); 250 251 // FIXME: This was moved from AMDGPUTargetLowering, I'm not sure if we 252 // need it for R600. 253 if (Subtarget->hasBFE()) 254 setHasExtractBitsInsn(true); 255 256 setOperationAction(ISD::GlobalAddress, MVT::i32, Custom); 257 258 const MVT ScalarIntVTs[] = { MVT::i32, MVT::i64 }; 259 for (MVT VT : ScalarIntVTs) { 260 setOperationAction(ISD::ADDC, VT, Expand); 261 setOperationAction(ISD::SUBC, VT, Expand); 262 setOperationAction(ISD::ADDE, VT, Expand); 263 setOperationAction(ISD::SUBE, VT, Expand); 264 } 265 266 // LLVM will expand these to atomic_cmp_swap(0) 267 // and atomic_swap, respectively. 268 setOperationAction(ISD::ATOMIC_LOAD, MVT::i32, Expand); 269 setOperationAction(ISD::ATOMIC_STORE, MVT::i32, Expand); 270 271 // We need to custom lower some of the intrinsics 272 setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom); 273 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom); 274 275 setSchedulingPreference(Sched::Source); 276 277 setTargetDAGCombine(ISD::FP_ROUND); 278 setTargetDAGCombine(ISD::FP_TO_SINT); 279 setTargetDAGCombine(ISD::EXTRACT_VECTOR_ELT); 280 setTargetDAGCombine(ISD::SELECT_CC); 281 setTargetDAGCombine(ISD::INSERT_VECTOR_ELT); 282 setTargetDAGCombine(ISD::LOAD); 283 } 284 285 static inline bool isEOP(MachineBasicBlock::iterator I) { 286 if (std::next(I) == I->getParent()->end()) 287 return false; 288 return std::next(I)->getOpcode() == R600::RETURN; 289 } 290 291 MachineBasicBlock * 292 R600TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI, 293 MachineBasicBlock *BB) const { 294 MachineFunction *MF = BB->getParent(); 295 MachineRegisterInfo &MRI = MF->getRegInfo(); 296 MachineBasicBlock::iterator I = MI; 297 const R600InstrInfo *TII = Subtarget->getInstrInfo(); 298 299 switch (MI.getOpcode()) { 300 default: 301 // Replace LDS_*_RET instruction that don't have any uses with the 302 // equivalent LDS_*_NORET instruction. 303 if (TII->isLDSRetInstr(MI.getOpcode())) { 304 int DstIdx = TII->getOperandIdx(MI.getOpcode(), R600::OpName::dst); 305 assert(DstIdx != -1); 306 MachineInstrBuilder NewMI; 307 // FIXME: getLDSNoRetOp method only handles LDS_1A1D LDS ops. Add 308 // LDS_1A2D support and remove this special case. 309 if (!MRI.use_empty(MI.getOperand(DstIdx).getReg()) || 310 MI.getOpcode() == R600::LDS_CMPST_RET) 311 return BB; 312 313 NewMI = BuildMI(*BB, I, BB->findDebugLoc(I), 314 TII->get(R600::getLDSNoRetOp(MI.getOpcode()))); 315 for (unsigned i = 1, e = MI.getNumOperands(); i < e; ++i) { 316 NewMI.add(MI.getOperand(i)); 317 } 318 } else { 319 return AMDGPUTargetLowering::EmitInstrWithCustomInserter(MI, BB); 320 } 321 break; 322 323 case R600::FABS_R600: { 324 MachineInstr *NewMI = TII->buildDefaultInstruction( 325 *BB, I, R600::MOV, MI.getOperand(0).getReg(), 326 MI.getOperand(1).getReg()); 327 TII->addFlag(*NewMI, 0, MO_FLAG_ABS); 328 break; 329 } 330 331 case R600::FNEG_R600: { 332 MachineInstr *NewMI = TII->buildDefaultInstruction( 333 *BB, I, R600::MOV, MI.getOperand(0).getReg(), 334 MI.getOperand(1).getReg()); 335 TII->addFlag(*NewMI, 0, MO_FLAG_NEG); 336 break; 337 } 338 339 case R600::MASK_WRITE: { 340 Register maskedRegister = MI.getOperand(0).getReg(); 341 assert(Register::isVirtualRegister(maskedRegister)); 342 MachineInstr * defInstr = MRI.getVRegDef(maskedRegister); 343 TII->addFlag(*defInstr, 0, MO_FLAG_MASK); 344 break; 345 } 346 347 case R600::MOV_IMM_F32: 348 TII->buildMovImm(*BB, I, MI.getOperand(0).getReg(), MI.getOperand(1) 349 .getFPImm() 350 ->getValueAPF() 351 .bitcastToAPInt() 352 .getZExtValue()); 353 break; 354 355 case R600::MOV_IMM_I32: 356 TII->buildMovImm(*BB, I, MI.getOperand(0).getReg(), 357 MI.getOperand(1).getImm()); 358 break; 359 360 case R600::MOV_IMM_GLOBAL_ADDR: { 361 //TODO: Perhaps combine this instruction with the next if possible 362 auto MIB = TII->buildDefaultInstruction( 363 *BB, MI, R600::MOV, MI.getOperand(0).getReg(), R600::ALU_LITERAL_X); 364 int Idx = TII->getOperandIdx(*MIB, R600::OpName::literal); 365 //TODO: Ugh this is rather ugly 366 MIB->getOperand(Idx) = MI.getOperand(1); 367 break; 368 } 369 370 case R600::CONST_COPY: { 371 MachineInstr *NewMI = TII->buildDefaultInstruction( 372 *BB, MI, R600::MOV, MI.getOperand(0).getReg(), R600::ALU_CONST); 373 TII->setImmOperand(*NewMI, R600::OpName::src0_sel, 374 MI.getOperand(1).getImm()); 375 break; 376 } 377 378 case R600::RAT_WRITE_CACHELESS_32_eg: 379 case R600::RAT_WRITE_CACHELESS_64_eg: 380 case R600::RAT_WRITE_CACHELESS_128_eg: 381 BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(MI.getOpcode())) 382 .add(MI.getOperand(0)) 383 .add(MI.getOperand(1)) 384 .addImm(isEOP(I)); // Set End of program bit 385 break; 386 387 case R600::RAT_STORE_TYPED_eg: 388 BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(MI.getOpcode())) 389 .add(MI.getOperand(0)) 390 .add(MI.getOperand(1)) 391 .add(MI.getOperand(2)) 392 .addImm(isEOP(I)); // Set End of program bit 393 break; 394 395 case R600::BRANCH: 396 BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(R600::JUMP)) 397 .add(MI.getOperand(0)); 398 break; 399 400 case R600::BRANCH_COND_f32: { 401 MachineInstr *NewMI = 402 BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(R600::PRED_X), 403 R600::PREDICATE_BIT) 404 .add(MI.getOperand(1)) 405 .addImm(R600::PRED_SETNE) 406 .addImm(0); // Flags 407 TII->addFlag(*NewMI, 0, MO_FLAG_PUSH); 408 BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(R600::JUMP_COND)) 409 .add(MI.getOperand(0)) 410 .addReg(R600::PREDICATE_BIT, RegState::Kill); 411 break; 412 } 413 414 case R600::BRANCH_COND_i32: { 415 MachineInstr *NewMI = 416 BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(R600::PRED_X), 417 R600::PREDICATE_BIT) 418 .add(MI.getOperand(1)) 419 .addImm(R600::PRED_SETNE_INT) 420 .addImm(0); // Flags 421 TII->addFlag(*NewMI, 0, MO_FLAG_PUSH); 422 BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(R600::JUMP_COND)) 423 .add(MI.getOperand(0)) 424 .addReg(R600::PREDICATE_BIT, RegState::Kill); 425 break; 426 } 427 428 case R600::EG_ExportSwz: 429 case R600::R600_ExportSwz: { 430 // Instruction is left unmodified if its not the last one of its type 431 bool isLastInstructionOfItsType = true; 432 unsigned InstExportType = MI.getOperand(1).getImm(); 433 for (MachineBasicBlock::iterator NextExportInst = std::next(I), 434 EndBlock = BB->end(); NextExportInst != EndBlock; 435 NextExportInst = std::next(NextExportInst)) { 436 if (NextExportInst->getOpcode() == R600::EG_ExportSwz || 437 NextExportInst->getOpcode() == R600::R600_ExportSwz) { 438 unsigned CurrentInstExportType = NextExportInst->getOperand(1) 439 .getImm(); 440 if (CurrentInstExportType == InstExportType) { 441 isLastInstructionOfItsType = false; 442 break; 443 } 444 } 445 } 446 bool EOP = isEOP(I); 447 if (!EOP && !isLastInstructionOfItsType) 448 return BB; 449 unsigned CfInst = (MI.getOpcode() == R600::EG_ExportSwz) ? 84 : 40; 450 BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(MI.getOpcode())) 451 .add(MI.getOperand(0)) 452 .add(MI.getOperand(1)) 453 .add(MI.getOperand(2)) 454 .add(MI.getOperand(3)) 455 .add(MI.getOperand(4)) 456 .add(MI.getOperand(5)) 457 .add(MI.getOperand(6)) 458 .addImm(CfInst) 459 .addImm(EOP); 460 break; 461 } 462 case R600::RETURN: { 463 return BB; 464 } 465 } 466 467 MI.eraseFromParent(); 468 return BB; 469 } 470 471 //===----------------------------------------------------------------------===// 472 // Custom DAG Lowering Operations 473 //===----------------------------------------------------------------------===// 474 475 SDValue R600TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const { 476 MachineFunction &MF = DAG.getMachineFunction(); 477 R600MachineFunctionInfo *MFI = MF.getInfo<R600MachineFunctionInfo>(); 478 switch (Op.getOpcode()) { 479 default: return AMDGPUTargetLowering::LowerOperation(Op, DAG); 480 case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG); 481 case ISD::INSERT_VECTOR_ELT: return LowerINSERT_VECTOR_ELT(Op, DAG); 482 case ISD::SHL_PARTS: return LowerSHLParts(Op, DAG); 483 case ISD::SRA_PARTS: 484 case ISD::SRL_PARTS: return LowerSRXParts(Op, DAG); 485 case ISD::UADDO: return LowerUADDSUBO(Op, DAG, ISD::ADD, AMDGPUISD::CARRY); 486 case ISD::USUBO: return LowerUADDSUBO(Op, DAG, ISD::SUB, AMDGPUISD::BORROW); 487 case ISD::FCOS: 488 case ISD::FSIN: return LowerTrig(Op, DAG); 489 case ISD::SELECT_CC: return LowerSELECT_CC(Op, DAG); 490 case ISD::STORE: return LowerSTORE(Op, DAG); 491 case ISD::LOAD: { 492 SDValue Result = LowerLOAD(Op, DAG); 493 assert((!Result.getNode() || 494 Result.getNode()->getNumValues() == 2) && 495 "Load should return a value and a chain"); 496 return Result; 497 } 498 499 case ISD::BRCOND: return LowerBRCOND(Op, DAG); 500 case ISD::GlobalAddress: return LowerGlobalAddress(MFI, Op, DAG); 501 case ISD::FrameIndex: return lowerFrameIndex(Op, DAG); 502 case ISD::INTRINSIC_VOID: { 503 SDValue Chain = Op.getOperand(0); 504 unsigned IntrinsicID = 505 cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue(); 506 switch (IntrinsicID) { 507 case Intrinsic::r600_store_swizzle: { 508 SDLoc DL(Op); 509 const SDValue Args[8] = { 510 Chain, 511 Op.getOperand(2), // Export Value 512 Op.getOperand(3), // ArrayBase 513 Op.getOperand(4), // Type 514 DAG.getConstant(0, DL, MVT::i32), // SWZ_X 515 DAG.getConstant(1, DL, MVT::i32), // SWZ_Y 516 DAG.getConstant(2, DL, MVT::i32), // SWZ_Z 517 DAG.getConstant(3, DL, MVT::i32) // SWZ_W 518 }; 519 return DAG.getNode(AMDGPUISD::R600_EXPORT, DL, Op.getValueType(), Args); 520 } 521 522 // default for switch(IntrinsicID) 523 default: break; 524 } 525 // break out of case ISD::INTRINSIC_VOID in switch(Op.getOpcode()) 526 break; 527 } 528 case ISD::INTRINSIC_WO_CHAIN: { 529 unsigned IntrinsicID = 530 cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); 531 EVT VT = Op.getValueType(); 532 SDLoc DL(Op); 533 switch (IntrinsicID) { 534 case Intrinsic::r600_tex: 535 case Intrinsic::r600_texc: { 536 unsigned TextureOp; 537 switch (IntrinsicID) { 538 case Intrinsic::r600_tex: 539 TextureOp = 0; 540 break; 541 case Intrinsic::r600_texc: 542 TextureOp = 1; 543 break; 544 default: 545 llvm_unreachable("unhandled texture operation"); 546 } 547 548 SDValue TexArgs[19] = { 549 DAG.getConstant(TextureOp, DL, MVT::i32), 550 Op.getOperand(1), 551 DAG.getConstant(0, DL, MVT::i32), 552 DAG.getConstant(1, DL, MVT::i32), 553 DAG.getConstant(2, DL, MVT::i32), 554 DAG.getConstant(3, DL, MVT::i32), 555 Op.getOperand(2), 556 Op.getOperand(3), 557 Op.getOperand(4), 558 DAG.getConstant(0, DL, MVT::i32), 559 DAG.getConstant(1, DL, MVT::i32), 560 DAG.getConstant(2, DL, MVT::i32), 561 DAG.getConstant(3, DL, MVT::i32), 562 Op.getOperand(5), 563 Op.getOperand(6), 564 Op.getOperand(7), 565 Op.getOperand(8), 566 Op.getOperand(9), 567 Op.getOperand(10) 568 }; 569 return DAG.getNode(AMDGPUISD::TEXTURE_FETCH, DL, MVT::v4f32, TexArgs); 570 } 571 case Intrinsic::r600_dot4: { 572 SDValue Args[8] = { 573 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, Op.getOperand(1), 574 DAG.getConstant(0, DL, MVT::i32)), 575 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, Op.getOperand(2), 576 DAG.getConstant(0, DL, MVT::i32)), 577 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, Op.getOperand(1), 578 DAG.getConstant(1, DL, MVT::i32)), 579 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, Op.getOperand(2), 580 DAG.getConstant(1, DL, MVT::i32)), 581 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, Op.getOperand(1), 582 DAG.getConstant(2, DL, MVT::i32)), 583 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, Op.getOperand(2), 584 DAG.getConstant(2, DL, MVT::i32)), 585 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, Op.getOperand(1), 586 DAG.getConstant(3, DL, MVT::i32)), 587 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, Op.getOperand(2), 588 DAG.getConstant(3, DL, MVT::i32)) 589 }; 590 return DAG.getNode(AMDGPUISD::DOT4, DL, MVT::f32, Args); 591 } 592 593 case Intrinsic::r600_implicitarg_ptr: { 594 MVT PtrVT = getPointerTy(DAG.getDataLayout(), AMDGPUAS::PARAM_I_ADDRESS); 595 uint32_t ByteOffset = getImplicitParameterOffset(MF, FIRST_IMPLICIT); 596 return DAG.getConstant(ByteOffset, DL, PtrVT); 597 } 598 case Intrinsic::r600_read_ngroups_x: 599 return LowerImplicitParameter(DAG, VT, DL, 0); 600 case Intrinsic::r600_read_ngroups_y: 601 return LowerImplicitParameter(DAG, VT, DL, 1); 602 case Intrinsic::r600_read_ngroups_z: 603 return LowerImplicitParameter(DAG, VT, DL, 2); 604 case Intrinsic::r600_read_global_size_x: 605 return LowerImplicitParameter(DAG, VT, DL, 3); 606 case Intrinsic::r600_read_global_size_y: 607 return LowerImplicitParameter(DAG, VT, DL, 4); 608 case Intrinsic::r600_read_global_size_z: 609 return LowerImplicitParameter(DAG, VT, DL, 5); 610 case Intrinsic::r600_read_local_size_x: 611 return LowerImplicitParameter(DAG, VT, DL, 6); 612 case Intrinsic::r600_read_local_size_y: 613 return LowerImplicitParameter(DAG, VT, DL, 7); 614 case Intrinsic::r600_read_local_size_z: 615 return LowerImplicitParameter(DAG, VT, DL, 8); 616 617 case Intrinsic::r600_read_tgid_x: 618 case Intrinsic::amdgcn_workgroup_id_x: 619 return CreateLiveInRegisterRaw(DAG, &R600::R600_TReg32RegClass, 620 R600::T1_X, VT); 621 case Intrinsic::r600_read_tgid_y: 622 case Intrinsic::amdgcn_workgroup_id_y: 623 return CreateLiveInRegisterRaw(DAG, &R600::R600_TReg32RegClass, 624 R600::T1_Y, VT); 625 case Intrinsic::r600_read_tgid_z: 626 case Intrinsic::amdgcn_workgroup_id_z: 627 return CreateLiveInRegisterRaw(DAG, &R600::R600_TReg32RegClass, 628 R600::T1_Z, VT); 629 case Intrinsic::r600_read_tidig_x: 630 case Intrinsic::amdgcn_workitem_id_x: 631 return CreateLiveInRegisterRaw(DAG, &R600::R600_TReg32RegClass, 632 R600::T0_X, VT); 633 case Intrinsic::r600_read_tidig_y: 634 case Intrinsic::amdgcn_workitem_id_y: 635 return CreateLiveInRegisterRaw(DAG, &R600::R600_TReg32RegClass, 636 R600::T0_Y, VT); 637 case Intrinsic::r600_read_tidig_z: 638 case Intrinsic::amdgcn_workitem_id_z: 639 return CreateLiveInRegisterRaw(DAG, &R600::R600_TReg32RegClass, 640 R600::T0_Z, VT); 641 642 case Intrinsic::r600_recipsqrt_ieee: 643 return DAG.getNode(AMDGPUISD::RSQ, DL, VT, Op.getOperand(1)); 644 645 case Intrinsic::r600_recipsqrt_clamped: 646 return DAG.getNode(AMDGPUISD::RSQ_CLAMP, DL, VT, Op.getOperand(1)); 647 default: 648 return Op; 649 } 650 651 // break out of case ISD::INTRINSIC_WO_CHAIN in switch(Op.getOpcode()) 652 break; 653 } 654 } // end switch(Op.getOpcode()) 655 return SDValue(); 656 } 657 658 void R600TargetLowering::ReplaceNodeResults(SDNode *N, 659 SmallVectorImpl<SDValue> &Results, 660 SelectionDAG &DAG) const { 661 switch (N->getOpcode()) { 662 default: 663 AMDGPUTargetLowering::ReplaceNodeResults(N, Results, DAG); 664 return; 665 case ISD::FP_TO_UINT: 666 if (N->getValueType(0) == MVT::i1) { 667 Results.push_back(lowerFP_TO_UINT(N->getOperand(0), DAG)); 668 return; 669 } 670 // Since we don't care about out of bounds values we can use FP_TO_SINT for 671 // uints too. The DAGLegalizer code for uint considers some extra cases 672 // which are not necessary here. 673 LLVM_FALLTHROUGH; 674 case ISD::FP_TO_SINT: { 675 if (N->getValueType(0) == MVT::i1) { 676 Results.push_back(lowerFP_TO_SINT(N->getOperand(0), DAG)); 677 return; 678 } 679 680 SDValue Result; 681 if (expandFP_TO_SINT(N, Result, DAG)) 682 Results.push_back(Result); 683 return; 684 } 685 case ISD::SDIVREM: { 686 SDValue Op = SDValue(N, 1); 687 SDValue RES = LowerSDIVREM(Op, DAG); 688 Results.push_back(RES); 689 Results.push_back(RES.getValue(1)); 690 break; 691 } 692 case ISD::UDIVREM: { 693 SDValue Op = SDValue(N, 0); 694 LowerUDIVREM64(Op, DAG, Results); 695 break; 696 } 697 } 698 } 699 700 SDValue R600TargetLowering::vectorToVerticalVector(SelectionDAG &DAG, 701 SDValue Vector) const { 702 SDLoc DL(Vector); 703 EVT VecVT = Vector.getValueType(); 704 EVT EltVT = VecVT.getVectorElementType(); 705 SmallVector<SDValue, 8> Args; 706 707 for (unsigned i = 0, e = VecVT.getVectorNumElements(); i != e; ++i) { 708 Args.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Vector, 709 DAG.getVectorIdxConstant(i, DL))); 710 } 711 712 return DAG.getNode(AMDGPUISD::BUILD_VERTICAL_VECTOR, DL, VecVT, Args); 713 } 714 715 SDValue R600TargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op, 716 SelectionDAG &DAG) const { 717 SDLoc DL(Op); 718 SDValue Vector = Op.getOperand(0); 719 SDValue Index = Op.getOperand(1); 720 721 if (isa<ConstantSDNode>(Index) || 722 Vector.getOpcode() == AMDGPUISD::BUILD_VERTICAL_VECTOR) 723 return Op; 724 725 Vector = vectorToVerticalVector(DAG, Vector); 726 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, Op.getValueType(), 727 Vector, Index); 728 } 729 730 SDValue R600TargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op, 731 SelectionDAG &DAG) const { 732 SDLoc DL(Op); 733 SDValue Vector = Op.getOperand(0); 734 SDValue Value = Op.getOperand(1); 735 SDValue Index = Op.getOperand(2); 736 737 if (isa<ConstantSDNode>(Index) || 738 Vector.getOpcode() == AMDGPUISD::BUILD_VERTICAL_VECTOR) 739 return Op; 740 741 Vector = vectorToVerticalVector(DAG, Vector); 742 SDValue Insert = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, Op.getValueType(), 743 Vector, Value, Index); 744 return vectorToVerticalVector(DAG, Insert); 745 } 746 747 SDValue R600TargetLowering::LowerGlobalAddress(AMDGPUMachineFunction *MFI, 748 SDValue Op, 749 SelectionDAG &DAG) const { 750 GlobalAddressSDNode *GSD = cast<GlobalAddressSDNode>(Op); 751 if (GSD->getAddressSpace() != AMDGPUAS::CONSTANT_ADDRESS) 752 return AMDGPUTargetLowering::LowerGlobalAddress(MFI, Op, DAG); 753 754 const DataLayout &DL = DAG.getDataLayout(); 755 const GlobalValue *GV = GSD->getGlobal(); 756 MVT ConstPtrVT = getPointerTy(DL, AMDGPUAS::CONSTANT_ADDRESS); 757 758 SDValue GA = DAG.getTargetGlobalAddress(GV, SDLoc(GSD), ConstPtrVT); 759 return DAG.getNode(AMDGPUISD::CONST_DATA_PTR, SDLoc(GSD), ConstPtrVT, GA); 760 } 761 762 SDValue R600TargetLowering::LowerTrig(SDValue Op, SelectionDAG &DAG) const { 763 // On hw >= R700, COS/SIN input must be between -1. and 1. 764 // Thus we lower them to TRIG ( FRACT ( x / 2Pi + 0.5) - 0.5) 765 EVT VT = Op.getValueType(); 766 SDValue Arg = Op.getOperand(0); 767 SDLoc DL(Op); 768 769 // TODO: Should this propagate fast-math-flags? 770 SDValue FractPart = DAG.getNode(AMDGPUISD::FRACT, DL, VT, 771 DAG.getNode(ISD::FADD, DL, VT, 772 DAG.getNode(ISD::FMUL, DL, VT, Arg, 773 DAG.getConstantFP(0.15915494309, DL, MVT::f32)), 774 DAG.getConstantFP(0.5, DL, MVT::f32))); 775 unsigned TrigNode; 776 switch (Op.getOpcode()) { 777 case ISD::FCOS: 778 TrigNode = AMDGPUISD::COS_HW; 779 break; 780 case ISD::FSIN: 781 TrigNode = AMDGPUISD::SIN_HW; 782 break; 783 default: 784 llvm_unreachable("Wrong trig opcode"); 785 } 786 SDValue TrigVal = DAG.getNode(TrigNode, DL, VT, 787 DAG.getNode(ISD::FADD, DL, VT, FractPart, 788 DAG.getConstantFP(-0.5, DL, MVT::f32))); 789 if (Gen >= AMDGPUSubtarget::R700) 790 return TrigVal; 791 // On R600 hw, COS/SIN input must be between -Pi and Pi. 792 return DAG.getNode(ISD::FMUL, DL, VT, TrigVal, 793 DAG.getConstantFP(numbers::pif, DL, MVT::f32)); 794 } 795 796 SDValue R600TargetLowering::LowerSHLParts(SDValue Op, SelectionDAG &DAG) const { 797 SDLoc DL(Op); 798 EVT VT = Op.getValueType(); 799 800 SDValue Lo = Op.getOperand(0); 801 SDValue Hi = Op.getOperand(1); 802 SDValue Shift = Op.getOperand(2); 803 SDValue Zero = DAG.getConstant(0, DL, VT); 804 SDValue One = DAG.getConstant(1, DL, VT); 805 806 SDValue Width = DAG.getConstant(VT.getSizeInBits(), DL, VT); 807 SDValue Width1 = DAG.getConstant(VT.getSizeInBits() - 1, DL, VT); 808 SDValue BigShift = DAG.getNode(ISD::SUB, DL, VT, Shift, Width); 809 SDValue CompShift = DAG.getNode(ISD::SUB, DL, VT, Width1, Shift); 810 811 // The dance around Width1 is necessary for 0 special case. 812 // Without it the CompShift might be 32, producing incorrect results in 813 // Overflow. So we do the shift in two steps, the alternative is to 814 // add a conditional to filter the special case. 815 816 SDValue Overflow = DAG.getNode(ISD::SRL, DL, VT, Lo, CompShift); 817 Overflow = DAG.getNode(ISD::SRL, DL, VT, Overflow, One); 818 819 SDValue HiSmall = DAG.getNode(ISD::SHL, DL, VT, Hi, Shift); 820 HiSmall = DAG.getNode(ISD::OR, DL, VT, HiSmall, Overflow); 821 SDValue LoSmall = DAG.getNode(ISD::SHL, DL, VT, Lo, Shift); 822 823 SDValue HiBig = DAG.getNode(ISD::SHL, DL, VT, Lo, BigShift); 824 SDValue LoBig = Zero; 825 826 Hi = DAG.getSelectCC(DL, Shift, Width, HiSmall, HiBig, ISD::SETULT); 827 Lo = DAG.getSelectCC(DL, Shift, Width, LoSmall, LoBig, ISD::SETULT); 828 829 return DAG.getNode(ISD::MERGE_VALUES, DL, DAG.getVTList(VT,VT), Lo, Hi); 830 } 831 832 SDValue R600TargetLowering::LowerSRXParts(SDValue Op, SelectionDAG &DAG) const { 833 SDLoc DL(Op); 834 EVT VT = Op.getValueType(); 835 836 SDValue Lo = Op.getOperand(0); 837 SDValue Hi = Op.getOperand(1); 838 SDValue Shift = Op.getOperand(2); 839 SDValue Zero = DAG.getConstant(0, DL, VT); 840 SDValue One = DAG.getConstant(1, DL, VT); 841 842 const bool SRA = Op.getOpcode() == ISD::SRA_PARTS; 843 844 SDValue Width = DAG.getConstant(VT.getSizeInBits(), DL, VT); 845 SDValue Width1 = DAG.getConstant(VT.getSizeInBits() - 1, DL, VT); 846 SDValue BigShift = DAG.getNode(ISD::SUB, DL, VT, Shift, Width); 847 SDValue CompShift = DAG.getNode(ISD::SUB, DL, VT, Width1, Shift); 848 849 // The dance around Width1 is necessary for 0 special case. 850 // Without it the CompShift might be 32, producing incorrect results in 851 // Overflow. So we do the shift in two steps, the alternative is to 852 // add a conditional to filter the special case. 853 854 SDValue Overflow = DAG.getNode(ISD::SHL, DL, VT, Hi, CompShift); 855 Overflow = DAG.getNode(ISD::SHL, DL, VT, Overflow, One); 856 857 SDValue HiSmall = DAG.getNode(SRA ? ISD::SRA : ISD::SRL, DL, VT, Hi, Shift); 858 SDValue LoSmall = DAG.getNode(ISD::SRL, DL, VT, Lo, Shift); 859 LoSmall = DAG.getNode(ISD::OR, DL, VT, LoSmall, Overflow); 860 861 SDValue LoBig = DAG.getNode(SRA ? ISD::SRA : ISD::SRL, DL, VT, Hi, BigShift); 862 SDValue HiBig = SRA ? DAG.getNode(ISD::SRA, DL, VT, Hi, Width1) : Zero; 863 864 Hi = DAG.getSelectCC(DL, Shift, Width, HiSmall, HiBig, ISD::SETULT); 865 Lo = DAG.getSelectCC(DL, Shift, Width, LoSmall, LoBig, ISD::SETULT); 866 867 return DAG.getNode(ISD::MERGE_VALUES, DL, DAG.getVTList(VT,VT), Lo, Hi); 868 } 869 870 SDValue R600TargetLowering::LowerUADDSUBO(SDValue Op, SelectionDAG &DAG, 871 unsigned mainop, unsigned ovf) const { 872 SDLoc DL(Op); 873 EVT VT = Op.getValueType(); 874 875 SDValue Lo = Op.getOperand(0); 876 SDValue Hi = Op.getOperand(1); 877 878 SDValue OVF = DAG.getNode(ovf, DL, VT, Lo, Hi); 879 // Extend sign. 880 OVF = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, OVF, 881 DAG.getValueType(MVT::i1)); 882 883 SDValue Res = DAG.getNode(mainop, DL, VT, Lo, Hi); 884 885 return DAG.getNode(ISD::MERGE_VALUES, DL, DAG.getVTList(VT, VT), Res, OVF); 886 } 887 888 SDValue R600TargetLowering::lowerFP_TO_UINT(SDValue Op, SelectionDAG &DAG) const { 889 SDLoc DL(Op); 890 return DAG.getNode( 891 ISD::SETCC, 892 DL, 893 MVT::i1, 894 Op, DAG.getConstantFP(1.0f, DL, MVT::f32), 895 DAG.getCondCode(ISD::SETEQ)); 896 } 897 898 SDValue R600TargetLowering::lowerFP_TO_SINT(SDValue Op, SelectionDAG &DAG) const { 899 SDLoc DL(Op); 900 return DAG.getNode( 901 ISD::SETCC, 902 DL, 903 MVT::i1, 904 Op, DAG.getConstantFP(-1.0f, DL, MVT::f32), 905 DAG.getCondCode(ISD::SETEQ)); 906 } 907 908 SDValue R600TargetLowering::LowerImplicitParameter(SelectionDAG &DAG, EVT VT, 909 const SDLoc &DL, 910 unsigned DwordOffset) const { 911 unsigned ByteOffset = DwordOffset * 4; 912 PointerType * PtrType = PointerType::get(VT.getTypeForEVT(*DAG.getContext()), 913 AMDGPUAS::PARAM_I_ADDRESS); 914 915 // We shouldn't be using an offset wider than 16-bits for implicit parameters. 916 assert(isInt<16>(ByteOffset)); 917 918 return DAG.getLoad(VT, DL, DAG.getEntryNode(), 919 DAG.getConstant(ByteOffset, DL, MVT::i32), // PTR 920 MachinePointerInfo(ConstantPointerNull::get(PtrType))); 921 } 922 923 bool R600TargetLowering::isZero(SDValue Op) const { 924 if(ConstantSDNode *Cst = dyn_cast<ConstantSDNode>(Op)) { 925 return Cst->isNullValue(); 926 } else if(ConstantFPSDNode *CstFP = dyn_cast<ConstantFPSDNode>(Op)){ 927 return CstFP->isZero(); 928 } else { 929 return false; 930 } 931 } 932 933 bool R600TargetLowering::isHWTrueValue(SDValue Op) const { 934 if (ConstantFPSDNode * CFP = dyn_cast<ConstantFPSDNode>(Op)) { 935 return CFP->isExactlyValue(1.0); 936 } 937 return isAllOnesConstant(Op); 938 } 939 940 bool R600TargetLowering::isHWFalseValue(SDValue Op) const { 941 if (ConstantFPSDNode * CFP = dyn_cast<ConstantFPSDNode>(Op)) { 942 return CFP->getValueAPF().isZero(); 943 } 944 return isNullConstant(Op); 945 } 946 947 SDValue R600TargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const { 948 SDLoc DL(Op); 949 EVT VT = Op.getValueType(); 950 951 SDValue LHS = Op.getOperand(0); 952 SDValue RHS = Op.getOperand(1); 953 SDValue True = Op.getOperand(2); 954 SDValue False = Op.getOperand(3); 955 SDValue CC = Op.getOperand(4); 956 SDValue Temp; 957 958 if (VT == MVT::f32) { 959 DAGCombinerInfo DCI(DAG, AfterLegalizeVectorOps, true, nullptr); 960 SDValue MinMax = combineFMinMaxLegacy(DL, VT, LHS, RHS, True, False, CC, DCI); 961 if (MinMax) 962 return MinMax; 963 } 964 965 // LHS and RHS are guaranteed to be the same value type 966 EVT CompareVT = LHS.getValueType(); 967 968 // Check if we can lower this to a native operation. 969 970 // Try to lower to a SET* instruction: 971 // 972 // SET* can match the following patterns: 973 // 974 // select_cc f32, f32, -1, 0, cc_supported 975 // select_cc f32, f32, 1.0f, 0.0f, cc_supported 976 // select_cc i32, i32, -1, 0, cc_supported 977 // 978 979 // Move hardware True/False values to the correct operand. 980 if (isHWTrueValue(False) && isHWFalseValue(True)) { 981 ISD::CondCode CCOpcode = cast<CondCodeSDNode>(CC)->get(); 982 ISD::CondCode InverseCC = ISD::getSetCCInverse(CCOpcode, CompareVT); 983 if (isCondCodeLegal(InverseCC, CompareVT.getSimpleVT())) { 984 std::swap(False, True); 985 CC = DAG.getCondCode(InverseCC); 986 } else { 987 ISD::CondCode SwapInvCC = ISD::getSetCCSwappedOperands(InverseCC); 988 if (isCondCodeLegal(SwapInvCC, CompareVT.getSimpleVT())) { 989 std::swap(False, True); 990 std::swap(LHS, RHS); 991 CC = DAG.getCondCode(SwapInvCC); 992 } 993 } 994 } 995 996 if (isHWTrueValue(True) && isHWFalseValue(False) && 997 (CompareVT == VT || VT == MVT::i32)) { 998 // This can be matched by a SET* instruction. 999 return DAG.getNode(ISD::SELECT_CC, DL, VT, LHS, RHS, True, False, CC); 1000 } 1001 1002 // Try to lower to a CND* instruction: 1003 // 1004 // CND* can match the following patterns: 1005 // 1006 // select_cc f32, 0.0, f32, f32, cc_supported 1007 // select_cc f32, 0.0, i32, i32, cc_supported 1008 // select_cc i32, 0, f32, f32, cc_supported 1009 // select_cc i32, 0, i32, i32, cc_supported 1010 // 1011 1012 // Try to move the zero value to the RHS 1013 if (isZero(LHS)) { 1014 ISD::CondCode CCOpcode = cast<CondCodeSDNode>(CC)->get(); 1015 // Try swapping the operands 1016 ISD::CondCode CCSwapped = ISD::getSetCCSwappedOperands(CCOpcode); 1017 if (isCondCodeLegal(CCSwapped, CompareVT.getSimpleVT())) { 1018 std::swap(LHS, RHS); 1019 CC = DAG.getCondCode(CCSwapped); 1020 } else { 1021 // Try inverting the conditon and then swapping the operands 1022 ISD::CondCode CCInv = ISD::getSetCCInverse(CCOpcode, CompareVT); 1023 CCSwapped = ISD::getSetCCSwappedOperands(CCInv); 1024 if (isCondCodeLegal(CCSwapped, CompareVT.getSimpleVT())) { 1025 std::swap(True, False); 1026 std::swap(LHS, RHS); 1027 CC = DAG.getCondCode(CCSwapped); 1028 } 1029 } 1030 } 1031 if (isZero(RHS)) { 1032 SDValue Cond = LHS; 1033 SDValue Zero = RHS; 1034 ISD::CondCode CCOpcode = cast<CondCodeSDNode>(CC)->get(); 1035 if (CompareVT != VT) { 1036 // Bitcast True / False to the correct types. This will end up being 1037 // a nop, but it allows us to define only a single pattern in the 1038 // .TD files for each CND* instruction rather than having to have 1039 // one pattern for integer True/False and one for fp True/False 1040 True = DAG.getNode(ISD::BITCAST, DL, CompareVT, True); 1041 False = DAG.getNode(ISD::BITCAST, DL, CompareVT, False); 1042 } 1043 1044 switch (CCOpcode) { 1045 case ISD::SETONE: 1046 case ISD::SETUNE: 1047 case ISD::SETNE: 1048 CCOpcode = ISD::getSetCCInverse(CCOpcode, CompareVT); 1049 Temp = True; 1050 True = False; 1051 False = Temp; 1052 break; 1053 default: 1054 break; 1055 } 1056 SDValue SelectNode = DAG.getNode(ISD::SELECT_CC, DL, CompareVT, 1057 Cond, Zero, 1058 True, False, 1059 DAG.getCondCode(CCOpcode)); 1060 return DAG.getNode(ISD::BITCAST, DL, VT, SelectNode); 1061 } 1062 1063 // If we make it this for it means we have no native instructions to handle 1064 // this SELECT_CC, so we must lower it. 1065 SDValue HWTrue, HWFalse; 1066 1067 if (CompareVT == MVT::f32) { 1068 HWTrue = DAG.getConstantFP(1.0f, DL, CompareVT); 1069 HWFalse = DAG.getConstantFP(0.0f, DL, CompareVT); 1070 } else if (CompareVT == MVT::i32) { 1071 HWTrue = DAG.getConstant(-1, DL, CompareVT); 1072 HWFalse = DAG.getConstant(0, DL, CompareVT); 1073 } 1074 else { 1075 llvm_unreachable("Unhandled value type in LowerSELECT_CC"); 1076 } 1077 1078 // Lower this unsupported SELECT_CC into a combination of two supported 1079 // SELECT_CC operations. 1080 SDValue Cond = DAG.getNode(ISD::SELECT_CC, DL, CompareVT, LHS, RHS, HWTrue, HWFalse, CC); 1081 1082 return DAG.getNode(ISD::SELECT_CC, DL, VT, 1083 Cond, HWFalse, 1084 True, False, 1085 DAG.getCondCode(ISD::SETNE)); 1086 } 1087 1088 /// LLVM generates byte-addressed pointers. For indirect addressing, we need to 1089 /// convert these pointers to a register index. Each register holds 1090 /// 16 bytes, (4 x 32bit sub-register), but we need to take into account the 1091 /// \p StackWidth, which tells us how many of the 4 sub-registrers will be used 1092 /// for indirect addressing. 1093 SDValue R600TargetLowering::stackPtrToRegIndex(SDValue Ptr, 1094 unsigned StackWidth, 1095 SelectionDAG &DAG) const { 1096 unsigned SRLPad; 1097 switch(StackWidth) { 1098 case 1: 1099 SRLPad = 2; 1100 break; 1101 case 2: 1102 SRLPad = 3; 1103 break; 1104 case 4: 1105 SRLPad = 4; 1106 break; 1107 default: llvm_unreachable("Invalid stack width"); 1108 } 1109 1110 SDLoc DL(Ptr); 1111 return DAG.getNode(ISD::SRL, DL, Ptr.getValueType(), Ptr, 1112 DAG.getConstant(SRLPad, DL, MVT::i32)); 1113 } 1114 1115 void R600TargetLowering::getStackAddress(unsigned StackWidth, 1116 unsigned ElemIdx, 1117 unsigned &Channel, 1118 unsigned &PtrIncr) const { 1119 switch (StackWidth) { 1120 default: 1121 case 1: 1122 Channel = 0; 1123 if (ElemIdx > 0) { 1124 PtrIncr = 1; 1125 } else { 1126 PtrIncr = 0; 1127 } 1128 break; 1129 case 2: 1130 Channel = ElemIdx % 2; 1131 if (ElemIdx == 2) { 1132 PtrIncr = 1; 1133 } else { 1134 PtrIncr = 0; 1135 } 1136 break; 1137 case 4: 1138 Channel = ElemIdx; 1139 PtrIncr = 0; 1140 break; 1141 } 1142 } 1143 1144 SDValue R600TargetLowering::lowerPrivateTruncStore(StoreSDNode *Store, 1145 SelectionDAG &DAG) const { 1146 SDLoc DL(Store); 1147 //TODO: Who creates the i8 stores? 1148 assert(Store->isTruncatingStore() 1149 || Store->getValue().getValueType() == MVT::i8); 1150 assert(Store->getAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS); 1151 1152 SDValue Mask; 1153 if (Store->getMemoryVT() == MVT::i8) { 1154 assert(Store->getAlignment() >= 1); 1155 Mask = DAG.getConstant(0xff, DL, MVT::i32); 1156 } else if (Store->getMemoryVT() == MVT::i16) { 1157 assert(Store->getAlignment() >= 2); 1158 Mask = DAG.getConstant(0xffff, DL, MVT::i32); 1159 } else { 1160 llvm_unreachable("Unsupported private trunc store"); 1161 } 1162 1163 SDValue OldChain = Store->getChain(); 1164 bool VectorTrunc = (OldChain.getOpcode() == AMDGPUISD::DUMMY_CHAIN); 1165 // Skip dummy 1166 SDValue Chain = VectorTrunc ? OldChain->getOperand(0) : OldChain; 1167 SDValue BasePtr = Store->getBasePtr(); 1168 SDValue Offset = Store->getOffset(); 1169 EVT MemVT = Store->getMemoryVT(); 1170 1171 SDValue LoadPtr = BasePtr; 1172 if (!Offset.isUndef()) { 1173 LoadPtr = DAG.getNode(ISD::ADD, DL, MVT::i32, BasePtr, Offset); 1174 } 1175 1176 // Get dword location 1177 // TODO: this should be eliminated by the future SHR ptr, 2 1178 SDValue Ptr = DAG.getNode(ISD::AND, DL, MVT::i32, LoadPtr, 1179 DAG.getConstant(0xfffffffc, DL, MVT::i32)); 1180 1181 // Load dword 1182 // TODO: can we be smarter about machine pointer info? 1183 MachinePointerInfo PtrInfo(AMDGPUAS::PRIVATE_ADDRESS); 1184 SDValue Dst = DAG.getLoad(MVT::i32, DL, Chain, Ptr, PtrInfo); 1185 1186 Chain = Dst.getValue(1); 1187 1188 // Get offset in dword 1189 SDValue ByteIdx = DAG.getNode(ISD::AND, DL, MVT::i32, LoadPtr, 1190 DAG.getConstant(0x3, DL, MVT::i32)); 1191 1192 // Convert byte offset to bit shift 1193 SDValue ShiftAmt = DAG.getNode(ISD::SHL, DL, MVT::i32, ByteIdx, 1194 DAG.getConstant(3, DL, MVT::i32)); 1195 1196 // TODO: Contrary to the name of the functiom, 1197 // it also handles sub i32 non-truncating stores (like i1) 1198 SDValue SExtValue = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i32, 1199 Store->getValue()); 1200 1201 // Mask the value to the right type 1202 SDValue MaskedValue = DAG.getZeroExtendInReg(SExtValue, DL, MemVT); 1203 1204 // Shift the value in place 1205 SDValue ShiftedValue = DAG.getNode(ISD::SHL, DL, MVT::i32, 1206 MaskedValue, ShiftAmt); 1207 1208 // Shift the mask in place 1209 SDValue DstMask = DAG.getNode(ISD::SHL, DL, MVT::i32, Mask, ShiftAmt); 1210 1211 // Invert the mask. NOTE: if we had native ROL instructions we could 1212 // use inverted mask 1213 DstMask = DAG.getNOT(DL, DstMask, MVT::i32); 1214 1215 // Cleanup the target bits 1216 Dst = DAG.getNode(ISD::AND, DL, MVT::i32, Dst, DstMask); 1217 1218 // Add the new bits 1219 SDValue Value = DAG.getNode(ISD::OR, DL, MVT::i32, Dst, ShiftedValue); 1220 1221 // Store dword 1222 // TODO: Can we be smarter about MachinePointerInfo? 1223 SDValue NewStore = DAG.getStore(Chain, DL, Value, Ptr, PtrInfo); 1224 1225 // If we are part of expanded vector, make our neighbors depend on this store 1226 if (VectorTrunc) { 1227 // Make all other vector elements depend on this store 1228 Chain = DAG.getNode(AMDGPUISD::DUMMY_CHAIN, DL, MVT::Other, NewStore); 1229 DAG.ReplaceAllUsesOfValueWith(OldChain, Chain); 1230 } 1231 return NewStore; 1232 } 1233 1234 SDValue R600TargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const { 1235 StoreSDNode *StoreNode = cast<StoreSDNode>(Op); 1236 unsigned AS = StoreNode->getAddressSpace(); 1237 1238 SDValue Chain = StoreNode->getChain(); 1239 SDValue Ptr = StoreNode->getBasePtr(); 1240 SDValue Value = StoreNode->getValue(); 1241 1242 EVT VT = Value.getValueType(); 1243 EVT MemVT = StoreNode->getMemoryVT(); 1244 EVT PtrVT = Ptr.getValueType(); 1245 1246 SDLoc DL(Op); 1247 1248 const bool TruncatingStore = StoreNode->isTruncatingStore(); 1249 1250 // Neither LOCAL nor PRIVATE can do vectors at the moment 1251 if ((AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::PRIVATE_ADDRESS || 1252 TruncatingStore) && 1253 VT.isVector()) { 1254 if ((AS == AMDGPUAS::PRIVATE_ADDRESS) && TruncatingStore) { 1255 // Add an extra level of chain to isolate this vector 1256 SDValue NewChain = DAG.getNode(AMDGPUISD::DUMMY_CHAIN, DL, MVT::Other, Chain); 1257 // TODO: can the chain be replaced without creating a new store? 1258 SDValue NewStore = DAG.getTruncStore( 1259 NewChain, DL, Value, Ptr, StoreNode->getPointerInfo(), 1260 MemVT, StoreNode->getAlignment(), 1261 StoreNode->getMemOperand()->getFlags(), StoreNode->getAAInfo()); 1262 StoreNode = cast<StoreSDNode>(NewStore); 1263 } 1264 1265 return scalarizeVectorStore(StoreNode, DAG); 1266 } 1267 1268 Align Alignment = StoreNode->getAlign(); 1269 if (Alignment < MemVT.getStoreSize() && 1270 !allowsMisalignedMemoryAccesses(MemVT, AS, Alignment.value(), 1271 StoreNode->getMemOperand()->getFlags(), 1272 nullptr)) { 1273 return expandUnalignedStore(StoreNode, DAG); 1274 } 1275 1276 SDValue DWordAddr = DAG.getNode(ISD::SRL, DL, PtrVT, Ptr, 1277 DAG.getConstant(2, DL, PtrVT)); 1278 1279 if (AS == AMDGPUAS::GLOBAL_ADDRESS) { 1280 // It is beneficial to create MSKOR here instead of combiner to avoid 1281 // artificial dependencies introduced by RMW 1282 if (TruncatingStore) { 1283 assert(VT.bitsLE(MVT::i32)); 1284 SDValue MaskConstant; 1285 if (MemVT == MVT::i8) { 1286 MaskConstant = DAG.getConstant(0xFF, DL, MVT::i32); 1287 } else { 1288 assert(MemVT == MVT::i16); 1289 assert(StoreNode->getAlignment() >= 2); 1290 MaskConstant = DAG.getConstant(0xFFFF, DL, MVT::i32); 1291 } 1292 1293 SDValue ByteIndex = DAG.getNode(ISD::AND, DL, PtrVT, Ptr, 1294 DAG.getConstant(0x00000003, DL, PtrVT)); 1295 SDValue BitShift = DAG.getNode(ISD::SHL, DL, VT, ByteIndex, 1296 DAG.getConstant(3, DL, VT)); 1297 1298 // Put the mask in correct place 1299 SDValue Mask = DAG.getNode(ISD::SHL, DL, VT, MaskConstant, BitShift); 1300 1301 // Put the value bits in correct place 1302 SDValue TruncValue = DAG.getNode(ISD::AND, DL, VT, Value, MaskConstant); 1303 SDValue ShiftedValue = DAG.getNode(ISD::SHL, DL, VT, TruncValue, BitShift); 1304 1305 // XXX: If we add a 64-bit ZW register class, then we could use a 2 x i32 1306 // vector instead. 1307 SDValue Src[4] = { 1308 ShiftedValue, 1309 DAG.getConstant(0, DL, MVT::i32), 1310 DAG.getConstant(0, DL, MVT::i32), 1311 Mask 1312 }; 1313 SDValue Input = DAG.getBuildVector(MVT::v4i32, DL, Src); 1314 SDValue Args[3] = { Chain, Input, DWordAddr }; 1315 return DAG.getMemIntrinsicNode(AMDGPUISD::STORE_MSKOR, DL, 1316 Op->getVTList(), Args, MemVT, 1317 StoreNode->getMemOperand()); 1318 } else if (Ptr->getOpcode() != AMDGPUISD::DWORDADDR && VT.bitsGE(MVT::i32)) { 1319 // Convert pointer from byte address to dword address. 1320 Ptr = DAG.getNode(AMDGPUISD::DWORDADDR, DL, PtrVT, DWordAddr); 1321 1322 if (StoreNode->isIndexed()) { 1323 llvm_unreachable("Indexed stores not supported yet"); 1324 } else { 1325 Chain = DAG.getStore(Chain, DL, Value, Ptr, StoreNode->getMemOperand()); 1326 } 1327 return Chain; 1328 } 1329 } 1330 1331 // GLOBAL_ADDRESS has been handled above, LOCAL_ADDRESS allows all sizes 1332 if (AS != AMDGPUAS::PRIVATE_ADDRESS) 1333 return SDValue(); 1334 1335 if (MemVT.bitsLT(MVT::i32)) 1336 return lowerPrivateTruncStore(StoreNode, DAG); 1337 1338 // Standard i32+ store, tag it with DWORDADDR to note that the address 1339 // has been shifted 1340 if (Ptr.getOpcode() != AMDGPUISD::DWORDADDR) { 1341 Ptr = DAG.getNode(AMDGPUISD::DWORDADDR, DL, PtrVT, DWordAddr); 1342 return DAG.getStore(Chain, DL, Value, Ptr, StoreNode->getMemOperand()); 1343 } 1344 1345 // Tagged i32+ stores will be matched by patterns 1346 return SDValue(); 1347 } 1348 1349 // return (512 + (kc_bank << 12) 1350 static int 1351 ConstantAddressBlock(unsigned AddressSpace) { 1352 switch (AddressSpace) { 1353 case AMDGPUAS::CONSTANT_BUFFER_0: 1354 return 512; 1355 case AMDGPUAS::CONSTANT_BUFFER_1: 1356 return 512 + 4096; 1357 case AMDGPUAS::CONSTANT_BUFFER_2: 1358 return 512 + 4096 * 2; 1359 case AMDGPUAS::CONSTANT_BUFFER_3: 1360 return 512 + 4096 * 3; 1361 case AMDGPUAS::CONSTANT_BUFFER_4: 1362 return 512 + 4096 * 4; 1363 case AMDGPUAS::CONSTANT_BUFFER_5: 1364 return 512 + 4096 * 5; 1365 case AMDGPUAS::CONSTANT_BUFFER_6: 1366 return 512 + 4096 * 6; 1367 case AMDGPUAS::CONSTANT_BUFFER_7: 1368 return 512 + 4096 * 7; 1369 case AMDGPUAS::CONSTANT_BUFFER_8: 1370 return 512 + 4096 * 8; 1371 case AMDGPUAS::CONSTANT_BUFFER_9: 1372 return 512 + 4096 * 9; 1373 case AMDGPUAS::CONSTANT_BUFFER_10: 1374 return 512 + 4096 * 10; 1375 case AMDGPUAS::CONSTANT_BUFFER_11: 1376 return 512 + 4096 * 11; 1377 case AMDGPUAS::CONSTANT_BUFFER_12: 1378 return 512 + 4096 * 12; 1379 case AMDGPUAS::CONSTANT_BUFFER_13: 1380 return 512 + 4096 * 13; 1381 case AMDGPUAS::CONSTANT_BUFFER_14: 1382 return 512 + 4096 * 14; 1383 case AMDGPUAS::CONSTANT_BUFFER_15: 1384 return 512 + 4096 * 15; 1385 default: 1386 return -1; 1387 } 1388 } 1389 1390 SDValue R600TargetLowering::lowerPrivateExtLoad(SDValue Op, 1391 SelectionDAG &DAG) const { 1392 SDLoc DL(Op); 1393 LoadSDNode *Load = cast<LoadSDNode>(Op); 1394 ISD::LoadExtType ExtType = Load->getExtensionType(); 1395 EVT MemVT = Load->getMemoryVT(); 1396 assert(Load->getAlignment() >= MemVT.getStoreSize()); 1397 1398 SDValue BasePtr = Load->getBasePtr(); 1399 SDValue Chain = Load->getChain(); 1400 SDValue Offset = Load->getOffset(); 1401 1402 SDValue LoadPtr = BasePtr; 1403 if (!Offset.isUndef()) { 1404 LoadPtr = DAG.getNode(ISD::ADD, DL, MVT::i32, BasePtr, Offset); 1405 } 1406 1407 // Get dword location 1408 // NOTE: this should be eliminated by the future SHR ptr, 2 1409 SDValue Ptr = DAG.getNode(ISD::AND, DL, MVT::i32, LoadPtr, 1410 DAG.getConstant(0xfffffffc, DL, MVT::i32)); 1411 1412 // Load dword 1413 // TODO: can we be smarter about machine pointer info? 1414 MachinePointerInfo PtrInfo(AMDGPUAS::PRIVATE_ADDRESS); 1415 SDValue Read = DAG.getLoad(MVT::i32, DL, Chain, Ptr, PtrInfo); 1416 1417 // Get offset within the register. 1418 SDValue ByteIdx = DAG.getNode(ISD::AND, DL, MVT::i32, 1419 LoadPtr, DAG.getConstant(0x3, DL, MVT::i32)); 1420 1421 // Bit offset of target byte (byteIdx * 8). 1422 SDValue ShiftAmt = DAG.getNode(ISD::SHL, DL, MVT::i32, ByteIdx, 1423 DAG.getConstant(3, DL, MVT::i32)); 1424 1425 // Shift to the right. 1426 SDValue Ret = DAG.getNode(ISD::SRL, DL, MVT::i32, Read, ShiftAmt); 1427 1428 // Eliminate the upper bits by setting them to ... 1429 EVT MemEltVT = MemVT.getScalarType(); 1430 1431 if (ExtType == ISD::SEXTLOAD) { // ... ones. 1432 SDValue MemEltVTNode = DAG.getValueType(MemEltVT); 1433 Ret = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i32, Ret, MemEltVTNode); 1434 } else { // ... or zeros. 1435 Ret = DAG.getZeroExtendInReg(Ret, DL, MemEltVT); 1436 } 1437 1438 SDValue Ops[] = { 1439 Ret, 1440 Read.getValue(1) // This should be our output chain 1441 }; 1442 1443 return DAG.getMergeValues(Ops, DL); 1444 } 1445 1446 SDValue R600TargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const { 1447 LoadSDNode *LoadNode = cast<LoadSDNode>(Op); 1448 unsigned AS = LoadNode->getAddressSpace(); 1449 EVT MemVT = LoadNode->getMemoryVT(); 1450 ISD::LoadExtType ExtType = LoadNode->getExtensionType(); 1451 1452 if (AS == AMDGPUAS::PRIVATE_ADDRESS && 1453 ExtType != ISD::NON_EXTLOAD && MemVT.bitsLT(MVT::i32)) { 1454 return lowerPrivateExtLoad(Op, DAG); 1455 } 1456 1457 SDLoc DL(Op); 1458 EVT VT = Op.getValueType(); 1459 SDValue Chain = LoadNode->getChain(); 1460 SDValue Ptr = LoadNode->getBasePtr(); 1461 1462 if ((LoadNode->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS || 1463 LoadNode->getAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS) && 1464 VT.isVector()) { 1465 SDValue Ops[2]; 1466 std::tie(Ops[0], Ops[1]) = scalarizeVectorLoad(LoadNode, DAG); 1467 return DAG.getMergeValues(Ops, DL); 1468 } 1469 1470 // This is still used for explicit load from addrspace(8) 1471 int ConstantBlock = ConstantAddressBlock(LoadNode->getAddressSpace()); 1472 if (ConstantBlock > -1 && 1473 ((LoadNode->getExtensionType() == ISD::NON_EXTLOAD) || 1474 (LoadNode->getExtensionType() == ISD::ZEXTLOAD))) { 1475 SDValue Result; 1476 if (isa<Constant>(LoadNode->getMemOperand()->getValue()) || 1477 isa<ConstantSDNode>(Ptr)) { 1478 return constBufferLoad(LoadNode, LoadNode->getAddressSpace(), DAG); 1479 } else { 1480 //TODO: Does this even work? 1481 // non-constant ptr can't be folded, keeps it as a v4f32 load 1482 Result = DAG.getNode(AMDGPUISD::CONST_ADDRESS, DL, MVT::v4i32, 1483 DAG.getNode(ISD::SRL, DL, MVT::i32, Ptr, 1484 DAG.getConstant(4, DL, MVT::i32)), 1485 DAG.getConstant(LoadNode->getAddressSpace() - 1486 AMDGPUAS::CONSTANT_BUFFER_0, DL, MVT::i32) 1487 ); 1488 } 1489 1490 if (!VT.isVector()) { 1491 Result = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, Result, 1492 DAG.getConstant(0, DL, MVT::i32)); 1493 } 1494 1495 SDValue MergedValues[2] = { 1496 Result, 1497 Chain 1498 }; 1499 return DAG.getMergeValues(MergedValues, DL); 1500 } 1501 1502 // For most operations returning SDValue() will result in the node being 1503 // expanded by the DAG Legalizer. This is not the case for ISD::LOAD, so we 1504 // need to manually expand loads that may be legal in some address spaces and 1505 // illegal in others. SEXT loads from CONSTANT_BUFFER_0 are supported for 1506 // compute shaders, since the data is sign extended when it is uploaded to the 1507 // buffer. However SEXT loads from other address spaces are not supported, so 1508 // we need to expand them here. 1509 if (LoadNode->getExtensionType() == ISD::SEXTLOAD) { 1510 assert(!MemVT.isVector() && (MemVT == MVT::i16 || MemVT == MVT::i8)); 1511 SDValue NewLoad = DAG.getExtLoad( 1512 ISD::EXTLOAD, DL, VT, Chain, Ptr, LoadNode->getPointerInfo(), MemVT, 1513 LoadNode->getAlignment(), LoadNode->getMemOperand()->getFlags()); 1514 SDValue Res = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, NewLoad, 1515 DAG.getValueType(MemVT)); 1516 1517 SDValue MergedValues[2] = { Res, Chain }; 1518 return DAG.getMergeValues(MergedValues, DL); 1519 } 1520 1521 if (LoadNode->getAddressSpace() != AMDGPUAS::PRIVATE_ADDRESS) { 1522 return SDValue(); 1523 } 1524 1525 // DWORDADDR ISD marks already shifted address 1526 if (Ptr.getOpcode() != AMDGPUISD::DWORDADDR) { 1527 assert(VT == MVT::i32); 1528 Ptr = DAG.getNode(ISD::SRL, DL, MVT::i32, Ptr, DAG.getConstant(2, DL, MVT::i32)); 1529 Ptr = DAG.getNode(AMDGPUISD::DWORDADDR, DL, MVT::i32, Ptr); 1530 return DAG.getLoad(MVT::i32, DL, Chain, Ptr, LoadNode->getMemOperand()); 1531 } 1532 return SDValue(); 1533 } 1534 1535 SDValue R600TargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const { 1536 SDValue Chain = Op.getOperand(0); 1537 SDValue Cond = Op.getOperand(1); 1538 SDValue Jump = Op.getOperand(2); 1539 1540 return DAG.getNode(AMDGPUISD::BRANCH_COND, SDLoc(Op), Op.getValueType(), 1541 Chain, Jump, Cond); 1542 } 1543 1544 SDValue R600TargetLowering::lowerFrameIndex(SDValue Op, 1545 SelectionDAG &DAG) const { 1546 MachineFunction &MF = DAG.getMachineFunction(); 1547 const R600FrameLowering *TFL = Subtarget->getFrameLowering(); 1548 1549 FrameIndexSDNode *FIN = cast<FrameIndexSDNode>(Op); 1550 1551 unsigned FrameIndex = FIN->getIndex(); 1552 Register IgnoredFrameReg; 1553 unsigned Offset = 1554 TFL->getFrameIndexReference(MF, FrameIndex, IgnoredFrameReg); 1555 return DAG.getConstant(Offset * 4 * TFL->getStackWidth(MF), SDLoc(Op), 1556 Op.getValueType()); 1557 } 1558 1559 CCAssignFn *R600TargetLowering::CCAssignFnForCall(CallingConv::ID CC, 1560 bool IsVarArg) const { 1561 switch (CC) { 1562 case CallingConv::AMDGPU_KERNEL: 1563 case CallingConv::SPIR_KERNEL: 1564 case CallingConv::C: 1565 case CallingConv::Fast: 1566 case CallingConv::Cold: 1567 llvm_unreachable("kernels should not be handled here"); 1568 case CallingConv::AMDGPU_VS: 1569 case CallingConv::AMDGPU_GS: 1570 case CallingConv::AMDGPU_PS: 1571 case CallingConv::AMDGPU_CS: 1572 case CallingConv::AMDGPU_HS: 1573 case CallingConv::AMDGPU_ES: 1574 case CallingConv::AMDGPU_LS: 1575 return CC_R600; 1576 default: 1577 report_fatal_error("Unsupported calling convention."); 1578 } 1579 } 1580 1581 /// XXX Only kernel functions are supported, so we can assume for now that 1582 /// every function is a kernel function, but in the future we should use 1583 /// separate calling conventions for kernel and non-kernel functions. 1584 SDValue R600TargetLowering::LowerFormalArguments( 1585 SDValue Chain, CallingConv::ID CallConv, bool isVarArg, 1586 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL, 1587 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const { 1588 SmallVector<CCValAssign, 16> ArgLocs; 1589 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), ArgLocs, 1590 *DAG.getContext()); 1591 MachineFunction &MF = DAG.getMachineFunction(); 1592 SmallVector<ISD::InputArg, 8> LocalIns; 1593 1594 if (AMDGPU::isShader(CallConv)) { 1595 CCInfo.AnalyzeFormalArguments(Ins, CCAssignFnForCall(CallConv, isVarArg)); 1596 } else { 1597 analyzeFormalArgumentsCompute(CCInfo, Ins); 1598 } 1599 1600 for (unsigned i = 0, e = Ins.size(); i < e; ++i) { 1601 CCValAssign &VA = ArgLocs[i]; 1602 const ISD::InputArg &In = Ins[i]; 1603 EVT VT = In.VT; 1604 EVT MemVT = VA.getLocVT(); 1605 if (!VT.isVector() && MemVT.isVector()) { 1606 // Get load source type if scalarized. 1607 MemVT = MemVT.getVectorElementType(); 1608 } 1609 1610 if (AMDGPU::isShader(CallConv)) { 1611 unsigned Reg = MF.addLiveIn(VA.getLocReg(), &R600::R600_Reg128RegClass); 1612 SDValue Register = DAG.getCopyFromReg(Chain, DL, Reg, VT); 1613 InVals.push_back(Register); 1614 continue; 1615 } 1616 1617 // i64 isn't a legal type, so the register type used ends up as i32, which 1618 // isn't expected here. It attempts to create this sextload, but it ends up 1619 // being invalid. Somehow this seems to work with i64 arguments, but breaks 1620 // for <1 x i64>. 1621 1622 // The first 36 bytes of the input buffer contains information about 1623 // thread group and global sizes. 1624 ISD::LoadExtType Ext = ISD::NON_EXTLOAD; 1625 if (MemVT.getScalarSizeInBits() != VT.getScalarSizeInBits()) { 1626 // FIXME: This should really check the extload type, but the handling of 1627 // extload vector parameters seems to be broken. 1628 1629 // Ext = In.Flags.isSExt() ? ISD::SEXTLOAD : ISD::ZEXTLOAD; 1630 Ext = ISD::SEXTLOAD; 1631 } 1632 1633 // Compute the offset from the value. 1634 // XXX - I think PartOffset should give you this, but it seems to give the 1635 // size of the register which isn't useful. 1636 1637 unsigned PartOffset = VA.getLocMemOffset(); 1638 unsigned Alignment = MinAlign(VT.getStoreSize(), PartOffset); 1639 1640 MachinePointerInfo PtrInfo(AMDGPUAS::PARAM_I_ADDRESS); 1641 SDValue Arg = DAG.getLoad( 1642 ISD::UNINDEXED, Ext, VT, DL, Chain, 1643 DAG.getConstant(PartOffset, DL, MVT::i32), DAG.getUNDEF(MVT::i32), 1644 PtrInfo, 1645 MemVT, Alignment, MachineMemOperand::MONonTemporal | 1646 MachineMemOperand::MODereferenceable | 1647 MachineMemOperand::MOInvariant); 1648 1649 InVals.push_back(Arg); 1650 } 1651 return Chain; 1652 } 1653 1654 EVT R600TargetLowering::getSetCCResultType(const DataLayout &DL, LLVMContext &, 1655 EVT VT) const { 1656 if (!VT.isVector()) 1657 return MVT::i32; 1658 return VT.changeVectorElementTypeToInteger(); 1659 } 1660 1661 bool R600TargetLowering::canMergeStoresTo(unsigned AS, EVT MemVT, 1662 const SelectionDAG &DAG) const { 1663 // Local and Private addresses do not handle vectors. Limit to i32 1664 if ((AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::PRIVATE_ADDRESS)) { 1665 return (MemVT.getSizeInBits() <= 32); 1666 } 1667 return true; 1668 } 1669 1670 bool R600TargetLowering::allowsMisalignedMemoryAccesses( 1671 EVT VT, unsigned AddrSpace, unsigned Align, MachineMemOperand::Flags Flags, 1672 bool *IsFast) const { 1673 if (IsFast) 1674 *IsFast = false; 1675 1676 if (!VT.isSimple() || VT == MVT::Other) 1677 return false; 1678 1679 if (VT.bitsLT(MVT::i32)) 1680 return false; 1681 1682 // TODO: This is a rough estimate. 1683 if (IsFast) 1684 *IsFast = true; 1685 1686 return VT.bitsGT(MVT::i32) && Align % 4 == 0; 1687 } 1688 1689 static SDValue CompactSwizzlableVector( 1690 SelectionDAG &DAG, SDValue VectorEntry, 1691 DenseMap<unsigned, unsigned> &RemapSwizzle) { 1692 assert(RemapSwizzle.empty()); 1693 1694 SDLoc DL(VectorEntry); 1695 EVT EltTy = VectorEntry.getValueType().getVectorElementType(); 1696 1697 SDValue NewBldVec[4]; 1698 for (unsigned i = 0; i < 4; i++) 1699 NewBldVec[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltTy, VectorEntry, 1700 DAG.getIntPtrConstant(i, DL)); 1701 1702 for (unsigned i = 0; i < 4; i++) { 1703 if (NewBldVec[i].isUndef()) 1704 // We mask write here to teach later passes that the ith element of this 1705 // vector is undef. Thus we can use it to reduce 128 bits reg usage, 1706 // break false dependencies and additionnaly make assembly easier to read. 1707 RemapSwizzle[i] = 7; // SEL_MASK_WRITE 1708 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(NewBldVec[i])) { 1709 if (C->isZero()) { 1710 RemapSwizzle[i] = 4; // SEL_0 1711 NewBldVec[i] = DAG.getUNDEF(MVT::f32); 1712 } else if (C->isExactlyValue(1.0)) { 1713 RemapSwizzle[i] = 5; // SEL_1 1714 NewBldVec[i] = DAG.getUNDEF(MVT::f32); 1715 } 1716 } 1717 1718 if (NewBldVec[i].isUndef()) 1719 continue; 1720 1721 for (unsigned j = 0; j < i; j++) { 1722 if (NewBldVec[i] == NewBldVec[j]) { 1723 NewBldVec[i] = DAG.getUNDEF(NewBldVec[i].getValueType()); 1724 RemapSwizzle[i] = j; 1725 break; 1726 } 1727 } 1728 } 1729 1730 return DAG.getBuildVector(VectorEntry.getValueType(), SDLoc(VectorEntry), 1731 NewBldVec); 1732 } 1733 1734 static SDValue ReorganizeVector(SelectionDAG &DAG, SDValue VectorEntry, 1735 DenseMap<unsigned, unsigned> &RemapSwizzle) { 1736 assert(RemapSwizzle.empty()); 1737 1738 SDLoc DL(VectorEntry); 1739 EVT EltTy = VectorEntry.getValueType().getVectorElementType(); 1740 1741 SDValue NewBldVec[4]; 1742 bool isUnmovable[4] = {false, false, false, false}; 1743 for (unsigned i = 0; i < 4; i++) 1744 NewBldVec[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltTy, VectorEntry, 1745 DAG.getIntPtrConstant(i, DL)); 1746 1747 for (unsigned i = 0; i < 4; i++) { 1748 RemapSwizzle[i] = i; 1749 if (NewBldVec[i].getOpcode() == ISD::EXTRACT_VECTOR_ELT) { 1750 unsigned Idx = dyn_cast<ConstantSDNode>(NewBldVec[i].getOperand(1)) 1751 ->getZExtValue(); 1752 if (i == Idx) 1753 isUnmovable[Idx] = true; 1754 } 1755 } 1756 1757 for (unsigned i = 0; i < 4; i++) { 1758 if (NewBldVec[i].getOpcode() == ISD::EXTRACT_VECTOR_ELT) { 1759 unsigned Idx = dyn_cast<ConstantSDNode>(NewBldVec[i].getOperand(1)) 1760 ->getZExtValue(); 1761 if (isUnmovable[Idx]) 1762 continue; 1763 // Swap i and Idx 1764 std::swap(NewBldVec[Idx], NewBldVec[i]); 1765 std::swap(RemapSwizzle[i], RemapSwizzle[Idx]); 1766 break; 1767 } 1768 } 1769 1770 return DAG.getBuildVector(VectorEntry.getValueType(), SDLoc(VectorEntry), 1771 NewBldVec); 1772 } 1773 1774 SDValue R600TargetLowering::OptimizeSwizzle(SDValue BuildVector, SDValue Swz[4], 1775 SelectionDAG &DAG, 1776 const SDLoc &DL) const { 1777 // Old -> New swizzle values 1778 DenseMap<unsigned, unsigned> SwizzleRemap; 1779 1780 BuildVector = CompactSwizzlableVector(DAG, BuildVector, SwizzleRemap); 1781 for (unsigned i = 0; i < 4; i++) { 1782 unsigned Idx = cast<ConstantSDNode>(Swz[i])->getZExtValue(); 1783 if (SwizzleRemap.find(Idx) != SwizzleRemap.end()) 1784 Swz[i] = DAG.getConstant(SwizzleRemap[Idx], DL, MVT::i32); 1785 } 1786 1787 SwizzleRemap.clear(); 1788 BuildVector = ReorganizeVector(DAG, BuildVector, SwizzleRemap); 1789 for (unsigned i = 0; i < 4; i++) { 1790 unsigned Idx = cast<ConstantSDNode>(Swz[i])->getZExtValue(); 1791 if (SwizzleRemap.find(Idx) != SwizzleRemap.end()) 1792 Swz[i] = DAG.getConstant(SwizzleRemap[Idx], DL, MVT::i32); 1793 } 1794 1795 return BuildVector; 1796 } 1797 1798 SDValue R600TargetLowering::constBufferLoad(LoadSDNode *LoadNode, int Block, 1799 SelectionDAG &DAG) const { 1800 SDLoc DL(LoadNode); 1801 EVT VT = LoadNode->getValueType(0); 1802 SDValue Chain = LoadNode->getChain(); 1803 SDValue Ptr = LoadNode->getBasePtr(); 1804 assert (isa<ConstantSDNode>(Ptr)); 1805 1806 //TODO: Support smaller loads 1807 if (LoadNode->getMemoryVT().getScalarType() != MVT::i32 || !ISD::isNON_EXTLoad(LoadNode)) 1808 return SDValue(); 1809 1810 if (LoadNode->getAlignment() < 4) 1811 return SDValue(); 1812 1813 int ConstantBlock = ConstantAddressBlock(Block); 1814 1815 SDValue Slots[4]; 1816 for (unsigned i = 0; i < 4; i++) { 1817 // We want Const position encoded with the following formula : 1818 // (((512 + (kc_bank << 12) + const_index) << 2) + chan) 1819 // const_index is Ptr computed by llvm using an alignment of 16. 1820 // Thus we add (((512 + (kc_bank << 12)) + chan ) * 4 here and 1821 // then div by 4 at the ISel step 1822 SDValue NewPtr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr, 1823 DAG.getConstant(4 * i + ConstantBlock * 16, DL, MVT::i32)); 1824 Slots[i] = DAG.getNode(AMDGPUISD::CONST_ADDRESS, DL, MVT::i32, NewPtr); 1825 } 1826 EVT NewVT = MVT::v4i32; 1827 unsigned NumElements = 4; 1828 if (VT.isVector()) { 1829 NewVT = VT; 1830 NumElements = VT.getVectorNumElements(); 1831 } 1832 SDValue Result = DAG.getBuildVector(NewVT, DL, makeArrayRef(Slots, NumElements)); 1833 if (!VT.isVector()) { 1834 Result = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, Result, 1835 DAG.getConstant(0, DL, MVT::i32)); 1836 } 1837 SDValue MergedValues[2] = { 1838 Result, 1839 Chain 1840 }; 1841 return DAG.getMergeValues(MergedValues, DL); 1842 } 1843 1844 //===----------------------------------------------------------------------===// 1845 // Custom DAG Optimizations 1846 //===----------------------------------------------------------------------===// 1847 1848 SDValue R600TargetLowering::PerformDAGCombine(SDNode *N, 1849 DAGCombinerInfo &DCI) const { 1850 SelectionDAG &DAG = DCI.DAG; 1851 SDLoc DL(N); 1852 1853 switch (N->getOpcode()) { 1854 // (f32 fp_round (f64 uint_to_fp a)) -> (f32 uint_to_fp a) 1855 case ISD::FP_ROUND: { 1856 SDValue Arg = N->getOperand(0); 1857 if (Arg.getOpcode() == ISD::UINT_TO_FP && Arg.getValueType() == MVT::f64) { 1858 return DAG.getNode(ISD::UINT_TO_FP, DL, N->getValueType(0), 1859 Arg.getOperand(0)); 1860 } 1861 break; 1862 } 1863 1864 // (i32 fp_to_sint (fneg (select_cc f32, f32, 1.0, 0.0 cc))) -> 1865 // (i32 select_cc f32, f32, -1, 0 cc) 1866 // 1867 // Mesa's GLSL frontend generates the above pattern a lot and we can lower 1868 // this to one of the SET*_DX10 instructions. 1869 case ISD::FP_TO_SINT: { 1870 SDValue FNeg = N->getOperand(0); 1871 if (FNeg.getOpcode() != ISD::FNEG) { 1872 return SDValue(); 1873 } 1874 SDValue SelectCC = FNeg.getOperand(0); 1875 if (SelectCC.getOpcode() != ISD::SELECT_CC || 1876 SelectCC.getOperand(0).getValueType() != MVT::f32 || // LHS 1877 SelectCC.getOperand(2).getValueType() != MVT::f32 || // True 1878 !isHWTrueValue(SelectCC.getOperand(2)) || 1879 !isHWFalseValue(SelectCC.getOperand(3))) { 1880 return SDValue(); 1881 } 1882 1883 return DAG.getNode(ISD::SELECT_CC, DL, N->getValueType(0), 1884 SelectCC.getOperand(0), // LHS 1885 SelectCC.getOperand(1), // RHS 1886 DAG.getConstant(-1, DL, MVT::i32), // True 1887 DAG.getConstant(0, DL, MVT::i32), // False 1888 SelectCC.getOperand(4)); // CC 1889 } 1890 1891 // insert_vector_elt (build_vector elt0, ... , eltN), NewEltIdx, idx 1892 // => build_vector elt0, ... , NewEltIdx, ... , eltN 1893 case ISD::INSERT_VECTOR_ELT: { 1894 SDValue InVec = N->getOperand(0); 1895 SDValue InVal = N->getOperand(1); 1896 SDValue EltNo = N->getOperand(2); 1897 1898 // If the inserted element is an UNDEF, just use the input vector. 1899 if (InVal.isUndef()) 1900 return InVec; 1901 1902 EVT VT = InVec.getValueType(); 1903 1904 // If we can't generate a legal BUILD_VECTOR, exit 1905 if (!isOperationLegal(ISD::BUILD_VECTOR, VT)) 1906 return SDValue(); 1907 1908 // Check that we know which element is being inserted 1909 if (!isa<ConstantSDNode>(EltNo)) 1910 return SDValue(); 1911 unsigned Elt = cast<ConstantSDNode>(EltNo)->getZExtValue(); 1912 1913 // Check that the operand is a BUILD_VECTOR (or UNDEF, which can essentially 1914 // be converted to a BUILD_VECTOR). Fill in the Ops vector with the 1915 // vector elements. 1916 SmallVector<SDValue, 8> Ops; 1917 if (InVec.getOpcode() == ISD::BUILD_VECTOR) { 1918 Ops.append(InVec.getNode()->op_begin(), 1919 InVec.getNode()->op_end()); 1920 } else if (InVec.isUndef()) { 1921 unsigned NElts = VT.getVectorNumElements(); 1922 Ops.append(NElts, DAG.getUNDEF(InVal.getValueType())); 1923 } else { 1924 return SDValue(); 1925 } 1926 1927 // Insert the element 1928 if (Elt < Ops.size()) { 1929 // All the operands of BUILD_VECTOR must have the same type; 1930 // we enforce that here. 1931 EVT OpVT = Ops[0].getValueType(); 1932 if (InVal.getValueType() != OpVT) 1933 InVal = OpVT.bitsGT(InVal.getValueType()) ? 1934 DAG.getNode(ISD::ANY_EXTEND, DL, OpVT, InVal) : 1935 DAG.getNode(ISD::TRUNCATE, DL, OpVT, InVal); 1936 Ops[Elt] = InVal; 1937 } 1938 1939 // Return the new vector 1940 return DAG.getBuildVector(VT, DL, Ops); 1941 } 1942 1943 // Extract_vec (Build_vector) generated by custom lowering 1944 // also needs to be customly combined 1945 case ISD::EXTRACT_VECTOR_ELT: { 1946 SDValue Arg = N->getOperand(0); 1947 if (Arg.getOpcode() == ISD::BUILD_VECTOR) { 1948 if (ConstantSDNode *Const = dyn_cast<ConstantSDNode>(N->getOperand(1))) { 1949 unsigned Element = Const->getZExtValue(); 1950 return Arg->getOperand(Element); 1951 } 1952 } 1953 if (Arg.getOpcode() == ISD::BITCAST && 1954 Arg.getOperand(0).getOpcode() == ISD::BUILD_VECTOR && 1955 (Arg.getOperand(0).getValueType().getVectorNumElements() == 1956 Arg.getValueType().getVectorNumElements())) { 1957 if (ConstantSDNode *Const = dyn_cast<ConstantSDNode>(N->getOperand(1))) { 1958 unsigned Element = Const->getZExtValue(); 1959 return DAG.getNode(ISD::BITCAST, DL, N->getVTList(), 1960 Arg->getOperand(0).getOperand(Element)); 1961 } 1962 } 1963 break; 1964 } 1965 1966 case ISD::SELECT_CC: { 1967 // Try common optimizations 1968 if (SDValue Ret = AMDGPUTargetLowering::PerformDAGCombine(N, DCI)) 1969 return Ret; 1970 1971 // fold selectcc (selectcc x, y, a, b, cc), b, a, b, seteq -> 1972 // selectcc x, y, a, b, inv(cc) 1973 // 1974 // fold selectcc (selectcc x, y, a, b, cc), b, a, b, setne -> 1975 // selectcc x, y, a, b, cc 1976 SDValue LHS = N->getOperand(0); 1977 if (LHS.getOpcode() != ISD::SELECT_CC) { 1978 return SDValue(); 1979 } 1980 1981 SDValue RHS = N->getOperand(1); 1982 SDValue True = N->getOperand(2); 1983 SDValue False = N->getOperand(3); 1984 ISD::CondCode NCC = cast<CondCodeSDNode>(N->getOperand(4))->get(); 1985 1986 if (LHS.getOperand(2).getNode() != True.getNode() || 1987 LHS.getOperand(3).getNode() != False.getNode() || 1988 RHS.getNode() != False.getNode()) { 1989 return SDValue(); 1990 } 1991 1992 switch (NCC) { 1993 default: return SDValue(); 1994 case ISD::SETNE: return LHS; 1995 case ISD::SETEQ: { 1996 ISD::CondCode LHSCC = cast<CondCodeSDNode>(LHS.getOperand(4))->get(); 1997 LHSCC = ISD::getSetCCInverse(LHSCC, LHS.getOperand(0).getValueType()); 1998 if (DCI.isBeforeLegalizeOps() || 1999 isCondCodeLegal(LHSCC, LHS.getOperand(0).getSimpleValueType())) 2000 return DAG.getSelectCC(DL, 2001 LHS.getOperand(0), 2002 LHS.getOperand(1), 2003 LHS.getOperand(2), 2004 LHS.getOperand(3), 2005 LHSCC); 2006 break; 2007 } 2008 } 2009 return SDValue(); 2010 } 2011 2012 case AMDGPUISD::R600_EXPORT: { 2013 SDValue Arg = N->getOperand(1); 2014 if (Arg.getOpcode() != ISD::BUILD_VECTOR) 2015 break; 2016 2017 SDValue NewArgs[8] = { 2018 N->getOperand(0), // Chain 2019 SDValue(), 2020 N->getOperand(2), // ArrayBase 2021 N->getOperand(3), // Type 2022 N->getOperand(4), // SWZ_X 2023 N->getOperand(5), // SWZ_Y 2024 N->getOperand(6), // SWZ_Z 2025 N->getOperand(7) // SWZ_W 2026 }; 2027 NewArgs[1] = OptimizeSwizzle(N->getOperand(1), &NewArgs[4], DAG, DL); 2028 return DAG.getNode(AMDGPUISD::R600_EXPORT, DL, N->getVTList(), NewArgs); 2029 } 2030 case AMDGPUISD::TEXTURE_FETCH: { 2031 SDValue Arg = N->getOperand(1); 2032 if (Arg.getOpcode() != ISD::BUILD_VECTOR) 2033 break; 2034 2035 SDValue NewArgs[19] = { 2036 N->getOperand(0), 2037 N->getOperand(1), 2038 N->getOperand(2), 2039 N->getOperand(3), 2040 N->getOperand(4), 2041 N->getOperand(5), 2042 N->getOperand(6), 2043 N->getOperand(7), 2044 N->getOperand(8), 2045 N->getOperand(9), 2046 N->getOperand(10), 2047 N->getOperand(11), 2048 N->getOperand(12), 2049 N->getOperand(13), 2050 N->getOperand(14), 2051 N->getOperand(15), 2052 N->getOperand(16), 2053 N->getOperand(17), 2054 N->getOperand(18), 2055 }; 2056 NewArgs[1] = OptimizeSwizzle(N->getOperand(1), &NewArgs[2], DAG, DL); 2057 return DAG.getNode(AMDGPUISD::TEXTURE_FETCH, DL, N->getVTList(), NewArgs); 2058 } 2059 2060 case ISD::LOAD: { 2061 LoadSDNode *LoadNode = cast<LoadSDNode>(N); 2062 SDValue Ptr = LoadNode->getBasePtr(); 2063 if (LoadNode->getAddressSpace() == AMDGPUAS::PARAM_I_ADDRESS && 2064 isa<ConstantSDNode>(Ptr)) 2065 return constBufferLoad(LoadNode, AMDGPUAS::CONSTANT_BUFFER_0, DAG); 2066 break; 2067 } 2068 2069 default: break; 2070 } 2071 2072 return AMDGPUTargetLowering::PerformDAGCombine(N, DCI); 2073 } 2074 2075 bool R600TargetLowering::FoldOperand(SDNode *ParentNode, unsigned SrcIdx, 2076 SDValue &Src, SDValue &Neg, SDValue &Abs, 2077 SDValue &Sel, SDValue &Imm, 2078 SelectionDAG &DAG) const { 2079 const R600InstrInfo *TII = Subtarget->getInstrInfo(); 2080 if (!Src.isMachineOpcode()) 2081 return false; 2082 2083 switch (Src.getMachineOpcode()) { 2084 case R600::FNEG_R600: 2085 if (!Neg.getNode()) 2086 return false; 2087 Src = Src.getOperand(0); 2088 Neg = DAG.getTargetConstant(1, SDLoc(ParentNode), MVT::i32); 2089 return true; 2090 case R600::FABS_R600: 2091 if (!Abs.getNode()) 2092 return false; 2093 Src = Src.getOperand(0); 2094 Abs = DAG.getTargetConstant(1, SDLoc(ParentNode), MVT::i32); 2095 return true; 2096 case R600::CONST_COPY: { 2097 unsigned Opcode = ParentNode->getMachineOpcode(); 2098 bool HasDst = TII->getOperandIdx(Opcode, R600::OpName::dst) > -1; 2099 2100 if (!Sel.getNode()) 2101 return false; 2102 2103 SDValue CstOffset = Src.getOperand(0); 2104 if (ParentNode->getValueType(0).isVector()) 2105 return false; 2106 2107 // Gather constants values 2108 int SrcIndices[] = { 2109 TII->getOperandIdx(Opcode, R600::OpName::src0), 2110 TII->getOperandIdx(Opcode, R600::OpName::src1), 2111 TII->getOperandIdx(Opcode, R600::OpName::src2), 2112 TII->getOperandIdx(Opcode, R600::OpName::src0_X), 2113 TII->getOperandIdx(Opcode, R600::OpName::src0_Y), 2114 TII->getOperandIdx(Opcode, R600::OpName::src0_Z), 2115 TII->getOperandIdx(Opcode, R600::OpName::src0_W), 2116 TII->getOperandIdx(Opcode, R600::OpName::src1_X), 2117 TII->getOperandIdx(Opcode, R600::OpName::src1_Y), 2118 TII->getOperandIdx(Opcode, R600::OpName::src1_Z), 2119 TII->getOperandIdx(Opcode, R600::OpName::src1_W) 2120 }; 2121 std::vector<unsigned> Consts; 2122 for (int OtherSrcIdx : SrcIndices) { 2123 int OtherSelIdx = TII->getSelIdx(Opcode, OtherSrcIdx); 2124 if (OtherSrcIdx < 0 || OtherSelIdx < 0) 2125 continue; 2126 if (HasDst) { 2127 OtherSrcIdx--; 2128 OtherSelIdx--; 2129 } 2130 if (RegisterSDNode *Reg = 2131 dyn_cast<RegisterSDNode>(ParentNode->getOperand(OtherSrcIdx))) { 2132 if (Reg->getReg() == R600::ALU_CONST) { 2133 ConstantSDNode *Cst 2134 = cast<ConstantSDNode>(ParentNode->getOperand(OtherSelIdx)); 2135 Consts.push_back(Cst->getZExtValue()); 2136 } 2137 } 2138 } 2139 2140 ConstantSDNode *Cst = cast<ConstantSDNode>(CstOffset); 2141 Consts.push_back(Cst->getZExtValue()); 2142 if (!TII->fitsConstReadLimitations(Consts)) { 2143 return false; 2144 } 2145 2146 Sel = CstOffset; 2147 Src = DAG.getRegister(R600::ALU_CONST, MVT::f32); 2148 return true; 2149 } 2150 case R600::MOV_IMM_GLOBAL_ADDR: 2151 // Check if the Imm slot is used. Taken from below. 2152 if (cast<ConstantSDNode>(Imm)->getZExtValue()) 2153 return false; 2154 Imm = Src.getOperand(0); 2155 Src = DAG.getRegister(R600::ALU_LITERAL_X, MVT::i32); 2156 return true; 2157 case R600::MOV_IMM_I32: 2158 case R600::MOV_IMM_F32: { 2159 unsigned ImmReg = R600::ALU_LITERAL_X; 2160 uint64_t ImmValue = 0; 2161 2162 if (Src.getMachineOpcode() == R600::MOV_IMM_F32) { 2163 ConstantFPSDNode *FPC = dyn_cast<ConstantFPSDNode>(Src.getOperand(0)); 2164 float FloatValue = FPC->getValueAPF().convertToFloat(); 2165 if (FloatValue == 0.0) { 2166 ImmReg = R600::ZERO; 2167 } else if (FloatValue == 0.5) { 2168 ImmReg = R600::HALF; 2169 } else if (FloatValue == 1.0) { 2170 ImmReg = R600::ONE; 2171 } else { 2172 ImmValue = FPC->getValueAPF().bitcastToAPInt().getZExtValue(); 2173 } 2174 } else { 2175 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Src.getOperand(0)); 2176 uint64_t Value = C->getZExtValue(); 2177 if (Value == 0) { 2178 ImmReg = R600::ZERO; 2179 } else if (Value == 1) { 2180 ImmReg = R600::ONE_INT; 2181 } else { 2182 ImmValue = Value; 2183 } 2184 } 2185 2186 // Check that we aren't already using an immediate. 2187 // XXX: It's possible for an instruction to have more than one 2188 // immediate operand, but this is not supported yet. 2189 if (ImmReg == R600::ALU_LITERAL_X) { 2190 if (!Imm.getNode()) 2191 return false; 2192 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Imm); 2193 assert(C); 2194 if (C->getZExtValue()) 2195 return false; 2196 Imm = DAG.getTargetConstant(ImmValue, SDLoc(ParentNode), MVT::i32); 2197 } 2198 Src = DAG.getRegister(ImmReg, MVT::i32); 2199 return true; 2200 } 2201 default: 2202 return false; 2203 } 2204 } 2205 2206 /// Fold the instructions after selecting them 2207 SDNode *R600TargetLowering::PostISelFolding(MachineSDNode *Node, 2208 SelectionDAG &DAG) const { 2209 const R600InstrInfo *TII = Subtarget->getInstrInfo(); 2210 if (!Node->isMachineOpcode()) 2211 return Node; 2212 2213 unsigned Opcode = Node->getMachineOpcode(); 2214 SDValue FakeOp; 2215 2216 std::vector<SDValue> Ops(Node->op_begin(), Node->op_end()); 2217 2218 if (Opcode == R600::DOT_4) { 2219 int OperandIdx[] = { 2220 TII->getOperandIdx(Opcode, R600::OpName::src0_X), 2221 TII->getOperandIdx(Opcode, R600::OpName::src0_Y), 2222 TII->getOperandIdx(Opcode, R600::OpName::src0_Z), 2223 TII->getOperandIdx(Opcode, R600::OpName::src0_W), 2224 TII->getOperandIdx(Opcode, R600::OpName::src1_X), 2225 TII->getOperandIdx(Opcode, R600::OpName::src1_Y), 2226 TII->getOperandIdx(Opcode, R600::OpName::src1_Z), 2227 TII->getOperandIdx(Opcode, R600::OpName::src1_W) 2228 }; 2229 int NegIdx[] = { 2230 TII->getOperandIdx(Opcode, R600::OpName::src0_neg_X), 2231 TII->getOperandIdx(Opcode, R600::OpName::src0_neg_Y), 2232 TII->getOperandIdx(Opcode, R600::OpName::src0_neg_Z), 2233 TII->getOperandIdx(Opcode, R600::OpName::src0_neg_W), 2234 TII->getOperandIdx(Opcode, R600::OpName::src1_neg_X), 2235 TII->getOperandIdx(Opcode, R600::OpName::src1_neg_Y), 2236 TII->getOperandIdx(Opcode, R600::OpName::src1_neg_Z), 2237 TII->getOperandIdx(Opcode, R600::OpName::src1_neg_W) 2238 }; 2239 int AbsIdx[] = { 2240 TII->getOperandIdx(Opcode, R600::OpName::src0_abs_X), 2241 TII->getOperandIdx(Opcode, R600::OpName::src0_abs_Y), 2242 TII->getOperandIdx(Opcode, R600::OpName::src0_abs_Z), 2243 TII->getOperandIdx(Opcode, R600::OpName::src0_abs_W), 2244 TII->getOperandIdx(Opcode, R600::OpName::src1_abs_X), 2245 TII->getOperandIdx(Opcode, R600::OpName::src1_abs_Y), 2246 TII->getOperandIdx(Opcode, R600::OpName::src1_abs_Z), 2247 TII->getOperandIdx(Opcode, R600::OpName::src1_abs_W) 2248 }; 2249 for (unsigned i = 0; i < 8; i++) { 2250 if (OperandIdx[i] < 0) 2251 return Node; 2252 SDValue &Src = Ops[OperandIdx[i] - 1]; 2253 SDValue &Neg = Ops[NegIdx[i] - 1]; 2254 SDValue &Abs = Ops[AbsIdx[i] - 1]; 2255 bool HasDst = TII->getOperandIdx(Opcode, R600::OpName::dst) > -1; 2256 int SelIdx = TII->getSelIdx(Opcode, OperandIdx[i]); 2257 if (HasDst) 2258 SelIdx--; 2259 SDValue &Sel = (SelIdx > -1) ? Ops[SelIdx] : FakeOp; 2260 if (FoldOperand(Node, i, Src, Neg, Abs, Sel, FakeOp, DAG)) 2261 return DAG.getMachineNode(Opcode, SDLoc(Node), Node->getVTList(), Ops); 2262 } 2263 } else if (Opcode == R600::REG_SEQUENCE) { 2264 for (unsigned i = 1, e = Node->getNumOperands(); i < e; i += 2) { 2265 SDValue &Src = Ops[i]; 2266 if (FoldOperand(Node, i, Src, FakeOp, FakeOp, FakeOp, FakeOp, DAG)) 2267 return DAG.getMachineNode(Opcode, SDLoc(Node), Node->getVTList(), Ops); 2268 } 2269 } else { 2270 if (!TII->hasInstrModifiers(Opcode)) 2271 return Node; 2272 int OperandIdx[] = { 2273 TII->getOperandIdx(Opcode, R600::OpName::src0), 2274 TII->getOperandIdx(Opcode, R600::OpName::src1), 2275 TII->getOperandIdx(Opcode, R600::OpName::src2) 2276 }; 2277 int NegIdx[] = { 2278 TII->getOperandIdx(Opcode, R600::OpName::src0_neg), 2279 TII->getOperandIdx(Opcode, R600::OpName::src1_neg), 2280 TII->getOperandIdx(Opcode, R600::OpName::src2_neg) 2281 }; 2282 int AbsIdx[] = { 2283 TII->getOperandIdx(Opcode, R600::OpName::src0_abs), 2284 TII->getOperandIdx(Opcode, R600::OpName::src1_abs), 2285 -1 2286 }; 2287 for (unsigned i = 0; i < 3; i++) { 2288 if (OperandIdx[i] < 0) 2289 return Node; 2290 SDValue &Src = Ops[OperandIdx[i] - 1]; 2291 SDValue &Neg = Ops[NegIdx[i] - 1]; 2292 SDValue FakeAbs; 2293 SDValue &Abs = (AbsIdx[i] > -1) ? Ops[AbsIdx[i] - 1] : FakeAbs; 2294 bool HasDst = TII->getOperandIdx(Opcode, R600::OpName::dst) > -1; 2295 int SelIdx = TII->getSelIdx(Opcode, OperandIdx[i]); 2296 int ImmIdx = TII->getOperandIdx(Opcode, R600::OpName::literal); 2297 if (HasDst) { 2298 SelIdx--; 2299 ImmIdx--; 2300 } 2301 SDValue &Sel = (SelIdx > -1) ? Ops[SelIdx] : FakeOp; 2302 SDValue &Imm = Ops[ImmIdx]; 2303 if (FoldOperand(Node, i, Src, Neg, Abs, Sel, Imm, DAG)) 2304 return DAG.getMachineNode(Opcode, SDLoc(Node), Node->getVTList(), Ops); 2305 } 2306 } 2307 2308 return Node; 2309 } 2310