Lines Matching +full:summation +full:- +full:disable
1 //===- OpenMPIRBuilder.cpp - Builder for LLVM-IR for OpenMP directives ----===//
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
13 //===----------------------------------------------------------------------===//
62 #define DEBUG_TYPE "openmp-ir-builder"
68 OptimisticAttributes("openmp-ir-builder-optimistic-attributes", cl::Hidden,
70 "'as-if' properties of runtime calls."),
74 "openmp-ir-builder-unroll-threshold-factor", cl::Hidden,
81 /// at position IP1 may change the meaning of IP2 or vice-versa. This is because
154 Kernel->getFnAttribute("target-features").getValueAsString(); in getGridValue()
229 // OpenMP 5.1, 2.11.4 Worksharing-Loop Construct, Description. in getOpenMPMonotonicityScheduleType()
273 if (Instruction *Term = Source->getTerminator()) { in redirectTo()
275 assert(!Br->isConditional() && in redirectTo()
277 BasicBlock *Succ = Br->getSuccessor(0); in redirectTo()
278 Succ->removePredecessor(Source, /*KeepOneInputPHIs=*/true); in redirectTo()
279 Br->setSuccessor(0, Target); in redirectTo()
284 NewBr->setDebugLoc(DL); in redirectTo()
289 assert(New->getFirstInsertionPt() == New->begin() && in spliceBB()
294 New->splice(New->begin(), Old, IP.getPoint(), Old->end()); in spliceBB()
306 Builder.SetInsertPoint(Old->getTerminator()); in spliceBB()
319 Old->getContext(), Name.isTriviallyEmpty() ? Old->getName() : Name, in splitBB()
320 Old->getParent(), Old->getNextNode()); in splitBB()
322 New->replaceSuccessorsPhiUsesWith(Old, New); in splitBB()
331 Builder.SetInsertPoint(Builder.GetInsertBlock()->getTerminator()); in splitBB()
345 Builder.SetInsertPoint(Builder.GetInsertBlock()->getTerminator()); in splitBB()
357 return splitBB(Builder, CreateBranch, Old->getName() + Suffix); in splitBBWithSuffix()
396 //===----------------------------------------------------------------------===//
398 //===----------------------------------------------------------------------===//
490 //===----------------------------------------------------------------------===//
492 //===----------------------------------------------------------------------===//
536 bool Param = true) -> void { in addAttributes()
601 if (!Fn->hasMetadata(LLVMContext::MD_callback)) { in getOrCreateRuntimeFunction()
602 LLVMContext &Ctx = Fn->getContext(); in getOrCreateRuntimeFunction()
605 // - The callback callee is argument number 2 (microtask). in getOrCreateRuntimeFunction()
606 // - The first two arguments of the callback callee are unknown (-1). in getOrCreateRuntimeFunction()
607 // - All variadic arguments to the runtime function are passed to the in getOrCreateRuntimeFunction()
609 Fn->addMetadata( in getOrCreateRuntimeFunction()
612 2, {-1, -1}, /* VarArgsArePassed */ true)})); in getOrCreateRuntimeFunction()
616 LLVM_DEBUG(dbgs() << "Created OpenMP runtime function " << Fn->getName() in getOrCreateRuntimeFunction()
617 << " with type " << *Fn->getFunctionType() << "\n"); in getOrCreateRuntimeFunction()
621 LLVM_DEBUG(dbgs() << "Found OpenMP runtime function " << Fn->getName() in getOrCreateRuntimeFunction()
622 << " with type " << *Fn->getFunctionType() << "\n"); in getOrCreateRuntimeFunction()
641 BasicBlock &EntryBlock = Function->getEntryBlock(); in raiseUserConstantDataAllocasToEntryBlock()
646 for (auto Block = std::next(Function->begin(), 1); Block != Function->end(); in raiseUserConstantDataAllocasToEntryBlock()
648 for (auto Inst = Block->getReverseIterator()->begin(); in raiseUserConstantDataAllocasToEntryBlock()
649 Inst != Block->getReverseIterator()->end();) { in raiseUserConstantDataAllocasToEntryBlock()
652 if (!isa<ConstantData>(AllocaInst->getArraySize())) in raiseUserConstantDataAllocasToEntryBlock()
654 AllocaInst->moveBeforePreserving(MoveLocInst); in raiseUserConstantDataAllocasToEntryBlock()
698 LLVM_DEBUG(dbgs() << "Entry " << OI.EntryBB->getName() in finalize()
699 << " Exit: " << OI.ExitBB->getName() << "\n"); in finalize()
708 // Forward target-cpu, target-features attributes to the outlined function. in finalize()
709 auto TargetCpuAttr = OuterFn->getFnAttribute("target-cpu"); in finalize()
711 OutlinedFn->addFnAttr(TargetCpuAttr); in finalize()
713 auto TargetFeaturesAttr = OuterFn->getFnAttribute("target-features"); in finalize()
715 OutlinedFn->addFnAttr(TargetFeaturesAttr); in finalize()
719 assert(OutlinedFn->getReturnType()->isVoidTy() && in finalize()
724 OutlinedFn->removeFromParent(); in finalize()
725 M.getFunctionList().insertAfter(OuterFn->getIterator(), OutlinedFn); in finalize()
730 BasicBlock &ArtificialEntry = OutlinedFn->getEntryBlock(); in finalize()
732 assert(OI.EntryBB->getUniquePredecessor() == &ArtificialEntry); in finalize()
733 // Move instructions from the to-be-deleted ArtificialEntry to the entry in finalize()
749 I.moveBeforePreserving(*OI.EntryBB, OI.EntryBB->getFirstInsertionPt()); in finalize()
752 OI.EntryBB->moveBefore(&ArtificialEntry); in finalize()
755 assert(&OutlinedFn->getEntryBlock() == OI.EntryBB); in finalize()
756 assert(OutlinedFn && OutlinedFn->getNumUses() == 1); in finalize()
780 // defined/read+writeable allocation sizes would be non-trivial in finalize()
790 const TargetRegionEntryInfo &EntryInfo) -> void { in finalize()
816 GV->setVisibility(GlobalValue::HiddenVisibility); in createGlobalFlag()
825 // Enable "C-mode". in getOrCreateIdent()
852 GV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global); in getOrCreateIdent()
853 GV->setAlignment(Align(8)); in getOrCreateIdent()
913 if (DIFile *DIF = DIL->getFile()) in getOrCreateSrcLocStr()
914 if (std::optional<StringRef> Source = DIF->getSource()) in getOrCreateSrcLocStr()
916 StringRef Function = DIL->getScope()->getSubprogram()->getName(); in getOrCreateSrcLocStr()
918 Function = F->getName(); in getOrCreateSrcLocStr()
919 return getOrCreateSrcLocStr(Function, FileName, DIL->getLine(), in getOrCreateSrcLocStr()
920 DIL->getColumn(), SrcLocStrSize); in getOrCreateSrcLocStr()
926 Loc.IP.getBlock()->getParent()); in getOrCreateSrcLocStr()
1033 Builder.SetInsertPoint(UI->getParent()); in createCancel()
1034 UI->eraseFromParent(); in createCancel()
1056 M.getDataLayout().getPrefTypeAlign(KernelArgs[I]->getType())); in emitTargetKernel()
1122 auto CurFn = Builder.GetInsertBlock()->getParent(); in emitKernelLaunch()
1139 if (Builder.GetInsertPoint() == BB->end()) { in emitCancelationCheckImpl()
1143 BB->getContext(), BB->getName() + ".cont", BB->getParent()); in emitCancelationCheckImpl()
1146 BB->getTerminator()->eraseFromParent(); in emitCancelationCheckImpl()
1150 BB->getContext(), BB->getName() + ".cncl", BB->getParent()); in emitCancelationCheckImpl()
1166 Builder.SetInsertPoint(NonCancellationBlock, NonCancellationBlock->begin()); in emitCancelationCheckImpl()
1179 IRBuilder<> &Builder = OMPIRBuilder->Builder; in targetParallelCallback()
1188 unsigned NumCapturedVars = OutlinedFn.arg_size() - /* tid & bounded tid */ 2; in targetParallelCallback()
1192 CI->getParent()->setName("omp_parallel"); in targetParallelCallback()
1195 Type *PtrTy = OMPIRBuilder->VoidPtr; in targetParallelCallback()
1200 Builder.SetInsertPoint(OuterAllocaBB, OuterAllocaBB->getFirstInsertionPt()); in targetParallelCallback()
1206 if (ArgsAlloca->getAddressSpace()) in targetParallelCallback()
1212 Value *V = *(CI->arg_begin() + 2 + Idx); in targetParallelCallback()
1219 IfCondition ? Builder.CreateSExtOrTrunc(IfCondition, OMPIRBuilder->Int32) in targetParallelCallback()
1227 /* number of threads */ NumThreads ? NumThreads : Builder.getInt32(-1), in targetParallelCallback()
1228 /* Proc bind */ Builder.getInt32(-1), in targetParallelCallback()
1230 Builder.CreateBitCast(&OutlinedFn, OMPIRBuilder->ParallelTaskPtr), in targetParallelCallback()
1236 OMPIRBuilder->getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_parallel_51); in targetParallelCallback()
1241 << *Builder.GetInsertBlock()->getParent() << "\n"); in targetParallelCallback()
1246 Builder.CreateStore(Builder.CreateLoad(OMPIRBuilder->Int32, OutlinedAI), in targetParallelCallback()
1250 CI->eraseFromParent(); in targetParallelCallback()
1253 I->eraseFromParent(); in targetParallelCallback()
1266 IRBuilder<> &Builder = OMPIRBuilder->Builder; in hostParallelCallback()
1270 OMPIRBuilder->getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_fork_call_if); in hostParallelCallback()
1273 OMPIRBuilder->getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_fork_call); in hostParallelCallback()
1276 if (!F->hasMetadata(LLVMContext::MD_callback)) { in hostParallelCallback()
1277 LLVMContext &Ctx = F->getContext(); in hostParallelCallback()
1280 // - The callback callee is argument number 2 (microtask). in hostParallelCallback()
1281 // - The first two arguments of the callback callee are unknown (-1). in hostParallelCallback()
1282 // - All variadic arguments to the __kmpc_fork_call are passed to the in hostParallelCallback()
1284 F->addMetadata(LLVMContext::MD_callback, in hostParallelCallback()
1286 2, {-1, -1}, in hostParallelCallback()
1297 unsigned NumCapturedVars = OutlinedFn.arg_size() - /* tid & bounded tid */ 2; in hostParallelCallback()
1300 CI->getParent()->setName("omp_parallel"); in hostParallelCallback()
1306 Builder.CreateBitCast(&OutlinedFn, OMPIRBuilder->ParallelTaskPtr)}; in hostParallelCallback()
1311 Value *Cond = Builder.CreateSExtOrTrunc(IfCondition, OMPIRBuilder->Int32); in hostParallelCallback()
1314 RealArgs.append(CI->arg_begin() + /* tid & bound tid */ 2, CI->arg_end()); in hostParallelCallback()
1318 auto PtrTy = OMPIRBuilder->VoidPtr; in hostParallelCallback()
1323 if (IfCondition && RealArgs.back()->getType() != PtrTy) in hostParallelCallback()
1329 << *Builder.GetInsertBlock()->getParent() << "\n"); in hostParallelCallback()
1334 Builder.CreateStore(Builder.CreateLoad(OMPIRBuilder->Int32, OutlinedAI), in hostParallelCallback()
1338 CI->eraseFromParent(); in hostParallelCallback()
1341 I->eraseFromParent(); in hostParallelCallback()
1386 Function *OuterFn = InsertBB->getParent(); in createParallel()
1398 InsertPointTy NewOuter(OuterAllocaBlock, OuterAllocaBlock->begin()); in createParallel()
1409 TIDAddr->insertAfter(TIDAddrAlloca); in createParallel()
1414 ZeroAddr->insertAfter(ZeroAddrAlloca); in createParallel()
1427 BasicBlock *EntryBB = UI->getParent(); in createParallel()
1428 BasicBlock *PRegEntryBB = EntryBB->splitBasicBlock(UI, "omp.par.entry"); in createParallel()
1429 BasicBlock *PRegBodyBB = PRegEntryBB->splitBasicBlock(UI, "omp.par.region"); in createParallel()
1431 PRegBodyBB->splitBasicBlock(UI, "omp.par.pre_finalize"); in createParallel()
1432 BasicBlock *PRegExitBB = PRegPreFiniBB->splitBasicBlock(UI, "omp.par.exit"); in createParallel()
1435 // Hide "open-ended" blocks from the given FiniCB by setting the right jump in createParallel()
1437 if (IP.getBlock()->end() == IP.getPoint()) { in createParallel()
1441 IP = InsertPointTy(I->getParent(), I->getIterator()); in createParallel()
1443 assert(IP.getBlock()->getTerminator()->getNumSuccessors() == 1 && in createParallel()
1444 IP.getBlock()->getTerminator()->getSuccessor(0) == PRegExitBB && in createParallel()
1453 Builder.SetInsertPoint(PRegEntryBB->getTerminator()); in createParallel()
1469 // PRegionEntryBB <- Privatization allocas are placed here. in createParallel()
1472 // PRegionBodyBB <- BodeGen is invoked here. in createParallel()
1475 // PRegPreFiniBB <- The block we will start finalization from. in createParallel()
1478 // PRegionExitBB <- A common exit to simplify block collection. in createParallel()
1485 InsertPointTy CodeGenIP(PRegBodyBB, PRegBodyBB->begin()); in createParallel()
1520 PRegExitBB = SplitBlock(PRegExitBB, &*PRegExitBB->getFirstInsertionPt()); in createParallel()
1521 PRegOutlinedExitBB->setName("omp.par.outlined.exit"); in createParallel()
1555 if (ParallelRegionBlockSet.count(UserI->getParent())) in createParallel()
1560 // value onto stack and load it back inside the to-be-outlined region. This in createParallel()
1565 if (!V.getType()->isPointerTy()) { in createParallel()
1574 // block of the to-be-outlined region. in createParallel()
1576 InsertBB->getTerminator()->getIterator()); in createParallel()
1579 // Load back next to allocations in the to-be-outlined region. in createParallel()
1586 if (CI && CI->getCalledFunction() == TIDRTLFn.getCallee()) { in createParallel()
1593 InnerAllocaIP.getBlock()->getTerminator()->getIterator()}; in createParallel()
1602 UPtr->set(ReplacementValue); in createParallel()
1606 // wrapped into pointers before passing them into the to-be-outlined region. in createParallel()
1609 // OpenMP-related values (thread ID and zero address pointers) remain leading in createParallel()
1612 ZeroAddrUse->getParent(), ZeroAddrUse->getNextNode()->getIterator()); in createParallel()
1617 OuterAllocaBlock, OuterAllocaBlock->getFirstInsertionPt()); in createParallel()
1628 "OpenMP outlining should not produce live-out values!"); in createParallel()
1633 dbgs() << " PBR: " << BB->getName() << "\n"; in createParallel()
1637 // finalize function a last time to finalize values between the pre-fini in createParallel()
1644 Instruction *PRegPreFiniTI = PRegPreFiniBB->getTerminator(); in createParallel()
1646 InsertPointTy PreFiniIP(PRegPreFiniBB, PRegPreFiniTI->getIterator()); in createParallel()
1652 InsertPointTy AfterIP(UI->getParent(), UI->getParent()->end()); in createParallel()
1653 UI->eraseFromParent(); in createParallel()
1711 // - Allocates space on the stack of an array of DependInfo objects
1712 // - Populates each DependInfo object with relevant information of
1714 // - All code is inserted in the entry block of the current function.
1745 OldIP.getBlock()->getParent()->getEntryBlock().getTerminator()); in emitTaskDependencies()
1812 InsertPointTy(TaskAllocaBB, TaskAllocaBB->begin()); in createTask()
1813 InsertPointTy TaskBodyIP = InsertPointTy(TaskBodyBB, TaskBodyBB->begin()); in createTask()
1835 bool HasShareds = StaleCI->arg_size() > 1; in createTask()
1843 // Arguments - `loc_ref` (Ident) and `gtid` (ThreadID) in createTask()
1847 // Argument - `flags` in createTask()
1860 // Argument - `sizeof_kmp_task_t` (TaskSize) in createTask()
1867 // Argument - `sizeof_shareds` (SharedsSize) in createTask()
1873 dyn_cast<AllocaInst>(StaleCI->getArgOperand(1)); in createTask()
1878 dyn_cast<StructType>(ArgStructAlloca->getAllocatedType()); in createTask()
1894 Value *Shareds = StaleCI->getArgOperand(1); in createTask()
1895 Align Alignment = TaskData->getPointerAlignment(M.getDataLayout()); in createTask()
1905 &OldIP.getBlock()->getParent()->getEntryBlock().back()); in createTask()
1964 Builder.GetInsertPoint()->getParent()->getTerminator(); in createTask()
1990 CI->setDebugLoc(StaleCI->getDebugLoc()); in createTask()
2010 StaleCI->eraseFromParent(); in createTask()
2012 Builder.SetInsertPoint(TaskAllocaBB, TaskAllocaBB->begin()); in createTask()
2015 OutlinedFn.getArg(1)->replaceUsesWithIf( in createTask()
2020 [](Instruction *I) { I->eraseFromParent(); }); in createTask()
2024 Builder.SetInsertPoint(TaskExitBB, TaskExitBB->begin()); in createTask()
2068 if (IP.getBlock()->end() != IP.getPoint()) in createSections()
2079 auto *CaseBB = IP.getBlock()->getSinglePredecessor(); in createSections()
2080 auto *CondBB = CaseBB->getSinglePredecessor()->getSinglePredecessor(); in createSections()
2081 auto *ExitBB = CondBB->getTerminator()->getSuccessor(1); in createSections()
2083 IP = InsertPointTy(I->getParent(), I->getIterator()); in createSections()
2091 // -> OMP.createSection() which generates the IR for each section in createSections()
2098 // case <NumSection> - 1: in createSections()
2099 // <SectionStmt[<NumSection> - 1]>; in createSections()
2109 Function *CurFn = Continue->getParent(); in createSections()
2116 SwitchStmt->addCase(Builder.getInt32(CaseNumber), CaseBB); in createSections()
2120 {CaseEndBr->getParent(), CaseEndBr->getIterator()}); in createSections()
2146 AfterIP = {FiniBB, FiniBB->begin()}; in createSections()
2160 if (IP.getBlock()->end() != IP.getPoint()) in createSection()
2172 auto *CondBB = CaseBB->getSinglePredecessor()->getSinglePredecessor(); in createSection()
2173 auto *ExitBB = CondBB->getTerminator()->getSuccessor(1); in createSection()
2175 IP = InsertPointTy(I->getParent(), I->getIterator()); in createSection()
2190 return OpenMPIRBuilder::InsertPointTy(I->getParent(), IT); in getInsertPointAfterInstr()
2212 GV->setSection("llvm.metadata"); in emitUsed()
2235 unsigned LaneIDMask = ~0u >> (32u - LaneIDBits); in getNVPTXLaneID()
2242 Type *FromType = From->getType(); in castValueToType()
2251 if (ToType->isIntegerTy() && FromType->isIntegerTy()) in castValueToType()
2259 CastItem, FromType->getPointerTo()); in castValueToType()
2271 // Cast all types to 32- or 64-bit values before calling shuffle routines. in createRuntimeShuffleFunction()
2309 Ptr, IntType->getPointerTo(), Ptr->getName() + ".ascast"); in shuffleAndStore()
2313 ElemPtr, IntType->getPointerTo(), ElemPtr->getName() + ".ascast"); in shuffleAndStore()
2315 Function *CurFunc = Builder.GetInsertBlock()->getParent(); in shuffleAndStore()
2326 Builder.CreatePHI(Ptr->getType(), /*NumReservedValues=*/2); in shuffleAndStore()
2327 PhiSrc->addIncoming(Ptr, CurrentBB); in shuffleAndStore()
2329 Builder.CreatePHI(ElemPtr->getType(), /*NumReservedValues=*/2); in shuffleAndStore()
2330 PhiDest->addIncoming(ElemPtr, CurrentBB); in shuffleAndStore()
2337 Builder.CreateICmpSGT(PtrDiff, Builder.getInt64(IntSize - 1)), ThenBB, in shuffleAndStore()
2351 PhiSrc->addIncoming(LocalPtr, ThenBB); in shuffleAndStore()
2352 PhiDest->addIncoming(LocalElemPtr, ThenBB); in shuffleAndStore()
2358 if (ElemType->isIntegerTy() && ElemType->getScalarSizeInBits() < in shuffleAndStore()
2359 Res->getType()->getScalarSizeInBits()) in shuffleAndStore()
2378 // Iterates, element-by-element, through the source Reduce list and in emitReductionListCopy()
2408 DestAlloca->setAlignment( in emitReductionListCopy()
2413 DestElementAddr->getName() + ".ascast"); in emitReductionListCopy()
2443 RI.ElementType->getStructElementType(0), SrcRealPtr, ".real"); in emitReductionListCopy()
2447 RI.ElementType->getStructElementType(1), SrcImgPtr, ".imag"); in emitReductionListCopy()
2477 DestElementAddr->getName() + ".ascast"); in emitReductionListCopy()
2494 WcFunc->setAttributes(FuncAttrs); in emitInterWarpCopyFunction()
2495 WcFunc->addParamAttr(0, Attribute::NoUndef); in emitInterWarpCopyFunction()
2496 WcFunc->addParamAttr(1, Attribute::NoUndef); in emitInterWarpCopyFunction()
2503 Argument *ReduceListArg = WcFunc->getArg(0); in emitInterWarpCopyFunction()
2506 Argument *NumWarpsArg = WcFunc->getArg(1); in emitInterWarpCopyFunction()
2537 Builder.GetInsertBlock()->getFirstInsertionPt()); in emitInterWarpCopyFunction()
2538 Type *Arg0Type = ReduceListArg->getType(); in emitInterWarpCopyFunction()
2539 Type *Arg1Type = NumWarpsArg->getType(); in emitInterWarpCopyFunction()
2542 Arg0Type, nullptr, ReduceListArg->getName() + ".addr"); in emitInterWarpCopyFunction()
2544 Builder.CreateAlloca(Arg1Type, nullptr, NumWarpsArg->getName() + ".addr"); in emitInterWarpCopyFunction()
2546 ReduceListAlloca, Arg0Type, ReduceListAlloca->getName() + ".ascast"); in emitInterWarpCopyFunction()
2548 NumWarpsAlloca, Arg1Type->getPointerTo(), in emitInterWarpCopyFunction()
2549 NumWarpsAlloca->getName() + ".ascast"); in emitInterWarpCopyFunction()
2554 getInsertPointAfterInstr(&Builder.GetInsertBlock()->back()); in emitInterWarpCopyFunction()
2584 CntAddr->getName() + ".ascast"); in emitInterWarpCopyFunction()
2592 emitBlock(PrecondBB, Builder.GetInsertBlock()->getParent()); in emitInterWarpCopyFunction()
2598 emitBlock(BodyBB, Builder.GetInsertBlock()->getParent()); in emitInterWarpCopyFunction()
2613 emitBlock(ThenBB, Builder.GetInsertBlock()->getParent()); in emitInterWarpCopyFunction()
2642 emitBlock(ElseBB, Builder.GetInsertBlock()->getParent()); in emitInterWarpCopyFunction()
2646 emitBlock(MergeBB, Builder.GetInsertBlock()->getParent()); in emitInterWarpCopyFunction()
2664 emitBlock(W0ThenBB, Builder.GetInsertBlock()->getParent()); in emitInterWarpCopyFunction()
2688 emitBlock(W0ElseBB, Builder.GetInsertBlock()->getParent()); in emitInterWarpCopyFunction()
2691 emitBlock(W0MergeBB, Builder.GetInsertBlock()->getParent()); in emitInterWarpCopyFunction()
2698 auto *CurFn = Builder.GetInsertBlock()->getParent(); in emitInterWarpCopyFunction()
2724 SarFunc->setAttributes(FuncAttrs); in emitShuffleAndReduceFunction()
2725 SarFunc->addParamAttr(0, Attribute::NoUndef); in emitShuffleAndReduceFunction()
2726 SarFunc->addParamAttr(1, Attribute::NoUndef); in emitShuffleAndReduceFunction()
2727 SarFunc->addParamAttr(2, Attribute::NoUndef); in emitShuffleAndReduceFunction()
2728 SarFunc->addParamAttr(3, Attribute::NoUndef); in emitShuffleAndReduceFunction()
2729 SarFunc->addParamAttr(1, Attribute::SExt); in emitShuffleAndReduceFunction()
2730 SarFunc->addParamAttr(2, Attribute::SExt); in emitShuffleAndReduceFunction()
2731 SarFunc->addParamAttr(3, Attribute::SExt); in emitShuffleAndReduceFunction()
2736 Argument *ReduceListArg = SarFunc->getArg(0); in emitShuffleAndReduceFunction()
2738 Argument *LaneIDArg = SarFunc->getArg(1); in emitShuffleAndReduceFunction()
2740 Argument *RemoteLaneOffsetArg = SarFunc->getArg(2); in emitShuffleAndReduceFunction()
2742 Argument *AlgoVerArg = SarFunc->getArg(3); in emitShuffleAndReduceFunction()
2744 Type *ReduceListArgType = ReduceListArg->getType(); in emitShuffleAndReduceFunction()
2745 Type *LaneIDArgType = LaneIDArg->getType(); in emitShuffleAndReduceFunction()
2746 Type *LaneIDArgPtrType = LaneIDArg->getType()->getPointerTo(); in emitShuffleAndReduceFunction()
2748 ReduceListArgType, nullptr, ReduceListArg->getName() + ".addr"); in emitShuffleAndReduceFunction()
2750 LaneIDArg->getName() + ".addr"); in emitShuffleAndReduceFunction()
2752 LaneIDArgType, nullptr, RemoteLaneOffsetArg->getName() + ".addr"); in emitShuffleAndReduceFunction()
2754 AlgoVerArg->getName() + ".addr"); in emitShuffleAndReduceFunction()
2758 // Create a local thread-private variable to host the Reduce list in emitShuffleAndReduceFunction()
2765 ReduceListAlloca->getName() + ".ascast"); in emitShuffleAndReduceFunction()
2767 LaneIdAlloca, LaneIDArgPtrType, LaneIdAlloca->getName() + ".ascast"); in emitShuffleAndReduceFunction()
2770 RemoteLaneOffsetAlloca->getName() + ".ascast"); in emitShuffleAndReduceFunction()
2772 AlgoVerAlloca, LaneIDArgPtrType, AlgoVerAlloca->getName() + ".ascast"); in emitShuffleAndReduceFunction()
2775 RemoteReductionListAlloca->getName() + ".ascast"); in emitShuffleAndReduceFunction()
2837 emitBlock(ThenBB, Builder.GetInsertBlock()->getParent()); in emitShuffleAndReduceFunction()
2843 ->addFnAttr(Attribute::NoUnwind); in emitShuffleAndReduceFunction()
2846 emitBlock(ElseBB, Builder.GetInsertBlock()->getParent()); in emitShuffleAndReduceFunction()
2849 emitBlock(MergeBB, Builder.GetInsertBlock()->getParent()); in emitShuffleAndReduceFunction()
2862 emitBlock(CpyThenBB, Builder.GetInsertBlock()->getParent()); in emitShuffleAndReduceFunction()
2867 emitBlock(CpyElseBB, Builder.GetInsertBlock()->getParent()); in emitShuffleAndReduceFunction()
2870 emitBlock(CpyMergeBB, Builder.GetInsertBlock()->getParent()); in emitShuffleAndReduceFunction()
2889 LtGCFunc->setAttributes(FuncAttrs); in emitListToGlobalCopyFunction()
2890 LtGCFunc->addParamAttr(0, Attribute::NoUndef); in emitListToGlobalCopyFunction()
2891 LtGCFunc->addParamAttr(1, Attribute::NoUndef); in emitListToGlobalCopyFunction()
2892 LtGCFunc->addParamAttr(2, Attribute::NoUndef); in emitListToGlobalCopyFunction()
2898 Argument *BufferArg = LtGCFunc->getArg(0); in emitListToGlobalCopyFunction()
2900 Argument *IdxArg = LtGCFunc->getArg(1); in emitListToGlobalCopyFunction()
2902 Argument *ReduceListArg = LtGCFunc->getArg(2); in emitListToGlobalCopyFunction()
2905 BufferArg->getName() + ".addr"); in emitListToGlobalCopyFunction()
2907 IdxArg->getName() + ".addr"); in emitListToGlobalCopyFunction()
2909 Builder.getPtrTy(), nullptr, ReduceListArg->getName() + ".addr"); in emitListToGlobalCopyFunction()
2912 BufferArgAlloca->getName() + ".ascast"); in emitListToGlobalCopyFunction()
2914 IdxArgAlloca, Builder.getPtrTy(), IdxArgAlloca->getName() + ".ascast"); in emitListToGlobalCopyFunction()
2917 ReduceListArgAlloca->getName() + ".ascast"); in emitListToGlobalCopyFunction()
2957 RI.ElementType->getStructElementType(0), SrcRealPtr, ".real"); in emitListToGlobalCopyFunction()
2961 RI.ElementType->getStructElementType(1), SrcImgPtr, ".imag"); in emitListToGlobalCopyFunction()
2999 LtGRFunc->setAttributes(FuncAttrs); in emitListToGlobalReduceFunction()
3000 LtGRFunc->addParamAttr(0, Attribute::NoUndef); in emitListToGlobalReduceFunction()
3001 LtGRFunc->addParamAttr(1, Attribute::NoUndef); in emitListToGlobalReduceFunction()
3002 LtGRFunc->addParamAttr(2, Attribute::NoUndef); in emitListToGlobalReduceFunction()
3008 Argument *BufferArg = LtGRFunc->getArg(0); in emitListToGlobalReduceFunction()
3010 Argument *IdxArg = LtGRFunc->getArg(1); in emitListToGlobalReduceFunction()
3012 Argument *ReduceListArg = LtGRFunc->getArg(2); in emitListToGlobalReduceFunction()
3015 BufferArg->getName() + ".addr"); in emitListToGlobalReduceFunction()
3017 IdxArg->getName() + ".addr"); in emitListToGlobalReduceFunction()
3019 Builder.getPtrTy(), nullptr, ReduceListArg->getName() + ".addr"); in emitListToGlobalReduceFunction()
3024 // void *RedList[<n>] = {<ReductionVars>[0], ..., <ReductionVars>[<n>-1]}; in emitListToGlobalReduceFunction()
3030 BufferArgAlloca->getName() + ".ascast"); in emitListToGlobalReduceFunction()
3032 IdxArgAlloca, Builder.getPtrTy(), IdxArgAlloca->getName() + ".ascast"); in emitListToGlobalReduceFunction()
3035 ReduceListArgAlloca->getName() + ".ascast"); in emitListToGlobalReduceFunction()
3038 LocalReduceList->getName() + ".ascast"); in emitListToGlobalReduceFunction()
3064 ->addFnAttr(Attribute::NoUnwind); in emitListToGlobalReduceFunction()
3082 LtGCFunc->setAttributes(FuncAttrs); in emitGlobalToListCopyFunction()
3083 LtGCFunc->addParamAttr(0, Attribute::NoUndef); in emitGlobalToListCopyFunction()
3084 LtGCFunc->addParamAttr(1, Attribute::NoUndef); in emitGlobalToListCopyFunction()
3085 LtGCFunc->addParamAttr(2, Attribute::NoUndef); in emitGlobalToListCopyFunction()
3091 Argument *BufferArg = LtGCFunc->getArg(0); in emitGlobalToListCopyFunction()
3093 Argument *IdxArg = LtGCFunc->getArg(1); in emitGlobalToListCopyFunction()
3095 Argument *ReduceListArg = LtGCFunc->getArg(2); in emitGlobalToListCopyFunction()
3098 BufferArg->getName() + ".addr"); in emitGlobalToListCopyFunction()
3100 IdxArg->getName() + ".addr"); in emitGlobalToListCopyFunction()
3102 Builder.getPtrTy(), nullptr, ReduceListArg->getName() + ".addr"); in emitGlobalToListCopyFunction()
3105 BufferArgAlloca->getName() + ".ascast"); in emitGlobalToListCopyFunction()
3107 IdxArgAlloca, Builder.getPtrTy(), IdxArgAlloca->getName() + ".ascast"); in emitGlobalToListCopyFunction()
3110 ReduceListArgAlloca->getName() + ".ascast"); in emitGlobalToListCopyFunction()
3147 RI.ElementType->getStructElementType(0), SrcRealPtr, ".real"); in emitGlobalToListCopyFunction()
3151 RI.ElementType->getStructElementType(1), SrcImgPtr, ".imag"); in emitGlobalToListCopyFunction()
3190 LtGRFunc->setAttributes(FuncAttrs); in emitGlobalToListReduceFunction()
3191 LtGRFunc->addParamAttr(0, Attribute::NoUndef); in emitGlobalToListReduceFunction()
3192 LtGRFunc->addParamAttr(1, Attribute::NoUndef); in emitGlobalToListReduceFunction()
3193 LtGRFunc->addParamAttr(2, Attribute::NoUndef); in emitGlobalToListReduceFunction()
3199 Argument *BufferArg = LtGRFunc->getArg(0); in emitGlobalToListReduceFunction()
3201 Argument *IdxArg = LtGRFunc->getArg(1); in emitGlobalToListReduceFunction()
3203 Argument *ReduceListArg = LtGRFunc->getArg(2); in emitGlobalToListReduceFunction()
3206 BufferArg->getName() + ".addr"); in emitGlobalToListReduceFunction()
3208 IdxArg->getName() + ".addr"); in emitGlobalToListReduceFunction()
3210 Builder.getPtrTy(), nullptr, ReduceListArg->getName() + ".addr"); in emitGlobalToListReduceFunction()
3215 // void *RedList[<n>] = {<ReductionVars>[0], ..., <ReductionVars>[<n>-1]}; in emitGlobalToListReduceFunction()
3221 BufferArgAlloca->getName() + ".ascast"); in emitGlobalToListReduceFunction()
3223 IdxArgAlloca, Builder.getPtrTy(), IdxArgAlloca->getName() + ".ascast"); in emitGlobalToListReduceFunction()
3226 ReduceListArgAlloca->getName() + ".ascast"); in emitGlobalToListReduceFunction()
3229 LocalReduceList->getName() + ".ascast"); in emitGlobalToListReduceFunction()
3255 ->addFnAttr(Attribute::NoUnwind); in emitGlobalToListReduceFunction()
3276 ReductionFunc->setAttributes(FuncAttrs); in createReductionFunction()
3277 ReductionFunc->addParamAttr(0, Attribute::NoUndef); in createReductionFunction()
3278 ReductionFunc->addParamAttr(1, Attribute::NoUndef); in createReductionFunction()
3287 Argument *Arg0 = ReductionFunc->getArg(0); in createReductionFunction()
3288 Argument *Arg1 = ReductionFunc->getArg(1); in createReductionFunction()
3289 Type *Arg0Type = Arg0->getType(); in createReductionFunction()
3290 Type *Arg1Type = Arg1->getType(); in createReductionFunction()
3293 Builder.CreateAlloca(Arg0Type, nullptr, Arg0->getName() + ".addr"); in createReductionFunction()
3295 Builder.CreateAlloca(Arg1Type, nullptr, Arg1->getName() + ".addr"); in createReductionFunction()
3297 LHSAlloca, Arg0Type, LHSAlloca->getName() + ".ascast"); in createReductionFunction()
3299 RHSAlloca, Arg1Type, RHSAlloca->getName() + ".ascast"); in createReductionFunction()
3316 RHSI8Ptr, RI.PrivateVariable->getType(), in createReductionFunction()
3317 RHSI8Ptr->getName() + ".ascast"); in createReductionFunction()
3324 LHSI8Ptr, RI.Variable->getType(), LHSI8Ptr->getName() + ".ascast"); in createReductionFunction()
3350 LHSFixupPtr->replaceUsesWithIf( in createReductionFunction()
3352 return cast<Instruction>(U.getUser())->getParent()->getParent() == in createReductionFunction()
3355 RHSFixupPtr->replaceUsesWithIf( in createReductionFunction()
3357 return cast<Instruction>(U.getUser())->getParent()->getParent() == in createReductionFunction()
3371 assert(RI.Variable && "expected non-null variable"); in checkReductionInfos()
3372 assert(RI.PrivateVariable && "expected non-null private variable"); in checkReductionInfos()
3374 "expected non-null reduction generator callback"); in checkReductionInfos()
3377 RI.Variable->getType() == RI.PrivateVariable->getType() && in checkReductionInfos()
3381 assert(RI.Variable->getType()->isPointerTy() && in checkReductionInfos()
3408 Function *CurFunc = Builder.GetInsertBlock()->getParent(); in createReductionsGPU()
3411 for (auto Attr : CurFunc->getAttributes().getFnAttrs()) in createReductionsGPU()
3419 createReductionFunction(Builder.GetInsertBlock()->getParent()->getName(), in createReductionsGPU()
3441 // void *RedList[<n>] = {<ReductionVars>[0], ..., <ReductionVars>[<n>-1]}; in createReductionsGPU()
3450 ReductionListAlloca, PtrTy, ReductionListAlloca->getName() + ".ascast"); in createReductionsGPU()
3552 LHSPtr->replaceUsesWithIf(LHS, [ReductionFunc](const Use &U) { in createReductionsGPU()
3553 return cast<Instruction>(U.getUser())->getParent()->getParent() == in createReductionsGPU()
3556 RHSPtr->replaceUsesWithIf(RHS, [ReductionFunc](const Use &U) { in createReductionsGPU()
3557 return cast<Instruction>(U.getUser())->getParent()->getParent() == in createReductionsGPU()
3588 assert(RI.Variable && "expected non-null variable"); in createReductions()
3589 assert(RI.PrivateVariable && "expected non-null private variable"); in createReductions()
3590 assert(RI.ReductionGen && "expected non-null reduction generator callback"); in createReductions()
3591 assert(RI.Variable->getType() == RI.PrivateVariable->getType() && in createReductions()
3594 assert(RI.Variable->getType()->isPointerTy() && in createReductions()
3603 InsertBlock->splitBasicBlock(Loc.IP.getPoint(), "reduce.finalize"); in createReductions()
3604 InsertBlock->getTerminator()->eraseFromParent(); in createReductions()
3606 // Create and populate array of type-erased pointers to private reduction in createReductions()
3610 Builder.SetInsertPoint(AllocaIP.getBlock()->getTerminator()); in createReductions()
3613 Builder.SetInsertPoint(InsertBlock, InsertBlock->end()); in createReductions()
3625 Function *Func = Builder.GetInsertBlock()->getParent(); in createReductions()
3626 Module *Module = Func->getParent(); in createReductions()
3638 const DataLayout &DL = Module->getDataLayout(); in createReductions()
3652 // Create final reduction entry blocks for the atomic and non-atomic case. in createReductions()
3656 BasicBlock::Create(Module->getContext(), "reduce.switch.nonatomic", Func); in createReductions()
3658 BasicBlock::Create(Module->getContext(), "reduce.switch.atomic", Func); in createReductions()
3661 Switch->addCase(Builder.getInt32(1), NonAtomicRedBlock); in createReductions()
3662 Switch->addCase(Builder.getInt32(2), AtomicRedBlock); in createReductions()
3664 // Populate the non-atomic reduction using the elementwise reduction function. in createReductions()
3671 // We have one less load for by-ref case because that load is now inside of in createReductions()
3691 // for by-ref case, the load is inside of the reduction region in createReductions()
3718 // function. Partial values are extracted from the type-erased array of in createReductions()
3721 BasicBlock::Create(Module->getContext(), "", ReductionFunc); in createReductions()
3723 Value *LHSArrayPtr = ReductionFunc->getArg(0); in createReductions()
3724 Value *RHSArrayPtr = ReductionFunc->getArg(1); in createReductions()
3731 Value *LHSPtr = Builder.CreateBitCast(LHSI8Ptr, RI.Variable->getType()); in createReductions()
3737 Builder.CreateBitCast(RHSI8Ptr, RI.PrivateVariable->getType()); in createReductions()
3743 // store is inside of the reduction region when using by-ref in createReductions()
3806 Module *M = F->getParent(); in createLoopSkeleton()
3807 LLVMContext &Ctx = M->getContext(); in createLoopSkeleton()
3808 Type *IndVarTy = TripCount->getType(); in createLoopSkeleton()
3834 IndVarPHI->addIncoming(ConstantInt::get(IndVarTy, 0), Preheader); in createLoopSkeleton()
3849 IndVarPHI->addIncoming(Next, Latch); in createLoopSkeleton()
3858 CL->Header = Header; in createLoopSkeleton()
3859 CL->Cond = Cond; in createLoopSkeleton()
3860 CL->Latch = Latch; in createLoopSkeleton()
3861 CL->Exit = Exit; in createLoopSkeleton()
3864 CL->assertOK(); in createLoopSkeleton()
3874 BasicBlock *NextBB = BB->getNextNode(); in createCanonicalLoop()
3876 CanonicalLoopInfo *CL = createLoopSkeleton(Loc.DL, TripCount, BB->getParent(), in createCanonicalLoop()
3878 BasicBlock *After = CL->getAfter(); in createCanonicalLoop()
3886 Builder.CreateBr(CL->getPreheader()); in createCanonicalLoop()
3891 BodyGenCB(CL->getBodyIP(), CL->getIndVar()); in createCanonicalLoop()
3894 CL->assertOK(); in createCanonicalLoop()
3904 // Consider the following difficulties (assuming 8-bit signed integers): in createCanonicalLoop()
3908 // DO I = 100, 0, -128 in createCanonicalLoop()
3911 auto *IndVarTy = cast<IntegerType>(Start->getType()); in createCanonicalLoop()
3912 assert(IndVarTy == Stop->getType() && "Stop type mismatch"); in createCanonicalLoop()
3913 assert(IndVarTy == Step->getType() && "Step type mismatch"); in createCanonicalLoop()
3976 unsigned Bitwidth = Ty->getIntegerBitWidth(); in getKmpcForStaticInitForType()
3990 assert(CLI->isValid() && "Requires a valid canonical loop"); in applyStaticWorkshareLoop()
3991 assert(!isConflictIP(AllocaIP, CLI->getPreheaderIP()) && in applyStaticWorkshareLoop()
3995 Builder.restoreIP(CLI->getPreheaderIP()); in applyStaticWorkshareLoop()
4003 Value *IV = CLI->getIndVar(); in applyStaticWorkshareLoop()
4004 Type *IVTy = IV->getType(); in applyStaticWorkshareLoop()
4010 Builder.SetInsertPoint(AllocaIP.getBlock()->getFirstNonPHIOrDbgOrAlloca()); in applyStaticWorkshareLoop()
4020 // always iterates from 0 to trip-count with step 1. Note that "init" expects in applyStaticWorkshareLoop()
4022 Builder.SetInsertPoint(CLI->getPreheader()->getTerminator()); in applyStaticWorkshareLoop()
4026 Value *UpperBound = Builder.CreateSub(CLI->getTripCount(), One); in applyStaticWorkshareLoop()
4044 CLI->setTripCount(TripCount); in applyStaticWorkshareLoop()
4050 CLI->mapIndVar([&](Instruction *OldIV) -> Value * { in applyStaticWorkshareLoop()
4051 Builder.SetInsertPoint(CLI->getBody(), in applyStaticWorkshareLoop()
4052 CLI->getBody()->getFirstInsertionPt()); in applyStaticWorkshareLoop()
4058 Builder.SetInsertPoint(CLI->getExit(), in applyStaticWorkshareLoop()
4059 CLI->getExit()->getTerminator()->getIterator()); in applyStaticWorkshareLoop()
4068 InsertPointTy AfterIP = CLI->getAfterIP(); in applyStaticWorkshareLoop()
4069 CLI->invalidate(); in applyStaticWorkshareLoop()
4077 assert(CLI->isValid() && "Requires a valid canonical loop"); in applyStaticChunkedWorkshareLoop()
4080 LLVMContext &Ctx = CLI->getFunction()->getContext(); in applyStaticChunkedWorkshareLoop()
4081 Value *IV = CLI->getIndVar(); in applyStaticChunkedWorkshareLoop()
4082 Value *OrigTripCount = CLI->getTripCount(); in applyStaticChunkedWorkshareLoop()
4083 Type *IVTy = IV->getType(); in applyStaticChunkedWorkshareLoop()
4084 assert(IVTy->getIntegerBitWidth() <= 64 && in applyStaticChunkedWorkshareLoop()
4086 Type *InternalIVTy = IVTy->getIntegerBitWidth() <= 32 ? Type::getInt32Ty(Ctx) in applyStaticChunkedWorkshareLoop()
4109 Builder.restoreIP(CLI->getPreheaderIP()); in applyStaticChunkedWorkshareLoop()
4112 // TODO: Detect overflow in ubsan or max-out with current tripcount. in applyStaticChunkedWorkshareLoop()
4161 BasicBlock *DispatchBody = DispatchCLI->getBody(); in applyStaticChunkedWorkshareLoop()
4162 BasicBlock *DispatchLatch = DispatchCLI->getLatch(); in applyStaticChunkedWorkshareLoop()
4163 BasicBlock *DispatchExit = DispatchCLI->getExit(); in applyStaticChunkedWorkshareLoop()
4164 BasicBlock *DispatchAfter = DispatchCLI->getAfter(); in applyStaticChunkedWorkshareLoop()
4165 DispatchCLI->invalidate(); in applyStaticChunkedWorkshareLoop()
4168 redirectTo(DispatchAfter, CLI->getAfter(), DL); in applyStaticChunkedWorkshareLoop()
4169 redirectTo(CLI->getExit(), DispatchLatch, DL); in applyStaticChunkedWorkshareLoop()
4173 Builder.restoreIP(CLI->getPreheaderIP()); in applyStaticChunkedWorkshareLoop()
4177 Builder.SetInsertPoint(CLI->getPreheader()->getTerminator()); in applyStaticChunkedWorkshareLoop()
4187 CLI->setTripCount(BackcastedChunkTC); in applyStaticChunkedWorkshareLoop()
4194 CLI->mapIndVar([&](Instruction *) -> Value * { in applyStaticChunkedWorkshareLoop()
4195 Builder.restoreIP(CLI->getBodyIP()); in applyStaticChunkedWorkshareLoop()
4200 Builder.SetInsertPoint(DispatchExit, DispatchExit->getFirstInsertionPt()); in applyStaticChunkedWorkshareLoop()
4211 CLI->assertOK(); in applyStaticChunkedWorkshareLoop()
4214 return {DispatchAfter, DispatchAfter->getFirstInsertionPt()}; in applyStaticChunkedWorkshareLoop()
4223 unsigned Bitwidth = Ty->getIntegerBitWidth(); in getKmpcForStaticLoopForType()
4224 Module &M = OMPBuilder->M; in getKmpcForStaticLoopForType()
4228 return OMPBuilder->getOrCreateRuntimeFunction( in getKmpcForStaticLoopForType()
4231 return OMPBuilder->getOrCreateRuntimeFunction( in getKmpcForStaticLoopForType()
4236 return OMPBuilder->getOrCreateRuntimeFunction( in getKmpcForStaticLoopForType()
4239 return OMPBuilder->getOrCreateRuntimeFunction( in getKmpcForStaticLoopForType()
4244 return OMPBuilder->getOrCreateRuntimeFunction( in getKmpcForStaticLoopForType()
4247 return OMPBuilder->getOrCreateRuntimeFunction( in getKmpcForStaticLoopForType()
4263 Type *TripCountTy = TripCount->getType(); in createTargetLoopWorkshareCall()
4264 Module &M = OMPBuilder->M; in createTargetLoopWorkshareCall()
4265 IRBuilder<> &Builder = OMPBuilder->Builder; in createTargetLoopWorkshareCall()
4278 FunctionCallee RTLNumThreads = OMPBuilder->getOrCreateRuntimeFunction( in createTargetLoopWorkshareCall()
4280 Builder.restoreIP({InsertBlock, std::prev(InsertBlock->end())}); in createTargetLoopWorkshareCall()
4299 IRBuilder<> &Builder = OMPIRBuilder->Builder; in workshareLoopTargetCallback()
4300 BasicBlock *Preheader = CLI->getPreheader(); in workshareLoopTargetCallback()
4301 Value *TripCount = CLI->getTripCount(); in workshareLoopTargetCallback()
4307 Preheader->splice(std::prev(Preheader->end()), CLI->getBody(), in workshareLoopTargetCallback()
4308 CLI->getBody()->begin(), std::prev(CLI->getBody()->end())); in workshareLoopTargetCallback()
4313 Builder.restoreIP({Preheader, Preheader->end()}); in workshareLoopTargetCallback()
4314 Preheader->getTerminator()->eraseFromParent(); in workshareLoopTargetCallback()
4315 Builder.CreateBr(CLI->getExit()); in workshareLoopTargetCallback()
4321 CleanUpInfo.EntryBB = CLI->getHeader(); in workshareLoopTargetCallback()
4322 CleanUpInfo.ExitBB = CLI->getExit(); in workshareLoopTargetCallback()
4334 assert((OutlinedFnCallInstruction->getParent() == Preheader) && in workshareLoopTargetCallback()
4337 if (OutlinedFnCallInstruction->arg_size() > 1) in workshareLoopTargetCallback()
4338 LoopBodyArg = OutlinedFnCallInstruction->getArgOperand(1); in workshareLoopTargetCallback()
4341 OutlinedFnCallInstruction->eraseFromParent(); in workshareLoopTargetCallback()
4348 ToBeDeletedItem->eraseFromParent(); in workshareLoopTargetCallback()
4349 CLI->invalidate(); in workshareLoopTargetCallback()
4361 OI.OuterAllocaBB = CLI->getPreheader(); in applyWorkshareLoopTarget()
4362 Function *OuterFn = CLI->getPreheader()->getParent(); in applyWorkshareLoopTarget()
4370 OI.EntryBB = CLI->getBody(); in applyWorkshareLoopTarget()
4371 OI.ExitBB = CLI->getLatch()->splitBasicBlock(CLI->getLatch()->begin(), in applyWorkshareLoopTarget()
4375 Builder.restoreIP({CLI->getPreheader(), CLI->getPreheader()->begin()}); in applyWorkshareLoopTarget()
4379 AllocaInst *NewLoopCnt = Builder.CreateAlloca(CLI->getIndVarType(), 0, ""); in applyWorkshareLoopTarget()
4381 Builder.CreateLoad(CLI->getIndVarType(), NewLoopCnt); in applyWorkshareLoopTarget()
4405 /* AllocationBlock */ CLI->getPreheader(), in applyWorkshareLoopTarget()
4419 SmallVector<User *> Users(CLI->getIndVar()->user_begin(), in applyWorkshareLoopTarget()
4420 CLI->getIndVar()->user_end()); in applyWorkshareLoopTarget()
4423 if (ParallelRegionBlockSet.count(Inst->getParent())) { in applyWorkshareLoopTarget()
4424 Inst->replaceUsesOfWith(CLI->getIndVar(), NewLoopCntLoad); in applyWorkshareLoopTarget()
4443 return CLI->getAfterIP(); in applyWorkshareLoopTarget()
4462 assert(!ChunkSize && "No chunk size with static-chunked schedule"); in applyWorkshareLoop()
4485 "schedule type does not support user-defined chunk sizes"); in applyWorkshareLoop()
4506 unsigned Bitwidth = Ty->getIntegerBitWidth(); in getKmpcForDynamicInitForType()
4522 unsigned Bitwidth = Ty->getIntegerBitWidth(); in getKmpcForDynamicNextForType()
4537 unsigned Bitwidth = Ty->getIntegerBitWidth(); in getKmpcForDynamicFiniForType()
4550 assert(CLI->isValid() && "Requires a valid canonical loop"); in applyDynamicWorkshareLoop()
4551 assert(!isConflictIP(AllocaIP, CLI->getPreheaderIP()) && in applyDynamicWorkshareLoop()
4567 Value *IV = CLI->getIndVar(); in applyDynamicWorkshareLoop()
4568 Type *IVTy = IV->getType(); in applyDynamicWorkshareLoop()
4573 Builder.SetInsertPoint(AllocaIP.getBlock()->getFirstNonPHIOrDbgOrAlloca()); in applyDynamicWorkshareLoop()
4582 // always iterates from 0 to trip-count with step 1. Note that "init" expects in applyDynamicWorkshareLoop()
4584 BasicBlock *PreHeader = CLI->getPreheader(); in applyDynamicWorkshareLoop()
4585 Builder.SetInsertPoint(PreHeader->getTerminator()); in applyDynamicWorkshareLoop()
4588 Value *UpperBound = CLI->getTripCount(); in applyDynamicWorkshareLoop()
4592 BasicBlock *Header = CLI->getHeader(); in applyDynamicWorkshareLoop()
4593 BasicBlock *Exit = CLI->getExit(); in applyDynamicWorkshareLoop()
4594 BasicBlock *Cond = CLI->getCond(); in applyDynamicWorkshareLoop()
4595 BasicBlock *Latch = CLI->getLatch(); in applyDynamicWorkshareLoop()
4596 InsertPointTy AfterIP = CLI->getAfterIP(); in applyDynamicWorkshareLoop()
4616 PreHeader->getContext(), Twine(PreHeader->getName()) + ".outer.cond", in applyDynamicWorkshareLoop()
4617 PreHeader->getParent()); in applyDynamicWorkshareLoop()
4618 // This needs to be 32-bit always, so can't use the IVTy Zero above. in applyDynamicWorkshareLoop()
4619 Builder.SetInsertPoint(OuterCond, OuterCond->getFirstInsertionPt()); in applyDynamicWorkshareLoop()
4629 // Change PHI-node in loop header to use outer cond rather than preheader, in applyDynamicWorkshareLoop()
4631 Instruction *Phi = &Header->front(); in applyDynamicWorkshareLoop()
4633 PI->setIncomingBlock(0, OuterCond); in applyDynamicWorkshareLoop()
4634 PI->setIncomingValue(0, LowerBound); in applyDynamicWorkshareLoop()
4636 // Then set the pre-header to jump to the OuterCond in applyDynamicWorkshareLoop()
4637 Instruction *Term = PreHeader->getTerminator(); in applyDynamicWorkshareLoop()
4639 Br->setSuccessor(0, OuterCond); in applyDynamicWorkshareLoop()
4644 Builder.SetInsertPoint(Cond, Cond->getFirstInsertionPt()); in applyDynamicWorkshareLoop()
4648 CI->setOperand(1, UpperBound); in applyDynamicWorkshareLoop()
4650 Instruction *Branch = &Cond->back(); in applyDynamicWorkshareLoop()
4652 assert(BI->getSuccessor(1) == Exit); in applyDynamicWorkshareLoop()
4653 BI->setSuccessor(1, OuterCond); in applyDynamicWorkshareLoop()
4657 Builder.SetInsertPoint(&Latch->back()); in applyDynamicWorkshareLoop()
4664 Builder.SetInsertPoint(&Exit->back()); in applyDynamicWorkshareLoop()
4670 CLI->invalidate(); in applyDynamicWorkshareLoop()
4687 for (Use &U : BB->uses()) { in removeUnusedBlocksFromParent()
4691 if (BBsToErase.count(UseInst->getParent())) in removeUnusedBlocksFromParent()
4718 BasicBlock *OrigPreheader = Outermost->getPreheader(); in collapseLoops()
4719 BasicBlock *OrigAfter = Outermost->getAfter(); in collapseLoops()
4720 Function *F = OrigPreheader->getParent(); in collapseLoops()
4726 Loop->collectControlBlocks(OldControlBBs); in collapseLoops()
4733 Builder.restoreIP(Outermost->getPreheaderIP()); in collapseLoops()
4739 assert(L->isValid() && in collapseLoops()
4741 Value *OrigTripCount = L->getTripCount(); in collapseLoops()
4755 OrigPreheader->getNextNode(), OrigAfter, "collapsed"); in collapseLoops()
4761 Builder.restoreIP(Result->getBodyIP()); in collapseLoops()
4763 Value *Leftover = Result->getIndVar(); in collapseLoops()
4766 for (int i = NumLoops - 1; i >= 1; --i) { in collapseLoops()
4767 Value *OrigTripCount = Loops[i]->getTripCount(); in collapseLoops()
4779 // the control flow, from the leading in-between code, the loop nest body, the in collapseLoops()
4780 // trailing in-between code, and rejoining the collapsed loop's latch. in collapseLoops()
4784 BasicBlock *ContinueBlock = Result->getBody(); in collapseLoops()
4800 // the in-between code is and instantiate it only once per thread. in collapseLoops()
4801 for (size_t i = 0; i < NumLoops - 1; ++i) in collapseLoops()
4802 ContinueWith(Loops[i]->getBody(), Loops[i + 1]->getHeader()); in collapseLoops()
4805 ContinueWith(Innermost->getBody(), Innermost->getLatch()); in collapseLoops()
4808 for (size_t i = NumLoops - 1; i > 0; --i) in collapseLoops()
4809 ContinueWith(Loops[i]->getAfter(), Loops[i - 1]->getLatch()); in collapseLoops()
4812 ContinueWith(Result->getLatch(), nullptr); in collapseLoops()
4815 redirectTo(Outermost->getPreheader(), Result->getPreheader(), DL); in collapseLoops()
4816 redirectTo(Result->getAfter(), Outermost->getAfter(), DL); in collapseLoops()
4820 Loops[i]->getIndVar()->replaceAllUsesWith(NewIndVars[i]); in collapseLoops()
4826 L->invalidate(); in collapseLoops()
4829 Result->assertOK(); in collapseLoops()
4844 Function *F = OutermostLoop->getBody()->getParent(); in tileLoops()
4845 BasicBlock *InnerEnter = InnermostLoop->getBody(); in tileLoops()
4846 BasicBlock *InnerLatch = InnermostLoop->getLatch(); in tileLoops()
4852 Loop->collectControlBlocks(OldControlBBs); in tileLoops()
4860 assert(L->isValid() && "All input loops must be valid canonical loops"); in tileLoops()
4861 OrigTripCounts.push_back(L->getTripCount()); in tileLoops()
4862 OrigIndVars.push_back(L->getIndVar()); in tileLoops()
4872 for (int i = 0; i < NumLoops - 1; ++i) { in tileLoops()
4876 BasicBlock *EnterBB = Surrounding->getBody(); in tileLoops()
4877 BasicBlock *ExitBB = Nested->getHeader(); in tileLoops()
4883 Builder.restoreIP(OutermostLoop->getPreheaderIP()); in tileLoops()
4888 Type *IVType = OrigTripCount->getType(); in tileLoops()
4896 // Unfortunately we cannot just use the roundup-formula in tileLoops()
4897 // (tripcount + tilesize - 1)/tilesize in tileLoops()
4898 // because the summation might overflow. We do not want introduce undefined in tileLoops()
4919 BasicBlock *Enter = OutermostLoop->getPreheader(); in tileLoops()
4923 BasicBlock *Continue = OutermostLoop->getAfter(); in tileLoops()
4926 BasicBlock *OutroInsertBefore = InnermostLoop->getExit(); in tileLoops()
4930 Value *TripCount, const Twine &Name) -> CanonicalLoopInfo * { in tileLoops()
4933 redirectTo(Enter, EmbeddedLoop->getPreheader(), DL); in tileLoops()
4934 redirectTo(EmbeddedLoop->getAfter(), Continue, DL); in tileLoops()
4937 Enter = EmbeddedLoop->getBody(); in tileLoops()
4938 Continue = EmbeddedLoop->getLatch(); in tileLoops()
4939 OutroInsertBefore = EmbeddedLoop->getLatch(); in tileLoops()
4956 Builder.SetInsertPoint(Enter->getTerminator()); in tileLoops()
4963 Builder.CreateICmpEQ(FloorLoop->getIndVar(), FloorCount[i]); in tileLoops()
4998 Builder.restoreIP(Result.back()->getBodyIP()); in tileLoops()
5006 Builder.CreateMul(Size, FloorLoop->getIndVar(), {}, /*HasNUW=*/true); in tileLoops()
5008 Builder.CreateAdd(Scale, TileLoop->getIndVar(), {}, /*HasNUW=*/true); in tileLoops()
5009 OrigIndVar->replaceAllUsesWith(Shift); in tileLoops()
5016 L->invalidate(); in tileLoops()
5020 GenL->assertOK(); in tileLoops()
5033 LLVMContext &Ctx = BB->getContext(); in addBasicBlockMetadata()
5038 MDNode *Existing = BB->getTerminator()->getMetadata(LLVMContext::MD_loop); in addBasicBlockMetadata()
5040 append_range(NewProperties, drop_begin(Existing->operands(), 1)); in addBasicBlockMetadata()
5044 BasicBlockID->replaceOperandWith(0, BasicBlockID); in addBasicBlockMetadata()
5046 BB->getTerminator()->setMetadata(LLVMContext::MD_loop, BasicBlockID); in addBasicBlockMetadata()
5053 assert(Loop->isValid() && "Expecting a valid CanonicalLoopInfo"); in addLoopMetadata()
5056 BasicBlock *Latch = Loop->getLatch(); in addLoopMetadata()
5092 Function *F = CanonicalLoop->getFunction(); in createIfVersion()
5099 SplitBefore = CanonicalLoop->getPreheader()->getTerminator(); in createIfVersion()
5114 Loop *L = LI.getLoopFor(CanonicalLoop->getHeader()); in createIfVersion()
5117 BasicBlock *Head = SplitBefore->getParent(); in createIfVersion()
5118 Instruction *HeadOldTerm = Head->getTerminator(); in createIfVersion()
5119 llvm::LLVMContext &C = Head->getContext(); in createIfVersion()
5121 C, NamePrefix + ".if.then", Head->getParent(), Head->getNextNode()); in createIfVersion()
5123 C, NamePrefix + ".if.else", Head->getParent(), CanonicalLoop->getExit()); in createIfVersion()
5129 InsertPointTy IP{BrInstr->getParent(), ++BrInstr->getIterator()}; in createIfVersion()
5132 ThenBlock->replaceSuccessorsPhiUsesWith(Head, ThenBlock); in createIfVersion()
5139 VMap[CanonicalLoop->getPreheader()] = ElseBlock; in createIfVersion()
5140 for (BasicBlock *Block : L->getBlocks()) { in createIfVersion()
5142 NewBB->moveBefore(CanonicalLoop->getExit()); in createIfVersion()
5173 Function *F = CanonicalLoop->getFunction(); in applySimd()
5187 Loop *L = LI.getLoopFor(CanonicalLoop->getHeader()); in applySimd()
5190 Builder.SetInsertPoint(CanonicalLoop->getPreheader()->getTerminator()); in applySimd()
5194 Builder.CreateAlignmentAssumption(F->getDataLayout(), in applySimd()
5204 Value *MappedLatch = VMap.lookup(CanonicalLoop->getLatch()); in applySimd()
5224 for (BasicBlock *Block : L->getBlocks()) { in applySimd()
5225 if (Block == CanonicalLoop->getCond() || in applySimd()
5226 Block == CanonicalLoop->getHeader()) in applySimd()
5234 // the memory instructions parallel, because loop-carried in applySimd()
5239 // Add access group metadata to memory-access instructions. in applySimd()
5272 /// Ideally, this would be passed from the front-end to the OpenMPBuilder, but
5281 /// might become be worth requiring front-ends to pass on their TargetMachine,
5283 /// have just a single main TargetMachine per translation unit, "target-cpu" and
5284 /// "target-features" that determine the TargetMachine are per-function and can
5288 Module *M = F->getParent(); in createTargetMachine()
5290 StringRef CPU = F->getFnAttribute("target-cpu").getValueAsString(); in createTargetMachine()
5291 StringRef Features = F->getFnAttribute("target-features").getValueAsString(); in createTargetMachine()
5292 const std::string &Triple = M->getTargetTriple(); in createTargetMachine()
5300 return std::unique_ptr<TargetMachine>(TheTarget->createTargetMachine( in createTargetMachine()
5305 /// Heuristically determine the best-performant unroll factor for \p CLI. This
5306 /// depends on the target processor. We are re-using the same heuristics as the
5309 Function *F = CLI->getFunction(); in computeHeuristicUnrollFactor()
5326 [&](const Function &F) { return TM->getTargetTransformInfo(F); }); in computeHeuristicUnrollFactor()
5340 Loop *L = LI.getLoopFor(CLI->getHeader()); in computeHeuristicUnrollFactor()
5373 // Disable peeling. in computeHeuristicUnrollFactor()
5386 for (BasicBlock *BB : L->blocks()) { in computeHeuristicUnrollFactor()
5390 Ptr = Load->getPointerOperand(); in computeHeuristicUnrollFactor()
5392 Ptr = Store->getPointerOperand(); in computeHeuristicUnrollFactor()
5396 Ptr = Ptr->stripPointerCasts(); in computeHeuristicUnrollFactor()
5399 if (Alloca->getParent() == &F->getEntryBlock()) in computeHeuristicUnrollFactor()
5441 Function *F = Loop->getFunction(); in unrollLoopPartial()
5442 LLVMContext &Ctx = F->getContext(); in unrollLoopPartial()
5444 // If the unrolled loop is not used for another loop-associated directive, it in unrollLoopPartial()
5475 Type *IndVarTy = Loop->getIndVarType(); in unrollLoopPartial()
5477 // Apply partial unrolling by tiling the loop by the unroll-factor, then fully in unrollLoopPartial()
5480 ConstantInt::get(IndVarTy, APInt(IndVarTy->getIntegerBitWidth(), Factor, in unrollLoopPartial()
5500 (*UnrolledCLI)->assertOK(); in unrollLoopPartial()
5632 [](Value *SV) { return SV->getType()->isIntegerTy(64); }) && in createOrderedDepend()
5642 ArgsBase->setAlignment(Align(8)); in createOrderedDepend()
5650 STInst->setAlignment(Align(8)); in createOrderedDepend()
5712 Instruction *SplitPos = EntryBB->getTerminator(); in EmitOMPInlinedRegion()
5715 BasicBlock *ExitBB = EntryBB->splitBasicBlock(SplitPos, "omp_region.end"); in EmitOMPInlinedRegion()
5717 EntryBB->splitBasicBlock(EntryBB->getTerminator(), "omp_region.finalize"); in EmitOMPInlinedRegion()
5719 Builder.SetInsertPoint(EntryBB->getTerminator()); in EmitOMPInlinedRegion()
5727 auto FinIP = InsertPointTy(FiniBB, FiniBB->getFirstInsertionPt()); in EmitOMPInlinedRegion()
5728 assert(FiniBB->getTerminator()->getNumSuccessors() == 1 && in EmitOMPInlinedRegion()
5729 FiniBB->getTerminator()->getSuccessor(0) == ExitBB && in EmitOMPInlinedRegion()
5732 assert(FiniBB->getUniquePredecessor()->getUniqueSuccessor() == FiniBB && in EmitOMPInlinedRegion()
5738 assert(SplitPos->getParent() == ExitBB && in EmitOMPInlinedRegion()
5741 BasicBlock *ExitPredBB = SplitPos->getParent(); in EmitOMPInlinedRegion()
5744 SplitPos->eraseFromParent(); in EmitOMPInlinedRegion()
5763 Function *CurFn = EntryBB->getParent(); in emitCommonDirectiveEntry()
5764 CurFn->insert(std::next(EntryBB->getIterator()), ThenBB); in emitCommonDirectiveEntry()
5767 // branch (If-stmt) in emitCommonDirectiveEntry()
5768 Instruction *EntryBBTI = EntryBB->getTerminator(); in emitCommonDirectiveEntry()
5770 EntryBBTI->removeFromParent(); in emitCommonDirectiveEntry()
5773 UI->eraseFromParent(); in emitCommonDirectiveEntry()
5774 Builder.SetInsertPoint(ThenBB->getTerminator()); in emitCommonDirectiveEntry()
5777 return IRBuilder<>::InsertPoint(ExitBB, ExitBB->getFirstInsertionPt()); in emitCommonDirectiveEntry()
5797 Instruction *FiniBBTI = FiniBB->getTerminator(); in emitCommonDirectiveExit()
5807 ExitCall->removeFromParent(); in emitCommonDirectiveExit()
5810 return IRBuilder<>::InsertPoint(ExitCall->getParent(), in emitCommonDirectiveExit()
5811 ExitCall->getIterator()); in emitCommonDirectiveExit()
5835 Function *CurFn = OMP_Entry->getParent(); in createCopyinClauseBlocks()
5842 if (isa_and_nonnull<BranchInst>(OMP_Entry->getTerminator())) { in createCopyinClauseBlocks()
5843 CopyEnd = OMP_Entry->splitBasicBlock(OMP_Entry->getTerminator(), in createCopyinClauseBlocks()
5845 OMP_Entry->getTerminator()->eraseFromParent(); in createCopyinClauseBlocks()
5908 Device = ConstantInt::get(Int32, -1); in createOMPInteropInit()
5936 Device = ConstantInt::get(Int32, -1); in createOMPInteropDestroy()
5964 Device = ConstantInt::get(Int32, -1); in createOMPInteropUse()
6016 Function *Kernel = Builder.GetInsertBlock()->getParent(); in createTargetInit()
6039 StringRef KernelName = Kernel->getName(); in createTargetInit()
6046 const DataLayout &DL = Fn->getDataLayout(); in createTargetInit()
6056 DynamicEnvironmentGV->setVisibility(GlobalValue::ProtectedVisibility); in createTargetInit()
6059 DynamicEnvironmentGV->getType() == DynamicEnvironmentPtr in createTargetInit()
6089 KernelEnvironmentGV->setVisibility(GlobalValue::ProtectedVisibility); in createTargetInit()
6092 KernelEnvironmentGV->getType() == KernelEnvironmentPtr in createTargetInit()
6096 Value *KernelLaunchEnvironment = Kernel->getArg(0); in createTargetInit()
6101 ThreadKind, ConstantInt::get(ThreadKind->getType(), -1), in createTargetInit()
6105 // if (ThreadKind == -1) in createTargetInit()
6111 BasicBlock *CheckBB = UI->getParent(); in createTargetInit()
6112 BasicBlock *UserCodeEntryBB = CheckBB->splitBasicBlock(UI, "user_code.entry"); in createTargetInit()
6115 CheckBB->getContext(), "worker.exit", CheckBB->getParent()); in createTargetInit()
6119 auto *CheckBBTI = CheckBB->getTerminator(); in createTargetInit()
6121 Builder.CreateCondBr(ExecUserCode, UI->getParent(), WorkerExitBB); in createTargetInit()
6123 CheckBBTI->eraseFromParent(); in createTargetInit()
6124 UI->eraseFromParent(); in createTargetInit()
6128 return InsertPointTy(UserCodeEntryBB, UserCodeEntryBB->getFirstInsertionPt()); in createTargetInit()
6145 Function *Kernel = Builder.GetInsertBlock()->getParent(); in createTargetDeinit()
6147 StringRef KernelName = Kernel->getName(); in createTargetDeinit()
6154 auto *KernelEnvironmentInitializer = KernelEnvironmentGV->getInitializer(); in createTargetDeinit()
6161 KernelEnvironmentGV->setInitializer(NewInitializer); in createTargetDeinit()
6167 for (auto *Op : MD->operands()) { in getNVPTXMDNode()
6168 if (Op->getNumOperands() != 3) in getNVPTXMDNode()
6170 auto *KernelOp = dyn_cast<ConstantAsMetadata>(Op->getOperand(0)); in getNVPTXMDNode()
6171 if (!KernelOp || KernelOp->getValue() != &Kernel) in getNVPTXMDNode()
6173 auto *Prop = dyn_cast<MDString>(Op->getOperand(1)); in getNVPTXMDNode()
6174 if (!Prop || Prop->getString() != Name) in getNVPTXMDNode()
6186 auto *OldVal = cast<ConstantAsMetadata>(ExistingOp->getOperand(2)); in updateNVPTXMetadata()
6187 int32_t OldLimit = cast<ConstantInt>(OldVal->getValue())->getZExtValue(); in updateNVPTXMetadata()
6188 ExistingOp->replaceOperandWith( in updateNVPTXMetadata()
6190 OldVal->getValue()->getType(), in updateNVPTXMetadata()
6201 MD->addOperand(MDNode::get(Ctx, MDVals)); in updateNVPTXMetadata()
6211 const auto &Attr = Kernel.getFnAttribute("amdgpu-flat-work-group-size"); in readThreadBoundsForKernel()
6225 auto *OldVal = cast<ConstantAsMetadata>(ExistingOp->getOperand(2)); in readThreadBoundsForKernel()
6226 int32_t UB = cast<ConstantInt>(OldVal->getValue())->getZExtValue(); in readThreadBoundsForKernel()
6238 Kernel.addFnAttr("amdgpu-flat-work-group-size", in writeThreadBoundsForKernel()
6258 Kernel.addFnAttr("amdgpu-max-num-workgroups", llvm::utostr(LB) + ",1,1"); in writeTeamsForKernel()
6266 OutlinedFn->setLinkage(GlobalValue::WeakODRLinkage); in setOutlinedTargetRegionFunctionAttributes()
6268 OutlinedFn->setDSOLocal(false); in setOutlinedTargetRegionFunctionAttributes()
6269 OutlinedFn->setVisibility(GlobalValue::ProtectedVisibility); in setOutlinedTargetRegionFunctionAttributes()
6271 OutlinedFn->setCallingConv(CallingConv::AMDGPU_KERNEL); in setOutlinedTargetRegionFunctionAttributes()
6351 // Disable TargetData CodeGen on Device pass. in createTargetData()
6372 !MapInfo->Names.empty()); in createTargetData()
6425 emitOffloadingArraysArgument(Builder, RTArgs, Info, !MapInfo->Names.empty(), in createTargetData()
6557 // or i64. This assumes 64-bit address spaces/pointers. in createOutlinedFunction()
6559 ParameterTypes.push_back(Arg->getType()->isPointerTy() in createOutlinedFunction()
6560 ? Arg->getType() in createOutlinedFunction()
6564 ParameterTypes.push_back(Arg->getType()); in createOutlinedFunction()
6570 Builder.GetInsertBlock()->getModule()); in createOutlinedFunction()
6600 Builder.SetInsertPoint(EntryBB->getFirstNonPHI()); in createOutlinedFunction()
6603 Builder.SetInsertPoint(UserCodeEntryBB->getFirstNonPHIOrDbg()); in createOutlinedFunction()
6608 ? make_range(Func->arg_begin() + 1, Func->arg_end()) in createOutlinedFunction()
6609 : Func->args(); in createOutlinedFunction()
6627 // from MLIR to LLVM-IR and the MLIR lowering may still require the original in createOutlinedFunction()
6633 for (User *User : make_early_inc_range(Input->users())) in createOutlinedFunction()
6635 if (Instr->getFunction() == Func) in createOutlinedFunction()
6636 Instr->replaceUsesOfWith(Input, InputCopy); in createOutlinedFunction()
6657 // purposes. If we have mapped a segment that requires a GEP into the 0-th in createOutlinedFunction()
6699 Function *KernelLaunchFunction = StaleCI->getCalledFunction(); in emitTargetTaskProxyFunction()
6717 OpenMPIRBuilder::InsertPointTy IP(StaleCI->getParent(), in emitTargetTaskProxyFunction()
6718 StaleCI->getIterator()); in emitTargetTaskProxyFunction()
6719 LLVMContext &Ctx = StaleCI->getParent()->getContext(); in emitTargetTaskProxyFunction()
6728 Builder.GetInsertBlock()->getModule()); in emitTargetTaskProxyFunction()
6729 ProxyFn->getArg(0)->setName("thread.id"); in emitTargetTaskProxyFunction()
6730 ProxyFn->getArg(1)->setName("task"); in emitTargetTaskProxyFunction()
6736 bool HasShareds = StaleCI->arg_size() > 1; in emitTargetTaskProxyFunction()
6741 assert((!HasShareds || (StaleCI->arg_size() == 2)) && in emitTargetTaskProxyFunction()
6744 auto *ArgStructAlloca = dyn_cast<AllocaInst>(StaleCI->getArgOperand(1)); in emitTargetTaskProxyFunction()
6749 dyn_cast<StructType>(ArgStructAlloca->getAllocatedType()); in emitTargetTaskProxyFunction()
6753 Value *TaskT = ProxyFn->getArg(1); in emitTargetTaskProxyFunction()
6754 Value *ThreadId = ProxyFn->getArg(0); in emitTargetTaskProxyFunction()
6763 NewArgStructAlloca, NewArgStructAlloca->getAlign(), LoadShared, in emitTargetTaskProxyFunction()
6764 LoadShared->getPointerAlignment(M.getDataLayout()), SharedsSize); in emitTargetTaskProxyFunction()
6798 // -------------------------------------------------- in emitTargetTask()
6804 // -------------------------------------------------- in emitTargetTask()
6808 // -------------------------------------------------- in emitTargetTask()
6843 // memcpy(proxy_target_task->shareds, %structArg, sizeof(structArg)) in emitTargetTask()
6888 TargetTaskAllocaBB->begin()); in emitTargetTask()
6889 InsertPointTy TargetTaskBodyIP(TargetTaskBodyBB, TargetTaskBodyBB->begin()); in emitTargetTask()
6919 bool HasShareds = StaleCI->arg_size() > 1; in emitTargetTask()
6938 // Arguments - `loc_ref` (Ident) and `gtid` (ThreadID) in emitTargetTask()
6942 // Argument - `sizeof_kmp_task_t` (TaskSize) in emitTargetTask()
6949 // Argument - `sizeof_shareds` (SharedsSize) in emitTargetTask()
6954 auto *ArgStructAlloca = dyn_cast<AllocaInst>(StaleCI->getArgOperand(1)); in emitTargetTask()
6959 dyn_cast<StructType>(ArgStructAlloca->getAllocatedType()); in emitTargetTask()
6966 // Argument - `flags` in emitTargetTask()
6983 Value *Shareds = StaleCI->getArgOperand(1); in emitTargetTask()
6984 Align Alignment = TaskData->getPointerAlignment(M.getDataLayout()); in emitTargetTask()
6992 // --------------------------------------------------------------- in emitTargetTask()
6997 // --------------------------------------------------------------- in emitTargetTask()
7024 CI->setDebugLoc(StaleCI->getDebugLoc()); in emitTargetTask()
7027 // HasNoWait - meaning the task may be deferred. Call in emitTargetTask()
7043 StaleCI->eraseFromParent(); in emitTargetTask()
7045 [](Instruction *I) { I->eraseFromParent(); }); in emitTargetTask()
7052 << *(Builder.GetInsertBlock()->getParent()->getParent()) in emitTargetTask()
7078 [&](OpenMPIRBuilder::InsertPointTy IP) -> OpenMPIRBuilder::InsertPointTy { in __anon46338d6b3702()
7173 assert(Elem.second->getValueType() == Ty && in getOrCreateInternalVariable()
7180 auto Linkage = this->M.getTargetTriple().rfind("wasm32") == 0 in getOrCreateInternalVariable()
7190 GV->setAlignment(std::max(TypeAlign, PtrAlign)); in getOrCreateInternalVariable()
7206 Constant::getNullValue(PointerType::getUnqual(BasePtr->getContext())); in getSizeInBytes()
7208 Builder.CreateGEP(BasePtr->getType(), Null, Builder.getInt32(1)); in getSizeInBytes()
7219 M, MaptypesArrayInit->getType(), in createOffloadMaptypes()
7222 MaptypesArrayGlobal->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global); in createOffloadMaptypes()
7268 Constant::getNullValue(PointerType::getUnqual(Int8Ptr->getContext())); in emitMapperCall()
7325 // If there is no user-defined mapper, set the mapper array to nullptr to in emitOffloadingArraysArgument()
7357 // equal to the size of base declaration that is non-contiguous. in emitNonContiguousDescriptor()
7360 // non-contiguous. in emitNonContiguousDescriptor()
7369 unsigned RevIdx = EE - II - 1; in emitNonContiguousDescriptor()
7371 DimsAddr->getAllocatedType(), DimsAddr, in emitNonContiguousDescriptor()
7377 M.getDataLayout().getPrefTypeAlign(OffsetLVal->getType())); in emitNonContiguousDescriptor()
7382 M.getDataLayout().getPrefTypeAlign(CountLVal->getType())); in emitNonContiguousDescriptor()
7387 M.getDataLayout().getPrefTypeAlign(CountLVal->getType())); in emitNonContiguousDescriptor()
7464 new GlobalVariable(M, SizesArrayInit->getType(), /*isConstant=*/true, in emitOffloadingArrays()
7466 SizesArrayGbl->setUnnamedAddr(GlobalValue::UnnamedAddr::Global); in emitOffloadingArrays()
7476 Buffer->setAlignment(OffloadSizeAlign); in emitOffloadingArrays()
7479 Buffer, M.getDataLayout().getPrefTypeAlign(Buffer->getType()), in emitOffloadingArrays()
7483 Buffer->getAllocationSize(M.getDataLayout())->getFixedValue())); in emitOffloadingArrays()
7579 MappersArray->getAllocatedType(), MappersArray, in emitOffloadingArrays()
7582 MFunc, MAddr, M.getDataLayout().getPrefTypeAlign(MAddr->getType())); in emitOffloadingArrays()
7594 if (!CurBB || CurBB->getTerminator()) { in emitBranch()
7598 // Otherwise, create a fall-through branch. in emitBranch()
7612 if (IsFinished && BB->use_empty()) { in emitBlock()
7613 BB->eraseFromParent(); in emitBlock()
7619 if (CurBB && CurBB->getParent()) in emitBlock()
7620 CurFn->insert(std::next(CurBB->getIterator()), BB); in emitBlock()
7622 CurFn->insert(CurFn->end(), BB); in emitBlock()
7632 auto CondConstant = CI->getSExtValue(); in emitIfClause()
7640 Function *CurFn = Builder.GetInsertBlock()->getParent(); in emitIfClause()
7704 // do nothing - leave silently. in checkAndEmitFlushAfterAtomic()
7730 assert(X.Var->getType()->isPointerTy() && in createAtomicRead()
7733 assert((XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || in createAtomicRead()
7734 XElemTy->isPointerTy()) && in createAtomicRead()
7739 if (XElemTy->isIntegerTy()) { in createAtomicRead()
7742 XLD->setAtomic(AO); in createAtomicRead()
7747 IntegerType::get(M.getContext(), XElemTy->getScalarSizeInBits()); in createAtomicRead()
7750 XLoad->setAtomic(AO); in createAtomicRead()
7751 if (XElemTy->isFloatingPointTy()) { in createAtomicRead()
7769 assert(X.Var->getType()->isPointerTy() && in createAtomicWrite()
7772 assert((XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || in createAtomicWrite()
7773 XElemTy->isPointerTy()) && in createAtomicWrite()
7776 if (XElemTy->isIntegerTy()) { in createAtomicWrite()
7778 XSt->setAtomic(AO); in createAtomicWrite()
7782 IntegerType::get(M.getContext(), XElemTy->getScalarSizeInBits()); in createAtomicWrite()
7786 XSt->setAtomic(AO); in createAtomicWrite()
7802 Type *XTy = X.Var->getType(); in createAtomicUpdate()
7803 assert(XTy->isPointerTy() && in createAtomicUpdate()
7806 assert((XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || in createAtomicUpdate()
7807 XElemTy->isPointerTy()) && in createAtomicUpdate()
7857 // TODO: handle the case where XElemTy is not byte-sized or not a power of 2 in emitAtomicUpdate()
7875 emitRMWOp &= XElemTy->isIntegerTy(); in emitAtomicUpdate()
7889 IntegerType::get(M.getContext(), XElemTy->getScalarSizeInBits()); in emitAtomicUpdate()
7891 Builder.CreateLoad(IntCastTy, X, X->getName() + ".atomic.load"); in emitAtomicUpdate()
7892 OldVal->setAtomic(AO); in emitAtomicUpdate()
7894 // | /---\ in emitAtomicUpdate()
7896 // | \---/ in emitAtomicUpdate()
7899 Instruction *CurBBTI = CurBB->getTerminator(); in emitAtomicUpdate()
7902 CurBB->splitBasicBlock(CurBBTI, X->getName() + ".atomic.exit"); in emitAtomicUpdate()
7903 BasicBlock *ContBB = CurBB->splitBasicBlock(CurBB->getTerminator(), in emitAtomicUpdate()
7904 X->getName() + ".atomic.cont"); in emitAtomicUpdate()
7905 ContBB->getTerminator()->eraseFromParent(); in emitAtomicUpdate()
7908 NewAtomicAddr->setName(X->getName() + "x.new.val"); in emitAtomicUpdate()
7910 llvm::PHINode *PHI = Builder.CreatePHI(OldVal->getType(), 2); in emitAtomicUpdate()
7911 PHI->addIncoming(OldVal, CurBB); in emitAtomicUpdate()
7912 bool IsIntTy = XElemTy->isIntegerTy(); in emitAtomicUpdate()
7915 if (XElemTy->isFloatingPointTy()) { in emitAtomicUpdate()
7917 X->getName() + ".atomic.fltCast"); in emitAtomicUpdate()
7920 X->getName() + ".atomic.ptrCast"); in emitAtomicUpdate()
7931 Result->setVolatile(VolatileX); in emitAtomicUpdate()
7934 PHI->addIncoming(PreviousVal, Builder.GetInsertBlock()); in emitAtomicUpdate()
7942 dyn_cast<UnreachableInst>(ExitBB->getTerminator())) { in emitAtomicUpdate()
7943 CurBBTI->eraseFromParent(); in emitAtomicUpdate()
7962 Type *XTy = X.Var->getType(); in createAtomicCapture()
7963 assert(XTy->isPointerTy() && in createAtomicCapture()
7966 assert((XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || in createAtomicCapture()
7967 XElemTy->isPointerTy()) && in createAtomicCapture()
8007 assert(X.Var->getType()->isPointerTy() && in createAtomicCompare()
8011 assert(V.Var->getType()->isPointerTy() && "v.var must be of pointer type"); in createAtomicCompare()
8015 bool IsInteger = E->getType()->isIntegerTy(); in createAtomicCompare()
8021 IntegerType::get(M.getContext(), X.ElemTy->getScalarSizeInBits()); in createAtomicCompare()
8035 assert(OldValue->getType() == V.ElemTy && in createAtomicCompare()
8042 // CurBB---- in createAtomicCompare()
8048 // ExitBB <- in createAtomicCompare()
8052 Instruction *CurBBTI = CurBB->getTerminator(); in createAtomicCompare()
8054 BasicBlock *ExitBB = CurBB->splitBasicBlock( in createAtomicCompare()
8055 CurBBTI, X.Var->getName() + ".atomic.exit"); in createAtomicCompare()
8056 BasicBlock *ContBB = CurBB->splitBasicBlock( in createAtomicCompare()
8057 CurBB->getTerminator(), X.Var->getName() + ".atomic.cont"); in createAtomicCompare()
8058 ContBB->getTerminator()->eraseFromParent(); in createAtomicCompare()
8059 CurBB->getTerminator()->eraseFromParent(); in createAtomicCompare()
8068 dyn_cast<UnreachableInst>(ExitBB->getTerminator())) { in createAtomicCompare()
8069 CurBBTI->eraseFromParent(); in createAtomicCompare()
8083 assert(R.Var->getType()->isPointerTy() && in createAtomicCompare()
8085 assert(R.ElemTy->isIntegerTy() && "r must be of integral type"); in createAtomicCompare()
8186 Function *CurrentFunction = Builder.GetInsertBlock()->getParent(); in createTeams()
8189 BasicBlock &OuterAllocaBB = CurrentFunction->getEntryBlock(); in createTeams()
8192 Builder.SetInsertPoint(BodyBB, BodyBB->begin()); in createTeams()
8222 "if lowerbound is non-null, then upperbound must also be non-null " in createTeams()
8232 assert(IfExpr->getType()->isIntegerTy() && in createTeams()
8236 if (IfExpr->getType() != Int1) in createTeams()
8238 ConstantInt::get(IfExpr->getType(), 0)); in createTeams()
8256 InsertPointTy AllocaIP(AllocaBB, AllocaBB->begin()); in createTeams()
8257 InsertPointTy CodeGenIP(BodyBB, BodyBB->begin()); in createTeams()
8288 OutlinedFn.getArg(0)->setName("global.tid.ptr"); in createTeams()
8289 OutlinedFn.getArg(1)->setName("bound.tid.ptr"); in createTeams()
8291 OutlinedFn.getArg(2)->setName("data"); in createTeams()
8294 assert(StaleCI && "Error while outlining - no CallInst user found for the " in createTeams()
8298 Ident, Builder.getInt32(StaleCI->arg_size() - 2), &OutlinedFn}; in createTeams()
8300 Args.push_back(StaleCI->getArgOperand(2)); in createTeams()
8306 [](Instruction *I) { I->eraseFromParent(); }); in createTeams()
8315 Builder.SetInsertPoint(ExitBB, ExitBB->begin()); in createTeams()
8328 M, MapNamesArrayInit->getType(), in createOffloadMapnames()
8378 M, ID, Name.empty() ? Addr->getName() : Name, Size, Flags, /*Data=*/0, in createOffloadEntry()
8388 Module &M = *(Fn->getParent()); in createOffloadEntry()
8398 MD->addOperand(MDNode::get(Ctx, MDVals)); in createOffloadEntry()
8401 Fn->addFnAttr(Attribute::get(Ctx, "kernel")); in createOffloadEntry()
8403 Fn->addFnAttr("uniform-work-group-size", "true"); in createOffloadEntry()
8404 Fn->addFnAttr(Attribute::MustProgress); in createOffloadEntry()
8436 // - Entry 0 -> Kind of this type of metadata (0). in createOffloadEntriesAndInfoMetadata()
8437 // - Entry 1 -> Device ID of the file where the entry was identified. in createOffloadEntriesAndInfoMetadata()
8438 // - Entry 2 -> File ID of the file where the entry was identified. in createOffloadEntriesAndInfoMetadata()
8439 // - Entry 3 -> Mangled name of the function where the entry was in createOffloadEntriesAndInfoMetadata()
8441 // - Entry 4 -> Line in the file where the entry was identified. in createOffloadEntriesAndInfoMetadata()
8442 // - Entry 5 -> Count of regions at this DeviceID/FilesID/Line. in createOffloadEntriesAndInfoMetadata()
8443 // - Entry 6 -> Order the entry was created. in createOffloadEntriesAndInfoMetadata()
8455 MD->addOperand(MDNode::get(C, Ops)); in createOffloadEntriesAndInfoMetadata()
8467 // - Entry 0 -> Kind of this type of metadata (1). in createOffloadEntriesAndInfoMetadata()
8468 // - Entry 1 -> Mangled name of the variable. in createOffloadEntriesAndInfoMetadata()
8469 // - Entry 2 -> Declare target kind. in createOffloadEntriesAndInfoMetadata()
8470 // - Entry 3 -> Order the entry was created. in createOffloadEntriesAndInfoMetadata()
8480 MD->addOperand(MDNode::get(C, Ops)); in createOffloadEntriesAndInfoMetadata()
8491 if (!CE->getID() || !CE->getAddress()) { in createOffloadEntriesAndInfoMetadata()
8500 createOffloadEntry(CE->getID(), CE->getAddress(), in createOffloadEntriesAndInfoMetadata()
8501 /*Size=*/0, CE->getFlags(), in createOffloadEntriesAndInfoMetadata()
8508 CE->getFlags()); in createOffloadEntriesAndInfoMetadata()
8514 if (!CE->getAddress()) { in createOffloadEntriesAndInfoMetadata()
8518 // The vaiable has no definition - no need to add the entry. in createOffloadEntriesAndInfoMetadata()
8519 if (CE->getVarSize() == 0) in createOffloadEntriesAndInfoMetadata()
8523 assert(((Config.isTargetDevice() && !CE->getAddress()) || in createOffloadEntriesAndInfoMetadata()
8524 (!Config.isTargetDevice() && CE->getAddress())) && in createOffloadEntriesAndInfoMetadata()
8528 if (!CE->getAddress()) { in createOffloadEntriesAndInfoMetadata()
8540 if (auto *GV = dyn_cast<GlobalValue>(CE->getAddress())) in createOffloadEntriesAndInfoMetadata()
8541 if ((GV->hasLocalLinkage() || GV->hasHiddenVisibility()) && in createOffloadEntriesAndInfoMetadata()
8548 createOffloadEntry(CE->getAddress(), CE->getAddress(), CE->getVarSize(), in createOffloadEntriesAndInfoMetadata()
8549 Flags, CE->getLinkage(), CE->getVarName()); in createOffloadEntriesAndInfoMetadata()
8551 createOffloadEntry(CE->getAddress(), CE->getAddress(), CE->getVarSize(), in createOffloadEntriesAndInfoMetadata()
8552 Flags, CE->getLinkage()); in createOffloadEntriesAndInfoMetadata()
8561 // TODO: This reduces the offloading entries to a 32-bit integer. Offloading in createOffloadEntriesAndInfoMetadata()
8562 // entries should be redesigned to better suit this use-case. in createOffloadEntriesAndInfoMetadata()
8676 GV->setLinkage(GlobalValue::WeakAnyLinkage); in getAddrOfDeclareTargetVar()
8680 GV->setInitializer(GlobalInitializer()); in getAddrOfDeclareTargetVar()
8682 GV->setInitializer(GlobalValue); in getAddrOfDeclareTargetVar()
8726 M.getDataLayout().getTypeSizeInBits(LlvmVal->getValueType()), 8); in registerTargetGlobalVariable()
8729 Linkage = (VariableLinkage) ? VariableLinkage() : LlvmVal->getLinkage(); in registerTargetGlobalVariable()
8735 // Do not create a "ref-variable" if the original is not also available in registerTargetGlobalVariable()
8744 getOrCreateInternalVariable(Addr->getType(), RefName); in registerTargetGlobalVariable()
8746 GvAddrRef->setConstant(true); in registerTargetGlobalVariable()
8747 GvAddrRef->setLinkage(GlobalValue::InternalLinkage); in registerTargetGlobalVariable()
8748 GvAddrRef->setInitializer(Addr); in registerTargetGlobalVariable()
8759 VarName = (Addr) ? Addr->getName() : ""; in registerTargetGlobalVariable()
8766 VarName = (Addr) ? Addr->getName() : ""; in registerTargetGlobalVariable()
8786 for (MDNode *MN : MD->operands()) { in loadOffloadInfoMetadata()
8788 auto *V = cast<ConstantAsMetadata>(MN->getOperand(Idx)); in loadOffloadInfoMetadata()
8789 return cast<ConstantInt>(V->getValue())->getZExtValue(); in loadOffloadInfoMetadata()
8793 auto *V = cast<MDString>(MN->getOperand(Idx)); in loadOffloadInfoMetadata()
8794 return V->getString(); in loadOffloadInfoMetadata()
8838 Ctx, parseBitcodeFile(Buf.get()->getMemBufferRef(), Ctx)); in loadOffloadInfoMetadata()
8848 //===----------------------------------------------------------------------===//
8850 //===----------------------------------------------------------------------===//
8863 return It->second; in getTargetRegionEntryInfoCount()
8891 if (OMPBuilder->Config.isTargetDevice()) { in registerTargetRegionEntryInfo()
8924 if (!IgnoreAddressId && (It->second.getAddress() || It->second.getID())) in hasTargetRegionEntryInfo()
8946 if (OMPBuilder->Config.isTargetDevice()) { in registerDeviceGlobalVarEntryInfo()
8990 //===----------------------------------------------------------------------===//
8992 //===----------------------------------------------------------------------===//
9016 Instruction *CmpI = &getCond()->front(); in setTripCount()
9018 CmpI->setOperand(1, TripCount); in setTripCount()
9035 for (Use &U : OldIV->uses()) { in mapIndVar()
9039 if (User->getParent() == getCond()) in mapIndVar()
9041 if (User->getParent() == getLatch()) in mapIndVar()
9051 U->set(NewIV); in mapIndVar()
9068 // Verify standard control-flow we use for OpenMP loops. in assertOK()
9070 assert(isa<BranchInst>(Preheader->getTerminator()) && in assertOK()
9072 assert(Preheader->getSingleSuccessor() == Header && in assertOK()
9076 assert(isa<BranchInst>(Header->getTerminator()) && in assertOK()
9078 assert(Header->getSingleSuccessor() == Cond && in assertOK()
9082 assert(Cond->getSinglePredecessor() == Header && in assertOK()
9085 assert(isa<BranchInst>(Cond->getTerminator()) && in assertOK()
9089 assert(cast<BranchInst>(Cond->getTerminator())->getSuccessor(0) == Body && in assertOK()
9091 assert(cast<BranchInst>(Cond->getTerminator())->getSuccessor(1) == Exit && in assertOK()
9095 assert(Body->getSinglePredecessor() == Cond && in assertOK()
9097 assert(!isa<PHINode>(Body->front())); in assertOK()
9100 assert(isa<BranchInst>(Latch->getTerminator()) && in assertOK()
9102 assert(Latch->getSingleSuccessor() == Header && "Latch must jump to header"); in assertOK()
9105 assert(Latch->getSinglePredecessor() != nullptr); in assertOK()
9106 assert(!isa<PHINode>(Latch->front())); in assertOK()
9109 assert(isa<BranchInst>(Exit->getTerminator()) && in assertOK()
9111 assert(Exit->getSingleSuccessor() == After && in assertOK()
9115 assert(After->getSinglePredecessor() == Exit && in assertOK()
9117 assert(After->empty() || !isa<PHINode>(After->front())); in assertOK()
9121 assert(isa<IntegerType>(IndVar->getType()) && in assertOK()
9123 assert(cast<PHINode>(IndVar)->getParent() == Header && in assertOK()
9125 assert(cast<PHINode>(IndVar)->getIncomingBlock(0) == Preheader); in assertOK()
9127 cast<ConstantInt>(cast<PHINode>(IndVar)->getIncomingValue(0))->isZero()); in assertOK()
9128 assert(cast<PHINode>(IndVar)->getIncomingBlock(1) == Latch); in assertOK()
9130 auto *NextIndVar = cast<PHINode>(IndVar)->getIncomingValue(1); in assertOK()
9131 assert(cast<Instruction>(NextIndVar)->getParent() == Latch); in assertOK()
9132 assert(cast<BinaryOperator>(NextIndVar)->getOpcode() == BinaryOperator::Add); in assertOK()
9133 assert(cast<BinaryOperator>(NextIndVar)->getOperand(0) == IndVar); in assertOK()
9134 assert(cast<ConstantInt>(cast<BinaryOperator>(NextIndVar)->getOperand(1)) in assertOK()
9135 ->isOne()); in assertOK()
9139 assert(IndVar->getType() == TripCount->getType() && in assertOK()
9142 auto *CmpI = cast<CmpInst>(&Cond->front()); in assertOK()
9143 assert(CmpI->getPredicate() == CmpInst::ICMP_ULT && in assertOK()
9144 "Exit condition must be a signed less-than comparison"); in assertOK()
9145 assert(CmpI->getOperand(0) == IndVar && in assertOK()
9147 assert(CmpI->getOperand(1) == TripCount && in assertOK()