1 //===----- CGOpenMPRuntime.h - Interface to OpenMP Runtimes -----*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This provides a class for OpenMP runtime code generation. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #ifndef LLVM_CLANG_LIB_CODEGEN_CGOPENMPRUNTIME_H 14 #define LLVM_CLANG_LIB_CODEGEN_CGOPENMPRUNTIME_H 15 16 #include "CGValue.h" 17 #include "clang/AST/DeclOpenMP.h" 18 #include "clang/AST/GlobalDecl.h" 19 #include "clang/AST/Type.h" 20 #include "clang/Basic/OpenMPKinds.h" 21 #include "clang/Basic/SourceLocation.h" 22 #include "llvm/ADT/DenseMap.h" 23 #include "llvm/ADT/PointerIntPair.h" 24 #include "llvm/ADT/SmallPtrSet.h" 25 #include "llvm/ADT/StringMap.h" 26 #include "llvm/ADT/StringSet.h" 27 #include "llvm/Frontend/OpenMP/OMPConstants.h" 28 #include "llvm/Frontend/OpenMP/OMPIRBuilder.h" 29 #include "llvm/IR/Function.h" 30 #include "llvm/IR/ValueHandle.h" 31 #include "llvm/Support/AtomicOrdering.h" 32 33 namespace llvm { 34 class ArrayType; 35 class Constant; 36 class FunctionType; 37 class GlobalVariable; 38 class StructType; 39 class Type; 40 class Value; 41 class OpenMPIRBuilder; 42 } // namespace llvm 43 44 namespace clang { 45 class Expr; 46 class OMPDependClause; 47 class OMPExecutableDirective; 48 class OMPLoopDirective; 49 class VarDecl; 50 class OMPDeclareReductionDecl; 51 class IdentifierInfo; 52 53 namespace CodeGen { 54 class Address; 55 class CodeGenFunction; 56 class CodeGenModule; 57 58 /// A basic class for pre|post-action for advanced codegen sequence for OpenMP 59 /// region. 60 class PrePostActionTy { 61 public: 62 explicit PrePostActionTy() {} 63 virtual void Enter(CodeGenFunction &CGF) {} 64 virtual void Exit(CodeGenFunction &CGF) {} 65 virtual ~PrePostActionTy() {} 66 }; 67 68 /// Class provides a way to call simple version of codegen for OpenMP region, or 69 /// an advanced with possible pre|post-actions in codegen. 70 class RegionCodeGenTy final { 71 intptr_t CodeGen; 72 typedef void (*CodeGenTy)(intptr_t, CodeGenFunction &, PrePostActionTy &); 73 CodeGenTy Callback; 74 mutable PrePostActionTy *PrePostAction; 75 RegionCodeGenTy() = delete; 76 RegionCodeGenTy &operator=(const RegionCodeGenTy &) = delete; 77 template <typename Callable> 78 static void CallbackFn(intptr_t CodeGen, CodeGenFunction &CGF, 79 PrePostActionTy &Action) { 80 return (*reinterpret_cast<Callable *>(CodeGen))(CGF, Action); 81 } 82 83 public: 84 template <typename Callable> 85 RegionCodeGenTy( 86 Callable &&CodeGen, 87 std::enable_if_t<!std::is_same<std::remove_reference_t<Callable>, 88 RegionCodeGenTy>::value> * = nullptr) 89 : CodeGen(reinterpret_cast<intptr_t>(&CodeGen)), 90 Callback(CallbackFn<std::remove_reference_t<Callable>>), 91 PrePostAction(nullptr) {} 92 void setAction(PrePostActionTy &Action) const { PrePostAction = &Action; } 93 void operator()(CodeGenFunction &CGF) const; 94 }; 95 96 struct OMPTaskDataTy final { 97 SmallVector<const Expr *, 4> PrivateVars; 98 SmallVector<const Expr *, 4> PrivateCopies; 99 SmallVector<const Expr *, 4> FirstprivateVars; 100 SmallVector<const Expr *, 4> FirstprivateCopies; 101 SmallVector<const Expr *, 4> FirstprivateInits; 102 SmallVector<const Expr *, 4> LastprivateVars; 103 SmallVector<const Expr *, 4> LastprivateCopies; 104 SmallVector<const Expr *, 4> ReductionVars; 105 SmallVector<const Expr *, 4> ReductionOrigs; 106 SmallVector<const Expr *, 4> ReductionCopies; 107 SmallVector<const Expr *, 4> ReductionOps; 108 struct DependData { 109 OpenMPDependClauseKind DepKind = OMPC_DEPEND_unknown; 110 const Expr *IteratorExpr = nullptr; 111 SmallVector<const Expr *, 4> DepExprs; 112 explicit DependData() = default; 113 DependData(OpenMPDependClauseKind DepKind, const Expr *IteratorExpr) 114 : DepKind(DepKind), IteratorExpr(IteratorExpr) {} 115 }; 116 SmallVector<DependData, 4> Dependences; 117 llvm::PointerIntPair<llvm::Value *, 1, bool> Final; 118 llvm::PointerIntPair<llvm::Value *, 1, bool> Schedule; 119 llvm::PointerIntPair<llvm::Value *, 1, bool> Priority; 120 llvm::Value *Reductions = nullptr; 121 unsigned NumberOfParts = 0; 122 bool Tied = true; 123 bool Nogroup = false; 124 bool IsReductionWithTaskMod = false; 125 bool IsWorksharingReduction = false; 126 }; 127 128 /// Class intended to support codegen of all kind of the reduction clauses. 129 class ReductionCodeGen { 130 private: 131 /// Data required for codegen of reduction clauses. 132 struct ReductionData { 133 /// Reference to the item shared between tasks to reduce into. 134 const Expr *Shared = nullptr; 135 /// Reference to the original item. 136 const Expr *Ref = nullptr; 137 /// Helper expression for generation of private copy. 138 const Expr *Private = nullptr; 139 /// Helper expression for generation reduction operation. 140 const Expr *ReductionOp = nullptr; 141 ReductionData(const Expr *Shared, const Expr *Ref, const Expr *Private, 142 const Expr *ReductionOp) 143 : Shared(Shared), Ref(Ref), Private(Private), ReductionOp(ReductionOp) { 144 } 145 }; 146 /// List of reduction-based clauses. 147 SmallVector<ReductionData, 4> ClausesData; 148 149 /// List of addresses of shared variables/expressions. 150 SmallVector<std::pair<LValue, LValue>, 4> SharedAddresses; 151 /// List of addresses of original variables/expressions. 152 SmallVector<std::pair<LValue, LValue>, 4> OrigAddresses; 153 /// Sizes of the reduction items in chars. 154 SmallVector<std::pair<llvm::Value *, llvm::Value *>, 4> Sizes; 155 /// Base declarations for the reduction items. 156 SmallVector<const VarDecl *, 4> BaseDecls; 157 158 /// Emits lvalue for shared expression. 159 LValue emitSharedLValue(CodeGenFunction &CGF, const Expr *E); 160 /// Emits upper bound for shared expression (if array section). 161 LValue emitSharedLValueUB(CodeGenFunction &CGF, const Expr *E); 162 /// Performs aggregate initialization. 163 /// \param N Number of reduction item in the common list. 164 /// \param PrivateAddr Address of the corresponding private item. 165 /// \param SharedLVal Address of the original shared variable. 166 /// \param DRD Declare reduction construct used for reduction item. 167 void emitAggregateInitialization(CodeGenFunction &CGF, unsigned N, 168 Address PrivateAddr, LValue SharedLVal, 169 const OMPDeclareReductionDecl *DRD); 170 171 public: 172 ReductionCodeGen(ArrayRef<const Expr *> Shareds, ArrayRef<const Expr *> Origs, 173 ArrayRef<const Expr *> Privates, 174 ArrayRef<const Expr *> ReductionOps); 175 /// Emits lvalue for the shared and original reduction item. 176 /// \param N Number of the reduction item. 177 void emitSharedOrigLValue(CodeGenFunction &CGF, unsigned N); 178 /// Emits the code for the variable-modified type, if required. 179 /// \param N Number of the reduction item. 180 void emitAggregateType(CodeGenFunction &CGF, unsigned N); 181 /// Emits the code for the variable-modified type, if required. 182 /// \param N Number of the reduction item. 183 /// \param Size Size of the type in chars. 184 void emitAggregateType(CodeGenFunction &CGF, unsigned N, llvm::Value *Size); 185 /// Performs initialization of the private copy for the reduction item. 186 /// \param N Number of the reduction item. 187 /// \param PrivateAddr Address of the corresponding private item. 188 /// \param DefaultInit Default initialization sequence that should be 189 /// performed if no reduction specific initialization is found. 190 /// \param SharedLVal Address of the original shared variable. 191 void 192 emitInitialization(CodeGenFunction &CGF, unsigned N, Address PrivateAddr, 193 LValue SharedLVal, 194 llvm::function_ref<bool(CodeGenFunction &)> DefaultInit); 195 /// Returns true if the private copy requires cleanups. 196 bool needCleanups(unsigned N); 197 /// Emits cleanup code for the reduction item. 198 /// \param N Number of the reduction item. 199 /// \param PrivateAddr Address of the corresponding private item. 200 void emitCleanups(CodeGenFunction &CGF, unsigned N, Address PrivateAddr); 201 /// Adjusts \p PrivatedAddr for using instead of the original variable 202 /// address in normal operations. 203 /// \param N Number of the reduction item. 204 /// \param PrivateAddr Address of the corresponding private item. 205 Address adjustPrivateAddress(CodeGenFunction &CGF, unsigned N, 206 Address PrivateAddr); 207 /// Returns LValue for the reduction item. 208 LValue getSharedLValue(unsigned N) const { return SharedAddresses[N].first; } 209 /// Returns LValue for the original reduction item. 210 LValue getOrigLValue(unsigned N) const { return OrigAddresses[N].first; } 211 /// Returns the size of the reduction item (in chars and total number of 212 /// elements in the item), or nullptr, if the size is a constant. 213 std::pair<llvm::Value *, llvm::Value *> getSizes(unsigned N) const { 214 return Sizes[N]; 215 } 216 /// Returns the base declaration of the reduction item. 217 const VarDecl *getBaseDecl(unsigned N) const { return BaseDecls[N]; } 218 /// Returns the base declaration of the reduction item. 219 const Expr *getRefExpr(unsigned N) const { return ClausesData[N].Ref; } 220 /// Returns true if the initialization of the reduction item uses initializer 221 /// from declare reduction construct. 222 bool usesReductionInitializer(unsigned N) const; 223 }; 224 225 class CGOpenMPRuntime { 226 public: 227 /// Allows to disable automatic handling of functions used in target regions 228 /// as those marked as `omp declare target`. 229 class DisableAutoDeclareTargetRAII { 230 CodeGenModule &CGM; 231 bool SavedShouldMarkAsGlobal; 232 233 public: 234 DisableAutoDeclareTargetRAII(CodeGenModule &CGM); 235 ~DisableAutoDeclareTargetRAII(); 236 }; 237 238 /// Manages list of nontemporal decls for the specified directive. 239 class NontemporalDeclsRAII { 240 CodeGenModule &CGM; 241 const bool NeedToPush; 242 243 public: 244 NontemporalDeclsRAII(CodeGenModule &CGM, const OMPLoopDirective &S); 245 ~NontemporalDeclsRAII(); 246 }; 247 248 /// Maps the expression for the lastprivate variable to the global copy used 249 /// to store new value because original variables are not mapped in inner 250 /// parallel regions. Only private copies are captured but we need also to 251 /// store private copy in shared address. 252 /// Also, stores the expression for the private loop counter and it 253 /// threaprivate name. 254 struct LastprivateConditionalData { 255 llvm::MapVector<CanonicalDeclPtr<const Decl>, SmallString<16>> 256 DeclToUniqueName; 257 LValue IVLVal; 258 llvm::Function *Fn = nullptr; 259 bool Disabled = false; 260 }; 261 /// Manages list of lastprivate conditional decls for the specified directive. 262 class LastprivateConditionalRAII { 263 enum class ActionToDo { 264 DoNotPush, 265 PushAsLastprivateConditional, 266 DisableLastprivateConditional, 267 }; 268 CodeGenModule &CGM; 269 ActionToDo Action = ActionToDo::DoNotPush; 270 271 /// Check and try to disable analysis of inner regions for changes in 272 /// lastprivate conditional. 273 void tryToDisableInnerAnalysis(const OMPExecutableDirective &S, 274 llvm::DenseSet<CanonicalDeclPtr<const Decl>> 275 &NeedToAddForLPCsAsDisabled) const; 276 277 LastprivateConditionalRAII(CodeGenFunction &CGF, 278 const OMPExecutableDirective &S); 279 280 public: 281 explicit LastprivateConditionalRAII(CodeGenFunction &CGF, 282 const OMPExecutableDirective &S, 283 LValue IVLVal); 284 static LastprivateConditionalRAII disable(CodeGenFunction &CGF, 285 const OMPExecutableDirective &S); 286 ~LastprivateConditionalRAII(); 287 }; 288 289 llvm::OpenMPIRBuilder &getOMPBuilder() { return OMPBuilder; } 290 291 protected: 292 CodeGenModule &CGM; 293 StringRef FirstSeparator, Separator; 294 295 /// Constructor allowing to redefine the name separator for the variables. 296 explicit CGOpenMPRuntime(CodeGenModule &CGM, StringRef FirstSeparator, 297 StringRef Separator); 298 299 /// Creates offloading entry for the provided entry ID \a ID, 300 /// address \a Addr, size \a Size, and flags \a Flags. 301 virtual void createOffloadEntry(llvm::Constant *ID, llvm::Constant *Addr, 302 uint64_t Size, int32_t Flags, 303 llvm::GlobalValue::LinkageTypes Linkage); 304 305 /// Helper to emit outlined function for 'target' directive. 306 /// \param D Directive to emit. 307 /// \param ParentName Name of the function that encloses the target region. 308 /// \param OutlinedFn Outlined function value to be defined by this call. 309 /// \param OutlinedFnID Outlined function ID value to be defined by this call. 310 /// \param IsOffloadEntry True if the outlined function is an offload entry. 311 /// \param CodeGen Lambda codegen specific to an accelerator device. 312 /// An outlined function may not be an entry if, e.g. the if clause always 313 /// evaluates to false. 314 virtual void emitTargetOutlinedFunctionHelper(const OMPExecutableDirective &D, 315 StringRef ParentName, 316 llvm::Function *&OutlinedFn, 317 llvm::Constant *&OutlinedFnID, 318 bool IsOffloadEntry, 319 const RegionCodeGenTy &CodeGen); 320 321 /// Emits object of ident_t type with info for source location. 322 /// \param Flags Flags for OpenMP location. 323 /// 324 llvm::Value *emitUpdateLocation(CodeGenFunction &CGF, SourceLocation Loc, 325 unsigned Flags = 0); 326 327 /// Returns pointer to ident_t type. 328 llvm::Type *getIdentTyPointerTy(); 329 330 /// Gets thread id value for the current thread. 331 /// 332 llvm::Value *getThreadID(CodeGenFunction &CGF, SourceLocation Loc); 333 334 /// Get the function name of an outlined region. 335 // The name can be customized depending on the target. 336 // 337 virtual StringRef getOutlinedHelperName() const { return ".omp_outlined."; } 338 339 /// Emits \p Callee function call with arguments \p Args with location \p Loc. 340 void emitCall(CodeGenFunction &CGF, SourceLocation Loc, 341 llvm::FunctionCallee Callee, 342 ArrayRef<llvm::Value *> Args = llvm::None) const; 343 344 /// Emits address of the word in a memory where current thread id is 345 /// stored. 346 virtual Address emitThreadIDAddress(CodeGenFunction &CGF, SourceLocation Loc); 347 348 void setLocThreadIdInsertPt(CodeGenFunction &CGF, 349 bool AtCurrentPoint = false); 350 void clearLocThreadIdInsertPt(CodeGenFunction &CGF); 351 352 /// Check if the default location must be constant. 353 /// Default is false to support OMPT/OMPD. 354 virtual bool isDefaultLocationConstant() const { return false; } 355 356 /// Returns additional flags that can be stored in reserved_2 field of the 357 /// default location. 358 virtual unsigned getDefaultLocationReserved2Flags() const { return 0; } 359 360 /// Returns default flags for the barriers depending on the directive, for 361 /// which this barier is going to be emitted. 362 static unsigned getDefaultFlagsForBarriers(OpenMPDirectiveKind Kind); 363 364 /// Get the LLVM type for the critical name. 365 llvm::ArrayType *getKmpCriticalNameTy() const {return KmpCriticalNameTy;} 366 367 /// Returns corresponding lock object for the specified critical region 368 /// name. If the lock object does not exist it is created, otherwise the 369 /// reference to the existing copy is returned. 370 /// \param CriticalName Name of the critical region. 371 /// 372 llvm::Value *getCriticalRegionLock(StringRef CriticalName); 373 374 private: 375 /// An OpenMP-IR-Builder instance. 376 llvm::OpenMPIRBuilder OMPBuilder; 377 /// Default const ident_t object used for initialization of all other 378 /// ident_t objects. 379 llvm::Constant *DefaultOpenMPPSource = nullptr; 380 using FlagsTy = std::pair<unsigned, unsigned>; 381 /// Map of flags and corresponding default locations. 382 using OpenMPDefaultLocMapTy = llvm::DenseMap<FlagsTy, llvm::Value *>; 383 OpenMPDefaultLocMapTy OpenMPDefaultLocMap; 384 Address getOrCreateDefaultLocation(unsigned Flags); 385 386 QualType IdentQTy; 387 llvm::StructType *IdentTy = nullptr; 388 /// Map for SourceLocation and OpenMP runtime library debug locations. 389 typedef llvm::DenseMap<unsigned, llvm::Value *> OpenMPDebugLocMapTy; 390 OpenMPDebugLocMapTy OpenMPDebugLocMap; 391 /// The type for a microtask which gets passed to __kmpc_fork_call(). 392 /// Original representation is: 393 /// typedef void (kmpc_micro)(kmp_int32 global_tid, kmp_int32 bound_tid,...); 394 llvm::FunctionType *Kmpc_MicroTy = nullptr; 395 /// Stores debug location and ThreadID for the function. 396 struct DebugLocThreadIdTy { 397 llvm::Value *DebugLoc; 398 llvm::Value *ThreadID; 399 /// Insert point for the service instructions. 400 llvm::AssertingVH<llvm::Instruction> ServiceInsertPt = nullptr; 401 }; 402 /// Map of local debug location, ThreadId and functions. 403 typedef llvm::DenseMap<llvm::Function *, DebugLocThreadIdTy> 404 OpenMPLocThreadIDMapTy; 405 OpenMPLocThreadIDMapTy OpenMPLocThreadIDMap; 406 /// Map of UDRs and corresponding combiner/initializer. 407 typedef llvm::DenseMap<const OMPDeclareReductionDecl *, 408 std::pair<llvm::Function *, llvm::Function *>> 409 UDRMapTy; 410 UDRMapTy UDRMap; 411 /// Map of functions and locally defined UDRs. 412 typedef llvm::DenseMap<llvm::Function *, 413 SmallVector<const OMPDeclareReductionDecl *, 4>> 414 FunctionUDRMapTy; 415 FunctionUDRMapTy FunctionUDRMap; 416 /// Map from the user-defined mapper declaration to its corresponding 417 /// functions. 418 llvm::DenseMap<const OMPDeclareMapperDecl *, llvm::Function *> UDMMap; 419 /// Map of functions and their local user-defined mappers. 420 using FunctionUDMMapTy = 421 llvm::DenseMap<llvm::Function *, 422 SmallVector<const OMPDeclareMapperDecl *, 4>>; 423 FunctionUDMMapTy FunctionUDMMap; 424 /// Maps local variables marked as lastprivate conditional to their internal 425 /// types. 426 llvm::DenseMap<llvm::Function *, 427 llvm::DenseMap<CanonicalDeclPtr<const Decl>, 428 std::tuple<QualType, const FieldDecl *, 429 const FieldDecl *, LValue>>> 430 LastprivateConditionalToTypes; 431 /// Type kmp_critical_name, originally defined as typedef kmp_int32 432 /// kmp_critical_name[8]; 433 llvm::ArrayType *KmpCriticalNameTy; 434 /// An ordered map of auto-generated variables to their unique names. 435 /// It stores variables with the following names: 1) ".gomp_critical_user_" + 436 /// <critical_section_name> + ".var" for "omp critical" directives; 2) 437 /// <mangled_name_for_global_var> + ".cache." for cache for threadprivate 438 /// variables. 439 llvm::StringMap<llvm::AssertingVH<llvm::Constant>, llvm::BumpPtrAllocator> 440 InternalVars; 441 /// Type typedef kmp_int32 (* kmp_routine_entry_t)(kmp_int32, void *); 442 llvm::Type *KmpRoutineEntryPtrTy = nullptr; 443 QualType KmpRoutineEntryPtrQTy; 444 /// Type typedef struct kmp_task { 445 /// void * shareds; /**< pointer to block of pointers to 446 /// shared vars */ 447 /// kmp_routine_entry_t routine; /**< pointer to routine to call for 448 /// executing task */ 449 /// kmp_int32 part_id; /**< part id for the task */ 450 /// kmp_routine_entry_t destructors; /* pointer to function to invoke 451 /// deconstructors of firstprivate C++ objects */ 452 /// } kmp_task_t; 453 QualType KmpTaskTQTy; 454 /// Saved kmp_task_t for task directive. 455 QualType SavedKmpTaskTQTy; 456 /// Saved kmp_task_t for taskloop-based directive. 457 QualType SavedKmpTaskloopTQTy; 458 /// Type typedef struct kmp_depend_info { 459 /// kmp_intptr_t base_addr; 460 /// size_t len; 461 /// struct { 462 /// bool in:1; 463 /// bool out:1; 464 /// } flags; 465 /// } kmp_depend_info_t; 466 QualType KmpDependInfoTy; 467 /// Type typedef struct kmp_task_affinity_info { 468 /// kmp_intptr_t base_addr; 469 /// size_t len; 470 /// struct { 471 /// bool flag1 : 1; 472 /// bool flag2 : 1; 473 /// kmp_int32 reserved : 30; 474 /// } flags; 475 /// } kmp_task_affinity_info_t; 476 QualType KmpTaskAffinityInfoTy; 477 /// struct kmp_dim { // loop bounds info casted to kmp_int64 478 /// kmp_int64 lo; // lower 479 /// kmp_int64 up; // upper 480 /// kmp_int64 st; // stride 481 /// }; 482 QualType KmpDimTy; 483 /// Type struct __tgt_offload_entry{ 484 /// void *addr; // Pointer to the offload entry info. 485 /// // (function or global) 486 /// char *name; // Name of the function or global. 487 /// size_t size; // Size of the entry info (0 if it a function). 488 /// int32_t flags; 489 /// int32_t reserved; 490 /// }; 491 QualType TgtOffloadEntryQTy; 492 /// Entity that registers the offloading constants that were emitted so 493 /// far. 494 class OffloadEntriesInfoManagerTy { 495 CodeGenModule &CGM; 496 497 /// Number of entries registered so far. 498 unsigned OffloadingEntriesNum = 0; 499 500 public: 501 /// Base class of the entries info. 502 class OffloadEntryInfo { 503 public: 504 /// Kind of a given entry. 505 enum OffloadingEntryInfoKinds : unsigned { 506 /// Entry is a target region. 507 OffloadingEntryInfoTargetRegion = 0, 508 /// Entry is a declare target variable. 509 OffloadingEntryInfoDeviceGlobalVar = 1, 510 /// Invalid entry info. 511 OffloadingEntryInfoInvalid = ~0u 512 }; 513 514 protected: 515 OffloadEntryInfo() = delete; 516 explicit OffloadEntryInfo(OffloadingEntryInfoKinds Kind) : Kind(Kind) {} 517 explicit OffloadEntryInfo(OffloadingEntryInfoKinds Kind, unsigned Order, 518 uint32_t Flags) 519 : Flags(Flags), Order(Order), Kind(Kind) {} 520 ~OffloadEntryInfo() = default; 521 522 public: 523 bool isValid() const { return Order != ~0u; } 524 unsigned getOrder() const { return Order; } 525 OffloadingEntryInfoKinds getKind() const { return Kind; } 526 uint32_t getFlags() const { return Flags; } 527 void setFlags(uint32_t NewFlags) { Flags = NewFlags; } 528 llvm::Constant *getAddress() const { 529 return cast_or_null<llvm::Constant>(Addr); 530 } 531 void setAddress(llvm::Constant *V) { 532 assert(!Addr.pointsToAliveValue() && "Address has been set before!"); 533 Addr = V; 534 } 535 static bool classof(const OffloadEntryInfo *Info) { return true; } 536 537 private: 538 /// Address of the entity that has to be mapped for offloading. 539 llvm::WeakTrackingVH Addr; 540 541 /// Flags associated with the device global. 542 uint32_t Flags = 0u; 543 544 /// Order this entry was emitted. 545 unsigned Order = ~0u; 546 547 OffloadingEntryInfoKinds Kind = OffloadingEntryInfoInvalid; 548 }; 549 550 /// Return true if a there are no entries defined. 551 bool empty() const; 552 /// Return number of entries defined so far. 553 unsigned size() const { return OffloadingEntriesNum; } 554 OffloadEntriesInfoManagerTy(CodeGenModule &CGM) : CGM(CGM) {} 555 556 // 557 // Target region entries related. 558 // 559 560 /// Kind of the target registry entry. 561 enum OMPTargetRegionEntryKind : uint32_t { 562 /// Mark the entry as target region. 563 OMPTargetRegionEntryTargetRegion = 0x0, 564 /// Mark the entry as a global constructor. 565 OMPTargetRegionEntryCtor = 0x02, 566 /// Mark the entry as a global destructor. 567 OMPTargetRegionEntryDtor = 0x04, 568 }; 569 570 /// Target region entries info. 571 class OffloadEntryInfoTargetRegion final : public OffloadEntryInfo { 572 /// Address that can be used as the ID of the entry. 573 llvm::Constant *ID = nullptr; 574 575 public: 576 OffloadEntryInfoTargetRegion() 577 : OffloadEntryInfo(OffloadingEntryInfoTargetRegion) {} 578 explicit OffloadEntryInfoTargetRegion(unsigned Order, 579 llvm::Constant *Addr, 580 llvm::Constant *ID, 581 OMPTargetRegionEntryKind Flags) 582 : OffloadEntryInfo(OffloadingEntryInfoTargetRegion, Order, Flags), 583 ID(ID) { 584 setAddress(Addr); 585 } 586 587 llvm::Constant *getID() const { return ID; } 588 void setID(llvm::Constant *V) { 589 assert(!ID && "ID has been set before!"); 590 ID = V; 591 } 592 static bool classof(const OffloadEntryInfo *Info) { 593 return Info->getKind() == OffloadingEntryInfoTargetRegion; 594 } 595 }; 596 597 /// Initialize target region entry. 598 void initializeTargetRegionEntryInfo(unsigned DeviceID, unsigned FileID, 599 StringRef ParentName, unsigned LineNum, 600 unsigned Order); 601 /// Register target region entry. 602 void registerTargetRegionEntryInfo(unsigned DeviceID, unsigned FileID, 603 StringRef ParentName, unsigned LineNum, 604 llvm::Constant *Addr, llvm::Constant *ID, 605 OMPTargetRegionEntryKind Flags); 606 /// Return true if a target region entry with the provided information 607 /// exists. 608 bool hasTargetRegionEntryInfo(unsigned DeviceID, unsigned FileID, 609 StringRef ParentName, unsigned LineNum) const; 610 /// brief Applies action \a Action on all registered entries. 611 typedef llvm::function_ref<void(unsigned, unsigned, StringRef, unsigned, 612 const OffloadEntryInfoTargetRegion &)> 613 OffloadTargetRegionEntryInfoActTy; 614 void actOnTargetRegionEntriesInfo( 615 const OffloadTargetRegionEntryInfoActTy &Action); 616 617 // 618 // Device global variable entries related. 619 // 620 621 /// Kind of the global variable entry.. 622 enum OMPTargetGlobalVarEntryKind : uint32_t { 623 /// Mark the entry as a to declare target. 624 OMPTargetGlobalVarEntryTo = 0x0, 625 /// Mark the entry as a to declare target link. 626 OMPTargetGlobalVarEntryLink = 0x1, 627 }; 628 629 /// Device global variable entries info. 630 class OffloadEntryInfoDeviceGlobalVar final : public OffloadEntryInfo { 631 /// Type of the global variable. 632 CharUnits VarSize; 633 llvm::GlobalValue::LinkageTypes Linkage; 634 635 public: 636 OffloadEntryInfoDeviceGlobalVar() 637 : OffloadEntryInfo(OffloadingEntryInfoDeviceGlobalVar) {} 638 explicit OffloadEntryInfoDeviceGlobalVar(unsigned Order, 639 OMPTargetGlobalVarEntryKind Flags) 640 : OffloadEntryInfo(OffloadingEntryInfoDeviceGlobalVar, Order, Flags) {} 641 explicit OffloadEntryInfoDeviceGlobalVar( 642 unsigned Order, llvm::Constant *Addr, CharUnits VarSize, 643 OMPTargetGlobalVarEntryKind Flags, 644 llvm::GlobalValue::LinkageTypes Linkage) 645 : OffloadEntryInfo(OffloadingEntryInfoDeviceGlobalVar, Order, Flags), 646 VarSize(VarSize), Linkage(Linkage) { 647 setAddress(Addr); 648 } 649 650 CharUnits getVarSize() const { return VarSize; } 651 void setVarSize(CharUnits Size) { VarSize = Size; } 652 llvm::GlobalValue::LinkageTypes getLinkage() const { return Linkage; } 653 void setLinkage(llvm::GlobalValue::LinkageTypes LT) { Linkage = LT; } 654 static bool classof(const OffloadEntryInfo *Info) { 655 return Info->getKind() == OffloadingEntryInfoDeviceGlobalVar; 656 } 657 }; 658 659 /// Initialize device global variable entry. 660 void initializeDeviceGlobalVarEntryInfo(StringRef Name, 661 OMPTargetGlobalVarEntryKind Flags, 662 unsigned Order); 663 664 /// Register device global variable entry. 665 void 666 registerDeviceGlobalVarEntryInfo(StringRef VarName, llvm::Constant *Addr, 667 CharUnits VarSize, 668 OMPTargetGlobalVarEntryKind Flags, 669 llvm::GlobalValue::LinkageTypes Linkage); 670 /// Checks if the variable with the given name has been registered already. 671 bool hasDeviceGlobalVarEntryInfo(StringRef VarName) const { 672 return OffloadEntriesDeviceGlobalVar.count(VarName) > 0; 673 } 674 /// Applies action \a Action on all registered entries. 675 typedef llvm::function_ref<void(StringRef, 676 const OffloadEntryInfoDeviceGlobalVar &)> 677 OffloadDeviceGlobalVarEntryInfoActTy; 678 void actOnDeviceGlobalVarEntriesInfo( 679 const OffloadDeviceGlobalVarEntryInfoActTy &Action); 680 681 private: 682 // Storage for target region entries kind. The storage is to be indexed by 683 // file ID, device ID, parent function name and line number. 684 typedef llvm::DenseMap<unsigned, OffloadEntryInfoTargetRegion> 685 OffloadEntriesTargetRegionPerLine; 686 typedef llvm::StringMap<OffloadEntriesTargetRegionPerLine> 687 OffloadEntriesTargetRegionPerParentName; 688 typedef llvm::DenseMap<unsigned, OffloadEntriesTargetRegionPerParentName> 689 OffloadEntriesTargetRegionPerFile; 690 typedef llvm::DenseMap<unsigned, OffloadEntriesTargetRegionPerFile> 691 OffloadEntriesTargetRegionPerDevice; 692 typedef OffloadEntriesTargetRegionPerDevice OffloadEntriesTargetRegionTy; 693 OffloadEntriesTargetRegionTy OffloadEntriesTargetRegion; 694 /// Storage for device global variable entries kind. The storage is to be 695 /// indexed by mangled name. 696 typedef llvm::StringMap<OffloadEntryInfoDeviceGlobalVar> 697 OffloadEntriesDeviceGlobalVarTy; 698 OffloadEntriesDeviceGlobalVarTy OffloadEntriesDeviceGlobalVar; 699 }; 700 OffloadEntriesInfoManagerTy OffloadEntriesInfoManager; 701 702 bool ShouldMarkAsGlobal = true; 703 /// List of the emitted declarations. 704 llvm::DenseSet<CanonicalDeclPtr<const Decl>> AlreadyEmittedTargetDecls; 705 /// List of the global variables with their addresses that should not be 706 /// emitted for the target. 707 llvm::StringMap<llvm::WeakTrackingVH> EmittedNonTargetVariables; 708 709 /// List of variables that can become declare target implicitly and, thus, 710 /// must be emitted. 711 llvm::SmallDenseSet<const VarDecl *> DeferredGlobalVariables; 712 713 using NontemporalDeclsSet = llvm::SmallDenseSet<CanonicalDeclPtr<const Decl>>; 714 /// Stack for list of declarations in current context marked as nontemporal. 715 /// The set is the union of all current stack elements. 716 llvm::SmallVector<NontemporalDeclsSet, 4> NontemporalDeclsStack; 717 718 /// Stack for list of addresses of declarations in current context marked as 719 /// lastprivate conditional. The set is the union of all current stack 720 /// elements. 721 llvm::SmallVector<LastprivateConditionalData, 4> LastprivateConditionalStack; 722 723 /// Flag for keeping track of weather a requires unified_shared_memory 724 /// directive is present. 725 bool HasRequiresUnifiedSharedMemory = false; 726 727 /// Atomic ordering from the omp requires directive. 728 llvm::AtomicOrdering RequiresAtomicOrdering = llvm::AtomicOrdering::Monotonic; 729 730 /// Flag for keeping track of weather a target region has been emitted. 731 bool HasEmittedTargetRegion = false; 732 733 /// Flag for keeping track of weather a device routine has been emitted. 734 /// Device routines are specific to the 735 bool HasEmittedDeclareTargetRegion = false; 736 737 /// Loads all the offload entries information from the host IR 738 /// metadata. 739 void loadOffloadInfoMetadata(); 740 741 /// Returns __tgt_offload_entry type. 742 QualType getTgtOffloadEntryQTy(); 743 744 /// Start scanning from statement \a S and and emit all target regions 745 /// found along the way. 746 /// \param S Starting statement. 747 /// \param ParentName Name of the function declaration that is being scanned. 748 void scanForTargetRegionsFunctions(const Stmt *S, StringRef ParentName); 749 750 /// Build type kmp_routine_entry_t (if not built yet). 751 void emitKmpRoutineEntryT(QualType KmpInt32Ty); 752 753 /// Returns pointer to kmpc_micro type. 754 llvm::Type *getKmpc_MicroPointerTy(); 755 756 /// Returns __kmpc_for_static_init_* runtime function for the specified 757 /// size \a IVSize and sign \a IVSigned. 758 llvm::FunctionCallee createForStaticInitFunction(unsigned IVSize, 759 bool IVSigned); 760 761 /// Returns __kmpc_dispatch_init_* runtime function for the specified 762 /// size \a IVSize and sign \a IVSigned. 763 llvm::FunctionCallee createDispatchInitFunction(unsigned IVSize, 764 bool IVSigned); 765 766 /// Returns __kmpc_dispatch_next_* runtime function for the specified 767 /// size \a IVSize and sign \a IVSigned. 768 llvm::FunctionCallee createDispatchNextFunction(unsigned IVSize, 769 bool IVSigned); 770 771 /// Returns __kmpc_dispatch_fini_* runtime function for the specified 772 /// size \a IVSize and sign \a IVSigned. 773 llvm::FunctionCallee createDispatchFiniFunction(unsigned IVSize, 774 bool IVSigned); 775 776 /// If the specified mangled name is not in the module, create and 777 /// return threadprivate cache object. This object is a pointer's worth of 778 /// storage that's reserved for use by the OpenMP runtime. 779 /// \param VD Threadprivate variable. 780 /// \return Cache variable for the specified threadprivate. 781 llvm::Constant *getOrCreateThreadPrivateCache(const VarDecl *VD); 782 783 /// Gets (if variable with the given name already exist) or creates 784 /// internal global variable with the specified Name. The created variable has 785 /// linkage CommonLinkage by default and is initialized by null value. 786 /// \param Ty Type of the global variable. If it is exist already the type 787 /// must be the same. 788 /// \param Name Name of the variable. 789 llvm::Constant *getOrCreateInternalVariable(llvm::Type *Ty, 790 const llvm::Twine &Name, 791 unsigned AddressSpace = 0); 792 793 /// Set of threadprivate variables with the generated initializer. 794 llvm::StringSet<> ThreadPrivateWithDefinition; 795 796 /// Set of declare target variables with the generated initializer. 797 llvm::StringSet<> DeclareTargetWithDefinition; 798 799 /// Emits initialization code for the threadprivate variables. 800 /// \param VDAddr Address of the global variable \a VD. 801 /// \param Ctor Pointer to a global init function for \a VD. 802 /// \param CopyCtor Pointer to a global copy function for \a VD. 803 /// \param Dtor Pointer to a global destructor function for \a VD. 804 /// \param Loc Location of threadprivate declaration. 805 void emitThreadPrivateVarInit(CodeGenFunction &CGF, Address VDAddr, 806 llvm::Value *Ctor, llvm::Value *CopyCtor, 807 llvm::Value *Dtor, SourceLocation Loc); 808 809 /// Emit the array initialization or deletion portion for user-defined mapper 810 /// code generation. 811 void emitUDMapperArrayInitOrDel(CodeGenFunction &MapperCGF, 812 llvm::Value *Handle, llvm::Value *BasePtr, 813 llvm::Value *Ptr, llvm::Value *Size, 814 llvm::Value *MapType, CharUnits ElementSize, 815 llvm::BasicBlock *ExitBB, bool IsInit); 816 817 struct TaskResultTy { 818 llvm::Value *NewTask = nullptr; 819 llvm::Function *TaskEntry = nullptr; 820 llvm::Value *NewTaskNewTaskTTy = nullptr; 821 LValue TDBase; 822 const RecordDecl *KmpTaskTQTyRD = nullptr; 823 llvm::Value *TaskDupFn = nullptr; 824 }; 825 /// Emit task region for the task directive. The task region is emitted in 826 /// several steps: 827 /// 1. Emit a call to kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32 828 /// gtid, kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds, 829 /// kmp_routine_entry_t *task_entry). Here task_entry is a pointer to the 830 /// function: 831 /// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) { 832 /// TaskFunction(gtid, tt->part_id, tt->shareds); 833 /// return 0; 834 /// } 835 /// 2. Copy a list of shared variables to field shareds of the resulting 836 /// structure kmp_task_t returned by the previous call (if any). 837 /// 3. Copy a pointer to destructions function to field destructions of the 838 /// resulting structure kmp_task_t. 839 /// \param D Current task directive. 840 /// \param TaskFunction An LLVM function with type void (*)(i32 /*gtid*/, i32 841 /// /*part_id*/, captured_struct */*__context*/); 842 /// \param SharedsTy A type which contains references the shared variables. 843 /// \param Shareds Context with the list of shared variables from the \p 844 /// TaskFunction. 845 /// \param Data Additional data for task generation like tiednsee, final 846 /// state, list of privates etc. 847 TaskResultTy emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc, 848 const OMPExecutableDirective &D, 849 llvm::Function *TaskFunction, QualType SharedsTy, 850 Address Shareds, const OMPTaskDataTy &Data); 851 852 /// Returns default address space for the constant firstprivates, 0 by 853 /// default. 854 virtual unsigned getDefaultFirstprivateAddressSpace() const { return 0; } 855 856 /// Emit code that pushes the trip count of loops associated with constructs 857 /// 'target teams distribute' and 'teams distribute parallel for'. 858 /// \param SizeEmitter Emits the int64 value for the number of iterations of 859 /// the associated loop. 860 void emitTargetNumIterationsCall( 861 CodeGenFunction &CGF, const OMPExecutableDirective &D, 862 llvm::Value *DeviceID, 863 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF, 864 const OMPLoopDirective &D)> 865 SizeEmitter); 866 867 /// Emit update for lastprivate conditional data. 868 void emitLastprivateConditionalUpdate(CodeGenFunction &CGF, LValue IVLVal, 869 StringRef UniqueDeclName, LValue LVal, 870 SourceLocation Loc); 871 872 /// Returns the number of the elements and the address of the depobj 873 /// dependency array. 874 /// \return Number of elements in depobj array and the pointer to the array of 875 /// dependencies. 876 std::pair<llvm::Value *, LValue> getDepobjElements(CodeGenFunction &CGF, 877 LValue DepobjLVal, 878 SourceLocation Loc); 879 880 public: 881 explicit CGOpenMPRuntime(CodeGenModule &CGM) 882 : CGOpenMPRuntime(CGM, ".", ".") {} 883 virtual ~CGOpenMPRuntime() {} 884 virtual void clear(); 885 886 /// Emits code for OpenMP 'if' clause using specified \a CodeGen 887 /// function. Here is the logic: 888 /// if (Cond) { 889 /// ThenGen(); 890 /// } else { 891 /// ElseGen(); 892 /// } 893 void emitIfClause(CodeGenFunction &CGF, const Expr *Cond, 894 const RegionCodeGenTy &ThenGen, 895 const RegionCodeGenTy &ElseGen); 896 897 /// Checks if the \p Body is the \a CompoundStmt and returns its child 898 /// statement iff there is only one that is not evaluatable at the compile 899 /// time. 900 static const Stmt *getSingleCompoundChild(ASTContext &Ctx, const Stmt *Body); 901 902 /// Get the platform-specific name separator. 903 std::string getName(ArrayRef<StringRef> Parts) const; 904 905 /// Emit code for the specified user defined reduction construct. 906 virtual void emitUserDefinedReduction(CodeGenFunction *CGF, 907 const OMPDeclareReductionDecl *D); 908 /// Get combiner/initializer for the specified user-defined reduction, if any. 909 virtual std::pair<llvm::Function *, llvm::Function *> 910 getUserDefinedReduction(const OMPDeclareReductionDecl *D); 911 912 /// Emit the function for the user defined mapper construct. 913 void emitUserDefinedMapper(const OMPDeclareMapperDecl *D, 914 CodeGenFunction *CGF = nullptr); 915 916 /// Emits outlined function for the specified OpenMP parallel directive 917 /// \a D. This outlined function has type void(*)(kmp_int32 *ThreadID, 918 /// kmp_int32 BoundID, struct context_vars*). 919 /// \param D OpenMP directive. 920 /// \param ThreadIDVar Variable for thread id in the current OpenMP region. 921 /// \param InnermostKind Kind of innermost directive (for simple directives it 922 /// is a directive itself, for combined - its innermost directive). 923 /// \param CodeGen Code generation sequence for the \a D directive. 924 virtual llvm::Function *emitParallelOutlinedFunction( 925 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, 926 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen); 927 928 /// Emits outlined function for the specified OpenMP teams directive 929 /// \a D. This outlined function has type void(*)(kmp_int32 *ThreadID, 930 /// kmp_int32 BoundID, struct context_vars*). 931 /// \param D OpenMP directive. 932 /// \param ThreadIDVar Variable for thread id in the current OpenMP region. 933 /// \param InnermostKind Kind of innermost directive (for simple directives it 934 /// is a directive itself, for combined - its innermost directive). 935 /// \param CodeGen Code generation sequence for the \a D directive. 936 virtual llvm::Function *emitTeamsOutlinedFunction( 937 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, 938 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen); 939 940 /// Emits outlined function for the OpenMP task directive \a D. This 941 /// outlined function has type void(*)(kmp_int32 ThreadID, struct task_t* 942 /// TaskT). 943 /// \param D OpenMP directive. 944 /// \param ThreadIDVar Variable for thread id in the current OpenMP region. 945 /// \param PartIDVar Variable for partition id in the current OpenMP untied 946 /// task region. 947 /// \param TaskTVar Variable for task_t argument. 948 /// \param InnermostKind Kind of innermost directive (for simple directives it 949 /// is a directive itself, for combined - its innermost directive). 950 /// \param CodeGen Code generation sequence for the \a D directive. 951 /// \param Tied true if task is generated for tied task, false otherwise. 952 /// \param NumberOfParts Number of parts in untied task. Ignored for tied 953 /// tasks. 954 /// 955 virtual llvm::Function *emitTaskOutlinedFunction( 956 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, 957 const VarDecl *PartIDVar, const VarDecl *TaskTVar, 958 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, 959 bool Tied, unsigned &NumberOfParts); 960 961 /// Cleans up references to the objects in finished function. 962 /// 963 virtual void functionFinished(CodeGenFunction &CGF); 964 965 /// Emits code for parallel or serial call of the \a OutlinedFn with 966 /// variables captured in a record which address is stored in \a 967 /// CapturedStruct. 968 /// \param OutlinedFn Outlined function to be run in parallel threads. Type of 969 /// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*). 970 /// \param CapturedVars A pointer to the record with the references to 971 /// variables used in \a OutlinedFn function. 972 /// \param IfCond Condition in the associated 'if' clause, if it was 973 /// specified, nullptr otherwise. 974 /// 975 virtual void emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc, 976 llvm::Function *OutlinedFn, 977 ArrayRef<llvm::Value *> CapturedVars, 978 const Expr *IfCond); 979 980 /// Emits a critical region. 981 /// \param CriticalName Name of the critical region. 982 /// \param CriticalOpGen Generator for the statement associated with the given 983 /// critical region. 984 /// \param Hint Value of the 'hint' clause (optional). 985 virtual void emitCriticalRegion(CodeGenFunction &CGF, StringRef CriticalName, 986 const RegionCodeGenTy &CriticalOpGen, 987 SourceLocation Loc, 988 const Expr *Hint = nullptr); 989 990 /// Emits a master region. 991 /// \param MasterOpGen Generator for the statement associated with the given 992 /// master region. 993 virtual void emitMasterRegion(CodeGenFunction &CGF, 994 const RegionCodeGenTy &MasterOpGen, 995 SourceLocation Loc); 996 997 /// Emits code for a taskyield directive. 998 virtual void emitTaskyieldCall(CodeGenFunction &CGF, SourceLocation Loc); 999 1000 /// Emit a taskgroup region. 1001 /// \param TaskgroupOpGen Generator for the statement associated with the 1002 /// given taskgroup region. 1003 virtual void emitTaskgroupRegion(CodeGenFunction &CGF, 1004 const RegionCodeGenTy &TaskgroupOpGen, 1005 SourceLocation Loc); 1006 1007 /// Emits a single region. 1008 /// \param SingleOpGen Generator for the statement associated with the given 1009 /// single region. 1010 virtual void emitSingleRegion(CodeGenFunction &CGF, 1011 const RegionCodeGenTy &SingleOpGen, 1012 SourceLocation Loc, 1013 ArrayRef<const Expr *> CopyprivateVars, 1014 ArrayRef<const Expr *> DestExprs, 1015 ArrayRef<const Expr *> SrcExprs, 1016 ArrayRef<const Expr *> AssignmentOps); 1017 1018 /// Emit an ordered region. 1019 /// \param OrderedOpGen Generator for the statement associated with the given 1020 /// ordered region. 1021 virtual void emitOrderedRegion(CodeGenFunction &CGF, 1022 const RegionCodeGenTy &OrderedOpGen, 1023 SourceLocation Loc, bool IsThreads); 1024 1025 /// Emit an implicit/explicit barrier for OpenMP threads. 1026 /// \param Kind Directive for which this implicit barrier call must be 1027 /// generated. Must be OMPD_barrier for explicit barrier generation. 1028 /// \param EmitChecks true if need to emit checks for cancellation barriers. 1029 /// \param ForceSimpleCall true simple barrier call must be emitted, false if 1030 /// runtime class decides which one to emit (simple or with cancellation 1031 /// checks). 1032 /// 1033 virtual void emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc, 1034 OpenMPDirectiveKind Kind, 1035 bool EmitChecks = true, 1036 bool ForceSimpleCall = false); 1037 1038 /// Check if the specified \a ScheduleKind is static non-chunked. 1039 /// This kind of worksharing directive is emitted without outer loop. 1040 /// \param ScheduleKind Schedule kind specified in the 'schedule' clause. 1041 /// \param Chunked True if chunk is specified in the clause. 1042 /// 1043 virtual bool isStaticNonchunked(OpenMPScheduleClauseKind ScheduleKind, 1044 bool Chunked) const; 1045 1046 /// Check if the specified \a ScheduleKind is static non-chunked. 1047 /// This kind of distribute directive is emitted without outer loop. 1048 /// \param ScheduleKind Schedule kind specified in the 'dist_schedule' clause. 1049 /// \param Chunked True if chunk is specified in the clause. 1050 /// 1051 virtual bool isStaticNonchunked(OpenMPDistScheduleClauseKind ScheduleKind, 1052 bool Chunked) const; 1053 1054 /// Check if the specified \a ScheduleKind is static chunked. 1055 /// \param ScheduleKind Schedule kind specified in the 'schedule' clause. 1056 /// \param Chunked True if chunk is specified in the clause. 1057 /// 1058 virtual bool isStaticChunked(OpenMPScheduleClauseKind ScheduleKind, 1059 bool Chunked) const; 1060 1061 /// Check if the specified \a ScheduleKind is static non-chunked. 1062 /// \param ScheduleKind Schedule kind specified in the 'dist_schedule' clause. 1063 /// \param Chunked True if chunk is specified in the clause. 1064 /// 1065 virtual bool isStaticChunked(OpenMPDistScheduleClauseKind ScheduleKind, 1066 bool Chunked) const; 1067 1068 /// Check if the specified \a ScheduleKind is dynamic. 1069 /// This kind of worksharing directive is emitted without outer loop. 1070 /// \param ScheduleKind Schedule Kind specified in the 'schedule' clause. 1071 /// 1072 virtual bool isDynamic(OpenMPScheduleClauseKind ScheduleKind) const; 1073 1074 /// struct with the values to be passed to the dispatch runtime function 1075 struct DispatchRTInput { 1076 /// Loop lower bound 1077 llvm::Value *LB = nullptr; 1078 /// Loop upper bound 1079 llvm::Value *UB = nullptr; 1080 /// Chunk size specified using 'schedule' clause (nullptr if chunk 1081 /// was not specified) 1082 llvm::Value *Chunk = nullptr; 1083 DispatchRTInput() = default; 1084 DispatchRTInput(llvm::Value *LB, llvm::Value *UB, llvm::Value *Chunk) 1085 : LB(LB), UB(UB), Chunk(Chunk) {} 1086 }; 1087 1088 /// Call the appropriate runtime routine to initialize it before start 1089 /// of loop. 1090 1091 /// This is used for non static scheduled types and when the ordered 1092 /// clause is present on the loop construct. 1093 /// Depending on the loop schedule, it is necessary to call some runtime 1094 /// routine before start of the OpenMP loop to get the loop upper / lower 1095 /// bounds \a LB and \a UB and stride \a ST. 1096 /// 1097 /// \param CGF Reference to current CodeGenFunction. 1098 /// \param Loc Clang source location. 1099 /// \param ScheduleKind Schedule kind, specified by the 'schedule' clause. 1100 /// \param IVSize Size of the iteration variable in bits. 1101 /// \param IVSigned Sign of the iteration variable. 1102 /// \param Ordered true if loop is ordered, false otherwise. 1103 /// \param DispatchValues struct containing llvm values for lower bound, upper 1104 /// bound, and chunk expression. 1105 /// For the default (nullptr) value, the chunk 1 will be used. 1106 /// 1107 virtual void emitForDispatchInit(CodeGenFunction &CGF, SourceLocation Loc, 1108 const OpenMPScheduleTy &ScheduleKind, 1109 unsigned IVSize, bool IVSigned, bool Ordered, 1110 const DispatchRTInput &DispatchValues); 1111 1112 /// Struct with the values to be passed to the static runtime function 1113 struct StaticRTInput { 1114 /// Size of the iteration variable in bits. 1115 unsigned IVSize = 0; 1116 /// Sign of the iteration variable. 1117 bool IVSigned = false; 1118 /// true if loop is ordered, false otherwise. 1119 bool Ordered = false; 1120 /// Address of the output variable in which the flag of the last iteration 1121 /// is returned. 1122 Address IL = Address::invalid(); 1123 /// Address of the output variable in which the lower iteration number is 1124 /// returned. 1125 Address LB = Address::invalid(); 1126 /// Address of the output variable in which the upper iteration number is 1127 /// returned. 1128 Address UB = Address::invalid(); 1129 /// Address of the output variable in which the stride value is returned 1130 /// necessary to generated the static_chunked scheduled loop. 1131 Address ST = Address::invalid(); 1132 /// Value of the chunk for the static_chunked scheduled loop. For the 1133 /// default (nullptr) value, the chunk 1 will be used. 1134 llvm::Value *Chunk = nullptr; 1135 StaticRTInput(unsigned IVSize, bool IVSigned, bool Ordered, Address IL, 1136 Address LB, Address UB, Address ST, 1137 llvm::Value *Chunk = nullptr) 1138 : IVSize(IVSize), IVSigned(IVSigned), Ordered(Ordered), IL(IL), LB(LB), 1139 UB(UB), ST(ST), Chunk(Chunk) {} 1140 }; 1141 /// Call the appropriate runtime routine to initialize it before start 1142 /// of loop. 1143 /// 1144 /// This is used only in case of static schedule, when the user did not 1145 /// specify a ordered clause on the loop construct. 1146 /// Depending on the loop schedule, it is necessary to call some runtime 1147 /// routine before start of the OpenMP loop to get the loop upper / lower 1148 /// bounds LB and UB and stride ST. 1149 /// 1150 /// \param CGF Reference to current CodeGenFunction. 1151 /// \param Loc Clang source location. 1152 /// \param DKind Kind of the directive. 1153 /// \param ScheduleKind Schedule kind, specified by the 'schedule' clause. 1154 /// \param Values Input arguments for the construct. 1155 /// 1156 virtual void emitForStaticInit(CodeGenFunction &CGF, SourceLocation Loc, 1157 OpenMPDirectiveKind DKind, 1158 const OpenMPScheduleTy &ScheduleKind, 1159 const StaticRTInput &Values); 1160 1161 /// 1162 /// \param CGF Reference to current CodeGenFunction. 1163 /// \param Loc Clang source location. 1164 /// \param SchedKind Schedule kind, specified by the 'dist_schedule' clause. 1165 /// \param Values Input arguments for the construct. 1166 /// 1167 virtual void emitDistributeStaticInit(CodeGenFunction &CGF, 1168 SourceLocation Loc, 1169 OpenMPDistScheduleClauseKind SchedKind, 1170 const StaticRTInput &Values); 1171 1172 /// Call the appropriate runtime routine to notify that we finished 1173 /// iteration of the ordered loop with the dynamic scheduling. 1174 /// 1175 /// \param CGF Reference to current CodeGenFunction. 1176 /// \param Loc Clang source location. 1177 /// \param IVSize Size of the iteration variable in bits. 1178 /// \param IVSigned Sign of the iteration variable. 1179 /// 1180 virtual void emitForOrderedIterationEnd(CodeGenFunction &CGF, 1181 SourceLocation Loc, unsigned IVSize, 1182 bool IVSigned); 1183 1184 /// Call the appropriate runtime routine to notify that we finished 1185 /// all the work with current loop. 1186 /// 1187 /// \param CGF Reference to current CodeGenFunction. 1188 /// \param Loc Clang source location. 1189 /// \param DKind Kind of the directive for which the static finish is emitted. 1190 /// 1191 virtual void emitForStaticFinish(CodeGenFunction &CGF, SourceLocation Loc, 1192 OpenMPDirectiveKind DKind); 1193 1194 /// Call __kmpc_dispatch_next( 1195 /// ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter, 1196 /// kmp_int[32|64] *p_lower, kmp_int[32|64] *p_upper, 1197 /// kmp_int[32|64] *p_stride); 1198 /// \param IVSize Size of the iteration variable in bits. 1199 /// \param IVSigned Sign of the iteration variable. 1200 /// \param IL Address of the output variable in which the flag of the 1201 /// last iteration is returned. 1202 /// \param LB Address of the output variable in which the lower iteration 1203 /// number is returned. 1204 /// \param UB Address of the output variable in which the upper iteration 1205 /// number is returned. 1206 /// \param ST Address of the output variable in which the stride value is 1207 /// returned. 1208 virtual llvm::Value *emitForNext(CodeGenFunction &CGF, SourceLocation Loc, 1209 unsigned IVSize, bool IVSigned, 1210 Address IL, Address LB, 1211 Address UB, Address ST); 1212 1213 /// Emits call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32 1214 /// global_tid, kmp_int32 num_threads) to generate code for 'num_threads' 1215 /// clause. 1216 /// \param NumThreads An integer value of threads. 1217 virtual void emitNumThreadsClause(CodeGenFunction &CGF, 1218 llvm::Value *NumThreads, 1219 SourceLocation Loc); 1220 1221 /// Emit call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32 1222 /// global_tid, int proc_bind) to generate code for 'proc_bind' clause. 1223 virtual void emitProcBindClause(CodeGenFunction &CGF, 1224 llvm::omp::ProcBindKind ProcBind, 1225 SourceLocation Loc); 1226 1227 /// Returns address of the threadprivate variable for the current 1228 /// thread. 1229 /// \param VD Threadprivate variable. 1230 /// \param VDAddr Address of the global variable \a VD. 1231 /// \param Loc Location of the reference to threadprivate var. 1232 /// \return Address of the threadprivate variable for the current thread. 1233 virtual Address getAddrOfThreadPrivate(CodeGenFunction &CGF, 1234 const VarDecl *VD, 1235 Address VDAddr, 1236 SourceLocation Loc); 1237 1238 /// Returns the address of the variable marked as declare target with link 1239 /// clause OR as declare target with to clause and unified memory. 1240 virtual Address getAddrOfDeclareTargetVar(const VarDecl *VD); 1241 1242 /// Emit a code for initialization of threadprivate variable. It emits 1243 /// a call to runtime library which adds initial value to the newly created 1244 /// threadprivate variable (if it is not constant) and registers destructor 1245 /// for the variable (if any). 1246 /// \param VD Threadprivate variable. 1247 /// \param VDAddr Address of the global variable \a VD. 1248 /// \param Loc Location of threadprivate declaration. 1249 /// \param PerformInit true if initialization expression is not constant. 1250 virtual llvm::Function * 1251 emitThreadPrivateVarDefinition(const VarDecl *VD, Address VDAddr, 1252 SourceLocation Loc, bool PerformInit, 1253 CodeGenFunction *CGF = nullptr); 1254 1255 /// Emit a code for initialization of declare target variable. 1256 /// \param VD Declare target variable. 1257 /// \param Addr Address of the global variable \a VD. 1258 /// \param PerformInit true if initialization expression is not constant. 1259 virtual bool emitDeclareTargetVarDefinition(const VarDecl *VD, 1260 llvm::GlobalVariable *Addr, 1261 bool PerformInit); 1262 1263 /// Creates artificial threadprivate variable with name \p Name and type \p 1264 /// VarType. 1265 /// \param VarType Type of the artificial threadprivate variable. 1266 /// \param Name Name of the artificial threadprivate variable. 1267 virtual Address getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF, 1268 QualType VarType, 1269 StringRef Name); 1270 1271 /// Emit flush of the variables specified in 'omp flush' directive. 1272 /// \param Vars List of variables to flush. 1273 virtual void emitFlush(CodeGenFunction &CGF, ArrayRef<const Expr *> Vars, 1274 SourceLocation Loc, llvm::AtomicOrdering AO); 1275 1276 /// Emit task region for the task directive. The task region is 1277 /// emitted in several steps: 1278 /// 1. Emit a call to kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32 1279 /// gtid, kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds, 1280 /// kmp_routine_entry_t *task_entry). Here task_entry is a pointer to the 1281 /// function: 1282 /// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) { 1283 /// TaskFunction(gtid, tt->part_id, tt->shareds); 1284 /// return 0; 1285 /// } 1286 /// 2. Copy a list of shared variables to field shareds of the resulting 1287 /// structure kmp_task_t returned by the previous call (if any). 1288 /// 3. Copy a pointer to destructions function to field destructions of the 1289 /// resulting structure kmp_task_t. 1290 /// 4. Emit a call to kmp_int32 __kmpc_omp_task(ident_t *, kmp_int32 gtid, 1291 /// kmp_task_t *new_task), where new_task is a resulting structure from 1292 /// previous items. 1293 /// \param D Current task directive. 1294 /// \param TaskFunction An LLVM function with type void (*)(i32 /*gtid*/, i32 1295 /// /*part_id*/, captured_struct */*__context*/); 1296 /// \param SharedsTy A type which contains references the shared variables. 1297 /// \param Shareds Context with the list of shared variables from the \p 1298 /// TaskFunction. 1299 /// \param IfCond Not a nullptr if 'if' clause was specified, nullptr 1300 /// otherwise. 1301 /// \param Data Additional data for task generation like tiednsee, final 1302 /// state, list of privates etc. 1303 virtual void emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc, 1304 const OMPExecutableDirective &D, 1305 llvm::Function *TaskFunction, QualType SharedsTy, 1306 Address Shareds, const Expr *IfCond, 1307 const OMPTaskDataTy &Data); 1308 1309 /// Emit task region for the taskloop directive. The taskloop region is 1310 /// emitted in several steps: 1311 /// 1. Emit a call to kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32 1312 /// gtid, kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds, 1313 /// kmp_routine_entry_t *task_entry). Here task_entry is a pointer to the 1314 /// function: 1315 /// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) { 1316 /// TaskFunction(gtid, tt->part_id, tt->shareds); 1317 /// return 0; 1318 /// } 1319 /// 2. Copy a list of shared variables to field shareds of the resulting 1320 /// structure kmp_task_t returned by the previous call (if any). 1321 /// 3. Copy a pointer to destructions function to field destructions of the 1322 /// resulting structure kmp_task_t. 1323 /// 4. Emit a call to void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t 1324 /// *task, int if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int 1325 /// nogroup, int sched, kmp_uint64 grainsize, void *task_dup ), where new_task 1326 /// is a resulting structure from 1327 /// previous items. 1328 /// \param D Current task directive. 1329 /// \param TaskFunction An LLVM function with type void (*)(i32 /*gtid*/, i32 1330 /// /*part_id*/, captured_struct */*__context*/); 1331 /// \param SharedsTy A type which contains references the shared variables. 1332 /// \param Shareds Context with the list of shared variables from the \p 1333 /// TaskFunction. 1334 /// \param IfCond Not a nullptr if 'if' clause was specified, nullptr 1335 /// otherwise. 1336 /// \param Data Additional data for task generation like tiednsee, final 1337 /// state, list of privates etc. 1338 virtual void emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc, 1339 const OMPLoopDirective &D, 1340 llvm::Function *TaskFunction, 1341 QualType SharedsTy, Address Shareds, 1342 const Expr *IfCond, const OMPTaskDataTy &Data); 1343 1344 /// Emit code for the directive that does not require outlining. 1345 /// 1346 /// \param InnermostKind Kind of innermost directive (for simple directives it 1347 /// is a directive itself, for combined - its innermost directive). 1348 /// \param CodeGen Code generation sequence for the \a D directive. 1349 /// \param HasCancel true if region has inner cancel directive, false 1350 /// otherwise. 1351 virtual void emitInlinedDirective(CodeGenFunction &CGF, 1352 OpenMPDirectiveKind InnermostKind, 1353 const RegionCodeGenTy &CodeGen, 1354 bool HasCancel = false); 1355 1356 /// Emits reduction function. 1357 /// \param ArgsType Array type containing pointers to reduction variables. 1358 /// \param Privates List of private copies for original reduction arguments. 1359 /// \param LHSExprs List of LHS in \a ReductionOps reduction operations. 1360 /// \param RHSExprs List of RHS in \a ReductionOps reduction operations. 1361 /// \param ReductionOps List of reduction operations in form 'LHS binop RHS' 1362 /// or 'operator binop(LHS, RHS)'. 1363 llvm::Function *emitReductionFunction(SourceLocation Loc, 1364 llvm::Type *ArgsType, 1365 ArrayRef<const Expr *> Privates, 1366 ArrayRef<const Expr *> LHSExprs, 1367 ArrayRef<const Expr *> RHSExprs, 1368 ArrayRef<const Expr *> ReductionOps); 1369 1370 /// Emits single reduction combiner 1371 void emitSingleReductionCombiner(CodeGenFunction &CGF, 1372 const Expr *ReductionOp, 1373 const Expr *PrivateRef, 1374 const DeclRefExpr *LHS, 1375 const DeclRefExpr *RHS); 1376 1377 struct ReductionOptionsTy { 1378 bool WithNowait; 1379 bool SimpleReduction; 1380 OpenMPDirectiveKind ReductionKind; 1381 }; 1382 /// Emit a code for reduction clause. Next code should be emitted for 1383 /// reduction: 1384 /// \code 1385 /// 1386 /// static kmp_critical_name lock = { 0 }; 1387 /// 1388 /// void reduce_func(void *lhs[<n>], void *rhs[<n>]) { 1389 /// ... 1390 /// *(Type<i>*)lhs[i] = RedOp<i>(*(Type<i>*)lhs[i], *(Type<i>*)rhs[i]); 1391 /// ... 1392 /// } 1393 /// 1394 /// ... 1395 /// void *RedList[<n>] = {&<RHSExprs>[0], ..., &<RHSExprs>[<n>-1]}; 1396 /// switch (__kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList), 1397 /// RedList, reduce_func, &<lock>)) { 1398 /// case 1: 1399 /// ... 1400 /// <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]); 1401 /// ... 1402 /// __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>); 1403 /// break; 1404 /// case 2: 1405 /// ... 1406 /// Atomic(<LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i])); 1407 /// ... 1408 /// break; 1409 /// default:; 1410 /// } 1411 /// \endcode 1412 /// 1413 /// \param Privates List of private copies for original reduction arguments. 1414 /// \param LHSExprs List of LHS in \a ReductionOps reduction operations. 1415 /// \param RHSExprs List of RHS in \a ReductionOps reduction operations. 1416 /// \param ReductionOps List of reduction operations in form 'LHS binop RHS' 1417 /// or 'operator binop(LHS, RHS)'. 1418 /// \param Options List of options for reduction codegen: 1419 /// WithNowait true if parent directive has also nowait clause, false 1420 /// otherwise. 1421 /// SimpleReduction Emit reduction operation only. Used for omp simd 1422 /// directive on the host. 1423 /// ReductionKind The kind of reduction to perform. 1424 virtual void emitReduction(CodeGenFunction &CGF, SourceLocation Loc, 1425 ArrayRef<const Expr *> Privates, 1426 ArrayRef<const Expr *> LHSExprs, 1427 ArrayRef<const Expr *> RHSExprs, 1428 ArrayRef<const Expr *> ReductionOps, 1429 ReductionOptionsTy Options); 1430 1431 /// Emit a code for initialization of task reduction clause. Next code 1432 /// should be emitted for reduction: 1433 /// \code 1434 /// 1435 /// _taskred_item_t red_data[n]; 1436 /// ... 1437 /// red_data[i].shar = &shareds[i]; 1438 /// red_data[i].orig = &origs[i]; 1439 /// red_data[i].size = sizeof(origs[i]); 1440 /// red_data[i].f_init = (void*)RedInit<i>; 1441 /// red_data[i].f_fini = (void*)RedDest<i>; 1442 /// red_data[i].f_comb = (void*)RedOp<i>; 1443 /// red_data[i].flags = <Flag_i>; 1444 /// ... 1445 /// void* tg1 = __kmpc_taskred_init(gtid, n, red_data); 1446 /// \endcode 1447 /// For reduction clause with task modifier it emits the next call: 1448 /// \code 1449 /// 1450 /// _taskred_item_t red_data[n]; 1451 /// ... 1452 /// red_data[i].shar = &shareds[i]; 1453 /// red_data[i].orig = &origs[i]; 1454 /// red_data[i].size = sizeof(origs[i]); 1455 /// red_data[i].f_init = (void*)RedInit<i>; 1456 /// red_data[i].f_fini = (void*)RedDest<i>; 1457 /// red_data[i].f_comb = (void*)RedOp<i>; 1458 /// red_data[i].flags = <Flag_i>; 1459 /// ... 1460 /// void* tg1 = __kmpc_taskred_modifier_init(loc, gtid, is_worksharing, n, 1461 /// red_data); 1462 /// \endcode 1463 /// \param LHSExprs List of LHS in \a Data.ReductionOps reduction operations. 1464 /// \param RHSExprs List of RHS in \a Data.ReductionOps reduction operations. 1465 /// \param Data Additional data for task generation like tiedness, final 1466 /// state, list of privates, reductions etc. 1467 virtual llvm::Value *emitTaskReductionInit(CodeGenFunction &CGF, 1468 SourceLocation Loc, 1469 ArrayRef<const Expr *> LHSExprs, 1470 ArrayRef<const Expr *> RHSExprs, 1471 const OMPTaskDataTy &Data); 1472 1473 /// Emits the following code for reduction clause with task modifier: 1474 /// \code 1475 /// __kmpc_task_reduction_modifier_fini(loc, gtid, is_worksharing); 1476 /// \endcode 1477 virtual void emitTaskReductionFini(CodeGenFunction &CGF, SourceLocation Loc, 1478 bool IsWorksharingReduction); 1479 1480 /// Required to resolve existing problems in the runtime. Emits threadprivate 1481 /// variables to store the size of the VLAs/array sections for 1482 /// initializer/combiner/finalizer functions. 1483 /// \param RCG Allows to reuse an existing data for the reductions. 1484 /// \param N Reduction item for which fixups must be emitted. 1485 virtual void emitTaskReductionFixups(CodeGenFunction &CGF, SourceLocation Loc, 1486 ReductionCodeGen &RCG, unsigned N); 1487 1488 /// Get the address of `void *` type of the privatue copy of the reduction 1489 /// item specified by the \p SharedLVal. 1490 /// \param ReductionsPtr Pointer to the reduction data returned by the 1491 /// emitTaskReductionInit function. 1492 /// \param SharedLVal Address of the original reduction item. 1493 virtual Address getTaskReductionItem(CodeGenFunction &CGF, SourceLocation Loc, 1494 llvm::Value *ReductionsPtr, 1495 LValue SharedLVal); 1496 1497 /// Emit code for 'taskwait' directive. 1498 virtual void emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc); 1499 1500 /// Emit code for 'cancellation point' construct. 1501 /// \param CancelRegion Region kind for which the cancellation point must be 1502 /// emitted. 1503 /// 1504 virtual void emitCancellationPointCall(CodeGenFunction &CGF, 1505 SourceLocation Loc, 1506 OpenMPDirectiveKind CancelRegion); 1507 1508 /// Emit code for 'cancel' construct. 1509 /// \param IfCond Condition in the associated 'if' clause, if it was 1510 /// specified, nullptr otherwise. 1511 /// \param CancelRegion Region kind for which the cancel must be emitted. 1512 /// 1513 virtual void emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc, 1514 const Expr *IfCond, 1515 OpenMPDirectiveKind CancelRegion); 1516 1517 /// Emit outilined function for 'target' directive. 1518 /// \param D Directive to emit. 1519 /// \param ParentName Name of the function that encloses the target region. 1520 /// \param OutlinedFn Outlined function value to be defined by this call. 1521 /// \param OutlinedFnID Outlined function ID value to be defined by this call. 1522 /// \param IsOffloadEntry True if the outlined function is an offload entry. 1523 /// \param CodeGen Code generation sequence for the \a D directive. 1524 /// An outlined function may not be an entry if, e.g. the if clause always 1525 /// evaluates to false. 1526 virtual void emitTargetOutlinedFunction(const OMPExecutableDirective &D, 1527 StringRef ParentName, 1528 llvm::Function *&OutlinedFn, 1529 llvm::Constant *&OutlinedFnID, 1530 bool IsOffloadEntry, 1531 const RegionCodeGenTy &CodeGen); 1532 1533 /// Emit the target offloading code associated with \a D. The emitted 1534 /// code attempts offloading the execution to the device, an the event of 1535 /// a failure it executes the host version outlined in \a OutlinedFn. 1536 /// \param D Directive to emit. 1537 /// \param OutlinedFn Host version of the code to be offloaded. 1538 /// \param OutlinedFnID ID of host version of the code to be offloaded. 1539 /// \param IfCond Expression evaluated in if clause associated with the target 1540 /// directive, or null if no if clause is used. 1541 /// \param Device Expression evaluated in device clause associated with the 1542 /// target directive, or null if no device clause is used and device modifier. 1543 /// \param SizeEmitter Callback to emit number of iterations for loop-based 1544 /// directives. 1545 virtual void emitTargetCall( 1546 CodeGenFunction &CGF, const OMPExecutableDirective &D, 1547 llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond, 1548 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier> Device, 1549 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF, 1550 const OMPLoopDirective &D)> 1551 SizeEmitter); 1552 1553 /// Emit the target regions enclosed in \a GD function definition or 1554 /// the function itself in case it is a valid device function. Returns true if 1555 /// \a GD was dealt with successfully. 1556 /// \param GD Function to scan. 1557 virtual bool emitTargetFunctions(GlobalDecl GD); 1558 1559 /// Emit the global variable if it is a valid device global variable. 1560 /// Returns true if \a GD was dealt with successfully. 1561 /// \param GD Variable declaration to emit. 1562 virtual bool emitTargetGlobalVariable(GlobalDecl GD); 1563 1564 /// Checks if the provided global decl \a GD is a declare target variable and 1565 /// registers it when emitting code for the host. 1566 virtual void registerTargetGlobalVariable(const VarDecl *VD, 1567 llvm::Constant *Addr); 1568 1569 /// Registers provided target firstprivate variable as global on the 1570 /// target. 1571 llvm::Constant *registerTargetFirstprivateCopy(CodeGenFunction &CGF, 1572 const VarDecl *VD); 1573 1574 /// Emit the global \a GD if it is meaningful for the target. Returns 1575 /// if it was emitted successfully. 1576 /// \param GD Global to scan. 1577 virtual bool emitTargetGlobal(GlobalDecl GD); 1578 1579 /// Creates and returns a registration function for when at least one 1580 /// requires directives was used in the current module. 1581 llvm::Function *emitRequiresDirectiveRegFun(); 1582 1583 /// Creates all the offload entries in the current compilation unit 1584 /// along with the associated metadata. 1585 void createOffloadEntriesAndInfoMetadata(); 1586 1587 /// Emits code for teams call of the \a OutlinedFn with 1588 /// variables captured in a record which address is stored in \a 1589 /// CapturedStruct. 1590 /// \param OutlinedFn Outlined function to be run by team masters. Type of 1591 /// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*). 1592 /// \param CapturedVars A pointer to the record with the references to 1593 /// variables used in \a OutlinedFn function. 1594 /// 1595 virtual void emitTeamsCall(CodeGenFunction &CGF, 1596 const OMPExecutableDirective &D, 1597 SourceLocation Loc, llvm::Function *OutlinedFn, 1598 ArrayRef<llvm::Value *> CapturedVars); 1599 1600 /// Emits call to void __kmpc_push_num_teams(ident_t *loc, kmp_int32 1601 /// global_tid, kmp_int32 num_teams, kmp_int32 thread_limit) to generate code 1602 /// for num_teams clause. 1603 /// \param NumTeams An integer expression of teams. 1604 /// \param ThreadLimit An integer expression of threads. 1605 virtual void emitNumTeamsClause(CodeGenFunction &CGF, const Expr *NumTeams, 1606 const Expr *ThreadLimit, SourceLocation Loc); 1607 1608 /// Struct that keeps all the relevant information that should be kept 1609 /// throughout a 'target data' region. 1610 class TargetDataInfo { 1611 /// Set to true if device pointer information have to be obtained. 1612 bool RequiresDevicePointerInfo = false; 1613 1614 public: 1615 /// The array of base pointer passed to the runtime library. 1616 llvm::Value *BasePointersArray = nullptr; 1617 /// The array of section pointers passed to the runtime library. 1618 llvm::Value *PointersArray = nullptr; 1619 /// The array of sizes passed to the runtime library. 1620 llvm::Value *SizesArray = nullptr; 1621 /// The array of map types passed to the runtime library. 1622 llvm::Value *MapTypesArray = nullptr; 1623 /// The total number of pointers passed to the runtime library. 1624 unsigned NumberOfPtrs = 0u; 1625 /// Map between the a declaration of a capture and the corresponding base 1626 /// pointer address where the runtime returns the device pointers. 1627 llvm::DenseMap<const ValueDecl *, Address> CaptureDeviceAddrMap; 1628 1629 explicit TargetDataInfo() {} 1630 explicit TargetDataInfo(bool RequiresDevicePointerInfo) 1631 : RequiresDevicePointerInfo(RequiresDevicePointerInfo) {} 1632 /// Clear information about the data arrays. 1633 void clearArrayInfo() { 1634 BasePointersArray = nullptr; 1635 PointersArray = nullptr; 1636 SizesArray = nullptr; 1637 MapTypesArray = nullptr; 1638 NumberOfPtrs = 0u; 1639 } 1640 /// Return true if the current target data information has valid arrays. 1641 bool isValid() { 1642 return BasePointersArray && PointersArray && SizesArray && 1643 MapTypesArray && NumberOfPtrs; 1644 } 1645 bool requiresDevicePointerInfo() { return RequiresDevicePointerInfo; } 1646 }; 1647 1648 /// Emit the target data mapping code associated with \a D. 1649 /// \param D Directive to emit. 1650 /// \param IfCond Expression evaluated in if clause associated with the 1651 /// target directive, or null if no device clause is used. 1652 /// \param Device Expression evaluated in device clause associated with the 1653 /// target directive, or null if no device clause is used. 1654 /// \param Info A record used to store information that needs to be preserved 1655 /// until the region is closed. 1656 virtual void emitTargetDataCalls(CodeGenFunction &CGF, 1657 const OMPExecutableDirective &D, 1658 const Expr *IfCond, const Expr *Device, 1659 const RegionCodeGenTy &CodeGen, 1660 TargetDataInfo &Info); 1661 1662 /// Emit the data mapping/movement code associated with the directive 1663 /// \a D that should be of the form 'target [{enter|exit} data | update]'. 1664 /// \param D Directive to emit. 1665 /// \param IfCond Expression evaluated in if clause associated with the target 1666 /// directive, or null if no if clause is used. 1667 /// \param Device Expression evaluated in device clause associated with the 1668 /// target directive, or null if no device clause is used. 1669 virtual void emitTargetDataStandAloneCall(CodeGenFunction &CGF, 1670 const OMPExecutableDirective &D, 1671 const Expr *IfCond, 1672 const Expr *Device); 1673 1674 /// Marks function \a Fn with properly mangled versions of vector functions. 1675 /// \param FD Function marked as 'declare simd'. 1676 /// \param Fn LLVM function that must be marked with 'declare simd' 1677 /// attributes. 1678 virtual void emitDeclareSimdFunction(const FunctionDecl *FD, 1679 llvm::Function *Fn); 1680 1681 /// Emit initialization for doacross loop nesting support. 1682 /// \param D Loop-based construct used in doacross nesting construct. 1683 virtual void emitDoacrossInit(CodeGenFunction &CGF, const OMPLoopDirective &D, 1684 ArrayRef<Expr *> NumIterations); 1685 1686 /// Emit code for doacross ordered directive with 'depend' clause. 1687 /// \param C 'depend' clause with 'sink|source' dependency kind. 1688 virtual void emitDoacrossOrdered(CodeGenFunction &CGF, 1689 const OMPDependClause *C); 1690 1691 /// Translates the native parameter of outlined function if this is required 1692 /// for target. 1693 /// \param FD Field decl from captured record for the parameter. 1694 /// \param NativeParam Parameter itself. 1695 virtual const VarDecl *translateParameter(const FieldDecl *FD, 1696 const VarDecl *NativeParam) const { 1697 return NativeParam; 1698 } 1699 1700 /// Gets the address of the native argument basing on the address of the 1701 /// target-specific parameter. 1702 /// \param NativeParam Parameter itself. 1703 /// \param TargetParam Corresponding target-specific parameter. 1704 virtual Address getParameterAddress(CodeGenFunction &CGF, 1705 const VarDecl *NativeParam, 1706 const VarDecl *TargetParam) const; 1707 1708 /// Choose default schedule type and chunk value for the 1709 /// dist_schedule clause. 1710 virtual void getDefaultDistScheduleAndChunk(CodeGenFunction &CGF, 1711 const OMPLoopDirective &S, OpenMPDistScheduleClauseKind &ScheduleKind, 1712 llvm::Value *&Chunk) const {} 1713 1714 /// Choose default schedule type and chunk value for the 1715 /// schedule clause. 1716 virtual void getDefaultScheduleAndChunk(CodeGenFunction &CGF, 1717 const OMPLoopDirective &S, OpenMPScheduleClauseKind &ScheduleKind, 1718 const Expr *&ChunkExpr) const; 1719 1720 /// Emits call of the outlined function with the provided arguments, 1721 /// translating these arguments to correct target-specific arguments. 1722 virtual void 1723 emitOutlinedFunctionCall(CodeGenFunction &CGF, SourceLocation Loc, 1724 llvm::FunctionCallee OutlinedFn, 1725 ArrayRef<llvm::Value *> Args = llvm::None) const; 1726 1727 /// Emits OpenMP-specific function prolog. 1728 /// Required for device constructs. 1729 virtual void emitFunctionProlog(CodeGenFunction &CGF, const Decl *D); 1730 1731 /// Gets the OpenMP-specific address of the local variable. 1732 virtual Address getAddressOfLocalVariable(CodeGenFunction &CGF, 1733 const VarDecl *VD); 1734 1735 /// Marks the declaration as already emitted for the device code and returns 1736 /// true, if it was marked already, and false, otherwise. 1737 bool markAsGlobalTarget(GlobalDecl GD); 1738 1739 /// Emit deferred declare target variables marked for deferred emission. 1740 void emitDeferredTargetDecls() const; 1741 1742 /// Adjust some parameters for the target-based directives, like addresses of 1743 /// the variables captured by reference in lambdas. 1744 virtual void 1745 adjustTargetSpecificDataForLambdas(CodeGenFunction &CGF, 1746 const OMPExecutableDirective &D) const; 1747 1748 /// Perform check on requires decl to ensure that target architecture 1749 /// supports unified addressing 1750 virtual void processRequiresDirective(const OMPRequiresDecl *D); 1751 1752 /// Gets default memory ordering as specified in requires directive. 1753 llvm::AtomicOrdering getDefaultMemoryOrdering() const; 1754 1755 /// Checks if the variable has associated OMPAllocateDeclAttr attribute with 1756 /// the predefined allocator and translates it into the corresponding address 1757 /// space. 1758 virtual bool hasAllocateAttributeForGlobalVar(const VarDecl *VD, LangAS &AS); 1759 1760 /// Return whether the unified_shared_memory has been specified. 1761 bool hasRequiresUnifiedSharedMemory() const; 1762 1763 /// Checks if the \p VD variable is marked as nontemporal declaration in 1764 /// current context. 1765 bool isNontemporalDecl(const ValueDecl *VD) const; 1766 1767 /// Create specialized alloca to handle lastprivate conditionals. 1768 Address emitLastprivateConditionalInit(CodeGenFunction &CGF, 1769 const VarDecl *VD); 1770 1771 /// Checks if the provided \p LVal is lastprivate conditional and emits the 1772 /// code to update the value of the original variable. 1773 /// \code 1774 /// lastprivate(conditional: a) 1775 /// ... 1776 /// <type> a; 1777 /// lp_a = ...; 1778 /// #pragma omp critical(a) 1779 /// if (last_iv_a <= iv) { 1780 /// last_iv_a = iv; 1781 /// global_a = lp_a; 1782 /// } 1783 /// \endcode 1784 virtual void checkAndEmitLastprivateConditional(CodeGenFunction &CGF, 1785 const Expr *LHS); 1786 1787 /// Checks if the lastprivate conditional was updated in inner region and 1788 /// writes the value. 1789 /// \code 1790 /// lastprivate(conditional: a) 1791 /// ... 1792 /// <type> a;bool Fired = false; 1793 /// #pragma omp ... shared(a) 1794 /// { 1795 /// lp_a = ...; 1796 /// Fired = true; 1797 /// } 1798 /// if (Fired) { 1799 /// #pragma omp critical(a) 1800 /// if (last_iv_a <= iv) { 1801 /// last_iv_a = iv; 1802 /// global_a = lp_a; 1803 /// } 1804 /// Fired = false; 1805 /// } 1806 /// \endcode 1807 virtual void checkAndEmitSharedLastprivateConditional( 1808 CodeGenFunction &CGF, const OMPExecutableDirective &D, 1809 const llvm::DenseSet<CanonicalDeclPtr<const VarDecl>> &IgnoredDecls); 1810 1811 /// Gets the address of the global copy used for lastprivate conditional 1812 /// update, if any. 1813 /// \param PrivLVal LValue for the private copy. 1814 /// \param VD Original lastprivate declaration. 1815 virtual void emitLastprivateConditionalFinalUpdate(CodeGenFunction &CGF, 1816 LValue PrivLVal, 1817 const VarDecl *VD, 1818 SourceLocation Loc); 1819 1820 /// Emits list of dependecies based on the provided data (array of 1821 /// dependence/expression pairs). 1822 /// \returns Pointer to the first element of the array casted to VoidPtr type. 1823 std::pair<llvm::Value *, Address> 1824 emitDependClause(CodeGenFunction &CGF, 1825 ArrayRef<OMPTaskDataTy::DependData> Dependencies, 1826 SourceLocation Loc); 1827 1828 /// Emits list of dependecies based on the provided data (array of 1829 /// dependence/expression pairs) for depobj construct. In this case, the 1830 /// variable is allocated in dynamically. \returns Pointer to the first 1831 /// element of the array casted to VoidPtr type. 1832 Address emitDepobjDependClause(CodeGenFunction &CGF, 1833 const OMPTaskDataTy::DependData &Dependencies, 1834 SourceLocation Loc); 1835 1836 /// Emits the code to destroy the dependency object provided in depobj 1837 /// directive. 1838 void emitDestroyClause(CodeGenFunction &CGF, LValue DepobjLVal, 1839 SourceLocation Loc); 1840 1841 /// Updates the dependency kind in the specified depobj object. 1842 /// \param DepobjLVal LValue for the main depobj object. 1843 /// \param NewDepKind New dependency kind. 1844 void emitUpdateClause(CodeGenFunction &CGF, LValue DepobjLVal, 1845 OpenMPDependClauseKind NewDepKind, SourceLocation Loc); 1846 1847 /// Initializes user defined allocators specified in the uses_allocators 1848 /// clauses. 1849 void emitUsesAllocatorsInit(CodeGenFunction &CGF, const Expr *Allocator, 1850 const Expr *AllocatorTraits); 1851 1852 /// Destroys user defined allocators specified in the uses_allocators clause. 1853 void emitUsesAllocatorsFini(CodeGenFunction &CGF, const Expr *Allocator); 1854 }; 1855 1856 /// Class supports emissionof SIMD-only code. 1857 class CGOpenMPSIMDRuntime final : public CGOpenMPRuntime { 1858 public: 1859 explicit CGOpenMPSIMDRuntime(CodeGenModule &CGM) : CGOpenMPRuntime(CGM) {} 1860 ~CGOpenMPSIMDRuntime() override {} 1861 1862 /// Emits outlined function for the specified OpenMP parallel directive 1863 /// \a D. This outlined function has type void(*)(kmp_int32 *ThreadID, 1864 /// kmp_int32 BoundID, struct context_vars*). 1865 /// \param D OpenMP directive. 1866 /// \param ThreadIDVar Variable for thread id in the current OpenMP region. 1867 /// \param InnermostKind Kind of innermost directive (for simple directives it 1868 /// is a directive itself, for combined - its innermost directive). 1869 /// \param CodeGen Code generation sequence for the \a D directive. 1870 llvm::Function * 1871 emitParallelOutlinedFunction(const OMPExecutableDirective &D, 1872 const VarDecl *ThreadIDVar, 1873 OpenMPDirectiveKind InnermostKind, 1874 const RegionCodeGenTy &CodeGen) override; 1875 1876 /// Emits outlined function for the specified OpenMP teams directive 1877 /// \a D. This outlined function has type void(*)(kmp_int32 *ThreadID, 1878 /// kmp_int32 BoundID, struct context_vars*). 1879 /// \param D OpenMP directive. 1880 /// \param ThreadIDVar Variable for thread id in the current OpenMP region. 1881 /// \param InnermostKind Kind of innermost directive (for simple directives it 1882 /// is a directive itself, for combined - its innermost directive). 1883 /// \param CodeGen Code generation sequence for the \a D directive. 1884 llvm::Function * 1885 emitTeamsOutlinedFunction(const OMPExecutableDirective &D, 1886 const VarDecl *ThreadIDVar, 1887 OpenMPDirectiveKind InnermostKind, 1888 const RegionCodeGenTy &CodeGen) override; 1889 1890 /// Emits outlined function for the OpenMP task directive \a D. This 1891 /// outlined function has type void(*)(kmp_int32 ThreadID, struct task_t* 1892 /// TaskT). 1893 /// \param D OpenMP directive. 1894 /// \param ThreadIDVar Variable for thread id in the current OpenMP region. 1895 /// \param PartIDVar Variable for partition id in the current OpenMP untied 1896 /// task region. 1897 /// \param TaskTVar Variable for task_t argument. 1898 /// \param InnermostKind Kind of innermost directive (for simple directives it 1899 /// is a directive itself, for combined - its innermost directive). 1900 /// \param CodeGen Code generation sequence for the \a D directive. 1901 /// \param Tied true if task is generated for tied task, false otherwise. 1902 /// \param NumberOfParts Number of parts in untied task. Ignored for tied 1903 /// tasks. 1904 /// 1905 llvm::Function *emitTaskOutlinedFunction( 1906 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, 1907 const VarDecl *PartIDVar, const VarDecl *TaskTVar, 1908 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, 1909 bool Tied, unsigned &NumberOfParts) override; 1910 1911 /// Emits code for parallel or serial call of the \a OutlinedFn with 1912 /// variables captured in a record which address is stored in \a 1913 /// CapturedStruct. 1914 /// \param OutlinedFn Outlined function to be run in parallel threads. Type of 1915 /// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*). 1916 /// \param CapturedVars A pointer to the record with the references to 1917 /// variables used in \a OutlinedFn function. 1918 /// \param IfCond Condition in the associated 'if' clause, if it was 1919 /// specified, nullptr otherwise. 1920 /// 1921 void emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc, 1922 llvm::Function *OutlinedFn, 1923 ArrayRef<llvm::Value *> CapturedVars, 1924 const Expr *IfCond) override; 1925 1926 /// Emits a critical region. 1927 /// \param CriticalName Name of the critical region. 1928 /// \param CriticalOpGen Generator for the statement associated with the given 1929 /// critical region. 1930 /// \param Hint Value of the 'hint' clause (optional). 1931 void emitCriticalRegion(CodeGenFunction &CGF, StringRef CriticalName, 1932 const RegionCodeGenTy &CriticalOpGen, 1933 SourceLocation Loc, 1934 const Expr *Hint = nullptr) override; 1935 1936 /// Emits a master region. 1937 /// \param MasterOpGen Generator for the statement associated with the given 1938 /// master region. 1939 void emitMasterRegion(CodeGenFunction &CGF, 1940 const RegionCodeGenTy &MasterOpGen, 1941 SourceLocation Loc) override; 1942 1943 /// Emits code for a taskyield directive. 1944 void emitTaskyieldCall(CodeGenFunction &CGF, SourceLocation Loc) override; 1945 1946 /// Emit a taskgroup region. 1947 /// \param TaskgroupOpGen Generator for the statement associated with the 1948 /// given taskgroup region. 1949 void emitTaskgroupRegion(CodeGenFunction &CGF, 1950 const RegionCodeGenTy &TaskgroupOpGen, 1951 SourceLocation Loc) override; 1952 1953 /// Emits a single region. 1954 /// \param SingleOpGen Generator for the statement associated with the given 1955 /// single region. 1956 void emitSingleRegion(CodeGenFunction &CGF, 1957 const RegionCodeGenTy &SingleOpGen, SourceLocation Loc, 1958 ArrayRef<const Expr *> CopyprivateVars, 1959 ArrayRef<const Expr *> DestExprs, 1960 ArrayRef<const Expr *> SrcExprs, 1961 ArrayRef<const Expr *> AssignmentOps) override; 1962 1963 /// Emit an ordered region. 1964 /// \param OrderedOpGen Generator for the statement associated with the given 1965 /// ordered region. 1966 void emitOrderedRegion(CodeGenFunction &CGF, 1967 const RegionCodeGenTy &OrderedOpGen, 1968 SourceLocation Loc, bool IsThreads) override; 1969 1970 /// Emit an implicit/explicit barrier for OpenMP threads. 1971 /// \param Kind Directive for which this implicit barrier call must be 1972 /// generated. Must be OMPD_barrier for explicit barrier generation. 1973 /// \param EmitChecks true if need to emit checks for cancellation barriers. 1974 /// \param ForceSimpleCall true simple barrier call must be emitted, false if 1975 /// runtime class decides which one to emit (simple or with cancellation 1976 /// checks). 1977 /// 1978 void emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc, 1979 OpenMPDirectiveKind Kind, bool EmitChecks = true, 1980 bool ForceSimpleCall = false) override; 1981 1982 /// This is used for non static scheduled types and when the ordered 1983 /// clause is present on the loop construct. 1984 /// Depending on the loop schedule, it is necessary to call some runtime 1985 /// routine before start of the OpenMP loop to get the loop upper / lower 1986 /// bounds \a LB and \a UB and stride \a ST. 1987 /// 1988 /// \param CGF Reference to current CodeGenFunction. 1989 /// \param Loc Clang source location. 1990 /// \param ScheduleKind Schedule kind, specified by the 'schedule' clause. 1991 /// \param IVSize Size of the iteration variable in bits. 1992 /// \param IVSigned Sign of the iteration variable. 1993 /// \param Ordered true if loop is ordered, false otherwise. 1994 /// \param DispatchValues struct containing llvm values for lower bound, upper 1995 /// bound, and chunk expression. 1996 /// For the default (nullptr) value, the chunk 1 will be used. 1997 /// 1998 void emitForDispatchInit(CodeGenFunction &CGF, SourceLocation Loc, 1999 const OpenMPScheduleTy &ScheduleKind, 2000 unsigned IVSize, bool IVSigned, bool Ordered, 2001 const DispatchRTInput &DispatchValues) override; 2002 2003 /// Call the appropriate runtime routine to initialize it before start 2004 /// of loop. 2005 /// 2006 /// This is used only in case of static schedule, when the user did not 2007 /// specify a ordered clause on the loop construct. 2008 /// Depending on the loop schedule, it is necessary to call some runtime 2009 /// routine before start of the OpenMP loop to get the loop upper / lower 2010 /// bounds LB and UB and stride ST. 2011 /// 2012 /// \param CGF Reference to current CodeGenFunction. 2013 /// \param Loc Clang source location. 2014 /// \param DKind Kind of the directive. 2015 /// \param ScheduleKind Schedule kind, specified by the 'schedule' clause. 2016 /// \param Values Input arguments for the construct. 2017 /// 2018 void emitForStaticInit(CodeGenFunction &CGF, SourceLocation Loc, 2019 OpenMPDirectiveKind DKind, 2020 const OpenMPScheduleTy &ScheduleKind, 2021 const StaticRTInput &Values) override; 2022 2023 /// 2024 /// \param CGF Reference to current CodeGenFunction. 2025 /// \param Loc Clang source location. 2026 /// \param SchedKind Schedule kind, specified by the 'dist_schedule' clause. 2027 /// \param Values Input arguments for the construct. 2028 /// 2029 void emitDistributeStaticInit(CodeGenFunction &CGF, SourceLocation Loc, 2030 OpenMPDistScheduleClauseKind SchedKind, 2031 const StaticRTInput &Values) override; 2032 2033 /// Call the appropriate runtime routine to notify that we finished 2034 /// iteration of the ordered loop with the dynamic scheduling. 2035 /// 2036 /// \param CGF Reference to current CodeGenFunction. 2037 /// \param Loc Clang source location. 2038 /// \param IVSize Size of the iteration variable in bits. 2039 /// \param IVSigned Sign of the iteration variable. 2040 /// 2041 void emitForOrderedIterationEnd(CodeGenFunction &CGF, SourceLocation Loc, 2042 unsigned IVSize, bool IVSigned) override; 2043 2044 /// Call the appropriate runtime routine to notify that we finished 2045 /// all the work with current loop. 2046 /// 2047 /// \param CGF Reference to current CodeGenFunction. 2048 /// \param Loc Clang source location. 2049 /// \param DKind Kind of the directive for which the static finish is emitted. 2050 /// 2051 void emitForStaticFinish(CodeGenFunction &CGF, SourceLocation Loc, 2052 OpenMPDirectiveKind DKind) override; 2053 2054 /// Call __kmpc_dispatch_next( 2055 /// ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter, 2056 /// kmp_int[32|64] *p_lower, kmp_int[32|64] *p_upper, 2057 /// kmp_int[32|64] *p_stride); 2058 /// \param IVSize Size of the iteration variable in bits. 2059 /// \param IVSigned Sign of the iteration variable. 2060 /// \param IL Address of the output variable in which the flag of the 2061 /// last iteration is returned. 2062 /// \param LB Address of the output variable in which the lower iteration 2063 /// number is returned. 2064 /// \param UB Address of the output variable in which the upper iteration 2065 /// number is returned. 2066 /// \param ST Address of the output variable in which the stride value is 2067 /// returned. 2068 llvm::Value *emitForNext(CodeGenFunction &CGF, SourceLocation Loc, 2069 unsigned IVSize, bool IVSigned, Address IL, 2070 Address LB, Address UB, Address ST) override; 2071 2072 /// Emits call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32 2073 /// global_tid, kmp_int32 num_threads) to generate code for 'num_threads' 2074 /// clause. 2075 /// \param NumThreads An integer value of threads. 2076 void emitNumThreadsClause(CodeGenFunction &CGF, llvm::Value *NumThreads, 2077 SourceLocation Loc) override; 2078 2079 /// Emit call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32 2080 /// global_tid, int proc_bind) to generate code for 'proc_bind' clause. 2081 void emitProcBindClause(CodeGenFunction &CGF, 2082 llvm::omp::ProcBindKind ProcBind, 2083 SourceLocation Loc) override; 2084 2085 /// Returns address of the threadprivate variable for the current 2086 /// thread. 2087 /// \param VD Threadprivate variable. 2088 /// \param VDAddr Address of the global variable \a VD. 2089 /// \param Loc Location of the reference to threadprivate var. 2090 /// \return Address of the threadprivate variable for the current thread. 2091 Address getAddrOfThreadPrivate(CodeGenFunction &CGF, const VarDecl *VD, 2092 Address VDAddr, SourceLocation Loc) override; 2093 2094 /// Emit a code for initialization of threadprivate variable. It emits 2095 /// a call to runtime library which adds initial value to the newly created 2096 /// threadprivate variable (if it is not constant) and registers destructor 2097 /// for the variable (if any). 2098 /// \param VD Threadprivate variable. 2099 /// \param VDAddr Address of the global variable \a VD. 2100 /// \param Loc Location of threadprivate declaration. 2101 /// \param PerformInit true if initialization expression is not constant. 2102 llvm::Function * 2103 emitThreadPrivateVarDefinition(const VarDecl *VD, Address VDAddr, 2104 SourceLocation Loc, bool PerformInit, 2105 CodeGenFunction *CGF = nullptr) override; 2106 2107 /// Creates artificial threadprivate variable with name \p Name and type \p 2108 /// VarType. 2109 /// \param VarType Type of the artificial threadprivate variable. 2110 /// \param Name Name of the artificial threadprivate variable. 2111 Address getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF, 2112 QualType VarType, 2113 StringRef Name) override; 2114 2115 /// Emit flush of the variables specified in 'omp flush' directive. 2116 /// \param Vars List of variables to flush. 2117 void emitFlush(CodeGenFunction &CGF, ArrayRef<const Expr *> Vars, 2118 SourceLocation Loc, llvm::AtomicOrdering AO) override; 2119 2120 /// Emit task region for the task directive. The task region is 2121 /// emitted in several steps: 2122 /// 1. Emit a call to kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32 2123 /// gtid, kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds, 2124 /// kmp_routine_entry_t *task_entry). Here task_entry is a pointer to the 2125 /// function: 2126 /// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) { 2127 /// TaskFunction(gtid, tt->part_id, tt->shareds); 2128 /// return 0; 2129 /// } 2130 /// 2. Copy a list of shared variables to field shareds of the resulting 2131 /// structure kmp_task_t returned by the previous call (if any). 2132 /// 3. Copy a pointer to destructions function to field destructions of the 2133 /// resulting structure kmp_task_t. 2134 /// 4. Emit a call to kmp_int32 __kmpc_omp_task(ident_t *, kmp_int32 gtid, 2135 /// kmp_task_t *new_task), where new_task is a resulting structure from 2136 /// previous items. 2137 /// \param D Current task directive. 2138 /// \param TaskFunction An LLVM function with type void (*)(i32 /*gtid*/, i32 2139 /// /*part_id*/, captured_struct */*__context*/); 2140 /// \param SharedsTy A type which contains references the shared variables. 2141 /// \param Shareds Context with the list of shared variables from the \p 2142 /// TaskFunction. 2143 /// \param IfCond Not a nullptr if 'if' clause was specified, nullptr 2144 /// otherwise. 2145 /// \param Data Additional data for task generation like tiednsee, final 2146 /// state, list of privates etc. 2147 void emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc, 2148 const OMPExecutableDirective &D, 2149 llvm::Function *TaskFunction, QualType SharedsTy, 2150 Address Shareds, const Expr *IfCond, 2151 const OMPTaskDataTy &Data) override; 2152 2153 /// Emit task region for the taskloop directive. The taskloop region is 2154 /// emitted in several steps: 2155 /// 1. Emit a call to kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32 2156 /// gtid, kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds, 2157 /// kmp_routine_entry_t *task_entry). Here task_entry is a pointer to the 2158 /// function: 2159 /// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) { 2160 /// TaskFunction(gtid, tt->part_id, tt->shareds); 2161 /// return 0; 2162 /// } 2163 /// 2. Copy a list of shared variables to field shareds of the resulting 2164 /// structure kmp_task_t returned by the previous call (if any). 2165 /// 3. Copy a pointer to destructions function to field destructions of the 2166 /// resulting structure kmp_task_t. 2167 /// 4. Emit a call to void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t 2168 /// *task, int if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int 2169 /// nogroup, int sched, kmp_uint64 grainsize, void *task_dup ), where new_task 2170 /// is a resulting structure from 2171 /// previous items. 2172 /// \param D Current task directive. 2173 /// \param TaskFunction An LLVM function with type void (*)(i32 /*gtid*/, i32 2174 /// /*part_id*/, captured_struct */*__context*/); 2175 /// \param SharedsTy A type which contains references the shared variables. 2176 /// \param Shareds Context with the list of shared variables from the \p 2177 /// TaskFunction. 2178 /// \param IfCond Not a nullptr if 'if' clause was specified, nullptr 2179 /// otherwise. 2180 /// \param Data Additional data for task generation like tiednsee, final 2181 /// state, list of privates etc. 2182 void emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc, 2183 const OMPLoopDirective &D, llvm::Function *TaskFunction, 2184 QualType SharedsTy, Address Shareds, const Expr *IfCond, 2185 const OMPTaskDataTy &Data) override; 2186 2187 /// Emit a code for reduction clause. Next code should be emitted for 2188 /// reduction: 2189 /// \code 2190 /// 2191 /// static kmp_critical_name lock = { 0 }; 2192 /// 2193 /// void reduce_func(void *lhs[<n>], void *rhs[<n>]) { 2194 /// ... 2195 /// *(Type<i>*)lhs[i] = RedOp<i>(*(Type<i>*)lhs[i], *(Type<i>*)rhs[i]); 2196 /// ... 2197 /// } 2198 /// 2199 /// ... 2200 /// void *RedList[<n>] = {&<RHSExprs>[0], ..., &<RHSExprs>[<n>-1]}; 2201 /// switch (__kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList), 2202 /// RedList, reduce_func, &<lock>)) { 2203 /// case 1: 2204 /// ... 2205 /// <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]); 2206 /// ... 2207 /// __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>); 2208 /// break; 2209 /// case 2: 2210 /// ... 2211 /// Atomic(<LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i])); 2212 /// ... 2213 /// break; 2214 /// default:; 2215 /// } 2216 /// \endcode 2217 /// 2218 /// \param Privates List of private copies for original reduction arguments. 2219 /// \param LHSExprs List of LHS in \a ReductionOps reduction operations. 2220 /// \param RHSExprs List of RHS in \a ReductionOps reduction operations. 2221 /// \param ReductionOps List of reduction operations in form 'LHS binop RHS' 2222 /// or 'operator binop(LHS, RHS)'. 2223 /// \param Options List of options for reduction codegen: 2224 /// WithNowait true if parent directive has also nowait clause, false 2225 /// otherwise. 2226 /// SimpleReduction Emit reduction operation only. Used for omp simd 2227 /// directive on the host. 2228 /// ReductionKind The kind of reduction to perform. 2229 void emitReduction(CodeGenFunction &CGF, SourceLocation Loc, 2230 ArrayRef<const Expr *> Privates, 2231 ArrayRef<const Expr *> LHSExprs, 2232 ArrayRef<const Expr *> RHSExprs, 2233 ArrayRef<const Expr *> ReductionOps, 2234 ReductionOptionsTy Options) override; 2235 2236 /// Emit a code for initialization of task reduction clause. Next code 2237 /// should be emitted for reduction: 2238 /// \code 2239 /// 2240 /// _taskred_item_t red_data[n]; 2241 /// ... 2242 /// red_data[i].shar = &shareds[i]; 2243 /// red_data[i].orig = &origs[i]; 2244 /// red_data[i].size = sizeof(origs[i]); 2245 /// red_data[i].f_init = (void*)RedInit<i>; 2246 /// red_data[i].f_fini = (void*)RedDest<i>; 2247 /// red_data[i].f_comb = (void*)RedOp<i>; 2248 /// red_data[i].flags = <Flag_i>; 2249 /// ... 2250 /// void* tg1 = __kmpc_taskred_init(gtid, n, red_data); 2251 /// \endcode 2252 /// For reduction clause with task modifier it emits the next call: 2253 /// \code 2254 /// 2255 /// _taskred_item_t red_data[n]; 2256 /// ... 2257 /// red_data[i].shar = &shareds[i]; 2258 /// red_data[i].orig = &origs[i]; 2259 /// red_data[i].size = sizeof(origs[i]); 2260 /// red_data[i].f_init = (void*)RedInit<i>; 2261 /// red_data[i].f_fini = (void*)RedDest<i>; 2262 /// red_data[i].f_comb = (void*)RedOp<i>; 2263 /// red_data[i].flags = <Flag_i>; 2264 /// ... 2265 /// void* tg1 = __kmpc_taskred_modifier_init(loc, gtid, is_worksharing, n, 2266 /// red_data); 2267 /// \endcode 2268 /// \param LHSExprs List of LHS in \a Data.ReductionOps reduction operations. 2269 /// \param RHSExprs List of RHS in \a Data.ReductionOps reduction operations. 2270 /// \param Data Additional data for task generation like tiedness, final 2271 /// state, list of privates, reductions etc. 2272 llvm::Value *emitTaskReductionInit(CodeGenFunction &CGF, SourceLocation Loc, 2273 ArrayRef<const Expr *> LHSExprs, 2274 ArrayRef<const Expr *> RHSExprs, 2275 const OMPTaskDataTy &Data) override; 2276 2277 /// Emits the following code for reduction clause with task modifier: 2278 /// \code 2279 /// __kmpc_task_reduction_modifier_fini(loc, gtid, is_worksharing); 2280 /// \endcode 2281 void emitTaskReductionFini(CodeGenFunction &CGF, SourceLocation Loc, 2282 bool IsWorksharingReduction) override; 2283 2284 /// Required to resolve existing problems in the runtime. Emits threadprivate 2285 /// variables to store the size of the VLAs/array sections for 2286 /// initializer/combiner/finalizer functions + emits threadprivate variable to 2287 /// store the pointer to the original reduction item for the custom 2288 /// initializer defined by declare reduction construct. 2289 /// \param RCG Allows to reuse an existing data for the reductions. 2290 /// \param N Reduction item for which fixups must be emitted. 2291 void emitTaskReductionFixups(CodeGenFunction &CGF, SourceLocation Loc, 2292 ReductionCodeGen &RCG, unsigned N) override; 2293 2294 /// Get the address of `void *` type of the privatue copy of the reduction 2295 /// item specified by the \p SharedLVal. 2296 /// \param ReductionsPtr Pointer to the reduction data returned by the 2297 /// emitTaskReductionInit function. 2298 /// \param SharedLVal Address of the original reduction item. 2299 Address getTaskReductionItem(CodeGenFunction &CGF, SourceLocation Loc, 2300 llvm::Value *ReductionsPtr, 2301 LValue SharedLVal) override; 2302 2303 /// Emit code for 'taskwait' directive. 2304 void emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc) override; 2305 2306 /// Emit code for 'cancellation point' construct. 2307 /// \param CancelRegion Region kind for which the cancellation point must be 2308 /// emitted. 2309 /// 2310 void emitCancellationPointCall(CodeGenFunction &CGF, SourceLocation Loc, 2311 OpenMPDirectiveKind CancelRegion) override; 2312 2313 /// Emit code for 'cancel' construct. 2314 /// \param IfCond Condition in the associated 'if' clause, if it was 2315 /// specified, nullptr otherwise. 2316 /// \param CancelRegion Region kind for which the cancel must be emitted. 2317 /// 2318 void emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc, 2319 const Expr *IfCond, 2320 OpenMPDirectiveKind CancelRegion) override; 2321 2322 /// Emit outilined function for 'target' directive. 2323 /// \param D Directive to emit. 2324 /// \param ParentName Name of the function that encloses the target region. 2325 /// \param OutlinedFn Outlined function value to be defined by this call. 2326 /// \param OutlinedFnID Outlined function ID value to be defined by this call. 2327 /// \param IsOffloadEntry True if the outlined function is an offload entry. 2328 /// \param CodeGen Code generation sequence for the \a D directive. 2329 /// An outlined function may not be an entry if, e.g. the if clause always 2330 /// evaluates to false. 2331 void emitTargetOutlinedFunction(const OMPExecutableDirective &D, 2332 StringRef ParentName, 2333 llvm::Function *&OutlinedFn, 2334 llvm::Constant *&OutlinedFnID, 2335 bool IsOffloadEntry, 2336 const RegionCodeGenTy &CodeGen) override; 2337 2338 /// Emit the target offloading code associated with \a D. The emitted 2339 /// code attempts offloading the execution to the device, an the event of 2340 /// a failure it executes the host version outlined in \a OutlinedFn. 2341 /// \param D Directive to emit. 2342 /// \param OutlinedFn Host version of the code to be offloaded. 2343 /// \param OutlinedFnID ID of host version of the code to be offloaded. 2344 /// \param IfCond Expression evaluated in if clause associated with the target 2345 /// directive, or null if no if clause is used. 2346 /// \param Device Expression evaluated in device clause associated with the 2347 /// target directive, or null if no device clause is used and device modifier. 2348 void emitTargetCall( 2349 CodeGenFunction &CGF, const OMPExecutableDirective &D, 2350 llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond, 2351 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier> Device, 2352 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF, 2353 const OMPLoopDirective &D)> 2354 SizeEmitter) override; 2355 2356 /// Emit the target regions enclosed in \a GD function definition or 2357 /// the function itself in case it is a valid device function. Returns true if 2358 /// \a GD was dealt with successfully. 2359 /// \param GD Function to scan. 2360 bool emitTargetFunctions(GlobalDecl GD) override; 2361 2362 /// Emit the global variable if it is a valid device global variable. 2363 /// Returns true if \a GD was dealt with successfully. 2364 /// \param GD Variable declaration to emit. 2365 bool emitTargetGlobalVariable(GlobalDecl GD) override; 2366 2367 /// Emit the global \a GD if it is meaningful for the target. Returns 2368 /// if it was emitted successfully. 2369 /// \param GD Global to scan. 2370 bool emitTargetGlobal(GlobalDecl GD) override; 2371 2372 /// Emits code for teams call of the \a OutlinedFn with 2373 /// variables captured in a record which address is stored in \a 2374 /// CapturedStruct. 2375 /// \param OutlinedFn Outlined function to be run by team masters. Type of 2376 /// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*). 2377 /// \param CapturedVars A pointer to the record with the references to 2378 /// variables used in \a OutlinedFn function. 2379 /// 2380 void emitTeamsCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, 2381 SourceLocation Loc, llvm::Function *OutlinedFn, 2382 ArrayRef<llvm::Value *> CapturedVars) override; 2383 2384 /// Emits call to void __kmpc_push_num_teams(ident_t *loc, kmp_int32 2385 /// global_tid, kmp_int32 num_teams, kmp_int32 thread_limit) to generate code 2386 /// for num_teams clause. 2387 /// \param NumTeams An integer expression of teams. 2388 /// \param ThreadLimit An integer expression of threads. 2389 void emitNumTeamsClause(CodeGenFunction &CGF, const Expr *NumTeams, 2390 const Expr *ThreadLimit, SourceLocation Loc) override; 2391 2392 /// Emit the target data mapping code associated with \a D. 2393 /// \param D Directive to emit. 2394 /// \param IfCond Expression evaluated in if clause associated with the 2395 /// target directive, or null if no device clause is used. 2396 /// \param Device Expression evaluated in device clause associated with the 2397 /// target directive, or null if no device clause is used. 2398 /// \param Info A record used to store information that needs to be preserved 2399 /// until the region is closed. 2400 void emitTargetDataCalls(CodeGenFunction &CGF, 2401 const OMPExecutableDirective &D, const Expr *IfCond, 2402 const Expr *Device, const RegionCodeGenTy &CodeGen, 2403 TargetDataInfo &Info) override; 2404 2405 /// Emit the data mapping/movement code associated with the directive 2406 /// \a D that should be of the form 'target [{enter|exit} data | update]'. 2407 /// \param D Directive to emit. 2408 /// \param IfCond Expression evaluated in if clause associated with the target 2409 /// directive, or null if no if clause is used. 2410 /// \param Device Expression evaluated in device clause associated with the 2411 /// target directive, or null if no device clause is used. 2412 void emitTargetDataStandAloneCall(CodeGenFunction &CGF, 2413 const OMPExecutableDirective &D, 2414 const Expr *IfCond, 2415 const Expr *Device) override; 2416 2417 /// Emit initialization for doacross loop nesting support. 2418 /// \param D Loop-based construct used in doacross nesting construct. 2419 void emitDoacrossInit(CodeGenFunction &CGF, const OMPLoopDirective &D, 2420 ArrayRef<Expr *> NumIterations) override; 2421 2422 /// Emit code for doacross ordered directive with 'depend' clause. 2423 /// \param C 'depend' clause with 'sink|source' dependency kind. 2424 void emitDoacrossOrdered(CodeGenFunction &CGF, 2425 const OMPDependClause *C) override; 2426 2427 /// Translates the native parameter of outlined function if this is required 2428 /// for target. 2429 /// \param FD Field decl from captured record for the parameter. 2430 /// \param NativeParam Parameter itself. 2431 const VarDecl *translateParameter(const FieldDecl *FD, 2432 const VarDecl *NativeParam) const override; 2433 2434 /// Gets the address of the native argument basing on the address of the 2435 /// target-specific parameter. 2436 /// \param NativeParam Parameter itself. 2437 /// \param TargetParam Corresponding target-specific parameter. 2438 Address getParameterAddress(CodeGenFunction &CGF, const VarDecl *NativeParam, 2439 const VarDecl *TargetParam) const override; 2440 2441 /// Gets the OpenMP-specific address of the local variable. 2442 Address getAddressOfLocalVariable(CodeGenFunction &CGF, 2443 const VarDecl *VD) override { 2444 return Address::invalid(); 2445 } 2446 }; 2447 2448 } // namespace CodeGen 2449 } // namespace clang 2450 2451 #endif 2452