30#include "llvm/ADT/ArrayRef.h"
31#include "llvm/ADT/SetOperations.h"
32#include "llvm/ADT/SmallBitVector.h"
33#include "llvm/ADT/StringExtras.h"
34#include "llvm/Bitcode/BitcodeReader.h"
35#include "llvm/IR/Constants.h"
36#include "llvm/IR/DerivedTypes.h"
37#include "llvm/IR/GlobalValue.h"
38#include "llvm/IR/InstrTypes.h"
39#include "llvm/IR/Value.h"
40#include "llvm/Support/AtomicOrdering.h"
41#include "llvm/Support/Format.h"
42#include "llvm/Support/raw_ostream.h"
49using namespace CodeGen;
50using namespace llvm::omp;
54class CGOpenMPRegionInfo :
public CodeGenFunction::CGCapturedStmtInfo {
57 enum CGOpenMPRegionKind {
60 ParallelOutlinedRegion,
71 const CGOpenMPRegionKind RegionKind,
74 : CGCapturedStmtInfo(CS,
CR_OpenMP), RegionKind(RegionKind),
75 CodeGen(CodeGen),
Kind(
Kind), HasCancel(HasCancel) {}
77 CGOpenMPRegionInfo(
const CGOpenMPRegionKind RegionKind,
80 : CGCapturedStmtInfo(
CR_OpenMP), RegionKind(RegionKind), CodeGen(CodeGen),
85 virtual const VarDecl *getThreadIDVariable()
const = 0;
96 CGOpenMPRegionKind getRegionKind()
const {
return RegionKind; }
100 bool hasCancel()
const {
return HasCancel; }
102 static bool classof(
const CGCapturedStmtInfo *Info) {
106 ~CGOpenMPRegionInfo()
override =
default;
109 CGOpenMPRegionKind RegionKind;
116class CGOpenMPOutlinedRegionInfo final :
public CGOpenMPRegionInfo {
121 StringRef HelperName)
122 : CGOpenMPRegionInfo(CS, ParallelOutlinedRegion, CodeGen,
Kind,
124 ThreadIDVar(ThreadIDVar), HelperName(HelperName) {
125 assert(ThreadIDVar !=
nullptr &&
"No ThreadID in OpenMP region.");
130 const VarDecl *getThreadIDVariable()
const override {
return ThreadIDVar; }
133 StringRef getHelperName()
const override {
return HelperName; }
135 static bool classof(
const CGCapturedStmtInfo *Info) {
136 return CGOpenMPRegionInfo::classof(Info) &&
137 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() ==
138 ParallelOutlinedRegion;
145 StringRef HelperName;
149class CGOpenMPTaskOutlinedRegionInfo final :
public CGOpenMPRegionInfo {
155 llvm::SwitchInst *UntiedSwitch =
nullptr;
158 UntiedTaskActionTy(
bool Tied,
const VarDecl *PartIDVar,
160 : Untied(!Tied), PartIDVar(PartIDVar), UntiedCodeGen(UntiedCodeGen) {}
170 UntiedSwitch = CGF.
Builder.CreateSwitch(Res, DoneBB);
174 UntiedSwitch->addCase(CGF.
Builder.getInt32(0),
176 emitUntiedSwitch(CGF);
187 CodeGenFunction::JumpDest CurPoint =
191 UntiedSwitch->addCase(CGF.
Builder.getInt32(UntiedSwitch->getNumCases()),
197 unsigned getNumberOfParts()
const {
return UntiedSwitch->getNumCases(); }
203 const UntiedTaskActionTy &Action)
204 : CGOpenMPRegionInfo(CS, TaskOutlinedRegion, CodeGen,
Kind, HasCancel),
205 ThreadIDVar(ThreadIDVar), Action(Action) {
206 assert(ThreadIDVar !=
nullptr &&
"No ThreadID in OpenMP region.");
211 const VarDecl *getThreadIDVariable()
const override {
return ThreadIDVar; }
217 StringRef getHelperName()
const override {
return ".omp_outlined."; }
220 Action.emitUntiedSwitch(CGF);
223 static bool classof(
const CGCapturedStmtInfo *Info) {
224 return CGOpenMPRegionInfo::classof(Info) &&
225 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() ==
234 const UntiedTaskActionTy &Action;
239class CGOpenMPInlinedRegionInfo :
public CGOpenMPRegionInfo {
241 CGOpenMPInlinedRegionInfo(CodeGenFunction::CGCapturedStmtInfo *OldCSI,
244 : CGOpenMPRegionInfo(InlinedRegion, CodeGen,
Kind, HasCancel),
246 OuterRegionInfo(dyn_cast_or_null<CGOpenMPRegionInfo>(OldCSI)) {}
249 llvm::Value *getContextValue()
const override {
251 return OuterRegionInfo->getContextValue();
252 llvm_unreachable(
"No context value for inlined OpenMP region");
255 void setContextValue(llvm::Value *
V)
override {
256 if (OuterRegionInfo) {
257 OuterRegionInfo->setContextValue(
V);
260 llvm_unreachable(
"No context value for inlined OpenMP region");
266 return OuterRegionInfo->lookup(VD);
272 FieldDecl *getThisFieldDecl()
const override {
274 return OuterRegionInfo->getThisFieldDecl();
280 const VarDecl *getThreadIDVariable()
const override {
282 return OuterRegionInfo->getThreadIDVariable();
289 return OuterRegionInfo->getThreadIDVariableLValue(CGF);
290 llvm_unreachable(
"No LValue for inlined OpenMP construct");
294 StringRef getHelperName()
const override {
295 if (
auto *OuterRegionInfo = getOldCSI())
296 return OuterRegionInfo->getHelperName();
297 llvm_unreachable(
"No helper name for inlined OpenMP construct");
302 OuterRegionInfo->emitUntiedSwitch(CGF);
305 CodeGenFunction::CGCapturedStmtInfo *getOldCSI()
const {
return OldCSI; }
307 static bool classof(
const CGCapturedStmtInfo *Info) {
308 return CGOpenMPRegionInfo::classof(Info) &&
309 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == InlinedRegion;
312 ~CGOpenMPInlinedRegionInfo()
override =
default;
316 CodeGenFunction::CGCapturedStmtInfo *OldCSI;
317 CGOpenMPRegionInfo *OuterRegionInfo;
325class CGOpenMPTargetRegionInfo final :
public CGOpenMPRegionInfo {
329 : CGOpenMPRegionInfo(CS, TargetRegion, CodeGen, OMPD_target,
331 HelperName(HelperName) {}
335 const VarDecl *getThreadIDVariable()
const override {
return nullptr; }
338 StringRef getHelperName()
const override {
return HelperName; }
340 static bool classof(
const CGCapturedStmtInfo *Info) {
341 return CGOpenMPRegionInfo::classof(Info) &&
342 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == TargetRegion;
346 StringRef HelperName;
350 llvm_unreachable(
"No codegen for expressions");
354class CGOpenMPInnerExprInfo final :
public CGOpenMPInlinedRegionInfo {
357 : CGOpenMPInlinedRegionInfo(CGF.CapturedStmtInfo, EmptyCodeGen,
365 if (!
C.capturesVariable() && !
C.capturesVariableByCopy())
368 const VarDecl *VD =
C.getCapturedVar();
378 (void)PrivScope.Privatize();
383 if (
const FieldDecl *FD = CGOpenMPInlinedRegionInfo::lookup(VD))
390 llvm_unreachable(
"No body for expressions");
395 const VarDecl *getThreadIDVariable()
const override {
396 llvm_unreachable(
"No thread id for expressions");
400 StringRef getHelperName()
const override {
401 llvm_unreachable(
"No helper name for expressions");
404 static bool classof(
const CGCapturedStmtInfo *Info) {
return false; }
408 CodeGenFunction::OMPPrivateScope PrivScope;
412class InlinedOpenMPRegionRAII {
414 llvm::DenseMap<const ValueDecl *, FieldDecl *> LambdaCaptureFields;
415 FieldDecl *LambdaThisCaptureField =
nullptr;
417 bool NoInheritance =
false;
426 bool NoInheritance =
true)
427 : CGF(CGF), NoInheritance(NoInheritance) {
440 ~InlinedOpenMPRegionRAII() {
457enum OpenMPLocationFlags :
unsigned {
459 OMP_IDENT_IMD = 0x01,
461 OMP_IDENT_KMPC = 0x02,
463 OMP_ATOMIC_REDUCE = 0x10,
465 OMP_IDENT_BARRIER_EXPL = 0x20,
467 OMP_IDENT_BARRIER_IMPL = 0x40,
469 OMP_IDENT_BARRIER_IMPL_FOR = 0x40,
471 OMP_IDENT_BARRIER_IMPL_SECTIONS = 0xC0,
473 OMP_IDENT_BARRIER_IMPL_SINGLE = 0x140,
475 OMP_IDENT_WORK_LOOP = 0x200,
477 OMP_IDENT_WORK_SECTIONS = 0x400,
479 OMP_IDENT_WORK_DISTRIBUTE = 0x800,
509enum IdentFieldIndex {
511 IdentField_Reserved_1,
515 IdentField_Reserved_2,
517 IdentField_Reserved_3,
526enum OpenMPSchedType {
529 OMP_sch_static_chunked = 33,
531 OMP_sch_dynamic_chunked = 35,
532 OMP_sch_guided_chunked = 36,
533 OMP_sch_runtime = 37,
536 OMP_sch_static_balanced_chunked = 45,
539 OMP_ord_static_chunked = 65,
541 OMP_ord_dynamic_chunked = 67,
542 OMP_ord_guided_chunked = 68,
543 OMP_ord_runtime = 69,
545 OMP_sch_default = OMP_sch_static,
547 OMP_dist_sch_static_chunked = 91,
548 OMP_dist_sch_static = 92,
551 OMP_sch_modifier_monotonic = (1 << 29),
553 OMP_sch_modifier_nonmonotonic = (1 << 30),
576 Callback(CodeGen, CGF, *PrePostAction);
579 Callback(CodeGen, CGF, Action);
587 if (
const auto *CE = dyn_cast<CallExpr>(ReductionOp))
588 if (
const auto *OVE = dyn_cast<OpaqueValueExpr>(CE->getCallee()))
589 if (
const auto *DRE =
590 dyn_cast<DeclRefExpr>(OVE->getSourceExpr()->IgnoreImpCasts()))
591 if (
const auto *DRD = dyn_cast<OMPDeclareReductionDecl>(DRE->getDecl()))
602 std::pair<llvm::Function *, llvm::Function *>
Reduction =
604 const auto *CE = cast<CallExpr>(InitOp);
605 const auto *OVE = cast<OpaqueValueExpr>(CE->getCallee());
609 cast<DeclRefExpr>(cast<UnaryOperator>(LHS)->getSubExpr());
611 cast<DeclRefExpr>(cast<UnaryOperator>(RHS)->getSubExpr());
612 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
613 PrivateScope.addPrivate(cast<VarDecl>(LHSDRE->getDecl()),
Private);
614 PrivateScope.addPrivate(cast<VarDecl>(RHSDRE->getDecl()), Original);
615 (void)PrivateScope.Privatize();
617 CodeGenFunction::OpaqueValueMapping Map(CGF, OVE,
Func);
622 auto *GV =
new llvm::GlobalVariable(
624 llvm::GlobalValue::PrivateLinkage,
Init, Name);
637 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE, LV);
644 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE, InitRVal);
665 llvm::Value *NumElements = CGF.
emitArrayLength(ArrayTy, ElementTy, DestAddr);
669 llvm::Value *SrcBegin =
nullptr;
671 SrcBegin = SrcAddr.emitRawPointer(CGF);
674 llvm::Value *DestEnd =
679 llvm::Value *IsEmpty =
680 CGF.
Builder.CreateICmpEQ(DestBegin, DestEnd,
"omp.arrayinit.isempty");
681 CGF.
Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
684 llvm::BasicBlock *EntryBB = CGF.
Builder.GetInsertBlock();
689 llvm::PHINode *SrcElementPHI =
nullptr;
692 SrcElementPHI = CGF.
Builder.CreatePHI(SrcBegin->getType(), 2,
693 "omp.arraycpy.srcElementPast");
694 SrcElementPHI->addIncoming(SrcBegin, EntryBB);
696 Address(SrcElementPHI, SrcAddr.getElementType(),
697 SrcAddr.getAlignment().alignmentOfArrayElement(ElementSize));
699 llvm::PHINode *DestElementPHI = CGF.
Builder.CreatePHI(
700 DestBegin->getType(), 2,
"omp.arraycpy.destElementPast");
701 DestElementPHI->addIncoming(DestBegin, EntryBB);
708 CodeGenFunction::RunCleanupsScope InitScope(CGF);
709 if (EmitDeclareReductionInit) {
711 SrcElementCurrent, ElementTy);
719 llvm::Value *SrcElementNext = CGF.
Builder.CreateConstGEP1_32(
720 SrcAddr.getElementType(), SrcElementPHI, 1,
721 "omp.arraycpy.dest.element");
722 SrcElementPHI->addIncoming(SrcElementNext, CGF.
Builder.GetInsertBlock());
726 llvm::Value *DestElementNext = CGF.
Builder.CreateConstGEP1_32(
728 "omp.arraycpy.dest.element");
731 CGF.
Builder.CreateICmpEQ(DestElementNext, DestEnd,
"omp.arraycpy.done");
732 CGF.
Builder.CreateCondBr(Done, DoneBB, BodyBB);
733 DestElementPHI->addIncoming(DestElementNext, CGF.
Builder.GetInsertBlock());
745 if (
const auto *OASE = dyn_cast<ArraySectionExpr>(E))
750void ReductionCodeGen::emitAggregateInitialization(
756 const auto *PrivateVD =
757 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].
Private)->getDecl());
758 bool EmitDeclareReductionInit =
761 EmitDeclareReductionInit,
762 EmitDeclareReductionInit ? ClausesData[N].ReductionOp
763 : PrivateVD->getInit(),
771 ClausesData.reserve(Shareds.size());
772 SharedAddresses.reserve(Shareds.size());
773 Sizes.reserve(Shareds.size());
774 BaseDecls.reserve(Shareds.size());
775 const auto *IOrig = Origs.begin();
776 const auto *IPriv = Privates.begin();
777 const auto *IRed = ReductionOps.begin();
778 for (
const Expr *Ref : Shareds) {
779 ClausesData.emplace_back(Ref, *IOrig, *IPriv, *IRed);
780 std::advance(IOrig, 1);
781 std::advance(IPriv, 1);
782 std::advance(IRed, 1);
787 assert(SharedAddresses.size() == N && OrigAddresses.size() == N &&
788 "Number of generated lvalues must be exactly N.");
789 LValue First = emitSharedLValue(CGF, ClausesData[N].Shared);
790 LValue Second = emitSharedLValueUB(CGF, ClausesData[N].Shared);
791 SharedAddresses.emplace_back(
First, Second);
792 if (ClausesData[N].Shared == ClausesData[N].Ref) {
793 OrigAddresses.emplace_back(
First, Second);
795 LValue First = emitSharedLValue(CGF, ClausesData[N].Ref);
796 LValue Second = emitSharedLValueUB(CGF, ClausesData[N].Ref);
797 OrigAddresses.emplace_back(
First, Second);
803 bool AsArraySection = isa<ArraySectionExpr>(ClausesData[N].Ref);
806 CGF.
getTypeSize(OrigAddresses[N].first.getType().getNonReferenceType()),
811 llvm::Value *SizeInChars;
812 auto *ElemType = OrigAddresses[N].first.getAddress().getElementType();
813 auto *ElemSizeOf = llvm::ConstantExpr::getSizeOf(ElemType);
814 if (AsArraySection) {
815 Size = CGF.
Builder.CreatePtrDiff(ElemType,
816 OrigAddresses[N].second.getPointer(CGF),
817 OrigAddresses[N].first.getPointer(CGF));
818 Size = CGF.
Builder.CreateNUWAdd(
819 Size, llvm::ConstantInt::get(Size->getType(), 1));
820 SizeInChars = CGF.
Builder.CreateNUWMul(Size, ElemSizeOf);
823 CGF.
getTypeSize(OrigAddresses[N].first.getType().getNonReferenceType());
824 Size = CGF.
Builder.CreateExactUDiv(SizeInChars, ElemSizeOf);
826 Sizes.emplace_back(SizeInChars, Size);
829 cast<OpaqueValueExpr>(
839 assert(!Size && !Sizes[N].second &&
840 "Size should be nullptr for non-variably modified reduction "
846 cast<OpaqueValueExpr>(
855 assert(SharedAddresses.size() > N &&
"No variable was generated");
856 const auto *PrivateVD =
857 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].
Private)->getDecl());
862 (void)DefaultInit(CGF);
863 emitAggregateInitialization(CGF, N, PrivateAddr, SharedAddr, DRD);
864 }
else if (DRD && (DRD->
getInitializer() || !PrivateVD->hasInit())) {
865 (void)DefaultInit(CGF);
866 QualType SharedType = SharedAddresses[N].first.getType();
868 PrivateAddr, SharedAddr, SharedType);
869 }
else if (!DefaultInit(CGF) && PrivateVD->hasInit() &&
872 PrivateVD->
getType().getQualifiers(),
890 CGF.
pushDestroy(DTorKind, PrivateAddr, PrivateType);
914 Address OriginalBaseAddress, llvm::Value *Addr) {
938 Addr, OriginalBaseAddress.
getType());
943 const VarDecl *OrigVD =
nullptr;
944 if (
const auto *OASE = dyn_cast<ArraySectionExpr>(Ref)) {
945 const Expr *
Base = OASE->getBase()->IgnoreParenImpCasts();
946 while (
const auto *TempOASE = dyn_cast<ArraySectionExpr>(
Base))
947 Base = TempOASE->getBase()->IgnoreParenImpCasts();
948 while (
const auto *TempASE = dyn_cast<ArraySubscriptExpr>(
Base))
949 Base = TempASE->getBase()->IgnoreParenImpCasts();
950 DE = cast<DeclRefExpr>(
Base);
951 OrigVD = cast<VarDecl>(DE->
getDecl());
952 }
else if (
const auto *ASE = dyn_cast<ArraySubscriptExpr>(Ref)) {
953 const Expr *
Base = ASE->getBase()->IgnoreParenImpCasts();
954 while (
const auto *TempASE = dyn_cast<ArraySubscriptExpr>(
Base))
955 Base = TempASE->getBase()->IgnoreParenImpCasts();
956 DE = cast<DeclRefExpr>(
Base);
957 OrigVD = cast<VarDecl>(DE->
getDecl());
966 BaseDecls.emplace_back(OrigVD);
969 loadToBegin(CGF, OrigVD->getType(), SharedAddresses[N].first.getType(),
971 Address SharedAddr = SharedAddresses[N].first.getAddress();
972 llvm::Value *Adjustment = CGF.
Builder.CreatePtrDiff(
975 llvm::Value *PrivatePointer =
981 SharedAddresses[N].first.getType(),
984 BaseDecls.emplace_back(
985 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Ref)->getDecl()));
998 getThreadIDVariable()->getType()->castAs<PointerType>());
1016LValue CGOpenMPTaskOutlinedRegionInfo::getThreadIDVariableLValue(
1019 getThreadIDVariable()->getType(),
1035 : CGM(CGM), OMPBuilder(CGM.getModule()) {
1037 llvm::OpenMPIRBuilderConfig Config(
1052 OMPBuilder.Config.setHasRequiresUnifiedSharedMemory(
true);
1060 if (!
Data.getValue().pointsToAliveValue())
1062 auto *GV = dyn_cast<llvm::GlobalVariable>(
Data.getValue());
1065 if (!GV->isDeclaration() || GV->getNumUses() > 0)
1067 GV->eraseFromParent();
1072 return OMPBuilder.createPlatformSpecificName(Parts);
1075static llvm::Function *
1077 const Expr *CombinerInitializer,
const VarDecl *In,
1078 const VarDecl *Out,
bool IsCombiner) {
1081 QualType PtrTy =
C.getPointerType(Ty).withRestrict();
1087 Args.push_back(&OmpOutParm);
1088 Args.push_back(&OmpInParm);
1093 {IsCombiner ?
"omp_combiner" :
"omp_initializer",
""});
1094 auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,
1098 Fn->removeFnAttr(llvm::Attribute::NoInline);
1099 Fn->removeFnAttr(llvm::Attribute::OptimizeNone);
1100 Fn->addFnAttr(llvm::Attribute::AlwaysInline);
1106 Out->getLocation());
1107 CodeGenFunction::OMPPrivateScope
Scope(CGF);
1116 (void)
Scope.Privatize();
1117 if (!IsCombiner && Out->hasInit() &&
1120 Out->getType().getQualifiers(),
1123 if (CombinerInitializer)
1125 Scope.ForceCleanup();
1136 cast<VarDecl>(cast<DeclRefExpr>(D->
getCombinerIn())->getDecl()),
1137 cast<VarDecl>(cast<DeclRefExpr>(D->
getCombinerOut())->getDecl()),
1145 cast<VarDecl>(cast<DeclRefExpr>(D->
getInitOrig())->getDecl()),
1146 cast<VarDecl>(cast<DeclRefExpr>(D->
getInitPriv())->getDecl()),
1152 Decls.second.push_back(D);
1156std::pair<llvm::Function *, llvm::Function *>
1168struct PushAndPopStackRAII {
1169 PushAndPopStackRAII(llvm::OpenMPIRBuilder *OMPBuilder,
CodeGenFunction &CGF,
1170 bool HasCancel, llvm::omp::Directive Kind)
1171 : OMPBuilder(OMPBuilder) {
1187 auto FiniCB = [&CGF](llvm::OpenMPIRBuilder::InsertPointTy IP) {
1188 assert(IP.getBlock()->end() == IP.getPoint() &&
1189 "Clang CG should cause non-terminated block!");
1190 CGBuilderTy::InsertPointGuard IPG(CGF.
Builder);
1199 llvm::OpenMPIRBuilder::FinalizationInfo FI({FiniCB, Kind, HasCancel});
1200 OMPBuilder->pushFinalizationCB(std::move(FI));
1202 ~PushAndPopStackRAII() {
1204 OMPBuilder->popFinalizationCB();
1206 llvm::OpenMPIRBuilder *OMPBuilder;
1215 "thread id variable must be of type kmp_int32 *");
1217 bool HasCancel =
false;
1218 if (
const auto *OPD = dyn_cast<OMPParallelDirective>(&D))
1219 HasCancel = OPD->hasCancel();
1220 else if (
const auto *OPD = dyn_cast<OMPTargetParallelDirective>(&D))
1221 HasCancel = OPD->hasCancel();
1222 else if (
const auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&D))
1223 HasCancel = OPSD->hasCancel();
1224 else if (
const auto *OPFD = dyn_cast<OMPParallelForDirective>(&D))
1225 HasCancel = OPFD->hasCancel();
1226 else if (
const auto *OPFD = dyn_cast<OMPTargetParallelForDirective>(&D))
1227 HasCancel = OPFD->hasCancel();
1228 else if (
const auto *OPFD = dyn_cast<OMPDistributeParallelForDirective>(&D))
1229 HasCancel = OPFD->hasCancel();
1230 else if (
const auto *OPFD =
1231 dyn_cast<OMPTeamsDistributeParallelForDirective>(&D))
1232 HasCancel = OPFD->hasCancel();
1233 else if (
const auto *OPFD =
1234 dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&D))
1235 HasCancel = OPFD->hasCancel();
1240 PushAndPopStackRAII PSR(&OMPBuilder, CGF, HasCancel, InnermostKind);
1241 CGOpenMPOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen, InnermostKind,
1242 HasCancel, OutlinedHelperName);
1243 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
1248 std::string Suffix =
getName({
"omp_outlined"});
1249 return (Name + Suffix).str();
1257 std::string Suffix =
getName({
"omp",
"reduction",
"reduction_func"});
1258 return (Name + Suffix).str();
1285 bool Tied,
unsigned &NumberOfParts) {
1290 llvm::Value *TaskArgs[] = {
1292 CGF.EmitLoadOfPointerLValue(CGF.GetAddrOfLocalVar(TaskTVar),
1295 CGF.EmitRuntimeCall(
OMPBuilder.getOrCreateRuntimeFunction(
1299 CGOpenMPTaskOutlinedRegionInfo::UntiedTaskActionTy Action(Tied, PartIDVar,
1303 "thread id variable must be of type kmp_int32 for tasks");
1308 bool HasCancel =
false;
1309 if (
const auto *TD = dyn_cast<OMPTaskDirective>(&D))
1310 HasCancel = TD->hasCancel();
1311 else if (
const auto *TD = dyn_cast<OMPTaskLoopDirective>(&D))
1312 HasCancel = TD->hasCancel();
1313 else if (
const auto *TD = dyn_cast<OMPMasterTaskLoopDirective>(&D))
1314 HasCancel = TD->hasCancel();
1315 else if (
const auto *TD = dyn_cast<OMPParallelMasterTaskLoopDirective>(&D))
1316 HasCancel = TD->hasCancel();
1319 CGOpenMPTaskOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen,
1320 InnermostKind, HasCancel, Action);
1322 llvm::Function *Res = CGF.GenerateCapturedStmtFunction(*CS);
1324 NumberOfParts = Action.getNumberOfParts();
1329 bool AtCurrentPoint) {
1331 assert(!Elem.second.ServiceInsertPt &&
"Insert point is set already.");
1333 llvm::Value *Undef = llvm::UndefValue::get(CGF.
Int32Ty);
1334 if (AtCurrentPoint) {
1335 Elem.second.ServiceInsertPt =
new llvm::BitCastInst(
1338 Elem.second.ServiceInsertPt =
1339 new llvm::BitCastInst(Undef, CGF.
Int32Ty,
"svcpt");
1346 if (Elem.second.ServiceInsertPt) {
1347 llvm::Instruction *Ptr = Elem.second.ServiceInsertPt;
1348 Elem.second.ServiceInsertPt =
nullptr;
1349 Ptr->eraseFromParent();
1356 llvm::raw_svector_ostream OS(Buffer);
1360 if (
const auto *FD = dyn_cast_or_null<FunctionDecl>(CGF.
CurFuncDecl))
1361 OS << FD->getQualifiedNameAsString();
1368 unsigned Flags,
bool EmitLoc) {
1369 uint32_t SrcLocStrSize;
1370 llvm::Constant *SrcLocStr;
1372 llvm::codegenoptions::NoDebugInfo) ||
1374 SrcLocStr =
OMPBuilder.getOrCreateDefaultSrcLocStr(SrcLocStrSize);
1376 std::string FunctionName;
1377 if (
const auto *FD = dyn_cast_or_null<FunctionDecl>(CGF.
CurFuncDecl))
1378 FunctionName = FD->getQualifiedNameAsString();
1388 SrcLocStr, SrcLocStrSize, llvm::omp::IdentFlag(Flags), Reserved2Flags);
1393 assert(CGF.
CurFn &&
"No function in current CodeGenFunction.");
1399 uint32_t SrcLocStrSize;
1400 auto *SrcLocStr =
OMPBuilder.getOrCreateSrcLocStr(
1403 OMPBuilder.getOrCreateIdent(SrcLocStr, SrcLocStrSize));
1406 llvm::Value *ThreadID =
nullptr;
1411 ThreadID = I->second.ThreadID;
1412 if (ThreadID !=
nullptr)
1416 if (
auto *OMPRegionInfo =
1418 if (OMPRegionInfo->getThreadIDVariable()) {
1420 LValue LVal = OMPRegionInfo->getThreadIDVariableLValue(CGF);
1424 CGF.
Builder.GetInsertBlock() == TopBlock ||
1425 !isa<llvm::Instruction>(LVal.
getPointer(CGF)) ||
1426 cast<llvm::Instruction>(LVal.
getPointer(CGF))->getParent() ==
1428 cast<llvm::Instruction>(LVal.
getPointer(CGF))->getParent() ==
1429 CGF.
Builder.GetInsertBlock()) {
1433 if (CGF.
Builder.GetInsertBlock() == TopBlock) {
1435 Elem.second.ThreadID = ThreadID;
1447 if (!Elem.second.ServiceInsertPt)
1449 CGBuilderTy::InsertPointGuard IPG(CGF.
Builder);
1450 CGF.
Builder.SetInsertPoint(Elem.second.ServiceInsertPt);
1454 OMPRTL___kmpc_global_thread_num),
1457 Elem.second.ThreadID =
Call;
1462 assert(CGF.
CurFn &&
"No function in current CodeGenFunction.");
1474 for(
const auto *D : I->second)
1489 llvm::Type *MicroParams[] = {llvm::PointerType::getUnqual(
CGM.
Int32Ty),
1496llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseKind
1498 std::optional<OMPDeclareTargetDeclAttr::DevTypeTy> DevTy =
1499 OMPDeclareTargetDeclAttr::getDeviceType(VD);
1501 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseNone;
1503 switch ((
int)*DevTy) {
1504 case OMPDeclareTargetDeclAttr::DT_Host:
1505 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseHost;
1507 case OMPDeclareTargetDeclAttr::DT_NoHost:
1508 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseNoHost;
1510 case OMPDeclareTargetDeclAttr::DT_Any:
1511 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseAny;
1514 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseNone;
1519llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind
1521 std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> MapType =
1522 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
1524 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryNone;
1525 switch ((
int)*MapType) {
1526 case OMPDeclareTargetDeclAttr::MapTypeTy::MT_To:
1527 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo;
1529 case OMPDeclareTargetDeclAttr::MapTypeTy::MT_Enter:
1530 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryEnter;
1532 case OMPDeclareTargetDeclAttr::MapTypeTy::MT_Link:
1533 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryLink;
1536 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryNone;
1545 auto FileInfoCallBack = [&]() {
1549 llvm::sys::fs::UniqueID ID;
1550 if (llvm::sys::fs::getUniqueID(PLoc.
getFilename(), ID)) {
1551 PLoc =
SM.getPresumedLoc(BeginLoc,
false);
1557 return OMPBuilder.getTargetEntryUniqueInfo(FileInfoCallBack, ParentName);
1563 auto LinkageForVariable = [&VD,
this]() {
1567 std::vector<llvm::GlobalVariable *> GeneratedRefs;
1571 llvm::Constant *addr =
OMPBuilder.getAddrOfDeclareTargetVar(
1579 LinkageForVariable);
1591 std::string Suffix =
getName({
"cache",
""});
1592 return OMPBuilder.getOrCreateInternalVariable(
1605 llvm::Value *Args[] = {
1629 llvm::Value *Args[] = {
1632 Ctor, CopyCtor, Dtor};
1635 CGM.
getModule(), OMPRTL___kmpc_threadprivate_register),
1650 llvm::Value *Ctor =
nullptr, *CopyCtor =
nullptr, *Dtor =
nullptr;
1660 Args.push_back(&Dst);
1665 std::string Name =
getName({
"__kmpc_global_ctor_",
""});
1666 llvm::Function *Fn =
1692 Args.push_back(&Dst);
1697 std::string Name =
getName({
"__kmpc_global_dtor_",
""});
1698 llvm::Function *Fn =
1720 auto *CopyCtorTy = llvm::FunctionType::get(
CGM.
VoidPtrTy, CopyCtorTyArgs,
1726 CopyCtor = llvm::Constant::getNullValue(CopyCtorTy);
1727 if (Ctor ==
nullptr) {
1731 Ctor = llvm::Constant::getNullValue(CtorTy);
1733 if (Dtor ==
nullptr) {
1737 Dtor = llvm::Constant::getNullValue(DtorTy);
1740 auto *InitFunctionTy =
1741 llvm::FunctionType::get(
CGM.
VoidTy,
false);
1742 std::string Name =
getName({
"__omp_threadprivate_init_",
""});
1752 return InitFunction;
1760 llvm::GlobalValue *GV) {
1761 std::optional<OMPDeclareTargetDeclAttr *> ActiveAttr =
1762 OMPDeclareTargetDeclAttr::getActiveAttr(FD);
1765 if (!ActiveAttr || !(*ActiveAttr)->getIndirect())
1772 OMPBuilder.OffloadInfoManager.getTargetRegionEntryFnName(Name, EntryInfo);
1778 llvm::GlobalValue *Addr = GV;
1780 Addr =
new llvm::GlobalVariable(
1782 true, llvm::GlobalValue::ExternalLinkage, GV, Name,
1783 nullptr, llvm::GlobalValue::NotThreadLocal,
1784 CGM.
getModule().getDataLayout().getDefaultGlobalsAddressSpace());
1785 Addr->setVisibility(llvm::GlobalValue::ProtectedVisibility);
1788 OMPBuilder.OffloadInfoManager.registerDeviceGlobalVarEntryInfo(
1790 llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirect,
1791 llvm::GlobalValue::WeakODRLinkage);
1797 std::string Suffix =
getName({
"artificial",
""});
1799 llvm::GlobalVariable *GAddr =
OMPBuilder.getOrCreateInternalVariable(
1800 VarLVType, Twine(Name).concat(Suffix).str());
1803 GAddr->setThreadLocal(
true);
1804 return Address(GAddr, GAddr->getValueType(),
1807 std::string CacheSuffix =
getName({
"cache",
""});
1808 llvm::Value *Args[] = {
1816 Twine(Name).concat(Suffix).concat(CacheSuffix).str())};
1823 VarLVType->getPointerTo(0)),
1867 llvm::Function *OutlinedFn,
1870 llvm::Value *NumThreads) {
1875 auto &&ThenGen = [&M, OutlinedFn, CapturedVars, RTLoc,
1879 llvm::Value *Args[] = {
1881 CGF.
Builder.getInt32(CapturedVars.size()),
1884 RealArgs.append(std::begin(Args), std::end(Args));
1885 RealArgs.append(CapturedVars.begin(), CapturedVars.end());
1887 llvm::FunctionCallee RTLFn =
1888 OMPBuilder.getOrCreateRuntimeFunction(M, OMPRTL___kmpc_fork_call);
1891 auto &&ElseGen = [&M, OutlinedFn, CapturedVars, RTLoc,
Loc,
1897 llvm::Value *Args[] = {RTLoc, ThreadID};
1899 M, OMPRTL___kmpc_serialized_parallel),
1906 ".bound.zero.addr");
1911 OutlinedFnArgs.push_back(ZeroAddrBound.
getPointer());
1912 OutlinedFnArgs.append(CapturedVars.begin(), CapturedVars.end());
1920 OutlinedFn->removeFnAttr(llvm::Attribute::AlwaysInline);
1921 OutlinedFn->addFnAttr(llvm::Attribute::NoInline);
1927 M, OMPRTL___kmpc_end_serialized_parallel),
1946 if (
auto *OMPRegionInfo =
1948 if (OMPRegionInfo->getThreadIDVariable())
1949 return OMPRegionInfo->getThreadIDVariableLValue(CGF).getAddress();
1958 return ThreadIDTemp;
1962 std::string Prefix = Twine(
"gomp_critical_user_", CriticalName).str();
1963 std::string Name =
getName({Prefix,
"var"});
1970 llvm::FunctionCallee EnterCallee;
1972 llvm::FunctionCallee ExitCallee;
1975 llvm::BasicBlock *ContBlock =
nullptr;
1978 CommonActionTy(llvm::FunctionCallee EnterCallee,
1980 llvm::FunctionCallee ExitCallee,
1982 : EnterCallee(EnterCallee), EnterArgs(EnterArgs), ExitCallee(ExitCallee),
1983 ExitArgs(ExitArgs), Conditional(Conditional) {}
1987 llvm::Value *CallBool = CGF.
Builder.CreateIsNotNull(EnterRes);
1991 CGF.
Builder.CreateCondBr(CallBool, ThenBlock, ContBlock);
2007 StringRef CriticalName,
2021 EnterArgs.push_back(CGF.
Builder.CreateIntCast(
2024 CommonActionTy Action(
2027 Hint ? OMPRTL___kmpc_critical_with_hint : OMPRTL___kmpc_critical),
2030 OMPRTL___kmpc_end_critical),
2047 CommonActionTy Action(
OMPBuilder.getOrCreateRuntimeFunction(
2069 llvm::Value *FilterVal = Filter
2076 CommonActionTy Action(
OMPBuilder.getOrCreateRuntimeFunction(
2096 llvm::Value *Args[] = {
2098 llvm::ConstantInt::get(
CGM.
IntTy, 0,
true)};
2104 if (
auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.
CapturedStmtInfo))
2105 Region->emitUntiedSwitch(CGF);
2118 CommonActionTy Action(
OMPBuilder.getOrCreateRuntimeFunction(
2131 unsigned Index,
const VarDecl *Var) {
2139 Ptr, ElemTy->getPointerTo(Ptr->getType()->getPointerAddressSpace())),
2155 Args.push_back(&LHSArg);
2156 Args.push_back(&RHSArg);
2162 llvm::GlobalValue::InternalLinkage, Name,
2165 Fn->setDoesNotRecurse();
2172 ArgsElemType->getPointerTo()),
2176 ArgsElemType->getPointerTo()),
2182 for (
unsigned I = 0, E = AssignmentOps.size(); I < E; ++I) {
2183 const auto *DestVar =
2184 cast<VarDecl>(cast<DeclRefExpr>(DestExprs[I])->getDecl());
2187 const auto *SrcVar =
2188 cast<VarDecl>(cast<DeclRefExpr>(SrcExprs[I])->getDecl());
2191 const auto *VD = cast<DeclRefExpr>(CopyprivateVars[I])->getDecl();
2193 CGF.
EmitOMPCopy(
Type, DestAddr, SrcAddr, DestVar, SrcVar, AssignmentOps[I]);
2208 assert(CopyprivateVars.size() == SrcExprs.size() &&
2209 CopyprivateVars.size() == DstExprs.size() &&
2210 CopyprivateVars.size() == AssignmentOps.size());
2222 if (!CopyprivateVars.empty()) {
2225 C.getIntTypeForBitwidth(32, 1);
2226 DidIt = CGF.
CreateMemTemp(KmpInt32Ty,
".omp.copyprivate.did_it");
2231 CommonActionTy Action(
OMPBuilder.getOrCreateRuntimeFunction(
2248 llvm::APInt ArraySize(32, CopyprivateVars.size());
2249 QualType CopyprivateArrayTy =
C.getConstantArrayType(
2254 CGF.
CreateMemTemp(CopyprivateArrayTy,
".omp.copyprivate.cpr_list");
2255 for (
unsigned I = 0, E = CopyprivateVars.size(); I < E; ++I) {
2267 SrcExprs, DstExprs, AssignmentOps,
Loc);
2268 llvm::Value *BufSize = CGF.
getTypeSize(CopyprivateArrayTy);
2272 llvm::Value *Args[] = {
2297 CommonActionTy Action(
OMPBuilder.getOrCreateRuntimeFunction(
2312 if (Kind == OMPD_for)
2313 Flags = OMP_IDENT_BARRIER_IMPL_FOR;
2314 else if (Kind == OMPD_sections)
2315 Flags = OMP_IDENT_BARRIER_IMPL_SECTIONS;
2316 else if (Kind == OMPD_single)
2317 Flags = OMP_IDENT_BARRIER_IMPL_SINGLE;
2318 else if (Kind == OMPD_barrier)
2319 Flags = OMP_IDENT_BARRIER_EXPL;
2321 Flags = OMP_IDENT_BARRIER_IMPL;
2333 ScheduleKind = OMPC_SCHEDULE_static;
2335 llvm::APInt ChunkSize(32, 1);
2345 bool ForceSimpleCall) {
2347 auto *OMPRegionInfo =
2351 CGF.
Builder, Kind, ForceSimpleCall, EmitChecks));
2364 if (OMPRegionInfo) {
2365 if (!ForceSimpleCall && OMPRegionInfo->hasCancel()) {
2368 OMPRTL___kmpc_cancel_barrier),
2377 CGF.
Builder.CreateCondBr(Cmp, ExitBB, ContBB);
2394 Expr *ME,
bool IsFatal) {
2397 : llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
2400 llvm::Value *Args[] = {
2402 llvm::ConstantInt::get(
CGM.
Int32Ty, IsFatal ? 2 : 1),
2411 bool Chunked,
bool Ordered) {
2412 switch (ScheduleKind) {
2413 case OMPC_SCHEDULE_static:
2414 return Chunked ? (Ordered ? OMP_ord_static_chunked : OMP_sch_static_chunked)
2415 : (Ordered ? OMP_ord_static : OMP_sch_static);
2416 case OMPC_SCHEDULE_dynamic:
2417 return Ordered ? OMP_ord_dynamic_chunked : OMP_sch_dynamic_chunked;
2418 case OMPC_SCHEDULE_guided:
2419 return Ordered ? OMP_ord_guided_chunked : OMP_sch_guided_chunked;
2420 case OMPC_SCHEDULE_runtime:
2421 return Ordered ? OMP_ord_runtime : OMP_sch_runtime;
2422 case OMPC_SCHEDULE_auto:
2423 return Ordered ? OMP_ord_auto : OMP_sch_auto;
2425 assert(!Chunked &&
"chunk was specified but schedule kind not known");
2426 return Ordered ? OMP_ord_static : OMP_sch_static;
2428 llvm_unreachable(
"Unexpected runtime schedule");
2432static OpenMPSchedType
2435 return Chunked ? OMP_dist_sch_static_chunked : OMP_dist_sch_static;
2439 bool Chunked)
const {
2440 OpenMPSchedType Schedule =
2442 return Schedule == OMP_sch_static;
2448 return Schedule == OMP_dist_sch_static;
2452 bool Chunked)
const {
2453 OpenMPSchedType Schedule =
2455 return Schedule == OMP_sch_static_chunked;
2461 return Schedule == OMP_dist_sch_static_chunked;
2465 OpenMPSchedType Schedule =
2467 assert(Schedule != OMP_sch_static_chunked &&
"cannot be chunked here");
2468 return Schedule != OMP_sch_static;
2476 case OMPC_SCHEDULE_MODIFIER_monotonic:
2477 Modifier = OMP_sch_modifier_monotonic;
2479 case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
2480 Modifier = OMP_sch_modifier_nonmonotonic;
2482 case OMPC_SCHEDULE_MODIFIER_simd:
2483 if (Schedule == OMP_sch_static_chunked)
2484 Schedule = OMP_sch_static_balanced_chunked;
2491 case OMPC_SCHEDULE_MODIFIER_monotonic:
2492 Modifier = OMP_sch_modifier_monotonic;
2494 case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
2495 Modifier = OMP_sch_modifier_nonmonotonic;
2497 case OMPC_SCHEDULE_MODIFIER_simd:
2498 if (Schedule == OMP_sch_static_chunked)
2499 Schedule = OMP_sch_static_balanced_chunked;
2511 if (CGM.
getLangOpts().OpenMP >= 50 && Modifier == 0) {
2512 if (!(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static ||
2513 Schedule == OMP_sch_static_balanced_chunked ||
2514 Schedule == OMP_ord_static_chunked || Schedule == OMP_ord_static ||
2515 Schedule == OMP_dist_sch_static_chunked ||
2516 Schedule == OMP_dist_sch_static))
2517 Modifier = OMP_sch_modifier_nonmonotonic;
2519 return Schedule | Modifier;
2529 ScheduleKind.
Schedule, DispatchValues.
Chunk !=
nullptr, Ordered);
2531 (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked &&
2532 Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked &&
2533 Schedule != OMP_sch_static_balanced_chunked));
2540 llvm::Value *Chunk = DispatchValues.
Chunk ? DispatchValues.
Chunk
2541 : CGF.
Builder.getIntN(IVSize, 1);
2542 llvm::Value *Args[] = {
2546 CGM, Schedule, ScheduleKind.
M1, ScheduleKind.
M2)),
2549 CGF.
Builder.getIntN(IVSize, 1),
2557 CodeGenFunction &CGF, llvm::Value *UpdateLocation, llvm::Value *ThreadId,
2558 llvm::FunctionCallee ForStaticInitFunction, OpenMPSchedType Schedule,
2565 assert(Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked ||
2566 Schedule == OMP_sch_static_balanced_chunked ||
2567 Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked ||
2568 Schedule == OMP_dist_sch_static ||
2569 Schedule == OMP_dist_sch_static_chunked);
2576 llvm::Value *Chunk = Values.
Chunk;
2577 if (Chunk ==
nullptr) {
2578 assert((Schedule == OMP_sch_static || Schedule == OMP_ord_static ||
2579 Schedule == OMP_dist_sch_static) &&
2580 "expected static non-chunked schedule");
2584 assert((Schedule == OMP_sch_static_chunked ||
2585 Schedule == OMP_sch_static_balanced_chunked ||
2586 Schedule == OMP_ord_static_chunked ||
2587 Schedule == OMP_dist_sch_static_chunked) &&
2588 "expected static chunked schedule");
2590 llvm::Value *Args[] = {
2613 "Expected loop-based or sections-based directive.");
2616 ? OMP_IDENT_WORK_LOOP
2617 : OMP_IDENT_WORK_SECTIONS);
2619 llvm::FunctionCallee StaticInitFunction =
2624 ScheduleNum, ScheduleKind.
M1, ScheduleKind.
M2, Values);
2631 OpenMPSchedType ScheduleNum =
2633 llvm::Value *UpdatedLocation =
2636 llvm::FunctionCallee StaticInitFunction;
2637 bool isGPUDistribute =
2640 StaticInitFunction =
OMPBuilder.createForStaticInitFunction(
2651 assert((DKind == OMPD_distribute || DKind == OMPD_for ||
2652 DKind == OMPD_sections) &&
2653 "Expected distribute, for, or sections directive kind");
2657 llvm::Value *Args[] = {
2660 (DKind == OMPD_target_teams_loop)
2661 ? OMP_IDENT_WORK_DISTRIBUTE
2663 ? OMP_IDENT_WORK_LOOP
2664 : OMP_IDENT_WORK_SECTIONS),
2672 CGM.
getModule(), OMPRTL___kmpc_distribute_static_fini),
2701 llvm::Value *Args[] = {
2709 OMPBuilder.createDispatchNextFunction(IVSize, IVSigned), Args);
2716 llvm::Value *NumThreads,
2721 llvm::Value *Args[] = {
2730 ProcBindKind ProcBind,
2734 assert(ProcBind != OMP_PROC_BIND_unknown &&
"Unsupported proc_bind value.");
2736 llvm::Value *Args[] = {
2738 llvm::ConstantInt::get(
CGM.
IntTy,
unsigned(ProcBind),
true)};
2760enum KmpTaskTFields {
2790 llvm::OpenMPIRBuilder::EmitMetadataErrorReportFunctionTy &&ErrorReportFn =
2791 [
this](llvm::OpenMPIRBuilder::EmitMetadataErrorKind Kind,
2792 const llvm::TargetRegionEntryInfo &EntryInfo) ->
void {
2794 if (Kind != llvm::OpenMPIRBuilder::EMIT_MD_GLOBAL_VAR_LINK_ERROR) {
2798 if (I->getFirst().getUniqueID().getDevice() == EntryInfo.DeviceID &&
2799 I->getFirst().getUniqueID().getFile() == EntryInfo.FileID) {
2801 I->getFirst(), EntryInfo.Line, 1);
2807 case llvm::OpenMPIRBuilder::EMIT_MD_TARGET_REGION_ERROR: {
2810 "%0 is incorrect: either the "
2811 "address or the ID is invalid.");
2814 case llvm::OpenMPIRBuilder::EMIT_MD_DECLARE_TARGET_ERROR: {
2817 "variable %0 is incorrect: the "
2818 "address is invalid.");
2821 case llvm::OpenMPIRBuilder::EMIT_MD_GLOBAL_VAR_LINK_ERROR: {
2824 "Offloading entry for declare target variable is incorrect: the "
2825 "address is invalid.");
2831 OMPBuilder.createOffloadEntriesAndInfoMetadata(ErrorReportFn);
2838 QualType KmpRoutineEntryTyArgs[] = {KmpInt32Ty,
C.VoidPtrTy};
2841 C.getFunctionType(KmpInt32Ty, KmpRoutineEntryTyArgs, EPI));
2847struct PrivateHelpersTy {
2848 PrivateHelpersTy(
const Expr *OriginalRef,
const VarDecl *Original,
2850 : OriginalRef(OriginalRef), Original(Original), PrivateCopy(PrivateCopy),
2851 PrivateElemInit(PrivateElemInit) {}
2852 PrivateHelpersTy(
const VarDecl *Original) : Original(Original) {}
2853 const Expr *OriginalRef =
nullptr;
2854 const VarDecl *Original =
nullptr;
2855 const VarDecl *PrivateCopy =
nullptr;
2856 const VarDecl *PrivateElemInit =
nullptr;
2857 bool isLocalPrivate()
const {
2858 return !OriginalRef && !PrivateCopy && !PrivateElemInit;
2861typedef std::pair<
CharUnits , PrivateHelpersTy> PrivateDataTy;
2866 if (!CVD->
hasAttr<OMPAllocateDeclAttr>())
2868 const auto *AA = CVD->
getAttr<OMPAllocateDeclAttr>();
2870 return !(AA->getAllocatorType() == OMPAllocateDeclAttr::OMPDefaultMemAlloc &&
2871 !AA->getAllocator());
2876 if (!Privates.empty()) {
2881 RecordDecl *RD =
C.buildImplicitRecord(
".kmp_privates.t");
2883 for (
const auto &Pair : Privates) {
2884 const VarDecl *VD = Pair.second.Original;
2888 if (Pair.second.isLocalPrivate()) {
2911 QualType KmpRoutineEntryPointerQTy) {
2931 QualType KmpCmplrdataTy =
C.getRecordType(UD);
2932 RecordDecl *RD =
C.buildImplicitRecord(
"kmp_task_t");
2962 RecordDecl *RD =
C.buildImplicitRecord(
"kmp_task_t_with_privates");
2982static llvm::Function *
2985 QualType KmpTaskTWithPrivatesPtrQTy,
2987 QualType SharedsPtrTy, llvm::Function *TaskFunction,
2988 llvm::Value *TaskPrivatesMap) {
2996 Args.push_back(&GtidArg);
2997 Args.push_back(&TaskTypeArg);
2998 const auto &TaskEntryFnInfo =
3000 llvm::FunctionType *TaskEntryTy =
3003 auto *TaskEntry = llvm::Function::Create(
3004 TaskEntryTy, llvm::GlobalValue::InternalLinkage, Name, &CGM.
getModule());
3006 TaskEntry->setDoesNotRecurse();
3021 const auto *KmpTaskTWithPrivatesQTyRD =
3022 cast<RecordDecl>(KmpTaskTWithPrivatesQTy->
getAsTagDecl());
3025 const auto *KmpTaskTQTyRD = cast<RecordDecl>(KmpTaskTQTy->
getAsTagDecl());
3026 auto PartIdFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTPartId);
3028 llvm::Value *PartidParam = PartIdLVal.
getPointer(CGF);
3030 auto SharedsFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTShareds);
3036 auto PrivatesFI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin(), 1);
3037 llvm::Value *PrivatesParam;
3038 if (PrivatesFI != KmpTaskTWithPrivatesQTyRD->field_end()) {
3043 PrivatesParam = llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
3046 llvm::Value *CommonArgs[] = {
3047 GtidParam, PartidParam, PrivatesParam, TaskPrivatesMap,
3053 std::end(CommonArgs));
3055 auto LBFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLowerBound);
3058 auto UBFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTUpperBound);
3061 auto StFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTStride);
3064 auto LIFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLastIter);
3067 auto RFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTReductions);
3070 CallArgs.push_back(LBParam);
3071 CallArgs.push_back(UBParam);
3072 CallArgs.push_back(StParam);
3073 CallArgs.push_back(LIParam);
3074 CallArgs.push_back(RParam);
3076 CallArgs.push_back(SharedsParam);
3089 QualType KmpTaskTWithPrivatesPtrQTy,
3090 QualType KmpTaskTWithPrivatesQTy) {
3098 Args.push_back(&GtidArg);
3099 Args.push_back(&TaskTypeArg);
3100 const auto &DestructorFnInfo =
3102 llvm::FunctionType *DestructorFnTy =
3106 auto *DestructorFn =
3107 llvm::Function::Create(DestructorFnTy, llvm::GlobalValue::InternalLinkage,
3111 DestructorFn->setDoesNotRecurse();
3119 const auto *KmpTaskTWithPrivatesQTyRD =
3120 cast<RecordDecl>(KmpTaskTWithPrivatesQTy->
getAsTagDecl());
3121 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
3123 for (
const auto *Field :
3124 cast<RecordDecl>(FI->getType()->getAsTagDecl())->fields()) {
3126 Field->getType().isDestructedType()) {
3132 return DestructorFn;
3152 C,
nullptr,
Loc,
nullptr,
3153 C.getPointerType(PrivatesQTy).withConst().withRestrict(),
3155 Args.push_back(&TaskPrivatesArg);
3156 llvm::DenseMap<CanonicalDeclPtr<const VarDecl>,
unsigned> PrivateVarsPos;
3157 unsigned Counter = 1;
3158 for (
const Expr *E :
Data.PrivateVars) {
3160 C,
nullptr,
Loc,
nullptr,
3161 C.getPointerType(
C.getPointerType(E->
getType()))
3165 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3166 PrivateVarsPos[VD] = Counter;
3169 for (
const Expr *E :
Data.FirstprivateVars) {
3171 C,
nullptr,
Loc,
nullptr,
3172 C.getPointerType(
C.getPointerType(E->
getType()))
3176 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3177 PrivateVarsPos[VD] = Counter;
3180 for (
const Expr *E :
Data.LastprivateVars) {
3182 C,
nullptr,
Loc,
nullptr,
3183 C.getPointerType(
C.getPointerType(E->
getType()))
3187 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3188 PrivateVarsPos[VD] = Counter;
3194 Ty =
C.getPointerType(Ty);
3196 Ty =
C.getPointerType(Ty);
3198 C,
nullptr,
Loc,
nullptr,
3199 C.getPointerType(
C.getPointerType(Ty)).withConst().withRestrict(),
3201 PrivateVarsPos[VD] = Counter;
3204 const auto &TaskPrivatesMapFnInfo =
3206 llvm::FunctionType *TaskPrivatesMapTy =
3210 auto *TaskPrivatesMap = llvm::Function::Create(
3211 TaskPrivatesMapTy, llvm::GlobalValue::InternalLinkage, Name,
3214 TaskPrivatesMapFnInfo);
3216 TaskPrivatesMap->removeFnAttr(llvm::Attribute::NoInline);
3217 TaskPrivatesMap->removeFnAttr(llvm::Attribute::OptimizeNone);
3218 TaskPrivatesMap->addFnAttr(llvm::Attribute::AlwaysInline);
3222 TaskPrivatesMapFnInfo, Args,
Loc,
Loc);
3228 const auto *PrivatesQTyRD = cast<RecordDecl>(PrivatesQTy->
getAsTagDecl());
3230 for (
const FieldDecl *Field : PrivatesQTyRD->fields()) {
3232 const VarDecl *VD = Args[PrivateVarsPos[Privates[Counter].second.Original]];
3241 return TaskPrivatesMap;
3253 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->
field_begin());
3259 CodeGenFunction::CGCapturedStmtInfo CapturesInfo(CS);
3267 if ((!IsTargetTask && !
Data.FirstprivateVars.empty() && ForDup) ||
3268 (IsTargetTask && KmpTaskSharedsPtr.
isValid())) {
3275 FI = cast<RecordDecl>(FI->getType()->getAsTagDecl())->field_begin();
3276 for (
const PrivateDataTy &Pair : Privates) {
3278 if (Pair.second.isLocalPrivate()) {
3282 const VarDecl *VD = Pair.second.PrivateCopy;
3284 if (
Init && (!ForDup || (isa<CXXConstructExpr>(
Init) &&
3287 if (
const VarDecl *Elem = Pair.second.PrivateElemInit) {
3288 const VarDecl *OriginalVD = Pair.second.Original;
3293 const FieldDecl *SharedField = CapturesInfo.lookup(OriginalVD);
3294 if (IsTargetTask && !SharedField) {
3295 assert(isa<ImplicitParamDecl>(OriginalVD) &&
3298 ->getNumParams() == 0 &&
3299 isa<TranslationUnitDecl>(
3301 ->getDeclContext()) &&
3302 "Expected artificial target data variable.");
3305 }
else if (ForDup) {
3309 C.getDeclAlign(OriginalVD)),
3313 Pair.second.Original->getCanonicalDecl()) > 0 ||
3315 SharedRefLValue = CGF.
EmitLValue(Pair.second.OriginalRef);
3318 InlinedOpenMPRegionRAII Region(
3321 SharedRefLValue = CGF.
EmitLValue(Pair.second.OriginalRef);
3333 [&CGF, Elem,
Init, &CapturesInfo](
Address DestElement,
3336 CodeGenFunction::OMPPrivateScope InitScope(CGF);
3337 InitScope.addPrivate(Elem, SrcElement);
3338 (void)InitScope.Privatize();
3340 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(
3341 CGF, &CapturesInfo);
3342 CGF.EmitAnyExprToMem(Init, DestElement,
3343 Init->getType().getQualifiers(),
3348 CodeGenFunction::OMPPrivateScope InitScope(CGF);
3349 InitScope.addPrivate(Elem, SharedRefLValue.
getAddress());
3350 (void)InitScope.Privatize();
3351 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CapturesInfo);
3366 bool InitRequired =
false;
3367 for (
const PrivateDataTy &Pair : Privates) {
3368 if (Pair.second.isLocalPrivate())
3370 const VarDecl *VD = Pair.second.PrivateCopy;
3372 InitRequired = InitRequired || (isa_and_nonnull<CXXConstructExpr>(
Init) &&
3377 return InitRequired;
3394 QualType KmpTaskTWithPrivatesPtrQTy,
3402 KmpTaskTWithPrivatesPtrQTy,
3405 KmpTaskTWithPrivatesPtrQTy,
3409 Args.push_back(&DstArg);
3410 Args.push_back(&SrcArg);
3411 Args.push_back(&LastprivArg);
3412 const auto &TaskDupFnInfo =
3416 auto *TaskDup = llvm::Function::Create(
3417 TaskDupTy, llvm::GlobalValue::InternalLinkage, Name, &CGM.
getModule());
3419 TaskDup->setDoesNotRecurse();
3429 auto LIFI = std::next(KmpTaskTQTyRD->
field_begin(), KmpTaskTLastIter);
3431 TDBase, *KmpTaskTWithPrivatesQTyRD->
field_begin());
3439 assert(!Privates.empty());
3441 if (!
Data.FirstprivateVars.empty()) {
3446 TDBase, *KmpTaskTWithPrivatesQTyRD->
field_begin());
3454 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, TDBase, KmpTaskTWithPrivatesQTyRD,
3455 SharedsTy, SharedsPtrTy,
Data, Privates,
true);
3465 for (
const PrivateDataTy &
P : Privates) {
3466 if (
P.second.isLocalPrivate())
3468 QualType Ty =
P.second.Original->getType().getNonReferenceType();
3477class OMPIteratorGeneratorScope final
3478 :
public CodeGenFunction::OMPPrivateScope {
3483 OMPIteratorGeneratorScope() =
delete;
3484 OMPIteratorGeneratorScope(OMPIteratorGeneratorScope &) =
delete;
3512 CodeGenFunction::JumpDest &ContDest =
3514 CodeGenFunction::JumpDest &ExitDest =
3517 llvm::Value *N = Uppers[I];
3525 ? CGF.
Builder.CreateICmpSLT(CVal, N)
3526 : CGF.
Builder.CreateICmpULT(CVal, N);
3528 CGF.
Builder.CreateCondBr(Cmp, BodyBB, ExitDest.getBlock());
3535 ~OMPIteratorGeneratorScope() {
3545 CGF.
EmitBlock(ExitDests[I - 1].getBlock(), I == 1);
3551static std::pair<llvm::Value *, llvm::Value *>
3553 const auto *OASE = dyn_cast<OMPArrayShapingExpr>(E);
3556 const Expr *
Base = OASE->getBase();
3561 llvm::Value *SizeVal;
3564 SizeVal = CGF.
getTypeSize(OASE->getBase()->getType()->getPointeeType());
3565 for (
const Expr *SE : OASE->getDimensions()) {
3569 SizeVal = CGF.
Builder.CreateNUWMul(SizeVal, Sz);
3571 }
else if (
const auto *ASE =
3575 llvm::Value *UpAddr = CGF.
Builder.CreateConstGEP1_32(
3578 llvm::Value *LowIntPtr = CGF.
Builder.CreatePtrToInt(Addr, CGF.
SizeTy);
3579 llvm::Value *UpIntPtr = CGF.
Builder.CreatePtrToInt(UpAddr, CGF.
SizeTy);
3580 SizeVal = CGF.
Builder.CreateNUWSub(UpIntPtr, LowIntPtr);
3584 return std::make_pair(Addr, SizeVal);
3589 QualType FlagsTy =
C.getIntTypeForBitwidth(32,
false);
3590 if (KmpTaskAffinityInfoTy.
isNull()) {
3592 C.buildImplicitRecord(
"kmp_task_affinity_info_t");
3598 KmpTaskAffinityInfoTy =
C.getRecordType(KmpAffinityInfoRD);
3605 llvm::Function *TaskFunction,
QualType SharedsTy,
3610 const auto *I =
Data.PrivateCopies.begin();
3611 for (
const Expr *E :
Data.PrivateVars) {
3612 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3613 Privates.emplace_back(
3615 PrivateHelpersTy(E, VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
3619 I =
Data.FirstprivateCopies.begin();
3620 const auto *IElemInitRef =
Data.FirstprivateInits.begin();
3621 for (
const Expr *E :
Data.FirstprivateVars) {
3622 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3623 Privates.emplace_back(
3626 E, VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
3627 cast<VarDecl>(cast<DeclRefExpr>(*IElemInitRef)->getDecl())));
3631 I =
Data.LastprivateCopies.begin();
3632 for (
const Expr *E :
Data.LastprivateVars) {
3633 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3634 Privates.emplace_back(
3636 PrivateHelpersTy(E, VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
3644 Privates.emplace_back(
C.getDeclAlign(VD), PrivateHelpersTy(VD));
3646 llvm::stable_sort(Privates,
3647 [](
const PrivateDataTy &L,
const PrivateDataTy &R) {
3648 return L.first > R.first;
3650 QualType KmpInt32Ty =
C.getIntTypeForBitwidth(32, 1);
3664 "Expected taskloop, task or target directive");
3673 const RecordDecl *KmpTaskTWithPrivatesQTyRD =
3675 QualType KmpTaskTWithPrivatesQTy =
C.getRecordType(KmpTaskTWithPrivatesQTyRD);
3676 QualType KmpTaskTWithPrivatesPtrQTy =
3677 C.getPointerType(KmpTaskTWithPrivatesQTy);
3678 llvm::Type *KmpTaskTWithPrivatesTy = CGF.
ConvertType(KmpTaskTWithPrivatesQTy);
3679 llvm::Type *KmpTaskTWithPrivatesPtrTy =
3680 KmpTaskTWithPrivatesTy->getPointerTo();
3681 llvm::Value *KmpTaskTWithPrivatesTySize =
3683 QualType SharedsPtrTy =
C.getPointerType(SharedsTy);
3686 llvm::Value *TaskPrivatesMap =
nullptr;
3687 llvm::Type *TaskPrivatesMapTy =
3688 std::next(TaskFunction->arg_begin(), 3)->getType();
3689 if (!Privates.empty()) {
3690 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->
field_begin());
3694 TaskPrivatesMap, TaskPrivatesMapTy);
3696 TaskPrivatesMap = llvm::ConstantPointerNull::get(
3697 cast<llvm::PointerType>(TaskPrivatesMapTy));
3703 KmpTaskTWithPrivatesQTy,
KmpTaskTQTy, SharedsPtrTy, TaskFunction,
3715 DestructorsFlag = 0x8,
3716 PriorityFlag = 0x20,
3717 DetachableFlag = 0x40,
3719 unsigned Flags =
Data.Tied ? TiedFlag : 0;
3720 bool NeedsCleanup =
false;
3721 if (!Privates.empty()) {
3725 Flags = Flags | DestructorsFlag;
3727 if (
Data.Priority.getInt())
3728 Flags = Flags | PriorityFlag;
3730 Flags = Flags | DetachableFlag;
3731 llvm::Value *TaskFlags =
3732 Data.Final.getPointer()
3733 ? CGF.
Builder.CreateSelect(
Data.Final.getPointer(),
3734 CGF.
Builder.getInt32(FinalFlag),
3736 : CGF.
Builder.getInt32(
Data.Final.getInt() ? FinalFlag : 0);
3737 TaskFlags = CGF.
Builder.CreateOr(TaskFlags, CGF.
Builder.getInt32(Flags));
3738 llvm::Value *SharedsSize =
CGM.
getSize(
C.getTypeSizeInChars(SharedsTy));
3743 llvm::Value *NewTask;
3750 llvm::Value *DeviceID;
3755 DeviceID = CGF.
Builder.getInt64(OMP_DEVICEID_UNDEF);
3756 AllocArgs.push_back(DeviceID);
3777 llvm::Value *Tid =
getThreadID(CGF, DC->getBeginLoc());
3778 Tid = CGF.
Builder.CreateIntCast(Tid, CGF.
IntTy,
false);
3781 CGM.
getModule(), OMPRTL___kmpc_task_allow_completion_event),
3782 {Loc, Tid, NewTask});
3793 llvm::Value *NumOfElements =
nullptr;
3794 unsigned NumAffinities = 0;
3796 if (
const Expr *Modifier =
C->getModifier()) {
3797 const auto *IE = cast<OMPIteratorExpr>(Modifier->IgnoreParenImpCasts());
3798 for (
unsigned I = 0, E = IE->numOfIterators(); I < E; ++I) {
3802 NumOfElements ? CGF.
Builder.CreateNUWMul(NumOfElements, Sz) : Sz;
3805 NumAffinities +=
C->varlist_size();
3810 enum RTLAffinityInfoFieldsTy { BaseAddr, Len, Flags };
3812 QualType KmpTaskAffinityInfoArrayTy;
3813 if (NumOfElements) {
3814 NumOfElements = CGF.
Builder.CreateNUWAdd(
3815 llvm::ConstantInt::get(CGF.
SizeTy, NumAffinities), NumOfElements);
3818 C.getIntTypeForBitwidth(
C.getTypeSize(
C.getSizeType()), 0),
3822 KmpTaskAffinityInfoArrayTy =
C.getVariableArrayType(
3830 NumOfElements = CGF.
Builder.CreateIntCast(NumOfElements, CGF.
Int32Ty,
3833 KmpTaskAffinityInfoArrayTy =
C.getConstantArrayType(
3835 llvm::APInt(
C.getTypeSize(
C.getSizeType()), NumAffinities),
nullptr,
3838 CGF.
CreateMemTemp(KmpTaskAffinityInfoArrayTy,
".affs.arr.addr");
3840 NumOfElements = llvm::ConstantInt::get(
CGM.
Int32Ty, NumAffinities,
3847 bool HasIterator =
false;
3849 if (
C->getModifier()) {
3853 for (
const Expr *E :
C->varlists()) {
3862 Base, *std::next(KmpAffinityInfoRD->field_begin(), BaseAddr));
3867 Base, *std::next(KmpAffinityInfoRD->field_begin(), Len));
3881 const Expr *Modifier =
C->getModifier();
3884 OMPIteratorGeneratorScope IteratorScope(
3886 for (
const Expr *E :
C->varlists()) {
3896 Base, *std::next(KmpAffinityInfoRD->field_begin(), BaseAddr));
3901 Base, *std::next(KmpAffinityInfoRD->field_begin(), Len));
3903 Idx = CGF.
Builder.CreateNUWAdd(
3904 Idx, llvm::ConstantInt::get(Idx->getType(), 1));
3919 CGM.
getModule(), OMPRTL___kmpc_omp_reg_task_with_affinity),
3920 {LocRef, GTid, NewTask, NumOfElements, AffinListPtr});
3922 llvm::Value *NewTaskNewTaskTTy =
3924 NewTask, KmpTaskTWithPrivatesPtrTy);
3926 KmpTaskTWithPrivatesQTy);
3937 *std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTShareds)),
3946 if (!Privates.empty()) {
3948 SharedsTy, SharedsPtrTy,
Data, Privates,
3953 CGM,
Loc, D, KmpTaskTWithPrivatesPtrQTy, KmpTaskTWithPrivatesQTyRD,
3954 KmpTaskTQTyRD, SharedsTy, SharedsPtrTy,
Data, Privates,
3955 !
Data.LastprivateVars.empty());
3959 enum {
Priority = 0, Destructors = 1 };
3961 auto FI = std::next(KmpTaskTQTyRD->field_begin(), Data1);
3963 (*FI)->getType()->getAsUnionType()->getDecl();
3966 CGM,
Loc, KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy,
3967 KmpTaskTWithPrivatesQTy);
3970 Data1LV, *std::next(KmpCmplrdataUD->
field_begin(), Destructors));
3976 if (
Data.Priority.getInt()) {
3978 TDBase, *std::next(KmpTaskTQTyRD->field_begin(), Data2));
3983 Result.NewTask = NewTask;
3984 Result.TaskEntry = TaskEntry;
3985 Result.NewTaskNewTaskTTy = NewTaskNewTaskTTy;
3987 Result.KmpTaskTQTyRD = KmpTaskTQTyRD;
3993 RTLDependenceKindTy DepKind;
3995 case OMPC_DEPEND_in:
3996 DepKind = RTLDependenceKindTy::DepIn;
3999 case OMPC_DEPEND_out:
4000 case OMPC_DEPEND_inout:
4001 DepKind = RTLDependenceKindTy::DepInOut;
4003 case OMPC_DEPEND_mutexinoutset:
4004 DepKind = RTLDependenceKindTy::DepMutexInOutSet;
4006 case OMPC_DEPEND_inoutset:
4007 DepKind = RTLDependenceKindTy::DepInOutSet;
4009 case OMPC_DEPEND_outallmemory:
4010 DepKind = RTLDependenceKindTy::DepOmpAllMem;
4012 case OMPC_DEPEND_source:
4013 case OMPC_DEPEND_sink:
4014 case OMPC_DEPEND_depobj:
4015 case OMPC_DEPEND_inoutallmemory:
4017 llvm_unreachable(
"Unknown task dependence type");
4025 FlagsTy =
C.getIntTypeForBitwidth(
C.getTypeSize(
C.BoolTy),
false);
4026 if (KmpDependInfoTy.
isNull()) {
4027 RecordDecl *KmpDependInfoRD =
C.buildImplicitRecord(
"kmp_depend_info");
4033 KmpDependInfoTy =
C.getRecordType(KmpDependInfoRD);
4037std::pair<llvm::Value *, LValue>
4051 CGF,
Base.getAddress(),
4052 llvm::ConstantInt::get(CGF.
IntPtrTy, -1,
true));
4059 static_cast<unsigned int>(RTLDependInfoFields::BaseAddr)));
4061 return std::make_pair(NumDeps,
Base);
4065 llvm::PointerUnion<unsigned *, LValue *> Pos,
4076 OMPIteratorGeneratorScope IteratorScope(
4077 CGF, cast_or_null<OMPIteratorExpr>(
4078 Data.IteratorExpr ?
Data.IteratorExpr->IgnoreParenImpCasts()
4080 for (
const Expr *E :
Data.DepExprs) {
4089 Addr = llvm::ConstantInt::get(CGF.
IntPtrTy, 0);
4090 Size = llvm::ConstantInt::get(CGF.
SizeTy, 0);
4093 if (
unsigned *
P = Pos.dyn_cast<
unsigned *>()) {
4097 assert(E &&
"Expected a non-null expression");
4107 static_cast<unsigned int>(RTLDependInfoFields::BaseAddr)));
4112 static_cast<unsigned int>(RTLDependInfoFields::Len)));
4119 static_cast<unsigned int>(RTLDependInfoFields::Flags)));
4121 llvm::ConstantInt::get(LLVMFlagsTy,
static_cast<unsigned int>(DepKind)),
4123 if (
unsigned *
P = Pos.dyn_cast<
unsigned *>()) {
4128 Idx = CGF.
Builder.CreateNUWAdd(Idx,
4129 llvm::ConstantInt::get(Idx->getType(), 1));
4138 assert(
Data.DepKind == OMPC_DEPEND_depobj &&
4139 "Expected depobj dependency kind.");
4144 OMPIteratorGeneratorScope IteratorScope(
4145 CGF, cast_or_null<OMPIteratorExpr>(
4146 Data.IteratorExpr ?
Data.IteratorExpr->IgnoreParenImpCasts()
4148 for (
const Expr *E :
Data.DepExprs) {
4149 llvm::Value *NumDeps;
4152 std::tie(NumDeps,
Base) =
4156 C.getUIntPtrType());
4160 llvm::Value *Add = CGF.
Builder.CreateNUWAdd(PrevVal, NumDeps);
4162 SizeLVals.push_back(NumLVal);
4165 for (
unsigned I = 0, E = SizeLVals.size(); I < E; ++I) {
4168 Sizes.push_back(Size);
4178 assert(
Data.DepKind == OMPC_DEPEND_depobj &&
4179 "Expected depobj dependency kind.");
4182 OMPIteratorGeneratorScope IteratorScope(
4183 CGF, cast_or_null<OMPIteratorExpr>(
4184 Data.IteratorExpr ?
Data.IteratorExpr->IgnoreParenImpCasts()
4186 for (
unsigned I = 0, End =
Data.DepExprs.size(); I < End; ++I) {
4188 llvm::Value *NumDeps;
4191 std::tie(NumDeps,
Base) =
4195 llvm::Value *Size = CGF.
Builder.CreateNUWMul(
4204 llvm::Value *Add = CGF.
Builder.CreateNUWAdd(Pos, NumDeps);
4220 llvm::Value *NumOfElements =
nullptr;
4221 unsigned NumDependencies = std::accumulate(
4222 Dependencies.begin(), Dependencies.end(), 0,
4224 return D.DepKind == OMPC_DEPEND_depobj
4226 : (V + (D.IteratorExpr ? 0 : D.DepExprs.size()));
4230 bool HasDepobjDeps =
false;
4231 bool HasRegularWithIterators =
false;
4232 llvm::Value *NumOfDepobjElements = llvm::ConstantInt::get(CGF.
IntPtrTy, 0);
4233 llvm::Value *NumOfRegularWithIterators =
4234 llvm::ConstantInt::get(CGF.
IntPtrTy, 0);
4238 if (D.
DepKind == OMPC_DEPEND_depobj) {
4241 for (llvm::Value *Size : Sizes) {
4242 NumOfDepobjElements =
4243 CGF.
Builder.CreateNUWAdd(NumOfDepobjElements, Size);
4245 HasDepobjDeps =
true;
4250 if (
const auto *IE = cast_or_null<OMPIteratorExpr>(D.
IteratorExpr)) {
4254 llvm::Value *NumClauseDeps = CGF.
Builder.CreateNUWMul(
4256 NumOfRegularWithIterators =
4257 CGF.
Builder.CreateNUWAdd(NumOfRegularWithIterators, NumClauseDeps);
4259 HasRegularWithIterators =
true;
4265 if (HasDepobjDeps || HasRegularWithIterators) {
4266 NumOfElements = llvm::ConstantInt::get(
CGM.
IntPtrTy, NumDependencies,
4268 if (HasDepobjDeps) {
4270 CGF.
Builder.CreateNUWAdd(NumOfDepobjElements, NumOfElements);
4272 if (HasRegularWithIterators) {
4274 CGF.
Builder.CreateNUWAdd(NumOfRegularWithIterators, NumOfElements);
4277 Loc,
C.getIntTypeForBitwidth(64, 0),
4281 KmpDependInfoArrayTy =
4290 NumOfElements = CGF.
Builder.CreateIntCast(NumOfElements, CGF.
Int32Ty,
4293 KmpDependInfoArrayTy =
C.getConstantArrayType(
4299 NumOfElements = llvm::ConstantInt::get(
CGM.
Int32Ty, NumDependencies,
4303 for (
unsigned I = 0, End = Dependencies.size(); I < End; ++I) {
4304 if (Dependencies[I].DepKind == OMPC_DEPEND_depobj ||
4305 Dependencies[I].IteratorExpr)
4312 CGF.
CreateMemTemp(
C.getSizeType(),
"dep.counter.addr"),
C.getSizeType());
4314 for (
unsigned I = 0, End = Dependencies.size(); I < End; ++I) {
4315 if (Dependencies[I].DepKind == OMPC_DEPEND_depobj ||
4316 !Dependencies[I].IteratorExpr)
4322 if (HasDepobjDeps) {
4323 for (
unsigned I = 0, End = Dependencies.size(); I < End; ++I) {
4324 if (Dependencies[I].DepKind != OMPC_DEPEND_depobj)
4332 return std::make_pair(NumOfElements, DependenciesArray);
4343 unsigned NumDependencies = Dependencies.
DepExprs.size();
4354 llvm::Value *NumDepsVal;
4356 if (
const auto *IE =
4357 cast_or_null<OMPIteratorExpr>(Dependencies.
IteratorExpr)) {
4358 NumDepsVal = llvm::ConstantInt::get(CGF.
SizeTy, 1);
4362 NumDepsVal = CGF.
Builder.CreateNUWMul(NumDepsVal, Sz);
4364 Size = CGF.
Builder.CreateNUWAdd(llvm::ConstantInt::get(CGF.
SizeTy, 1),
4368 llvm::Value *RecSize =
CGM.
getSize(SizeInBytes);
4369 Size = CGF.
Builder.CreateNUWMul(Size, RecSize);
4373 QualType KmpDependInfoArrayTy =
C.getConstantArrayType(
4376 CharUnits Sz =
C.getTypeSizeInChars(KmpDependInfoArrayTy);
4378 NumDepsVal = llvm::ConstantInt::get(CGF.
IntPtrTy, NumDependencies);
4383 llvm::Value *Allocator = llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
4384 llvm::Value *Args[] = {ThreadID, Size, Allocator};
4389 Args,
".dep.arr.addr");
4392 Addr, KmpDependInfoLlvmTy->getPointerTo());
4393 DependenciesArray =
Address(Addr, KmpDependInfoLlvmTy, Align);
4400 static_cast<unsigned int>(RTLDependInfoFields::BaseAddr)));
4402 llvm::PointerUnion<unsigned *, LValue *> Pos;
4419 return DependenciesArray;
4435 llvm::ConstantInt::get(CGF.
IntPtrTy, -1,
true));
4440 llvm::Value *Allocator = llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
4441 llvm::Value *Args[] = {ThreadID, DepObjAddr, Allocator};
4458 llvm::Value *NumDeps;
4465 Begin.emitRawPointer(CGF), NumDeps);
4469 llvm::BasicBlock *EntryBB = CGF.
Builder.GetInsertBlock();
4471 llvm::PHINode *ElementPHI =
4472 CGF.
Builder.CreatePHI(
Begin.getType(), 2,
"omp.elementPast");
4473 ElementPHI->addIncoming(
Begin.emitRawPointer(CGF), EntryBB);
4476 Base.getTBAAInfo());
4481 static_cast<unsigned int>(RTLDependInfoFields::Flags)));
4483 llvm::ConstantInt::get(LLVMFlagsTy,
static_cast<unsigned int>(DepKind)),
4487 llvm::Value *ElementNext =
4490 ElementPHI->addIncoming(ElementNext, CGF.
Builder.GetInsertBlock());
4491 llvm::Value *IsEmpty =
4492 CGF.
Builder.CreateICmpEQ(ElementNext, End,
"omp.isempty");
4493 CGF.
Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
4500 llvm::Function *TaskFunction,
4509 llvm::Value *NewTask =
Result.NewTask;
4510 llvm::Function *TaskEntry =
Result.TaskEntry;
4511 llvm::Value *NewTaskNewTaskTTy =
Result.NewTaskNewTaskTTy;
4516 llvm::Value *NumOfElements;
4517 std::tie(NumOfElements, DependenciesArray) =
4528 llvm::Value *TaskArgs[] = { UpLoc, ThreadID, NewTask };
4529 llvm::Value *DepTaskArgs[7];
4530 if (!
Data.Dependences.empty()) {
4531 DepTaskArgs[0] = UpLoc;
4532 DepTaskArgs[1] = ThreadID;
4533 DepTaskArgs[2] = NewTask;
4534 DepTaskArgs[3] = NumOfElements;
4536 DepTaskArgs[5] = CGF.
Builder.getInt32(0);
4537 DepTaskArgs[6] = llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
4539 auto &&ThenCodeGen = [
this, &
Data, TDBase, KmpTaskTQTyRD, &TaskArgs,
4542 auto PartIdFI = std::next(KmpTaskTQTyRD->
field_begin(), KmpTaskTPartId);
4546 if (!
Data.Dependences.empty()) {
4559 Region->emitUntiedSwitch(CGF);
4562 llvm::Value *DepWaitTaskArgs[7];
4563 if (!
Data.Dependences.empty()) {
4564 DepWaitTaskArgs[0] = UpLoc;
4565 DepWaitTaskArgs[1] = ThreadID;
4566 DepWaitTaskArgs[2] = NumOfElements;
4568 DepWaitTaskArgs[4] = CGF.
Builder.getInt32(0);
4569 DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
4570 DepWaitTaskArgs[6] =
4571 llvm::ConstantInt::get(CGF.
Int32Ty,
Data.HasNowaitClause);
4574 auto &&ElseCodeGen = [
this, &M, &TaskArgs, ThreadID, NewTaskNewTaskTTy,
4575 TaskEntry, &
Data, &DepWaitTaskArgs,
4582 if (!
Data.Dependences.empty())
4584 M, OMPRTL___kmpc_omp_taskwait_deps_51),
4587 auto &&CodeGen = [TaskEntry, ThreadID, NewTaskNewTaskTTy,
4590 llvm::Value *OutlinedFnArgs[] = {ThreadID, NewTaskNewTaskTTy};
4591 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF,
Loc, TaskEntry,
4600 CommonActionTy Action(
OMPBuilder.getOrCreateRuntimeFunction(
4601 M, OMPRTL___kmpc_omp_task_begin_if0),
4604 M, OMPRTL___kmpc_omp_task_complete_if0),
4620 llvm::Function *TaskFunction,
4640 IfVal = llvm::ConstantInt::getSigned(CGF.
IntTy, 1);
4645 *std::next(
Result.KmpTaskTQTyRD->field_begin(), KmpTaskTLowerBound));
4652 *std::next(
Result.KmpTaskTQTyRD->field_begin(), KmpTaskTUpperBound));
4659 *std::next(
Result.KmpTaskTQTyRD->field_begin(), KmpTaskTStride));
4667 *std::next(
Result.KmpTaskTQTyRD->field_begin(), KmpTaskTReductions));
4668 if (
Data.Reductions) {
4674 enum { NoSchedule = 0, Grainsize = 1, NumTasks = 2 };
4675 llvm::Value *TaskArgs[] = {
4683 llvm::ConstantInt::getSigned(
4685 llvm::ConstantInt::getSigned(
4687 ?
Data.Schedule.getInt() ? NumTasks : Grainsize
4689 Data.Schedule.getPointer()
4692 : llvm::ConstantInt::get(CGF.
Int64Ty, 0),
4695 : llvm::ConstantPointerNull::get(CGF.
VoidPtrTy)};
4714 const Expr *,
const Expr *)> &RedOpGen,
4715 const Expr *XExpr =
nullptr,
const Expr *EExpr =
nullptr,
4716 const Expr *UpExpr =
nullptr) {
4724 llvm::Value *NumElements = CGF.
emitArrayLength(ArrayTy, ElementTy, LHSAddr);
4729 llvm::Value *LHSEnd =
4734 llvm::Value *IsEmpty =
4735 CGF.
Builder.CreateICmpEQ(LHSBegin, LHSEnd,
"omp.arraycpy.isempty");
4736 CGF.
Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
4739 llvm::BasicBlock *EntryBB = CGF.
Builder.GetInsertBlock();
4744 llvm::PHINode *RHSElementPHI = CGF.
Builder.CreatePHI(
4745 RHSBegin->getType(), 2,
"omp.arraycpy.srcElementPast");
4746 RHSElementPHI->addIncoming(RHSBegin, EntryBB);
4751 llvm::PHINode *LHSElementPHI = CGF.
Builder.CreatePHI(
4752 LHSBegin->getType(), 2,
"omp.arraycpy.destElementPast");
4753 LHSElementPHI->addIncoming(LHSBegin, EntryBB);
4759 CodeGenFunction::OMPPrivateScope
Scope(CGF);
4760 Scope.addPrivate(LHSVar, LHSElementCurrent);
4761 Scope.addPrivate(RHSVar, RHSElementCurrent);
4763 RedOpGen(CGF, XExpr, EExpr, UpExpr);
4764 Scope.ForceCleanup();
4767 llvm::Value *LHSElementNext = CGF.
Builder.CreateConstGEP1_32(
4769 "omp.arraycpy.dest.element");
4770 llvm::Value *RHSElementNext = CGF.
Builder.CreateConstGEP1_32(
4772 "omp.arraycpy.src.element");
4775 CGF.
Builder.CreateICmpEQ(LHSElementNext, LHSEnd,
"omp.arraycpy.done");
4776 CGF.
Builder.CreateCondBr(Done, DoneBB, BodyBB);
4777 LHSElementPHI->addIncoming(LHSElementNext, CGF.
Builder.GetInsertBlock());
4778 RHSElementPHI->addIncoming(RHSElementNext, CGF.
Builder.GetInsertBlock());
4788 const Expr *ReductionOp) {
4789 if (
const auto *CE = dyn_cast<CallExpr>(ReductionOp))
4790 if (
const auto *OVE = dyn_cast<OpaqueValueExpr>(CE->getCallee()))
4791 if (
const auto *DRE =
4792 dyn_cast<DeclRefExpr>(OVE->getSourceExpr()->IgnoreImpCasts()))
4793 if (
const auto *DRD =
4794 dyn_cast<OMPDeclareReductionDecl>(DRE->getDecl())) {
4795 std::pair<llvm::Function *, llvm::Function *>
Reduction =
4798 CodeGenFunction::OpaqueValueMapping Map(CGF, OVE,
Func);
4817 Args.push_back(&LHSArg);
4818 Args.push_back(&RHSArg);
4823 llvm::GlobalValue::InternalLinkage, Name,
4826 Fn->setDoesNotRecurse();
4834 ArgsElemType->getPointerTo()),
4838 ArgsElemType->getPointerTo()),
4845 const auto *IPriv = Privates.begin();
4847 for (
unsigned I = 0, E = ReductionOps.size(); I < E; ++I, ++IPriv, ++Idx) {
4848 const auto *RHSVar =
4849 cast<VarDecl>(cast<DeclRefExpr>(RHSExprs[I])->getDecl());
4851 const auto *LHSVar =
4852 cast<VarDecl>(cast<DeclRefExpr>(LHSExprs[I])->getDecl());
4854 QualType PrivTy = (*IPriv)->getType();
4862 const auto *OVE = cast<OpaqueValueExpr>(VLA->
getSizeExpr());
4869 IPriv = Privates.begin();
4870 const auto *ILHS = LHSExprs.begin();
4871 const auto *IRHS = RHSExprs.begin();
4872 for (
const Expr *E : ReductionOps) {
4873 if ((*IPriv)->getType()->isArrayType()) {
4875 const auto *LHSVar = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
4876 const auto *RHSVar = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
4878 CGF, (*IPriv)->getType(), LHSVar, RHSVar,
4880 emitReductionCombiner(CGF, E);
4890 Scope.ForceCleanup();
4896 const Expr *ReductionOp,
4897 const Expr *PrivateRef,
4902 const auto *LHSVar = cast<VarDecl>(LHS->
getDecl());
4903 const auto *RHSVar = cast<VarDecl>(RHS->
getDecl());
4905 CGF, PrivateRef->
getType(), LHSVar, RHSVar,
4907 emitReductionCombiner(CGF, ReductionOp);
4924 bool WithNowait = Options.WithNowait;
4925 bool SimpleReduction = Options.SimpleReduction;
4964 if (SimpleReduction) {
4966 const auto *IPriv = Privates.begin();
4967 const auto *ILHS = LHSExprs.begin();
4968 const auto *IRHS = RHSExprs.begin();
4969 for (
const Expr *E : ReductionOps) {
4971 cast<DeclRefExpr>(*IRHS));
4981 auto Size = RHSExprs.size();
4982 for (
const Expr *E : Privates) {
4987 llvm::APInt ArraySize(32, Size);
4988 QualType ReductionArrayTy =
C.getConstantArrayType(
4992 CGF.
CreateMemTemp(ReductionArrayTy,
".omp.reduction.red_list");
4993 const auto *IPriv = Privates.begin();
4995 for (
unsigned I = 0, E = RHSExprs.size(); I < E; ++I, ++IPriv, ++Idx) {
5001 if ((*IPriv)->getType()->isVariablyModifiedType()) {
5005 llvm::Value *Size = CGF.
Builder.CreateIntCast(
5018 Privates, LHSExprs, RHSExprs, ReductionOps);
5021 std::string Name =
getName({
"reduction"});
5028 llvm::Value *ReductionArrayTySize = CGF.
getTypeSize(ReductionArrayTy);
5031 llvm::Value *Args[] = {
5034 CGF.
Builder.getInt32(RHSExprs.size()),
5035 ReductionArrayTySize,
5043 WithNowait ? OMPRTL___kmpc_reduce_nowait : OMPRTL___kmpc_reduce),
5047 llvm::BasicBlock *DefaultBB = CGF.
createBasicBlock(
".omp.reduction.default");
5048 llvm::SwitchInst *SwInst =
5049 CGF.
Builder.CreateSwitch(Res, DefaultBB, 2);
5058 SwInst->addCase(CGF.
Builder.getInt32(1), Case1BB);
5062 llvm::Value *EndArgs[] = {
5067 auto &&CodeGen = [Privates, LHSExprs, RHSExprs, ReductionOps](
5070 const auto *IPriv = Privates.begin();
5071 const auto *ILHS = LHSExprs.begin();
5072 const auto *IRHS = RHSExprs.begin();
5073 for (
const Expr *E : ReductionOps) {
5075 cast<DeclRefExpr>(*IRHS));
5082 CommonActionTy Action(
5083 nullptr, std::nullopt,
5085 CGM.
getModule(), WithNowait ? OMPRTL___kmpc_end_reduce_nowait
5086 : OMPRTL___kmpc_end_reduce),
5099 SwInst->addCase(CGF.
Builder.getInt32(2), Case2BB);
5102 auto &&AtomicCodeGen = [
Loc, Privates, LHSExprs, RHSExprs, ReductionOps](
5104 const auto *ILHS = LHSExprs.begin();
5105 const auto *IRHS = RHSExprs.begin();
5106 const auto *IPriv = Privates.begin();
5107 for (
const Expr *E : ReductionOps) {
5108 const Expr *XExpr =
nullptr;
5109 const Expr *EExpr =
nullptr;
5110 const Expr *UpExpr =
nullptr;
5112 if (
const auto *BO = dyn_cast<BinaryOperator>(E)) {
5113 if (BO->getOpcode() == BO_Assign) {
5114 XExpr = BO->getLHS();
5115 UpExpr = BO->getRHS();
5119 const Expr *RHSExpr = UpExpr;
5122 if (
const auto *ACO = dyn_cast<AbstractConditionalOperator>(
5126 RHSExpr = ACO->getCond();
5128 if (
const auto *BORHS =
5130 EExpr = BORHS->getRHS();
5131 BO = BORHS->getOpcode();
5135 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
5136 auto &&AtomicRedGen = [BO, VD,
5138 const Expr *EExpr,
const Expr *UpExpr) {
5139 LValue X = CGF.EmitLValue(XExpr);
5142 E = CGF.EmitAnyExpr(EExpr);
5143 CGF.EmitOMPAtomicSimpleUpdateExpr(
5145 llvm::AtomicOrdering::Monotonic,
Loc,
5146 [&CGF, UpExpr, VD,
Loc](
RValue XRValue) {
5148 Address LHSTemp = CGF.CreateMemTemp(VD->getType());
5149 CGF.emitOMPSimpleStore(
5150 CGF.MakeAddrLValue(LHSTemp, VD->
getType()), XRValue,
5151 VD->getType().getNonReferenceType(),
Loc);
5154 return CGF.EmitAnyExpr(UpExpr);
5157 if ((*IPriv)->getType()->isArrayType()) {
5159 const auto *RHSVar =
5160 cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
5162 AtomicRedGen, XExpr, EExpr, UpExpr);
5165 AtomicRedGen(CGF, XExpr, EExpr, UpExpr);
5172 std::string Name = RT.
getName({
"atomic_reduction"});
5181 if ((*IPriv)->getType()->isArrayType()) {
5182 const auto *LHSVar =
5183 cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
5184 const auto *RHSVar =
5185 cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
5189 CritRedGen(CGF,
nullptr,
nullptr,
nullptr);
5200 llvm::Value *EndArgs[] = {
5205 CommonActionTy Action(
nullptr, std::nullopt,
5224 llvm::raw_svector_ostream Out(Buffer);
5228 D = cast<VarDecl>(cast<DeclRefExpr>(Ref)->getDecl());
5232 Out << Prefix << Name <<
"_"
5234 return std::string(Out.str());
5256 Args.emplace_back(&Param);
5257 Args.emplace_back(&ParamOrig);
5258 const auto &FnInfo =
5262 auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,
5265 Fn->setDoesNotRecurse();
5273 llvm::Value *Size =
nullptr;
5316 const Expr *ReductionOp,
5318 const Expr *PrivateRef) {
5320 const auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(LHS)->getDecl());
5321 const auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(RHS)->getDecl());
5327 Args.emplace_back(&ParamInOut);
5328 Args.emplace_back(&ParamIn);
5329 const auto &FnInfo =
5333 auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,
5336 Fn->setDoesNotRecurse();
5339 llvm::Value *Size =
nullptr;
5353 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
5354 PrivateScope.addPrivate(
5361 C.getPointerType(LHSVD->getType())->castAs<
PointerType>()));
5362 PrivateScope.addPrivate(
5368 C.getPointerType(RHSVD->getType())->castAs<
PointerType>()));
5369 PrivateScope.Privatize();
5374 CGF, ReductionOp, PrivateRef, cast<DeclRefExpr>(LHS),
5375 cast<DeclRefExpr>(RHS));
5397 Args.emplace_back(&Param);
5398 const auto &FnInfo =
5402 auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,
5405 Fn->setDoesNotRecurse();
5410 llvm::Value *Size =
nullptr;
5445 RecordDecl *RD =
C.buildImplicitRecord(
"kmp_taskred_input_t");
5454 C, RD,
C.getIntTypeForBitwidth(32,
false));
5457 unsigned Size =
Data.ReductionVars.size();
5458 llvm::APInt ArraySize(64, Size);
5460 C.getConstantArrayType(RDType, ArraySize,
nullptr,
5465 Data.ReductionCopies,
Data.ReductionOps);
5466 for (
unsigned Cnt = 0; Cnt < Size; ++Cnt) {
5468 llvm::Value *Idxs[] = {llvm::ConstantInt::get(
CGM.
SizeTy, 0),
5469 llvm::ConstantInt::get(
CGM.
SizeTy, Cnt)};
5485 llvm::Value *SizeValInChars;
5486 llvm::Value *SizeVal;
5487 std::tie(SizeValInChars, SizeVal) = RCG.
getSizes(Cnt);
5493 bool DelayedCreation = !!SizeVal;
5505 llvm::Value *FiniAddr =
5506 Fini ? Fini : llvm::ConstantPointerNull::get(
CGM.
VoidPtrTy);
5511 CGM,
Loc, RCG, Cnt,
Data.ReductionOps[Cnt], LHSExprs[Cnt],
5512 RHSExprs[Cnt],
Data.ReductionCopies[Cnt]);
5516 if (DelayedCreation) {
5518 llvm::ConstantInt::get(
CGM.
Int32Ty, 1,
true),
5523 if (
Data.IsReductionWithTaskMod) {
5529 llvm::Value *Args[] = {
5531 llvm::ConstantInt::get(
CGM.
IntTy,
Data.IsWorksharingReduction ? 1 : 0,
5533 llvm::ConstantInt::get(
CGM.
IntTy, Size,
true),
5542 llvm::Value *Args[] = {
5545 llvm::ConstantInt::get(
CGM.
IntTy, Size,
true),
5555 bool IsWorksharingReduction) {
5561 llvm::Value *Args[] = {IdentTLoc, GTid,
5563 IsWorksharingReduction ? 1 : 0,
5567 CGM.
getModule(), OMPRTL___kmpc_task_reduction_modifier_fini),
5590 llvm::Value *ReductionsPtr,
5603 CGM.
getModule(), OMPRTL___kmpc_task_reduction_get_th_data),
5621 llvm::Value *NumOfElements;
5622 std::tie(NumOfElements, DependenciesArray) =
5624 if (!
Data.Dependences.empty()) {
5625 llvm::Value *DepWaitTaskArgs[7];
5626 DepWaitTaskArgs[0] = UpLoc;
5627 DepWaitTaskArgs[1] = ThreadID;
5628 DepWaitTaskArgs[2] = NumOfElements;
5630 DepWaitTaskArgs[4] = CGF.
Builder.getInt32(0);
5631 DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
5632 DepWaitTaskArgs[6] =
5633 llvm::ConstantInt::get(CGF.
Int32Ty,
Data.HasNowaitClause);
5642 M, OMPRTL___kmpc_omp_taskwait_deps_51),
5649 llvm::Value *Args[] = {UpLoc, ThreadID};
5652 OMPBuilder.getOrCreateRuntimeFunction(M, OMPRTL___kmpc_omp_taskwait),
5657 if (
auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.
CapturedStmtInfo))
5658 Region->emitUntiedSwitch(CGF);
5667 InlinedOpenMPRegionRAII Region(CGF, CodeGen, InnerKind, HasCancel,
5668 InnerKind != OMPD_critical &&
5669 InnerKind != OMPD_master &&
5670 InnerKind != OMPD_masked);
5685 RTCancelKind CancelKind = CancelNoreq;
5686 if (CancelRegion == OMPD_parallel)
5687 CancelKind = CancelParallel;
5688 else if (CancelRegion == OMPD_for)
5689 CancelKind = CancelLoop;
5690 else if (CancelRegion == OMPD_sections)
5691 CancelKind = CancelSections;
5693 assert(CancelRegion == OMPD_taskgroup);
5694 CancelKind = CancelTaskgroup;
5706 if (
auto *OMPRegionInfo =
5710 if (CancelRegion == OMPD_taskgroup || OMPRegionInfo->hasCancel()) {
5711 llvm::Value *Args[] = {
5726 CGF.
Builder.CreateCondBr(Cmp, ExitBB, ContBB);
5728 if (CancelRegion == OMPD_parallel)
5747 if (
auto *OMPRegionInfo =
5749 auto &&ThenGen = [
this, &M,
Loc, CancelRegion,
5752 llvm::Value *Args[] = {
5756 llvm::Value *
Result = CGF.EmitRuntimeCall(
5757 OMPBuilder.getOrCreateRuntimeFunction(M, OMPRTL___kmpc_cancel), Args);
5762 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(
".cancel.exit");
5763 llvm::BasicBlock *ContBB = CGF.createBasicBlock(
".cancel.continue");
5764 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(
Result);
5765 CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB);
5766 CGF.EmitBlock(ExitBB);
5767 if (CancelRegion == OMPD_parallel)
5771 CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind());
5772 CGF.EmitBranchThroughCleanup(CancelDest);
5773 CGF.EmitBlock(ContBB,
true);
5791 OMPUsesAllocatorsActionTy(
5792 ArrayRef<std::pair<const Expr *, const Expr *>> Allocators)
5793 : Allocators(Allocators) {}
5797 for (
const auto &AllocatorData : Allocators) {
5799 CGF, AllocatorData.first, AllocatorData.second);
5805 for (
const auto &AllocatorData : Allocators) {
5807 AllocatorData.first);
5815 llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,
5817 assert(!ParentName.empty() &&
"Invalid target entry parent name!");
5821 for (
unsigned I = 0, E =
C->getNumberOfAllocators(); I < E; ++I) {
5828 OMPUsesAllocatorsActionTy UsesAllocatorAction(Allocators);
5831 IsOffloadEntry, CodeGen);
5835 const Expr *Allocator,
5836 const Expr *AllocatorTraits) {
5837 llvm::Value *ThreadId =
getThreadID(CGF, Allocator->getExprLoc());
5838 ThreadId = CGF.
Builder.CreateIntCast(ThreadId, CGF.
IntTy,
true);
5840 llvm::Value *MemSpaceHandle = llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
5841 llvm::Value *NumTraits = llvm::ConstantInt::get(
5842 CGF.
IntTy, cast<ConstantArrayType>(
5845 .getLimitedValue());
5854 llvm::Value *AllocatorVal =
5857 {ThreadId, MemSpaceHandle, NumTraits, Traits});
5860 cast<DeclRefExpr>(Allocator->IgnoreParenImpCasts())->getDecl()));
5864 Allocator->getType(), Allocator->getExprLoc());
5869 const Expr *Allocator) {
5870 llvm::Value *ThreadId =
getThreadID(CGF, Allocator->getExprLoc());
5871 ThreadId = CGF.
Builder.CreateIntCast(ThreadId, CGF.
IntTy,
true);
5873 llvm::Value *AllocatorVal =
5877 Allocator->getExprLoc());
5880 OMPRTL___kmpc_destroy_allocator),
5881 {ThreadId, AllocatorVal});
5886 int32_t &MinThreadsVal, int32_t &MaxThreadsVal, int32_t &MinTeamsVal,
5887 int32_t &MaxTeamsVal) {
5894 for (
auto *A :
C->getAttrs()) {
5895 int32_t AttrMinThreadsVal = 1, AttrMaxThreadsVal = -1;
5896 int32_t AttrMinBlocksVal = 1, AttrMaxBlocksVal = -1;
5897 if (
auto *
Attr = dyn_cast<CUDALaunchBoundsAttr>(A))
5899 &AttrMinBlocksVal, &AttrMaxBlocksVal);
5900 else if (
auto *
Attr = dyn_cast<AMDGPUFlatWorkGroupSizeAttr>(A))
5902 nullptr,
Attr,
nullptr, &AttrMinThreadsVal,
5903 &AttrMaxThreadsVal);
5907 MinThreadsVal = std::max(MinThreadsVal, AttrMinThreadsVal);
5908 if (AttrMaxThreadsVal > 0)
5909 MaxThreadsVal = MaxThreadsVal > 0
5910 ? std::min(MaxThreadsVal, AttrMaxThreadsVal)
5911 : AttrMaxThreadsVal;
5912 MinTeamsVal = std::max(MinTeamsVal, AttrMinBlocksVal);
5913 if (AttrMaxBlocksVal > 0)
5914 MaxTeamsVal = MaxTeamsVal > 0 ? std::min(MaxTeamsVal, AttrMaxBlocksVal)
5922 llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,
5925 llvm::TargetRegionEntryInfo EntryInfo =
5929 llvm::OpenMPIRBuilder::FunctionGenCallback &&GenerateOutlinedFunction =
5930 [&CGF, &D, &CodeGen](StringRef EntryFnName) {
5933 CGOpenMPTargetRegionInfo CGInfo(CS, CodeGen, EntryFnName);
5938 OMPBuilder.emitTargetRegionFunction(EntryInfo, GenerateOutlinedFunction,
5939 IsOffloadEntry, OutlinedFn, OutlinedFnID);
5947 for (
auto *A :
C->getAttrs()) {
5948 if (
auto *
Attr = dyn_cast<AMDGPUWavesPerEUAttr>(A))
5967 while (
const auto *
C = dyn_cast_or_null<CompoundStmt>(Child)) {
5969 for (
const Stmt *S :
C->body()) {
5970 if (
const auto *E = dyn_cast<Expr>(S)) {
5975 if (isa<AsmStmt>(S) || isa<NullStmt>(S) || isa<OMPFlushDirective>(S) ||
5976 isa<OMPBarrierDirective>(S) || isa<OMPTaskyieldDirective>(S))
5979 if (
const auto *DS = dyn_cast<DeclStmt>(S)) {
5980 if (llvm::all_of(DS->decls(), [](
const Decl *D) {
5981 if (isa<EmptyDecl>(D) || isa<DeclContext>(D) ||
5982 isa<TypeDecl>(D) || isa<PragmaCommentDecl>(D) ||
5983 isa<PragmaDetectMismatchDecl>(D) || isa<UsingDecl>(D) ||
5984 isa<UsingDirectiveDecl>(D) ||
5985 isa<OMPDeclareReductionDecl>(D) ||
5986 isa<OMPThreadPrivateDecl>(D) || isa<OMPAllocateDecl>(D))
5988 const auto *VD = dyn_cast<VarDecl>(D);
5991 return VD->hasGlobalStorage() || !VD->isUsed();
6001 Child = Child->IgnoreContainers();
6008 int32_t &MaxTeamsVal) {
6012 "Expected target-based executable directive.");
6013 switch (DirectiveKind) {
6018 const Stmt *ChildStmt =
6020 if (
const auto *NestedDir =
6021 dyn_cast_or_null<OMPExecutableDirective>(ChildStmt)) {
6024 const Expr *NumTeams =
6029 MinTeamsVal = MaxTeamsVal = Constant->getExtValue();
6032 MinTeamsVal = MaxTeamsVal = 0;
6037 MinTeamsVal = MaxTeamsVal = 1;
6040 MinTeamsVal = MaxTeamsVal = 1;
6044 MinTeamsVal = MaxTeamsVal = -1;
6047 case OMPD_target_teams_loop:
6048 case OMPD_target_teams:
6049 case OMPD_target_teams_distribute:
6050 case OMPD_target_teams_distribute_simd:
6051 case OMPD_target_teams_distribute_parallel_for:
6052 case OMPD_target_teams_distribute_parallel_for_simd: {
6054 const Expr *NumTeams =
6058 MinTeamsVal = MaxTeamsVal = Constant->getExtValue();
6061 MinTeamsVal = MaxTeamsVal = 0;
6064 case OMPD_target_parallel:
6065 case OMPD_target_parallel_for:
6066 case OMPD_target_parallel_for_simd:
6067 case OMPD_target_parallel_loop:
6068 case OMPD_target_simd:
6069 MinTeamsVal = MaxTeamsVal = 1;
6073 case OMPD_parallel_for:
6074 case OMPD_parallel_loop:
6075 case OMPD_parallel_master:
6076 case OMPD_parallel_sections:
6078 case OMPD_parallel_for_simd:
6080 case OMPD_cancellation_point:
6082 case OMPD_threadprivate:
6093 case OMPD_taskyield:
6096 case OMPD_taskgroup:
6102 case OMPD_target_data:
6103 case OMPD_target_exit_data:
6104 case OMPD_target_enter_data:
6105 case OMPD_distribute:
6106 case OMPD_distribute_simd:
6107 case OMPD_distribute_parallel_for:
6108 case OMPD_distribute_parallel_for_simd:
6109 case OMPD_teams_distribute:
6110 case OMPD_teams_distribute_simd:
6111 case OMPD_teams_distribute_parallel_for:
6112 case OMPD_teams_distribute_parallel_for_simd:
6113 case OMPD_target_update:
6114 case OMPD_declare_simd:
6115 case OMPD_declare_variant:
6116 case OMPD_begin_declare_variant:
6117 case OMPD_end_declare_variant:
6118 case OMPD_declare_target:
6119 case OMPD_end_declare_target:
6120 case OMPD_declare_reduction:
6121 case OMPD_declare_mapper:
6123 case OMPD_taskloop_simd:
6124 case OMPD_master_taskloop:
6125 case OMPD_master_taskloop_simd:
6126 case OMPD_parallel_master_taskloop:
6127 case OMPD_parallel_master_taskloop_simd:
6129 case OMPD_metadirective:
6135 llvm_unreachable(
"Unexpected directive kind.");
6141 "Clauses associated with the teams directive expected to be emitted "
6142 "only for the host!");
6144 int32_t MinNT = -1, MaxNT = -1;
6145 const Expr *NumTeams =
6147 if (NumTeams !=
nullptr) {
6150 switch (DirectiveKind) {
6153 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
6157 return Bld.CreateIntCast(NumTeamsVal, CGF.
Int32Ty,
6160 case OMPD_target_teams:
6161 case OMPD_target_teams_distribute:
6162 case OMPD_target_teams_distribute_simd:
6163 case OMPD_target_teams_distribute_parallel_for:
6164 case OMPD_target_teams_distribute_parallel_for_simd: {
6168 return Bld.CreateIntCast(NumTeamsVal, CGF.
Int32Ty,
6176 assert(MinNT == MaxNT &&
"Num threads ranges require handling here.");
6177 return llvm::ConstantInt::get(CGF.
Int32Ty, MinNT);
6185 const Expr **E, int32_t &UpperBound,
6186 bool UpperBoundOnly, llvm::Value **CondVal) {
6189 const auto *Dir = dyn_cast_or_null<OMPExecutableDirective>(Child);
6196 if (CondVal && Dir->hasClausesOfKind<
OMPIfClause>()) {
6197 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
6198 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
6200 for (
const auto *
C : Dir->getClausesOfKind<
OMPIfClause>()) {
6201 if (
C->getNameModifier() == OMPD_unknown ||
6202 C->getNameModifier() == OMPD_parallel) {
6217 if (
const auto *PreInit =
6219 for (
const auto *I : PreInit->decls()) {
6220 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
6223 CodeGenFunction::AutoVarEmission Emission =
6236 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
6237 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
6238 const auto *NumThreadsClause =
6240 const Expr *NTExpr = NumThreadsClause->getNumThreads();
6241 if (NTExpr->isIntegerConstantExpr(CGF.
getContext()))
6242 if (
auto Constant = NTExpr->getIntegerConstantExpr(CGF.
getContext()))
6245 ? Constant->getZExtValue()
6246 : std::min(UpperBound,
6247 static_cast<int32_t
>(Constant->getZExtValue()));
6250 if (UpperBound == -1)
6254 CodeGenFunction::LexicalScope
Scope(CGF, NTExpr->getSourceRange());
6255 if (
const auto *PreInit =
6256 cast_or_null<DeclStmt>(NumThreadsClause->getPreInitStmt())) {
6257 for (
const auto *I : PreInit->decls()) {
6258 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
6261 CodeGenFunction::AutoVarEmission Emission =
6277 bool UpperBoundOnly, llvm::Value **CondVal,
const Expr **ThreadLimitExpr) {
6278 assert((!CGF.
getLangOpts().OpenMPIsTargetDevice || UpperBoundOnly) &&
6279 "Clauses associated with the teams directive expected to be emitted "
6280 "only for the host!");
6283 "Expected target-based executable directive.");
6285 const Expr *NT =
nullptr;
6286 const Expr **NTPtr = UpperBoundOnly ? nullptr : &NT;
6288 auto CheckForConstExpr = [&](
const Expr *E,
const Expr **EPtr) {
6291 UpperBound = UpperBound ? Constant->getZExtValue()
6292 : std::min(UpperBound,
6293 int32_t(Constant->getZExtValue()));
6297 if (UpperBound == -1)
6303 auto ReturnSequential = [&]() {
6308 switch (DirectiveKind) {
6311 getNumThreads(CGF, CS, NTPtr, UpperBound, UpperBoundOnly, CondVal);
6317 if (
const auto *Dir = dyn_cast_or_null<OMPExecutableDirective>(Child)) {
6319 ThreadLimitClause = TLC;
6320 if (ThreadLimitExpr) {
6321 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
6324 CGF, ThreadLimitClause->getThreadLimit()->getSourceRange());
6325 if (
const auto *PreInit =
6326 cast_or_null<DeclStmt>(ThreadLimitClause->getPreInitStmt())) {
6327 for (
const auto *I : PreInit->decls()) {
6328 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
6340 if (ThreadLimitClause)
6341 CheckForConstExpr(ThreadLimitClause->getThreadLimit(), ThreadLimitExpr);
6342 if (
const auto *Dir = dyn_cast_or_null<OMPExecutableDirective>(Child)) {
6345 CS = Dir->getInnermostCapturedStmt();
6348 Dir = dyn_cast_or_null<OMPExecutableDirective>(Child);
6351 CS = Dir->getInnermostCapturedStmt();
6352 getNumThreads(CGF, CS, NTPtr, UpperBound, UpperBoundOnly, CondVal);
6354 return ReturnSequential();
6358 case OMPD_target_teams: {
6362 CheckForConstExpr(ThreadLimitClause->getThreadLimit(), ThreadLimitExpr);
6365 getNumThreads(CGF, CS, NTPtr, UpperBound, UpperBoundOnly, CondVal);
6368 if (
const auto *Dir = dyn_cast_or_null<OMPExecutableDirective>(Child)) {
6369 if (Dir->getDirectiveKind() == OMPD_distribute) {
6370 CS = Dir->getInnermostCapturedStmt();
6371 getNumThreads(CGF, CS, NTPtr, UpperBound, UpperBoundOnly, CondVal);
6376 case OMPD_target_teams_distribute:
6380 CheckForConstExpr(ThreadLimitClause->getThreadLimit(), ThreadLimitExpr);
6383 UpperBoundOnly, CondVal);
6385 case OMPD_target_teams_loop:
6386 case OMPD_target_parallel_loop:
6387 case OMPD_target_parallel:
6388 case OMPD_target_parallel_for:
6389 case OMPD_target_parallel_for_simd:
6390 case OMPD_target_teams_distribute_parallel_for:
6391 case OMPD_target_teams_distribute_parallel_for_simd: {
6395 if (
C->getNameModifier() == OMPD_unknown ||
6396 C->getNameModifier() == OMPD_parallel) {
6406 return ReturnSequential();
6416 CheckForConstExpr(ThreadLimitClause->getThreadLimit(), ThreadLimitExpr);
6421 CheckForConstExpr(NumThreadsClause->getNumThreads(),
nullptr);
6422 return NumThreadsClause->getNumThreads();
6426 case OMPD_target_teams_distribute_simd:
6427 case OMPD_target_simd:
6428 return ReturnSequential();
6432 llvm_unreachable(
"Unsupported directive kind.");
6437 llvm::Value *NumThreadsVal =
nullptr;
6438 llvm::Value *CondVal =
nullptr;
6439 llvm::Value *ThreadLimitVal =
nullptr;
6440 const Expr *ThreadLimitExpr =
nullptr;
6441 int32_t UpperBound = -1;
6444 CGF, D, UpperBound,
false, &CondVal,
6448 if (ThreadLimitExpr) {
6451 ThreadLimitVal = CGF.
Builder.CreateIntCast(ThreadLimitVal, CGF.
Int32Ty,
6456 if (UpperBound == 1) {
6457 NumThreadsVal = CGF.
Builder.getInt32(UpperBound);
6460 NumThreadsVal = CGF.
Builder.CreateIntCast(NumThreadsVal, CGF.
Int32Ty,
6462 }
else if (ThreadLimitVal) {
6465 NumThreadsVal = ThreadLimitVal;
6466 ThreadLimitVal =
nullptr;
6469 assert(!ThreadLimitVal &&
"Default not applicable with thread limit value");
6470 NumThreadsVal = CGF.
Builder.getInt32(0);
6477 NumThreadsVal = CGF.
Builder.CreateSelect(CondVal, NumThreadsVal,
6483 if (ThreadLimitVal) {
6484 NumThreadsVal = CGF.
Builder.CreateSelect(
6485 CGF.
Builder.CreateICmpULT(ThreadLimitVal, NumThreadsVal),
6486 ThreadLimitVal, NumThreadsVal);
6489 return NumThreadsVal;
6499class MappableExprsHandler {
6502 static unsigned getFlagMemberOffset() {
6503 unsigned Offset = 0;
6504 for (uint64_t Remain =
6505 static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>
>(
6506 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF);
6507 !(Remain & 1); Remain = Remain >> 1)
6514 class MappingExprInfo {
6519 const Expr *MapExpr =
nullptr;
6522 MappingExprInfo(
const ValueDecl *MapDecl,
const Expr *MapExpr =
nullptr)
6523 : MapDecl(MapDecl), MapExpr(MapExpr) {}
6525 const ValueDecl *getMapDecl()
const {
return MapDecl; }
6526 const Expr *getMapExpr()
const {
return MapExpr; }
6529 using DeviceInfoTy = llvm::OpenMPIRBuilder::DeviceInfoTy;
6530 using MapBaseValuesArrayTy = llvm::OpenMPIRBuilder::MapValuesArrayTy;
6531 using MapValuesArrayTy = llvm::OpenMPIRBuilder::MapValuesArrayTy;
6532 using MapFlagsArrayTy = llvm::OpenMPIRBuilder::MapFlagsArrayTy;
6533 using MapDimArrayTy = llvm::OpenMPIRBuilder::MapDimArrayTy;
6534 using MapNonContiguousArrayTy =
6535 llvm::OpenMPIRBuilder::MapNonContiguousArrayTy;
6542 struct MapCombinedInfoTy : llvm::OpenMPIRBuilder::MapInfosTy {
6543 MapExprsArrayTy Exprs;
6544 MapValueDeclsArrayTy Mappers;
6545 MapValueDeclsArrayTy DevicePtrDecls;
6548 void append(MapCombinedInfoTy &CurInfo) {
6549 Exprs.append(CurInfo.Exprs.begin(), CurInfo.Exprs.end());
6550 DevicePtrDecls.append(CurInfo.DevicePtrDecls.begin(),
6551 CurInfo.DevicePtrDecls.end());
6552 Mappers.append(CurInfo.Mappers.begin(), CurInfo.Mappers.end());
6553 llvm::OpenMPIRBuilder::MapInfosTy::append(CurInfo);
6561 struct StructRangeInfoTy {
6562 MapCombinedInfoTy PreliminaryMapData;
6563 std::pair<
unsigned ,
Address > LowestElem = {
6565 std::pair<
unsigned ,
Address > HighestElem = {
6569 bool IsArraySection =
false;
6570 bool HasCompleteRecord =
false;
6580 bool ReturnDevicePointer =
false;
6581 bool IsImplicit =
false;
6583 const Expr *VarRef =
nullptr;
6584 bool ForDeviceAddr =
false;
6586 MapInfo() =
default;
6592 bool ReturnDevicePointer,
bool IsImplicit,
6593 const ValueDecl *Mapper =
nullptr,
const Expr *VarRef =
nullptr,
6594 bool ForDeviceAddr =
false)
6595 : Components(Components), MapType(MapType), MapModifiers(MapModifiers),
6596 MotionModifiers(MotionModifiers),
6597 ReturnDevicePointer(ReturnDevicePointer), IsImplicit(IsImplicit),
6598 Mapper(Mapper), VarRef(VarRef), ForDeviceAddr(ForDeviceAddr) {}
6604 struct DeferredDevicePtrEntryTy {
6605 const Expr *IE =
nullptr;
6607 bool ForDeviceAddr =
false;
6609 DeferredDevicePtrEntryTy(
const Expr *IE,
const ValueDecl *VD,
6611 : IE(IE), VD(VD), ForDeviceAddr(ForDeviceAddr) {}
6626 llvm::DenseMap<CanonicalDeclPtr<const VarDecl>,
bool> FirstPrivateDecls;
6643 llvm::DenseMap<const ValueDecl *, const OMPMapClause *> LambdasMap;
6645 llvm::Value *getExprTypeSize(
const Expr *E)
const {
6649 if (
const auto *OAE = dyn_cast<OMPArrayShapingExpr>(E)) {
6651 CGF.
getTypeSize(OAE->getBase()->getType()->getPointeeType());
6652 for (
const Expr *SE : OAE->getDimensions()) {
6669 if (
const auto *OAE = dyn_cast<ArraySectionExpr>(E)) {
6671 OAE->getBase()->IgnoreParenImpCasts())
6677 if (!OAE->getLength() && OAE->getColonLocFirst().isValid() &&
6678 !OAE->getLowerBound())
6681 llvm::Value *ElemSize;
6683 ElemSize = CGF.
getTypeSize(PTy->getPointeeType().getCanonicalType());
6685 const auto *ATy = cast<ArrayType>(BaseTy.
getTypePtr());
6686 assert(ATy &&
"Expecting array type if not a pointer type.");
6687 ElemSize = CGF.
getTypeSize(ATy->getElementType().getCanonicalType());
6692 if (!OAE->getLength() && OAE->getColonLocFirst().isInvalid())
6695 if (
const Expr *LenExpr = OAE->getLength()) {
6699 LenExpr->getExprLoc());
6700 return CGF.
Builder.CreateNUWMul(LengthVal, ElemSize);
6702 assert(!OAE->getLength() && OAE->getColonLocFirst().isValid() &&
6703 OAE->getLowerBound() &&
"expected array_section[lb:].");
6709 OAE->getLowerBound()->getExprLoc());
6710 LBVal = CGF.
Builder.CreateNUWMul(LBVal, ElemSize);
6711 llvm::Value *Cmp = CGF.
Builder.CreateICmpUGT(LengthVal, LBVal);
6712 llvm::Value *TrueVal = CGF.
Builder.CreateNUWSub(LengthVal, LBVal);
6713 LengthVal = CGF.
Builder.CreateSelect(
6714 Cmp, TrueVal, llvm::ConstantInt::get(CGF.
SizeTy, 0));
6724 OpenMPOffloadMappingFlags getMapTypeBits(
6727 bool AddPtrFlag,
bool AddIsTargetParamFlag,
bool IsNonContiguous)
const {
6728 OpenMPOffloadMappingFlags Bits =
6729 IsImplicit ? OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT
6730 : OpenMPOffloadMappingFlags::OMP_MAP_NONE;
6732 case OMPC_MAP_alloc:
6733 case OMPC_MAP_release:
6740 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_TO;
6743 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_FROM;
6745 case OMPC_MAP_tofrom:
6746 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_TO |
6747 OpenMPOffloadMappingFlags::OMP_MAP_FROM;
6749 case OMPC_MAP_delete:
6750 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_DELETE;
6753 llvm_unreachable(
"Unexpected map type!");
6756 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ;
6757 if (AddIsTargetParamFlag)
6758 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM;
6759 if (llvm::is_contained(MapModifiers, OMPC_MAP_MODIFIER_always))
6760 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_ALWAYS;
6761 if (llvm::is_contained(MapModifiers, OMPC_MAP_MODIFIER_close))
6762 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_CLOSE;
6763 if (llvm::is_contained(MapModifiers, OMPC_MAP_MODIFIER_present) ||
6764 llvm::is_contained(MotionModifiers, OMPC_MOTION_MODIFIER_present))
6765 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_PRESENT;
6766 if (llvm::is_contained(MapModifiers, OMPC_MAP_MODIFIER_ompx_hold))
6767 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_OMPX_HOLD;
6768 if (IsNonContiguous)
6769 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_NON_CONTIG;
6775 bool isFinalArraySectionExpression(
const Expr *E)
const {
6776 const auto *OASE = dyn_cast<ArraySectionExpr>(E);
6783 if (OASE->getColonLocFirst().isInvalid())
6786 const Expr *Length = OASE->getLength();
6793 OASE->getBase()->IgnoreParenImpCasts())
6795 if (
const auto *ATy = dyn_cast<ConstantArrayType>(BaseQTy.
getTypePtr()))
6796 return ATy->getSExtSize() != 1;
6805 if (!Length->EvaluateAsInt(Result, CGF.
getContext()))
6808 llvm::APSInt ConstLength = Result.Val.getInt();
6809 return ConstLength.getSExtValue() != 1;
6817 void generateInfoForComponentList(
6821 MapCombinedInfoTy &CombinedInfo,
6822 MapCombinedInfoTy &StructBaseCombinedInfo,
6823 StructRangeInfoTy &PartialStruct,
bool IsFirstComponentList,
6824 bool IsImplicit,
bool GenerateAllInfoForClauses,
6825 const ValueDecl *Mapper =
nullptr,
bool ForDeviceAddr =
false,
6826 const ValueDecl *BaseDecl =
nullptr,
const Expr *MapExpr =
nullptr,
6828 OverlappedElements = std::nullopt,
6829 bool AreBothBasePtrAndPteeMapped =
false)
const {
7011 bool IsCaptureFirstInfo = IsFirstComponentList;
7015 bool RequiresReference =
false;
7018 auto CI = Components.rbegin();
7019 auto CE = Components.rend();
7024 bool IsExpressionFirstInfo =
true;
7025 bool FirstPointerInComplexData =
false;
7027 const Expr *AssocExpr = I->getAssociatedExpression();
7028 const auto *AE = dyn_cast<ArraySubscriptExpr>(AssocExpr);
7029 const auto *OASE = dyn_cast<ArraySectionExpr>(AssocExpr);
7030 const auto *OAShE = dyn_cast<OMPArrayShapingExpr>(AssocExpr);
7032 if (AreBothBasePtrAndPteeMapped && std::next(I) == CE)
7034 if (isa<MemberExpr>(AssocExpr)) {
7038 }
else if ((AE && isa<CXXThisExpr>(AE->getBase()->IgnoreParenImpCasts())) ||
7040 isa<CXXThisExpr>(OASE->getBase()->IgnoreParenImpCasts()))) {
7043 isa<CXXThisExpr>(OAShE->getBase()->IgnoreParenCasts())) {
7052 if (
const auto *VD =
7053 dyn_cast_or_null<VarDecl>(I->getAssociatedDeclaration())) {
7054 if (std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
7055 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD)) {
7056 if ((*Res == OMPDeclareTargetDeclAttr::MT_Link) ||
7057 ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
7058 *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
7060 RequiresReference =
true;
7070 I->getAssociatedDeclaration()->getType().getNonReferenceType();
7075 const auto *VD = dyn_cast<VarDecl>(I->getAssociatedDeclaration());
7076 if (!AreBothBasePtrAndPteeMapped &&
7081 FirstPointerInComplexData =
true;
7100 bool ShouldBeMemberOf =
false;
7115 bool IsNonContiguous = CombinedInfo.NonContigInfo.IsNonContiguous;
7116 bool IsPrevMemberReference =
false;
7123 bool IsMappingWholeStruct =
true;
7124 if (!GenerateAllInfoForClauses) {
7125 IsMappingWholeStruct =
false;
7127 for (
auto TempI = I; TempI != CE; ++TempI) {
7129 dyn_cast<MemberExpr>(TempI->getAssociatedExpression());
7131 IsMappingWholeStruct =
false;
7137 for (; I != CE; ++I) {
7139 if (!EncounteredME) {
7140 EncounteredME = dyn_cast<MemberExpr>(I->getAssociatedExpression());
7143 if (EncounteredME) {
7144 ShouldBeMemberOf =
true;
7147 if (FirstPointerInComplexData) {
7149 ->getAssociatedDeclaration()
7151 .getNonReferenceType();
7153 FirstPointerInComplexData =
false;
7158 auto Next = std::next(I);
7168 bool IsFinalArraySection =
7170 isFinalArraySectionExpression(I->getAssociatedExpression());
7174 const ValueDecl *MapDecl = (I->getAssociatedDeclaration())
7175 ? I->getAssociatedDeclaration()
7177 MapExpr = (I->getAssociatedExpression()) ? I->getAssociatedExpression()
7184 dyn_cast<ArraySectionExpr>(I->getAssociatedExpression());
7186 dyn_cast<OMPArrayShapingExpr>(I->getAssociatedExpression());
7187 const auto *UO = dyn_cast<UnaryOperator>(I->getAssociatedExpression());
7188 const auto *BO = dyn_cast<BinaryOperator>(I->getAssociatedExpression());
7194 I->getAssociatedExpression()->getType()->isAnyPointerType();
7195 bool IsMemberReference = isa<MemberExpr>(I->getAssociatedExpression()) &&
7198 bool IsNonDerefPointer = IsPointer &&
7199 !(UO && UO->getOpcode() != UO_Deref) && !BO &&
7205 if (Next == CE || IsMemberReference || IsNonDerefPointer ||
7206 IsFinalArraySection) {
7209 assert((Next == CE ||
7210 isa<MemberExpr>(Next->getAssociatedExpression()) ||
7211 isa<ArraySubscriptExpr>(Next->getAssociatedExpression()) ||
7212 isa<ArraySectionExpr>(Next->getAssociatedExpression()) ||
7213 isa<OMPArrayShapingExpr>(Next->getAssociatedExpression()) ||
7214 isa<UnaryOperator>(Next->getAssociatedExpression()) ||
7215 isa<BinaryOperator>(Next->getAssociatedExpression())) &&
7216 "Unexpected expression");
7222 const Expr *BaseExpr = E->getBase();
7242 OAShE->getBase()->getType()->getPointeeType()),
7244 OAShE->getBase()->getType()));
7245 }
else if (IsMemberReference) {
7246 const auto *ME = cast<MemberExpr>(I->getAssociatedExpression());
7247 LValue BaseLVal = EmitMemberExprBase(CGF, ME);
7249 BaseLVal, cast<FieldDecl>(MapDecl))
7262 bool IsMemberPointerOrAddr =
7264 (((IsPointer || ForDeviceAddr) &&
7265 I->getAssociatedExpression() == EncounteredME) ||
7266 (IsPrevMemberReference && !IsPointer) ||
7267 (IsMemberReference && Next != CE &&
7268 !Next->getAssociatedExpression()->getType()->isPointerType()));
7269 if (!OverlappedElements.empty() && Next == CE) {
7271 assert(!PartialStruct.Base.isValid() &&
"The base element is set.");
7272 assert(!IsPointer &&
7273 "Unexpected base element with the pointer type.");
7276 PartialStruct.LowestElem = {0, LowestElem};
7278 I->getAssociatedExpression()->getType());
7283 PartialStruct.HighestElem = {
7284 std::numeric_limits<
decltype(
7285 PartialStruct.HighestElem.first)>
::max(),
7287 PartialStruct.Base = BP;
7288 PartialStruct.LB = LB;
7290 PartialStruct.PreliminaryMapData.BasePointers.empty() &&
7291 "Overlapped elements must be used only once for the variable.");
7292 std::swap(PartialStruct.PreliminaryMapData, CombinedInfo);
7294 OpenMPOffloadMappingFlags Flags =
7295 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF |
7296 getMapTypeBits(MapType, MapModifiers, MotionModifiers, IsImplicit,
7298 false, IsNonContiguous);
7299 llvm::Value *
Size =
nullptr;
7302 Component : OverlappedElements) {
7306 if (
const ValueDecl *VD = MC.getAssociatedDeclaration()) {
7307 const auto *FD = dyn_cast<FieldDecl>(VD);
7308 if (FD && FD->getType()->isLValueReferenceType()) {
7310 cast<MemberExpr>(MC.getAssociatedExpression());
7311 LValue BaseLVal = EmitMemberExprBase(CGF, ME);
7327 assert(Size &&
"Failed to determine structure size");
7328 CombinedInfo.Exprs.emplace_back(MapDecl, MapExpr);
7330 CombinedInfo.DevicePtrDecls.push_back(
nullptr);
7331 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
7333 CombinedInfo.Sizes.push_back(CGF.
Builder.CreateIntCast(
7335 CombinedInfo.Types.push_back(Flags);
7336 CombinedInfo.Mappers.push_back(
nullptr);
7337 CombinedInfo.NonContigInfo.Dims.push_back(IsNonContiguous ? DimSize
7341 CombinedInfo.Exprs.emplace_back(MapDecl, MapExpr);
7343 CombinedInfo.DevicePtrDecls.push_back(
nullptr);
7344 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
7350 CombinedInfo.Sizes.push_back(
7352 CombinedInfo.Types.push_back(Flags);
7353 CombinedInfo.Mappers.push_back(
nullptr);
7354 CombinedInfo.NonContigInfo.Dims.push_back(IsNonContiguous ? DimSize
7358 llvm::Value *
Size = getExprTypeSize(I->getAssociatedExpression());
7363 if (!IsMemberPointerOrAddr ||
7365 if (!IsMappingWholeStruct) {
7366 CombinedInfo.Exprs.emplace_back(MapDecl, MapExpr);
7368 CombinedInfo.DevicePtrDecls.push_back(
nullptr);
7369 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
7371 CombinedInfo.Sizes.push_back(CGF.
Builder.CreateIntCast(
7373 CombinedInfo.NonContigInfo.Dims.push_back(IsNonContiguous ? DimSize
7376 StructBaseCombinedInfo.Exprs.emplace_back(MapDecl, MapExpr);
7377 StructBaseCombinedInfo.BasePointers.push_back(
7379 StructBaseCombinedInfo.DevicePtrDecls.push_back(
nullptr);
7380 StructBaseCombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
7381 StructBaseCombinedInfo.Pointers.push_back(LB.
emitRawPointer(CGF));
7382 StructBaseCombinedInfo.Sizes.push_back(CGF.
Builder.CreateIntCast(
7384 StructBaseCombinedInfo.NonContigInfo.Dims.push_back(
7385 IsNonContiguous ? DimSize : 1);
7389 bool HasMapper = Mapper && Next == CE;
7390 if (!IsMappingWholeStruct)
7391 CombinedInfo.Mappers.push_back(HasMapper ? Mapper :
nullptr);
7393 StructBaseCombinedInfo.Mappers.push_back(HasMapper ? Mapper
7400 OpenMPOffloadMappingFlags Flags =
7401 getMapTypeBits(MapType, MapModifiers, MotionModifiers, IsImplicit,
7402 !IsExpressionFirstInfo || RequiresReference ||
7403 FirstPointerInComplexData || IsMemberReference,
7404 AreBothBasePtrAndPteeMapped ||
7405 (IsCaptureFirstInfo && !RequiresReference),
7408 if (!IsExpressionFirstInfo || IsMemberReference) {
7411 if (IsPointer || (IsMemberReference && Next != CE))
7412 Flags &= ~(OpenMPOffloadMappingFlags::OMP_MAP_TO |
7413 OpenMPOffloadMappingFlags::OMP_MAP_FROM |
7414 OpenMPOffloadMappingFlags::OMP_MAP_ALWAYS |
7415 OpenMPOffloadMappingFlags::OMP_MAP_DELETE |
7416 OpenMPOffloadMappingFlags::OMP_MAP_CLOSE);
7418 if (ShouldBeMemberOf) {
7421 Flags |= OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF;
7424 ShouldBeMemberOf =
false;
7428 if (!IsMappingWholeStruct)
7429 CombinedInfo.Types.push_back(Flags);
7431 StructBaseCombinedInfo.Types.push_back(Flags);
7437 if (EncounteredME) {
7438 const auto *FD = cast<FieldDecl>(EncounteredME->
getMemberDecl());
7439 unsigned FieldIndex = FD->getFieldIndex();
7442 if (!PartialStruct.Base.isValid()) {
7443 PartialStruct.LowestElem = {FieldIndex, LowestElem};
7444 if (IsFinalArraySection) {
7448 PartialStruct.HighestElem = {FieldIndex, HB};
7450 PartialStruct.HighestElem = {FieldIndex, LowestElem};
7452 PartialStruct.Base = BP;
7453 PartialStruct.LB = BP;
7454 }
else if (FieldIndex < PartialStruct.LowestElem.first) {
7455 PartialStruct.LowestElem = {FieldIndex, LowestElem};
7456 }
else if (FieldIndex > PartialStruct.HighestElem.first) {
7457 if (IsFinalArraySection) {
7461 PartialStruct.HighestElem = {FieldIndex, HB};
7463 PartialStruct.HighestElem = {FieldIndex, LowestElem};
7469 if (IsFinalArraySection || IsNonContiguous)
7470 PartialStruct.IsArraySection =
true;
7473 if (IsFinalArraySection)
7478 BP = IsMemberReference ? LowestElem : LB;
7480 IsExpressionFirstInfo =
false;
7481 IsCaptureFirstInfo =
false;
7482 FirstPointerInComplexData =
false;
7483 IsPrevMemberReference = IsMemberReference;
7484 }
else if (FirstPointerInComplexData) {
7486 ->getAssociatedDeclaration()
7488 .getNonReferenceType();
7490 FirstPointerInComplexData =
false;
7496 PartialStruct.HasCompleteRecord =
true;
7498 if (!IsNonContiguous)
7505 MapValuesArrayTy CurOffsets = {llvm::ConstantInt::get(CGF.
CGM.
Int64Ty, 0)};
7506 MapValuesArrayTy CurCounts = {llvm::ConstantInt::get(CGF.
CGM.
Int64Ty, 1)};
7507 MapValuesArrayTy CurStrides;
7508 MapValuesArrayTy DimSizes{llvm::ConstantInt::get(CGF.
CGM.
Int64Ty, 1)};
7516 const Expr *AssocExpr = Component.getAssociatedExpression();
7517 const auto *OASE = dyn_cast<ArraySectionExpr>(AssocExpr);
7527 assert((VAT || CAT || &Component == &*Components.begin()) &&
7528 "Should be either ConstantArray or VariableArray if not the "
7532 if (CurStrides.empty()) {
7533 const Type *ElementType =
nullptr;
7535 ElementType = CAT->getElementType().getTypePtr();
7537 ElementType = VAT->getElementType().getTypePtr();
7539 assert(&Component == &*Components.begin() &&
7540 "Only expect pointer (non CAT or VAT) when this is the "
7548 if (&Component != &*Components.begin())
7552 CurStrides.push_back(
7553 llvm::ConstantInt::get(CGF.
Int64Ty, ElementTypeSize));
7558 if (DimSizes.size() < Components.size() - 1) {
7561 llvm::ConstantInt::get(CGF.
Int64Ty, CAT->getZExtSize()));
7563 DimSizes.push_back(CGF.
Builder.CreateIntCast(
7570 auto *DI = DimSizes.begin() + 1;
7572 llvm::Value *DimProd =
7573 llvm::ConstantInt::get(CGF.
CGM.
Int64Ty, ElementTypeSize);
7584 const Expr *AssocExpr = Component.getAssociatedExpression();
7586 if (
const auto *AE = dyn_cast<ArraySubscriptExpr>(AssocExpr)) {
7587 llvm::Value *Offset = CGF.
Builder.CreateIntCast(
7590 CurOffsets.push_back(Offset);
7591 CurCounts.push_back(llvm::ConstantInt::get(CGF.
Int64Ty, 1));
7592 CurStrides.push_back(CurStrides.back());
7596 const auto *OASE = dyn_cast<ArraySectionExpr>(AssocExpr);
7602 const Expr *OffsetExpr = OASE->getLowerBound();
7603 llvm::Value *Offset =
nullptr;
7606 Offset = llvm::ConstantInt::get(CGF.
Int64Ty, 0);
7612 CurOffsets.push_back(Offset);
7615 const Expr *CountExpr = OASE->getLength();
7616 llvm::Value *Count =
nullptr;
7622 if (!OASE->getColonLocFirst().isValid() &&
7623 !OASE->getColonLocSecond().isValid()) {
7624 Count = llvm::ConstantInt::get(CGF.
Int64Ty, 1);
7630 const Expr *StrideExpr = OASE->getStride();
7631 llvm::Value *Stride =
7637 Count = CGF.
Builder.CreateUDiv(
7638 CGF.
Builder.CreateNUWSub(*DI, Offset), Stride);
7640 Count = CGF.
Builder.CreateNUWSub(*DI, Offset);
7646 CurCounts.push_back(Count);
7655 const Expr *StrideExpr = OASE->getStride();
7656 llvm::Value *Stride =
7661 DimProd = CGF.
Builder.CreateNUWMul(DimProd, *(DI - 1));
7663 CurStrides.push_back(CGF.
Builder.CreateNUWMul(DimProd, Stride));
7665 CurStrides.push_back(DimProd);
7666 if (DI != DimSizes.end())
7670 CombinedInfo.NonContigInfo.Offsets.push_back(CurOffsets);
7671 CombinedInfo.NonContigInfo.Counts.push_back(CurCounts);
7672 CombinedInfo.NonContigInfo.Strides.push_back(CurStrides);
7678 OpenMPOffloadMappingFlags
7687 return OpenMPOffloadMappingFlags::OMP_MAP_TO |
7688 OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ;
7689 return OpenMPOffloadMappingFlags::OMP_MAP_PRIVATE |
7690 OpenMPOffloadMappingFlags::OMP_MAP_TO;
7693 if (I != LambdasMap.end())
7695 return getMapTypeBits(
7696 I->getSecond()->getMapType(), I->getSecond()->getMapTypeModifiers(),
7697 std::nullopt, I->getSecond()->isImplicit(),
7701 return OpenMPOffloadMappingFlags::OMP_MAP_TO |
7702 OpenMPOffloadMappingFlags::OMP_MAP_FROM;
7707 bool AsBase)
const {
7710 llvm::StructType *St =
7713 unsigned NumElements = St->getNumElements();
7715 llvm::PointerUnion<const CXXRecordDecl *, const FieldDecl *>, 4>
7716 RecordLayout(NumElements);
7719 for (
const auto &I : RD->
bases()) {
7722 const auto *
Base = I.getType()->getAsCXXRecordDecl();
7731 RecordLayout[FieldIndex] =
Base;
7734 for (
const auto &I : RD->
vbases()) {
7735 const auto *
Base = I.getType()->getAsCXXRecordDecl();
7737 if (
Base->isEmpty())
7740 if (RecordLayout[FieldIndex])
7742 RecordLayout[FieldIndex] =
Base;
7745 assert(!RD->
isUnion() &&
"Unexpected union.");
7746 for (
const auto *Field : RD->
fields()) {
7751 RecordLayout[FieldIndex] =
Field;
7754 for (
const llvm::PointerUnion<const CXXRecordDecl *, const FieldDecl *>
7755 &
Data : RecordLayout) {
7759 getPlainLayout(
Base, Layout,
true);
7770 void generateAllInfoForClauses(
7772 llvm::OpenMPIRBuilder &OMPBuilder,
7779 llvm::MapVector<CanonicalDeclPtr<const Decl>,
7786 [&Info, &SkipVarSet](
7792 bool ReturnDevicePointer,
bool IsImplicit,
const ValueDecl *Mapper,
7793 const Expr *VarRef =
nullptr,
bool ForDeviceAddr =
false) {
7794 if (SkipVarSet.contains(D))
7796 auto It = Info.find(D);
7797 if (It == Info.end())
7799 .insert(std::make_pair(
7802 It->second[
Kind].emplace_back(
7803 L, MapType, MapModifiers, MotionModifiers, ReturnDevicePointer,
7804 IsImplicit, Mapper, VarRef, ForDeviceAddr);
7807 for (
const auto *
Cl : Clauses) {
7808 const auto *
C = dyn_cast<OMPMapClause>(
Cl);
7812 if (llvm::is_contained(
C->getMapTypeModifiers(),
7813 OMPC_MAP_MODIFIER_present))
7815 else if (
C->getMapType() == OMPC_MAP_alloc)
7817 const auto *EI =
C->getVarRefs().begin();
7818 for (
const auto L :
C->component_lists()) {
7819 const Expr *E = (
C->getMapLoc().isValid()) ? *EI :
nullptr;
7820 InfoGen(std::get<0>(L), Kind, std::get<1>(L),
C->getMapType(),
7821 C->getMapTypeModifiers(), std::nullopt,
7822 false,
C->isImplicit(), std::get<2>(L),
7827 for (
const auto *
Cl : Clauses) {
7828 const auto *
C = dyn_cast<OMPToClause>(
Cl);
7832 if (llvm::is_contained(
C->getMotionModifiers(),
7833 OMPC_MOTION_MODIFIER_present))
7835 const auto *EI =
C->getVarRefs().begin();
7836 for (
const auto L :
C->component_lists()) {
7837 InfoGen(std::get<0>(L), Kind, std::get<1>(L), OMPC_MAP_to, std::nullopt,
7838 C->getMotionModifiers(),
false,
7839 C->isImplicit(), std::get<2>(L), *EI);
7843 for (
const auto *
Cl : Clauses) {
7844 const auto *
C = dyn_cast<OMPFromClause>(
Cl);
7848 if (llvm::is_contained(
C->getMotionModifiers(),
7849 OMPC_MOTION_MODIFIER_present))
7851 const auto *EI =
C->getVarRefs().begin();
7852 for (
const auto L :
C->component_lists()) {
7853 InfoGen(std::get<0>(L), Kind, std::get<1>(L), OMPC_MAP_from,
7854 std::nullopt,
C->getMotionModifiers(),
7855 false,
C->isImplicit(), std::get<2>(L),
7868 llvm::MapVector<CanonicalDeclPtr<const Decl>,
7871 MapCombinedInfoTy UseDeviceDataCombinedInfo;
7873 auto &&UseDeviceDataCombinedInfoGen =
7874 [&UseDeviceDataCombinedInfo](
const ValueDecl *VD, llvm::Value *Ptr,
7876 UseDeviceDataCombinedInfo.Exprs.push_back(VD);
7877 UseDeviceDataCombinedInfo.BasePointers.emplace_back(Ptr);
7878 UseDeviceDataCombinedInfo.DevicePtrDecls.emplace_back(VD);
7879 UseDeviceDataCombinedInfo.DevicePointers.emplace_back(
7880 IsDevAddr ? DeviceInfoTy::Address : DeviceInfoTy::Pointer);
7881 UseDeviceDataCombinedInfo.Pointers.push_back(Ptr);
7882 UseDeviceDataCombinedInfo.Sizes.push_back(
7883 llvm::Constant::getNullValue(CGF.Int64Ty));
7884 UseDeviceDataCombinedInfo.Types.push_back(
7885 OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM);
7886 UseDeviceDataCombinedInfo.Mappers.push_back(
nullptr);
7890 [&DeferredInfo, &UseDeviceDataCombinedInfoGen,
7894 bool IsImplicit,
bool IsDevAddr) {
7898 if (isa<MemberExpr>(IE)) {
7907 std::nullopt,
false, IsImplicit,
7908 nullptr,
nullptr, IsDevAddr);
7909 DeferredInfo[
nullptr].emplace_back(IE, VD, IsDevAddr);
7913 if (IE->isGLValue())
7914 Ptr = CGF.EmitLValue(IE).getPointer(CGF);
7916 Ptr = CGF.EmitScalarExpr(IE);
7918 Ptr = CGF.EmitLoadOfScalar(CGF.EmitLValue(IE), IE->getExprLoc());
7920 UseDeviceDataCombinedInfoGen(VD, Ptr, CGF, IsDevAddr);
7925 const Expr *IE,
bool IsDevAddr) ->
bool {
7932 auto It = Info.find(isa<MemberExpr>(IE) ? nullptr : VD);
7933 if (It != Info.end()) {
7935 for (
auto &
Data : It->second) {
7936 auto *CI = llvm::find_if(
Data, [VD](
const MapInfo &MI) {
7937 return MI.Components.back().getAssociatedDeclaration() == VD;
7945 if (CI !=
Data.end()) {
7947 CI->ForDeviceAddr = IsDevAddr;
7948 CI->ReturnDevicePointer =
true;
7952 auto PrevCI = std::next(CI->Components.rbegin());
7953 const auto *VarD = dyn_cast<VarDecl>(VD);
7955 isa<MemberExpr>(IE) ||
7956 !VD->getType().getNonReferenceType()->isPointerType() ||
7957 PrevCI == CI->Components.rend() ||
7958 isa<MemberExpr>(PrevCI->getAssociatedExpression()) || !VarD ||
7959 VarD->hasLocalStorage()) {
7960 CI->ForDeviceAddr = IsDevAddr;
7961 CI->ReturnDevicePointer =
true;
7979 for (
const auto *
Cl : Clauses) {
7980 const auto *
C = dyn_cast<OMPUseDevicePtrClause>(
Cl);
7983 for (
const auto L :
C->component_lists()) {
7986 assert(!Components.empty() &&
7987 "Not expecting empty list of components!");
7988 const ValueDecl *VD = Components.back().getAssociatedDeclaration();
7990 const Expr *IE = Components.back().getAssociatedExpression();
7991 if (IsMapInfoExist(CGF, VD, IE,
false))
7993 MapInfoGen(CGF, IE, VD, Components,
C->isImplicit(),
7998 llvm::SmallDenseSet<CanonicalDeclPtr<const Decl>, 4> Processed;
7999 for (
const auto *
Cl : Clauses) {
8000 const auto *
C = dyn_cast<OMPUseDeviceAddrClause>(
Cl);
8003 for (
const auto L :
C->component_lists()) {
8006 assert(!std::get<1>(L).empty() &&
8007 "Not expecting empty list of components!");
8008 const ValueDecl *VD = std::get<1>(L).back().getAssociatedDeclaration();
8009 if (!Processed.insert(VD).second)
8012 const Expr *IE = std::get<1>(L).back().getAssociatedExpression();
8013 if (IsMapInfoExist(CGF, VD, IE,
true))
8015 MapInfoGen(CGF, IE, VD, Components,
C->isImplicit(),
8020 for (
const auto &
Data : Info) {
8021 StructRangeInfoTy PartialStruct;
8023 MapCombinedInfoTy CurInfo;
8025 MapCombinedInfoTy StructBaseCurInfo;
8027 const ValueDecl *VD = cast_or_null<ValueDecl>(D);
8028 for (
const auto &M :
Data.second) {
8029 for (
const MapInfo &L : M) {
8030 assert(!L.Components.empty() &&
8031 "Not expecting declaration with no component lists.");
8034 unsigned CurrentBasePointersIdx = CurInfo.BasePointers.size();
8035 unsigned StructBasePointersIdx =
8036 StructBaseCurInfo.BasePointers.size();
8037 CurInfo.NonContigInfo.IsNonContiguous =
8038 L.Components.back().isNonContiguous();
8039 generateInfoForComponentList(
8040 L.MapType, L.MapModifiers, L.MotionModifiers, L.Components,
8041 CurInfo, StructBaseCurInfo, PartialStruct,
8042 false, L.IsImplicit,
8043 true, L.Mapper, L.ForDeviceAddr, VD,
8048 if (L.ReturnDevicePointer) {
8052 assert((CurrentBasePointersIdx < CurInfo.BasePointers.size() ||
8053 StructBasePointersIdx <
8054 StructBaseCurInfo.BasePointers.size()) &&
8055 "Unexpected number of mapped base pointers.");
8059 L.Components.back().getAssociatedDeclaration();
8060 assert(RelevantVD &&
8061 "No relevant declaration related with device pointer??");
8068 if (StructBasePointersIdx < StructBaseCurInfo.BasePointers.size()) {
8069 StructBaseCurInfo.DevicePtrDecls[StructBasePointersIdx] =
8071 StructBaseCurInfo.DevicePointers[StructBasePointersIdx] =
8072 L.ForDeviceAddr ? DeviceInfoTy::Address
8073 : DeviceInfoTy::Pointer;
8074 StructBaseCurInfo.Types[StructBasePointersIdx] |=
8075 OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM;
8077 CurInfo.DevicePtrDecls[CurrentBasePointersIdx] = RelevantVD;
8078 CurInfo.DevicePointers[CurrentBasePointersIdx] =
8079 L.ForDeviceAddr ? DeviceInfoTy::Address
8080 : DeviceInfoTy::Pointer;
8081 CurInfo.Types[CurrentBasePointersIdx] |=
8082 OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM;
8090 auto CI = DeferredInfo.find(
Data.first);
8091 if (CI != DeferredInfo.end()) {
8092 for (
const DeferredDevicePtrEntryTy &L : CI->second) {
8093 llvm::Value *BasePtr;
8095 if (L.ForDeviceAddr) {
8096 if (L.IE->isGLValue())
8104 CurInfo.Types.push_back(
8105 OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM |
8106 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF);
8110 L.IE->getExprLoc());
8114 CurInfo.Types.push_back(
8115 OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ |
8116 OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM |
8117 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF);
8119 CurInfo.Exprs.push_back(L.VD);
8120 CurInfo.BasePointers.emplace_back(BasePtr);
8121 CurInfo.DevicePtrDecls.emplace_back(L.VD);
8122 CurInfo.DevicePointers.emplace_back(
8123 L.ForDeviceAddr ? DeviceInfoTy::Address : DeviceInfoTy::Pointer);
8124 CurInfo.Pointers.push_back(Ptr);
8125 CurInfo.Sizes.push_back(
8126 llvm::Constant::getNullValue(this->CGF.
Int64Ty));
8127 CurInfo.Mappers.push_back(
nullptr);
8133 MapCombinedInfoTy UnionCurInfo;
8134 UnionCurInfo.append(StructBaseCurInfo);
8135 UnionCurInfo.append(CurInfo);
8139 if (PartialStruct.Base.isValid()) {
8140 UnionCurInfo.NonContigInfo.Dims.push_back(0);
8142 emitCombinedEntry(CombinedInfo, UnionCurInfo.Types, PartialStruct,
8143 !VD, OMPBuilder, VD);
8147 CombinedInfo.append(UnionCurInfo);
8150 CombinedInfo.append(UseDeviceDataCombinedInfo);
8155 : CurDir(&Dir), CGF(CGF) {
8158 for (
const auto *D :
C->varlists())
8159 FirstPrivateDecls.try_emplace(
8160 cast<VarDecl>(cast<DeclRefExpr>(D)->getDecl()),
C->isImplicit());
8163 for (
unsigned I = 0, E =
C->getNumberOfAllocators(); I < E; ++I) {
8165 if (
const auto *DRE = dyn_cast_or_null<DeclRefExpr>(D.
AllocatorTraits))
8166 FirstPrivateDecls.try_emplace(cast<VarDecl>(DRE->getDecl()),
8168 else if (
const auto *VD = dyn_cast<VarDecl>(
8171 FirstPrivateDecls.try_emplace(VD,
true);
8176 for (
auto L :
C->component_lists())
8177 DevPointersMap[std::get<0>(L)].push_back(std::get<1>(L));
8180 for (
auto L :
C->component_lists())
8181 HasDevAddrsMap[std::get<0>(L)].push_back(std::get<1>(L));
8184 if (
C->getMapType() != OMPC_MAP_to)
8186 for (
auto L :
C->component_lists()) {
8188 const auto *RD = VD ? VD->
getType()
8194 LambdasMap.try_emplace(std::get<0>(L),
C);
8201 : CurDir(&Dir), CGF(CGF) {}
8206 void emitCombinedEntry(MapCombinedInfoTy &CombinedInfo,
8207 MapFlagsArrayTy &CurTypes,
8208 const StructRangeInfoTy &PartialStruct,
bool IsMapThis,
8209 llvm::OpenMPIRBuilder &OMPBuilder,
8211 bool NotTargetParams =
true)
const {
8212 if (CurTypes.size() == 1 &&
8213 ((CurTypes.back() & OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF) !=
8214 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF) &&
8215 !PartialStruct.IsArraySection)
8217 Address LBAddr = PartialStruct.LowestElem.second;
8218 Address HBAddr = PartialStruct.HighestElem.second;
8219 if (PartialStruct.HasCompleteRecord) {
8220 LBAddr = PartialStruct.LB;
8221 HBAddr = PartialStruct.LB;
8223 CombinedInfo.Exprs.push_back(VD);
8225 CombinedInfo.BasePointers.push_back(PartialStruct.Base.emitRawPointer(CGF));
8226 CombinedInfo.DevicePtrDecls.push_back(
nullptr);
8227 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
8233 bool HasBaseClass = RD && IsMapThis ? RD->
getNumBases() > 0 :
false;
8243 CombinedInfo.Pointers.push_back(PartialStruct.Base.emitRawPointer(CGF));
8248 CombinedInfo.Sizes.push_back(Size);
8250 CombinedInfo.Pointers.push_back(LB);
8253 llvm::Value *HAddr = CGF.
Builder.CreateConstGEP1_32(
8257 llvm::Value *Diff = CGF.
Builder.CreatePtrDiff(CGF.
Int8Ty, CHAddr, CLAddr);
8260 CombinedInfo.Sizes.push_back(Size);
8262 CombinedInfo.Mappers.push_back(
nullptr);
8264 CombinedInfo.Types.push_back(
8265 NotTargetParams ? OpenMPOffloadMappingFlags::OMP_MAP_NONE
8266 : OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM);
8269 if (CurTypes.end() !=
8270 llvm::find_if(CurTypes, [](OpenMPOffloadMappingFlags
Type) {
8271 return static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
8272 Type & OpenMPOffloadMappingFlags::OMP_MAP_PRESENT);
8274 CombinedInfo.Types.back() |= OpenMPOffloadMappingFlags::OMP_MAP_PRESENT;
8276 (*CurTypes.begin()) &= ~OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM;
8283 if (CurTypes.end() !=
8284 llvm::find_if(CurTypes, [](OpenMPOffloadMappingFlags
Type) {
8285 return static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>
>(
8286 Type & OpenMPOffloadMappingFlags::OMP_MAP_OMPX_HOLD);
8288 CombinedInfo.Types.back() |= OpenMPOffloadMappingFlags::OMP_MAP_OMPX_HOLD;
8289 for (
auto &M : CurTypes)
8290 M |= OpenMPOffloadMappingFlags::OMP_MAP_OMPX_HOLD;
8296 OpenMPOffloadMappingFlags MemberOfFlag =
8297 OMPBuilder.getMemberOfFlag(CombinedInfo.BasePointers.size() - 1);
8298 for (
auto &M : CurTypes)
8299 OMPBuilder.setCorrectMemberOfFlag(M, MemberOfFlag);
8307 void generateAllInfo(
8308 MapCombinedInfoTy &CombinedInfo, llvm::OpenMPIRBuilder &OMPBuilder,
8312 "Expect a executable directive");
8314 generateAllInfoForClauses(CurExecDir->clauses(), CombinedInfo, OMPBuilder,
8321 void generateAllInfoForMapper(MapCombinedInfoTy &CombinedInfo,
8322 llvm::OpenMPIRBuilder &OMPBuilder)
const {
8324 "Expect a declare mapper directive");
8326 generateAllInfoForClauses(CurMapperDir->clauses(), CombinedInfo,
8331 void generateInfoForLambdaCaptures(
8332 const ValueDecl *VD, llvm::Value *Arg, MapCombinedInfoTy &CombinedInfo,
8333 llvm::DenseMap<llvm::Value *, llvm::Value *> &LambdaPointers)
const {
8341 llvm::DenseMap<const ValueDecl *, FieldDecl *> Captures;
8348 LambdaPointers.try_emplace(ThisLVal.
getPointer(CGF),
8350 CombinedInfo.Exprs.push_back(VD);
8351 CombinedInfo.BasePointers.push_back(ThisLVal.
getPointer(CGF));
8352 CombinedInfo.DevicePtrDecls.push_back(
nullptr);
8353 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
8354 CombinedInfo.Pointers.push_back(ThisLValVal.
getPointer(CGF));
8355 CombinedInfo.Sizes.push_back(
8358 CombinedInfo.Types.push_back(
8359 OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ |
8360 OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |
8361 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF |
8362 OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT);
8363 CombinedInfo.Mappers.push_back(
nullptr);
8366 if (!LC.capturesVariable())
8368 const VarDecl *VD = cast<VarDecl>(LC.getCapturedVar());
8371 auto It = Captures.find(VD);
8372 assert(It != Captures.end() &&
"Found lambda capture without field.");
8376 LambdaPointers.try_emplace(VarLVal.
getPointer(CGF),
8378 CombinedInfo.Exprs.push_back(VD);
8379 CombinedInfo.BasePointers.push_back(VarLVal.
getPointer(CGF));
8380 CombinedInfo.DevicePtrDecls.push_back(
nullptr);
8381 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
8382 CombinedInfo.Pointers.push_back(VarLValVal.
getPointer(CGF));
8383 CombinedInfo.Sizes.push_back(CGF.
Builder.CreateIntCast(
8389 LambdaPointers.try_emplace(VarLVal.
getPointer(CGF),
8391 CombinedInfo.Exprs.push_back(VD);
8392 CombinedInfo.BasePointers.push_back(VarLVal.
getPointer(CGF));
8393 CombinedInfo.DevicePtrDecls.push_back(
nullptr);
8394 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
8395 CombinedInfo.Pointers.push_back(VarRVal.
getScalarVal());
8396 CombinedInfo.Sizes.push_back(llvm::ConstantInt::get(CGF.
Int64Ty, 0));
8398 CombinedInfo.Types.push_back(
8399 OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ |
8400 OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |
8401 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF |
8402 OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT);
8403 CombinedInfo.Mappers.push_back(
nullptr);
8408 void adjustMemberOfForLambdaCaptures(
8409 llvm::OpenMPIRBuilder &OMPBuilder,
8410 const llvm::DenseMap<llvm::Value *, llvm::Value *> &LambdaPointers,
8411 MapBaseValuesArrayTy &BasePointers, MapValuesArrayTy &Pointers,
8412 MapFlagsArrayTy &Types)
const {
8413 for (
unsigned I = 0, E = Types.size(); I < E; ++I) {
8415 if (Types[I] != (OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ |
8416 OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |
8417 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF |
8418 OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT))
8420 llvm::Value *BasePtr = LambdaPointers.lookup(BasePointers[I]);
8421 assert(BasePtr &&
"Unable to find base lambda address.");
8423 for (
unsigned J = I; J > 0; --J) {
8424 unsigned Idx = J - 1;
8425 if (Pointers[Idx] != BasePtr)
8430 assert(TgtIdx != -1 &&
"Unable to find parent lambda.");
8434 OpenMPOffloadMappingFlags MemberOfFlag =
8435 OMPBuilder.getMemberOfFlag(TgtIdx);
8436 OMPBuilder.setCorrectMemberOfFlag(Types[I], MemberOfFlag);
8443 llvm::Value *Arg, MapCombinedInfoTy &CombinedInfo,
8444 StructRangeInfoTy &PartialStruct)
const {
8446 "Not expecting to generate map info for a variable array type!");
8455 if (LambdasMap.count(VD))
8461 if (VD && (DevPointersMap.count(VD) || HasDevAddrsMap.count(VD))) {
8462 CombinedInfo.Exprs.push_back(VD);
8463 CombinedInfo.BasePointers.emplace_back(Arg);
8464 CombinedInfo.DevicePtrDecls.emplace_back(VD);
8465 CombinedInfo.DevicePointers.emplace_back(DeviceInfoTy::Pointer);
8466 CombinedInfo.Pointers.push_back(Arg);
8467 CombinedInfo.Sizes.push_back(CGF.
Builder.CreateIntCast(
8470 CombinedInfo.Types.push_back(
8471 OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |
8472 OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM);
8473 CombinedInfo.Mappers.push_back(
nullptr);
8485 auto It = DevPointersMap.find(VD);
8486 if (It != DevPointersMap.end())
8487 for (
const auto &MCL : It->second)
8488 DeclComponentLists.emplace_back(MCL, OMPC_MAP_to,
Unknown,
8491 auto I = HasDevAddrsMap.find(VD);
8492 if (I != HasDevAddrsMap.end())
8493 for (
const auto &MCL : I->second)
8494 DeclComponentLists.emplace_back(MCL, OMPC_MAP_tofrom,
Unknown,
8498 "Expect a executable directive");
8500 bool HasMapBasePtr =
false;
8501 bool HasMapArraySec =
false;
8502 for (
const auto *
C : CurExecDir->getClausesOfKind<
OMPMapClause>()) {
8503 const auto *EI =
C->getVarRefs().begin();
8504 for (
const auto L :
C->decl_component_lists(VD)) {
8507 const Expr *E = (
C->getMapLoc().isValid()) ? *EI :
nullptr;
8509 std::tie(VDecl, Components, Mapper) = L;
8510 assert(VDecl == VD &&
"We got information for the wrong declaration??");
8511 assert(!Components.empty() &&
8512 "Not expecting declaration with no component lists.");
8514 HasMapBasePtr =
true;
8516 (isa<ArraySectionExpr>(E) || isa<ArraySubscriptExpr>(E)))
8517 HasMapArraySec =
true;
8518 DeclComponentLists.emplace_back(Components,
C->getMapType(),
8519 C->getMapTypeModifiers(),
8520 C->isImplicit(), Mapper, E);
8524 llvm::stable_sort(DeclComponentLists, [](
const MapData &LHS,
8525 const MapData &RHS) {
8529 llvm::is_contained(MapModifiers, clang::OMPC_MAP_MODIFIER_present);
8530 bool HasAllocs = MapType == OMPC_MAP_alloc;
8531 MapModifiers = std::get<2>(RHS);
8532 MapType = std::get<1>(LHS);
8534 llvm::is_contained(MapModifiers, clang::OMPC_MAP_MODIFIER_present);
8535 bool HasAllocsR = MapType == OMPC_MAP_alloc;
8536 return (HasPresent && !HasPresentR) || (HasAllocs && !HasAllocsR);
8540 llvm::SmallDenseMap<
8547 for (
const MapData &L : DeclComponentLists) {
8554 std::tie(Components, MapType, MapModifiers, IsImplicit, Mapper, VarRef) =
8557 for (
const MapData &L1 :
ArrayRef(DeclComponentLists).slice(Count)) {
8559 std::tie(Components1, MapType, MapModifiers, IsImplicit, Mapper,
8561 auto CI = Components.rbegin();
8562 auto CE = Components.rend();
8563 auto SI = Components1.rbegin();
8564 auto SE = Components1.rend();
8565 for (; CI != CE && SI != SE; ++CI, ++SI) {
8566 if (CI->getAssociatedExpression()->getStmtClass() !=
8567 SI->getAssociatedExpression()->getStmtClass())
8570 if (CI->getAssociatedDeclaration() != SI->getAssociatedDeclaration())
8575 if (CI == CE || SI == SE) {
8577 if (CI == CE && SI == SE)
8579 const auto It = (SI == SE) ? CI : SI;
8585 if (!isa<MemberExpr>(It->getAssociatedExpression()) ||
8586 (std::prev(It)->getAssociatedDeclaration() &&
8588 ->getAssociatedDeclaration()
8590 ->isPointerType()) ||
8591 (It->getAssociatedDeclaration() &&
8592 It->getAssociatedDeclaration()->getType()->isPointerType() &&
8593 std::next(It) != CE && std::next(It) != SE))
8595 const MapData &BaseData = CI == CE ? L : L1;
8597 SI == SE ? Components : Components1;
8598 auto &OverlappedElements = OverlappedData.FindAndConstruct(&BaseData);
8599 OverlappedElements.getSecond().push_back(SubData);
8605 if (!OverlappedData.empty()) {
8608 while (BaseType != OrigType) {
8614 getPlainLayout(CRD, Layout,
false);
8620 for (
auto &Pair : OverlappedData) {
8627 auto CI = First.rbegin();
8628 auto CE = First.rend();
8629 auto SI = Second.rbegin();
8630 auto SE = Second.rend();
8631 for (; CI != CE && SI != SE; ++CI, ++SI) {
8632 if (CI->getAssociatedExpression()->getStmtClass() !=
8633 SI->getAssociatedExpression()->getStmtClass())
8636 if (CI->getAssociatedDeclaration() !=
8637 SI->getAssociatedDeclaration())
8642 if (CI == CE && SI == SE)
8646 if (CI == CE || SI == SE)
8649 const auto *FD1 = cast<FieldDecl>(CI->getAssociatedDeclaration());
8650 const auto *FD2 = cast<FieldDecl>(SI->getAssociatedDeclaration());
8651 if (FD1->getParent() == FD2->getParent())
8652 return FD1->getFieldIndex() < FD2->getFieldIndex();
8654 llvm::find_if(Layout, [FD1, FD2](
const FieldDecl *FD) {
8655 return FD == FD1 || FD == FD2;
8663 bool IsFirstComponentList =
true;
8664 MapCombinedInfoTy StructBaseCombinedInfo;
8665 for (
const auto &Pair : OverlappedData) {
8666 const MapData &L = *Pair.getFirst();
8673 std::tie(Components, MapType, MapModifiers, IsImplicit, Mapper, VarRef) =
8676 OverlappedComponents = Pair.getSecond();
8677 generateInfoForComponentList(
8678 MapType, MapModifiers, std::nullopt, Components, CombinedInfo,
8679 StructBaseCombinedInfo, PartialStruct, IsFirstComponentList,
8680 IsImplicit,
false, Mapper,
8681 false, VD, VarRef, OverlappedComponents);
8682 IsFirstComponentList =
false;
8685 for (
const MapData &L : DeclComponentLists) {
8692 std::tie(Components, MapType, MapModifiers, IsImplicit, Mapper, VarRef) =
8694 auto It = OverlappedData.find(&L);
8695 if (It == OverlappedData.end())
8696 generateInfoForComponentList(
8697 MapType, MapModifiers, std::nullopt, Components, CombinedInfo,
8698 StructBaseCombinedInfo, PartialStruct, IsFirstComponentList,
8699 IsImplicit,
false, Mapper,
8702 HasMapBasePtr && HasMapArraySec);
8703 IsFirstComponentList =
false;
8711 MapCombinedInfoTy &CombinedInfo)
const {
8712 bool IsImplicit =
true;
8715 CombinedInfo.Exprs.push_back(
nullptr);
8716 CombinedInfo.BasePointers.push_back(CV);
8717 CombinedInfo.DevicePtrDecls.push_back(
nullptr);
8718 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
8719 CombinedInfo.Pointers.push_back(CV);
8721 CombinedInfo.Sizes.push_back(
8725 CombinedInfo.Types.push_back(OpenMPOffloadMappingFlags::OMP_MAP_TO |
8726 OpenMPOffloadMappingFlags::OMP_MAP_FROM);
8730 CombinedInfo.BasePointers.push_back(CV);
8731 CombinedInfo.DevicePtrDecls.push_back(
nullptr);
8732 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
8733 CombinedInfo.Pointers.push_back(CV);
8737 CombinedInfo.Types.push_back(
8738 OpenMPOffloadMappingFlags::OMP_MAP_LITERAL);
8739 CombinedInfo.Sizes.push_back(CGF.
Builder.CreateIntCast(
8744 CombinedInfo.Types.push_back(OpenMPOffloadMappingFlags::OMP_MAP_NONE);
8745 CombinedInfo.Sizes.push_back(llvm::Constant::getNullValue(CGF.
Int64Ty));
8747 auto I = FirstPrivateDecls.find(VD);
8748 if (I != FirstPrivateDecls.end())
8749 IsImplicit = I->getSecond();
8754 CombinedInfo.Sizes.push_back(CGF.
Builder.CreateIntCast(
8759 CombinedInfo.Types.push_back(getMapModifiersForPrivateClauses(CI));
8761 auto I = FirstPrivateDecls.find(VD);
8763 CombinedInfo.BasePointers.push_back(CV);
8764 CombinedInfo.DevicePtrDecls.push_back(
nullptr);
8765 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
8772 CombinedInfo.Pointers.push_back(CV);
8774 if (I != FirstPrivateDecls.end())
8775 IsImplicit = I->getSecond();
8778 CombinedInfo.Types.back() |=
8779 OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM;
8783 CombinedInfo.Types.back() |= OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT;
8786 CombinedInfo.Mappers.push_back(
nullptr);
8798 dyn_cast<MemberExpr>(OASE->getBase()->IgnoreParenImpCasts()))
8799 return ME->getMemberDecl();
8807 MappableExprsHandler::MappingExprInfo &MapExprs) {
8809 uint32_t SrcLocStrSize;
8810 if (!MapExprs.getMapDecl() && !MapExprs.getMapExpr())
8811 return OMPBuilder.getOrCreateDefaultSrcLocStr(SrcLocStrSize);
8814 if (!MapExprs.getMapDecl() && MapExprs.getMapExpr()) {
8818 Loc = MapExprs.getMapExpr()->getExprLoc();
8820 Loc = MapExprs.getMapDecl()->getLocation();
8823 std::string ExprName;
8824 if (MapExprs.getMapExpr()) {
8826 llvm::raw_string_ostream OS(ExprName);
8827 MapExprs.getMapExpr()->printPretty(OS,
nullptr,
P);
8830 ExprName = MapExprs.getMapDecl()->getNameAsString();
8834 return OMPBuilder.getOrCreateSrcLocStr(PLoc.
getFilename(), ExprName,
8843 CodeGenFunction &CGF, MappableExprsHandler::MapCombinedInfoTy &CombinedInfo,
8845 bool IsNonContiguous =
false) {
8849 Info.clearArrayInfo();
8850 Info.NumberOfPtrs = CombinedInfo.BasePointers.size();
8852 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
8855 InsertPointTy CodeGenIP(CGF.
Builder.GetInsertBlock(),
8856 CGF.
Builder.GetInsertPoint());
8858 auto FillInfoMap = [&](MappableExprsHandler::MappingExprInfo &MapExpr) {
8862 llvm::codegenoptions::NoDebugInfo) {
8863 CombinedInfo.Names.resize(CombinedInfo.Exprs.size());
8864 llvm::transform(CombinedInfo.Exprs, CombinedInfo.Names.begin(),
8868 auto DeviceAddrCB = [&](
unsigned int I, llvm::Value *NewDecl) {
8869 if (
const ValueDecl *DevVD = CombinedInfo.DevicePtrDecls[I]) {
8874 auto CustomMapperCB = [&](
unsigned int I) {
8875 llvm::Value *MFunc =
nullptr;
8876 if (CombinedInfo.Mappers[I]) {
8877 Info.HasMapper =
true;
8879 cast<OMPDeclareMapperDecl>(CombinedInfo.Mappers[I]));
8883 OMPBuilder.emitOffloadingArrays(AllocaIP, CodeGenIP, CombinedInfo, Info,
8894 const Stmt *ChildStmt =
8897 if (
const auto *NestedDir =
8898 dyn_cast_or_null<OMPExecutableDirective>(ChildStmt)) {
8906 if (DKind == OMPD_teams) {
8907 Body = NestedDir->getInnermostCapturedStmt()->IgnoreContainers(
8912 if (
const auto *NND =
8913 dyn_cast_or_null<OMPExecutableDirective>(ChildStmt)) {
8914 DKind = NND->getDirectiveKind();
8920 case OMPD_target_teams:
8924 case OMPD_target_parallel:
8925 case OMPD_target_simd:
8926 case OMPD_target_parallel_for:
8927 case OMPD_target_parallel_for_simd:
8929 case OMPD_target_teams_distribute:
8930 case OMPD_target_teams_distribute_simd:
8931 case OMPD_target_teams_distribute_parallel_for:
8932 case OMPD_target_teams_distribute_parallel_for_simd:
8935 case OMPD_parallel_for:
8936 case OMPD_parallel_master:
8937 case OMPD_parallel_sections:
8939 case OMPD_parallel_for_simd:
8941 case OMPD_cancellation_point:
8943 case OMPD_threadprivate:
8954 case OMPD_taskyield:
8957 case OMPD_taskgroup:
8963 case OMPD_target_data:
8964 case OMPD_target_exit_data:
8965 case OMPD_target_enter_data:
8966 case OMPD_distribute:
8967 case OMPD_distribute_simd:
8968 case OMPD_distribute_parallel_for:
8969 case OMPD_distribute_parallel_for_simd:
8970 case OMPD_teams_distribute:
8971 case OMPD_teams_distribute_simd:
8972 case OMPD_teams_distribute_parallel_for:
8973 case OMPD_teams_distribute_parallel_for_simd:
8974 case OMPD_target_update:
8975 case OMPD_declare_simd:
8976 case OMPD_declare_variant:
8977 case OMPD_begin_declare_variant:
8978 case OMPD_end_declare_variant:
8979 case OMPD_declare_target:
8980 case OMPD_end_declare_target:
8981 case OMPD_declare_reduction:
8982 case OMPD_declare_mapper:
8984 case OMPD_taskloop_simd:
8985 case OMPD_master_taskloop:
8986 case OMPD_master_taskloop_simd:
8987 case OMPD_parallel_master_taskloop:
8988 case OMPD_parallel_master_taskloop_simd:
8990 case OMPD_metadirective:
8993 llvm_unreachable(
"Unexpected directive.");
9038 QualType PtrTy =
C.getPointerType(Ty).withRestrict();
9039 QualType Int64Ty =
C.getIntTypeForBitwidth(64,
true);
9040 auto *MapperVarDecl =
9043 CharUnits ElementSize =
C.getTypeSizeInChars(Ty);
9060 Args.push_back(&HandleArg);
9061 Args.push_back(&BaseArg);
9062 Args.push_back(&BeginArg);
9063 Args.push_back(&SizeArg);
9064 Args.push_back(&TypeArg);
9065 Args.push_back(&NameArg);
9070 llvm::raw_svector_ostream Out(TyStr);
9073 auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,
9076 Fn->removeFnAttr(llvm::Attribute::OptimizeNone);
9083 C.getPointerType(Int64Ty),
Loc);
9087 false,
C.getPointerType(
C.VoidPtrTy),
Loc);
9090 false,
C.getPointerType(
C.VoidPtrTy),
Loc);
9093 false,
C.getPointerType(
C.VoidPtrTy),
Loc);
9095 Size = MapperCGF.
Builder.CreateExactUDiv(
9097 llvm::Value *PtrBegin = MapperCGF.
Builder.CreateBitCast(
9102 C.getPointerType(Int64Ty),
Loc);
9105 false,
C.getPointerType(
C.VoidPtrTy),
Loc);
9109 llvm::BasicBlock *HeadBB = MapperCGF.
createBasicBlock(
"omp.arraymap.head");
9111 MapName, ElementSize, HeadBB,
true);
9117 llvm::BasicBlock *BodyBB = MapperCGF.
createBasicBlock(
"omp.arraymap.body");
9120 llvm::Value *IsEmpty =
9121 MapperCGF.
Builder.CreateICmpEQ(PtrBegin, PtrEnd,
"omp.arraymap.isempty");
9122 MapperCGF.
Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
9123 llvm::BasicBlock *EntryBB = MapperCGF.
Builder.GetInsertBlock();
9127 llvm::BasicBlock *LastBB = BodyBB;
9128 llvm::PHINode *PtrPHI = MapperCGF.
Builder.CreatePHI(
9129 PtrBegin->getType(), 2,
"omp.arraymap.ptrcurrent");
9130 PtrPHI->addIncoming(PtrBegin, EntryBB);
9131 Address PtrCurrent(PtrPHI, ElemTy,
9137 Scope.addPrivate(MapperVarDecl, PtrCurrent);
9138 (void)
Scope.Privatize();
9141 MappableExprsHandler::MapCombinedInfoTy Info;
9142 MappableExprsHandler MEHandler(*D, MapperCGF);
9143 MEHandler.generateAllInfoForMapper(Info,
OMPBuilder);
9147 llvm::Value *OffloadingArgs[] = {Handle};
9150 OMPRTL___tgt_mapper_num_components),
9152 llvm::Value *ShiftedPreviousSize = MapperCGF.
Builder.CreateShl(
9154 MapperCGF.
Builder.getInt64(MappableExprsHandler::getFlagMemberOffset()));
9157 for (
unsigned I = 0; I < Info.BasePointers.size(); ++I) {
9158 llvm::Value *CurBaseArg = MapperCGF.
Builder.CreateBitCast(
9160 llvm::Value *CurBeginArg = MapperCGF.
Builder.CreateBitCast(
9162 llvm::Value *CurSizeArg = Info.Sizes[I];
9163 llvm::Value *CurNameArg =
9165 llvm::codegenoptions::NoDebugInfo)
9170 llvm::Value *OriMapType = MapperCGF.
Builder.getInt64(
9171 static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>
>(
9173 llvm::Value *MemberMapType =
9174 MapperCGF.
Builder.CreateNUWAdd(OriMapType, ShiftedPreviousSize);
9188 llvm::Value *LeftToFrom = MapperCGF.
Builder.CreateAnd(
9191 static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>
>(
9192 OpenMPOffloadMappingFlags::OMP_MAP_TO |
9193 OpenMPOffloadMappingFlags::OMP_MAP_FROM)));
9195 llvm::BasicBlock *AllocElseBB =
9198 llvm::BasicBlock *ToElseBB = MapperCGF.
createBasicBlock(
"omp.type.to.else");
9202 MapperCGF.
Builder.CreateCondBr(IsAlloc, AllocBB, AllocElseBB);
9205 llvm::Value *AllocMapType = MapperCGF.
Builder.CreateAnd(
9208 ~
static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>
>(
9209 OpenMPOffloadMappingFlags::OMP_MAP_TO |
9210 OpenMPOffloadMappingFlags::OMP_MAP_FROM)));
9211 MapperCGF.
Builder.CreateBr(EndBB);
9213 llvm::Value *IsTo = MapperCGF.
Builder.CreateICmpEQ(
9216 static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>
>(
9217 OpenMPOffloadMappingFlags::OMP_MAP_TO)));
9218 MapperCGF.
Builder.CreateCondBr(IsTo, ToBB, ToElseBB);
9221 llvm::Value *ToMapType = MapperCGF.
Builder.CreateAnd(
9224 ~
static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>
>(
9225 OpenMPOffloadMappingFlags::OMP_MAP_FROM)));
9226 MapperCGF.
Builder.CreateBr(EndBB);
9228 llvm::Value *IsFrom = MapperCGF.
Builder.CreateICmpEQ(
9231 static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>
>(
9232 OpenMPOffloadMappingFlags::OMP_MAP_FROM)));
9233 MapperCGF.
Builder.CreateCondBr(IsFrom, FromBB, EndBB);
9236 llvm::Value *FromMapType = MapperCGF.
Builder.CreateAnd(
9239 ~
static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>
>(
9240 OpenMPOffloadMappingFlags::OMP_MAP_TO)));
9244 llvm::PHINode *CurMapType =
9246 CurMapType->addIncoming(AllocMapType, AllocBB);
9247 CurMapType->addIncoming(ToMapType, ToBB);
9248 CurMapType->addIncoming(FromMapType, FromBB);
9249 CurMapType->addIncoming(MemberMapType, ToElseBB);
9251 llvm::Value *OffloadingArgs[] = {Handle, CurBaseArg, CurBeginArg,
9252 CurSizeArg, CurMapType, CurNameArg};
9253 if (Info.Mappers[I]) {
9256 cast<OMPDeclareMapperDecl>(Info.Mappers[I]));
9257 assert(MapperFunc &&
"Expect a valid mapper function is available.");
9271 llvm::Value *PtrNext = MapperCGF.
Builder.CreateConstGEP1_32(
9272 ElemTy, PtrPHI, 1,
"omp.arraymap.next");
9273 PtrPHI->addIncoming(PtrNext, LastBB);
9274 llvm::Value *IsDone =
9275 MapperCGF.
Builder.CreateICmpEQ(PtrNext, PtrEnd,
"omp.arraymap.isdone");
9276 llvm::BasicBlock *ExitBB = MapperCGF.
createBasicBlock(
"omp.arraymap.exit");
9277 MapperCGF.
Builder.CreateCondBr(IsDone, ExitBB, BodyBB);
9283 MapName, ElementSize, DoneBB,
false);
9288 UDMMap.try_emplace(D, Fn);
9291 Decls.second.push_back(D);
9303 llvm::Value *
Begin, llvm::Value *Size, llvm::Value *MapType,
9304 llvm::Value *MapName,
CharUnits ElementSize, llvm::BasicBlock *ExitBB,
9306 StringRef Prefix = IsInit ?
".init" :
".del";
9309 llvm::BasicBlock *BodyBB =
9311 llvm::Value *IsArray = MapperCGF.
Builder.CreateICmpSGT(
9312 Size, MapperCGF.
Builder.getInt64(1),
"omp.arrayinit.isarray");
9313 llvm::Value *DeleteBit = MapperCGF.
Builder.CreateAnd(
9316 static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>
>(
9317 OpenMPOffloadMappingFlags::OMP_MAP_DELETE)));
9318 llvm::Value *DeleteCond;
9324 llvm::Value *PtrAndObjBit = MapperCGF.
Builder.CreateAnd(
9327 static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>
>(
9328 OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ)));
9329 PtrAndObjBit = MapperCGF.
Builder.CreateIsNotNull(PtrAndObjBit);
9330 BaseIsBegin = MapperCGF.
Builder.CreateAnd(BaseIsBegin, PtrAndObjBit);
9331 Cond = MapperCGF.
Builder.CreateOr(IsArray, BaseIsBegin);
9333 DeleteBit,
getName({
"omp.array", Prefix,
".delete"}));
9336 DeleteCond = MapperCGF.
Builder.CreateIsNotNull(
9337 DeleteBit,
getName({
"omp.array", Prefix,
".delete"}));
9339 Cond = MapperCGF.
Builder.CreateAnd(Cond, DeleteCond);
9340 MapperCGF.
Builder.CreateCondBr(Cond, BodyBB, ExitBB);
9345 llvm::Value *ArraySize = MapperCGF.
Builder.CreateNUWMul(
9349 llvm::Value *MapTypeArg = MapperCGF.
Builder.CreateAnd(
9352 ~
static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>
>(
9353 OpenMPOffloadMappingFlags::OMP_MAP_TO |
9354 OpenMPOffloadMappingFlags::OMP_MAP_FROM)));
9355 MapTypeArg = MapperCGF.
Builder.CreateOr(
9358 static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>
>(
9359 OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT)));
9363 llvm::Value *OffloadingArgs[] = {Handle,
Base,
Begin,
9364 ArraySize, MapTypeArg, MapName};
9367 OMPRTL___tgt_push_mapper_component),
9390 Kind != OMPD_target_teams_loop)
9393 return llvm::ConstantInt::get(CGF.
Int64Ty, 0);
9395 const auto *LD = cast<OMPLoopDirective>(TD);
9396 if (llvm::Value *NumIterations = SizeEmitter(CGF, *LD))
9397 return NumIterations;
9398 return llvm::ConstantInt::get(CGF.
Int64Ty, 0);
9407 if (OffloadingMandatory) {
9408 CGF.
Builder.CreateUnreachable();
9410 if (RequiresOuterTask) {
9411 CapturedVars.clear();
9420 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier>
Device,
9423 llvm::Value *DeviceID;
9424 if (
Device.getPointer()) {
9426 Device.getInt() == OMPC_DEVICE_device_num) &&
9427 "Expected device_num modifier.");
9432 DeviceID = CGF.
Builder.getInt64(OMP_DEVICEID_UNDEF);
9439 llvm::Value *DynCGroupMem = CGF.
Builder.getInt32(0);
9442 CodeGenFunction::RunCleanupsScope DynCGroupMemScope(CGF);
9444 DynMemClause->getSize(),
true);
9445 DynCGroupMem = CGF.
Builder.CreateIntCast(DynCGroupMemVal, CGF.
Int32Ty,
9448 return DynCGroupMem;
9456 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier>
Device,
9457 llvm::Value *OutlinedFnID, CodeGenFunction::OMPTargetDataInfo &InputInfo,
9458 llvm::Value *&MapTypesArray, llvm::Value *&MapNamesArray,
9463 llvm::OpenMPIRBuilder &OMPBuilder = OMPRuntime->
getOMPBuilder();
9466 MappableExprsHandler::MapCombinedInfoTy CombinedInfo;
9469 MappableExprsHandler MEHandler(D, CGF);
9470 llvm::DenseMap<llvm::Value *, llvm::Value *> LambdaPointers;
9474 auto *CV = CapturedVars.begin();
9477 CI != CE; ++CI, ++RI, ++CV) {
9478 MappableExprsHandler::MapCombinedInfoTy CurInfo;
9479 MappableExprsHandler::StructRangeInfoTy PartialStruct;
9484 CurInfo.Exprs.push_back(
nullptr);
9485 CurInfo.BasePointers.push_back(*CV);
9486 CurInfo.DevicePtrDecls.push_back(
nullptr);
9487 CurInfo.DevicePointers.push_back(
9488 MappableExprsHandler::DeviceInfoTy::None);
9489 CurInfo.Pointers.push_back(*CV);
9490 CurInfo.Sizes.push_back(CGF.
Builder.CreateIntCast(
9493 CurInfo.Types.push_back(OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |
9494 OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM |
9495 OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT);
9496 CurInfo.Mappers.push_back(
nullptr);
9500 MEHandler.generateInfoForCapture(CI, *CV, CurInfo, PartialStruct);
9504 MappedVarSet.insert(
nullptr);
9505 if (CurInfo.BasePointers.empty() && !PartialStruct.Base.isValid())
9506 MEHandler.generateDefaultMapInfo(*CI, **RI, *CV, CurInfo);
9510 MEHandler.generateInfoForLambdaCaptures(CI->
getCapturedVar(), *CV,
9511 CurInfo, LambdaPointers);
9514 assert((!CurInfo.BasePointers.empty() || PartialStruct.Base.isValid()) &&
9515 "Non-existing map pointer for capture!");
9516 assert(CurInfo.BasePointers.size() == CurInfo.Pointers.size() &&
9517 CurInfo.BasePointers.size() == CurInfo.Sizes.size() &&
9518 CurInfo.BasePointers.size() == CurInfo.Types.size() &&
9519 CurInfo.BasePointers.size() == CurInfo.Mappers.size() &&
9520 "Inconsistent map information sizes!");
9524 if (PartialStruct.Base.isValid()) {
9525 CombinedInfo.append(PartialStruct.PreliminaryMapData);
9526 MEHandler.emitCombinedEntry(
9527 CombinedInfo, CurInfo.Types, PartialStruct, CI->
capturesThis(),
9528 OMPBuilder,
nullptr,
9529 !PartialStruct.PreliminaryMapData.BasePointers.empty());
9533 CombinedInfo.append(CurInfo);
9536 MEHandler.adjustMemberOfForLambdaCaptures(
9537 OMPBuilder, LambdaPointers, CombinedInfo.BasePointers,
9538 CombinedInfo.Pointers, CombinedInfo.Types);
9541 MEHandler.generateAllInfo(CombinedInfo, OMPBuilder, MappedVarSet);
9547 llvm::codegenoptions::NoDebugInfo;
9548 OMPBuilder.emitOffloadingArraysArgument(CGF.
Builder, Info.RTArgs, Info,
9552 InputInfo.NumberOfTargetItems = Info.NumberOfPtrs;
9553 InputInfo.BasePointersArray =
Address(Info.RTArgs.BasePointersArray,
9555 InputInfo.PointersArray =
9557 InputInfo.SizesArray =
9559 InputInfo.MappersArray =
9561 MapTypesArray = Info.RTArgs.MapTypesArray;
9562 MapNamesArray = Info.RTArgs.MapNamesArray;
9564 auto &&ThenGen = [&OMPRuntime, OutlinedFn, &D, &CapturedVars,
9565 RequiresOuterTask, &CS, OffloadingMandatory,
Device,
9566 OutlinedFnID, &InputInfo, &MapTypesArray, &MapNamesArray,
9568 bool IsReverseOffloading =
Device.getInt() == OMPC_DEVICE_ancestor;
9570 if (IsReverseOffloading) {
9576 RequiresOuterTask, CS, OffloadingMandatory, CGF);
9581 unsigned NumTargetItems = InputInfo.NumberOfTargetItems;
9583 llvm::Value *BasePointersArray =
9584 InputInfo.BasePointersArray.emitRawPointer(CGF);
9585 llvm::Value *PointersArray = InputInfo.PointersArray.emitRawPointer(CGF);
9586 llvm::Value *SizesArray = InputInfo.SizesArray.emitRawPointer(CGF);
9587 llvm::Value *MappersArray = InputInfo.MappersArray.emitRawPointer(CGF);
9589 auto &&EmitTargetCallFallbackCB =
9590 [&OMPRuntime, OutlinedFn, &D, &CapturedVars, RequiresOuterTask, &CS,
9591 OffloadingMandatory, &CGF](llvm::OpenMPIRBuilder::InsertPointTy IP)
9592 -> llvm::OpenMPIRBuilder::InsertPointTy {
9595 RequiresOuterTask, CS, OffloadingMandatory, CGF);
9601 llvm::Value *NumThreads =
9604 llvm::Value *NumIterations =
9607 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
9610 llvm::OpenMPIRBuilder::TargetDataRTArgs RTArgs(
9611 BasePointersArray, PointersArray, SizesArray, MapTypesArray,
9612 nullptr , MappersArray, MapNamesArray);
9614 llvm::OpenMPIRBuilder::TargetKernelArgs Args(
9615 NumTargetItems, RTArgs, NumIterations, NumTeams, NumThreads,
9616 DynCGGroupMem, HasNoWait);
9619 CGF.
Builder, OutlinedFn, OutlinedFnID, EmitTargetCallFallbackCB, Args,
9620 DeviceID, RTLoc, AllocaIP));
9623 if (RequiresOuterTask)
9638 [&OMPRuntime, OutlinedFn, &D, &CapturedVars, RequiresOuterTask, &CS,
9641 RequiresOuterTask, CS, OffloadingMandatory, CGF);
9644 if (RequiresOuterTask) {
9645 CodeGenFunction::OMPTargetDataInfo InputInfo;
9654 llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID,
const Expr *IfCond,
9655 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier>
Device,
9662 const bool OffloadingMandatory = !
CGM.
getLangOpts().OpenMPIsTargetDevice &&
9665 assert((OffloadingMandatory || OutlinedFn) &&
"Invalid outlined function!");
9667 const bool RequiresOuterTask =
9683 llvm::Value *MapTypesArray =
nullptr;
9684 llvm::Value *MapNamesArray =
nullptr;
9686 auto &&TargetThenGen = [
this, OutlinedFn, &D, &CapturedVars,
9687 RequiresOuterTask, &CS, OffloadingMandatory,
Device,
9688 OutlinedFnID, &InputInfo, &MapTypesArray,
9692 RequiresOuterTask, CS, OffloadingMandatory,
9693 Device, OutlinedFnID, InputInfo, MapTypesArray,
9694 MapNamesArray, SizeEmitter, CGF,
CGM);
9697 auto &&TargetElseGen =
9698 [
this, OutlinedFn, &D, &CapturedVars, RequiresOuterTask, &CS,
9701 CS, OffloadingMandatory, CGF);
9710 emitIfClause(CGF, IfCond, TargetThenGen, TargetElseGen);
9722 StringRef ParentName) {
9727 bool RequiresDeviceCodegen =
9728 isa<OMPExecutableDirective>(S) &&
9730 cast<OMPExecutableDirective>(S)->getDirectiveKind());
9732 if (RequiresDeviceCodegen) {
9733 const auto &E = *cast<OMPExecutableDirective>(S);
9740 if (!
OMPBuilder.OffloadInfoManager.hasTargetRegionEntryInfo(EntryInfo))
9743 switch (E.getDirectiveKind()) {
9746 cast<OMPTargetDirective>(E));
9748 case OMPD_target_parallel:
9750 CGM, ParentName, cast<OMPTargetParallelDirective>(E));
9752 case OMPD_target_teams:
9754 CGM, ParentName, cast<OMPTargetTeamsDirective>(E));
9756 case OMPD_target_teams_distribute:
9758 CGM, ParentName, cast<OMPTargetTeamsDistributeDirective>(E));
9760 case OMPD_target_teams_distribute_simd:
9762 CGM, ParentName, cast<OMPTargetTeamsDistributeSimdDirective>(E));
9764 case OMPD_target_parallel_for:
9766 CGM, ParentName, cast<OMPTargetParallelForDirective>(E));
9768 case OMPD_target_parallel_for_simd:
9770 CGM, ParentName, cast<OMPTargetParallelForSimdDirective>(E));
9772 case OMPD_target_simd:
9774 CGM, ParentName, cast<OMPTargetSimdDirective>(E));
9776 case OMPD_target_teams_distribute_parallel_for:
9779 cast<OMPTargetTeamsDistributeParallelForDirective>(E));
9781 case OMPD_target_teams_distribute_parallel_for_simd:
9785 cast<OMPTargetTeamsDistributeParallelForSimdDirective>(E));
9787 case OMPD_target_teams_loop:
9789 CGM, ParentName, cast<OMPTargetTeamsGenericLoopDirective>(E));
9791 case OMPD_target_parallel_loop:
9793 CGM, ParentName, cast<OMPTargetParallelGenericLoopDirective>(E));
9797 case OMPD_parallel_for:
9798 case OMPD_parallel_master:
9799 case OMPD_parallel_sections:
9801 case OMPD_parallel_for_simd:
9803 case OMPD_cancellation_point:
9805 case OMPD_threadprivate:
9816 case OMPD_taskyield:
9819 case OMPD_taskgroup:
9825 case OMPD_target_data:
9826 case OMPD_target_exit_data:
9827 case OMPD_target_enter_data:
9828 case OMPD_distribute:
9829 case OMPD_distribute_simd:
9830 case OMPD_distribute_parallel_for:
9831 case OMPD_distribute_parallel_for_simd:
9832 case OMPD_teams_distribute:
9833 case OMPD_teams_distribute_simd:
9834 case OMPD_teams_distribute_parallel_for:
9835 case OMPD_teams_distribute_parallel_for_simd:
9836 case OMPD_target_update:
9837 case OMPD_declare_simd:
9838 case OMPD_declare_variant:
9839 case OMPD_begin_declare_variant:
9840 case OMPD_end_declare_variant:
9841 case OMPD_declare_target:
9842 case OMPD_end_declare_target:
9843 case OMPD_declare_reduction:
9844 case OMPD_declare_mapper:
9846 case OMPD_taskloop_simd:
9847 case OMPD_master_taskloop:
9848 case OMPD_master_taskloop_simd:
9849 case OMPD_parallel_master_taskloop:
9850 case OMPD_parallel_master_taskloop_simd:
9852 case OMPD_metadirective:
9855 llvm_unreachable(
"Unknown target directive for OpenMP device codegen.");
9860 if (
const auto *E = dyn_cast<OMPExecutableDirective>(S)) {
9861 if (!E->hasAssociatedStmt() || !E->getAssociatedStmt())
9869 if (
const auto *L = dyn_cast<LambdaExpr>(S))
9873 for (
const Stmt *II : S->children())
9878 std::optional<OMPDeclareTargetDeclAttr::DevTypeTy> DevTy =
9879 OMPDeclareTargetDeclAttr::getDeviceType(VD);
9883 if (!IsDevice && DevTy == OMPDeclareTargetDeclAttr::DT_NoHost)
9886 if (IsDevice && DevTy == OMPDeclareTargetDeclAttr::DT_Host)
9895 if (
const auto *FD = dyn_cast<FunctionDecl>(GD.
getDecl()))
9904 if (
const auto *FD = dyn_cast<FunctionDecl>(VD)) {
9913 return !OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD) &&
9931 StringRef ParentName =
9936 StringRef ParentName =
9943 std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
9944 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(
9946 if (!Res || *Res == OMPDeclareTargetDeclAttr::MT_Link ||
9947 ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
9948 *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
9957 llvm::Constant *Addr) {
9962 std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
9963 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
9967 if (Res && *Res != OMPDeclareTargetDeclAttr::MT_Link &&
9982 auto LinkageForVariable = [&VD,
this]() {
9986 std::vector<llvm::GlobalVariable *> GeneratedRefs;
9999 for (
auto *ref : GeneratedRefs)
10004 if (isa<FunctionDecl>(GD.
getDecl()) ||
10005 isa<OMPDeclareReductionDecl>(GD.
getDecl()))
10013 std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
10014 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
10017 if ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
10018 *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
10022 assert((*Res == OMPDeclareTargetDeclAttr::MT_Link ||
10023 ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
10024 *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
10026 "Expected link clause or to clause with unified memory.");
10035 " Expected target-based directive.");
10040 if (Clause->getClauseKind() == OMPC_unified_shared_memory) {
10042 OMPBuilder.Config.setHasRequiresUnifiedSharedMemory(
true);
10043 }
else if (
const auto *AC =
10044 dyn_cast<OMPAtomicDefaultMemOrderClause>(Clause)) {
10045 switch (AC->getAtomicDefaultMemOrderKind()) {
10046 case OMPC_ATOMIC_DEFAULT_MEM_ORDER_acq_rel:
10049 case OMPC_ATOMIC_DEFAULT_MEM_ORDER_seq_cst:
10052 case OMPC_ATOMIC_DEFAULT_MEM_ORDER_relaxed:
10068 if (!VD || !VD->
hasAttr<OMPAllocateDeclAttr>())
10070 const auto *A = VD->
getAttr<OMPAllocateDeclAttr>();
10071 switch(A->getAllocatorType()) {
10072 case OMPAllocateDeclAttr::OMPNullMemAlloc:
10073 case OMPAllocateDeclAttr::OMPDefaultMemAlloc:
10075 case OMPAllocateDeclAttr::OMPLargeCapMemAlloc:
10076 case OMPAllocateDeclAttr::OMPCGroupMemAlloc:
10077 case OMPAllocateDeclAttr::OMPHighBWMemAlloc:
10078 case OMPAllocateDeclAttr::OMPLowLatMemAlloc:
10079 case OMPAllocateDeclAttr::OMPThreadMemAlloc:
10080 case OMPAllocateDeclAttr::OMPConstMemAlloc:
10081 case OMPAllocateDeclAttr::OMPPTeamMemAlloc:
10084 case OMPAllocateDeclAttr::OMPUserDefinedMemAlloc:
10085 llvm_unreachable(
"Expected predefined allocator for the variables with the "
10086 "static storage.");
10113 const auto *D = cast<FunctionDecl>(GD.
getDecl());
10116 if (OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(D)) {
10118 if (
auto *F = dyn_cast_or_null<llvm::Function>(
10120 return !F->isDeclaration();
10132 llvm::Function *OutlinedFn,
10141 llvm::Value *Args[] = {
10143 CGF.
Builder.getInt32(CapturedVars.size()),
10146 RealArgs.append(std::begin(Args), std::end(Args));
10147 RealArgs.append(CapturedVars.begin(), CapturedVars.end());
10149 llvm::FunctionCallee RTLFn =
OMPBuilder.getOrCreateRuntimeFunction(
10155 const Expr *NumTeams,
10156 const Expr *ThreadLimit,
10163 llvm::Value *NumTeamsVal =
10169 llvm::Value *ThreadLimitVal =
10176 llvm::Value *PushNumTeamsArgs[] = {RTLoc,
getThreadID(CGF,
Loc), NumTeamsVal,
10184 const Expr *ThreadLimit,
10187 llvm::Value *ThreadLimitVal =
10212 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
10214 llvm::Value *IfCondVal =
nullptr;
10219 llvm::Value *DeviceID =
nullptr;
10224 DeviceID = CGF.
Builder.getInt64(OMP_DEVICEID_UNDEF);
10228 MappableExprsHandler::MapCombinedInfoTy CombinedInfo;
10229 auto GenMapInfoCB =
10230 [&](InsertPointTy CodeGenIP) -> llvm::OpenMPIRBuilder::MapInfosTy & {
10231 CGF.
Builder.restoreIP(CodeGenIP);
10233 MappableExprsHandler MEHandler(D, CGF);
10234 MEHandler.generateAllInfo(CombinedInfo,
OMPBuilder);
10236 auto FillInfoMap = [&](MappableExprsHandler::MappingExprInfo &MapExpr) {
10240 llvm::codegenoptions::NoDebugInfo) {
10241 CombinedInfo.Names.resize(CombinedInfo.Exprs.size());
10242 llvm::transform(CombinedInfo.Exprs, CombinedInfo.Names.begin(),
10246 return CombinedInfo;
10248 using BodyGenTy = llvm::OpenMPIRBuilder::BodyGenTy;
10249 auto BodyCB = [&](InsertPointTy CodeGenIP, BodyGenTy BodyGenType) {
10250 CGF.
Builder.restoreIP(CodeGenIP);
10251 switch (BodyGenType) {
10252 case BodyGenTy::Priv:
10256 case BodyGenTy::DupNoPriv:
10262 case BodyGenTy::NoPriv:
10269 return InsertPointTy(CGF.
Builder.GetInsertBlock(),
10270 CGF.
Builder.GetInsertPoint());
10273 auto DeviceAddrCB = [&](
unsigned int I, llvm::Value *NewDecl) {
10274 if (
const ValueDecl *DevVD = CombinedInfo.DevicePtrDecls[I]) {
10279 auto CustomMapperCB = [&](
unsigned int I) {
10280 llvm::Value *MFunc =
nullptr;
10281 if (CombinedInfo.Mappers[I]) {
10282 Info.HasMapper =
true;
10284 cast<OMPDeclareMapperDecl>(CombinedInfo.Mappers[I]));
10294 InsertPointTy CodeGenIP(CGF.
Builder.GetInsertBlock(),
10295 CGF.
Builder.GetInsertPoint());
10296 llvm::OpenMPIRBuilder::LocationDescription OmpLoc(CodeGenIP);
10298 OmpLoc, AllocaIP, CodeGenIP, DeviceID, IfCondVal, Info, GenMapInfoCB,
10299 nullptr, BodyCB, DeviceAddrCB, CustomMapperCB, RTLoc));
10308 assert((isa<OMPTargetEnterDataDirective>(D) ||
10309 isa<OMPTargetExitDataDirective>(D) ||
10310 isa<OMPTargetUpdateDirective>(D)) &&
10311 "Expecting either target enter, exit data, or update directives.");
10314 llvm::Value *MapTypesArray =
nullptr;
10315 llvm::Value *MapNamesArray =
nullptr;
10317 auto &&ThenGen = [
this, &D,
Device, &InputInfo, &MapTypesArray,
10320 llvm::Value *DeviceID =
nullptr;
10325 DeviceID = CGF.
Builder.getInt64(OMP_DEVICEID_UNDEF);
10329 llvm::Constant *PointerNum =
10335 llvm::Value *OffloadingArgs[] = {
10349 RuntimeFunction RTLFn;
10351 case OMPD_target_enter_data:
10352 RTLFn = HasNowait ? OMPRTL___tgt_target_data_begin_nowait_mapper
10353 : OMPRTL___tgt_target_data_begin_mapper;
10355 case OMPD_target_exit_data:
10356 RTLFn = HasNowait ? OMPRTL___tgt_target_data_end_nowait_mapper
10357 : OMPRTL___tgt_target_data_end_mapper;
10359 case OMPD_target_update:
10360 RTLFn = HasNowait ? OMPRTL___tgt_target_data_update_nowait_mapper
10361 : OMPRTL___tgt_target_data_update_mapper;
10363 case OMPD_parallel:
10365 case OMPD_parallel_for:
10366 case OMPD_parallel_master:
10367 case OMPD_parallel_sections:
10368 case OMPD_for_simd:
10369 case OMPD_parallel_for_simd:
10371 case OMPD_cancellation_point:
10373 case OMPD_threadprivate:
10374 case OMPD_allocate:
10379 case OMPD_sections:
10383 case OMPD_critical:
10384 case OMPD_taskyield:
10386 case OMPD_taskwait:
10387 case OMPD_taskgroup:
10393 case OMPD_target_data:
10394 case OMPD_distribute:
10395 case OMPD_distribute_simd:
10396 case OMPD_distribute_parallel_for:
10397 case OMPD_distribute_parallel_for_simd:
10398 case OMPD_teams_distribute:
10399 case OMPD_teams_distribute_simd:
10400 case OMPD_teams_distribute_parallel_for:
10401 case OMPD_teams_distribute_parallel_for_simd:
10402 case OMPD_declare_simd:
10403 case OMPD_declare_variant:
10404 case OMPD_begin_declare_variant:
10405 case OMPD_end_declare_variant:
10406 case OMPD_declare_target:
10407 case OMPD_end_declare_target:
10408 case OMPD_declare_reduction:
10409 case OMPD_declare_mapper:
10410 case OMPD_taskloop:
10411 case OMPD_taskloop_simd:
10412 case OMPD_master_taskloop:
10413 case OMPD_master_taskloop_simd:
10414 case OMPD_parallel_master_taskloop:
10415 case OMPD_parallel_master_taskloop_simd:
10417 case OMPD_target_simd:
10418 case OMPD_target_teams_distribute:
10419 case OMPD_target_teams_distribute_simd:
10420 case OMPD_target_teams_distribute_parallel_for:
10421 case OMPD_target_teams_distribute_parallel_for_simd:
10422 case OMPD_target_teams:
10423 case OMPD_target_parallel:
10424 case OMPD_target_parallel_for:
10425 case OMPD_target_parallel_for_simd:
10426 case OMPD_requires:
10427 case OMPD_metadirective:
10430 llvm_unreachable(
"Unexpected standalone target data directive.");
10438 auto &&TargetThenGen = [
this, &ThenGen, &D, &InputInfo, &MapTypesArray,
10442 MappableExprsHandler::MapCombinedInfoTy CombinedInfo;
10445 MappableExprsHandler MEHandler(D, CGF);
10446 MEHandler.generateAllInfo(CombinedInfo,
OMPBuilder);
10455 llvm::codegenoptions::NoDebugInfo;
10468 MapTypesArray = Info.RTArgs.MapTypesArray;
10469 MapNamesArray = Info.RTArgs.MapNamesArray;
10470 if (RequiresOuterTask)
10496struct ParamAttrTy {
10497 ParamKindTy Kind =
Vector;
10498 llvm::APSInt StrideOrArg;
10499 llvm::APSInt Alignment;
10500 bool HasVarStride =
false;
10533 unsigned Offset = 0;
10534 if (
const auto *MD = dyn_cast<CXXMethodDecl>(FD)) {
10535 if (ParamAttrs[Offset].Kind ==
Vector)
10536 CDT =
C.getPointerType(
C.getRecordType(MD->
getParent()));
10540 for (
unsigned I = 0, E = FD->
getNumParams(); I < E; ++I) {
10541 if (ParamAttrs[I + Offset].Kind ==
Vector) {
10553 return C.getTypeSize(CDT);
10561 llvm::raw_svector_ostream Out(Buffer);
10562 for (
const auto &ParamAttr : ParamAttrs) {
10563 switch (ParamAttr.Kind) {
10583 if (ParamAttr.HasVarStride)
10584 Out <<
"s" << ParamAttr.StrideOrArg;
10585 else if (ParamAttr.Kind == Linear || ParamAttr.Kind == LinearRef ||
10586 ParamAttr.Kind == LinearUVal || ParamAttr.Kind == LinearVal) {
10589 if (ParamAttr.StrideOrArg < 0)
10590 Out <<
'n' << -ParamAttr.StrideOrArg;
10591 else if (ParamAttr.StrideOrArg != 1)
10592 Out << ParamAttr.StrideOrArg;
10595 if (!!ParamAttr.Alignment)
10596 Out <<
'a' << ParamAttr.Alignment;
10599 return std::string(Out.str());
10604 const llvm::APSInt &VLENVal,
10606 OMPDeclareSimdDeclAttr::BranchStateTy State) {
10609 unsigned VecRegSize;
10611 ISADataTy ISAData[] = {
10627 case OMPDeclareSimdDeclAttr::BS_Undefined:
10628 Masked.push_back(
'N');
10629 Masked.push_back(
'M');
10631 case OMPDeclareSimdDeclAttr::BS_Notinbranch:
10632 Masked.push_back(
'N');
10634 case OMPDeclareSimdDeclAttr::BS_Inbranch:
10635 Masked.push_back(
'M');
10638 for (
char Mask : Masked) {
10639 for (
const ISADataTy &
Data : ISAData) {
10641 llvm::raw_svector_ostream Out(Buffer);
10642 Out <<
"_ZGV" <<
Data.ISA << Mask;
10645 assert(NumElts &&
"Non-zero simdlen/cdtsize expected");
10646 Out << llvm::APSInt::getUnsigned(
Data.VecRegSize / NumElts);
10651 Out <<
'_' << Fn->getName();
10652 Fn->addFnAttr(Out.str());
10670 if (Kind == ParamKindTy::Uniform)
10673 if (Kind == ParamKindTy::LinearUVal || Kind == ParamKindTy::LinearRef)
10676 if ((Kind == ParamKindTy::Linear || Kind == ParamKindTy::LinearVal) &&
10686 unsigned Size =
C.getTypeSize(QT);
10689 if (Size != 8 && Size != 16 && Size != 32 && Size != 64 && Size != 128)
10713 return C.getTypeSize(PTy);
10716 return C.getTypeSize(QT);
10718 return C.getTypeSize(
C.getUIntPtrType());
10724static std::tuple<unsigned, unsigned, bool>
10730 bool OutputBecomesInput =
false;
10734 Sizes.push_back(
getAArch64LS(RetType, ParamKindTy::Vector,
C));
10736 OutputBecomesInput =
true;
10738 for (
unsigned I = 0, E = FD->
getNumParams(); I < E; ++I) {
10743 assert(!Sizes.empty() &&
"Unable to determine NDS and WDS.");
10746 assert(llvm::all_of(Sizes,
10747 [](
unsigned Size) {
10748 return Size == 8 || Size == 16 || Size == 32 ||
10749 Size == 64 || Size == 128;
10753 return std::make_tuple(*std::min_element(std::begin(Sizes), std::end(Sizes)),
10754 *std::max_element(std::begin(Sizes), std::end(Sizes)),
10755 OutputBecomesInput);
10761template <
typename T>
10763 char ISA, StringRef ParSeq,
10764 StringRef MangledName,
bool OutputBecomesInput,
10765 llvm::Function *Fn) {
10767 llvm::raw_svector_ostream Out(Buffer);
10768 Out << Prefix << ISA << LMask << VLEN;
10769 if (OutputBecomesInput)
10771 Out << ParSeq <<
"_" << MangledName;
10772 Fn->addFnAttr(Out.str());
10778 StringRef Prefix,
char ISA,
10779 StringRef ParSeq, StringRef MangledName,
10780 bool OutputBecomesInput,
10781 llvm::Function *Fn) {
10785 OutputBecomesInput, Fn);
10787 OutputBecomesInput, Fn);
10791 OutputBecomesInput, Fn);
10793 OutputBecomesInput, Fn);
10797 OutputBecomesInput, Fn);
10799 OutputBecomesInput, Fn);
10804 OutputBecomesInput, Fn);
10807 llvm_unreachable(
"Scalar type is too wide.");
10815 OMPDeclareSimdDeclAttr::BranchStateTy State, StringRef MangledName,
10816 char ISA,
unsigned VecRegSize, llvm::Function *Fn,
SourceLocation SLoc) {
10820 const unsigned NDS = std::get<0>(
Data);
10821 const unsigned WDS = std::get<1>(
Data);
10822 const bool OutputBecomesInput = std::get<2>(
Data);
10826 if (UserVLEN == 1) {
10829 "The clause simdlen(1) has no effect when targeting aarch64.");
10836 if (ISA ==
'n' && UserVLEN && !llvm::isPowerOf2_32(UserVLEN)) {
10839 "power of 2 when targeting Advanced SIMD.");
10846 if (ISA ==
's' && UserVLEN != 0) {
10847 if ((UserVLEN * WDS > 2048) || (UserVLEN * WDS % 128 != 0)) {
10850 "lanes in the architectural constraints "
10851 "for SVE (min is 128-bit, max is "
10852 "2048-bit, by steps of 128-bit)");
10860 StringRef Prefix =
"_ZGV";
10866 OutputBecomesInput, Fn);
10868 assert(ISA ==
'n' &&
"Expected ISA either 's' or 'n'.");
10872 case OMPDeclareSimdDeclAttr::BS_Undefined:
10874 OutputBecomesInput, Fn);
10876 OutputBecomesInput, Fn);
10878 case OMPDeclareSimdDeclAttr::BS_Notinbranch:
10880 OutputBecomesInput, Fn);
10882 case OMPDeclareSimdDeclAttr::BS_Inbranch:
10884 OutputBecomesInput, Fn);
10894 OutputBecomesInput, Fn);
10896 assert(ISA ==
'n' &&
"Expected ISA either 's' or 'n'.");
10901 case OMPDeclareSimdDeclAttr::BS_Undefined:
10903 OutputBecomesInput, Fn);
10905 OutputBecomesInput, Fn);
10907 case OMPDeclareSimdDeclAttr::BS_Notinbranch:
10909 OutputBecomesInput, Fn);
10911 case OMPDeclareSimdDeclAttr::BS_Inbranch:
10913 OutputBecomesInput, Fn);
10921 llvm::Function *Fn) {
10926 llvm::DenseMap<const Decl *, unsigned> ParamPositions;
10927 if (isa<CXXMethodDecl>(FD))
10928 ParamPositions.try_emplace(FD, 0);
10929 unsigned ParamPos = ParamPositions.size();
10931 ParamPositions.try_emplace(
P->getCanonicalDecl(), ParamPos);
10937 for (
const Expr *E :
Attr->uniforms()) {
10940 if (isa<CXXThisExpr>(E)) {
10941 Pos = ParamPositions[FD];
10943 const auto *PVD = cast<ParmVarDecl>(cast<DeclRefExpr>(E)->getDecl())
10944 ->getCanonicalDecl();
10945 auto It = ParamPositions.find(PVD);
10946 assert(It != ParamPositions.end() &&
"Function parameter not found");
10949 ParamAttrs[Pos].Kind = Uniform;
10952 auto *NI =
Attr->alignments_begin();
10953 for (
const Expr *E :
Attr->aligneds()) {
10957 if (isa<CXXThisExpr>(E)) {
10958 Pos = ParamPositions[FD];
10961 const auto *PVD = cast<ParmVarDecl>(cast<DeclRefExpr>(E)->getDecl())
10962 ->getCanonicalDecl();
10963 auto It = ParamPositions.find(PVD);
10964 assert(It != ParamPositions.end() &&
"Function parameter not found");
10966 ParmTy = PVD->getType();
10968 ParamAttrs[Pos].Alignment =
10970 ? (*NI)->EvaluateKnownConstInt(
C)
10971 : llvm::APSInt::getUnsigned(
10972 C.toCharUnitsFromBits(
C.getOpenMPDefaultSimdAlign(ParmTy))
10977 auto *SI =
Attr->steps_begin();
10978 auto *MI =
Attr->modifiers_begin();
10979 for (
const Expr *E :
Attr->linears()) {
10982 bool IsReferenceType =
false;
10985 unsigned PtrRescalingFactor = 1;
10986 if (isa<CXXThisExpr>(E)) {
10987 Pos = ParamPositions[FD];
10988 auto *
P = cast<PointerType>(E->
getType());
10993 const auto *PVD = cast<ParmVarDecl>(cast<DeclRefExpr>(E)->getDecl())
10994 ->getCanonicalDecl();
10995 auto It = ParamPositions.find(PVD);
10996 assert(It != ParamPositions.end() &&
"Function parameter not found");
10998 if (
auto *
P = dyn_cast<PointerType>(PVD->getType()))
11002 else if (PVD->getType()->isReferenceType()) {
11003 IsReferenceType =
true;
11004 PtrRescalingFactor =
11010 ParamAttrTy &ParamAttr = ParamAttrs[Pos];
11011 if (*MI == OMPC_LINEAR_ref)
11012 ParamAttr.Kind = LinearRef;
11013 else if (*MI == OMPC_LINEAR_uval)
11014 ParamAttr.Kind = LinearUVal;
11015 else if (IsReferenceType)
11016 ParamAttr.Kind = LinearVal;
11018 ParamAttr.Kind = Linear;
11020 ParamAttr.StrideOrArg = llvm::APSInt::getUnsigned(1);
11024 if (
const auto *DRE =
11025 cast<DeclRefExpr>((*SI)->IgnoreParenImpCasts())) {
11026 if (
const auto *StridePVD =
11027 dyn_cast<ParmVarDecl>(DRE->getDecl())) {
11028 ParamAttr.HasVarStride =
true;
11029 auto It = ParamPositions.find(StridePVD->getCanonicalDecl());
11030 assert(It != ParamPositions.end() &&
11031 "Function parameter not found");
11032 ParamAttr.StrideOrArg = llvm::APSInt::getUnsigned(It->second);
11036 ParamAttr.StrideOrArg =
Result.Val.getInt();
11042 if (!ParamAttr.HasVarStride &&
11043 (ParamAttr.Kind == Linear || ParamAttr.Kind == LinearRef))
11044 ParamAttr.StrideOrArg = ParamAttr.StrideOrArg * PtrRescalingFactor;
11048 llvm::APSInt VLENVal;
11050 const Expr *VLENExpr =
Attr->getSimdlen();
11055 OMPDeclareSimdDeclAttr::BranchStateTy State =
Attr->getBranchState();
11058 }
else if (
CGM.
getTriple().getArch() == llvm::Triple::aarch64) {
11059 unsigned VLEN = VLENVal.getExtValue();
11060 StringRef MangledName = Fn->getName();
11063 MangledName,
's', 128, Fn, ExprLoc);
11066 MangledName,
'n', 128, Fn, ExprLoc);
11077 static const int DoacrossFinArgs = 2;
11080 llvm::FunctionCallee RTLFn;
11081 llvm::Value *Args[DoacrossFinArgs];
11084 DoacrossCleanupTy(llvm::FunctionCallee RTLFn,
11087 assert(CallArgs.size() == DoacrossFinArgs);
11088 std::copy(CallArgs.begin(), CallArgs.end(), std::begin(Args));
11105 QualType Int64Ty =
C.getIntTypeForBitwidth(64,
true);
11113 RD =
C.buildImplicitRecord(
"kmp_dim");
11123 llvm::APInt Size(32, NumIterations.size());
11129 enum { LowerFD = 0, UpperFD, StrideFD };
11131 for (
unsigned I = 0, E = NumIterations.size(); I < E; ++I) {
11136 DimsLVal, *std::next(RD->
field_begin(), UpperFD));
11138 CGF.
EmitScalarExpr(NumIterations[I]), NumIterations[I]->getType(),
11139 Int64Ty, NumIterations[I]->getExprLoc());
11143 DimsLVal, *std::next(RD->
field_begin(), StrideFD));
11150 llvm::Value *Args[] = {
11153 llvm::ConstantInt::getSigned(
CGM.
Int32Ty, NumIterations.size()),
11158 llvm::FunctionCallee RTLFn =
OMPBuilder.getOrCreateRuntimeFunction(
11161 llvm::Value *FiniArgs[DoacrossCleanupTy::DoacrossFinArgs] = {
11163 llvm::FunctionCallee FiniRTLFn =
OMPBuilder.getOrCreateRuntimeFunction(
11169template <
typename T>
11171 const T *
C, llvm::Value *ULoc,
11172 llvm::Value *ThreadID) {
11175 llvm::APInt Size(32,
C->getNumLoops());
11179 for (
unsigned I = 0, E =
C->getNumLoops(); I < E; ++I) {
11180 const Expr *CounterVal =
C->getLoopData(I);
11181 assert(CounterVal);
11188 llvm::Value *Args[] = {
11191 llvm::FunctionCallee RTLFn;
11193 OMPDoacrossKind<T> ODK;
11194 if (ODK.isSource(
C)) {
11196 OMPRTL___kmpc_doacross_post);
11198 assert(ODK.isSink(
C) &&
"Expect sink modifier.");
11200 OMPRTL___kmpc_doacross_wait);
11207 return EmitDoacrossOrdered<OMPDependClause>(
11214 return EmitDoacrossOrdered<OMPDoacrossClause>(
11220 llvm::FunctionCallee Callee,
11222 assert(
Loc.
isValid() &&
"Outlined function call location must be valid.");
11225 if (
auto *Fn = dyn_cast<llvm::Function>(Callee.getCallee())) {
11226 if (Fn->doesNotThrow()) {
11241 if (
const auto *FD = dyn_cast<FunctionDecl>(D))
11242 if (OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(FD))
11248 const VarDecl *TargetParam)
const {
11255 const Expr *Allocator) {
11256 llvm::Value *AllocVal;
11263 Allocator->getExprLoc());
11266 AllocVal = llvm::Constant::getNullValue(
11276 if (!AllocateAlignment)
11279 return llvm::ConstantInt::get(
CGM.
SizeTy, AllocateAlignment->getQuantity());
11292 auto I = UntiedData.find(VD);
11293 if (I != UntiedData.end()) {
11294 UntiedAddr = I->second.first;
11295 UntiedRealAddr = I->second.second;
11299 if (CVD->
hasAttr<OMPAllocateDeclAttr>()) {
11308 Size = CGF.
Builder.CreateNUWAdd(
11317 const auto *AA = CVD->
getAttr<OMPAllocateDeclAttr>();
11318 const Expr *Allocator = AA->getAllocator();
11322 Args.push_back(ThreadID);
11324 Args.push_back(Alignment);
11325 Args.push_back(Size);
11326 Args.push_back(AllocVal);
11327 llvm::omp::RuntimeFunction FnID =
11328 Alignment ? OMPRTL___kmpc_aligned_alloc : OMPRTL___kmpc_alloc;
11332 llvm::FunctionCallee FiniRTLFn =
OMPBuilder.getOrCreateRuntimeFunction(
11342 llvm::FunctionCallee RTLFn;
11345 const Expr *AllocExpr;
11348 OMPAllocateCleanupTy(llvm::FunctionCallee RTLFn,
11350 const Expr *AllocExpr)
11351 : RTLFn(RTLFn), LocEncoding(LocEncoding), Addr(Addr),
11352 AllocExpr(AllocExpr) {}
11356 llvm::Value *Args[3];
11362 Args[2] = AllocVal;
11370 CGF.
EHStack.pushCleanup<OMPAllocateCleanupTy>(
11372 VDAddr, Allocator);
11373 if (UntiedRealAddr.
isValid())
11376 Region->emitUntiedSwitch(CGF);
11393 assert(CGM.
getLangOpts().OpenMP &&
"Not in OpenMP mode.");
11399 for (
const Stmt *Ref :
C->private_refs()) {
11400 const auto *SimpleRefExpr = cast<Expr>(Ref)->IgnoreParenImpCasts();
11402 if (
const auto *DRE = dyn_cast<DeclRefExpr>(SimpleRefExpr)) {
11403 VD = DRE->getDecl();
11405 const auto *ME = cast<MemberExpr>(SimpleRefExpr);
11406 assert((ME->isImplicitCXXThis() ||
11407 isa<CXXThisExpr>(ME->getBase()->IgnoreParenImpCasts())) &&
11408 "Expected member of current class.");
11409 VD = ME->getMemberDecl();
11425 std::pair<Address, Address>> &LocalVars)
11426 :
CGM(CGF.
CGM), NeedToPush(!LocalVars.empty()) {
11443 return llvm::any_of(
11448void CGOpenMPRuntime::LastprivateConditionalRAII::tryToDisableInnerAnalysis(
11458 const CapturedStmt *CS = S.getCapturedStmt(CaptureRegions.front());
11466 for (
const Expr *Ref :
C->varlists()) {
11467 if (!Ref->getType()->isScalarType())
11469 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
11472 NeedToCheckForLPCs.insert(DRE->getDecl());
11476 for (
const Expr *Ref :
C->varlists()) {
11477 if (!Ref->getType()->isScalarType())
11479 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
11482 NeedToCheckForLPCs.insert(DRE->getDecl());
11486 for (
const Expr *Ref :
C->varlists()) {
11487 if (!Ref->getType()->isScalarType())
11489 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
11492 NeedToCheckForLPCs.insert(DRE->getDecl());
11496 for (
const Expr *Ref :
C->varlists()) {
11497 if (!Ref->getType()->isScalarType())
11499 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
11502 NeedToCheckForLPCs.insert(DRE->getDecl());
11506 for (
const Expr *Ref :
C->varlists()) {
11507 if (!Ref->getType()->isScalarType())
11509 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
11512 NeedToCheckForLPCs.insert(DRE->getDecl());
11515 for (
const Decl *VD : NeedToCheckForLPCs) {
11516 for (
const LastprivateConditionalData &
Data :
11518 if (
Data.DeclToUniqueName.count(VD) > 0) {
11519 if (!
Data.Disabled)
11520 NeedToAddForLPCsAsDisabled.insert(VD);
11527CGOpenMPRuntime::LastprivateConditionalRAII::LastprivateConditionalRAII(
11530 Action((
CGM.getLangOpts().OpenMP >= 50 &&
11533 return C->getKind() ==
11534 OMPC_LASTPRIVATE_conditional;
11536 ? ActionToDo::PushAsLastprivateConditional
11537 : ActionToDo::DoNotPush) {
11539 if (
CGM.
getLangOpts().OpenMP < 50 || Action == ActionToDo::DoNotPush)
11541 assert(Action == ActionToDo::PushAsLastprivateConditional &&
11542 "Expected a push action.");
11546 if (
C->getKind() != OMPC_LASTPRIVATE_conditional)
11549 for (
const Expr *Ref :
C->varlists()) {
11550 Data.DeclToUniqueName.insert(std::make_pair(
11551 cast<DeclRefExpr>(Ref->IgnoreParenImpCasts())->getDecl(),
11555 Data.IVLVal = IVLVal;
11559CGOpenMPRuntime::LastprivateConditionalRAII::LastprivateConditionalRAII(
11561 :
CGM(CGF.
CGM), Action(ActionToDo::DoNotPush) {
11562 assert(CGM.
getLangOpts().OpenMP &&
"Not in OpenMP mode.");
11566 tryToDisableInnerAnalysis(S, NeedToAddForLPCsAsDisabled);
11567 if (!NeedToAddForLPCsAsDisabled.empty()) {
11568 Action = ActionToDo::DisableLastprivateConditional;
11569 LastprivateConditionalData &
Data =
11571 for (
const Decl *VD : NeedToAddForLPCsAsDisabled)
11574 Data.Disabled =
true;
11587 if (Action == ActionToDo::DisableLastprivateConditional) {
11589 "Expected list of disabled private vars.");
11592 if (Action == ActionToDo::PushAsLastprivateConditional) {
11595 "Expected list of lastprivate conditional vars.");
11610 auto VI = I->getSecond().find(VD);
11611 if (VI == I->getSecond().end()) {
11612 RecordDecl *RD =
C.buildImplicitRecord(
"lasprivate.conditional");
11617 NewType =
C.getRecordType(RD);
11620 I->getSecond().try_emplace(VD, NewType, VDField, FiredField, BaseLVal);
11622 NewType = std::get<0>(VI->getSecond());
11623 VDField = std::get<1>(VI->getSecond());
11624 FiredField = std::get<2>(VI->getSecond());
11625 BaseLVal = std::get<3>(VI->getSecond());
11637class LastprivateConditionalRefChecker final
11640 const Expr *FoundE =
nullptr;
11641 const Decl *FoundD =
nullptr;
11642 StringRef UniqueDeclName;
11644 llvm::Function *FoundFn =
nullptr;
11650 llvm::reverse(LPM)) {
11651 auto It = D.DeclToUniqueName.find(E->
getDecl());
11652 if (It == D.DeclToUniqueName.end())
11658 UniqueDeclName = It->second;
11663 return FoundE == E;
11666 if (!CodeGenFunction::IsWrappedCXXThis(E->
getBase()))
11669 llvm::reverse(LPM)) {
11671 if (It == D.DeclToUniqueName.end())
11677 UniqueDeclName = It->second;
11682 return FoundE == E;
11684 bool VisitStmt(
const Stmt *S) {
11685 for (
const Stmt *Child : S->children()) {
11688 if (
const auto *E = dyn_cast<Expr>(Child))
11696 explicit LastprivateConditionalRefChecker(
11699 std::tuple<const Expr *, const Decl *, StringRef, LValue, llvm::Function *>
11700 getFoundData()
const {
11701 return std::make_tuple(FoundE, FoundD, UniqueDeclName, IVLVal, FoundFn);
11708 StringRef UniqueDeclName,
11714 llvm::Constant *LastIV =
OMPBuilder.getOrCreateInternalVariable(
11715 LLIVTy,
getName({UniqueDeclName,
"iv"}));
11716 cast<llvm::GlobalVariable>(LastIV)->setAlignment(
11723 llvm::GlobalVariable *
Last =
OMPBuilder.getOrCreateInternalVariable(
11725 cast<llvm::GlobalVariable>(
Last)->setAlignment(
11739 auto &&CodeGen = [&LastIVLVal, &IVLVal, IVVal, &LVal, &LastLVal,
11745 llvm::Value *CmpRes;
11747 CmpRes = CGF.
Builder.CreateICmpSLE(LastIVVal, IVVal);
11750 "Loop iteration variable must be integer.");
11751 CmpRes = CGF.
Builder.CreateICmpULE(LastIVVal, IVVal);
11755 CGF.
Builder.CreateCondBr(CmpRes, ThenBB, ExitBB);
11776 "Aggregates are not supported in lastprivate conditional.");
11799 if (!Checker.Visit(LHS))
11801 const Expr *FoundE;
11802 const Decl *FoundD;
11803 StringRef UniqueDeclName;
11805 llvm::Function *FoundFn;
11806 std::tie(FoundE, FoundD, UniqueDeclName, IVLVal, FoundFn) =
11807 Checker.getFoundData();
11808 if (FoundFn != CGF.
CurFn) {
11813 "Lastprivate conditional is not found in outer region.");
11814 QualType StructTy = std::get<0>(It->getSecond());
11815 const FieldDecl* FiredDecl = std::get<2>(It->getSecond());
11826 FiredLVal, llvm::AtomicOrdering::Unordered,
11844 auto It = llvm::find_if(
11846 if (It ==
Range.end() || It->Fn != CGF.
CurFn)
11850 "Lastprivates must be registered already.");
11854 for (
const auto &Pair : It->DeclToUniqueName) {
11855 const auto *VD = cast<VarDecl>(Pair.first->getCanonicalDecl());
11858 auto I = LPCI->getSecond().find(Pair.first);
11859 assert(I != LPCI->getSecond().end() &&
11860 "Lastprivate must be rehistered already.");
11862 LValue BaseLVal = std::get<3>(I->getSecond());
11866 llvm::Value *Cmp = CGF.
Builder.CreateIsNotNull(Res);
11870 CGF.
Builder.CreateCondBr(Cmp, ThenBB, DoneBB);
11895 "Unknown lastprivate conditional variable.");
11896 StringRef UniqueName = It->second;
11897 llvm::GlobalVariable *GV =
CGM.
getModule().getNamedGlobal(UniqueName);
11911 llvm_unreachable(
"Not supported in SIMD-only mode");
11918 llvm_unreachable(
"Not supported in SIMD-only mode");
11925 bool Tied,
unsigned &NumberOfParts) {
11926 llvm_unreachable(
"Not supported in SIMD-only mode");
11931 llvm::Function *OutlinedFn,
11933 const Expr *IfCond,
11934 llvm::Value *NumThreads) {
11935 llvm_unreachable(
"Not supported in SIMD-only mode");
11941 const Expr *Hint) {
11942 llvm_unreachable(
"Not supported in SIMD-only mode");
11948 llvm_unreachable(
"Not supported in SIMD-only mode");
11954 const Expr *Filter) {
11955 llvm_unreachable(
"Not supported in SIMD-only mode");
11960 llvm_unreachable(
"Not supported in SIMD-only mode");
11966 llvm_unreachable(
"Not supported in SIMD-only mode");
11974 llvm_unreachable(
"Not supported in SIMD-only mode");
11981 llvm_unreachable(
"Not supported in SIMD-only mode");
11988 bool ForceSimpleCall) {
11989 llvm_unreachable(
"Not supported in SIMD-only mode");
11996 llvm_unreachable(
"Not supported in SIMD-only mode");
12002 llvm_unreachable(
"Not supported in SIMD-only mode");
12008 llvm_unreachable(
"Not supported in SIMD-only mode");
12015 llvm_unreachable(
"Not supported in SIMD-only mode");
12021 llvm_unreachable(
"Not supported in SIMD-only mode");
12026 unsigned IVSize,
bool IVSigned,
12029 llvm_unreachable(
"Not supported in SIMD-only mode");
12033 llvm::Value *NumThreads,
12035 llvm_unreachable(
"Not supported in SIMD-only mode");
12039 ProcBindKind ProcBind,
12041 llvm_unreachable(
"Not supported in SIMD-only mode");
12048 llvm_unreachable(
"Not supported in SIMD-only mode");
12054 llvm_unreachable(
"Not supported in SIMD-only mode");
12059 llvm_unreachable(
"Not supported in SIMD-only mode");
12065 llvm::AtomicOrdering AO) {
12066 llvm_unreachable(
"Not supported in SIMD-only mode");
12071 llvm::Function *TaskFunction,
12073 const Expr *IfCond,
12075 llvm_unreachable(
"Not supported in SIMD-only mode");
12082 llvm_unreachable(
"Not supported in SIMD-only mode");
12089 assert(Options.SimpleReduction &&
"Only simple reduction is expected.");
12091 ReductionOps, Options);
12097 llvm_unreachable(
"Not supported in SIMD-only mode");
12102 bool IsWorksharingReduction) {
12103 llvm_unreachable(
"Not supported in SIMD-only mode");
12110 llvm_unreachable(
"Not supported in SIMD-only mode");
12115 llvm::Value *ReductionsPtr,
12117 llvm_unreachable(
"Not supported in SIMD-only mode");
12123 llvm_unreachable(
"Not supported in SIMD-only mode");
12129 llvm_unreachable(
"Not supported in SIMD-only mode");
12135 llvm_unreachable(
"Not supported in SIMD-only mode");
12140 llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,
12142 llvm_unreachable(
"Not supported in SIMD-only mode");
12147 llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID,
const Expr *IfCond,
12148 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier>
Device,
12152 llvm_unreachable(
"Not supported in SIMD-only mode");
12156 llvm_unreachable(
"Not supported in SIMD-only mode");
12160 llvm_unreachable(
"Not supported in SIMD-only mode");
12170 llvm::Function *OutlinedFn,
12172 llvm_unreachable(
"Not supported in SIMD-only mode");
12176 const Expr *NumTeams,
12177 const Expr *ThreadLimit,
12179 llvm_unreachable(
"Not supported in SIMD-only mode");
12186 llvm_unreachable(
"Not supported in SIMD-only mode");
12192 llvm_unreachable(
"Not supported in SIMD-only mode");
12198 llvm_unreachable(
"Not supported in SIMD-only mode");
12203 llvm_unreachable(
"Not supported in SIMD-only mode");
12208 llvm_unreachable(
"Not supported in SIMD-only mode");
12213 const VarDecl *NativeParam)
const {
12214 llvm_unreachable(
"Not supported in SIMD-only mode");
12220 const VarDecl *TargetParam)
const {
12221 llvm_unreachable(
"Not supported in SIMD-only mode");
Provides LLVM's BitmaskEnum facility to enumeration types declared in namespace clang.
static llvm::Value * emitCopyprivateCopyFunction(CodeGenModule &CGM, llvm::Type *ArgsElemType, ArrayRef< const Expr * > CopyprivateVars, ArrayRef< const Expr * > DestExprs, ArrayRef< const Expr * > SrcExprs, ArrayRef< const Expr * > AssignmentOps, SourceLocation Loc)
static StringRef getIdentStringFromSourceLocation(CodeGenFunction &CGF, SourceLocation Loc, SmallString< 128 > &Buffer)
llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind convertCaptureClause(const VarDecl *VD)
static void emitOffloadingArrays(CodeGenFunction &CGF, MappableExprsHandler::MapCombinedInfoTy &CombinedInfo, CGOpenMPRuntime::TargetDataInfo &Info, llvm::OpenMPIRBuilder &OMPBuilder, bool IsNonContiguous=false)
Emit the arrays used to pass the captures and map information to the offloading runtime library.
static RecordDecl * createKmpTaskTWithPrivatesRecordDecl(CodeGenModule &CGM, QualType KmpTaskTQTy, ArrayRef< PrivateDataTy > Privates)
llvm::Value * emitDynCGGroupMem(const OMPExecutableDirective &D, CodeGenFunction &CGF)
static void emitInitWithReductionInitializer(CodeGenFunction &CGF, const OMPDeclareReductionDecl *DRD, const Expr *InitOp, Address Private, Address Original, QualType Ty)
static Address castToBase(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy, Address OriginalBaseAddress, llvm::Value *Addr)
static void emitPrivatesInit(CodeGenFunction &CGF, const OMPExecutableDirective &D, Address KmpTaskSharedsPtr, LValue TDBase, const RecordDecl *KmpTaskTWithPrivatesQTyRD, QualType SharedsTy, QualType SharedsPtrTy, const OMPTaskDataTy &Data, ArrayRef< PrivateDataTy > Privates, bool ForDup)
Emit initialization for private variables in task-based directives.
static llvm::Value * emitDestructorsFunction(CodeGenModule &CGM, SourceLocation Loc, QualType KmpInt32Ty, QualType KmpTaskTWithPrivatesPtrQTy, QualType KmpTaskTWithPrivatesQTy)
static unsigned evaluateCDTSize(const FunctionDecl *FD, ArrayRef< ParamAttrTy > ParamAttrs)
static void EmitOMPAggregateReduction(CodeGenFunction &CGF, QualType Type, const VarDecl *LHSVar, const VarDecl *RHSVar, const llvm::function_ref< void(CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *)> &RedOpGen, const Expr *XExpr=nullptr, const Expr *EExpr=nullptr, const Expr *UpExpr=nullptr)
Emit reduction operation for each element of array (required for array sections) LHS op = RHS.
static void emitTargetCallFallback(CGOpenMPRuntime *OMPRuntime, llvm::Function *OutlinedFn, const OMPExecutableDirective &D, llvm::SmallVectorImpl< llvm::Value * > &CapturedVars, bool RequiresOuterTask, const CapturedStmt &CS, bool OffloadingMandatory, CodeGenFunction &CGF)
static llvm::Value * emitReduceInitFunction(CodeGenModule &CGM, SourceLocation Loc, ReductionCodeGen &RCG, unsigned N)
Emits reduction initializer function:
static RTCancelKind getCancellationKind(OpenMPDirectiveKind CancelRegion)
static void emitDependData(CodeGenFunction &CGF, QualType &KmpDependInfoTy, llvm::PointerUnion< unsigned *, LValue * > Pos, const OMPTaskDataTy::DependData &Data, Address DependenciesArray)
static llvm::Value * emitTaskPrivateMappingFunction(CodeGenModule &CGM, SourceLocation Loc, const OMPTaskDataTy &Data, QualType PrivatesQTy, ArrayRef< PrivateDataTy > Privates)
Emit a privates mapping function for correct handling of private and firstprivate variables.
static llvm::Value * emitReduceCombFunction(CodeGenModule &CGM, SourceLocation Loc, ReductionCodeGen &RCG, unsigned N, const Expr *ReductionOp, const Expr *LHS, const Expr *RHS, const Expr *PrivateRef)
Emits reduction combiner function:
static RecordDecl * createPrivatesRecordDecl(CodeGenModule &CGM, ArrayRef< PrivateDataTy > Privates)
static llvm::Value * getAllocatorVal(CodeGenFunction &CGF, const Expr *Allocator)
Return allocator value from expression, or return a null allocator (default when no allocator specifi...
static llvm::Function * emitProxyTaskFunction(CodeGenModule &CGM, SourceLocation Loc, OpenMPDirectiveKind Kind, QualType KmpInt32Ty, QualType KmpTaskTWithPrivatesPtrQTy, QualType KmpTaskTWithPrivatesQTy, QualType KmpTaskTQTy, QualType SharedsPtrTy, llvm::Function *TaskFunction, llvm::Value *TaskPrivatesMap)
Emit a proxy function which accepts kmp_task_t as the second argument.
static void addAArch64VectorName(T VLEN, StringRef LMask, StringRef Prefix, char ISA, StringRef ParSeq, StringRef MangledName, bool OutputBecomesInput, llvm::Function *Fn)
static bool isAllocatableDecl(const VarDecl *VD)
static llvm::Value * getAlignmentValue(CodeGenModule &CGM, const VarDecl *VD)
Return the alignment from an allocate directive if present.
static void emitTargetCallKernelLaunch(CGOpenMPRuntime *OMPRuntime, llvm::Function *OutlinedFn, const OMPExecutableDirective &D, llvm::SmallVectorImpl< llvm::Value * > &CapturedVars, bool RequiresOuterTask, const CapturedStmt &CS, bool OffloadingMandatory, llvm::PointerIntPair< const Expr *, 2, OpenMPDeviceClauseModifier > Device, llvm::Value *OutlinedFnID, CodeGenFunction::OMPTargetDataInfo &InputInfo, llvm::Value *&MapTypesArray, llvm::Value *&MapNamesArray, llvm::function_ref< llvm::Value *(CodeGenFunction &CGF, const OMPLoopDirective &D)> SizeEmitter, CodeGenFunction &CGF, CodeGenModule &CGM)
static std::tuple< unsigned, unsigned, bool > getNDSWDS(const FunctionDecl *FD, ArrayRef< ParamAttrTy > ParamAttrs)
static const OMPExecutableDirective * getNestedDistributeDirective(ASTContext &Ctx, const OMPExecutableDirective &D)
Check for inner distribute directive.
static std::pair< llvm::Value *, llvm::Value * > getPointerAndSize(CodeGenFunction &CGF, const Expr *E)
static const VarDecl * getBaseDecl(const Expr *Ref, const DeclRefExpr *&DE)
static bool isTrivial(ASTContext &Ctx, const Expr *E)
Checks if the expression is constant or does not have non-trivial function calls.
static OpenMPSchedType getRuntimeSchedule(OpenMPScheduleClauseKind ScheduleKind, bool Chunked, bool Ordered)
Map the OpenMP loop schedule to the runtime enumeration.
static void getNumThreads(CodeGenFunction &CGF, const CapturedStmt *CS, const Expr **E, int32_t &UpperBound, bool UpperBoundOnly, llvm::Value **CondVal)
Check for a num threads constant value (stored in DefaultVal), or expression (stored in E).
static llvm::Value * emitDeviceID(llvm::PointerIntPair< const Expr *, 2, OpenMPDeviceClauseModifier > Device, CodeGenFunction &CGF)
static const OMPDeclareReductionDecl * getReductionInit(const Expr *ReductionOp)
Check if the combiner is a call to UDR combiner and if it is so return the UDR decl used for reductio...
static bool checkInitIsRequired(CodeGenFunction &CGF, ArrayRef< PrivateDataTy > Privates)
Check if duplication function is required for taskloops.
static bool checkDestructorsRequired(const RecordDecl *KmpTaskTWithPrivatesQTyRD, ArrayRef< PrivateDataTy > Privates)
Checks if destructor function is required to be generated.
static llvm::TargetRegionEntryInfo getEntryInfoFromPresumedLoc(CodeGenModule &CGM, llvm::OpenMPIRBuilder &OMPBuilder, SourceLocation BeginLoc, llvm::StringRef ParentName="")
static std::string generateUniqueName(CodeGenModule &CGM, StringRef Prefix, const Expr *Ref)
Generates unique name for artificial threadprivate variables.
static void emitForStaticInitCall(CodeGenFunction &CGF, llvm::Value *UpdateLocation, llvm::Value *ThreadId, llvm::FunctionCallee ForStaticInitFunction, OpenMPSchedType Schedule, OpenMPScheduleClauseModifier M1, OpenMPScheduleClauseModifier M2, const CGOpenMPRuntime::StaticRTInput &Values)
static LValue loadToBegin(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy, LValue BaseLV)
static void getKmpAffinityType(ASTContext &C, QualType &KmpTaskAffinityInfoTy)
Builds kmp_depend_info, if it is not built yet, and builds flags type.
static void getDependTypes(ASTContext &C, QualType &KmpDependInfoTy, QualType &FlagsTy)
Builds kmp_depend_info, if it is not built yet, and builds flags type.
static llvm::Value * emitTaskDupFunction(CodeGenModule &CGM, SourceLocation Loc, const OMPExecutableDirective &D, QualType KmpTaskTWithPrivatesPtrQTy, const RecordDecl *KmpTaskTWithPrivatesQTyRD, const RecordDecl *KmpTaskTQTyRD, QualType SharedsTy, QualType SharedsPtrTy, const OMPTaskDataTy &Data, ArrayRef< PrivateDataTy > Privates, bool WithLastIter)
Emit task_dup function (for initialization of private/firstprivate/lastprivate vars and last_iter fla...
static llvm::Value * emitReduceFiniFunction(CodeGenModule &CGM, SourceLocation Loc, ReductionCodeGen &RCG, unsigned N)
Emits reduction finalizer function:
static void EmitOMPAggregateInit(CodeGenFunction &CGF, Address DestAddr, QualType Type, bool EmitDeclareReductionInit, const Expr *Init, const OMPDeclareReductionDecl *DRD, Address SrcAddr=Address::invalid())
Emit initialization of arrays of complex types.
static bool getAArch64PBV(QualType QT, ASTContext &C)
Pass By Value (PBV), as defined in 3.1.2 of the AAVFABI.
static unsigned getAArch64LS(QualType QT, ParamKindTy Kind, ASTContext &C)
Computes the lane size (LS) of a return type or of an input parameter, as defined by LS(P) in 3....
static void EmitDoacrossOrdered(CodeGenFunction &CGF, CodeGenModule &CGM, const T *C, llvm::Value *ULoc, llvm::Value *ThreadID)
static RTLDependenceKindTy translateDependencyKind(OpenMPDependClauseKind K)
Translates internal dependency kind into the runtime kind.
static void emitTargetCallElse(CGOpenMPRuntime *OMPRuntime, llvm::Function *OutlinedFn, const OMPExecutableDirective &D, llvm::SmallVectorImpl< llvm::Value * > &CapturedVars, bool RequiresOuterTask, const CapturedStmt &CS, bool OffloadingMandatory, CodeGenFunction &CGF)
static llvm::Function * emitCombinerOrInitializer(CodeGenModule &CGM, QualType Ty, const Expr *CombinerInitializer, const VarDecl *In, const VarDecl *Out, bool IsCombiner)
static void emitX86DeclareSimdFunction(const FunctionDecl *FD, llvm::Function *Fn, const llvm::APSInt &VLENVal, ArrayRef< ParamAttrTy > ParamAttrs, OMPDeclareSimdDeclAttr::BranchStateTy State)
static void emitReductionCombiner(CodeGenFunction &CGF, const Expr *ReductionOp)
Emit reduction combiner.
static std::string mangleVectorParameters(ArrayRef< ParamAttrTy > ParamAttrs)
Mangle the parameter part of the vector function name according to their OpenMP classification.
static llvm::Function * emitParallelOrTeamsOutlinedFunction(CodeGenModule &CGM, const OMPExecutableDirective &D, const CapturedStmt *CS, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const StringRef OutlinedHelperName, const RegionCodeGenTy &CodeGen)
static void emitAArch64DeclareSimdFunction(CodeGenModule &CGM, const FunctionDecl *FD, unsigned UserVLEN, ArrayRef< ParamAttrTy > ParamAttrs, OMPDeclareSimdDeclAttr::BranchStateTy State, StringRef MangledName, char ISA, unsigned VecRegSize, llvm::Function *Fn, SourceLocation SLoc)
Emit vector function attributes for AArch64, as defined in the AAVFABI.
static Address emitAddrOfVarFromArray(CodeGenFunction &CGF, Address Array, unsigned Index, const VarDecl *Var)
Given an array of pointers to variables, project the address of a given variable.
llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseKind convertDeviceClause(const VarDecl *VD)
static bool isAssumedToBeNotEmitted(const ValueDecl *VD, bool IsDevice)
static void addAArch64AdvSIMDNDSNames(unsigned NDS, StringRef Mask, StringRef Prefix, char ISA, StringRef ParSeq, StringRef MangledName, bool OutputBecomesInput, llvm::Function *Fn)
static FieldDecl * addFieldToRecordDecl(ASTContext &C, DeclContext *DC, QualType FieldTy)
static ValueDecl * getDeclFromThisExpr(const Expr *E)
llvm::Constant * emitMappingInformation(CodeGenFunction &CGF, llvm::OpenMPIRBuilder &OMPBuilder, MappableExprsHandler::MappingExprInfo &MapExprs)
Emit a string constant containing the names of the values mapped to the offloading runtime library.
static RecordDecl * createKmpTaskTRecordDecl(CodeGenModule &CGM, OpenMPDirectiveKind Kind, QualType KmpInt32Ty, QualType KmpRoutineEntryPointerQTy)
static int addMonoNonMonoModifier(CodeGenModule &CGM, OpenMPSchedType Schedule, OpenMPScheduleClauseModifier M1, OpenMPScheduleClauseModifier M2)
static bool getAArch64MTV(QualType QT, ParamKindTy Kind)
Maps To Vector (MTV), as defined in 4.1.1 of the AAVFABI (2021Q1).
Defines the clang::FileManager interface and associated types.
This file defines OpenMP AST classes for clauses.
Defines some OpenMP-specific enums and functions.
Defines the SourceManager interface.
This file defines OpenMP AST classes for executable directives and clauses.
__DEVICE__ int max(int __a, int __b)
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
SourceManager & getSourceManager()
const ConstantArrayType * getAsConstantArrayType(QualType T) const
CharUnits getTypeAlignInChars(QualType T) const
Return the ABI-specified alignment of a (complete) type T, in characters.
const ASTRecordLayout & getASTRecordLayout(const RecordDecl *D) const
Get or compute information about the layout of the specified record (struct/union/class) D,...
bool hasSameType(QualType T1, QualType T2) const
Determine whether the given types T1 and T2 are equivalent.
QualType getPointerType(QualType T) const
Return the uniqued reference to the type for a pointer to the specified type.
QualType getConstantArrayType(QualType EltTy, const llvm::APInt &ArySize, const Expr *SizeExpr, ArraySizeModifier ASM, unsigned IndexTypeQuals) const
Return the unique reference to the type for a constant array of the specified element type.
const LangOptions & getLangOpts() const
QualType getIntTypeForBitwidth(unsigned DestWidth, unsigned Signed) const
getIntTypeForBitwidth - sets integer QualTy according to specified details: bitwidth,...
CanQualType getSizeType() const
Return the unique type for "size_t" (C99 7.17), defined in <stddef.h>.
CharUnits getDeclAlign(const Decl *D, bool ForAlignof=false) const
Return a conservative estimate of the alignment of the specified decl D.
const ArrayType * getAsArrayType(QualType T) const
Type Query functions.
CharUnits getTypeSizeInChars(QualType T) const
Return the size of the specified (complete) type T, in characters.
const VariableArrayType * getAsVariableArrayType(QualType T) const
const TargetInfo & getTargetInfo() const
CharUnits getNonVirtualSize() const
getNonVirtualSize - Get the non-virtual size (in chars) of an object, which is the size of the object...
static QualType getBaseOriginalType(const Expr *Base)
Return original type of the base expression for array section.
Represents an array type, per C99 6.7.5.2 - Array Declarators.
Attr - This represents one attribute.
Represents a C++ constructor within a class.
Represents a C++ destructor within a class.
Represents a static or instance method of a struct/union/class.
const CXXRecordDecl * getParent() const
Return the parent of this method declaration, which is the class in which this method is defined.
QualType getFunctionObjectParameterType() const
Represents a C++ struct/union/class.
bool isLambda() const
Determine whether this class describes a lambda function object.
void getCaptureFields(llvm::DenseMap< const ValueDecl *, FieldDecl * > &Captures, FieldDecl *&ThisCapture) const
For a closure type, retrieve the mapping from captured variables and this to the non-static data memb...
unsigned getNumBases() const
Retrieves the number of base classes of this class.
base_class_range vbases()
capture_const_range captures() const
CXXDestructorDecl * getDestructor() const
Returns the destructor decl for this class.
CanProxy< U > castAs() const
A wrapper class around a pointer that always points to its canonical declaration.
Describes the capture of either a variable, or 'this', or variable-length array type.
bool capturesVariableByCopy() const
Determine whether this capture handles a variable by copy.
VarDecl * getCapturedVar() const
Retrieve the declaration of the variable being captured.
bool capturesVariableArrayType() const
Determine whether this capture handles a variable-length array type.
bool capturesThis() const
Determine whether this capture handles the C++ 'this' pointer.
bool capturesVariable() const
Determine whether this capture handles a variable (by reference).
This captures a statement into a function.
capture_iterator capture_end() const
Retrieve an iterator pointing past the end of the sequence of captures.
const RecordDecl * getCapturedRecordDecl() const
Retrieve the record declaration for captured variables.
Stmt * getCapturedStmt()
Retrieve the statement being captured.
bool capturesVariable(const VarDecl *Var) const
True if this variable has been captured.
capture_iterator capture_begin()
Retrieve an iterator pointing to the first capture.
CharUnits - This is an opaque type for sizes expressed in character units.
bool isZero() const
isZero - Test whether the quantity equals zero.
llvm::Align getAsAlign() const
getAsAlign - Returns Quantity as a valid llvm::Align, Beware llvm::Align assumes power of two 8-bit b...
QuantityType getQuantity() const
getQuantity - Get the raw integer representation of this quantity.
CharUnits alignmentOfArrayElement(CharUnits elementSize) const
Given that this is the alignment of the first element of an array, return the minimum alignment of an...
static CharUnits fromQuantity(QuantityType Quantity)
fromQuantity - Construct a CharUnits quantity from a raw integer type.
CharUnits alignTo(const CharUnits &Align) const
alignTo - Returns the next integer (mod 2**64) that is greater than or equal to this quantity and is ...
Like RawAddress, an abstract representation of an aligned address, but the pointer contained in this ...
llvm::Value * emitRawPointer(CodeGenFunction &CGF) const
Return the pointer contained in this class after authenticating it and adding offset to it if necessa...
CharUnits getAlignment() const
llvm::Type * getElementType() const
Return the type of the values stored in this address.
Address withPointer(llvm::Value *NewPointer, KnownNonNull_t IsKnownNonNull) const
Return address with different pointer, but same element type and alignment.
Address withElementType(llvm::Type *ElemTy) const
Return address with different element type, but same pointer and alignment.
Address withAlignment(CharUnits NewAlignment) const
Return address with different alignment, but same pointer and element type.
llvm::PointerType * getType() const
Return the type of the pointer value.
static ApplyDebugLocation CreateArtificial(CodeGenFunction &CGF)
Apply TemporaryLocation if it is valid.
static ApplyDebugLocation CreateDefaultArtificial(CodeGenFunction &CGF, SourceLocation TemporaryLocation)
Apply TemporaryLocation if it is valid.
static ApplyDebugLocation CreateEmpty(CodeGenFunction &CGF)
Set the IRBuilder to not attach debug locations.
CGBlockInfo - Information to generate a block literal.
llvm::StoreInst * CreateStore(llvm::Value *Val, Address Addr, bool IsVolatile=false)
llvm::Value * CreateIsNull(Address Addr, const Twine &Name="")
Address CreateGEP(CodeGenFunction &CGF, Address Addr, llvm::Value *Index, const llvm::Twine &Name="")
Address CreatePointerBitCastOrAddrSpaceCast(Address Addr, llvm::Type *Ty, llvm::Type *ElementTy, const llvm::Twine &Name="")
Address CreateConstArrayGEP(Address Addr, uint64_t Index, const llvm::Twine &Name="")
Given addr = [n x T]* ... produce name = getelementptr inbounds addr, i64 0, i64 index where i64 is a...
llvm::LoadInst * CreateLoad(Address Addr, const llvm::Twine &Name="")
llvm::CallInst * CreateMemCpy(Address Dest, Address Src, llvm::Value *Size, bool IsVolatile=false)
Address CreateConstGEP(Address Addr, uint64_t Index, const llvm::Twine &Name="")
Given addr = T* ... produce name = getelementptr inbounds addr, i64 index where i64 is actually the t...
MangleContext & getMangleContext()
Gets the mangle context.
CGFunctionInfo - Class to encapsulate the information about a function definition.
DisableAutoDeclareTargetRAII(CodeGenModule &CGM)
~DisableAutoDeclareTargetRAII()
Manages list of lastprivate conditional decls for the specified directive.
~LastprivateConditionalRAII()
static LastprivateConditionalRAII disable(CodeGenFunction &CGF, const OMPExecutableDirective &S)
NontemporalDeclsRAII(CodeGenModule &CGM, const OMPLoopDirective &S)
Struct that keeps all the relevant information that should be kept throughout a 'target data' region.
llvm::DenseMap< const ValueDecl *, llvm::Value * > CaptureDeviceAddrMap
Map between the a declaration of a capture and the corresponding new llvm address where the runtime r...
~UntiedTaskLocalDeclsRAII()
UntiedTaskLocalDeclsRAII(CodeGenFunction &CGF, const llvm::MapVector< CanonicalDeclPtr< const VarDecl >, std::pair< Address, Address > > &LocalVars)
virtual Address emitThreadIDAddress(CodeGenFunction &CGF, SourceLocation Loc)
Emits address of the word in a memory where current thread id is stored.
llvm::FunctionType * Kmpc_MicroTy
The type for a microtask which gets passed to __kmpc_fork_call().
llvm::StringSet ThreadPrivateWithDefinition
Set of threadprivate variables with the generated initializer.
CGOpenMPRuntime(CodeGenModule &CGM)
virtual void emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPExecutableDirective &D, llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds, const Expr *IfCond, const OMPTaskDataTy &Data)
Emit task region for the task directive.
void createOffloadEntriesAndInfoMetadata()
Creates all the offload entries in the current compilation unit along with the associated metadata.
const Expr * getNumTeamsExprForTargetDirective(CodeGenFunction &CGF, const OMPExecutableDirective &D, int32_t &MinTeamsVal, int32_t &MaxTeamsVal)
Emit the number of teams for a target directive.
virtual Address getAddrOfThreadPrivate(CodeGenFunction &CGF, const VarDecl *VD, Address VDAddr, SourceLocation Loc)
Returns address of the threadprivate variable for the current thread.
void emitDeferredTargetDecls() const
Emit deferred declare target variables marked for deferred emission.
virtual llvm::Value * emitForNext(CodeGenFunction &CGF, SourceLocation Loc, unsigned IVSize, bool IVSigned, Address IL, Address LB, Address UB, Address ST)
Call __kmpc_dispatch_next( ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter, kmp_int[32|64] *p_lowe...
bool markAsGlobalTarget(GlobalDecl GD)
Marks the declaration as already emitted for the device code and returns true, if it was marked alrea...
virtual void emitTargetDataStandAloneCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, const Expr *Device)
Emit the data mapping/movement code associated with the directive D that should be of the form 'targe...
virtual void emitNumThreadsClause(CodeGenFunction &CGF, llvm::Value *NumThreads, SourceLocation Loc)
Emits call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_threads)...
QualType SavedKmpTaskloopTQTy
Saved kmp_task_t for taskloop-based directive.
virtual void emitSingleRegion(CodeGenFunction &CGF, const RegionCodeGenTy &SingleOpGen, SourceLocation Loc, ArrayRef< const Expr * > CopyprivateVars, ArrayRef< const Expr * > DestExprs, ArrayRef< const Expr * > SrcExprs, ArrayRef< const Expr * > AssignmentOps)
Emits a single region.
virtual bool emitTargetGlobal(GlobalDecl GD)
Emit the global GD if it is meaningful for the target.
void setLocThreadIdInsertPt(CodeGenFunction &CGF, bool AtCurrentPoint=false)
std::string getOutlinedHelperName(StringRef Name) const
Get the function name of an outlined region.
bool HasEmittedDeclareTargetRegion
Flag for keeping track of weather a device routine has been emitted.
llvm::Constant * getOrCreateThreadPrivateCache(const VarDecl *VD)
If the specified mangled name is not in the module, create and return threadprivate cache object.
virtual Address getTaskReductionItem(CodeGenFunction &CGF, SourceLocation Loc, llvm::Value *ReductionsPtr, LValue SharedLVal)
Get the address of void * type of the privatue copy of the reduction item specified by the SharedLVal...
virtual void getDefaultScheduleAndChunk(CodeGenFunction &CGF, const OMPLoopDirective &S, OpenMPScheduleClauseKind &ScheduleKind, const Expr *&ChunkExpr) const
Choose default schedule type and chunk value for the schedule clause.
virtual std::pair< llvm::Function *, llvm::Function * > getUserDefinedReduction(const OMPDeclareReductionDecl *D)
Get combiner/initializer for the specified user-defined reduction, if any.
virtual bool isGPU() const
Returns true if the current target is a GPU.
static const Stmt * getSingleCompoundChild(ASTContext &Ctx, const Stmt *Body)
Checks if the Body is the CompoundStmt and returns its child statement iff there is only one that is ...
virtual void emitDeclareTargetFunction(const FunctionDecl *FD, llvm::GlobalValue *GV)
Emit code for handling declare target functions in the runtime.
llvm::Type * getKmpc_MicroPointerTy()
Returns pointer to kmpc_micro type.
bool HasRequiresUnifiedSharedMemory
Flag for keeping track of weather a requires unified_shared_memory directive is present.
llvm::Value * emitUpdateLocation(CodeGenFunction &CGF, SourceLocation Loc, unsigned Flags=0, bool EmitLoc=false)
Emits object of ident_t type with info for source location.
bool isLocalVarInUntiedTask(CodeGenFunction &CGF, const VarDecl *VD) const
Returns true if the variable is a local variable in untied task.
virtual void emitTeamsCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, SourceLocation Loc, llvm::Function *OutlinedFn, ArrayRef< llvm::Value * > CapturedVars)
Emits code for teams call of the OutlinedFn with variables captured in a record which address is stor...
virtual void emitCancellationPointCall(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind CancelRegion)
Emit code for 'cancellation point' construct.
void emitUDMapperArrayInitOrDel(CodeGenFunction &MapperCGF, llvm::Value *Handle, llvm::Value *BasePtr, llvm::Value *Ptr, llvm::Value *Size, llvm::Value *MapType, llvm::Value *MapName, CharUnits ElementSize, llvm::BasicBlock *ExitBB, bool IsInit)
Emit the array initialization or deletion portion for user-defined mapper code generation.
virtual llvm::Function * emitThreadPrivateVarDefinition(const VarDecl *VD, Address VDAddr, SourceLocation Loc, bool PerformInit, CodeGenFunction *CGF=nullptr)
Emit a code for initialization of threadprivate variable.
FunctionUDMMapTy FunctionUDMMap
virtual ConstantAddress getAddrOfDeclareTargetVar(const VarDecl *VD)
Returns the address of the variable marked as declare target with link clause OR as declare target wi...
llvm::MapVector< CanonicalDeclPtr< const VarDecl >, std::pair< Address, Address > > UntiedLocalVarsAddressesMap
llvm::Function * getOrCreateUserDefinedMapperFunc(const OMPDeclareMapperDecl *D)
Get the function for the specified user-defined mapper.
OpenMPLocThreadIDMapTy OpenMPLocThreadIDMap
virtual void functionFinished(CodeGenFunction &CGF)
Cleans up references to the objects in finished function.
void emitCall(CodeGenFunction &CGF, SourceLocation Loc, llvm::FunctionCallee Callee, ArrayRef< llvm::Value * > Args=std::nullopt) const
Emits Callee function call with arguments Args with location Loc.
virtual llvm::Function * emitTeamsOutlinedFunction(CodeGenFunction &CGF, const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen)
Emits outlined function for the specified OpenMP teams directive D.
QualType KmpTaskTQTy
Type typedef struct kmp_task { void * shareds; /**< pointer to block of pointers to shared vars / k...
llvm::OpenMPIRBuilder OMPBuilder
An OpenMP-IR-Builder instance.
virtual void emitDoacrossInit(CodeGenFunction &CGF, const OMPLoopDirective &D, ArrayRef< Expr * > NumIterations)
Emit initialization for doacross loop nesting support.
virtual void adjustTargetSpecificDataForLambdas(CodeGenFunction &CGF, const OMPExecutableDirective &D) const
Adjust some parameters for the target-based directives, like addresses of the variables captured by r...
FunctionUDRMapTy FunctionUDRMap
virtual void emitTargetDataCalls(CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, const Expr *Device, const RegionCodeGenTy &CodeGen, CGOpenMPRuntime::TargetDataInfo &Info)
Emit the target data mapping code associated with D.
virtual unsigned getDefaultLocationReserved2Flags() const
Returns additional flags that can be stored in reserved_2 field of the default location.
void computeMinAndMaxThreadsAndTeams(const OMPExecutableDirective &D, CodeGenFunction &CGF, int32_t &MinThreadsVal, int32_t &MaxThreadsVal, int32_t &MinTeamsVal, int32_t &MaxTeamsVal)
Helper to determine the min/max number of threads/teams for D.
virtual Address getParameterAddress(CodeGenFunction &CGF, const VarDecl *NativeParam, const VarDecl *TargetParam) const
Gets the address of the native argument basing on the address of the target-specific parameter.
void emitUsesAllocatorsFini(CodeGenFunction &CGF, const Expr *Allocator)
Destroys user defined allocators specified in the uses_allocators clause.
QualType KmpTaskAffinityInfoTy
Type typedef struct kmp_task_affinity_info { kmp_intptr_t base_addr; size_t len; struct { bool flag1 ...
llvm::SmallVector< NontemporalDeclsSet, 4 > NontemporalDeclsStack
Stack for list of declarations in current context marked as nontemporal.
llvm::Value * emitNumTeamsForTargetDirective(CodeGenFunction &CGF, const OMPExecutableDirective &D)
virtual void emitTargetOutlinedFunctionHelper(const OMPExecutableDirective &D, StringRef ParentName, llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID, bool IsOffloadEntry, const RegionCodeGenTy &CodeGen)
Helper to emit outlined function for 'target' directive.
void scanForTargetRegionsFunctions(const Stmt *S, StringRef ParentName)
Start scanning from statement S and emit all target regions found along the way.
SmallVector< llvm::Value *, 4 > emitDepobjElementsSizes(CodeGenFunction &CGF, QualType &KmpDependInfoTy, const OMPTaskDataTy::DependData &Data)
virtual void emitTaskgroupRegion(CodeGenFunction &CGF, const RegionCodeGenTy &TaskgroupOpGen, SourceLocation Loc)
Emit a taskgroup region.
llvm::DenseMap< llvm::Function *, llvm::DenseMap< CanonicalDeclPtr< const Decl >, std::tuple< QualType, const FieldDecl *, const FieldDecl *, LValue > > > LastprivateConditionalToTypes
Maps local variables marked as lastprivate conditional to their internal types.
virtual bool emitTargetGlobalVariable(GlobalDecl GD)
Emit the global variable if it is a valid device global variable.
virtual void emitNumTeamsClause(CodeGenFunction &CGF, const Expr *NumTeams, const Expr *ThreadLimit, SourceLocation Loc)
Emits call to void __kmpc_push_num_teams(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_teams,...
bool hasRequiresUnifiedSharedMemory() const
Return whether the unified_shared_memory has been specified.
virtual Address getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF, QualType VarType, StringRef Name)
Creates artificial threadprivate variable with name Name and type VarType.
void emitUserDefinedMapper(const OMPDeclareMapperDecl *D, CodeGenFunction *CGF=nullptr)
Emit the function for the user defined mapper construct.
bool HasEmittedTargetRegion
Flag for keeping track of weather a target region has been emitted.
void emitDepobjElements(CodeGenFunction &CGF, QualType &KmpDependInfoTy, LValue PosLVal, const OMPTaskDataTy::DependData &Data, Address DependenciesArray)
std::string getReductionFuncName(StringRef Name) const
Get the function name of a reduction function.
virtual void processRequiresDirective(const OMPRequiresDecl *D)
Perform check on requires decl to ensure that target architecture supports unified addressing.
llvm::DenseSet< CanonicalDeclPtr< const Decl > > AlreadyEmittedTargetDecls
List of the emitted declarations.
virtual llvm::Value * emitTaskReductionInit(CodeGenFunction &CGF, SourceLocation Loc, ArrayRef< const Expr * > LHSExprs, ArrayRef< const Expr * > RHSExprs, const OMPTaskDataTy &Data)
Emit a code for initialization of task reduction clause.
llvm::Value * getThreadID(CodeGenFunction &CGF, SourceLocation Loc)
Gets thread id value for the current thread.
void emitUpdateClause(CodeGenFunction &CGF, LValue DepobjLVal, OpenMPDependClauseKind NewDepKind, SourceLocation Loc)
Updates the dependency kind in the specified depobj object.
virtual void emitLastprivateConditionalFinalUpdate(CodeGenFunction &CGF, LValue PrivLVal, const VarDecl *VD, SourceLocation Loc)
Gets the address of the global copy used for lastprivate conditional update, if any.
virtual void emitErrorCall(CodeGenFunction &CGF, SourceLocation Loc, Expr *ME, bool IsFatal)
Emit __kmpc_error call for error directive extern void __kmpc_error(ident_t *loc, int severity,...
void clearLocThreadIdInsertPt(CodeGenFunction &CGF)
virtual void emitTaskyieldCall(CodeGenFunction &CGF, SourceLocation Loc)
Emits code for a taskyield directive.
std::string getName(ArrayRef< StringRef > Parts) const
Get the platform-specific name separator.
QualType KmpRoutineEntryPtrQTy
virtual void emitFlush(CodeGenFunction &CGF, ArrayRef< const Expr * > Vars, SourceLocation Loc, llvm::AtomicOrdering AO)
Emit flush of the variables specified in 'omp flush' directive.
virtual void emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPTaskDataTy &Data)
Emit code for 'taskwait' directive.
virtual void emitProcBindClause(CodeGenFunction &CGF, llvm::omp::ProcBindKind ProcBind, SourceLocation Loc)
Emit call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32 global_tid, int proc_bind) to generat...
void emitLastprivateConditionalUpdate(CodeGenFunction &CGF, LValue IVLVal, StringRef UniqueDeclName, LValue LVal, SourceLocation Loc)
Emit update for lastprivate conditional data.
virtual void emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPLoopDirective &D, llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds, const Expr *IfCond, const OMPTaskDataTy &Data)
Emit task region for the taskloop directive.
virtual void emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind Kind, bool EmitChecks=true, bool ForceSimpleCall=false)
Emit an implicit/explicit barrier for OpenMP threads.
static unsigned getDefaultFlagsForBarriers(OpenMPDirectiveKind Kind)
Returns default flags for the barriers depending on the directive, for which this barier is going to ...
virtual bool emitTargetFunctions(GlobalDecl GD)
Emit the target regions enclosed in GD function definition or the function itself in case it is a val...
TaskResultTy emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc, const OMPExecutableDirective &D, llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds, const OMPTaskDataTy &Data)
Emit task region for the task directive.
llvm::Value * emitTargetNumIterationsCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, llvm::function_ref< llvm::Value *(CodeGenFunction &CGF, const OMPLoopDirective &D)> SizeEmitter)
Return the trip count of loops associated with constructs / 'target teams distribute' and 'teams dist...
llvm::StringMap< llvm::AssertingVH< llvm::GlobalVariable >, llvm::BumpPtrAllocator > InternalVars
An ordered map of auto-generated variables to their unique names.
virtual void emitDistributeStaticInit(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDistScheduleClauseKind SchedKind, const StaticRTInput &Values)
llvm::SmallVector< UntiedLocalVarsAddressesMap, 4 > UntiedLocalVarsStack
virtual void emitForStaticFinish(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind)
Call the appropriate runtime routine to notify that we finished all the work with current loop.
virtual void emitThreadLimitClause(CodeGenFunction &CGF, const Expr *ThreadLimit, SourceLocation Loc)
Emits call to void __kmpc_set_thread_limit(ident_t *loc, kmp_int32 global_tid, kmp_int32 thread_limit...
void emitIfClause(CodeGenFunction &CGF, const Expr *Cond, const RegionCodeGenTy &ThenGen, const RegionCodeGenTy &ElseGen)
Emits code for OpenMP 'if' clause using specified CodeGen function.
Address emitDepobjDependClause(CodeGenFunction &CGF, const OMPTaskDataTy::DependData &Dependencies, SourceLocation Loc)
Emits list of dependecies based on the provided data (array of dependence/expression pairs) for depob...
bool isNontemporalDecl(const ValueDecl *VD) const
Checks if the VD variable is marked as nontemporal declaration in current context.
virtual llvm::Function * emitParallelOutlinedFunction(CodeGenFunction &CGF, const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen)
Emits outlined function for the specified OpenMP parallel directive D.
const Expr * getNumThreadsExprForTargetDirective(CodeGenFunction &CGF, const OMPExecutableDirective &D, int32_t &UpperBound, bool UpperBoundOnly, llvm::Value **CondExpr=nullptr, const Expr **ThreadLimitExpr=nullptr)
Check for a number of threads upper bound constant value (stored in UpperBound), or expression (retur...
llvm::SmallVector< LastprivateConditionalData, 4 > LastprivateConditionalStack
Stack for list of addresses of declarations in current context marked as lastprivate conditional.
virtual void emitForStaticInit(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind, const OpenMPScheduleTy &ScheduleKind, const StaticRTInput &Values)
Call the appropriate runtime routine to initialize it before start of loop.
virtual void emitDeclareSimdFunction(const FunctionDecl *FD, llvm::Function *Fn)
Marks function Fn with properly mangled versions of vector functions.
llvm::AtomicOrdering getDefaultMemoryOrdering() const
Gets default memory ordering as specified in requires directive.
llvm::SmallDenseSet< CanonicalDeclPtr< const Decl > > NontemporalDeclsSet
virtual bool isStaticNonchunked(OpenMPScheduleClauseKind ScheduleKind, bool Chunked) const
Check if the specified ScheduleKind is static non-chunked.
llvm::Value * getCriticalRegionLock(StringRef CriticalName)
Returns corresponding lock object for the specified critical region name.
virtual void emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc, const Expr *IfCond, OpenMPDirectiveKind CancelRegion)
Emit code for 'cancel' construct.
QualType SavedKmpTaskTQTy
Saved kmp_task_t for task directive.
virtual void emitMasterRegion(CodeGenFunction &CGF, const RegionCodeGenTy &MasterOpGen, SourceLocation Loc)
Emits a master region.
virtual llvm::Function * emitTaskOutlinedFunction(const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, const VarDecl *PartIDVar, const VarDecl *TaskTVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, bool Tied, unsigned &NumberOfParts)
Emits outlined function for the OpenMP task directive D.
llvm::DenseMap< llvm::Function *, unsigned > FunctionToUntiedTaskStackMap
Maps function to the position of the untied task locals stack.
void emitDestroyClause(CodeGenFunction &CGF, LValue DepobjLVal, SourceLocation Loc)
Emits the code to destroy the dependency object provided in depobj directive.
virtual void emitTaskReductionFixups(CodeGenFunction &CGF, SourceLocation Loc, ReductionCodeGen &RCG, unsigned N)
Required to resolve existing problems in the runtime.
llvm::ArrayType * KmpCriticalNameTy
Type kmp_critical_name, originally defined as typedef kmp_int32 kmp_critical_name[8];.
virtual void emitDoacrossOrdered(CodeGenFunction &CGF, const OMPDependClause *C)
Emit code for doacross ordered directive with 'depend' clause.
llvm::DenseMap< const OMPDeclareMapperDecl *, llvm::Function * > UDMMap
Map from the user-defined mapper declaration to its corresponding functions.
virtual void checkAndEmitLastprivateConditional(CodeGenFunction &CGF, const Expr *LHS)
Checks if the provided LVal is lastprivate conditional and emits the code to update the value of the ...
std::pair< llvm::Value *, LValue > getDepobjElements(CodeGenFunction &CGF, LValue DepobjLVal, SourceLocation Loc)
Returns the number of the elements and the address of the depobj dependency array.
llvm::SmallDenseSet< const VarDecl * > DeferredGlobalVariables
List of variables that can become declare target implicitly and, thus, must be emitted.
void emitUsesAllocatorsInit(CodeGenFunction &CGF, const Expr *Allocator, const Expr *AllocatorTraits)
Initializes user defined allocators specified in the uses_allocators clauses.
llvm::Type * KmpRoutineEntryPtrTy
Type typedef kmp_int32 (* kmp_routine_entry_t)(kmp_int32, void *);.
llvm::Type * getIdentTyPointerTy()
Returns pointer to ident_t type.
void emitSingleReductionCombiner(CodeGenFunction &CGF, const Expr *ReductionOp, const Expr *PrivateRef, const DeclRefExpr *LHS, const DeclRefExpr *RHS)
Emits single reduction combiner.
llvm::OpenMPIRBuilder & getOMPBuilder()
virtual void emitTargetOutlinedFunction(const OMPExecutableDirective &D, StringRef ParentName, llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID, bool IsOffloadEntry, const RegionCodeGenTy &CodeGen)
Emit outilined function for 'target' directive.
virtual void emitCriticalRegion(CodeGenFunction &CGF, StringRef CriticalName, const RegionCodeGenTy &CriticalOpGen, SourceLocation Loc, const Expr *Hint=nullptr)
Emits a critical region.
virtual void emitForOrderedIterationEnd(CodeGenFunction &CGF, SourceLocation Loc, unsigned IVSize, bool IVSigned)
Call the appropriate runtime routine to notify that we finished iteration of the ordered loop with th...
llvm::Value * emitNumThreadsForTargetDirective(CodeGenFunction &CGF, const OMPExecutableDirective &D)
Emit an expression that denotes the number of threads a target region shall use.
void emitThreadPrivateVarInit(CodeGenFunction &CGF, Address VDAddr, llvm::Value *Ctor, llvm::Value *CopyCtor, llvm::Value *Dtor, SourceLocation Loc)
Emits initialization code for the threadprivate variables.
virtual void emitUserDefinedReduction(CodeGenFunction *CGF, const OMPDeclareReductionDecl *D)
Emit code for the specified user defined reduction construct.
virtual void checkAndEmitSharedLastprivateConditional(CodeGenFunction &CGF, const OMPExecutableDirective &D, const llvm::DenseSet< CanonicalDeclPtr< const VarDecl > > &IgnoredDecls)
Checks if the lastprivate conditional was updated in inner region and writes the value.
QualType KmpDimTy
struct kmp_dim { // loop bounds info casted to kmp_int64 kmp_int64 lo; // lower kmp_int64 up; // uppe...
virtual void emitInlinedDirective(CodeGenFunction &CGF, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, bool HasCancel=false)
Emit code for the directive that does not require outlining.
virtual void registerTargetGlobalVariable(const VarDecl *VD, llvm::Constant *Addr)
Checks if the provided global decl GD is a declare target variable and registers it when emitting cod...
virtual void emitFunctionProlog(CodeGenFunction &CGF, const Decl *D)
Emits OpenMP-specific function prolog.
virtual void emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc, llvm::Function *OutlinedFn, ArrayRef< llvm::Value * > CapturedVars, const Expr *IfCond, llvm::Value *NumThreads)
Emits code for parallel or serial call of the OutlinedFn with variables captured in a record which ad...
void emitKmpRoutineEntryT(QualType KmpInt32Ty)
Build type kmp_routine_entry_t (if not built yet).
virtual bool isStaticChunked(OpenMPScheduleClauseKind ScheduleKind, bool Chunked) const
Check if the specified ScheduleKind is static chunked.
virtual void emitTargetCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond, llvm::PointerIntPair< const Expr *, 2, OpenMPDeviceClauseModifier > Device, llvm::function_ref< llvm::Value *(CodeGenFunction &CGF, const OMPLoopDirective &D)> SizeEmitter)
Emit the target offloading code associated with D.
virtual bool hasAllocateAttributeForGlobalVar(const VarDecl *VD, LangAS &AS)
Checks if the variable has associated OMPAllocateDeclAttr attribute with the predefined allocator and...
llvm::AtomicOrdering RequiresAtomicOrdering
Atomic ordering from the omp requires directive.
virtual void emitReduction(CodeGenFunction &CGF, SourceLocation Loc, ArrayRef< const Expr * > Privates, ArrayRef< const Expr * > LHSExprs, ArrayRef< const Expr * > RHSExprs, ArrayRef< const Expr * > ReductionOps, ReductionOptionsTy Options)
Emit a code for reduction clause.
std::pair< llvm::Value *, Address > emitDependClause(CodeGenFunction &CGF, ArrayRef< OMPTaskDataTy::DependData > Dependencies, SourceLocation Loc)
Emits list of dependecies based on the provided data (array of dependence/expression pairs).
llvm::StringMap< llvm::WeakTrackingVH > EmittedNonTargetVariables
List of the global variables with their addresses that should not be emitted for the target.
virtual bool isDynamic(OpenMPScheduleClauseKind ScheduleKind) const
Check if the specified ScheduleKind is dynamic.
Address emitLastprivateConditionalInit(CodeGenFunction &CGF, const VarDecl *VD)
Create specialized alloca to handle lastprivate conditionals.
virtual void emitOrderedRegion(CodeGenFunction &CGF, const RegionCodeGenTy &OrderedOpGen, SourceLocation Loc, bool IsThreads)
Emit an ordered region.
virtual void emitOutlinedFunctionCall(CodeGenFunction &CGF, SourceLocation Loc, llvm::FunctionCallee OutlinedFn, ArrayRef< llvm::Value * > Args=std::nullopt) const
Emits call of the outlined function with the provided arguments, translating these arguments to corre...
virtual Address getAddressOfLocalVariable(CodeGenFunction &CGF, const VarDecl *VD)
Gets the OpenMP-specific address of the local variable.
virtual void emitTaskReductionFini(CodeGenFunction &CGF, SourceLocation Loc, bool IsWorksharingReduction)
Emits the following code for reduction clause with task modifier:
virtual void emitMaskedRegion(CodeGenFunction &CGF, const RegionCodeGenTy &MaskedOpGen, SourceLocation Loc, const Expr *Filter=nullptr)
Emits a masked region.
QualType KmpDependInfoTy
Type typedef struct kmp_depend_info { kmp_intptr_t base_addr; size_t len; struct { bool in:1; bool ou...
llvm::Function * emitReductionFunction(StringRef ReducerName, SourceLocation Loc, llvm::Type *ArgsElemType, ArrayRef< const Expr * > Privates, ArrayRef< const Expr * > LHSExprs, ArrayRef< const Expr * > RHSExprs, ArrayRef< const Expr * > ReductionOps)
Emits reduction function.
virtual void emitForDispatchInit(CodeGenFunction &CGF, SourceLocation Loc, const OpenMPScheduleTy &ScheduleKind, unsigned IVSize, bool IVSigned, bool Ordered, const DispatchRTInput &DispatchValues)
Call the appropriate runtime routine to initialize it before start of loop.
Address getTaskReductionItem(CodeGenFunction &CGF, SourceLocation Loc, llvm::Value *ReductionsPtr, LValue SharedLVal) override
Get the address of void * type of the privatue copy of the reduction item specified by the SharedLVal...
void emitCriticalRegion(CodeGenFunction &CGF, StringRef CriticalName, const RegionCodeGenTy &CriticalOpGen, SourceLocation Loc, const Expr *Hint=nullptr) override
Emits a critical region.
void emitDistributeStaticInit(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDistScheduleClauseKind SchedKind, const StaticRTInput &Values) override
void emitForStaticInit(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind, const OpenMPScheduleTy &ScheduleKind, const StaticRTInput &Values) override
Call the appropriate runtime routine to initialize it before start of loop.
bool emitTargetGlobalVariable(GlobalDecl GD) override
Emit the global variable if it is a valid device global variable.
llvm::Value * emitForNext(CodeGenFunction &CGF, SourceLocation Loc, unsigned IVSize, bool IVSigned, Address IL, Address LB, Address UB, Address ST) override
Call __kmpc_dispatch_next( ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter, kmp_int[32|64] *p_lowe...
llvm::Function * emitThreadPrivateVarDefinition(const VarDecl *VD, Address VDAddr, SourceLocation Loc, bool PerformInit, CodeGenFunction *CGF=nullptr) override
Emit a code for initialization of threadprivate variable.
void emitTargetDataStandAloneCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, const Expr *Device) override
Emit the data mapping/movement code associated with the directive D that should be of the form 'targe...
llvm::Function * emitTeamsOutlinedFunction(CodeGenFunction &CGF, const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) override
Emits outlined function for the specified OpenMP teams directive D.
void emitReduction(CodeGenFunction &CGF, SourceLocation Loc, ArrayRef< const Expr * > Privates, ArrayRef< const Expr * > LHSExprs, ArrayRef< const Expr * > RHSExprs, ArrayRef< const Expr * > ReductionOps, ReductionOptionsTy Options) override
Emit a code for reduction clause.
void emitFlush(CodeGenFunction &CGF, ArrayRef< const Expr * > Vars, SourceLocation Loc, llvm::AtomicOrdering AO) override
Emit flush of the variables specified in 'omp flush' directive.
void emitDoacrossOrdered(CodeGenFunction &CGF, const OMPDependClause *C) override
Emit code for doacross ordered directive with 'depend' clause.
void emitTaskyieldCall(CodeGenFunction &CGF, SourceLocation Loc) override
Emits a masked region.
Address getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF, QualType VarType, StringRef Name) override
Creates artificial threadprivate variable with name Name and type VarType.
Address getAddrOfThreadPrivate(CodeGenFunction &CGF, const VarDecl *VD, Address VDAddr, SourceLocation Loc) override
Returns address of the threadprivate variable for the current thread.
void emitSingleRegion(CodeGenFunction &CGF, const RegionCodeGenTy &SingleOpGen, SourceLocation Loc, ArrayRef< const Expr * > CopyprivateVars, ArrayRef< const Expr * > DestExprs, ArrayRef< const Expr * > SrcExprs, ArrayRef< const Expr * > AssignmentOps) override
Emits a single region.
void emitTaskReductionFixups(CodeGenFunction &CGF, SourceLocation Loc, ReductionCodeGen &RCG, unsigned N) override
Required to resolve existing problems in the runtime.
llvm::Function * emitParallelOutlinedFunction(CodeGenFunction &CGF, const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) override
Emits outlined function for the specified OpenMP parallel directive D.
void emitCancellationPointCall(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind CancelRegion) override
Emit code for 'cancellation point' construct.
void emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind Kind, bool EmitChecks=true, bool ForceSimpleCall=false) override
Emit an implicit/explicit barrier for OpenMP threads.
Address getParameterAddress(CodeGenFunction &CGF, const VarDecl *NativeParam, const VarDecl *TargetParam) const override
Gets the address of the native argument basing on the address of the target-specific parameter.
void emitTeamsCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, SourceLocation Loc, llvm::Function *OutlinedFn, ArrayRef< llvm::Value * > CapturedVars) override
Emits code for teams call of the OutlinedFn with variables captured in a record which address is stor...
void emitForOrderedIterationEnd(CodeGenFunction &CGF, SourceLocation Loc, unsigned IVSize, bool IVSigned) override
Call the appropriate runtime routine to notify that we finished iteration of the ordered loop with th...
bool emitTargetGlobal(GlobalDecl GD) override
Emit the global GD if it is meaningful for the target.
void emitTaskReductionFini(CodeGenFunction &CGF, SourceLocation Loc, bool IsWorksharingReduction) override
Emits the following code for reduction clause with task modifier:
void emitOrderedRegion(CodeGenFunction &CGF, const RegionCodeGenTy &OrderedOpGen, SourceLocation Loc, bool IsThreads) override
Emit an ordered region.
void emitForStaticFinish(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind) override
Call the appropriate runtime routine to notify that we finished all the work with current loop.
llvm::Value * emitTaskReductionInit(CodeGenFunction &CGF, SourceLocation Loc, ArrayRef< const Expr * > LHSExprs, ArrayRef< const Expr * > RHSExprs, const OMPTaskDataTy &Data) override
Emit a code for initialization of task reduction clause.
void emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc, llvm::Function *OutlinedFn, ArrayRef< llvm::Value * > CapturedVars, const Expr *IfCond, llvm::Value *NumThreads) override
Emits code for parallel or serial call of the OutlinedFn with variables captured in a record which ad...
void emitProcBindClause(CodeGenFunction &CGF, llvm::omp::ProcBindKind ProcBind, SourceLocation Loc) override
Emit call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32 global_tid, int proc_bind) to generat...
void emitTargetOutlinedFunction(const OMPExecutableDirective &D, StringRef ParentName, llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID, bool IsOffloadEntry, const RegionCodeGenTy &CodeGen) override
Emit outilined function for 'target' directive.
void emitMasterRegion(CodeGenFunction &CGF, const RegionCodeGenTy &MasterOpGen, SourceLocation Loc) override
Emits a master region.
void emitNumTeamsClause(CodeGenFunction &CGF, const Expr *NumTeams, const Expr *ThreadLimit, SourceLocation Loc) override
Emits call to void __kmpc_push_num_teams(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_teams,...
const VarDecl * translateParameter(const FieldDecl *FD, const VarDecl *NativeParam) const override
Translates the native parameter of outlined function if this is required for target.
void emitMaskedRegion(CodeGenFunction &CGF, const RegionCodeGenTy &MaskedOpGen, SourceLocation Loc, const Expr *Filter=nullptr) override
Emits a masked region.
void emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPExecutableDirective &D, llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds, const Expr *IfCond, const OMPTaskDataTy &Data) override
Emit task region for the task directive.
void emitTargetCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond, llvm::PointerIntPair< const Expr *, 2, OpenMPDeviceClauseModifier > Device, llvm::function_ref< llvm::Value *(CodeGenFunction &CGF, const OMPLoopDirective &D)> SizeEmitter) override
Emit the target offloading code associated with D.
bool emitTargetFunctions(GlobalDecl GD) override
Emit the target regions enclosed in GD function definition or the function itself in case it is a val...
void emitNumThreadsClause(CodeGenFunction &CGF, llvm::Value *NumThreads, SourceLocation Loc) override
Emits call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_threads)...
void emitDoacrossInit(CodeGenFunction &CGF, const OMPLoopDirective &D, ArrayRef< Expr * > NumIterations) override
Emit initialization for doacross loop nesting support.
void emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc, const Expr *IfCond, OpenMPDirectiveKind CancelRegion) override
Emit code for 'cancel' construct.
void emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPTaskDataTy &Data) override
Emit code for 'taskwait' directive.
void emitTaskgroupRegion(CodeGenFunction &CGF, const RegionCodeGenTy &TaskgroupOpGen, SourceLocation Loc) override
Emit a taskgroup region.
void emitTargetDataCalls(CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, const Expr *Device, const RegionCodeGenTy &CodeGen, CGOpenMPRuntime::TargetDataInfo &Info) override
Emit the target data mapping code associated with D.
void emitForDispatchInit(CodeGenFunction &CGF, SourceLocation Loc, const OpenMPScheduleTy &ScheduleKind, unsigned IVSize, bool IVSigned, bool Ordered, const DispatchRTInput &DispatchValues) override
This is used for non static scheduled types and when the ordered clause is present on the loop constr...
llvm::Function * emitTaskOutlinedFunction(const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, const VarDecl *PartIDVar, const VarDecl *TaskTVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, bool Tied, unsigned &NumberOfParts) override
Emits outlined function for the OpenMP task directive D.
void emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPLoopDirective &D, llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds, const Expr *IfCond, const OMPTaskDataTy &Data) override
Emit task region for the taskloop directive.
CGRecordLayout - This class handles struct and union layout info while lowering AST types to LLVM typ...
unsigned getNonVirtualBaseLLVMFieldNo(const CXXRecordDecl *RD) const
llvm::StructType * getLLVMType() const
Return the "complete object" LLVM type associated with this record.
llvm::StructType * getBaseSubobjectLLVMType() const
Return the "base subobject" LLVM type associated with this record.
unsigned getLLVMFieldNo(const FieldDecl *FD) const
Return llvm::StructType element number that corresponds to the field FD.
unsigned getVirtualBaseIndex(const CXXRecordDecl *base) const
Return the LLVM field index corresponding to the given virtual base.
virtual void EmitBody(CodeGenFunction &CGF, const Stmt *S)
Emit the captured statement body.
RAII for correct setting/restoring of CapturedStmtInfo.
The scope used to remap some variables as private in the OpenMP loop body (or other captured region e...
bool Privatize()
Privatizes local variables previously registered as private.
bool addPrivate(const VarDecl *LocalVD, Address Addr)
Registers LocalVD variable as a private with Addr as the address of the corresponding private variabl...
An RAII object to set (and then clear) a mapping for an OpaqueValueExpr.
Enters a new scope for capturing cleanups, all of which will be executed once the scope is exited.
CodeGenFunction - This class organizes the per-function state that is used while generating LLVM code...
void FinishFunction(SourceLocation EndLoc=SourceLocation())
FinishFunction - Complete IR generation of the current function.
void EmitNullInitialization(Address DestPtr, QualType Ty)
EmitNullInitialization - Generate code to set a value of the given type to null, If the type contains...
void EmitOMPAggregateAssign(Address DestAddr, Address SrcAddr, QualType OriginalType, const llvm::function_ref< void(Address, Address)> CopyGen)
Perform element by element copying of arrays with type OriginalType from SrcAddr to DestAddr using co...
static TypeEvaluationKind getEvaluationKind(QualType T)
getEvaluationKind - Return the TypeEvaluationKind of QualType T.
void EmitBranchOnBoolExpr(const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock, uint64_t TrueCount, Stmt::Likelihood LH=Stmt::LH_None, const Expr *ConditionalOp=nullptr)
EmitBranchOnBoolExpr - Emit a branch on a boolean condition (e.g.
JumpDest getJumpDestInCurrentScope(llvm::BasicBlock *Target)
The given basic block lies in the current EH scope, but may be a target of a potentially scope-crossi...
void EmitOMPCopy(QualType OriginalType, Address DestAddr, Address SrcAddr, const VarDecl *DestVD, const VarDecl *SrcVD, const Expr *Copy)
Emit proper copying of data from one variable to another.
void EmitStoreThroughLValue(RValue Src, LValue Dst, bool isInit=false)
EmitStoreThroughLValue - Store the specified rvalue into the specified lvalue, where both are guarant...
static void EmitOMPTargetTeamsDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDirective &S)
Emit device code for the target teams directive.
FieldDecl * LambdaThisCaptureField
CGCapturedStmtInfo * CapturedStmtInfo
static void EmitOMPTargetTeamsDistributeDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeDirective &S)
Emit device code for the target teams distribute directive.
Address EmitLoadOfPointer(Address Ptr, const PointerType *PtrTy, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr)
Load a pointer with type PtrTy stored at address Ptr.
RawAddress CreateDefaultAlignTempAlloca(llvm::Type *Ty, const Twine &Name="tmp")
CreateDefaultAlignedTempAlloca - This creates an alloca with the default ABI alignment of the given L...
static void EmitOMPTargetParallelForSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelForSimdDirective &S)
Emit device code for the target parallel for simd directive.
llvm::Value * emitArrayLength(const ArrayType *arrayType, QualType &baseType, Address &addr)
emitArrayLength - Compute the length of an array, even if it's a VLA, and drill down to the base elem...
VlaSizePair getVLASize(const VariableArrayType *vla)
Returns an LLVM value that corresponds to the size, in non-variably-sized elements,...
JumpDest getOMPCancelDestination(OpenMPDirectiveKind Kind)
LValue EmitLValue(const Expr *E, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
EmitLValue - Emit code to compute a designator that specifies the location of the expression.
static void EmitOMPTargetDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetDirective &S)
Emit device code for the target directive.
void EmitVariablyModifiedType(QualType Ty)
EmitVLASize - Capture all the sizes for the VLA expressions in the given variably-modified type and s...
llvm::BasicBlock * createBasicBlock(const Twine &name="", llvm::Function *parent=nullptr, llvm::BasicBlock *before=nullptr)
createBasicBlock - Create an LLVM basic block.
const LangOptions & getLangOpts() const
static void EmitOMPTargetSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetSimdDirective &S)
Emit device code for the target simd directive.
LValue EmitLValueForFieldInitialization(LValue Base, const FieldDecl *Field)
EmitLValueForFieldInitialization - Like EmitLValueForField, except that if the Field is a reference,...
void EmitBlock(llvm::BasicBlock *BB, bool IsFinished=false)
EmitBlock - Emit the given block.
const CodeGen::CGBlockInfo * BlockInfo
Address EmitLoadOfReference(LValue RefLVal, LValueBaseInfo *PointeeBaseInfo=nullptr, TBAAAccessInfo *PointeeTBAAInfo=nullptr)
void EmitExprAsInit(const Expr *init, const ValueDecl *D, LValue lvalue, bool capturedByInit)
EmitExprAsInit - Emits the code necessary to initialize a location in memory with the given initializ...
RValue EmitLoadOfLValue(LValue V, SourceLocation Loc)
EmitLoadOfLValue - Given an expression that represents a value lvalue, this method emits the address ...
void EmitAnyExprToMem(const Expr *E, Address Location, Qualifiers Quals, bool IsInitializer)
EmitAnyExprToMem - Emits the code necessary to evaluate an arbitrary expression into the given memory...
void EmitIgnoredExpr(const Expr *E)
EmitIgnoredExpr - Emit an expression in a context which ignores the result.
llvm::Type * ConvertTypeForMem(QualType T)
const Decl * CurCodeDecl
CurCodeDecl - This is the inner-most code context, which includes blocks.
llvm::AssertingVH< llvm::Instruction > AllocaInsertPt
AllocaInsertPoint - This is an instruction in the entry block before which we prefer to insert alloca...
void EmitAggregateAssign(LValue Dest, LValue Src, QualType EltTy)
Emit an aggregate assignment.
void GenerateOpenMPCapturedVars(const CapturedStmt &S, SmallVectorImpl< llvm::Value * > &CapturedVars)
JumpDest ReturnBlock
ReturnBlock - Unified return block.
static void EmitOMPTargetTeamsGenericLoopDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsGenericLoopDirective &S)
Emit device code for the target teams loop directive.
LValue EmitLValueForField(LValue Base, const FieldDecl *Field)
RawAddress CreateMemTemp(QualType T, const Twine &Name="tmp", RawAddress *Alloca=nullptr)
CreateMemTemp - Create a temporary memory object of the given type, with appropriate alignmen and cas...
Destroyer * getDestroyer(QualType::DestructionKind destructionKind)
static void EmitOMPTargetTeamsDistributeParallelForDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeParallelForDirective &S)
llvm::Value * getTypeSize(QualType Ty)
Returns calculated size of the specified type.
void emitDestroy(Address addr, QualType type, Destroyer *destroyer, bool useEHCleanupForArray)
Address EmitPointerWithAlignment(const Expr *Addr, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
EmitPointerWithAlignment - Given an expression with a pointer type, emit the value and compute our be...
void StartFunction(GlobalDecl GD, QualType RetTy, llvm::Function *Fn, const CGFunctionInfo &FnInfo, const FunctionArgList &Args, SourceLocation Loc=SourceLocation(), SourceLocation StartLoc=SourceLocation())
Emit code for the start of a function.
ComplexPairTy EmitLoadOfComplex(LValue src, SourceLocation loc)
EmitLoadOfComplex - Load a complex number from the specified l-value.
bool HaveInsertPoint() const
HaveInsertPoint - True if an insertion point is defined.
bool isTrivialInitializer(const Expr *Init)
Determine whether the given initializer is trivial in the sense that it requires no code to be genera...
void EmitBranch(llvm::BasicBlock *Block)
EmitBranch - Emit a branch to the specified basic block from the current insert block,...
LValue MakeRawAddrLValue(llvm::Value *V, QualType T, CharUnits Alignment, AlignmentSource Source=AlignmentSource::Type)
Same as MakeAddrLValue above except that the pointer is known to be unsigned.
void EmitAggregateCopy(LValue Dest, LValue Src, QualType EltTy, AggValueSlot::Overlap_t MayOverlap, bool isVolatile=false)
EmitAggregateCopy - Emit an aggregate copy.
LValue MakeNaturalAlignRawAddrLValue(llvm::Value *V, QualType T)
void EmitOMPTargetTaskBasedDirective(const OMPExecutableDirective &S, const RegionCodeGenTy &BodyGen, OMPTargetDataInfo &InputInfo)
static void EmitOMPTargetParallelForDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelForDirective &S)
Emit device code for the target parallel for directive.
void EmitVarDecl(const VarDecl &D)
EmitVarDecl - Emit a local variable declaration.
llvm::CallInst * EmitNounwindRuntimeCall(llvm::FunctionCallee callee, const Twine &name="")
ASTContext & getContext() const
llvm::Value * EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty, SourceLocation Loc, AlignmentSource Source=AlignmentSource::Type, bool isNontemporal=false)
EmitLoadOfScalar - Load a scalar value from an address, taking care to appropriately convert from the...
const Decl * CurFuncDecl
CurFuncDecl - Holds the Decl for the current outermost non-closure context.
LValue EmitLoadOfPointerLValue(Address Ptr, const PointerType *PtrTy)
static void EmitOMPTargetTeamsDistributeSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeSimdDirective &S)
Emit device code for the target teams distribute simd directive.
void EmitBranchThroughCleanup(JumpDest Dest)
EmitBranchThroughCleanup - Emit a branch from the current insert block through the normal cleanup han...
AutoVarEmission EmitAutoVarAlloca(const VarDecl &var)
void pushDestroy(QualType::DestructionKind dtorKind, Address addr, QualType type)
bool ConstantFoldsToSimpleInteger(const Expr *Cond, bool &Result, bool AllowLabels=false)
ConstantFoldsToSimpleInteger - If the specified expression does not fold to a constant,...
void EmitAutoVarCleanups(const AutoVarEmission &emission)
bool needsEHCleanup(QualType::DestructionKind kind)
Determines whether an EH cleanup is required to destroy a type with the given destruction kind.
llvm::DenseMap< const ValueDecl *, FieldDecl * > LambdaCaptureFields
llvm::CallInst * EmitRuntimeCall(llvm::FunctionCallee callee, const Twine &name="")
llvm::Type * ConvertType(QualType T)
CodeGenTypes & getTypes() const
LValue EmitArraySectionExpr(const ArraySectionExpr *E, bool IsLowerBound=true)
static void EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeParallelForSimdDirective &S)
Emit device code for the target teams distribute parallel for simd directive.
LValue EmitStringLiteralLValue(const StringLiteral *E)
llvm::Value * EvaluateExprAsBool(const Expr *E)
EvaluateExprAsBool - Perform the usual unary conversions on the specified expression and compare the ...
Address LoadCXXThisAddress()
LValue EmitOMPSharedLValue(const Expr *E)
Emits the lvalue for the expression with possibly captured variable.
llvm::Value * EmitCheckedInBoundsGEP(llvm::Type *ElemTy, llvm::Value *Ptr, ArrayRef< llvm::Value * > IdxList, bool SignedIndices, bool IsSubtraction, SourceLocation Loc, const Twine &Name="")
Same as IRBuilder::CreateInBoundsGEP, but additionally emits a check to detect undefined behavior whe...
llvm::Function * GenerateOpenMPCapturedStmtFunction(const CapturedStmt &S, SourceLocation Loc)
LValue MakeAddrLValue(Address Addr, QualType T, AlignmentSource Source=AlignmentSource::Type)
void EmitStoreOfComplex(ComplexPairTy V, LValue dest, bool isInit)
EmitStoreOfComplex - Store a complex number into the specified l-value.
static void EmitOMPTargetParallelGenericLoopDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelGenericLoopDirective &S)
Emit device code for the target parallel loop directive.
LValue EmitLoadOfReferenceLValue(LValue RefLVal)
Address GetAddrOfLocalVar(const VarDecl *VD)
GetAddrOfLocalVar - Return the address of a local variable.
void EmitAtomicStore(RValue rvalue, LValue lvalue, bool isInit)
llvm::Value * EmitScalarConversion(llvm::Value *Src, QualType SrcTy, QualType DstTy, SourceLocation Loc)
Emit a conversion from the specified type to the specified destination type, both of which are LLVM s...
std::pair< llvm::Value *, llvm::Value * > ComplexPairTy
Address ReturnValue
ReturnValue - The temporary alloca to hold the return value.
static void EmitOMPTargetParallelDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelDirective &S)
llvm::Value * EmitScalarExpr(const Expr *E, bool IgnoreResultAssign=false)
EmitScalarExpr - Emit the computation of the specified expression of LLVM scalar type,...
void incrementProfileCounter(const Stmt *S, llvm::Value *StepV=nullptr)
Increment the profiler's counter for the given statement by StepV.
void EmitStoreOfScalar(llvm::Value *Value, Address Addr, bool Volatile, QualType Ty, AlignmentSource Source=AlignmentSource::Type, bool isInit=false, bool isNontemporal=false)
EmitStoreOfScalar - Store a scalar value to an address, taking care to appropriately convert from the...
This class organizes the cross-function state that is used while generating LLVM code.
void handleCUDALaunchBoundsAttr(llvm::Function *F, const CUDALaunchBoundsAttr *A, int32_t *MaxThreadsVal=nullptr, int32_t *MinBlocksVal=nullptr, int32_t *MaxClusterRankVal=nullptr)
Emit the IR encoding to attach the CUDA launch bounds attribute to F.
void SetInternalFunctionAttributes(GlobalDecl GD, llvm::Function *F, const CGFunctionInfo &FI)
Set the attributes on the LLVM function for the given decl and function info.
llvm::Module & getModule() const
void addCompilerUsedGlobal(llvm::GlobalValue *GV)
Add a global to a list to be added to the llvm.compiler.used metadata.
CharUnits GetTargetTypeStoreSize(llvm::Type *Ty) const
Return the store size, in character units, of the given LLVM type.
void handleAMDGPUWavesPerEUAttr(llvm::Function *F, const AMDGPUWavesPerEUAttr *A)
Emit the IR encoding to attach the AMD GPU waves-per-eu attribute to F.
DiagnosticsEngine & getDiags() const
const LangOptions & getLangOpts() const
CharUnits getNaturalTypeAlignment(QualType T, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr, bool forPointeeType=false)
CodeGenTypes & getTypes()
const TargetInfo & getTarget() const
void EmitGlobal(GlobalDecl D)
Emit code for a single global function or var decl.
void handleAMDGPUFlatWorkGroupSizeAttr(llvm::Function *F, const AMDGPUFlatWorkGroupSizeAttr *A, const ReqdWorkGroupSizeAttr *ReqdWGS=nullptr, int32_t *MinThreadsVal=nullptr, int32_t *MaxThreadsVal=nullptr)
Emit the IR encoding to attach the AMD GPU flat-work-group-size attribute to F.
llvm::GlobalValue::LinkageTypes getLLVMLinkageVarDefinition(const VarDecl *VD)
Returns LLVM linkage for a declarator.
CGCXXABI & getCXXABI() const
CGOpenMPRuntime & getOpenMPRuntime()
Return a reference to the configured OpenMP runtime.
const llvm::Triple & getTriple() const
TBAAAccessInfo getTBAAInfoForSubobject(LValue Base, QualType AccessType)
getTBAAInfoForSubobject - Get TBAA information for an access with a given base lvalue.
llvm::Constant * GetAddrOfGlobal(GlobalDecl GD, ForDefinition_t IsForDefinition=NotForDefinition)
ASTContext & getContext() const
const TargetCodeGenInfo & getTargetCodeGenInfo()
const CodeGenOptions & getCodeGenOpts() const
StringRef getMangledName(GlobalDecl GD)
std::optional< CharUnits > getOMPAllocateAlignment(const VarDecl *VD)
Return the alignment specified in an allocate directive, if present.
llvm::GlobalValue * GetGlobalValue(StringRef Ref)
llvm::Constant * EmitNullConstant(QualType T)
Return the result of value-initializing the given type, i.e.
llvm::Function * CreateGlobalInitOrCleanUpFunction(llvm::FunctionType *ty, const Twine &name, const CGFunctionInfo &FI, SourceLocation Loc=SourceLocation(), bool TLS=false, llvm::GlobalVariable::LinkageTypes Linkage=llvm::GlobalVariable::InternalLinkage)
llvm::ConstantInt * getSize(CharUnits numChars)
Emit the given number of characters as a value of type size_t.
llvm::Type * ConvertType(QualType T)
ConvertType - Convert type T into a llvm::Type.
llvm::FunctionType * GetFunctionType(const CGFunctionInfo &Info)
GetFunctionType - Get the LLVM function type for.
const CGFunctionInfo & arrangeBuiltinFunctionDeclaration(QualType resultType, const FunctionArgList &args)
A builtin function is a freestanding function using the default C conventions.
const CGRecordLayout & getCGRecordLayout(const RecordDecl *)
getCGRecordLayout - Return record layout info for the given record decl.
llvm::Type * ConvertTypeForMem(QualType T, bool ForBitField=false)
ConvertTypeForMem - Convert type T into a llvm::Type.
const CGFunctionInfo & arrangeNullaryFunction()
A nullary function is a freestanding function of type 'void ()'.
A specialization of Address that requires the address to be an LLVM Constant.
static ConstantAddress invalid()
Information for lazily generating a cleanup.
bool requiresLandingPad() const
void popTerminate()
Pops a terminate handler off the stack.
void pushTerminate()
Push a terminate handler on the stack.
FunctionArgList - Type for representing both the decl and type of parameters to a function.
LValue - This represents an lvalue references.
CharUnits getAlignment() const
const Qualifiers & getQuals() const
Address getAddress() const
LValueBaseInfo getBaseInfo() const
llvm::Value * getPointer(CodeGenFunction &CGF) const
TBAAAccessInfo getTBAAInfo() const
A basic class for pre|post-action for advanced codegen sequence for OpenMP region.
virtual void Enter(CodeGenFunction &CGF)
RValue - This trivial value class is used to represent the result of an expression that is evaluated.
static RValue get(llvm::Value *V)
static RValue getComplex(llvm::Value *V1, llvm::Value *V2)
llvm::Value * getScalarVal() const
getScalarVal() - Return the Value* of this scalar value.
An abstract representation of an aligned address.
llvm::Type * getElementType() const
Return the type of the values stored in this address.
llvm::Value * getPointer() const
static RawAddress invalid()
Class intended to support codegen of all kind of the reduction clauses.
LValue getSharedLValue(unsigned N) const
Returns LValue for the reduction item.
const Expr * getRefExpr(unsigned N) const
Returns the base declaration of the reduction item.
LValue getOrigLValue(unsigned N) const
Returns LValue for the original reduction item.
bool needCleanups(unsigned N)
Returns true if the private copy requires cleanups.
void emitAggregateType(CodeGenFunction &CGF, unsigned N)
Emits the code for the variable-modified type, if required.
const VarDecl * getBaseDecl(unsigned N) const
Returns the base declaration of the reduction item.
QualType getPrivateType(unsigned N) const
Return the type of the private item.
bool usesReductionInitializer(unsigned N) const
Returns true if the initialization of the reduction item uses initializer from declare reduction cons...
void emitSharedOrigLValue(CodeGenFunction &CGF, unsigned N)
Emits lvalue for the shared and original reduction item.
void emitInitialization(CodeGenFunction &CGF, unsigned N, Address PrivateAddr, Address SharedAddr, llvm::function_ref< bool(CodeGenFunction &)> DefaultInit)
Performs initialization of the private copy for the reduction item.
std::pair< llvm::Value *, llvm::Value * > getSizes(unsigned N) const
Returns the size of the reduction item (in chars and total number of elements in the item),...
ReductionCodeGen(ArrayRef< const Expr * > Shareds, ArrayRef< const Expr * > Origs, ArrayRef< const Expr * > Privates, ArrayRef< const Expr * > ReductionOps)
void emitCleanups(CodeGenFunction &CGF, unsigned N, Address PrivateAddr)
Emits cleanup code for the reduction item.
Address adjustPrivateAddress(CodeGenFunction &CGF, unsigned N, Address PrivateAddr)
Adjusts PrivatedAddr for using instead of the original variable address in normal operations.
Class provides a way to call simple version of codegen for OpenMP region, or an advanced with possibl...
void operator()(CodeGenFunction &CGF) const
void setAction(PrePostActionTy &Action) const
virtual void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &M) const
setTargetAttributes - Provides a convenient hook to handle extra target-specific attributes for the g...
ConstStmtVisitor - This class implements a simple visitor for Stmt subclasses.
DeclContext - This is used only as base class of specific decl types that can act as declaration cont...
void addDecl(Decl *D)
Add the declaration D into this context.
A reference to a declared variable, function, enum, etc.
Decl - This represents one declaration (or definition), e.g.
ASTContext & getASTContext() const LLVM_READONLY
llvm::iterator_range< specific_attr_iterator< T > > specific_attrs() const
SourceLocation getLocation() const
DeclContext * getDeclContext()
virtual Decl * getCanonicalDecl()
Retrieves the "canonical" declaration of the given declaration.
SourceLocation getBeginLoc() const LLVM_READONLY
DiagnosticBuilder Report(SourceLocation Loc, unsigned DiagID)
Issue the message to the client.
unsigned getCustomDiagID(Level L, const char(&FormatString)[N])
Return an ID for a diagnostic with the specified format string and level.
The return type of classify().
This represents one expression.
@ SE_AllowSideEffects
Allow any unmodeled side effect.
@ SE_AllowUndefinedBehavior
Allow UB that we can give a value, but not arbitrary unmodeled side effects.
Expr * IgnoreParenCasts() LLVM_READONLY
Skip past any parentheses and casts which might surround this expression until reaching a fixed point...
llvm::APSInt EvaluateKnownConstInt(const ASTContext &Ctx, SmallVectorImpl< PartialDiagnosticAt > *Diag=nullptr) const
EvaluateKnownConstInt - Call EvaluateAsRValue and return the folded integer.
Expr * IgnoreParenImpCasts() LLVM_READONLY
Skip past any parentheses and implicit casts which might surround this expression until reaching a fi...
bool isEvaluatable(const ASTContext &Ctx, SideEffectsKind AllowSideEffects=SE_NoSideEffects) const
isEvaluatable - Call EvaluateAsRValue to see if this expression can be constant folded without side-e...
bool HasSideEffects(const ASTContext &Ctx, bool IncludePossibleEffects=true) const
HasSideEffects - This routine returns true for all those expressions which have any effect other than...
bool isIntegerConstantExpr(const ASTContext &Ctx, SourceLocation *Loc=nullptr) const
bool EvaluateAsBooleanCondition(bool &Result, const ASTContext &Ctx, bool InConstantContext=false) const
EvaluateAsBooleanCondition - Return true if this is a constant which we can fold and convert to a boo...
SourceLocation getExprLoc() const LLVM_READONLY
getExprLoc - Return the preferred location for the arrow when diagnosing a problem with a generic exp...
std::optional< llvm::APSInt > getIntegerConstantExpr(const ASTContext &Ctx, SourceLocation *Loc=nullptr) const
isIntegerConstantExpr - Return the value if this expression is a valid integer constant expression.
bool hasNonTrivialCall(const ASTContext &Ctx) const
Determine whether this expression involves a call to any function that is not trivial.
Represents a member of a struct/union/class.
static FieldDecl * Create(const ASTContext &C, DeclContext *DC, SourceLocation StartLoc, SourceLocation IdLoc, const IdentifierInfo *Id, QualType T, TypeSourceInfo *TInfo, Expr *BW, bool Mutable, InClassInitStyle InitStyle)
Represents a function declaration or definition.
const ParmVarDecl * getParamDecl(unsigned i) const
QualType getReturnType() const
ArrayRef< ParmVarDecl * > parameters() const
FunctionDecl * getCanonicalDecl() override
Retrieves the "canonical" declaration of the given declaration.
unsigned getNumParams() const
Return the number of parameters this function must have based on its FunctionType.
GlobalDecl - represents a global declaration.
const Decl * getDecl() const
static ImplicitParamDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation IdLoc, IdentifierInfo *Id, QualType T, ImplicitParamKind ParamKind)
Create implicit parameter.
static IntegerLiteral * Create(const ASTContext &C, const llvm::APInt &V, QualType type, SourceLocation l)
Returns a new integer literal with value 'V' and type 'type'.
Describes the capture of a variable or of this, or of a C++1y init-capture.
std::string OMPHostIRFile
Name of the IR file that contains the result of the OpenMP target host code generation.
std::vector< llvm::Triple > OMPTargetTriples
Triples of the OpenMP targets that the host code codegen should take into account in order to generat...
virtual void mangleCanonicalTypeName(QualType T, raw_ostream &, bool NormalizeIntegers=false)=0
Generates a unique string for an externally visible type for use with TBAA or type uniquing.
MemberExpr - [C99 6.5.2.3] Structure and Union Members.
ValueDecl * getMemberDecl() const
Retrieve the member declaration to which this expression refers.
StringRef getName() const
Get the name of identifier for this declaration as a StringRef.
bool isExternallyVisible() const
This represents clause 'affinity' in the '#pragma omp task'-based directives.
Class that represents a component of a mappable expression.
ArrayRef< MappableComponent > MappableExprComponentListRef
const Stmt * getPreInitStmt() const
Get pre-initialization statement for the clause.
This is a basic class for representing single OpenMP clause.
This represents '#pragma omp declare mapper ...' directive.
Expr * getMapperVarRef()
Get the variable declared in the mapper.
This represents '#pragma omp declare reduction ...' directive.
Expr * getInitializer()
Get initializer expression (if specified) of the declare reduction construct.
Expr * getInitPriv()
Get Priv variable of the initializer.
Expr * getCombinerOut()
Get Out variable of the combiner.
Expr * getCombinerIn()
Get In variable of the combiner.
Expr * getCombiner()
Get combiner expression of the declare reduction construct.
Expr * getInitOrig()
Get Orig variable of the initializer.
OMPDeclareReductionInitKind getInitializerKind() const
Get initializer kind.
This represents implicit clause 'depend' for the '#pragma omp task' directive.
This represents 'detach' clause in the '#pragma omp task' directive.
This represents 'device' clause in the '#pragma omp ...' directive.
This represents the 'doacross' clause for the '#pragma omp ordered' directive.
This is a basic class for representing single OpenMP executable directive.
CapturedStmt * getInnermostCapturedStmt()
Get innermost captured statement for the construct.
const CapturedStmt * getCapturedStmt(OpenMPDirectiveKind RegionKind) const
Returns the captured statement associated with the component region within the (combined) directive.
OpenMPDirectiveKind getDirectiveKind() const
SourceLocation getBeginLoc() const
Returns starting location of directive kind.
bool hasClausesOfKind() const
Returns true if the current directive has one or more clauses of a specific kind.
SourceLocation getEndLoc() const
Returns ending location of directive.
static const SpecificClause * getSingleClause(ArrayRef< OMPClause * > Clauses)
Gets a single clause of the specified kind associated with the current directive iff there is only on...
static llvm::iterator_range< specific_clause_iterator< SpecificClause > > getClausesOfKind(ArrayRef< OMPClause * > Clauses)
This represents clause 'firstprivate' in the '#pragma omp ...' directives.
This represents clause 'has_device_ptr' in the '#pragma omp ...' directives.
This represents 'if' clause in the '#pragma omp ...' directive.
Expr * getCondition() const
Returns condition.
This represents clause 'in_reduction' in the '#pragma omp task' directives.
This represents clause 'is_device_ptr' in the '#pragma omp ...' directives.
OpenMP 5.0 [2.1.6 Iterators] Iterators are identifiers that expand to multiple values in the clause o...
OMPIteratorHelperData & getHelper(unsigned I)
Fetches helper data for the specified iteration space.
unsigned numOfIterators() const
Returns number of iterator definitions.
Decl * getIteratorDecl(unsigned I)
Gets the iterator declaration for the given iterator.
This represents clause 'lastprivate' in the '#pragma omp ...' directives.
This represents clause 'linear' in the '#pragma omp ...' directives.
This is a common base class for loop directives ('omp simd', 'omp for', 'omp for simd' etc....
Expr * getStrideVariable() const
Expr * getUpperBoundVariable() const
Expr * getLowerBoundVariable() const
This represents clause 'map' in the '#pragma omp ...' directives.
This represents clause 'nontemporal' in the '#pragma omp ...' directives.
This represents 'nowait' clause in the '#pragma omp ...' directive.
This represents 'num_teams' clause in the '#pragma omp ...' directive.
This represents 'num_threads' clause in the '#pragma omp ...' directive.
This represents 'ordered' clause in the '#pragma omp ...' directive.
This represents clause 'private' in the '#pragma omp ...' directives.
This represents clause 'reduction' in the '#pragma omp ...' directives.
This represents '#pragma omp requires...' directive.
clauselist_range clauselists()
This represents 'thread_limit' clause in the '#pragma omp ...' directive.
This represents clause 'uses_allocators' in the '#pragma omp target'-based directives.
This represents 'ompx_attribute' clause in a directive that might generate an outlined function.
This represents 'ompx_dyn_cgroup_mem' clause in the '#pragma omp target ...' directive.
OpaqueValueExpr - An expression referring to an opaque object of a fixed type and value class.
Represents a parameter to a function.
PointerType - C99 6.7.5.1 - Pointer Declarators.
Represents an unpacked "presumed" location which can be presented to the user.
unsigned getColumn() const
Return the presumed column number of this location.
const char * getFilename() const
Return the presumed filename of this location.
unsigned getLine() const
Return the presumed line number of this location.
A (possibly-)qualified type.
void addRestrict()
Add the restrict qualifier to this QualType.
QualType withRestrict() const
bool isNull() const
Return true if this QualType doesn't point to a type yet.
const Type * getTypePtr() const
Retrieves a pointer to the underlying (unqualified) type.
Qualifiers getQualifiers() const
Retrieve the set of qualifiers applied to this type.
QualType getNonReferenceType() const
If Type is a reference type (e.g., const int&), returns the type that the reference refers to ("const...
QualType getCanonicalType() const
DestructionKind isDestructedType() const
Returns a nonzero value if objects of this type require non-trivial work to clean up after.
Represents a struct/union/class.
field_iterator field_end() const
field_range fields() const
virtual void completeDefinition()
Note that the definition of this type is now complete.
field_iterator field_begin() const
RecordDecl * getDecl() const
decl_type * getPreviousDecl()
Return the previous declaration of this declaration or NULL if this is the first declaration.
decl_type * getMostRecentDecl()
Returns the most recent (re)declaration of this declaration.
Base for LValueReferenceType and RValueReferenceType.
Scope - A scope is a transient data structure that is used while parsing the program.
Encodes a location in the source.
static SourceLocation getFromRawEncoding(UIntTy Encoding)
Turn a raw encoding of a SourceLocation object into a real SourceLocation.
bool isValid() const
Return true if this is a valid SourceLocation object.
UIntTy getRawEncoding() const
When a SourceLocation itself cannot be used, this returns an (opaque) 32-bit integer encoding for it.
This class handles loading and caching of source files into memory.
PresumedLoc getPresumedLoc(SourceLocation Loc, bool UseLineDirectives=true) const
Returns the "presumed" location of a SourceLocation specifies.
fileinfo_iterator fileinfo_end() const
SourceLocation translateFileLineCol(const FileEntry *SourceFile, unsigned Line, unsigned Col) const
Get the source location for the given file:line:col triplet.
fileinfo_iterator fileinfo_begin() const
A trivial tuple used to represent a source range.
Stmt - This represents one statement.
SourceRange getSourceRange() const LLVM_READONLY
SourceLocation tokens are not useful in isolation - they are low level value objects created/interpre...
Stmt * IgnoreContainers(bool IgnoreCaptured=false)
Skip no-op (attributed, compound) container stmts and skip captured stmt at the top,...
SourceLocation getBeginLoc() const LLVM_READONLY
void startDefinition()
Starts the definition of this tag declaration.
bool isTLSSupported() const
Whether the target supports thread-local storage.
virtual bool hasFeature(StringRef Feature) const
Determine whether the given target has the given feature.
The base class of the type hierarchy.
CXXRecordDecl * getAsCXXRecordDecl() const
Retrieves the CXXRecordDecl that this type refers to, either because the type is a RecordType or beca...
bool isSignedIntegerOrEnumerationType() const
Determines whether this is an integer type that is signed or an enumeration types whose underlying ty...
const Type * getPointeeOrArrayElementType() const
If this is a pointer type, return the pointee type.
bool isSignedIntegerType() const
Return true if this is an integer type that is signed, according to C99 6.2.5p4 [char,...
bool isPointerType() const
CanQualType getCanonicalTypeUnqualified() const
bool isIntegerType() const
isIntegerType() does not include complex integers (a GCC extension).
const T * castAs() const
Member-template castAs<specific type>.
bool isReferenceType() const
QualType getPointeeType() const
If this is a pointer, ObjC object pointer, or block pointer, this returns the respective pointee.
bool isLValueReferenceType() const
QualType getCanonicalTypeInternal() const
const RecordType * getAsStructureType() const
const Type * getBaseElementTypeUnsafe() const
Get the base element type of this type, potentially discarding type qualifiers.
bool isVariablyModifiedType() const
Whether this type is a variably-modified type (C99 6.7.5).
const ArrayType * getAsArrayTypeUnsafe() const
A variant of getAs<> for array types which silently discards qualifiers from the outermost type.
bool isFloatingType() const
bool isUnsignedIntegerType() const
Return true if this is an integer type that is unsigned, according to C99 6.2.5p6 [which returns true...
bool isAnyPointerType() const
const T * getAs() const
Member-template getAs<specific type>'.
bool isRecordType() const
TagDecl * getAsTagDecl() const
Retrieves the TagDecl that this type refers to, either because the type is a TagType or because it is...
RecordDecl * getAsRecordDecl() const
Retrieves the RecordDecl this type refers to.
Represent the declaration of a variable (in which case it is an lvalue) a function (in which case it ...
Represents a variable declaration or definition.
VarDecl * getCanonicalDecl() override
Retrieves the "canonical" declaration of the given declaration.
VarDecl * getDefinition(ASTContext &)
Get the real (not just tentative) definition for this declaration.
bool hasExternalStorage() const
Returns true if a variable has extern or private_extern storage.
bool hasLocalStorage() const
Returns true if a variable with function scope is a non-static local variable.
@ DeclarationOnly
This declaration is only a declaration.
DefinitionKind hasDefinition(ASTContext &) const
Check whether this variable is defined in this translation unit.
bool isLocalVarDeclOrParm() const
Similar to isLocalVarDecl but also includes parameters.
const Expr * getAnyInitializer() const
Get the initializer for this variable, no matter which declaration it is attached to.
Represents a C array with a specified size that is not an integer-constant-expression.
Expr * getSizeExpr() const
specific_attr_iterator - Iterates over a subrange of an AttrVec, only providing attributes that are o...
@ Decl
The l-value was an access to a declared entity or something equivalently strong, like the address of ...
@ LLVM_MARK_AS_BITMASK_ENUM
The JSON file list parser is used to communicate input to InstallAPI.
bool isOpenMPWorksharingDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a worksharing directive.
@ Private
'private' clause, allowed on 'parallel', 'serial', 'loop', 'parallel loop', and 'serial loop' constru...
@ Vector
'vector' clause, allowed on 'loop', Combined, and 'routine' directives.
@ Reduction
'reduction' clause, allowed on Parallel, Serial, Loop, and the combined constructs.
@ Present
'present' clause, allowed on Compute and Combined constructs, plus 'data' and 'declare'.
bool needsTaskBasedThreadLimit(OpenMPDirectiveKind DKind)
Checks if the specified target directive, combined or not, needs task based thread_limit.
@ Ctor_Complete
Complete object ctor.
if(T->getSizeExpr()) TRY_TO(TraverseStmt(const_cast< Expr * >(T -> getSizeExpr())))
bool isOpenMPTargetDataManagementDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a target data offload directive.
llvm::omp::Directive OpenMPDirectiveKind
OpenMP directives.
@ ICIS_NoInit
No in-class initializer.
bool isOpenMPDistributeDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a distribute directive.
@ LCK_ByRef
Capturing by reference.
LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE()
OpenMPScheduleClauseModifier
OpenMP modifiers for 'schedule' clause.
@ OMPC_SCHEDULE_MODIFIER_last
@ OMPC_SCHEDULE_MODIFIER_unknown
bool isOpenMPParallelDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a parallel-kind directive.
OpenMPDistScheduleClauseKind
OpenMP attributes for 'dist_schedule' clause.
bool isOpenMPTaskingDirective(OpenMPDirectiveKind Kind)
Checks if the specified directive kind is one of tasking directives - task, taskloop,...
bool isOpenMPTargetExecutionDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a target code offload directive.
@ Result
The result type of a method or function.
bool isOpenMPTeamsDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a teams-kind directive.
OpenMPDependClauseKind
OpenMP attributes for 'depend' clause.
@ Dtor_Complete
Complete object dtor.
@ Union
The "union" keyword.
bool isOpenMPLoopDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a directive with an associated loop construct.
LangAS
Defines the address space values used by the address space qualifier of QualType.
bool isOpenMPSimdDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a simd directive.
@ VK_PRValue
A pr-value expression (in the C++11 taxonomy) produces a temporary value.
@ VK_LValue
An l-value expression is a reference to an object with independent storage.
const FunctionProtoType * T
void getOpenMPCaptureRegions(llvm::SmallVectorImpl< OpenMPDirectiveKind > &CaptureRegions, OpenMPDirectiveKind DKind)
Return the captured regions of an OpenMP directive.
@ OMPC_ATOMIC_DEFAULT_MEM_ORDER_unknown
OpenMPMapModifierKind
OpenMP modifier kind for 'map' clause.
@ OMPC_MAP_MODIFIER_unknown
@ Other
Other implicit parameter.
OpenMPScheduleClauseKind
OpenMP attributes for 'schedule' clause.
bool isOpenMPTaskLoopDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a taskloop directive.
OpenMPMapClauseKind
OpenMP mapping kind for 'map' clause.
Diagnostic wrappers for TextAPI types for error reporting.
Maps the expression for the lastprivate variable to the global copy used to store new value because o...
A jump destination is an abstract label, branching to which may require a jump out through normal cle...
llvm::BasicBlock * getBlock() const
unsigned NumberOfTargetItems
Address BasePointersArray
llvm::PointerType * VoidPtrTy
llvm::IntegerType * Int64Ty
llvm::PointerType * Int8PtrPtrTy
llvm::IntegerType * Int8Ty
i8, i16, i32, and i64
llvm::CallingConv::ID getRuntimeCC() const
llvm::IntegerType * SizeTy
llvm::PointerType * VoidPtrPtrTy
llvm::IntegerType * Int32Ty
llvm::IntegerType * IntPtrTy
llvm::IntegerType * IntTy
int
llvm::PointerType * Int8PtrTy
CharUnits getPointerAlign() const
OpenMPDependClauseKind DepKind
const Expr * IteratorExpr
SmallVector< const Expr *, 4 > DepExprs
EvalResult is a struct with detailed info about an evaluated expression.
Extra information about a function prototype.
Helper expressions and declaration for OMPIteratorExpr class for each iteration space.
Expr * CounterUpdate
Updater for the internal counter: ++CounterVD;.
Expr * Upper
Normalized upper bound.
Expr * Update
Update expression for the originally specified iteration variable, calculated as VD = Begin + Counter...
VarDecl * CounterVD
Internal normalized counter.
Data for list of allocators.
Expr * AllocatorTraits
Allocator traits.
Expr * Allocator
Allocator.
Scheduling data for loop-based OpenMP directives.
OpenMPScheduleClauseModifier M2
OpenMPScheduleClauseModifier M1
OpenMPScheduleClauseKind Schedule
Describes how types, statements, expressions, and declarations should be printed.