14#include "llvm/IR/BasicBlock.h"
15#include "llvm/IR/CFG.h"
16#include "llvm/IR/Constants.h"
17#include "llvm/IR/InstrTypes.h"
18#include "llvm/IR/Instructions.h"
19#include "llvm/IR/Metadata.h"
26 LLVMContext &Ctx = Header->getContext();
28 NewLoopProperties.push_back(
nullptr);
29 NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
31 MDNode *LoopID = MDNode::getDistinct(Ctx, NewLoopProperties);
32 LoopID->replaceOperandWith(0, LoopID);
36MDNode *LoopInfo::createPipeliningMetadata(
const LoopAttributes &Attrs,
38 bool &HasUserTransforms) {
39 LLVMContext &Ctx = Header->getContext();
41 std::optional<bool> Enabled;
47 if (Enabled !=
true) {
49 if (Enabled ==
false) {
50 NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
51 NewLoopProperties.push_back(
52 MDNode::get(Ctx, {MDString::get(Ctx,
"llvm.loop.pipeline.disable"),
53 ConstantAsMetadata::get(ConstantInt::get(
54 llvm::Type::getInt1Ty(Ctx), 1))}));
55 LoopProperties = NewLoopProperties;
57 return createLoopPropertiesMetadata(LoopProperties);
61 Args.push_back(
nullptr);
62 Args.append(LoopProperties.begin(), LoopProperties.end());
66 MDString::get(Ctx,
"llvm.loop.pipeline.initiationinterval"),
67 ConstantAsMetadata::get(ConstantInt::get(
69 Args.push_back(MDNode::get(Ctx, Vals));
74 MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
75 LoopID->replaceOperandWith(0, LoopID);
76 HasUserTransforms =
true;
83 bool &HasUserTransforms) {
84 LLVMContext &Ctx = Header->getContext();
86 std::optional<bool> Enabled;
90 Enabled = std::nullopt;
95 if (Enabled !=
true) {
98 return createPipeliningMetadata(Attrs, LoopProperties, HasUserTransforms);
104 FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
107 FollowupLoopProperties.push_back(
108 MDNode::get(Ctx, MDString::get(Ctx,
"llvm.loop.unroll.disable")));
110 bool FollowupHasTransforms =
false;
111 MDNode *Followup = createPipeliningMetadata(Attrs, FollowupLoopProperties,
112 FollowupHasTransforms);
115 Args.push_back(
nullptr);
116 Args.append(LoopProperties.begin(), LoopProperties.end());
120 Metadata *Vals[] = {MDString::get(Ctx,
"llvm.loop.unroll.count"),
121 ConstantAsMetadata::get(ConstantInt::get(
123 Args.push_back(MDNode::get(Ctx, Vals));
128 Metadata *Vals[] = {MDString::get(Ctx,
"llvm.loop.unroll.enable")};
129 Args.push_back(MDNode::get(Ctx, Vals));
132 if (FollowupHasTransforms)
133 Args.push_back(MDNode::get(
134 Ctx, {MDString::get(Ctx,
"llvm.loop.unroll.followup_all"), Followup}));
136 MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
137 LoopID->replaceOperandWith(0, LoopID);
138 HasUserTransforms =
true;
145 bool &HasUserTransforms) {
146 LLVMContext &Ctx = Header->getContext();
148 std::optional<bool> Enabled;
155 if (Enabled !=
true) {
157 if (Enabled ==
false) {
158 NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
159 NewLoopProperties.push_back(MDNode::get(
160 Ctx, MDString::get(Ctx,
"llvm.loop.unroll_and_jam.disable")));
161 LoopProperties = NewLoopProperties;
163 return createPartialUnrollMetadata(Attrs, LoopProperties,
168 FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
169 FollowupLoopProperties.push_back(
170 MDNode::get(Ctx, MDString::get(Ctx,
"llvm.loop.unroll_and_jam.disable")));
172 bool FollowupHasTransforms =
false;
173 MDNode *Followup = createPartialUnrollMetadata(Attrs, FollowupLoopProperties,
174 FollowupHasTransforms);
177 Args.push_back(
nullptr);
178 Args.append(LoopProperties.begin(), LoopProperties.end());
183 MDString::get(Ctx,
"llvm.loop.unroll_and_jam.count"),
184 ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
186 Args.push_back(MDNode::get(Ctx, Vals));
190 Metadata *Vals[] = {MDString::get(Ctx,
"llvm.loop.unroll_and_jam.enable")};
191 Args.push_back(MDNode::get(Ctx, Vals));
194 if (FollowupHasTransforms)
195 Args.push_back(MDNode::get(
196 Ctx, {MDString::get(Ctx,
"llvm.loop.unroll_and_jam.followup_outer"),
199 if (UnrollAndJamInnerFollowup)
200 Args.push_back(MDNode::get(
201 Ctx, {MDString::get(Ctx,
"llvm.loop.unroll_and_jam.followup_inner"),
202 UnrollAndJamInnerFollowup}));
204 MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
205 LoopID->replaceOperandWith(0, LoopID);
206 HasUserTransforms =
true;
213 bool &HasUserTransforms) {
214 LLVMContext &Ctx = Header->getContext();
216 std::optional<bool> Enabled;
225 if (Enabled !=
true) {
227 if (Enabled ==
false) {
228 NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
229 NewLoopProperties.push_back(
230 MDNode::get(Ctx, {MDString::get(Ctx,
"llvm.loop.vectorize.enable"),
231 ConstantAsMetadata::get(ConstantInt::get(
232 llvm::Type::getInt1Ty(Ctx), 0))}));
233 LoopProperties = NewLoopProperties;
235 return createUnrollAndJamMetadata(Attrs, LoopProperties, HasUserTransforms);
240 FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
243 FollowupLoopProperties.push_back(
244 MDNode::get(Ctx, MDString::get(Ctx,
"llvm.loop.isvectorized")));
246 bool FollowupHasTransforms =
false;
247 MDNode *Followup = createUnrollAndJamMetadata(Attrs, FollowupLoopProperties,
248 FollowupHasTransforms);
251 Args.push_back(
nullptr);
252 Args.append(LoopProperties.begin(), LoopProperties.end());
256 bool IsVectorPredicateEnabled =
false;
258 IsVectorPredicateEnabled =
262 MDString::get(Ctx,
"llvm.loop.vectorize.predicate.enable"),
263 ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt1Ty(Ctx),
264 IsVectorPredicateEnabled))};
265 Args.push_back(MDNode::get(Ctx, Vals));
271 MDString::get(Ctx,
"llvm.loop.vectorize.width"),
272 ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
275 Args.push_back(MDNode::get(Ctx, Vals));
281 MDString::get(Ctx,
"llvm.loop.vectorize.scalable.enable"),
282 ConstantAsMetadata::get(
283 ConstantInt::get(llvm::Type::getInt1Ty(Ctx), IsScalable))};
284 Args.push_back(MDNode::get(Ctx, Vals));
290 MDString::get(Ctx,
"llvm.loop.interleave.count"),
291 ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
293 Args.push_back(MDNode::get(Ctx, Vals));
312 MDNode::get(Ctx, {MDString::get(Ctx,
"llvm.loop.vectorize.enable"),
313 ConstantAsMetadata::get(ConstantInt::get(
314 llvm::Type::getInt1Ty(Ctx), AttrVal))}));
317 if (FollowupHasTransforms)
318 Args.push_back(MDNode::get(
320 {MDString::get(Ctx,
"llvm.loop.vectorize.followup_all"), Followup}));
322 MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
323 LoopID->replaceOperandWith(0, LoopID);
324 HasUserTransforms =
true;
329LoopInfo::createLoopDistributeMetadata(
const LoopAttributes &Attrs,
331 bool &HasUserTransforms) {
332 LLVMContext &Ctx = Header->getContext();
334 std::optional<bool> Enabled;
340 if (Enabled !=
true) {
342 if (Enabled ==
false) {
343 NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
344 NewLoopProperties.push_back(
345 MDNode::get(Ctx, {MDString::get(Ctx,
"llvm.loop.distribute.enable"),
346 ConstantAsMetadata::get(ConstantInt::get(
347 llvm::Type::getInt1Ty(Ctx), 0))}));
348 LoopProperties = NewLoopProperties;
350 return createLoopVectorizeMetadata(Attrs, LoopProperties,
354 bool FollowupHasTransforms =
false;
356 createLoopVectorizeMetadata(Attrs, LoopProperties, FollowupHasTransforms);
359 Args.push_back(
nullptr);
360 Args.append(LoopProperties.begin(), LoopProperties.end());
362 Metadata *Vals[] = {MDString::get(Ctx,
"llvm.loop.distribute.enable"),
363 ConstantAsMetadata::get(ConstantInt::get(
364 llvm::Type::getInt1Ty(Ctx),
366 Args.push_back(MDNode::get(Ctx, Vals));
368 if (FollowupHasTransforms)
369 Args.push_back(MDNode::get(
371 {MDString::get(Ctx,
"llvm.loop.distribute.followup_all"), Followup}));
373 MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
374 LoopID->replaceOperandWith(0, LoopID);
375 HasUserTransforms =
true;
379MDNode *LoopInfo::createFullUnrollMetadata(
const LoopAttributes &Attrs,
381 bool &HasUserTransforms) {
382 LLVMContext &Ctx = Header->getContext();
384 std::optional<bool> Enabled;
390 if (Enabled !=
true) {
392 if (Enabled ==
false) {
393 NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
394 NewLoopProperties.push_back(
395 MDNode::get(Ctx, MDString::get(Ctx,
"llvm.loop.unroll.disable")));
396 LoopProperties = NewLoopProperties;
398 return createLoopDistributeMetadata(Attrs, LoopProperties,
403 Args.push_back(
nullptr);
404 Args.append(LoopProperties.begin(), LoopProperties.end());
405 Args.push_back(MDNode::get(Ctx, MDString::get(Ctx,
"llvm.loop.unroll.full")));
410 MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
411 LoopID->replaceOperandWith(0, LoopID);
412 HasUserTransforms =
true;
416MDNode *LoopInfo::createMetadata(
419 bool &HasUserTransforms) {
424 LoopProperties.push_back(StartLoc.getAsMDNode());
428 LoopProperties.push_back(EndLoc.getAsMDNode());
431 LLVMContext &Ctx = Header->getContext();
433 LoopProperties.push_back(
434 MDNode::get(Ctx, MDString::get(Ctx,
"llvm.loop.mustprogress")));
437 "There must be an access group iff the loop is parallel");
439 LoopProperties.push_back(MDNode::get(
440 Ctx, {MDString::get(Ctx,
"llvm.loop.parallel_accesses"), AccGroup}));
445 Metadata *Vals[] = {MDString::get(Ctx,
"llvm.loop.align"),
446 ConstantAsMetadata::get(ConstantInt::get(
447 llvm::Type::getInt32Ty(Ctx), Attrs.
CodeAlign))};
448 LoopProperties.push_back(MDNode::get(Ctx, Vals));
451 LoopProperties.insert(LoopProperties.end(), AdditionalLoopProperties.begin(),
452 AdditionalLoopProperties.end());
453 return createFullUnrollMetadata(Attrs, LoopProperties, HasUserTransforms);
462 UnrollCount(0), UnrollAndJamCount(0),
464 PipelineInitiationInterval(0), CodeAlign(0), MustProgress(
false) {}
485 const llvm::DebugLoc &StartLoc,
const llvm::DebugLoc &EndLoc,
487 : Header(Header), Attrs(Attrs), StartLoc(StartLoc), EndLoc(EndLoc),
492 LLVMContext &Ctx = Header->getContext();
493 AccGroup = MDNode::getDistinct(Ctx, {});
509 TempLoopID = MDNode::getTemporary(Header->getContext(), std::nullopt);
520 LLVMContext &Ctx = Header->getContext();
568 if (!Parent->UnrollAndJamInnerFollowup) {
578 BeforeLoopProperties.push_back(
579 MDNode::get(Ctx, MDString::get(Ctx,
"llvm.loop.isvectorized")));
581 bool InnerFollowupHasTransform =
false;
582 MDNode *InnerFollowup = createMetadata(AfterJam, BeforeLoopProperties,
583 InnerFollowupHasTransform);
584 if (InnerFollowupHasTransform)
585 Parent->UnrollAndJamInnerFollowup = InnerFollowup;
588 CurLoopAttr = BeforeJam;
591 bool HasUserTransforms =
false;
592 LoopID = createMetadata(CurLoopAttr, {}, HasUserTransforms);
593 TempLoopID->replaceAllUsesWith(LoopID);
597 const llvm::DebugLoc &EndLoc) {
599 new LoopInfo(Header, StagedAttrs, StartLoc, EndLoc,
600 Active.empty() ?
nullptr : Active.back().get()));
608 const llvm::DebugLoc &StartLoc,
609 const llvm::DebugLoc &EndLoc,
bool MustProgress) {
611 for (
const auto *
Attr : Attrs) {
612 const LoopHintAttr *LH = dyn_cast<LoopHintAttr>(
Attr);
613 const OpenCLUnrollHintAttr *OpenCLHint =
614 dyn_cast<OpenCLUnrollHintAttr>(
Attr);
615 const HLSLLoopHintAttr *HLSLLoopHint = dyn_cast<HLSLLoopHintAttr>(
Attr);
617 if (!LH && !OpenCLHint && !HLSLLoopHint) {
621 LoopHintAttr::OptionType Option = LoopHintAttr::Unroll;
622 LoopHintAttr::LoopHintState State = LoopHintAttr::Disable;
623 unsigned ValueInt = 1;
631 ValueInt = OpenCLHint->getUnrollHint();
633 State = LoopHintAttr::Enable;
634 }
else if (ValueInt != 1) {
635 Option = LoopHintAttr::UnrollCount;
636 State = LoopHintAttr::Numeric;
638 }
else if (HLSLLoopHint) {
639 ValueInt = HLSLLoopHint->getDirective();
640 if (HLSLLoopHint->getSemanticSpelling() ==
641 HLSLLoopHintAttr::Spelling::Microsoft_unroll) {
643 State = LoopHintAttr::Enable;
645 Option = LoopHintAttr::UnrollCount;
646 State = LoopHintAttr::Numeric;
650 auto *ValueExpr = LH->getValue();
652 llvm::APSInt ValueAPS = ValueExpr->EvaluateKnownConstInt(Ctx);
653 ValueInt = ValueAPS.getSExtValue();
656 Option = LH->getOption();
657 State = LH->getState();
660 case LoopHintAttr::Disable:
662 case LoopHintAttr::Vectorize:
667 case LoopHintAttr::Interleave:
671 case LoopHintAttr::Unroll:
674 case LoopHintAttr::UnrollAndJam:
677 case LoopHintAttr::VectorizePredicate:
680 case LoopHintAttr::Distribute:
683 case LoopHintAttr::PipelineDisabled:
686 case LoopHintAttr::UnrollCount:
687 case LoopHintAttr::UnrollAndJamCount:
688 case LoopHintAttr::VectorizeWidth:
689 case LoopHintAttr::InterleaveCount:
690 case LoopHintAttr::PipelineInitiationInterval:
691 llvm_unreachable(
"Options cannot be disabled.");
695 case LoopHintAttr::Enable:
697 case LoopHintAttr::Vectorize:
698 case LoopHintAttr::Interleave:
701 case LoopHintAttr::Unroll:
704 case LoopHintAttr::UnrollAndJam:
707 case LoopHintAttr::VectorizePredicate:
710 case LoopHintAttr::Distribute:
713 case LoopHintAttr::UnrollCount:
714 case LoopHintAttr::UnrollAndJamCount:
715 case LoopHintAttr::VectorizeWidth:
716 case LoopHintAttr::InterleaveCount:
717 case LoopHintAttr::PipelineDisabled:
718 case LoopHintAttr::PipelineInitiationInterval:
719 llvm_unreachable(
"Options cannot enabled.");
723 case LoopHintAttr::AssumeSafety:
725 case LoopHintAttr::Vectorize:
726 case LoopHintAttr::Interleave:
731 case LoopHintAttr::Unroll:
732 case LoopHintAttr::UnrollAndJam:
733 case LoopHintAttr::VectorizePredicate:
734 case LoopHintAttr::UnrollCount:
735 case LoopHintAttr::UnrollAndJamCount:
736 case LoopHintAttr::VectorizeWidth:
737 case LoopHintAttr::InterleaveCount:
738 case LoopHintAttr::Distribute:
739 case LoopHintAttr::PipelineDisabled:
740 case LoopHintAttr::PipelineInitiationInterval:
741 llvm_unreachable(
"Options cannot be used to assume mem safety.");
745 case LoopHintAttr::Full:
747 case LoopHintAttr::Unroll:
750 case LoopHintAttr::UnrollAndJam:
753 case LoopHintAttr::Vectorize:
754 case LoopHintAttr::Interleave:
755 case LoopHintAttr::UnrollCount:
756 case LoopHintAttr::UnrollAndJamCount:
757 case LoopHintAttr::VectorizeWidth:
758 case LoopHintAttr::InterleaveCount:
759 case LoopHintAttr::Distribute:
760 case LoopHintAttr::PipelineDisabled:
761 case LoopHintAttr::PipelineInitiationInterval:
762 case LoopHintAttr::VectorizePredicate:
763 llvm_unreachable(
"Options cannot be used with 'full' hint.");
767 case LoopHintAttr::FixedWidth:
768 case LoopHintAttr::ScalableWidth:
770 case LoopHintAttr::VectorizeWidth:
778 llvm_unreachable(
"Options cannot be used with 'scalable' hint.");
782 case LoopHintAttr::Numeric:
784 case LoopHintAttr::InterleaveCount:
787 case LoopHintAttr::UnrollCount:
790 case LoopHintAttr::UnrollAndJamCount:
793 case LoopHintAttr::PipelineInitiationInterval:
796 case LoopHintAttr::Unroll:
797 case LoopHintAttr::UnrollAndJam:
798 case LoopHintAttr::VectorizePredicate:
799 case LoopHintAttr::Vectorize:
800 case LoopHintAttr::VectorizeWidth:
801 case LoopHintAttr::Interleave:
802 case LoopHintAttr::Distribute:
803 case LoopHintAttr::PipelineDisabled:
804 llvm_unreachable(
"Options cannot be assigned a value.");
814 if (
const auto *CodeAlign = getSpecificAttr<const CodeAlignAttr>(Attrs)) {
815 const auto *CE = cast<ConstantExpr>(CodeAlign->getAlignment());
816 llvm::APSInt ArgVal = CE->getResultAsAPSInt();
822 if (CGOpts.OptimizationLevel > 0)
825 if (!CGOpts.UnrollLoops &&
831 push(Header, StartLoc, EndLoc);
835 assert(!Active.empty() &&
"No active loops to pop");
836 Active.back()->finish();
841 if (I->mayReadOrWriteMemory()) {
843 for (
const auto &AL : Active) {
845 if (MDNode *Group = AL->getAccessGroup())
846 AccessGroups.push_back(Group);
848 MDNode *UnionMD =
nullptr;
849 if (AccessGroups.size() == 1)
850 UnionMD = cast<MDNode>(AccessGroups[0]);
851 else if (AccessGroups.size() >= 2)
852 UnionMD = MDNode::get(I->getContext(), AccessGroups);
853 I->setMetadata(
"llvm.access.group", UnionMD);
863 if (I->isTerminator()) {
864 for (BasicBlock *Succ : successors(I))
866 I->setMetadata(llvm::LLVMContext::MD_loop, L.
getLoopID());
Defines the clang::ASTContext interface.
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
Attr - This represents one attribute.
CodeGenOptions - Track various options which control how the code is optimized and passed to the back...
void setPipelineDisabled(bool S)
Set the pipeline disabled state.
void setUnrollCount(unsigned C)
Set the unroll count for the next loop pushed.
bool hasInfo() const
Returns true if there is LoopInfo on the stack.
void setVectorizeWidth(unsigned W)
Set the vectorize width for the next loop pushed.
void InsertHelper(llvm::Instruction *I) const
Function called by the CodeGenFunction when an instruction is created.
void setDistributeState(bool Enable=true)
Set the next pushed loop as a distribution candidate.
void setParallel(bool Enable=true)
Set the next pushed loop as parallel.
void setInterleaveCount(unsigned C)
Set the interleave count for the next loop pushed.
void setUnrollState(const LoopAttributes::LVEnableState &State)
Set the next pushed loop unroll state.
void setVectorizeScalable(const LoopAttributes::LVEnableState &State)
void setVectorizePredicateState(const LoopAttributes::LVEnableState &State)
Set the next pushed vectorize predicate state.
void pop()
End the current loop.
void setCodeAlign(unsigned C)
Set value of code align for the next loop pushed.
void push(llvm::BasicBlock *Header, const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc)
Begin a new structured loop.
void setMustProgress(bool P)
Set no progress for the next loop pushed.
void setUnrollAndJamState(const LoopAttributes::LVEnableState &State)
Set the next pushed loop unroll_and_jam state.
void setUnrollAndJamCount(unsigned C)
Set the unroll count for the next loop pushed.
const LoopInfo & getInfo() const
Return the LoopInfo for the current loop.
void setPipelineInitiationInterval(unsigned C)
Set the pipeline initiation interval.
void setVectorizeEnable(bool Enable=true)
Set the next pushed loop 'vectorize.enable'.
Information used when generating a structured loop.
llvm::BasicBlock * getHeader() const
Get the header block of this loop.
LoopInfo(llvm::BasicBlock *Header, const LoopAttributes &Attrs, const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc, LoopInfo *Parent)
Construct a new LoopInfo for the loop with entry Header.
llvm::MDNode * getLoopID() const
Get the loop id metadata for this loop.
void finish()
Create the loop's metadata.
Diagnostic wrappers for TextAPI types for error reporting.
Attributes that may be specified on loops.
unsigned UnrollCount
llvm.unroll.
bool MustProgress
Value for whether the loop is required to make progress.
unsigned InterleaveCount
Value for llvm.loop.interleave.count metadata.
LoopAttributes(bool IsParallel=false)
bool IsParallel
Generate llvm.loop.parallel metadata for loads and stores.
LVEnableState VectorizeScalable
LVEnableState UnrollAndJamEnable
Value for llvm.loop.unroll_and_jam.* metadata (enable, disable, or full).
unsigned UnrollAndJamCount
llvm.unroll.
LVEnableState VectorizePredicateEnable
Value for llvm.loop.vectorize.predicate metadata.
LVEnableState DistributeEnable
Value for llvm.loop.distribute.enable metadata.
bool PipelineDisabled
Value for llvm.loop.pipeline.disable metadata.
unsigned CodeAlign
Value for 'llvm.loop.align' metadata.
LVEnableState UnrollEnable
Value for llvm.loop.unroll.* metadata (enable, disable, or full).
unsigned VectorizeWidth
Value for llvm.loop.vectorize.width metadata.
unsigned PipelineInitiationInterval
Value for llvm.loop.pipeline.iicount metadata.
LVEnableState VectorizeEnable
Value for llvm.loop.vectorize.enable metadata.