35#include "llvm/ADT/APFloat.h"
36#include "llvm/ADT/APInt.h"
37#include "llvm/ADT/FloatingPointMode.h"
38#include "llvm/ADT/SmallPtrSet.h"
39#include "llvm/ADT/StringExtras.h"
40#include "llvm/Analysis/ValueTracking.h"
41#include "llvm/IR/DataLayout.h"
42#include "llvm/IR/InlineAsm.h"
43#include "llvm/IR/Intrinsics.h"
44#include "llvm/IR/IntrinsicsAArch64.h"
45#include "llvm/IR/IntrinsicsAMDGPU.h"
46#include "llvm/IR/IntrinsicsARM.h"
47#include "llvm/IR/IntrinsicsBPF.h"
48#include "llvm/IR/IntrinsicsDirectX.h"
49#include "llvm/IR/IntrinsicsHexagon.h"
50#include "llvm/IR/IntrinsicsNVPTX.h"
51#include "llvm/IR/IntrinsicsPowerPC.h"
52#include "llvm/IR/IntrinsicsR600.h"
53#include "llvm/IR/IntrinsicsRISCV.h"
54#include "llvm/IR/IntrinsicsS390.h"
55#include "llvm/IR/IntrinsicsVE.h"
56#include "llvm/IR/IntrinsicsWebAssembly.h"
57#include "llvm/IR/IntrinsicsX86.h"
58#include "llvm/IR/MDBuilder.h"
59#include "llvm/IR/MatrixBuilder.h"
60#include "llvm/Support/ConvertUTF.h"
61#include "llvm/Support/MathExtras.h"
62#include "llvm/Support/ScopedPrinter.h"
63#include "llvm/TargetParser/AArch64TargetParser.h"
64#include "llvm/TargetParser/X86TargetParser.h"
69using namespace CodeGen;
73 Align AlignmentInBytes) {
75 switch (CGF.
getLangOpts().getTrivialAutoVarInit()) {
76 case LangOptions::TrivialAutoVarInitKind::Uninitialized:
79 case LangOptions::TrivialAutoVarInitKind::Zero:
80 Byte = CGF.
Builder.getInt8(0x00);
82 case LangOptions::TrivialAutoVarInitKind::Pattern: {
84 Byte = llvm::dyn_cast<llvm::ConstantInt>(
92 I->addAnnotationMetadata(
"auto-init");
107 static SmallDenseMap<unsigned, StringRef, 64> F128Builtins{
108 {Builtin::BI__builtin___fprintf_chk,
"__fprintf_chkieee128"},
109 {Builtin::BI__builtin___printf_chk,
"__printf_chkieee128"},
110 {Builtin::BI__builtin___snprintf_chk,
"__snprintf_chkieee128"},
111 {Builtin::BI__builtin___sprintf_chk,
"__sprintf_chkieee128"},
112 {Builtin::BI__builtin___vfprintf_chk,
"__vfprintf_chkieee128"},
113 {Builtin::BI__builtin___vprintf_chk,
"__vprintf_chkieee128"},
114 {Builtin::BI__builtin___vsnprintf_chk,
"__vsnprintf_chkieee128"},
115 {Builtin::BI__builtin___vsprintf_chk,
"__vsprintf_chkieee128"},
116 {Builtin::BI__builtin_fprintf,
"__fprintfieee128"},
117 {Builtin::BI__builtin_printf,
"__printfieee128"},
118 {Builtin::BI__builtin_snprintf,
"__snprintfieee128"},
119 {Builtin::BI__builtin_sprintf,
"__sprintfieee128"},
120 {Builtin::BI__builtin_vfprintf,
"__vfprintfieee128"},
121 {Builtin::BI__builtin_vprintf,
"__vprintfieee128"},
122 {Builtin::BI__builtin_vsnprintf,
"__vsnprintfieee128"},
123 {Builtin::BI__builtin_vsprintf,
"__vsprintfieee128"},
124 {Builtin::BI__builtin_fscanf,
"__fscanfieee128"},
125 {Builtin::BI__builtin_scanf,
"__scanfieee128"},
126 {Builtin::BI__builtin_sscanf,
"__sscanfieee128"},
127 {Builtin::BI__builtin_vfscanf,
"__vfscanfieee128"},
128 {Builtin::BI__builtin_vscanf,
"__vscanfieee128"},
129 {Builtin::BI__builtin_vsscanf,
"__vsscanfieee128"},
130 {Builtin::BI__builtin_nexttowardf128,
"__nexttowardieee128"},
136 static SmallDenseMap<unsigned, StringRef, 4> AIXLongDouble64Builtins{
137 {Builtin::BI__builtin_frexpl,
"frexp"},
138 {Builtin::BI__builtin_ldexpl,
"ldexp"},
139 {Builtin::BI__builtin_modfl,
"modf"},
145 if (FD->
hasAttr<AsmLabelAttr>())
151 &
getTarget().getLongDoubleFormat() == &llvm::APFloat::IEEEquad() &&
152 F128Builtins.contains(BuiltinID))
153 Name = F128Builtins[BuiltinID];
156 &llvm::APFloat::IEEEdouble() &&
157 AIXLongDouble64Builtins.contains(BuiltinID))
158 Name = AIXLongDouble64Builtins[BuiltinID];
163 llvm::FunctionType *Ty =
166 return GetOrCreateLLVMFunction(Name, Ty, D,
false);
172 QualType T, llvm::IntegerType *IntType) {
175 if (
V->getType()->isPointerTy())
176 return CGF.
Builder.CreatePtrToInt(
V, IntType);
178 assert(
V->getType() == IntType);
186 if (ResultType->isPointerTy())
187 return CGF.
Builder.CreateIntToPtr(
V, ResultType);
189 assert(
V->getType() == ResultType);
200 if (Align % Bytes != 0) {
213 AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent) {
223 llvm::IntegerType *IntType = llvm::IntegerType::get(
227 llvm::Type *ValueType = Val->getType();
255 llvm::AtomicRMWInst::BinOp Kind,
264 llvm::AtomicRMWInst::BinOp Kind,
266 Instruction::BinaryOps Op,
267 bool Invert =
false) {
276 llvm::IntegerType *IntType = llvm::IntegerType::get(
280 llvm::Type *ValueType = Val->getType();
284 Kind, DestAddr, Val, llvm::AtomicOrdering::SequentiallyConsistent);
289 llvm::ConstantInt::getAllOnesValue(IntType));
313 llvm::IntegerType *IntType = llvm::IntegerType::get(
317 llvm::Type *ValueType = Cmp->getType();
322 DestAddr, Cmp, New, llvm::AtomicOrdering::SequentiallyConsistent,
323 llvm::AtomicOrdering::SequentiallyConsistent);
326 return CGF.
Builder.CreateZExt(CGF.
Builder.CreateExtractValue(Pair, 1),
349 AtomicOrdering SuccessOrdering = AtomicOrdering::SequentiallyConsistent) {
364 auto FailureOrdering = SuccessOrdering == AtomicOrdering::Release ?
365 AtomicOrdering::Monotonic :
373 DestAddr, Comparand, Exchange, SuccessOrdering, FailureOrdering);
374 Result->setVolatile(
true);
392 AtomicOrdering SuccessOrdering) {
399 assert(DestPtr->getType()->isPointerTy());
400 assert(!ExchangeHigh->getType()->isPointerTy());
401 assert(!ExchangeLow->getType()->isPointerTy());
404 auto FailureOrdering = SuccessOrdering == AtomicOrdering::Release
405 ? AtomicOrdering::Monotonic
410 llvm::Type *Int128Ty = llvm::IntegerType::get(CGF.
getLLVMContext(), 128);
411 Address DestAddr(DestPtr, Int128Ty,
416 ExchangeHigh = CGF.
Builder.CreateZExt(ExchangeHigh, Int128Ty);
417 ExchangeLow = CGF.
Builder.CreateZExt(ExchangeLow, Int128Ty);
419 CGF.
Builder.CreateShl(ExchangeHigh, llvm::ConstantInt::get(Int128Ty, 64));
420 llvm::Value *Exchange = CGF.
Builder.CreateOr(ExchangeHigh, ExchangeLow);
426 SuccessOrdering, FailureOrdering);
432 CXI->setVolatile(
true);
444 AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent) {
450 AtomicRMWInst::Add, DestAddr, ConstantInt::get(IntTy, 1), Ordering);
451 return CGF.
Builder.CreateAdd(
Result, ConstantInt::get(IntTy, 1));
456 AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent) {
462 AtomicRMWInst::Sub, DestAddr, ConstantInt::get(IntTy, 1), Ordering);
463 return CGF.
Builder.CreateSub(
Result, ConstantInt::get(IntTy, 1));
474 Load->setVolatile(
true);
484 llvm::StoreInst *Store =
486 Store->setVolatile(
true);
494 const CallExpr *E,
unsigned IntrinsicID,
495 unsigned ConstrainedIntrinsicID) {
498 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
499 if (CGF.
Builder.getIsFPConstrained()) {
501 return CGF.
Builder.CreateConstrainedFPCall(F, { Src0 });
504 return CGF.
Builder.CreateCall(F, Src0);
511 const CallExpr *E,
unsigned IntrinsicID,
512 unsigned ConstrainedIntrinsicID) {
516 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
517 if (CGF.
Builder.getIsFPConstrained()) {
519 return CGF.
Builder.CreateConstrainedFPCall(F, { Src0, Src1 });
522 return CGF.
Builder.CreateCall(F, { Src0, Src1 });
529 llvm::Intrinsic::ID ConstrainedIntrinsicID) {
533 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
534 if (CGF.
Builder.getIsFPConstrained()) {
536 {Src0->getType(), Src1->getType()});
537 return CGF.
Builder.CreateConstrainedFPCall(F, {Src0, Src1});
542 return CGF.
Builder.CreateCall(F, {Src0, Src1});
548 const CallExpr *E,
unsigned IntrinsicID,
549 unsigned ConstrainedIntrinsicID) {
554 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
555 if (CGF.
Builder.getIsFPConstrained()) {
557 return CGF.
Builder.CreateConstrainedFPCall(F, { Src0, Src1, Src2 });
560 return CGF.
Builder.CreateCall(F, { Src0, Src1, Src2 });
567 unsigned IntrinsicID,
568 unsigned ConstrainedIntrinsicID,
572 if (CGF.
Builder.getIsFPConstrained())
577 if (CGF.
Builder.getIsFPConstrained())
578 return CGF.
Builder.CreateConstrainedFPCall(F, Args);
580 return CGF.
Builder.CreateCall(F, Args);
586 unsigned IntrinsicID,
587 llvm::StringRef Name =
"") {
591 return CGF.
Builder.CreateCall(F, Src0, Name);
597 unsigned IntrinsicID) {
602 return CGF.
Builder.CreateCall(F, { Src0, Src1 });
608 unsigned IntrinsicID) {
614 return CGF.
Builder.CreateCall(F, { Src0, Src1, Src2 });
620 unsigned IntrinsicID) {
625 return CGF.
Builder.CreateCall(F, {Src0, Src1});
631 unsigned IntrinsicID,
632 unsigned ConstrainedIntrinsicID) {
636 if (CGF.
Builder.getIsFPConstrained()) {
637 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
639 {ResultType, Src0->getType()});
640 return CGF.
Builder.CreateConstrainedFPCall(F, {Src0});
644 return CGF.
Builder.CreateCall(F, Src0);
649 llvm::Intrinsic::ID IntrinsicID) {
657 llvm::Value *
Call = CGF.
Builder.CreateCall(F, Src0);
659 llvm::Value *Exp = CGF.
Builder.CreateExtractValue(
Call, 1);
670 Call->setDoesNotAccessMemory();
679 llvm::Type *Ty =
V->getType();
680 int Width = Ty->getPrimitiveSizeInBits();
681 llvm::Type *IntTy = llvm::IntegerType::get(
C, Width);
683 if (Ty->isPPC_FP128Ty()) {
693 Value *ShiftCst = llvm::ConstantInt::get(IntTy, Width);
698 IntTy = llvm::IntegerType::get(
C, Width);
701 Value *Zero = llvm::Constant::getNullValue(IntTy);
702 return CGF.
Builder.CreateICmpSLT(
V, Zero);
706 const CallExpr *E, llvm::Constant *calleeValue) {
707 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
722 const llvm::Intrinsic::ID IntrinsicID,
723 llvm::Value *
X, llvm::Value *Y,
724 llvm::Value *&Carry) {
726 assert(
X->getType() == Y->getType() &&
727 "Arguments must be the same type. (Did you forget to make sure both "
728 "arguments have the same integer width?)");
731 llvm::Value *Tmp = CGF.
Builder.CreateCall(Callee, {
X, Y});
732 Carry = CGF.
Builder.CreateExtractValue(Tmp, 1);
733 return CGF.
Builder.CreateExtractValue(Tmp, 0);
737 unsigned IntrinsicID,
740 llvm::MDNode *RNode = MDHelper.createRange(APInt(32, low), APInt(32, high));
742 llvm::Instruction *
Call = CGF.
Builder.CreateCall(F);
743 Call->setMetadata(llvm::LLVMContext::MD_range, RNode);
744 Call->setMetadata(llvm::LLVMContext::MD_noundef,
750 struct WidthAndSignedness {
756static WidthAndSignedness
770static struct WidthAndSignedness
772 assert(Types.size() > 0 &&
"Empty list of types.");
776 for (
const auto &
Type : Types) {
785 for (
const auto &
Type : Types) {
787 if (Width < MinWidth) {
796 Intrinsic::ID inst = IsStart ? Intrinsic::vastart : Intrinsic::vaend;
807 return From == To || (From == 0 && To == 1) || (From == 3 && To == 2);
812 return ConstantInt::get(ResType, (
Type & 2) ? 0 : -1,
true);
816CodeGenFunction::evaluateOrEmitBuiltinObjectSize(
const Expr *E,
unsigned Type,
817 llvm::IntegerType *ResType,
818 llvm::Value *EmittedE,
822 return emitBuiltinObjectSize(E,
Type, ResType, EmittedE, IsDynamic);
823 return ConstantInt::get(ResType, ObjectSize,
true);
831 uint32_t FieldNo = 0;
837 if ((!FAMDecl || FD == FAMDecl) &&
839 Ctx, FD, FD->getType(), StrictFlexArraysLevel,
867 if (FD->getType()->isCountAttributedType())
879CodeGenFunction::emitFlexibleArrayMemberSize(
const Expr *E,
unsigned Type,
880 llvm::IntegerType *ResType) {
909 const Expr *Idx =
nullptr;
911 if (
const auto *UO = dyn_cast<UnaryOperator>(
Base);
912 UO && UO->getOpcode() == UO_AddrOf) {
914 if (
const auto *ASE = dyn_cast<ArraySubscriptExpr>(SubExpr)) {
915 Base = ASE->getBase()->IgnoreParenImpCasts();
918 if (
const auto *IL = dyn_cast<IntegerLiteral>(Idx)) {
919 int64_t Val = IL->getValue().getSExtValue();
936 if (
const auto *ME = dyn_cast<MemberExpr>(
Base)) {
938 const ValueDecl *VD = ME->getMemberDecl();
940 FAMDecl = dyn_cast<FieldDecl>(VD);
943 }
else if (
const auto *DRE = dyn_cast<DeclRefExpr>(
Base)) {
945 QualType Ty = DRE->getDecl()->getType();
1004 CountedByInst =
Builder.CreateIntCast(CountedByInst, ResType, IsSigned);
1007 Value *IdxInst =
nullptr;
1015 IdxInst =
Builder.CreateIntCast(IdxInst, ResType, IdxSigned);
1020 Builder.CreateSub(CountedByInst, IdxInst,
"", !IsSigned, IsSigned);
1026 llvm::Constant *ElemSize =
1027 llvm::ConstantInt::get(ResType,
Size.getQuantity(), IsSigned);
1029 Builder.CreateMul(CountedByInst, ElemSize,
"", !IsSigned, IsSigned);
1030 FAMSize =
Builder.CreateIntCast(FAMSize, ResType, IsSigned);
1031 Value *Res = FAMSize;
1033 if (isa<DeclRefExpr>(
Base)) {
1038 llvm::Constant *FAMOffset = ConstantInt::get(ResType, Offset, IsSigned);
1039 Value *OffsetAndFAMSize =
1040 Builder.CreateAdd(FAMOffset, Res,
"", !IsSigned, IsSigned);
1043 llvm::Constant *SizeofStruct =
1049 ?
Builder.CreateBinaryIntrinsic(llvm::Intrinsic::smax,
1050 OffsetAndFAMSize, SizeofStruct)
1051 :
Builder.CreateBinaryIntrinsic(llvm::Intrinsic::umax,
1052 OffsetAndFAMSize, SizeofStruct);
1062 return Builder.CreateSelect(Cmp, Res, ConstantInt::get(ResType, 0, IsSigned));
1075CodeGenFunction::emitBuiltinObjectSize(
const Expr *E,
unsigned Type,
1076 llvm::IntegerType *ResType,
1077 llvm::Value *EmittedE,
bool IsDynamic) {
1081 auto *Param = dyn_cast<ParmVarDecl>(D->getDecl());
1082 auto *PS = D->getDecl()->getAttr<PassObjectSizeAttr>();
1083 if (Param !=
nullptr && PS !=
nullptr &&
1085 auto Iter = SizeArguments.find(Param);
1086 assert(
Iter != SizeArguments.end());
1089 auto DIter = LocalDeclMap.find(D);
1090 assert(DIter != LocalDeclMap.end());
1100 if (
Value *
V = emitFlexibleArrayMemberSize(E,
Type, ResType))
1111 assert(Ptr->
getType()->isPointerTy() &&
1112 "Non-pointer passed to __builtin_object_size?");
1128 enum ActionKind : uint8_t { TestOnly, Complement, Reset,
Set };
1129 enum InterlockingKind : uint8_t {
1138 InterlockingKind Interlocking;
1141 static BitTest decodeBitTestBuiltin(
unsigned BuiltinID);
1146BitTest BitTest::decodeBitTestBuiltin(
unsigned BuiltinID) {
1147 switch (BuiltinID) {
1149 case Builtin::BI_bittest:
1150 return {TestOnly, Unlocked,
false};
1151 case Builtin::BI_bittestandcomplement:
1152 return {Complement, Unlocked,
false};
1153 case Builtin::BI_bittestandreset:
1154 return {Reset, Unlocked,
false};
1155 case Builtin::BI_bittestandset:
1156 return {
Set, Unlocked,
false};
1157 case Builtin::BI_interlockedbittestandreset:
1158 return {Reset, Sequential,
false};
1159 case Builtin::BI_interlockedbittestandset:
1160 return {
Set, Sequential,
false};
1163 case Builtin::BI_bittest64:
1164 return {TestOnly, Unlocked,
true};
1165 case Builtin::BI_bittestandcomplement64:
1166 return {Complement, Unlocked,
true};
1167 case Builtin::BI_bittestandreset64:
1168 return {Reset, Unlocked,
true};
1169 case Builtin::BI_bittestandset64:
1170 return {
Set, Unlocked,
true};
1171 case Builtin::BI_interlockedbittestandreset64:
1172 return {Reset, Sequential,
true};
1173 case Builtin::BI_interlockedbittestandset64:
1174 return {
Set, Sequential,
true};
1177 case Builtin::BI_interlockedbittestandset_acq:
1178 return {
Set, Acquire,
false};
1179 case Builtin::BI_interlockedbittestandset_rel:
1180 return {
Set, Release,
false};
1181 case Builtin::BI_interlockedbittestandset_nf:
1182 return {
Set, NoFence,
false};
1183 case Builtin::BI_interlockedbittestandreset_acq:
1184 return {Reset, Acquire,
false};
1185 case Builtin::BI_interlockedbittestandreset_rel:
1186 return {Reset, Release,
false};
1187 case Builtin::BI_interlockedbittestandreset_nf:
1188 return {Reset, NoFence,
false};
1190 llvm_unreachable(
"expected only bittest intrinsics");
1195 case BitTest::TestOnly:
return '\0';
1196 case BitTest::Complement:
return 'c';
1197 case BitTest::Reset:
return 'r';
1198 case BitTest::Set:
return 's';
1200 llvm_unreachable(
"invalid action");
1208 char SizeSuffix = BT.Is64Bit ?
'q' :
'l';
1212 raw_svector_ostream AsmOS(
Asm);
1213 if (BT.Interlocking != BitTest::Unlocked)
1218 AsmOS << SizeSuffix <<
" $2, ($1)";
1221 std::string Constraints =
"={@ccc},r,r,~{cc},~{memory}";
1223 if (!MachineClobbers.empty()) {
1225 Constraints += MachineClobbers;
1227 llvm::IntegerType *IntType = llvm::IntegerType::get(
1230 llvm::FunctionType *FTy =
1231 llvm::FunctionType::get(CGF.
Int8Ty, {CGF.UnqualPtrTy, IntType},
false);
1233 llvm::InlineAsm *IA =
1234 llvm::InlineAsm::get(FTy,
Asm, Constraints,
true);
1235 return CGF.
Builder.CreateCall(IA, {BitBase, BitPos});
1238static llvm::AtomicOrdering
1241 case BitTest::Unlocked:
return llvm::AtomicOrdering::NotAtomic;
1242 case BitTest::Sequential:
return llvm::AtomicOrdering::SequentiallyConsistent;
1243 case BitTest::Acquire:
return llvm::AtomicOrdering::Acquire;
1244 case BitTest::Release:
return llvm::AtomicOrdering::Release;
1245 case BitTest::NoFence:
return llvm::AtomicOrdering::Monotonic;
1247 llvm_unreachable(
"invalid interlocking");
1260 BitTest BT = BitTest::decodeBitTestBuiltin(BuiltinID);
1272 BitPos, llvm::ConstantInt::get(BitPos->
getType(), 3),
"bittest.byteidx");
1275 ByteIndex,
"bittest.byteaddr"),
1279 llvm::ConstantInt::get(CGF.
Int8Ty, 0x7));
1282 Value *Mask =
nullptr;
1283 if (BT.Action != BitTest::TestOnly) {
1284 Mask = CGF.
Builder.CreateShl(llvm::ConstantInt::get(CGF.
Int8Ty, 1), PosLow,
1291 Value *OldByte =
nullptr;
1292 if (Ordering != llvm::AtomicOrdering::NotAtomic) {
1295 llvm::AtomicRMWInst::BinOp RMWOp = llvm::AtomicRMWInst::Or;
1296 if (BT.Action == BitTest::Reset) {
1297 Mask = CGF.
Builder.CreateNot(Mask);
1298 RMWOp = llvm::AtomicRMWInst::And;
1304 Value *NewByte =
nullptr;
1305 switch (BT.Action) {
1306 case BitTest::TestOnly:
1309 case BitTest::Complement:
1310 NewByte = CGF.
Builder.CreateXor(OldByte, Mask);
1312 case BitTest::Reset:
1313 NewByte = CGF.
Builder.CreateAnd(OldByte, CGF.
Builder.CreateNot(Mask));
1316 NewByte = CGF.
Builder.CreateOr(OldByte, Mask);
1325 Value *ShiftedByte = CGF.
Builder.CreateLShr(OldByte, PosLow,
"bittest.shr");
1327 ShiftedByte, llvm::ConstantInt::get(CGF.
Int8Ty, 1),
"bittest.res");
1336 raw_svector_ostream AsmOS(
Asm);
1337 llvm::IntegerType *RetType = CGF.
Int32Ty;
1339 switch (BuiltinID) {
1340 case clang::PPC::BI__builtin_ppc_ldarx:
1344 case clang::PPC::BI__builtin_ppc_lwarx:
1348 case clang::PPC::BI__builtin_ppc_lharx:
1352 case clang::PPC::BI__builtin_ppc_lbarx:
1357 llvm_unreachable(
"Expected only PowerPC load reserve intrinsics");
1360 AsmOS <<
"$0, ${1:y}";
1362 std::string Constraints =
"=r,*Z,~{memory}";
1364 if (!MachineClobbers.empty()) {
1366 Constraints += MachineClobbers;
1370 llvm::FunctionType *FTy = llvm::FunctionType::get(RetType, {PtrType},
false);
1372 llvm::InlineAsm *IA =
1373 llvm::InlineAsm::get(FTy,
Asm, Constraints,
true);
1374 llvm::CallInst *CI = CGF.
Builder.CreateCall(IA, {Addr});
1376 0, Attribute::get(CGF.
getLLVMContext(), Attribute::ElementType, RetType));
1381enum class MSVCSetJmpKind {
1393 llvm::Value *Arg1 =
nullptr;
1394 llvm::Type *Arg1Ty =
nullptr;
1396 bool IsVarArg =
false;
1397 if (SJKind == MSVCSetJmpKind::_setjmp3) {
1400 Arg1 = llvm::ConstantInt::get(CGF.
IntTy, 0);
1403 Name = SJKind == MSVCSetJmpKind::_setjmp ?
"_setjmp" :
"_setjmpex";
1406 Arg1 = CGF.
Builder.CreateCall(
1409 Arg1 = CGF.
Builder.CreateCall(
1411 llvm::ConstantInt::get(CGF.
Int32Ty, 0));
1415 llvm::Type *ArgTypes[2] = {CGF.
Int8PtrTy, Arg1Ty};
1416 llvm::AttributeList ReturnsTwiceAttr = llvm::AttributeList::get(
1418 llvm::Attribute::ReturnsTwice);
1420 llvm::FunctionType::get(CGF.
IntTy, ArgTypes, IsVarArg), Name,
1421 ReturnsTwiceAttr,
true);
1423 llvm::Value *Buf = CGF.
Builder.CreateBitOrPointerCast(
1425 llvm::Value *Args[] = {Buf, Arg1};
1427 CB->setAttributes(ReturnsTwiceAttr);
1475static std::optional<CodeGenFunction::MSVCIntrin>
1478 switch (BuiltinID) {
1480 return std::nullopt;
1481 case clang::ARM::BI_BitScanForward:
1482 case clang::ARM::BI_BitScanForward64:
1483 return MSVCIntrin::_BitScanForward;
1484 case clang::ARM::BI_BitScanReverse:
1485 case clang::ARM::BI_BitScanReverse64:
1486 return MSVCIntrin::_BitScanReverse;
1487 case clang::ARM::BI_InterlockedAnd64:
1488 return MSVCIntrin::_InterlockedAnd;
1489 case clang::ARM::BI_InterlockedExchange64:
1490 return MSVCIntrin::_InterlockedExchange;
1491 case clang::ARM::BI_InterlockedExchangeAdd64:
1492 return MSVCIntrin::_InterlockedExchangeAdd;
1493 case clang::ARM::BI_InterlockedExchangeSub64:
1494 return MSVCIntrin::_InterlockedExchangeSub;
1495 case clang::ARM::BI_InterlockedOr64:
1496 return MSVCIntrin::_InterlockedOr;
1497 case clang::ARM::BI_InterlockedXor64:
1498 return MSVCIntrin::_InterlockedXor;
1499 case clang::ARM::BI_InterlockedDecrement64:
1500 return MSVCIntrin::_InterlockedDecrement;
1501 case clang::ARM::BI_InterlockedIncrement64:
1502 return MSVCIntrin::_InterlockedIncrement;
1503 case clang::ARM::BI_InterlockedExchangeAdd8_acq:
1504 case clang::ARM::BI_InterlockedExchangeAdd16_acq:
1505 case clang::ARM::BI_InterlockedExchangeAdd_acq:
1506 case clang::ARM::BI_InterlockedExchangeAdd64_acq:
1507 return MSVCIntrin::_InterlockedExchangeAdd_acq;
1508 case clang::ARM::BI_InterlockedExchangeAdd8_rel:
1509 case clang::ARM::BI_InterlockedExchangeAdd16_rel:
1510 case clang::ARM::BI_InterlockedExchangeAdd_rel:
1511 case clang::ARM::BI_InterlockedExchangeAdd64_rel:
1512 return MSVCIntrin::_InterlockedExchangeAdd_rel;
1513 case clang::ARM::BI_InterlockedExchangeAdd8_nf:
1514 case clang::ARM::BI_InterlockedExchangeAdd16_nf:
1515 case clang::ARM::BI_InterlockedExchangeAdd_nf:
1516 case clang::ARM::BI_InterlockedExchangeAdd64_nf:
1517 return MSVCIntrin::_InterlockedExchangeAdd_nf;
1518 case clang::ARM::BI_InterlockedExchange8_acq:
1519 case clang::ARM::BI_InterlockedExchange16_acq:
1520 case clang::ARM::BI_InterlockedExchange_acq:
1521 case clang::ARM::BI_InterlockedExchange64_acq:
1522 return MSVCIntrin::_InterlockedExchange_acq;
1523 case clang::ARM::BI_InterlockedExchange8_rel:
1524 case clang::ARM::BI_InterlockedExchange16_rel:
1525 case clang::ARM::BI_InterlockedExchange_rel:
1526 case clang::ARM::BI_InterlockedExchange64_rel:
1527 return MSVCIntrin::_InterlockedExchange_rel;
1528 case clang::ARM::BI_InterlockedExchange8_nf:
1529 case clang::ARM::BI_InterlockedExchange16_nf:
1530 case clang::ARM::BI_InterlockedExchange_nf:
1531 case clang::ARM::BI_InterlockedExchange64_nf:
1532 return MSVCIntrin::_InterlockedExchange_nf;
1533 case clang::ARM::BI_InterlockedCompareExchange8_acq:
1534 case clang::ARM::BI_InterlockedCompareExchange16_acq:
1535 case clang::ARM::BI_InterlockedCompareExchange_acq:
1536 case clang::ARM::BI_InterlockedCompareExchange64_acq:
1537 return MSVCIntrin::_InterlockedCompareExchange_acq;
1538 case clang::ARM::BI_InterlockedCompareExchange8_rel:
1539 case clang::ARM::BI_InterlockedCompareExchange16_rel:
1540 case clang::ARM::BI_InterlockedCompareExchange_rel:
1541 case clang::ARM::BI_InterlockedCompareExchange64_rel:
1542 return MSVCIntrin::_InterlockedCompareExchange_rel;
1543 case clang::ARM::BI_InterlockedCompareExchange8_nf:
1544 case clang::ARM::BI_InterlockedCompareExchange16_nf:
1545 case clang::ARM::BI_InterlockedCompareExchange_nf:
1546 case clang::ARM::BI_InterlockedCompareExchange64_nf:
1547 return MSVCIntrin::_InterlockedCompareExchange_nf;
1548 case clang::ARM::BI_InterlockedOr8_acq:
1549 case clang::ARM::BI_InterlockedOr16_acq:
1550 case clang::ARM::BI_InterlockedOr_acq:
1551 case clang::ARM::BI_InterlockedOr64_acq:
1552 return MSVCIntrin::_InterlockedOr_acq;
1553 case clang::ARM::BI_InterlockedOr8_rel:
1554 case clang::ARM::BI_InterlockedOr16_rel:
1555 case clang::ARM::BI_InterlockedOr_rel:
1556 case clang::ARM::BI_InterlockedOr64_rel:
1557 return MSVCIntrin::_InterlockedOr_rel;
1558 case clang::ARM::BI_InterlockedOr8_nf:
1559 case clang::ARM::BI_InterlockedOr16_nf:
1560 case clang::ARM::BI_InterlockedOr_nf:
1561 case clang::ARM::BI_InterlockedOr64_nf:
1562 return MSVCIntrin::_InterlockedOr_nf;
1563 case clang::ARM::BI_InterlockedXor8_acq:
1564 case clang::ARM::BI_InterlockedXor16_acq:
1565 case clang::ARM::BI_InterlockedXor_acq:
1566 case clang::ARM::BI_InterlockedXor64_acq:
1567 return MSVCIntrin::_InterlockedXor_acq;
1568 case clang::ARM::BI_InterlockedXor8_rel:
1569 case clang::ARM::BI_InterlockedXor16_rel:
1570 case clang::ARM::BI_InterlockedXor_rel:
1571 case clang::ARM::BI_InterlockedXor64_rel:
1572 return MSVCIntrin::_InterlockedXor_rel;
1573 case clang::ARM::BI_InterlockedXor8_nf:
1574 case clang::ARM::BI_InterlockedXor16_nf:
1575 case clang::ARM::BI_InterlockedXor_nf:
1576 case clang::ARM::BI_InterlockedXor64_nf:
1577 return MSVCIntrin::_InterlockedXor_nf;
1578 case clang::ARM::BI_InterlockedAnd8_acq:
1579 case clang::ARM::BI_InterlockedAnd16_acq:
1580 case clang::ARM::BI_InterlockedAnd_acq:
1581 case clang::ARM::BI_InterlockedAnd64_acq:
1582 return MSVCIntrin::_InterlockedAnd_acq;
1583 case clang::ARM::BI_InterlockedAnd8_rel:
1584 case clang::ARM::BI_InterlockedAnd16_rel:
1585 case clang::ARM::BI_InterlockedAnd_rel:
1586 case clang::ARM::BI_InterlockedAnd64_rel:
1587 return MSVCIntrin::_InterlockedAnd_rel;
1588 case clang::ARM::BI_InterlockedAnd8_nf:
1589 case clang::ARM::BI_InterlockedAnd16_nf:
1590 case clang::ARM::BI_InterlockedAnd_nf:
1591 case clang::ARM::BI_InterlockedAnd64_nf:
1592 return MSVCIntrin::_InterlockedAnd_nf;
1593 case clang::ARM::BI_InterlockedIncrement16_acq:
1594 case clang::ARM::BI_InterlockedIncrement_acq:
1595 case clang::ARM::BI_InterlockedIncrement64_acq:
1596 return MSVCIntrin::_InterlockedIncrement_acq;
1597 case clang::ARM::BI_InterlockedIncrement16_rel:
1598 case clang::ARM::BI_InterlockedIncrement_rel:
1599 case clang::ARM::BI_InterlockedIncrement64_rel:
1600 return MSVCIntrin::_InterlockedIncrement_rel;
1601 case clang::ARM::BI_InterlockedIncrement16_nf:
1602 case clang::ARM::BI_InterlockedIncrement_nf:
1603 case clang::ARM::BI_InterlockedIncrement64_nf:
1604 return MSVCIntrin::_InterlockedIncrement_nf;
1605 case clang::ARM::BI_InterlockedDecrement16_acq:
1606 case clang::ARM::BI_InterlockedDecrement_acq:
1607 case clang::ARM::BI_InterlockedDecrement64_acq:
1608 return MSVCIntrin::_InterlockedDecrement_acq;
1609 case clang::ARM::BI_InterlockedDecrement16_rel:
1610 case clang::ARM::BI_InterlockedDecrement_rel:
1611 case clang::ARM::BI_InterlockedDecrement64_rel:
1612 return MSVCIntrin::_InterlockedDecrement_rel;
1613 case clang::ARM::BI_InterlockedDecrement16_nf:
1614 case clang::ARM::BI_InterlockedDecrement_nf:
1615 case clang::ARM::BI_InterlockedDecrement64_nf:
1616 return MSVCIntrin::_InterlockedDecrement_nf;
1618 llvm_unreachable(
"must return from switch");
1621static std::optional<CodeGenFunction::MSVCIntrin>
1624 switch (BuiltinID) {
1626 return std::nullopt;
1627 case clang::AArch64::BI_BitScanForward:
1628 case clang::AArch64::BI_BitScanForward64:
1629 return MSVCIntrin::_BitScanForward;
1630 case clang::AArch64::BI_BitScanReverse:
1631 case clang::AArch64::BI_BitScanReverse64:
1632 return MSVCIntrin::_BitScanReverse;
1633 case clang::AArch64::BI_InterlockedAnd64:
1634 return MSVCIntrin::_InterlockedAnd;
1635 case clang::AArch64::BI_InterlockedExchange64:
1636 return MSVCIntrin::_InterlockedExchange;
1637 case clang::AArch64::BI_InterlockedExchangeAdd64:
1638 return MSVCIntrin::_InterlockedExchangeAdd;
1639 case clang::AArch64::BI_InterlockedExchangeSub64:
1640 return MSVCIntrin::_InterlockedExchangeSub;
1641 case clang::AArch64::BI_InterlockedOr64:
1642 return MSVCIntrin::_InterlockedOr;
1643 case clang::AArch64::BI_InterlockedXor64:
1644 return MSVCIntrin::_InterlockedXor;
1645 case clang::AArch64::BI_InterlockedDecrement64:
1646 return MSVCIntrin::_InterlockedDecrement;
1647 case clang::AArch64::BI_InterlockedIncrement64:
1648 return MSVCIntrin::_InterlockedIncrement;
1649 case clang::AArch64::BI_InterlockedExchangeAdd8_acq:
1650 case clang::AArch64::BI_InterlockedExchangeAdd16_acq:
1651 case clang::AArch64::BI_InterlockedExchangeAdd_acq:
1652 case clang::AArch64::BI_InterlockedExchangeAdd64_acq:
1653 return MSVCIntrin::_InterlockedExchangeAdd_acq;
1654 case clang::AArch64::BI_InterlockedExchangeAdd8_rel:
1655 case clang::AArch64::BI_InterlockedExchangeAdd16_rel:
1656 case clang::AArch64::BI_InterlockedExchangeAdd_rel:
1657 case clang::AArch64::BI_InterlockedExchangeAdd64_rel:
1658 return MSVCIntrin::_InterlockedExchangeAdd_rel;
1659 case clang::AArch64::BI_InterlockedExchangeAdd8_nf:
1660 case clang::AArch64::BI_InterlockedExchangeAdd16_nf:
1661 case clang::AArch64::BI_InterlockedExchangeAdd_nf:
1662 case clang::AArch64::BI_InterlockedExchangeAdd64_nf:
1663 return MSVCIntrin::_InterlockedExchangeAdd_nf;
1664 case clang::AArch64::BI_InterlockedExchange8_acq:
1665 case clang::AArch64::BI_InterlockedExchange16_acq:
1666 case clang::AArch64::BI_InterlockedExchange_acq:
1667 case clang::AArch64::BI_InterlockedExchange64_acq:
1668 return MSVCIntrin::_InterlockedExchange_acq;
1669 case clang::AArch64::BI_InterlockedExchange8_rel:
1670 case clang::AArch64::BI_InterlockedExchange16_rel:
1671 case clang::AArch64::BI_InterlockedExchange_rel:
1672 case clang::AArch64::BI_InterlockedExchange64_rel:
1673 return MSVCIntrin::_InterlockedExchange_rel;
1674 case clang::AArch64::BI_InterlockedExchange8_nf:
1675 case clang::AArch64::BI_InterlockedExchange16_nf:
1676 case clang::AArch64::BI_InterlockedExchange_nf:
1677 case clang::AArch64::BI_InterlockedExchange64_nf:
1678 return MSVCIntrin::_InterlockedExchange_nf;
1679 case clang::AArch64::BI_InterlockedCompareExchange8_acq:
1680 case clang::AArch64::BI_InterlockedCompareExchange16_acq:
1681 case clang::AArch64::BI_InterlockedCompareExchange_acq:
1682 case clang::AArch64::BI_InterlockedCompareExchange64_acq:
1683 return MSVCIntrin::_InterlockedCompareExchange_acq;
1684 case clang::AArch64::BI_InterlockedCompareExchange8_rel:
1685 case clang::AArch64::BI_InterlockedCompareExchange16_rel:
1686 case clang::AArch64::BI_InterlockedCompareExchange_rel:
1687 case clang::AArch64::BI_InterlockedCompareExchange64_rel:
1688 return MSVCIntrin::_InterlockedCompareExchange_rel;
1689 case clang::AArch64::BI_InterlockedCompareExchange8_nf:
1690 case clang::AArch64::BI_InterlockedCompareExchange16_nf:
1691 case clang::AArch64::BI_InterlockedCompareExchange_nf:
1692 case clang::AArch64::BI_InterlockedCompareExchange64_nf:
1693 return MSVCIntrin::_InterlockedCompareExchange_nf;
1694 case clang::AArch64::BI_InterlockedCompareExchange128:
1695 return MSVCIntrin::_InterlockedCompareExchange128;
1696 case clang::AArch64::BI_InterlockedCompareExchange128_acq:
1697 return MSVCIntrin::_InterlockedCompareExchange128_acq;
1698 case clang::AArch64::BI_InterlockedCompareExchange128_nf:
1699 return MSVCIntrin::_InterlockedCompareExchange128_nf;
1700 case clang::AArch64::BI_InterlockedCompareExchange128_rel:
1701 return MSVCIntrin::_InterlockedCompareExchange128_rel;
1702 case clang::AArch64::BI_InterlockedOr8_acq:
1703 case clang::AArch64::BI_InterlockedOr16_acq:
1704 case clang::AArch64::BI_InterlockedOr_acq:
1705 case clang::AArch64::BI_InterlockedOr64_acq:
1706 return MSVCIntrin::_InterlockedOr_acq;
1707 case clang::AArch64::BI_InterlockedOr8_rel:
1708 case clang::AArch64::BI_InterlockedOr16_rel:
1709 case clang::AArch64::BI_InterlockedOr_rel:
1710 case clang::AArch64::BI_InterlockedOr64_rel:
1711 return MSVCIntrin::_InterlockedOr_rel;
1712 case clang::AArch64::BI_InterlockedOr8_nf:
1713 case clang::AArch64::BI_InterlockedOr16_nf:
1714 case clang::AArch64::BI_InterlockedOr_nf:
1715 case clang::AArch64::BI_InterlockedOr64_nf:
1716 return MSVCIntrin::_InterlockedOr_nf;
1717 case clang::AArch64::BI_InterlockedXor8_acq:
1718 case clang::AArch64::BI_InterlockedXor16_acq:
1719 case clang::AArch64::BI_InterlockedXor_acq:
1720 case clang::AArch64::BI_InterlockedXor64_acq:
1721 return MSVCIntrin::_InterlockedXor_acq;
1722 case clang::AArch64::BI_InterlockedXor8_rel:
1723 case clang::AArch64::BI_InterlockedXor16_rel:
1724 case clang::AArch64::BI_InterlockedXor_rel:
1725 case clang::AArch64::BI_InterlockedXor64_rel:
1726 return MSVCIntrin::_InterlockedXor_rel;
1727 case clang::AArch64::BI_InterlockedXor8_nf:
1728 case clang::AArch64::BI_InterlockedXor16_nf:
1729 case clang::AArch64::BI_InterlockedXor_nf:
1730 case clang::AArch64::BI_InterlockedXor64_nf:
1731 return MSVCIntrin::_InterlockedXor_nf;
1732 case clang::AArch64::BI_InterlockedAnd8_acq:
1733 case clang::AArch64::BI_InterlockedAnd16_acq:
1734 case clang::AArch64::BI_InterlockedAnd_acq:
1735 case clang::AArch64::BI_InterlockedAnd64_acq:
1736 return MSVCIntrin::_InterlockedAnd_acq;
1737 case clang::AArch64::BI_InterlockedAnd8_rel:
1738 case clang::AArch64::BI_InterlockedAnd16_rel:
1739 case clang::AArch64::BI_InterlockedAnd_rel:
1740 case clang::AArch64::BI_InterlockedAnd64_rel:
1741 return MSVCIntrin::_InterlockedAnd_rel;
1742 case clang::AArch64::BI_InterlockedAnd8_nf:
1743 case clang::AArch64::BI_InterlockedAnd16_nf:
1744 case clang::AArch64::BI_InterlockedAnd_nf:
1745 case clang::AArch64::BI_InterlockedAnd64_nf:
1746 return MSVCIntrin::_InterlockedAnd_nf;
1747 case clang::AArch64::BI_InterlockedIncrement16_acq:
1748 case clang::AArch64::BI_InterlockedIncrement_acq:
1749 case clang::AArch64::BI_InterlockedIncrement64_acq:
1750 return MSVCIntrin::_InterlockedIncrement_acq;
1751 case clang::AArch64::BI_InterlockedIncrement16_rel:
1752 case clang::AArch64::BI_InterlockedIncrement_rel:
1753 case clang::AArch64::BI_InterlockedIncrement64_rel:
1754 return MSVCIntrin::_InterlockedIncrement_rel;
1755 case clang::AArch64::BI_InterlockedIncrement16_nf:
1756 case clang::AArch64::BI_InterlockedIncrement_nf:
1757 case clang::AArch64::BI_InterlockedIncrement64_nf:
1758 return MSVCIntrin::_InterlockedIncrement_nf;
1759 case clang::AArch64::BI_InterlockedDecrement16_acq:
1760 case clang::AArch64::BI_InterlockedDecrement_acq:
1761 case clang::AArch64::BI_InterlockedDecrement64_acq:
1762 return MSVCIntrin::_InterlockedDecrement_acq;
1763 case clang::AArch64::BI_InterlockedDecrement16_rel:
1764 case clang::AArch64::BI_InterlockedDecrement_rel:
1765 case clang::AArch64::BI_InterlockedDecrement64_rel:
1766 return MSVCIntrin::_InterlockedDecrement_rel;
1767 case clang::AArch64::BI_InterlockedDecrement16_nf:
1768 case clang::AArch64::BI_InterlockedDecrement_nf:
1769 case clang::AArch64::BI_InterlockedDecrement64_nf:
1770 return MSVCIntrin::_InterlockedDecrement_nf;
1772 llvm_unreachable(
"must return from switch");
1775static std::optional<CodeGenFunction::MSVCIntrin>
1778 switch (BuiltinID) {
1780 return std::nullopt;
1781 case clang::X86::BI_BitScanForward:
1782 case clang::X86::BI_BitScanForward64:
1783 return MSVCIntrin::_BitScanForward;
1784 case clang::X86::BI_BitScanReverse:
1785 case clang::X86::BI_BitScanReverse64:
1786 return MSVCIntrin::_BitScanReverse;
1787 case clang::X86::BI_InterlockedAnd64:
1788 return MSVCIntrin::_InterlockedAnd;
1789 case clang::X86::BI_InterlockedCompareExchange128:
1790 return MSVCIntrin::_InterlockedCompareExchange128;
1791 case clang::X86::BI_InterlockedExchange64:
1792 return MSVCIntrin::_InterlockedExchange;
1793 case clang::X86::BI_InterlockedExchangeAdd64:
1794 return MSVCIntrin::_InterlockedExchangeAdd;
1795 case clang::X86::BI_InterlockedExchangeSub64:
1796 return MSVCIntrin::_InterlockedExchangeSub;
1797 case clang::X86::BI_InterlockedOr64:
1798 return MSVCIntrin::_InterlockedOr;
1799 case clang::X86::BI_InterlockedXor64:
1800 return MSVCIntrin::_InterlockedXor;
1801 case clang::X86::BI_InterlockedDecrement64:
1802 return MSVCIntrin::_InterlockedDecrement;
1803 case clang::X86::BI_InterlockedIncrement64:
1804 return MSVCIntrin::_InterlockedIncrement;
1806 llvm_unreachable(
"must return from switch");
1812 switch (BuiltinID) {
1813 case MSVCIntrin::_BitScanForward:
1814 case MSVCIntrin::_BitScanReverse: {
1818 llvm::Type *ArgType = ArgValue->
getType();
1819 llvm::Type *IndexType = IndexAddress.getElementType();
1822 Value *ArgZero = llvm::Constant::getNullValue(ArgType);
1823 Value *ResZero = llvm::Constant::getNullValue(ResultType);
1824 Value *ResOne = llvm::ConstantInt::get(ResultType, 1);
1829 PHINode *
Result =
Builder.CreatePHI(ResultType, 2,
"bitscan_result");
1832 Value *IsZero =
Builder.CreateICmpEQ(ArgValue, ArgZero);
1834 Builder.CreateCondBr(IsZero, End, NotZero);
1837 Builder.SetInsertPoint(NotZero);
1839 if (BuiltinID == MSVCIntrin::_BitScanForward) {
1842 ZeroCount =
Builder.CreateIntCast(ZeroCount, IndexType,
false);
1845 unsigned ArgWidth = cast<llvm::IntegerType>(ArgType)->getBitWidth();
1846 Value *ArgTypeLastIndex = llvm::ConstantInt::get(IndexType, ArgWidth - 1);
1850 ZeroCount =
Builder.CreateIntCast(ZeroCount, IndexType,
false);
1851 Value *Index =
Builder.CreateNSWSub(ArgTypeLastIndex, ZeroCount);
1855 Result->addIncoming(ResOne, NotZero);
1860 case MSVCIntrin::_InterlockedAnd:
1862 case MSVCIntrin::_InterlockedExchange:
1864 case MSVCIntrin::_InterlockedExchangeAdd:
1866 case MSVCIntrin::_InterlockedExchangeSub:
1868 case MSVCIntrin::_InterlockedOr:
1870 case MSVCIntrin::_InterlockedXor:
1872 case MSVCIntrin::_InterlockedExchangeAdd_acq:
1874 AtomicOrdering::Acquire);
1875 case MSVCIntrin::_InterlockedExchangeAdd_rel:
1877 AtomicOrdering::Release);
1878 case MSVCIntrin::_InterlockedExchangeAdd_nf:
1880 AtomicOrdering::Monotonic);
1881 case MSVCIntrin::_InterlockedExchange_acq:
1883 AtomicOrdering::Acquire);
1884 case MSVCIntrin::_InterlockedExchange_rel:
1886 AtomicOrdering::Release);
1887 case MSVCIntrin::_InterlockedExchange_nf:
1889 AtomicOrdering::Monotonic);
1890 case MSVCIntrin::_InterlockedCompareExchange_acq:
1892 case MSVCIntrin::_InterlockedCompareExchange_rel:
1894 case MSVCIntrin::_InterlockedCompareExchange_nf:
1896 case MSVCIntrin::_InterlockedCompareExchange128:
1898 *
this, E, AtomicOrdering::SequentiallyConsistent);
1899 case MSVCIntrin::_InterlockedCompareExchange128_acq:
1901 case MSVCIntrin::_InterlockedCompareExchange128_rel:
1903 case MSVCIntrin::_InterlockedCompareExchange128_nf:
1905 case MSVCIntrin::_InterlockedOr_acq:
1907 AtomicOrdering::Acquire);
1908 case MSVCIntrin::_InterlockedOr_rel:
1910 AtomicOrdering::Release);
1911 case MSVCIntrin::_InterlockedOr_nf:
1913 AtomicOrdering::Monotonic);
1914 case MSVCIntrin::_InterlockedXor_acq:
1916 AtomicOrdering::Acquire);
1917 case MSVCIntrin::_InterlockedXor_rel:
1919 AtomicOrdering::Release);
1920 case MSVCIntrin::_InterlockedXor_nf:
1922 AtomicOrdering::Monotonic);
1923 case MSVCIntrin::_InterlockedAnd_acq:
1925 AtomicOrdering::Acquire);
1926 case MSVCIntrin::_InterlockedAnd_rel:
1928 AtomicOrdering::Release);
1929 case MSVCIntrin::_InterlockedAnd_nf:
1931 AtomicOrdering::Monotonic);
1932 case MSVCIntrin::_InterlockedIncrement_acq:
1934 case MSVCIntrin::_InterlockedIncrement_rel:
1936 case MSVCIntrin::_InterlockedIncrement_nf:
1938 case MSVCIntrin::_InterlockedDecrement_acq:
1940 case MSVCIntrin::_InterlockedDecrement_rel:
1942 case MSVCIntrin::_InterlockedDecrement_nf:
1945 case MSVCIntrin::_InterlockedDecrement:
1947 case MSVCIntrin::_InterlockedIncrement:
1950 case MSVCIntrin::__fastfail: {
1955 StringRef
Asm, Constraints;
1960 case llvm::Triple::x86:
1961 case llvm::Triple::x86_64:
1963 Constraints =
"{cx}";
1965 case llvm::Triple::thumb:
1967 Constraints =
"{r0}";
1969 case llvm::Triple::aarch64:
1970 Asm =
"brk #0xF003";
1971 Constraints =
"{w0}";
1973 llvm::FunctionType *FTy = llvm::FunctionType::get(
VoidTy, {
Int32Ty},
false);
1974 llvm::InlineAsm *IA =
1975 llvm::InlineAsm::get(FTy,
Asm, Constraints,
true);
1976 llvm::AttributeList NoReturnAttr = llvm::AttributeList::get(
1978 llvm::Attribute::NoReturn);
1980 CI->setAttributes(NoReturnAttr);
1984 llvm_unreachable(
"Incorrect MSVC intrinsic!");
1990 CallObjCArcUse(llvm::Value *
object) : object(object) {}
1991 llvm::Value *object;
2000 BuiltinCheckKind Kind) {
2002 &&
"Unsupported builtin check kind");
2008 SanitizerScope SanScope(
this);
2010 ArgValue, llvm::Constant::getNullValue(ArgValue->
getType()));
2011 EmitCheck(std::make_pair(Cond, SanitizerKind::Builtin),
2012 SanitizerHandler::InvalidBuiltin,
2014 llvm::ConstantInt::get(
Builder.getInt8Ty(), Kind)},
2020 return CGF.
Builder.CreateBinaryIntrinsic(
2021 Intrinsic::abs, ArgValue,
2022 ConstantInt::get(CGF.
Builder.getInt1Ty(), HasNSW));
2026 bool SanitizeOverflow) {
2030 if (
const auto *VCI = dyn_cast<llvm::ConstantInt>(ArgValue)) {
2031 if (!VCI->isMinSignedValue())
2032 return EmitAbs(CGF, ArgValue,
true);
2035 CodeGenFunction::SanitizerScope SanScope(&CGF);
2037 Constant *Zero = Constant::getNullValue(ArgValue->
getType());
2038 Value *ResultAndOverflow = CGF.
Builder.CreateBinaryIntrinsic(
2039 Intrinsic::ssub_with_overflow, Zero, ArgValue);
2042 CGF.
Builder.CreateExtractValue(ResultAndOverflow, 1));
2045 if (SanitizeOverflow) {
2046 CGF.
EmitCheck({{NotOverflow, SanitizerKind::SignedIntegerOverflow}},
2047 SanitizerHandler::NegateOverflow,
2052 CGF.
EmitTrapCheck(NotOverflow, SanitizerHandler::SubOverflow);
2054 Value *CmpResult = CGF.
Builder.CreateICmpSLT(ArgValue, Zero,
"abscond");
2055 return CGF.
Builder.CreateSelect(CmpResult,
Result, ArgValue,
"abs");
2060 QualType UnsignedTy =
C.getIntTypeForBitwidth(Size * 8,
false);
2061 return C.getCanonicalType(UnsignedTy);
2071 raw_svector_ostream OS(Name);
2072 OS <<
"__os_log_helper";
2076 for (
const auto &Item : Layout.
Items)
2077 OS <<
"_" <<
int(Item.getSizeByte()) <<
"_"
2078 <<
int(Item.getDescriptorByte());
2081 if (llvm::Function *F =
CGM.
getModule().getFunction(Name))
2091 for (
unsigned int I = 0, E = Layout.
Items.size(); I < E; ++I) {
2092 char Size = Layout.
Items[I].getSizeByte();
2099 &Ctx.
Idents.
get(std::string(
"arg") + llvm::to_string(I)), ArgTy,
2101 ArgTys.emplace_back(ArgTy);
2112 llvm::Function *Fn = llvm::Function::Create(
2113 FuncTy, llvm::GlobalValue::LinkOnceODRLinkage, Name, &
CGM.
getModule());
2114 Fn->setVisibility(llvm::GlobalValue::HiddenVisibility);
2117 Fn->setDoesNotThrow();
2121 Fn->addFnAttr(llvm::Attribute::NoInline);
2139 for (
const auto &Item : Layout.
Items) {
2141 Builder.getInt8(Item.getDescriptorByte()),
2144 Builder.getInt8(Item.getSizeByte()),
2148 if (!
Size.getQuantity())
2166 "__builtin_os_log_format takes at least 2 arguments");
2177 for (
const auto &Item : Layout.
Items) {
2178 int Size = Item.getSizeByte();
2182 llvm::Value *ArgVal;
2186 for (
unsigned I = 0, E = Item.getMaskType().size(); I < E; ++I)
2187 Val |= ((
uint64_t)Item.getMaskType()[I]) << I * 8;
2188 ArgVal = llvm::Constant::getIntegerValue(
Int64Ty, llvm::APInt(64, Val));
2189 }
else if (
const Expr *TheExpr = Item.getExpr()) {
2195 auto LifetimeExtendObject = [&](
const Expr *E) {
2203 if (isa<CallExpr>(E) || isa<ObjCMessageExpr>(E))
2208 if (TheExpr->getType()->isObjCRetainableType() &&
2209 getLangOpts().ObjCAutoRefCount && LifetimeExtendObject(TheExpr)) {
2211 "Only scalar can be a ObjC retainable type");
2212 if (!isa<Constant>(ArgVal)) {
2226 pushCleanupAfterFullExpr<CallObjCArcUse>(Cleanup, ArgVal);
2230 ArgVal =
Builder.getInt32(Item.getConstValue().getQuantity());
2233 unsigned ArgValSize =
2237 ArgVal =
Builder.CreateBitOrPointerCast(ArgVal,
IntTy);
2253 unsigned BuiltinID, WidthAndSignedness Op1Info, WidthAndSignedness Op2Info,
2254 WidthAndSignedness ResultInfo) {
2255 return BuiltinID == Builtin::BI__builtin_mul_overflow &&
2256 Op1Info.Width == Op2Info.Width && Op2Info.Width == ResultInfo.Width &&
2257 !Op1Info.Signed && !Op2Info.Signed && ResultInfo.Signed;
2262 const clang::Expr *Op2, WidthAndSignedness Op2Info,
2264 WidthAndSignedness ResultInfo) {
2266 Builtin::BI__builtin_mul_overflow, Op1Info, Op2Info, ResultInfo) &&
2267 "Cannot specialize this multiply");
2272 llvm::Value *HasOverflow;
2274 CGF, llvm::Intrinsic::umul_with_overflow, V1, V2, HasOverflow);
2279 auto IntMax = llvm::APInt::getSignedMaxValue(ResultInfo.Width);
2280 llvm::Value *IntMaxValue = llvm::ConstantInt::get(
Result->getType(), IntMax);
2282 llvm::Value *IntMaxOverflow = CGF.
Builder.CreateICmpUGT(
Result, IntMaxValue);
2283 HasOverflow = CGF.
Builder.CreateOr(HasOverflow, IntMaxOverflow);
2295 WidthAndSignedness Op1Info,
2296 WidthAndSignedness Op2Info,
2297 WidthAndSignedness ResultInfo) {
2298 return BuiltinID == Builtin::BI__builtin_mul_overflow &&
2299 std::max(Op1Info.Width, Op2Info.Width) >= ResultInfo.Width &&
2300 Op1Info.Signed != Op2Info.Signed;
2307 WidthAndSignedness Op1Info,
const clang::Expr *Op2,
2308 WidthAndSignedness Op2Info,
2310 WidthAndSignedness ResultInfo) {
2312 Op2Info, ResultInfo) &&
2313 "Not a mixed-sign multipliction we can specialize");
2316 const clang::Expr *SignedOp = Op1Info.Signed ? Op1 : Op2;
2317 const clang::Expr *UnsignedOp = Op1Info.Signed ? Op2 : Op1;
2320 unsigned SignedOpWidth = Op1Info.Signed ? Op1Info.Width : Op2Info.Width;
2321 unsigned UnsignedOpWidth = Op1Info.Signed ? Op2Info.Width : Op1Info.Width;
2324 if (SignedOpWidth < UnsignedOpWidth)
2326 if (UnsignedOpWidth < SignedOpWidth)
2329 llvm::Type *OpTy =
Signed->getType();
2330 llvm::Value *Zero = llvm::Constant::getNullValue(OpTy);
2333 unsigned OpWidth = std::max(Op1Info.Width, Op2Info.Width);
2336 llvm::Value *IsNegative = CGF.
Builder.CreateICmpSLT(
Signed, Zero);
2337 llvm::Value *AbsOfNegative = CGF.
Builder.CreateSub(Zero,
Signed);
2338 llvm::Value *AbsSigned =
2339 CGF.
Builder.CreateSelect(IsNegative, AbsOfNegative,
Signed);
2342 llvm::Value *UnsignedOverflow;
2343 llvm::Value *UnsignedResult =
2347 llvm::Value *Overflow, *
Result;
2348 if (ResultInfo.Signed) {
2352 llvm::APInt::getSignedMaxValue(ResultInfo.Width).zext(OpWidth);
2353 llvm::Value *MaxResult =
2354 CGF.
Builder.CreateAdd(llvm::ConstantInt::get(OpTy, IntMax),
2355 CGF.
Builder.CreateZExt(IsNegative, OpTy));
2356 llvm::Value *SignedOverflow =
2357 CGF.
Builder.CreateICmpUGT(UnsignedResult, MaxResult);
2358 Overflow = CGF.
Builder.CreateOr(UnsignedOverflow, SignedOverflow);
2361 llvm::Value *NegativeResult = CGF.
Builder.CreateNeg(UnsignedResult);
2362 llvm::Value *SignedResult =
2363 CGF.
Builder.CreateSelect(IsNegative, NegativeResult, UnsignedResult);
2367 llvm::Value *Underflow = CGF.
Builder.CreateAnd(
2368 IsNegative, CGF.
Builder.CreateIsNotNull(UnsignedResult));
2369 Overflow = CGF.
Builder.CreateOr(UnsignedOverflow, Underflow);
2370 if (ResultInfo.Width < OpWidth) {
2372 llvm::APInt::getMaxValue(ResultInfo.Width).zext(OpWidth);
2373 llvm::Value *TruncOverflow = CGF.
Builder.CreateICmpUGT(
2374 UnsignedResult, llvm::ConstantInt::get(OpTy, IntMax));
2375 Overflow = CGF.
Builder.CreateOr(Overflow, TruncOverflow);
2380 IsNegative, CGF.
Builder.CreateNeg(UnsignedResult), UnsignedResult);
2384 assert(Overflow &&
Result &&
"Missing overflow or result");
2395 llvm::SmallPtrSetImpl<const Decl *> &Seen) {
2404 if (!Seen.insert(
Record).second)
2407 assert(
Record->hasDefinition() &&
2408 "Incomplete types should already be diagnosed");
2410 if (
Record->isDynamicClass())
2435 llvm::Type *Ty = Src->getType();
2436 ShiftAmt =
Builder.CreateIntCast(ShiftAmt, Ty,
false);
2439 unsigned IID = IsRotateRight ? Intrinsic::fshr : Intrinsic::fshl;
2446 switch (BuiltinID) {
2447#define MUTATE_LDBL(func) \
2448 case Builtin::BI__builtin_##func##l: \
2449 return Builtin::BI__builtin_##func##f128;
2518 if (CGF.
Builder.getIsFPConstrained() &&
2519 CGF.
Builder.getDefaultConstrainedExcept() != fp::ebIgnore) {
2531 auto UBF = CGF->
CGM.
getModule().getOrInsertFunction(Name, FnTy);
2534 for (
auto &&FormalTy : FnTy->params())
2535 Args.push_back(llvm::PoisonValue::get(FormalTy));
2548 !
Result.hasSideEffects()) {
2552 if (
Result.Val.isFloat())
2561 if (
getTarget().getTriple().isPPC64() &&
2562 &
getTarget().getLongDoubleFormat() == &llvm::APFloat::IEEEquad())
2569 const unsigned BuiltinIDIfNoAsmLabel =
2570 FD->
hasAttr<AsmLabelAttr>() ? 0 : BuiltinID;
2572 std::optional<bool> ErrnoOverriden;
2578 if (OP.hasMathErrnoOverride())
2579 ErrnoOverriden = OP.getMathErrnoOverride();
2588 bool ErrnoOverridenToFalseWithOpt =
2589 ErrnoOverriden.has_value() && !ErrnoOverriden.value() && !OptNone &&
2607 switch (BuiltinID) {
2608 case Builtin::BI__builtin_fma:
2609 case Builtin::BI__builtin_fmaf:
2610 case Builtin::BI__builtin_fmal:
2611 case Builtin::BIfma:
2612 case Builtin::BIfmaf:
2613 case Builtin::BIfmal: {
2615 if (Trip.isGNUEnvironment() || Trip.isOSMSVCRT())
2623 bool ConstWithoutErrnoAndExceptions =
2625 bool ConstWithoutExceptions =
2643 bool ConstWithoutErrnoOrExceptions =
2644 ConstWithoutErrnoAndExceptions || ConstWithoutExceptions;
2645 bool GenerateIntrinsics =
2646 (ConstAlways && !OptNone) ||
2648 !(ErrnoOverriden.has_value() && ErrnoOverriden.value()) && !OptNone);
2649 if (!GenerateIntrinsics) {
2650 GenerateIntrinsics =
2651 ConstWithoutErrnoOrExceptions && !ConstWithoutErrnoAndExceptions;
2652 if (!GenerateIntrinsics)
2653 GenerateIntrinsics =
2654 ConstWithoutErrnoOrExceptions &&
2656 !(ErrnoOverriden.has_value() && ErrnoOverriden.value()) && !OptNone);
2657 if (!GenerateIntrinsics)
2658 GenerateIntrinsics =
2659 ConstWithoutErrnoOrExceptions && ErrnoOverridenToFalseWithOpt;
2661 if (GenerateIntrinsics) {
2662 switch (BuiltinIDIfNoAsmLabel) {
2663 case Builtin::BIceil:
2664 case Builtin::BIceilf:
2665 case Builtin::BIceill:
2666 case Builtin::BI__builtin_ceil:
2667 case Builtin::BI__builtin_ceilf:
2668 case Builtin::BI__builtin_ceilf16:
2669 case Builtin::BI__builtin_ceill:
2670 case Builtin::BI__builtin_ceilf128:
2673 Intrinsic::experimental_constrained_ceil));
2675 case Builtin::BIcopysign:
2676 case Builtin::BIcopysignf:
2677 case Builtin::BIcopysignl:
2678 case Builtin::BI__builtin_copysign:
2679 case Builtin::BI__builtin_copysignf:
2680 case Builtin::BI__builtin_copysignf16:
2681 case Builtin::BI__builtin_copysignl:
2682 case Builtin::BI__builtin_copysignf128:
2685 case Builtin::BIcos:
2686 case Builtin::BIcosf:
2687 case Builtin::BIcosl:
2688 case Builtin::BI__builtin_cos:
2689 case Builtin::BI__builtin_cosf:
2690 case Builtin::BI__builtin_cosf16:
2691 case Builtin::BI__builtin_cosl:
2692 case Builtin::BI__builtin_cosf128:
2695 Intrinsic::experimental_constrained_cos));
2697 case Builtin::BIexp:
2698 case Builtin::BIexpf:
2699 case Builtin::BIexpl:
2700 case Builtin::BI__builtin_exp:
2701 case Builtin::BI__builtin_expf:
2702 case Builtin::BI__builtin_expf16:
2703 case Builtin::BI__builtin_expl:
2704 case Builtin::BI__builtin_expf128:
2707 Intrinsic::experimental_constrained_exp));
2709 case Builtin::BIexp2:
2710 case Builtin::BIexp2f:
2711 case Builtin::BIexp2l:
2712 case Builtin::BI__builtin_exp2:
2713 case Builtin::BI__builtin_exp2f:
2714 case Builtin::BI__builtin_exp2f16:
2715 case Builtin::BI__builtin_exp2l:
2716 case Builtin::BI__builtin_exp2f128:
2719 Intrinsic::experimental_constrained_exp2));
2720 case Builtin::BI__builtin_exp10:
2721 case Builtin::BI__builtin_exp10f:
2722 case Builtin::BI__builtin_exp10f16:
2723 case Builtin::BI__builtin_exp10l:
2724 case Builtin::BI__builtin_exp10f128: {
2726 if (
Builder.getIsFPConstrained())
2730 case Builtin::BIfabs:
2731 case Builtin::BIfabsf:
2732 case Builtin::BIfabsl:
2733 case Builtin::BI__builtin_fabs:
2734 case Builtin::BI__builtin_fabsf:
2735 case Builtin::BI__builtin_fabsf16:
2736 case Builtin::BI__builtin_fabsl:
2737 case Builtin::BI__builtin_fabsf128:
2740 case Builtin::BIfloor:
2741 case Builtin::BIfloorf:
2742 case Builtin::BIfloorl:
2743 case Builtin::BI__builtin_floor:
2744 case Builtin::BI__builtin_floorf:
2745 case Builtin::BI__builtin_floorf16:
2746 case Builtin::BI__builtin_floorl:
2747 case Builtin::BI__builtin_floorf128:
2750 Intrinsic::experimental_constrained_floor));
2752 case Builtin::BIfma:
2753 case Builtin::BIfmaf:
2754 case Builtin::BIfmal:
2755 case Builtin::BI__builtin_fma:
2756 case Builtin::BI__builtin_fmaf:
2757 case Builtin::BI__builtin_fmaf16:
2758 case Builtin::BI__builtin_fmal:
2759 case Builtin::BI__builtin_fmaf128:
2762 Intrinsic::experimental_constrained_fma));
2764 case Builtin::BIfmax:
2765 case Builtin::BIfmaxf:
2766 case Builtin::BIfmaxl:
2767 case Builtin::BI__builtin_fmax:
2768 case Builtin::BI__builtin_fmaxf:
2769 case Builtin::BI__builtin_fmaxf16:
2770 case Builtin::BI__builtin_fmaxl:
2771 case Builtin::BI__builtin_fmaxf128:
2774 Intrinsic::experimental_constrained_maxnum));
2776 case Builtin::BIfmin:
2777 case Builtin::BIfminf:
2778 case Builtin::BIfminl:
2779 case Builtin::BI__builtin_fmin:
2780 case Builtin::BI__builtin_fminf:
2781 case Builtin::BI__builtin_fminf16:
2782 case Builtin::BI__builtin_fminl:
2783 case Builtin::BI__builtin_fminf128:
2786 Intrinsic::experimental_constrained_minnum));
2790 case Builtin::BIfmod:
2791 case Builtin::BIfmodf:
2792 case Builtin::BIfmodl:
2793 case Builtin::BI__builtin_fmod:
2794 case Builtin::BI__builtin_fmodf:
2795 case Builtin::BI__builtin_fmodf16:
2796 case Builtin::BI__builtin_fmodl:
2797 case Builtin::BI__builtin_fmodf128: {
2798 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
2804 case Builtin::BIlog:
2805 case Builtin::BIlogf:
2806 case Builtin::BIlogl:
2807 case Builtin::BI__builtin_log:
2808 case Builtin::BI__builtin_logf:
2809 case Builtin::BI__builtin_logf16:
2810 case Builtin::BI__builtin_logl:
2811 case Builtin::BI__builtin_logf128:
2814 Intrinsic::experimental_constrained_log));
2816 case Builtin::BIlog10:
2817 case Builtin::BIlog10f:
2818 case Builtin::BIlog10l:
2819 case Builtin::BI__builtin_log10:
2820 case Builtin::BI__builtin_log10f:
2821 case Builtin::BI__builtin_log10f16:
2822 case Builtin::BI__builtin_log10l:
2823 case Builtin::BI__builtin_log10f128:
2826 Intrinsic::experimental_constrained_log10));
2828 case Builtin::BIlog2:
2829 case Builtin::BIlog2f:
2830 case Builtin::BIlog2l:
2831 case Builtin::BI__builtin_log2:
2832 case Builtin::BI__builtin_log2f:
2833 case Builtin::BI__builtin_log2f16:
2834 case Builtin::BI__builtin_log2l:
2835 case Builtin::BI__builtin_log2f128:
2838 Intrinsic::experimental_constrained_log2));
2840 case Builtin::BInearbyint:
2841 case Builtin::BInearbyintf:
2842 case Builtin::BInearbyintl:
2843 case Builtin::BI__builtin_nearbyint:
2844 case Builtin::BI__builtin_nearbyintf:
2845 case Builtin::BI__builtin_nearbyintl:
2846 case Builtin::BI__builtin_nearbyintf128:
2848 Intrinsic::nearbyint,
2849 Intrinsic::experimental_constrained_nearbyint));
2851 case Builtin::BIpow:
2852 case Builtin::BIpowf:
2853 case Builtin::BIpowl:
2854 case Builtin::BI__builtin_pow:
2855 case Builtin::BI__builtin_powf:
2856 case Builtin::BI__builtin_powf16:
2857 case Builtin::BI__builtin_powl:
2858 case Builtin::BI__builtin_powf128:
2861 Intrinsic::experimental_constrained_pow));
2863 case Builtin::BIrint:
2864 case Builtin::BIrintf:
2865 case Builtin::BIrintl:
2866 case Builtin::BI__builtin_rint:
2867 case Builtin::BI__builtin_rintf:
2868 case Builtin::BI__builtin_rintf16:
2869 case Builtin::BI__builtin_rintl:
2870 case Builtin::BI__builtin_rintf128:
2873 Intrinsic::experimental_constrained_rint));
2875 case Builtin::BIround:
2876 case Builtin::BIroundf:
2877 case Builtin::BIroundl:
2878 case Builtin::BI__builtin_round:
2879 case Builtin::BI__builtin_roundf:
2880 case Builtin::BI__builtin_roundf16:
2881 case Builtin::BI__builtin_roundl:
2882 case Builtin::BI__builtin_roundf128:
2885 Intrinsic::experimental_constrained_round));
2887 case Builtin::BIroundeven:
2888 case Builtin::BIroundevenf:
2889 case Builtin::BIroundevenl:
2890 case Builtin::BI__builtin_roundeven:
2891 case Builtin::BI__builtin_roundevenf:
2892 case Builtin::BI__builtin_roundevenf16:
2893 case Builtin::BI__builtin_roundevenl:
2894 case Builtin::BI__builtin_roundevenf128:
2896 Intrinsic::roundeven,
2897 Intrinsic::experimental_constrained_roundeven));
2899 case Builtin::BIsin:
2900 case Builtin::BIsinf:
2901 case Builtin::BIsinl:
2902 case Builtin::BI__builtin_sin:
2903 case Builtin::BI__builtin_sinf:
2904 case Builtin::BI__builtin_sinf16:
2905 case Builtin::BI__builtin_sinl:
2906 case Builtin::BI__builtin_sinf128:
2909 Intrinsic::experimental_constrained_sin));
2911 case Builtin::BIsqrt:
2912 case Builtin::BIsqrtf:
2913 case Builtin::BIsqrtl:
2914 case Builtin::BI__builtin_sqrt:
2915 case Builtin::BI__builtin_sqrtf:
2916 case Builtin::BI__builtin_sqrtf16:
2917 case Builtin::BI__builtin_sqrtl:
2918 case Builtin::BI__builtin_sqrtf128:
2919 case Builtin::BI__builtin_elementwise_sqrt: {
2921 *
this, E, Intrinsic::sqrt, Intrinsic::experimental_constrained_sqrt);
2925 case Builtin::BItrunc:
2926 case Builtin::BItruncf:
2927 case Builtin::BItruncl:
2928 case Builtin::BI__builtin_trunc:
2929 case Builtin::BI__builtin_truncf:
2930 case Builtin::BI__builtin_truncf16:
2931 case Builtin::BI__builtin_truncl:
2932 case Builtin::BI__builtin_truncf128:
2935 Intrinsic::experimental_constrained_trunc));
2937 case Builtin::BIlround:
2938 case Builtin::BIlroundf:
2939 case Builtin::BIlroundl:
2940 case Builtin::BI__builtin_lround:
2941 case Builtin::BI__builtin_lroundf:
2942 case Builtin::BI__builtin_lroundl:
2943 case Builtin::BI__builtin_lroundf128:
2945 *
this, E, Intrinsic::lround,
2946 Intrinsic::experimental_constrained_lround));
2948 case Builtin::BIllround:
2949 case Builtin::BIllroundf:
2950 case Builtin::BIllroundl:
2951 case Builtin::BI__builtin_llround:
2952 case Builtin::BI__builtin_llroundf:
2953 case Builtin::BI__builtin_llroundl:
2954 case Builtin::BI__builtin_llroundf128:
2956 *
this, E, Intrinsic::llround,
2957 Intrinsic::experimental_constrained_llround));
2959 case Builtin::BIlrint:
2960 case Builtin::BIlrintf:
2961 case Builtin::BIlrintl:
2962 case Builtin::BI__builtin_lrint:
2963 case Builtin::BI__builtin_lrintf:
2964 case Builtin::BI__builtin_lrintl:
2965 case Builtin::BI__builtin_lrintf128:
2967 *
this, E, Intrinsic::lrint,
2968 Intrinsic::experimental_constrained_lrint));
2970 case Builtin::BIllrint:
2971 case Builtin::BIllrintf:
2972 case Builtin::BIllrintl:
2973 case Builtin::BI__builtin_llrint:
2974 case Builtin::BI__builtin_llrintf:
2975 case Builtin::BI__builtin_llrintl:
2976 case Builtin::BI__builtin_llrintf128:
2978 *
this, E, Intrinsic::llrint,
2979 Intrinsic::experimental_constrained_llrint));
2980 case Builtin::BI__builtin_ldexp:
2981 case Builtin::BI__builtin_ldexpf:
2982 case Builtin::BI__builtin_ldexpl:
2983 case Builtin::BI__builtin_ldexpf16:
2984 case Builtin::BI__builtin_ldexpf128: {
2986 *
this, E, Intrinsic::ldexp,
2987 Intrinsic::experimental_constrained_ldexp));
2997 Value *Val = A.emitRawPointer(*
this);
3003 SkippedChecks.
set(SanitizerKind::All);
3004 SkippedChecks.
clear(SanitizerKind::Alignment);
3007 if (
auto *CE = dyn_cast<ImplicitCastExpr>(Arg))
3008 if (CE->getCastKind() == CK_BitCast)
3009 Arg = CE->getSubExpr();
3015 switch (BuiltinIDIfNoAsmLabel) {
3017 case Builtin::BI__builtin___CFStringMakeConstantString:
3018 case Builtin::BI__builtin___NSStringMakeConstantString:
3020 case Builtin::BI__builtin_stdarg_start:
3021 case Builtin::BI__builtin_va_start:
3022 case Builtin::BI__va_start:
3023 case Builtin::BI__builtin_va_end:
3027 BuiltinID != Builtin::BI__builtin_va_end);
3029 case Builtin::BI__builtin_va_copy: {
3036 case Builtin::BIabs:
3037 case Builtin::BIlabs:
3038 case Builtin::BIllabs:
3039 case Builtin::BI__builtin_abs:
3040 case Builtin::BI__builtin_labs:
3041 case Builtin::BI__builtin_llabs: {
3042 bool SanitizeOverflow =
SanOpts.
has(SanitizerKind::SignedIntegerOverflow);
3045 switch (
getLangOpts().getSignedOverflowBehavior()) {
3050 if (!SanitizeOverflow) {
3062 case Builtin::BI__builtin_complex: {
3067 case Builtin::BI__builtin_conj:
3068 case Builtin::BI__builtin_conjf:
3069 case Builtin::BI__builtin_conjl:
3070 case Builtin::BIconj:
3071 case Builtin::BIconjf:
3072 case Builtin::BIconjl: {
3074 Value *Real = ComplexVal.first;
3075 Value *Imag = ComplexVal.second;
3076 Imag =
Builder.CreateFNeg(Imag,
"neg");
3079 case Builtin::BI__builtin_creal:
3080 case Builtin::BI__builtin_crealf:
3081 case Builtin::BI__builtin_creall:
3082 case Builtin::BIcreal:
3083 case Builtin::BIcrealf:
3084 case Builtin::BIcreall: {
3089 case Builtin::BI__builtin_preserve_access_index: {
3110 case Builtin::BI__builtin_cimag:
3111 case Builtin::BI__builtin_cimagf:
3112 case Builtin::BI__builtin_cimagl:
3113 case Builtin::BIcimag:
3114 case Builtin::BIcimagf:
3115 case Builtin::BIcimagl: {
3120 case Builtin::BI__builtin_clrsb:
3121 case Builtin::BI__builtin_clrsbl:
3122 case Builtin::BI__builtin_clrsbll: {
3126 llvm::Type *ArgType = ArgValue->
getType();
3130 Value *
Zero = llvm::Constant::getNullValue(ArgType);
3131 Value *IsNeg =
Builder.CreateICmpSLT(ArgValue, Zero,
"isneg");
3133 Value *Tmp =
Builder.CreateSelect(IsNeg, Inverse, ArgValue);
3140 case Builtin::BI__builtin_ctzs:
3141 case Builtin::BI__builtin_ctz:
3142 case Builtin::BI__builtin_ctzl:
3143 case Builtin::BI__builtin_ctzll:
3144 case Builtin::BI__builtin_ctzg: {
3145 bool HasFallback = BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_ctzg &&
3152 llvm::Type *ArgType = ArgValue->
getType();
3159 if (
Result->getType() != ResultType)
3165 Value *
Zero = Constant::getNullValue(ArgType);
3166 Value *IsZero =
Builder.CreateICmpEQ(ArgValue, Zero,
"iszero");
3168 Value *ResultOrFallback =
3169 Builder.CreateSelect(IsZero, FallbackValue,
Result,
"ctzg");
3172 case Builtin::BI__builtin_clzs:
3173 case Builtin::BI__builtin_clz:
3174 case Builtin::BI__builtin_clzl:
3175 case Builtin::BI__builtin_clzll:
3176 case Builtin::BI__builtin_clzg: {
3177 bool HasFallback = BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_clzg &&
3184 llvm::Type *ArgType = ArgValue->
getType();
3191 if (
Result->getType() != ResultType)
3197 Value *
Zero = Constant::getNullValue(ArgType);
3198 Value *IsZero =
Builder.CreateICmpEQ(ArgValue, Zero,
"iszero");
3200 Value *ResultOrFallback =
3201 Builder.CreateSelect(IsZero, FallbackValue,
Result,
"clzg");
3204 case Builtin::BI__builtin_ffs:
3205 case Builtin::BI__builtin_ffsl:
3206 case Builtin::BI__builtin_ffsll: {
3210 llvm::Type *ArgType = ArgValue->
getType();
3215 Builder.CreateAdd(
Builder.CreateCall(F, {ArgValue, Builder.getTrue()}),
3216 llvm::ConstantInt::get(ArgType, 1));
3217 Value *
Zero = llvm::Constant::getNullValue(ArgType);
3218 Value *IsZero =
Builder.CreateICmpEQ(ArgValue, Zero,
"iszero");
3220 if (
Result->getType() != ResultType)
3225 case Builtin::BI__builtin_parity:
3226 case Builtin::BI__builtin_parityl:
3227 case Builtin::BI__builtin_parityll: {
3231 llvm::Type *ArgType = ArgValue->
getType();
3237 if (
Result->getType() != ResultType)
3242 case Builtin::BI__lzcnt16:
3243 case Builtin::BI__lzcnt:
3244 case Builtin::BI__lzcnt64: {
3247 llvm::Type *ArgType = ArgValue->
getType();
3252 if (
Result->getType() != ResultType)
3257 case Builtin::BI__popcnt16:
3258 case Builtin::BI__popcnt:
3259 case Builtin::BI__popcnt64:
3260 case Builtin::BI__builtin_popcount:
3261 case Builtin::BI__builtin_popcountl:
3262 case Builtin::BI__builtin_popcountll:
3263 case Builtin::BI__builtin_popcountg: {
3266 llvm::Type *ArgType = ArgValue->
getType();
3271 if (
Result->getType() != ResultType)
3276 case Builtin::BI__builtin_unpredictable: {
3282 case Builtin::BI__builtin_expect: {
3284 llvm::Type *ArgType = ArgValue->
getType();
3295 Builder.CreateCall(FnExpect, {ArgValue, ExpectedValue},
"expval");
3298 case Builtin::BI__builtin_expect_with_probability: {
3300 llvm::Type *ArgType = ArgValue->
getType();
3303 llvm::APFloat Probability(0.0);
3306 assert(EvalSucceed &&
"probability should be able to evaluate as float");
3308 bool LoseInfo =
false;
3309 Probability.convert(llvm::APFloat::IEEEdouble(),
3310 llvm::RoundingMode::Dynamic, &LoseInfo);
3312 Constant *Confidence = ConstantFP::get(Ty, Probability);
3322 FnExpect, {ArgValue, ExpectedValue, Confidence},
"expval");
3325 case Builtin::BI__builtin_assume_aligned: {
3328 Value *OffsetValue =
3332 ConstantInt *AlignmentCI = cast<ConstantInt>(AlignmentValue);
3333 if (AlignmentCI->getValue().ugt(llvm::Value::MaximumAlignment))
3334 AlignmentCI = ConstantInt::get(AlignmentCI->getIntegerType(),
3335 llvm::Value::MaximumAlignment);
3339 AlignmentCI, OffsetValue);
3342 case Builtin::BI__assume:
3343 case Builtin::BI__builtin_assume: {
3349 Builder.CreateCall(FnAssume, ArgValue);
3352 case Builtin::BI__builtin_assume_separate_storage: {
3359 Value *Values[] = {Value0, Value1};
3360 OperandBundleDefT<Value *> OBD(
"separate_storage", Values);
3364 case Builtin::BI__builtin_allow_runtime_check: {
3368 llvm::Value *Allow =
Builder.CreateCall(
3370 llvm::MetadataAsValue::get(Ctx, llvm::MDString::get(Ctx, Kind)));
3373 case Builtin::BI__arithmetic_fence: {
3376 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
3377 llvm::FastMathFlags FMF =
Builder.getFastMathFlags();
3378 bool isArithmeticFenceEnabled =
3379 FMF.allowReassoc() &&
3383 if (isArithmeticFenceEnabled) {
3386 Value *Real =
Builder.CreateArithmeticFence(ComplexVal.first,
3388 Value *Imag =
Builder.CreateArithmeticFence(ComplexVal.second,
3393 Value *Real = ComplexVal.first;
3394 Value *Imag = ComplexVal.second;
3398 if (isArithmeticFenceEnabled)
3403 case Builtin::BI__builtin_bswap16:
3404 case Builtin::BI__builtin_bswap32:
3405 case Builtin::BI__builtin_bswap64:
3406 case Builtin::BI_byteswap_ushort:
3407 case Builtin::BI_byteswap_ulong:
3408 case Builtin::BI_byteswap_uint64: {
3411 case Builtin::BI__builtin_bitreverse8:
3412 case Builtin::BI__builtin_bitreverse16:
3413 case Builtin::BI__builtin_bitreverse32:
3414 case Builtin::BI__builtin_bitreverse64: {
3417 case Builtin::BI__builtin_rotateleft8:
3418 case Builtin::BI__builtin_rotateleft16:
3419 case Builtin::BI__builtin_rotateleft32:
3420 case Builtin::BI__builtin_rotateleft64:
3421 case Builtin::BI_rotl8:
3422 case Builtin::BI_rotl16:
3423 case Builtin::BI_rotl:
3424 case Builtin::BI_lrotl:
3425 case Builtin::BI_rotl64:
3428 case Builtin::BI__builtin_rotateright8:
3429 case Builtin::BI__builtin_rotateright16:
3430 case Builtin::BI__builtin_rotateright32:
3431 case Builtin::BI__builtin_rotateright64:
3432 case Builtin::BI_rotr8:
3433 case Builtin::BI_rotr16:
3434 case Builtin::BI_rotr:
3435 case Builtin::BI_lrotr:
3436 case Builtin::BI_rotr64:
3439 case Builtin::BI__builtin_constant_p: {
3450 return RValue::get(ConstantInt::get(ResultType, 0));
3455 return RValue::get(ConstantInt::get(ResultType, 0));
3467 if (
Result->getType() != ResultType)
3471 case Builtin::BI__builtin_dynamic_object_size:
3472 case Builtin::BI__builtin_object_size: {
3479 bool IsDynamic = BuiltinID == Builtin::BI__builtin_dynamic_object_size;
3481 nullptr, IsDynamic));
3483 case Builtin::BI__builtin_prefetch: {
3487 llvm::ConstantInt::get(
Int32Ty, 0);
3489 llvm::ConstantInt::get(
Int32Ty, 3);
3495 case Builtin::BI__builtin_readcyclecounter: {
3499 case Builtin::BI__builtin_readsteadycounter: {
3503 case Builtin::BI__builtin___clear_cache: {
3509 case Builtin::BI__builtin_trap:
3512 case Builtin::BI__debugbreak:
3515 case Builtin::BI__builtin_unreachable: {
3524 case Builtin::BI__builtin_powi:
3525 case Builtin::BI__builtin_powif:
3526 case Builtin::BI__builtin_powil: {
3530 if (
Builder.getIsFPConstrained()) {
3533 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
3540 { Src0->getType(), Src1->getType() });
3543 case Builtin::BI__builtin_frexpl: {
3547 if (&
getTarget().getLongDoubleFormat() == &llvm::APFloat::PPCDoubleDouble())
3551 case Builtin::BI__builtin_frexp:
3552 case Builtin::BI__builtin_frexpf:
3553 case Builtin::BI__builtin_frexpf128:
3554 case Builtin::BI__builtin_frexpf16:
3556 case Builtin::BI__builtin_isgreater:
3557 case Builtin::BI__builtin_isgreaterequal:
3558 case Builtin::BI__builtin_isless:
3559 case Builtin::BI__builtin_islessequal:
3560 case Builtin::BI__builtin_islessgreater:
3561 case Builtin::BI__builtin_isunordered: {
3564 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
3568 switch (BuiltinID) {
3569 default: llvm_unreachable(
"Unknown ordered comparison");
3570 case Builtin::BI__builtin_isgreater:
3571 LHS =
Builder.CreateFCmpOGT(LHS, RHS,
"cmp");
3573 case Builtin::BI__builtin_isgreaterequal:
3574 LHS =
Builder.CreateFCmpOGE(LHS, RHS,
"cmp");
3576 case Builtin::BI__builtin_isless:
3577 LHS =
Builder.CreateFCmpOLT(LHS, RHS,
"cmp");
3579 case Builtin::BI__builtin_islessequal:
3580 LHS =
Builder.CreateFCmpOLE(LHS, RHS,
"cmp");
3582 case Builtin::BI__builtin_islessgreater:
3583 LHS =
Builder.CreateFCmpONE(LHS, RHS,
"cmp");
3585 case Builtin::BI__builtin_isunordered:
3586 LHS =
Builder.CreateFCmpUNO(LHS, RHS,
"cmp");
3593 case Builtin::BI__builtin_isnan: {
3594 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
3603 case Builtin::BI__builtin_issignaling: {
3604 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
3611 case Builtin::BI__builtin_isinf: {
3612 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
3621 case Builtin::BIfinite:
3622 case Builtin::BI__finite:
3623 case Builtin::BIfinitef:
3624 case Builtin::BI__finitef:
3625 case Builtin::BIfinitel:
3626 case Builtin::BI__finitel:
3627 case Builtin::BI__builtin_isfinite: {
3628 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
3637 case Builtin::BI__builtin_isnormal: {
3638 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
3645 case Builtin::BI__builtin_issubnormal: {
3646 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
3649 Builder.CreateZExt(
Builder.createIsFPClass(
V, FPClassTest::fcSubnormal),
3653 case Builtin::BI__builtin_iszero: {
3654 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
3661 case Builtin::BI__builtin_isfpclass: {
3666 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
3672 case Builtin::BI__builtin_nondeterministic_value: {
3681 case Builtin::BI__builtin_elementwise_abs: {
3686 QT = VecTy->getElementType();
3690 Builder.getFalse(),
nullptr,
"elt.abs");
3697 case Builtin::BI__builtin_elementwise_ceil:
3700 case Builtin::BI__builtin_elementwise_exp:
3703 case Builtin::BI__builtin_elementwise_exp2:
3706 case Builtin::BI__builtin_elementwise_log:
3709 case Builtin::BI__builtin_elementwise_log2:
3712 case Builtin::BI__builtin_elementwise_log10:
3715 case Builtin::BI__builtin_elementwise_pow: {
3718 case Builtin::BI__builtin_elementwise_bitreverse:
3721 case Builtin::BI__builtin_elementwise_cos:
3724 case Builtin::BI__builtin_elementwise_floor:
3727 case Builtin::BI__builtin_elementwise_roundeven:
3730 case Builtin::BI__builtin_elementwise_round:
3733 case Builtin::BI__builtin_elementwise_rint:
3736 case Builtin::BI__builtin_elementwise_nearbyint:
3739 case Builtin::BI__builtin_elementwise_sin:
3742 case Builtin::BI__builtin_elementwise_tan:
3745 case Builtin::BI__builtin_elementwise_trunc:
3748 case Builtin::BI__builtin_elementwise_canonicalize:
3750 emitUnaryBuiltin(*
this, E, llvm::Intrinsic::canonicalize,
"elt.canonicalize"));
3751 case Builtin::BI__builtin_elementwise_copysign:
3753 case Builtin::BI__builtin_elementwise_fma:
3755 case Builtin::BI__builtin_elementwise_add_sat:
3756 case Builtin::BI__builtin_elementwise_sub_sat: {
3760 assert(Op0->
getType()->isIntOrIntVectorTy() &&
"integer type expected");
3763 Ty = VecTy->getElementType();
3766 if (BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_elementwise_add_sat)
3767 Opc = IsSigned ? llvm::Intrinsic::sadd_sat : llvm::Intrinsic::uadd_sat;
3769 Opc = IsSigned ? llvm::Intrinsic::ssub_sat : llvm::Intrinsic::usub_sat;
3770 Result =
Builder.CreateBinaryIntrinsic(Opc, Op0, Op1,
nullptr,
"elt.sat");
3774 case Builtin::BI__builtin_elementwise_max: {
3778 if (Op0->
getType()->isIntOrIntVectorTy()) {
3781 Ty = VecTy->getElementType();
3783 ? llvm::Intrinsic::smax
3784 : llvm::Intrinsic::umax,
3785 Op0, Op1,
nullptr,
"elt.max");
3790 case Builtin::BI__builtin_elementwise_min: {
3794 if (Op0->
getType()->isIntOrIntVectorTy()) {
3797 Ty = VecTy->getElementType();
3799 ? llvm::Intrinsic::smin
3800 : llvm::Intrinsic::umin,
3801 Op0, Op1,
nullptr,
"elt.min");
3807 case Builtin::BI__builtin_reduce_max: {
3808 auto GetIntrinsicID = [
this](
QualType QT) {
3810 QT = VecTy->getElementType();
3815 return llvm::Intrinsic::vector_reduce_smax;
3817 return llvm::Intrinsic::vector_reduce_umax;
3819 return llvm::Intrinsic::vector_reduce_fmax;
3822 *
this, E, GetIntrinsicID(E->
getArg(0)->
getType()),
"rdx.min"));
3825 case Builtin::BI__builtin_reduce_min: {
3826 auto GetIntrinsicID = [
this](
QualType QT) {
3828 QT = VecTy->getElementType();
3833 return llvm::Intrinsic::vector_reduce_smin;
3835 return llvm::Intrinsic::vector_reduce_umin;
3837 return llvm::Intrinsic::vector_reduce_fmin;
3841 *
this, E, GetIntrinsicID(E->
getArg(0)->
getType()),
"rdx.min"));
3844 case Builtin::BI__builtin_reduce_add:
3846 *
this, E, llvm::Intrinsic::vector_reduce_add,
"rdx.add"));
3847 case Builtin::BI__builtin_reduce_mul:
3849 *
this, E, llvm::Intrinsic::vector_reduce_mul,
"rdx.mul"));
3850 case Builtin::BI__builtin_reduce_xor:
3852 *
this, E, llvm::Intrinsic::vector_reduce_xor,
"rdx.xor"));
3853 case Builtin::BI__builtin_reduce_or:
3855 *
this, E, llvm::Intrinsic::vector_reduce_or,
"rdx.or"));
3856 case Builtin::BI__builtin_reduce_and:
3858 *
this, E, llvm::Intrinsic::vector_reduce_and,
"rdx.and"));
3860 case Builtin::BI__builtin_matrix_transpose: {
3864 Value *
Result = MB.CreateMatrixTranspose(MatValue, MatrixTy->getNumRows(),
3865 MatrixTy->getNumColumns());
3869 case Builtin::BI__builtin_matrix_column_major_load: {
3875 assert(PtrTy &&
"arg0 must be of pointer type");
3885 ResultTy->getNumRows(), ResultTy->getNumColumns(),
"matrix");
3889 case Builtin::BI__builtin_matrix_column_major_store: {
3897 assert(PtrTy &&
"arg1 must be of pointer type");
3906 MatrixTy->getNumRows(), MatrixTy->getNumColumns());
3910 case Builtin::BI__builtin_isinf_sign: {
3912 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
3917 AbsArg, ConstantFP::getInfinity(Arg->
getType()),
"isinf");
3923 Value *NegativeOne = ConstantInt::get(
IntTy, -1);
3924 Value *SignResult =
Builder.CreateSelect(IsNeg, NegativeOne, One);
3929 case Builtin::BI__builtin_flt_rounds: {
3934 if (
Result->getType() != ResultType)
3940 case Builtin::BI__builtin_set_flt_rounds: {
3948 case Builtin::BI__builtin_fpclassify: {
3949 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
3960 "fpclassify_result");
3964 Value *IsZero =
Builder.CreateFCmpOEQ(
V, Constant::getNullValue(Ty),
3968 Builder.CreateCondBr(IsZero, End, NotZero);
3972 Builder.SetInsertPoint(NotZero);
3976 Builder.CreateCondBr(IsNan, End, NotNan);
3977 Result->addIncoming(NanLiteral, NotZero);
3980 Builder.SetInsertPoint(NotNan);
3983 Builder.CreateFCmpOEQ(VAbs, ConstantFP::getInfinity(
V->getType()),
3987 Builder.CreateCondBr(IsInf, End, NotInf);
3988 Result->addIncoming(InfLiteral, NotNan);
3991 Builder.SetInsertPoint(NotInf);
3992 APFloat Smallest = APFloat::getSmallestNormalized(
3995 Builder.CreateFCmpUGE(VAbs, ConstantFP::get(
V->getContext(), Smallest),
3997 Value *NormalResult =
4001 Result->addIncoming(NormalResult, NotInf);
4014 case Builtin::BIalloca:
4015 case Builtin::BI_alloca:
4016 case Builtin::BI__builtin_alloca_uninitialized:
4017 case Builtin::BI__builtin_alloca: {
4021 const Align SuitableAlignmentInBytes =
4025 AllocaInst *AI =
Builder.CreateAlloca(
Builder.getInt8Ty(), Size);
4026 AI->setAlignment(SuitableAlignmentInBytes);
4027 if (BuiltinID != Builtin::BI__builtin_alloca_uninitialized)
4039 case Builtin::BI__builtin_alloca_with_align_uninitialized:
4040 case Builtin::BI__builtin_alloca_with_align: {
4043 auto *AlignmentInBitsCI = cast<ConstantInt>(AlignmentInBitsValue);
4044 unsigned AlignmentInBits = AlignmentInBitsCI->getZExtValue();
4045 const Align AlignmentInBytes =
4047 AllocaInst *AI =
Builder.CreateAlloca(
Builder.getInt8Ty(), Size);
4048 AI->setAlignment(AlignmentInBytes);
4049 if (BuiltinID != Builtin::BI__builtin_alloca_with_align_uninitialized)
4061 case Builtin::BIbzero:
4062 case Builtin::BI__builtin_bzero: {
4071 case Builtin::BIbcopy:
4072 case Builtin::BI__builtin_bcopy: {
4086 case Builtin::BImemcpy:
4087 case Builtin::BI__builtin_memcpy:
4088 case Builtin::BImempcpy:
4089 case Builtin::BI__builtin_mempcpy: {
4096 if (BuiltinID == Builtin::BImempcpy ||
4097 BuiltinID == Builtin::BI__builtin_mempcpy)
4104 case Builtin::BI__builtin_memcpy_inline: {
4115 case Builtin::BI__builtin_char_memchr:
4116 BuiltinID = Builtin::BI__builtin_memchr;
4119 case Builtin::BI__builtin___memcpy_chk: {
4126 llvm::APSInt DstSize = DstSizeResult.
Val.
getInt();
4127 if (
Size.ugt(DstSize))
4131 Value *SizeVal = llvm::ConstantInt::get(
Builder.getContext(), Size);
4136 case Builtin::BI__builtin_objc_memmove_collectable: {
4141 DestAddr, SrcAddr, SizeVal);
4145 case Builtin::BI__builtin___memmove_chk: {
4152 llvm::APSInt DstSize = DstSizeResult.
Val.
getInt();
4153 if (
Size.ugt(DstSize))
4157 Value *SizeVal = llvm::ConstantInt::get(
Builder.getContext(), Size);
4162 case Builtin::BImemmove:
4163 case Builtin::BI__builtin_memmove: {
4172 case Builtin::BImemset:
4173 case Builtin::BI__builtin_memset: {
4183 case Builtin::BI__builtin_memset_inline: {
4195 case Builtin::BI__builtin___memset_chk: {
4202 llvm::APSInt DstSize = DstSizeResult.
Val.
getInt();
4203 if (
Size.ugt(DstSize))
4208 Value *SizeVal = llvm::ConstantInt::get(
Builder.getContext(), Size);
4212 case Builtin::BI__builtin_wmemchr: {
4215 if (!
getTarget().getTriple().isOSMSVCRT())
4223 BasicBlock *Entry =
Builder.GetInsertBlock();
4228 Builder.CreateCondBr(SizeEq0, Exit, CmpEq);
4232 StrPhi->addIncoming(Str, Entry);
4234 SizePhi->addIncoming(Size, Entry);
4238 Value *FoundChr =
Builder.CreateConstInBoundsGEP1_32(WCharTy, StrPhi, 0);
4240 Builder.CreateCondBr(StrEqChr, Exit, Next);
4243 Value *NextStr =
Builder.CreateConstInBoundsGEP1_32(WCharTy, StrPhi, 1);
4245 Value *NextSizeEq0 =
4246 Builder.CreateICmpEQ(NextSize, ConstantInt::get(
SizeTy, 0));
4247 Builder.CreateCondBr(NextSizeEq0, Exit, CmpEq);
4248 StrPhi->addIncoming(NextStr, Next);
4249 SizePhi->addIncoming(NextSize, Next);
4253 Ret->addIncoming(llvm::Constant::getNullValue(Str->
getType()), Entry);
4254 Ret->addIncoming(llvm::Constant::getNullValue(Str->
getType()), Next);
4255 Ret->addIncoming(FoundChr, CmpEq);
4258 case Builtin::BI__builtin_wmemcmp: {
4261 if (!
getTarget().getTriple().isOSMSVCRT())
4270 BasicBlock *Entry =
Builder.GetInsertBlock();
4276 Builder.CreateCondBr(SizeEq0, Exit, CmpGT);
4280 DstPhi->addIncoming(Dst, Entry);
4282 SrcPhi->addIncoming(Src, Entry);
4284 SizePhi->addIncoming(Size, Entry);
4290 Builder.CreateCondBr(DstGtSrc, Exit, CmpLT);
4294 Builder.CreateCondBr(DstLtSrc, Exit, Next);
4297 Value *NextDst =
Builder.CreateConstInBoundsGEP1_32(WCharTy, DstPhi, 1);
4298 Value *NextSrc =
Builder.CreateConstInBoundsGEP1_32(WCharTy, SrcPhi, 1);
4300 Value *NextSizeEq0 =
4301 Builder.CreateICmpEQ(NextSize, ConstantInt::get(
SizeTy, 0));
4302 Builder.CreateCondBr(NextSizeEq0, Exit, CmpGT);
4303 DstPhi->addIncoming(NextDst, Next);
4304 SrcPhi->addIncoming(NextSrc, Next);
4305 SizePhi->addIncoming(NextSize, Next);
4309 Ret->addIncoming(ConstantInt::get(
IntTy, 0), Entry);
4310 Ret->addIncoming(ConstantInt::get(
IntTy, 1), CmpGT);
4311 Ret->addIncoming(ConstantInt::get(
IntTy, -1), CmpLT);
4312 Ret->addIncoming(ConstantInt::get(
IntTy, 0), Next);
4315 case Builtin::BI__builtin_dwarf_cfa: {
4328 llvm::ConstantInt::get(
Int32Ty, Offset)));
4330 case Builtin::BI__builtin_return_address: {
4336 case Builtin::BI_ReturnAddress: {
4340 case Builtin::BI__builtin_frame_address: {
4346 case Builtin::BI__builtin_extract_return_addr: {
4351 case Builtin::BI__builtin_frob_return_addr: {
4356 case Builtin::BI__builtin_dwarf_sp_column: {
4357 llvm::IntegerType *Ty
4366 case Builtin::BI__builtin_init_dwarf_reg_size_table: {
4372 case Builtin::BI__builtin_eh_return: {
4376 llvm::IntegerType *
IntTy = cast<llvm::IntegerType>(
Int->getType());
4377 assert((
IntTy->getBitWidth() == 32 ||
IntTy->getBitWidth() == 64) &&
4378 "LLVM's __builtin_eh_return only supports 32- and 64-bit variants");
4381 : Intrinsic::eh_return_i64);
4390 case Builtin::BI__builtin_unwind_init: {
4395 case Builtin::BI__builtin_extend_pointer: {
4420 case Builtin::BI__builtin_setjmp: {
4427 ConstantInt::get(
Int32Ty, 0));
4441 case Builtin::BI__builtin_longjmp: {
4455 case Builtin::BI__builtin_launder: {
4464 case Builtin::BI__sync_fetch_and_add:
4465 case Builtin::BI__sync_fetch_and_sub:
4466 case Builtin::BI__sync_fetch_and_or:
4467 case Builtin::BI__sync_fetch_and_and:
4468 case Builtin::BI__sync_fetch_and_xor:
4469 case Builtin::BI__sync_fetch_and_nand:
4470 case Builtin::BI__sync_add_and_fetch:
4471 case Builtin::BI__sync_sub_and_fetch:
4472 case Builtin::BI__sync_and_and_fetch:
4473 case Builtin::BI__sync_or_and_fetch:
4474 case Builtin::BI__sync_xor_and_fetch:
4475 case Builtin::BI__sync_nand_and_fetch:
4476 case Builtin::BI__sync_val_compare_and_swap:
4477 case Builtin::BI__sync_bool_compare_and_swap:
4478 case Builtin::BI__sync_lock_test_and_set:
4479 case Builtin::BI__sync_lock_release:
4480 case Builtin::BI__sync_swap:
4481 llvm_unreachable(
"Shouldn't make it through sema");
4482 case Builtin::BI__sync_fetch_and_add_1:
4483 case Builtin::BI__sync_fetch_and_add_2:
4484 case Builtin::BI__sync_fetch_and_add_4:
4485 case Builtin::BI__sync_fetch_and_add_8:
4486 case Builtin::BI__sync_fetch_and_add_16:
4488 case Builtin::BI__sync_fetch_and_sub_1:
4489 case Builtin::BI__sync_fetch_and_sub_2:
4490 case Builtin::BI__sync_fetch_and_sub_4:
4491 case Builtin::BI__sync_fetch_and_sub_8:
4492 case Builtin::BI__sync_fetch_and_sub_16:
4494 case Builtin::BI__sync_fetch_and_or_1:
4495 case Builtin::BI__sync_fetch_and_or_2:
4496 case Builtin::BI__sync_fetch_and_or_4:
4497 case Builtin::BI__sync_fetch_and_or_8:
4498 case Builtin::BI__sync_fetch_and_or_16:
4500 case Builtin::BI__sync_fetch_and_and_1:
4501 case Builtin::BI__sync_fetch_and_and_2:
4502 case Builtin::BI__sync_fetch_and_and_4:
4503 case Builtin::BI__sync_fetch_and_and_8:
4504 case Builtin::BI__sync_fetch_and_and_16:
4506 case Builtin::BI__sync_fetch_and_xor_1:
4507 case Builtin::BI__sync_fetch_and_xor_2:
4508 case Builtin::BI__sync_fetch_and_xor_4:
4509 case Builtin::BI__sync_fetch_and_xor_8:
4510 case Builtin::BI__sync_fetch_and_xor_16:
4512 case Builtin::BI__sync_fetch_and_nand_1:
4513 case Builtin::BI__sync_fetch_and_nand_2:
4514 case Builtin::BI__sync_fetch_and_nand_4:
4515 case Builtin::BI__sync_fetch_and_nand_8:
4516 case Builtin::BI__sync_fetch_and_nand_16:
4520 case Builtin::BI__sync_fetch_and_min:
4522 case Builtin::BI__sync_fetch_and_max:
4524 case Builtin::BI__sync_fetch_and_umin:
4526 case Builtin::BI__sync_fetch_and_umax:
4529 case Builtin::BI__sync_add_and_fetch_1:
4530 case Builtin::BI__sync_add_and_fetch_2:
4531 case Builtin::BI__sync_add_and_fetch_4:
4532 case Builtin::BI__sync_add_and_fetch_8:
4533 case Builtin::BI__sync_add_and_fetch_16:
4535 llvm::Instruction::Add);
4536 case Builtin::BI__sync_sub_and_fetch_1:
4537 case Builtin::BI__sync_sub_and_fetch_2:
4538 case Builtin::BI__sync_sub_and_fetch_4:
4539 case Builtin::BI__sync_sub_and_fetch_8:
4540 case Builtin::BI__sync_sub_and_fetch_16:
4542 llvm::Instruction::Sub);
4543 case Builtin::BI__sync_and_and_fetch_1:
4544 case Builtin::BI__sync_and_and_fetch_2:
4545 case Builtin::BI__sync_and_and_fetch_4:
4546 case Builtin::BI__sync_and_and_fetch_8:
4547 case Builtin::BI__sync_and_and_fetch_16:
4549 llvm::Instruction::And);
4550 case Builtin::BI__sync_or_and_fetch_1:
4551 case Builtin::BI__sync_or_and_fetch_2:
4552 case Builtin::BI__sync_or_and_fetch_4:
4553 case Builtin::BI__sync_or_and_fetch_8:
4554 case Builtin::BI__sync_or_and_fetch_16:
4556 llvm::Instruction::Or);
4557 case Builtin::BI__sync_xor_and_fetch_1:
4558 case Builtin::BI__sync_xor_and_fetch_2:
4559 case Builtin::BI__sync_xor_and_fetch_4:
4560 case Builtin::BI__sync_xor_and_fetch_8:
4561 case Builtin::BI__sync_xor_and_fetch_16:
4563 llvm::Instruction::Xor);
4564 case Builtin::BI__sync_nand_and_fetch_1:
4565 case Builtin::BI__sync_nand_and_fetch_2:
4566 case Builtin::BI__sync_nand_and_fetch_4:
4567 case Builtin::BI__sync_nand_and_fetch_8:
4568 case Builtin::BI__sync_nand_and_fetch_16:
4570 llvm::Instruction::And,
true);
4572 case Builtin::BI__sync_val_compare_and_swap_1:
4573 case Builtin::BI__sync_val_compare_and_swap_2:
4574 case Builtin::BI__sync_val_compare_and_swap_4:
4575 case Builtin::BI__sync_val_compare_and_swap_8:
4576 case Builtin::BI__sync_val_compare_and_swap_16:
4579 case Builtin::BI__sync_bool_compare_and_swap_1:
4580 case Builtin::BI__sync_bool_compare_and_swap_2:
4581 case Builtin::BI__sync_bool_compare_and_swap_4:
4582 case Builtin::BI__sync_bool_compare_and_swap_8:
4583 case Builtin::BI__sync_bool_compare_and_swap_16:
4586 case Builtin::BI__sync_swap_1:
4587 case Builtin::BI__sync_swap_2:
4588 case Builtin::BI__sync_swap_4:
4589 case Builtin::BI__sync_swap_8:
4590 case Builtin::BI__sync_swap_16:
4593 case Builtin::BI__sync_lock_test_and_set_1:
4594 case Builtin::BI__sync_lock_test_and_set_2:
4595 case Builtin::BI__sync_lock_test_and_set_4:
4596 case Builtin::BI__sync_lock_test_and_set_8:
4597 case Builtin::BI__sync_lock_test_and_set_16:
4600 case Builtin::BI__sync_lock_release_1:
4601 case Builtin::BI__sync_lock_release_2:
4602 case Builtin::BI__sync_lock_release_4:
4603 case Builtin::BI__sync_lock_release_8:
4604 case Builtin::BI__sync_lock_release_16: {
4610 llvm::StoreInst *
Store =
4612 Store->setAtomic(llvm::AtomicOrdering::Release);
4616 case Builtin::BI__sync_synchronize: {
4624 Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent);
4628 case Builtin::BI__builtin_nontemporal_load:
4630 case Builtin::BI__builtin_nontemporal_store:
4632 case Builtin::BI__c11_atomic_is_lock_free:
4633 case Builtin::BI__atomic_is_lock_free: {
4637 const char *LibCallName =
"__atomic_is_lock_free";
4641 if (BuiltinID == Builtin::BI__atomic_is_lock_free)
4655 case Builtin::BI__atomic_test_and_set: {
4667 if (isa<llvm::ConstantInt>(Order)) {
4668 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
4669 AtomicRMWInst *
Result =
nullptr;
4674 llvm::AtomicOrdering::Monotonic);
4679 llvm::AtomicOrdering::Acquire);
4683 llvm::AtomicOrdering::Release);
4688 llvm::AtomicOrdering::AcquireRelease);
4692 llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
4693 llvm::AtomicOrdering::SequentiallyConsistent);
4696 Result->setVolatile(Volatile);
4702 llvm::BasicBlock *BBs[5] = {
4709 llvm::AtomicOrdering Orders[5] = {
4710 llvm::AtomicOrdering::Monotonic, llvm::AtomicOrdering::Acquire,
4711 llvm::AtomicOrdering::Release, llvm::AtomicOrdering::AcquireRelease,
4712 llvm::AtomicOrdering::SequentiallyConsistent};
4714 Order =
Builder.CreateIntCast(Order,
Builder.getInt32Ty(),
false);
4715 llvm::SwitchInst *SI =
Builder.CreateSwitch(Order, BBs[0]);
4717 Builder.SetInsertPoint(ContBB);
4720 for (
unsigned i = 0; i < 5; ++i) {
4721 Builder.SetInsertPoint(BBs[i]);
4723 Ptr, NewVal, Orders[i]);
4724 RMW->setVolatile(Volatile);
4725 Result->addIncoming(RMW, BBs[i]);
4729 SI->addCase(
Builder.getInt32(0), BBs[0]);
4730 SI->addCase(
Builder.getInt32(1), BBs[1]);
4731 SI->addCase(
Builder.getInt32(2), BBs[1]);
4732 SI->addCase(
Builder.getInt32(3), BBs[2]);
4733 SI->addCase(
Builder.getInt32(4), BBs[3]);
4734 SI->addCase(
Builder.getInt32(5), BBs[4]);
4736 Builder.SetInsertPoint(ContBB);
4740 case Builtin::BI__atomic_clear: {
4749 if (isa<llvm::ConstantInt>(Order)) {
4750 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
4755 Store->setOrdering(llvm::AtomicOrdering::Monotonic);
4758 Store->setOrdering(llvm::AtomicOrdering::Release);
4761 Store->setOrdering(llvm::AtomicOrdering::SequentiallyConsistent);
4769 llvm::BasicBlock *BBs[3] = {
4774 llvm::AtomicOrdering Orders[3] = {
4775 llvm::AtomicOrdering::Monotonic, llvm::AtomicOrdering::Release,
4776 llvm::AtomicOrdering::SequentiallyConsistent};
4778 Order =
Builder.CreateIntCast(Order,
Builder.getInt32Ty(),
false);
4779 llvm::SwitchInst *SI =
Builder.CreateSwitch(Order, BBs[0]);
4781 for (
unsigned i = 0; i < 3; ++i) {
4782 Builder.SetInsertPoint(BBs[i]);
4784 Store->setOrdering(Orders[i]);
4788 SI->addCase(
Builder.getInt32(0), BBs[0]);
4789 SI->addCase(
Builder.getInt32(3), BBs[1]);
4790 SI->addCase(
Builder.getInt32(5), BBs[2]);
4792 Builder.SetInsertPoint(ContBB);
4796 case Builtin::BI__atomic_thread_fence:
4797 case Builtin::BI__atomic_signal_fence:
4798 case Builtin::BI__c11_atomic_thread_fence:
4799 case Builtin::BI__c11_atomic_signal_fence: {
4800 llvm::SyncScope::ID SSID;
4801 if (BuiltinID == Builtin::BI__atomic_signal_fence ||
4802 BuiltinID == Builtin::BI__c11_atomic_signal_fence)
4803 SSID = llvm::SyncScope::SingleThread;
4805 SSID = llvm::SyncScope::System;
4807 if (isa<llvm::ConstantInt>(Order)) {
4808 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
4815 Builder.CreateFence(llvm::AtomicOrdering::Acquire, SSID);
4818 Builder.CreateFence(llvm::AtomicOrdering::Release, SSID);
4821 Builder.CreateFence(llvm::AtomicOrdering::AcquireRelease, SSID);
4824 Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent, SSID);
4830 llvm::BasicBlock *AcquireBB, *ReleaseBB, *AcqRelBB, *SeqCstBB;
4837 Order =
Builder.CreateIntCast(Order,
Builder.getInt32Ty(),
false);
4838 llvm::SwitchInst *SI =
Builder.CreateSwitch(Order, ContBB);
4840 Builder.SetInsertPoint(AcquireBB);
4841 Builder.CreateFence(llvm::AtomicOrdering::Acquire, SSID);
4843 SI->addCase(
Builder.getInt32(1), AcquireBB);
4844 SI->addCase(
Builder.getInt32(2), AcquireBB);
4846 Builder.SetInsertPoint(ReleaseBB);
4847 Builder.CreateFence(llvm::AtomicOrdering::Release, SSID);
4849 SI->addCase(
Builder.getInt32(3), ReleaseBB);
4851 Builder.SetInsertPoint(AcqRelBB);
4852 Builder.CreateFence(llvm::AtomicOrdering::AcquireRelease, SSID);
4854 SI->addCase(
Builder.getInt32(4), AcqRelBB);
4856 Builder.SetInsertPoint(SeqCstBB);
4857 Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent, SSID);
4859 SI->addCase(
Builder.getInt32(5), SeqCstBB);
4861 Builder.SetInsertPoint(ContBB);
4865 case Builtin::BI__builtin_signbit:
4866 case Builtin::BI__builtin_signbitf:
4867 case Builtin::BI__builtin_signbitl: {
4872 case Builtin::BI__warn_memset_zero_len:
4874 case Builtin::BI__annotation: {
4879 assert(Str->getCharByteWidth() == 2);
4880 StringRef WideBytes = Str->getBytes();
4881 std::string StrUtf8;
4882 if (!convertUTF16ToUTF8String(
4883 ArrayRef(WideBytes.data(), WideBytes.size()), StrUtf8)) {
4887 Strings.push_back(llvm::MDString::get(
getLLVMContext(), StrUtf8));
4897 case Builtin::BI__builtin_annotation: {
4906 StringRef Str = cast<StringLiteral>(AnnotationStrExpr)->getString();
4910 case Builtin::BI__builtin_addcb:
4911 case Builtin::BI__builtin_addcs:
4912 case Builtin::BI__builtin_addc:
4913 case Builtin::BI__builtin_addcl:
4914 case Builtin::BI__builtin_addcll:
4915 case Builtin::BI__builtin_subcb:
4916 case Builtin::BI__builtin_subcs:
4917 case Builtin::BI__builtin_subc:
4918 case Builtin::BI__builtin_subcl:
4919 case Builtin::BI__builtin_subcll: {
4945 llvm::Intrinsic::ID IntrinsicId;
4946 switch (BuiltinID) {
4947 default: llvm_unreachable(
"Unknown multiprecision builtin id.");
4948 case Builtin::BI__builtin_addcb:
4949 case Builtin::BI__builtin_addcs:
4950 case Builtin::BI__builtin_addc:
4951 case Builtin::BI__builtin_addcl:
4952 case Builtin::BI__builtin_addcll:
4953 IntrinsicId = llvm::Intrinsic::uadd_with_overflow;
4955 case Builtin::BI__builtin_subcb:
4956 case Builtin::BI__builtin_subcs:
4957 case Builtin::BI__builtin_subc:
4958 case Builtin::BI__builtin_subcl:
4959 case Builtin::BI__builtin_subcll:
4960 IntrinsicId = llvm::Intrinsic::usub_with_overflow;
4965 llvm::Value *Carry1;
4968 llvm::Value *Carry2;
4970 Sum1, Carryin, Carry2);
4971 llvm::Value *CarryOut =
Builder.CreateZExt(
Builder.CreateOr(Carry1, Carry2),
4977 case Builtin::BI__builtin_add_overflow:
4978 case Builtin::BI__builtin_sub_overflow:
4979 case Builtin::BI__builtin_mul_overflow: {
4987 WidthAndSignedness LeftInfo =
4989 WidthAndSignedness RightInfo =
4991 WidthAndSignedness ResultInfo =
4998 RightInfo, ResultArg, ResultQTy,
5004 *
this, LeftArg, LeftInfo, RightArg, RightInfo, ResultArg, ResultQTy,
5007 WidthAndSignedness EncompassingInfo =
5010 llvm::Type *EncompassingLLVMTy =
5015 llvm::Intrinsic::ID IntrinsicId;
5016 switch (BuiltinID) {
5018 llvm_unreachable(
"Unknown overflow builtin id.");
5019 case Builtin::BI__builtin_add_overflow:
5020 IntrinsicId = EncompassingInfo.Signed
5021 ? llvm::Intrinsic::sadd_with_overflow
5022 : llvm::Intrinsic::uadd_with_overflow;
5024 case Builtin::BI__builtin_sub_overflow:
5025 IntrinsicId = EncompassingInfo.Signed
5026 ? llvm::Intrinsic::ssub_with_overflow
5027 : llvm::Intrinsic::usub_with_overflow;
5029 case Builtin::BI__builtin_mul_overflow:
5030 IntrinsicId = EncompassingInfo.Signed
5031 ? llvm::Intrinsic::smul_with_overflow
5032 : llvm::Intrinsic::umul_with_overflow;
5041 Left =
Builder.CreateIntCast(Left, EncompassingLLVMTy, LeftInfo.Signed);
5042 Right =
Builder.CreateIntCast(Right, EncompassingLLVMTy, RightInfo.Signed);
5045 llvm::Value *Overflow, *
Result;
5048 if (EncompassingInfo.Width > ResultInfo.Width) {
5051 llvm::Value *ResultTrunc =
Builder.CreateTrunc(
Result, ResultLLVMTy);
5055 llvm::Value *ResultTruncExt =
Builder.CreateIntCast(
5056 ResultTrunc, EncompassingLLVMTy, ResultInfo.Signed);
5057 llvm::Value *TruncationOverflow =
5060 Overflow =
Builder.CreateOr(Overflow, TruncationOverflow);
5072 case Builtin::BI__builtin_uadd_overflow:
5073 case Builtin::BI__builtin_uaddl_overflow:
5074 case Builtin::BI__builtin_uaddll_overflow:
5075 case Builtin::BI__builtin_usub_overflow:
5076 case Builtin::BI__builtin_usubl_overflow:
5077 case Builtin::BI__builtin_usubll_overflow:
5078 case Builtin::BI__builtin_umul_overflow:
5079 case Builtin::BI__builtin_umull_overflow:
5080 case Builtin::BI__builtin_umulll_overflow:
5081 case Builtin::BI__builtin_sadd_overflow:
5082 case Builtin::BI__builtin_saddl_overflow:
5083 case Builtin::BI__builtin_saddll_overflow:
5084 case Builtin::BI__builtin_ssub_overflow:
5085 case Builtin::BI__builtin_ssubl_overflow:
5086 case Builtin::BI__builtin_ssubll_overflow:
5087 case Builtin::BI__builtin_smul_overflow:
5088 case Builtin::BI__builtin_smull_overflow:
5089 case Builtin::BI__builtin_smulll_overflow: {
5099 llvm::Intrinsic::ID IntrinsicId;
5100 switch (BuiltinID) {
5101 default: llvm_unreachable(
"Unknown overflow builtin id.");
5102 case Builtin::BI__builtin_uadd_overflow:
5103 case Builtin::BI__builtin_uaddl_overflow:
5104 case Builtin::BI__builtin_uaddll_overflow:
5105 IntrinsicId = llvm::Intrinsic::uadd_with_overflow;
5107 case Builtin::BI__builtin_usub_overflow:
5108 case Builtin::BI__builtin_usubl_overflow:
5109 case Builtin::BI__builtin_usubll_overflow:
5110 IntrinsicId = llvm::Intrinsic::usub_with_overflow;
5112 case Builtin::BI__builtin_umul_overflow:
5113 case Builtin::BI__builtin_umull_overflow:
5114 case Builtin::BI__builtin_umulll_overflow:
5115 IntrinsicId = llvm::Intrinsic::umul_with_overflow;
5117 case Builtin::BI__builtin_sadd_overflow:
5118 case Builtin::BI__builtin_saddl_overflow:
5119 case Builtin::BI__builtin_saddll_overflow:
5120 IntrinsicId = llvm::Intrinsic::sadd_with_overflow;
5122 case Builtin::BI__builtin_ssub_overflow:
5123 case Builtin::BI__builtin_ssubl_overflow:
5124 case Builtin::BI__builtin_ssubll_overflow:
5125 IntrinsicId = llvm::Intrinsic::ssub_with_overflow;
5127 case Builtin::BI__builtin_smul_overflow:
5128 case Builtin::BI__builtin_smull_overflow:
5129 case Builtin::BI__builtin_smulll_overflow:
5130 IntrinsicId = llvm::Intrinsic::smul_with_overflow;
5141 case Builtin::BIaddressof:
5142 case Builtin::BI__addressof:
5143 case Builtin::BI__builtin_addressof:
5145 case Builtin::BI__builtin_function_start:
5148 case Builtin::BI__builtin_operator_new:
5151 case Builtin::BI__builtin_operator_delete:
5156 case Builtin::BI__builtin_is_aligned:
5158 case Builtin::BI__builtin_align_up:
5160 case Builtin::BI__builtin_align_down:
5163 case Builtin::BI__noop:
5166 case Builtin::BI__builtin_call_with_static_chain: {
5173 case Builtin::BI_InterlockedExchange8:
5174 case Builtin::BI_InterlockedExchange16:
5175 case Builtin::BI_InterlockedExchange:
5176 case Builtin::BI_InterlockedExchangePointer:
5179 case Builtin::BI_InterlockedCompareExchangePointer:
5180 case Builtin::BI_InterlockedCompareExchangePointer_nf: {
5182 llvm::IntegerType *IntType = IntegerType::get(
5188 RTy = Exchange->getType();
5189 Exchange =
Builder.CreatePtrToInt(Exchange, IntType);
5191 llvm::Value *Comparand =
5195 BuiltinID == Builtin::BI_InterlockedCompareExchangePointer_nf ?
5196 AtomicOrdering::Monotonic : AtomicOrdering::SequentiallyConsistent;
5199 Ordering, Ordering);
5200 Result->setVolatile(
true);
5206 case Builtin::BI_InterlockedCompareExchange8:
5207 case Builtin::BI_InterlockedCompareExchange16:
5208 case Builtin::BI_InterlockedCompareExchange:
5209 case Builtin::BI_InterlockedCompareExchange64:
5211 case Builtin::BI_InterlockedIncrement16:
5212 case Builtin::BI_InterlockedIncrement:
5215 case Builtin::BI_InterlockedDecrement16:
5216 case Builtin::BI_InterlockedDecrement:
5219 case Builtin::BI_InterlockedAnd8:
5220 case Builtin::BI_InterlockedAnd16:
5221 case Builtin::BI_InterlockedAnd:
5223 case Builtin::BI_InterlockedExchangeAdd8:
5224 case Builtin::BI_InterlockedExchangeAdd16:
5225 case Builtin::BI_InterlockedExchangeAdd:
5228 case Builtin::BI_InterlockedExchangeSub8:
5229 case Builtin::BI_InterlockedExchangeSub16:
5230 case Builtin::BI_InterlockedExchangeSub:
5233 case Builtin::BI_InterlockedOr8:
5234 case Builtin::BI_InterlockedOr16:
5235 case Builtin::BI_InterlockedOr:
5237 case Builtin::BI_InterlockedXor8:
5238 case Builtin::BI_InterlockedXor16:
5239 case Builtin::BI_InterlockedXor:
5242 case Builtin::BI_bittest64:
5243 case Builtin::BI_bittest:
5244 case Builtin::BI_bittestandcomplement64:
5245 case Builtin::BI_bittestandcomplement:
5246 case Builtin::BI_bittestandreset64:
5247 case Builtin::BI_bittestandreset:
5248 case Builtin::BI_bittestandset64:
5249 case Builtin::BI_bittestandset:
5250 case Builtin::BI_interlockedbittestandreset:
5251 case Builtin::BI_interlockedbittestandreset64:
5252 case Builtin::BI_interlockedbittestandset64:
5253 case Builtin::BI_interlockedbittestandset:
5254 case Builtin::BI_interlockedbittestandset_acq:
5255 case Builtin::BI_interlockedbittestandset_rel:
5256 case Builtin::BI_interlockedbittestandset_nf:
5257 case Builtin::BI_interlockedbittestandreset_acq:
5258 case Builtin::BI_interlockedbittestandreset_rel:
5259 case Builtin::BI_interlockedbittestandreset_nf:
5264 case Builtin::BI__iso_volatile_load8:
5265 case Builtin::BI__iso_volatile_load16:
5266 case Builtin::BI__iso_volatile_load32:
5267 case Builtin::BI__iso_volatile_load64:
5269 case Builtin::BI__iso_volatile_store8:
5270 case Builtin::BI__iso_volatile_store16:
5271 case Builtin::BI__iso_volatile_store32:
5272 case Builtin::BI__iso_volatile_store64:
5275 case Builtin::BI__builtin_ptrauth_auth:
5276 case Builtin::BI__builtin_ptrauth_auth_and_resign:
5277 case Builtin::BI__builtin_ptrauth_blend_discriminator:
5278 case Builtin::BI__builtin_ptrauth_sign_generic_data:
5279 case Builtin::BI__builtin_ptrauth_sign_unauthenticated:
5280 case Builtin::BI__builtin_ptrauth_strip: {
5287 llvm::Type *OrigValueType = Args[0]->getType();
5288 if (OrigValueType->isPointerTy())
5291 switch (BuiltinID) {
5292 case Builtin::BI__builtin_ptrauth_auth_and_resign:
5293 if (Args[4]->getType()->isPointerTy())
5297 case Builtin::BI__builtin_ptrauth_auth:
5298 case Builtin::BI__builtin_ptrauth_sign_unauthenticated:
5299 if (Args[2]->getType()->isPointerTy())
5303 case Builtin::BI__builtin_ptrauth_sign_generic_data:
5304 if (Args[1]->getType()->isPointerTy())
5308 case Builtin::BI__builtin_ptrauth_blend_discriminator:
5309 case Builtin::BI__builtin_ptrauth_strip:
5314 auto IntrinsicID = [&]() ->
unsigned {
5315 switch (BuiltinID) {
5316 case Builtin::BI__builtin_ptrauth_auth:
5317 return llvm::Intrinsic::ptrauth_auth;
5318 case Builtin::BI__builtin_ptrauth_auth_and_resign:
5319 return llvm::Intrinsic::ptrauth_resign;
5320 case Builtin::BI__builtin_ptrauth_blend_discriminator:
5321 return llvm::Intrinsic::ptrauth_blend;
5322 case Builtin::BI__builtin_ptrauth_sign_generic_data:
5323 return llvm::Intrinsic::ptrauth_sign_generic;
5324 case Builtin::BI__builtin_ptrauth_sign_unauthenticated:
5325 return llvm::Intrinsic::ptrauth_sign;
5326 case Builtin::BI__builtin_ptrauth_strip:
5327 return llvm::Intrinsic::ptrauth_strip;
5329 llvm_unreachable(
"bad ptrauth intrinsic");
5334 if (BuiltinID != Builtin::BI__builtin_ptrauth_sign_generic_data &&
5335 BuiltinID != Builtin::BI__builtin_ptrauth_blend_discriminator &&
5336 OrigValueType->isPointerTy()) {
5342 case Builtin::BI__exception_code:
5343 case Builtin::BI_exception_code:
5345 case Builtin::BI__exception_info:
5346 case Builtin::BI_exception_info:
5348 case Builtin::BI__abnormal_termination:
5349 case Builtin::BI_abnormal_termination:
5351 case Builtin::BI_setjmpex:
5356 case Builtin::BI_setjmp:
5359 if (
getTarget().getTriple().getArch() == llvm::Triple::x86)
5361 else if (
getTarget().getTriple().getArch() == llvm::Triple::aarch64)
5368 case Builtin::BImove:
5369 case Builtin::BImove_if_noexcept:
5370 case Builtin::BIforward:
5371 case Builtin::BIforward_like:
5372 case Builtin::BIas_const:
5374 case Builtin::BI__GetExceptionInfo: {
5375 if (llvm::GlobalVariable *GV =
5381 case Builtin::BI__fastfail:
5384 case Builtin::BI__builtin_coro_id:
5386 case Builtin::BI__builtin_coro_promise:
5388 case Builtin::BI__builtin_coro_resume:
5391 case Builtin::BI__builtin_coro_frame:
5393 case Builtin::BI__builtin_coro_noop:
5395 case Builtin::BI__builtin_coro_free:
5397 case Builtin::BI__builtin_coro_destroy:
5400 case Builtin::BI__builtin_coro_done:
5402 case Builtin::BI__builtin_coro_alloc:
5404 case Builtin::BI__builtin_coro_begin:
5406 case Builtin::BI__builtin_coro_end:
5408 case Builtin::BI__builtin_coro_suspend:
5410 case Builtin::BI__builtin_coro_size:
5412 case Builtin::BI__builtin_coro_align:
5416 case Builtin::BIread_pipe:
5417 case Builtin::BIwrite_pipe: {
5421 Value *PacketSize = OpenCLRT.getPipeElemSize(E->
getArg(0));
5422 Value *PacketAlign = OpenCLRT.getPipeElemAlign(E->
getArg(0));
5425 unsigned GenericAS =
5427 llvm::Type *I8PTy = llvm::PointerType::get(
getLLVMContext(), GenericAS);
5431 const char *Name = (BuiltinID == Builtin::BIread_pipe) ?
"__read_pipe_2"
5436 llvm::FunctionType *FTy = llvm::FunctionType::get(
5441 {Arg0, BCast, PacketSize, PacketAlign}));
5444 "Illegal number of parameters to pipe function");
5445 const char *Name = (BuiltinID == Builtin::BIread_pipe) ?
"__read_pipe_4"
5452 llvm::FunctionType *FTy = llvm::FunctionType::get(
5461 {Arg0, Arg1, Arg2, BCast, PacketSize, PacketAlign}));
5466 case Builtin::BIreserve_read_pipe:
5467 case Builtin::BIreserve_write_pipe:
5468 case Builtin::BIwork_group_reserve_read_pipe:
5469 case Builtin::BIwork_group_reserve_write_pipe:
5470 case Builtin::BIsub_group_reserve_read_pipe:
5471 case Builtin::BIsub_group_reserve_write_pipe: {
5474 if (BuiltinID == Builtin::BIreserve_read_pipe)
5475 Name =
"__reserve_read_pipe";
5476 else if (BuiltinID == Builtin::BIreserve_write_pipe)
5477 Name =
"__reserve_write_pipe";
5478 else if (BuiltinID == Builtin::BIwork_group_reserve_read_pipe)
5479 Name =
"__work_group_reserve_read_pipe";
5480 else if (BuiltinID == Builtin::BIwork_group_reserve_write_pipe)
5481 Name =
"__work_group_reserve_write_pipe";
5482 else if (BuiltinID == Builtin::BIsub_group_reserve_read_pipe)
5483 Name =
"__sub_group_reserve_read_pipe";
5485 Name =
"__sub_group_reserve_write_pipe";
5491 Value *PacketSize = OpenCLRT.getPipeElemSize(E->
getArg(0));
5492 Value *PacketAlign = OpenCLRT.getPipeElemAlign(E->
getArg(0));
5496 llvm::FunctionType *FTy = llvm::FunctionType::get(
5503 {Arg0, Arg1, PacketSize, PacketAlign}));
5507 case Builtin::BIcommit_read_pipe:
5508 case Builtin::BIcommit_write_pipe:
5509 case Builtin::BIwork_group_commit_read_pipe:
5510 case Builtin::BIwork_group_commit_write_pipe:
5511 case Builtin::BIsub_group_commit_read_pipe:
5512 case Builtin::BIsub_group_commit_write_pipe: {
5514 if (BuiltinID == Builtin::BIcommit_read_pipe)
5515 Name =
"__commit_read_pipe";
5516 else if (BuiltinID == Builtin::BIcommit_write_pipe)
5517 Name =
"__commit_write_pipe";
5518 else if (BuiltinID == Builtin::BIwork_group_commit_read_pipe)
5519 Name =
"__work_group_commit_read_pipe";
5520 else if (BuiltinID == Builtin::BIwork_group_commit_write_pipe)
5521 Name =
"__work_group_commit_write_pipe";
5522 else if (BuiltinID == Builtin::BIsub_group_commit_read_pipe)
5523 Name =
"__sub_group_commit_read_pipe";
5525 Name =
"__sub_group_commit_write_pipe";
5530 Value *PacketSize = OpenCLRT.getPipeElemSize(E->
getArg(0));
5531 Value *PacketAlign = OpenCLRT.getPipeElemAlign(E->
getArg(0));
5535 llvm::FunctionType *FTy =
5540 {Arg0, Arg1, PacketSize, PacketAlign}));
5543 case Builtin::BIget_pipe_num_packets:
5544 case Builtin::BIget_pipe_max_packets: {
5545 const char *BaseName;
5547 if (BuiltinID == Builtin::BIget_pipe_num_packets)
5548 BaseName =
"__get_pipe_num_packets";
5550 BaseName =
"__get_pipe_max_packets";
5551 std::string Name = std::string(BaseName) +
5552 std::string(PipeTy->isReadOnly() ?
"_ro" :
"_wo");
5557 Value *PacketSize = OpenCLRT.getPipeElemSize(E->
getArg(0));
5558 Value *PacketAlign = OpenCLRT.getPipeElemAlign(E->
getArg(0));
5560 llvm::FunctionType *FTy = llvm::FunctionType::get(
5564 {Arg0, PacketSize, PacketAlign}));
5568 case Builtin::BIto_global:
5569 case Builtin::BIto_local:
5570 case Builtin::BIto_private: {
5572 auto NewArgT = llvm::PointerType::get(
5575 auto NewRetT = llvm::PointerType::get(
5579 auto FTy = llvm::FunctionType::get(NewRetT, {NewArgT},
false);
5580 llvm::Value *NewArg;
5581 if (Arg0->
getType()->getPointerAddressSpace() !=
5582 NewArgT->getPointerAddressSpace())
5585 NewArg =
Builder.CreateBitOrPointerCast(Arg0, NewArgT);
5601 case Builtin::BIenqueue_kernel: {
5606 llvm::Type *GenericVoidPtrTy =
Builder.getPtrTy(
5618 Name =
"__enqueue_kernel_basic";
5619 llvm::Type *ArgTys[] = {QueueTy,
Int32Ty, RangeTy, GenericVoidPtrTy,
5621 llvm::FunctionType *FTy = llvm::FunctionType::get(
5627 Builder.CreatePointerCast(Info.KernelHandle, GenericVoidPtrTy);
5628 llvm::Value *
Block =
5629 Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
5631 AttrBuilder B(
Builder.getContext());
5633 llvm::AttributeList ByValAttrSet =
5634 llvm::AttributeList::get(
CGM.
getModule().getContext(), 3U, B);
5638 {Queue, Flags, Range, Kernel, Block});
5639 RTCall->setAttributes(ByValAttrSet);
5642 assert(NumArgs >= 5 &&
"Invalid enqueue_kernel signature");
5646 auto CreateArrayForSizeVar = [=](
unsigned First)
5647 -> std::tuple<llvm::Value *, llvm::Value *, llvm::Value *> {
5648 llvm::APInt ArraySize(32, NumArgs -
First);
5650 getContext().getSizeType(), ArraySize,
nullptr,
5654 llvm::Value *TmpPtr = Tmp.getPointer();
5657 llvm::Value *ElemPtr;
5660 auto *
Zero = llvm::ConstantInt::get(
IntTy, 0);
5661 for (
unsigned I =
First; I < NumArgs; ++I) {
5662 auto *Index = llvm::ConstantInt::get(
IntTy, I -
First);
5672 return std::tie(ElemPtr, TmpSize, TmpPtr);
5678 Name =
"__enqueue_kernel_varargs";
5682 Builder.CreatePointerCast(Info.KernelHandle, GenericVoidPtrTy);
5683 auto *
Block =
Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
5684 llvm::Value *ElemPtr, *TmpSize, *TmpPtr;
5685 std::tie(ElemPtr, TmpSize, TmpPtr) = CreateArrayForSizeVar(4);
5689 llvm::Value *
const Args[] = {Queue, Flags,
5693 llvm::Type *
const ArgTys[] = {
5694 QueueTy,
IntTy, RangeTy, GenericVoidPtrTy,
5695 GenericVoidPtrTy,
IntTy, ElemPtr->getType()};
5697 llvm::FunctionType *FTy = llvm::FunctionType::get(
Int32Ty, ArgTys,
false);
5706 llvm::PointerType *PtrTy = llvm::PointerType::get(
5710 llvm::Value *NumEvents =
5716 llvm::Value *EventWaitList =
nullptr;
5719 EventWaitList = llvm::ConstantPointerNull::get(PtrTy);
5726 EventWaitList =
Builder.CreatePointerCast(EventWaitList, PtrTy);
5728 llvm::Value *EventRet =
nullptr;
5731 EventRet = llvm::ConstantPointerNull::get(PtrTy);
5740 Builder.CreatePointerCast(Info.KernelHandle, GenericVoidPtrTy);
5741 llvm::Value *
Block =
5742 Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
5744 std::vector<llvm::Type *> ArgTys = {
5746 PtrTy, PtrTy, GenericVoidPtrTy, GenericVoidPtrTy};
5748 std::vector<llvm::Value *> Args = {Queue, Flags,
Range,
5749 NumEvents, EventWaitList, EventRet,
5754 Name =
"__enqueue_kernel_basic_events";
5755 llvm::FunctionType *FTy = llvm::FunctionType::get(
5763 Args.push_back(ConstantInt::get(
Int32Ty, NumArgs - 7));
5765 Name =
"__enqueue_kernel_events_varargs";
5767 llvm::Value *ElemPtr, *TmpSize, *TmpPtr;
5768 std::tie(ElemPtr, TmpSize, TmpPtr) = CreateArrayForSizeVar(7);
5769 Args.push_back(ElemPtr);
5770 ArgTys.push_back(ElemPtr->getType());
5772 llvm::FunctionType *FTy = llvm::FunctionType::get(
5781 llvm_unreachable(
"Unexpected enqueue_kernel signature");
5785 case Builtin::BIget_kernel_work_group_size: {
5786 llvm::Type *GenericVoidPtrTy =
Builder.getPtrTy(
5791 Builder.CreatePointerCast(Info.KernelHandle, GenericVoidPtrTy);
5792 Value *Arg =
Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
5795 llvm::FunctionType::get(
IntTy, {GenericVoidPtrTy, GenericVoidPtrTy},
5797 "__get_kernel_work_group_size_impl"),
5800 case Builtin::BIget_kernel_preferred_work_group_size_multiple: {
5801 llvm::Type *GenericVoidPtrTy =
Builder.getPtrTy(
5806 Builder.CreatePointerCast(Info.KernelHandle, GenericVoidPtrTy);
5807 Value *Arg =
Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
5810 llvm::FunctionType::get(
IntTy, {GenericVoidPtrTy, GenericVoidPtrTy},
5812 "__get_kernel_preferred_work_group_size_multiple_impl"),
5815 case Builtin::BIget_kernel_max_sub_group_size_for_ndrange:
5816 case Builtin::BIget_kernel_sub_group_count_for_ndrange: {
5817 llvm::Type *GenericVoidPtrTy =
Builder.getPtrTy(
5824 Builder.CreatePointerCast(Info.KernelHandle, GenericVoidPtrTy);
5827 BuiltinID == Builtin::BIget_kernel_max_sub_group_size_for_ndrange
5828 ?
"__get_kernel_max_sub_group_size_for_ndrange_impl"
5829 :
"__get_kernel_sub_group_count_for_ndrange_impl";
5832 llvm::FunctionType::get(
5833 IntTy, {NDRange->getType(), GenericVoidPtrTy, GenericVoidPtrTy},
5836 {NDRange, Kernel, Block}));
5838 case Builtin::BI__builtin_store_half:
5839 case Builtin::BI__builtin_store_halff: {
5846 case Builtin::BI__builtin_load_half: {
5851 case Builtin::BI__builtin_load_halff: {
5856 case Builtin::BI__builtin_printf:
5857 case Builtin::BIprintf:
5858 if (
getTarget().getTriple().isNVPTX() ||
5869 case Builtin::BI__builtin_canonicalize:
5870 case Builtin::BI__builtin_canonicalizef:
5871 case Builtin::BI__builtin_canonicalizef16:
5872 case Builtin::BI__builtin_canonicalizel:
5875 case Builtin::BI__builtin_thread_pointer: {
5876 if (!
getContext().getTargetInfo().isTLSSupported())
5881 case Builtin::BI__builtin_os_log_format:
5884 case Builtin::BI__xray_customevent: {
5897 auto FTy = F->getFunctionType();
5898 auto Arg0 = E->
getArg(0);
5900 auto Arg0Ty = Arg0->
getType();
5901 auto PTy0 = FTy->getParamType(0);
5902 if (PTy0 != Arg0Val->getType()) {
5903 if (Arg0Ty->isArrayType())
5906 Arg0Val =
Builder.CreatePointerCast(Arg0Val, PTy0);
5909 auto PTy1 = FTy->getParamType(1);
5911 Arg1 =
Builder.CreateTruncOrBitCast(Arg1, PTy1);
5915 case Builtin::BI__xray_typedevent: {
5931 auto FTy = F->getFunctionType();
5933 auto PTy0 = FTy->getParamType(0);
5935 Arg0 =
Builder.CreateTruncOrBitCast(Arg0, PTy0);
5936 auto Arg1 = E->
getArg(1);
5938 auto Arg1Ty = Arg1->
getType();
5939 auto PTy1 = FTy->getParamType(1);
5940 if (PTy1 != Arg1Val->getType()) {
5941 if (Arg1Ty->isArrayType())
5944 Arg1Val =
Builder.CreatePointerCast(Arg1Val, PTy1);
5947 auto PTy2 = FTy->getParamType(2);
5949 Arg2 =
Builder.CreateTruncOrBitCast(Arg2, PTy2);
5953 case Builtin::BI__builtin_ms_va_start:
5954 case Builtin::BI__builtin_ms_va_end:
5957 BuiltinID == Builtin::BI__builtin_ms_va_start));
5959 case Builtin::BI__builtin_ms_va_copy: {
5976 case Builtin::BI__builtin_get_device_side_mangled_name: {
5980 llvm::Constant *Zeros[] = {llvm::ConstantInt::get(
SizeTy, 0),
5981 llvm::ConstantInt::get(
SizeTy, 0)};
5982 auto *Ptr = llvm::ConstantExpr::getGetElementPtr(Str.getElementType(),
5983 Str.getPointer(), Zeros);
6009 LargestVectorWidth = std::max(LargestVectorWidth, VectorWidth);
6013 Intrinsic::ID IntrinsicID = Intrinsic::not_intrinsic;
6015 llvm::Triple::getArchTypePrefix(
getTarget().getTriple().getArch());
6016 if (!Prefix.empty()) {
6017 IntrinsicID = Intrinsic::getIntrinsicForClangBuiltin(Prefix.data(), Name);
6021 if (IntrinsicID == Intrinsic::not_intrinsic)
6022 IntrinsicID = Intrinsic::getIntrinsicForMSBuiltin(Prefix.data(), Name);
6025 if (IntrinsicID != Intrinsic::not_intrinsic) {
6030 unsigned ICEArguments = 0;
6036 llvm::FunctionType *FTy = F->getFunctionType();
6038 for (
unsigned i = 0, e = E->
getNumArgs(); i != e; ++i) {
6042 llvm::Type *PTy = FTy->getParamType(i);
6043 if (PTy != ArgValue->
getType()) {
6045 if (
auto *PtrTy = dyn_cast<llvm::PointerType>(PTy)) {
6046 if (PtrTy->getAddressSpace() !=
6047 ArgValue->
getType()->getPointerAddressSpace()) {
6050 PtrTy->getAddressSpace()));
6056 if (PTy->isX86_AMXTy())
6057 ArgValue =
Builder.CreateIntrinsic(Intrinsic::x86_cast_vector_to_tile,
6058 {ArgValue->
getType()}, {ArgValue});
6060 ArgValue =
Builder.CreateBitCast(ArgValue, PTy);
6063 Args.push_back(ArgValue);
6069 llvm::Type *RetTy =
VoidTy;
6073 if (RetTy !=
V->getType()) {
6075 if (
auto *PtrTy = dyn_cast<llvm::PointerType>(RetTy)) {
6076 if (PtrTy->getAddressSpace() !=
V->getType()->getPointerAddressSpace()) {
6079 PtrTy->getAddressSpace()));
6085 if (
V->getType()->isX86_AMXTy())
6086 V =
Builder.CreateIntrinsic(Intrinsic::x86_cast_tile_to_vector, {RetTy},
6092 if (RetTy->isVoidTy())
6112 if (
V->getType()->isVoidTy())
6119 llvm_unreachable(
"No current target builtin returns complex");
6121 llvm_unreachable(
"Bad evaluation kind in EmitBuiltinExpr");
6138 unsigned BuiltinID,
const CallExpr *E,
6140 llvm::Triple::ArchType Arch) {
6152 case llvm::Triple::arm:
6153 case llvm::Triple::armeb:
6154 case llvm::Triple::thumb:
6155 case llvm::Triple::thumbeb:
6157 case llvm::Triple::aarch64:
6158 case llvm::Triple::aarch64_32:
6159 case llvm::Triple::aarch64_be:
6161 case llvm::Triple::bpfeb:
6162 case llvm::Triple::bpfel:
6164 case llvm::Triple::x86:
6165 case llvm::Triple::x86_64:
6167 case llvm::Triple::ppc:
6168 case llvm::Triple::ppcle:
6169 case llvm::Triple::ppc64:
6170 case llvm::Triple::ppc64le:
6172 case llvm::Triple::r600:
6173 case llvm::Triple::amdgcn:
6175 case llvm::Triple::systemz:
6177 case llvm::Triple::nvptx:
6178 case llvm::Triple::nvptx64:
6180 case llvm::Triple::wasm32:
6181 case llvm::Triple::wasm64:
6183 case llvm::Triple::hexagon:
6185 case llvm::Triple::riscv32:
6186 case llvm::Triple::riscv64:
6197 assert(
getContext().getAuxTargetInfo() &&
"Missing aux target info");
6209 bool HasLegalHalfType =
true,
6211 bool AllowBFloatArgsAndRet =
true) {
6212 int IsQuad = TypeFlags.
isQuad();
6216 return llvm::FixedVectorType::get(CGF->
Int8Ty, V1Ty ? 1 : (8 << IsQuad));
6219 return llvm::FixedVectorType::get(CGF->
Int16Ty, V1Ty ? 1 : (4 << IsQuad));
6221 if (AllowBFloatArgsAndRet)
6222 return llvm::FixedVectorType::get(CGF->
BFloatTy, V1Ty ? 1 : (4 << IsQuad));
6224 return llvm::FixedVectorType::get(CGF->
Int16Ty, V1Ty ? 1 : (4 << IsQuad));
6226 if (HasLegalHalfType)
6227 return llvm::FixedVectorType::get(CGF->
HalfTy, V1Ty ? 1 : (4 << IsQuad));
6229 return llvm::FixedVectorType::get(CGF->
Int16Ty, V1Ty ? 1 : (4 << IsQuad));
6231 return llvm::FixedVectorType::get(CGF->
Int32Ty, V1Ty ? 1 : (2 << IsQuad));
6234 return llvm::FixedVectorType::get(CGF->
Int64Ty, V1Ty ? 1 : (1 << IsQuad));
6239 return llvm::FixedVectorType::get(CGF->
Int8Ty, 16);
6241 return llvm::FixedVectorType::get(CGF->
FloatTy, V1Ty ? 1 : (2 << IsQuad));
6243 return llvm::FixedVectorType::get(CGF->
DoubleTy, V1Ty ? 1 : (1 << IsQuad));
6245 llvm_unreachable(
"Unknown vector element type!");
6250 int IsQuad = IntTypeFlags.
isQuad();
6253 return llvm::FixedVectorType::get(CGF->
HalfTy, (4 << IsQuad));
6255 return llvm::FixedVectorType::get(CGF->
FloatTy, (2 << IsQuad));
6257 return llvm::FixedVectorType::get(CGF->
DoubleTy, (1 << IsQuad));
6259 llvm_unreachable(
"Type can't be converted to floating-point!");
6264 const ElementCount &Count) {
6265 Value *SV = llvm::ConstantVector::getSplat(Count,
C);
6266 return Builder.CreateShuffleVector(
V,
V, SV,
"lane");
6270 ElementCount EC = cast<llvm::VectorType>(
V->getType())->getElementCount();
6276 unsigned shift,
bool rightshift) {
6278 for (Function::const_arg_iterator ai = F->arg_begin(), ae = F->arg_end();
6279 ai != ae; ++ai, ++j) {
6280 if (F->isConstrainedFPIntrinsic())
6281 if (ai->getType()->isMetadataTy())
6283 if (shift > 0 && shift == j)
6286 Ops[j] =
Builder.CreateBitCast(Ops[j], ai->getType(), name);
6289 if (F->isConstrainedFPIntrinsic())
6290 return Builder.CreateConstrainedFPCall(F, Ops, name);
6292 return Builder.CreateCall(F, Ops, name);
6297 int SV = cast<ConstantInt>(
V)->getSExtValue();
6298 return ConstantInt::get(Ty, neg ? -SV : SV);
6303 llvm::Type *Ty,
bool usgn,
6305 llvm::VectorType *VTy = cast<llvm::VectorType>(Ty);
6307 int ShiftAmt = cast<ConstantInt>(Shift)->getSExtValue();
6308 int EltSize = VTy->getScalarSizeInBits();
6310 Vec =
Builder.CreateBitCast(Vec, Ty);
6314 if (ShiftAmt == EltSize) {
6317 return llvm::ConstantAggregateZero::get(VTy);
6322 Shift = ConstantInt::get(VTy->getElementType(), ShiftAmt);
6328 return Builder.CreateLShr(Vec, Shift, name);
6330 return Builder.CreateAShr(Vec, Shift, name);
6356struct ARMVectorIntrinsicInfo {
6357 const char *NameHint;
6359 unsigned LLVMIntrinsic;
6360 unsigned AltLLVMIntrinsic;
6363 bool operator<(
unsigned RHSBuiltinID)
const {
6364 return BuiltinID < RHSBuiltinID;
6366 bool operator<(
const ARMVectorIntrinsicInfo &TE)
const {
6367 return BuiltinID < TE.BuiltinID;
6372#define NEONMAP0(NameBase) \
6373 { #NameBase, NEON::BI__builtin_neon_ ## NameBase, 0, 0, 0 }
6375#define NEONMAP1(NameBase, LLVMIntrinsic, TypeModifier) \
6376 { #NameBase, NEON:: BI__builtin_neon_ ## NameBase, \
6377 Intrinsic::LLVMIntrinsic, 0, TypeModifier }
6379#define NEONMAP2(NameBase, LLVMIntrinsic, AltLLVMIntrinsic, TypeModifier) \
6380 { #NameBase, NEON:: BI__builtin_neon_ ## NameBase, \
6381 Intrinsic::LLVMIntrinsic, Intrinsic::AltLLVMIntrinsic, \
6385 NEONMAP1(__a32_vcvt_bf16_f32, arm_neon_vcvtfp2bf, 0),
6392 NEONMAP1(vabs_v, arm_neon_vabs, 0),
6393 NEONMAP1(vabsq_v, arm_neon_vabs, 0),
6397 NEONMAP1(vaesdq_u8, arm_neon_aesd, 0),
6398 NEONMAP1(vaeseq_u8, arm_neon_aese, 0),
6399 NEONMAP1(vaesimcq_u8, arm_neon_aesimc, 0),
6400 NEONMAP1(vaesmcq_u8, arm_neon_aesmc, 0),
6401 NEONMAP1(vbfdot_f32, arm_neon_bfdot, 0),
6402 NEONMAP1(vbfdotq_f32, arm_neon_bfdot, 0),
6403 NEONMAP1(vbfmlalbq_f32, arm_neon_bfmlalb, 0),
6404 NEONMAP1(vbfmlaltq_f32, arm_neon_bfmlalt, 0),
6405 NEONMAP1(vbfmmlaq_f32, arm_neon_bfmmla, 0),
6418 NEONMAP1(vcage_v, arm_neon_vacge, 0),
6419 NEONMAP1(vcageq_v, arm_neon_vacge, 0),
6420 NEONMAP1(vcagt_v, arm_neon_vacgt, 0),
6421 NEONMAP1(vcagtq_v, arm_neon_vacgt, 0),
6422 NEONMAP1(vcale_v, arm_neon_vacge, 0),
6423 NEONMAP1(vcaleq_v, arm_neon_vacge, 0),
6424 NEONMAP1(vcalt_v, arm_neon_vacgt, 0),
6425 NEONMAP1(vcaltq_v, arm_neon_vacgt, 0),
6442 NEONMAP1(vcvt_f16_f32, arm_neon_vcvtfp2hf, 0),
6445 NEONMAP1(vcvt_f32_f16, arm_neon_vcvthf2fp, 0),
6447 NEONMAP1(vcvt_n_f16_s16, arm_neon_vcvtfxs2fp, 0),
6448 NEONMAP1(vcvt_n_f16_u16, arm_neon_vcvtfxu2fp, 0),
6449 NEONMAP2(vcvt_n_f32_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0),
6450 NEONMAP1(vcvt_n_s16_f16, arm_neon_vcvtfp2fxs, 0),
6451 NEONMAP1(vcvt_n_s32_v, arm_neon_vcvtfp2fxs, 0),
6452 NEONMAP1(vcvt_n_s64_v, arm_neon_vcvtfp2fxs, 0),
6453 NEONMAP1(vcvt_n_u16_f16, arm_neon_vcvtfp2fxu, 0),
6454 NEONMAP1(vcvt_n_u32_v, arm_neon_vcvtfp2fxu, 0),
6455 NEONMAP1(vcvt_n_u64_v, arm_neon_vcvtfp2fxu, 0),
6462 NEONMAP1(vcvta_s16_f16, arm_neon_vcvtas, 0),
6463 NEONMAP1(vcvta_s32_v, arm_neon_vcvtas, 0),
6464 NEONMAP1(vcvta_s64_v, arm_neon_vcvtas, 0),
6465 NEONMAP1(vcvta_u16_f16, arm_neon_vcvtau, 0),
6466 NEONMAP1(vcvta_u32_v, arm_neon_vcvtau, 0),
6467 NEONMAP1(vcvta_u64_v, arm_neon_vcvtau, 0),
6468 NEONMAP1(vcvtaq_s16_f16, arm_neon_vcvtas, 0),
6469 NEONMAP1(vcvtaq_s32_v, arm_neon_vcvtas, 0),
6470 NEONMAP1(vcvtaq_s64_v, arm_neon_vcvtas, 0),
6471 NEONMAP1(vcvtaq_u16_f16, arm_neon_vcvtau, 0),
6472 NEONMAP1(vcvtaq_u32_v, arm_neon_vcvtau, 0),
6473 NEONMAP1(vcvtaq_u64_v, arm_neon_vcvtau, 0),
6474 NEONMAP1(vcvth_bf16_f32, arm_neon_vcvtbfp2bf, 0),
6475 NEONMAP1(vcvtm_s16_f16, arm_neon_vcvtms, 0),
6476 NEONMAP1(vcvtm_s32_v, arm_neon_vcvtms, 0),
6477 NEONMAP1(vcvtm_s64_v, arm_neon_vcvtms, 0),
6478 NEONMAP1(vcvtm_u16_f16, arm_neon_vcvtmu, 0),
6479 NEONMAP1(vcvtm_u32_v, arm_neon_vcvtmu, 0),
6480 NEONMAP1(vcvtm_u64_v, arm_neon_vcvtmu, 0),
6481 NEONMAP1(vcvtmq_s16_f16, arm_neon_vcvtms, 0),
6482 NEONMAP1(vcvtmq_s32_v, arm_neon_vcvtms, 0),
6483 NEONMAP1(vcvtmq_s64_v, arm_neon_vcvtms, 0),
6484 NEONMAP1(vcvtmq_u16_f16, arm_neon_vcvtmu, 0),
6485 NEONMAP1(vcvtmq_u32_v, arm_neon_vcvtmu, 0),
6486 NEONMAP1(vcvtmq_u64_v, arm_neon_vcvtmu, 0),
6487 NEONMAP1(vcvtn_s16_f16, arm_neon_vcvtns, 0),
6488 NEONMAP1(vcvtn_s32_v, arm_neon_vcvtns, 0),
6489 NEONMAP1(vcvtn_s64_v, arm_neon_vcvtns, 0),
6490 NEONMAP1(vcvtn_u16_f16, arm_neon_vcvtnu, 0),
6491 NEONMAP1(vcvtn_u32_v, arm_neon_vcvtnu, 0),
6492 NEONMAP1(vcvtn_u64_v, arm_neon_vcvtnu, 0),
6493 NEONMAP1(vcvtnq_s16_f16, arm_neon_vcvtns, 0),
6494 NEONMAP1(vcvtnq_s32_v, arm_neon_vcvtns, 0),
6495 NEONMAP1(vcvtnq_s64_v, arm_neon_vcvtns, 0),
6496 NEONMAP1(vcvtnq_u16_f16, arm_neon_vcvtnu, 0),
6497 NEONMAP1(vcvtnq_u32_v, arm_neon_vcvtnu, 0),
6498 NEONMAP1(vcvtnq_u64_v, arm_neon_vcvtnu, 0),
6499 NEONMAP1(vcvtp_s16_f16, arm_neon_vcvtps, 0),
6500 NEONMAP1(vcvtp_s32_v, arm_neon_vcvtps, 0),
6501 NEONMAP1(vcvtp_s64_v, arm_neon_vcvtps, 0),
6502 NEONMAP1(vcvtp_u16_f16, arm_neon_vcvtpu, 0),
6503 NEONMAP1(vcvtp_u32_v, arm_neon_vcvtpu, 0),
6504 NEONMAP1(vcvtp_u64_v, arm_neon_vcvtpu, 0),
6505 NEONMAP1(vcvtpq_s16_f16, arm_neon_vcvtps, 0),
6506 NEONMAP1(vcvtpq_s32_v, arm_neon_vcvtps, 0),
6507 NEONMAP1(vcvtpq_s64_v, arm_neon_vcvtps, 0),
6508 NEONMAP1(vcvtpq_u16_f16, arm_neon_vcvtpu, 0),
6509 NEONMAP1(vcvtpq_u32_v, arm_neon_vcvtpu, 0),
6510 NEONMAP1(vcvtpq_u64_v, arm_neon_vcvtpu, 0),
6514 NEONMAP1(vcvtq_n_f16_s16, arm_neon_vcvtfxs2fp, 0),
6515 NEONMAP1(vcvtq_n_f16_u16, arm_neon_vcvtfxu2fp, 0),
6516 NEONMAP2(vcvtq_n_f32_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0),
6517 NEONMAP1(vcvtq_n_s16_f16, arm_neon_vcvtfp2fxs, 0),
6518 NEONMAP1(vcvtq_n_s32_v, arm_neon_vcvtfp2fxs, 0),
6519 NEONMAP1(vcvtq_n_s64_v, arm_neon_vcvtfp2fxs, 0),
6520 NEONMAP1(vcvtq_n_u16_f16, arm_neon_vcvtfp2fxu, 0),
6521 NEONMAP1(vcvtq_n_u32_v, arm_neon_vcvtfp2fxu, 0),
6522 NEONMAP1(vcvtq_n_u64_v, arm_neon_vcvtfp2fxu, 0),
6529 NEONMAP1(vdot_s32, arm_neon_sdot, 0),
6530 NEONMAP1(vdot_u32, arm_neon_udot, 0),
6531 NEONMAP1(vdotq_s32, arm_neon_sdot, 0),
6532 NEONMAP1(vdotq_u32, arm_neon_udot, 0),
6542 NEONMAP1(vld1_v, arm_neon_vld1, 0),
6543 NEONMAP1(vld1_x2_v, arm_neon_vld1x2, 0),
6544 NEONMAP1(vld1_x3_v, arm_neon_vld1x3, 0),
6545 NEONMAP1(vld1_x4_v, arm_neon_vld1x4, 0),
6547 NEONMAP1(vld1q_v, arm_neon_vld1, 0),
6548 NEONMAP1(vld1q_x2_v, arm_neon_vld1x2, 0),
6549 NEONMAP1(vld1q_x3_v, arm_neon_vld1x3, 0),
6550 NEONMAP1(vld1q_x4_v, arm_neon_vld1x4, 0),
6551 NEONMAP1(vld2_dup_v, arm_neon_vld2dup, 0),
6552 NEONMAP1(vld2_lane_v, arm_neon_vld2lane, 0),
6553 NEONMAP1(vld2_v, arm_neon_vld2, 0),
6554 NEONMAP1(vld2q_dup_v, arm_neon_vld2dup, 0),
6555 NEONMAP1(vld2q_lane_v, arm_neon_vld2lane, 0),
6556 NEONMAP1(vld2q_v, arm_neon_vld2, 0),
6557 NEONMAP1(vld3_dup_v, arm_neon_vld3dup, 0),
6558 NEONMAP1(vld3_lane_v, arm_neon_vld3lane, 0),
6559 NEONMAP1(vld3_v, arm_neon_vld3, 0),
6560 NEONMAP1(vld3q_dup_v, arm_neon_vld3dup, 0),
6561 NEONMAP1(vld3q_lane_v, arm_neon_vld3lane, 0),
6562 NEONMAP1(vld3q_v, arm_neon_vld3, 0),
6563 NEONMAP1(vld4_dup_v, arm_neon_vld4dup, 0),
6564 NEONMAP1(vld4_lane_v, arm_neon_vld4lane, 0),
6565 NEONMAP1(vld4_v, arm_neon_vld4, 0),
6566 NEONMAP1(vld4q_dup_v, arm_neon_vld4dup, 0),
6567 NEONMAP1(vld4q_lane_v, arm_neon_vld4lane, 0),
6568 NEONMAP1(vld4q_v, arm_neon_vld4, 0),
6577 NEONMAP1(vmmlaq_s32, arm_neon_smmla, 0),
6578 NEONMAP1(vmmlaq_u32, arm_neon_ummla, 0),
6596 NEONMAP2(vqdmlal_v, arm_neon_vqdmull, sadd_sat, 0),
6597 NEONMAP2(vqdmlsl_v, arm_neon_vqdmull, ssub_sat, 0),
6621 NEONMAP1(vqshlu_n_v, arm_neon_vqshiftsu, 0),
6622 NEONMAP1(vqshluq_n_v, arm_neon_vqshiftsu, 0),
6626 NEONMAP2(vrecpe_v, arm_neon_vrecpe, arm_neon_vrecpe, 0),
6627 NEONMAP2(vrecpeq_v, arm_neon_vrecpe, arm_neon_vrecpe, 0),
6650 NEONMAP2(vrsqrte_v, arm_neon_vrsqrte, arm_neon_vrsqrte, 0),
6651 NEONMAP2(vrsqrteq_v, arm_neon_vrsqrte, arm_neon_vrsqrte, 0),
6655 NEONMAP1(vsha1su0q_u32, arm_neon_sha1su0, 0),
6656 NEONMAP1(vsha1su1q_u32, arm_neon_sha1su1, 0),
6657 NEONMAP1(vsha256h2q_u32, arm_neon_sha256h2, 0),
6658 NEONMAP1(vsha256hq_u32, arm_neon_sha256h, 0),
6659 NEONMAP1(vsha256su0q_u32, arm_neon_sha256su0, 0),
6660 NEONMAP1(vsha256su1q_u32, arm_neon_sha256su1, 0),
6669 NEONMAP1(vst1_v, arm_neon_vst1, 0),
6670 NEONMAP1(vst1_x2_v, arm_neon_vst1x2, 0),
6671 NEONMAP1(vst1_x3_v, arm_neon_vst1x3, 0),
6672 NEONMAP1(vst1_x4_v, arm_neon_vst1x4, 0),
6673 NEONMAP1(vst1q_v, arm_neon_vst1, 0),
6674 NEONMAP1(vst1q_x2_v, arm_neon_vst1x2, 0),
6675 NEONMAP1(vst1q_x3_v, arm_neon_vst1x3, 0),
6676 NEONMAP1(vst1q_x4_v, arm_neon_vst1x4, 0),
6677 NEONMAP1(vst2_lane_v, arm_neon_vst2lane, 0),
6678 NEONMAP1(vst2_v, arm_neon_vst2, 0),
6679 NEONMAP1(vst2q_lane_v, arm_neon_vst2lane, 0),
6680 NEONMAP1(vst2q_v, arm_neon_vst2, 0),
6681 NEONMAP1(vst3_lane_v, arm_neon_vst3lane, 0),
6682 NEONMAP1(vst3_v, arm_neon_vst3, 0),
6683 NEONMAP1(vst3q_lane_v, arm_neon_vst3lane, 0),
6684 NEONMAP1(vst3q_v, arm_neon_vst3, 0),
6685 NEONMAP1(vst4_lane_v, arm_neon_vst4lane, 0),
6686 NEONMAP1(vst4_v, arm_neon_vst4, 0),
6687 NEONMAP1(vst4q_lane_v, arm_neon_vst4lane, 0),
6688 NEONMAP1(vst4q_v, arm_neon_vst4, 0),
6694 NEONMAP1(vusdot_s32, arm_neon_usdot, 0),
6695 NEONMAP1(vusdotq_s32, arm_neon_usdot, 0),
6696 NEONMAP1(vusmmlaq_s32, arm_neon_usmmla, 0),
6704 NEONMAP1(__a64_vcvtq_low_bf16_f32, aarch64_neon_bfcvtn, 0),
6709 NEONMAP1(vabs_v, aarch64_neon_abs, 0),
6710 NEONMAP1(vabsq_v, aarch64_neon_abs, 0),
6715 NEONMAP1(vaesdq_u8, aarch64_crypto_aesd, 0),
6716 NEONMAP1(vaeseq_u8, aarch64_crypto_aese, 0),
6717 NEONMAP1(vaesimcq_u8, aarch64_crypto_aesimc, 0),
6718 NEONMAP1(vaesmcq_u8, aarch64_crypto_aesmc, 0),
6727 NEONMAP1(vbfdot_f32, aarch64_neon_bfdot, 0),
6728 NEONMAP1(vbfdotq_f32, aarch64_neon_bfdot, 0),
6729 NEONMAP1(vbfmlalbq_f32, aarch64_neon_bfmlalb, 0),
6730 NEONMAP1(vbfmlaltq_f32, aarch64_neon_bfmlalt, 0),
6731 NEONMAP1(vbfmmlaq_f32, aarch64_neon_bfmmla, 0),
6742 NEONMAP1(vcage_v, aarch64_neon_facge, 0),
6743 NEONMAP1(vcageq_v, aarch64_neon_facge, 0),
6744 NEONMAP1(vcagt_v, aarch64_neon_facgt, 0),
6745 NEONMAP1(vcagtq_v, aarch64_neon_facgt, 0),
6746 NEONMAP1(vcale_v, aarch64_neon_facge, 0),
6747 NEONMAP1(vcaleq_v, aarch64_neon_facge, 0),
6748 NEONMAP1(vcalt_v, aarch64_neon_facgt, 0),
6749 NEONMAP1(vcaltq_v, aarch64_neon_facgt, 0),
6786 NEONMAP1(vcvt_f16_f32, aarch64_neon_vcvtfp2hf, 0),
6789 NEONMAP1(vcvt_f32_f16, aarch64_neon_vcvthf2fp, 0),
6791 NEONMAP1(vcvt_n_f16_s16, aarch64_neon_vcvtfxs2fp, 0),
6792 NEONMAP1(vcvt_n_f16_u16, aarch64_neon_vcvtfxu2fp, 0),
6793 NEONMAP2(vcvt_n_f32_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
6794 NEONMAP2(vcvt_n_f64_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
6795 NEONMAP1(vcvt_n_s16_f16, aarch64_neon_vcvtfp2fxs, 0),
6796 NEONMAP1(vcvt_n_s32_v, aarch64_neon_vcvtfp2fxs, 0),
6797 NEONMAP1(vcvt_n_s64_v, aarch64_neon_vcvtfp2fxs, 0),
6798 NEONMAP1(vcvt_n_u16_f16, aarch64_neon_vcvtfp2fxu, 0),
6799 NEONMAP1(vcvt_n_u32_v, aarch64_neon_vcvtfp2fxu, 0),
6800 NEONMAP1(vcvt_n_u64_v, aarch64_neon_vcvtfp2fxu, 0),
6804 NEONMAP1(vcvtq_high_bf16_f32, aarch64_neon_bfcvtn2, 0),
6805 NEONMAP1(vcvtq_n_f16_s16, aarch64_neon_vcvtfxs2fp, 0),
6806 NEONMAP1(vcvtq_n_f16_u16, aarch64_neon_vcvtfxu2fp, 0),
6807 NEONMAP2(vcvtq_n_f32_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
6808 NEONMAP2(vcvtq_n_f64_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
6809 NEONMAP1(vcvtq_n_s16_f16, aarch64_neon_vcvtfp2fxs, 0),
6810 NEONMAP1(vcvtq_n_s32_v, aarch64_neon_vcvtfp2fxs, 0),
6811 NEONMAP1(vcvtq_n_s64_v, aarch64_neon_vcvtfp2fxs, 0),
6812 NEONMAP1(vcvtq_n_u16_f16, aarch64_neon_vcvtfp2fxu, 0),
6813 NEONMAP1(vcvtq_n_u32_v, aarch64_neon_vcvtfp2fxu, 0),
6814 NEONMAP1(vcvtq_n_u64_v, aarch64_neon_vcvtfp2fxu, 0),
6816 NEONMAP1(vdot_s32, aarch64_neon_sdot, 0),
6817 NEONMAP1(vdot_u32, aarch64_neon_udot, 0),
6818 NEONMAP1(vdotq_s32, aarch64_neon_sdot, 0),
6819 NEONMAP1(vdotq_u32, aarch64_neon_udot, 0),
6832 NEONMAP1(vfmlal_high_f16, aarch64_neon_fmlal2, 0),
6833 NEONMAP1(vfmlal_low_f16, aarch64_neon_fmlal, 0),
6834 NEONMAP1(vfmlalq_high_f16, aarch64_neon_fmlal2, 0),
6835 NEONMAP1(vfmlalq_low_f16, aarch64_neon_fmlal, 0),
6836 NEONMAP1(vfmlsl_high_f16, aarch64_neon_fmlsl2, 0),
6837 NEONMAP1(vfmlsl_low_f16, aarch64_neon_fmlsl, 0),
6838 NEONMAP1(vfmlslq_high_f16, aarch64_neon_fmlsl2, 0),
6839 NEONMAP1(vfmlslq_low_f16, aarch64_neon_fmlsl, 0),
6844 NEONMAP1(vld1_x2_v, aarch64_neon_ld1x2, 0),
6845 NEONMAP1(vld1_x3_v, aarch64_neon_ld1x3, 0),
6846 NEONMAP1(vld1_x4_v, aarch64_neon_ld1x4, 0),
6847 NEONMAP1(vld1q_x2_v, aarch64_neon_ld1x2, 0),
6848 NEONMAP1(vld1q_x3_v, aarch64_neon_ld1x3, 0),
6849 NEONMAP1(vld1q_x4_v, aarch64_neon_ld1x4, 0),
6850 NEONMAP1(vmmlaq_s32, aarch64_neon_smmla, 0),
6851 NEONMAP1(vmmlaq_u32, aarch64_neon_ummla, 0),
6864 NEONMAP2(vqdmlal_v, aarch64_neon_sqdmull, aarch64_neon_sqadd, 0),
6865 NEONMAP2(vqdmlsl_v, aarch64_neon_sqdmull, aarch64_neon_sqsub, 0),
6866 NEONMAP1(vqdmulh_lane_v, aarch64_neon_sqdmulh_lane, 0),
6867 NEONMAP1(vqdmulh_laneq_v, aarch64_neon_sqdmulh_laneq, 0),
6869 NEONMAP1(vqdmulhq_lane_v, aarch64_neon_sqdmulh_lane, 0),
6870 NEONMAP1(vqdmulhq_laneq_v, aarch64_neon_sqdmulh_laneq, 0),
6885 NEONMAP1(vqrdmulh_lane_v, aarch64_neon_sqrdmulh_lane, 0),
6886 NEONMAP1(vqrdmulh_laneq_v, aarch64_neon_sqrdmulh_laneq, 0),
6888 NEONMAP1(vqrdmulhq_lane_v, aarch64_neon_sqrdmulh_lane, 0),
6889 NEONMAP1(vqrdmulhq_laneq_v, aarch64_neon_sqrdmulh_laneq, 0),
6897 NEONMAP1(vqshlu_n_v, aarch64_neon_sqshlu, 0),
6898 NEONMAP1(vqshluq_n_v, aarch64_neon_sqshlu, 0),
6902 NEONMAP1(vrax1q_u64, aarch64_crypto_rax1, 0),
6903 NEONMAP2(vrecpe_v, aarch64_neon_frecpe, aarch64_neon_urecpe, 0),
6904 NEONMAP2(vrecpeq_v, aarch64_neon_frecpe, aarch64_neon_urecpe, 0),
6931 NEONMAP2(vrsqrte_v, aarch64_neon_frsqrte, aarch64_neon_ursqrte, 0),
6932 NEONMAP2(vrsqrteq_v, aarch64_neon_frsqrte, aarch64_neon_ursqrte, 0),
6936 NEONMAP1(vsha1su0q_u32, aarch64_crypto_sha1su0, 0),
6937 NEONMAP1(vsha1su1q_u32, aarch64_crypto_sha1su1, 0),
6938 NEONMAP1(vsha256h2q_u32, aarch64_crypto_sha256h2, 0),
6939 NEONMAP1(vsha256hq_u32, aarch64_crypto_sha256h, 0),
6940 NEONMAP1(vsha256su0q_u32, aarch64_crypto_sha256su0, 0),
6941 NEONMAP1(vsha256su1q_u32, aarch64_crypto_sha256su1, 0),
6942 NEONMAP1(vsha512h2q_u64, aarch64_crypto_sha512h2, 0),
6943 NEONMAP1(vsha512hq_u64, aarch64_crypto_sha512h, 0),
6944 NEONMAP1(vsha512su0q_u64, aarch64_crypto_sha512su0, 0),
6945 NEONMAP1(vsha512su1q_u64, aarch64_crypto_sha512su1, 0),
6954 NEONMAP1(vsm3partw1q_u32, aarch64_crypto_sm3partw1, 0),
6955 NEONMAP1(vsm3partw2q_u32, aarch64_crypto_sm3partw2, 0),
6956 NEONMAP1(vsm3ss1q_u32, aarch64_crypto_sm3ss1, 0),
6957 NEONMAP1(vsm3tt1aq_u32, aarch64_crypto_sm3tt1a, 0),
6958 NEONMAP1(vsm3tt1bq_u32, aarch64_crypto_sm3tt1b, 0),
6959 NEONMAP1(vsm3tt2aq_u32, aarch64_crypto_sm3tt2a, 0),
6960 NEONMAP1(vsm3tt2bq_u32, aarch64_crypto_sm3tt2b, 0),
6961 NEONMAP1(vsm4ekeyq_u32, aarch64_crypto_sm4ekey, 0),
6962 NEONMAP1(vsm4eq_u32, aarch64_crypto_sm4e, 0),
6963 NEONMAP1(vst1_x2_v, aarch64_neon_st1x2, 0),
6964 NEONMAP1(vst1_x3_v, aarch64_neon_st1x3, 0),
6965 NEONMAP1(vst1_x4_v, aarch64_neon_st1x4, 0),
6966 NEONMAP1(vst1q_x2_v, aarch64_neon_st1x2, 0),
6967 NEONMAP1(vst1q_x3_v, aarch64_neon_st1x3, 0),
6968 NEONMAP1(vst1q_x4_v, aarch64_neon_st1x4, 0),
6972 NEONMAP1(vusdot_s32, aarch64_neon_usdot, 0),
6973 NEONMAP1(vusdotq_s32, aarch64_neon_usdot, 0),
6974 NEONMAP1(vusmmlaq_s32, aarch64_neon_usmmla, 0),
6975 NEONMAP1(vxarq_u64, aarch64_crypto_xar, 0),
7013 NEONMAP1(vcvth_bf16_f32, aarch64_neon_bfcvt, 0),
7032 NEONMAP1(vcvtxd_f32_f64, aarch64_sisd_fcvtxn, 0),
7053 NEONMAP1(vmull_p64, aarch64_neon_pmull64, 0),
7081 NEONMAP1(vqdmulls_s32, aarch64_neon_sqdmulls_scalar, 0),
7162 NEONMAP1(vsha1cq_u32, aarch64_crypto_sha1c, 0),
7163 NEONMAP1(vsha1h_u32, aarch64_crypto_sha1h, 0),
7164 NEONMAP1(vsha1mq_u32, aarch64_crypto_sha1m, 0),
7165 NEONMAP1(vsha1pq_u32, aarch64_crypto_sha1p, 0),
7219 { NEON::BI__builtin_neon_splat_lane_bf16, NEON::BI__builtin_neon_splat_lane_v, },
7220 { NEON::BI__builtin_neon_splat_laneq_bf16, NEON::BI__builtin_neon_splat_laneq_v, },
7221 { NEON::BI__builtin_neon_splatq_lane_bf16, NEON::BI__builtin_neon_splatq_lane_v, },
7222 { NEON::BI__builtin_neon_splatq_laneq_bf16, NEON::BI__builtin_neon_splatq_laneq_v, },
7223 { NEON::BI__builtin_neon_vabd_f16, NEON::BI__builtin_neon_vabd_v, },
7224 { NEON::BI__builtin_neon_vabdq_f16, NEON::BI__builtin_neon_vabdq_v, },
7225 { NEON::BI__builtin_neon_vabs_f16, NEON::BI__builtin_neon_vabs_v, },
7226 { NEON::BI__builtin_neon_vabsq_f16, NEON::BI__builtin_neon_vabsq_v, },
7227 { NEON::BI__builtin_neon_vcage_f16, NEON::BI__builtin_neon_vcage_v, },
7228 { NEON::BI__builtin_neon_vcageq_f16, NEON::BI__builtin_neon_vcageq_v, },
7229 { NEON::BI__builtin_neon_vcagt_f16, NEON::BI__builtin_neon_vcagt_v, },
7230 { NEON::BI__builtin_neon_vcagtq_f16, NEON::BI__builtin_neon_vcagtq_v, },
7231 { NEON::BI__builtin_neon_vcale_f16, NEON::BI__builtin_neon_vcale_v, },
7232 { NEON::BI__builtin_neon_vcaleq_f16, NEON::BI__builtin_neon_vcaleq_v, },
7233 { NEON::BI__builtin_neon_vcalt_f16, NEON::BI__builtin_neon_vcalt_v, },
7234 { NEON::BI__builtin_neon_vcaltq_f16, NEON::BI__builtin_neon_vcaltq_v, },
7235 { NEON::BI__builtin_neon_vceqz_f16, NEON::BI__builtin_neon_vceqz_v, },
7236 { NEON::BI__builtin_neon_vceqzq_f16, NEON::BI__builtin_neon_vceqzq_v, },
7237 { NEON::BI__builtin_neon_vcgez_f16, NEON::BI__builtin_neon_vcgez_v, },
7238 { NEON::BI__builtin_neon_vcgezq_f16, NEON::BI__builtin_neon_vcgezq_v, },
7239 { NEON::BI__builtin_neon_vcgtz_f16, NEON::BI__builtin_neon_vcgtz_v, },
7240 { NEON::BI__builtin_neon_vcgtzq_f16, NEON::BI__builtin_neon_vcgtzq_v, },
7241 { NEON::BI__builtin_neon_vclez_f16, NEON::BI__builtin_neon_vclez_v, },
7242 { NEON::BI__builtin_neon_vclezq_f16, NEON::BI__builtin_neon_vclezq_v, },
7243 { NEON::BI__builtin_neon_vcltz_f16, NEON::BI__builtin_neon_vcltz_v, },
7244 { NEON::BI__builtin_neon_vcltzq_f16, NEON::BI__builtin_neon_vcltzq_v, },
7245 { NEON::BI__builtin_neon_vfma_f16, NEON::BI__builtin_neon_vfma_v, },
7246 { NEON::BI__builtin_neon_vfma_lane_f16, NEON::BI__builtin_neon_vfma_lane_v, },
7247 { NEON::BI__builtin_neon_vfma_laneq_f16, NEON::BI__builtin_neon_vfma_laneq_v, },
7248 { NEON::BI__builtin_neon_vfmaq_f16, NEON::BI__builtin_neon_vfmaq_v, },
7249 { NEON::BI__builtin_neon_vfmaq_lane_f16, NEON::BI__builtin_neon_vfmaq_lane_v, },
7250 { NEON::BI__builtin_neon_vfmaq_laneq_f16, NEON::BI__builtin_neon_vfmaq_laneq_v, },
7251 { NEON::BI__builtin_neon_vld1_bf16_x2, NEON::BI__builtin_neon_vld1_x2_v },
7252 { NEON::BI__builtin_neon_vld1_bf16_x3, NEON::BI__builtin_neon_vld1_x3_v },
7253 { NEON::BI__builtin_neon_vld1_bf16_x4, NEON::BI__builtin_neon_vld1_x4_v },
7254 { NEON::BI__builtin_neon_vld1_bf16, NEON::BI__builtin_neon_vld1_v },
7255 { NEON::BI__builtin_neon_vld1_dup_bf16, NEON::BI__builtin_neon_vld1_dup_v },
7256 { NEON::BI__builtin_neon_vld1_lane_bf16, NEON::BI__builtin_neon_vld1_lane_v },
7257 { NEON::BI__builtin_neon_vld1q_bf16_x2, NEON::BI__builtin_neon_vld1q_x2_v },
7258 { NEON::BI__builtin_neon_vld1q_bf16_x3, NEON::BI__builtin_neon_vld1q_x3_v },
7259 { NEON::BI__builtin_neon_vld1q_bf16_x4, NEON::BI__builtin_neon_vld1q_x4_v },
7260 { NEON::BI__builtin_neon_vld1q_bf16, NEON::BI__builtin_neon_vld1q_v },
7261 { NEON::BI__builtin_neon_vld1q_dup_bf16, NEON::BI__builtin_neon_vld1q_dup_v },
7262 { NEON::BI__builtin_neon_vld1q_lane_bf16, NEON::BI__builtin_neon_vld1q_lane_v },
7263 { NEON::BI__builtin_neon_vld2_bf16, NEON::BI__builtin_neon_vld2_v },
7264 { NEON::BI__builtin_neon_vld2_dup_bf16, NEON::BI__builtin_neon_vld2_dup_v },
7265 { NEON::BI__builtin_neon_vld2_lane_bf16, NEON::BI__builtin_neon_vld2_lane_v },
7266 { NEON::BI__builtin_neon_vld2q_bf16, NEON::BI__builtin_neon_vld2q_v },
7267 { NEON::BI__builtin_neon_vld2q_dup_bf16, NEON::BI__builtin_neon_vld2q_dup_v },
7268 { NEON::BI__builtin_neon_vld2q_lane_bf16, NEON::BI__builtin_neon_vld2q_lane_v },
7269 { NEON::BI__builtin_neon_vld3_bf16, NEON::BI__builtin_neon_vld3_v },
7270 { NEON::BI__builtin_neon_vld3_dup_bf16, NEON::BI__builtin_neon_vld3_dup_v },
7271 { NEON::BI__builtin_neon_vld3_lane_bf16, NEON::BI__builtin_neon_vld3_lane_v },
7272 { NEON::BI__builtin_neon_vld3q_bf16, NEON::BI__builtin_neon_vld3q_v },
7273 { NEON::BI__builtin_neon_vld3q_dup_bf16, NEON::BI__builtin_neon_vld3q_dup_v },
7274 { NEON::BI__builtin_neon_vld3q_lane_bf16, NEON::BI__builtin_neon_vld3q_lane_v },
7275 { NEON::BI__builtin_neon_vld4_bf16, NEON::BI__builtin_neon_vld4_v },
7276 { NEON::BI__builtin_neon_vld4_dup_bf16, NEON::BI__builtin_neon_vld4_dup_v },
7277 { NEON::BI__builtin_neon_vld4_lane_bf16, NEON::BI__builtin_neon_vld4_lane_v },
7278 { NEON::BI__builtin_neon_vld4q_bf16, NEON::BI__builtin_neon_vld4q_v },
7279 { NEON::BI__builtin_neon_vld4q_dup_bf16, NEON::BI__builtin_neon_vld4q_dup_v },
7280 { NEON::BI__builtin_neon_vld4q_lane_bf16, NEON::BI__builtin_neon_vld4q_lane_v },
7281 { NEON::BI__builtin_neon_vmax_f16, NEON::BI__builtin_neon_vmax_v, },
7282 { NEON::BI__builtin_neon_vmaxnm_f16, NEON::BI__builtin_neon_vmaxnm_v, },
7283 { NEON::BI__builtin_neon_vmaxnmq_f16, NEON::BI__builtin_neon_vmaxnmq_v, },
7284 { NEON::BI__builtin_neon_vmaxq_f16, NEON::BI__builtin_neon_vmaxq_v, },
7285 { NEON::BI__builtin_neon_vmin_f16, NEON::BI__builtin_neon_vmin_v, },
7286 { NEON::BI__builtin_neon_vminnm_f16, NEON::BI__builtin_neon_vminnm_v, },
7287 { NEON::BI__builtin_neon_vminnmq_f16, NEON::BI__builtin_neon_vminnmq_v, },
7288 { NEON::BI__builtin_neon_vminq_f16, NEON::BI__builtin_neon_vminq_v, },
7289 { NEON::BI__builtin_neon_vmulx_f16, NEON::BI__builtin_neon_vmulx_v, },
7290 { NEON::BI__builtin_neon_vmulxq_f16, NEON::BI__builtin_neon_vmulxq_v, },
7291 { NEON::BI__builtin_neon_vpadd_f16, NEON::BI__builtin_neon_vpadd_v, },
7292 { NEON::BI__builtin_neon_vpaddq_f16, NEON::BI__builtin_neon_vpaddq_v, },
7293 { NEON::BI__builtin_neon_vpmax_f16, NEON::BI__builtin_neon_vpmax_v, },
7294 { NEON::BI__builtin_neon_vpmaxnm_f16, NEON::BI__builtin_neon_vpmaxnm_v, },
7295 { NEON::BI__builtin_neon_vpmaxnmq_f16, NEON::BI__builtin_neon_vpmaxnmq_v, },
7296 { NEON::BI__builtin_neon_vpmaxq_f16, NEON::BI__builtin_neon_vpmaxq_v, },
7297 { NEON::BI__builtin_neon_vpmin_f16, NEON::BI__builtin_neon_vpmin_v, },
7298 { NEON::BI__builtin_neon_vpminnm_f16, NEON::BI__builtin_neon_vpminnm_v, },
7299 { NEON::BI__builtin_neon_vpminnmq_f16, NEON::BI__builtin_neon_vpminnmq_v, },
7300 { NEON::BI__builtin_neon_vpminq_f16, NEON::BI__builtin_neon_vpminq_v, },
7301 { NEON::BI__builtin_neon_vrecpe_f16, NEON::BI__builtin_neon_vrecpe_v, },
7302 { NEON::BI__builtin_neon_vrecpeq_f16, NEON::BI__builtin_neon_vrecpeq_v, },
7303 { NEON::BI__builtin_neon_vrecps_f16, NEON::BI__builtin_neon_vrecps_v, },
7304 { NEON::BI__builtin_neon_vrecpsq_f16, NEON::BI__builtin_neon_vrecpsq_v, },
7305 { NEON::BI__builtin_neon_vrnd_f16, NEON::BI__builtin_neon_vrnd_v, },
7306 { NEON::BI__builtin_neon_vrnda_f16, NEON::BI__builtin_neon_vrnda_v, },
7307 { NEON::BI__builtin_neon_vrndaq_f16, NEON::BI__builtin_neon_vrndaq_v, },
7308 { NEON::BI__builtin_neon_vrndi_f16, NEON::BI__builtin_neon_vrndi_v, },
7309 { NEON::BI__builtin_neon_vrndiq_f16, NEON::BI__builtin_neon_vrndiq_v, },
7310 { NEON::BI__builtin_neon_vrndm_f16, NEON::BI__builtin_neon_vrndm_v, },
7311 { NEON::BI__builtin_neon_vrndmq_f16, NEON::BI__builtin_neon_vrndmq_v, },
7312 { NEON::BI__builtin_neon_vrndn_f16, NEON::BI__builtin_neon_vrndn_v, },
7313 { NEON::BI__builtin_neon_vrndnq_f16, NEON::BI__builtin_neon_vrndnq_v, },
7314 { NEON::BI__builtin_neon_vrndp_f16, NEON::BI__builtin_neon_vrndp_v, },
7315 { NEON::BI__builtin_neon_vrndpq_f16, NEON::BI__builtin_neon_vrndpq_v, },
7316 { NEON::BI__builtin_neon_vrndq_f16, NEON::BI__builtin_neon_vrndq_v, },
7317 { NEON::BI__builtin_neon_vrndx_f16, NEON::BI__builtin_neon_vrndx_v, },
7318 { NEON::BI__builtin_neon_vrndxq_f16, NEON::BI__builtin_neon_vrndxq_v, },
7319 { NEON::BI__builtin_neon_vrsqrte_f16, NEON::BI__builtin_neon_vrsqrte_v, },
7320 { NEON::BI__builtin_neon_vrsqrteq_f16, NEON::BI__builtin_neon_vrsqrteq_v, },
7321 { NEON::BI__builtin_neon_vrsqrts_f16, NEON::BI__builtin_neon_vrsqrts_v, },
7322 { NEON::BI__builtin_neon_vrsqrtsq_f16, NEON::BI__builtin_neon_vrsqrtsq_v, },
7323 { NEON::BI__builtin_neon_vsqrt_f16, NEON::BI__builtin_neon_vsqrt_v, },
7324 { NEON::BI__builtin_neon_vsqrtq_f16, NEON::BI__builtin_neon_vsqrtq_v, },
7325 { NEON::BI__builtin_neon_vst1_bf16_x2, NEON::BI__builtin_neon_vst1_x2_v },
7326 { NEON::BI__builtin_neon_vst1_bf16_x3, NEON::BI__builtin_neon_vst1_x3_v },
7327 { NEON::BI__builtin_neon_vst1_bf16_x4, NEON::BI__builtin_neon_vst1_x4_v },
7328 { NEON::BI__builtin_neon_vst1_bf16, NEON::BI__builtin_neon_vst1_v },
7329 { NEON::BI__builtin_neon_vst1_lane_bf16, NEON::BI__builtin_neon_vst1_lane_v },
7330 { NEON::BI__builtin_neon_vst1q_bf16_x2, NEON::BI__builtin_neon_vst1q_x2_v },
7331 { NEON::BI__builtin_neon_vst1q_bf16_x3, NEON::BI__builtin_neon_vst1q_x3_v },
7332 { NEON::BI__builtin_neon_vst1q_bf16_x4, NEON::BI__builtin_neon_vst1q_x4_v },
7333 { NEON::BI__builtin_neon_vst1q_bf16, NEON::BI__builtin_neon_vst1q_v },
7334 { NEON::BI__builtin_neon_vst1q_lane_bf16, NEON::BI__builtin_neon_vst1q_lane_v },
7335 { NEON::BI__builtin_neon_vst2_bf16, NEON::BI__builtin_neon_vst2_v },
7336 { NEON::BI__builtin_neon_vst2_lane_bf16, NEON::BI__builtin_neon_vst2_lane_v },
7337 { NEON::BI__builtin_neon_vst2q_bf16, NEON::BI__builtin_neon_vst2q_v },
7338 { NEON::BI__builtin_neon_vst2q_lane_bf16, NEON::BI__builtin_neon_vst2q_lane_v },
7339 { NEON::BI__builtin_neon_vst3_bf16, NEON::BI__builtin_neon_vst3_v },
7340 { NEON::BI__builtin_neon_vst3_lane_bf16, NEON::BI__builtin_neon_vst3_lane_v },
7341 { NEON::BI__builtin_neon_vst3q_bf16, NEON::BI__builtin_neon_vst3q_v },
7342 { NEON::BI__builtin_neon_vst3q_lane_bf16, NEON::BI__builtin_neon_vst3q_lane_v },
7343 { NEON::BI__builtin_neon_vst4_bf16, NEON::BI__builtin_neon_vst4_v },
7344 { NEON::BI__builtin_neon_vst4_lane_bf16, NEON::BI__builtin_neon_vst4_lane_v },
7345 { NEON::BI__builtin_neon_vst4q_bf16, NEON::BI__builtin_neon_vst4q_v },
7346 { NEON::BI__builtin_neon_vst4q_lane_bf16, NEON::BI__builtin_neon_vst4q_lane_v },
7350 { NEON::BI__builtin_neon_vldap1_lane_u64, NEON::BI__builtin_neon_vldap1_lane_s64 },
7351 { NEON::BI__builtin_neon_vldap1_lane_f64, NEON::BI__builtin_neon_vldap1_lane_s64 },
7352 { NEON::BI__builtin_neon_vldap1_lane_p64, NEON::BI__builtin_neon_vldap1_lane_s64 },
7353 { NEON::BI__builtin_neon_vldap1q_lane_u64, NEON::BI__builtin_neon_vldap1q_lane_s64 },
7354 { NEON::BI__builtin_neon_vldap1q_lane_f64, NEON::BI__builtin_neon_vldap1q_lane_s64 },
7355 { NEON::BI__builtin_neon_vldap1q_lane_p64, NEON::BI__builtin_neon_vldap1q_lane_s64 },
7356 { NEON::BI__builtin_neon_vstl1_lane_u64, NEON::BI__builtin_neon_vstl1_lane_s64 },
7357 { NEON::BI__builtin_neon_vstl1_lane_f64, NEON::BI__builtin_neon_vstl1_lane_s64 },
7358 { NEON::BI__builtin_neon_vstl1_lane_p64, NEON::BI__builtin_neon_vstl1_lane_s64 },
7359 { NEON::BI__builtin_neon_vstl1q_lane_u64, NEON::BI__builtin_neon_vstl1q_lane_s64 },
7360 { NEON::BI__builtin_neon_vstl1q_lane_f64, NEON::BI__builtin_neon_vstl1q_lane_s64 },
7361 { NEON::BI__builtin_neon_vstl1q_lane_p64, NEON::BI__builtin_neon_vstl1q_lane_s64 },
7368#define SVEMAP1(NameBase, LLVMIntrinsic, TypeModifier) \
7370 #NameBase, SVE::BI__builtin_sve_##NameBase, Intrinsic::LLVMIntrinsic, 0, \
7374#define SVEMAP2(NameBase, TypeModifier) \
7375 { #NameBase, SVE::BI__builtin_sve_##NameBase, 0, 0, TypeModifier }
7377#define GET_SVE_LLVM_INTRINSIC_MAP
7378#include "clang/Basic/arm_sve_builtin_cg.inc"
7379#include "clang/Basic/BuiltinsAArch64NeonSVEBridge_cg.def"
7380#undef GET_SVE_LLVM_INTRINSIC_MAP
7386#define SMEMAP1(NameBase, LLVMIntrinsic, TypeModifier) \
7388 #NameBase, SME::BI__builtin_sme_##NameBase, Intrinsic::LLVMIntrinsic, 0, \
7392#define SMEMAP2(NameBase, TypeModifier) \
7393 { #NameBase, SME::BI__builtin_sme_##NameBase, 0, 0, TypeModifier }
7395#define GET_SME_LLVM_INTRINSIC_MAP
7396#include "clang/Basic/arm_sme_builtin_cg.inc"
7397#undef GET_SME_LLVM_INTRINSIC_MAP
7410static const ARMVectorIntrinsicInfo *
7412 unsigned BuiltinID,
bool &MapProvenSorted) {
7415 if (!MapProvenSorted) {
7416 assert(llvm::is_sorted(IntrinsicMap));
7417 MapProvenSorted =
true;
7421 const ARMVectorIntrinsicInfo *Builtin =
7422 llvm::lower_bound(IntrinsicMap, BuiltinID);
7424 if (Builtin != IntrinsicMap.end() && Builtin->BuiltinID == BuiltinID)
7432 llvm::Type *ArgType,
7445 Ty = llvm::FixedVectorType::get(
7446 Ty, VectorSize ? VectorSize / Ty->getPrimitiveSizeInBits() : 1);
7453 int Elts = VectorSize ? VectorSize / ArgType->getPrimitiveSizeInBits() : 1;
7454 ArgType = llvm::FixedVectorType::get(ArgType, Elts);
7458 Tys.push_back(ArgType);
7461 Tys.push_back(ArgType);
7472 unsigned BuiltinID = SISDInfo.BuiltinID;
7473 unsigned int Int = SISDInfo.LLVMIntrinsic;
7474 unsigned Modifier = SISDInfo.TypeModifier;
7475 const char *
s = SISDInfo.NameHint;
7477 switch (BuiltinID) {
7478 case NEON::BI__builtin_neon_vcled_s64:
7479 case NEON::BI__builtin_neon_vcled_u64:
7480 case NEON::BI__builtin_neon_vcles_f32:
7481 case NEON::BI__builtin_neon_vcled_f64:
7482 case NEON::BI__builtin_neon_vcltd_s64:
7483 case NEON::BI__builtin_neon_vcltd_u64:
7484 case NEON::BI__builtin_neon_vclts_f32:
7485 case NEON::BI__builtin_neon_vcltd_f64:
7486 case NEON::BI__builtin_neon_vcales_f32:
7487 case NEON::BI__builtin_neon_vcaled_f64:
7488 case NEON::BI__builtin_neon_vcalts_f32:
7489 case NEON::BI__builtin_neon_vcaltd_f64:
7493 std::swap(Ops[0], Ops[1]);
7497 assert(Int &&
"Generic code assumes a valid intrinsic");
7505 ConstantInt *C0 = ConstantInt::get(CGF.
SizeTy, 0);
7506 for (Function::const_arg_iterator ai = F->arg_begin(), ae = F->arg_end();
7507 ai != ae; ++ai, ++j) {
7508 llvm::Type *ArgTy = ai->getType();
7509 if (Ops[j]->getType()->getPrimitiveSizeInBits() ==
7510 ArgTy->getPrimitiveSizeInBits())
7513 assert(ArgTy->isVectorTy() && !Ops[j]->getType()->isVectorTy());
7516 Ops[j] = CGF.
Builder.CreateTruncOrBitCast(
7517 Ops[j], cast<llvm::VectorType>(ArgTy)->getElementType());
7519 CGF.
Builder.CreateInsertElement(PoisonValue::get(ArgTy), Ops[j], C0);
7524 if (ResultType->getPrimitiveSizeInBits().getFixedValue() <
7525 Result->getType()->getPrimitiveSizeInBits().getFixedValue())
7532 unsigned BuiltinID,
unsigned LLVMIntrinsic,
unsigned AltLLVMIntrinsic,
7533 const char *NameHint,
unsigned Modifier,
const CallExpr *E,
7535 llvm::Triple::ArchType Arch) {
7538 std::optional<llvm::APSInt> NeonTypeConst =
7545 bool Usgn =
Type.isUnsigned();
7546 bool Quad =
Type.isQuad();
7548 const bool AllowBFloatArgsAndRet =
7551 llvm::FixedVectorType *VTy =
7552 GetNeonType(
this,
Type, HasLegalHalfType,
false, AllowBFloatArgsAndRet);
7553 llvm::Type *Ty = VTy;
7557 auto getAlignmentValue32 = [&](
Address addr) ->
Value* {
7558 return Builder.getInt32(addr.getAlignment().getQuantity());
7561 unsigned Int = LLVMIntrinsic;
7563 Int = AltLLVMIntrinsic;
7565 switch (BuiltinID) {
7567 case NEON::BI__builtin_neon_splat_lane_v:
7568 case NEON::BI__builtin_neon_splat_laneq_v:
7569 case NEON::BI__builtin_neon_splatq_lane_v:
7570 case NEON::BI__builtin_neon_splatq_laneq_v: {
7571 auto NumElements = VTy->getElementCount();
7572 if (BuiltinID == NEON::BI__builtin_neon_splatq_lane_v)
7573 NumElements = NumElements * 2;
7574 if (BuiltinID == NEON::BI__builtin_neon_splat_laneq_v)
7575 NumElements = NumElements.divideCoefficientBy(2);
7577 Ops[0] =
Builder.CreateBitCast(Ops[0], VTy);
7578 return EmitNeonSplat(Ops[0], cast<ConstantInt>(Ops[1]), NumElements);
7580 case NEON::BI__builtin_neon_vpadd_v:
7581 case NEON::BI__builtin_neon_vpaddq_v:
7583 if (VTy->getElementType()->isFloatingPointTy() &&
7584 Int == Intrinsic::aarch64_neon_addp)
7585 Int = Intrinsic::aarch64_neon_faddp;
7587 case NEON::BI__builtin_neon_vabs_v:
7588 case NEON::BI__builtin_neon_vabsq_v:
7589 if (VTy->getElementType()->isFloatingPointTy())
7592 case NEON::BI__builtin_neon_vadd_v:
7593 case NEON::BI__builtin_neon_vaddq_v: {
7594 llvm::Type *VTy = llvm::FixedVectorType::get(
Int8Ty, Quad ? 16 : 8);
7595 Ops[0] =
Builder.CreateBitCast(Ops[0], VTy);
7596 Ops[1] =
Builder.CreateBitCast(Ops[1], VTy);
7597 Ops[0] =
Builder.CreateXor(Ops[0], Ops[1]);
7598 return Builder.CreateBitCast(Ops[0], Ty);
7600 case NEON::BI__builtin_neon_vaddhn_v: {
7601 llvm::FixedVectorType *SrcTy =
7602 llvm::FixedVectorType::getExtendedElementVectorType(VTy);
7605 Ops[0] =
Builder.CreateBitCast(Ops[0], SrcTy);
7606 Ops[1] =
Builder.CreateBitCast(Ops[1], SrcTy);
7607 Ops[0] =
Builder.CreateAdd(Ops[0], Ops[1],
"vaddhn");
7610 Constant *ShiftAmt =
7611 ConstantInt::get(SrcTy, SrcTy->getScalarSizeInBits() / 2);
7612 Ops[0] =
Builder.CreateLShr(Ops[0], ShiftAmt,
"vaddhn");
7615 return Builder.CreateTrunc(Ops[0], VTy,
"vaddhn");
7617 case NEON::BI__builtin_neon_vcale_v:
7618 case NEON::BI__builtin_neon_vcaleq_v:
7619 case NEON::BI__builtin_neon_vcalt_v:
7620 case NEON::BI__builtin_neon_vcaltq_v:
7621 std::swap(Ops[0], Ops[1]);
7623 case NEON::BI__builtin_neon_vcage_v:
7624 case NEON::BI__builtin_neon_vcageq_v:
7625 case NEON::BI__builtin_neon_vcagt_v:
7626 case NEON::BI__builtin_neon_vcagtq_v: {
7628 switch (VTy->getScalarSizeInBits()) {
7629 default: llvm_unreachable(
"unexpected type");
7640 auto *VecFlt = llvm::FixedVectorType::get(Ty, VTy->getNumElements());
7641 llvm::Type *Tys[] = { VTy, VecFlt };
7645 case NEON::BI__builtin_neon_vceqz_v:
7646 case NEON::BI__builtin_neon_vceqzq_v:
7648 ICmpInst::ICMP_EQ,
"vceqz");
7649 case NEON::BI__builtin_neon_vcgez_v:
7650 case NEON::BI__builtin_neon_vcgezq_v:
7652 ICmpInst::ICMP_SGE,
"vcgez");
7653 case NEON::BI__builtin_neon_vclez_v:
7654 case NEON::BI__builtin_neon_vclezq_v:
7656 ICmpInst::ICMP_SLE,
"vclez");
7657 case NEON::BI__builtin_neon_vcgtz_v:
7658 case NEON::BI__builtin_neon_vcgtzq_v:
7660 ICmpInst::ICMP_SGT,
"vcgtz");
7661 case NEON::BI__builtin_neon_vcltz_v:
7662 case NEON::BI__builtin_neon_vcltzq_v:
7664 ICmpInst::ICMP_SLT,
"vcltz");
7665 case NEON::BI__builtin_neon_vclz_v:
7666 case NEON::BI__builtin_neon_vclzq_v:
7671 case NEON::BI__builtin_neon_vcvt_f32_v:
7672 case NEON::BI__builtin_neon_vcvtq_f32_v:
7673 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
7676 return Usgn ?
Builder.CreateUIToFP(Ops[0], Ty,
"vcvt")
7677 :
Builder.CreateSIToFP(Ops[0], Ty,
"vcvt");
7678 case NEON::BI__builtin_neon_vcvt_f16_s16:
7679 case NEON::BI__builtin_neon_vcvt_f16_u16:
7680 case NEON::BI__builtin_neon_vcvtq_f16_s16:
7681 case NEON::BI__builtin_neon_vcvtq_f16_u16:
7682 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
7685 return Usgn ?
Builder.CreateUIToFP(Ops[0], Ty,
"vcvt")
7686 :
Builder.CreateSIToFP(Ops[0], Ty,
"vcvt");
7687 case NEON::BI__builtin_neon_vcvt_n_f16_s16:
7688 case NEON::BI__builtin_neon_vcvt_n_f16_u16:
7689 case NEON::BI__builtin_neon_vcvtq_n_f16_s16:
7690 case NEON::BI__builtin_neon_vcvtq_n_f16_u16: {
7695 case NEON::BI__builtin_neon_vcvt_n_f32_v:
7696 case NEON::BI__builtin_neon_vcvt_n_f64_v:
7697 case NEON::BI__builtin_neon_vcvtq_n_f32_v:
7698 case NEON::BI__builtin_neon_vcvtq_n_f64_v: {
7700 Int = Usgn ? LLVMIntrinsic : AltLLVMIntrinsic;
7704 case NEON::BI__builtin_neon_vcvt_n_s16_f16:
7705 case NEON::BI__builtin_neon_vcvt_n_s32_v:
7706 case NEON::BI__builtin_neon_vcvt_n_u16_f16:
7707 case NEON::BI__builtin_neon_vcvt_n_u32_v:
7708 case NEON::BI__builtin_neon_vcvt_n_s64_v:
7709 case NEON::BI__builtin_neon_vcvt_n_u64_v:
7710 case NEON::BI__builtin_neon_vcvtq_n_s16_f16:
7711 case NEON::BI__builtin_neon_vcvtq_n_s32_v:
7712 case NEON::BI__builtin_neon_vcvtq_n_u16_f16:
7713 case NEON::BI__builtin_neon_vcvtq_n_u32_v:
7714 case NEON::BI__builtin_neon_vcvtq_n_s64_v:
7715 case NEON::BI__builtin_neon_vcvtq_n_u64_v: {
7720 case NEON::BI__builtin_neon_vcvt_s32_v:
7721 case NEON::BI__builtin_neon_vcvt_u32_v:
7722 case NEON::BI__builtin_neon_vcvt_s64_v:
7723 case NEON::BI__builtin_neon_vcvt_u64_v:
7724 case NEON::BI__builtin_neon_vcvt_s16_f16:
7725 case NEON::BI__builtin_neon_vcvt_u16_f16:
7726 case NEON::BI__builtin_neon_vcvtq_s32_v:
7727 case NEON::BI__builtin_neon_vcvtq_u32_v:
7728 case NEON::BI__builtin_neon_vcvtq_s64_v:
7729 case NEON::BI__builtin_neon_vcvtq_u64_v:
7730 case NEON::BI__builtin_neon_vcvtq_s16_f16:
7731 case NEON::BI__builtin_neon_vcvtq_u16_f16: {
7733 return Usgn ?
Builder.CreateFPToUI(Ops[0], Ty,
"vcvt")
7734 :
Builder.CreateFPToSI(Ops[0], Ty,
"vcvt");
7736 case NEON::BI__builtin_neon_vcvta_s16_f16:
7737 case NEON::BI__builtin_neon_vcvta_s32_v:
7738 case NEON::BI__builtin_neon_vcvta_s64_v:
7739 case NEON::BI__builtin_neon_vcvta_u16_f16:
7740 case NEON::BI__builtin_neon_vcvta_u32_v:
7741 case NEON::BI__builtin_neon_vcvta_u64_v:
7742 case NEON::BI__builtin_neon_vcvtaq_s16_f16:
7743 case NEON::BI__builtin_neon_vcvtaq_s32_v:
7744 case NEON::BI__builtin_neon_vcvtaq_s64_v:
7745 case NEON::BI__builtin_neon_vcvtaq_u16_f16:
7746 case NEON::BI__builtin_neon_vcvtaq_u32_v:
7747 case NEON::BI__builtin_neon_vcvtaq_u64_v:
7748 case NEON::BI__builtin_neon_vcvtn_s16_f16:
7749 case NEON::BI__builtin_neon_vcvtn_s32_v:
7750 case NEON::BI__builtin_neon_vcvtn_s64_v:
7751 case NEON::BI__builtin_neon_vcvtn_u16_f16:
7752 case NEON::BI__builtin_neon_vcvtn_u32_v:
7753 case NEON::BI__builtin_neon_vcvtn_u64_v:
7754 case NEON::BI__builtin_neon_vcvtnq_s16_f16:
7755 case NEON::BI__builtin_neon_vcvtnq_s32_v:
7756 case NEON::BI__builtin_neon_vcvtnq_s64_v:
7757 case NEON::BI__builtin_neon_vcvtnq_u16_f16:
7758 case NEON::BI__builtin_neon_vcvtnq_u32_v:
7759 case NEON::BI__builtin_neon_vcvtnq_u64_v:
7760 case NEON::BI__builtin_neon_vcvtp_s16_f16:
7761 case NEON::BI__builtin_neon_vcvtp_s32_v:
7762 case NEON::BI__builtin_neon_vcvtp_s64_v:
7763 case NEON::BI__builtin_neon_vcvtp_u16_f16:
7764 case NEON::BI__builtin_neon_vcvtp_u32_v:
7765 case NEON::BI__builtin_neon_vcvtp_u64_v:
7766 case NEON::BI__builtin_neon_vcvtpq_s16_f16:
7767 case NEON::BI__builtin_neon_vcvtpq_s32_v:
7768 case NEON::BI__builtin_neon_vcvtpq_s64_v:
7769 case NEON::BI__builtin_neon_vcvtpq_u16_f16:
7770 case NEON::BI__builtin_neon_vcvtpq_u32_v:
7771 case NEON::BI__builtin_neon_vcvtpq_u64_v:
7772 case NEON::BI__builtin_neon_vcvtm_s16_f16:
7773 case NEON::BI__builtin_neon_vcvtm_s32_v:
7774 case NEON::BI__builtin_neon_vcvtm_s64_v:
7775 case NEON::BI__builtin_neon_vcvtm_u16_f16:
7776 case NEON::BI__builtin_neon_vcvtm_u32_v:
7777 case NEON::BI__builtin_neon_vcvtm_u64_v:
7778 case NEON::BI__builtin_neon_vcvtmq_s16_f16:
7779 case NEON::BI__builtin_neon_vcvtmq_s32_v:
7780 case NEON::BI__builtin_neon_vcvtmq_s64_v:
7781 case NEON::BI__builtin_neon_vcvtmq_u16_f16:
7782 case NEON::BI__builtin_neon_vcvtmq_u32_v:
7783 case NEON::BI__builtin_neon_vcvtmq_u64_v: {
7787 case NEON::BI__builtin_neon_vcvtx_f32_v: {
7788 llvm::Type *Tys[2] = { VTy->getTruncatedElementVectorType(VTy), Ty};
7792 case NEON::BI__builtin_neon_vext_v:
7793 case NEON::BI__builtin_neon_vextq_v: {
7794 int CV = cast<ConstantInt>(Ops[2])->getSExtValue();
7796 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
7797 Indices.push_back(i+CV);
7799 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
7800 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
7801 return Builder.CreateShuffleVector(Ops[0], Ops[1], Indices,
"vext");
7803 case NEON::BI__builtin_neon_vfma_v:
7804 case NEON::BI__builtin_neon_vfmaq_v: {
7805 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
7806 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
7807 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
7811 *
this, Intrinsic::fma, Intrinsic::experimental_constrained_fma, Ty,
7812 {Ops[1], Ops[2], Ops[0]});
7814 case NEON::BI__builtin_neon_vld1_v:
7815 case NEON::BI__builtin_neon_vld1q_v: {
7817 Ops.push_back(getAlignmentValue32(PtrOp0));
7820 case NEON::BI__builtin_neon_vld1_x2_v:
7821 case NEON::BI__builtin_neon_vld1q_x2_v:
7822 case NEON::BI__builtin_neon_vld1_x3_v:
7823 case NEON::BI__builtin_neon_vld1q_x3_v:
7824 case NEON::BI__builtin_neon_vld1_x4_v:
7825 case NEON::BI__builtin_neon_vld1q_x4_v: {
7828 Ops[1] =
Builder.CreateCall(F, Ops[1],
"vld1xN");
7831 case NEON::BI__builtin_neon_vld2_v:
7832 case NEON::BI__builtin_neon_vld2q_v:
7833 case NEON::BI__builtin_neon_vld3_v:
7834 case NEON::BI__builtin_neon_vld3q_v:
7835 case NEON::BI__builtin_neon_vld4_v:
7836 case NEON::BI__builtin_neon_vld4q_v:
7837 case NEON::BI__builtin_neon_vld2_dup_v:
7838 case NEON::BI__builtin_neon_vld2q_dup_v:
7839 case NEON::BI__builtin_neon_vld3_dup_v:
7840 case NEON::BI__builtin_neon_vld3q_dup_v:
7841 case NEON::BI__builtin_neon_vld4_dup_v:
7842 case NEON::BI__builtin_neon_vld4q_dup_v: {
7845 Value *Align = getAlignmentValue32(PtrOp1);
7846 Ops[1] =
Builder.CreateCall(F, {Ops[1], Align}, NameHint);
7849 case NEON::BI__builtin_neon_vld1_dup_v:
7850 case NEON::BI__builtin_neon_vld1q_dup_v: {
7851 Value *
V = PoisonValue::get(Ty);
7854 llvm::Constant *CI = ConstantInt::get(
SizeTy, 0);
7855 Ops[0] =
Builder.CreateInsertElement(
V, Ld, CI);
7858 case NEON::BI__builtin_neon_vld2_lane_v:
7859 case NEON::BI__builtin_neon_vld2q_lane_v:
7860 case NEON::BI__builtin_neon_vld3_lane_v:
7861 case NEON::BI__builtin_neon_vld3q_lane_v:
7862 case NEON::BI__builtin_neon_vld4_lane_v:
7863 case NEON::BI__builtin_neon_vld4q_lane_v: {
7866 for (
unsigned I = 2; I < Ops.size() - 1; ++I)
7867 Ops[I] =
Builder.CreateBitCast(Ops[I], Ty);
7868 Ops.push_back(getAlignmentValue32(PtrOp1));
7872 case NEON::BI__builtin_neon_vmovl_v: {
7873 llvm::FixedVectorType *DTy =
7874 llvm::FixedVectorType::getTruncatedElementVectorType(VTy);
7875 Ops[0] =
Builder.CreateBitCast(Ops[0], DTy);
7877 return Builder.CreateZExt(Ops[0], Ty,
"vmovl");
7878 return Builder.CreateSExt(Ops[0], Ty,
"vmovl");
7880 case NEON::BI__builtin_neon_vmovn_v: {
7881 llvm::FixedVectorType *QTy =
7882 llvm::FixedVectorType::getExtendedElementVectorType(VTy);
7883 Ops[0] =
Builder.CreateBitCast(Ops[0], QTy);
7884 return Builder.CreateTrunc(Ops[0], Ty,
"vmovn");
7886 case NEON::BI__builtin_neon_vmull_v:
7892 Int = Usgn ? Intrinsic::arm_neon_vmullu : Intrinsic::arm_neon_vmulls;
7895 case NEON::BI__builtin_neon_vpadal_v:
7896 case NEON::BI__builtin_neon_vpadalq_v: {
7898 unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits();
7902 llvm::FixedVectorType::get(EltTy, VTy->getNumElements() * 2);
7903 llvm::Type *Tys[2] = { Ty, NarrowTy };
7906 case NEON::BI__builtin_neon_vpaddl_v:
7907 case NEON::BI__builtin_neon_vpaddlq_v: {
7909 unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits();
7910 llvm::Type *EltTy = llvm::IntegerType::get(
getLLVMContext(), EltBits / 2);
7912 llvm::FixedVectorType::get(EltTy, VTy->getNumElements() * 2);
7913 llvm::Type *Tys[2] = { Ty, NarrowTy };
7916 case NEON::BI__builtin_neon_vqdmlal_v:
7917 case NEON::BI__builtin_neon_vqdmlsl_v: {
7924 case NEON::BI__builtin_neon_vqdmulhq_lane_v:
7925 case NEON::BI__builtin_neon_vqdmulh_lane_v:
7926 case NEON::BI__builtin_neon_vqrdmulhq_lane_v:
7927 case NEON::BI__builtin_neon_vqrdmulh_lane_v: {
7928 auto *RTy = cast<llvm::FixedVectorType>(Ty);
7929 if (BuiltinID == NEON::BI__builtin_neon_vqdmulhq_lane_v ||
7930 BuiltinID == NEON::BI__builtin_neon_vqrdmulhq_lane_v)
7931 RTy = llvm::FixedVectorType::get(RTy->getElementType(),
7932 RTy->getNumElements() * 2);
7933 llvm::Type *Tys[2] = {
7938 case NEON::BI__builtin_neon_vqdmulhq_laneq_v:
7939 case NEON::BI__builtin_neon_vqdmulh_laneq_v:
7940 case NEON::BI__builtin_neon_vqrdmulhq_laneq_v:
7941 case NEON::BI__builtin_neon_vqrdmulh_laneq_v: {
7942 llvm::Type *Tys[2] = {
7947 case NEON::BI__builtin_neon_vqshl_n_v:
7948 case NEON::BI__builtin_neon_vqshlq_n_v:
7951 case NEON::BI__builtin_neon_vqshlu_n_v:
7952 case NEON::BI__builtin_neon_vqshluq_n_v:
7955 case NEON::BI__builtin_neon_vrecpe_v:
7956 case NEON::BI__builtin_neon_vrecpeq_v:
7957 case NEON::BI__builtin_neon_vrsqrte_v:
7958 case NEON::BI__builtin_neon_vrsqrteq_v:
7959 Int = Ty->isFPOrFPVectorTy() ? LLVMIntrinsic : AltLLVMIntrinsic;
7961 case NEON::BI__builtin_neon_vrndi_v:
7962 case NEON::BI__builtin_neon_vrndiq_v:
7964 ? Intrinsic::experimental_constrained_nearbyint
7965 : Intrinsic::nearbyint;
7967 case NEON::BI__builtin_neon_vrshr_n_v:
7968 case NEON::BI__builtin_neon_vrshrq_n_v:
7971 case NEON::BI__builtin_neon_vsha512hq_u64:
7972 case NEON::BI__builtin_neon_vsha512h2q_u64:
7973 case NEON::BI__builtin_neon_vsha512su0q_u64:
7974 case NEON::BI__builtin_neon_vsha512su1q_u64: {
7978 case NEON::BI__builtin_neon_vshl_n_v:
7979 case NEON::BI__builtin_neon_vshlq_n_v:
7981 return Builder.CreateShl(
Builder.CreateBitCast(Ops[0],Ty), Ops[1],
7983 case NEON::BI__builtin_neon_vshll_n_v: {
7984 llvm::FixedVectorType *SrcTy =
7985 llvm::FixedVectorType::getTruncatedElementVectorType(VTy);
7986 Ops[0] =
Builder.CreateBitCast(Ops[0], SrcTy);
7988 Ops[0] =
Builder.CreateZExt(Ops[0], VTy);
7990 Ops[0] =
Builder.CreateSExt(Ops[0], VTy);
7992 return Builder.CreateShl(Ops[0], Ops[1],
"vshll_n");
7994 case NEON::BI__builtin_neon_vshrn_n_v: {
7995 llvm::FixedVectorType *SrcTy =
7996 llvm::FixedVectorType::getExtendedElementVectorType(VTy);
7997 Ops[0] =
Builder.CreateBitCast(Ops[0], SrcTy);
8000 Ops[0] =
Builder.CreateLShr(Ops[0], Ops[1]);
8002 Ops[0] =
Builder.CreateAShr(Ops[0], Ops[1]);
8003 return Builder.CreateTrunc(Ops[0], Ty,
"vshrn_n");
8005 case NEON::BI__builtin_neon_vshr_n_v:
8006 case NEON::BI__builtin_neon_vshrq_n_v:
8008 case NEON::BI__builtin_neon_vst1_v:
8009 case NEON::BI__builtin_neon_vst1q_v:
8010 case NEON::BI__builtin_neon_vst2_v:
8011 case NEON::BI__builtin_neon_vst2q_v:
8012 case NEON::BI__builtin_neon_vst3_v:
8013 case NEON::BI__builtin_neon_vst3q_v:
8014 case NEON::BI__builtin_neon_vst4_v:
8015 case NEON::BI__builtin_neon_vst4q_v:
8016 case NEON::BI__builtin_neon_vst2_lane_v:
8017 case NEON::BI__builtin_neon_vst2q_lane_v:
8018 case NEON::BI__builtin_neon_vst3_lane_v:
8019 case NEON::BI__builtin_neon_vst3q_lane_v:
8020 case NEON::BI__builtin_neon_vst4_lane_v:
8021 case NEON::BI__builtin_neon_vst4q_lane_v: {
8023 Ops.push_back(getAlignmentValue32(PtrOp0));
8026 case NEON::BI__builtin_neon_vsm3partw1q_u32:
8027 case NEON::BI__builtin_neon_vsm3partw2q_u32:
8028 case NEON::BI__builtin_neon_vsm3ss1q_u32:
8029 case NEON::BI__builtin_neon_vsm4ekeyq_u32:
8030 case NEON::BI__builtin_neon_vsm4eq_u32: {
8034 case NEON::BI__builtin_neon_vsm3tt1aq_u32:
8035 case NEON::BI__builtin_neon_vsm3tt1bq_u32:
8036 case NEON::BI__builtin_neon_vsm3tt2aq_u32:
8037 case NEON::BI__builtin_neon_vsm3tt2bq_u32: {
8042 case NEON::BI__builtin_neon_vst1_x2_v:
8043 case NEON::BI__builtin_neon_vst1q_x2_v:
8044 case NEON::BI__builtin_neon_vst1_x3_v:
8045 case NEON::BI__builtin_neon_vst1q_x3_v:
8046 case NEON::BI__builtin_neon_vst1_x4_v:
8047 case NEON::BI__builtin_neon_vst1q_x4_v: {
8050 if (Arch == llvm::Triple::aarch64 || Arch == llvm::Triple::aarch64_be ||
8051 Arch == llvm::Triple::aarch64_32) {
8053 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
8059 case NEON::BI__builtin_neon_vsubhn_v: {
8060 llvm::FixedVectorType *SrcTy =
8061 llvm::FixedVectorType::getExtendedElementVectorType(VTy);
8064 Ops[0] =
Builder.CreateBitCast(Ops[0], SrcTy);
8065 Ops[1] =
Builder.CreateBitCast(Ops[1], SrcTy);
8066 Ops[0] =
Builder.CreateSub(Ops[0], Ops[1],
"vsubhn");
8069 Constant *ShiftAmt =
8070 ConstantInt::get(SrcTy, SrcTy->getScalarSizeInBits() / 2);
8071 Ops[0] =
Builder.CreateLShr(Ops[0], ShiftAmt,
"vsubhn");
8074 return Builder.CreateTrunc(Ops[0], VTy,
"vsubhn");
8076 case NEON::BI__builtin_neon_vtrn_v:
8077 case NEON::BI__builtin_neon_vtrnq_v: {
8078 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
8079 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
8080 Value *SV =
nullptr;
8082 for (
unsigned vi = 0; vi != 2; ++vi) {
8084 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
8085 Indices.push_back(i+vi);
8086 Indices.push_back(i+e+vi);
8088 Value *Addr =
Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
8089 SV =
Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vtrn");
8094 case NEON::BI__builtin_neon_vtst_v:
8095 case NEON::BI__builtin_neon_vtstq_v: {
8096 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
8097 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
8098 Ops[0] =
Builder.CreateAnd(Ops[0], Ops[1]);
8099 Ops[0] =
Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0],
8100 ConstantAggregateZero::get(Ty));
8101 return Builder.CreateSExt(Ops[0], Ty,
"vtst");
8103 case NEON::BI__builtin_neon_vuzp_v:
8104 case NEON::BI__builtin_neon_vuzpq_v: {
8105 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
8106 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
8107 Value *SV =
nullptr;
8109 for (
unsigned vi = 0; vi != 2; ++vi) {
8111 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
8112 Indices.push_back(2*i+vi);
8114 Value *Addr =
Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
8115 SV =
Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vuzp");
8120 case NEON::BI__builtin_neon_vxarq_u64: {
8125 case NEON::BI__builtin_neon_vzip_v:
8126 case NEON::BI__builtin_neon_vzipq_v: {
8127 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
8128 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
8129 Value *SV =
nullptr;
8131 for (
unsigned vi = 0; vi != 2; ++vi) {
8133 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
8134 Indices.push_back((i + vi*e) >> 1);
8135 Indices.push_back(((i + vi*e) >> 1)+e);
8137 Value *Addr =
Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
8138 SV =
Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vzip");
8143 case NEON::BI__builtin_neon_vdot_s32:
8144 case NEON::BI__builtin_neon_vdot_u32:
8145 case NEON::BI__builtin_neon_vdotq_s32:
8146 case NEON::BI__builtin_neon_vdotq_u32: {
8148 llvm::FixedVectorType::get(
Int8Ty, Ty->getPrimitiveSizeInBits() / 8);
8149 llvm::Type *Tys[2] = { Ty, InputTy };
8152 case NEON::BI__builtin_neon_vfmlal_low_f16:
8153 case NEON::BI__builtin_neon_vfmlalq_low_f16: {
8155 llvm::FixedVectorType::get(
HalfTy, Ty->getPrimitiveSizeInBits() / 16);
8156 llvm::Type *Tys[2] = { Ty, InputTy };
8159 case NEON::BI__builtin_neon_vfmlsl_low_f16:
8160 case NEON::BI__builtin_neon_vfmlslq_low_f16: {
8162 llvm::FixedVectorType::get(
HalfTy, Ty->getPrimitiveSizeInBits() / 16);
8163 llvm::Type *Tys[2] = { Ty, InputTy };
8166 case NEON::BI__builtin_neon_vfmlal_high_f16:
8167 case NEON::BI__builtin_neon_vfmlalq_high_f16: {
8169 llvm::FixedVectorType::get(
HalfTy, Ty->getPrimitiveSizeInBits() / 16);
8170 llvm::Type *Tys[2] = { Ty, InputTy };
8173 case NEON::BI__builtin_neon_vfmlsl_high_f16:
8174 case NEON::BI__builtin_neon_vfmlslq_high_f16: {
8176 llvm::FixedVectorType::get(
HalfTy, Ty->getPrimitiveSizeInBits() / 16);
8177 llvm::Type *Tys[2] = { Ty, InputTy };
8180 case NEON::BI__builtin_neon_vmmlaq_s32:
8181 case NEON::BI__builtin_neon_vmmlaq_u32: {
8183 llvm::FixedVectorType::get(
Int8Ty, Ty->getPrimitiveSizeInBits() / 8);
8184 llvm::Type *Tys[2] = { Ty, InputTy };
8187 case NEON::BI__builtin_neon_vusmmlaq_s32: {
8189 llvm::FixedVectorType::get(
Int8Ty, Ty->getPrimitiveSizeInBits() / 8);
8190 llvm::Type *Tys[2] = { Ty, InputTy };
8193 case NEON::BI__builtin_neon_vusdot_s32:
8194 case NEON::BI__builtin_neon_vusdotq_s32: {
8196 llvm::FixedVectorType::get(
Int8Ty, Ty->getPrimitiveSizeInBits() / 8);
8197 llvm::Type *Tys[2] = { Ty, InputTy };
8200 case NEON::BI__builtin_neon_vbfdot_f32:
8201 case NEON::BI__builtin_neon_vbfdotq_f32: {
8202 llvm::Type *InputTy =
8203 llvm::FixedVectorType::get(
BFloatTy, Ty->getPrimitiveSizeInBits() / 16);
8204 llvm::Type *Tys[2] = { Ty, InputTy };
8207 case NEON::BI__builtin_neon___a32_vcvt_bf16_f32: {
8208 llvm::Type *Tys[1] = { Ty };
8215 assert(Int &&
"Expected valid intrinsic number");
8228 Value *Op, llvm::Type *Ty,
const CmpInst::Predicate Fp,
8229 const CmpInst::Predicate Ip,
const Twine &Name) {
8230 llvm::Type *OTy = Op->
getType();
8236 if (BitCastInst *BI = dyn_cast<BitCastInst>(Op))
8237 OTy = BI->getOperand(0)->getType();
8239 Op =
Builder.CreateBitCast(Op, OTy);
8240 if (OTy->getScalarType()->isFloatingPointTy()) {
8241 if (Fp == CmpInst::FCMP_OEQ)
8242 Op =
Builder.CreateFCmp(Fp, Op, Constant::getNullValue(OTy));
8244 Op =
Builder.CreateFCmpS(Fp, Op, Constant::getNullValue(OTy));
8246 Op =
Builder.CreateICmp(Ip, Op, Constant::getNullValue(OTy));
8248 return Builder.CreateSExt(Op, Ty, Name);
8253 llvm::Type *ResTy,
unsigned IntID,
8257 TblOps.push_back(ExtOp);
8261 auto *TblTy = cast<llvm::FixedVectorType>(Ops[0]->getType());
8262 for (
unsigned i = 0, e = TblTy->getNumElements(); i != e; ++i) {
8263 Indices.push_back(2*i);
8264 Indices.push_back(2*i+1);
8267 int PairPos = 0, End = Ops.size() - 1;
8268 while (PairPos < End) {
8269 TblOps.push_back(CGF.
Builder.CreateShuffleVector(Ops[PairPos],
8270 Ops[PairPos+1], Indices,
8277 if (PairPos == End) {
8278 Value *ZeroTbl = ConstantAggregateZero::get(TblTy);
8279 TblOps.push_back(CGF.
Builder.CreateShuffleVector(Ops[PairPos],
8280 ZeroTbl, Indices, Name));
8284 TblOps.push_back(IndexOp);
8290Value *CodeGenFunction::GetValueForARMHint(
unsigned BuiltinID) {
8292 switch (BuiltinID) {
8295 case clang::ARM::BI__builtin_arm_nop:
8298 case clang::ARM::BI__builtin_arm_yield:
8299 case clang::ARM::BI__yield:
8302 case clang::ARM::BI__builtin_arm_wfe:
8303 case clang::ARM::BI__wfe:
8306 case clang::ARM::BI__builtin_arm_wfi:
8307 case clang::ARM::BI__wfi:
8310 case clang::ARM::BI__builtin_arm_sev:
8311 case clang::ARM::BI__sev:
8314 case clang::ARM::BI__builtin_arm_sevl:
8315 case clang::ARM::BI__sevl:
8333 llvm::Type *RegisterType,
8334 llvm::Type *ValueType,
bool isExecHi) {
8339 llvm::Value *
Call = Builder.CreateCall(F, {Builder.getInt1(
true)});
8342 Value *Rt2 = Builder.CreateLShr(
Call, 32);
8343 Rt2 = Builder.CreateTrunc(Rt2, CGF.
Int32Ty);
8355 llvm::Type *RegisterType,
8356 llvm::Type *ValueType,
8358 StringRef SysReg =
"") {
8360 assert((RegisterType->isIntegerTy(32) || RegisterType->isIntegerTy(64) ||
8361 RegisterType->isIntegerTy(128)) &&
8362 "Unsupported size for register.");
8368 if (SysReg.empty()) {
8370 SysReg = cast<clang::StringLiteral>(SysRegStrExpr)->getString();
8373 llvm::Metadata *Ops[] = { llvm::MDString::get(Context, SysReg) };
8374 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
8375 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
8377 llvm::Type *Types[] = { RegisterType };
8379 bool MixedTypes = RegisterType->isIntegerTy(64) && ValueType->isIntegerTy(32);
8380 assert(!(RegisterType->isIntegerTy(32) && ValueType->isIntegerTy(64))
8381 &&
"Can't fit 64-bit value in 32-bit register");
8383 if (AccessKind !=
Write) {
8386 AccessKind ==
VolatileRead ? llvm::Intrinsic::read_volatile_register
8387 : llvm::Intrinsic::read_register,
8389 llvm::Value *
Call = Builder.CreateCall(F, Metadata);
8393 return Builder.CreateTrunc(
Call, ValueType);
8395 if (ValueType->isPointerTy())
8397 return Builder.CreateIntToPtr(
Call, ValueType);
8402 llvm::Function *F = CGM.
getIntrinsic(llvm::Intrinsic::write_register, Types);
8406 ArgValue = Builder.CreateZExt(ArgValue, RegisterType);
8407 return Builder.CreateCall(F, { Metadata, ArgValue });
8410 if (ValueType->isPointerTy()) {
8412 ArgValue = Builder.CreatePtrToInt(ArgValue, RegisterType);
8413 return Builder.CreateCall(F, { Metadata, ArgValue });
8416 return Builder.CreateCall(F, { Metadata, ArgValue });
8422 switch (BuiltinID) {
8424 case NEON::BI__builtin_neon_vget_lane_i8:
8425 case NEON::BI__builtin_neon_vget_lane_i16:
8426 case NEON::BI__builtin_neon_vget_lane_bf16:
8427 case NEON::BI__builtin_neon_vget_lane_i32:
8428 case NEON::BI__builtin_neon_vget_lane_i64:
8429 case NEON::BI__builtin_neon_vget_lane_f32:
8430 case NEON::BI__builtin_neon_vgetq_lane_i8:
8431 case NEON::BI__builtin_neon_vgetq_lane_i16:
8432 case NEON::BI__builtin_neon_vgetq_lane_bf16:
8433 case NEON::BI__builtin_neon_vgetq_lane_i32:
8434 case NEON::BI__builtin_neon_vgetq_lane_i64:
8435 case NEON::BI__builtin_neon_vgetq_lane_f32:
8436 case NEON::BI__builtin_neon_vduph_lane_bf16:
8437 case NEON::BI__builtin_neon_vduph_laneq_bf16:
8438 case NEON::BI__builtin_neon_vset_lane_i8:
8439 case NEON::BI__builtin_neon_vset_lane_i16:
8440 case NEON::BI__builtin_neon_vset_lane_bf16:
8441 case NEON::BI__builtin_neon_vset_lane_i32:
8442 case NEON::BI__builtin_neon_vset_lane_i64:
8443 case NEON::BI__builtin_neon_vset_lane_f32:
8444 case NEON::BI__builtin_neon_vsetq_lane_i8:
8445 case NEON::BI__builtin_neon_vsetq_lane_i16:
8446 case NEON::BI__builtin_neon_vsetq_lane_bf16:
8447 case NEON::BI__builtin_neon_vsetq_lane_i32:
8448 case NEON::BI__builtin_neon_vsetq_lane_i64:
8449 case NEON::BI__builtin_neon_vsetq_lane_f32:
8450 case NEON::BI__builtin_neon_vsha1h_u32:
8451 case NEON::BI__builtin_neon_vsha1cq_u32:
8452 case NEON::BI__builtin_neon_vsha1pq_u32:
8453 case NEON::BI__builtin_neon_vsha1mq_u32:
8454 case NEON::BI__builtin_neon_vcvth_bf16_f32:
8455 case clang::ARM::BI_MoveToCoprocessor:
8456 case clang::ARM::BI_MoveToCoprocessor2:
8465 llvm::Triple::ArchType Arch) {
8466 if (
auto Hint = GetValueForARMHint(BuiltinID))
8469 if (BuiltinID == clang::ARM::BI__emit) {
8471 llvm::FunctionType *FTy =
8472 llvm::FunctionType::get(
VoidTy,
false);
8476 llvm_unreachable(
"Sema will ensure that the parameter is constant");
8479 uint64_t ZExtValue =
Value.zextOrTrunc(IsThumb ? 16 : 32).getZExtValue();
8481 llvm::InlineAsm *Emit =
8482 IsThumb ? InlineAsm::get(FTy,
".inst.n 0x" + utohexstr(ZExtValue),
"",
8484 : InlineAsm::get(FTy,
".inst 0x" + utohexstr(ZExtValue),
"",
8487 return Builder.CreateCall(Emit);
8490 if (BuiltinID == clang::ARM::BI__builtin_arm_dbg) {
8495 if (BuiltinID == clang::ARM::BI__builtin_arm_prefetch) {
8507 if (BuiltinID == clang::ARM::BI__builtin_arm_rbit) {
8510 CGM.
getIntrinsic(Intrinsic::bitreverse, Arg->getType()), Arg,
"rbit");
8513 if (BuiltinID == clang::ARM::BI__builtin_arm_clz ||
8514 BuiltinID == clang::ARM::BI__builtin_arm_clz64) {
8518 if (BuiltinID == clang::ARM::BI__builtin_arm_clz64)
8524 if (BuiltinID == clang::ARM::BI__builtin_arm_cls) {
8528 if (BuiltinID == clang::ARM::BI__builtin_arm_cls64) {
8534 if (BuiltinID == clang::ARM::BI__clear_cache) {
8535 assert(E->
getNumArgs() == 2 &&
"__clear_cache takes 2 arguments");
8538 for (
unsigned i = 0; i < 2; i++)
8541 llvm::FunctionType *FTy = cast<llvm::FunctionType>(Ty);
8542 StringRef Name = FD->
getName();
8546 if (BuiltinID == clang::ARM::BI__builtin_arm_mcrr ||
8547 BuiltinID == clang::ARM::BI__builtin_arm_mcrr2) {
8550 switch (BuiltinID) {
8551 default: llvm_unreachable(
"unexpected builtin");
8552 case clang::ARM::BI__builtin_arm_mcrr:
8555 case clang::ARM::BI__builtin_arm_mcrr2:
8577 return Builder.CreateCall(F, {Coproc, Opc1, Rt, Rt2, CRm});
8580 if (BuiltinID == clang::ARM::BI__builtin_arm_mrrc ||
8581 BuiltinID == clang::ARM::BI__builtin_arm_mrrc2) {
8584 switch (BuiltinID) {
8585 default: llvm_unreachable(
"unexpected builtin");
8586 case clang::ARM::BI__builtin_arm_mrrc:
8589 case clang::ARM::BI__builtin_arm_mrrc2:
8597 Value *RtAndRt2 =
Builder.CreateCall(F, {Coproc, Opc1, CRm});
8607 Value *ShiftCast = llvm::ConstantInt::get(
Int64Ty, 32);
8608 RtAndRt2 =
Builder.CreateShl(Rt, ShiftCast,
"shl",
true);
8609 RtAndRt2 =
Builder.CreateOr(RtAndRt2, Rt1);
8614 if (BuiltinID == clang::ARM::BI__builtin_arm_ldrexd ||
8615 ((BuiltinID == clang::ARM::BI__builtin_arm_ldrex ||
8616 BuiltinID == clang::ARM::BI__builtin_arm_ldaex) &&
8618 BuiltinID == clang::ARM::BI__ldrexd) {
8621 switch (BuiltinID) {
8622 default: llvm_unreachable(
"unexpected builtin");
8623 case clang::ARM::BI__builtin_arm_ldaex:
8626 case clang::ARM::BI__builtin_arm_ldrexd:
8627 case clang::ARM::BI__builtin_arm_ldrex:
8628 case clang::ARM::BI__ldrexd:
8642 Val =
Builder.CreateShl(Val0, ShiftCst,
"shl",
true );
8643 Val =
Builder.CreateOr(Val, Val1);
8647 if (BuiltinID == clang::ARM::BI__builtin_arm_ldrex ||
8648 BuiltinID == clang::ARM::BI__builtin_arm_ldaex) {
8657 BuiltinID == clang::ARM::BI__builtin_arm_ldaex ? Intrinsic::arm_ldaex
8658 : Intrinsic::arm_ldrex,
8660 CallInst *Val =
Builder.CreateCall(F, LoadAddr,
"ldrex");
8664 if (RealResTy->isPointerTy())
8665 return Builder.CreateIntToPtr(Val, RealResTy);
8667 llvm::Type *IntResTy = llvm::IntegerType::get(
8669 return Builder.CreateBitCast(
Builder.CreateTruncOrBitCast(Val, IntResTy),
8674 if (BuiltinID == clang::ARM::BI__builtin_arm_strexd ||
8675 ((BuiltinID == clang::ARM::BI__builtin_arm_stlex ||
8676 BuiltinID == clang::ARM::BI__builtin_arm_strex) &&
8679 BuiltinID == clang::ARM::BI__builtin_arm_stlex ? Intrinsic::arm_stlexd
8680 : Intrinsic::arm_strexd);
8693 return Builder.CreateCall(F, {Arg0, Arg1, StPtr},
"strexd");
8696 if (BuiltinID == clang::ARM::BI__builtin_arm_strex ||
8697 BuiltinID == clang::ARM::BI__builtin_arm_stlex) {
8702 llvm::Type *StoreTy =
8705 if (StoreVal->
getType()->isPointerTy())
8708 llvm::Type *
IntTy = llvm::IntegerType::get(
8716 BuiltinID == clang::ARM::BI__builtin_arm_stlex ? Intrinsic::arm_stlex
8717 : Intrinsic::arm_strex,
8720 CallInst *CI =
Builder.CreateCall(F, {StoreVal, StoreAddr},
"strex");
8722 1, Attribute::get(
getLLVMContext(), Attribute::ElementType, StoreTy));
8726 if (BuiltinID == clang::ARM::BI__builtin_arm_clrex) {
8732 Intrinsic::ID CRCIntrinsicID = Intrinsic::not_intrinsic;
8733 switch (BuiltinID) {
8734 case clang::ARM::BI__builtin_arm_crc32b:
8735 CRCIntrinsicID = Intrinsic::arm_crc32b;
break;
8736 case clang::ARM::BI__builtin_arm_crc32cb:
8737 CRCIntrinsicID = Intrinsic::arm_crc32cb;
break;
8738 case clang::ARM::BI__builtin_arm_crc32h:
8739 CRCIntrinsicID = Intrinsic::arm_crc32h;
break;
8740 case clang::ARM::BI__builtin_arm_crc32ch:
8741 CRCIntrinsicID = Intrinsic::arm_crc32ch;
break;
8742 case clang::ARM::BI__builtin_arm_crc32w:
8743 case clang::ARM::BI__builtin_arm_crc32d:
8744 CRCIntrinsicID = Intrinsic::arm_crc32w;
break;
8745 case clang::ARM::BI__builtin_arm_crc32cw:
8746 case clang::ARM::BI__builtin_arm_crc32cd:
8747 CRCIntrinsicID = Intrinsic::arm_crc32cw;
break;
8750 if (CRCIntrinsicID != Intrinsic::not_intrinsic) {
8756 if (BuiltinID == clang::ARM::BI__builtin_arm_crc32d ||
8757 BuiltinID == clang::ARM::BI__builtin_arm_crc32cd) {
8765 return Builder.CreateCall(F, {Res, Arg1b});
8770 return Builder.CreateCall(F, {Arg0, Arg1});
8774 if (BuiltinID == clang::ARM::BI__builtin_arm_rsr ||
8775 BuiltinID == clang::ARM::BI__builtin_arm_rsr64 ||
8776 BuiltinID == clang::ARM::BI__builtin_arm_rsrp ||
8777 BuiltinID == clang::ARM::BI__builtin_arm_wsr ||
8778 BuiltinID == clang::ARM::BI__builtin_arm_wsr64 ||
8779 BuiltinID == clang::ARM::BI__builtin_arm_wsrp) {
8782 if (BuiltinID == clang::ARM::BI__builtin_arm_rsr ||
8783 BuiltinID == clang::ARM::BI__builtin_arm_rsr64 ||
8784 BuiltinID == clang::ARM::BI__builtin_arm_rsrp)
8787 bool IsPointerBuiltin = BuiltinID == clang::ARM::BI__builtin_arm_rsrp ||
8788 BuiltinID == clang::ARM::BI__builtin_arm_wsrp;
8790 bool Is64Bit = BuiltinID == clang::ARM::BI__builtin_arm_rsr64 ||
8791 BuiltinID == clang::ARM::BI__builtin_arm_wsr64;
8793 llvm::Type *ValueType;
8794 llvm::Type *RegisterType;
8795 if (IsPointerBuiltin) {
8798 }
else if (Is64Bit) {
8799 ValueType = RegisterType =
Int64Ty;
8801 ValueType = RegisterType =
Int32Ty;
8808 if (BuiltinID == ARM::BI__builtin_sponentry) {
8827 return P.first == BuiltinID;
8830 BuiltinID = It->second;
8834 unsigned ICEArguments = 0;
8839 auto getAlignmentValue32 = [&](
Address addr) ->
Value* {
8840 return Builder.getInt32(addr.getAlignment().getQuantity());
8847 unsigned NumArgs = E->
getNumArgs() - (HasExtraArg ? 1 : 0);
8848 for (
unsigned i = 0, e = NumArgs; i != e; i++) {
8850 switch (BuiltinID) {
8851 case NEON::BI__builtin_neon_vld1_v:
8852 case NEON::BI__builtin_neon_vld1q_v:
8853 case NEON::BI__builtin_neon_vld1q_lane_v:
8854 case NEON::BI__builtin_neon_vld1_lane_v:
8855 case NEON::BI__builtin_neon_vld1_dup_v:
8856 case NEON::BI__builtin_neon_vld1q_dup_v:
8857 case NEON::BI__builtin_neon_vst1_v:
8858 case NEON::BI__builtin_neon_vst1q_v:
8859 case NEON::BI__builtin_neon_vst1q_lane_v:
8860 case NEON::BI__builtin_neon_vst1_lane_v:
8861 case NEON::BI__builtin_neon_vst2_v:
8862 case NEON::BI__builtin_neon_vst2q_v:
8863 case NEON::BI__builtin_neon_vst2_lane_v:
8864 case NEON::BI__builtin_neon_vst2q_lane_v:
8865 case NEON::BI__builtin_neon_vst3_v:
8866 case NEON::BI__builtin_neon_vst3q_v:
8867 case NEON::BI__builtin_neon_vst3_lane_v:
8868 case NEON::BI__builtin_neon_vst3q_lane_v:
8869 case NEON::BI__builtin_neon_vst4_v:
8870 case NEON::BI__builtin_neon_vst4q_v:
8871 case NEON::BI__builtin_neon_vst4_lane_v:
8872 case NEON::BI__builtin_neon_vst4q_lane_v:
8881 switch (BuiltinID) {
8882 case NEON::BI__builtin_neon_vld2_v:
8883 case NEON::BI__builtin_neon_vld2q_v:
8884 case NEON::BI__builtin_neon_vld3_v:
8885 case NEON::BI__builtin_neon_vld3q_v:
8886 case NEON::BI__builtin_neon_vld4_v:
8887 case NEON::BI__builtin_neon_vld4q_v:
8888 case NEON::BI__builtin_neon_vld2_lane_v:
8889 case NEON::BI__builtin_neon_vld2q_lane_v:
8890 case NEON::BI__builtin_neon_vld3_lane_v:
8891 case NEON::BI__builtin_neon_vld3q_lane_v:
8892 case NEON::BI__builtin_neon_vld4_lane_v:
8893 case NEON::BI__builtin_neon_vld4q_lane_v:
8894 case NEON::BI__builtin_neon_vld2_dup_v:
8895 case NEON::BI__builtin_neon_vld2q_dup_v:
8896 case NEON::BI__builtin_neon_vld3_dup_v:
8897 case NEON::BI__builtin_neon_vld3q_dup_v:
8898 case NEON::BI__builtin_neon_vld4_dup_v:
8899 case NEON::BI__builtin_neon_vld4q_dup_v:
8911 switch (BuiltinID) {
8914 case NEON::BI__builtin_neon_vget_lane_i8:
8915 case NEON::BI__builtin_neon_vget_lane_i16:
8916 case NEON::BI__builtin_neon_vget_lane_i32:
8917 case NEON::BI__builtin_neon_vget_lane_i64:
8918 case NEON::BI__builtin_neon_vget_lane_bf16:
8919 case NEON::BI__builtin_neon_vget_lane_f32:
8920 case NEON::BI__builtin_neon_vgetq_lane_i8:
8921 case NEON::BI__builtin_neon_vgetq_lane_i16:
8922 case NEON::BI__builtin_neon_vgetq_lane_i32:
8923 case NEON::BI__builtin_neon_vgetq_lane_i64:
8924 case NEON::BI__builtin_neon_vgetq_lane_bf16:
8925 case NEON::BI__builtin_neon_vgetq_lane_f32:
8926 case NEON::BI__builtin_neon_vduph_lane_bf16:
8927 case NEON::BI__builtin_neon_vduph_laneq_bf16:
8928 return Builder.CreateExtractElement(Ops[0], Ops[1],
"vget_lane");
8930 case NEON::BI__builtin_neon_vrndns_f32: {
8932 llvm::Type *Tys[] = {Arg->
getType()};
8934 return Builder.CreateCall(F, {Arg},
"vrndn"); }
8936 case NEON::BI__builtin_neon_vset_lane_i8:
8937 case NEON::BI__builtin_neon_vset_lane_i16:
8938 case NEON::BI__builtin_neon_vset_lane_i32:
8939 case NEON::BI__builtin_neon_vset_lane_i64:
8940 case NEON::BI__builtin_neon_vset_lane_bf16:
8941 case NEON::BI__builtin_neon_vset_lane_f32:
8942 case NEON::BI__builtin_neon_vsetq_lane_i8:
8943 case NEON::BI__builtin_neon_vsetq_lane_i16:
8944 case NEON::BI__builtin_neon_vsetq_lane_i32:
8945 case NEON::BI__builtin_neon_vsetq_lane_i64:
8946 case NEON::BI__builtin_neon_vsetq_lane_bf16:
8947 case NEON::BI__builtin_neon_vsetq_lane_f32:
8948 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vset_lane");
8950 case NEON::BI__builtin_neon_vsha1h_u32:
8953 case NEON::BI__builtin_neon_vsha1cq_u32:
8956 case NEON::BI__builtin_neon_vsha1pq_u32:
8959 case NEON::BI__builtin_neon_vsha1mq_u32:
8963 case NEON::BI__builtin_neon_vcvth_bf16_f32: {
8970 case clang::ARM::BI_MoveToCoprocessor:
8971 case clang::ARM::BI_MoveToCoprocessor2: {
8973 ? Intrinsic::arm_mcr
8974 : Intrinsic::arm_mcr2);
8975 return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0],
8976 Ops[3], Ops[4], Ops[5]});
8981 assert(HasExtraArg);
8983 std::optional<llvm::APSInt>
Result =
8988 if (BuiltinID == clang::ARM::BI__builtin_arm_vcvtr_f ||
8989 BuiltinID == clang::ARM::BI__builtin_arm_vcvtr_d) {
8992 if (BuiltinID == clang::ARM::BI__builtin_arm_vcvtr_f)
8998 bool usgn =
Result->getZExtValue() == 1;
8999 unsigned Int = usgn ? Intrinsic::arm_vcvtru : Intrinsic::arm_vcvtr;
9003 return Builder.CreateCall(F, Ops,
"vcvtr");
9008 bool usgn =
Type.isUnsigned();
9009 bool rightShift =
false;
9011 llvm::FixedVectorType *VTy =
9014 llvm::Type *Ty = VTy;
9025 Builtin->BuiltinID, Builtin->LLVMIntrinsic, Builtin->AltLLVMIntrinsic,
9026 Builtin->NameHint, Builtin->TypeModifier, E, Ops, PtrOp0, PtrOp1, Arch);
9029 switch (BuiltinID) {
9030 default:
return nullptr;
9031 case NEON::BI__builtin_neon_vld1q_lane_v:
9034 if (VTy->getElementType()->isIntegerTy(64)) {
9036 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
9037 int Lane = cast<ConstantInt>(Ops[2])->getZExtValue();
9038 Value *SV = llvm::ConstantVector::get(ConstantInt::get(
Int32Ty, 1-Lane));
9039 Ops[1] =
Builder.CreateShuffleVector(Ops[1], Ops[1], SV);
9041 Ty = llvm::FixedVectorType::get(VTy->getElementType(), 1);
9044 Value *Align = getAlignmentValue32(PtrOp0);
9047 int Indices[] = {1 - Lane, Lane};
9048 return Builder.CreateShuffleVector(Ops[1], Ld, Indices,
"vld1q_lane");
9051 case NEON::BI__builtin_neon_vld1_lane_v: {
9052 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
9055 return Builder.CreateInsertElement(Ops[1], Ld, Ops[2],
"vld1_lane");
9057 case NEON::BI__builtin_neon_vqrshrn_n_v:
9059 usgn ? Intrinsic::arm_neon_vqrshiftnu : Intrinsic::arm_neon_vqrshiftns;
9062 case NEON::BI__builtin_neon_vqrshrun_n_v:
9064 Ops,
"vqrshrun_n", 1,
true);
9065 case NEON::BI__builtin_neon_vqshrn_n_v:
9066 Int = usgn ? Intrinsic::arm_neon_vqshiftnu : Intrinsic::arm_neon_vqshiftns;
9069 case NEON::BI__builtin_neon_vqshrun_n_v:
9071 Ops,
"vqshrun_n", 1,
true);
9072 case NEON::BI__builtin_neon_vrecpe_v:
9073 case NEON::BI__builtin_neon_vrecpeq_v:
9076 case NEON::BI__builtin_neon_vrshrn_n_v:
9078 Ops,
"vrshrn_n", 1,
true);
9079 case NEON::BI__builtin_neon_vrsra_n_v:
9080 case NEON::BI__builtin_neon_vrsraq_n_v:
9081 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
9082 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
9084 Int = usgn ? Intrinsic::arm_neon_vrshiftu : Intrinsic::arm_neon_vrshifts;
9086 return Builder.CreateAdd(Ops[0], Ops[1],
"vrsra_n");
9087 case NEON::BI__builtin_neon_vsri_n_v:
9088 case NEON::BI__builtin_neon_vsriq_n_v:
9091 case NEON::BI__builtin_neon_vsli_n_v:
9092 case NEON::BI__builtin_neon_vsliq_n_v:
9096 case NEON::BI__builtin_neon_vsra_n_v:
9097 case NEON::BI__builtin_neon_vsraq_n_v:
9098 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
9100 return Builder.CreateAdd(Ops[0], Ops[1]);
9101 case NEON::BI__builtin_neon_vst1q_lane_v:
9104 if (VTy->getElementType()->isIntegerTy(64)) {
9105 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
9106 Value *SV = llvm::ConstantVector::get(cast<llvm::Constant>(Ops[2]));
9107 Ops[1] =
Builder.CreateShuffleVector(Ops[1], Ops[1], SV);
9108 Ops[2] = getAlignmentValue32(PtrOp0);
9109 llvm::Type *Tys[] = {
Int8PtrTy, Ops[1]->getType()};
9114 case NEON::BI__builtin_neon_vst1_lane_v: {
9115 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
9116 Ops[1] =
Builder.CreateExtractElement(Ops[1], Ops[2]);
9120 case NEON::BI__builtin_neon_vtbl1_v:
9123 case NEON::BI__builtin_neon_vtbl2_v:
9126 case NEON::BI__builtin_neon_vtbl3_v:
9129 case NEON::BI__builtin_neon_vtbl4_v:
9132 case NEON::BI__builtin_neon_vtbx1_v:
9135 case NEON::BI__builtin_neon_vtbx2_v:
9138 case NEON::BI__builtin_neon_vtbx3_v:
9141 case NEON::BI__builtin_neon_vtbx4_v:
9147template<
typename Integer>
9156 return Unsigned ? Builder.CreateZExt(
V,
T) : Builder.CreateSExt(
V,
T);
9166 unsigned LaneBits = cast<llvm::VectorType>(
V->getType())
9168 ->getPrimitiveSizeInBits();
9169 if (Shift == LaneBits) {
9174 return llvm::Constant::getNullValue(
V->getType());
9178 return Unsigned ? Builder.CreateLShr(
V, Shift) : Builder.CreateAShr(
V, Shift);
9185 unsigned Elements = 128 /
V->getType()->getPrimitiveSizeInBits();
9186 return Builder.CreateVectorSplat(Elements,
V);
9192 llvm::Type *DestType) {
9205 V->getType()->getScalarSizeInBits() != DestType->getScalarSizeInBits()) {
9206 return Builder.CreateCall(
9208 {DestType, V->getType()}),
9211 return Builder.CreateBitCast(
V, DestType);
9219 unsigned InputElements =
9220 cast<llvm::FixedVectorType>(
V->getType())->getNumElements();
9221 for (
unsigned i = 0; i < InputElements; i += 2)
9222 Indices.push_back(i + Odd);
9223 return Builder.CreateShuffleVector(
V, Indices);
9229 assert(V0->getType() == V1->getType() &&
"Can't zip different vector types");
9231 unsigned InputElements =
9232 cast<llvm::FixedVectorType>(V0->getType())->getNumElements();
9233 for (
unsigned i = 0; i < InputElements; i++) {
9234 Indices.push_back(i);
9235 Indices.push_back(i + InputElements);
9237 return Builder.CreateShuffleVector(V0, V1, Indices);
9240template<
unsigned HighBit,
unsigned OtherBits>
9244 llvm::Type *
T = cast<llvm::VectorType>(VT)->getElementType();
9245 unsigned LaneBits =
T->getPrimitiveSizeInBits();
9246 uint32_t
Value = HighBit << (LaneBits - 1);
9248 Value |= (1UL << (LaneBits - 1)) - 1;
9249 llvm::Value *Lane = llvm::ConstantInt::get(
T,
Value);
9255 unsigned ReverseWidth) {
9259 unsigned LaneSize =
V->getType()->getScalarSizeInBits();
9260 unsigned Elements = 128 / LaneSize;
9261 unsigned Mask = ReverseWidth / LaneSize - 1;
9262 for (
unsigned i = 0; i < Elements; i++)
9263 Indices.push_back(i ^ Mask);
9264 return Builder.CreateShuffleVector(
V, Indices);
9270 llvm::Triple::ArchType Arch) {
9271 enum class CustomCodeGen { VLD24, VST24 } CustomCodeGenType;
9272 Intrinsic::ID IRIntr;
9273 unsigned NumVectors;
9276 switch (BuiltinID) {
9277 #include "clang/Basic/arm_mve_builtin_cg.inc"
9288 switch (CustomCodeGenType) {
9290 case CustomCodeGen::VLD24: {
9294 auto MvecCType = E->
getType();
9296 assert(MvecLType->isStructTy() &&
9297 "Return type for vld[24]q should be a struct");
9298 assert(MvecLType->getStructNumElements() == 1 &&
9299 "Return-type struct for vld[24]q should have one element");
9300 auto MvecLTypeInner = MvecLType->getStructElementType(0);
9301 assert(MvecLTypeInner->isArrayTy() &&
9302 "Return-type struct for vld[24]q should contain an array");
9303 assert(MvecLTypeInner->getArrayNumElements() == NumVectors &&
9304 "Array member of return-type struct vld[24]q has wrong length");
9305 auto VecLType = MvecLTypeInner->getArrayElementType();
9307 Tys.push_back(VecLType);
9309 auto Addr = E->
getArg(0);
9315 Value *MvecOut = PoisonValue::get(MvecLType);
9316 for (
unsigned i = 0; i < NumVectors; ++i) {
9317 Value *Vec =
Builder.CreateExtractValue(LoadResult, i);
9318 MvecOut =
Builder.CreateInsertValue(MvecOut, Vec, {0, i});
9327 case CustomCodeGen::VST24: {
9331 auto Addr = E->
getArg(0);
9337 assert(MvecLType->isStructTy() &&
"Data type for vst2q should be a struct");
9338 assert(MvecLType->getStructNumElements() == 1 &&
9339 "Data-type struct for vst2q should have one element");
9340 auto MvecLTypeInner = MvecLType->getStructElementType(0);
9341 assert(MvecLTypeInner->isArrayTy() &&
9342 "Data-type struct for vst2q should contain an array");
9343 assert(MvecLTypeInner->getArrayNumElements() == NumVectors &&
9344 "Array member of return-type struct vld[24]q has wrong length");
9345 auto VecLType = MvecLTypeInner->getArrayElementType();
9347 Tys.push_back(VecLType);
9352 for (
unsigned i = 0; i < NumVectors; i++)
9353 Ops.push_back(
Builder.CreateExtractValue(Mvec, {0, i}));
9356 Value *ToReturn =
nullptr;
9357 for (
unsigned i = 0; i < NumVectors; i++) {
9358 Ops.push_back(llvm::ConstantInt::get(
Int32Ty, i));
9359 ToReturn =
Builder.CreateCall(F, Ops);
9365 llvm_unreachable(
"unknown custom codegen type.");
9371 llvm::Triple::ArchType Arch) {
9372 switch (BuiltinID) {
9375#include "clang/Basic/arm_cde_builtin_cg.inc"
9382 llvm::Triple::ArchType Arch) {
9383 unsigned int Int = 0;
9384 const char *
s =
nullptr;
9386 switch (BuiltinID) {
9389 case NEON::BI__builtin_neon_vtbl1_v:
9390 case NEON::BI__builtin_neon_vqtbl1_v:
9391 case NEON::BI__builtin_neon_vqtbl1q_v:
9392 case NEON::BI__builtin_neon_vtbl2_v:
9393 case NEON::BI__builtin_neon_vqtbl2_v:
9394 case NEON::BI__builtin_neon_vqtbl2q_v:
9395 case NEON::BI__builtin_neon_vtbl3_v:
9396 case NEON::BI__builtin_neon_vqtbl3_v:
9397 case NEON::BI__builtin_neon_vqtbl3q_v:
9398 case NEON::BI__builtin_neon_vtbl4_v:
9399 case NEON::BI__builtin_neon_vqtbl4_v:
9400 case NEON::BI__builtin_neon_vqtbl4q_v:
9402 case NEON::BI__builtin_neon_vtbx1_v:
9403 case NEON::BI__builtin_neon_vqtbx1_v:
9404 case NEON::BI__builtin_neon_vqtbx1q_v:
9405 case NEON::BI__builtin_neon_vtbx2_v:
9406 case NEON::BI__builtin_neon_vqtbx2_v:
9407 case NEON::BI__builtin_neon_vqtbx2q_v:
9408 case NEON::BI__builtin_neon_vtbx3_v:
9409 case NEON::BI__builtin_neon_vqtbx3_v:
9410 case NEON::BI__builtin_neon_vqtbx3q_v:
9411 case NEON::BI__builtin_neon_vtbx4_v:
9412 case NEON::BI__builtin_neon_vqtbx4_v:
9413 case NEON::BI__builtin_neon_vqtbx4q_v:
9421 std::optional<llvm::APSInt>
Result =
9436 switch (BuiltinID) {
9437 case NEON::BI__builtin_neon_vtbl1_v: {
9439 Ty, Intrinsic::aarch64_neon_tbl1,
"vtbl1");
9441 case NEON::BI__builtin_neon_vtbl2_v: {
9443 Ty, Intrinsic::aarch64_neon_tbl1,
"vtbl1");
9445 case NEON::BI__builtin_neon_vtbl3_v: {
9447 Ty, Intrinsic::aarch64_neon_tbl2,
"vtbl2");
9449 case NEON::BI__builtin_neon_vtbl4_v: {
9451 Ty, Intrinsic::aarch64_neon_tbl2,
"vtbl2");
9453 case NEON::BI__builtin_neon_vtbx1_v: {
9456 Intrinsic::aarch64_neon_tbl1,
"vtbl1");
9458 llvm::Constant *EightV = ConstantInt::get(Ty, 8);
9459 Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[2], EightV);
9460 CmpRes = Builder.CreateSExt(CmpRes, Ty);
9462 Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]);
9463 Value *EltsFromTbl = Builder.CreateAnd(Builder.CreateNot(CmpRes), TblRes);
9464 return Builder.CreateOr(EltsFromInput, EltsFromTbl,
"vtbx");
9466 case NEON::BI__builtin_neon_vtbx2_v: {
9468 Ty, Intrinsic::aarch64_neon_tbx1,
"vtbx1");
9470 case NEON::BI__builtin_neon_vtbx3_v: {
9473 Intrinsic::aarch64_neon_tbl2,
"vtbl2");
9475 llvm::Constant *TwentyFourV = ConstantInt::get(Ty, 24);
9476 Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[4],
9478 CmpRes = Builder.CreateSExt(CmpRes, Ty);
9480 Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]);
9481 Value *EltsFromTbl = Builder.CreateAnd(Builder.CreateNot(CmpRes), TblRes);
9482 return Builder.CreateOr(EltsFromInput, EltsFromTbl,
"vtbx");
9484 case NEON::BI__builtin_neon_vtbx4_v: {
9486 Ty, Intrinsic::aarch64_neon_tbx2,
"vtbx2");
9488 case NEON::BI__builtin_neon_vqtbl1_v:
9489 case NEON::BI__builtin_neon_vqtbl1q_v:
9490 Int = Intrinsic::aarch64_neon_tbl1;
s =
"vtbl1";
break;
9491 case NEON::BI__builtin_neon_vqtbl2_v:
9492 case NEON::BI__builtin_neon_vqtbl2q_v: {
9493 Int = Intrinsic::aarch64_neon_tbl2;
s =
"vtbl2";
break;
9494 case NEON::BI__builtin_neon_vqtbl3_v:
9495 case NEON::BI__builtin_neon_vqtbl3q_v:
9496 Int = Intrinsic::aarch64_neon_tbl3;
s =
"vtbl3";
break;
9497 case NEON::BI__builtin_neon_vqtbl4_v:
9498 case NEON::BI__builtin_neon_vqtbl4q_v:
9499 Int = Intrinsic::aarch64_neon_tbl4;
s =
"vtbl4";
break;
9500 case NEON::BI__builtin_neon_vqtbx1_v:
9501 case NEON::BI__builtin_neon_vqtbx1q_v:
9502 Int = Intrinsic::aarch64_neon_tbx1;
s =
"vtbx1";
break;
9503 case NEON::BI__builtin_neon_vqtbx2_v:
9504 case NEON::BI__builtin_neon_vqtbx2q_v:
9505 Int = Intrinsic::aarch64_neon_tbx2;
s =
"vtbx2";
break;
9506 case NEON::BI__builtin_neon_vqtbx3_v:
9507 case NEON::BI__builtin_neon_vqtbx3q_v:
9508 Int = Intrinsic::aarch64_neon_tbx3;
s =
"vtbx3";
break;
9509 case NEON::BI__builtin_neon_vqtbx4_v:
9510 case NEON::BI__builtin_neon_vqtbx4q_v:
9511 Int = Intrinsic::aarch64_neon_tbx4;
s =
"vtbx4";
break;
9523 auto *VTy = llvm::FixedVectorType::get(
Int16Ty, 4);
9525 Value *
V = PoisonValue::get(VTy);
9526 llvm::Constant *CI = ConstantInt::get(
SizeTy, 0);
9527 Op =
Builder.CreateInsertElement(
V, Op, CI);
9536 case SVETypeFlags::MemEltTyDefault:
9538 case SVETypeFlags::MemEltTyInt8:
9540 case SVETypeFlags::MemEltTyInt16:
9542 case SVETypeFlags::MemEltTyInt32:
9544 case SVETypeFlags::MemEltTyInt64:
9547 llvm_unreachable(
"Unknown MemEltType");
9553 llvm_unreachable(
"Invalid SVETypeFlag!");
9555 case SVETypeFlags::EltTyInt8:
9557 case SVETypeFlags::EltTyInt16:
9559 case SVETypeFlags::EltTyInt32:
9561 case SVETypeFlags::EltTyInt64:
9563 case SVETypeFlags::EltTyInt128:
9566 case SVETypeFlags::EltTyFloat16:
9568 case SVETypeFlags::EltTyFloat32:
9570 case SVETypeFlags::EltTyFloat64:
9573 case SVETypeFlags::EltTyBFloat16:
9576 case SVETypeFlags::EltTyBool8:
9577 case SVETypeFlags::EltTyBool16:
9578 case SVETypeFlags::EltTyBool32:
9579 case SVETypeFlags::EltTyBool64:
9586llvm::ScalableVectorType *
9589 default: llvm_unreachable(
"Unhandled SVETypeFlag!");
9591 case SVETypeFlags::EltTyInt8:
9592 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 16);
9593 case SVETypeFlags::EltTyInt16:
9594 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 8);
9595 case SVETypeFlags::EltTyInt32:
9596 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 4);
9597 case SVETypeFlags::EltTyInt64:
9598 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 2);
9600 case SVETypeFlags::EltTyBFloat16:
9601 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 8);
9602 case SVETypeFlags::EltTyFloat16:
9603 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 8);
9604 case SVETypeFlags::EltTyFloat32:
9605 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 4);
9606 case SVETypeFlags::EltTyFloat64:
9607 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 2);
9609 case SVETypeFlags::EltTyBool8:
9610 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 16);
9611 case SVETypeFlags::EltTyBool16:
9612 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 8);
9613 case SVETypeFlags::EltTyBool32:
9614 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 4);
9615 case SVETypeFlags::EltTyBool64:
9616 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 2);
9621llvm::ScalableVectorType *
9625 llvm_unreachable(
"Invalid SVETypeFlag!");
9627 case SVETypeFlags::EltTyInt8:
9628 return llvm::ScalableVectorType::get(
Builder.getInt8Ty(), 16);
9629 case SVETypeFlags::EltTyInt16:
9630 return llvm::ScalableVectorType::get(
Builder.getInt16Ty(), 8);
9631 case SVETypeFlags::EltTyInt32:
9632 return llvm::ScalableVectorType::get(
Builder.getInt32Ty(), 4);
9633 case SVETypeFlags::EltTyInt64:
9634 return llvm::ScalableVectorType::get(
Builder.getInt64Ty(), 2);
9636 case SVETypeFlags::EltTyFloat16:
9637 return llvm::ScalableVectorType::get(
Builder.getHalfTy(), 8);
9638 case SVETypeFlags::EltTyBFloat16:
9639 return llvm::ScalableVectorType::get(
Builder.getBFloatTy(), 8);
9640 case SVETypeFlags::EltTyFloat32:
9641 return llvm::ScalableVectorType::get(
Builder.getFloatTy(), 4);
9642 case SVETypeFlags::EltTyFloat64:
9643 return llvm::ScalableVectorType::get(
Builder.getDoubleTy(), 2);
9645 case SVETypeFlags::EltTyBool8:
9646 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 16);
9647 case SVETypeFlags::EltTyBool16:
9648 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 8);
9649 case SVETypeFlags::EltTyBool32:
9650 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 4);
9651 case SVETypeFlags::EltTyBool64:
9652 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 2);
9667 return llvm::ScalableVectorType::get(EltTy, NumElts);
9673 llvm::ScalableVectorType *VTy) {
9675 if (isa<TargetExtType>(Pred->
getType()) &&
9676 cast<TargetExtType>(Pred->
getType())->getName() ==
"aarch64.svcount")
9679 auto *RTy = llvm::VectorType::get(IntegerType::get(
getLLVMContext(), 1), VTy);
9684 llvm::Type *IntrinsicTy;
9685 switch (VTy->getMinNumElements()) {
9687 llvm_unreachable(
"unsupported element count!");
9692 IntID = Intrinsic::aarch64_sve_convert_from_svbool;
9696 IntID = Intrinsic::aarch64_sve_convert_to_svbool;
9697 IntrinsicTy = Pred->
getType();
9703 assert(
C->getType() == RTy &&
"Unexpected return type!");
9711 auto *OverloadedTy =
9715 if (Ops[1]->getType()->isVectorTy())
9735 Ops[0], cast<llvm::ScalableVectorType>(F->getArg(0)->getType()));
9740 if (Ops.size() == 2) {
9741 assert(Ops[1]->getType()->isVectorTy() &&
"Scalar base requires an offset");
9742 Ops.push_back(ConstantInt::get(
Int64Ty, 0));
9747 if (!TypeFlags.
isByteIndexed() && Ops[1]->getType()->isVectorTy()) {
9748 unsigned BytesPerElt =
9749 OverloadedTy->getElementType()->getScalarSizeInBits() / 8;
9750 Ops[2] =
Builder.CreateShl(Ops[2], Log2_32(BytesPerElt));
9765 auto *OverloadedTy =
9770 Ops.insert(Ops.begin(), Ops.pop_back_val());
9773 if (Ops[2]->getType()->isVectorTy())
9788 if (Ops.size() == 3) {
9789 assert(Ops[1]->getType()->isVectorTy() &&
"Scalar base requires an offset");
9790 Ops.push_back(ConstantInt::get(
Int64Ty, 0));
9795 Ops[0] =
Builder.CreateTrunc(Ops[0], OverloadedTy);
9805 Ops[1], cast<llvm::ScalableVectorType>(F->getArg(1)->getType()));
9809 if (!TypeFlags.
isByteIndexed() && Ops[2]->getType()->isVectorTy()) {
9810 unsigned BytesPerElt =
9811 OverloadedTy->getElementType()->getScalarSizeInBits() / 8;
9812 Ops[3] =
Builder.CreateShl(Ops[3], Log2_32(BytesPerElt));
9815 return Builder.CreateCall(F, Ops);
9823 auto *OverloadedTy = dyn_cast<llvm::ScalableVectorType>(Ops[1]->getType());
9825 OverloadedTy = cast<llvm::ScalableVectorType>(Ops[2]->getType());
9831 if (Ops[1]->getType()->isVectorTy()) {
9832 if (Ops.size() == 3) {
9834 Ops.push_back(ConstantInt::get(
Int64Ty, 0));
9837 std::swap(Ops[2], Ops[3]);
9841 unsigned BytesPerElt = MemEltTy->getPrimitiveSizeInBits() / 8;
9842 if (BytesPerElt > 1)
9843 Ops[2] =
Builder.CreateShl(Ops[2], Log2_32(BytesPerElt));
9848 return Builder.CreateCall(F, Ops);
9854 llvm::ScalableVectorType *VTy =
getSVEType(TypeFlags);
9858 case Intrinsic::aarch64_sve_ld2_sret:
9859 case Intrinsic::aarch64_sve_ld1_pn_x2:
9860 case Intrinsic::aarch64_sve_ldnt1_pn_x2:
9861 case Intrinsic::aarch64_sve_ld2q_sret:
9864 case Intrinsic::aarch64_sve_ld3_sret:
9865 case Intrinsic::aarch64_sve_ld3q_sret:
9868 case Intrinsic::aarch64_sve_ld4_sret:
9869 case Intrinsic::aarch64_sve_ld1_pn_x4:
9870 case Intrinsic::aarch64_sve_ldnt1_pn_x4:
9871 case Intrinsic::aarch64_sve_ld4q_sret:
9875 llvm_unreachable(
"unknown intrinsic!");
9877 auto RetTy = llvm::VectorType::get(VTy->getElementType(),
9878 VTy->getElementCount() * N);
9881 Value *BasePtr = Ops[1];
9889 unsigned MinElts = VTy->getMinNumElements();
9890 Value *
Ret = llvm::PoisonValue::get(RetTy);
9891 for (
unsigned I = 0; I < N; I++) {
9894 Ret =
Builder.CreateInsertVector(RetTy, Ret, SRet, Idx);
9902 llvm::ScalableVectorType *VTy =
getSVEType(TypeFlags);
9906 case Intrinsic::aarch64_sve_st2:
9907 case Intrinsic::aarch64_sve_st1_pn_x2:
9908 case Intrinsic::aarch64_sve_stnt1_pn_x2:
9909 case Intrinsic::aarch64_sve_st2q:
9912 case Intrinsic::aarch64_sve_st3:
9913 case Intrinsic::aarch64_sve_st3q:
9916 case Intrinsic::aarch64_sve_st4:
9917 case Intrinsic::aarch64_sve_st1_pn_x4:
9918 case Intrinsic::aarch64_sve_stnt1_pn_x4:
9919 case Intrinsic::aarch64_sve_st4q:
9923 llvm_unreachable(
"unknown intrinsic!");
9927 Value *BasePtr = Ops[1];
9930 if (Ops.size() > (2 + N))
9936 for (
unsigned I = Ops.size() - N; I < Ops.size(); ++I)
9937 Operands.push_back(Ops[I]);
9938 Operands.append({Predicate, BasePtr});
9941 return Builder.CreateCall(F, Operands);
9949 unsigned BuiltinID) {
9961 llvm::ScalableVectorType *Ty =
getSVEType(TypeFlags);
9967 llvm::Type *OverloadedTy =
getSVEType(TypeFlags);
9974 unsigned BuiltinID) {
9977 auto *MemoryTy = llvm::ScalableVectorType::get(MemEltTy, VectorTy);
9980 Value *BasePtr = Ops[1];
9986 Value *PrfOp = Ops.back();
9989 return Builder.CreateCall(F, {Predicate, BasePtr, PrfOp});
9993 llvm::Type *ReturnTy,
9995 unsigned IntrinsicID,
9996 bool IsZExtReturn) {
10003 auto VectorTy = cast<llvm::ScalableVectorType>(ReturnTy);
10004 llvm::ScalableVectorType *MemoryTy =
nullptr;
10005 llvm::ScalableVectorType *PredTy =
nullptr;
10006 bool IsQuadLoad =
false;
10007 switch (IntrinsicID) {
10008 case Intrinsic::aarch64_sve_ld1uwq:
10009 case Intrinsic::aarch64_sve_ld1udq:
10010 MemoryTy = llvm::ScalableVectorType::get(MemEltTy, 1);
10011 PredTy = llvm::ScalableVectorType::get(
10016 MemoryTy = llvm::ScalableVectorType::get(MemEltTy, VectorTy);
10022 Value *BasePtr = Ops[1];
10025 if (Ops.size() > 2)
10030 cast<llvm::Instruction>(
Builder.CreateCall(F, {Predicate, BasePtr}));
10037 return IsZExtReturn ?
Builder.CreateZExt(Load, VectorTy)
10038 :
Builder.CreateSExt(Load, VectorTy);
10043 unsigned IntrinsicID) {
10050 auto VectorTy = cast<llvm::ScalableVectorType>(Ops.back()->getType());
10051 auto MemoryTy = llvm::ScalableVectorType::get(MemEltTy, VectorTy);
10053 auto PredTy = MemoryTy;
10054 auto AddrMemoryTy = MemoryTy;
10055 bool IsQuadStore =
false;
10057 switch (IntrinsicID) {
10058 case Intrinsic::aarch64_sve_st1wq:
10059 case Intrinsic::aarch64_sve_st1dq:
10060 AddrMemoryTy = llvm::ScalableVectorType::get(MemEltTy, 1);
10062 llvm::ScalableVectorType::get(IntegerType::get(
getLLVMContext(), 1), 1);
10063 IsQuadStore =
true;
10069 Value *BasePtr = Ops[1];
10072 if (Ops.size() == 4)
10077 IsQuadStore ? Ops.back() :
Builder.CreateTrunc(Ops.back(), MemoryTy);
10082 cast<llvm::Instruction>(
Builder.CreateCall(F, {Val, Predicate, BasePtr}));
10095 NewOps.push_back(Ops[2]);
10097 llvm::Value *BasePtr = Ops[3];
10101 if (Ops.size() == 5) {
10104 llvm::Value *StreamingVectorLengthCall =
10105 Builder.CreateCall(StreamingVectorLength);
10106 llvm::Value *Mulvl =
10107 Builder.CreateMul(StreamingVectorLengthCall, Ops[4],
"mulvl");
10111 NewOps.push_back(BasePtr);
10112 NewOps.push_back(Ops[0]);
10113 NewOps.push_back(Ops[1]);
10115 return Builder.CreateCall(F, NewOps);
10127 return Builder.CreateCall(F, Ops);
10134 if (Ops.size() == 0)
10135 Ops.push_back(llvm::ConstantInt::get(
Int32Ty, 255));
10137 return Builder.CreateCall(F, Ops);
10143 if (Ops.size() == 2)
10144 Ops.push_back(
Builder.getInt32(0));
10148 return Builder.CreateCall(F, Ops);
10154 return Builder.CreateVectorSplat(
10155 cast<llvm::VectorType>(Ty)->getElementCount(), Scalar);
10169 return Builder.CreateBitCast(Val, Ty);
10174 auto *SplatZero = Constant::getNullValue(Ty);
10175 Ops.insert(Ops.begin(), SplatZero);
10180 auto *SplatUndef = UndefValue::get(Ty);
10181 Ops.insert(Ops.begin(), SplatUndef);
10186 llvm::Type *ResultType,
10191 llvm::Type *DefaultType =
getSVEType(TypeFlags);
10194 return {DefaultType, Ops[1]->getType()};
10200 return {Ops[0]->getType(), Ops.back()->getType()};
10202 if (TypeFlags.
isReductionQV() && !ResultType->isScalableTy() &&
10203 ResultType->isVectorTy())
10204 return {ResultType, Ops[1]->getType()};
10207 return {DefaultType};
10214 "Expects TypleFlag isTupleSet or TypeFlags.isTupleSet()");
10216 unsigned I = cast<ConstantInt>(Ops[1])->getSExtValue();
10217 auto *SingleVecTy = dyn_cast<llvm::ScalableVectorType>(
10218 TypeFlags.
isTupleSet() ? Ops[2]->getType() : Ty);
10220 I * SingleVecTy->getMinNumElements());
10223 return Builder.CreateInsertVector(Ty, Ops[0], Ops[2], Idx);
10224 return Builder.CreateExtractVector(Ty, Ops[0], Idx);
10230 assert(TypeFlags.
isTupleCreate() &&
"Expects TypleFlag isTupleCreate");
10232 auto *SrcTy = dyn_cast<llvm::ScalableVectorType>(Ops[0]->getType());
10233 unsigned MinElts = SrcTy->getMinNumElements();
10234 Value *
Call = llvm::PoisonValue::get(Ty);
10235 for (
unsigned I = 0; I < Ops.size(); I++) {
10246 auto *StructTy = dyn_cast<StructType>(
Call->getType());
10250 auto *VTy = dyn_cast<ScalableVectorType>(StructTy->getTypeAtIndex(0
U));
10253 unsigned N = StructTy->getNumElements();
10256 bool IsPredTy = VTy->getElementType()->isIntegerTy(1);
10257 unsigned MinElts = IsPredTy ? 16 : VTy->getMinNumElements();
10259 ScalableVectorType *WideVTy =
10260 ScalableVectorType::get(VTy->getElementType(), MinElts * N);
10261 Value *
Ret = llvm::PoisonValue::get(WideVTy);
10262 for (
unsigned I = 0; I < N; ++I) {
10264 assert(SRet->
getType() == VTy &&
"Unexpected type for result value");
10269 SRet, ScalableVectorType::get(
Builder.getInt1Ty(), 16));
10271 Ret =
Builder.CreateInsertVector(WideVTy, Ret, SRet, Idx);
10282 unsigned ICEArguments = 0;
10291 for (
unsigned i = 0, e = E->
getNumArgs(); i != e; i++) {
10292 bool IsICE = ICEArguments & (1 << i);
10298 std::optional<llvm::APSInt>
Result =
10300 assert(
Result &&
"Expected argument to be a constant");
10310 if (IsTupleGetOrSet || !isa<ScalableVectorType>(Arg->getType())) {
10311 Ops.push_back(Arg);
10315 auto *VTy = cast<ScalableVectorType>(Arg->getType());
10316 unsigned MinElts = VTy->getMinNumElements();
10317 bool IsPred = VTy->getElementType()->isIntegerTy(1);
10318 unsigned N = (MinElts * VTy->getScalarSizeInBits()) / (IsPred ? 16 : 128);
10321 Ops.push_back(Arg);
10325 for (
unsigned I = 0; I < N; ++I) {
10328 ScalableVectorType::get(VTy->getElementType(), MinElts / N);
10329 Ops.push_back(
Builder.CreateExtractVector(NewVTy, Arg, Idx));
10337 if (BuiltinID >= SVE::BI__builtin_sve_reinterpret_s8_s8 &&
10338 BuiltinID <= SVE::BI__builtin_sve_reinterpret_f64_f64_x4) {
10353 else if (TypeFlags.
isStore())
10371 else if (TypeFlags.
isUndef())
10372 return UndefValue::get(Ty);
10373 else if (Builtin->LLVMIntrinsic != 0) {
10374 if (TypeFlags.
getMergeType() == SVETypeFlags::MergeZeroExp)
10377 if (TypeFlags.
getMergeType() == SVETypeFlags::MergeAnyExp)
10383 Ops.push_back(
Builder.getInt32( 31));
10385 Ops.insert(&Ops[1],
Builder.getInt32( 31));
10388 for (
unsigned i = 0, e = Ops.size(); i != e; ++i)
10389 if (
auto PredTy = dyn_cast<llvm::VectorType>(Ops[i]->getType()))
10390 if (PredTy->getElementType()->isIntegerTy(1))
10400 std::swap(Ops[1], Ops[2]);
10402 std::swap(Ops[1], Ops[2]);
10405 std::swap(Ops[1], Ops[2]);
10408 std::swap(Ops[1], Ops[3]);
10411 if (TypeFlags.
getMergeType() == SVETypeFlags::MergeZero) {
10412 llvm::Type *OpndTy = Ops[1]->getType();
10413 auto *SplatZero = Constant::getNullValue(OpndTy);
10414 Ops[1] =
Builder.CreateSelect(Ops[0], Ops[1], SplatZero);
10422 if (
auto PredTy = dyn_cast<llvm::VectorType>(
Call->getType()))
10423 if (PredTy->getScalarType()->isIntegerTy(1))
10429 switch (BuiltinID) {
10433 case SVE::BI__builtin_sve_svreinterpret_b: {
10437 CGM.
getIntrinsic(Intrinsic::aarch64_sve_convert_to_svbool, SVCountTy);
10438 return Builder.CreateCall(CastFromSVCountF, Ops[0]);
10440 case SVE::BI__builtin_sve_svreinterpret_c: {
10444 CGM.
getIntrinsic(Intrinsic::aarch64_sve_convert_from_svbool, SVCountTy);
10445 return Builder.CreateCall(CastToSVCountF, Ops[0]);
10448 case SVE::BI__builtin_sve_svpsel_lane_b8:
10449 case SVE::BI__builtin_sve_svpsel_lane_b16:
10450 case SVE::BI__builtin_sve_svpsel_lane_b32:
10451 case SVE::BI__builtin_sve_svpsel_lane_b64:
10452 case SVE::BI__builtin_sve_svpsel_lane_c8:
10453 case SVE::BI__builtin_sve_svpsel_lane_c16:
10454 case SVE::BI__builtin_sve_svpsel_lane_c32:
10455 case SVE::BI__builtin_sve_svpsel_lane_c64: {
10456 bool IsSVCount = isa<TargetExtType>(Ops[0]->getType());
10457 assert(((!IsSVCount || cast<TargetExtType>(Ops[0]->getType())->
getName() ==
10458 "aarch64.svcount")) &&
10459 "Unexpected TargetExtType");
10463 CGM.
getIntrinsic(Intrinsic::aarch64_sve_convert_to_svbool, SVCountTy);
10465 CGM.
getIntrinsic(Intrinsic::aarch64_sve_convert_from_svbool, SVCountTy);
10469 llvm::Value *Ops0 =
10470 IsSVCount ?
Builder.CreateCall(CastFromSVCountF, Ops[0]) : Ops[0];
10472 llvm::Value *PSel =
Builder.CreateCall(F, {Ops0, Ops1, Ops[2]});
10473 return IsSVCount ?
Builder.CreateCall(CastToSVCountF, PSel) : PSel;
10475 case SVE::BI__builtin_sve_svmov_b_z: {
10478 llvm::Type* OverloadedTy =
getSVEType(TypeFlags);
10480 return Builder.CreateCall(F, {Ops[0], Ops[1], Ops[1]});
10483 case SVE::BI__builtin_sve_svnot_b_z: {
10486 llvm::Type* OverloadedTy =
getSVEType(TypeFlags);
10488 return Builder.CreateCall(F, {Ops[0], Ops[1], Ops[0]});
10491 case SVE::BI__builtin_sve_svmovlb_u16:
10492 case SVE::BI__builtin_sve_svmovlb_u32:
10493 case SVE::BI__builtin_sve_svmovlb_u64:
10494 return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_ushllb);
10496 case SVE::BI__builtin_sve_svmovlb_s16:
10497 case SVE::BI__builtin_sve_svmovlb_s32:
10498 case SVE::BI__builtin_sve_svmovlb_s64:
10499 return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_sshllb);
10501 case SVE::BI__builtin_sve_svmovlt_u16:
10502 case SVE::BI__builtin_sve_svmovlt_u32:
10503 case SVE::BI__builtin_sve_svmovlt_u64:
10504 return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_ushllt);
10506 case SVE::BI__builtin_sve_svmovlt_s16:
10507 case SVE::BI__builtin_sve_svmovlt_s32:
10508 case SVE::BI__builtin_sve_svmovlt_s64:
10509 return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_sshllt);
10511 case SVE::BI__builtin_sve_svpmullt_u16:
10512 case SVE::BI__builtin_sve_svpmullt_u64:
10513 case SVE::BI__builtin_sve_svpmullt_n_u16:
10514 case SVE::BI__builtin_sve_svpmullt_n_u64:
10515 return EmitSVEPMull(TypeFlags, Ops, Intrinsic::aarch64_sve_pmullt_pair);
10517 case SVE::BI__builtin_sve_svpmullb_u16:
10518 case SVE::BI__builtin_sve_svpmullb_u64:
10519 case SVE::BI__builtin_sve_svpmullb_n_u16:
10520 case SVE::BI__builtin_sve_svpmullb_n_u64:
10521 return EmitSVEPMull(TypeFlags, Ops, Intrinsic::aarch64_sve_pmullb_pair);
10523 case SVE::BI__builtin_sve_svdup_n_b8:
10524 case SVE::BI__builtin_sve_svdup_n_b16:
10525 case SVE::BI__builtin_sve_svdup_n_b32:
10526 case SVE::BI__builtin_sve_svdup_n_b64: {
10528 Builder.CreateICmpNE(Ops[0], Constant::getNullValue(Ops[0]->getType()));
10529 llvm::ScalableVectorType *OverloadedTy =
getSVEType(TypeFlags);
10534 case SVE::BI__builtin_sve_svdupq_n_b8:
10535 case SVE::BI__builtin_sve_svdupq_n_b16:
10536 case SVE::BI__builtin_sve_svdupq_n_b32:
10537 case SVE::BI__builtin_sve_svdupq_n_b64:
10538 case SVE::BI__builtin_sve_svdupq_n_u8:
10539 case SVE::BI__builtin_sve_svdupq_n_s8:
10540 case SVE::BI__builtin_sve_svdupq_n_u64:
10541 case SVE::BI__builtin_sve_svdupq_n_f64:
10542 case SVE::BI__builtin_sve_svdupq_n_s64:
10543 case SVE::BI__builtin_sve_svdupq_n_u16:
10544 case SVE::BI__builtin_sve_svdupq_n_f16:
10545 case SVE::BI__builtin_sve_svdupq_n_bf16:
10546 case SVE::BI__builtin_sve_svdupq_n_s16:
10547 case SVE::BI__builtin_sve_svdupq_n_u32:
10548 case SVE::BI__builtin_sve_svdupq_n_f32:
10549 case SVE::BI__builtin_sve_svdupq_n_s32: {
10552 unsigned NumOpnds = Ops.size();
10555 cast<llvm::VectorType>(Ty)->getElementType()->isIntegerTy(1);
10560 llvm::Type *EltTy = Ops[0]->getType();
10565 for (
unsigned I = 0; I < NumOpnds; ++I)
10566 VecOps.push_back(
Builder.CreateZExt(Ops[I], EltTy));
10571 OverloadedTy, PoisonValue::get(OverloadedTy), Vec,
Builder.getInt64(0));
10586 : Intrinsic::aarch64_sve_cmpne_wide,
10593 case SVE::BI__builtin_sve_svpfalse_b:
10594 return ConstantInt::getFalse(Ty);
10596 case SVE::BI__builtin_sve_svpfalse_c: {
10597 auto SVBoolTy = ScalableVectorType::get(
Builder.getInt1Ty(), 16);
10600 return Builder.CreateCall(CastToSVCountF, ConstantInt::getFalse(SVBoolTy));
10603 case SVE::BI__builtin_sve_svlen_bf16:
10604 case SVE::BI__builtin_sve_svlen_f16:
10605 case SVE::BI__builtin_sve_svlen_f32:
10606 case SVE::BI__builtin_sve_svlen_f64:
10607 case SVE::BI__builtin_sve_svlen_s8:
10608 case SVE::BI__builtin_sve_svlen_s16:
10609 case SVE::BI__builtin_sve_svlen_s32:
10610 case SVE::BI__builtin_sve_svlen_s64:
10611 case SVE::BI__builtin_sve_svlen_u8:
10612 case SVE::BI__builtin_sve_svlen_u16:
10613 case SVE::BI__builtin_sve_svlen_u32:
10614 case SVE::BI__builtin_sve_svlen_u64: {
10616 auto VTy = cast<llvm::VectorType>(
getSVEType(TF));
10618 llvm::ConstantInt::get(Ty, VTy->getElementCount().getKnownMinValue());
10624 case SVE::BI__builtin_sve_svtbl2_u8:
10625 case SVE::BI__builtin_sve_svtbl2_s8:
10626 case SVE::BI__builtin_sve_svtbl2_u16:
10627 case SVE::BI__builtin_sve_svtbl2_s16:
10628 case SVE::BI__builtin_sve_svtbl2_u32:
10629 case SVE::BI__builtin_sve_svtbl2_s32:
10630 case SVE::BI__builtin_sve_svtbl2_u64:
10631 case SVE::BI__builtin_sve_svtbl2_s64:
10632 case SVE::BI__builtin_sve_svtbl2_f16:
10633 case SVE::BI__builtin_sve_svtbl2_bf16:
10634 case SVE::BI__builtin_sve_svtbl2_f32:
10635 case SVE::BI__builtin_sve_svtbl2_f64: {
10637 auto VTy = cast<llvm::ScalableVectorType>(
getSVEType(TF));
10639 return Builder.CreateCall(F, Ops);
10642 case SVE::BI__builtin_sve_svset_neonq_s8:
10643 case SVE::BI__builtin_sve_svset_neonq_s16:
10644 case SVE::BI__builtin_sve_svset_neonq_s32:
10645 case SVE::BI__builtin_sve_svset_neonq_s64:
10646 case SVE::BI__builtin_sve_svset_neonq_u8:
10647 case SVE::BI__builtin_sve_svset_neonq_u16:
10648 case SVE::BI__builtin_sve_svset_neonq_u32:
10649 case SVE::BI__builtin_sve_svset_neonq_u64:
10650 case SVE::BI__builtin_sve_svset_neonq_f16:
10651 case SVE::BI__builtin_sve_svset_neonq_f32:
10652 case SVE::BI__builtin_sve_svset_neonq_f64:
10653 case SVE::BI__builtin_sve_svset_neonq_bf16: {
10654 return Builder.CreateInsertVector(Ty, Ops[0], Ops[1],
Builder.getInt64(0));
10657 case SVE::BI__builtin_sve_svget_neonq_s8:
10658 case SVE::BI__builtin_sve_svget_neonq_s16:
10659 case SVE::BI__builtin_sve_svget_neonq_s32:
10660 case SVE::BI__builtin_sve_svget_neonq_s64:
10661 case SVE::BI__builtin_sve_svget_neonq_u8:
10662 case SVE::BI__builtin_sve_svget_neonq_u16:
10663 case SVE::BI__builtin_sve_svget_neonq_u32:
10664 case SVE::BI__builtin_sve_svget_neonq_u64:
10665 case SVE::BI__builtin_sve_svget_neonq_f16:
10666 case SVE::BI__builtin_sve_svget_neonq_f32:
10667 case SVE::BI__builtin_sve_svget_neonq_f64:
10668 case SVE::BI__builtin_sve_svget_neonq_bf16: {
10669 return Builder.CreateExtractVector(Ty, Ops[0],
Builder.getInt64(0));
10672 case SVE::BI__builtin_sve_svdup_neonq_s8:
10673 case SVE::BI__builtin_sve_svdup_neonq_s16:
10674 case SVE::BI__builtin_sve_svdup_neonq_s32:
10675 case SVE::BI__builtin_sve_svdup_neonq_s64:
10676 case SVE::BI__builtin_sve_svdup_neonq_u8:
10677 case SVE::BI__builtin_sve_svdup_neonq_u16:
10678 case SVE::BI__builtin_sve_svdup_neonq_u32:
10679 case SVE::BI__builtin_sve_svdup_neonq_u64:
10680 case SVE::BI__builtin_sve_svdup_neonq_f16:
10681 case SVE::BI__builtin_sve_svdup_neonq_f32:
10682 case SVE::BI__builtin_sve_svdup_neonq_f64:
10683 case SVE::BI__builtin_sve_svdup_neonq_bf16: {
10686 return Builder.CreateIntrinsic(Intrinsic::aarch64_sve_dupq_lane, {Ty},
10698 switch (BuiltinID) {
10701 case SME::BI__builtin_sme_svsumla_za32_s8_vg4x1:
10704 case SME::BI__builtin_sme_svsumla_za32_s8_vg4x2:
10705 case SME::BI__builtin_sme_svsudot_za32_s8_vg1x2:
10708 case SME::BI__builtin_sme_svsudot_za32_s8_vg1x4:
10709 case SME::BI__builtin_sme_svsumla_za32_s8_vg4x4:
10715 for (
unsigned I = 0; I < MultiVec; ++I)
10716 std::swap(Ops[I + 1], Ops[I + 1 + MultiVec]);
10729 return EmitSMELd1St1(TypeFlags, Ops, Builtin->LLVMIntrinsic);
10732 else if (BuiltinID == SME::BI__builtin_sme_svzero_mask_za ||
10733 BuiltinID == SME::BI__builtin_sme_svzero_za)
10734 return EmitSMEZero(TypeFlags, Ops, Builtin->LLVMIntrinsic);
10735 else if (BuiltinID == SME::BI__builtin_sme_svldr_vnum_za ||
10736 BuiltinID == SME::BI__builtin_sme_svstr_vnum_za ||
10737 BuiltinID == SME::BI__builtin_sme_svldr_za ||
10738 BuiltinID == SME::BI__builtin_sme_svstr_za)
10739 return EmitSMELdrStr(TypeFlags, Ops, Builtin->LLVMIntrinsic);
10745 if (Builtin->LLVMIntrinsic == 0)
10749 for (
unsigned i = 0, e = Ops.size(); i != e; ++i)
10750 if (
auto PredTy = dyn_cast<llvm::VectorType>(Ops[i]->getType()))
10751 if (PredTy->getElementType()->isIntegerTy(1))
10765 llvm::Triple::ArchType Arch) {
10774 if (BuiltinID == Builtin::BI__builtin_cpu_supports)
10775 return EmitAArch64CpuSupports(E);
10777 unsigned HintID =
static_cast<unsigned>(-1);
10778 switch (BuiltinID) {
10780 case clang::AArch64::BI__builtin_arm_nop:
10783 case clang::AArch64::BI__builtin_arm_yield:
10784 case clang::AArch64::BI__yield:
10787 case clang::AArch64::BI__builtin_arm_wfe:
10788 case clang::AArch64::BI__wfe:
10791 case clang::AArch64::BI__builtin_arm_wfi:
10792 case clang::AArch64::BI__wfi:
10795 case clang::AArch64::BI__builtin_arm_sev:
10796 case clang::AArch64::BI__sev:
10799 case clang::AArch64::BI__builtin_arm_sevl:
10800 case clang::AArch64::BI__sevl:
10805 if (HintID !=
static_cast<unsigned>(-1)) {
10807 return Builder.CreateCall(F, llvm::ConstantInt::get(
Int32Ty, HintID));
10810 if (BuiltinID == clang::AArch64::BI__builtin_arm_trap) {
10816 if (BuiltinID == clang::AArch64::BI__builtin_arm_get_sme_state) {
10821 "__arm_sme_state"));
10823 "aarch64_pstate_sm_compatible");
10824 CI->setAttributes(Attrs);
10825 CI->setCallingConv(
10826 llvm::CallingConv::
10827 AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2);
10834 if (BuiltinID == clang::AArch64::BI__builtin_arm_rbit) {
10836 "rbit of unusual size!");
10839 CGM.
getIntrinsic(Intrinsic::bitreverse, Arg->getType()), Arg,
"rbit");
10841 if (BuiltinID == clang::AArch64::BI__builtin_arm_rbit64) {
10843 "rbit of unusual size!");
10846 CGM.
getIntrinsic(Intrinsic::bitreverse, Arg->getType()), Arg,
"rbit");
10849 if (BuiltinID == clang::AArch64::BI__builtin_arm_clz ||
10850 BuiltinID == clang::AArch64::BI__builtin_arm_clz64) {
10854 if (BuiltinID == clang::AArch64::BI__builtin_arm_clz64)
10859 if (BuiltinID == clang::AArch64::BI__builtin_arm_cls) {
10864 if (BuiltinID == clang::AArch64::BI__builtin_arm_cls64) {
10870 if (BuiltinID == clang::AArch64::BI__builtin_arm_rint32zf ||
10871 BuiltinID == clang::AArch64::BI__builtin_arm_rint32z) {
10873 llvm::Type *Ty = Arg->getType();
10878 if (BuiltinID == clang::AArch64::BI__builtin_arm_rint64zf ||
10879 BuiltinID == clang::AArch64::BI__builtin_arm_rint64z) {
10881 llvm::Type *Ty = Arg->getType();
10886 if (BuiltinID == clang::AArch64::BI__builtin_arm_rint32xf ||
10887 BuiltinID == clang::AArch64::BI__builtin_arm_rint32x) {
10889 llvm::Type *Ty = Arg->getType();
10894 if (BuiltinID == clang::AArch64::BI__builtin_arm_rint64xf ||
10895 BuiltinID == clang::AArch64::BI__builtin_arm_rint64x) {
10897 llvm::Type *Ty = Arg->getType();
10902 if (BuiltinID == clang::AArch64::BI__builtin_arm_jcvt) {
10904 "__jcvt of unusual size!");
10910 if (BuiltinID == clang::AArch64::BI__builtin_arm_ld64b ||
10911 BuiltinID == clang::AArch64::BI__builtin_arm_st64b ||
10912 BuiltinID == clang::AArch64::BI__builtin_arm_st64bv ||
10913 BuiltinID == clang::AArch64::BI__builtin_arm_st64bv0) {
10917 if (BuiltinID == clang::AArch64::BI__builtin_arm_ld64b) {
10921 llvm::Value *Val =
Builder.CreateCall(F, MemAddr);
10922 llvm::Value *ToRet;
10923 for (
size_t i = 0; i < 8; i++) {
10924 llvm::Value *ValOffsetPtr =
10935 Args.push_back(MemAddr);
10936 for (
size_t i = 0; i < 8; i++) {
10937 llvm::Value *ValOffsetPtr =
10944 auto Intr = (BuiltinID == clang::AArch64::BI__builtin_arm_st64b
10945 ? Intrinsic::aarch64_st64b
10946 : BuiltinID == clang::AArch64::BI__builtin_arm_st64bv
10947 ? Intrinsic::aarch64_st64bv
10948 : Intrinsic::aarch64_st64bv0);
10950 return Builder.CreateCall(F, Args);
10954 if (BuiltinID == clang::AArch64::BI__builtin_arm_rndr ||
10955 BuiltinID == clang::AArch64::BI__builtin_arm_rndrrs) {
10957 auto Intr = (BuiltinID == clang::AArch64::BI__builtin_arm_rndr
10958 ? Intrinsic::aarch64_rndr
10959 : Intrinsic::aarch64_rndrrs);
10961 llvm::Value *Val =
Builder.CreateCall(F);
10962 Value *RandomValue =
Builder.CreateExtractValue(Val, 0);
10971 if (BuiltinID == clang::AArch64::BI__clear_cache) {
10972 assert(E->
getNumArgs() == 2 &&
"__clear_cache takes 2 arguments");
10975 for (
unsigned i = 0; i < 2; i++)
10978 llvm::FunctionType *FTy = cast<llvm::FunctionType>(Ty);
10979 StringRef Name = FD->
getName();
10983 if ((BuiltinID == clang::AArch64::BI__builtin_arm_ldrex ||
10984 BuiltinID == clang::AArch64::BI__builtin_arm_ldaex) &&
10988 ? Intrinsic::aarch64_ldaxp
10989 : Intrinsic::aarch64_ldxp);
10996 llvm::Type *Int128Ty = llvm::IntegerType::get(
getLLVMContext(), 128);
10997 Val0 =
Builder.CreateZExt(Val0, Int128Ty);
10998 Val1 =
Builder.CreateZExt(Val1, Int128Ty);
11000 Value *ShiftCst = llvm::ConstantInt::get(Int128Ty, 64);
11001 Val =
Builder.CreateShl(Val0, ShiftCst,
"shl",
true );
11002 Val =
Builder.CreateOr(Val, Val1);
11004 }
else if (BuiltinID == clang::AArch64::BI__builtin_arm_ldrex ||
11005 BuiltinID == clang::AArch64::BI__builtin_arm_ldaex) {
11010 llvm::Type *
IntTy =
11015 ? Intrinsic::aarch64_ldaxr
11016 : Intrinsic::aarch64_ldxr,
11018 CallInst *Val =
Builder.CreateCall(F, LoadAddr,
"ldxr");
11022 if (RealResTy->isPointerTy())
11023 return Builder.CreateIntToPtr(Val, RealResTy);
11025 llvm::Type *IntResTy = llvm::IntegerType::get(
11027 return Builder.CreateBitCast(
Builder.CreateTruncOrBitCast(Val, IntResTy),
11031 if ((BuiltinID == clang::AArch64::BI__builtin_arm_strex ||
11032 BuiltinID == clang::AArch64::BI__builtin_arm_stlex) &&
11036 ? Intrinsic::aarch64_stlxp
11037 : Intrinsic::aarch64_stxp);
11049 return Builder.CreateCall(F, {Arg0, Arg1, StPtr},
"stxp");
11052 if (BuiltinID == clang::AArch64::BI__builtin_arm_strex ||
11053 BuiltinID == clang::AArch64::BI__builtin_arm_stlex) {
11058 llvm::Type *StoreTy =
11061 if (StoreVal->
getType()->isPointerTy())
11064 llvm::Type *
IntTy = llvm::IntegerType::get(
11073 ? Intrinsic::aarch64_stlxr
11074 : Intrinsic::aarch64_stxr,
11076 CallInst *CI =
Builder.CreateCall(F, {StoreVal, StoreAddr},
"stxr");
11078 1, Attribute::get(
getLLVMContext(), Attribute::ElementType, StoreTy));
11082 if (BuiltinID == clang::AArch64::BI__getReg) {
11085 llvm_unreachable(
"Sema will ensure that the parameter is constant");
11091 llvm::Metadata *Ops[] = {llvm::MDString::get(Context, Reg)};
11092 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
11093 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
11095 llvm::Function *F =
11097 return Builder.CreateCall(F, Metadata);
11100 if (BuiltinID == clang::AArch64::BI__break) {
11103 llvm_unreachable(
"Sema will ensure that the parameter is constant");
11105 llvm::Function *F =
CGM.
getIntrinsic(llvm::Intrinsic::aarch64_break);
11109 if (BuiltinID == clang::AArch64::BI__builtin_arm_clrex) {
11111 return Builder.CreateCall(F);
11114 if (BuiltinID == clang::AArch64::BI_ReadWriteBarrier)
11115 return Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent,
11116 llvm::SyncScope::SingleThread);
11119 Intrinsic::ID CRCIntrinsicID = Intrinsic::not_intrinsic;
11120 switch (BuiltinID) {
11121 case clang::AArch64::BI__builtin_arm_crc32b:
11122 CRCIntrinsicID = Intrinsic::aarch64_crc32b;
break;
11123 case clang::AArch64::BI__builtin_arm_crc32cb:
11124 CRCIntrinsicID = Intrinsic::aarch64_crc32cb;
break;
11125 case clang::AArch64::BI__builtin_arm_crc32h:
11126 CRCIntrinsicID = Intrinsic::aarch64_crc32h;
break;
11127 case clang::AArch64::BI__builtin_arm_crc32ch:
11128 CRCIntrinsicID = Intrinsic::aarch64_crc32ch;
break;
11129 case clang::AArch64::BI__builtin_arm_crc32w:
11130 CRCIntrinsicID = Intrinsic::aarch64_crc32w;
break;
11131 case clang::AArch64::BI__builtin_arm_crc32cw:
11132 CRCIntrinsicID = Intrinsic::aarch64_crc32cw;
break;
11133 case clang::AArch64::BI__builtin_arm_crc32d:
11134 CRCIntrinsicID = Intrinsic::aarch64_crc32x;
break;
11135 case clang::AArch64::BI__builtin_arm_crc32cd:
11136 CRCIntrinsicID = Intrinsic::aarch64_crc32cx;
break;
11139 if (CRCIntrinsicID != Intrinsic::not_intrinsic) {
11144 llvm::Type *DataTy = F->getFunctionType()->getParamType(1);
11145 Arg1 =
Builder.CreateZExtOrBitCast(Arg1, DataTy);
11147 return Builder.CreateCall(F, {Arg0, Arg1});
11151 if (BuiltinID == AArch64::BI__builtin_arm_mops_memset_tag) {
11159 CGM.
getIntrinsic(Intrinsic::aarch64_mops_memset_tag), {Dst, Val, Size});
11163 Intrinsic::ID MTEIntrinsicID = Intrinsic::not_intrinsic;
11164 switch (BuiltinID) {
11165 case clang::AArch64::BI__builtin_arm_irg:
11166 MTEIntrinsicID = Intrinsic::aarch64_irg;
break;
11167 case clang::AArch64::BI__builtin_arm_addg:
11168 MTEIntrinsicID = Intrinsic::aarch64_addg;
break;
11169 case clang::AArch64::BI__builtin_arm_gmi:
11170 MTEIntrinsicID = Intrinsic::aarch64_gmi;
break;
11171 case clang::AArch64::BI__builtin_arm_ldg:
11172 MTEIntrinsicID = Intrinsic::aarch64_ldg;
break;
11173 case clang::AArch64::BI__builtin_arm_stg:
11174 MTEIntrinsicID = Intrinsic::aarch64_stg;
break;
11175 case clang::AArch64::BI__builtin_arm_subp:
11176 MTEIntrinsicID = Intrinsic::aarch64_subp;
break;
11179 if (MTEIntrinsicID != Intrinsic::not_intrinsic) {
11182 if (MTEIntrinsicID == Intrinsic::aarch64_irg) {
11190 return Builder.CreatePointerCast(RV,
T);
11192 if (MTEIntrinsicID == Intrinsic::aarch64_addg) {
11200 return Builder.CreatePointerCast(RV,
T);
11202 if (MTEIntrinsicID == Intrinsic::aarch64_gmi) {
11214 if (MTEIntrinsicID == Intrinsic::aarch64_ldg) {
11219 return Builder.CreatePointerCast(RV,
T);
11224 if (MTEIntrinsicID == Intrinsic::aarch64_stg) {
11230 if (MTEIntrinsicID == Intrinsic::aarch64_subp) {
11240 if (BuiltinID == clang::AArch64::BI__builtin_arm_rsr ||
11241 BuiltinID == clang::AArch64::BI__builtin_arm_rsr64 ||
11242 BuiltinID == clang::AArch64::BI__builtin_arm_rsr128 ||
11243 BuiltinID == clang::AArch64::BI__builtin_arm_rsrp ||
11244 BuiltinID == clang::AArch64::BI__builtin_arm_wsr ||
11245 BuiltinID == clang::AArch64::BI__builtin_arm_wsr64 ||
11246 BuiltinID == clang::AArch64::BI__builtin_arm_wsr128 ||
11247 BuiltinID == clang::AArch64::BI__builtin_arm_wsrp) {
11250 if (BuiltinID == clang::AArch64::BI__builtin_arm_rsr ||
11251 BuiltinID == clang::AArch64::BI__builtin_arm_rsr64 ||
11252 BuiltinID == clang::AArch64::BI__builtin_arm_rsr128 ||
11253 BuiltinID == clang::AArch64::BI__builtin_arm_rsrp)
11256 bool IsPointerBuiltin = BuiltinID == clang::AArch64::BI__builtin_arm_rsrp ||
11257 BuiltinID == clang::AArch64::BI__builtin_arm_wsrp;
11259 bool Is32Bit = BuiltinID == clang::AArch64::BI__builtin_arm_rsr ||
11260 BuiltinID == clang::AArch64::BI__builtin_arm_wsr;
11262 bool Is128Bit = BuiltinID == clang::AArch64::BI__builtin_arm_rsr128 ||
11263 BuiltinID == clang::AArch64::BI__builtin_arm_wsr128;
11265 llvm::Type *ValueType;
11266 llvm::Type *RegisterType =
Int64Ty;
11269 }
else if (Is128Bit) {
11270 llvm::Type *Int128Ty =
11272 ValueType = Int128Ty;
11273 RegisterType = Int128Ty;
11274 }
else if (IsPointerBuiltin) {
11284 if (BuiltinID == clang::AArch64::BI_ReadStatusReg ||
11285 BuiltinID == clang::AArch64::BI_WriteStatusReg) {
11291 std::string SysRegStr;
11292 llvm::raw_string_ostream(SysRegStr) <<
11293 ((1 << 1) | ((SysReg >> 14) & 1)) <<
":" <<
11294 ((SysReg >> 11) & 7) <<
":" <<
11295 ((SysReg >> 7) & 15) <<
":" <<
11296 ((SysReg >> 3) & 15) <<
":" <<
11299 llvm::Metadata *Ops[] = { llvm::MDString::get(Context, SysRegStr) };
11300 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
11301 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
11303 llvm::Type *RegisterType =
Int64Ty;
11304 llvm::Type *Types[] = { RegisterType };
11306 if (BuiltinID == clang::AArch64::BI_ReadStatusReg) {
11307 llvm::Function *F =
CGM.
getIntrinsic(llvm::Intrinsic::read_register, Types);
11309 return Builder.CreateCall(F, Metadata);
11312 llvm::Function *F =
CGM.
getIntrinsic(llvm::Intrinsic::write_register, Types);
11315 return Builder.CreateCall(F, { Metadata, ArgValue });
11318 if (BuiltinID == clang::AArch64::BI_AddressOfReturnAddress) {
11319 llvm::Function *F =
11321 return Builder.CreateCall(F);
11324 if (BuiltinID == clang::AArch64::BI__builtin_sponentry) {
11326 return Builder.CreateCall(F);
11329 if (BuiltinID == clang::AArch64::BI__mulh ||
11330 BuiltinID == clang::AArch64::BI__umulh) {
11332 llvm::Type *Int128Ty = llvm::IntegerType::get(
getLLVMContext(), 128);
11334 bool IsSigned = BuiltinID == clang::AArch64::BI__mulh;
11340 Value *MulResult, *HigherBits;
11342 MulResult =
Builder.CreateNSWMul(LHS, RHS);
11343 HigherBits =
Builder.CreateAShr(MulResult, 64);
11345 MulResult =
Builder.CreateNUWMul(LHS, RHS);
11346 HigherBits =
Builder.CreateLShr(MulResult, 64);
11348 HigherBits =
Builder.CreateIntCast(HigherBits, ResType, IsSigned);
11353 if (BuiltinID == AArch64::BI__writex18byte ||
11354 BuiltinID == AArch64::BI__writex18word ||
11355 BuiltinID == AArch64::BI__writex18dword ||
11356 BuiltinID == AArch64::BI__writex18qword) {
11359 llvm::Metadata *Ops[] = {llvm::MDString::get(Context,
"x18")};
11360 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
11361 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
11362 llvm::Function *F =
11364 llvm::Value *X18 =
Builder.CreateCall(F, Metadata);
11375 if (BuiltinID == AArch64::BI__readx18byte ||
11376 BuiltinID == AArch64::BI__readx18word ||
11377 BuiltinID == AArch64::BI__readx18dword ||
11378 BuiltinID == AArch64::BI__readx18qword) {
11383 llvm::Metadata *Ops[] = {llvm::MDString::get(Context,
"x18")};
11384 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
11385 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
11386 llvm::Function *F =
11388 llvm::Value *X18 =
Builder.CreateCall(F, Metadata);
11398 if (BuiltinID == AArch64::BI_CopyDoubleFromInt64 ||
11399 BuiltinID == AArch64::BI_CopyFloatFromInt32 ||
11400 BuiltinID == AArch64::BI_CopyInt32FromFloat ||
11401 BuiltinID == AArch64::BI_CopyInt64FromDouble) {
11404 return Builder.CreateBitCast(Arg, RetTy);
11407 if (BuiltinID == AArch64::BI_CountLeadingOnes ||
11408 BuiltinID == AArch64::BI_CountLeadingOnes64 ||
11409 BuiltinID == AArch64::BI_CountLeadingZeros ||
11410 BuiltinID == AArch64::BI_CountLeadingZeros64) {
11412 llvm::Type *ArgType = Arg->
getType();
11414 if (BuiltinID == AArch64::BI_CountLeadingOnes ||
11415 BuiltinID == AArch64::BI_CountLeadingOnes64)
11416 Arg =
Builder.CreateXor(Arg, Constant::getAllOnesValue(ArgType));
11421 if (BuiltinID == AArch64::BI_CountLeadingOnes64 ||
11422 BuiltinID == AArch64::BI_CountLeadingZeros64)
11427 if (BuiltinID == AArch64::BI_CountLeadingSigns ||
11428 BuiltinID == AArch64::BI_CountLeadingSigns64) {
11431 Function *F = (BuiltinID == AArch64::BI_CountLeadingSigns)
11436 if (BuiltinID == AArch64::BI_CountLeadingSigns64)
11441 if (BuiltinID == AArch64::BI_CountOneBits ||
11442 BuiltinID == AArch64::BI_CountOneBits64) {
11444 llvm::Type *ArgType = ArgValue->
getType();
11448 if (BuiltinID == AArch64::BI_CountOneBits64)
11453 if (BuiltinID == AArch64::BI__prefetch) {
11464 if (std::optional<MSVCIntrin> MsvcIntId =
11470 return P.first == BuiltinID;
11473 BuiltinID = It->second;
11477 unsigned ICEArguments = 0;
11484 for (
unsigned i = 0, e = E->
getNumArgs() - 1; i != e; i++) {
11486 switch (BuiltinID) {
11487 case NEON::BI__builtin_neon_vld1_v:
11488 case NEON::BI__builtin_neon_vld1q_v:
11489 case NEON::BI__builtin_neon_vld1_dup_v:
11490 case NEON::BI__builtin_neon_vld1q_dup_v:
11491 case NEON::BI__builtin_neon_vld1_lane_v:
11492 case NEON::BI__builtin_neon_vld1q_lane_v:
11493 case NEON::BI__builtin_neon_vst1_v:
11494 case NEON::BI__builtin_neon_vst1q_v:
11495 case NEON::BI__builtin_neon_vst1_lane_v:
11496 case NEON::BI__builtin_neon_vst1q_lane_v:
11497 case NEON::BI__builtin_neon_vldap1_lane_s64:
11498 case NEON::BI__builtin_neon_vldap1q_lane_s64:
11499 case NEON::BI__builtin_neon_vstl1_lane_s64:
11500 case NEON::BI__builtin_neon_vstl1q_lane_s64:
11518 assert(
Result &&
"SISD intrinsic should have been handled");
11524 if (std::optional<llvm::APSInt>
Result =
11529 bool usgn =
Type.isUnsigned();
11530 bool quad =
Type.isQuad();
11533 switch (BuiltinID) {
11535 case NEON::BI__builtin_neon_vabsh_f16:
11538 case NEON::BI__builtin_neon_vaddq_p128: {
11541 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
11542 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
11543 Ops[0] =
Builder.CreateXor(Ops[0], Ops[1]);
11544 llvm::Type *Int128Ty = llvm::Type::getIntNTy(
getLLVMContext(), 128);
11545 return Builder.CreateBitCast(Ops[0], Int128Ty);
11547 case NEON::BI__builtin_neon_vldrq_p128: {
11548 llvm::Type *Int128Ty = llvm::Type::getIntNTy(
getLLVMContext(), 128);
11553 case NEON::BI__builtin_neon_vstrq_p128: {
11554 Value *Ptr = Ops[0];
11557 case NEON::BI__builtin_neon_vcvts_f32_u32:
11558 case NEON::BI__builtin_neon_vcvtd_f64_u64:
11561 case NEON::BI__builtin_neon_vcvts_f32_s32:
11562 case NEON::BI__builtin_neon_vcvtd_f64_s64: {
11564 bool Is64 = Ops[0]->getType()->getPrimitiveSizeInBits() == 64;
11567 Ops[0] =
Builder.CreateBitCast(Ops[0], InTy);
11569 return Builder.CreateUIToFP(Ops[0], FTy);
11570 return Builder.CreateSIToFP(Ops[0], FTy);
11572 case NEON::BI__builtin_neon_vcvth_f16_u16:
11573 case NEON::BI__builtin_neon_vcvth_f16_u32:
11574 case NEON::BI__builtin_neon_vcvth_f16_u64:
11577 case NEON::BI__builtin_neon_vcvth_f16_s16:
11578 case NEON::BI__builtin_neon_vcvth_f16_s32:
11579 case NEON::BI__builtin_neon_vcvth_f16_s64: {
11581 llvm::Type *FTy =
HalfTy;
11583 if (Ops[0]->getType()->getPrimitiveSizeInBits() == 64)
11585 else if (Ops[0]->getType()->getPrimitiveSizeInBits() == 32)
11589 Ops[0] =
Builder.CreateBitCast(Ops[0], InTy);
11591 return Builder.CreateUIToFP(Ops[0], FTy);
11592 return Builder.CreateSIToFP(Ops[0], FTy);
11594 case NEON::BI__builtin_neon_vcvtah_u16_f16:
11595 case NEON::BI__builtin_neon_vcvtmh_u16_f16:
11596 case NEON::BI__builtin_neon_vcvtnh_u16_f16:
11597 case NEON::BI__builtin_neon_vcvtph_u16_f16:
11598 case NEON::BI__builtin_neon_vcvth_u16_f16:
11599 case NEON::BI__builtin_neon_vcvtah_s16_f16:
11600 case NEON::BI__builtin_neon_vcvtmh_s16_f16:
11601 case NEON::BI__builtin_neon_vcvtnh_s16_f16:
11602 case NEON::BI__builtin_neon_vcvtph_s16_f16:
11603 case NEON::BI__builtin_neon_vcvth_s16_f16: {
11606 llvm::Type* FTy =
HalfTy;
11607 llvm::Type *Tys[2] = {InTy, FTy};
11609 switch (BuiltinID) {
11610 default: llvm_unreachable(
"missing builtin ID in switch!");
11611 case NEON::BI__builtin_neon_vcvtah_u16_f16:
11612 Int = Intrinsic::aarch64_neon_fcvtau;
break;
11613 case NEON::BI__builtin_neon_vcvtmh_u16_f16:
11614 Int = Intrinsic::aarch64_neon_fcvtmu;
break;
11615 case NEON::BI__builtin_neon_vcvtnh_u16_f16:
11616 Int = Intrinsic::aarch64_neon_fcvtnu;
break;
11617 case NEON::BI__builtin_neon_vcvtph_u16_f16:
11618 Int = Intrinsic::aarch64_neon_fcvtpu;
break;
11619 case NEON::BI__builtin_neon_vcvth_u16_f16:
11620 Int = Intrinsic::aarch64_neon_fcvtzu;
break;
11621 case NEON::BI__builtin_neon_vcvtah_s16_f16:
11622 Int = Intrinsic::aarch64_neon_fcvtas;
break;
11623 case NEON::BI__builtin_neon_vcvtmh_s16_f16:
11624 Int = Intrinsic::aarch64_neon_fcvtms;
break;
11625 case NEON::BI__builtin_neon_vcvtnh_s16_f16:
11626 Int = Intrinsic::aarch64_neon_fcvtns;
break;
11627 case NEON::BI__builtin_neon_vcvtph_s16_f16:
11628 Int = Intrinsic::aarch64_neon_fcvtps;
break;
11629 case NEON::BI__builtin_neon_vcvth_s16_f16:
11630 Int = Intrinsic::aarch64_neon_fcvtzs;
break;
11635 case NEON::BI__builtin_neon_vcaleh_f16:
11636 case NEON::BI__builtin_neon_vcalth_f16:
11637 case NEON::BI__builtin_neon_vcageh_f16:
11638 case NEON::BI__builtin_neon_vcagth_f16: {
11641 llvm::Type* FTy =
HalfTy;
11642 llvm::Type *Tys[2] = {InTy, FTy};
11644 switch (BuiltinID) {
11645 default: llvm_unreachable(
"missing builtin ID in switch!");
11646 case NEON::BI__builtin_neon_vcageh_f16:
11647 Int = Intrinsic::aarch64_neon_facge;
break;
11648 case NEON::BI__builtin_neon_vcagth_f16:
11649 Int = Intrinsic::aarch64_neon_facgt;
break;
11650 case NEON::BI__builtin_neon_vcaleh_f16:
11651 Int = Intrinsic::aarch64_neon_facge; std::swap(Ops[0], Ops[1]);
break;
11652 case NEON::BI__builtin_neon_vcalth_f16:
11653 Int = Intrinsic::aarch64_neon_facgt; std::swap(Ops[0], Ops[1]);
break;
11658 case NEON::BI__builtin_neon_vcvth_n_s16_f16:
11659 case NEON::BI__builtin_neon_vcvth_n_u16_f16: {
11662 llvm::Type* FTy =
HalfTy;
11663 llvm::Type *Tys[2] = {InTy, FTy};
11665 switch (BuiltinID) {
11666 default: llvm_unreachable(
"missing builtin ID in switch!");
11667 case NEON::BI__builtin_neon_vcvth_n_s16_f16:
11668 Int = Intrinsic::aarch64_neon_vcvtfp2fxs;
break;
11669 case NEON::BI__builtin_neon_vcvth_n_u16_f16:
11670 Int = Intrinsic::aarch64_neon_vcvtfp2fxu;
break;
11675 case NEON::BI__builtin_neon_vcvth_n_f16_s16:
11676 case NEON::BI__builtin_neon_vcvth_n_f16_u16: {
11678 llvm::Type* FTy =
HalfTy;
11680 llvm::Type *Tys[2] = {FTy, InTy};
11682 switch (BuiltinID) {
11683 default: llvm_unreachable(
"missing builtin ID in switch!");
11684 case NEON::BI__builtin_neon_vcvth_n_f16_s16:
11685 Int = Intrinsic::aarch64_neon_vcvtfxs2fp;
11686 Ops[0] =
Builder.CreateSExt(Ops[0], InTy,
"sext");
11688 case NEON::BI__builtin_neon_vcvth_n_f16_u16:
11689 Int = Intrinsic::aarch64_neon_vcvtfxu2fp;
11690 Ops[0] =
Builder.CreateZExt(Ops[0], InTy);
11695 case NEON::BI__builtin_neon_vpaddd_s64: {
11696 auto *Ty = llvm::FixedVectorType::get(
Int64Ty, 2);
11699 Vec =
Builder.CreateBitCast(Vec, Ty,
"v2i64");
11700 llvm::Value *Idx0 = llvm::ConstantInt::get(
SizeTy, 0);
11701 llvm::Value *Idx1 = llvm::ConstantInt::get(
SizeTy, 1);
11702 Value *Op0 =
Builder.CreateExtractElement(Vec, Idx0,
"lane0");
11703 Value *Op1 =
Builder.CreateExtractElement(Vec, Idx1,
"lane1");
11705 return Builder.CreateAdd(Op0, Op1,
"vpaddd");
11707 case NEON::BI__builtin_neon_vpaddd_f64: {
11708 auto *Ty = llvm::FixedVectorType::get(
DoubleTy, 2);
11711 Vec =
Builder.CreateBitCast(Vec, Ty,
"v2f64");
11712 llvm::Value *Idx0 = llvm::ConstantInt::get(
SizeTy, 0);
11713 llvm::Value *Idx1 = llvm::ConstantInt::get(
SizeTy, 1);
11714 Value *Op0 =
Builder.CreateExtractElement(Vec, Idx0,
"lane0");
11715 Value *Op1 =
Builder.CreateExtractElement(Vec, Idx1,
"lane1");
11717 return Builder.CreateFAdd(Op0, Op1,
"vpaddd");
11719 case NEON::BI__builtin_neon_vpadds_f32: {
11720 auto *Ty = llvm::FixedVectorType::get(
FloatTy, 2);
11723 Vec =
Builder.CreateBitCast(Vec, Ty,
"v2f32");
11724 llvm::Value *Idx0 = llvm::ConstantInt::get(
SizeTy, 0);
11725 llvm::Value *Idx1 = llvm::ConstantInt::get(
SizeTy, 1);
11726 Value *Op0 =
Builder.CreateExtractElement(Vec, Idx0,
"lane0");
11727 Value *Op1 =
Builder.CreateExtractElement(Vec, Idx1,
"lane1");
11729 return Builder.CreateFAdd(Op0, Op1,
"vpaddd");
11731 case NEON::BI__builtin_neon_vceqzd_s64:
11732 case NEON::BI__builtin_neon_vceqzd_f64:
11733 case NEON::BI__builtin_neon_vceqzs_f32:
11734 case NEON::BI__builtin_neon_vceqzh_f16:
11738 ICmpInst::FCMP_OEQ, ICmpInst::ICMP_EQ,
"vceqz");
11739 case NEON::BI__builtin_neon_vcgezd_s64:
11740 case NEON::BI__builtin_neon_vcgezd_f64:
11741 case NEON::BI__builtin_neon_vcgezs_f32:
11742 case NEON::BI__builtin_neon_vcgezh_f16:
11746 ICmpInst::FCMP_OGE, ICmpInst::ICMP_SGE,
"vcgez");
11747 case NEON::BI__builtin_neon_vclezd_s64:
11748 case NEON::BI__builtin_neon_vclezd_f64:
11749 case NEON::BI__builtin_neon_vclezs_f32:
11750 case NEON::BI__builtin_neon_vclezh_f16:
11754 ICmpInst::FCMP_OLE, ICmpInst::ICMP_SLE,
"vclez");
11755 case NEON::BI__builtin_neon_vcgtzd_s64:
11756 case NEON::BI__builtin_neon_vcgtzd_f64:
11757 case NEON::BI__builtin_neon_vcgtzs_f32:
11758 case NEON::BI__builtin_neon_vcgtzh_f16:
11762 ICmpInst::FCMP_OGT, ICmpInst::ICMP_SGT,
"vcgtz");
11763 case NEON::BI__builtin_neon_vcltzd_s64:
11764 case NEON::BI__builtin_neon_vcltzd_f64:
11765 case NEON::BI__builtin_neon_vcltzs_f32:
11766 case NEON::BI__builtin_neon_vcltzh_f16:
11770 ICmpInst::FCMP_OLT, ICmpInst::ICMP_SLT,
"vcltz");
11772 case NEON::BI__builtin_neon_vceqzd_u64: {
11776 Builder.CreateICmpEQ(Ops[0], llvm::Constant::getNullValue(
Int64Ty));
11779 case NEON::BI__builtin_neon_vceqd_f64:
11780 case NEON::BI__builtin_neon_vcled_f64:
11781 case NEON::BI__builtin_neon_vcltd_f64:
11782 case NEON::BI__builtin_neon_vcged_f64:
11783 case NEON::BI__builtin_neon_vcgtd_f64: {
11784 llvm::CmpInst::Predicate
P;
11785 switch (BuiltinID) {
11786 default: llvm_unreachable(
"missing builtin ID in switch!");
11787 case NEON::BI__builtin_neon_vceqd_f64:
P = llvm::FCmpInst::FCMP_OEQ;
break;
11788 case NEON::BI__builtin_neon_vcled_f64:
P = llvm::FCmpInst::FCMP_OLE;
break;
11789 case NEON::BI__builtin_neon_vcltd_f64:
P = llvm::FCmpInst::FCMP_OLT;
break;
11790 case NEON::BI__builtin_neon_vcged_f64:
P = llvm::FCmpInst::FCMP_OGE;
break;
11791 case NEON::BI__builtin_neon_vcgtd_f64:
P = llvm::FCmpInst::FCMP_OGT;
break;
11796 if (
P == llvm::FCmpInst::FCMP_OEQ)
11797 Ops[0] =
Builder.CreateFCmp(
P, Ops[0], Ops[1]);
11799 Ops[0] =
Builder.CreateFCmpS(
P, Ops[0], Ops[1]);
11802 case NEON::BI__builtin_neon_vceqs_f32:
11803 case NEON::BI__builtin_neon_vcles_f32:
11804 case NEON::BI__builtin_neon_vclts_f32:
11805 case NEON::BI__builtin_neon_vcges_f32:
11806 case NEON::BI__builtin_neon_vcgts_f32: {
11807 llvm::CmpInst::Predicate
P;
11808 switch (BuiltinID) {
11809 default: llvm_unreachable(
"missing builtin ID in switch!");
11810 case NEON::BI__builtin_neon_vceqs_f32:
P = llvm::FCmpInst::FCMP_OEQ;
break;
11811 case NEON::BI__builtin_neon_vcles_f32:
P = llvm::FCmpInst::FCMP_OLE;
break;
11812 case NEON::BI__builtin_neon_vclts_f32:
P = llvm::FCmpInst::FCMP_OLT;
break;
11813 case NEON::BI__builtin_neon_vcges_f32:
P = llvm::FCmpInst::FCMP_OGE;
break;
11814 case NEON::BI__builtin_neon_vcgts_f32:
P = llvm::FCmpInst::FCMP_OGT;
break;
11819 if (
P == llvm::FCmpInst::FCMP_OEQ)
11820 Ops[0] =
Builder.CreateFCmp(
P, Ops[0], Ops[1]);
11822 Ops[0] =
Builder.CreateFCmpS(
P, Ops[0], Ops[1]);
11825 case NEON::BI__builtin_neon_vceqh_f16:
11826 case NEON::BI__builtin_neon_vcleh_f16:
11827 case NEON::BI__builtin_neon_vclth_f16:
11828 case NEON::BI__builtin_neon_vcgeh_f16:
11829 case NEON::BI__builtin_neon_vcgth_f16: {
11830 llvm::CmpInst::Predicate
P;
11831 switch (BuiltinID) {
11832 default: llvm_unreachable(
"missing builtin ID in switch!");
11833 case NEON::BI__builtin_neon_vceqh_f16:
P = llvm::FCmpInst::FCMP_OEQ;
break;
11834 case NEON::BI__builtin_neon_vcleh_f16:
P = llvm::FCmpInst::FCMP_OLE;
break;
11835 case NEON::BI__builtin_neon_vclth_f16:
P = llvm::FCmpInst::FCMP_OLT;
break;
11836 case NEON::BI__builtin_neon_vcgeh_f16:
P = llvm::FCmpInst::FCMP_OGE;
break;
11837 case NEON::BI__builtin_neon_vcgth_f16:
P = llvm::FCmpInst::FCMP_OGT;
break;
11842 if (
P == llvm::FCmpInst::FCMP_OEQ)
11843 Ops[0] =
Builder.CreateFCmp(
P, Ops[0], Ops[1]);
11845 Ops[0] =
Builder.CreateFCmpS(
P, Ops[0], Ops[1]);
11848 case NEON::BI__builtin_neon_vceqd_s64:
11849 case NEON::BI__builtin_neon_vceqd_u64:
11850 case NEON::BI__builtin_neon_vcgtd_s64:
11851 case NEON::BI__builtin_neon_vcgtd_u64:
11852 case NEON::BI__builtin_neon_vcltd_s64:
11853 case NEON::BI__builtin_neon_vcltd_u64:
11854 case NEON::BI__builtin_neon_vcged_u64:
11855 case NEON::BI__builtin_neon_vcged_s64:
11856 case NEON::BI__builtin_neon_vcled_u64:
11857 case NEON::BI__builtin_neon_vcled_s64: {
11858 llvm::CmpInst::Predicate
P;
11859 switch (BuiltinID) {
11860 default: llvm_unreachable(
"missing builtin ID in switch!");
11861 case NEON::BI__builtin_neon_vceqd_s64:
11862 case NEON::BI__builtin_neon_vceqd_u64:
P = llvm::ICmpInst::ICMP_EQ;
break;
11863 case NEON::BI__builtin_neon_vcgtd_s64:
P = llvm::ICmpInst::ICMP_SGT;
break;
11864 case NEON::BI__builtin_neon_vcgtd_u64:
P = llvm::ICmpInst::ICMP_UGT;
break;
11865 case NEON::BI__builtin_neon_vcltd_s64:
P = llvm::ICmpInst::ICMP_SLT;
break;
11866 case NEON::BI__builtin_neon_vcltd_u64:
P = llvm::ICmpInst::ICMP_ULT;
break;
11867 case NEON::BI__builtin_neon_vcged_u64:
P = llvm::ICmpInst::ICMP_UGE;
break;
11868 case NEON::BI__builtin_neon_vcged_s64:
P = llvm::ICmpInst::ICMP_SGE;
break;
11869 case NEON::BI__builtin_neon_vcled_u64:
P = llvm::ICmpInst::ICMP_ULE;
break;
11870 case NEON::BI__builtin_neon_vcled_s64:
P = llvm::ICmpInst::ICMP_SLE;
break;
11875 Ops[0] =
Builder.CreateICmp(
P, Ops[0], Ops[1]);
11878 case NEON::BI__builtin_neon_vtstd_s64:
11879 case NEON::BI__builtin_neon_vtstd_u64: {
11883 Ops[0] =
Builder.CreateAnd(Ops[0], Ops[1]);
11884 Ops[0] =
Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0],
11885 llvm::Constant::getNullValue(
Int64Ty));
11888 case NEON::BI__builtin_neon_vset_lane_i8:
11889 case NEON::BI__builtin_neon_vset_lane_i16:
11890 case NEON::BI__builtin_neon_vset_lane_i32:
11891 case NEON::BI__builtin_neon_vset_lane_i64:
11892 case NEON::BI__builtin_neon_vset_lane_bf16:
11893 case NEON::BI__builtin_neon_vset_lane_f32:
11894 case NEON::BI__builtin_neon_vsetq_lane_i8:
11895 case NEON::BI__builtin_neon_vsetq_lane_i16:
11896 case NEON::BI__builtin_neon_vsetq_lane_i32:
11897 case NEON::BI__builtin_neon_vsetq_lane_i64:
11898 case NEON::BI__builtin_neon_vsetq_lane_bf16:
11899 case NEON::BI__builtin_neon_vsetq_lane_f32:
11901 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vset_lane");
11902 case NEON::BI__builtin_neon_vset_lane_f64:
11905 Builder.CreateBitCast(Ops[1], llvm::FixedVectorType::get(
DoubleTy, 1));
11907 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vset_lane");
11908 case NEON::BI__builtin_neon_vsetq_lane_f64:
11911 Builder.CreateBitCast(Ops[1], llvm::FixedVectorType::get(
DoubleTy, 2));
11913 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vset_lane");
11915 case NEON::BI__builtin_neon_vget_lane_i8:
11916 case NEON::BI__builtin_neon_vdupb_lane_i8:
11918 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
Int8Ty, 8));
11921 case NEON::BI__builtin_neon_vgetq_lane_i8:
11922 case NEON::BI__builtin_neon_vdupb_laneq_i8:
11924 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
Int8Ty, 16));
11927 case NEON::BI__builtin_neon_vget_lane_i16:
11928 case NEON::BI__builtin_neon_vduph_lane_i16:
11930 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
Int16Ty, 4));
11933 case NEON::BI__builtin_neon_vgetq_lane_i16:
11934 case NEON::BI__builtin_neon_vduph_laneq_i16:
11936 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
Int16Ty, 8));
11939 case NEON::BI__builtin_neon_vget_lane_i32:
11940 case NEON::BI__builtin_neon_vdups_lane_i32:
11942 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
Int32Ty, 2));
11945 case NEON::BI__builtin_neon_vdups_lane_f32:
11947 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
FloatTy, 2));
11950 case NEON::BI__builtin_neon_vgetq_lane_i32:
11951 case NEON::BI__builtin_neon_vdups_laneq_i32:
11953 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
Int32Ty, 4));
11956 case NEON::BI__builtin_neon_vget_lane_i64:
11957 case NEON::BI__builtin_neon_vdupd_lane_i64:
11959 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
Int64Ty, 1));
11962 case NEON::BI__builtin_neon_vdupd_lane_f64:
11964 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
DoubleTy, 1));
11967 case NEON::BI__builtin_neon_vgetq_lane_i64:
11968 case NEON::BI__builtin_neon_vdupd_laneq_i64:
11970 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
Int64Ty, 2));
11973 case NEON::BI__builtin_neon_vget_lane_f32:
11975 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
FloatTy, 2));
11978 case NEON::BI__builtin_neon_vget_lane_f64:
11980 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
DoubleTy, 1));
11983 case NEON::BI__builtin_neon_vgetq_lane_f32:
11984 case NEON::BI__builtin_neon_vdups_laneq_f32:
11986 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
FloatTy, 4));
11989 case NEON::BI__builtin_neon_vgetq_lane_f64:
11990 case NEON::BI__builtin_neon_vdupd_laneq_f64:
11992 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
DoubleTy, 2));
11995 case NEON::BI__builtin_neon_vaddh_f16:
11997 return Builder.CreateFAdd(Ops[0], Ops[1],
"vaddh");
11998 case NEON::BI__builtin_neon_vsubh_f16:
12000 return Builder.CreateFSub(Ops[0], Ops[1],
"vsubh");
12001 case NEON::BI__builtin_neon_vmulh_f16:
12003 return Builder.CreateFMul(Ops[0], Ops[1],
"vmulh");
12004 case NEON::BI__builtin_neon_vdivh_f16:
12006 return Builder.CreateFDiv(Ops[0], Ops[1],
"vdivh");
12007 case NEON::BI__builtin_neon_vfmah_f16:
12010 *
this, Intrinsic::fma, Intrinsic::experimental_constrained_fma,
HalfTy,
12012 case NEON::BI__builtin_neon_vfmsh_f16: {
12017 *
this, Intrinsic::fma, Intrinsic::experimental_constrained_fma,
HalfTy,
12020 case NEON::BI__builtin_neon_vaddd_s64:
12021 case NEON::BI__builtin_neon_vaddd_u64:
12023 case NEON::BI__builtin_neon_vsubd_s64:
12024 case NEON::BI__builtin_neon_vsubd_u64:
12026 case NEON::BI__builtin_neon_vqdmlalh_s16:
12027 case NEON::BI__builtin_neon_vqdmlslh_s16: {
12031 auto *VTy = llvm::FixedVectorType::get(
Int32Ty, 4);
12033 ProductOps,
"vqdmlXl");
12034 Constant *CI = ConstantInt::get(
SizeTy, 0);
12035 Ops[1] =
Builder.CreateExtractElement(Ops[1], CI,
"lane0");
12037 unsigned AccumInt = BuiltinID == NEON::BI__builtin_neon_vqdmlalh_s16
12038 ? Intrinsic::aarch64_neon_sqadd
12039 : Intrinsic::aarch64_neon_sqsub;
12042 case NEON::BI__builtin_neon_vqshlud_n_s64: {
12048 case NEON::BI__builtin_neon_vqshld_n_u64:
12049 case NEON::BI__builtin_neon_vqshld_n_s64: {
12050 unsigned Int = BuiltinID == NEON::BI__builtin_neon_vqshld_n_u64
12051 ? Intrinsic::aarch64_neon_uqshl
12052 : Intrinsic::aarch64_neon_sqshl;
12057 case NEON::BI__builtin_neon_vrshrd_n_u64:
12058 case NEON::BI__builtin_neon_vrshrd_n_s64: {
12059 unsigned Int = BuiltinID == NEON::BI__builtin_neon_vrshrd_n_u64
12060 ? Intrinsic::aarch64_neon_urshl
12061 : Intrinsic::aarch64_neon_srshl;
12063 int SV = cast<ConstantInt>(Ops[1])->getSExtValue();
12064 Ops[1] = ConstantInt::get(
Int64Ty, -SV);
12067 case NEON::BI__builtin_neon_vrsrad_n_u64:
12068 case NEON::BI__builtin_neon_vrsrad_n_s64: {
12069 unsigned Int = BuiltinID == NEON::BI__builtin_neon_vrsrad_n_u64
12070 ? Intrinsic::aarch64_neon_urshl
12071 : Intrinsic::aarch64_neon_srshl;
12075 {Ops[1], Builder.CreateSExt(Ops[2], Int64Ty)});
12078 case NEON::BI__builtin_neon_vshld_n_s64:
12079 case NEON::BI__builtin_neon_vshld_n_u64: {
12082 Ops[0], ConstantInt::get(
Int64Ty, Amt->getZExtValue()),
"shld_n");
12084 case NEON::BI__builtin_neon_vshrd_n_s64: {
12087 Ops[0], ConstantInt::get(
Int64Ty, std::min(
static_cast<uint64_t>(63),
12088 Amt->getZExtValue())),
12091 case NEON::BI__builtin_neon_vshrd_n_u64: {
12093 uint64_t ShiftAmt = Amt->getZExtValue();
12095 if (ShiftAmt == 64)
12096 return ConstantInt::get(
Int64Ty, 0);
12097 return Builder.CreateLShr(Ops[0], ConstantInt::get(
Int64Ty, ShiftAmt),
12100 case NEON::BI__builtin_neon_vsrad_n_s64: {
12103 Ops[1], ConstantInt::get(
Int64Ty, std::min(
static_cast<uint64_t>(63),
12104 Amt->getZExtValue())),
12106 return Builder.CreateAdd(Ops[0], Ops[1]);
12108 case NEON::BI__builtin_neon_vsrad_n_u64: {
12110 uint64_t ShiftAmt = Amt->getZExtValue();
12113 if (ShiftAmt == 64)
12115 Ops[1] =
Builder.CreateLShr(Ops[1], ConstantInt::get(
Int64Ty, ShiftAmt),
12117 return Builder.CreateAdd(Ops[0], Ops[1]);
12119 case NEON::BI__builtin_neon_vqdmlalh_lane_s16:
12120 case NEON::BI__builtin_neon_vqdmlalh_laneq_s16:
12121 case NEON::BI__builtin_neon_vqdmlslh_lane_s16:
12122 case NEON::BI__builtin_neon_vqdmlslh_laneq_s16: {
12128 auto *VTy = llvm::FixedVectorType::get(
Int32Ty, 4);
12130 ProductOps,
"vqdmlXl");
12131 Constant *CI = ConstantInt::get(
SizeTy, 0);
12132 Ops[1] =
Builder.CreateExtractElement(Ops[1], CI,
"lane0");
12135 unsigned AccInt = (BuiltinID == NEON::BI__builtin_neon_vqdmlalh_lane_s16 ||
12136 BuiltinID == NEON::BI__builtin_neon_vqdmlalh_laneq_s16)
12137 ? Intrinsic::aarch64_neon_sqadd
12138 : Intrinsic::aarch64_neon_sqsub;
12141 case NEON::BI__builtin_neon_vqdmlals_s32:
12142 case NEON::BI__builtin_neon_vqdmlsls_s32: {
12144 ProductOps.push_back(Ops[1]);
12148 ProductOps,
"vqdmlXl");
12150 unsigned AccumInt = BuiltinID == NEON::BI__builtin_neon_vqdmlals_s32
12151 ? Intrinsic::aarch64_neon_sqadd
12152 : Intrinsic::aarch64_neon_sqsub;
12155 case NEON::BI__builtin_neon_vqdmlals_lane_s32:
12156 case NEON::BI__builtin_neon_vqdmlals_laneq_s32:
12157 case NEON::BI__builtin_neon_vqdmlsls_lane_s32:
12158 case NEON::BI__builtin_neon_vqdmlsls_laneq_s32: {
12162 ProductOps.push_back(Ops[1]);
12163 ProductOps.push_back(Ops[2]);
12166 ProductOps,
"vqdmlXl");
12169 unsigned AccInt = (BuiltinID == NEON::BI__builtin_neon_vqdmlals_lane_s32 ||
12170 BuiltinID == NEON::BI__builtin_neon_vqdmlals_laneq_s32)
12171 ? Intrinsic::aarch64_neon_sqadd
12172 : Intrinsic::aarch64_neon_sqsub;
12175 case NEON::BI__builtin_neon_vget_lane_bf16:
12176 case NEON::BI__builtin_neon_vduph_lane_bf16:
12177 case NEON::BI__builtin_neon_vduph_lane_f16: {
12181 case NEON::BI__builtin_neon_vgetq_lane_bf16:
12182 case NEON::BI__builtin_neon_vduph_laneq_bf16:
12183 case NEON::BI__builtin_neon_vduph_laneq_f16: {
12188 case clang::AArch64::BI_InterlockedAdd:
12189 case clang::AArch64::BI_InterlockedAdd64: {
12192 AtomicRMWInst *RMWI =
12194 llvm::AtomicOrdering::SequentiallyConsistent);
12195 return Builder.CreateAdd(RMWI, Val);
12200 llvm::Type *Ty = VTy;
12211 Builtin->BuiltinID, Builtin->LLVMIntrinsic, Builtin->AltLLVMIntrinsic,
12212 Builtin->NameHint, Builtin->TypeModifier, E, Ops,
12219 switch (BuiltinID) {
12220 default:
return nullptr;
12221 case NEON::BI__builtin_neon_vbsl_v:
12222 case NEON::BI__builtin_neon_vbslq_v: {
12223 llvm::Type *BitTy = llvm::VectorType::getInteger(VTy);
12224 Ops[0] =
Builder.CreateBitCast(Ops[0], BitTy,
"vbsl");
12225 Ops[1] =
Builder.CreateBitCast(Ops[1], BitTy,
"vbsl");
12226 Ops[2] =
Builder.CreateBitCast(Ops[2], BitTy,
"vbsl");
12228 Ops[1] =
Builder.CreateAnd(Ops[0], Ops[1],
"vbsl");
12229 Ops[2] =
Builder.CreateAnd(
Builder.CreateNot(Ops[0]), Ops[2],
"vbsl");
12230 Ops[0] =
Builder.CreateOr(Ops[1], Ops[2],
"vbsl");
12231 return Builder.CreateBitCast(Ops[0], Ty);
12233 case NEON::BI__builtin_neon_vfma_lane_v:
12234 case NEON::BI__builtin_neon_vfmaq_lane_v: {
12237 Value *Addend = Ops[0];
12238 Value *Multiplicand = Ops[1];
12239 Value *LaneSource = Ops[2];
12240 Ops[0] = Multiplicand;
12241 Ops[1] = LaneSource;
12245 auto *SourceTy = BuiltinID == NEON::BI__builtin_neon_vfmaq_lane_v
12246 ? llvm::FixedVectorType::get(VTy->getElementType(),
12247 VTy->getNumElements() / 2)
12249 llvm::Constant *cst = cast<Constant>(Ops[3]);
12250 Value *SV = llvm::ConstantVector::getSplat(VTy->getElementCount(), cst);
12251 Ops[1] =
Builder.CreateBitCast(Ops[1], SourceTy);
12252 Ops[1] =
Builder.CreateShuffleVector(Ops[1], Ops[1], SV,
"lane");
12255 Int =
Builder.getIsFPConstrained() ? Intrinsic::experimental_constrained_fma
12259 case NEON::BI__builtin_neon_vfma_laneq_v: {
12260 auto *VTy = cast<llvm::FixedVectorType>(Ty);
12262 if (VTy && VTy->getElementType() ==
DoubleTy) {
12265 llvm::FixedVectorType *VTy =
12267 Ops[2] =
Builder.CreateBitCast(Ops[2], VTy);
12268 Ops[2] =
Builder.CreateExtractElement(Ops[2], Ops[3],
"extract");
12271 *
this, Intrinsic::fma, Intrinsic::experimental_constrained_fma,
12272 DoubleTy, {Ops[1], Ops[2], Ops[0]});
12275 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
12276 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
12278 auto *STy = llvm::FixedVectorType::get(VTy->getElementType(),
12279 VTy->getNumElements() * 2);
12280 Ops[2] =
Builder.CreateBitCast(Ops[2], STy);
12281 Value *SV = llvm::ConstantVector::getSplat(VTy->getElementCount(),
12282 cast<ConstantInt>(Ops[3]));
12283 Ops[2] =
Builder.CreateShuffleVector(Ops[2], Ops[2], SV,
"lane");
12286 *
this, Intrinsic::fma, Intrinsic::experimental_constrained_fma, Ty,
12287 {Ops[2], Ops[1], Ops[0]});
12289 case NEON::BI__builtin_neon_vfmaq_laneq_v: {
12290 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
12291 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
12293 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
12296 *
this, Intrinsic::fma, Intrinsic::experimental_constrained_fma, Ty,
12297 {Ops[2], Ops[1], Ops[0]});
12299 case NEON::BI__builtin_neon_vfmah_lane_f16:
12300 case NEON::BI__builtin_neon_vfmas_lane_f32:
12301 case NEON::BI__builtin_neon_vfmah_laneq_f16:
12302 case NEON::BI__builtin_neon_vfmas_laneq_f32:
12303 case NEON::BI__builtin_neon_vfmad_lane_f64:
12304 case NEON::BI__builtin_neon_vfmad_laneq_f64: {
12307 Ops[2] =
Builder.CreateExtractElement(Ops[2], Ops[3],
"extract");
12309 *
this, Intrinsic::fma, Intrinsic::experimental_constrained_fma, Ty,
12310 {Ops[1], Ops[2], Ops[0]});
12312 case NEON::BI__builtin_neon_vmull_v:
12314 Int = usgn ? Intrinsic::aarch64_neon_umull : Intrinsic::aarch64_neon_smull;
12315 if (
Type.isPoly())
Int = Intrinsic::aarch64_neon_pmull;
12317 case NEON::BI__builtin_neon_vmax_v:
12318 case NEON::BI__builtin_neon_vmaxq_v:
12320 Int = usgn ? Intrinsic::aarch64_neon_umax : Intrinsic::aarch64_neon_smax;
12321 if (Ty->isFPOrFPVectorTy())
Int = Intrinsic::aarch64_neon_fmax;
12323 case NEON::BI__builtin_neon_vmaxh_f16: {
12325 Int = Intrinsic::aarch64_neon_fmax;
12328 case NEON::BI__builtin_neon_vmin_v:
12329 case NEON::BI__builtin_neon_vminq_v:
12331 Int = usgn ? Intrinsic::aarch64_neon_umin : Intrinsic::aarch64_neon_smin;
12332 if (Ty->isFPOrFPVectorTy())
Int = Intrinsic::aarch64_neon_fmin;
12334 case NEON::BI__builtin_neon_vminh_f16: {
12336 Int = Intrinsic::aarch64_neon_fmin;
12339 case NEON::BI__builtin_neon_vabd_v:
12340 case NEON::BI__builtin_neon_vabdq_v:
12342 Int = usgn ? Intrinsic::aarch64_neon_uabd : Intrinsic::aarch64_neon_sabd;
12343 if (Ty->isFPOrFPVectorTy())
Int = Intrinsic::aarch64_neon_fabd;
12345 case NEON::BI__builtin_neon_vpadal_v:
12346 case NEON::BI__builtin_neon_vpadalq_v: {
12347 unsigned ArgElts = VTy->getNumElements();
12348 llvm::IntegerType *EltTy = cast<IntegerType>(VTy->getElementType());
12349 unsigned BitWidth = EltTy->getBitWidth();
12350 auto *ArgTy = llvm::FixedVectorType::get(
12351 llvm::IntegerType::get(
getLLVMContext(), BitWidth / 2), 2 * ArgElts);
12352 llvm::Type* Tys[2] = { VTy, ArgTy };
12353 Int = usgn ? Intrinsic::aarch64_neon_uaddlp : Intrinsic::aarch64_neon_saddlp;
12355 TmpOps.push_back(Ops[1]);
12358 llvm::Value *addend =
Builder.CreateBitCast(Ops[0], tmp->getType());
12359 return Builder.CreateAdd(tmp, addend);
12361 case NEON::BI__builtin_neon_vpmin_v:
12362 case NEON::BI__builtin_neon_vpminq_v:
12364 Int = usgn ? Intrinsic::aarch64_neon_uminp : Intrinsic::aarch64_neon_sminp;
12365 if (Ty->isFPOrFPVectorTy())
Int = Intrinsic::aarch64_neon_fminp;
12367 case NEON::BI__builtin_neon_vpmax_v:
12368 case NEON::BI__builtin_neon_vpmaxq_v:
12370 Int = usgn ? Intrinsic::aarch64_neon_umaxp : Intrinsic::aarch64_neon_smaxp;
12371 if (Ty->isFPOrFPVectorTy())
Int = Intrinsic::aarch64_neon_fmaxp;
12373 case NEON::BI__builtin_neon_vminnm_v:
12374 case NEON::BI__builtin_neon_vminnmq_v:
12375 Int = Intrinsic::aarch64_neon_fminnm;
12377 case NEON::BI__builtin_neon_vminnmh_f16:
12379 Int = Intrinsic::aarch64_neon_fminnm;
12381 case NEON::BI__builtin_neon_vmaxnm_v:
12382 case NEON::BI__builtin_neon_vmaxnmq_v:
12383 Int = Intrinsic::aarch64_neon_fmaxnm;
12385 case NEON::BI__builtin_neon_vmaxnmh_f16:
12387 Int = Intrinsic::aarch64_neon_fmaxnm;
12389 case NEON::BI__builtin_neon_vrecpss_f32: {
12394 case NEON::BI__builtin_neon_vrecpsd_f64:
12398 case NEON::BI__builtin_neon_vrecpsh_f16:
12402 case NEON::BI__builtin_neon_vqshrun_n_v:
12403 Int = Intrinsic::aarch64_neon_sqshrun;
12405 case NEON::BI__builtin_neon_vqrshrun_n_v:
12406 Int = Intrinsic::aarch64_neon_sqrshrun;
12408 case NEON::BI__builtin_neon_vqshrn_n_v:
12409 Int = usgn ? Intrinsic::aarch64_neon_uqshrn : Intrinsic::aarch64_neon_sqshrn;
12411 case NEON::BI__builtin_neon_vrshrn_n_v:
12412 Int = Intrinsic::aarch64_neon_rshrn;
12414 case NEON::BI__builtin_neon_vqrshrn_n_v:
12415 Int = usgn ? Intrinsic::aarch64_neon_uqrshrn : Intrinsic::aarch64_neon_sqrshrn;
12417 case NEON::BI__builtin_neon_vrndah_f16: {
12420 ? Intrinsic::experimental_constrained_round
12421 : Intrinsic::round;
12424 case NEON::BI__builtin_neon_vrnda_v:
12425 case NEON::BI__builtin_neon_vrndaq_v: {
12427 ? Intrinsic::experimental_constrained_round
12428 : Intrinsic::round;
12431 case NEON::BI__builtin_neon_vrndih_f16: {
12434 ? Intrinsic::experimental_constrained_nearbyint
12435 : Intrinsic::nearbyint;
12438 case NEON::BI__builtin_neon_vrndmh_f16: {
12441 ? Intrinsic::experimental_constrained_floor
12442 : Intrinsic::floor;
12445 case NEON::BI__builtin_neon_vrndm_v:
12446 case NEON::BI__builtin_neon_vrndmq_v: {
12448 ? Intrinsic::experimental_constrained_floor
12449 : Intrinsic::floor;
12452 case NEON::BI__builtin_neon_vrndnh_f16: {
12455 ? Intrinsic::experimental_constrained_roundeven
12456 : Intrinsic::roundeven;
12459 case NEON::BI__builtin_neon_vrndn_v:
12460 case NEON::BI__builtin_neon_vrndnq_v: {
12462 ? Intrinsic::experimental_constrained_roundeven
12463 : Intrinsic::roundeven;
12466 case NEON::BI__builtin_neon_vrndns_f32: {
12469 ? Intrinsic::experimental_constrained_roundeven
12470 : Intrinsic::roundeven;
12473 case NEON::BI__builtin_neon_vrndph_f16: {
12476 ? Intrinsic::experimental_constrained_ceil
12480 case NEON::BI__builtin_neon_vrndp_v:
12481 case NEON::BI__builtin_neon_vrndpq_v: {
12483 ? Intrinsic::experimental_constrained_ceil
12487 case NEON::BI__builtin_neon_vrndxh_f16: {
12490 ? Intrinsic::experimental_constrained_rint
12494 case NEON::BI__builtin_neon_vrndx_v:
12495 case NEON::BI__builtin_neon_vrndxq_v: {
12497 ? Intrinsic::experimental_constrained_rint
12501 case NEON::BI__builtin_neon_vrndh_f16: {
12504 ? Intrinsic::experimental_constrained_trunc
12505 : Intrinsic::trunc;
12508 case NEON::BI__builtin_neon_vrnd32x_f32:
12509 case NEON::BI__builtin_neon_vrnd32xq_f32:
12510 case NEON::BI__builtin_neon_vrnd32x_f64:
12511 case NEON::BI__builtin_neon_vrnd32xq_f64: {
12513 Int = Intrinsic::aarch64_neon_frint32x;
12516 case NEON::BI__builtin_neon_vrnd32z_f32:
12517 case NEON::BI__builtin_neon_vrnd32zq_f32:
12518 case NEON::BI__builtin_neon_vrnd32z_f64:
12519 case NEON::BI__builtin_neon_vrnd32zq_f64: {
12521 Int = Intrinsic::aarch64_neon_frint32z;
12524 case NEON::BI__builtin_neon_vrnd64x_f32:
12525 case NEON::BI__builtin_neon_vrnd64xq_f32:
12526 case NEON::BI__builtin_neon_vrnd64x_f64:
12527 case NEON::BI__builtin_neon_vrnd64xq_f64: {
12529 Int = Intrinsic::aarch64_neon_frint64x;
12532 case NEON::BI__builtin_neon_vrnd64z_f32:
12533 case NEON::BI__builtin_neon_vrnd64zq_f32:
12534 case NEON::BI__builtin_neon_vrnd64z_f64:
12535 case NEON::BI__builtin_neon_vrnd64zq_f64: {
12537 Int = Intrinsic::aarch64_neon_frint64z;
12540 case NEON::BI__builtin_neon_vrnd_v:
12541 case NEON::BI__builtin_neon_vrndq_v: {
12543 ? Intrinsic::experimental_constrained_trunc
12544 : Intrinsic::trunc;
12547 case NEON::BI__builtin_neon_vcvt_f64_v:
12548 case NEON::BI__builtin_neon_vcvtq_f64_v:
12549 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
12551 return usgn ?
Builder.CreateUIToFP(Ops[0], Ty,
"vcvt")
12552 :
Builder.CreateSIToFP(Ops[0], Ty,
"vcvt");
12553 case NEON::BI__builtin_neon_vcvt_f64_f32: {
12555 "unexpected vcvt_f64_f32 builtin");
12559 return Builder.CreateFPExt(Ops[0], Ty,
"vcvt");
12561 case NEON::BI__builtin_neon_vcvt_f32_f64: {
12563 "unexpected vcvt_f32_f64 builtin");
12567 return Builder.CreateFPTrunc(Ops[0], Ty,
"vcvt");
12569 case NEON::BI__builtin_neon_vcvt_s32_v:
12570 case NEON::BI__builtin_neon_vcvt_u32_v:
12571 case NEON::BI__builtin_neon_vcvt_s64_v:
12572 case NEON::BI__builtin_neon_vcvt_u64_v:
12573 case NEON::BI__builtin_neon_vcvt_s16_f16:
12574 case NEON::BI__builtin_neon_vcvt_u16_f16:
12575 case NEON::BI__builtin_neon_vcvtq_s32_v:
12576 case NEON::BI__builtin_neon_vcvtq_u32_v:
12577 case NEON::BI__builtin_neon_vcvtq_s64_v:
12578 case NEON::BI__builtin_neon_vcvtq_u64_v:
12579 case NEON::BI__builtin_neon_vcvtq_s16_f16:
12580 case NEON::BI__builtin_neon_vcvtq_u16_f16: {
12582 usgn ? Intrinsic::aarch64_neon_fcvtzu : Intrinsic::aarch64_neon_fcvtzs;
12586 case NEON::BI__builtin_neon_vcvta_s16_f16:
12587 case NEON::BI__builtin_neon_vcvta_u16_f16:
12588 case NEON::BI__builtin_neon_vcvta_s32_v:
12589 case NEON::BI__builtin_neon_vcvtaq_s16_f16:
12590 case NEON::BI__builtin_neon_vcvtaq_s32_v:
12591 case NEON::BI__builtin_neon_vcvta_u32_v:
12592 case NEON::BI__builtin_neon_vcvtaq_u16_f16:
12593 case NEON::BI__builtin_neon_vcvtaq_u32_v:
12594 case NEON::BI__builtin_neon_vcvta_s64_v:
12595 case NEON::BI__builtin_neon_vcvtaq_s64_v:
12596 case NEON::BI__builtin_neon_vcvta_u64_v:
12597 case NEON::BI__builtin_neon_vcvtaq_u64_v: {
12598 Int = usgn ? Intrinsic::aarch64_neon_fcvtau : Intrinsic::aarch64_neon_fcvtas;
12602 case NEON::BI__builtin_neon_vcvtm_s16_f16:
12603 case NEON::BI__builtin_neon_vcvtm_s32_v:
12604 case NEON::BI__builtin_neon_vcvtmq_s16_f16:
12605 case NEON::BI__builtin_neon_vcvtmq_s32_v:
12606 case NEON::BI__builtin_neon_vcvtm_u16_f16:
12607 case NEON::BI__builtin_neon_vcvtm_u32_v:
12608 case NEON::BI__builtin_neon_vcvtmq_u16_f16:
12609 case NEON::BI__builtin_neon_vcvtmq_u32_v:
12610 case NEON::BI__builtin_neon_vcvtm_s64_v:
12611 case NEON::BI__builtin_neon_vcvtmq_s64_v:
12612 case NEON::BI__builtin_neon_vcvtm_u64_v:
12613 case NEON::BI__builtin_neon_vcvtmq_u64_v: {
12614 Int = usgn ? Intrinsic::aarch64_neon_fcvtmu : Intrinsic::aarch64_neon_fcvtms;
12618 case NEON::BI__builtin_neon_vcvtn_s16_f16:
12619 case NEON::BI__builtin_neon_vcvtn_s32_v:
12620 case NEON::BI__builtin_neon_vcvtnq_s16_f16:
12621 case NEON::BI__builtin_neon_vcvtnq_s32_v:
12622 case NEON::BI__builtin_neon_vcvtn_u16_f16:
12623 case NEON::BI__builtin_neon_vcvtn_u32_v:
12624 case NEON::BI__builtin_neon_vcvtnq_u16_f16:
12625 case NEON::BI__builtin_neon_vcvtnq_u32_v:
12626 case NEON::BI__builtin_neon_vcvtn_s64_v:
12627 case NEON::BI__builtin_neon_vcvtnq_s64_v:
12628 case NEON::BI__builtin_neon_vcvtn_u64_v:
12629 case NEON::BI__builtin_neon_vcvtnq_u64_v: {
12630 Int = usgn ? Intrinsic::aarch64_neon_fcvtnu : Intrinsic::aarch64_neon_fcvtns;
12634 case NEON::BI__builtin_neon_vcvtp_s16_f16:
12635 case NEON::BI__builtin_neon_vcvtp_s32_v:
12636 case NEON::BI__builtin_neon_vcvtpq_s16_f16:
12637 case NEON::BI__builtin_neon_vcvtpq_s32_v:
12638 case NEON::BI__builtin_neon_vcvtp_u16_f16:
12639 case NEON::BI__builtin_neon_vcvtp_u32_v:
12640 case NEON::BI__builtin_neon_vcvtpq_u16_f16:
12641 case NEON::BI__builtin_neon_vcvtpq_u32_v:
12642 case NEON::BI__builtin_neon_vcvtp_s64_v:
12643 case NEON::BI__builtin_neon_vcvtpq_s64_v:
12644 case NEON::BI__builtin_neon_vcvtp_u64_v:
12645 case NEON::BI__builtin_neon_vcvtpq_u64_v: {
12646 Int = usgn ? Intrinsic::aarch64_neon_fcvtpu : Intrinsic::aarch64_neon_fcvtps;
12650 case NEON::BI__builtin_neon_vmulx_v:
12651 case NEON::BI__builtin_neon_vmulxq_v: {
12652 Int = Intrinsic::aarch64_neon_fmulx;
12655 case NEON::BI__builtin_neon_vmulxh_lane_f16:
12656 case NEON::BI__builtin_neon_vmulxh_laneq_f16: {
12660 Ops[1] =
Builder.CreateExtractElement(Ops[1], Ops[2],
"extract");
12662 Int = Intrinsic::aarch64_neon_fmulx;
12665 case NEON::BI__builtin_neon_vmul_lane_v:
12666 case NEON::BI__builtin_neon_vmul_laneq_v: {
12669 if (BuiltinID == NEON::BI__builtin_neon_vmul_laneq_v)
12672 llvm::FixedVectorType *VTy =
12674 Ops[1] =
Builder.CreateBitCast(Ops[1], VTy);
12675 Ops[1] =
Builder.CreateExtractElement(Ops[1], Ops[2],
"extract");
12679 case NEON::BI__builtin_neon_vnegd_s64:
12681 case NEON::BI__builtin_neon_vnegh_f16:
12683 case NEON::BI__builtin_neon_vpmaxnm_v:
12684 case NEON::BI__builtin_neon_vpmaxnmq_v: {
12685 Int = Intrinsic::aarch64_neon_fmaxnmp;
12688 case NEON::BI__builtin_neon_vpminnm_v:
12689 case NEON::BI__builtin_neon_vpminnmq_v: {
12690 Int = Intrinsic::aarch64_neon_fminnmp;
12693 case NEON::BI__builtin_neon_vsqrth_f16: {
12696 ? Intrinsic::experimental_constrained_sqrt
12700 case NEON::BI__builtin_neon_vsqrt_v:
12701 case NEON::BI__builtin_neon_vsqrtq_v: {
12703 ? Intrinsic::experimental_constrained_sqrt
12705 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
12708 case NEON::BI__builtin_neon_vrbit_v:
12709 case NEON::BI__builtin_neon_vrbitq_v: {
12710 Int = Intrinsic::bitreverse;
12713 case NEON::BI__builtin_neon_vaddv_u8:
12717 case NEON::BI__builtin_neon_vaddv_s8: {
12718 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
12720 VTy = llvm::FixedVectorType::get(
Int8Ty, 8);
12721 llvm::Type *Tys[2] = { Ty, VTy };
12726 case NEON::BI__builtin_neon_vaddv_u16:
12729 case NEON::BI__builtin_neon_vaddv_s16: {
12730 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
12732 VTy = llvm::FixedVectorType::get(
Int16Ty, 4);
12733 llvm::Type *Tys[2] = { Ty, VTy };
12738 case NEON::BI__builtin_neon_vaddvq_u8:
12741 case NEON::BI__builtin_neon_vaddvq_s8: {
12742 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
12744 VTy = llvm::FixedVectorType::get(
Int8Ty, 16);
12745 llvm::Type *Tys[2] = { Ty, VTy };
12750 case NEON::BI__builtin_neon_vaddvq_u16:
12753 case NEON::BI__builtin_neon_vaddvq_s16: {
12754 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
12756 VTy = llvm::FixedVectorType::get(
Int16Ty, 8);
12757 llvm::Type *Tys[2] = { Ty, VTy };
12762 case NEON::BI__builtin_neon_vmaxv_u8: {
12763 Int = Intrinsic::aarch64_neon_umaxv;
12765 VTy = llvm::FixedVectorType::get(
Int8Ty, 8);
12766 llvm::Type *Tys[2] = { Ty, VTy };
12771 case NEON::BI__builtin_neon_vmaxv_u16: {
12772 Int = Intrinsic::aarch64_neon_umaxv;
12774 VTy = llvm::FixedVectorType::get(
Int16Ty, 4);
12775 llvm::Type *Tys[2] = { Ty, VTy };
12780 case NEON::BI__builtin_neon_vmaxvq_u8: {
12781 Int = Intrinsic::aarch64_neon_umaxv;
12783 VTy = llvm::FixedVectorType::get(
Int8Ty, 16);
12784 llvm::Type *Tys[2] = { Ty, VTy };
12789 case NEON::BI__builtin_neon_vmaxvq_u16: {
12790 Int = Intrinsic::aarch64_neon_umaxv;
12792 VTy = llvm::FixedVectorType::get(
Int16Ty, 8);
12793 llvm::Type *Tys[2] = { Ty, VTy };
12798 case NEON::BI__builtin_neon_vmaxv_s8: {
12799 Int = Intrinsic::aarch64_neon_smaxv;
12801 VTy = llvm::FixedVectorType::get(
Int8Ty, 8);
12802 llvm::Type *Tys[2] = { Ty, VTy };
12807 case NEON::BI__builtin_neon_vmaxv_s16: {
12808 Int = Intrinsic::aarch64_neon_smaxv;
12810 VTy = llvm::FixedVectorType::get(
Int16Ty, 4);
12811 llvm::Type *Tys[2] = { Ty, VTy };
12816 case NEON::BI__builtin_neon_vmaxvq_s8: {
12817 Int = Intrinsic::aarch64_neon_smaxv;
12819 VTy = llvm::FixedVectorType::get(
Int8Ty, 16);
12820 llvm::Type *Tys[2] = { Ty, VTy };
12825 case NEON::BI__builtin_neon_vmaxvq_s16: {
12826 Int = Intrinsic::aarch64_neon_smaxv;
12828 VTy = llvm::FixedVectorType::get(
Int16Ty, 8);
12829 llvm::Type *Tys[2] = { Ty, VTy };
12834 case NEON::BI__builtin_neon_vmaxv_f16: {
12835 Int = Intrinsic::aarch64_neon_fmaxv;
12837 VTy = llvm::FixedVectorType::get(
HalfTy, 4);
12838 llvm::Type *Tys[2] = { Ty, VTy };
12843 case NEON::BI__builtin_neon_vmaxvq_f16: {
12844 Int = Intrinsic::aarch64_neon_fmaxv;
12846 VTy = llvm::FixedVectorType::get(
HalfTy, 8);
12847 llvm::Type *Tys[2] = { Ty, VTy };
12852 case NEON::BI__builtin_neon_vminv_u8: {
12853 Int = Intrinsic::aarch64_neon_uminv;
12855 VTy = llvm::FixedVectorType::get(
Int8Ty, 8);
12856 llvm::Type *Tys[2] = { Ty, VTy };
12861 case NEON::BI__builtin_neon_vminv_u16: {
12862 Int = Intrinsic::aarch64_neon_uminv;
12864 VTy = llvm::FixedVectorType::get(
Int16Ty, 4);
12865 llvm::Type *Tys[2] = { Ty, VTy };
12870 case NEON::BI__builtin_neon_vminvq_u8: {
12871 Int = Intrinsic::aarch64_neon_uminv;
12873 VTy = llvm::FixedVectorType::get(
Int8Ty, 16);
12874 llvm::Type *Tys[2] = { Ty, VTy };
12879 case NEON::BI__builtin_neon_vminvq_u16: {
12880 Int = Intrinsic::aarch64_neon_uminv;
12882 VTy = llvm::FixedVectorType::get(
Int16Ty, 8);
12883 llvm::Type *Tys[2] = { Ty, VTy };
12888 case NEON::BI__builtin_neon_vminv_s8: {
12889 Int = Intrinsic::aarch64_neon_sminv;
12891 VTy = llvm::FixedVectorType::get(
Int8Ty, 8);
12892 llvm::Type *Tys[2] = { Ty, VTy };
12897 case NEON::BI__builtin_neon_vminv_s16: {
12898 Int = Intrinsic::aarch64_neon_sminv;
12900 VTy = llvm::FixedVectorType::get(
Int16Ty, 4);
12901 llvm::Type *Tys[2] = { Ty, VTy };
12906 case NEON::BI__builtin_neon_vminvq_s8: {
12907 Int = Intrinsic::aarch64_neon_sminv;
12909 VTy = llvm::FixedVectorType::get(
Int8Ty, 16);
12910 llvm::Type *Tys[2] = { Ty, VTy };
12915 case NEON::BI__builtin_neon_vminvq_s16: {
12916 Int = Intrinsic::aarch64_neon_sminv;
12918 VTy = llvm::FixedVectorType::get(
Int16Ty, 8);
12919 llvm::Type *Tys[2] = { Ty, VTy };
12924 case NEON::BI__builtin_neon_vminv_f16: {
12925 Int = Intrinsic::aarch64_neon_fminv;
12927 VTy = llvm::FixedVectorType::get(
HalfTy, 4);
12928 llvm::Type *Tys[2] = { Ty, VTy };
12933 case NEON::BI__builtin_neon_vminvq_f16: {
12934 Int = Intrinsic::aarch64_neon_fminv;
12936 VTy = llvm::FixedVectorType::get(
HalfTy, 8);
12937 llvm::Type *Tys[2] = { Ty, VTy };
12942 case NEON::BI__builtin_neon_vmaxnmv_f16: {
12943 Int = Intrinsic::aarch64_neon_fmaxnmv;
12945 VTy = llvm::FixedVectorType::get(
HalfTy, 4);
12946 llvm::Type *Tys[2] = { Ty, VTy };
12951 case NEON::BI__builtin_neon_vmaxnmvq_f16: {
12952 Int = Intrinsic::aarch64_neon_fmaxnmv;
12954 VTy = llvm::FixedVectorType::get(
HalfTy, 8);
12955 llvm::Type *Tys[2] = { Ty, VTy };
12960 case NEON::BI__builtin_neon_vminnmv_f16: {
12961 Int = Intrinsic::aarch64_neon_fminnmv;
12963 VTy = llvm::FixedVectorType::get(
HalfTy, 4);
12964 llvm::Type *Tys[2] = { Ty, VTy };
12969 case NEON::BI__builtin_neon_vminnmvq_f16: {
12970 Int = Intrinsic::aarch64_neon_fminnmv;
12972 VTy = llvm::FixedVectorType::get(
HalfTy, 8);
12973 llvm::Type *Tys[2] = { Ty, VTy };
12978 case NEON::BI__builtin_neon_vmul_n_f64: {
12981 return Builder.CreateFMul(Ops[0], RHS);
12983 case NEON::BI__builtin_neon_vaddlv_u8: {
12984 Int = Intrinsic::aarch64_neon_uaddlv;
12986 VTy = llvm::FixedVectorType::get(
Int8Ty, 8);
12987 llvm::Type *Tys[2] = { Ty, VTy };
12992 case NEON::BI__builtin_neon_vaddlv_u16: {
12993 Int = Intrinsic::aarch64_neon_uaddlv;
12995 VTy = llvm::FixedVectorType::get(
Int16Ty, 4);
12996 llvm::Type *Tys[2] = { Ty, VTy };
13000 case NEON::BI__builtin_neon_vaddlvq_u8: {
13001 Int = Intrinsic::aarch64_neon_uaddlv;
13003 VTy = llvm::FixedVectorType::get(
Int8Ty, 16);
13004 llvm::Type *Tys[2] = { Ty, VTy };
13009 case NEON::BI__builtin_neon_vaddlvq_u16: {
13010 Int = Intrinsic::aarch64_neon_uaddlv;
13012 VTy = llvm::FixedVectorType::get(
Int16Ty, 8);
13013 llvm::Type *Tys[2] = { Ty, VTy };
13017 case NEON::BI__builtin_neon_vaddlv_s8: {
13018 Int = Intrinsic::aarch64_neon_saddlv;
13020 VTy = llvm::FixedVectorType::get(
Int8Ty, 8);
13021 llvm::Type *Tys[2] = { Ty, VTy };
13026 case NEON::BI__builtin_neon_vaddlv_s16: {
13027 Int = Intrinsic::aarch64_neon_saddlv;
13029 VTy = llvm::FixedVectorType::get(
Int16Ty, 4);
13030 llvm::Type *Tys[2] = { Ty, VTy };
13034 case NEON::BI__builtin_neon_vaddlvq_s8: {
13035 Int = Intrinsic::aarch64_neon_saddlv;
13037 VTy = llvm::FixedVectorType::get(
Int8Ty, 16);
13038 llvm::Type *Tys[2] = { Ty, VTy };
13043 case NEON::BI__builtin_neon_vaddlvq_s16: {
13044 Int = Intrinsic::aarch64_neon_saddlv;
13046 VTy = llvm::FixedVectorType::get(
Int16Ty, 8);
13047 llvm::Type *Tys[2] = { Ty, VTy };
13051 case NEON::BI__builtin_neon_vsri_n_v:
13052 case NEON::BI__builtin_neon_vsriq_n_v: {
13053 Int = Intrinsic::aarch64_neon_vsri;
13057 case NEON::BI__builtin_neon_vsli_n_v:
13058 case NEON::BI__builtin_neon_vsliq_n_v: {
13059 Int = Intrinsic::aarch64_neon_vsli;
13063 case NEON::BI__builtin_neon_vsra_n_v:
13064 case NEON::BI__builtin_neon_vsraq_n_v:
13065 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
13067 return Builder.CreateAdd(Ops[0], Ops[1]);
13068 case NEON::BI__builtin_neon_vrsra_n_v:
13069 case NEON::BI__builtin_neon_vrsraq_n_v: {
13070 Int = usgn ? Intrinsic::aarch64_neon_urshl : Intrinsic::aarch64_neon_srshl;
13072 TmpOps.push_back(Ops[1]);
13073 TmpOps.push_back(Ops[2]);
13075 llvm::Value *tmp =
EmitNeonCall(F, TmpOps,
"vrshr_n", 1,
true);
13076 Ops[0] =
Builder.CreateBitCast(Ops[0], VTy);
13077 return Builder.CreateAdd(Ops[0], tmp);
13079 case NEON::BI__builtin_neon_vld1_v:
13080 case NEON::BI__builtin_neon_vld1q_v: {
13083 case NEON::BI__builtin_neon_vst1_v:
13084 case NEON::BI__builtin_neon_vst1q_v:
13085 Ops[1] =
Builder.CreateBitCast(Ops[1], VTy);
13087 case NEON::BI__builtin_neon_vld1_lane_v:
13088 case NEON::BI__builtin_neon_vld1q_lane_v: {
13089 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
13092 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vld1_lane");
13094 case NEON::BI__builtin_neon_vldap1_lane_s64:
13095 case NEON::BI__builtin_neon_vldap1q_lane_s64: {
13096 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
13098 VTy->getElementType(), Ops[0], PtrOp0.
getAlignment());
13099 LI->setAtomic(llvm::AtomicOrdering::Acquire);
13101 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vldap1_lane");
13103 case NEON::BI__builtin_neon_vld1_dup_v:
13104 case NEON::BI__builtin_neon_vld1q_dup_v: {
13105 Value *
V = PoisonValue::get(Ty);
13108 llvm::Constant *CI = ConstantInt::get(
Int32Ty, 0);
13109 Ops[0] =
Builder.CreateInsertElement(
V, Ops[0], CI);
13112 case NEON::BI__builtin_neon_vst1_lane_v:
13113 case NEON::BI__builtin_neon_vst1q_lane_v:
13114 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
13115 Ops[1] =
Builder.CreateExtractElement(Ops[1], Ops[2]);
13117 case NEON::BI__builtin_neon_vstl1_lane_s64:
13118 case NEON::BI__builtin_neon_vstl1q_lane_s64: {
13119 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
13120 Ops[1] =
Builder.CreateExtractElement(Ops[1], Ops[2]);
13121 llvm::StoreInst *SI =
13123 SI->setAtomic(llvm::AtomicOrdering::Release);
13126 case NEON::BI__builtin_neon_vld2_v:
13127 case NEON::BI__builtin_neon_vld2q_v: {
13130 Ops[1] =
Builder.CreateCall(F, Ops[1],
"vld2");
13133 case NEON::BI__builtin_neon_vld3_v:
13134 case NEON::BI__builtin_neon_vld3q_v: {
13137 Ops[1] =
Builder.CreateCall(F, Ops[1],
"vld3");
13140 case NEON::BI__builtin_neon_vld4_v:
13141 case NEON::BI__builtin_neon_vld4q_v: {
13144 Ops[1] =
Builder.CreateCall(F, Ops[1],
"vld4");
13147 case NEON::BI__builtin_neon_vld2_dup_v:
13148 case NEON::BI__builtin_neon_vld2q_dup_v: {
13151 Ops[1] =
Builder.CreateCall(F, Ops[1],
"vld2");
13154 case NEON::BI__builtin_neon_vld3_dup_v:
13155 case NEON::BI__builtin_neon_vld3q_dup_v: {
13158 Ops[1] =
Builder.CreateCall(F, Ops[1],
"vld3");
13161 case NEON::BI__builtin_neon_vld4_dup_v:
13162 case NEON::BI__builtin_neon_vld4q_dup_v: {
13165 Ops[1] =
Builder.CreateCall(F, Ops[1],
"vld4");
13168 case NEON::BI__builtin_neon_vld2_lane_v:
13169 case NEON::BI__builtin_neon_vld2q_lane_v: {
13170 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
13172 std::rotate(Ops.begin() + 1, Ops.begin() + 2, Ops.end());
13173 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
13174 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
13179 case NEON::BI__builtin_neon_vld3_lane_v:
13180 case NEON::BI__builtin_neon_vld3q_lane_v: {
13181 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
13183 std::rotate(Ops.begin() + 1, Ops.begin() + 2, Ops.end());
13184 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
13185 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
13186 Ops[3] =
Builder.CreateBitCast(Ops[3], Ty);
13191 case NEON::BI__builtin_neon_vld4_lane_v:
13192 case NEON::BI__builtin_neon_vld4q_lane_v: {
13193 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
13195 std::rotate(Ops.begin() + 1, Ops.begin() + 2, Ops.end());
13196 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
13197 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
13198 Ops[3] =
Builder.CreateBitCast(Ops[3], Ty);
13199 Ops[4] =
Builder.CreateBitCast(Ops[4], Ty);
13204 case NEON::BI__builtin_neon_vst2_v:
13205 case NEON::BI__builtin_neon_vst2q_v: {
13206 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
13207 llvm::Type *Tys[2] = { VTy, Ops[2]->getType() };
13211 case NEON::BI__builtin_neon_vst2_lane_v:
13212 case NEON::BI__builtin_neon_vst2q_lane_v: {
13213 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
13215 llvm::Type *Tys[2] = { VTy, Ops[3]->getType() };
13219 case NEON::BI__builtin_neon_vst3_v:
13220 case NEON::BI__builtin_neon_vst3q_v: {
13221 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
13222 llvm::Type *Tys[2] = { VTy, Ops[3]->getType() };
13226 case NEON::BI__builtin_neon_vst3_lane_v:
13227 case NEON::BI__builtin_neon_vst3q_lane_v: {
13228 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
13230 llvm::Type *Tys[2] = { VTy, Ops[4]->getType() };
13234 case NEON::BI__builtin_neon_vst4_v:
13235 case NEON::BI__builtin_neon_vst4q_v: {
13236 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
13237 llvm::Type *Tys[2] = { VTy, Ops[4]->getType() };
13241 case NEON::BI__builtin_neon_vst4_lane_v:
13242 case NEON::BI__builtin_neon_vst4q_lane_v: {
13243 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
13245 llvm::Type *Tys[2] = { VTy, Ops[5]->getType() };
13249 case NEON::BI__builtin_neon_vtrn_v:
13250 case NEON::BI__builtin_neon_vtrnq_v: {
13251 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
13252 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
13253 Value *SV =
nullptr;
13255 for (
unsigned vi = 0; vi != 2; ++vi) {
13257 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
13258 Indices.push_back(i+vi);
13259 Indices.push_back(i+e+vi);
13261 Value *Addr =
Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
13262 SV =
Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vtrn");
13267 case NEON::BI__builtin_neon_vuzp_v:
13268 case NEON::BI__builtin_neon_vuzpq_v: {
13269 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
13270 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
13271 Value *SV =
nullptr;
13273 for (
unsigned vi = 0; vi != 2; ++vi) {
13275 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
13276 Indices.push_back(2*i+vi);
13278 Value *Addr =
Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
13279 SV =
Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vuzp");
13284 case NEON::BI__builtin_neon_vzip_v:
13285 case NEON::BI__builtin_neon_vzipq_v: {
13286 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
13287 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
13288 Value *SV =
nullptr;
13290 for (
unsigned vi = 0; vi != 2; ++vi) {
13292 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
13293 Indices.push_back((i + vi*e) >> 1);
13294 Indices.push_back(((i + vi*e) >> 1)+e);
13296 Value *Addr =
Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
13297 SV =
Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vzip");
13302 case NEON::BI__builtin_neon_vqtbl1q_v: {
13306 case NEON::BI__builtin_neon_vqtbl2q_v: {
13310 case NEON::BI__builtin_neon_vqtbl3q_v: {
13314 case NEON::BI__builtin_neon_vqtbl4q_v: {
13318 case NEON::BI__builtin_neon_vqtbx1q_v: {
13322 case NEON::BI__builtin_neon_vqtbx2q_v: {
13326 case NEON::BI__builtin_neon_vqtbx3q_v: {
13330 case NEON::BI__builtin_neon_vqtbx4q_v: {
13334 case NEON::BI__builtin_neon_vsqadd_v:
13335 case NEON::BI__builtin_neon_vsqaddq_v: {
13336 Int = Intrinsic::aarch64_neon_usqadd;
13339 case NEON::BI__builtin_neon_vuqadd_v:
13340 case NEON::BI__builtin_neon_vuqaddq_v: {
13341 Int = Intrinsic::aarch64_neon_suqadd;
13349 assert((BuiltinID == BPF::BI__builtin_preserve_field_info ||
13350 BuiltinID == BPF::BI__builtin_btf_type_id ||
13351 BuiltinID == BPF::BI__builtin_preserve_type_info ||
13352 BuiltinID == BPF::BI__builtin_preserve_enum_value) &&
13353 "unexpected BPF builtin");
13358 static uint32_t BuiltinSeqNum;
13360 switch (BuiltinID) {
13362 llvm_unreachable(
"Unexpected BPF builtin");
13363 case BPF::BI__builtin_preserve_field_info: {
13369 "using __builtin_preserve_field_info() without -g");
13382 Value *InfoKind = ConstantInt::get(
Int64Ty,
C->getSExtValue());
13385 llvm::Function *FnGetFieldInfo = llvm::Intrinsic::getDeclaration(
13386 &
CGM.
getModule(), llvm::Intrinsic::bpf_preserve_field_info,
13387 {FieldAddr->getType()});
13388 return Builder.CreateCall(FnGetFieldInfo, {FieldAddr, InfoKind});
13390 case BPF::BI__builtin_btf_type_id:
13391 case BPF::BI__builtin_preserve_type_info: {
13402 Value *FlagValue = ConstantInt::get(
Int64Ty, Flag->getSExtValue());
13403 Value *SeqNumVal = ConstantInt::get(
Int32Ty, BuiltinSeqNum++);
13405 llvm::Function *FnDecl;
13406 if (BuiltinID == BPF::BI__builtin_btf_type_id)
13407 FnDecl = llvm::Intrinsic::getDeclaration(
13408 &
CGM.
getModule(), llvm::Intrinsic::bpf_btf_type_id, {});
13410 FnDecl = llvm::Intrinsic::getDeclaration(
13411 &
CGM.
getModule(), llvm::Intrinsic::bpf_preserve_type_info, {});
13412 CallInst *Fn =
Builder.CreateCall(FnDecl, {SeqNumVal, FlagValue});
13413 Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
13416 case BPF::BI__builtin_preserve_enum_value: {
13427 const auto *UO = cast<UnaryOperator>(Arg0->
IgnoreParens());
13428 const auto *CE = cast<CStyleCastExpr>(UO->getSubExpr());
13429 const auto *DR = cast<DeclRefExpr>(CE->getSubExpr());
13430 const auto *Enumerator = cast<EnumConstantDecl>(DR->getDecl());
13432 auto InitVal = Enumerator->getInitVal();
13433 std::string InitValStr;
13434 if (InitVal.isNegative() || InitVal >
uint64_t(INT64_MAX))
13435 InitValStr = std::to_string(InitVal.getSExtValue());
13437 InitValStr = std::to_string(InitVal.getZExtValue());
13438 std::string EnumStr = Enumerator->getNameAsString() +
":" + InitValStr;
13439 Value *EnumStrVal =
Builder.CreateGlobalStringPtr(EnumStr);
13442 Value *FlagValue = ConstantInt::get(
Int64Ty, Flag->getSExtValue());
13443 Value *SeqNumVal = ConstantInt::get(
Int32Ty, BuiltinSeqNum++);
13445 llvm::Function *IntrinsicFn = llvm::Intrinsic::getDeclaration(
13446 &
CGM.
getModule(), llvm::Intrinsic::bpf_preserve_enum_value, {});
13448 Builder.CreateCall(IntrinsicFn, {SeqNumVal, EnumStrVal, FlagValue});
13449 Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
13457 assert((Ops.size() & (Ops.size() - 1)) == 0 &&
13458 "Not a power-of-two sized vector!");
13459 bool AllConstants =
true;
13460 for (
unsigned i = 0, e = Ops.size(); i != e && AllConstants; ++i)
13461 AllConstants &= isa<Constant>(Ops[i]);
13464 if (AllConstants) {
13466 for (
unsigned i = 0, e = Ops.size(); i != e; ++i)
13467 CstOps.push_back(cast<Constant>(Ops[i]));
13468 return llvm::ConstantVector::get(CstOps);
13473 llvm::FixedVectorType::get(Ops[0]->getType(), Ops.size()));
13475 for (
unsigned i = 0, e = Ops.size(); i != e; ++i)
13483 unsigned NumElts) {
13485 auto *MaskTy = llvm::FixedVectorType::get(
13487 cast<IntegerType>(Mask->
getType())->getBitWidth());
13488 Value *MaskVec = CGF.
Builder.CreateBitCast(Mask, MaskTy);
13494 for (
unsigned i = 0; i != NumElts; ++i)
13496 MaskVec = CGF.
Builder.CreateShuffleVector(
13497 MaskVec, MaskVec,
ArrayRef(Indices, NumElts),
"extract");
13504 Value *Ptr = Ops[0];
13508 cast<llvm::FixedVectorType>(Ops[1]->getType())->getNumElements());
13510 return CGF.
Builder.CreateMaskedStore(Ops[1], Ptr, Alignment, MaskVec);
13515 llvm::Type *Ty = Ops[1]->getType();
13516 Value *Ptr = Ops[0];
13519 CGF, Ops[2], cast<llvm::FixedVectorType>(Ty)->getNumElements());
13521 return CGF.
Builder.CreateMaskedLoad(Ty, Ptr, Alignment, MaskVec, Ops[1]);
13526 auto *ResultTy = cast<llvm::VectorType>(Ops[1]->getType());
13527 Value *Ptr = Ops[0];
13530 CGF, Ops[2], cast<FixedVectorType>(ResultTy)->getNumElements());
13532 llvm::Function *F = CGF.
CGM.
getIntrinsic(Intrinsic::masked_expandload,
13534 return CGF.
Builder.CreateCall(F, { Ptr, MaskVec, Ops[1] });
13540 auto *ResultTy = cast<llvm::FixedVectorType>(Ops[1]->getType());
13544 Intrinsic::ID IID = IsCompress ? Intrinsic::x86_avx512_mask_compress
13545 : Intrinsic::x86_avx512_mask_expand;
13547 return CGF.
Builder.CreateCall(F, { Ops[0], Ops[1], MaskVec });
13552 auto *ResultTy = cast<llvm::FixedVectorType>(Ops[1]->getType());
13553 Value *Ptr = Ops[0];
13557 llvm::Function *F = CGF.
CGM.
getIntrinsic(Intrinsic::masked_compressstore,
13559 return CGF.
Builder.CreateCall(F, { Ops[1], Ptr, MaskVec });
13564 bool InvertLHS =
false) {
13565 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
13570 LHS = CGF.
Builder.CreateNot(LHS);
13572 return CGF.
Builder.CreateBitCast(CGF.
Builder.CreateBinOp(Opc, LHS, RHS),
13573 Ops[0]->getType());
13577 Value *Amt,
bool IsRight) {
13578 llvm::Type *Ty = Op0->
getType();
13584 unsigned NumElts = cast<llvm::FixedVectorType>(Ty)->getNumElements();
13585 Amt = CGF.
Builder.CreateIntCast(Amt, Ty->getScalarType(),
false);
13586 Amt = CGF.
Builder.CreateVectorSplat(NumElts, Amt);
13589 unsigned IID = IsRight ? Intrinsic::fshr : Intrinsic::fshl;
13591 return CGF.
Builder.CreateCall(F, {Op0, Op1, Amt});
13596 Value *Op0 = Ops[0];
13597 Value *Op1 = Ops[1];
13598 llvm::Type *Ty = Op0->
getType();
13599 uint64_t Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x7;
13601 CmpInst::Predicate Pred;
13604 Pred = IsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
13607 Pred = IsSigned ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
13610 Pred = IsSigned ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
13613 Pred = IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
13616 Pred = ICmpInst::ICMP_EQ;
13619 Pred = ICmpInst::ICMP_NE;
13622 return llvm::Constant::getNullValue(Ty);
13624 return llvm::Constant::getAllOnesValue(Ty);
13626 llvm_unreachable(
"Unexpected XOP vpcom/vpcomu predicate");
13638 if (
const auto *
C = dyn_cast<Constant>(Mask))
13639 if (
C->isAllOnesValue())
13643 CGF, Mask, cast<llvm::FixedVectorType>(Op0->
getType())->getNumElements());
13645 return CGF.
Builder.CreateSelect(Mask, Op0, Op1);
13651 if (
const auto *
C = dyn_cast<Constant>(Mask))
13652 if (
C->isAllOnesValue())
13655 auto *MaskTy = llvm::FixedVectorType::get(
13656 CGF.
Builder.getInt1Ty(), Mask->
getType()->getIntegerBitWidth());
13657 Mask = CGF.
Builder.CreateBitCast(Mask, MaskTy);
13658 Mask = CGF.
Builder.CreateExtractElement(Mask, (uint64_t)0);
13659 return CGF.
Builder.CreateSelect(Mask, Op0, Op1);
13663 unsigned NumElts,
Value *MaskIn) {
13665 const auto *
C = dyn_cast<Constant>(MaskIn);
13666 if (!
C || !
C->isAllOnesValue())
13672 for (
unsigned i = 0; i != NumElts; ++i)
13674 for (
unsigned i = NumElts; i != 8; ++i)
13675 Indices[i] = i % NumElts + NumElts;
13676 Cmp = CGF.
Builder.CreateShuffleVector(
13677 Cmp, llvm::Constant::getNullValue(Cmp->getType()), Indices);
13680 return CGF.
Builder.CreateBitCast(Cmp,
13682 std::max(NumElts, 8U)));
13687 assert((Ops.size() == 2 || Ops.size() == 4) &&
13688 "Unexpected number of arguments");
13690 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
13694 Cmp = Constant::getNullValue(
13695 llvm::FixedVectorType::get(CGF.
Builder.getInt1Ty(), NumElts));
13696 }
else if (CC == 7) {
13697 Cmp = Constant::getAllOnesValue(
13698 llvm::FixedVectorType::get(CGF.
Builder.getInt1Ty(), NumElts));
13700 ICmpInst::Predicate Pred;
13702 default: llvm_unreachable(
"Unknown condition code");
13703 case 0: Pred = ICmpInst::ICMP_EQ;
break;
13704 case 1: Pred =
Signed ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
break;
13705 case 2: Pred =
Signed ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
break;
13706 case 4: Pred = ICmpInst::ICMP_NE;
break;
13707 case 5: Pred =
Signed ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
break;
13708 case 6: Pred =
Signed ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
break;
13710 Cmp = CGF.
Builder.CreateICmp(Pred, Ops[0], Ops[1]);
13713 Value *MaskIn =
nullptr;
13714 if (Ops.size() == 4)
13721 Value *Zero = Constant::getNullValue(In->getType());
13727 unsigned Rnd = cast<llvm::ConstantInt>(Ops[3])->getZExtValue();
13728 llvm::Type *Ty = Ops[1]->getType();
13732 Intrinsic::ID IID = IsSigned ? Intrinsic::x86_avx512_sitofp_round
13733 : Intrinsic::x86_avx512_uitofp_round;
13735 Res = CGF.
Builder.CreateCall(F, { Ops[0], Ops[3] });
13737 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
13738 Res = IsSigned ? CGF.
Builder.CreateSIToFP(Ops[0], Ty)
13739 : CGF.
Builder.CreateUIToFP(Ops[0], Ty);
13750 bool Subtract =
false;
13751 Intrinsic::ID IID = Intrinsic::not_intrinsic;
13752 switch (BuiltinID) {
13754 case clang::X86::BI__builtin_ia32_vfmsubph512_mask3:
13757 case clang::X86::BI__builtin_ia32_vfmaddph512_mask:
13758 case clang::X86::BI__builtin_ia32_vfmaddph512_maskz:
13759 case clang::X86::BI__builtin_ia32_vfmaddph512_mask3:
13760 IID = llvm::Intrinsic::x86_avx512fp16_vfmadd_ph_512;
13762 case clang::X86::BI__builtin_ia32_vfmsubaddph512_mask3:
13765 case clang::X86::BI__builtin_ia32_vfmaddsubph512_mask:
13766 case clang::X86::BI__builtin_ia32_vfmaddsubph512_maskz:
13767 case clang::X86::BI__builtin_ia32_vfmaddsubph512_mask3:
13768 IID = llvm::Intrinsic::x86_avx512fp16_vfmaddsub_ph_512;
13770 case clang::X86::BI__builtin_ia32_vfmsubps512_mask3:
13773 case clang::X86::BI__builtin_ia32_vfmaddps512_mask:
13774 case clang::X86::BI__builtin_ia32_vfmaddps512_maskz:
13775 case clang::X86::BI__builtin_ia32_vfmaddps512_mask3:
13776 IID = llvm::Intrinsic::x86_avx512_vfmadd_ps_512;
break;
13777 case clang::X86::BI__builtin_ia32_vfmsubpd512_mask3:
13780 case clang::X86::BI__builtin_ia32_vfmaddpd512_mask:
13781 case clang::X86::BI__builtin_ia32_vfmaddpd512_maskz:
13782 case clang::X86::BI__builtin_ia32_vfmaddpd512_mask3:
13783 IID = llvm::Intrinsic::x86_avx512_vfmadd_pd_512;
break;
13784 case clang::X86::BI__builtin_ia32_vfmsubaddps512_mask3:
13787 case clang::X86::BI__builtin_ia32_vfmaddsubps512_mask:
13788 case clang::X86::BI__builtin_ia32_vfmaddsubps512_maskz:
13789 case clang::X86::BI__builtin_ia32_vfmaddsubps512_mask3:
13790 IID = llvm::Intrinsic::x86_avx512_vfmaddsub_ps_512;
13792 case clang::X86::BI__builtin_ia32_vfmsubaddpd512_mask3:
13795 case clang::X86::BI__builtin_ia32_vfmaddsubpd512_mask:
13796 case clang::X86::BI__builtin_ia32_vfmaddsubpd512_maskz:
13797 case clang::X86::BI__builtin_ia32_vfmaddsubpd512_mask3:
13798 IID = llvm::Intrinsic::x86_avx512_vfmaddsub_pd_512;
13812 if (IID != Intrinsic::not_intrinsic &&
13813 (cast<llvm::ConstantInt>(Ops.back())->getZExtValue() != (uint64_t)4 ||
13816 Res = CGF.
Builder.CreateCall(Intr, {A, B,
C, Ops.back() });
13818 llvm::Type *Ty = A->
getType();
13820 if (CGF.
Builder.getIsFPConstrained()) {
13821 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
13822 FMA = CGF.
CGM.
getIntrinsic(Intrinsic::experimental_constrained_fma, Ty);
13823 Res = CGF.
Builder.CreateConstrainedFPCall(FMA, {A, B,
C});
13826 Res = CGF.
Builder.CreateCall(FMA, {A, B,
C});
13831 Value *MaskFalseVal =
nullptr;
13832 switch (BuiltinID) {
13833 case clang::X86::BI__builtin_ia32_vfmaddph512_mask:
13834 case clang::X86::BI__builtin_ia32_vfmaddps512_mask:
13835 case clang::X86::BI__builtin_ia32_vfmaddpd512_mask:
13836 case clang::X86::BI__builtin_ia32_vfmaddsubph512_mask:
13837 case clang::X86::BI__builtin_ia32_vfmaddsubps512_mask:
13838 case clang::X86::BI__builtin_ia32_vfmaddsubpd512_mask:
13839 MaskFalseVal = Ops[0];
13841 case clang::X86::BI__builtin_ia32_vfmaddph512_maskz:
13842 case clang::X86::BI__builtin_ia32_vfmaddps512_maskz:
13843 case clang::X86::BI__builtin_ia32_vfmaddpd512_maskz:
13844 case clang::X86::BI__builtin_ia32_vfmaddsubph512_maskz:
13845 case clang::X86::BI__builtin_ia32_vfmaddsubps512_maskz:
13846 case clang::X86::BI__builtin_ia32_vfmaddsubpd512_maskz:
13847 MaskFalseVal = Constant::getNullValue(Ops[0]->getType());
13849 case clang::X86::BI__builtin_ia32_vfmsubph512_mask3:
13850 case clang::X86::BI__builtin_ia32_vfmaddph512_mask3:
13851 case clang::X86::BI__builtin_ia32_vfmsubps512_mask3:
13852 case clang::X86::BI__builtin_ia32_vfmaddps512_mask3:
13853 case clang::X86::BI__builtin_ia32_vfmsubpd512_mask3:
13854 case clang::X86::BI__builtin_ia32_vfmaddpd512_mask3:
13855 case clang::X86::BI__builtin_ia32_vfmsubaddph512_mask3:
13856 case clang::X86::BI__builtin_ia32_vfmaddsubph512_mask3:
13857 case clang::X86::BI__builtin_ia32_vfmsubaddps512_mask3:
13858 case clang::X86::BI__builtin_ia32_vfmaddsubps512_mask3:
13859 case clang::X86::BI__builtin_ia32_vfmsubaddpd512_mask3:
13860 case clang::X86::BI__builtin_ia32_vfmaddsubpd512_mask3:
13861 MaskFalseVal = Ops[2];
13873 bool ZeroMask =
false,
unsigned PTIdx = 0,
13874 bool NegAcc =
false) {
13876 if (Ops.size() > 4)
13877 Rnd = cast<llvm::ConstantInt>(Ops[4])->getZExtValue();
13880 Ops[2] = CGF.
Builder.CreateFNeg(Ops[2]);
13882 Ops[0] = CGF.
Builder.CreateExtractElement(Ops[0], (uint64_t)0);
13883 Ops[1] = CGF.
Builder.CreateExtractElement(Ops[1], (uint64_t)0);
13884 Ops[2] = CGF.
Builder.CreateExtractElement(Ops[2], (uint64_t)0);
13889 switch (Ops[0]->getType()->getPrimitiveSizeInBits()) {
13891 IID = Intrinsic::x86_avx512fp16_vfmadd_f16;
13894 IID = Intrinsic::x86_avx512_vfmadd_f32;
13897 IID = Intrinsic::x86_avx512_vfmadd_f64;
13900 llvm_unreachable(
"Unexpected size");
13903 {Ops[0], Ops[1], Ops[2], Ops[4]});
13904 }
else if (CGF.
Builder.getIsFPConstrained()) {
13905 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
13907 Intrinsic::experimental_constrained_fma, Ops[0]->getType());
13908 Res = CGF.
Builder.CreateConstrainedFPCall(FMA, Ops.slice(0, 3));
13911 Res = CGF.
Builder.CreateCall(FMA, Ops.slice(0, 3));
13914 if (Ops.size() > 3) {
13915 Value *PassThru = ZeroMask ? Constant::getNullValue(Res->
getType())
13921 if (NegAcc && PTIdx == 2)
13922 PassThru = CGF.
Builder.CreateExtractElement(Upper, (uint64_t)0);
13926 return CGF.
Builder.CreateInsertElement(Upper, Res, (uint64_t)0);
13931 llvm::Type *Ty = Ops[0]->getType();
13933 Ty = llvm::FixedVectorType::get(CGF.
Int64Ty,
13934 Ty->getPrimitiveSizeInBits() / 64);
13940 Constant *ShiftAmt = ConstantInt::get(Ty, 32);
13941 LHS = CGF.
Builder.CreateShl(LHS, ShiftAmt);
13942 LHS = CGF.
Builder.CreateAShr(LHS, ShiftAmt);
13943 RHS = CGF.
Builder.CreateShl(RHS, ShiftAmt);
13944 RHS = CGF.
Builder.CreateAShr(RHS, ShiftAmt);
13947 Constant *Mask = ConstantInt::get(Ty, 0xffffffff);
13948 LHS = CGF.
Builder.CreateAnd(LHS, Mask);
13949 RHS = CGF.
Builder.CreateAnd(RHS, Mask);
13952 return CGF.
Builder.CreateMul(LHS, RHS);
13960 llvm::Type *Ty = Ops[0]->getType();
13962 unsigned VecWidth = Ty->getPrimitiveSizeInBits();
13963 unsigned EltWidth = Ty->getScalarSizeInBits();
13965 if (VecWidth == 128 && EltWidth == 32)
13966 IID = Intrinsic::x86_avx512_pternlog_d_128;
13967 else if (VecWidth == 256 && EltWidth == 32)
13968 IID = Intrinsic::x86_avx512_pternlog_d_256;
13969 else if (VecWidth == 512 && EltWidth == 32)
13970 IID = Intrinsic::x86_avx512_pternlog_d_512;
13971 else if (VecWidth == 128 && EltWidth == 64)
13972 IID = Intrinsic::x86_avx512_pternlog_q_128;
13973 else if (VecWidth == 256 && EltWidth == 64)
13974 IID = Intrinsic::x86_avx512_pternlog_q_256;
13975 else if (VecWidth == 512 && EltWidth == 64)
13976 IID = Intrinsic::x86_avx512_pternlog_q_512;
13978 llvm_unreachable(
"Unexpected intrinsic");
13982 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(Ty) : Ops[0];
13987 llvm::Type *DstTy) {
13988 unsigned NumberOfElements =
13989 cast<llvm::FixedVectorType>(DstTy)->getNumElements();
13991 return CGF.
Builder.CreateSExt(Mask, DstTy,
"vpmovm2");
13996 StringRef CPUStr = cast<clang::StringLiteral>(CPUExpr)->getString();
13997 return EmitX86CpuIs(CPUStr);
14003 llvm::Type *DstTy) {
14004 assert((Ops.size() == 1 || Ops.size() == 3 || Ops.size() == 4) &&
14005 "Unknown cvtph2ps intrinsic");
14008 if (Ops.size() == 4 && cast<llvm::ConstantInt>(Ops[3])->getZExtValue() != 4) {
14011 return CGF.
Builder.CreateCall(F, {Ops[0], Ops[1], Ops[2], Ops[3]});
14014 unsigned NumDstElts = cast<llvm::FixedVectorType>(DstTy)->getNumElements();
14015 Value *Src = Ops[0];
14019 cast<llvm::FixedVectorType>(Src->
getType())->getNumElements()) {
14020 assert(NumDstElts == 4 &&
"Unexpected vector size");
14025 auto *HalfTy = llvm::FixedVectorType::get(
14027 Src = CGF.
Builder.CreateBitCast(Src, HalfTy);
14030 Value *Res = CGF.
Builder.CreateFPExt(Src, DstTy,
"cvtph2ps");
14032 if (Ops.size() >= 3)
14037Value *CodeGenFunction::EmitX86CpuIs(StringRef CPUStr) {
14048 llvm::ArrayType::get(
Int32Ty, 1));
14052 cast<llvm::GlobalValue>(CpuModel)->setDSOLocal(
true);
14058 std::tie(Index,
Value) = StringSwitch<std::pair<unsigned, unsigned>>(CPUStr)
14060 .Case(STRING, {0u,
static_cast<unsigned>(llvm::X86::ENUM)})
14062 .Case(
ALIAS, {1u,
static_cast<unsigned>(llvm::X86::ENUM)})
14064 .Case(STR, {1u,
static_cast<unsigned>(llvm::X86::ENUM)})
14066 .Case(
ALIAS, {2u,
static_cast<unsigned>(llvm::X86::ENUM)})
14068 .Case(STR, {2u,
static_cast<unsigned>(llvm::X86::ENUM)})
14069#include
"llvm/TargetParser/X86TargetParser.def"
14071 assert(
Value != 0 &&
"Invalid CPUStr passed to CpuIs");
14074 llvm::Value *Idxs[] = {ConstantInt::get(
Int32Ty, 0),
14075 ConstantInt::get(
Int32Ty, Index)};
14081 return Builder.CreateICmpEQ(CpuValue,
14085Value *CodeGenFunction::EmitX86CpuSupports(
const CallExpr *E) {
14087 StringRef FeatureStr = cast<StringLiteral>(FeatureExpr)->getString();
14088 if (!
getContext().getTargetInfo().validateCpuSupports(FeatureStr))
14090 return EmitX86CpuSupports(FeatureStr);
14094 return EmitX86CpuSupports(llvm::X86::getCpuSupportsMask(FeatureStrs));
14098CodeGenFunction::EmitX86CpuSupports(std::array<uint32_t, 4> FeatureMask) {
14100 if (FeatureMask[0] != 0) {
14108 llvm::ArrayType::get(
Int32Ty, 1));
14112 cast<llvm::GlobalValue>(CpuModel)->setDSOLocal(
true);
14129 llvm::Type *ATy = llvm::ArrayType::get(
Int32Ty, 3);
14130 llvm::Constant *CpuFeatures2 =
14132 cast<llvm::GlobalValue>(CpuFeatures2)->setDSOLocal(
true);
14133 for (
int i = 1; i != 4; ++i) {
14134 const uint32_t M = FeatureMask[i];
14151Value *CodeGenFunction::EmitAArch64CpuInit() {
14152 llvm::FunctionType *FTy = llvm::FunctionType::get(
VoidTy,
false);
14153 llvm::FunctionCallee
Func =
14155 cast<llvm::GlobalValue>(
Func.getCallee())->setDSOLocal(
true);
14156 cast<llvm::GlobalValue>(
Func.getCallee())
14157 ->setDLLStorageClass(llvm::GlobalValue::DefaultStorageClass);
14161Value *CodeGenFunction::EmitX86CpuInit() {
14162 llvm::FunctionType *FTy = llvm::FunctionType::get(
VoidTy,
14164 llvm::FunctionCallee
Func =
14166 cast<llvm::GlobalValue>(
Func.getCallee())->setDSOLocal(
true);
14167 cast<llvm::GlobalValue>(
Func.getCallee())
14168 ->setDLLStorageClass(llvm::GlobalValue::DefaultStorageClass);
14172Value *CodeGenFunction::EmitAArch64CpuSupports(
const CallExpr *E) {
14174 StringRef ArgStr = cast<StringLiteral>(ArgExpr)->getString();
14176 ArgStr.split(Features,
"+");
14177 for (
auto &Feature : Features) {
14178 Feature = Feature.trim();
14179 if (!llvm::AArch64::parseArchExtension(Feature))
14181 if (Feature !=
"default")
14182 Features.push_back(Feature);
14184 return EmitAArch64CpuSupports(Features);
14189 uint64_t FeaturesMask = llvm::AArch64::getCpuSupportsMask(FeaturesStrs);
14191 if (FeaturesMask != 0) {
14196 llvm::Type *STy = llvm::StructType::get(
Int64Ty);
14197 llvm::Constant *AArch64CPUFeatures =
14199 cast<llvm::GlobalValue>(AArch64CPUFeatures)->setDSOLocal(
true);
14201 STy, AArch64CPUFeatures,
14215 if (BuiltinID == Builtin::BI__builtin_cpu_is)
14216 return EmitX86CpuIs(E);
14217 if (BuiltinID == Builtin::BI__builtin_cpu_supports)
14218 return EmitX86CpuSupports(E);
14219 if (BuiltinID == Builtin::BI__builtin_cpu_init)
14220 return EmitX86CpuInit();
14228 bool IsMaskFCmp =
false;
14229 bool IsConjFMA =
false;
14232 unsigned ICEArguments = 0;
14237 for (
unsigned i = 0, e = E->
getNumArgs(); i != e; i++) {
14247 auto getCmpIntrinsicCall = [
this, &Ops](Intrinsic::ID
ID,
unsigned Imm) {
14248 Ops.push_back(llvm::ConstantInt::get(
Int8Ty, Imm));
14250 return Builder.CreateCall(F, Ops);
14258 auto getVectorFCmpIR = [
this, &Ops, E](CmpInst::Predicate Pred,
14259 bool IsSignaling) {
14260 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
14263 Cmp =
Builder.CreateFCmpS(Pred, Ops[0], Ops[1]);
14265 Cmp =
Builder.CreateFCmp(Pred, Ops[0], Ops[1]);
14266 llvm::VectorType *FPVecTy = cast<llvm::VectorType>(Ops[0]->getType());
14267 llvm::VectorType *IntVecTy = llvm::VectorType::getInteger(FPVecTy);
14269 return Builder.CreateBitCast(Sext, FPVecTy);
14272 switch (BuiltinID) {
14273 default:
return nullptr;
14274 case X86::BI_mm_prefetch: {
14276 ConstantInt *
C = cast<ConstantInt>(Ops[1]);
14277 Value *RW = ConstantInt::get(
Int32Ty, (
C->getZExtValue() >> 2) & 0x1);
14278 Value *Locality = ConstantInt::get(
Int32Ty,
C->getZExtValue() & 0x3);
14283 case X86::BI_mm_clflush: {
14287 case X86::BI_mm_lfence: {
14290 case X86::BI_mm_mfence: {
14293 case X86::BI_mm_sfence: {
14296 case X86::BI_mm_pause: {
14299 case X86::BI__rdtsc: {
14302 case X86::BI__builtin_ia32_rdtscp: {
14308 case X86::BI__builtin_ia32_lzcnt_u16:
14309 case X86::BI__builtin_ia32_lzcnt_u32:
14310 case X86::BI__builtin_ia32_lzcnt_u64: {
14314 case X86::BI__builtin_ia32_tzcnt_u16:
14315 case X86::BI__builtin_ia32_tzcnt_u32:
14316 case X86::BI__builtin_ia32_tzcnt_u64: {
14320 case X86::BI__builtin_ia32_undef128:
14321 case X86::BI__builtin_ia32_undef256:
14322 case X86::BI__builtin_ia32_undef512:
14329 case X86::BI__builtin_ia32_vec_init_v8qi:
14330 case X86::BI__builtin_ia32_vec_init_v4hi:
14331 case X86::BI__builtin_ia32_vec_init_v2si:
14334 case X86::BI__builtin_ia32_vec_ext_v2si:
14335 case X86::BI__builtin_ia32_vec_ext_v16qi:
14336 case X86::BI__builtin_ia32_vec_ext_v8hi:
14337 case X86::BI__builtin_ia32_vec_ext_v4si:
14338 case X86::BI__builtin_ia32_vec_ext_v4sf:
14339 case X86::BI__builtin_ia32_vec_ext_v2di:
14340 case X86::BI__builtin_ia32_vec_ext_v32qi:
14341 case X86::BI__builtin_ia32_vec_ext_v16hi:
14342 case X86::BI__builtin_ia32_vec_ext_v8si:
14343 case X86::BI__builtin_ia32_vec_ext_v4di: {
14345 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
14346 uint64_t Index = cast<ConstantInt>(Ops[1])->getZExtValue();
14347 Index &= NumElts - 1;
14350 return Builder.CreateExtractElement(Ops[0], Index);
14352 case X86::BI__builtin_ia32_vec_set_v16qi:
14353 case X86::BI__builtin_ia32_vec_set_v8hi:
14354 case X86::BI__builtin_ia32_vec_set_v4si:
14355 case X86::BI__builtin_ia32_vec_set_v2di:
14356 case X86::BI__builtin_ia32_vec_set_v32qi:
14357 case X86::BI__builtin_ia32_vec_set_v16hi:
14358 case X86::BI__builtin_ia32_vec_set_v8si:
14359 case X86::BI__builtin_ia32_vec_set_v4di: {
14361 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
14362 unsigned Index = cast<ConstantInt>(Ops[2])->getZExtValue();
14363 Index &= NumElts - 1;
14366 return Builder.CreateInsertElement(Ops[0], Ops[1], Index);
14368 case X86::BI_mm_setcsr:
14369 case X86::BI__builtin_ia32_ldmxcsr: {
14375 case X86::BI_mm_getcsr:
14376 case X86::BI__builtin_ia32_stmxcsr: {
14382 case X86::BI__builtin_ia32_xsave:
14383 case X86::BI__builtin_ia32_xsave64:
14384 case X86::BI__builtin_ia32_xrstor:
14385 case X86::BI__builtin_ia32_xrstor64:
14386 case X86::BI__builtin_ia32_xsaveopt:
14387 case X86::BI__builtin_ia32_xsaveopt64:
14388 case X86::BI__builtin_ia32_xrstors:
14389 case X86::BI__builtin_ia32_xrstors64:
14390 case X86::BI__builtin_ia32_xsavec:
14391 case X86::BI__builtin_ia32_xsavec64:
14392 case X86::BI__builtin_ia32_xsaves:
14393 case X86::BI__builtin_ia32_xsaves64:
14394 case X86::BI__builtin_ia32_xsetbv:
14395 case X86::BI_xsetbv: {
14397#define INTRINSIC_X86_XSAVE_ID(NAME) \
14398 case X86::BI__builtin_ia32_##NAME: \
14399 ID = Intrinsic::x86_##NAME; \
14401 switch (BuiltinID) {
14402 default: llvm_unreachable(
"Unsupported intrinsic!");
14416 case X86::BI_xsetbv:
14417 ID = Intrinsic::x86_xsetbv;
14420#undef INTRINSIC_X86_XSAVE_ID
14425 Ops.push_back(Mlo);
14428 case X86::BI__builtin_ia32_xgetbv:
14429 case X86::BI_xgetbv:
14431 case X86::BI__builtin_ia32_storedqudi128_mask:
14432 case X86::BI__builtin_ia32_storedqusi128_mask:
14433 case X86::BI__builtin_ia32_storedquhi128_mask:
14434 case X86::BI__builtin_ia32_storedquqi128_mask:
14435 case X86::BI__builtin_ia32_storeupd128_mask:
14436 case X86::BI__builtin_ia32_storeups128_mask:
14437 case X86::BI__builtin_ia32_storedqudi256_mask:
14438 case X86::BI__builtin_ia32_storedqusi256_mask:
14439 case X86::BI__builtin_ia32_storedquhi256_mask:
14440 case X86::BI__builtin_ia32_storedquqi256_mask:
14441 case X86::BI__builtin_ia32_storeupd256_mask:
14442 case X86::BI__builtin_ia32_storeups256_mask:
14443 case X86::BI__builtin_ia32_storedqudi512_mask:
14444 case X86::BI__builtin_ia32_storedqusi512_mask:
14445 case X86::BI__builtin_ia32_storedquhi512_mask:
14446 case X86::BI__builtin_ia32_storedquqi512_mask:
14447 case X86::BI__builtin_ia32_storeupd512_mask:
14448 case X86::BI__builtin_ia32_storeups512_mask:
14451 case X86::BI__builtin_ia32_storesh128_mask:
14452 case X86::BI__builtin_ia32_storess128_mask:
14453 case X86::BI__builtin_ia32_storesd128_mask:
14456 case X86::BI__builtin_ia32_vpopcntb_128:
14457 case X86::BI__builtin_ia32_vpopcntd_128:
14458 case X86::BI__builtin_ia32_vpopcntq_128:
14459 case X86::BI__builtin_ia32_vpopcntw_128:
14460 case X86::BI__builtin_ia32_vpopcntb_256:
14461 case X86::BI__builtin_ia32_vpopcntd_256:
14462 case X86::BI__builtin_ia32_vpopcntq_256:
14463 case X86::BI__builtin_ia32_vpopcntw_256:
14464 case X86::BI__builtin_ia32_vpopcntb_512:
14465 case X86::BI__builtin_ia32_vpopcntd_512:
14466 case X86::BI__builtin_ia32_vpopcntq_512:
14467 case X86::BI__builtin_ia32_vpopcntw_512: {
14470 return Builder.CreateCall(F, Ops);
14472 case X86::BI__builtin_ia32_cvtmask2b128:
14473 case X86::BI__builtin_ia32_cvtmask2b256:
14474 case X86::BI__builtin_ia32_cvtmask2b512:
14475 case X86::BI__builtin_ia32_cvtmask2w128:
14476 case X86::BI__builtin_ia32_cvtmask2w256:
14477 case X86::BI__builtin_ia32_cvtmask2w512:
14478 case X86::BI__builtin_ia32_cvtmask2d128:
14479 case X86::BI__builtin_ia32_cvtmask2d256:
14480 case X86::BI__builtin_ia32_cvtmask2d512:
14481 case X86::BI__builtin_ia32_cvtmask2q128:
14482 case X86::BI__builtin_ia32_cvtmask2q256:
14483 case X86::BI__builtin_ia32_cvtmask2q512:
14486 case X86::BI__builtin_ia32_cvtb2mask128:
14487 case X86::BI__builtin_ia32_cvtb2mask256:
14488 case X86::BI__builtin_ia32_cvtb2mask512:
14489 case X86::BI__builtin_ia32_cvtw2mask128:
14490 case X86::BI__builtin_ia32_cvtw2mask256:
14491 case X86::BI__builtin_ia32_cvtw2mask512:
14492 case X86::BI__builtin_ia32_cvtd2mask128:
14493 case X86::BI__builtin_ia32_cvtd2mask256:
14494 case X86::BI__builtin_ia32_cvtd2mask512:
14495 case X86::BI__builtin_ia32_cvtq2mask128:
14496 case X86::BI__builtin_ia32_cvtq2mask256:
14497 case X86::BI__builtin_ia32_cvtq2mask512:
14500 case X86::BI__builtin_ia32_cvtdq2ps512_mask:
14501 case X86::BI__builtin_ia32_cvtqq2ps512_mask:
14502 case X86::BI__builtin_ia32_cvtqq2pd512_mask:
14503 case X86::BI__builtin_ia32_vcvtw2ph512_mask:
14504 case X86::BI__builtin_ia32_vcvtdq2ph512_mask:
14505 case X86::BI__builtin_ia32_vcvtqq2ph512_mask:
14507 case X86::BI__builtin_ia32_cvtudq2ps512_mask:
14508 case X86::BI__builtin_ia32_cvtuqq2ps512_mask:
14509 case X86::BI__builtin_ia32_cvtuqq2pd512_mask:
14510 case X86::BI__builtin_ia32_vcvtuw2ph512_mask:
14511 case X86::BI__builtin_ia32_vcvtudq2ph512_mask:
14512 case X86::BI__builtin_ia32_vcvtuqq2ph512_mask:
14515 case X86::BI__builtin_ia32_vfmaddss3:
14516 case X86::BI__builtin_ia32_vfmaddsd3:
14517 case X86::BI__builtin_ia32_vfmaddsh3_mask:
14518 case X86::BI__builtin_ia32_vfmaddss3_mask:
14519 case X86::BI__builtin_ia32_vfmaddsd3_mask:
14521 case X86::BI__builtin_ia32_vfmaddss:
14522 case X86::BI__builtin_ia32_vfmaddsd:
14524 Constant::getNullValue(Ops[0]->getType()));
14525 case X86::BI__builtin_ia32_vfmaddsh3_maskz:
14526 case X86::BI__builtin_ia32_vfmaddss3_maskz:
14527 case X86::BI__builtin_ia32_vfmaddsd3_maskz:
14529 case X86::BI__builtin_ia32_vfmaddsh3_mask3:
14530 case X86::BI__builtin_ia32_vfmaddss3_mask3:
14531 case X86::BI__builtin_ia32_vfmaddsd3_mask3:
14533 case X86::BI__builtin_ia32_vfmsubsh3_mask3:
14534 case X86::BI__builtin_ia32_vfmsubss3_mask3:
14535 case X86::BI__builtin_ia32_vfmsubsd3_mask3:
14538 case X86::BI__builtin_ia32_vfmaddph:
14539 case X86::BI__builtin_ia32_vfmaddps:
14540 case X86::BI__builtin_ia32_vfmaddpd:
14541 case X86::BI__builtin_ia32_vfmaddph256:
14542 case X86::BI__builtin_ia32_vfmaddps256:
14543 case X86::BI__builtin_ia32_vfmaddpd256:
14544 case X86::BI__builtin_ia32_vfmaddph512_mask:
14545 case X86::BI__builtin_ia32_vfmaddph512_maskz:
14546 case X86::BI__builtin_ia32_vfmaddph512_mask3:
14547 case X86::BI__builtin_ia32_vfmaddps512_mask:
14548 case X86::BI__builtin_ia32_vfmaddps512_maskz:
14549 case X86::BI__builtin_ia32_vfmaddps512_mask3:
14550 case X86::BI__builtin_ia32_vfmsubps512_mask3:
14551 case X86::BI__builtin_ia32_vfmaddpd512_mask:
14552 case X86::BI__builtin_ia32_vfmaddpd512_maskz:
14553 case X86::BI__builtin_ia32_vfmaddpd512_mask3:
14554 case X86::BI__builtin_ia32_vfmsubpd512_mask3:
14555 case X86::BI__builtin_ia32_vfmsubph512_mask3:
14557 case X86::BI__builtin_ia32_vfmaddsubph512_mask:
14558 case X86::BI__builtin_ia32_vfmaddsubph512_maskz:
14559 case X86::BI__builtin_ia32_vfmaddsubph512_mask3:
14560 case X86::BI__builtin_ia32_vfmsubaddph512_mask3:
14561 case X86::BI__builtin_ia32_vfmaddsubps512_mask:
14562 case X86::BI__builtin_ia32_vfmaddsubps512_maskz:
14563 case X86::BI__builtin_ia32_vfmaddsubps512_mask3:
14564 case X86::BI__builtin_ia32_vfmsubaddps512_mask3:
14565 case X86::BI__builtin_ia32_vfmaddsubpd512_mask:
14566 case X86::BI__builtin_ia32_vfmaddsubpd512_maskz:
14567 case X86::BI__builtin_ia32_vfmaddsubpd512_mask3:
14568 case X86::BI__builtin_ia32_vfmsubaddpd512_mask3:
14571 case X86::BI__builtin_ia32_movdqa32store128_mask:
14572 case X86::BI__builtin_ia32_movdqa64store128_mask:
14573 case X86::BI__builtin_ia32_storeaps128_mask:
14574 case X86::BI__builtin_ia32_storeapd128_mask:
14575 case X86::BI__builtin_ia32_movdqa32store256_mask:
14576 case X86::BI__builtin_ia32_movdqa64store256_mask:
14577 case X86::BI__builtin_ia32_storeaps256_mask:
14578 case X86::BI__builtin_ia32_storeapd256_mask:
14579 case X86::BI__builtin_ia32_movdqa32store512_mask:
14580 case X86::BI__builtin_ia32_movdqa64store512_mask:
14581 case X86::BI__builtin_ia32_storeaps512_mask:
14582 case X86::BI__builtin_ia32_storeapd512_mask:
14587 case X86::BI__builtin_ia32_loadups128_mask:
14588 case X86::BI__builtin_ia32_loadups256_mask:
14589 case X86::BI__builtin_ia32_loadups512_mask:
14590 case X86::BI__builtin_ia32_loadupd128_mask:
14591 case X86::BI__builtin_ia32_loadupd256_mask:
14592 case X86::BI__builtin_ia32_loadupd512_mask:
14593 case X86::BI__builtin_ia32_loaddquqi128_mask:
14594 case X86::BI__builtin_ia32_loaddquqi256_mask:
14595 case X86::BI__builtin_ia32_loaddquqi512_mask:
14596 case X86::BI__builtin_ia32_loaddquhi128_mask:
14597 case X86::BI__builtin_ia32_loaddquhi256_mask:
14598 case X86::BI__builtin_ia32_loaddquhi512_mask:
14599 case X86::BI__builtin_ia32_loaddqusi128_mask:
14600 case X86::BI__builtin_ia32_loaddqusi256_mask:
14601 case X86::BI__builtin_ia32_loaddqusi512_mask:
14602 case X86::BI__builtin_ia32_loaddqudi128_mask:
14603 case X86::BI__builtin_ia32_loaddqudi256_mask:
14604 case X86::BI__builtin_ia32_loaddqudi512_mask:
14607 case X86::BI__builtin_ia32_loadsh128_mask:
14608 case X86::BI__builtin_ia32_loadss128_mask:
14609 case X86::BI__builtin_ia32_loadsd128_mask:
14612 case X86::BI__builtin_ia32_loadaps128_mask:
14613 case X86::BI__builtin_ia32_loadaps256_mask:
14614 case X86::BI__builtin_ia32_loadaps512_mask:
14615 case X86::BI__builtin_ia32_loadapd128_mask:
14616 case X86::BI__builtin_ia32_loadapd256_mask:
14617 case X86::BI__builtin_ia32_loadapd512_mask:
14618 case X86::BI__builtin_ia32_movdqa32load128_mask:
14619 case X86::BI__builtin_ia32_movdqa32load256_mask:
14620 case X86::BI__builtin_ia32_movdqa32load512_mask:
14621 case X86::BI__builtin_ia32_movdqa64load128_mask:
14622 case X86::BI__builtin_ia32_movdqa64load256_mask:
14623 case X86::BI__builtin_ia32_movdqa64load512_mask:
14628 case X86::BI__builtin_ia32_expandloaddf128_mask:
14629 case X86::BI__builtin_ia32_expandloaddf256_mask:
14630 case X86::BI__builtin_ia32_expandloaddf512_mask:
14631 case X86::BI__builtin_ia32_expandloadsf128_mask:
14632 case X86::BI__builtin_ia32_expandloadsf256_mask:
14633 case X86::BI__builtin_ia32_expandloadsf512_mask:
14634 case X86::BI__builtin_ia32_expandloaddi128_mask:
14635 case X86::BI__builtin_ia32_expandloaddi256_mask:
14636 case X86::BI__builtin_ia32_expandloaddi512_mask:
14637 case X86::BI__builtin_ia32_expandloadsi128_mask:
14638 case X86::BI__builtin_ia32_expandloadsi256_mask:
14639 case X86::BI__builtin_ia32_expandloadsi512_mask:
14640 case X86::BI__builtin_ia32_expandloadhi128_mask:
14641 case X86::BI__builtin_ia32_expandloadhi256_mask:
14642 case X86::BI__builtin_ia32_expandloadhi512_mask:
14643 case X86::BI__builtin_ia32_expandloadqi128_mask:
14644 case X86::BI__builtin_ia32_expandloadqi256_mask:
14645 case X86::BI__builtin_ia32_expandloadqi512_mask:
14648 case X86::BI__builtin_ia32_compressstoredf128_mask:
14649 case X86::BI__builtin_ia32_compressstoredf256_mask:
14650 case X86::BI__builtin_ia32_compressstoredf512_mask:
14651 case X86::BI__builtin_ia32_compressstoresf128_mask:
14652 case X86::BI__builtin_ia32_compressstoresf256_mask:
14653 case X86::BI__builtin_ia32_compressstoresf512_mask:
14654 case X86::BI__builtin_ia32_compressstoredi128_mask:
14655 case X86::BI__builtin_ia32_compressstoredi256_mask:
14656 case X86::BI__builtin_ia32_compressstoredi512_mask:
14657 case X86::BI__builtin_ia32_compressstoresi128_mask:
14658 case X86::BI__builtin_ia32_compressstoresi256_mask:
14659 case X86::BI__builtin_ia32_compressstoresi512_mask:
14660 case X86::BI__builtin_ia32_compressstorehi128_mask:
14661 case X86::BI__builtin_ia32_compressstorehi256_mask:
14662 case X86::BI__builtin_ia32_compressstorehi512_mask:
14663 case X86::BI__builtin_ia32_compressstoreqi128_mask:
14664 case X86::BI__builtin_ia32_compressstoreqi256_mask:
14665 case X86::BI__builtin_ia32_compressstoreqi512_mask:
14668 case X86::BI__builtin_ia32_expanddf128_mask:
14669 case X86::BI__builtin_ia32_expanddf256_mask:
14670 case X86::BI__builtin_ia32_expanddf512_mask:
14671 case X86::BI__builtin_ia32_expandsf128_mask:
14672 case X86::BI__builtin_ia32_expandsf256_mask:
14673 case X86::BI__builtin_ia32_expandsf512_mask:
14674 case X86::BI__builtin_ia32_expanddi128_mask:
14675 case X86::BI__builtin_ia32_expanddi256_mask:
14676 case X86::BI__builtin_ia32_expanddi512_mask:
14677 case X86::BI__builtin_ia32_expandsi128_mask:
14678 case X86::BI__builtin_ia32_expandsi256_mask:
14679 case X86::BI__builtin_ia32_expandsi512_mask:
14680 case X86::BI__builtin_ia32_expandhi128_mask:
14681 case X86::BI__builtin_ia32_expandhi256_mask:
14682 case X86::BI__builtin_ia32_expandhi512_mask:
14683 case X86::BI__builtin_ia32_expandqi128_mask:
14684 case X86::BI__builtin_ia32_expandqi256_mask:
14685 case X86::BI__builtin_ia32_expandqi512_mask:
14688 case X86::BI__builtin_ia32_compressdf128_mask:
14689 case X86::BI__builtin_ia32_compressdf256_mask:
14690 case X86::BI__builtin_ia32_compressdf512_mask:
14691 case X86::BI__builtin_ia32_compresssf128_mask:
14692 case X86::BI__builtin_ia32_compresssf256_mask:
14693 case X86::BI__builtin_ia32_compresssf512_mask:
14694 case X86::BI__builtin_ia32_compressdi128_mask:
14695 case X86::BI__builtin_ia32_compressdi256_mask:
14696 case X86::BI__builtin_ia32_compressdi512_mask:
14697 case X86::BI__builtin_ia32_compresssi128_mask:
14698 case X86::BI__builtin_ia32_compresssi256_mask:
14699 case X86::BI__builtin_ia32_compresssi512_mask:
14700 case X86::BI__builtin_ia32_compresshi128_mask:
14701 case X86::BI__builtin_ia32_compresshi256_mask:
14702 case X86::BI__builtin_ia32_compresshi512_mask:
14703 case X86::BI__builtin_ia32_compressqi128_mask:
14704 case X86::BI__builtin_ia32_compressqi256_mask:
14705 case X86::BI__builtin_ia32_compressqi512_mask:
14708 case X86::BI__builtin_ia32_gather3div2df:
14709 case X86::BI__builtin_ia32_gather3div2di:
14710 case X86::BI__builtin_ia32_gather3div4df:
14711 case X86::BI__builtin_ia32_gather3div4di:
14712 case X86::BI__builtin_ia32_gather3div4sf:
14713 case X86::BI__builtin_ia32_gather3div4si:
14714 case X86::BI__builtin_ia32_gather3div8sf:
14715 case X86::BI__builtin_ia32_gather3div8si:
14716 case X86::BI__builtin_ia32_gather3siv2df:
14717 case X86::BI__builtin_ia32_gather3siv2di:
14718 case X86::BI__builtin_ia32_gather3siv4df:
14719 case X86::BI__builtin_ia32_gather3siv4di:
14720 case X86::BI__builtin_ia32_gather3siv4sf:
14721 case X86::BI__builtin_ia32_gather3siv4si:
14722 case X86::BI__builtin_ia32_gather3siv8sf:
14723 case X86::BI__builtin_ia32_gather3siv8si:
14724 case X86::BI__builtin_ia32_gathersiv8df:
14725 case X86::BI__builtin_ia32_gathersiv16sf:
14726 case X86::BI__builtin_ia32_gatherdiv8df:
14727 case X86::BI__builtin_ia32_gatherdiv16sf:
14728 case X86::BI__builtin_ia32_gathersiv8di:
14729 case X86::BI__builtin_ia32_gathersiv16si:
14730 case X86::BI__builtin_ia32_gatherdiv8di:
14731 case X86::BI__builtin_ia32_gatherdiv16si: {
14733 switch (BuiltinID) {
14734 default: llvm_unreachable(
"Unexpected builtin");
14735 case X86::BI__builtin_ia32_gather3div2df:
14736 IID = Intrinsic::x86_avx512_mask_gather3div2_df;
14738 case X86::BI__builtin_ia32_gather3div2di:
14739 IID = Intrinsic::x86_avx512_mask_gather3div2_di;
14741 case X86::BI__builtin_ia32_gather3div4df:
14742 IID = Intrinsic::x86_avx512_mask_gather3div4_df;
14744 case X86::BI__builtin_ia32_gather3div4di:
14745 IID = Intrinsic::x86_avx512_mask_gather3div4_di;
14747 case X86::BI__builtin_ia32_gather3div4sf:
14748 IID = Intrinsic::x86_avx512_mask_gather3div4_sf;
14750 case X86::BI__builtin_ia32_gather3div4si:
14751 IID = Intrinsic::x86_avx512_mask_gather3div4_si;
14753 case X86::BI__builtin_ia32_gather3div8sf:
14754 IID = Intrinsic::x86_avx512_mask_gather3div8_sf;
14756 case X86::BI__builtin_ia32_gather3div8si:
14757 IID = Intrinsic::x86_avx512_mask_gather3div8_si;
14759 case X86::BI__builtin_ia32_gather3siv2df:
14760 IID = Intrinsic::x86_avx512_mask_gather3siv2_df;
14762 case X86::BI__builtin_ia32_gather3siv2di:
14763 IID = Intrinsic::x86_avx512_mask_gather3siv2_di;
14765 case X86::BI__builtin_ia32_gather3siv4df:
14766 IID = Intrinsic::x86_avx512_mask_gather3siv4_df;
14768 case X86::BI__builtin_ia32_gather3siv4di:
14769 IID = Intrinsic::x86_avx512_mask_gather3siv4_di;
14771 case X86::BI__builtin_ia32_gather3siv4sf:
14772 IID = Intrinsic::x86_avx512_mask_gather3siv4_sf;
14774 case X86::BI__builtin_ia32_gather3siv4si:
14775 IID = Intrinsic::x86_avx512_mask_gather3siv4_si;
14777 case X86::BI__builtin_ia32_gather3siv8sf:
14778 IID = Intrinsic::x86_avx512_mask_gather3siv8_sf;
14780 case X86::BI__builtin_ia32_gather3siv8si:
14781 IID = Intrinsic::x86_avx512_mask_gather3siv8_si;
14783 case X86::BI__builtin_ia32_gathersiv8df:
14784 IID = Intrinsic::x86_avx512_mask_gather_dpd_512;
14786 case X86::BI__builtin_ia32_gathersiv16sf:
14787 IID = Intrinsic::x86_avx512_mask_gather_dps_512;
14789 case X86::BI__builtin_ia32_gatherdiv8df:
14790 IID = Intrinsic::x86_avx512_mask_gather_qpd_512;
14792 case X86::BI__builtin_ia32_gatherdiv16sf:
14793 IID = Intrinsic::x86_avx512_mask_gather_qps_512;
14795 case X86::BI__builtin_ia32_gathersiv8di:
14796 IID = Intrinsic::x86_avx512_mask_gather_dpq_512;
14798 case X86::BI__builtin_ia32_gathersiv16si:
14799 IID = Intrinsic::x86_avx512_mask_gather_dpi_512;
14801 case X86::BI__builtin_ia32_gatherdiv8di:
14802 IID = Intrinsic::x86_avx512_mask_gather_qpq_512;
14804 case X86::BI__builtin_ia32_gatherdiv16si:
14805 IID = Intrinsic::x86_avx512_mask_gather_qpi_512;
14809 unsigned MinElts = std::min(
14810 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements(),
14811 cast<llvm::FixedVectorType>(Ops[2]->getType())->getNumElements());
14814 return Builder.CreateCall(Intr, Ops);
14817 case X86::BI__builtin_ia32_scattersiv8df:
14818 case X86::BI__builtin_ia32_scattersiv16sf:
14819 case X86::BI__builtin_ia32_scatterdiv8df:
14820 case X86::BI__builtin_ia32_scatterdiv16sf:
14821 case X86::BI__builtin_ia32_scattersiv8di:
14822 case X86::BI__builtin_ia32_scattersiv16si:
14823 case X86::BI__builtin_ia32_scatterdiv8di:
14824 case X86::BI__builtin_ia32_scatterdiv16si:
14825 case X86::BI__builtin_ia32_scatterdiv2df:
14826 case X86::BI__builtin_ia32_scatterdiv2di:
14827 case X86::BI__builtin_ia32_scatterdiv4df:
14828 case X86::BI__builtin_ia32_scatterdiv4di:
14829 case X86::BI__builtin_ia32_scatterdiv4sf:
14830 case X86::BI__builtin_ia32_scatterdiv4si:
14831 case X86::BI__builtin_ia32_scatterdiv8sf:
14832 case X86::BI__builtin_ia32_scatterdiv8si:
14833 case X86::BI__builtin_ia32_scattersiv2df:
14834 case X86::BI__builtin_ia32_scattersiv2di:
14835 case X86::BI__builtin_ia32_scattersiv4df:
14836 case X86::BI__builtin_ia32_scattersiv4di:
14837 case X86::BI__builtin_ia32_scattersiv4sf:
14838 case X86::BI__builtin_ia32_scattersiv4si:
14839 case X86::BI__builtin_ia32_scattersiv8sf:
14840 case X86::BI__builtin_ia32_scattersiv8si: {
14842 switch (BuiltinID) {
14843 default: llvm_unreachable(
"Unexpected builtin");
14844 case X86::BI__builtin_ia32_scattersiv8df:
14845 IID = Intrinsic::x86_avx512_mask_scatter_dpd_512;
14847 case X86::BI__builtin_ia32_scattersiv16sf:
14848 IID = Intrinsic::x86_avx512_mask_scatter_dps_512;
14850 case X86::BI__builtin_ia32_scatterdiv8df:
14851 IID = Intrinsic::x86_avx512_mask_scatter_qpd_512;
14853 case X86::BI__builtin_ia32_scatterdiv16sf:
14854 IID = Intrinsic::x86_avx512_mask_scatter_qps_512;
14856 case X86::BI__builtin_ia32_scattersiv8di:
14857 IID = Intrinsic::x86_avx512_mask_scatter_dpq_512;
14859 case X86::BI__builtin_ia32_scattersiv16si:
14860 IID = Intrinsic::x86_avx512_mask_scatter_dpi_512;
14862 case X86::BI__builtin_ia32_scatterdiv8di:
14863 IID = Intrinsic::x86_avx512_mask_scatter_qpq_512;
14865 case X86::BI__builtin_ia32_scatterdiv16si:
14866 IID = Intrinsic::x86_avx512_mask_scatter_qpi_512;
14868 case X86::BI__builtin_ia32_scatterdiv2df:
14869 IID = Intrinsic::x86_avx512_mask_scatterdiv2_df;
14871 case X86::BI__builtin_ia32_scatterdiv2di:
14872 IID = Intrinsic::x86_avx512_mask_scatterdiv2_di;
14874 case X86::BI__builtin_ia32_scatterdiv4df:
14875 IID = Intrinsic::x86_avx512_mask_scatterdiv4_df;
14877 case X86::BI__builtin_ia32_scatterdiv4di:
14878 IID = Intrinsic::x86_avx512_mask_scatterdiv4_di;
14880 case X86::BI__builtin_ia32_scatterdiv4sf:
14881 IID = Intrinsic::x86_avx512_mask_scatterdiv4_sf;
14883 case X86::BI__builtin_ia32_scatterdiv4si:
14884 IID = Intrinsic::x86_avx512_mask_scatterdiv4_si;
14886 case X86::BI__builtin_ia32_scatterdiv8sf:
14887 IID = Intrinsic::x86_avx512_mask_scatterdiv8_sf;
14889 case X86::BI__builtin_ia32_scatterdiv8si:
14890 IID = Intrinsic::x86_avx512_mask_scatterdiv8_si;
14892 case X86::BI__builtin_ia32_scattersiv2df:
14893 IID = Intrinsic::x86_avx512_mask_scattersiv2_df;
14895 case X86::BI__builtin_ia32_scattersiv2di:
14896 IID = Intrinsic::x86_avx512_mask_scattersiv2_di;
14898 case X86::BI__builtin_ia32_scattersiv4df:
14899 IID = Intrinsic::x86_avx512_mask_scattersiv4_df;
14901 case X86::BI__builtin_ia32_scattersiv4di:
14902 IID = Intrinsic::x86_avx512_mask_scattersiv4_di;
14904 case X86::BI__builtin_ia32_scattersiv4sf:
14905 IID = Intrinsic::x86_avx512_mask_scattersiv4_sf;
14907 case X86::BI__builtin_ia32_scattersiv4si:
14908 IID = Intrinsic::x86_avx512_mask_scattersiv4_si;
14910 case X86::BI__builtin_ia32_scattersiv8sf:
14911 IID = Intrinsic::x86_avx512_mask_scattersiv8_sf;
14913 case X86::BI__builtin_ia32_scattersiv8si:
14914 IID = Intrinsic::x86_avx512_mask_scattersiv8_si;
14918 unsigned MinElts = std::min(
14919 cast<llvm::FixedVectorType>(Ops[2]->getType())->getNumElements(),
14920 cast<llvm::FixedVectorType>(Ops[3]->getType())->getNumElements());
14923 return Builder.CreateCall(Intr, Ops);
14926 case X86::BI__builtin_ia32_vextractf128_pd256:
14927 case X86::BI__builtin_ia32_vextractf128_ps256:
14928 case X86::BI__builtin_ia32_vextractf128_si256:
14929 case X86::BI__builtin_ia32_extract128i256:
14930 case X86::BI__builtin_ia32_extractf64x4_mask:
14931 case X86::BI__builtin_ia32_extractf32x4_mask:
14932 case X86::BI__builtin_ia32_extracti64x4_mask:
14933 case X86::BI__builtin_ia32_extracti32x4_mask:
14934 case X86::BI__builtin_ia32_extractf32x8_mask:
14935 case X86::BI__builtin_ia32_extracti32x8_mask:
14936 case X86::BI__builtin_ia32_extractf32x4_256_mask:
14937 case X86::BI__builtin_ia32_extracti32x4_256_mask:
14938 case X86::BI__builtin_ia32_extractf64x2_256_mask:
14939 case X86::BI__builtin_ia32_extracti64x2_256_mask:
14940 case X86::BI__builtin_ia32_extractf64x2_512_mask:
14941 case X86::BI__builtin_ia32_extracti64x2_512_mask: {
14943 unsigned NumElts = DstTy->getNumElements();
14944 unsigned SrcNumElts =
14945 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
14946 unsigned SubVectors = SrcNumElts / NumElts;
14947 unsigned Index = cast<ConstantInt>(Ops[1])->getZExtValue();
14948 assert(llvm::isPowerOf2_32(SubVectors) &&
"Expected power of 2 subvectors");
14949 Index &= SubVectors - 1;
14953 for (
unsigned i = 0; i != NumElts; ++i)
14954 Indices[i] = i + Index;
14959 if (Ops.size() == 4)
14964 case X86::BI__builtin_ia32_vinsertf128_pd256:
14965 case X86::BI__builtin_ia32_vinsertf128_ps256:
14966 case X86::BI__builtin_ia32_vinsertf128_si256:
14967 case X86::BI__builtin_ia32_insert128i256:
14968 case X86::BI__builtin_ia32_insertf64x4:
14969 case X86::BI__builtin_ia32_insertf32x4:
14970 case X86::BI__builtin_ia32_inserti64x4:
14971 case X86::BI__builtin_ia32_inserti32x4:
14972 case X86::BI__builtin_ia32_insertf32x8:
14973 case X86::BI__builtin_ia32_inserti32x8:
14974 case X86::BI__builtin_ia32_insertf32x4_256:
14975 case X86::BI__builtin_ia32_inserti32x4_256:
14976 case X86::BI__builtin_ia32_insertf64x2_256:
14977 case X86::BI__builtin_ia32_inserti64x2_256:
14978 case X86::BI__builtin_ia32_insertf64x2_512:
14979 case X86::BI__builtin_ia32_inserti64x2_512: {
14980 unsigned DstNumElts =
14981 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
14982 unsigned SrcNumElts =
14983 cast<llvm::FixedVectorType>(Ops[1]->getType())->getNumElements();
14984 unsigned SubVectors = DstNumElts / SrcNumElts;
14985 unsigned Index = cast<ConstantInt>(Ops[2])->getZExtValue();
14986 assert(llvm::isPowerOf2_32(SubVectors) &&
"Expected power of 2 subvectors");
14987 Index &= SubVectors - 1;
14988 Index *= SrcNumElts;
14991 for (
unsigned i = 0; i != DstNumElts; ++i)
14992 Indices[i] = (i >= SrcNumElts) ? SrcNumElts + (i % SrcNumElts) : i;
14995 Ops[1],
ArrayRef(Indices, DstNumElts),
"widen");
14997 for (
unsigned i = 0; i != DstNumElts; ++i) {
14998 if (i >= Index && i < (Index + SrcNumElts))
14999 Indices[i] = (i - Index) + DstNumElts;
15004 return Builder.CreateShuffleVector(Ops[0], Op1,
15005 ArrayRef(Indices, DstNumElts),
"insert");
15007 case X86::BI__builtin_ia32_pmovqd512_mask:
15008 case X86::BI__builtin_ia32_pmovwb512_mask: {
15009 Value *Res =
Builder.CreateTrunc(Ops[0], Ops[1]->getType());
15012 case X86::BI__builtin_ia32_pmovdb512_mask:
15013 case X86::BI__builtin_ia32_pmovdw512_mask:
15014 case X86::BI__builtin_ia32_pmovqw512_mask: {
15015 if (
const auto *
C = dyn_cast<Constant>(Ops[2]))
15016 if (
C->isAllOnesValue())
15017 return Builder.CreateTrunc(Ops[0], Ops[1]->getType());
15020 switch (BuiltinID) {
15021 default: llvm_unreachable(
"Unsupported intrinsic!");
15022 case X86::BI__builtin_ia32_pmovdb512_mask:
15023 IID = Intrinsic::x86_avx512_mask_pmov_db_512;
15025 case X86::BI__builtin_ia32_pmovdw512_mask:
15026 IID = Intrinsic::x86_avx512_mask_pmov_dw_512;
15028 case X86::BI__builtin_ia32_pmovqw512_mask:
15029 IID = Intrinsic::x86_avx512_mask_pmov_qw_512;
15034 return Builder.CreateCall(Intr, Ops);
15036 case X86::BI__builtin_ia32_pblendw128:
15037 case X86::BI__builtin_ia32_blendpd:
15038 case X86::BI__builtin_ia32_blendps:
15039 case X86::BI__builtin_ia32_blendpd256:
15040 case X86::BI__builtin_ia32_blendps256:
15041 case X86::BI__builtin_ia32_pblendw256:
15042 case X86::BI__builtin_ia32_pblendd128:
15043 case X86::BI__builtin_ia32_pblendd256: {
15045 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
15046 unsigned Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
15051 for (
unsigned i = 0; i != NumElts; ++i)
15052 Indices[i] = ((Imm >> (i % 8)) & 0x1) ? NumElts + i : i;
15054 return Builder.CreateShuffleVector(Ops[0], Ops[1],
15055 ArrayRef(Indices, NumElts),
"blend");
15057 case X86::BI__builtin_ia32_pshuflw:
15058 case X86::BI__builtin_ia32_pshuflw256:
15059 case X86::BI__builtin_ia32_pshuflw512: {
15060 uint32_t Imm = cast<llvm::ConstantInt>(Ops[1])->getZExtValue();
15061 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType());
15062 unsigned NumElts = Ty->getNumElements();
15065 Imm = (Imm & 0xff) * 0x01010101;
15068 for (
unsigned l = 0; l != NumElts; l += 8) {
15069 for (
unsigned i = 0; i != 4; ++i) {
15070 Indices[l + i] = l + (Imm & 3);
15073 for (
unsigned i = 4; i != 8; ++i)
15074 Indices[l + i] = l + i;
15077 return Builder.CreateShuffleVector(Ops[0],
ArrayRef(Indices, NumElts),
15080 case X86::BI__builtin_ia32_pshufhw:
15081 case X86::BI__builtin_ia32_pshufhw256:
15082 case X86::BI__builtin_ia32_pshufhw512: {
15083 uint32_t Imm = cast<llvm::ConstantInt>(Ops[1])->getZExtValue();
15084 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType());
15085 unsigned NumElts = Ty->getNumElements();
15088 Imm = (Imm & 0xff) * 0x01010101;
15091 for (
unsigned l = 0; l != NumElts; l += 8) {
15092 for (
unsigned i = 0; i != 4; ++i)
15093 Indices[l + i] = l + i;
15094 for (
unsigned i = 4; i != 8; ++i) {
15095 Indices[l + i] = l + 4 + (Imm & 3);
15100 return Builder.CreateShuffleVector(Ops[0],
ArrayRef(Indices, NumElts),
15103 case X86::BI__builtin_ia32_pshufd:
15104 case X86::BI__builtin_ia32_pshufd256:
15105 case X86::BI__builtin_ia32_pshufd512:
15106 case X86::BI__builtin_ia32_vpermilpd:
15107 case X86::BI__builtin_ia32_vpermilps:
15108 case X86::BI__builtin_ia32_vpermilpd256:
15109 case X86::BI__builtin_ia32_vpermilps256:
15110 case X86::BI__builtin_ia32_vpermilpd512:
15111 case X86::BI__builtin_ia32_vpermilps512: {
15112 uint32_t Imm = cast<llvm::ConstantInt>(Ops[1])->getZExtValue();
15113 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType());
15114 unsigned NumElts = Ty->getNumElements();
15115 unsigned NumLanes = Ty->getPrimitiveSizeInBits() / 128;
15116 unsigned NumLaneElts = NumElts / NumLanes;
15119 Imm = (Imm & 0xff) * 0x01010101;
15122 for (
unsigned l = 0; l != NumElts; l += NumLaneElts) {
15123 for (
unsigned i = 0; i != NumLaneElts; ++i) {
15124 Indices[i + l] = (Imm % NumLaneElts) + l;
15125 Imm /= NumLaneElts;
15129 return Builder.CreateShuffleVector(Ops[0],
ArrayRef(Indices, NumElts),
15132 case X86::BI__builtin_ia32_shufpd:
15133 case X86::BI__builtin_ia32_shufpd256:
15134 case X86::BI__builtin_ia32_shufpd512:
15135 case X86::BI__builtin_ia32_shufps:
15136 case X86::BI__builtin_ia32_shufps256:
15137 case X86::BI__builtin_ia32_shufps512: {
15138 uint32_t Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
15139 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType());
15140 unsigned NumElts = Ty->getNumElements();
15141 unsigned NumLanes = Ty->getPrimitiveSizeInBits() / 128;
15142 unsigned NumLaneElts = NumElts / NumLanes;
15145 Imm = (Imm & 0xff) * 0x01010101;
15148 for (
unsigned l = 0; l != NumElts; l += NumLaneElts) {
15149 for (
unsigned i = 0; i != NumLaneElts; ++i) {
15150 unsigned Index = Imm % NumLaneElts;
15151 Imm /= NumLaneElts;
15152 if (i >= (NumLaneElts / 2))
15154 Indices[l + i] = l + Index;
15158 return Builder.CreateShuffleVector(Ops[0], Ops[1],
15159 ArrayRef(Indices, NumElts),
"shufp");
15161 case X86::BI__builtin_ia32_permdi256:
15162 case X86::BI__builtin_ia32_permdf256:
15163 case X86::BI__builtin_ia32_permdi512:
15164 case X86::BI__builtin_ia32_permdf512: {
15165 unsigned Imm = cast<llvm::ConstantInt>(Ops[1])->getZExtValue();
15166 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType());
15167 unsigned NumElts = Ty->getNumElements();
15171 for (
unsigned l = 0; l != NumElts; l += 4)
15172 for (
unsigned i = 0; i != 4; ++i)
15173 Indices[l + i] = l + ((Imm >> (2 * i)) & 0x3);
15175 return Builder.CreateShuffleVector(Ops[0],
ArrayRef(Indices, NumElts),
15178 case X86::BI__builtin_ia32_palignr128:
15179 case X86::BI__builtin_ia32_palignr256:
15180 case X86::BI__builtin_ia32_palignr512: {
15181 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0xff;
15184 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
15185 assert(NumElts % 16 == 0);
15189 if (ShiftVal >= 32)
15194 if (ShiftVal > 16) {
15197 Ops[0] = llvm::Constant::getNullValue(Ops[0]->getType());
15202 for (
unsigned l = 0; l != NumElts; l += 16) {
15203 for (
unsigned i = 0; i != 16; ++i) {
15204 unsigned Idx = ShiftVal + i;
15206 Idx += NumElts - 16;
15207 Indices[l + i] = Idx + l;
15211 return Builder.CreateShuffleVector(Ops[1], Ops[0],
15212 ArrayRef(Indices, NumElts),
"palignr");
15214 case X86::BI__builtin_ia32_alignd128:
15215 case X86::BI__builtin_ia32_alignd256:
15216 case X86::BI__builtin_ia32_alignd512:
15217 case X86::BI__builtin_ia32_alignq128:
15218 case X86::BI__builtin_ia32_alignq256:
15219 case X86::BI__builtin_ia32_alignq512: {
15221 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
15222 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0xff;
15225 ShiftVal &= NumElts - 1;
15228 for (
unsigned i = 0; i != NumElts; ++i)
15229 Indices[i] = i + ShiftVal;
15231 return Builder.CreateShuffleVector(Ops[1], Ops[0],
15232 ArrayRef(Indices, NumElts),
"valign");
15234 case X86::BI__builtin_ia32_shuf_f32x4_256:
15235 case X86::BI__builtin_ia32_shuf_f64x2_256:
15236 case X86::BI__builtin_ia32_shuf_i32x4_256:
15237 case X86::BI__builtin_ia32_shuf_i64x2_256:
15238 case X86::BI__builtin_ia32_shuf_f32x4:
15239 case X86::BI__builtin_ia32_shuf_f64x2:
15240 case X86::BI__builtin_ia32_shuf_i32x4:
15241 case X86::BI__builtin_ia32_shuf_i64x2: {
15242 unsigned Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
15243 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType());
15244 unsigned NumElts = Ty->getNumElements();
15245 unsigned NumLanes = Ty->getPrimitiveSizeInBits() == 512 ? 4 : 2;
15246 unsigned NumLaneElts = NumElts / NumLanes;
15249 for (
unsigned l = 0; l != NumElts; l += NumLaneElts) {
15250 unsigned Index = (Imm % NumLanes) * NumLaneElts;
15252 if (l >= (NumElts / 2))
15254 for (
unsigned i = 0; i != NumLaneElts; ++i) {
15255 Indices[l + i] = Index + i;
15259 return Builder.CreateShuffleVector(Ops[0], Ops[1],
15260 ArrayRef(Indices, NumElts),
"shuf");
15263 case X86::BI__builtin_ia32_vperm2f128_pd256:
15264 case X86::BI__builtin_ia32_vperm2f128_ps256:
15265 case X86::BI__builtin_ia32_vperm2f128_si256:
15266 case X86::BI__builtin_ia32_permti256: {
15267 unsigned Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
15269 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
15278 for (
unsigned l = 0; l != 2; ++l) {
15280 if (Imm & (1 << ((l * 4) + 3)))
15281 OutOps[l] = llvm::ConstantAggregateZero::get(Ops[0]->getType());
15282 else if (Imm & (1 << ((l * 4) + 1)))
15283 OutOps[l] = Ops[1];
15285 OutOps[l] = Ops[0];
15287 for (
unsigned i = 0; i != NumElts/2; ++i) {
15289 unsigned Idx = (l * NumElts) + i;
15292 if (Imm & (1 << (l * 4)))
15294 Indices[(l * (NumElts/2)) + i] = Idx;
15298 return Builder.CreateShuffleVector(OutOps[0], OutOps[1],
15299 ArrayRef(Indices, NumElts),
"vperm");
15302 case X86::BI__builtin_ia32_pslldqi128_byteshift:
15303 case X86::BI__builtin_ia32_pslldqi256_byteshift:
15304 case X86::BI__builtin_ia32_pslldqi512_byteshift: {
15305 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() & 0xff;
15306 auto *ResultType = cast<llvm::FixedVectorType>(Ops[0]->getType());
15308 unsigned NumElts = ResultType->getNumElements() * 8;
15311 if (ShiftVal >= 16)
15312 return llvm::Constant::getNullValue(ResultType);
15316 for (
unsigned l = 0; l != NumElts; l += 16) {
15317 for (
unsigned i = 0; i != 16; ++i) {
15318 unsigned Idx = NumElts + i - ShiftVal;
15319 if (Idx < NumElts) Idx -= NumElts - 16;
15320 Indices[l + i] = Idx + l;
15324 auto *VecTy = llvm::FixedVectorType::get(
Int8Ty, NumElts);
15326 Value *
Zero = llvm::Constant::getNullValue(VecTy);
15328 Zero, Cast,
ArrayRef(Indices, NumElts),
"pslldq");
15329 return Builder.CreateBitCast(SV, Ops[0]->getType(),
"cast");
15331 case X86::BI__builtin_ia32_psrldqi128_byteshift:
15332 case X86::BI__builtin_ia32_psrldqi256_byteshift:
15333 case X86::BI__builtin_ia32_psrldqi512_byteshift: {
15334 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() & 0xff;
15335 auto *ResultType = cast<llvm::FixedVectorType>(Ops[0]->getType());
15337 unsigned NumElts = ResultType->getNumElements() * 8;
15340 if (ShiftVal >= 16)
15341 return llvm::Constant::getNullValue(ResultType);
15345 for (
unsigned l = 0; l != NumElts; l += 16) {
15346 for (
unsigned i = 0; i != 16; ++i) {
15347 unsigned Idx = i + ShiftVal;
15348 if (Idx >= 16) Idx += NumElts - 16;
15349 Indices[l + i] = Idx + l;
15353 auto *VecTy = llvm::FixedVectorType::get(
Int8Ty, NumElts);
15355 Value *
Zero = llvm::Constant::getNullValue(VecTy);
15357 Cast, Zero,
ArrayRef(Indices, NumElts),
"psrldq");
15358 return Builder.CreateBitCast(SV, ResultType,
"cast");
15360 case X86::BI__builtin_ia32_kshiftliqi:
15361 case X86::BI__builtin_ia32_kshiftlihi:
15362 case X86::BI__builtin_ia32_kshiftlisi:
15363 case X86::BI__builtin_ia32_kshiftlidi: {
15364 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() & 0xff;
15365 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
15367 if (ShiftVal >= NumElts)
15368 return llvm::Constant::getNullValue(Ops[0]->getType());
15373 for (
unsigned i = 0; i != NumElts; ++i)
15374 Indices[i] = NumElts + i - ShiftVal;
15376 Value *
Zero = llvm::Constant::getNullValue(
In->getType());
15378 Zero, In,
ArrayRef(Indices, NumElts),
"kshiftl");
15379 return Builder.CreateBitCast(SV, Ops[0]->getType());
15381 case X86::BI__builtin_ia32_kshiftriqi:
15382 case X86::BI__builtin_ia32_kshiftrihi:
15383 case X86::BI__builtin_ia32_kshiftrisi:
15384 case X86::BI__builtin_ia32_kshiftridi: {
15385 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() & 0xff;
15386 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
15388 if (ShiftVal >= NumElts)
15389 return llvm::Constant::getNullValue(Ops[0]->getType());
15394 for (
unsigned i = 0; i != NumElts; ++i)
15395 Indices[i] = i + ShiftVal;
15397 Value *
Zero = llvm::Constant::getNullValue(
In->getType());
15399 In, Zero,
ArrayRef(Indices, NumElts),
"kshiftr");
15400 return Builder.CreateBitCast(SV, Ops[0]->getType());
15402 case X86::BI__builtin_ia32_movnti:
15403 case X86::BI__builtin_ia32_movnti64:
15404 case X86::BI__builtin_ia32_movntsd:
15405 case X86::BI__builtin_ia32_movntss: {
15406 llvm::MDNode *
Node = llvm::MDNode::get(
15409 Value *Ptr = Ops[0];
15410 Value *Src = Ops[1];
15413 if (BuiltinID == X86::BI__builtin_ia32_movntsd ||
15414 BuiltinID == X86::BI__builtin_ia32_movntss)
15415 Src =
Builder.CreateExtractElement(Src, (uint64_t)0,
"extract");
15419 SI->setMetadata(llvm::LLVMContext::MD_nontemporal,
Node);
15420 SI->setAlignment(llvm::Align(1));
15424 case X86::BI__builtin_ia32_vprotb:
15425 case X86::BI__builtin_ia32_vprotw:
15426 case X86::BI__builtin_ia32_vprotd:
15427 case X86::BI__builtin_ia32_vprotq:
15428 case X86::BI__builtin_ia32_vprotbi:
15429 case X86::BI__builtin_ia32_vprotwi:
15430 case X86::BI__builtin_ia32_vprotdi:
15431 case X86::BI__builtin_ia32_vprotqi:
15432 case X86::BI__builtin_ia32_prold128:
15433 case X86::BI__builtin_ia32_prold256:
15434 case X86::BI__builtin_ia32_prold512:
15435 case X86::BI__builtin_ia32_prolq128:
15436 case X86::BI__builtin_ia32_prolq256:
15437 case X86::BI__builtin_ia32_prolq512:
15438 case X86::BI__builtin_ia32_prolvd128:
15439 case X86::BI__builtin_ia32_prolvd256:
15440 case X86::BI__builtin_ia32_prolvd512:
15441 case X86::BI__builtin_ia32_prolvq128:
15442 case X86::BI__builtin_ia32_prolvq256:
15443 case X86::BI__builtin_ia32_prolvq512:
15445 case X86::BI__builtin_ia32_prord128:
15446 case X86::BI__builtin_ia32_prord256:
15447 case X86::BI__builtin_ia32_prord512:
15448 case X86::BI__builtin_ia32_prorq128:
15449 case X86::BI__builtin_ia32_prorq256:
15450 case X86::BI__builtin_ia32_prorq512:
15451 case X86::BI__builtin_ia32_prorvd128:
15452 case X86::BI__builtin_ia32_prorvd256:
15453 case X86::BI__builtin_ia32_prorvd512:
15454 case X86::BI__builtin_ia32_prorvq128:
15455 case X86::BI__builtin_ia32_prorvq256:
15456 case X86::BI__builtin_ia32_prorvq512:
15458 case X86::BI__builtin_ia32_selectb_128:
15459 case X86::BI__builtin_ia32_selectb_256:
15460 case X86::BI__builtin_ia32_selectb_512:
15461 case X86::BI__builtin_ia32_selectw_128:
15462 case X86::BI__builtin_ia32_selectw_256:
15463 case X86::BI__builtin_ia32_selectw_512:
15464 case X86::BI__builtin_ia32_selectd_128:
15465 case X86::BI__builtin_ia32_selectd_256:
15466 case X86::BI__builtin_ia32_selectd_512:
15467 case X86::BI__builtin_ia32_selectq_128:
15468 case X86::BI__builtin_ia32_selectq_256:
15469 case X86::BI__builtin_ia32_selectq_512:
15470 case X86::BI__builtin_ia32_selectph_128:
15471 case X86::BI__builtin_ia32_selectph_256:
15472 case X86::BI__builtin_ia32_selectph_512:
15473 case X86::BI__builtin_ia32_selectpbf_128:
15474 case X86::BI__builtin_ia32_selectpbf_256:
15475 case X86::BI__builtin_ia32_selectpbf_512:
15476 case X86::BI__builtin_ia32_selectps_128:
15477 case X86::BI__builtin_ia32_selectps_256:
15478 case X86::BI__builtin_ia32_selectps_512:
15479 case X86::BI__builtin_ia32_selectpd_128:
15480 case X86::BI__builtin_ia32_selectpd_256:
15481 case X86::BI__builtin_ia32_selectpd_512:
15483 case X86::BI__builtin_ia32_selectsh_128:
15484 case X86::BI__builtin_ia32_selectsbf_128:
15485 case X86::BI__builtin_ia32_selectss_128:
15486 case X86::BI__builtin_ia32_selectsd_128: {
15487 Value *A =
Builder.CreateExtractElement(Ops[1], (uint64_t)0);
15488 Value *B =
Builder.CreateExtractElement(Ops[2], (uint64_t)0);
15490 return Builder.CreateInsertElement(Ops[1], A, (uint64_t)0);
15492 case X86::BI__builtin_ia32_cmpb128_mask:
15493 case X86::BI__builtin_ia32_cmpb256_mask:
15494 case X86::BI__builtin_ia32_cmpb512_mask:
15495 case X86::BI__builtin_ia32_cmpw128_mask:
15496 case X86::BI__builtin_ia32_cmpw256_mask:
15497 case X86::BI__builtin_ia32_cmpw512_mask:
15498 case X86::BI__builtin_ia32_cmpd128_mask:
15499 case X86::BI__builtin_ia32_cmpd256_mask:
15500 case X86::BI__builtin_ia32_cmpd512_mask:
15501 case X86::BI__builtin_ia32_cmpq128_mask:
15502 case X86::BI__builtin_ia32_cmpq256_mask:
15503 case X86::BI__builtin_ia32_cmpq512_mask: {
15504 unsigned CC = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x7;
15507 case X86::BI__builtin_ia32_ucmpb128_mask:
15508 case X86::BI__builtin_ia32_ucmpb256_mask:
15509 case X86::BI__builtin_ia32_ucmpb512_mask:
15510 case X86::BI__builtin_ia32_ucmpw128_mask:
15511 case X86::BI__builtin_ia32_ucmpw256_mask:
15512 case X86::BI__builtin_ia32_ucmpw512_mask:
15513 case X86::BI__builtin_ia32_ucmpd128_mask:
15514 case X86::BI__builtin_ia32_ucmpd256_mask:
15515 case X86::BI__builtin_ia32_ucmpd512_mask:
15516 case X86::BI__builtin_ia32_ucmpq128_mask:
15517 case X86::BI__builtin_ia32_ucmpq256_mask:
15518 case X86::BI__builtin_ia32_ucmpq512_mask: {
15519 unsigned CC = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x7;
15522 case X86::BI__builtin_ia32_vpcomb:
15523 case X86::BI__builtin_ia32_vpcomw:
15524 case X86::BI__builtin_ia32_vpcomd:
15525 case X86::BI__builtin_ia32_vpcomq:
15527 case X86::BI__builtin_ia32_vpcomub:
15528 case X86::BI__builtin_ia32_vpcomuw:
15529 case X86::BI__builtin_ia32_vpcomud:
15530 case X86::BI__builtin_ia32_vpcomuq:
15533 case X86::BI__builtin_ia32_kortestcqi:
15534 case X86::BI__builtin_ia32_kortestchi:
15535 case X86::BI__builtin_ia32_kortestcsi:
15536 case X86::BI__builtin_ia32_kortestcdi: {
15538 Value *
C = llvm::Constant::getAllOnesValue(Ops[0]->getType());
15542 case X86::BI__builtin_ia32_kortestzqi:
15543 case X86::BI__builtin_ia32_kortestzhi:
15544 case X86::BI__builtin_ia32_kortestzsi:
15545 case X86::BI__builtin_ia32_kortestzdi: {
15547 Value *
C = llvm::Constant::getNullValue(Ops[0]->getType());
15552 case X86::BI__builtin_ia32_ktestcqi:
15553 case X86::BI__builtin_ia32_ktestzqi:
15554 case X86::BI__builtin_ia32_ktestchi:
15555 case X86::BI__builtin_ia32_ktestzhi:
15556 case X86::BI__builtin_ia32_ktestcsi:
15557 case X86::BI__builtin_ia32_ktestzsi:
15558 case X86::BI__builtin_ia32_ktestcdi:
15559 case X86::BI__builtin_ia32_ktestzdi: {
15561 switch (BuiltinID) {
15562 default: llvm_unreachable(
"Unsupported intrinsic!");
15563 case X86::BI__builtin_ia32_ktestcqi:
15564 IID = Intrinsic::x86_avx512_ktestc_b;
15566 case X86::BI__builtin_ia32_ktestzqi:
15567 IID = Intrinsic::x86_avx512_ktestz_b;
15569 case X86::BI__builtin_ia32_ktestchi:
15570 IID = Intrinsic::x86_avx512_ktestc_w;
15572 case X86::BI__builtin_ia32_ktestzhi:
15573 IID = Intrinsic::x86_avx512_ktestz_w;
15575 case X86::BI__builtin_ia32_ktestcsi:
15576 IID = Intrinsic::x86_avx512_ktestc_d;
15578 case X86::BI__builtin_ia32_ktestzsi:
15579 IID = Intrinsic::x86_avx512_ktestz_d;
15581 case X86::BI__builtin_ia32_ktestcdi:
15582 IID = Intrinsic::x86_avx512_ktestc_q;
15584 case X86::BI__builtin_ia32_ktestzdi:
15585 IID = Intrinsic::x86_avx512_ktestz_q;
15589 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
15593 return Builder.CreateCall(Intr, {LHS, RHS});
15596 case X86::BI__builtin_ia32_kaddqi:
15597 case X86::BI__builtin_ia32_kaddhi:
15598 case X86::BI__builtin_ia32_kaddsi:
15599 case X86::BI__builtin_ia32_kadddi: {
15601 switch (BuiltinID) {
15602 default: llvm_unreachable(
"Unsupported intrinsic!");
15603 case X86::BI__builtin_ia32_kaddqi:
15604 IID = Intrinsic::x86_avx512_kadd_b;
15606 case X86::BI__builtin_ia32_kaddhi:
15607 IID = Intrinsic::x86_avx512_kadd_w;
15609 case X86::BI__builtin_ia32_kaddsi:
15610 IID = Intrinsic::x86_avx512_kadd_d;
15612 case X86::BI__builtin_ia32_kadddi:
15613 IID = Intrinsic::x86_avx512_kadd_q;
15617 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
15622 return Builder.CreateBitCast(Res, Ops[0]->getType());
15624 case X86::BI__builtin_ia32_kandqi:
15625 case X86::BI__builtin_ia32_kandhi:
15626 case X86::BI__builtin_ia32_kandsi:
15627 case X86::BI__builtin_ia32_kanddi:
15629 case X86::BI__builtin_ia32_kandnqi:
15630 case X86::BI__builtin_ia32_kandnhi:
15631 case X86::BI__builtin_ia32_kandnsi:
15632 case X86::BI__builtin_ia32_kandndi:
15634 case X86::BI__builtin_ia32_korqi:
15635 case X86::BI__builtin_ia32_korhi:
15636 case X86::BI__builtin_ia32_korsi:
15637 case X86::BI__builtin_ia32_kordi:
15639 case X86::BI__builtin_ia32_kxnorqi:
15640 case X86::BI__builtin_ia32_kxnorhi:
15641 case X86::BI__builtin_ia32_kxnorsi:
15642 case X86::BI__builtin_ia32_kxnordi:
15644 case X86::BI__builtin_ia32_kxorqi:
15645 case X86::BI__builtin_ia32_kxorhi:
15646 case X86::BI__builtin_ia32_kxorsi:
15647 case X86::BI__builtin_ia32_kxordi:
15649 case X86::BI__builtin_ia32_knotqi:
15650 case X86::BI__builtin_ia32_knothi:
15651 case X86::BI__builtin_ia32_knotsi:
15652 case X86::BI__builtin_ia32_knotdi: {
15653 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
15656 Ops[0]->getType());
15658 case X86::BI__builtin_ia32_kmovb:
15659 case X86::BI__builtin_ia32_kmovw:
15660 case X86::BI__builtin_ia32_kmovd:
15661 case X86::BI__builtin_ia32_kmovq: {
15665 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
15667 return Builder.CreateBitCast(Res, Ops[0]->getType());
15670 case X86::BI__builtin_ia32_kunpckdi:
15671 case X86::BI__builtin_ia32_kunpcksi:
15672 case X86::BI__builtin_ia32_kunpckhi: {
15673 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
15677 for (
unsigned i = 0; i != NumElts; ++i)
15682 LHS =
Builder.CreateShuffleVector(LHS, LHS,
ArrayRef(Indices, NumElts / 2));
15683 RHS =
Builder.CreateShuffleVector(RHS, RHS,
ArrayRef(Indices, NumElts / 2));
15688 return Builder.CreateBitCast(Res, Ops[0]->getType());
15691 case X86::BI__builtin_ia32_vplzcntd_128:
15692 case X86::BI__builtin_ia32_vplzcntd_256:
15693 case X86::BI__builtin_ia32_vplzcntd_512:
15694 case X86::BI__builtin_ia32_vplzcntq_128:
15695 case X86::BI__builtin_ia32_vplzcntq_256:
15696 case X86::BI__builtin_ia32_vplzcntq_512: {
15700 case X86::BI__builtin_ia32_sqrtss:
15701 case X86::BI__builtin_ia32_sqrtsd: {
15702 Value *A =
Builder.CreateExtractElement(Ops[0], (uint64_t)0);
15704 if (
Builder.getIsFPConstrained()) {
15705 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
15708 A =
Builder.CreateConstrainedFPCall(F, {A});
15711 A =
Builder.CreateCall(F, {A});
15713 return Builder.CreateInsertElement(Ops[0], A, (uint64_t)0);
15715 case X86::BI__builtin_ia32_sqrtsh_round_mask:
15716 case X86::BI__builtin_ia32_sqrtsd_round_mask:
15717 case X86::BI__builtin_ia32_sqrtss_round_mask: {
15718 unsigned CC = cast<llvm::ConstantInt>(Ops[4])->getZExtValue();
15724 switch (BuiltinID) {
15726 llvm_unreachable(
"Unsupported intrinsic!");
15727 case X86::BI__builtin_ia32_sqrtsh_round_mask:
15728 IID = Intrinsic::x86_avx512fp16_mask_sqrt_sh;
15730 case X86::BI__builtin_ia32_sqrtsd_round_mask:
15731 IID = Intrinsic::x86_avx512_mask_sqrt_sd;
15733 case X86::BI__builtin_ia32_sqrtss_round_mask:
15734 IID = Intrinsic::x86_avx512_mask_sqrt_ss;
15739 Value *A =
Builder.CreateExtractElement(Ops[1], (uint64_t)0);
15741 if (
Builder.getIsFPConstrained()) {
15742 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
15745 A =
Builder.CreateConstrainedFPCall(F, A);
15748 A =
Builder.CreateCall(F, A);
15750 Value *Src =
Builder.CreateExtractElement(Ops[2], (uint64_t)0);
15752 return Builder.CreateInsertElement(Ops[0], A, (uint64_t)0);
15754 case X86::BI__builtin_ia32_sqrtpd256:
15755 case X86::BI__builtin_ia32_sqrtpd:
15756 case X86::BI__builtin_ia32_sqrtps256:
15757 case X86::BI__builtin_ia32_sqrtps:
15758 case X86::BI__builtin_ia32_sqrtph256:
15759 case X86::BI__builtin_ia32_sqrtph:
15760 case X86::BI__builtin_ia32_sqrtph512:
15761 case X86::BI__builtin_ia32_sqrtps512:
15762 case X86::BI__builtin_ia32_sqrtpd512: {
15763 if (Ops.size() == 2) {
15764 unsigned CC = cast<llvm::ConstantInt>(Ops[1])->getZExtValue();
15770 switch (BuiltinID) {
15772 llvm_unreachable(
"Unsupported intrinsic!");
15773 case X86::BI__builtin_ia32_sqrtph512:
15774 IID = Intrinsic::x86_avx512fp16_sqrt_ph_512;
15776 case X86::BI__builtin_ia32_sqrtps512:
15777 IID = Intrinsic::x86_avx512_sqrt_ps_512;
15779 case X86::BI__builtin_ia32_sqrtpd512:
15780 IID = Intrinsic::x86_avx512_sqrt_pd_512;
15786 if (
Builder.getIsFPConstrained()) {
15787 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
15789 Ops[0]->getType());
15790 return Builder.CreateConstrainedFPCall(F, Ops[0]);
15793 return Builder.CreateCall(F, Ops[0]);
15797 case X86::BI__builtin_ia32_pmuludq128:
15798 case X86::BI__builtin_ia32_pmuludq256:
15799 case X86::BI__builtin_ia32_pmuludq512:
15802 case X86::BI__builtin_ia32_pmuldq128:
15803 case X86::BI__builtin_ia32_pmuldq256:
15804 case X86::BI__builtin_ia32_pmuldq512:
15807 case X86::BI__builtin_ia32_pternlogd512_mask:
15808 case X86::BI__builtin_ia32_pternlogq512_mask:
15809 case X86::BI__builtin_ia32_pternlogd128_mask:
15810 case X86::BI__builtin_ia32_pternlogd256_mask:
15811 case X86::BI__builtin_ia32_pternlogq128_mask:
15812 case X86::BI__builtin_ia32_pternlogq256_mask:
15815 case X86::BI__builtin_ia32_pternlogd512_maskz:
15816 case X86::BI__builtin_ia32_pternlogq512_maskz:
15817 case X86::BI__builtin_ia32_pternlogd128_maskz:
15818 case X86::BI__builtin_ia32_pternlogd256_maskz:
15819 case X86::BI__builtin_ia32_pternlogq128_maskz:
15820 case X86::BI__builtin_ia32_pternlogq256_maskz:
15823 case X86::BI__builtin_ia32_vpshldd128:
15824 case X86::BI__builtin_ia32_vpshldd256:
15825 case X86::BI__builtin_ia32_vpshldd512:
15826 case X86::BI__builtin_ia32_vpshldq128:
15827 case X86::BI__builtin_ia32_vpshldq256:
15828 case X86::BI__builtin_ia32_vpshldq512:
15829 case X86::BI__builtin_ia32_vpshldw128:
15830 case X86::BI__builtin_ia32_vpshldw256:
15831 case X86::BI__builtin_ia32_vpshldw512:
15834 case X86::BI__builtin_ia32_vpshrdd128:
15835 case X86::BI__builtin_ia32_vpshrdd256:
15836 case X86::BI__builtin_ia32_vpshrdd512:
15837 case X86::BI__builtin_ia32_vpshrdq128:
15838 case X86::BI__builtin_ia32_vpshrdq256:
15839 case X86::BI__builtin_ia32_vpshrdq512:
15840 case X86::BI__builtin_ia32_vpshrdw128:
15841 case X86::BI__builtin_ia32_vpshrdw256:
15842 case X86::BI__builtin_ia32_vpshrdw512:
15846 case X86::BI__builtin_ia32_vpshldvd128:
15847 case X86::BI__builtin_ia32_vpshldvd256:
15848 case X86::BI__builtin_ia32_vpshldvd512:
15849 case X86::BI__builtin_ia32_vpshldvq128:
15850 case X86::BI__builtin_ia32_vpshldvq256:
15851 case X86::BI__builtin_ia32_vpshldvq512:
15852 case X86::BI__builtin_ia32_vpshldvw128:
15853 case X86::BI__builtin_ia32_vpshldvw256:
15854 case X86::BI__builtin_ia32_vpshldvw512:
15857 case X86::BI__builtin_ia32_vpshrdvd128:
15858 case X86::BI__builtin_ia32_vpshrdvd256:
15859 case X86::BI__builtin_ia32_vpshrdvd512:
15860 case X86::BI__builtin_ia32_vpshrdvq128:
15861 case X86::BI__builtin_ia32_vpshrdvq256:
15862 case X86::BI__builtin_ia32_vpshrdvq512:
15863 case X86::BI__builtin_ia32_vpshrdvw128:
15864 case X86::BI__builtin_ia32_vpshrdvw256:
15865 case X86::BI__builtin_ia32_vpshrdvw512:
15870 case X86::BI__builtin_ia32_reduce_fadd_pd512:
15871 case X86::BI__builtin_ia32_reduce_fadd_ps512:
15872 case X86::BI__builtin_ia32_reduce_fadd_ph512:
15873 case X86::BI__builtin_ia32_reduce_fadd_ph256:
15874 case X86::BI__builtin_ia32_reduce_fadd_ph128: {
15877 IRBuilder<>::FastMathFlagGuard FMFGuard(
Builder);
15878 Builder.getFastMathFlags().setAllowReassoc();
15879 return Builder.CreateCall(F, {Ops[0], Ops[1]});
15881 case X86::BI__builtin_ia32_reduce_fmul_pd512:
15882 case X86::BI__builtin_ia32_reduce_fmul_ps512:
15883 case X86::BI__builtin_ia32_reduce_fmul_ph512:
15884 case X86::BI__builtin_ia32_reduce_fmul_ph256:
15885 case X86::BI__builtin_ia32_reduce_fmul_ph128: {
15888 IRBuilder<>::FastMathFlagGuard FMFGuard(
Builder);
15889 Builder.getFastMathFlags().setAllowReassoc();
15890 return Builder.CreateCall(F, {Ops[0], Ops[1]});
15892 case X86::BI__builtin_ia32_reduce_fmax_pd512:
15893 case X86::BI__builtin_ia32_reduce_fmax_ps512:
15894 case X86::BI__builtin_ia32_reduce_fmax_ph512:
15895 case X86::BI__builtin_ia32_reduce_fmax_ph256:
15896 case X86::BI__builtin_ia32_reduce_fmax_ph128: {
15899 IRBuilder<>::FastMathFlagGuard FMFGuard(
Builder);
15900 Builder.getFastMathFlags().setNoNaNs();
15901 return Builder.CreateCall(F, {Ops[0]});
15903 case X86::BI__builtin_ia32_reduce_fmin_pd512:
15904 case X86::BI__builtin_ia32_reduce_fmin_ps512:
15905 case X86::BI__builtin_ia32_reduce_fmin_ph512:
15906 case X86::BI__builtin_ia32_reduce_fmin_ph256:
15907 case X86::BI__builtin_ia32_reduce_fmin_ph128: {
15910 IRBuilder<>::FastMathFlagGuard FMFGuard(
Builder);
15911 Builder.getFastMathFlags().setNoNaNs();
15912 return Builder.CreateCall(F, {Ops[0]});
15916 case X86::BI__builtin_ia32_pswapdsf:
15917 case X86::BI__builtin_ia32_pswapdsi: {
15919 Ops[0] =
Builder.CreateBitCast(Ops[0], MMXTy,
"cast");
15921 return Builder.CreateCall(F, Ops,
"pswapd");
15923 case X86::BI__builtin_ia32_rdrand16_step:
15924 case X86::BI__builtin_ia32_rdrand32_step:
15925 case X86::BI__builtin_ia32_rdrand64_step:
15926 case X86::BI__builtin_ia32_rdseed16_step:
15927 case X86::BI__builtin_ia32_rdseed32_step:
15928 case X86::BI__builtin_ia32_rdseed64_step: {
15930 switch (BuiltinID) {
15931 default: llvm_unreachable(
"Unsupported intrinsic!");
15932 case X86::BI__builtin_ia32_rdrand16_step:
15933 ID = Intrinsic::x86_rdrand_16;
15935 case X86::BI__builtin_ia32_rdrand32_step:
15936 ID = Intrinsic::x86_rdrand_32;
15938 case X86::BI__builtin_ia32_rdrand64_step:
15939 ID = Intrinsic::x86_rdrand_64;
15941 case X86::BI__builtin_ia32_rdseed16_step:
15942 ID = Intrinsic::x86_rdseed_16;
15944 case X86::BI__builtin_ia32_rdseed32_step:
15945 ID = Intrinsic::x86_rdseed_32;
15947 case X86::BI__builtin_ia32_rdseed64_step:
15948 ID = Intrinsic::x86_rdseed_64;
15957 case X86::BI__builtin_ia32_addcarryx_u32:
15958 case X86::BI__builtin_ia32_addcarryx_u64:
15959 case X86::BI__builtin_ia32_subborrow_u32:
15960 case X86::BI__builtin_ia32_subborrow_u64: {
15962 switch (BuiltinID) {
15963 default: llvm_unreachable(
"Unsupported intrinsic!");
15964 case X86::BI__builtin_ia32_addcarryx_u32:
15965 IID = Intrinsic::x86_addcarry_32;
15967 case X86::BI__builtin_ia32_addcarryx_u64:
15968 IID = Intrinsic::x86_addcarry_64;
15970 case X86::BI__builtin_ia32_subborrow_u32:
15971 IID = Intrinsic::x86_subborrow_32;
15973 case X86::BI__builtin_ia32_subborrow_u64:
15974 IID = Intrinsic::x86_subborrow_64;
15979 { Ops[0], Ops[1], Ops[2] });
15985 case X86::BI__builtin_ia32_fpclassps128_mask:
15986 case X86::BI__builtin_ia32_fpclassps256_mask:
15987 case X86::BI__builtin_ia32_fpclassps512_mask:
15988 case X86::BI__builtin_ia32_fpclassph128_mask:
15989 case X86::BI__builtin_ia32_fpclassph256_mask:
15990 case X86::BI__builtin_ia32_fpclassph512_mask:
15991 case X86::BI__builtin_ia32_fpclasspd128_mask:
15992 case X86::BI__builtin_ia32_fpclasspd256_mask:
15993 case X86::BI__builtin_ia32_fpclasspd512_mask: {
15995 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
15996 Value *MaskIn = Ops[2];
15997 Ops.erase(&Ops[2]);
16000 switch (BuiltinID) {
16001 default: llvm_unreachable(
"Unsupported intrinsic!");
16002 case X86::BI__builtin_ia32_fpclassph128_mask:
16003 ID = Intrinsic::x86_avx512fp16_fpclass_ph_128;
16005 case X86::BI__builtin_ia32_fpclassph256_mask:
16006 ID = Intrinsic::x86_avx512fp16_fpclass_ph_256;
16008 case X86::BI__builtin_ia32_fpclassph512_mask:
16009 ID = Intrinsic::x86_avx512fp16_fpclass_ph_512;
16011 case X86::BI__builtin_ia32_fpclassps128_mask:
16012 ID = Intrinsic::x86_avx512_fpclass_ps_128;
16014 case X86::BI__builtin_ia32_fpclassps256_mask:
16015 ID = Intrinsic::x86_avx512_fpclass_ps_256;
16017 case X86::BI__builtin_ia32_fpclassps512_mask:
16018 ID = Intrinsic::x86_avx512_fpclass_ps_512;
16020 case X86::BI__builtin_ia32_fpclasspd128_mask:
16021 ID = Intrinsic::x86_avx512_fpclass_pd_128;
16023 case X86::BI__builtin_ia32_fpclasspd256_mask:
16024 ID = Intrinsic::x86_avx512_fpclass_pd_256;
16026 case X86::BI__builtin_ia32_fpclasspd512_mask:
16027 ID = Intrinsic::x86_avx512_fpclass_pd_512;
16035 case X86::BI__builtin_ia32_vp2intersect_q_512:
16036 case X86::BI__builtin_ia32_vp2intersect_q_256:
16037 case X86::BI__builtin_ia32_vp2intersect_q_128:
16038 case X86::BI__builtin_ia32_vp2intersect_d_512:
16039 case X86::BI__builtin_ia32_vp2intersect_d_256:
16040 case X86::BI__builtin_ia32_vp2intersect_d_128: {
16042 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
16045 switch (BuiltinID) {
16046 default: llvm_unreachable(
"Unsupported intrinsic!");
16047 case X86::BI__builtin_ia32_vp2intersect_q_512:
16048 ID = Intrinsic::x86_avx512_vp2intersect_q_512;
16050 case X86::BI__builtin_ia32_vp2intersect_q_256:
16051 ID = Intrinsic::x86_avx512_vp2intersect_q_256;
16053 case X86::BI__builtin_ia32_vp2intersect_q_128:
16054 ID = Intrinsic::x86_avx512_vp2intersect_q_128;
16056 case X86::BI__builtin_ia32_vp2intersect_d_512:
16057 ID = Intrinsic::x86_avx512_vp2intersect_d_512;
16059 case X86::BI__builtin_ia32_vp2intersect_d_256:
16060 ID = Intrinsic::x86_avx512_vp2intersect_d_256;
16062 case X86::BI__builtin_ia32_vp2intersect_d_128:
16063 ID = Intrinsic::x86_avx512_vp2intersect_d_128;
16077 case X86::BI__builtin_ia32_vpmultishiftqb128:
16078 case X86::BI__builtin_ia32_vpmultishiftqb256:
16079 case X86::BI__builtin_ia32_vpmultishiftqb512: {
16081 switch (BuiltinID) {
16082 default: llvm_unreachable(
"Unsupported intrinsic!");
16083 case X86::BI__builtin_ia32_vpmultishiftqb128:
16084 ID = Intrinsic::x86_avx512_pmultishift_qb_128;
16086 case X86::BI__builtin_ia32_vpmultishiftqb256:
16087 ID = Intrinsic::x86_avx512_pmultishift_qb_256;
16089 case X86::BI__builtin_ia32_vpmultishiftqb512:
16090 ID = Intrinsic::x86_avx512_pmultishift_qb_512;
16097 case X86::BI__builtin_ia32_vpshufbitqmb128_mask:
16098 case X86::BI__builtin_ia32_vpshufbitqmb256_mask:
16099 case X86::BI__builtin_ia32_vpshufbitqmb512_mask: {
16101 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
16102 Value *MaskIn = Ops[2];
16103 Ops.erase(&Ops[2]);
16106 switch (BuiltinID) {
16107 default: llvm_unreachable(
"Unsupported intrinsic!");
16108 case X86::BI__builtin_ia32_vpshufbitqmb128_mask:
16109 ID = Intrinsic::x86_avx512_vpshufbitqmb_128;
16111 case X86::BI__builtin_ia32_vpshufbitqmb256_mask:
16112 ID = Intrinsic::x86_avx512_vpshufbitqmb_256;
16114 case X86::BI__builtin_ia32_vpshufbitqmb512_mask:
16115 ID = Intrinsic::x86_avx512_vpshufbitqmb_512;
16124 case X86::BI__builtin_ia32_cmpeqps:
16125 case X86::BI__builtin_ia32_cmpeqpd:
16126 return getVectorFCmpIR(CmpInst::FCMP_OEQ,
false);
16127 case X86::BI__builtin_ia32_cmpltps:
16128 case X86::BI__builtin_ia32_cmpltpd:
16129 return getVectorFCmpIR(CmpInst::FCMP_OLT,
true);
16130 case X86::BI__builtin_ia32_cmpleps:
16131 case X86::BI__builtin_ia32_cmplepd:
16132 return getVectorFCmpIR(CmpInst::FCMP_OLE,
true);
16133 case X86::BI__builtin_ia32_cmpunordps:
16134 case X86::BI__builtin_ia32_cmpunordpd:
16135 return getVectorFCmpIR(CmpInst::FCMP_UNO,
false);
16136 case X86::BI__builtin_ia32_cmpneqps:
16137 case X86::BI__builtin_ia32_cmpneqpd:
16138 return getVectorFCmpIR(CmpInst::FCMP_UNE,
false);
16139 case X86::BI__builtin_ia32_cmpnltps:
16140 case X86::BI__builtin_ia32_cmpnltpd:
16141 return getVectorFCmpIR(CmpInst::FCMP_UGE,
true);
16142 case X86::BI__builtin_ia32_cmpnleps:
16143 case X86::BI__builtin_ia32_cmpnlepd:
16144 return getVectorFCmpIR(CmpInst::FCMP_UGT,
true);
16145 case X86::BI__builtin_ia32_cmpordps:
16146 case X86::BI__builtin_ia32_cmpordpd:
16147 return getVectorFCmpIR(CmpInst::FCMP_ORD,
false);
16148 case X86::BI__builtin_ia32_cmpph128_mask:
16149 case X86::BI__builtin_ia32_cmpph256_mask:
16150 case X86::BI__builtin_ia32_cmpph512_mask:
16151 case X86::BI__builtin_ia32_cmpps128_mask:
16152 case X86::BI__builtin_ia32_cmpps256_mask:
16153 case X86::BI__builtin_ia32_cmpps512_mask:
16154 case X86::BI__builtin_ia32_cmppd128_mask:
16155 case X86::BI__builtin_ia32_cmppd256_mask:
16156 case X86::BI__builtin_ia32_cmppd512_mask:
16159 case X86::BI__builtin_ia32_cmpps:
16160 case X86::BI__builtin_ia32_cmpps256:
16161 case X86::BI__builtin_ia32_cmppd:
16162 case X86::BI__builtin_ia32_cmppd256: {
16170 unsigned CC = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x1f;
16175 FCmpInst::Predicate Pred;
16179 switch (CC & 0xf) {
16180 case 0x00: Pred = FCmpInst::FCMP_OEQ; IsSignaling =
false;
break;
16181 case 0x01: Pred = FCmpInst::FCMP_OLT; IsSignaling =
true;
break;
16182 case 0x02: Pred = FCmpInst::FCMP_OLE; IsSignaling =
true;
break;
16183 case 0x03: Pred = FCmpInst::FCMP_UNO; IsSignaling =
false;
break;
16184 case 0x04: Pred = FCmpInst::FCMP_UNE; IsSignaling =
false;
break;
16185 case 0x05: Pred = FCmpInst::FCMP_UGE; IsSignaling =
true;
break;
16186 case 0x06: Pred = FCmpInst::FCMP_UGT; IsSignaling =
true;
break;
16187 case 0x07: Pred = FCmpInst::FCMP_ORD; IsSignaling =
false;
break;
16188 case 0x08: Pred = FCmpInst::FCMP_UEQ; IsSignaling =
false;
break;
16189 case 0x09: Pred = FCmpInst::FCMP_ULT; IsSignaling =
true;
break;
16190 case 0x0a: Pred = FCmpInst::FCMP_ULE; IsSignaling =
true;
break;
16191 case 0x0b: Pred = FCmpInst::FCMP_FALSE; IsSignaling =
false;
break;
16192 case 0x0c: Pred = FCmpInst::FCMP_ONE; IsSignaling =
false;
break;
16193 case 0x0d: Pred = FCmpInst::FCMP_OGE; IsSignaling =
true;
break;
16194 case 0x0e: Pred = FCmpInst::FCMP_OGT; IsSignaling =
true;
break;
16195 case 0x0f: Pred = FCmpInst::FCMP_TRUE; IsSignaling =
false;
break;
16196 default: llvm_unreachable(
"Unhandled CC");
16201 IsSignaling = !IsSignaling;
16208 if (
Builder.getIsFPConstrained() &&
16209 (Pred == FCmpInst::FCMP_TRUE || Pred == FCmpInst::FCMP_FALSE ||
16213 switch (BuiltinID) {
16214 default: llvm_unreachable(
"Unexpected builtin");
16215 case X86::BI__builtin_ia32_cmpps:
16216 IID = Intrinsic::x86_sse_cmp_ps;
16218 case X86::BI__builtin_ia32_cmpps256:
16219 IID = Intrinsic::x86_avx_cmp_ps_256;
16221 case X86::BI__builtin_ia32_cmppd:
16222 IID = Intrinsic::x86_sse2_cmp_pd;
16224 case X86::BI__builtin_ia32_cmppd256:
16225 IID = Intrinsic::x86_avx_cmp_pd_256;
16227 case X86::BI__builtin_ia32_cmpph128_mask:
16228 IID = Intrinsic::x86_avx512fp16_mask_cmp_ph_128;
16230 case X86::BI__builtin_ia32_cmpph256_mask:
16231 IID = Intrinsic::x86_avx512fp16_mask_cmp_ph_256;
16233 case X86::BI__builtin_ia32_cmpph512_mask:
16234 IID = Intrinsic::x86_avx512fp16_mask_cmp_ph_512;
16236 case X86::BI__builtin_ia32_cmpps512_mask:
16237 IID = Intrinsic::x86_avx512_mask_cmp_ps_512;
16239 case X86::BI__builtin_ia32_cmppd512_mask:
16240 IID = Intrinsic::x86_avx512_mask_cmp_pd_512;
16242 case X86::BI__builtin_ia32_cmpps128_mask:
16243 IID = Intrinsic::x86_avx512_mask_cmp_ps_128;
16245 case X86::BI__builtin_ia32_cmpps256_mask:
16246 IID = Intrinsic::x86_avx512_mask_cmp_ps_256;
16248 case X86::BI__builtin_ia32_cmppd128_mask:
16249 IID = Intrinsic::x86_avx512_mask_cmp_pd_128;
16251 case X86::BI__builtin_ia32_cmppd256_mask:
16252 IID = Intrinsic::x86_avx512_mask_cmp_pd_256;
16259 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
16265 return Builder.CreateCall(Intr, Ops);
16276 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
16279 Cmp =
Builder.CreateFCmpS(Pred, Ops[0], Ops[1]);
16281 Cmp =
Builder.CreateFCmp(Pred, Ops[0], Ops[1]);
16285 return getVectorFCmpIR(Pred, IsSignaling);
16289 case X86::BI__builtin_ia32_cmpeqss:
16290 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 0);
16291 case X86::BI__builtin_ia32_cmpltss:
16292 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 1);
16293 case X86::BI__builtin_ia32_cmpless:
16294 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 2);
16295 case X86::BI__builtin_ia32_cmpunordss:
16296 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 3);
16297 case X86::BI__builtin_ia32_cmpneqss:
16298 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 4);
16299 case X86::BI__builtin_ia32_cmpnltss:
16300 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 5);
16301 case X86::BI__builtin_ia32_cmpnless:
16302 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 6);
16303 case X86::BI__builtin_ia32_cmpordss:
16304 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 7);
16305 case X86::BI__builtin_ia32_cmpeqsd:
16306 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 0);
16307 case X86::BI__builtin_ia32_cmpltsd:
16308 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 1);
16309 case X86::BI__builtin_ia32_cmplesd:
16310 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 2);
16311 case X86::BI__builtin_ia32_cmpunordsd:
16312 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 3);
16313 case X86::BI__builtin_ia32_cmpneqsd:
16314 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 4);
16315 case X86::BI__builtin_ia32_cmpnltsd:
16316 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 5);
16317 case X86::BI__builtin_ia32_cmpnlesd:
16318 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 6);
16319 case X86::BI__builtin_ia32_cmpordsd:
16320 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 7);
16323 case X86::BI__builtin_ia32_vcvtph2ps:
16324 case X86::BI__builtin_ia32_vcvtph2ps256:
16325 case X86::BI__builtin_ia32_vcvtph2ps_mask:
16326 case X86::BI__builtin_ia32_vcvtph2ps256_mask:
16327 case X86::BI__builtin_ia32_vcvtph2ps512_mask: {
16328 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
16333 case X86::BI__builtin_ia32_cvtneps2bf16_128_mask: {
16336 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements());
16337 Intrinsic::ID IID = Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128;
16340 case X86::BI__builtin_ia32_cvtsbf162ss_32:
16343 case X86::BI__builtin_ia32_cvtneps2bf16_256_mask:
16344 case X86::BI__builtin_ia32_cvtneps2bf16_512_mask: {
16346 switch (BuiltinID) {
16347 default: llvm_unreachable(
"Unsupported intrinsic!");
16348 case X86::BI__builtin_ia32_cvtneps2bf16_256_mask:
16349 IID = Intrinsic::x86_avx512bf16_cvtneps2bf16_256;
16351 case X86::BI__builtin_ia32_cvtneps2bf16_512_mask:
16352 IID = Intrinsic::x86_avx512bf16_cvtneps2bf16_512;
16359 case X86::BI__cpuid:
16360 case X86::BI__cpuidex: {
16362 Value *SubFuncId = BuiltinID == X86::BI__cpuidex
16366 llvm::StructType *CpuidRetTy =
16368 llvm::FunctionType *FTy =
16371 StringRef
Asm, Constraints;
16372 if (
getTarget().getTriple().getArch() == llvm::Triple::x86) {
16374 Constraints =
"={ax},={bx},={cx},={dx},{ax},{cx}";
16377 Asm =
"xchgq %rbx, ${1:q}\n"
16379 "xchgq %rbx, ${1:q}";
16380 Constraints =
"={ax},=r,={cx},={dx},0,2";
16383 llvm::InlineAsm *IA = llvm::InlineAsm::get(FTy,
Asm, Constraints,
16385 Value *IACall =
Builder.CreateCall(IA, {FuncId, SubFuncId});
16388 for (
unsigned i = 0; i < 4; i++) {
16389 Value *Extracted =
Builder.CreateExtractValue(IACall, i);
16399 case X86::BI__emul:
16400 case X86::BI__emulu: {
16402 bool isSigned = (BuiltinID == X86::BI__emul);
16405 return Builder.CreateMul(LHS, RHS,
"", !isSigned, isSigned);
16407 case X86::BI__mulh:
16408 case X86::BI__umulh:
16409 case X86::BI_mul128:
16410 case X86::BI_umul128: {
16412 llvm::Type *Int128Ty = llvm::IntegerType::get(
getLLVMContext(), 128);
16414 bool IsSigned = (BuiltinID == X86::BI__mulh || BuiltinID == X86::BI_mul128);
16415 Value *LHS =
Builder.CreateIntCast(Ops[0], Int128Ty, IsSigned);
16416 Value *RHS =
Builder.CreateIntCast(Ops[1], Int128Ty, IsSigned);
16418 Value *MulResult, *HigherBits;
16420 MulResult =
Builder.CreateNSWMul(LHS, RHS);
16421 HigherBits =
Builder.CreateAShr(MulResult, 64);
16423 MulResult =
Builder.CreateNUWMul(LHS, RHS);
16424 HigherBits =
Builder.CreateLShr(MulResult, 64);
16426 HigherBits =
Builder.CreateIntCast(HigherBits, ResType, IsSigned);
16428 if (BuiltinID == X86::BI__mulh || BuiltinID == X86::BI__umulh)
16433 return Builder.CreateIntCast(MulResult, ResType, IsSigned);
16436 case X86::BI__faststorefence: {
16437 return Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent,
16438 llvm::SyncScope::System);
16440 case X86::BI__shiftleft128:
16441 case X86::BI__shiftright128: {
16443 BuiltinID == X86::BI__shiftleft128 ? Intrinsic::fshl : Intrinsic::fshr,
16448 std::swap(Ops[0], Ops[1]);
16450 return Builder.CreateCall(F, Ops);
16452 case X86::BI_ReadWriteBarrier:
16453 case X86::BI_ReadBarrier:
16454 case X86::BI_WriteBarrier: {
16455 return Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent,
16456 llvm::SyncScope::SingleThread);
16459 case X86::BI_AddressOfReturnAddress: {
16462 return Builder.CreateCall(F);
16464 case X86::BI__stosb: {
16472 case X86::BI__int2c: {
16474 llvm::FunctionType *FTy = llvm::FunctionType::get(
VoidTy,
false);
16475 llvm::InlineAsm *IA =
16476 llvm::InlineAsm::get(FTy,
"int $$0x2c",
"",
true);
16477 llvm::AttributeList NoReturnAttr = llvm::AttributeList::get(
16479 llvm::Attribute::NoReturn);
16480 llvm::CallInst *CI =
Builder.CreateCall(IA);
16481 CI->setAttributes(NoReturnAttr);
16484 case X86::BI__readfsbyte:
16485 case X86::BI__readfsword:
16486 case X86::BI__readfsdword:
16487 case X86::BI__readfsqword: {
16493 Load->setVolatile(
true);
16496 case X86::BI__readgsbyte:
16497 case X86::BI__readgsword:
16498 case X86::BI__readgsdword:
16499 case X86::BI__readgsqword: {
16505 Load->setVolatile(
true);
16508 case X86::BI__builtin_ia32_encodekey128_u32: {
16509 Intrinsic::ID IID = Intrinsic::x86_encodekey128;
16513 for (
int i = 0; i < 3; ++i) {
16521 case X86::BI__builtin_ia32_encodekey256_u32: {
16522 Intrinsic::ID IID = Intrinsic::x86_encodekey256;
16527 for (
int i = 0; i < 4; ++i) {
16535 case X86::BI__builtin_ia32_aesenc128kl_u8:
16536 case X86::BI__builtin_ia32_aesdec128kl_u8:
16537 case X86::BI__builtin_ia32_aesenc256kl_u8:
16538 case X86::BI__builtin_ia32_aesdec256kl_u8: {
16540 StringRef BlockName;
16541 switch (BuiltinID) {
16543 llvm_unreachable(
"Unexpected builtin");
16544 case X86::BI__builtin_ia32_aesenc128kl_u8:
16545 IID = Intrinsic::x86_aesenc128kl;
16546 BlockName =
"aesenc128kl";
16548 case X86::BI__builtin_ia32_aesdec128kl_u8:
16549 IID = Intrinsic::x86_aesdec128kl;
16550 BlockName =
"aesdec128kl";
16552 case X86::BI__builtin_ia32_aesenc256kl_u8:
16553 IID = Intrinsic::x86_aesenc256kl;
16554 BlockName =
"aesenc256kl";
16556 case X86::BI__builtin_ia32_aesdec256kl_u8:
16557 IID = Intrinsic::x86_aesdec256kl;
16558 BlockName =
"aesdec256kl";
16564 BasicBlock *NoError =
16572 Builder.CreateCondBr(Succ, NoError, Error);
16574 Builder.SetInsertPoint(NoError);
16578 Builder.SetInsertPoint(Error);
16579 Constant *
Zero = llvm::Constant::getNullValue(Out->getType());
16586 case X86::BI__builtin_ia32_aesencwide128kl_u8:
16587 case X86::BI__builtin_ia32_aesdecwide128kl_u8:
16588 case X86::BI__builtin_ia32_aesencwide256kl_u8:
16589 case X86::BI__builtin_ia32_aesdecwide256kl_u8: {
16591 StringRef BlockName;
16592 switch (BuiltinID) {
16593 case X86::BI__builtin_ia32_aesencwide128kl_u8:
16594 IID = Intrinsic::x86_aesencwide128kl;
16595 BlockName =
"aesencwide128kl";
16597 case X86::BI__builtin_ia32_aesdecwide128kl_u8:
16598 IID = Intrinsic::x86_aesdecwide128kl;
16599 BlockName =
"aesdecwide128kl";
16601 case X86::BI__builtin_ia32_aesencwide256kl_u8:
16602 IID = Intrinsic::x86_aesencwide256kl;
16603 BlockName =
"aesencwide256kl";
16605 case X86::BI__builtin_ia32_aesdecwide256kl_u8:
16606 IID = Intrinsic::x86_aesdecwide256kl;
16607 BlockName =
"aesdecwide256kl";
16611 llvm::Type *Ty = FixedVectorType::get(
Builder.getInt64Ty(), 2);
16614 for (
int i = 0; i != 8; ++i) {
16615 Value *Ptr =
Builder.CreateConstGEP1_32(Ty, Ops[1], i);
16621 BasicBlock *NoError =
16628 Builder.CreateCondBr(Succ, NoError, Error);
16630 Builder.SetInsertPoint(NoError);
16631 for (
int i = 0; i != 8; ++i) {
16638 Builder.SetInsertPoint(Error);
16639 for (
int i = 0; i != 8; ++i) {
16641 Constant *
Zero = llvm::Constant::getNullValue(Out->getType());
16642 Value *Ptr =
Builder.CreateConstGEP1_32(Out->getType(), Ops[0], i);
16650 case X86::BI__builtin_ia32_vfcmaddcph512_mask:
16653 case X86::BI__builtin_ia32_vfmaddcph512_mask: {
16654 Intrinsic::ID IID = IsConjFMA
16655 ? Intrinsic::x86_avx512fp16_mask_vfcmadd_cph_512
16656 : Intrinsic::x86_avx512fp16_mask_vfmadd_cph_512;
16660 case X86::BI__builtin_ia32_vfcmaddcsh_round_mask:
16663 case X86::BI__builtin_ia32_vfmaddcsh_round_mask: {
16664 Intrinsic::ID IID = IsConjFMA ? Intrinsic::x86_avx512fp16_mask_vfcmadd_csh
16665 : Intrinsic::x86_avx512fp16_mask_vfmadd_csh;
16670 case X86::BI__builtin_ia32_vfcmaddcsh_round_mask3:
16673 case X86::BI__builtin_ia32_vfmaddcsh_round_mask3: {
16674 Intrinsic::ID IID = IsConjFMA ? Intrinsic::x86_avx512fp16_mask_vfcmadd_csh
16675 : Intrinsic::x86_avx512fp16_mask_vfmadd_csh;
16677 static constexpr int Mask[] = {0, 5, 6, 7};
16678 return Builder.CreateShuffleVector(
Call, Ops[2], Mask);
16680 case X86::BI__builtin_ia32_prefetchi:
16683 {Ops[0], llvm::ConstantInt::get(Int32Ty, 0), Ops[1],
16684 llvm::ConstantInt::get(Int32Ty, 0)});
16702 Intrinsic::ID
ID = Intrinsic::not_intrinsic;
16704#include "llvm/TargetParser/PPCTargetParser.def"
16705 auto GenAIXPPCBuiltinCpuExpr = [&](
unsigned SupportMethod,
unsigned FieldIdx,
16706 unsigned Mask, CmpInst::Predicate CompOp,
16707 unsigned OpValue) ->
Value * {
16708 if (SupportMethod == AIX_BUILTIN_PPC_FALSE)
16711 if (SupportMethod == AIX_BUILTIN_PPC_TRUE)
16714 assert(SupportMethod <= SYS_CALL &&
"Invalid value for SupportMethod.");
16716 llvm::Value *FieldValue =
nullptr;
16717 if (SupportMethod == USE_SYS_CONF) {
16718 llvm::Type *STy = llvm::StructType::get(PPC_SYSTEMCONFIG_TYPE);
16719 llvm::Constant *SysConf =
16723 llvm::Value *Idxs[] = {ConstantInt::get(
Int32Ty, 0),
16724 ConstantInt::get(
Int32Ty, FieldIdx)};
16729 }
else if (SupportMethod == SYS_CALL) {
16730 llvm::FunctionType *FTy =
16732 llvm::FunctionCallee
Func =
16738 assert(FieldValue &&
16739 "SupportMethod value is not defined in PPCTargetParser.def.");
16742 FieldValue =
Builder.CreateAnd(FieldValue, Mask);
16744 llvm::Type *ValueType = FieldValue->getType();
16745 bool IsValueType64Bit = ValueType->isIntegerTy(64);
16747 (IsValueType64Bit || ValueType->isIntegerTy(32)) &&
16748 "Only 32/64-bit integers are supported in GenAIXPPCBuiltinCpuExpr().");
16751 CompOp, FieldValue,
16752 ConstantInt::get(IsValueType64Bit ?
Int64Ty :
Int32Ty, OpValue));
16755 switch (BuiltinID) {
16756 default:
return nullptr;
16758 case Builtin::BI__builtin_cpu_is: {
16760 StringRef CPUStr = cast<clang::StringLiteral>(CPUExpr)->getString();
16763 if (Triple.isOSAIX()) {
16764 unsigned SupportMethod, FieldIdx, CpuIdValue;
16765 CmpInst::Predicate CompareOp;
16766 typedef std::tuple<unsigned, unsigned, CmpInst::Predicate, unsigned>
16768 std::tie(SupportMethod, FieldIdx, CompareOp, CpuIdValue) =
16769 static_cast<CPUType
>(StringSwitch<CPUType>(CPUStr)
16770#define PPC_AIX_CPU(NAME, SUPPORT_METHOD, INDEX, COMPARE_OP, VALUE) \
16771 .Case(NAME, {SUPPORT_METHOD, INDEX, COMPARE_OP, VALUE})
16772#include "llvm/TargetParser/PPCTargetParser.def"
16773 .Default({AIX_BUILTIN_PPC_FALSE, 0,
16774 CmpInst::Predicate(), 0}));
16775 return GenAIXPPCBuiltinCpuExpr(SupportMethod, FieldIdx, 0, CompareOp,
16779 assert(Triple.isOSLinux() &&
16780 "__builtin_cpu_is() is only supported for AIX and Linux.");
16781 unsigned NumCPUID = StringSwitch<unsigned>(CPUStr)
16782#define PPC_LNX_CPU(Name, NumericID) .Case(Name, NumericID)
16783#include "llvm/TargetParser/PPCTargetParser.def"
16785 assert(NumCPUID < -1U &&
"Invalid CPU name. Missed by SemaChecking?");
16786 Value *Op0 = llvm::ConstantInt::get(
Int32Ty, PPC_FAWORD_CPUID);
16788 Value *TheCall =
Builder.CreateCall(F, {Op0},
"cpu_is");
16789 return Builder.CreateICmpEQ(TheCall,
16790 llvm::ConstantInt::get(
Int32Ty, NumCPUID));
16792 case Builtin::BI__builtin_cpu_supports: {
16795 StringRef CPUStr = cast<clang::StringLiteral>(CPUExpr)->getString();
16796 if (Triple.isOSAIX()) {
16797 unsigned SupportMethod, FieldIdx, Mask,
Value;
16798 CmpInst::Predicate CompOp;
16802 std::tie(SupportMethod, FieldIdx, Mask, CompOp,
Value) =
16803 static_cast<CPUSupportType
>(StringSwitch<CPUSupportType>(CPUStr)
16804#define PPC_AIX_FEATURE(NAME, DESC, SUPPORT_METHOD, INDEX, MASK, COMP_OP, \
16806 .Case(NAME, {SUPPORT_METHOD, INDEX, MASK, COMP_OP, VALUE})
16807#include "llvm/TargetParser/PPCTargetParser.def"
16808 .Default({AIX_BUILTIN_PPC_FALSE, 0, 0,
16809 CmpInst::Predicate(), 0}));
16810 return GenAIXPPCBuiltinCpuExpr(SupportMethod, FieldIdx, Mask, CompOp,
16814 assert(Triple.isOSLinux() &&
16815 "__builtin_cpu_supports() is only supported for AIX and Linux.");
16816 unsigned FeatureWord;
16818 std::tie(FeatureWord, BitMask) =
16819 StringSwitch<std::pair<unsigned, unsigned>>(CPUStr)
16820#define
PPC_LNX_FEATURE(Name, Description, EnumName, Bitmask, FA_WORD) \
16821 .Case(Name, {FA_WORD, Bitmask})
16822#include
"llvm/TargetParser/PPCTargetParser.def"
16826 Value *Op0 = llvm::ConstantInt::get(
Int32Ty, FeatureWord);
16828 Value *TheCall =
Builder.CreateCall(F, {Op0},
"cpu_supports");
16830 Builder.CreateAnd(TheCall, llvm::ConstantInt::get(
Int32Ty, BitMask));
16831 return Builder.CreateICmpNE(Mask, llvm::Constant::getNullValue(
Int32Ty));
16832#undef PPC_FAWORD_HWCAP
16833#undef PPC_FAWORD_HWCAP2
16834#undef PPC_FAWORD_CPUID
16839 case PPC::BI__builtin_ppc_get_timebase:
16843 case PPC::BI__builtin_altivec_lvx:
16844 case PPC::BI__builtin_altivec_lvxl:
16845 case PPC::BI__builtin_altivec_lvebx:
16846 case PPC::BI__builtin_altivec_lvehx:
16847 case PPC::BI__builtin_altivec_lvewx:
16848 case PPC::BI__builtin_altivec_lvsl:
16849 case PPC::BI__builtin_altivec_lvsr:
16850 case PPC::BI__builtin_vsx_lxvd2x:
16851 case PPC::BI__builtin_vsx_lxvw4x:
16852 case PPC::BI__builtin_vsx_lxvd2x_be:
16853 case PPC::BI__builtin_vsx_lxvw4x_be:
16854 case PPC::BI__builtin_vsx_lxvl:
16855 case PPC::BI__builtin_vsx_lxvll:
16860 if (!(BuiltinID == PPC::BI__builtin_vsx_lxvl ||
16861 BuiltinID == PPC::BI__builtin_vsx_lxvll)) {
16866 switch (BuiltinID) {
16867 default: llvm_unreachable(
"Unsupported ld/lvsl/lvsr intrinsic!");
16868 case PPC::BI__builtin_altivec_lvx:
16869 ID = Intrinsic::ppc_altivec_lvx;
16871 case PPC::BI__builtin_altivec_lvxl:
16872 ID = Intrinsic::ppc_altivec_lvxl;
16874 case PPC::BI__builtin_altivec_lvebx:
16875 ID = Intrinsic::ppc_altivec_lvebx;
16877 case PPC::BI__builtin_altivec_lvehx:
16878 ID = Intrinsic::ppc_altivec_lvehx;
16880 case PPC::BI__builtin_altivec_lvewx:
16881 ID = Intrinsic::ppc_altivec_lvewx;
16883 case PPC::BI__builtin_altivec_lvsl:
16884 ID = Intrinsic::ppc_altivec_lvsl;
16886 case PPC::BI__builtin_altivec_lvsr:
16887 ID = Intrinsic::ppc_altivec_lvsr;
16889 case PPC::BI__builtin_vsx_lxvd2x:
16890 ID = Intrinsic::ppc_vsx_lxvd2x;
16892 case PPC::BI__builtin_vsx_lxvw4x:
16893 ID = Intrinsic::ppc_vsx_lxvw4x;
16895 case PPC::BI__builtin_vsx_lxvd2x_be:
16896 ID = Intrinsic::ppc_vsx_lxvd2x_be;
16898 case PPC::BI__builtin_vsx_lxvw4x_be:
16899 ID = Intrinsic::ppc_vsx_lxvw4x_be;
16901 case PPC::BI__builtin_vsx_lxvl:
16902 ID = Intrinsic::ppc_vsx_lxvl;
16904 case PPC::BI__builtin_vsx_lxvll:
16905 ID = Intrinsic::ppc_vsx_lxvll;
16909 return Builder.CreateCall(F, Ops,
"");
16913 case PPC::BI__builtin_altivec_stvx:
16914 case PPC::BI__builtin_altivec_stvxl:
16915 case PPC::BI__builtin_altivec_stvebx:
16916 case PPC::BI__builtin_altivec_stvehx:
16917 case PPC::BI__builtin_altivec_stvewx:
16918 case PPC::BI__builtin_vsx_stxvd2x:
16919 case PPC::BI__builtin_vsx_stxvw4x:
16920 case PPC::BI__builtin_vsx_stxvd2x_be:
16921 case PPC::BI__builtin_vsx_stxvw4x_be:
16922 case PPC::BI__builtin_vsx_stxvl:
16923 case PPC::BI__builtin_vsx_stxvll:
16929 if (!(BuiltinID == PPC::BI__builtin_vsx_stxvl ||
16930 BuiltinID == PPC::BI__builtin_vsx_stxvll)) {
16935 switch (BuiltinID) {
16936 default: llvm_unreachable(
"Unsupported st intrinsic!");
16937 case PPC::BI__builtin_altivec_stvx:
16938 ID = Intrinsic::ppc_altivec_stvx;
16940 case PPC::BI__builtin_altivec_stvxl:
16941 ID = Intrinsic::ppc_altivec_stvxl;
16943 case PPC::BI__builtin_altivec_stvebx:
16944 ID = Intrinsic::ppc_altivec_stvebx;
16946 case PPC::BI__builtin_altivec_stvehx:
16947 ID = Intrinsic::ppc_altivec_stvehx;
16949 case PPC::BI__builtin_altivec_stvewx:
16950 ID = Intrinsic::ppc_altivec_stvewx;
16952 case PPC::BI__builtin_vsx_stxvd2x:
16953 ID = Intrinsic::ppc_vsx_stxvd2x;
16955 case PPC::BI__builtin_vsx_stxvw4x:
16956 ID = Intrinsic::ppc_vsx_stxvw4x;
16958 case PPC::BI__builtin_vsx_stxvd2x_be:
16959 ID = Intrinsic::ppc_vsx_stxvd2x_be;
16961 case PPC::BI__builtin_vsx_stxvw4x_be:
16962 ID = Intrinsic::ppc_vsx_stxvw4x_be;
16964 case PPC::BI__builtin_vsx_stxvl:
16965 ID = Intrinsic::ppc_vsx_stxvl;
16967 case PPC::BI__builtin_vsx_stxvll:
16968 ID = Intrinsic::ppc_vsx_stxvll;
16972 return Builder.CreateCall(F, Ops,
"");
16974 case PPC::BI__builtin_vsx_ldrmb: {
16980 int64_t NumBytes = cast<ConstantInt>(Op1)->getZExtValue();
16985 if (NumBytes == 16) {
16993 for (
int Idx = 0; Idx < 16; Idx++)
16994 RevMask.push_back(15 - Idx);
16995 return Builder.CreateShuffleVector(LD, LD, RevMask);
16999 llvm::Function *Lvs =
CGM.
getIntrinsic(IsLE ? Intrinsic::ppc_altivec_lvsr
17000 : Intrinsic::ppc_altivec_lvsl);
17001 llvm::Function *Vperm =
CGM.
getIntrinsic(Intrinsic::ppc_altivec_vperm);
17003 Int8Ty, Op0, ConstantInt::get(Op1->
getType(), NumBytes - 1));
17005 Value *HiLd =
Builder.CreateCall(Lvx, HiMem,
"ld.hi");
17008 Op0 = IsLE ? HiLd : LoLd;
17009 Op1 = IsLE ? LoLd : HiLd;
17010 Value *AllElts =
Builder.CreateCall(Vperm, {Op0, Op1, Mask1},
"shuffle1");
17011 Constant *
Zero = llvm::Constant::getNullValue(IsLE ? ResTy : AllElts->
getType());
17015 for (
int Idx = 0; Idx < 16; Idx++) {
17016 int Val = (NumBytes - Idx - 1 >= 0) ? (NumBytes - Idx - 1)
17017 : 16 - (NumBytes - Idx);
17018 Consts.push_back(Val);
17020 return Builder.CreateShuffleVector(
Builder.CreateBitCast(AllElts, ResTy),
17024 for (
int Idx = 0; Idx < 16; Idx++)
17025 Consts.push_back(
Builder.getInt8(NumBytes + Idx));
17026 Value *Mask2 = ConstantVector::get(Consts);
17027 return Builder.CreateBitCast(
17028 Builder.CreateCall(Vperm, {Zero, AllElts, Mask2},
"shuffle2"), ResTy);
17030 case PPC::BI__builtin_vsx_strmb: {
17034 int64_t NumBytes = cast<ConstantInt>(Op1)->getZExtValue();
17036 auto StoreSubVec = [&](
unsigned Width,
unsigned Offset,
unsigned EltNo) {
17040 Value *StVec = Op2;
17043 for (
int Idx = 0; Idx < 16; Idx++)
17044 RevMask.push_back(15 - Idx);
17045 StVec =
Builder.CreateShuffleVector(Op2, Op2, RevMask);
17051 unsigned NumElts = 0;
17054 llvm_unreachable(
"width for stores must be a power of 2");
17073 Op2, llvm::FixedVectorType::get(ConvTy, NumElts));
17076 Value *Elt =
Builder.CreateExtractElement(Vec, EltNo);
17077 if (IsLE && Width > 1) {
17079 Elt =
Builder.CreateCall(F, Elt);
17084 unsigned Stored = 0;
17085 unsigned RemainingBytes = NumBytes;
17087 if (NumBytes == 16)
17088 return StoreSubVec(16, 0, 0);
17089 if (NumBytes >= 8) {
17090 Result = StoreSubVec(8, NumBytes - 8, IsLE ? 0 : 1);
17091 RemainingBytes -= 8;
17094 if (RemainingBytes >= 4) {
17095 Result = StoreSubVec(4, NumBytes - Stored - 4,
17096 IsLE ? (Stored >> 2) : 3 - (Stored >> 2));
17097 RemainingBytes -= 4;
17100 if (RemainingBytes >= 2) {
17101 Result = StoreSubVec(2, NumBytes - Stored - 2,
17102 IsLE ? (Stored >> 1) : 7 - (Stored >> 1));
17103 RemainingBytes -= 2;
17106 if (RemainingBytes)
17108 StoreSubVec(1, NumBytes - Stored - 1, IsLE ? Stored : 15 - Stored);
17112 case PPC::BI__builtin_vsx_xvsqrtsp:
17113 case PPC::BI__builtin_vsx_xvsqrtdp: {
17116 if (
Builder.getIsFPConstrained()) {
17118 Intrinsic::experimental_constrained_sqrt, ResultType);
17119 return Builder.CreateConstrainedFPCall(F,
X);
17126 case PPC::BI__builtin_altivec_vclzb:
17127 case PPC::BI__builtin_altivec_vclzh:
17128 case PPC::BI__builtin_altivec_vclzw:
17129 case PPC::BI__builtin_altivec_vclzd: {
17132 Value *Undef = ConstantInt::get(
Builder.getInt1Ty(),
false);
17134 return Builder.CreateCall(F, {
X, Undef});
17136 case PPC::BI__builtin_altivec_vctzb:
17137 case PPC::BI__builtin_altivec_vctzh:
17138 case PPC::BI__builtin_altivec_vctzw:
17139 case PPC::BI__builtin_altivec_vctzd: {
17142 Value *Undef = ConstantInt::get(
Builder.getInt1Ty(),
false);
17144 return Builder.CreateCall(F, {
X, Undef});
17146 case PPC::BI__builtin_altivec_vinsd:
17147 case PPC::BI__builtin_altivec_vinsw:
17148 case PPC::BI__builtin_altivec_vinsd_elt:
17149 case PPC::BI__builtin_altivec_vinsw_elt: {
17155 bool IsUnaligned = (BuiltinID == PPC::BI__builtin_altivec_vinsw ||
17156 BuiltinID == PPC::BI__builtin_altivec_vinsd);
17158 bool Is32bit = (BuiltinID == PPC::BI__builtin_altivec_vinsw ||
17159 BuiltinID == PPC::BI__builtin_altivec_vinsw_elt);
17162 ConstantInt *ArgCI = dyn_cast<ConstantInt>(Op2);
17164 "Third Arg to vinsw/vinsd intrinsic must be a constant integer!");
17168 int ValidMaxValue = 0;
17170 ValidMaxValue = (Is32bit) ? 12 : 8;
17172 ValidMaxValue = (Is32bit) ? 3 : 1;
17175 int64_t ConstArg = ArgCI->getSExtValue();
17178 std::string RangeErrMsg = IsUnaligned ?
"byte" :
"element";
17179 RangeErrMsg +=
" number " + llvm::to_string(ConstArg);
17180 RangeErrMsg +=
" is outside of the valid range [0, ";
17181 RangeErrMsg += llvm::to_string(ValidMaxValue) +
"]";
17184 if (ConstArg < 0 || ConstArg > ValidMaxValue)
17188 if (!IsUnaligned) {
17189 ConstArg *= Is32bit ? 4 : 8;
17192 ConstArg = (Is32bit ? 12 : 8) - ConstArg;
17195 ID = Is32bit ? Intrinsic::ppc_altivec_vinsw : Intrinsic::ppc_altivec_vinsd;
17196 Op2 = ConstantInt::getSigned(
Int32Ty, ConstArg);
17200 ?
Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(
Int32Ty, 4))
17202 llvm::FixedVectorType::get(
Int64Ty, 2));
17203 return Builder.CreateBitCast(
17206 case PPC::BI__builtin_altivec_vpopcntb:
17207 case PPC::BI__builtin_altivec_vpopcnth:
17208 case PPC::BI__builtin_altivec_vpopcntw:
17209 case PPC::BI__builtin_altivec_vpopcntd: {
17215 case PPC::BI__builtin_altivec_vadduqm:
17216 case PPC::BI__builtin_altivec_vsubuqm: {
17219 llvm::Type *Int128Ty = llvm::IntegerType::get(
getLLVMContext(), 128);
17220 Op0 =
Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(Int128Ty, 1));
17221 Op1 =
Builder.CreateBitCast(Op1, llvm::FixedVectorType::get(Int128Ty, 1));
17222 if (BuiltinID == PPC::BI__builtin_altivec_vadduqm)
17223 return Builder.CreateAdd(Op0, Op1,
"vadduqm");
17225 return Builder.CreateSub(Op0, Op1,
"vsubuqm");
17227 case PPC::BI__builtin_altivec_vaddcuq_c:
17228 case PPC::BI__builtin_altivec_vsubcuq_c: {
17232 llvm::Type *V1I128Ty = llvm::FixedVectorType::get(
17234 Ops.push_back(
Builder.CreateBitCast(Op0, V1I128Ty));
17235 Ops.push_back(
Builder.CreateBitCast(Op1, V1I128Ty));
17236 ID = (BuiltinID == PPC::BI__builtin_altivec_vaddcuq_c)
17237 ? Intrinsic::ppc_altivec_vaddcuq
17238 : Intrinsic::ppc_altivec_vsubcuq;
17241 case PPC::BI__builtin_altivec_vaddeuqm_c:
17242 case PPC::BI__builtin_altivec_vaddecuq_c:
17243 case PPC::BI__builtin_altivec_vsubeuqm_c:
17244 case PPC::BI__builtin_altivec_vsubecuq_c: {
17249 llvm::Type *V1I128Ty = llvm::FixedVectorType::get(
17251 Ops.push_back(
Builder.CreateBitCast(Op0, V1I128Ty));
17252 Ops.push_back(
Builder.CreateBitCast(Op1, V1I128Ty));
17253 Ops.push_back(
Builder.CreateBitCast(Op2, V1I128Ty));
17254 switch (BuiltinID) {
17256 llvm_unreachable(
"Unsupported intrinsic!");
17257 case PPC::BI__builtin_altivec_vaddeuqm_c:
17258 ID = Intrinsic::ppc_altivec_vaddeuqm;
17260 case PPC::BI__builtin_altivec_vaddecuq_c:
17261 ID = Intrinsic::ppc_altivec_vaddecuq;
17263 case PPC::BI__builtin_altivec_vsubeuqm_c:
17264 ID = Intrinsic::ppc_altivec_vsubeuqm;
17266 case PPC::BI__builtin_altivec_vsubecuq_c:
17267 ID = Intrinsic::ppc_altivec_vsubecuq;
17272 case PPC::BI__builtin_ppc_rldimi:
17273 case PPC::BI__builtin_ppc_rlwimi: {
17280 if (BuiltinID == PPC::BI__builtin_ppc_rldimi &&
17290 ? Intrinsic::ppc_rldimi
17291 : Intrinsic::ppc_rlwimi),
17292 {Op0, Op1, Op2, Op3});
17294 case PPC::BI__builtin_ppc_rlwnm: {
17301 case PPC::BI__builtin_ppc_poppar4:
17302 case PPC::BI__builtin_ppc_poppar8: {
17304 llvm::Type *ArgType = Op0->
getType();
17310 if (
Result->getType() != ResultType)
17315 case PPC::BI__builtin_ppc_cmpb: {
17318 if (
getTarget().getTriple().isPPC64()) {
17321 return Builder.CreateCall(F, {Op0, Op1},
"cmpb");
17341 Constant *ShiftAmt = ConstantInt::get(
Int64Ty, 32);
17350 Value *ResHi =
Builder.CreateShl(ResHiShift, ShiftAmt);
17351 return Builder.CreateOr(ResLo, ResHi);
17354 case PPC::BI__builtin_vsx_xvcpsgnsp:
17355 case PPC::BI__builtin_vsx_xvcpsgndp: {
17359 ID = Intrinsic::copysign;
17361 return Builder.CreateCall(F, {
X, Y});
17364 case PPC::BI__builtin_vsx_xvrspip:
17365 case PPC::BI__builtin_vsx_xvrdpip:
17366 case PPC::BI__builtin_vsx_xvrdpim:
17367 case PPC::BI__builtin_vsx_xvrspim:
17368 case PPC::BI__builtin_vsx_xvrdpi:
17369 case PPC::BI__builtin_vsx_xvrspi:
17370 case PPC::BI__builtin_vsx_xvrdpic:
17371 case PPC::BI__builtin_vsx_xvrspic:
17372 case PPC::BI__builtin_vsx_xvrdpiz:
17373 case PPC::BI__builtin_vsx_xvrspiz: {
17376 if (BuiltinID == PPC::BI__builtin_vsx_xvrdpim ||
17377 BuiltinID == PPC::BI__builtin_vsx_xvrspim)
17379 ? Intrinsic::experimental_constrained_floor
17380 : Intrinsic::floor;
17381 else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpi ||
17382 BuiltinID == PPC::BI__builtin_vsx_xvrspi)
17384 ? Intrinsic::experimental_constrained_round
17385 : Intrinsic::round;
17386 else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpic ||
17387 BuiltinID == PPC::BI__builtin_vsx_xvrspic)
17389 ? Intrinsic::experimental_constrained_rint
17391 else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpip ||
17392 BuiltinID == PPC::BI__builtin_vsx_xvrspip)
17394 ? Intrinsic::experimental_constrained_ceil
17396 else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpiz ||
17397 BuiltinID == PPC::BI__builtin_vsx_xvrspiz)
17399 ? Intrinsic::experimental_constrained_trunc
17400 : Intrinsic::trunc;
17402 return Builder.getIsFPConstrained() ?
Builder.CreateConstrainedFPCall(F,
X)
17407 case PPC::BI__builtin_vsx_xvabsdp:
17408 case PPC::BI__builtin_vsx_xvabssp: {
17416 case PPC::BI__builtin_ppc_recipdivf:
17417 case PPC::BI__builtin_ppc_recipdivd:
17418 case PPC::BI__builtin_ppc_rsqrtf:
17419 case PPC::BI__builtin_ppc_rsqrtd: {
17420 FastMathFlags FMF =
Builder.getFastMathFlags();
17421 Builder.getFastMathFlags().setFast();
17425 if (BuiltinID == PPC::BI__builtin_ppc_recipdivf ||
17426 BuiltinID == PPC::BI__builtin_ppc_recipdivd) {
17429 Builder.getFastMathFlags() &= (FMF);
17432 auto *One = ConstantFP::get(ResultType, 1.0);
17435 Builder.getFastMathFlags() &= (FMF);
17438 case PPC::BI__builtin_ppc_alignx: {
17441 ConstantInt *AlignmentCI = cast<ConstantInt>(Op0);
17442 if (AlignmentCI->getValue().ugt(llvm::Value::MaximumAlignment))
17443 AlignmentCI = ConstantInt::get(AlignmentCI->getIntegerType(),
17444 llvm::Value::MaximumAlignment);
17448 AlignmentCI,
nullptr);
17451 case PPC::BI__builtin_ppc_rdlam: {
17455 llvm::Type *Ty = Op0->
getType();
17456 Value *ShiftAmt =
Builder.CreateIntCast(Op1, Ty,
false);
17458 Value *Rotate =
Builder.CreateCall(F, {Op0, Op0, ShiftAmt});
17459 return Builder.CreateAnd(Rotate, Op2);
17461 case PPC::BI__builtin_ppc_load2r: {
17468 case PPC::BI__builtin_ppc_fnmsub:
17469 case PPC::BI__builtin_ppc_fnmsubs:
17470 case PPC::BI__builtin_vsx_xvmaddadp:
17471 case PPC::BI__builtin_vsx_xvmaddasp:
17472 case PPC::BI__builtin_vsx_xvnmaddadp:
17473 case PPC::BI__builtin_vsx_xvnmaddasp:
17474 case PPC::BI__builtin_vsx_xvmsubadp:
17475 case PPC::BI__builtin_vsx_xvmsubasp:
17476 case PPC::BI__builtin_vsx_xvnmsubadp:
17477 case PPC::BI__builtin_vsx_xvnmsubasp: {
17483 if (
Builder.getIsFPConstrained())
17484 F =
CGM.
getIntrinsic(Intrinsic::experimental_constrained_fma, ResultType);
17487 switch (BuiltinID) {
17488 case PPC::BI__builtin_vsx_xvmaddadp:
17489 case PPC::BI__builtin_vsx_xvmaddasp:
17490 if (
Builder.getIsFPConstrained())
17491 return Builder.CreateConstrainedFPCall(F, {
X, Y, Z});
17493 return Builder.CreateCall(F, {
X, Y, Z});
17494 case PPC::BI__builtin_vsx_xvnmaddadp:
17495 case PPC::BI__builtin_vsx_xvnmaddasp:
17496 if (
Builder.getIsFPConstrained())
17498 Builder.CreateConstrainedFPCall(F, {X, Y, Z}),
"neg");
17500 return Builder.CreateFNeg(
Builder.CreateCall(F, {X, Y, Z}),
"neg");
17501 case PPC::BI__builtin_vsx_xvmsubadp:
17502 case PPC::BI__builtin_vsx_xvmsubasp:
17503 if (
Builder.getIsFPConstrained())
17504 return Builder.CreateConstrainedFPCall(
17505 F, {
X, Y,
Builder.CreateFNeg(Z,
"neg")});
17508 case PPC::BI__builtin_ppc_fnmsub:
17509 case PPC::BI__builtin_ppc_fnmsubs:
17510 case PPC::BI__builtin_vsx_xvnmsubadp:
17511 case PPC::BI__builtin_vsx_xvnmsubasp:
17512 if (
Builder.getIsFPConstrained())
17514 Builder.CreateConstrainedFPCall(
17515 F, {X, Y, Builder.CreateFNeg(Z,
"neg")}),
17521 llvm_unreachable(
"Unknown FMA operation");
17525 case PPC::BI__builtin_vsx_insertword: {
17533 ConstantInt *ArgCI = dyn_cast<ConstantInt>(Op2);
17535 "Third arg to xxinsertw intrinsic must be constant integer");
17537 int64_t Index = std::clamp(ArgCI->getSExtValue(), (int64_t)0, MaxIndex);
17544 std::swap(Op0, Op1);
17548 Op1 =
Builder.CreateBitCast(Op1, llvm::FixedVectorType::get(
Int64Ty, 2));
17552 Op0 =
Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(
Int64Ty, 2));
17556 Index = MaxIndex - Index;
17560 Op0 =
Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(
Int32Ty, 4));
17561 Op2 = ConstantInt::getSigned(
Int32Ty, Index);
17562 return Builder.CreateCall(F, {Op0, Op1, Op2});
17565 case PPC::BI__builtin_vsx_extractuword: {
17568 llvm::Function *F =
CGM.
getIntrinsic(Intrinsic::ppc_vsx_xxextractuw);
17571 Op0 =
Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(
Int64Ty, 2));
17575 ConstantInt *ArgCI = dyn_cast<ConstantInt>(Op1);
17577 "Second Arg to xxextractuw intrinsic must be a constant integer!");
17579 int64_t Index = std::clamp(ArgCI->getSExtValue(), (int64_t)0, MaxIndex);
17583 Index = MaxIndex - Index;
17584 Op1 = ConstantInt::getSigned(
Int32Ty, Index);
17589 Value *ShuffleCall =
17591 return ShuffleCall;
17593 Op1 = ConstantInt::getSigned(
Int32Ty, Index);
17594 return Builder.CreateCall(F, {Op0, Op1});
17598 case PPC::BI__builtin_vsx_xxpermdi: {
17602 ConstantInt *ArgCI = dyn_cast<ConstantInt>(Op2);
17603 assert(ArgCI &&
"Third arg must be constant integer!");
17605 unsigned Index = ArgCI->getZExtValue();
17606 Op0 =
Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(
Int64Ty, 2));
17607 Op1 =
Builder.CreateBitCast(Op1, llvm::FixedVectorType::get(
Int64Ty, 2));
17612 int ElemIdx0 = (Index & 2) >> 1;
17613 int ElemIdx1 = 2 + (Index & 1);
17615 int ShuffleElts[2] = {ElemIdx0, ElemIdx1};
17616 Value *ShuffleCall =
Builder.CreateShuffleVector(Op0, Op1, ShuffleElts);
17619 return Builder.CreateBitCast(ShuffleCall, RetTy);
17622 case PPC::BI__builtin_vsx_xxsldwi: {
17626 ConstantInt *ArgCI = dyn_cast<ConstantInt>(Op2);
17627 assert(ArgCI &&
"Third argument must be a compile time constant");
17628 unsigned Index = ArgCI->getZExtValue() & 0x3;
17629 Op0 =
Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(
Int32Ty, 4));
17630 Op1 =
Builder.CreateBitCast(Op1, llvm::FixedVectorType::get(
Int32Ty, 4));
17641 ElemIdx0 = (8 - Index) % 8;
17642 ElemIdx1 = (9 - Index) % 8;
17643 ElemIdx2 = (10 - Index) % 8;
17644 ElemIdx3 = (11 - Index) % 8;
17648 ElemIdx1 = Index + 1;
17649 ElemIdx2 = Index + 2;
17650 ElemIdx3 = Index + 3;
17653 int ShuffleElts[4] = {ElemIdx0, ElemIdx1, ElemIdx2, ElemIdx3};
17654 Value *ShuffleCall =
Builder.CreateShuffleVector(Op0, Op1, ShuffleElts);
17657 return Builder.CreateBitCast(ShuffleCall, RetTy);
17660 case PPC::BI__builtin_pack_vector_int128: {
17664 Value *PoisonValue =
17665 llvm::PoisonValue::get(llvm::FixedVectorType::get(Op0->
getType(), 2));
17667 PoisonValue, Op0, (uint64_t)(isLittleEndian ? 1 : 0));
17668 Res =
Builder.CreateInsertElement(Res, Op1,
17669 (uint64_t)(isLittleEndian ? 0 : 1));
17673 case PPC::BI__builtin_unpack_vector_int128: {
17676 ConstantInt *Index = cast<ConstantInt>(Op1);
17682 ConstantInt::get(Index->getIntegerType(), 1 - Index->getZExtValue());
17684 return Builder.CreateExtractElement(Unpacked, Index);
17687 case PPC::BI__builtin_ppc_sthcx: {
17691 return Builder.CreateCall(F, {Op0, Op1});
17700#define CUSTOM_BUILTIN(Name, Intr, Types, Accumulate, Feature) \
17701 case PPC::BI__builtin_##Name:
17702#include "clang/Basic/BuiltinsPPC.def"
17705 for (
unsigned i = 0, e = E->
getNumArgs(); i != e; i++)
17715 if (BuiltinID == PPC::BI__builtin_mma_disassemble_acc ||
17716 BuiltinID == PPC::BI__builtin_vsx_disassemble_pair ||
17717 BuiltinID == PPC::BI__builtin_mma_disassemble_pair) {
17718 unsigned NumVecs = 2;
17719 auto Intrinsic = Intrinsic::ppc_vsx_disassemble_pair;
17720 if (BuiltinID == PPC::BI__builtin_mma_disassemble_acc) {
17722 Intrinsic = Intrinsic::ppc_mma_disassemble_acc;
17728 llvm::Type *VTy = llvm::FixedVectorType::get(
Int8Ty, 16);
17729 Value *Ptr = Ops[0];
17730 for (
unsigned i=0; i<NumVecs; i++) {
17732 llvm::ConstantInt* Index = llvm::ConstantInt::get(
IntTy, i);
17738 if (BuiltinID == PPC::BI__builtin_vsx_build_pair ||
17739 BuiltinID == PPC::BI__builtin_mma_build_acc) {
17747 std::reverse(Ops.begin() + 1, Ops.end());
17750 switch (BuiltinID) {
17751 #define CUSTOM_BUILTIN(Name, Intr, Types, Acc, Feature) \
17752 case PPC::BI__builtin_##Name: \
17753 ID = Intrinsic::ppc_##Intr; \
17754 Accumulate = Acc; \
17756 #include "clang/Basic/BuiltinsPPC.def"
17758 if (BuiltinID == PPC::BI__builtin_vsx_lxvp ||
17759 BuiltinID == PPC::BI__builtin_vsx_stxvp ||
17760 BuiltinID == PPC::BI__builtin_mma_lxvp ||
17761 BuiltinID == PPC::BI__builtin_mma_stxvp) {
17762 if (BuiltinID == PPC::BI__builtin_vsx_lxvp ||
17763 BuiltinID == PPC::BI__builtin_mma_lxvp) {
17770 return Builder.CreateCall(F, Ops,
"");
17776 CallOps.push_back(Acc);
17778 for (
unsigned i=1; i<Ops.size(); i++)
17779 CallOps.push_back(Ops[i]);
17785 case PPC::BI__builtin_ppc_compare_and_swap:
17786 case PPC::BI__builtin_ppc_compare_and_swaplp: {
17795 llvm::AtomicOrdering::Monotonic, llvm::AtomicOrdering::Monotonic,
true);
17803 Value *LoadedVal = Pair.first.getScalarVal();
17807 case PPC::BI__builtin_ppc_fetch_and_add:
17808 case PPC::BI__builtin_ppc_fetch_and_addlp: {
17810 llvm::AtomicOrdering::Monotonic);
17812 case PPC::BI__builtin_ppc_fetch_and_and:
17813 case PPC::BI__builtin_ppc_fetch_and_andlp: {
17815 llvm::AtomicOrdering::Monotonic);
17818 case PPC::BI__builtin_ppc_fetch_and_or:
17819 case PPC::BI__builtin_ppc_fetch_and_orlp: {
17821 llvm::AtomicOrdering::Monotonic);
17823 case PPC::BI__builtin_ppc_fetch_and_swap:
17824 case PPC::BI__builtin_ppc_fetch_and_swaplp: {
17826 llvm::AtomicOrdering::Monotonic);
17828 case PPC::BI__builtin_ppc_ldarx:
17829 case PPC::BI__builtin_ppc_lwarx:
17830 case PPC::BI__builtin_ppc_lharx:
17831 case PPC::BI__builtin_ppc_lbarx:
17833 case PPC::BI__builtin_ppc_mfspr: {
17839 return Builder.CreateCall(F, {Op0});
17841 case PPC::BI__builtin_ppc_mtspr: {
17848 return Builder.CreateCall(F, {Op0, Op1});
17850 case PPC::BI__builtin_ppc_popcntb: {
17852 llvm::Type *ArgType = ArgValue->
getType();
17854 return Builder.CreateCall(F, {ArgValue},
"popcntb");
17856 case PPC::BI__builtin_ppc_mtfsf: {
17866 case PPC::BI__builtin_ppc_swdiv_nochk:
17867 case PPC::BI__builtin_ppc_swdivs_nochk: {
17870 FastMathFlags FMF =
Builder.getFastMathFlags();
17871 Builder.getFastMathFlags().setFast();
17872 Value *FDiv =
Builder.CreateFDiv(Op0, Op1,
"swdiv_nochk");
17873 Builder.getFastMathFlags() &= (FMF);
17876 case PPC::BI__builtin_ppc_fric:
17878 *
this, E, Intrinsic::rint,
17879 Intrinsic::experimental_constrained_rint))
17881 case PPC::BI__builtin_ppc_frim:
17882 case PPC::BI__builtin_ppc_frims:
17884 *
this, E, Intrinsic::floor,
17885 Intrinsic::experimental_constrained_floor))
17887 case PPC::BI__builtin_ppc_frin:
17888 case PPC::BI__builtin_ppc_frins:
17890 *
this, E, Intrinsic::round,
17891 Intrinsic::experimental_constrained_round))
17893 case PPC::BI__builtin_ppc_frip:
17894 case PPC::BI__builtin_ppc_frips:
17896 *
this, E, Intrinsic::ceil,
17897 Intrinsic::experimental_constrained_ceil))
17899 case PPC::BI__builtin_ppc_friz:
17900 case PPC::BI__builtin_ppc_frizs:
17902 *
this, E, Intrinsic::trunc,
17903 Intrinsic::experimental_constrained_trunc))
17905 case PPC::BI__builtin_ppc_fsqrt:
17906 case PPC::BI__builtin_ppc_fsqrts:
17908 *
this, E, Intrinsic::sqrt,
17909 Intrinsic::experimental_constrained_sqrt))
17911 case PPC::BI__builtin_ppc_test_data_class: {
17916 {Op0, Op1},
"test_data_class");
17918 case PPC::BI__builtin_ppc_maxfe: {
17924 {Op0, Op1, Op2, Op3});
17926 case PPC::BI__builtin_ppc_maxfl: {
17932 {Op0, Op1, Op2, Op3});
17934 case PPC::BI__builtin_ppc_maxfs: {
17940 {Op0, Op1, Op2, Op3});
17942 case PPC::BI__builtin_ppc_minfe: {
17948 {Op0, Op1, Op2, Op3});
17950 case PPC::BI__builtin_ppc_minfl: {
17956 {Op0, Op1, Op2, Op3});
17958 case PPC::BI__builtin_ppc_minfs: {
17964 {Op0, Op1, Op2, Op3});
17966 case PPC::BI__builtin_ppc_swdiv:
17967 case PPC::BI__builtin_ppc_swdivs: {
17970 return Builder.CreateFDiv(Op0, Op1,
"swdiv");
17972 case PPC::BI__builtin_ppc_set_fpscr_rn:
17974 {EmitScalarExpr(E->getArg(0))});
17975 case PPC::BI__builtin_ppc_mffs:
17988 Attribute::getWithDereferenceableBytes(
Call->getContext(), 64));
17989 Call->addRetAttr(Attribute::getWithAlignment(
Call->getContext(), Align(4)));
17993 auto *RetTy = cast<llvm::PointerType>(CGF.
ConvertType(BuiltinRetType));
17994 if (RetTy ==
Call->getType())
18003 Attribute::getWithDereferenceableBytes(
Call->getContext(), 256));
18004 Call->addRetAttr(Attribute::getWithAlignment(
Call->getContext(), Align(8)));
18019 llvm::LoadInst *LD;
18023 if (Cov == CodeObjectVersionKind::COV_None) {
18024 StringRef Name =
"__oclc_ABI_version";
18025 auto *ABIVersionC = CGF.
CGM.
getModule().getNamedGlobal(Name);
18027 ABIVersionC =
new llvm::GlobalVariable(
18029 llvm::GlobalValue::ExternalLinkage,
nullptr, Name,
nullptr,
18030 llvm::GlobalVariable::NotThreadLocal,
18041 llvm::ConstantInt::get(CGF.
Int32Ty, CodeObjectVersionKind::COV_5));
18045 CGF.
Int8Ty, EmitAMDGPUImplicitArgPtr(CGF), 12 + Index * 2);
18049 CGF.
Int8Ty, EmitAMDGPUDispatchPtr(CGF), 4 + Index * 2);
18051 auto Result = CGF.
Builder.CreateSelect(IsCOV5, ImplicitGEP, DispatchGEP);
18055 Value *GEP =
nullptr;
18056 if (Cov >= CodeObjectVersionKind::COV_5) {
18058 GEP = CGF.
Builder.CreateConstGEP1_32(
18059 CGF.
Int8Ty, EmitAMDGPUImplicitArgPtr(CGF), 12 + Index * 2);
18062 GEP = CGF.
Builder.CreateConstGEP1_32(
18063 CGF.
Int8Ty, EmitAMDGPUDispatchPtr(CGF), 4 + Index * 2);
18070 llvm::MDNode *RNode = MDHelper.createRange(
APInt(16, 1),
18072 LD->setMetadata(llvm::LLVMContext::MD_range, RNode);
18073 LD->setMetadata(llvm::LLVMContext::MD_noundef,
18075 LD->setMetadata(llvm::LLVMContext::MD_invariant_load,
18082 const unsigned XOffset = 12;
18083 auto *DP = EmitAMDGPUDispatchPtr(CGF);
18085 auto *Offset = llvm::ConstantInt::get(CGF.
Int32Ty, XOffset + Index * 4);
18089 LD->setMetadata(llvm::LLVMContext::MD_invariant_load,
18102 llvm::AtomicOrdering &AO,
18103 llvm::SyncScope::ID &SSID) {
18104 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
18107 assert(llvm::isValidAtomicOrderingCABI(ord));
18108 switch (
static_cast<llvm::AtomicOrderingCABI
>(ord)) {
18109 case llvm::AtomicOrderingCABI::acquire:
18110 case llvm::AtomicOrderingCABI::consume:
18111 AO = llvm::AtomicOrdering::Acquire;
18113 case llvm::AtomicOrderingCABI::release:
18114 AO = llvm::AtomicOrdering::Release;
18116 case llvm::AtomicOrderingCABI::acq_rel:
18117 AO = llvm::AtomicOrdering::AcquireRelease;
18119 case llvm::AtomicOrderingCABI::seq_cst:
18120 AO = llvm::AtomicOrdering::SequentiallyConsistent;
18122 case llvm::AtomicOrderingCABI::relaxed:
18123 AO = llvm::AtomicOrdering::Monotonic;
18128 llvm::getConstantStringInfo(
Scope, scp);
18135 llvm::Value *Arg =
nullptr;
18136 if ((ICEArguments & (1 << Idx)) == 0) {
18141 std::optional<llvm::APSInt>
Result =
18143 assert(
Result &&
"Expected argument to be a constant");
18151 switch (elementCount) {
18153 return Intrinsic::dx_dot2;
18155 return Intrinsic::dx_dot3;
18157 return Intrinsic::dx_dot4;
18161 return Intrinsic::dx_sdot;
18164 return Intrinsic::dx_udot;
18172 switch (BuiltinID) {
18173 case Builtin::BI__builtin_hlsl_elementwise_all: {
18175 return Builder.CreateIntrinsic(
18180 case Builtin::BI__builtin_hlsl_elementwise_any: {
18182 return Builder.CreateIntrinsic(
18187 case Builtin::BI__builtin_hlsl_elementwise_clamp: {
18193 bool IsUnsigned =
false;
18195 Ty = VecTy->getElementType();
18197 return Builder.CreateIntrinsic(
18199 IsUnsigned ? Intrinsic::dx_uclamp : Intrinsic::dx_clamp,
18202 case Builtin::BI__builtin_hlsl_dot: {
18205 llvm::Type *T0 = Op0->
getType();
18206 llvm::Type *T1 = Op1->
getType();
18207 if (!T0->isVectorTy() && !T1->isVectorTy()) {
18208 if (T0->isFloatingPointTy())
18209 return Builder.CreateFMul(Op0, Op1,
"dx.dot");
18211 if (T0->isIntegerTy())
18212 return Builder.CreateMul(Op0, Op1,
"dx.dot");
18216 "Scalar dot product is only supported on ints and floats.");
18219 assert(T0->isVectorTy() && T1->isVectorTy() &&
18220 "Dot product of vector and scalar is not supported.");
18223 assert(T0->getScalarType() == T1->getScalarType() &&
18224 "Dot product of vectors need the same element types.");
18227 [[maybe_unused]]
auto *VecTy1 =
18231 "Dot product requires vectors to be of the same size.");
18233 return Builder.CreateIntrinsic(
18234 T0->getScalarType(),
18236 VecTy0->getNumElements()),
18239 case Builtin::BI__builtin_hlsl_lerp: {
18244 llvm_unreachable(
"lerp operand must have a float representation");
18245 return Builder.CreateIntrinsic(
18249 case Builtin::BI__builtin_hlsl_elementwise_frac: {
18252 llvm_unreachable(
"frac operand must have a float representation");
18253 return Builder.CreateIntrinsic(
18254 Op0->
getType(), Intrinsic::dx_frac,
18257 case Builtin::BI__builtin_hlsl_elementwise_isinf: {
18259 llvm::Type *Xty = Op0->
getType();
18260 llvm::Type *retType = llvm::Type::getInt1Ty(this->
getLLVMContext());
18261 if (Xty->isVectorTy()) {
18263 retType = llvm::VectorType::get(
18264 retType, ElementCount::getFixed(XVecTy->getNumElements()));
18267 llvm_unreachable(
"isinf operand must have a float representation");
18268 return Builder.CreateIntrinsic(retType, Intrinsic::dx_isinf,
18271 case Builtin::BI__builtin_hlsl_mad: {
18276 return Builder.CreateIntrinsic(
18277 M->
getType(), Intrinsic::fmuladd,
18282 return Builder.CreateIntrinsic(
18283 M->
getType(), Intrinsic::dx_imad,
18287 return Builder.CreateNSWAdd(Mul, B);
18291 return Builder.CreateIntrinsic(
18292 M->
getType(), Intrinsic::dx_umad,
18296 return Builder.CreateNUWAdd(Mul, B);
18298 case Builtin::BI__builtin_hlsl_elementwise_rcp: {
18301 llvm_unreachable(
"rcp operand must have a float representation");
18302 llvm::Type *Ty = Op0->
getType();
18303 llvm::Type *EltTy = Ty->getScalarType();
18306 ? ConstantVector::getSplat(
18307 ElementCount::getFixed(
18308 dyn_cast<FixedVectorType>(Ty)->getNumElements()),
18309 ConstantFP::get(EltTy, 1.0))
18310 : ConstantFP::get(EltTy, 1.0);
18311 return Builder.CreateFDiv(One, Op0,
"hlsl.rcp");
18313 case Builtin::BI__builtin_hlsl_elementwise_rsqrt: {
18316 llvm_unreachable(
"rsqrt operand must have a float representation");
18317 return Builder.CreateIntrinsic(
18318 Op0->
getType(), Intrinsic::dx_rsqrt,
18321 case Builtin::BI__builtin_hlsl_wave_get_lane_index: {
18323 llvm::FunctionType::get(
IntTy, {},
false),
"__hlsl_wave_get_lane_index",
18332 llvm::AtomicOrdering AO = llvm::AtomicOrdering::SequentiallyConsistent;
18333 llvm::SyncScope::ID SSID;
18334 switch (BuiltinID) {
18335 case AMDGPU::BI__builtin_amdgcn_div_scale:
18336 case AMDGPU::BI__builtin_amdgcn_div_scalef: {
18349 llvm::Value *Tmp =
Builder.CreateCall(Callee, {
X, Y, Z});
18352 llvm::Value *Flag =
Builder.CreateExtractValue(Tmp, 1);
18356 llvm::Value *FlagExt =
Builder.CreateZExt(Flag, RealFlagType);
18360 case AMDGPU::BI__builtin_amdgcn_div_fmas:
18361 case AMDGPU::BI__builtin_amdgcn_div_fmasf: {
18369 llvm::Value *Src3ToBool =
Builder.CreateIsNotNull(Src3);
18370 return Builder.CreateCall(F, {Src0, Src1, Src2, Src3ToBool});
18373 case AMDGPU::BI__builtin_amdgcn_ds_swizzle:
18375 case AMDGPU::BI__builtin_amdgcn_mov_dpp8:
18377 case AMDGPU::BI__builtin_amdgcn_mov_dpp:
18378 case AMDGPU::BI__builtin_amdgcn_update_dpp: {
18382 unsigned ICEArguments = 0;
18386 for (
unsigned I = 0; I != E->
getNumArgs(); ++I) {
18389 assert(Args.size() == 5 || Args.size() == 6);
18390 if (Args.size() == 5)
18391 Args.insert(Args.begin(), llvm::PoisonValue::get(Args[0]->getType()));
18394 return Builder.CreateCall(F, Args);
18396 case AMDGPU::BI__builtin_amdgcn_div_fixup:
18397 case AMDGPU::BI__builtin_amdgcn_div_fixupf:
18398 case AMDGPU::BI__builtin_amdgcn_div_fixuph:
18400 case AMDGPU::BI__builtin_amdgcn_trig_preop:
18401 case AMDGPU::BI__builtin_amdgcn_trig_preopf:
18403 case AMDGPU::BI__builtin_amdgcn_rcp:
18404 case AMDGPU::BI__builtin_amdgcn_rcpf:
18405 case AMDGPU::BI__builtin_amdgcn_rcph:
18407 case AMDGPU::BI__builtin_amdgcn_sqrt:
18408 case AMDGPU::BI__builtin_amdgcn_sqrtf:
18409 case AMDGPU::BI__builtin_amdgcn_sqrth:
18411 case AMDGPU::BI__builtin_amdgcn_rsq:
18412 case AMDGPU::BI__builtin_amdgcn_rsqf:
18413 case AMDGPU::BI__builtin_amdgcn_rsqh:
18415 case AMDGPU::BI__builtin_amdgcn_rsq_clamp:
18416 case AMDGPU::BI__builtin_amdgcn_rsq_clampf:
18418 case AMDGPU::BI__builtin_amdgcn_sinf:
18419 case AMDGPU::BI__builtin_amdgcn_sinh:
18421 case AMDGPU::BI__builtin_amdgcn_cosf:
18422 case AMDGPU::BI__builtin_amdgcn_cosh:
18424 case AMDGPU::BI__builtin_amdgcn_dispatch_ptr:
18425 return EmitAMDGPUDispatchPtr(*
this, E);
18426 case AMDGPU::BI__builtin_amdgcn_logf:
18428 case AMDGPU::BI__builtin_amdgcn_exp2f:
18430 case AMDGPU::BI__builtin_amdgcn_log_clampf:
18432 case AMDGPU::BI__builtin_amdgcn_ldexp:
18433 case AMDGPU::BI__builtin_amdgcn_ldexpf: {
18436 llvm::Function *F =
18437 CGM.
getIntrinsic(Intrinsic::ldexp, {Src0->getType(), Src1->getType()});
18438 return Builder.CreateCall(F, {Src0, Src1});
18440 case AMDGPU::BI__builtin_amdgcn_ldexph: {
18445 llvm::Function *F =
18449 case AMDGPU::BI__builtin_amdgcn_frexp_mant:
18450 case AMDGPU::BI__builtin_amdgcn_frexp_mantf:
18451 case AMDGPU::BI__builtin_amdgcn_frexp_manth:
18453 case AMDGPU::BI__builtin_amdgcn_frexp_exp:
18454 case AMDGPU::BI__builtin_amdgcn_frexp_expf: {
18458 return Builder.CreateCall(F, Src0);
18460 case AMDGPU::BI__builtin_amdgcn_frexp_exph: {
18464 return Builder.CreateCall(F, Src0);
18466 case AMDGPU::BI__builtin_amdgcn_fract:
18467 case AMDGPU::BI__builtin_amdgcn_fractf:
18468 case AMDGPU::BI__builtin_amdgcn_fracth:
18470 case AMDGPU::BI__builtin_amdgcn_lerp:
18472 case AMDGPU::BI__builtin_amdgcn_ubfe:
18474 case AMDGPU::BI__builtin_amdgcn_sbfe:
18476 case AMDGPU::BI__builtin_amdgcn_ballot_w32:
18477 case AMDGPU::BI__builtin_amdgcn_ballot_w64: {
18481 return Builder.CreateCall(F, { Src });
18483 case AMDGPU::BI__builtin_amdgcn_uicmp:
18484 case AMDGPU::BI__builtin_amdgcn_uicmpl:
18485 case AMDGPU::BI__builtin_amdgcn_sicmp:
18486 case AMDGPU::BI__builtin_amdgcn_sicmpl: {
18493 {
Builder.getInt64Ty(), Src0->getType() });
18494 return Builder.CreateCall(F, { Src0, Src1, Src2 });
18496 case AMDGPU::BI__builtin_amdgcn_fcmp:
18497 case AMDGPU::BI__builtin_amdgcn_fcmpf: {
18504 {
Builder.getInt64Ty(), Src0->getType() });
18505 return Builder.CreateCall(F, { Src0, Src1, Src2 });
18507 case AMDGPU::BI__builtin_amdgcn_class:
18508 case AMDGPU::BI__builtin_amdgcn_classf:
18509 case AMDGPU::BI__builtin_amdgcn_classh:
18511 case AMDGPU::BI__builtin_amdgcn_fmed3f:
18512 case AMDGPU::BI__builtin_amdgcn_fmed3h:
18514 case AMDGPU::BI__builtin_amdgcn_ds_append:
18515 case AMDGPU::BI__builtin_amdgcn_ds_consume: {
18516 Intrinsic::ID Intrin = BuiltinID == AMDGPU::BI__builtin_amdgcn_ds_append ?
18517 Intrinsic::amdgcn_ds_append : Intrinsic::amdgcn_ds_consume;
18522 case AMDGPU::BI__builtin_amdgcn_ds_faddf:
18523 case AMDGPU::BI__builtin_amdgcn_ds_fminf:
18524 case AMDGPU::BI__builtin_amdgcn_ds_fmaxf: {
18525 Intrinsic::ID Intrin;
18526 switch (BuiltinID) {
18527 case AMDGPU::BI__builtin_amdgcn_ds_faddf:
18528 Intrin = Intrinsic::amdgcn_ds_fadd;
18530 case AMDGPU::BI__builtin_amdgcn_ds_fminf:
18531 Intrin = Intrinsic::amdgcn_ds_fmin;
18533 case AMDGPU::BI__builtin_amdgcn_ds_fmaxf:
18534 Intrin = Intrinsic::amdgcn_ds_fmax;
18543 llvm::FunctionType *FTy = F->getFunctionType();
18544 llvm::Type *PTy = FTy->getParamType(0);
18546 return Builder.CreateCall(F, { Src0, Src1, Src2, Src3, Src4 });
18548 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_f64:
18549 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_f32:
18550 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_v2f16:
18551 case AMDGPU::BI__builtin_amdgcn_global_atomic_fmin_f64:
18552 case AMDGPU::BI__builtin_amdgcn_global_atomic_fmax_f64:
18553 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_f64:
18554 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fmin_f64:
18555 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fmax_f64:
18556 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_f32:
18557 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_v2f16: {
18560 switch (BuiltinID) {
18561 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_f32:
18563 IID = Intrinsic::amdgcn_global_atomic_fadd;
18565 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_v2f16:
18566 ArgTy = llvm::FixedVectorType::get(
18568 IID = Intrinsic::amdgcn_global_atomic_fadd;
18570 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_f64:
18571 IID = Intrinsic::amdgcn_global_atomic_fadd;
18573 case AMDGPU::BI__builtin_amdgcn_global_atomic_fmin_f64:
18574 IID = Intrinsic::amdgcn_global_atomic_fmin;
18576 case AMDGPU::BI__builtin_amdgcn_global_atomic_fmax_f64:
18577 IID = Intrinsic::amdgcn_global_atomic_fmax;
18579 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_f64:
18580 IID = Intrinsic::amdgcn_flat_atomic_fadd;
18582 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fmin_f64:
18583 IID = Intrinsic::amdgcn_flat_atomic_fmin;
18585 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fmax_f64:
18586 IID = Intrinsic::amdgcn_flat_atomic_fmax;
18588 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_f32:
18590 IID = Intrinsic::amdgcn_flat_atomic_fadd;
18592 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_v2f16:
18593 ArgTy = llvm::FixedVectorType::get(
18595 IID = Intrinsic::amdgcn_flat_atomic_fadd;
18600 llvm::Function *F =
18602 return Builder.CreateCall(F, {Addr, Val});
18604 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_v2bf16:
18605 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_v2bf16: {
18607 switch (BuiltinID) {
18608 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_v2bf16:
18609 IID = Intrinsic::amdgcn_global_atomic_fadd_v2bf16;
18611 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_v2bf16:
18612 IID = Intrinsic::amdgcn_flat_atomic_fadd_v2bf16;
18618 return Builder.CreateCall(F, {Addr, Val});
18620 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_f64:
18621 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_f32:
18622 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_v2f16: {
18625 switch (BuiltinID) {
18626 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_f32:
18628 IID = Intrinsic::amdgcn_ds_fadd;
18630 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_f64:
18632 IID = Intrinsic::amdgcn_ds_fadd;
18634 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_v2f16:
18635 ArgTy = llvm::FixedVectorType::get(
18637 IID = Intrinsic::amdgcn_ds_fadd;
18642 llvm::Constant *ZeroI32 = llvm::ConstantInt::getIntegerValue(
18644 llvm::Constant *ZeroI1 = llvm::ConstantInt::getIntegerValue(
18647 return Builder.CreateCall(F, {Addr, Val, ZeroI32, ZeroI32, ZeroI1});
18649 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b64_i32:
18650 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b64_v2i32:
18651 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4i16:
18652 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8i16: {
18655 switch (BuiltinID) {
18656 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b64_i32:
18657 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b64_v2i32:
18658 IID = Intrinsic::amdgcn_global_load_tr_b64;
18660 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4i16:
18661 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8i16:
18662 IID = Intrinsic::amdgcn_global_load_tr_b128;
18668 return Builder.CreateCall(F, {Addr});
18670 case AMDGPU::BI__builtin_amdgcn_get_fpenv: {
18673 return Builder.CreateCall(F);
18675 case AMDGPU::BI__builtin_amdgcn_set_fpenv: {
18681 case AMDGPU::BI__builtin_amdgcn_read_exec:
18683 case AMDGPU::BI__builtin_amdgcn_read_exec_lo:
18685 case AMDGPU::BI__builtin_amdgcn_read_exec_hi:
18687 case AMDGPU::BI__builtin_amdgcn_image_bvh_intersect_ray:
18688 case AMDGPU::BI__builtin_amdgcn_image_bvh_intersect_ray_h:
18689 case AMDGPU::BI__builtin_amdgcn_image_bvh_intersect_ray_l:
18690 case AMDGPU::BI__builtin_amdgcn_image_bvh_intersect_ray_lh: {
18700 RayOrigin =
Builder.CreateShuffleVector(RayOrigin, RayOrigin,
18704 RayInverseDir =
Builder.CreateShuffleVector(RayInverseDir, RayInverseDir,
18708 {NodePtr->getType(), RayDir->getType()});
18709 return Builder.CreateCall(F, {NodePtr, RayExtent, RayOrigin, RayDir,
18710 RayInverseDir, TextureDescr});
18713 case AMDGPU::BI__builtin_amdgcn_ds_bvh_stack_rtn: {
18715 for (
int i = 0, e = E->
getNumArgs(); i != e; ++i)
18723 Value *I0 =
Builder.CreateInsertElement(PoisonValue::get(RetTy), Rtn,
18725 return Builder.CreateInsertElement(I0, A, 1);
18728 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w32:
18729 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_tied_w32:
18730 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w64:
18731 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_tied_w64:
18732 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w32:
18733 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_tied_w32:
18734 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w64:
18735 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_tied_w64:
18736 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w32:
18737 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w64:
18738 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w32:
18739 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w64:
18740 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w32:
18741 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w64:
18742 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w32:
18743 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w64:
18744 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w32_gfx12:
18745 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w64_gfx12:
18746 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w32_gfx12:
18747 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w64_gfx12:
18748 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w32_gfx12:
18749 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w64_gfx12:
18750 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w32_gfx12:
18751 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w64_gfx12:
18752 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w32_gfx12:
18753 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w64_gfx12:
18754 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w32_gfx12:
18755 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w64_gfx12:
18756 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_fp8_w32_gfx12:
18757 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_fp8_w64_gfx12:
18758 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_bf8_w32_gfx12:
18759 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_bf8_w64_gfx12:
18760 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_fp8_w32_gfx12:
18761 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_fp8_w64_gfx12:
18762 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_bf8_w32_gfx12:
18763 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_bf8_w64_gfx12:
18764 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x32_iu4_w32_gfx12:
18765 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x32_iu4_w64_gfx12:
18766 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_f16_w32:
18767 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_f16_w64:
18768 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf16_w32:
18769 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf16_w64:
18770 case AMDGPU::BI__builtin_amdgcn_swmmac_f16_16x16x32_f16_w32:
18771 case AMDGPU::BI__builtin_amdgcn_swmmac_f16_16x16x32_f16_w64:
18772 case AMDGPU::BI__builtin_amdgcn_swmmac_bf16_16x16x32_bf16_w32:
18773 case AMDGPU::BI__builtin_amdgcn_swmmac_bf16_16x16x32_bf16_w64:
18774 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu8_w32:
18775 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu8_w64:
18776 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu4_w32:
18777 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu4_w64:
18778 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x64_iu4_w32:
18779 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x64_iu4_w64:
18780 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_fp8_w32:
18781 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_fp8_w64:
18782 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_bf8_w32:
18783 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_bf8_w64:
18784 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_fp8_w32:
18785 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_fp8_w64:
18786 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_bf8_w32:
18787 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_bf8_w64: {
18800 bool AppendFalseForOpselArg =
false;
18801 unsigned BuiltinWMMAOp;
18803 switch (BuiltinID) {
18804 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w32:
18805 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w64:
18806 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w32_gfx12:
18807 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w64_gfx12:
18808 ArgsForMatchingMatrixTypes = {2, 0};
18809 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x16_f16;
18811 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w32:
18812 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w64:
18813 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w32_gfx12:
18814 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w64_gfx12:
18815 ArgsForMatchingMatrixTypes = {2, 0};
18816 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x16_bf16;
18818 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w32_gfx12:
18819 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w64_gfx12:
18820 AppendFalseForOpselArg =
true;
18822 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w32:
18823 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w64:
18824 ArgsForMatchingMatrixTypes = {2, 0};
18825 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f16_16x16x16_f16;
18827 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w32_gfx12:
18828 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w64_gfx12:
18829 AppendFalseForOpselArg =
true;
18831 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w32:
18832 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w64:
18833 ArgsForMatchingMatrixTypes = {2, 0};
18834 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_bf16_16x16x16_bf16;
18836 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_tied_w32:
18837 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_tied_w64:
18838 ArgsForMatchingMatrixTypes = {2, 0};
18839 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f16_16x16x16_f16_tied;
18841 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_tied_w32:
18842 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_tied_w64:
18843 ArgsForMatchingMatrixTypes = {2, 0};
18844 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_bf16_16x16x16_bf16_tied;
18846 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w32:
18847 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w64:
18848 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w32_gfx12:
18849 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w64_gfx12:
18850 ArgsForMatchingMatrixTypes = {4, 1};
18851 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_i32_16x16x16_iu8;
18853 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w32:
18854 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w64:
18855 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w32_gfx12:
18856 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w64_gfx12:
18857 ArgsForMatchingMatrixTypes = {4, 1};
18858 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_i32_16x16x16_iu4;
18860 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_fp8_w32_gfx12:
18861 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_fp8_w64_gfx12:
18862 ArgsForMatchingMatrixTypes = {2, 0};
18863 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x16_fp8_fp8;
18865 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_bf8_w32_gfx12:
18866 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_bf8_w64_gfx12:
18867 ArgsForMatchingMatrixTypes = {2, 0};
18868 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x16_fp8_bf8;
18870 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_fp8_w32_gfx12:
18871 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_fp8_w64_gfx12:
18872 ArgsForMatchingMatrixTypes = {2, 0};
18873 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x16_bf8_fp8;
18875 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_bf8_w32_gfx12:
18876 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_bf8_w64_gfx12:
18877 ArgsForMatchingMatrixTypes = {2, 0};
18878 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x16_bf8_bf8;
18880 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x32_iu4_w32_gfx12:
18881 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x32_iu4_w64_gfx12:
18882 ArgsForMatchingMatrixTypes = {4, 1};
18883 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_i32_16x16x32_iu4;
18885 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_f16_w32:
18886 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_f16_w64:
18887 ArgsForMatchingMatrixTypes = {2, 0, 1, 3};
18888 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f32_16x16x32_f16;
18890 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf16_w32:
18891 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf16_w64:
18892 ArgsForMatchingMatrixTypes = {2, 0, 1, 3};
18893 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f32_16x16x32_bf16;
18895 case AMDGPU::BI__builtin_amdgcn_swmmac_f16_16x16x32_f16_w32:
18896 case AMDGPU::BI__builtin_amdgcn_swmmac_f16_16x16x32_f16_w64:
18897 ArgsForMatchingMatrixTypes = {2, 0, 1, 3};
18898 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f16_16x16x32_f16;
18900 case AMDGPU::BI__builtin_amdgcn_swmmac_bf16_16x16x32_bf16_w32:
18901 case AMDGPU::BI__builtin_amdgcn_swmmac_bf16_16x16x32_bf16_w64:
18902 ArgsForMatchingMatrixTypes = {2, 0, 1, 3};
18903 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_bf16_16x16x32_bf16;
18905 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu8_w32:
18906 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu8_w64:
18907 ArgsForMatchingMatrixTypes = {4, 1, 3, 5};
18908 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_i32_16x16x32_iu8;
18910 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu4_w32:
18911 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu4_w64:
18912 ArgsForMatchingMatrixTypes = {4, 1, 3, 5};
18913 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_i32_16x16x32_iu4;
18915 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x64_iu4_w32:
18916 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x64_iu4_w64:
18917 ArgsForMatchingMatrixTypes = {4, 1, 3, 5};
18918 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_i32_16x16x64_iu4;
18920 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_fp8_w32:
18921 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_fp8_w64:
18922 ArgsForMatchingMatrixTypes = {2, 0, 1, 3};
18923 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f32_16x16x32_fp8_fp8;
18925 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_bf8_w32:
18926 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_bf8_w64:
18927 ArgsForMatchingMatrixTypes = {2, 0, 1, 3};
18928 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f32_16x16x32_fp8_bf8;
18930 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_fp8_w32:
18931 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_fp8_w64:
18932 ArgsForMatchingMatrixTypes = {2, 0, 1, 3};
18933 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f32_16x16x32_bf8_fp8;
18935 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_bf8_w32:
18936 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_bf8_w64:
18937 ArgsForMatchingMatrixTypes = {2, 0, 1, 3};
18938 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f32_16x16x32_bf8_bf8;
18943 for (
int i = 0, e = E->
getNumArgs(); i != e; ++i)
18945 if (AppendFalseForOpselArg)
18946 Args.push_back(
Builder.getFalse());
18949 for (
auto ArgIdx : ArgsForMatchingMatrixTypes)
18950 ArgTypes.push_back(Args[ArgIdx]->getType());
18953 return Builder.CreateCall(F, Args);
18957 case AMDGPU::BI__builtin_amdgcn_workitem_id_x:
18959 case AMDGPU::BI__builtin_amdgcn_workitem_id_y:
18961 case AMDGPU::BI__builtin_amdgcn_workitem_id_z:
18965 case AMDGPU::BI__builtin_amdgcn_workgroup_size_x:
18966 return EmitAMDGPUWorkGroupSize(*
this, 0);
18967 case AMDGPU::BI__builtin_amdgcn_workgroup_size_y:
18968 return EmitAMDGPUWorkGroupSize(*
this, 1);
18969 case AMDGPU::BI__builtin_amdgcn_workgroup_size_z:
18970 return EmitAMDGPUWorkGroupSize(*
this, 2);
18973 case AMDGPU::BI__builtin_amdgcn_grid_size_x:
18974 return EmitAMDGPUGridSize(*
this, 0);
18975 case AMDGPU::BI__builtin_amdgcn_grid_size_y:
18976 return EmitAMDGPUGridSize(*
this, 1);
18977 case AMDGPU::BI__builtin_amdgcn_grid_size_z:
18978 return EmitAMDGPUGridSize(*
this, 2);
18981 case AMDGPU::BI__builtin_r600_recipsqrt_ieee:
18982 case AMDGPU::BI__builtin_r600_recipsqrt_ieeef:
18984 case AMDGPU::BI__builtin_r600_read_tidig_x:
18986 case AMDGPU::BI__builtin_r600_read_tidig_y:
18988 case AMDGPU::BI__builtin_r600_read_tidig_z:
18990 case AMDGPU::BI__builtin_amdgcn_alignbit: {
18995 return Builder.CreateCall(F, { Src0, Src1, Src2 });
18997 case AMDGPU::BI__builtin_amdgcn_fence: {
19000 return Builder.CreateFence(AO, SSID);
19002 case AMDGPU::BI__builtin_amdgcn_atomic_inc32:
19003 case AMDGPU::BI__builtin_amdgcn_atomic_inc64:
19004 case AMDGPU::BI__builtin_amdgcn_atomic_dec32:
19005 case AMDGPU::BI__builtin_amdgcn_atomic_dec64: {
19006 llvm::AtomicRMWInst::BinOp BinOp;
19007 switch (BuiltinID) {
19008 case AMDGPU::BI__builtin_amdgcn_atomic_inc32:
19009 case AMDGPU::BI__builtin_amdgcn_atomic_inc64:
19010 BinOp = llvm::AtomicRMWInst::UIncWrap;
19012 case AMDGPU::BI__builtin_amdgcn_atomic_dec32:
19013 case AMDGPU::BI__builtin_amdgcn_atomic_dec64:
19014 BinOp = llvm::AtomicRMWInst::UDecWrap;
19028 llvm::AtomicRMWInst *RMW =
19031 RMW->setVolatile(
true);
19034 case AMDGPU::BI__builtin_amdgcn_s_sendmsg_rtn:
19035 case AMDGPU::BI__builtin_amdgcn_s_sendmsg_rtnl: {
19041 return Builder.CreateCall(F, {Arg});
19052 unsigned IntrinsicID,
19056 for (
unsigned I = 0; I < NumArgs; ++I)
19068 switch (BuiltinID) {
19069 case SystemZ::BI__builtin_tbegin: {
19071 Value *Control = llvm::ConstantInt::get(
Int32Ty, 0xff0c);
19073 return Builder.CreateCall(F, {TDB, Control});
19075 case SystemZ::BI__builtin_tbegin_nofloat: {
19077 Value *Control = llvm::ConstantInt::get(
Int32Ty, 0xff0c);
19079 return Builder.CreateCall(F, {TDB, Control});
19081 case SystemZ::BI__builtin_tbeginc: {
19083 Value *Control = llvm::ConstantInt::get(
Int32Ty, 0xff08);
19085 return Builder.CreateCall(F, {TDB, Control});
19087 case SystemZ::BI__builtin_tabort: {
19092 case SystemZ::BI__builtin_non_tx_store: {
19104 case SystemZ::BI__builtin_s390_vpopctb:
19105 case SystemZ::BI__builtin_s390_vpopcth:
19106 case SystemZ::BI__builtin_s390_vpopctf:
19107 case SystemZ::BI__builtin_s390_vpopctg: {
19114 case SystemZ::BI__builtin_s390_vclzb:
19115 case SystemZ::BI__builtin_s390_vclzh:
19116 case SystemZ::BI__builtin_s390_vclzf:
19117 case SystemZ::BI__builtin_s390_vclzg: {
19120 Value *Undef = ConstantInt::get(
Builder.getInt1Ty(),
false);
19122 return Builder.CreateCall(F, {
X, Undef});
19125 case SystemZ::BI__builtin_s390_vctzb:
19126 case SystemZ::BI__builtin_s390_vctzh:
19127 case SystemZ::BI__builtin_s390_vctzf:
19128 case SystemZ::BI__builtin_s390_vctzg: {
19131 Value *Undef = ConstantInt::get(
Builder.getInt1Ty(),
false);
19133 return Builder.CreateCall(F, {
X, Undef});
19136 case SystemZ::BI__builtin_s390_verllb:
19137 case SystemZ::BI__builtin_s390_verllh:
19138 case SystemZ::BI__builtin_s390_verllf:
19139 case SystemZ::BI__builtin_s390_verllg: {
19144 unsigned NumElts = cast<llvm::FixedVectorType>(ResultType)->getNumElements();
19145 Amt =
Builder.CreateIntCast(Amt, ResultType->getScalarType(),
false);
19146 Amt =
Builder.CreateVectorSplat(NumElts, Amt);
19148 return Builder.CreateCall(F, { Src, Src, Amt });
19151 case SystemZ::BI__builtin_s390_verllvb:
19152 case SystemZ::BI__builtin_s390_verllvh:
19153 case SystemZ::BI__builtin_s390_verllvf:
19154 case SystemZ::BI__builtin_s390_verllvg: {
19159 return Builder.CreateCall(F, { Src, Src, Amt });
19162 case SystemZ::BI__builtin_s390_vfsqsb:
19163 case SystemZ::BI__builtin_s390_vfsqdb: {
19166 if (
Builder.getIsFPConstrained()) {
19168 return Builder.CreateConstrainedFPCall(F, {
X });
19174 case SystemZ::BI__builtin_s390_vfmasb:
19175 case SystemZ::BI__builtin_s390_vfmadb: {
19180 if (
Builder.getIsFPConstrained()) {
19182 return Builder.CreateConstrainedFPCall(F, {
X, Y, Z});
19185 return Builder.CreateCall(F, {
X, Y, Z});
19188 case SystemZ::BI__builtin_s390_vfmssb:
19189 case SystemZ::BI__builtin_s390_vfmsdb: {
19194 if (
Builder.getIsFPConstrained()) {
19196 return Builder.CreateConstrainedFPCall(F, {
X, Y,
Builder.CreateFNeg(Z,
"neg")});
19202 case SystemZ::BI__builtin_s390_vfnmasb:
19203 case SystemZ::BI__builtin_s390_vfnmadb: {
19208 if (
Builder.getIsFPConstrained()) {
19210 return Builder.CreateFNeg(
Builder.CreateConstrainedFPCall(F, {X, Y, Z}),
"neg");
19213 return Builder.CreateFNeg(
Builder.CreateCall(F, {X, Y, Z}),
"neg");
19216 case SystemZ::BI__builtin_s390_vfnmssb:
19217 case SystemZ::BI__builtin_s390_vfnmsdb: {
19222 if (
Builder.getIsFPConstrained()) {
19225 return Builder.CreateFNeg(
Builder.CreateConstrainedFPCall(F, {X, Y, NegZ}));
19232 case SystemZ::BI__builtin_s390_vflpsb:
19233 case SystemZ::BI__builtin_s390_vflpdb: {
19239 case SystemZ::BI__builtin_s390_vflnsb:
19240 case SystemZ::BI__builtin_s390_vflndb: {
19246 case SystemZ::BI__builtin_s390_vfisb:
19247 case SystemZ::BI__builtin_s390_vfidb: {
19255 Intrinsic::ID
ID = Intrinsic::not_intrinsic;
19257 switch (M4.getZExtValue()) {
19260 switch (M5.getZExtValue()) {
19262 case 0:
ID = Intrinsic::rint;
19263 CI = Intrinsic::experimental_constrained_rint;
break;
19267 switch (M5.getZExtValue()) {
19269 case 0:
ID = Intrinsic::nearbyint;
19270 CI = Intrinsic::experimental_constrained_nearbyint;
break;
19271 case 1:
ID = Intrinsic::round;
19272 CI = Intrinsic::experimental_constrained_round;
break;
19273 case 5:
ID = Intrinsic::trunc;
19274 CI = Intrinsic::experimental_constrained_trunc;
break;
19275 case 6:
ID = Intrinsic::ceil;
19276 CI = Intrinsic::experimental_constrained_ceil;
break;
19277 case 7:
ID = Intrinsic::floor;
19278 CI = Intrinsic::experimental_constrained_floor;
break;
19282 if (ID != Intrinsic::not_intrinsic) {
19283 if (
Builder.getIsFPConstrained()) {
19285 return Builder.CreateConstrainedFPCall(F,
X);
19291 switch (BuiltinID) {
19292 case SystemZ::BI__builtin_s390_vfisb:
ID = Intrinsic::s390_vfisb;
break;
19293 case SystemZ::BI__builtin_s390_vfidb:
ID = Intrinsic::s390_vfidb;
break;
19294 default: llvm_unreachable(
"Unknown BuiltinID");
19299 return Builder.CreateCall(F, {
X, M4Value, M5Value});
19301 case SystemZ::BI__builtin_s390_vfmaxsb:
19302 case SystemZ::BI__builtin_s390_vfmaxdb: {
19310 Intrinsic::ID
ID = Intrinsic::not_intrinsic;
19312 switch (M4.getZExtValue()) {
19314 case 4:
ID = Intrinsic::maxnum;
19315 CI = Intrinsic::experimental_constrained_maxnum;
break;
19317 if (ID != Intrinsic::not_intrinsic) {
19318 if (
Builder.getIsFPConstrained()) {
19320 return Builder.CreateConstrainedFPCall(F, {
X, Y});
19323 return Builder.CreateCall(F, {
X, Y});
19326 switch (BuiltinID) {
19327 case SystemZ::BI__builtin_s390_vfmaxsb:
ID = Intrinsic::s390_vfmaxsb;
break;
19328 case SystemZ::BI__builtin_s390_vfmaxdb:
ID = Intrinsic::s390_vfmaxdb;
break;
19329 default: llvm_unreachable(
"Unknown BuiltinID");
19333 return Builder.CreateCall(F, {
X, Y, M4Value});
19335 case SystemZ::BI__builtin_s390_vfminsb:
19336 case SystemZ::BI__builtin_s390_vfmindb: {
19344 Intrinsic::ID
ID = Intrinsic::not_intrinsic;
19346 switch (M4.getZExtValue()) {
19348 case 4:
ID = Intrinsic::minnum;
19349 CI = Intrinsic::experimental_constrained_minnum;
break;
19351 if (ID != Intrinsic::not_intrinsic) {
19352 if (
Builder.getIsFPConstrained()) {
19354 return Builder.CreateConstrainedFPCall(F, {
X, Y});
19357 return Builder.CreateCall(F, {
X, Y});
19360 switch (BuiltinID) {
19361 case SystemZ::BI__builtin_s390_vfminsb:
ID = Intrinsic::s390_vfminsb;
break;
19362 case SystemZ::BI__builtin_s390_vfmindb:
ID = Intrinsic::s390_vfmindb;
break;
19363 default: llvm_unreachable(
"Unknown BuiltinID");
19367 return Builder.CreateCall(F, {
X, Y, M4Value});
19370 case SystemZ::BI__builtin_s390_vlbrh:
19371 case SystemZ::BI__builtin_s390_vlbrf:
19372 case SystemZ::BI__builtin_s390_vlbrg: {
19381#define INTRINSIC_WITH_CC(NAME) \
19382 case SystemZ::BI__builtin_##NAME: \
19383 return EmitSystemZIntrinsicWithCC(*this, Intrinsic::NAME, E)
19462#undef INTRINSIC_WITH_CC
19471struct NVPTXMmaLdstInfo {
19472 unsigned NumResults;
19478#define MMA_INTR(geom_op_type, layout) \
19479 Intrinsic::nvvm_wmma_##geom_op_type##_##layout##_stride
19480#define MMA_LDST(n, geom_op_type) \
19481 { n, MMA_INTR(geom_op_type, col), MMA_INTR(geom_op_type, row) }
19483static NVPTXMmaLdstInfo getNVPTXMmaLdstInfo(
unsigned BuiltinID) {
19484 switch (BuiltinID) {
19486 case NVPTX::BI__hmma_m16n16k16_ld_a:
19487 return MMA_LDST(8, m16n16k16_load_a_f16);
19488 case NVPTX::BI__hmma_m16n16k16_ld_b:
19489 return MMA_LDST(8, m16n16k16_load_b_f16);
19490 case NVPTX::BI__hmma_m16n16k16_ld_c_f16:
19491 return MMA_LDST(4, m16n16k16_load_c_f16);
19492 case NVPTX::BI__hmma_m16n16k16_ld_c_f32:
19493 return MMA_LDST(8, m16n16k16_load_c_f32);
19494 case NVPTX::BI__hmma_m32n8k16_ld_a:
19495 return MMA_LDST(8, m32n8k16_load_a_f16);
19496 case NVPTX::BI__hmma_m32n8k16_ld_b:
19497 return MMA_LDST(8, m32n8k16_load_b_f16);
19498 case NVPTX::BI__hmma_m32n8k16_ld_c_f16:
19499 return MMA_LDST(4, m32n8k16_load_c_f16);
19500 case NVPTX::BI__hmma_m32n8k16_ld_c_f32:
19501 return MMA_LDST(8, m32n8k16_load_c_f32);
19502 case NVPTX::BI__hmma_m8n32k16_ld_a:
19503 return MMA_LDST(8, m8n32k16_load_a_f16);
19504 case NVPTX::BI__hmma_m8n32k16_ld_b:
19505 return MMA_LDST(8, m8n32k16_load_b_f16);
19506 case NVPTX::BI__hmma_m8n32k16_ld_c_f16:
19507 return MMA_LDST(4, m8n32k16_load_c_f16);
19508 case NVPTX::BI__hmma_m8n32k16_ld_c_f32:
19509 return MMA_LDST(8, m8n32k16_load_c_f32);
19512 case NVPTX::BI__imma_m16n16k16_ld_a_s8:
19513 return MMA_LDST(2, m16n16k16_load_a_s8);
19514 case NVPTX::BI__imma_m16n16k16_ld_a_u8:
19515 return MMA_LDST(2, m16n16k16_load_a_u8);
19516 case NVPTX::BI__imma_m16n16k16_ld_b_s8:
19517 return MMA_LDST(2, m16n16k16_load_b_s8);
19518 case NVPTX::BI__imma_m16n16k16_ld_b_u8:
19519 return MMA_LDST(2, m16n16k16_load_b_u8);
19520 case NVPTX::BI__imma_m16n16k16_ld_c:
19521 return MMA_LDST(8, m16n16k16_load_c_s32);
19522 case NVPTX::BI__imma_m32n8k16_ld_a_s8:
19523 return MMA_LDST(4, m32n8k16_load_a_s8);
19524 case NVPTX::BI__imma_m32n8k16_ld_a_u8:
19525 return MMA_LDST(4, m32n8k16_load_a_u8);
19526 case NVPTX::BI__imma_m32n8k16_ld_b_s8:
19527 return MMA_LDST(1, m32n8k16_load_b_s8);
19528 case NVPTX::BI__imma_m32n8k16_ld_b_u8:
19529 return MMA_LDST(1, m32n8k16_load_b_u8);
19530 case NVPTX::BI__imma_m32n8k16_ld_c:
19531 return MMA_LDST(8, m32n8k16_load_c_s32);
19532 case NVPTX::BI__imma_m8n32k16_ld_a_s8:
19533 return MMA_LDST(1, m8n32k16_load_a_s8);
19534 case NVPTX::BI__imma_m8n32k16_ld_a_u8:
19535 return MMA_LDST(1, m8n32k16_load_a_u8);
19536 case NVPTX::BI__imma_m8n32k16_ld_b_s8:
19537 return MMA_LDST(4, m8n32k16_load_b_s8);
19538 case NVPTX::BI__imma_m8n32k16_ld_b_u8:
19539 return MMA_LDST(4, m8n32k16_load_b_u8);
19540 case NVPTX::BI__imma_m8n32k16_ld_c:
19541 return MMA_LDST(8, m8n32k16_load_c_s32);
19545 case NVPTX::BI__imma_m8n8k32_ld_a_s4:
19546 return {1, 0,
MMA_INTR(m8n8k32_load_a_s4, row)};
19547 case NVPTX::BI__imma_m8n8k32_ld_a_u4:
19548 return {1, 0,
MMA_INTR(m8n8k32_load_a_u4, row)};
19549 case NVPTX::BI__imma_m8n8k32_ld_b_s4:
19550 return {1,
MMA_INTR(m8n8k32_load_b_s4, col), 0};
19551 case NVPTX::BI__imma_m8n8k32_ld_b_u4:
19552 return {1,
MMA_INTR(m8n8k32_load_b_u4, col), 0};
19553 case NVPTX::BI__imma_m8n8k32_ld_c:
19554 return MMA_LDST(2, m8n8k32_load_c_s32);
19555 case NVPTX::BI__bmma_m8n8k128_ld_a_b1:
19556 return {1, 0,
MMA_INTR(m8n8k128_load_a_b1, row)};
19557 case NVPTX::BI__bmma_m8n8k128_ld_b_b1:
19558 return {1,
MMA_INTR(m8n8k128_load_b_b1, col), 0};
19559 case NVPTX::BI__bmma_m8n8k128_ld_c:
19560 return MMA_LDST(2, m8n8k128_load_c_s32);
19563 case NVPTX::BI__dmma_m8n8k4_ld_a:
19564 return MMA_LDST(1, m8n8k4_load_a_f64);
19565 case NVPTX::BI__dmma_m8n8k4_ld_b:
19566 return MMA_LDST(1, m8n8k4_load_b_f64);
19567 case NVPTX::BI__dmma_m8n8k4_ld_c:
19568 return MMA_LDST(2, m8n8k4_load_c_f64);
19571 case NVPTX::BI__mma_bf16_m16n16k16_ld_a:
19572 return MMA_LDST(4, m16n16k16_load_a_bf16);
19573 case NVPTX::BI__mma_bf16_m16n16k16_ld_b:
19574 return MMA_LDST(4, m16n16k16_load_b_bf16);
19575 case NVPTX::BI__mma_bf16_m8n32k16_ld_a:
19576 return MMA_LDST(2, m8n32k16_load_a_bf16);
19577 case NVPTX::BI__mma_bf16_m8n32k16_ld_b:
19578 return MMA_LDST(8, m8n32k16_load_b_bf16);
19579 case NVPTX::BI__mma_bf16_m32n8k16_ld_a:
19580 return MMA_LDST(8, m32n8k16_load_a_bf16);
19581 case NVPTX::BI__mma_bf16_m32n8k16_ld_b:
19582 return MMA_LDST(2, m32n8k16_load_b_bf16);
19583 case NVPTX::BI__mma_tf32_m16n16k8_ld_a:
19584 return MMA_LDST(4, m16n16k8_load_a_tf32);
19585 case NVPTX::BI__mma_tf32_m16n16k8_ld_b:
19586 return MMA_LDST(4, m16n16k8_load_b_tf32);
19587 case NVPTX::BI__mma_tf32_m16n16k8_ld_c:
19588 return MMA_LDST(8, m16n16k8_load_c_f32);
19594 case NVPTX::BI__hmma_m16n16k16_st_c_f16:
19595 return MMA_LDST(4, m16n16k16_store_d_f16);
19596 case NVPTX::BI__hmma_m16n16k16_st_c_f32:
19597 return MMA_LDST(8, m16n16k16_store_d_f32);
19598 case NVPTX::BI__hmma_m32n8k16_st_c_f16:
19599 return MMA_LDST(4, m32n8k16_store_d_f16);
19600 case NVPTX::BI__hmma_m32n8k16_st_c_f32:
19601 return MMA_LDST(8, m32n8k16_store_d_f32);
19602 case NVPTX::BI__hmma_m8n32k16_st_c_f16:
19603 return MMA_LDST(4, m8n32k16_store_d_f16);
19604 case NVPTX::BI__hmma_m8n32k16_st_c_f32:
19605 return MMA_LDST(8, m8n32k16_store_d_f32);
19610 case NVPTX::BI__imma_m16n16k16_st_c_i32:
19611 return MMA_LDST(8, m16n16k16_store_d_s32);
19612 case NVPTX::BI__imma_m32n8k16_st_c_i32:
19613 return MMA_LDST(8, m32n8k16_store_d_s32);
19614 case NVPTX::BI__imma_m8n32k16_st_c_i32:
19615 return MMA_LDST(8, m8n32k16_store_d_s32);
19616 case NVPTX::BI__imma_m8n8k32_st_c_i32:
19617 return MMA_LDST(2, m8n8k32_store_d_s32);
19618 case NVPTX::BI__bmma_m8n8k128_st_c_i32:
19619 return MMA_LDST(2, m8n8k128_store_d_s32);
19622 case NVPTX::BI__dmma_m8n8k4_st_c_f64:
19623 return MMA_LDST(2, m8n8k4_store_d_f64);
19626 case NVPTX::BI__mma_m16n16k8_st_c_f32:
19627 return MMA_LDST(8, m16n16k8_store_d_f32);
19630 llvm_unreachable(
"Unknown MMA builtin");
19637struct NVPTXMmaInfo {
19646 std::array<unsigned, 8> Variants;
19648 unsigned getMMAIntrinsic(
int Layout,
bool Satf) {
19649 unsigned Index = Layout + 4 * Satf;
19650 if (Index >= Variants.size())
19652 return Variants[Index];
19658static NVPTXMmaInfo getNVPTXMmaInfo(
unsigned BuiltinID) {
19660#define MMA_VARIANTS(geom, type) \
19661 Intrinsic::nvvm_wmma_##geom##_mma_row_row_##type, \
19662 Intrinsic::nvvm_wmma_##geom##_mma_row_col_##type, \
19663 Intrinsic::nvvm_wmma_##geom##_mma_col_row_##type, \
19664 Intrinsic::nvvm_wmma_##geom##_mma_col_col_##type
19665#define MMA_SATF_VARIANTS(geom, type) \
19666 MMA_VARIANTS(geom, type), \
19667 Intrinsic::nvvm_wmma_##geom##_mma_row_row_##type##_satfinite, \
19668 Intrinsic::nvvm_wmma_##geom##_mma_row_col_##type##_satfinite, \
19669 Intrinsic::nvvm_wmma_##geom##_mma_col_row_##type##_satfinite, \
19670 Intrinsic::nvvm_wmma_##geom##_mma_col_col_##type##_satfinite
19672#define MMA_VARIANTS_I4(geom, type) \
19674 Intrinsic::nvvm_wmma_##geom##_mma_row_col_##type, \
19678 Intrinsic::nvvm_wmma_##geom##_mma_row_col_##type##_satfinite, \
19682#define MMA_VARIANTS_B1_XOR(geom, type) \
19684 Intrinsic::nvvm_wmma_##geom##_mma_xor_popc_row_col_##type, \
19691#define MMA_VARIANTS_B1_AND(geom, type) \
19693 Intrinsic::nvvm_wmma_##geom##_mma_and_popc_row_col_##type, \
19701 switch (BuiltinID) {
19705 case NVPTX::BI__hmma_m16n16k16_mma_f16f16:
19707 case NVPTX::BI__hmma_m16n16k16_mma_f32f16:
19709 case NVPTX::BI__hmma_m16n16k16_mma_f16f32:
19711 case NVPTX::BI__hmma_m16n16k16_mma_f32f32:
19713 case NVPTX::BI__hmma_m32n8k16_mma_f16f16:
19715 case NVPTX::BI__hmma_m32n8k16_mma_f32f16:
19717 case NVPTX::BI__hmma_m32n8k16_mma_f16f32:
19719 case NVPTX::BI__hmma_m32n8k16_mma_f32f32:
19721 case NVPTX::BI__hmma_m8n32k16_mma_f16f16:
19723 case NVPTX::BI__hmma_m8n32k16_mma_f32f16:
19725 case NVPTX::BI__hmma_m8n32k16_mma_f16f32:
19727 case NVPTX::BI__hmma_m8n32k16_mma_f32f32:
19731 case NVPTX::BI__imma_m16n16k16_mma_s8:
19733 case NVPTX::BI__imma_m16n16k16_mma_u8:
19735 case NVPTX::BI__imma_m32n8k16_mma_s8:
19737 case NVPTX::BI__imma_m32n8k16_mma_u8:
19739 case NVPTX::BI__imma_m8n32k16_mma_s8:
19741 case NVPTX::BI__imma_m8n32k16_mma_u8:
19745 case NVPTX::BI__imma_m8n8k32_mma_s4:
19747 case NVPTX::BI__imma_m8n8k32_mma_u4:
19749 case NVPTX::BI__bmma_m8n8k128_mma_xor_popc_b1:
19751 case NVPTX::BI__bmma_m8n8k128_mma_and_popc_b1:
19755 case NVPTX::BI__dmma_m8n8k4_mma_f64:
19759 case NVPTX::BI__mma_bf16_m16n16k16_mma_f32:
19760 return {4, 4, 8, 8, {{
MMA_VARIANTS(m16n16k16, bf16)}}};
19761 case NVPTX::BI__mma_bf16_m8n32k16_mma_f32:
19762 return {2, 8, 8, 8, {{
MMA_VARIANTS(m8n32k16, bf16)}}};
19763 case NVPTX::BI__mma_bf16_m32n8k16_mma_f32:
19764 return {8, 2, 8, 8, {{
MMA_VARIANTS(m32n8k16, bf16)}}};
19765 case NVPTX::BI__mma_tf32_m16n16k8_mma_f32:
19766 return {4, 4, 8, 8, {{
MMA_VARIANTS(m16n16k8, tf32)}}};
19768 llvm_unreachable(
"Unexpected builtin ID.");
19771#undef MMA_SATF_VARIANTS
19772#undef MMA_VARIANTS_I4
19773#undef MMA_VARIANTS_B1_AND
19774#undef MMA_VARIANTS_B1_XOR
19783 return CGF.
Builder.CreateCall(
19785 {Ptr, ConstantInt::get(CGF.Builder.getInt32Ty(), Align.getQuantity())});
19791 llvm::Type *ElemTy =
19793 return CGF.
Builder.CreateCall(
19795 {Ptr, CGF.EmitScalarExpr(E->getArg(1))});
19798static Value *MakeCpAsync(
unsigned IntrinsicID,
unsigned IntrinsicIDS,
19803 {CGF.EmitScalarExpr(E->getArg(0)),
19804 CGF.EmitScalarExpr(E->getArg(1)),
19805 CGF.EmitScalarExpr(E->getArg(2))})
19807 {CGF.EmitScalarExpr(E->getArg(0)),
19808 CGF.EmitScalarExpr(E->getArg(1))});
19811static Value *MakeHalfType(
unsigned IntrinsicID,
unsigned BuiltinID,
19814 if (!(
C.getLangOpts().NativeHalfType ||
19815 !
C.getTargetInfo().useFP16ConversionIntrinsics())) {
19817 " requires native half type support.");
19821 if (IntrinsicID == Intrinsic::nvvm_ldg_global_f ||
19822 IntrinsicID == Intrinsic::nvvm_ldu_global_f)
19823 return MakeLdgLdu(IntrinsicID, CGF, E);
19827 auto *FTy = F->getFunctionType();
19828 unsigned ICEArguments = 0;
19830 C.GetBuiltinType(BuiltinID, Error, &ICEArguments);
19832 for (
unsigned i = 0, e = E->
getNumArgs(); i != e; ++i) {
19833 assert((ICEArguments & (1 << i)) == 0);
19835 auto *PTy = FTy->getParamType(i);
19836 if (PTy != ArgValue->
getType())
19837 ArgValue = CGF.
Builder.CreateBitCast(ArgValue, PTy);
19838 Args.push_back(ArgValue);
19841 return CGF.
Builder.CreateCall(F, Args);
19847 switch (BuiltinID) {
19848 case NVPTX::BI__nvvm_atom_add_gen_i:
19849 case NVPTX::BI__nvvm_atom_add_gen_l:
19850 case NVPTX::BI__nvvm_atom_add_gen_ll:
19853 case NVPTX::BI__nvvm_atom_sub_gen_i:
19854 case NVPTX::BI__nvvm_atom_sub_gen_l:
19855 case NVPTX::BI__nvvm_atom_sub_gen_ll:
19858 case NVPTX::BI__nvvm_atom_and_gen_i:
19859 case NVPTX::BI__nvvm_atom_and_gen_l:
19860 case NVPTX::BI__nvvm_atom_and_gen_ll:
19863 case NVPTX::BI__nvvm_atom_or_gen_i:
19864 case NVPTX::BI__nvvm_atom_or_gen_l:
19865 case NVPTX::BI__nvvm_atom_or_gen_ll:
19868 case NVPTX::BI__nvvm_atom_xor_gen_i:
19869 case NVPTX::BI__nvvm_atom_xor_gen_l:
19870 case NVPTX::BI__nvvm_atom_xor_gen_ll:
19873 case NVPTX::BI__nvvm_atom_xchg_gen_i:
19874 case NVPTX::BI__nvvm_atom_xchg_gen_l:
19875 case NVPTX::BI__nvvm_atom_xchg_gen_ll:
19878 case NVPTX::BI__nvvm_atom_max_gen_i:
19879 case NVPTX::BI__nvvm_atom_max_gen_l:
19880 case NVPTX::BI__nvvm_atom_max_gen_ll:
19883 case NVPTX::BI__nvvm_atom_max_gen_ui:
19884 case NVPTX::BI__nvvm_atom_max_gen_ul:
19885 case NVPTX::BI__nvvm_atom_max_gen_ull:
19888 case NVPTX::BI__nvvm_atom_min_gen_i:
19889 case NVPTX::BI__nvvm_atom_min_gen_l:
19890 case NVPTX::BI__nvvm_atom_min_gen_ll:
19893 case NVPTX::BI__nvvm_atom_min_gen_ui:
19894 case NVPTX::BI__nvvm_atom_min_gen_ul:
19895 case NVPTX::BI__nvvm_atom_min_gen_ull:
19898 case NVPTX::BI__nvvm_atom_cas_gen_i:
19899 case NVPTX::BI__nvvm_atom_cas_gen_l:
19900 case NVPTX::BI__nvvm_atom_cas_gen_ll:
19905 case NVPTX::BI__nvvm_atom_add_gen_f:
19906 case NVPTX::BI__nvvm_atom_add_gen_d: {
19911 AtomicOrdering::SequentiallyConsistent);
19914 case NVPTX::BI__nvvm_atom_inc_gen_ui: {
19919 return Builder.CreateCall(FnALI32, {Ptr, Val});
19922 case NVPTX::BI__nvvm_atom_dec_gen_ui: {
19927 return Builder.CreateCall(FnALD32, {Ptr, Val});
19930 case NVPTX::BI__nvvm_ldg_c:
19931 case NVPTX::BI__nvvm_ldg_sc:
19932 case NVPTX::BI__nvvm_ldg_c2:
19933 case NVPTX::BI__nvvm_ldg_sc2:
19934 case NVPTX::BI__nvvm_ldg_c4:
19935 case NVPTX::BI__nvvm_ldg_sc4:
19936 case NVPTX::BI__nvvm_ldg_s:
19937 case NVPTX::BI__nvvm_ldg_s2:
19938 case NVPTX::BI__nvvm_ldg_s4:
19939 case NVPTX::BI__nvvm_ldg_i:
19940 case NVPTX::BI__nvvm_ldg_i2:
19941 case NVPTX::BI__nvvm_ldg_i4:
19942 case NVPTX::BI__nvvm_ldg_l:
19943 case NVPTX::BI__nvvm_ldg_l2:
19944 case NVPTX::BI__nvvm_ldg_ll:
19945 case NVPTX::BI__nvvm_ldg_ll2:
19946 case NVPTX::BI__nvvm_ldg_uc:
19947 case NVPTX::BI__nvvm_ldg_uc2:
19948 case NVPTX::BI__nvvm_ldg_uc4:
19949 case NVPTX::BI__nvvm_ldg_us:
19950 case NVPTX::BI__nvvm_ldg_us2:
19951 case NVPTX::BI__nvvm_ldg_us4:
19952 case NVPTX::BI__nvvm_ldg_ui:
19953 case NVPTX::BI__nvvm_ldg_ui2:
19954 case NVPTX::BI__nvvm_ldg_ui4:
19955 case NVPTX::BI__nvvm_ldg_ul:
19956 case NVPTX::BI__nvvm_ldg_ul2:
19957 case NVPTX::BI__nvvm_ldg_ull:
19958 case NVPTX::BI__nvvm_ldg_ull2:
19962 return MakeLdgLdu(Intrinsic::nvvm_ldg_global_i, *
this, E);
19963 case NVPTX::BI__nvvm_ldg_f:
19964 case NVPTX::BI__nvvm_ldg_f2:
19965 case NVPTX::BI__nvvm_ldg_f4:
19966 case NVPTX::BI__nvvm_ldg_d:
19967 case NVPTX::BI__nvvm_ldg_d2:
19968 return MakeLdgLdu(Intrinsic::nvvm_ldg_global_f, *
this, E);
19970 case NVPTX::BI__nvvm_ldu_c:
19971 case NVPTX::BI__nvvm_ldu_sc:
19972 case NVPTX::BI__nvvm_ldu_c2:
19973 case NVPTX::BI__nvvm_ldu_sc2:
19974 case NVPTX::BI__nvvm_ldu_c4:
19975 case NVPTX::BI__nvvm_ldu_sc4:
19976 case NVPTX::BI__nvvm_ldu_s:
19977 case NVPTX::BI__nvvm_ldu_s2:
19978 case NVPTX::BI__nvvm_ldu_s4:
19979 case NVPTX::BI__nvvm_ldu_i:
19980 case NVPTX::BI__nvvm_ldu_i2:
19981 case NVPTX::BI__nvvm_ldu_i4:
19982 case NVPTX::BI__nvvm_ldu_l:
19983 case NVPTX::BI__nvvm_ldu_l2:
19984 case NVPTX::BI__nvvm_ldu_ll:
19985 case NVPTX::BI__nvvm_ldu_ll2:
19986 case NVPTX::BI__nvvm_ldu_uc:
19987 case NVPTX::BI__nvvm_ldu_uc2:
19988 case NVPTX::BI__nvvm_ldu_uc4:
19989 case NVPTX::BI__nvvm_ldu_us:
19990 case NVPTX::BI__nvvm_ldu_us2:
19991 case NVPTX::BI__nvvm_ldu_us4:
19992 case NVPTX::BI__nvvm_ldu_ui:
19993 case NVPTX::BI__nvvm_ldu_ui2:
19994 case NVPTX::BI__nvvm_ldu_ui4:
19995 case NVPTX::BI__nvvm_ldu_ul:
19996 case NVPTX::BI__nvvm_ldu_ul2:
19997 case NVPTX::BI__nvvm_ldu_ull:
19998 case NVPTX::BI__nvvm_ldu_ull2:
19999 return MakeLdgLdu(Intrinsic::nvvm_ldu_global_i, *
this, E);
20000 case NVPTX::BI__nvvm_ldu_f:
20001 case NVPTX::BI__nvvm_ldu_f2:
20002 case NVPTX::BI__nvvm_ldu_f4:
20003 case NVPTX::BI__nvvm_ldu_d:
20004 case NVPTX::BI__nvvm_ldu_d2:
20005 return MakeLdgLdu(Intrinsic::nvvm_ldu_global_f, *
this, E);
20007 case NVPTX::BI__nvvm_atom_cta_add_gen_i:
20008 case NVPTX::BI__nvvm_atom_cta_add_gen_l:
20009 case NVPTX::BI__nvvm_atom_cta_add_gen_ll:
20010 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_i_cta, *
this, E);
20011 case NVPTX::BI__nvvm_atom_sys_add_gen_i:
20012 case NVPTX::BI__nvvm_atom_sys_add_gen_l:
20013 case NVPTX::BI__nvvm_atom_sys_add_gen_ll:
20014 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_i_sys, *
this, E);
20015 case NVPTX::BI__nvvm_atom_cta_add_gen_f:
20016 case NVPTX::BI__nvvm_atom_cta_add_gen_d:
20017 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_f_cta, *
this, E);
20018 case NVPTX::BI__nvvm_atom_sys_add_gen_f:
20019 case NVPTX::BI__nvvm_atom_sys_add_gen_d:
20020 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_f_sys, *
this, E);
20021 case NVPTX::BI__nvvm_atom_cta_xchg_gen_i:
20022 case NVPTX::BI__nvvm_atom_cta_xchg_gen_l:
20023 case NVPTX::BI__nvvm_atom_cta_xchg_gen_ll:
20024 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_gen_i_cta, *
this, E);
20025 case NVPTX::BI__nvvm_atom_sys_xchg_gen_i:
20026 case NVPTX::BI__nvvm_atom_sys_xchg_gen_l:
20027 case NVPTX::BI__nvvm_atom_sys_xchg_gen_ll:
20028 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_gen_i_sys, *
this, E);
20029 case NVPTX::BI__nvvm_atom_cta_max_gen_i:
20030 case NVPTX::BI__nvvm_atom_cta_max_gen_ui:
20031 case NVPTX::BI__nvvm_atom_cta_max_gen_l:
20032 case NVPTX::BI__nvvm_atom_cta_max_gen_ul:
20033 case NVPTX::BI__nvvm_atom_cta_max_gen_ll:
20034 case NVPTX::BI__nvvm_atom_cta_max_gen_ull:
20035 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_gen_i_cta, *
this, E);
20036 case NVPTX::BI__nvvm_atom_sys_max_gen_i:
20037 case NVPTX::BI__nvvm_atom_sys_max_gen_ui:
20038 case NVPTX::BI__nvvm_atom_sys_max_gen_l:
20039 case NVPTX::BI__nvvm_atom_sys_max_gen_ul:
20040 case NVPTX::BI__nvvm_atom_sys_max_gen_ll:
20041 case NVPTX::BI__nvvm_atom_sys_max_gen_ull:
20042 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_gen_i_sys, *
this, E);
20043 case NVPTX::BI__nvvm_atom_cta_min_gen_i:
20044 case NVPTX::BI__nvvm_atom_cta_min_gen_ui:
20045 case NVPTX::BI__nvvm_atom_cta_min_gen_l:
20046 case NVPTX::BI__nvvm_atom_cta_min_gen_ul:
20047 case NVPTX::BI__nvvm_atom_cta_min_gen_ll:
20048 case NVPTX::BI__nvvm_atom_cta_min_gen_ull:
20049 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_gen_i_cta, *
this, E);
20050 case NVPTX::BI__nvvm_atom_sys_min_gen_i:
20051 case NVPTX::BI__nvvm_atom_sys_min_gen_ui:
20052 case NVPTX::BI__nvvm_atom_sys_min_gen_l:
20053 case NVPTX::BI__nvvm_atom_sys_min_gen_ul:
20054 case NVPTX::BI__nvvm_atom_sys_min_gen_ll:
20055 case NVPTX::BI__nvvm_atom_sys_min_gen_ull:
20056 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_gen_i_sys, *
this, E);
20057 case NVPTX::BI__nvvm_atom_cta_inc_gen_ui:
20058 return MakeScopedAtomic(Intrinsic::nvvm_atomic_inc_gen_i_cta, *
this, E);
20059 case NVPTX::BI__nvvm_atom_cta_dec_gen_ui:
20060 return MakeScopedAtomic(Intrinsic::nvvm_atomic_dec_gen_i_cta, *
this, E);
20061 case NVPTX::BI__nvvm_atom_sys_inc_gen_ui:
20062 return MakeScopedAtomic(Intrinsic::nvvm_atomic_inc_gen_i_sys, *
this, E);
20063 case NVPTX::BI__nvvm_atom_sys_dec_gen_ui:
20064 return MakeScopedAtomic(Intrinsic::nvvm_atomic_dec_gen_i_sys, *
this, E);
20065 case NVPTX::BI__nvvm_atom_cta_and_gen_i:
20066 case NVPTX::BI__nvvm_atom_cta_and_gen_l:
20067 case NVPTX::BI__nvvm_atom_cta_and_gen_ll:
20068 return MakeScopedAtomic(Intrinsic::nvvm_atomic_and_gen_i_cta, *
this, E);
20069 case NVPTX::BI__nvvm_atom_sys_and_gen_i:
20070 case NVPTX::BI__nvvm_atom_sys_and_gen_l:
20071 case NVPTX::BI__nvvm_atom_sys_and_gen_ll:
20072 return MakeScopedAtomic(Intrinsic::nvvm_atomic_and_gen_i_sys, *
this, E);
20073 case NVPTX::BI__nvvm_atom_cta_or_gen_i:
20074 case NVPTX::BI__nvvm_atom_cta_or_gen_l:
20075 case NVPTX::BI__nvvm_atom_cta_or_gen_ll:
20076 return MakeScopedAtomic(Intrinsic::nvvm_atomic_or_gen_i_cta, *
this, E);
20077 case NVPTX::BI__nvvm_atom_sys_or_gen_i:
20078 case NVPTX::BI__nvvm_atom_sys_or_gen_l:
20079 case NVPTX::BI__nvvm_atom_sys_or_gen_ll:
20080 return MakeScopedAtomic(Intrinsic::nvvm_atomic_or_gen_i_sys, *
this, E);
20081 case NVPTX::BI__nvvm_atom_cta_xor_gen_i:
20082 case NVPTX::BI__nvvm_atom_cta_xor_gen_l:
20083 case NVPTX::BI__nvvm_atom_cta_xor_gen_ll:
20084 return MakeScopedAtomic(Intrinsic::nvvm_atomic_xor_gen_i_cta, *
this, E);
20085 case NVPTX::BI__nvvm_atom_sys_xor_gen_i:
20086 case NVPTX::BI__nvvm_atom_sys_xor_gen_l:
20087 case NVPTX::BI__nvvm_atom_sys_xor_gen_ll:
20088 return MakeScopedAtomic(Intrinsic::nvvm_atomic_xor_gen_i_sys, *
this, E);
20089 case NVPTX::BI__nvvm_atom_cta_cas_gen_i:
20090 case NVPTX::BI__nvvm_atom_cta_cas_gen_l:
20091 case NVPTX::BI__nvvm_atom_cta_cas_gen_ll: {
20093 llvm::Type *ElemTy =
20097 Intrinsic::nvvm_atomic_cas_gen_i_cta, {ElemTy, Ptr->getType()}),
20098 {Ptr, EmitScalarExpr(E->getArg(1)), EmitScalarExpr(E->getArg(2))});
20100 case NVPTX::BI__nvvm_atom_sys_cas_gen_i:
20101 case NVPTX::BI__nvvm_atom_sys_cas_gen_l:
20102 case NVPTX::BI__nvvm_atom_sys_cas_gen_ll: {
20104 llvm::Type *ElemTy =
20108 Intrinsic::nvvm_atomic_cas_gen_i_sys, {ElemTy, Ptr->getType()}),
20109 {Ptr, EmitScalarExpr(E->getArg(1)), EmitScalarExpr(E->getArg(2))});
20111 case NVPTX::BI__nvvm_match_all_sync_i32p:
20112 case NVPTX::BI__nvvm_match_all_sync_i64p: {
20118 ? Intrinsic::nvvm_match_all_sync_i32p
20119 : Intrinsic::nvvm_match_all_sync_i64p),
20124 return Builder.CreateExtractValue(ResultPair, 0);
20128 case NVPTX::BI__hmma_m16n16k16_ld_a:
20129 case NVPTX::BI__hmma_m16n16k16_ld_b:
20130 case NVPTX::BI__hmma_m16n16k16_ld_c_f16:
20131 case NVPTX::BI__hmma_m16n16k16_ld_c_f32:
20132 case NVPTX::BI__hmma_m32n8k16_ld_a:
20133 case NVPTX::BI__hmma_m32n8k16_ld_b:
20134 case NVPTX::BI__hmma_m32n8k16_ld_c_f16:
20135 case NVPTX::BI__hmma_m32n8k16_ld_c_f32:
20136 case NVPTX::BI__hmma_m8n32k16_ld_a:
20137 case NVPTX::BI__hmma_m8n32k16_ld_b:
20138 case NVPTX::BI__hmma_m8n32k16_ld_c_f16:
20139 case NVPTX::BI__hmma_m8n32k16_ld_c_f32:
20141 case NVPTX::BI__imma_m16n16k16_ld_a_s8:
20142 case NVPTX::BI__imma_m16n16k16_ld_a_u8:
20143 case NVPTX::BI__imma_m16n16k16_ld_b_s8:
20144 case NVPTX::BI__imma_m16n16k16_ld_b_u8:
20145 case NVPTX::BI__imma_m16n16k16_ld_c:
20146 case NVPTX::BI__imma_m32n8k16_ld_a_s8:
20147 case NVPTX::BI__imma_m32n8k16_ld_a_u8:
20148 case NVPTX::BI__imma_m32n8k16_ld_b_s8:
20149 case NVPTX::BI__imma_m32n8k16_ld_b_u8:
20150 case NVPTX::BI__imma_m32n8k16_ld_c:
20151 case NVPTX::BI__imma_m8n32k16_ld_a_s8:
20152 case NVPTX::BI__imma_m8n32k16_ld_a_u8:
20153 case NVPTX::BI__imma_m8n32k16_ld_b_s8:
20154 case NVPTX::BI__imma_m8n32k16_ld_b_u8:
20155 case NVPTX::BI__imma_m8n32k16_ld_c:
20157 case NVPTX::BI__imma_m8n8k32_ld_a_s4:
20158 case NVPTX::BI__imma_m8n8k32_ld_a_u4:
20159 case NVPTX::BI__imma_m8n8k32_ld_b_s4:
20160 case NVPTX::BI__imma_m8n8k32_ld_b_u4:
20161 case NVPTX::BI__imma_m8n8k32_ld_c:
20162 case NVPTX::BI__bmma_m8n8k128_ld_a_b1:
20163 case NVPTX::BI__bmma_m8n8k128_ld_b_b1:
20164 case NVPTX::BI__bmma_m8n8k128_ld_c:
20166 case NVPTX::BI__dmma_m8n8k4_ld_a:
20167 case NVPTX::BI__dmma_m8n8k4_ld_b:
20168 case NVPTX::BI__dmma_m8n8k4_ld_c:
20170 case NVPTX::BI__mma_bf16_m16n16k16_ld_a:
20171 case NVPTX::BI__mma_bf16_m16n16k16_ld_b:
20172 case NVPTX::BI__mma_bf16_m8n32k16_ld_a:
20173 case NVPTX::BI__mma_bf16_m8n32k16_ld_b:
20174 case NVPTX::BI__mma_bf16_m32n8k16_ld_a:
20175 case NVPTX::BI__mma_bf16_m32n8k16_ld_b:
20176 case NVPTX::BI__mma_tf32_m16n16k8_ld_a:
20177 case NVPTX::BI__mma_tf32_m16n16k8_ld_b:
20178 case NVPTX::BI__mma_tf32_m16n16k8_ld_c: {
20182 std::optional<llvm::APSInt> isColMajorArg =
20184 if (!isColMajorArg)
20186 bool isColMajor = isColMajorArg->getSExtValue();
20187 NVPTXMmaLdstInfo II = getNVPTXMmaLdstInfo(BuiltinID);
20188 unsigned IID = isColMajor ? II.IID_col : II.IID_row;
20196 assert(II.NumResults);
20197 if (II.NumResults == 1) {
20201 for (
unsigned i = 0; i < II.NumResults; ++i) {
20206 llvm::ConstantInt::get(
IntTy, i)),
20213 case NVPTX::BI__hmma_m16n16k16_st_c_f16:
20214 case NVPTX::BI__hmma_m16n16k16_st_c_f32:
20215 case NVPTX::BI__hmma_m32n8k16_st_c_f16:
20216 case NVPTX::BI__hmma_m32n8k16_st_c_f32:
20217 case NVPTX::BI__hmma_m8n32k16_st_c_f16:
20218 case NVPTX::BI__hmma_m8n32k16_st_c_f32:
20219 case NVPTX::BI__imma_m16n16k16_st_c_i32:
20220 case NVPTX::BI__imma_m32n8k16_st_c_i32:
20221 case NVPTX::BI__imma_m8n32k16_st_c_i32:
20222 case NVPTX::BI__imma_m8n8k32_st_c_i32:
20223 case NVPTX::BI__bmma_m8n8k128_st_c_i32:
20224 case NVPTX::BI__dmma_m8n8k4_st_c_f64:
20225 case NVPTX::BI__mma_m16n16k8_st_c_f32: {
20229 std::optional<llvm::APSInt> isColMajorArg =
20231 if (!isColMajorArg)
20233 bool isColMajor = isColMajorArg->getSExtValue();
20234 NVPTXMmaLdstInfo II = getNVPTXMmaLdstInfo(BuiltinID);
20235 unsigned IID = isColMajor ? II.IID_col : II.IID_row;
20240 llvm::Type *ParamType = Intrinsic->getFunctionType()->getParamType(1);
20242 for (
unsigned i = 0; i < II.NumResults; ++i) {
20246 llvm::ConstantInt::get(
IntTy, i)),
20248 Values.push_back(
Builder.CreateBitCast(
V, ParamType));
20250 Values.push_back(Ldm);
20257 case NVPTX::BI__hmma_m16n16k16_mma_f16f16:
20258 case NVPTX::BI__hmma_m16n16k16_mma_f32f16:
20259 case NVPTX::BI__hmma_m16n16k16_mma_f32f32:
20260 case NVPTX::BI__hmma_m16n16k16_mma_f16f32:
20261 case NVPTX::BI__hmma_m32n8k16_mma_f16f16:
20262 case NVPTX::BI__hmma_m32n8k16_mma_f32f16:
20263 case NVPTX::BI__hmma_m32n8k16_mma_f32f32:
20264 case NVPTX::BI__hmma_m32n8k16_mma_f16f32:
20265 case NVPTX::BI__hmma_m8n32k16_mma_f16f16:
20266 case NVPTX::BI__hmma_m8n32k16_mma_f32f16:
20267 case NVPTX::BI__hmma_m8n32k16_mma_f32f32:
20268 case NVPTX::BI__hmma_m8n32k16_mma_f16f32:
20269 case NVPTX::BI__imma_m16n16k16_mma_s8:
20270 case NVPTX::BI__imma_m16n16k16_mma_u8:
20271 case NVPTX::BI__imma_m32n8k16_mma_s8:
20272 case NVPTX::BI__imma_m32n8k16_mma_u8:
20273 case NVPTX::BI__imma_m8n32k16_mma_s8:
20274 case NVPTX::BI__imma_m8n32k16_mma_u8:
20275 case NVPTX::BI__imma_m8n8k32_mma_s4:
20276 case NVPTX::BI__imma_m8n8k32_mma_u4:
20277 case NVPTX::BI__bmma_m8n8k128_mma_xor_popc_b1:
20278 case NVPTX::BI__bmma_m8n8k128_mma_and_popc_b1:
20279 case NVPTX::BI__dmma_m8n8k4_mma_f64:
20280 case NVPTX::BI__mma_bf16_m16n16k16_mma_f32:
20281 case NVPTX::BI__mma_bf16_m8n32k16_mma_f32:
20282 case NVPTX::BI__mma_bf16_m32n8k16_mma_f32:
20283 case NVPTX::BI__mma_tf32_m16n16k8_mma_f32: {
20288 std::optional<llvm::APSInt> LayoutArg =
20292 int Layout = LayoutArg->getSExtValue();
20293 if (Layout < 0 || Layout > 3)
20295 llvm::APSInt SatfArg;
20296 if (BuiltinID == NVPTX::BI__bmma_m8n8k128_mma_xor_popc_b1 ||
20297 BuiltinID == NVPTX::BI__bmma_m8n8k128_mma_and_popc_b1)
20299 else if (std::optional<llvm::APSInt> OptSatfArg =
20301 SatfArg = *OptSatfArg;
20304 bool Satf = SatfArg.getSExtValue();
20305 NVPTXMmaInfo MI = getNVPTXMmaInfo(BuiltinID);
20306 unsigned IID = MI.getMMAIntrinsic(Layout, Satf);
20312 llvm::Type *AType = Intrinsic->getFunctionType()->getParamType(0);
20314 for (
unsigned i = 0; i < MI.NumEltsA; ++i) {
20318 llvm::ConstantInt::get(
IntTy, i)),
20320 Values.push_back(
Builder.CreateBitCast(
V, AType));
20323 llvm::Type *BType = Intrinsic->getFunctionType()->getParamType(MI.NumEltsA);
20324 for (
unsigned i = 0; i < MI.NumEltsB; ++i) {
20328 llvm::ConstantInt::get(
IntTy, i)),
20330 Values.push_back(
Builder.CreateBitCast(
V, BType));
20333 llvm::Type *CType =
20334 Intrinsic->getFunctionType()->getParamType(MI.NumEltsA + MI.NumEltsB);
20335 for (
unsigned i = 0; i < MI.NumEltsC; ++i) {
20339 llvm::ConstantInt::get(
IntTy, i)),
20341 Values.push_back(
Builder.CreateBitCast(
V, CType));
20345 for (
unsigned i = 0; i < MI.NumEltsD; ++i)
20349 llvm::ConstantInt::get(
IntTy, i)),
20354 case NVPTX::BI__nvvm_ex2_approx_f16:
20355 return MakeHalfType(Intrinsic::nvvm_ex2_approx_f16, BuiltinID, E, *
this);
20356 case NVPTX::BI__nvvm_ex2_approx_f16x2:
20357 return MakeHalfType(Intrinsic::nvvm_ex2_approx_f16x2, BuiltinID, E, *
this);
20358 case NVPTX::BI__nvvm_ff2f16x2_rn:
20359 return MakeHalfType(Intrinsic::nvvm_ff2f16x2_rn, BuiltinID, E, *
this);
20360 case NVPTX::BI__nvvm_ff2f16x2_rn_relu:
20361 return MakeHalfType(Intrinsic::nvvm_ff2f16x2_rn_relu, BuiltinID, E, *
this);
20362 case NVPTX::BI__nvvm_ff2f16x2_rz:
20363 return MakeHalfType(Intrinsic::nvvm_ff2f16x2_rz, BuiltinID, E, *
this);
20364 case NVPTX::BI__nvvm_ff2f16x2_rz_relu:
20365 return MakeHalfType(Intrinsic::nvvm_ff2f16x2_rz_relu, BuiltinID, E, *
this);
20366 case NVPTX::BI__nvvm_fma_rn_f16:
20367 return MakeHalfType(Intrinsic::nvvm_fma_rn_f16, BuiltinID, E, *
this);
20368 case NVPTX::BI__nvvm_fma_rn_f16x2:
20369 return MakeHalfType(Intrinsic::nvvm_fma_rn_f16x2, BuiltinID, E, *
this);
20370 case NVPTX::BI__nvvm_fma_rn_ftz_f16:
20371 return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_f16, BuiltinID, E, *
this);
20372 case NVPTX::BI__nvvm_fma_rn_ftz_f16x2:
20373 return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_f16x2, BuiltinID, E, *
this);
20374 case NVPTX::BI__nvvm_fma_rn_ftz_relu_f16:
20375 return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_relu_f16, BuiltinID, E,
20377 case NVPTX::BI__nvvm_fma_rn_ftz_relu_f16x2:
20378 return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_relu_f16x2, BuiltinID, E,
20380 case NVPTX::BI__nvvm_fma_rn_ftz_sat_f16:
20381 return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_sat_f16, BuiltinID, E,
20383 case NVPTX::BI__nvvm_fma_rn_ftz_sat_f16x2:
20384 return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_sat_f16x2, BuiltinID, E,
20386 case NVPTX::BI__nvvm_fma_rn_relu_f16:
20387 return MakeHalfType(Intrinsic::nvvm_fma_rn_relu_f16, BuiltinID, E, *
this);
20388 case NVPTX::BI__nvvm_fma_rn_relu_f16x2:
20389 return MakeHalfType(Intrinsic::nvvm_fma_rn_relu_f16x2, BuiltinID, E, *
this);
20390 case NVPTX::BI__nvvm_fma_rn_sat_f16:
20391 return MakeHalfType(Intrinsic::nvvm_fma_rn_sat_f16, BuiltinID, E, *
this);
20392 case NVPTX::BI__nvvm_fma_rn_sat_f16x2:
20393 return MakeHalfType(Intrinsic::nvvm_fma_rn_sat_f16x2, BuiltinID, E, *
this);
20394 case NVPTX::BI__nvvm_fmax_f16:
20395 return MakeHalfType(Intrinsic::nvvm_fmax_f16, BuiltinID, E, *
this);
20396 case NVPTX::BI__nvvm_fmax_f16x2:
20397 return MakeHalfType(Intrinsic::nvvm_fmax_f16x2, BuiltinID, E, *
this);
20398 case NVPTX::BI__nvvm_fmax_ftz_f16:
20399 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_f16, BuiltinID, E, *
this);
20400 case NVPTX::BI__nvvm_fmax_ftz_f16x2:
20401 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_f16x2, BuiltinID, E, *
this);
20402 case NVPTX::BI__nvvm_fmax_ftz_nan_f16:
20403 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_nan_f16, BuiltinID, E, *
this);
20404 case NVPTX::BI__nvvm_fmax_ftz_nan_f16x2:
20405 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_nan_f16x2, BuiltinID, E,
20407 case NVPTX::BI__nvvm_fmax_ftz_nan_xorsign_abs_f16:
20408 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_f16, BuiltinID,
20410 case NVPTX::BI__nvvm_fmax_ftz_nan_xorsign_abs_f16x2:
20411 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_f16x2,
20412 BuiltinID, E, *
this);
20413 case NVPTX::BI__nvvm_fmax_ftz_xorsign_abs_f16:
20414 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_xorsign_abs_f16, BuiltinID, E,
20416 case NVPTX::BI__nvvm_fmax_ftz_xorsign_abs_f16x2:
20417 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_xorsign_abs_f16x2, BuiltinID,
20419 case NVPTX::BI__nvvm_fmax_nan_f16:
20420 return MakeHalfType(Intrinsic::nvvm_fmax_nan_f16, BuiltinID, E, *
this);
20421 case NVPTX::BI__nvvm_fmax_nan_f16x2:
20422 return MakeHalfType(Intrinsic::nvvm_fmax_nan_f16x2, BuiltinID, E, *
this);
20423 case NVPTX::BI__nvvm_fmax_nan_xorsign_abs_f16:
20424 return MakeHalfType(Intrinsic::nvvm_fmax_nan_xorsign_abs_f16, BuiltinID, E,
20426 case NVPTX::BI__nvvm_fmax_nan_xorsign_abs_f16x2:
20427 return MakeHalfType(Intrinsic::nvvm_fmax_nan_xorsign_abs_f16x2, BuiltinID,
20429 case NVPTX::BI__nvvm_fmax_xorsign_abs_f16:
20430 return MakeHalfType(Intrinsic::nvvm_fmax_xorsign_abs_f16, BuiltinID, E,
20432 case NVPTX::BI__nvvm_fmax_xorsign_abs_f16x2:
20433 return MakeHalfType(Intrinsic::nvvm_fmax_xorsign_abs_f16x2, BuiltinID, E,
20435 case NVPTX::BI__nvvm_fmin_f16:
20436 return MakeHalfType(Intrinsic::nvvm_fmin_f16, BuiltinID, E, *
this);
20437 case NVPTX::BI__nvvm_fmin_f16x2:
20438 return MakeHalfType(Intrinsic::nvvm_fmin_f16x2, BuiltinID, E, *
this);
20439 case NVPTX::BI__nvvm_fmin_ftz_f16:
20440 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_f16, BuiltinID, E, *
this);
20441 case NVPTX::BI__nvvm_fmin_ftz_f16x2:
20442 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_f16x2, BuiltinID, E, *
this);
20443 case NVPTX::BI__nvvm_fmin_ftz_nan_f16:
20444 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_nan_f16, BuiltinID, E, *
this);
20445 case NVPTX::BI__nvvm_fmin_ftz_nan_f16x2:
20446 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_nan_f16x2, BuiltinID, E,
20448 case NVPTX::BI__nvvm_fmin_ftz_nan_xorsign_abs_f16:
20449 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_f16, BuiltinID,
20451 case NVPTX::BI__nvvm_fmin_ftz_nan_xorsign_abs_f16x2:
20452 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_f16x2,
20453 BuiltinID, E, *
this);
20454 case NVPTX::BI__nvvm_fmin_ftz_xorsign_abs_f16:
20455 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_xorsign_abs_f16, BuiltinID, E,
20457 case NVPTX::BI__nvvm_fmin_ftz_xorsign_abs_f16x2:
20458 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_xorsign_abs_f16x2, BuiltinID,
20460 case NVPTX::BI__nvvm_fmin_nan_f16:
20461 return MakeHalfType(Intrinsic::nvvm_fmin_nan_f16, BuiltinID, E, *
this);
20462 case NVPTX::BI__nvvm_fmin_nan_f16x2:
20463 return MakeHalfType(Intrinsic::nvvm_fmin_nan_f16x2, BuiltinID, E, *
this);
20464 case NVPTX::BI__nvvm_fmin_nan_xorsign_abs_f16:
20465 return MakeHalfType(Intrinsic::nvvm_fmin_nan_xorsign_abs_f16, BuiltinID, E,
20467 case NVPTX::BI__nvvm_fmin_nan_xorsign_abs_f16x2:
20468 return MakeHalfType(Intrinsic::nvvm_fmin_nan_xorsign_abs_f16x2, BuiltinID,
20470 case NVPTX::BI__nvvm_fmin_xorsign_abs_f16:
20471 return MakeHalfType(Intrinsic::nvvm_fmin_xorsign_abs_f16, BuiltinID, E,
20473 case NVPTX::BI__nvvm_fmin_xorsign_abs_f16x2:
20474 return MakeHalfType(Intrinsic::nvvm_fmin_xorsign_abs_f16x2, BuiltinID, E,
20476 case NVPTX::BI__nvvm_ldg_h:
20477 return MakeHalfType(Intrinsic::nvvm_ldg_global_f, BuiltinID, E, *
this);
20478 case NVPTX::BI__nvvm_ldg_h2:
20479 return MakeHalfType(Intrinsic::nvvm_ldg_global_f, BuiltinID, E, *
this);
20480 case NVPTX::BI__nvvm_ldu_h:
20481 return MakeHalfType(Intrinsic::nvvm_ldu_global_f, BuiltinID, E, *
this);
20482 case NVPTX::BI__nvvm_ldu_h2: {
20483 return MakeHalfType(Intrinsic::nvvm_ldu_global_f, BuiltinID, E, *
this);
20485 case NVPTX::BI__nvvm_cp_async_ca_shared_global_4:
20486 return MakeCpAsync(Intrinsic::nvvm_cp_async_ca_shared_global_4,
20487 Intrinsic::nvvm_cp_async_ca_shared_global_4_s, *
this, E,
20489 case NVPTX::BI__nvvm_cp_async_ca_shared_global_8:
20490 return MakeCpAsync(Intrinsic::nvvm_cp_async_ca_shared_global_8,
20491 Intrinsic::nvvm_cp_async_ca_shared_global_8_s, *
this, E,
20493 case NVPTX::BI__nvvm_cp_async_ca_shared_global_16:
20494 return MakeCpAsync(Intrinsic::nvvm_cp_async_ca_shared_global_16,
20495 Intrinsic::nvvm_cp_async_ca_shared_global_16_s, *
this, E,
20497 case NVPTX::BI__nvvm_cp_async_cg_shared_global_16:
20498 return MakeCpAsync(Intrinsic::nvvm_cp_async_cg_shared_global_16,
20499 Intrinsic::nvvm_cp_async_cg_shared_global_16_s, *
this, E,
20501 case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_x:
20504 case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_y:
20507 case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_z:
20510 case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_w:
20513 case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_x:
20516 case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_y:
20519 case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_z:
20522 case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_w:
20525 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_x:
20528 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_y:
20531 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_z:
20534 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_w:
20537 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_x:
20540 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_y:
20543 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_z:
20546 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_w:
20549 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctarank:
20552 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctarank:
20555 case NVPTX::BI__nvvm_is_explicit_cluster:
20558 case NVPTX::BI__nvvm_isspacep_shared_cluster:
20562 case NVPTX::BI__nvvm_mapa:
20565 {EmitScalarExpr(E->getArg(0)), EmitScalarExpr(E->getArg(1))});
20566 case NVPTX::BI__nvvm_mapa_shared_cluster:
20569 {EmitScalarExpr(E->getArg(0)), EmitScalarExpr(E->getArg(1))});
20570 case NVPTX::BI__nvvm_getctarank:
20574 case NVPTX::BI__nvvm_getctarank_shared_cluster:
20578 case NVPTX::BI__nvvm_barrier_cluster_arrive:
20581 case NVPTX::BI__nvvm_barrier_cluster_arrive_relaxed:
20584 case NVPTX::BI__nvvm_barrier_cluster_wait:
20587 case NVPTX::BI__nvvm_fence_sc_cluster:
20596struct BuiltinAlignArgs {
20597 llvm::Value *Src =
nullptr;
20598 llvm::Type *SrcType =
nullptr;
20599 llvm::Value *Alignment =
nullptr;
20600 llvm::Value *Mask =
nullptr;
20601 llvm::IntegerType *IntType =
nullptr;
20609 SrcType = Src->getType();
20610 if (SrcType->isPointerTy()) {
20611 IntType = IntegerType::get(
20615 assert(SrcType->isIntegerTy());
20616 IntType = cast<llvm::IntegerType>(SrcType);
20619 Alignment = CGF.
Builder.CreateZExtOrTrunc(Alignment, IntType,
"alignment");
20620 auto *One = llvm::ConstantInt::get(IntType, 1);
20621 Mask = CGF.
Builder.CreateSub(Alignment, One,
"mask");
20628 BuiltinAlignArgs Args(E, *
this);
20629 llvm::Value *SrcAddress = Args.Src;
20630 if (Args.SrcType->isPointerTy())
20632 Builder.CreateBitOrPointerCast(Args.Src, Args.IntType,
"src_addr");
20634 Builder.CreateAnd(SrcAddress, Args.Mask,
"set_bits"),
20635 llvm::Constant::getNullValue(Args.IntType),
"is_aligned"));
20642 BuiltinAlignArgs Args(E, *
this);
20643 llvm::Value *SrcForMask = Args.Src;
20649 if (Args.Src->getType()->isPointerTy()) {
20659 SrcForMask =
Builder.CreateAdd(SrcForMask, Args.Mask,
"over_boundary");
20663 llvm::Value *InvertedMask =
Builder.CreateNot(Args.Mask,
"inverted_mask");
20664 llvm::Value *
Result =
nullptr;
20665 if (Args.Src->getType()->isPointerTy()) {
20667 Intrinsic::ptrmask, {Args.SrcType, Args.IntType},
20668 {SrcForMask, InvertedMask},
nullptr,
"aligned_result");
20670 Result =
Builder.CreateAnd(SrcForMask, InvertedMask,
"aligned_result");
20672 assert(
Result->getType() == Args.SrcType);
20678 switch (BuiltinID) {
20679 case WebAssembly::BI__builtin_wasm_memory_size: {
20684 return Builder.CreateCall(Callee, I);
20686 case WebAssembly::BI__builtin_wasm_memory_grow: {
20692 return Builder.CreateCall(Callee, Args);
20694 case WebAssembly::BI__builtin_wasm_tls_size: {
20697 return Builder.CreateCall(Callee);
20699 case WebAssembly::BI__builtin_wasm_tls_align: {
20702 return Builder.CreateCall(Callee);
20704 case WebAssembly::BI__builtin_wasm_tls_base: {
20706 return Builder.CreateCall(Callee);
20708 case WebAssembly::BI__builtin_wasm_throw: {
20712 return Builder.CreateCall(Callee, {Tag, Obj});
20714 case WebAssembly::BI__builtin_wasm_rethrow: {
20716 return Builder.CreateCall(Callee);
20718 case WebAssembly::BI__builtin_wasm_memory_atomic_wait32: {
20725 case WebAssembly::BI__builtin_wasm_memory_atomic_wait64: {
20732 case WebAssembly::BI__builtin_wasm_memory_atomic_notify: {
20736 return Builder.CreateCall(Callee, {Addr, Count});
20738 case WebAssembly::BI__builtin_wasm_trunc_s_i32_f32:
20739 case WebAssembly::BI__builtin_wasm_trunc_s_i32_f64:
20740 case WebAssembly::BI__builtin_wasm_trunc_s_i64_f32:
20741 case WebAssembly::BI__builtin_wasm_trunc_s_i64_f64: {
20746 return Builder.CreateCall(Callee, {Src});
20748 case WebAssembly::BI__builtin_wasm_trunc_u_i32_f32:
20749 case WebAssembly::BI__builtin_wasm_trunc_u_i32_f64:
20750 case WebAssembly::BI__builtin_wasm_trunc_u_i64_f32:
20751 case WebAssembly::BI__builtin_wasm_trunc_u_i64_f64: {
20756 return Builder.CreateCall(Callee, {Src});
20758 case WebAssembly::BI__builtin_wasm_trunc_saturate_s_i32_f32:
20759 case WebAssembly::BI__builtin_wasm_trunc_saturate_s_i32_f64:
20760 case WebAssembly::BI__builtin_wasm_trunc_saturate_s_i64_f32:
20761 case WebAssembly::BI__builtin_wasm_trunc_saturate_s_i64_f64:
20762 case WebAssembly::BI__builtin_wasm_trunc_saturate_s_i32x4_f32x4: {
20767 return Builder.CreateCall(Callee, {Src});
20769 case WebAssembly::BI__builtin_wasm_trunc_saturate_u_i32_f32:
20770 case WebAssembly::BI__builtin_wasm_trunc_saturate_u_i32_f64:
20771 case WebAssembly::BI__builtin_wasm_trunc_saturate_u_i64_f32:
20772 case WebAssembly::BI__builtin_wasm_trunc_saturate_u_i64_f64:
20773 case WebAssembly::BI__builtin_wasm_trunc_saturate_u_i32x4_f32x4: {
20778 return Builder.CreateCall(Callee, {Src});
20780 case WebAssembly::BI__builtin_wasm_min_f32:
20781 case WebAssembly::BI__builtin_wasm_min_f64:
20782 case WebAssembly::BI__builtin_wasm_min_f32x4:
20783 case WebAssembly::BI__builtin_wasm_min_f64x2: {
20788 return Builder.CreateCall(Callee, {LHS, RHS});
20790 case WebAssembly::BI__builtin_wasm_max_f32:
20791 case WebAssembly::BI__builtin_wasm_max_f64:
20792 case WebAssembly::BI__builtin_wasm_max_f32x4:
20793 case WebAssembly::BI__builtin_wasm_max_f64x2: {
20798 return Builder.CreateCall(Callee, {LHS, RHS});
20800 case WebAssembly::BI__builtin_wasm_pmin_f32x4:
20801 case WebAssembly::BI__builtin_wasm_pmin_f64x2: {
20806 return Builder.CreateCall(Callee, {LHS, RHS});
20808 case WebAssembly::BI__builtin_wasm_pmax_f32x4:
20809 case WebAssembly::BI__builtin_wasm_pmax_f64x2: {
20814 return Builder.CreateCall(Callee, {LHS, RHS});
20816 case WebAssembly::BI__builtin_wasm_ceil_f32x4:
20817 case WebAssembly::BI__builtin_wasm_floor_f32x4:
20818 case WebAssembly::BI__builtin_wasm_trunc_f32x4:
20819 case WebAssembly::BI__builtin_wasm_nearest_f32x4:
20820 case WebAssembly::BI__builtin_wasm_ceil_f64x2:
20821 case WebAssembly::BI__builtin_wasm_floor_f64x2:
20822 case WebAssembly::BI__builtin_wasm_trunc_f64x2:
20823 case WebAssembly::BI__builtin_wasm_nearest_f64x2: {
20825 switch (BuiltinID) {
20826 case WebAssembly::BI__builtin_wasm_ceil_f32x4:
20827 case WebAssembly::BI__builtin_wasm_ceil_f64x2:
20828 IntNo = Intrinsic::ceil;
20830 case WebAssembly::BI__builtin_wasm_floor_f32x4:
20831 case WebAssembly::BI__builtin_wasm_floor_f64x2:
20832 IntNo = Intrinsic::floor;
20834 case WebAssembly::BI__builtin_wasm_trunc_f32x4:
20835 case WebAssembly::BI__builtin_wasm_trunc_f64x2:
20836 IntNo = Intrinsic::trunc;
20838 case WebAssembly::BI__builtin_wasm_nearest_f32x4:
20839 case WebAssembly::BI__builtin_wasm_nearest_f64x2:
20840 IntNo = Intrinsic::nearbyint;
20843 llvm_unreachable(
"unexpected builtin ID");
20849 case WebAssembly::BI__builtin_wasm_ref_null_extern: {
20851 return Builder.CreateCall(Callee);
20853 case WebAssembly::BI__builtin_wasm_ref_null_func: {
20855 return Builder.CreateCall(Callee);
20857 case WebAssembly::BI__builtin_wasm_swizzle_i8x16: {
20861 return Builder.CreateCall(Callee, {Src, Indices});
20863 case WebAssembly::BI__builtin_wasm_add_sat_s_i8x16:
20864 case WebAssembly::BI__builtin_wasm_add_sat_u_i8x16:
20865 case WebAssembly::BI__builtin_wasm_add_sat_s_i16x8:
20866 case WebAssembly::BI__builtin_wasm_add_sat_u_i16x8:
20867 case WebAssembly::BI__builtin_wasm_sub_sat_s_i8x16:
20868 case WebAssembly::BI__builtin_wasm_sub_sat_u_i8x16:
20869 case WebAssembly::BI__builtin_wasm_sub_sat_s_i16x8:
20870 case WebAssembly::BI__builtin_wasm_sub_sat_u_i16x8: {
20872 switch (BuiltinID) {
20873 case WebAssembly::BI__builtin_wasm_add_sat_s_i8x16:
20874 case WebAssembly::BI__builtin_wasm_add_sat_s_i16x8:
20875 IntNo = Intrinsic::sadd_sat;
20877 case WebAssembly::BI__builtin_wasm_add_sat_u_i8x16:
20878 case WebAssembly::BI__builtin_wasm_add_sat_u_i16x8:
20879 IntNo = Intrinsic::uadd_sat;
20881 case WebAssembly::BI__builtin_wasm_sub_sat_s_i8x16:
20882 case WebAssembly::BI__builtin_wasm_sub_sat_s_i16x8:
20883 IntNo = Intrinsic::wasm_sub_sat_signed;
20885 case WebAssembly::BI__builtin_wasm_sub_sat_u_i8x16:
20886 case WebAssembly::BI__builtin_wasm_sub_sat_u_i16x8:
20887 IntNo = Intrinsic::wasm_sub_sat_unsigned;
20890 llvm_unreachable(
"unexpected builtin ID");
20895 return Builder.CreateCall(Callee, {LHS, RHS});
20897 case WebAssembly::BI__builtin_wasm_abs_i8x16:
20898 case WebAssembly::BI__builtin_wasm_abs_i16x8:
20899 case WebAssembly::BI__builtin_wasm_abs_i32x4:
20900 case WebAssembly::BI__builtin_wasm_abs_i64x2: {
20903 Constant *
Zero = llvm::Constant::getNullValue(Vec->
getType());
20904 Value *ICmp =
Builder.CreateICmpSLT(Vec, Zero,
"abscond");
20905 return Builder.CreateSelect(ICmp, Neg, Vec,
"abs");
20907 case WebAssembly::BI__builtin_wasm_min_s_i8x16:
20908 case WebAssembly::BI__builtin_wasm_min_u_i8x16:
20909 case WebAssembly::BI__builtin_wasm_max_s_i8x16:
20910 case WebAssembly::BI__builtin_wasm_max_u_i8x16:
20911 case WebAssembly::BI__builtin_wasm_min_s_i16x8:
20912 case WebAssembly::BI__builtin_wasm_min_u_i16x8:
20913 case WebAssembly::BI__builtin_wasm_max_s_i16x8:
20914 case WebAssembly::BI__builtin_wasm_max_u_i16x8:
20915 case WebAssembly::BI__builtin_wasm_min_s_i32x4:
20916 case WebAssembly::BI__builtin_wasm_min_u_i32x4:
20917 case WebAssembly::BI__builtin_wasm_max_s_i32x4:
20918 case WebAssembly::BI__builtin_wasm_max_u_i32x4: {
20922 switch (BuiltinID) {
20923 case WebAssembly::BI__builtin_wasm_min_s_i8x16:
20924 case WebAssembly::BI__builtin_wasm_min_s_i16x8:
20925 case WebAssembly::BI__builtin_wasm_min_s_i32x4:
20926 ICmp =
Builder.CreateICmpSLT(LHS, RHS);
20928 case WebAssembly::BI__builtin_wasm_min_u_i8x16:
20929 case WebAssembly::BI__builtin_wasm_min_u_i16x8:
20930 case WebAssembly::BI__builtin_wasm_min_u_i32x4:
20931 ICmp =
Builder.CreateICmpULT(LHS, RHS);
20933 case WebAssembly::BI__builtin_wasm_max_s_i8x16:
20934 case WebAssembly::BI__builtin_wasm_max_s_i16x8:
20935 case WebAssembly::BI__builtin_wasm_max_s_i32x4:
20936 ICmp =
Builder.CreateICmpSGT(LHS, RHS);
20938 case WebAssembly::BI__builtin_wasm_max_u_i8x16:
20939 case WebAssembly::BI__builtin_wasm_max_u_i16x8:
20940 case WebAssembly::BI__builtin_wasm_max_u_i32x4:
20941 ICmp =
Builder.CreateICmpUGT(LHS, RHS);
20944 llvm_unreachable(
"unexpected builtin ID");
20946 return Builder.CreateSelect(ICmp, LHS, RHS);
20948 case WebAssembly::BI__builtin_wasm_avgr_u_i8x16:
20949 case WebAssembly::BI__builtin_wasm_avgr_u_i16x8: {
20954 return Builder.CreateCall(Callee, {LHS, RHS});
20956 case WebAssembly::BI__builtin_wasm_q15mulr_sat_s_i16x8: {
20960 return Builder.CreateCall(Callee, {LHS, RHS});
20962 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i8x16_s_i16x8:
20963 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i8x16_u_i16x8:
20964 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i16x8_s_i32x4:
20965 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i16x8_u_i32x4: {
20968 switch (BuiltinID) {
20969 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i8x16_s_i16x8:
20970 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i16x8_s_i32x4:
20971 IntNo = Intrinsic::wasm_extadd_pairwise_signed;
20973 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i8x16_u_i16x8:
20974 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i16x8_u_i32x4:
20975 IntNo = Intrinsic::wasm_extadd_pairwise_unsigned;
20978 llvm_unreachable(
"unexpected builtin ID");
20982 return Builder.CreateCall(Callee, Vec);
20984 case WebAssembly::BI__builtin_wasm_bitselect: {
20990 return Builder.CreateCall(Callee, {V1, V2,
C});
20992 case WebAssembly::BI__builtin_wasm_dot_s_i32x4_i16x8: {
20996 return Builder.CreateCall(Callee, {LHS, RHS});
20998 case WebAssembly::BI__builtin_wasm_popcnt_i8x16: {
21002 return Builder.CreateCall(Callee, {Vec});
21004 case WebAssembly::BI__builtin_wasm_any_true_v128:
21005 case WebAssembly::BI__builtin_wasm_all_true_i8x16:
21006 case WebAssembly::BI__builtin_wasm_all_true_i16x8:
21007 case WebAssembly::BI__builtin_wasm_all_true_i32x4:
21008 case WebAssembly::BI__builtin_wasm_all_true_i64x2: {
21010 switch (BuiltinID) {
21011 case WebAssembly::BI__builtin_wasm_any_true_v128:
21012 IntNo = Intrinsic::wasm_anytrue;
21014 case WebAssembly::BI__builtin_wasm_all_true_i8x16:
21015 case WebAssembly::BI__builtin_wasm_all_true_i16x8:
21016 case WebAssembly::BI__builtin_wasm_all_true_i32x4:
21017 case WebAssembly::BI__builtin_wasm_all_true_i64x2:
21018 IntNo = Intrinsic::wasm_alltrue;
21021 llvm_unreachable(
"unexpected builtin ID");
21025 return Builder.CreateCall(Callee, {Vec});
21027 case WebAssembly::BI__builtin_wasm_bitmask_i8x16:
21028 case WebAssembly::BI__builtin_wasm_bitmask_i16x8:
21029 case WebAssembly::BI__builtin_wasm_bitmask_i32x4:
21030 case WebAssembly::BI__builtin_wasm_bitmask_i64x2: {
21034 return Builder.CreateCall(Callee, {Vec});
21036 case WebAssembly::BI__builtin_wasm_abs_f32x4:
21037 case WebAssembly::BI__builtin_wasm_abs_f64x2: {
21040 return Builder.CreateCall(Callee, {Vec});
21042 case WebAssembly::BI__builtin_wasm_sqrt_f32x4:
21043 case WebAssembly::BI__builtin_wasm_sqrt_f64x2: {
21046 return Builder.CreateCall(Callee, {Vec});
21048 case WebAssembly::BI__builtin_wasm_narrow_s_i8x16_i16x8:
21049 case WebAssembly::BI__builtin_wasm_narrow_u_i8x16_i16x8:
21050 case WebAssembly::BI__builtin_wasm_narrow_s_i16x8_i32x4:
21051 case WebAssembly::BI__builtin_wasm_narrow_u_i16x8_i32x4: {
21055 switch (BuiltinID) {
21056 case WebAssembly::BI__builtin_wasm_narrow_s_i8x16_i16x8:
21057 case WebAssembly::BI__builtin_wasm_narrow_s_i16x8_i32x4:
21058 IntNo = Intrinsic::wasm_narrow_signed;
21060 case WebAssembly::BI__builtin_wasm_narrow_u_i8x16_i16x8:
21061 case WebAssembly::BI__builtin_wasm_narrow_u_i16x8_i32x4:
21062 IntNo = Intrinsic::wasm_narrow_unsigned;
21065 llvm_unreachable(
"unexpected builtin ID");
21069 return Builder.CreateCall(Callee, {Low, High});
21071 case WebAssembly::BI__builtin_wasm_trunc_sat_s_zero_f64x2_i32x4:
21072 case WebAssembly::BI__builtin_wasm_trunc_sat_u_zero_f64x2_i32x4: {
21075 switch (BuiltinID) {
21076 case WebAssembly::BI__builtin_wasm_trunc_sat_s_zero_f64x2_i32x4:
21077 IntNo = Intrinsic::fptosi_sat;
21079 case WebAssembly::BI__builtin_wasm_trunc_sat_u_zero_f64x2_i32x4:
21080 IntNo = Intrinsic::fptoui_sat;
21083 llvm_unreachable(
"unexpected builtin ID");
21085 llvm::Type *SrcT = Vec->
getType();
21086 llvm::Type *TruncT = SrcT->getWithNewType(
Builder.getInt32Ty());
21089 Value *Splat = Constant::getNullValue(TruncT);
21092 case WebAssembly::BI__builtin_wasm_shuffle_i8x16: {
21097 while (OpIdx < 18) {
21098 std::optional<llvm::APSInt> LaneConst =
21100 assert(LaneConst &&
"Constant arg isn't actually constant?");
21101 Ops[OpIdx++] = llvm::ConstantInt::get(
getLLVMContext(), *LaneConst);
21104 return Builder.CreateCall(Callee, Ops);
21106 case WebAssembly::BI__builtin_wasm_relaxed_madd_f32x4:
21107 case WebAssembly::BI__builtin_wasm_relaxed_nmadd_f32x4:
21108 case WebAssembly::BI__builtin_wasm_relaxed_madd_f64x2:
21109 case WebAssembly::BI__builtin_wasm_relaxed_nmadd_f64x2: {
21114 switch (BuiltinID) {
21115 case WebAssembly::BI__builtin_wasm_relaxed_madd_f32x4:
21116 case WebAssembly::BI__builtin_wasm_relaxed_madd_f64x2:
21117 IntNo = Intrinsic::wasm_relaxed_madd;
21119 case WebAssembly::BI__builtin_wasm_relaxed_nmadd_f32x4:
21120 case WebAssembly::BI__builtin_wasm_relaxed_nmadd_f64x2:
21121 IntNo = Intrinsic::wasm_relaxed_nmadd;
21124 llvm_unreachable(
"unexpected builtin ID");
21127 return Builder.CreateCall(Callee, {A, B,
C});
21129 case WebAssembly::BI__builtin_wasm_relaxed_laneselect_i8x16:
21130 case WebAssembly::BI__builtin_wasm_relaxed_laneselect_i16x8:
21131 case WebAssembly::BI__builtin_wasm_relaxed_laneselect_i32x4:
21132 case WebAssembly::BI__builtin_wasm_relaxed_laneselect_i64x2: {
21138 return Builder.CreateCall(Callee, {A, B,
C});
21140 case WebAssembly::BI__builtin_wasm_relaxed_swizzle_i8x16: {
21144 return Builder.CreateCall(Callee, {Src, Indices});
21146 case WebAssembly::BI__builtin_wasm_relaxed_min_f32x4:
21147 case WebAssembly::BI__builtin_wasm_relaxed_max_f32x4:
21148 case WebAssembly::BI__builtin_wasm_relaxed_min_f64x2:
21149 case WebAssembly::BI__builtin_wasm_relaxed_max_f64x2: {
21153 switch (BuiltinID) {
21154 case WebAssembly::BI__builtin_wasm_relaxed_min_f32x4:
21155 case WebAssembly::BI__builtin_wasm_relaxed_min_f64x2:
21156 IntNo = Intrinsic::wasm_relaxed_min;
21158 case WebAssembly::BI__builtin_wasm_relaxed_max_f32x4:
21159 case WebAssembly::BI__builtin_wasm_relaxed_max_f64x2:
21160 IntNo = Intrinsic::wasm_relaxed_max;
21163 llvm_unreachable(
"unexpected builtin ID");
21166 return Builder.CreateCall(Callee, {LHS, RHS});
21168 case WebAssembly::BI__builtin_wasm_relaxed_trunc_s_i32x4_f32x4:
21169 case WebAssembly::BI__builtin_wasm_relaxed_trunc_u_i32x4_f32x4:
21170 case WebAssembly::BI__builtin_wasm_relaxed_trunc_s_zero_i32x4_f64x2:
21171 case WebAssembly::BI__builtin_wasm_relaxed_trunc_u_zero_i32x4_f64x2: {
21174 switch (BuiltinID) {
21175 case WebAssembly::BI__builtin_wasm_relaxed_trunc_s_i32x4_f32x4:
21176 IntNo = Intrinsic::wasm_relaxed_trunc_signed;
21178 case WebAssembly::BI__builtin_wasm_relaxed_trunc_u_i32x4_f32x4:
21179 IntNo = Intrinsic::wasm_relaxed_trunc_unsigned;
21181 case WebAssembly::BI__builtin_wasm_relaxed_trunc_s_zero_i32x4_f64x2:
21182 IntNo = Intrinsic::wasm_relaxed_trunc_signed_zero;
21184 case WebAssembly::BI__builtin_wasm_relaxed_trunc_u_zero_i32x4_f64x2:
21185 IntNo = Intrinsic::wasm_relaxed_trunc_unsigned_zero;
21188 llvm_unreachable(
"unexpected builtin ID");
21191 return Builder.CreateCall(Callee, {Vec});
21193 case WebAssembly::BI__builtin_wasm_relaxed_q15mulr_s_i16x8: {
21197 return Builder.CreateCall(Callee, {LHS, RHS});
21199 case WebAssembly::BI__builtin_wasm_relaxed_dot_i8x16_i7x16_s_i16x8: {
21204 return Builder.CreateCall(Callee, {LHS, RHS});
21206 case WebAssembly::BI__builtin_wasm_relaxed_dot_i8x16_i7x16_add_s_i32x4: {
21211 CGM.
getIntrinsic(Intrinsic::wasm_relaxed_dot_i8x16_i7x16_add_signed);
21212 return Builder.CreateCall(Callee, {LHS, RHS, Acc});
21214 case WebAssembly::BI__builtin_wasm_relaxed_dot_bf16x8_add_f32_f32x4: {
21220 return Builder.CreateCall(Callee, {LHS, RHS, Acc});
21222 case WebAssembly::BI__builtin_wasm_loadf16_f32: {
21225 return Builder.CreateCall(Callee, {Addr});
21227 case WebAssembly::BI__builtin_wasm_storef16_f32: {
21231 return Builder.CreateCall(Callee, {Val, Addr});
21233 case WebAssembly::BI__builtin_wasm_table_get: {
21244 "Unexpected reference type for __builtin_wasm_table_get");
21245 return Builder.CreateCall(Callee, {Table, Index});
21247 case WebAssembly::BI__builtin_wasm_table_set: {
21259 "Unexpected reference type for __builtin_wasm_table_set");
21260 return Builder.CreateCall(Callee, {Table, Index, Val});
21262 case WebAssembly::BI__builtin_wasm_table_size: {
21268 case WebAssembly::BI__builtin_wasm_table_grow: {
21281 "Unexpected reference type for __builtin_wasm_table_grow");
21283 return Builder.CreateCall(Callee, {Table, Val, NElems});
21285 case WebAssembly::BI__builtin_wasm_table_fill: {
21299 "Unexpected reference type for __builtin_wasm_table_fill");
21301 return Builder.CreateCall(Callee, {Table, Index, Val, NElems});
21303 case WebAssembly::BI__builtin_wasm_table_copy: {
21313 return Builder.CreateCall(Callee, {TableX, TableY, SrcIdx, DstIdx, NElems});
21320static std::pair<Intrinsic::ID, unsigned>
21323 unsigned BuiltinID;
21324 Intrinsic::ID IntrinsicID;
21327 static Info Infos[] = {
21328#define CUSTOM_BUILTIN_MAPPING(x,s) \
21329 { Hexagon::BI__builtin_HEXAGON_##x, Intrinsic::hexagon_##x, s },
21361#include "clang/Basic/BuiltinsHexagonMapCustomDep.def"
21362#undef CUSTOM_BUILTIN_MAPPING
21365 auto CmpInfo = [] (Info A, Info B) {
return A.BuiltinID < B.BuiltinID; };
21366 static const bool SortOnce = (llvm::sort(Infos, CmpInfo),
true);
21369 const Info *F = llvm::lower_bound(Infos, Info{BuiltinID, 0, 0}, CmpInfo);
21370 if (F == std::end(Infos) || F->BuiltinID != BuiltinID)
21371 return {Intrinsic::not_intrinsic, 0};
21373 return {F->IntrinsicID, F->VecLen};
21382 auto MakeCircOp = [
this, E](
unsigned IntID,
bool IsLoad) {
21396 for (
unsigned i = 1, e = E->
getNumArgs(); i != e; ++i)
21402 llvm::Value *NewBase = IsLoad ?
Builder.CreateExtractValue(
Result, 1)
21406 llvm::Value *RetVal =
21416 auto MakeBrevLd = [
this, E](
unsigned IntID, llvm::Type *DestTy) {
21433 CGM.
getIntrinsic(IntID), {BaseAddress, EmitScalarExpr(E->getArg(2))});
21436 llvm::Value *DestVal =
Builder.CreateExtractValue(
Result, 0);
21441 DestVal =
Builder.CreateTrunc(DestVal, DestTy);
21448 auto V2Q = [
this, VecLen] (llvm::Value *Vec) {
21449 Intrinsic::ID
ID = VecLen == 128 ? Intrinsic::hexagon_V6_vandvrt_128B
21450 : Intrinsic::hexagon_V6_vandvrt;
21452 {Vec,
Builder.getInt32(-1)});
21454 auto Q2V = [
this, VecLen] (llvm::Value *Pred) {
21455 Intrinsic::ID
ID = VecLen == 128 ? Intrinsic::hexagon_V6_vandqrt_128B
21456 : Intrinsic::hexagon_V6_vandqrt;
21458 {Pred,
Builder.getInt32(-1)});
21461 switch (BuiltinID) {
21465 case Hexagon::BI__builtin_HEXAGON_V6_vaddcarry:
21466 case Hexagon::BI__builtin_HEXAGON_V6_vaddcarry_128B:
21467 case Hexagon::BI__builtin_HEXAGON_V6_vsubcarry:
21468 case Hexagon::BI__builtin_HEXAGON_V6_vsubcarry_128B: {
21475 {EmitScalarExpr(E->getArg(0)), EmitScalarExpr(E->getArg(1)), PredIn});
21477 llvm::Value *PredOut =
Builder.CreateExtractValue(
Result, 1);
21485 case Hexagon::BI__builtin_HEXAGON_V6_vaddcarryo:
21486 case Hexagon::BI__builtin_HEXAGON_V6_vaddcarryo_128B:
21487 case Hexagon::BI__builtin_HEXAGON_V6_vsubcarryo:
21488 case Hexagon::BI__builtin_HEXAGON_V6_vsubcarryo_128B: {
21494 {EmitScalarExpr(E->getArg(0)), EmitScalarExpr(E->getArg(1))});
21496 llvm::Value *PredOut =
Builder.CreateExtractValue(
Result, 1);
21502 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstoreq:
21503 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstorenq:
21504 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstorentq:
21505 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstorentnq:
21506 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstoreq_128B:
21507 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstorenq_128B:
21508 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstorentq_128B:
21509 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstorentnq_128B: {
21513 if (
auto *Cast = dyn_cast<ImplicitCastExpr>(PredOp)) {
21514 if (
Cast->getCastKind() == CK_BitCast)
21515 PredOp =
Cast->getSubExpr();
21518 for (
int i = 1, e = E->
getNumArgs(); i != e; ++i)
21523 case Hexagon::BI__builtin_HEXAGON_L2_loadrub_pci:
21524 case Hexagon::BI__builtin_HEXAGON_L2_loadrb_pci:
21525 case Hexagon::BI__builtin_HEXAGON_L2_loadruh_pci:
21526 case Hexagon::BI__builtin_HEXAGON_L2_loadrh_pci:
21527 case Hexagon::BI__builtin_HEXAGON_L2_loadri_pci:
21528 case Hexagon::BI__builtin_HEXAGON_L2_loadrd_pci:
21529 case Hexagon::BI__builtin_HEXAGON_L2_loadrub_pcr:
21530 case Hexagon::BI__builtin_HEXAGON_L2_loadrb_pcr:
21531 case Hexagon::BI__builtin_HEXAGON_L2_loadruh_pcr:
21532 case Hexagon::BI__builtin_HEXAGON_L2_loadrh_pcr:
21533 case Hexagon::BI__builtin_HEXAGON_L2_loadri_pcr:
21534 case Hexagon::BI__builtin_HEXAGON_L2_loadrd_pcr:
21535 return MakeCircOp(ID,
true);
21536 case Hexagon::BI__builtin_HEXAGON_S2_storerb_pci:
21537 case Hexagon::BI__builtin_HEXAGON_S2_storerh_pci:
21538 case Hexagon::BI__builtin_HEXAGON_S2_storerf_pci:
21539 case Hexagon::BI__builtin_HEXAGON_S2_storeri_pci:
21540 case Hexagon::BI__builtin_HEXAGON_S2_storerd_pci:
21541 case Hexagon::BI__builtin_HEXAGON_S2_storerb_pcr:
21542 case Hexagon::BI__builtin_HEXAGON_S2_storerh_pcr:
21543 case Hexagon::BI__builtin_HEXAGON_S2_storerf_pcr:
21544 case Hexagon::BI__builtin_HEXAGON_S2_storeri_pcr:
21545 case Hexagon::BI__builtin_HEXAGON_S2_storerd_pcr:
21546 return MakeCircOp(ID,
false);
21547 case Hexagon::BI__builtin_brev_ldub:
21548 return MakeBrevLd(Intrinsic::hexagon_L2_loadrub_pbr,
Int8Ty);
21549 case Hexagon::BI__builtin_brev_ldb:
21550 return MakeBrevLd(Intrinsic::hexagon_L2_loadrb_pbr,
Int8Ty);
21551 case Hexagon::BI__builtin_brev_lduh:
21552 return MakeBrevLd(Intrinsic::hexagon_L2_loadruh_pbr,
Int16Ty);
21553 case Hexagon::BI__builtin_brev_ldh:
21554 return MakeBrevLd(Intrinsic::hexagon_L2_loadrh_pbr,
Int16Ty);
21555 case Hexagon::BI__builtin_brev_ldw:
21556 return MakeBrevLd(Intrinsic::hexagon_L2_loadri_pbr,
Int32Ty);
21557 case Hexagon::BI__builtin_brev_ldd:
21558 return MakeBrevLd(Intrinsic::hexagon_L2_loadrd_pbr,
Int64Ty);
21571 unsigned ICEArguments = 0;
21579 if (BuiltinID == RISCVVector::BI__builtin_rvv_vget_v ||
21580 BuiltinID == RISCVVector::BI__builtin_rvv_vset_v)
21581 ICEArguments = 1 << 1;
21586 if (BuiltinID == RISCV::BI__builtin_riscv_ntl_load)
21587 ICEArguments |= (1 << 1);
21588 if (BuiltinID == RISCV::BI__builtin_riscv_ntl_store)
21589 ICEArguments |= (1 << 2);
21591 for (
unsigned i = 0, e = E->
getNumArgs(); i != e; i++) {
21596 Ops.push_back(AggValue);
21602 Intrinsic::ID
ID = Intrinsic::not_intrinsic;
21606 constexpr unsigned RVV_VTA = 0x1;
21607 constexpr unsigned RVV_VMA = 0x2;
21608 int PolicyAttrs = 0;
21609 bool IsMasked =
false;
21613 switch (BuiltinID) {
21614 default: llvm_unreachable(
"unexpected builtin ID");
21615 case RISCV::BI__builtin_riscv_orc_b_32:
21616 case RISCV::BI__builtin_riscv_orc_b_64:
21617 case RISCV::BI__builtin_riscv_clz_32:
21618 case RISCV::BI__builtin_riscv_clz_64:
21619 case RISCV::BI__builtin_riscv_ctz_32:
21620 case RISCV::BI__builtin_riscv_ctz_64:
21621 case RISCV::BI__builtin_riscv_clmul_32:
21622 case RISCV::BI__builtin_riscv_clmul_64:
21623 case RISCV::BI__builtin_riscv_clmulh_32:
21624 case RISCV::BI__builtin_riscv_clmulh_64:
21625 case RISCV::BI__builtin_riscv_clmulr_32:
21626 case RISCV::BI__builtin_riscv_clmulr_64:
21627 case RISCV::BI__builtin_riscv_xperm4_32:
21628 case RISCV::BI__builtin_riscv_xperm4_64:
21629 case RISCV::BI__builtin_riscv_xperm8_32:
21630 case RISCV::BI__builtin_riscv_xperm8_64:
21631 case RISCV::BI__builtin_riscv_brev8_32:
21632 case RISCV::BI__builtin_riscv_brev8_64:
21633 case RISCV::BI__builtin_riscv_zip_32:
21634 case RISCV::BI__builtin_riscv_unzip_32: {
21635 switch (BuiltinID) {
21636 default: llvm_unreachable(
"unexpected builtin ID");
21638 case RISCV::BI__builtin_riscv_orc_b_32:
21639 case RISCV::BI__builtin_riscv_orc_b_64:
21640 ID = Intrinsic::riscv_orc_b;
21642 case RISCV::BI__builtin_riscv_clz_32:
21643 case RISCV::BI__builtin_riscv_clz_64: {
21646 if (
Result->getType() != ResultType)
21651 case RISCV::BI__builtin_riscv_ctz_32:
21652 case RISCV::BI__builtin_riscv_ctz_64: {
21655 if (
Result->getType() != ResultType)
21662 case RISCV::BI__builtin_riscv_clmul_32:
21663 case RISCV::BI__builtin_riscv_clmul_64:
21664 ID = Intrinsic::riscv_clmul;
21666 case RISCV::BI__builtin_riscv_clmulh_32:
21667 case RISCV::BI__builtin_riscv_clmulh_64:
21668 ID = Intrinsic::riscv_clmulh;
21670 case RISCV::BI__builtin_riscv_clmulr_32:
21671 case RISCV::BI__builtin_riscv_clmulr_64:
21672 ID = Intrinsic::riscv_clmulr;
21676 case RISCV::BI__builtin_riscv_xperm8_32:
21677 case RISCV::BI__builtin_riscv_xperm8_64:
21678 ID = Intrinsic::riscv_xperm8;
21680 case RISCV::BI__builtin_riscv_xperm4_32:
21681 case RISCV::BI__builtin_riscv_xperm4_64:
21682 ID = Intrinsic::riscv_xperm4;
21686 case RISCV::BI__builtin_riscv_brev8_32:
21687 case RISCV::BI__builtin_riscv_brev8_64:
21688 ID = Intrinsic::riscv_brev8;
21690 case RISCV::BI__builtin_riscv_zip_32:
21691 ID = Intrinsic::riscv_zip;
21693 case RISCV::BI__builtin_riscv_unzip_32:
21694 ID = Intrinsic::riscv_unzip;
21698 IntrinsicTypes = {ResultType};
21705 case RISCV::BI__builtin_riscv_sha256sig0:
21706 ID = Intrinsic::riscv_sha256sig0;
21708 case RISCV::BI__builtin_riscv_sha256sig1:
21709 ID = Intrinsic::riscv_sha256sig1;
21711 case RISCV::BI__builtin_riscv_sha256sum0:
21712 ID = Intrinsic::riscv_sha256sum0;
21714 case RISCV::BI__builtin_riscv_sha256sum1:
21715 ID = Intrinsic::riscv_sha256sum1;
21719 case RISCV::BI__builtin_riscv_sm4ks:
21720 ID = Intrinsic::riscv_sm4ks;
21722 case RISCV::BI__builtin_riscv_sm4ed:
21723 ID = Intrinsic::riscv_sm4ed;
21727 case RISCV::BI__builtin_riscv_sm3p0:
21728 ID = Intrinsic::riscv_sm3p0;
21730 case RISCV::BI__builtin_riscv_sm3p1:
21731 ID = Intrinsic::riscv_sm3p1;
21735 case RISCV::BI__builtin_riscv_ntl_load: {
21737 unsigned DomainVal = 5;
21738 if (Ops.size() == 2)
21739 DomainVal = cast<ConstantInt>(Ops[1])->getZExtValue();
21741 llvm::MDNode *RISCVDomainNode = llvm::MDNode::get(
21743 llvm::ConstantAsMetadata::get(
Builder.getInt32(DomainVal)));
21744 llvm::MDNode *NontemporalNode = llvm::MDNode::get(
21748 if(ResTy->isScalableTy()) {
21749 const ScalableVectorType *SVTy = cast<ScalableVectorType>(ResTy);
21750 llvm::Type *ScalarTy = ResTy->getScalarType();
21751 Width = ScalarTy->getPrimitiveSizeInBits() *
21752 SVTy->getElementCount().getKnownMinValue();
21754 Width = ResTy->getPrimitiveSizeInBits();
21758 Load->setMetadata(llvm::LLVMContext::MD_nontemporal, NontemporalNode);
21759 Load->setMetadata(
CGM.
getModule().getMDKindID(
"riscv-nontemporal-domain"),
21764 case RISCV::BI__builtin_riscv_ntl_store: {
21765 unsigned DomainVal = 5;
21766 if (Ops.size() == 3)
21767 DomainVal = cast<ConstantInt>(Ops[2])->getZExtValue();
21769 llvm::MDNode *RISCVDomainNode = llvm::MDNode::get(
21771 llvm::ConstantAsMetadata::get(
Builder.getInt32(DomainVal)));
21772 llvm::MDNode *NontemporalNode = llvm::MDNode::get(
21776 Store->setMetadata(llvm::LLVMContext::MD_nontemporal, NontemporalNode);
21784#include "clang/Basic/riscv_vector_builtin_cg.inc"
21786#include "clang/Basic/riscv_sifive_vector_builtin_cg.inc"
21789 assert(ID != Intrinsic::not_intrinsic);
21792 return Builder.CreateCall(F, Ops,
"");
Defines the clang::ASTContext interface.
#define PPC_LNX_FEATURE(NAME, DESC, ENUMNAME, ENUMVAL, HWCAPN)
#define X86_CPU_SUBTYPE(ENUM, STR)
#define X86_CPU_SUBTYPE_ALIAS(ENUM, ALIAS)
#define X86_VENDOR(ENUM, STRING)
#define X86_CPU_TYPE_ALIAS(ENUM, ALIAS)
#define X86_CPU_TYPE(ENUM, STR)
static constexpr Builtin::Info BuiltinInfo[]
static void Accumulate(SMap &SM, CFGBlock *B)
static Value * EmitSpecialRegisterBuiltin(CodeGenFunction &CGF, const CallExpr *E, llvm::Type *RegisterType, llvm::Type *ValueType, SpecialRegisterAccessKind AccessKind, StringRef SysReg="")
static llvm::Value * ARMMVEVectorReinterpret(CGBuilderTy &Builder, CodeGenFunction *CGF, llvm::Value *V, llvm::Type *DestType)
static Value * MakeBinaryAtomicValue(CodeGenFunction &CGF, llvm::AtomicRMWInst::BinOp Kind, const CallExpr *E, AtomicOrdering Ordering=AtomicOrdering::SequentiallyConsistent)
Utility to insert an atomic instruction based on Intrinsic::ID and the expression node.
static char bitActionToX86BTCode(BitTest::ActionKind A)
#define NEONMAP2(NameBase, LLVMIntrinsic, AltLLVMIntrinsic, TypeModifier)
static Value * EmitAtomicCmpXchg128ForMSIntrin(CodeGenFunction &CGF, const CallExpr *E, AtomicOrdering SuccessOrdering)
#define INTRINSIC_X86_XSAVE_ID(NAME)
static CanQualType getOSLogArgType(ASTContext &C, int Size)
Get the argument type for arguments to os_log_helper.
static Value * EmitOverflowCheckedAbs(CodeGenFunction &CGF, const CallExpr *E, bool SanitizeOverflow)
static llvm::VectorType * GetFloatNeonType(CodeGenFunction *CGF, NeonTypeFlags IntTypeFlags)
static Value * tryUseTestFPKind(CodeGenFunction &CGF, unsigned BuiltinID, Value *V)
static llvm::Value * MVEImmediateShr(CGBuilderTy &Builder, llvm::Value *V, uint32_t Shift, bool Unsigned)
static bool areBOSTypesCompatible(int From, int To)
Checks if using the result of __builtin_object_size(p, From) in place of __builtin_object_size(p,...
static llvm::Value * SignOrZeroExtend(CGBuilderTy &Builder, llvm::Value *V, llvm::Type *T, bool Unsigned)
static void InsertExplicitZeroOperand(CGBuilderTy &Builder, llvm::Type *Ty, SmallVectorImpl< Value * > &Ops)
static const ARMVectorIntrinsicInfo AArch64SMEIntrinsicMap[]
static Value * EmitAArch64TblBuiltinExpr(CodeGenFunction &CGF, unsigned BuiltinID, const CallExpr *E, SmallVectorImpl< Value * > &Ops, llvm::Triple::ArchType Arch)
#define MMA_VARIANTS_B1_AND(geom, type)
static void swapCommutativeSMEOperands(unsigned BuiltinID, SmallVectorImpl< Value * > &Ops)
static bool AArch64SISDIntrinsicsProvenSorted
static Value * EmitX86CompressExpand(CodeGenFunction &CGF, ArrayRef< Value * > Ops, bool IsCompress)
static const ARMVectorIntrinsicInfo AArch64SVEIntrinsicMap[]
static bool HasExtraNeonArgument(unsigned BuiltinID)
Return true if BuiltinID is an overloaded Neon intrinsic with an extra argument that specifies the ve...
static bool TypeRequiresBuiltinLaunderImp(const ASTContext &Ctx, QualType Ty, llvm::SmallPtrSetImpl< const Decl * > &Seen)
static Value * EmitAtomicIncrementValue(CodeGenFunction &CGF, const CallExpr *E, AtomicOrdering Ordering=AtomicOrdering::SequentiallyConsistent)
static std::pair< Intrinsic::ID, unsigned > getIntrinsicForHexagonNonClangBuiltin(unsigned BuiltinID)
static Value * emitRangedBuiltin(CodeGenFunction &CGF, unsigned IntrinsicID, int low, int high)
#define MMA_INTR(geom_op_type, layout)
static Value * EmitX86MaskedCompare(CodeGenFunction &CGF, unsigned CC, bool Signed, ArrayRef< Value * > Ops)
static Value * emitUnaryMaybeConstrainedFPBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID, unsigned ConstrainedIntrinsicID)
#define NEONMAP1(NameBase, LLVMIntrinsic, TypeModifier)
static bool AArch64SVEIntrinsicsProvenSorted
static RValue EmitMSVCRTSetJmp(CodeGenFunction &CGF, MSVCSetJmpKind SJKind, const CallExpr *E)
MSVC handles setjmp a bit differently on different platforms.
static const ARMVectorIntrinsicInfo * findARMVectorIntrinsicInMap(ArrayRef< ARMVectorIntrinsicInfo > IntrinsicMap, unsigned BuiltinID, bool &MapProvenSorted)
static Value * EmitScalarFMAExpr(CodeGenFunction &CGF, const CallExpr *E, MutableArrayRef< Value * > Ops, Value *Upper, bool ZeroMask=false, unsigned PTIdx=0, bool NegAcc=false)
#define MUTATE_LDBL(func)
static Value * EmitX86ExpandLoad(CodeGenFunction &CGF, ArrayRef< Value * > Ops)
static unsigned CountCountedByAttrs(const RecordDecl *RD)
static Value * emitMaybeConstrainedFPToIntRoundBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID, unsigned ConstrainedIntrinsicID)
static Value * EmitX86MaskedLoad(CodeGenFunction &CGF, ArrayRef< Value * > Ops, Align Alignment)
static bool TypeRequiresBuiltinLaunder(CodeGenModule &CGM, QualType Ty)
Determine if the specified type requires laundering by checking if it is a dynamic class type or cont...
static void InsertExplicitUndefOperand(CGBuilderTy &Builder, llvm::Type *Ty, SmallVectorImpl< Value * > &Ops)
static Value * EmitISOVolatileLoad(CodeGenFunction &CGF, const CallExpr *E)
static struct WidthAndSignedness EncompassingIntegerType(ArrayRef< struct WidthAndSignedness > Types)
static Integer GetIntegerConstantValue(const Expr *E, ASTContext &Context)
#define MMA_VARIANTS(geom, type)
static bool AArch64SMEIntrinsicsProvenSorted
static llvm::Value * VectorZip(CGBuilderTy &Builder, llvm::Value *V0, llvm::Value *V1)
static Value * EmitTargetArchBuiltinExpr(CodeGenFunction *CGF, unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue, llvm::Triple::ArchType Arch)
constexpr unsigned SVEBitsPerBlock
static std::optional< CodeGenFunction::MSVCIntrin > translateX86ToMsvcIntrin(unsigned BuiltinID)
static const std::pair< unsigned, unsigned > NEONEquivalentIntrinsicMap[]
#define NEONMAP0(NameBase)
static Value * EmitX86MaskedStore(CodeGenFunction &CGF, ArrayRef< Value * > Ops, Align Alignment)
static Value * emitBinaryBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID)
static Value * emitBinaryMaybeConstrainedFPBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID, unsigned ConstrainedIntrinsicID)
static RValue EmitBinaryAtomicPost(CodeGenFunction &CGF, llvm::AtomicRMWInst::BinOp Kind, const CallExpr *E, Instruction::BinaryOps Op, bool Invert=false)
Utility to insert an atomic instruction based Intrinsic::ID and the expression node,...
static Value * EmitX86Muldq(CodeGenFunction &CGF, bool IsSigned, ArrayRef< Value * > Ops)
static llvm::Value * ARMMVEVectorElementReverse(CGBuilderTy &Builder, llvm::Value *V, unsigned ReverseWidth)
#define MMA_SATF_VARIANTS(geom, type)
static std::optional< CodeGenFunction::MSVCIntrin > translateAarch64ToMsvcIntrin(unsigned BuiltinID)
static std::optional< CodeGenFunction::MSVCIntrin > translateArmToMsvcIntrin(unsigned BuiltinID)
static llvm::Value * EmitBitTestIntrinsic(CodeGenFunction &CGF, unsigned BuiltinID, const CallExpr *E)
Emit a _bittest* intrinsic.
static const ARMVectorIntrinsicInfo ARMSIMDIntrinsicMap[]
static Value * EmitSignBit(CodeGenFunction &CGF, Value *V)
Emit the computation of the sign bit for a floating point value.
static Value * EmitFAbs(CodeGenFunction &CGF, Value *V)
EmitFAbs - Emit a call to @llvm.fabs().
#define CUSTOM_BUILTIN_MAPPING(x, s)
static Value * EmitX86CvtF16ToFloatExpr(CodeGenFunction &CGF, ArrayRef< Value * > Ops, llvm::Type *DstTy)
static bool isSpecialUnsignedMultiplySignedResult(unsigned BuiltinID, WidthAndSignedness Op1Info, WidthAndSignedness Op2Info, WidthAndSignedness ResultInfo)
static llvm::Value * getDefaultBuiltinObjectSizeResult(unsigned Type, llvm::IntegerType *ResType)
static llvm::Value * emitPPCLoadReserveIntrinsic(CodeGenFunction &CGF, unsigned BuiltinID, const CallExpr *E)
static llvm::Value * VectorUnzip(CGBuilderTy &Builder, llvm::Value *V, bool Odd)
static Value * EmitX86SExtMask(CodeGenFunction &CGF, Value *Op, llvm::Type *DstTy)
static Value * emitTernaryMaybeConstrainedFPBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID, unsigned ConstrainedIntrinsicID)
static WidthAndSignedness getIntegerWidthAndSignedness(const clang::ASTContext &context, const clang::QualType Type)
static Value * EmitX86FunnelShift(CodeGenFunction &CGF, Value *Op0, Value *Op1, Value *Amt, bool IsRight)
static RValue EmitCheckedMixedSignMultiply(CodeGenFunction &CGF, const clang::Expr *Op1, WidthAndSignedness Op1Info, const clang::Expr *Op2, WidthAndSignedness Op2Info, const clang::Expr *ResultArg, QualType ResultQTy, WidthAndSignedness ResultInfo)
Emit a checked mixed-sign multiply.
static llvm::ScalableVectorType * getSVEVectorForElementType(llvm::Type *EltTy)
static unsigned mutateLongDoubleBuiltin(unsigned BuiltinID)
#define INTRINSIC_WITH_CC(NAME)
static llvm::FixedVectorType * GetNeonType(CodeGenFunction *CGF, NeonTypeFlags TypeFlags, bool HasLegalHalfType=true, bool V1Ty=false, bool AllowBFloatArgsAndRet=true)
static RValue EmitBinaryAtomic(CodeGenFunction &CGF, llvm::AtomicRMWInst::BinOp Kind, const CallExpr *E)
static llvm::Value * ARMMVEConstantSplat(CGBuilderTy &Builder, llvm::Type *VT)
static Value * EmitX86MaskLogic(CodeGenFunction &CGF, Instruction::BinaryOps Opc, ArrayRef< Value * > Ops, bool InvertLHS=false)
static Value * EmitFromInt(CodeGenFunction &CGF, llvm::Value *V, QualType T, llvm::Type *ResultType)
static Value * EmitAMDGCNBallotForExec(CodeGenFunction &CGF, const CallExpr *E, llvm::Type *RegisterType, llvm::Type *ValueType, bool isExecHi)
static void initializeAlloca(CodeGenFunction &CGF, AllocaInst *AI, Value *Size, Align AlignmentInBytes)
static Value * EmitX86Select(CodeGenFunction &CGF, Value *Mask, Value *Op0, Value *Op1)
SpecialRegisterAccessKind
static Value * EmitAtomicCmpXchgForMSIntrin(CodeGenFunction &CGF, const CallExpr *E, AtomicOrdering SuccessOrdering=AtomicOrdering::SequentiallyConsistent)
This function should be invoked to emit atomic cmpxchg for Microsoft's _InterlockedCompareExchange* i...
static Address CheckAtomicAlignment(CodeGenFunction &CGF, const CallExpr *E)
static Value * EmitX86FMAExpr(CodeGenFunction &CGF, const CallExpr *E, ArrayRef< Value * > Ops, unsigned BuiltinID, bool IsAddSub)
static Value * getMaskVecValue(CodeGenFunction &CGF, Value *Mask, unsigned NumElts)
static bool isSpecialMixedSignMultiply(unsigned BuiltinID, WidthAndSignedness Op1Info, WidthAndSignedness Op2Info, WidthAndSignedness ResultInfo)
Determine if a binop is a checked mixed-sign multiply we can specialize.
static Value * MakeAtomicCmpXchgValue(CodeGenFunction &CGF, const CallExpr *E, bool ReturnBool)
Utility to insert an atomic cmpxchg instruction.
static Value * emitBinaryExpMaybeConstrainedFPBuiltin(CodeGenFunction &CGF, const CallExpr *E, llvm::Intrinsic::ID IntrinsicID, llvm::Intrinsic::ID ConstrainedIntrinsicID)
static Value * EmitToInt(CodeGenFunction &CGF, llvm::Value *V, QualType T, llvm::IntegerType *IntType)
Emit the conversions required to turn the given value into an integer of the given size.
static llvm::Value * ARMMVEVectorSplat(CGBuilderTy &Builder, llvm::Value *V)
static Value * EmitX86MaskedCompareResult(CodeGenFunction &CGF, Value *Cmp, unsigned NumElts, Value *MaskIn)
static Value * EmitX86CompressStore(CodeGenFunction &CGF, ArrayRef< Value * > Ops)
static bool NEONSIMDIntrinsicsProvenSorted
static const ARMVectorIntrinsicInfo AArch64SIMDIntrinsicMap[]
static Value * EmitNontemporalStore(CodeGenFunction &CGF, const CallExpr *E)
static llvm::Value * EmitOverflowIntrinsic(CodeGenFunction &CGF, const llvm::Intrinsic::ID IntrinsicID, llvm::Value *X, llvm::Value *Y, llvm::Value *&Carry)
Emit a call to llvm.
static Value * EmitCommonNeonSISDBuiltinExpr(CodeGenFunction &CGF, const ARMVectorIntrinsicInfo &SISDInfo, SmallVectorImpl< Value * > &Ops, const CallExpr *E)
static Value * emitFPIntBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID)
static Value * emitTernaryBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID)
#define MMA_LDST(n, geom_op_type)
static Value * EmitX86vpcom(CodeGenFunction &CGF, ArrayRef< Value * > Ops, bool IsSigned)
static Value * emitFrexpBuiltin(CodeGenFunction &CGF, const CallExpr *E, llvm::Intrinsic::ID IntrinsicID)
@ _InterlockedExchangeAdd_rel
@ _InterlockedIncrement_acq
@ _InterlockedExchange_nf
@ _InterlockedIncrement_nf
@ _InterlockedExchange_acq
@ _InterlockedCompareExchange128_rel
@ _InterlockedCompareExchange128_acq
@ _InterlockedCompareExchange_acq
@ _InterlockedExchangeAdd_nf
@ _InterlockedCompareExchange_nf
@ _InterlockedDecrement_rel
@ _InterlockedExchangeSub
@ _InterlockedExchangeAdd_acq
@ _InterlockedIncrement_rel
@ _InterlockedCompareExchange128_nf
@ _InterlockedCompareExchange128
@ _InterlockedExchange_rel
@ _InterlockedDecrement_nf
@ _InterlockedExchangeAdd
@ _InterlockedDecrement_acq
@ _InterlockedCompareExchange_rel
static Value * EmitX86ConvertToMask(CodeGenFunction &CGF, Value *In)
static Value * EmitISOVolatileStore(CodeGenFunction &CGF, const CallExpr *E)
static Value * EmitSystemZIntrinsicWithCC(CodeGenFunction &CGF, unsigned IntrinsicID, const CallExpr *E)
Handle a SystemZ function in which the final argument is a pointer to an int that receives the post-i...
static RValue EmitHipStdParUnsupportedBuiltin(CodeGenFunction *CGF, const FunctionDecl *FD)
static llvm::Value * EmitX86BitTestIntrinsic(CodeGenFunction &CGF, BitTest BT, const CallExpr *E, Value *BitBase, Value *BitPos)
static RValue EmitCheckedUnsignedMultiplySignedResult(CodeGenFunction &CGF, const clang::Expr *Op1, WidthAndSignedness Op1Info, const clang::Expr *Op2, WidthAndSignedness Op2Info, const clang::Expr *ResultArg, QualType ResultQTy, WidthAndSignedness ResultInfo)
static Value * emitUnaryBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID, llvm::StringRef Name="")
static Value * emitCallMaybeConstrainedFPBuiltin(CodeGenFunction &CGF, unsigned IntrinsicID, unsigned ConstrainedIntrinsicID, llvm::Type *Ty, ArrayRef< Value * > Args)
static RValue emitLibraryCall(CodeGenFunction &CGF, const FunctionDecl *FD, const CallExpr *E, llvm::Constant *calleeValue)
static Value * EmitNontemporalLoad(CodeGenFunction &CGF, const CallExpr *E)
static Value * EmitX86Ternlog(CodeGenFunction &CGF, bool ZeroMask, ArrayRef< Value * > Ops)
static llvm::AtomicOrdering getBitTestAtomicOrdering(BitTest::InterlockingKind I)
#define MMA_VARIANTS_B1_XOR(geom, type)
#define MMA_VARIANTS_I4(geom, type)
Intrinsic::ID getDotProductIntrinsic(QualType QT, int elementCount)
static Value * EmitX86ConvertIntToFp(CodeGenFunction &CGF, const CallExpr *E, ArrayRef< Value * > Ops, bool IsSigned)
static Value * packTBLDVectorList(CodeGenFunction &CGF, ArrayRef< Value * > Ops, Value *ExtOp, Value *IndexOp, llvm::Type *ResTy, unsigned IntID, const char *Name)
static Value * EmitAbs(CodeGenFunction &CGF, Value *ArgValue, bool HasNSW)
static Value * EmitX86ScalarSelect(CodeGenFunction &CGF, Value *Mask, Value *Op0, Value *Op1)
static Value * EmitAtomicDecrementValue(CodeGenFunction &CGF, const CallExpr *E, AtomicOrdering Ordering=AtomicOrdering::SequentiallyConsistent)
static bool AArch64SIMDIntrinsicsProvenSorted
static const ARMVectorIntrinsicInfo AArch64SISDIntrinsicMap[]
CodeGenFunction::ComplexPairTy ComplexPairTy
#define ALIAS(NAME, TOK, FLAGS)
llvm::MachO::Record Record
static std::string getName(const CallEvent &Call)
static std::string toString(const clang::SanitizerSet &Sanitizers)
Produce a string containing comma-separated names of sanitizers in Sanitizers set.
Enumerates target-specific builtins in their own namespaces within namespace clang.
Defines the clang::TargetOptions class.
__DEVICE__ float modf(float __x, float *__iptr)
__DEVICE__ double nan(const char *)
__device__ __2f16 float __ockl_bool s
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
CharUnits getTypeAlignInChars(QualType T) const
Return the ABI-specified alignment of a (complete) type T, in characters.
unsigned getIntWidth(QualType T) const
const ASTRecordLayout & getASTRecordLayout(const RecordDecl *D) const
Get or compute information about the layout of the specified record (struct/union/class) D,...
Builtin::Context & BuiltinInfo
QualType getConstantArrayType(QualType EltTy, const llvm::APInt &ArySize, const Expr *SizeExpr, ArraySizeModifier ASM, unsigned IndexTypeQuals) const
Return the unique reference to the type for a constant array of the specified element type.
QualType getBaseElementType(const ArrayType *VAT) const
Return the innermost element type of an array type.
TypeInfo getTypeInfo(const Type *T) const
Get the size and alignment of the specified complete type in bits.
QualType getObjCIdType() const
Represents the Objective-CC id type.
bool hasSameUnqualifiedType(QualType T1, QualType T2) const
Determine whether the given types are equivalent after cvr-qualifiers have been removed.
const ArrayType * getAsArrayType(QualType T) const
Type Query functions.
uint64_t getTypeSize(QualType T) const
Return the size of the specified (complete) type T, in bits.
CharUnits getTypeSizeInChars(QualType T) const
Return the size of the specified (complete) type T, in characters.
QualType GetBuiltinType(unsigned ID, GetBuiltinTypeError &Error, unsigned *IntegerConstantArgs=nullptr) const
Return the type for the specified builtin.
const TargetInfo & getTargetInfo() const
CharUnits toCharUnitsFromBits(int64_t BitSize) const
Convert a size in bits to a size in characters.
unsigned getTargetAddressSpace(LangAS AS) const
@ GE_Missing_type
Missing a type.
ASTRecordLayout - This class contains layout information for one RecordDecl, which is a struct/union/...
CharUnits getSize() const
getSize - Get the record size in characters.
uint64_t getFieldOffset(unsigned FieldNo) const
getFieldOffset - Get the offset of the given field index, in bits.
Represents an array type, per C99 6.7.5.2 - Array Declarators.
QualType getElementType() const
bool isLibFunction(unsigned ID) const
Return true if this is a builtin for a libc/libm function, with a "__builtin_" prefix (e....
llvm::StringRef getName(unsigned ID) const
Return the identifier name for the specified builtin, e.g.
bool isConstWithoutErrnoAndExceptions(unsigned ID) const
Return true if this function has no side effects and doesn't read memory, except for possibly errno o...
bool isConstWithoutExceptions(unsigned ID) const
bool isConst(unsigned ID) const
Return true if this function has no side effects and doesn't read memory.
CallExpr - Represents a function call (C99 6.5.2.2, C++ [expr.call]).
Expr * getArg(unsigned Arg)
getArg - Return the specified argument.
bool hasStoredFPFeatures() const
SourceLocation getBeginLoc() const LLVM_READONLY
FunctionDecl * getDirectCallee()
If the callee is a FunctionDecl, return it. Otherwise return null.
FPOptionsOverride getFPFeatures() const
unsigned getNumArgs() const
getNumArgs - Return the number of actual arguments to this call.
QualType getCallReturnType(const ASTContext &Ctx) const
getCallReturnType - Get the return type of the call expr.
CharUnits - This is an opaque type for sizes expressed in character units.
llvm::Align getAsAlign() const
getAsAlign - Returns Quantity as a valid llvm::Align, Beware llvm::Align assumes power of two 8-bit b...
QuantityType getQuantity() const
getQuantity - Get the raw integer representation of this quantity.
static CharUnits One()
One - Construct a CharUnits quantity of one.
static CharUnits fromQuantity(QuantityType Quantity)
fromQuantity - Construct a CharUnits quantity from a raw integer type.
XRayInstrSet XRayInstrumentationBundle
Set of XRay instrumentation kinds to emit.
Like RawAddress, an abstract representation of an aligned address, but the pointer contained in this ...
llvm::Value * emitRawPointer(CodeGenFunction &CGF) const
Return the pointer contained in this class after authenticating it and adding offset to it if necessa...
CharUnits getAlignment() const
llvm::Type * getElementType() const
Return the type of the values stored in this address.
Address withElementType(llvm::Type *ElemTy) const
Return address with different element type, but same pointer and alignment.
Address withAlignment(CharUnits NewAlignment) const
Return address with different alignment, but same pointer and element type.
llvm::PointerType * getType() const
Return the type of the pointer value.
Address getAddress() const
static ApplyDebugLocation CreateArtificial(CodeGenFunction &CGF)
Apply TemporaryLocation if it is valid.
static ApplyDebugLocation CreateEmpty(CodeGenFunction &CGF)
Set the IRBuilder to not attach debug locations.
llvm::StoreInst * CreateStore(llvm::Value *Val, Address Addr, bool IsVolatile=false)
llvm::StoreInst * CreateAlignedStore(llvm::Value *Val, llvm::Value *Addr, CharUnits Align, bool IsVolatile=false)
Address CreateGEP(CodeGenFunction &CGF, Address Addr, llvm::Value *Index, const llvm::Twine &Name="")
llvm::CallInst * CreateMemMove(Address Dest, Address Src, llvm::Value *Size, bool IsVolatile=false)
Address CreatePointerBitCastOrAddrSpaceCast(Address Addr, llvm::Type *Ty, llvm::Type *ElementTy, const llvm::Twine &Name="")
llvm::CallInst * CreateMemCpyInline(Address Dest, Address Src, uint64_t Size)
llvm::AtomicRMWInst * CreateAtomicRMW(llvm::AtomicRMWInst::BinOp Op, Address Addr, llvm::Value *Val, llvm::AtomicOrdering Ordering, llvm::SyncScope::ID SSID=llvm::SyncScope::System)
llvm::CallInst * CreateMemSetInline(Address Dest, llvm::Value *Value, uint64_t Size)
llvm::StoreInst * CreateDefaultAlignedStore(llvm::Value *Val, llvm::Value *Addr, bool IsVolatile=false)
llvm::CallInst * CreateMemSet(Address Dest, llvm::Value *Value, llvm::Value *Size, bool IsVolatile=false)
llvm::AtomicCmpXchgInst * CreateAtomicCmpXchg(Address Addr, llvm::Value *Cmp, llvm::Value *New, llvm::AtomicOrdering SuccessOrdering, llvm::AtomicOrdering FailureOrdering, llvm::SyncScope::ID SSID=llvm::SyncScope::System)
llvm::LoadInst * CreateLoad(Address Addr, const llvm::Twine &Name="")
Address CreateConstByteGEP(Address Addr, CharUnits Offset, const llvm::Twine &Name="")
Address CreateLaunderInvariantGroup(Address Addr)
llvm::CallInst * CreateMemCpy(Address Dest, Address Src, llvm::Value *Size, bool IsVolatile=false)
llvm::LoadInst * CreateAlignedLoad(llvm::Type *Ty, llvm::Value *Addr, CharUnits Align, const llvm::Twine &Name="")
Address CreateAddrSpaceCast(Address Addr, llvm::Type *Ty, llvm::Type *ElementTy, const llvm::Twine &Name="")
Address CreateConstInBoundsGEP(Address Addr, uint64_t Index, const llvm::Twine &Name="")
Given addr = T* ... produce name = getelementptr inbounds addr, i64 index where i64 is actually the t...
Address CreateInBoundsGEP(Address Addr, ArrayRef< llvm::Value * > IdxList, llvm::Type *ElementType, CharUnits Align, const Twine &Name="")
virtual std::string getDeviceSideName(const NamedDecl *ND)=0
Returns function or variable name on device side even if the current compilation is for host.
virtual llvm::GlobalVariable * getThrowInfo(QualType T)
All available information about a concrete callee.
static CGCallee forDirect(llvm::Constant *functionPtr, const CGCalleeInfo &abstractInfo=CGCalleeInfo())
llvm::DIType * getOrCreateStandaloneType(QualType Ty, SourceLocation Loc)
Emit standalone debug info for a type.
CGFunctionInfo - Class to encapsulate the information about a function definition.
virtual void EmitGCMemmoveCollectable(CodeGen::CodeGenFunction &CGF, Address DestPtr, Address SrcPtr, llvm::Value *Size)=0
EnqueuedBlockInfo emitOpenCLEnqueuedBlock(CodeGenFunction &CGF, const Expr *E)
CallArgList - Type for representing both the value and type of arguments in a call.
void add(RValue rvalue, QualType type)
CodeGenFunction - This class organizes the per-function state that is used while generating LLVM code...
llvm::Value * EmitAArch64BuiltinExpr(unsigned BuiltinID, const CallExpr *E, llvm::Triple::ArchType Arch)
llvm::Value * EmitFromMemory(llvm::Value *Value, QualType Ty)
EmitFromMemory - Change a scalar value from its memory representation to its value representation.
llvm::Value * EmitMSVCBuiltinExpr(MSVCIntrin BuiltinID, const CallExpr *E)
void FinishFunction(SourceLocation EndLoc=SourceLocation())
FinishFunction - Complete IR generation of the current function.
llvm::Value * EmitLifetimeStart(llvm::TypeSize Size, llvm::Value *Addr)
std::pair< RValue, llvm::Value * > EmitAtomicCompareExchange(LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc, llvm::AtomicOrdering Success=llvm::AtomicOrdering::SequentiallyConsistent, llvm::AtomicOrdering Failure=llvm::AtomicOrdering::SequentiallyConsistent, bool IsWeak=false, AggValueSlot Slot=AggValueSlot::ignored())
static TypeEvaluationKind getEvaluationKind(QualType T)
getEvaluationKind - Return the TypeEvaluationKind of QualType T.
llvm::Value * EmitSVEPredicateCast(llvm::Value *Pred, llvm::ScalableVectorType *VTy)
llvm::CallInst * EmitTrapCall(llvm::Intrinsic::ID IntrID)
Emit a call to trap or debugtrap and attach function attribute "trap-func-name" if specified.
SanitizerSet SanOpts
Sanitizers enabled for this function.
RValue EmitBuiltinIsAligned(const CallExpr *E)
Emit IR for __builtin_is_aligned.
LValue EmitAggExprToLValue(const Expr *E)
EmitAggExprToLValue - Emit the computation of the specified expression of aggregate type into a tempo...
void EmitNonNullArgCheck(RValue RV, QualType ArgType, SourceLocation ArgLoc, AbstractCallee AC, unsigned ParmNum)
Create a check for a function parameter that may potentially be declared as non-null.
llvm::Value * EmitHexagonBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
void GetAArch64SVEProcessedOperands(unsigned BuiltinID, const CallExpr *E, SmallVectorImpl< llvm::Value * > &Ops, SVETypeFlags TypeFlags)
llvm::Value * EmitAMDGPUBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
void EmitLifetimeEnd(llvm::Value *Size, llvm::Value *Addr)
void pushLifetimeExtendedDestroy(CleanupKind kind, Address addr, QualType type, Destroyer *destroyer, bool useEHCleanupForArray)
llvm::Value * EmitCheckedArgForBuiltin(const Expr *E, BuiltinCheckKind Kind)
Emits an argument for a call to a builtin.
llvm::Value * EmitSVEGatherLoad(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
CleanupKind getARCCleanupKind()
Retrieves the default cleanup kind for an ARC cleanup.
llvm::Value * EmitNVPTXBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
llvm::Value * EmitHLSLBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
llvm::Value * EmitAArch64SVEBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
RValue emitBuiltinOSLogFormat(const CallExpr &E)
Emit IR for __builtin_os_log_format.
LValue EmitLValue(const Expr *E, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
EmitLValue - Emit code to compute a designator that specifies the location of the expression.
llvm::Value * EmitVAStartEnd(llvm::Value *ArgValue, bool IsStart)
Emits a call to an LLVM variable-argument intrinsic, either llvm.va_start or llvm....
llvm::Value * EmitSVEMaskedStore(const CallExpr *, SmallVectorImpl< llvm::Value * > &Ops, unsigned BuiltinID)
llvm::Value * EmitSVEReinterpret(llvm::Value *Val, llvm::Type *Ty)
llvm::BasicBlock * createBasicBlock(const Twine &name="", llvm::Function *parent=nullptr, llvm::BasicBlock *before=nullptr)
createBasicBlock - Create an LLVM basic block.
llvm::Value * EmitSEHExceptionInfo()
RValue EmitBuiltinAlignTo(const CallExpr *E, bool AlignUp)
Emit IR for __builtin_align_up/__builtin_align_down.
const LangOptions & getLangOpts() const
void ProcessOrderScopeAMDGCN(llvm::Value *Order, llvm::Value *Scope, llvm::AtomicOrdering &AO, llvm::SyncScope::ID &SSID)
llvm::Constant * EmitCheckTypeDescriptor(QualType T)
Emit a description of a type in a format suitable for passing to a runtime sanitizer handler.
void EmitBlock(llvm::BasicBlock *BB, bool IsFinished=false)
EmitBlock - Emit the given block.
void EmitTrapCheck(llvm::Value *Checked, SanitizerHandler CheckHandlerID)
Create a basic block that will call the trap intrinsic, and emit a conditional branch to it,...
void EmitUnreachable(SourceLocation Loc)
Emit a reached-unreachable diagnostic if Loc is valid and runtime checking is enabled.
llvm::Value * EmitSVETupleCreate(const SVETypeFlags &TypeFlags, llvm::Type *ReturnType, ArrayRef< llvm::Value * > Ops)
Address makeNaturalAddressForPointer(llvm::Value *Ptr, QualType T, CharUnits Alignment=CharUnits::Zero(), bool ForPointeeType=false, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
Construct an address with the natural alignment of T.
ComplexPairTy EmitComplexExpr(const Expr *E, bool IgnoreReal=false, bool IgnoreImag=false)
EmitComplexExpr - Emit the computation of the specified expression of complex type,...
void EmitAnyExprToMem(const Expr *E, Address Location, Qualifiers Quals, bool IsInitializer)
EmitAnyExprToMem - Emits the code necessary to evaluate an arbitrary expression into the given memory...
TypeCheckKind
Situations in which we might emit a check for the suitability of a pointer or glvalue.
@ TCK_Store
Checking the destination of a store. Must be suitably sized and aligned.
@ TCK_Load
Checking the operand of a load. Must be suitably sized and aligned.
llvm::Value * EmitSMELdrStr(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
RValue EmitCall(const CGFunctionInfo &CallInfo, const CGCallee &Callee, ReturnValueSlot ReturnValue, const CallArgList &Args, llvm::CallBase **callOrInvoke, bool IsMustTail, SourceLocation Loc)
EmitCall - Generate a call of the given function, expecting the given result type,...
llvm::Value * FormSVEBuiltinResult(llvm::Value *Call)
FormSVEBuiltinResult - Returns the struct of scalable vectors as a wider vector.
llvm::Value * EmitNeonSplat(llvm::Value *V, llvm::Constant *Idx, const llvm::ElementCount &Count)
llvm::Type * ConvertTypeForMem(QualType T)
llvm::Value * EmitSVEMaskedLoad(const CallExpr *, llvm::Type *ReturnTy, SmallVectorImpl< llvm::Value * > &Ops, unsigned BuiltinID, bool IsZExtReturn)
bool AlwaysEmitXRayCustomEvents() const
AlwaysEmitXRayCustomEvents - Return true if we must unconditionally emit XRay custom event handling c...
llvm::Value * EmitSVEDupX(llvm::Value *Scalar)
RawAddress CreateMemTemp(QualType T, const Twine &Name="tmp", RawAddress *Alloca=nullptr)
CreateMemTemp - Create a temporary memory object of the given type, with appropriate alignmen and cas...
@ Default
! No language constraints on evaluation order.
const TargetInfo & getTarget() const
llvm::Value * vectorWrapScalar16(llvm::Value *Op)
llvm::Function * LookupNeonLLVMIntrinsic(unsigned IntrinsicID, unsigned Modifier, llvm::Type *ArgTy, const CallExpr *E)
llvm::Value * getTypeSize(QualType Ty)
Returns calculated size of the specified type.
llvm::Value * EmitSEHExceptionCode()
Address EmitPointerWithAlignment(const Expr *Addr, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
EmitPointerWithAlignment - Given an expression with a pointer type, emit the value and compute our be...
llvm::Value * EmitTargetBuiltinExpr(unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue)
EmitTargetBuiltinExpr - Emit the given builtin call.
llvm::Value * EmitCountedByFieldExpr(const Expr *Base, const FieldDecl *FAMDecl, const FieldDecl *CountDecl)
Build an expression accessing the "counted_by" field.
RValue EmitAnyExprToTemp(const Expr *E)
EmitAnyExprToTemp - Similarly to EmitAnyExpr(), however, the result will always be accessible even if...
RValue EmitCoroutineIntrinsic(const CallExpr *E, unsigned int IID)
llvm::Value * EmitAArch64SMEBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
RValue EmitAMDGPUDevicePrintfCallExpr(const CallExpr *E)
Address EmitArrayToPointerDecay(const Expr *Array, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr)
void EmitCheck(ArrayRef< std::pair< llvm::Value *, SanitizerMask > > Checked, SanitizerHandler Check, ArrayRef< llvm::Constant * > StaticArgs, ArrayRef< llvm::Value * > DynamicArgs)
Create a basic block that will either trap or call a handler function in the UBSan runtime with the p...
RValue EmitBuiltinNewDeleteCall(const FunctionProtoType *Type, const CallExpr *TheCallExpr, bool IsDelete)
llvm::Value * EmitSVETupleSetOrGet(const SVETypeFlags &TypeFlags, llvm::Type *ReturnType, ArrayRef< llvm::Value * > Ops)
RValue EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue)
llvm::Value * EmitSVEPrefetchLoad(const SVETypeFlags &TypeFlags, SmallVectorImpl< llvm::Value * > &Ops, unsigned BuiltinID)
llvm::Value * EmitAArch64CompareBuiltinExpr(llvm::Value *Op, llvm::Type *Ty, const llvm::CmpInst::Predicate Fp, const llvm::CmpInst::Predicate Ip, const llvm::Twine &Name="")
void StartFunction(GlobalDecl GD, QualType RetTy, llvm::Function *Fn, const CGFunctionInfo &FnInfo, const FunctionArgList &Args, SourceLocation Loc=SourceLocation(), SourceLocation StartLoc=SourceLocation())
Emit code for the start of a function.
AggValueSlot CreateAggTemp(QualType T, const Twine &Name="tmp", RawAddress *Alloca=nullptr)
CreateAggTemp - Create a temporary memory object for the given aggregate type.
llvm::ScalableVectorType * getSVEType(const SVETypeFlags &TypeFlags)
RValue emitRotate(const CallExpr *E, bool IsRotateRight)
llvm::Constant * EmitCheckSourceLocation(SourceLocation Loc)
Emit a description of a source location in a format suitable for passing to a runtime sanitizer handl...
void ErrorUnsupported(const Stmt *S, const char *Type)
ErrorUnsupported - Print out an error that codegen doesn't support the specified stmt yet.
CGDebugInfo * getDebugInfo()
const FieldDecl * FindFlexibleArrayMemberFieldAndOffset(ASTContext &Ctx, const RecordDecl *RD, const FieldDecl *FAMDecl, uint64_t &Offset)
Address EmitVAListRef(const Expr *E)
llvm::Value * EmitNeonShiftVector(llvm::Value *V, llvm::Type *Ty, bool negateForRightShift)
llvm::Value * EmitSVEMovl(const SVETypeFlags &TypeFlags, llvm::ArrayRef< llvm::Value * > Ops, unsigned BuiltinID)
void emitAlignmentAssumption(llvm::Value *PtrValue, QualType Ty, SourceLocation Loc, SourceLocation AssumptionLoc, llvm::Value *Alignment, llvm::Value *OffsetValue=nullptr)
const TargetCodeGenInfo & getTargetHooks() const
llvm::Value * EmitARMBuiltinExpr(unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue, llvm::Triple::ArchType Arch)
llvm::Type * getEltType(const SVETypeFlags &TypeFlags)
void EmitAggExpr(const Expr *E, AggValueSlot AS)
EmitAggExpr - Emit the computation of the specified expression of aggregate type.
bool ShouldXRayInstrumentFunction() const
ShouldXRayInstrument - Return true if the current function should be instrumented with XRay nop sleds...
llvm::Value * EmitSVEPMull(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned BuiltinID)
llvm::Value * EmitToMemory(llvm::Value *Value, QualType Ty)
EmitToMemory - Change a scalar value from its value representation to its in-memory representation.
RValue EmitOpenMPDevicePrintfCallExpr(const CallExpr *E)
bool IsInPreservedAIRegion
True if CodeGen currently emits code inside presereved access index region.
llvm::Value * EmitARCRetain(QualType type, llvm::Value *value)
bool AlwaysEmitXRayTypedEvents() const
AlwaysEmitXRayTypedEvents - Return true if clang must unconditionally emit XRay typed event handling ...
void SetSqrtFPAccuracy(llvm::Value *Val)
Set the minimum required accuracy of the given sqrt operation based on CodeGenOpts.
llvm::Value * EmitSVEScatterStore(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
llvm::CallInst * EmitNounwindRuntimeCall(llvm::FunctionCallee callee, const Twine &name="")
ASTContext & getContext() const
llvm::Function * generateBuiltinOSLogHelperFunction(const analyze_os_log::OSLogBufferLayout &Layout, CharUnits BufferAlignment)
llvm::Value * EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty, SourceLocation Loc, AlignmentSource Source=AlignmentSource::Type, bool isNontemporal=false)
EmitLoadOfScalar - Load a scalar value from an address, taking care to appropriately convert from the...
CGCallee EmitCallee(const Expr *E)
const Decl * CurFuncDecl
CurFuncDecl - Holds the Decl for the current outermost non-closure context.
llvm::Value * EmitScalarOrConstFoldImmArg(unsigned ICEArguments, unsigned Idx, const CallExpr *E)
void checkTargetFeatures(const CallExpr *E, const FunctionDecl *TargetDecl)
llvm::Value * BuildVector(ArrayRef< llvm::Value * > Ops)
llvm::Value * EmitWebAssemblyBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
llvm::Value * EmitARMCDEBuiltinExpr(unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue, llvm::Triple::ArchType Arch)
llvm::CallInst * EmitRuntimeCall(llvm::FunctionCallee callee, const Twine &name="")
llvm::Type * ConvertType(QualType T)
llvm::CallBase * EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee, ArrayRef< llvm::Value * > args, const Twine &name="")
llvm::Value * EmitSystemZBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
llvm::Value * EmitSMEReadWrite(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
void EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, LValue LV, QualType Type, SanitizerSet SkippedChecks=SanitizerSet(), llvm::Value *ArraySize=nullptr)
LValue MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T)
llvm::Value * EmitSMELd1St1(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
llvm::Value * EmitPPCBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
static Destroyer destroyARCStrongPrecise
void EmitARCIntrinsicUse(ArrayRef< llvm::Value * > values)
RValue EmitNVPTXDevicePrintfCallExpr(const CallExpr *E)
llvm::Value * EmitSVEStructLoad(const SVETypeFlags &TypeFlags, SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
Address EmitMSVAListRef(const Expr *E)
Emit a "reference" to a __builtin_ms_va_list; this is always the value of the expression,...
llvm::Value * EmitCheckedInBoundsGEP(llvm::Type *ElemTy, llvm::Value *Ptr, ArrayRef< llvm::Value * > IdxList, bool SignedIndices, bool IsSubtraction, SourceLocation Loc, const Twine &Name="")
Same as IRBuilder::CreateInBoundsGEP, but additionally emits a check to detect undefined behavior whe...
llvm::Value * EmitNeonRShiftImm(llvm::Value *Vec, llvm::Value *Amt, llvm::Type *Ty, bool usgn, const char *name)
SmallVector< llvm::Type *, 2 > getSVEOverloadTypes(const SVETypeFlags &TypeFlags, llvm::Type *ReturnType, ArrayRef< llvm::Value * > Ops)
static bool hasAggregateEvaluationKind(QualType T)
const FieldDecl * FindCountedByField(const FieldDecl *FD)
Find the FieldDecl specified in a FAM's "counted_by" attribute.
LValue MakeAddrLValue(Address Addr, QualType T, AlignmentSource Source=AlignmentSource::Type)
llvm::Value * EmitARMMVEBuiltinExpr(unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue, llvm::Triple::ArchType Arch)
llvm::Value * EmitSVEStructStore(const SVETypeFlags &TypeFlags, SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
Address GetAddrOfLocalVar(const VarDecl *VD)
GetAddrOfLocalVar - Return the address of a local variable.
llvm::Value * EmitSEHAbnormalTermination()
llvm::Value * EmitX86BuiltinExpr(unsigned BuiltinID, const CallExpr *E)
Address ReturnValue
ReturnValue - The temporary alloca to hold the return value.
llvm::Value * EmitSVEAllTruePred(const SVETypeFlags &TypeFlags)
RValue GetUndefRValue(QualType Ty)
GetUndefRValue - Get an appropriate 'undef' rvalue for the given type.
llvm::Type * SVEBuiltinMemEltTy(const SVETypeFlags &TypeFlags)
SVEBuiltinMemEltTy - Returns the memory element type for this memory access builtin.
llvm::LLVMContext & getLLVMContext()
llvm::Value * EmitScalarExpr(const Expr *E, bool IgnoreResultAssign=false)
EmitScalarExpr - Emit the computation of the specified expression of LLVM scalar type,...
llvm::Value * EmitSMEZero(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
llvm::Value * EmitRISCVBuiltinExpr(unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue)
llvm::Value * EmitCommonNeonBuiltinExpr(unsigned BuiltinID, unsigned LLVMIntrinsic, unsigned AltLLVMIntrinsic, const char *NameHint, unsigned Modifier, const CallExpr *E, SmallVectorImpl< llvm::Value * > &Ops, Address PtrOp0, Address PtrOp1, llvm::Triple::ArchType Arch)
llvm::Value * EmitNeonCall(llvm::Function *F, SmallVectorImpl< llvm::Value * > &O, const char *name, unsigned shift=0, bool rightshift=false)
llvm::Value * EmitAnnotationCall(llvm::Function *AnnotationFn, llvm::Value *AnnotatedVal, StringRef AnnotationStr, SourceLocation Location, const AnnotateAttr *Attr)
Emit an annotation call (intrinsic).
llvm::ScalableVectorType * getSVEPredType(const SVETypeFlags &TypeFlags)
llvm::Value * EmitSVEGatherPrefetch(const SVETypeFlags &TypeFlags, SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
void EmitStoreOfScalar(llvm::Value *Value, Address Addr, bool Volatile, QualType Ty, AlignmentSource Source=AlignmentSource::Type, bool isInit=false, bool isNontemporal=false)
EmitStoreOfScalar - Store a scalar value to an address, taking care to appropriately convert from the...
llvm::Value * EmitBPFBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
This class organizes the cross-function state that is used while generating LLVM code.
CGHLSLRuntime & getHLSLRuntime()
Return a reference to the configured HLSL runtime.
llvm::Module & getModule() const
llvm::FunctionCallee CreateRuntimeFunction(llvm::FunctionType *Ty, StringRef Name, llvm::AttributeList ExtraAttrs=llvm::AttributeList(), bool Local=false, bool AssumeConvergent=false)
Create or return a runtime function declaration with the specified type and name.
llvm::Constant * getBuiltinLibFunction(const FunctionDecl *FD, unsigned BuiltinID)
Given a builtin id for a function like "__builtin_fabsf", return a Function* for "fabsf".
DiagnosticsEngine & getDiags() const
void ErrorUnsupported(const Stmt *S, const char *Type)
Print out an error that codegen doesn't support the specified stmt yet.
CGCUDARuntime & getCUDARuntime()
Return a reference to the configured CUDA runtime.
CodeGenTypes & getTypes()
CGOpenCLRuntime & getOpenCLRuntime()
Return a reference to the configured OpenCL runtime.
const TargetInfo & getTarget() const
const llvm::DataLayout & getDataLayout() const
void Error(SourceLocation loc, StringRef error)
Emit a general error that something can't be done.
CGCXXABI & getCXXABI() const
llvm::Constant * GetFunctionStart(const ValueDecl *Decl)
const llvm::Triple & getTriple() const
void DecorateInstructionWithTBAA(llvm::Instruction *Inst, TBAAAccessInfo TBAAInfo)
DecorateInstructionWithTBAA - Decorate the instruction with a TBAA tag.
llvm::Constant * CreateRuntimeVariable(llvm::Type *Ty, StringRef Name)
Create a new runtime global variable with the specified type and name.
TBAAAccessInfo getTBAAAccessInfo(QualType AccessType)
getTBAAAccessInfo - Get TBAA information that describes an access to an object of the given type.
ASTContext & getContext() const
const CodeGenOptions & getCodeGenOpts() const
StringRef getMangledName(GlobalDecl GD)
CharUnits getNaturalPointeeTypeAlignment(QualType T, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr)
llvm::LLVMContext & getLLVMContext()
CGObjCRuntime & getObjCRuntime()
Return a reference to the configured Objective-C runtime.
void SetLLVMFunctionAttributes(GlobalDecl GD, const CGFunctionInfo &Info, llvm::Function *F, bool IsThunk)
Set the LLVM function attributes (sext, zext, etc).
void SetLLVMFunctionAttributesForDefinition(const Decl *D, llvm::Function *F)
Set the LLVM function attributes which only apply to a function definition.
llvm::Function * getIntrinsic(unsigned IID, ArrayRef< llvm::Type * > Tys=std::nullopt)
ConstantAddress GetAddrOfConstantCString(const std::string &Str, const char *GlobalName=nullptr)
Returns a pointer to a character array containing the literal and a terminating '\0' character.
llvm::Type * ConvertType(QualType T)
ConvertType - Convert type T into a llvm::Type.
llvm::FunctionType * GetFunctionType(const CGFunctionInfo &Info)
GetFunctionType - Get the LLVM function type for.
const CGFunctionInfo & arrangeBuiltinFunctionDeclaration(QualType resultType, const FunctionArgList &args)
A builtin function is a freestanding function using the default C conventions.
const CGFunctionInfo & arrangeBuiltinFunctionCall(QualType resultType, const CallArgList &args)
llvm::Constant * emitAbstract(const Expr *E, QualType T)
Emit the result of the given expression as an abstract constant, asserting that it succeeded.
Information for lazily generating a cleanup.
FunctionArgList - Type for representing both the decl and type of parameters to a function.
LValue - This represents an lvalue references.
llvm::Value * getRawBitFieldPointer(CodeGenFunction &CGF) const
Address getAddress(CodeGenFunction &CGF) const
llvm::Value * emitRawPointer(CodeGenFunction &CGF) const
void setNontemporal(bool Value)
llvm::Value * getPointer(CodeGenFunction &CGF) const
RValue - This trivial value class is used to represent the result of an expression that is evaluated.
static RValue getIgnored()
static RValue get(llvm::Value *V)
static RValue getAggregate(Address addr, bool isVolatile=false)
Convert an Address to an RValue.
static RValue getComplex(llvm::Value *V1, llvm::Value *V2)
llvm::Value * getScalarVal() const
getScalarVal() - Return the Value* of this scalar value.
An abstract representation of an aligned address.
llvm::Value * getPointer() const
static RawAddress invalid()
ReturnValueSlot - Contains the address where the return value of a function can be stored,...
virtual llvm::Value * encodeReturnAddress(CodeGen::CodeGenFunction &CGF, llvm::Value *Address) const
Performs the code-generation required to convert the address of an instruction into a return address ...
virtual llvm::Value * decodeReturnAddress(CodeGen::CodeGenFunction &CGF, llvm::Value *Address) const
Performs the code-generation required to convert a return address as stored by the system into the ac...
const T & getABIInfo() const
virtual int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const
Determines the DWARF register number for the stack pointer, for exception-handling purposes.
virtual llvm::Value * testFPKind(llvm::Value *V, unsigned BuiltinID, CGBuilderTy &Builder, CodeGenModule &CGM) const
Performs a target specific test of a floating point value for things like IsNaN, Infinity,...
Complex values, per C99 6.2.5p11.
Represents a concrete matrix type with constant number of rows and columns.
RecordDecl * getOuterLexicalRecordContext()
Retrieve the outermost lexically enclosing record context.
bool isImplicit() const
isImplicit - Indicates whether the declaration was implicitly generated by the implementation.
FunctionDecl * getAsFunction() LLVM_READONLY
Returns the function itself, or the templated function if this is a function template.
DeclContext * getDeclContext()
static bool isFlexibleArrayMemberLike(ASTContext &Context, const Decl *D, QualType Ty, LangOptions::StrictFlexArraysLevelKind StrictFlexArraysLevel, bool IgnoreTemplateOrMacroSubstitution)
Whether it resembles a flexible array member.
Concrete class used by the front-end to report problems and issues.
DiagnosticBuilder Report(SourceLocation Loc, unsigned DiagID)
Issue the message to the client.
This represents one expression.
bool EvaluateAsInt(EvalResult &Result, const ASTContext &Ctx, SideEffectsKind AllowSideEffects=SE_NoSideEffects, bool InConstantContext=false) const
EvaluateAsInt - Return true if this is a constant which we can fold and convert to an integer,...
Expr * IgnoreParenCasts() LLVM_READONLY
Skip past any parentheses and casts which might surround this expression until reaching a fixed point...
llvm::APSInt EvaluateKnownConstInt(const ASTContext &Ctx, SmallVectorImpl< PartialDiagnosticAt > *Diag=nullptr) const
EvaluateKnownConstInt - Call EvaluateAsRValue and return the folded integer.
Expr * IgnoreParenImpCasts() LLVM_READONLY
Skip past any parentheses and implicit casts which might surround this expression until reaching a fi...
bool EvaluateAsFloat(llvm::APFloat &Result, const ASTContext &Ctx, SideEffectsKind AllowSideEffects=SE_NoSideEffects, bool InConstantContext=false) const
EvaluateAsFloat - Return true if this is a constant which we can fold and convert to a floating point...
Expr * IgnoreParens() LLVM_READONLY
Skip past any parentheses which might surround this expression until reaching a fixed point.
@ NPC_ValueDependentIsNotNull
Specifies that a value-dependent expression should be considered to never be a null pointer constant.
ExprObjectKind getObjectKind() const
getObjectKind - The object kind that this expression produces.
bool EvaluateAsRValue(EvalResult &Result, const ASTContext &Ctx, bool InConstantContext=false) const
EvaluateAsRValue - Return true if this is a constant which we can fold to an rvalue using any crazy t...
bool HasSideEffects(const ASTContext &Ctx, bool IncludePossibleEffects=true) const
HasSideEffects - This routine returns true for all those expressions which have any effect other than...
Expr * IgnoreImpCasts() LLVM_READONLY
Skip past any implicit casts which might surround this expression until reaching a fixed point.
NullPointerConstantKind isNullPointerConstant(ASTContext &Ctx, NullPointerConstantValueDependence NPC) const
isNullPointerConstant - C99 6.3.2.3p3 - Test if this reduces down to a Null pointer constant.
SourceLocation getExprLoc() const LLVM_READONLY
getExprLoc - Return the preferred location for the arrow when diagnosing a problem with a generic exp...
std::optional< llvm::APSInt > getIntegerConstantExpr(const ASTContext &Ctx, SourceLocation *Loc=nullptr) const
isIntegerConstantExpr - Return the value if this expression is a valid integer constant expression.
bool tryEvaluateObjectSize(uint64_t &Result, ASTContext &Ctx, unsigned Type) const
If the current Expr is a pointer, this will try to statically determine the number of bytes available...
const ValueDecl * getAsBuiltinConstantDeclRef(const ASTContext &Context) const
If this expression is an unambiguous reference to a single declaration, in the style of __builtin_fun...
Represents difference between two FPOptions values.
Represents a member of a struct/union/class.
Represents a function declaration or definition.
const ParmVarDecl * getParamDecl(unsigned i) const
Represents a prototype with parameter type info, e.g.
GlobalDecl - represents a global declaration.
const Decl * getDecl() const
IdentifierInfo & get(StringRef Name)
Return the identifier token info for the specified named identifier.
static ImplicitParamDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation IdLoc, IdentifierInfo *Id, QualType T, ImplicitParamKind ParamKind)
Create implicit parameter.
StrictFlexArraysLevelKind
StringRef getName() const
Get the name of identifier for this declaration as a StringRef.
std::string getNameAsString() const
Get a human-readable name for the declaration, even if it is one of the special kinds of names (C++ c...
Flags to identify the types for overloaded Neon builtins.
EltType getEltType() const
PointerType - C99 6.7.5.1 - Pointer Declarators.
QualType getPointeeType() const
A (possibly-)qualified type.
bool isVolatileQualified() const
Determine whether this type is volatile-qualified.
bool isWebAssemblyFuncrefType() const
Returns true if it is a WebAssembly Funcref Type.
LangAS getAddressSpace() const
Return the address space of this type.
bool isWebAssemblyExternrefType() const
Returns true if it is a WebAssembly Externref Type.
The collection of all-type qualifiers we support.
Represents a struct/union/class.
field_range fields() const
Flags to identify the types for overloaded SVE builtins.
bool isZExtReturn() const
bool isReverseUSDOT() const
bool isOverloadNone() const
MemEltType getMemEltType() const
bool isGatherLoad() const
bool isOverloadCvt() const
EltType getEltType() const
bool isOverloadDefault() const
bool isOverloadWhileRW() const
bool isReverseMergeAnyAccOp() const
bool isReductionQV() const
bool isInsertOp1SVALL() const
bool isAppendSVALL() const
bool isReverseMergeAnyBinOp() const
bool isStructStore() const
bool isTupleCreate() const
bool isGatherPrefetch() const
bool hasSplatOperand() const
MergeType getMergeType() const
bool isByteIndexed() const
bool isStructLoad() const
bool isOverloadWhileOrMultiVecCvt() const
unsigned getSplatOperand() const
bool isScatterStore() const
bool isReverseCompare() const
Scope - A scope is a transient data structure that is used while parsing the program.
Encodes a location in the source.
SourceLocation getBeginLoc() const LLVM_READONLY
Exposes information about the current target.
TargetOptions & getTargetOpts() const
Retrieve the target options.
virtual bool hasLegalHalfType() const
Determine whether _Float16 is supported on this target.
const llvm::Triple & getTriple() const
Returns the target triple of the primary target.
bool isLittleEndian() const
unsigned getMaxOpenCLWorkGroupSize() const
virtual bool checkArithmeticFenceSupported() const
Controls if __arithmetic_fence is supported in the targeted backend.
unsigned getSuitableAlign() const
Return the alignment that is the largest alignment ever used for any scalar/SIMD data type on the tar...
virtual std::string_view getClobbers() const =0
Returns a string of target-specific clobbers, in LLVM format.
llvm::CodeObjectVersionKind CodeObjectVersion
Code object version for AMDGPU.
The base class of the type hierarchy.
CXXRecordDecl * getAsCXXRecordDecl() const
Retrieves the CXXRecordDecl that this type refers to, either because the type is a RecordType or beca...
bool isBlockPointerType() const
bool isBooleanType() const
bool isSignedIntegerType() const
Return true if this is an integer type that is signed, according to C99 6.2.5p4 [char,...
bool isComplexType() const
isComplexType() does not include complex integers (a GCC extension).
bool isCountAttributedType() const
bool isPointerType() const
bool isIntegerType() const
isIntegerType() does not include complex integers (a GCC extension).
const T * castAs() const
Member-template castAs<specific type>.
QualType getPointeeType() const
If this is a pointer, ObjC object pointer, or block pointer, this returns the respective pointee.
bool isIntegralOrEnumerationType() const
Determine whether this type is an integral or enumeration type.
bool hasUnsignedIntegerRepresentation() const
Determine whether this type has an unsigned integer representation of some sort, e....
bool isBitIntType() const
bool hasSignedIntegerRepresentation() const
Determine whether this type has an signed integer representation of some sort, e.g....
bool isObjCObjectPointerType() const
bool hasFloatingRepresentation() const
Determine whether this type has a floating-point representation of some sort, e.g....
bool isFloatingType() const
bool isUnsignedIntegerType() const
Return true if this is an integer type that is unsigned, according to C99 6.2.5p6 [which returns true...
const T * getAs() const
Member-template getAs<specific type>'.
bool isRecordType() const
bool isSizelessVectorType() const
Returns true for all scalable vector types.
QualType getSizelessVectorEltType(const ASTContext &Ctx) const
Returns the representative type for the element of a sizeless vector builtin type.
RecordDecl * getAsRecordDecl() const
Retrieves the RecordDecl this type refers to.
Represent the declaration of a variable (in which case it is an lvalue) a function (in which case it ...
Represents a GCC generic vector type.
unsigned getNumElements() const
SmallVector< OSLogBufferItem, 4 > Items
unsigned char getNumArgsByte() const
unsigned char getSummaryByte() const
Defines the clang::TargetInfo interface.
@ Type
The l-value was considered opaque, so the alignment was determined from a type.
llvm::Constant * initializationPatternFor(CodeGenModule &, llvm::Type *)
TypeEvaluationKind
The kind of evaluation to perform on values of a particular type.
@ EHCleanup
Denotes a cleanup that should run when a scope is exited using exceptional control flow (a throw stat...
constexpr XRayInstrMask Typed
constexpr XRayInstrMask Custom
bool computeOSLogBufferLayout(clang::ASTContext &Ctx, const clang::CallExpr *E, OSLogBufferLayout &layout)
const void * Store
Store - This opaque type encapsulates an immutable mapping from locations to values.
bool Dup(InterpState &S, CodePtr OpPC)
bool Ret(InterpState &S, CodePtr &PC, APValue &Result)
bool Zero(InterpState &S, CodePtr OpPC)
bool Mul(InterpState &S, CodePtr OpPC)
bool Neg(InterpState &S, CodePtr OpPC)
bool Load(InterpState &S, CodePtr OpPC)
bool Cast(InterpState &S, CodePtr OpPC)
The JSON file list parser is used to communicate input to InstallAPI.
@ DType
'dtype' clause, an alias for 'device_type', stored separately for diagnostic purposes.
@ OK_BitField
A bitfield object is a bitfield on a C or C++ record.
bool operator<(DeclarationName LHS, DeclarationName RHS)
Ordering on two declaration names.
@ Asm
Assembly: we accept this only so that we can preprocess it.
@ Result
The result type of a method or function.
LangAS
Defines the address space values used by the address space qualifier of QualType.
const FunctionProtoType * T
@ Success
Template argument deduction was successful.
@ Other
Other implicit parameter.
Diagnostic wrappers for TextAPI types for error reporting.
llvm::PointerType * VoidPtrTy
llvm::IntegerType * Int64Ty
llvm::PointerType * ConstGlobalsPtrTy
void* in the address space for constant globals
CharUnits getIntAlign() const
llvm::IntegerType * Int8Ty
i8, i16, i32, and i64
llvm::Type * HalfTy
half, bfloat, float, double
llvm::IntegerType * SizeTy
llvm::IntegerType * Int32Ty
llvm::IntegerType * IntPtrTy
llvm::IntegerType * IntTy
int
llvm::IntegerType * Int16Ty
llvm::PointerType * Int8PtrTy
llvm::PointerType * UnqualPtrTy
llvm::PointerType * AllocaInt8PtrTy
LangAS getASTAllocaAddressSpace() const
EvalResult is a struct with detailed info about an evaluated expression.
APValue Val
Val - This is the value the expression can be folded to.
void clear(SanitizerMask K=SanitizerKind::All)
Disable the sanitizers specified in K.
void set(SanitizerMask K, bool Value)
Enable or disable a certain (single) sanitizer.
bool has(SanitizerMask K) const
Check if a certain (single) sanitizer is enabled.
bool has(XRayInstrMask K) const
#define scalbln(__x, __y)
#define copysign(__x, __y)
#define remquo(__x, __y, __z)
#define nextafter(__x, __y)
#define nexttoward(__x, __y)
#define remainder(__x, __y)
#define fma(__x, __y, __z)