38#include "llvm/ADT/APFloat.h"
39#include "llvm/ADT/APInt.h"
40#include "llvm/ADT/FloatingPointMode.h"
41#include "llvm/ADT/SmallPtrSet.h"
42#include "llvm/ADT/StringExtras.h"
43#include "llvm/Analysis/ValueTracking.h"
44#include "llvm/IR/DataLayout.h"
45#include "llvm/IR/InlineAsm.h"
46#include "llvm/IR/Intrinsics.h"
47#include "llvm/IR/IntrinsicsAArch64.h"
48#include "llvm/IR/IntrinsicsAMDGPU.h"
49#include "llvm/IR/IntrinsicsARM.h"
50#include "llvm/IR/IntrinsicsBPF.h"
51#include "llvm/IR/IntrinsicsDirectX.h"
52#include "llvm/IR/IntrinsicsHexagon.h"
53#include "llvm/IR/IntrinsicsNVPTX.h"
54#include "llvm/IR/IntrinsicsPowerPC.h"
55#include "llvm/IR/IntrinsicsR600.h"
56#include "llvm/IR/IntrinsicsRISCV.h"
57#include "llvm/IR/IntrinsicsS390.h"
58#include "llvm/IR/IntrinsicsWebAssembly.h"
59#include "llvm/IR/IntrinsicsX86.h"
60#include "llvm/IR/MDBuilder.h"
61#include "llvm/IR/MatrixBuilder.h"
62#include "llvm/IR/MemoryModelRelaxationAnnotations.h"
63#include "llvm/Support/AMDGPUAddrSpace.h"
64#include "llvm/Support/ConvertUTF.h"
65#include "llvm/Support/MathExtras.h"
66#include "llvm/Support/ScopedPrinter.h"
67#include "llvm/TargetParser/AArch64TargetParser.h"
68#include "llvm/TargetParser/RISCVISAInfo.h"
69#include "llvm/TargetParser/RISCVTargetParser.h"
70#include "llvm/TargetParser/X86TargetParser.h"
75using namespace CodeGen;
79 Align AlignmentInBytes) {
81 switch (CGF.
getLangOpts().getTrivialAutoVarInit()) {
82 case LangOptions::TrivialAutoVarInitKind::Uninitialized:
85 case LangOptions::TrivialAutoVarInitKind::Zero:
86 Byte = CGF.
Builder.getInt8(0x00);
88 case LangOptions::TrivialAutoVarInitKind::Pattern: {
90 Byte = llvm::dyn_cast<llvm::ConstantInt>(
98 I->addAnnotationMetadata(
"auto-init");
104 Constant *FZeroConst = ConstantFP::getZero(CGF->
FloatTy);
109 FZeroConst = ConstantVector::getSplat(
110 ElementCount::getFixed(VecTy->getNumElements()), FZeroConst);
111 auto *FCompInst = CGF->
Builder.CreateFCmpOLT(Op0, FZeroConst);
112 CMP = CGF->
Builder.CreateIntrinsic(
114 {FCompInst},
nullptr);
116 CMP = CGF->
Builder.CreateFCmpOLT(Op0, FZeroConst);
119 LastInstr = CGF->
Builder.CreateIntrinsic(
120 CGF->
VoidTy, llvm::Intrinsic::dx_discard, {CMP},
nullptr);
125 CGF->
Builder.CreateCondBr(CMP, LT0, End);
127 CGF->
Builder.SetInsertPoint(LT0);
129 CGF->
Builder.CreateIntrinsic(CGF->
VoidTy, llvm::Intrinsic::spv_discard, {},
132 LastInstr = CGF->
Builder.CreateBr(End);
134 CGF->
Builder.SetInsertPoint(End);
136 llvm_unreachable(
"Backend Codegen not supported.");
144 const auto *OutArg1 = dyn_cast<HLSLOutArgExpr>(
E->getArg(1));
145 const auto *OutArg2 = dyn_cast<HLSLOutArgExpr>(
E->getArg(2));
156 Value *LowBits =
nullptr;
157 Value *HighBits =
nullptr;
161 llvm::Type *RetElementTy = CGF->
Int32Ty;
163 RetElementTy = llvm::VectorType::get(
164 CGF->
Int32Ty, ElementCount::getFixed(Op0VecTy->getNumElements()));
165 auto *RetTy = llvm::StructType::get(RetElementTy, RetElementTy);
167 CallInst *CI = CGF->
Builder.CreateIntrinsic(
168 RetTy, Intrinsic::dx_splitdouble, {Op0},
nullptr,
"hlsl.splitdouble");
170 LowBits = CGF->
Builder.CreateExtractValue(CI, 0);
171 HighBits = CGF->
Builder.CreateExtractValue(CI, 1);
176 if (!Op0->
getType()->isVectorTy()) {
177 FixedVectorType *DestTy = FixedVectorType::get(CGF->
Int32Ty, 2);
178 Value *Bitcast = CGF->
Builder.CreateBitCast(Op0, DestTy);
180 LowBits = CGF->
Builder.CreateExtractElement(Bitcast, (uint64_t)0);
181 HighBits = CGF->
Builder.CreateExtractElement(Bitcast, 1);
184 if (
const auto *VecTy =
186 NumElements = VecTy->getNumElements();
188 FixedVectorType *Uint32VecTy =
189 FixedVectorType::get(CGF->
Int32Ty, NumElements * 2);
190 Value *Uint32Vec = CGF->
Builder.CreateBitCast(Op0, Uint32VecTy);
191 if (NumElements == 1) {
192 LowBits = CGF->
Builder.CreateExtractElement(Uint32Vec, (uint64_t)0);
193 HighBits = CGF->
Builder.CreateExtractElement(Uint32Vec, 1);
196 for (
int I = 0,
E = NumElements; I !=
E; ++I) {
197 EvenMask.push_back(I * 2);
198 OddMask.push_back(I * 2 + 1);
200 LowBits = CGF->
Builder.CreateShuffleVector(Uint32Vec, EvenMask);
201 HighBits = CGF->
Builder.CreateShuffleVector(Uint32Vec, OddMask);
215 "asdouble operands types mismatch");
219 llvm::Type *ResultType = CGF.
DoubleTy;
222 N = VTy->getNumElements();
223 ResultType = llvm::FixedVectorType::get(CGF.
DoubleTy, N);
227 return CGF.
Builder.CreateIntrinsic(
228 ResultType, Intrinsic::dx_asdouble,
232 OpLowBits = CGF.
Builder.CreateVectorSplat(1, OpLowBits);
233 OpHighBits = CGF.
Builder.CreateVectorSplat(1, OpHighBits);
237 for (
int i = 0; i < N; i++) {
239 Mask.push_back(i + N);
242 Value *BitVec = CGF.
Builder.CreateShuffleVector(OpLowBits, OpHighBits, Mask);
244 return CGF.
Builder.CreateBitCast(BitVec, ResultType);
251 llvm::Metadata *Ops[] = {llvm::MDString::get(Context,
"x18")};
252 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
253 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
256 llvm::Value *X18 = CGF.
Builder.CreateCall(F, Metadata);
263 unsigned BuiltinID) {
272 static SmallDenseMap<unsigned, StringRef, 64> F128Builtins{
273 {Builtin::BI__builtin___fprintf_chk,
"__fprintf_chkieee128"},
274 {Builtin::BI__builtin___printf_chk,
"__printf_chkieee128"},
275 {Builtin::BI__builtin___snprintf_chk,
"__snprintf_chkieee128"},
276 {Builtin::BI__builtin___sprintf_chk,
"__sprintf_chkieee128"},
277 {Builtin::BI__builtin___vfprintf_chk,
"__vfprintf_chkieee128"},
278 {Builtin::BI__builtin___vprintf_chk,
"__vprintf_chkieee128"},
279 {Builtin::BI__builtin___vsnprintf_chk,
"__vsnprintf_chkieee128"},
280 {Builtin::BI__builtin___vsprintf_chk,
"__vsprintf_chkieee128"},
281 {Builtin::BI__builtin_fprintf,
"__fprintfieee128"},
282 {Builtin::BI__builtin_printf,
"__printfieee128"},
283 {Builtin::BI__builtin_snprintf,
"__snprintfieee128"},
284 {Builtin::BI__builtin_sprintf,
"__sprintfieee128"},
285 {Builtin::BI__builtin_vfprintf,
"__vfprintfieee128"},
286 {Builtin::BI__builtin_vprintf,
"__vprintfieee128"},
287 {Builtin::BI__builtin_vsnprintf,
"__vsnprintfieee128"},
288 {Builtin::BI__builtin_vsprintf,
"__vsprintfieee128"},
289 {Builtin::BI__builtin_fscanf,
"__fscanfieee128"},
290 {Builtin::BI__builtin_scanf,
"__scanfieee128"},
291 {Builtin::BI__builtin_sscanf,
"__sscanfieee128"},
292 {Builtin::BI__builtin_vfscanf,
"__vfscanfieee128"},
293 {Builtin::BI__builtin_vscanf,
"__vscanfieee128"},
294 {Builtin::BI__builtin_vsscanf,
"__vsscanfieee128"},
295 {Builtin::BI__builtin_nexttowardf128,
"__nexttowardieee128"},
301 static SmallDenseMap<unsigned, StringRef, 4> AIXLongDouble64Builtins{
302 {Builtin::BI__builtin_frexpl,
"frexp"},
303 {Builtin::BI__builtin_ldexpl,
"ldexp"},
304 {Builtin::BI__builtin_modfl,
"modf"},
310 if (FD->
hasAttr<AsmLabelAttr>())
316 &
getTarget().getLongDoubleFormat() == &llvm::APFloat::IEEEquad() &&
317 F128Builtins.contains(BuiltinID))
318 Name = F128Builtins[BuiltinID];
321 &llvm::APFloat::IEEEdouble() &&
322 AIXLongDouble64Builtins.contains(BuiltinID))
323 Name = AIXLongDouble64Builtins[BuiltinID];
328 llvm::FunctionType *Ty =
331 return GetOrCreateLLVMFunction(Name, Ty,
D,
false);
337 QualType T, llvm::IntegerType *IntType) {
340 if (
V->getType()->isPointerTy())
341 return CGF.
Builder.CreatePtrToInt(
V, IntType);
343 assert(
V->getType() == IntType);
351 if (ResultType->isPointerTy())
352 return CGF.
Builder.CreateIntToPtr(
V, ResultType);
354 assert(
V->getType() == ResultType);
365 if (Align % Bytes != 0) {
378 AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent) {
388 llvm::IntegerType *IntType = llvm::IntegerType::get(
392 llvm::Type *ValueType = Val->getType();
420 llvm::AtomicRMWInst::BinOp Kind,
429 llvm::AtomicRMWInst::BinOp Kind,
431 Instruction::BinaryOps Op,
432 bool Invert =
false) {
441 llvm::IntegerType *IntType = llvm::IntegerType::get(
445 llvm::Type *ValueType = Val->getType();
449 Kind, DestAddr, Val, llvm::AtomicOrdering::SequentiallyConsistent);
454 llvm::ConstantInt::getAllOnesValue(IntType));
478 llvm::IntegerType *IntType = llvm::IntegerType::get(
482 llvm::Type *ValueType = Cmp->getType();
487 DestAddr, Cmp, New, llvm::AtomicOrdering::SequentiallyConsistent,
488 llvm::AtomicOrdering::SequentiallyConsistent);
491 return CGF.
Builder.CreateZExt(CGF.
Builder.CreateExtractValue(Pair, 1),
514 AtomicOrdering SuccessOrdering = AtomicOrdering::SequentiallyConsistent) {
526 auto *RTy = Exchange->getType();
530 if (RTy->isPointerTy()) {
536 auto FailureOrdering = SuccessOrdering == AtomicOrdering::Release ?
537 AtomicOrdering::Monotonic :
545 DestAddr, Comparand, Exchange, SuccessOrdering, FailureOrdering);
546 CmpXchg->setVolatile(
true);
549 if (RTy->isPointerTy()) {
570 AtomicOrdering SuccessOrdering) {
571 assert(
E->getNumArgs() == 4);
577 assert(DestPtr->getType()->isPointerTy());
578 assert(!ExchangeHigh->getType()->isPointerTy());
579 assert(!ExchangeLow->getType()->isPointerTy());
582 auto FailureOrdering = SuccessOrdering == AtomicOrdering::Release
583 ? AtomicOrdering::Monotonic
588 llvm::Type *Int128Ty = llvm::IntegerType::get(CGF.
getLLVMContext(), 128);
589 Address DestAddr(DestPtr, Int128Ty,
594 ExchangeHigh = CGF.
Builder.CreateZExt(ExchangeHigh, Int128Ty);
595 ExchangeLow = CGF.
Builder.CreateZExt(ExchangeLow, Int128Ty);
597 CGF.
Builder.CreateShl(ExchangeHigh, llvm::ConstantInt::get(Int128Ty, 64));
598 llvm::Value *Exchange = CGF.
Builder.CreateOr(ExchangeHigh, ExchangeLow);
604 SuccessOrdering, FailureOrdering);
610 CXI->setVolatile(
true);
622 AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent) {
628 AtomicRMWInst::Add, DestAddr, ConstantInt::get(IntTy, 1), Ordering);
629 return CGF.
Builder.CreateAdd(
Result, ConstantInt::get(IntTy, 1));
634 AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent) {
640 AtomicRMWInst::Sub, DestAddr, ConstantInt::get(IntTy, 1), Ordering);
641 return CGF.
Builder.CreateSub(
Result, ConstantInt::get(IntTy, 1));
652 Load->setVolatile(
true);
662 llvm::StoreInst *Store =
664 Store->setVolatile(
true);
673 unsigned ConstrainedIntrinsicID) {
676 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF,
E);
677 if (CGF.
Builder.getIsFPConstrained()) {
679 return CGF.
Builder.CreateConstrainedFPCall(F, { Src0 });
682 return CGF.
Builder.CreateCall(F, Src0);
690 unsigned ConstrainedIntrinsicID) {
694 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF,
E);
695 if (CGF.
Builder.getIsFPConstrained()) {
697 return CGF.
Builder.CreateConstrainedFPCall(F, { Src0, Src1 });
700 return CGF.
Builder.CreateCall(F, { Src0, Src1 });
707 llvm::Intrinsic::ID ConstrainedIntrinsicID) {
711 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF,
E);
712 if (CGF.
Builder.getIsFPConstrained()) {
714 {Src0->getType(), Src1->getType()});
715 return CGF.
Builder.CreateConstrainedFPCall(F, {Src0, Src1});
720 return CGF.
Builder.CreateCall(F, {Src0, Src1});
727 unsigned ConstrainedIntrinsicID) {
732 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF,
E);
733 if (CGF.
Builder.getIsFPConstrained()) {
735 return CGF.
Builder.CreateConstrainedFPCall(F, { Src0, Src1, Src2 });
738 return CGF.
Builder.CreateCall(F, { Src0, Src1, Src2 });
745 unsigned IntrinsicID,
746 unsigned ConstrainedIntrinsicID,
750 if (CGF.
Builder.getIsFPConstrained())
755 if (CGF.
Builder.getIsFPConstrained())
756 return CGF.
Builder.CreateConstrainedFPCall(F, Args);
758 return CGF.
Builder.CreateCall(F, Args);
767 unsigned IntrinsicID,
768 llvm::StringRef Name =
"") {
769 static_assert(N,
"expect non-empty argument");
771 for (
unsigned I = 0; I < N; ++I)
774 return CGF.
Builder.CreateCall(F, Args, Name);
780 unsigned IntrinsicID) {
785 return CGF.
Builder.CreateCall(F, {Src0, Src1});
791 unsigned IntrinsicID,
792 unsigned ConstrainedIntrinsicID) {
796 if (CGF.
Builder.getIsFPConstrained()) {
797 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF,
E);
799 {ResultType, Src0->getType()});
800 return CGF.
Builder.CreateConstrainedFPCall(F, {Src0});
804 return CGF.
Builder.CreateCall(F, Src0);
809 llvm::Intrinsic::ID IntrinsicID) {
817 llvm::Value *
Call = CGF.
Builder.CreateCall(F, Src0);
819 llvm::Value *Exp = CGF.
Builder.CreateExtractValue(
Call, 1);
827 llvm::Intrinsic::ID IntrinsicID) {
832 llvm::Function *F = CGF.
CGM.
getIntrinsic(IntrinsicID, {Val->getType()});
833 llvm::Value *
Call = CGF.
Builder.CreateCall(F, Val);
835 llvm::Value *SinResult = CGF.
Builder.CreateExtractValue(
Call, 0);
836 llvm::Value *CosResult = CGF.
Builder.CreateExtractValue(
Call, 1);
842 llvm::StoreInst *StoreSin =
844 llvm::StoreInst *StoreCos =
851 MDNode *
Domain = MDHelper.createAnonymousAliasScopeDomain();
852 MDNode *AliasScope = MDHelper.createAnonymousAliasScope(
Domain);
853 MDNode *AliasScopeList = MDNode::get(
Call->getContext(), AliasScope);
854 StoreSin->setMetadata(LLVMContext::MD_alias_scope, AliasScopeList);
855 StoreCos->setMetadata(LLVMContext::MD_noalias, AliasScopeList);
862 Call->setDoesNotAccessMemory();
871 llvm::Type *Ty =
V->getType();
872 int Width = Ty->getPrimitiveSizeInBits();
873 llvm::Type *IntTy = llvm::IntegerType::get(
C, Width);
875 if (Ty->isPPC_FP128Ty()) {
885 Value *ShiftCst = llvm::ConstantInt::get(IntTy, Width);
890 IntTy = llvm::IntegerType::get(
C, Width);
893 Value *Zero = llvm::Constant::getNullValue(IntTy);
894 return CGF.
Builder.CreateICmpSLT(
V, Zero);
903 auto IsIndirect = [&](
ABIArgInfo const &info) {
904 return info.isIndirect() || info.isIndirectAliased() || info.isInAlloca();
909 return IsIndirect(ArgInfo.info);
914 const CallExpr *
E, llvm::Constant *calleeValue) {
915 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF,
E);
917 llvm::CallBase *callOrInvoke =
nullptr;
921 nullptr, &callOrInvoke, &FnInfo);
926 bool ConstWithoutErrnoAndExceptions =
930 if (ConstWithoutErrnoAndExceptions && CGF.
CGM.
getLangOpts().MathErrno &&
931 !CGF.
Builder.getIsFPConstrained() &&
Call.isScalar() &&
952 const llvm::Intrinsic::ID IntrinsicID,
953 llvm::Value *
X, llvm::Value *Y,
954 llvm::Value *&Carry) {
956 assert(
X->getType() == Y->getType() &&
957 "Arguments must be the same type. (Did you forget to make sure both "
958 "arguments have the same integer width?)");
961 llvm::Value *Tmp = CGF.
Builder.CreateCall(Callee, {
X, Y});
962 Carry = CGF.
Builder.CreateExtractValue(Tmp, 1);
963 return CGF.
Builder.CreateExtractValue(Tmp, 0);
970 llvm::ConstantRange CR(APInt(32, low), APInt(32, high));
971 Call->addRangeRetAttr(CR);
972 Call->addRetAttr(llvm::Attribute::AttrKind::NoUndef);
977 struct WidthAndSignedness {
983static WidthAndSignedness
995static struct WidthAndSignedness
997 assert(Types.size() > 0 &&
"Empty list of types.");
1001 for (
const auto &
Type : Types) {
1010 for (
const auto &
Type : Types) {
1012 if (Width < MinWidth) {
1021 Intrinsic::ID inst = IsStart ? Intrinsic::vastart : Intrinsic::vaend;
1032 return From == To || (From == 0 && To == 1) || (From == 3 && To == 2);
1037 return ConstantInt::get(ResType, (
Type & 2) ? 0 : -1,
true);
1041CodeGenFunction::evaluateOrEmitBuiltinObjectSize(
const Expr *
E,
unsigned Type,
1042 llvm::IntegerType *ResType,
1043 llvm::Value *EmittedE,
1047 return emitBuiltinObjectSize(
E,
Type, ResType, EmittedE, IsDynamic);
1048 return ConstantInt::get(ResType, ObjectSize,
true);
1062 if ((!FAMDecl || FD == FAMDecl) &&
1064 Ctx, FD, FD->getType(), StrictFlexArraysLevel,
1092 if (FD->getType()->isCountAttributedType())
1104CodeGenFunction::emitFlexibleArrayMemberSize(
const Expr *
E,
unsigned Type,
1105 llvm::IntegerType *ResType) {
1134 const Expr *Idx =
nullptr;
1136 if (
const auto *UO = dyn_cast<UnaryOperator>(
Base);
1137 UO && UO->getOpcode() == UO_AddrOf) {
1139 if (
const auto *ASE = dyn_cast<ArraySubscriptExpr>(SubExpr)) {
1140 Base = ASE->getBase()->IgnoreParenImpCasts();
1143 if (
const auto *IL = dyn_cast<IntegerLiteral>(Idx)) {
1144 int64_t Val = IL->getValue().getSExtValue();
1161 if (
const auto *ME = dyn_cast<MemberExpr>(
Base)) {
1163 const ValueDecl *VD = ME->getMemberDecl();
1165 FAMDecl = dyn_cast<FieldDecl>(VD);
1168 }
else if (
const auto *DRE = dyn_cast<DeclRefExpr>(
Base)) {
1170 QualType Ty = DRE->getDecl()->getType();
1223 if (isa<DeclRefExpr>(
Base))
1247 CountedByInst =
Builder.CreateIntCast(CountedByInst, ResType, IsSigned);
1250 Value *IdxInst =
nullptr;
1258 IdxInst =
Builder.CreateIntCast(IdxInst, ResType, IdxSigned);
1263 Builder.CreateSub(CountedByInst, IdxInst,
"", !IsSigned, IsSigned);
1269 llvm::Constant *ElemSize =
1270 llvm::ConstantInt::get(ResType,
Size.getQuantity(), IsSigned);
1272 Builder.CreateMul(CountedByInst, ElemSize,
"", !IsSigned, IsSigned);
1273 Res =
Builder.CreateIntCast(Res, ResType, IsSigned);
1282 return Builder.CreateSelect(Cmp, Res, ConstantInt::get(ResType, 0, IsSigned));
1295CodeGenFunction::emitBuiltinObjectSize(
const Expr *
E,
unsigned Type,
1296 llvm::IntegerType *ResType,
1297 llvm::Value *EmittedE,
bool IsDynamic) {
1301 auto *Param = dyn_cast<ParmVarDecl>(
D->getDecl());
1302 auto *PS =
D->getDecl()->
getAttr<PassObjectSizeAttr>();
1303 if (Param !=
nullptr && PS !=
nullptr &&
1305 auto Iter = SizeArguments.find(Param);
1306 assert(
Iter != SizeArguments.end());
1309 auto DIter = LocalDeclMap.find(
D);
1310 assert(DIter != LocalDeclMap.end());
1320 if (
Value *
V = emitFlexibleArrayMemberSize(
E,
Type, ResType))
1331 assert(Ptr->
getType()->isPointerTy() &&
1332 "Non-pointer passed to __builtin_object_size?");
1348 enum ActionKind : uint8_t { TestOnly, Complement, Reset,
Set };
1349 enum InterlockingKind : uint8_t {
1358 InterlockingKind Interlocking;
1361 static BitTest decodeBitTestBuiltin(
unsigned BuiltinID);
1366BitTest BitTest::decodeBitTestBuiltin(
unsigned BuiltinID) {
1367 switch (BuiltinID) {
1369 case Builtin::BI_bittest:
1370 return {TestOnly, Unlocked,
false};
1371 case Builtin::BI_bittestandcomplement:
1372 return {Complement, Unlocked,
false};
1373 case Builtin::BI_bittestandreset:
1374 return {Reset, Unlocked,
false};
1375 case Builtin::BI_bittestandset:
1376 return {
Set, Unlocked,
false};
1377 case Builtin::BI_interlockedbittestandreset:
1378 return {Reset, Sequential,
false};
1379 case Builtin::BI_interlockedbittestandset:
1380 return {
Set, Sequential,
false};
1383 case Builtin::BI_bittest64:
1384 return {TestOnly, Unlocked,
true};
1385 case Builtin::BI_bittestandcomplement64:
1386 return {Complement, Unlocked,
true};
1387 case Builtin::BI_bittestandreset64:
1388 return {Reset, Unlocked,
true};
1389 case Builtin::BI_bittestandset64:
1390 return {
Set, Unlocked,
true};
1391 case Builtin::BI_interlockedbittestandreset64:
1392 return {Reset, Sequential,
true};
1393 case Builtin::BI_interlockedbittestandset64:
1394 return {
Set, Sequential,
true};
1397 case Builtin::BI_interlockedbittestandset_acq:
1398 return {
Set, Acquire,
false};
1399 case Builtin::BI_interlockedbittestandset_rel:
1400 return {
Set, Release,
false};
1401 case Builtin::BI_interlockedbittestandset_nf:
1402 return {
Set, NoFence,
false};
1403 case Builtin::BI_interlockedbittestandreset_acq:
1404 return {Reset, Acquire,
false};
1405 case Builtin::BI_interlockedbittestandreset_rel:
1406 return {Reset, Release,
false};
1407 case Builtin::BI_interlockedbittestandreset_nf:
1408 return {Reset, NoFence,
false};
1410 llvm_unreachable(
"expected only bittest intrinsics");
1415 case BitTest::TestOnly:
return '\0';
1416 case BitTest::Complement:
return 'c';
1417 case BitTest::Reset:
return 'r';
1418 case BitTest::Set:
return 's';
1420 llvm_unreachable(
"invalid action");
1428 char SizeSuffix = BT.Is64Bit ?
'q' :
'l';
1432 raw_svector_ostream AsmOS(
Asm);
1433 if (BT.Interlocking != BitTest::Unlocked)
1438 AsmOS << SizeSuffix <<
" $2, ($1)";
1441 std::string Constraints =
"={@ccc},r,r,~{cc},~{memory}";
1443 if (!MachineClobbers.empty()) {
1445 Constraints += MachineClobbers;
1447 llvm::IntegerType *IntType = llvm::IntegerType::get(
1450 llvm::FunctionType *FTy =
1451 llvm::FunctionType::get(CGF.
Int8Ty, {CGF.UnqualPtrTy, IntType},
false);
1453 llvm::InlineAsm *IA =
1454 llvm::InlineAsm::get(FTy,
Asm, Constraints,
true);
1455 return CGF.
Builder.CreateCall(IA, {BitBase, BitPos});
1458static llvm::AtomicOrdering
1461 case BitTest::Unlocked:
return llvm::AtomicOrdering::NotAtomic;
1462 case BitTest::Sequential:
return llvm::AtomicOrdering::SequentiallyConsistent;
1463 case BitTest::Acquire:
return llvm::AtomicOrdering::Acquire;
1464 case BitTest::Release:
return llvm::AtomicOrdering::Release;
1465 case BitTest::NoFence:
return llvm::AtomicOrdering::Monotonic;
1467 llvm_unreachable(
"invalid interlocking");
1480 BitTest BT = BitTest::decodeBitTestBuiltin(BuiltinID);
1492 BitPos, llvm::ConstantInt::get(BitPos->
getType(), 3),
"bittest.byteidx");
1494 "bittest.byteaddr"),
1498 llvm::ConstantInt::get(CGF.
Int8Ty, 0x7));
1501 Value *Mask =
nullptr;
1502 if (BT.Action != BitTest::TestOnly) {
1503 Mask = CGF.
Builder.CreateShl(llvm::ConstantInt::get(CGF.
Int8Ty, 1), PosLow,
1510 Value *OldByte =
nullptr;
1511 if (Ordering != llvm::AtomicOrdering::NotAtomic) {
1514 llvm::AtomicRMWInst::BinOp RMWOp = llvm::AtomicRMWInst::Or;
1515 if (BT.Action == BitTest::Reset) {
1516 Mask = CGF.
Builder.CreateNot(Mask);
1517 RMWOp = llvm::AtomicRMWInst::And;
1523 Value *NewByte =
nullptr;
1524 switch (BT.Action) {
1525 case BitTest::TestOnly:
1528 case BitTest::Complement:
1529 NewByte = CGF.
Builder.CreateXor(OldByte, Mask);
1531 case BitTest::Reset:
1532 NewByte = CGF.
Builder.CreateAnd(OldByte, CGF.
Builder.CreateNot(Mask));
1535 NewByte = CGF.
Builder.CreateOr(OldByte, Mask);
1544 Value *ShiftedByte = CGF.
Builder.CreateLShr(OldByte, PosLow,
"bittest.shr");
1546 ShiftedByte, llvm::ConstantInt::get(CGF.
Int8Ty, 1),
"bittest.res");
1555 raw_svector_ostream AsmOS(
Asm);
1556 llvm::IntegerType *RetType = CGF.
Int32Ty;
1558 switch (BuiltinID) {
1559 case clang::PPC::BI__builtin_ppc_ldarx:
1563 case clang::PPC::BI__builtin_ppc_lwarx:
1567 case clang::PPC::BI__builtin_ppc_lharx:
1571 case clang::PPC::BI__builtin_ppc_lbarx:
1576 llvm_unreachable(
"Expected only PowerPC load reserve intrinsics");
1579 AsmOS <<
"$0, ${1:y}";
1581 std::string Constraints =
"=r,*Z,~{memory}";
1583 if (!MachineClobbers.empty()) {
1585 Constraints += MachineClobbers;
1589 llvm::FunctionType *FTy = llvm::FunctionType::get(RetType, {PtrType},
false);
1591 llvm::InlineAsm *IA =
1592 llvm::InlineAsm::get(FTy,
Asm, Constraints,
true);
1593 llvm::CallInst *CI = CGF.
Builder.CreateCall(IA, {Addr});
1595 0, Attribute::get(CGF.
getLLVMContext(), Attribute::ElementType, RetType));
1600enum class MSVCSetJmpKind {
1612 llvm::Value *Arg1 =
nullptr;
1613 llvm::Type *Arg1Ty =
nullptr;
1615 bool IsVarArg =
false;
1616 if (SJKind == MSVCSetJmpKind::_setjmp3) {
1619 Arg1 = llvm::ConstantInt::get(CGF.
IntTy, 0);
1622 Name = SJKind == MSVCSetJmpKind::_setjmp ?
"_setjmp" :
"_setjmpex";
1625 Arg1 = CGF.
Builder.CreateCall(
1628 Arg1 = CGF.
Builder.CreateCall(
1630 llvm::ConstantInt::get(CGF.
Int32Ty, 0));
1634 llvm::Type *ArgTypes[2] = {CGF.
Int8PtrTy, Arg1Ty};
1635 llvm::AttributeList ReturnsTwiceAttr = llvm::AttributeList::get(
1637 llvm::Attribute::ReturnsTwice);
1639 llvm::FunctionType::get(CGF.
IntTy, ArgTypes, IsVarArg), Name,
1640 ReturnsTwiceAttr,
true);
1642 llvm::Value *Buf = CGF.
Builder.CreateBitOrPointerCast(
1644 llvm::Value *Args[] = {Buf, Arg1};
1646 CB->setAttributes(ReturnsTwiceAttr);
1695static std::optional<CodeGenFunction::MSVCIntrin>
1698 switch (BuiltinID) {
1700 return std::nullopt;
1701 case clang::ARM::BI_BitScanForward:
1702 case clang::ARM::BI_BitScanForward64:
1703 return MSVCIntrin::_BitScanForward;
1704 case clang::ARM::BI_BitScanReverse:
1705 case clang::ARM::BI_BitScanReverse64:
1706 return MSVCIntrin::_BitScanReverse;
1707 case clang::ARM::BI_InterlockedAnd64:
1708 return MSVCIntrin::_InterlockedAnd;
1709 case clang::ARM::BI_InterlockedExchange64:
1710 return MSVCIntrin::_InterlockedExchange;
1711 case clang::ARM::BI_InterlockedExchangeAdd64:
1712 return MSVCIntrin::_InterlockedExchangeAdd;
1713 case clang::ARM::BI_InterlockedExchangeSub64:
1714 return MSVCIntrin::_InterlockedExchangeSub;
1715 case clang::ARM::BI_InterlockedOr64:
1716 return MSVCIntrin::_InterlockedOr;
1717 case clang::ARM::BI_InterlockedXor64:
1718 return MSVCIntrin::_InterlockedXor;
1719 case clang::ARM::BI_InterlockedDecrement64:
1720 return MSVCIntrin::_InterlockedDecrement;
1721 case clang::ARM::BI_InterlockedIncrement64:
1722 return MSVCIntrin::_InterlockedIncrement;
1723 case clang::ARM::BI_InterlockedExchangeAdd8_acq:
1724 case clang::ARM::BI_InterlockedExchangeAdd16_acq:
1725 case clang::ARM::BI_InterlockedExchangeAdd_acq:
1726 case clang::ARM::BI_InterlockedExchangeAdd64_acq:
1727 return MSVCIntrin::_InterlockedExchangeAdd_acq;
1728 case clang::ARM::BI_InterlockedExchangeAdd8_rel:
1729 case clang::ARM::BI_InterlockedExchangeAdd16_rel:
1730 case clang::ARM::BI_InterlockedExchangeAdd_rel:
1731 case clang::ARM::BI_InterlockedExchangeAdd64_rel:
1732 return MSVCIntrin::_InterlockedExchangeAdd_rel;
1733 case clang::ARM::BI_InterlockedExchangeAdd8_nf:
1734 case clang::ARM::BI_InterlockedExchangeAdd16_nf:
1735 case clang::ARM::BI_InterlockedExchangeAdd_nf:
1736 case clang::ARM::BI_InterlockedExchangeAdd64_nf:
1737 return MSVCIntrin::_InterlockedExchangeAdd_nf;
1738 case clang::ARM::BI_InterlockedExchange8_acq:
1739 case clang::ARM::BI_InterlockedExchange16_acq:
1740 case clang::ARM::BI_InterlockedExchange_acq:
1741 case clang::ARM::BI_InterlockedExchange64_acq:
1742 case clang::ARM::BI_InterlockedExchangePointer_acq:
1743 return MSVCIntrin::_InterlockedExchange_acq;
1744 case clang::ARM::BI_InterlockedExchange8_rel:
1745 case clang::ARM::BI_InterlockedExchange16_rel:
1746 case clang::ARM::BI_InterlockedExchange_rel:
1747 case clang::ARM::BI_InterlockedExchange64_rel:
1748 case clang::ARM::BI_InterlockedExchangePointer_rel:
1749 return MSVCIntrin::_InterlockedExchange_rel;
1750 case clang::ARM::BI_InterlockedExchange8_nf:
1751 case clang::ARM::BI_InterlockedExchange16_nf:
1752 case clang::ARM::BI_InterlockedExchange_nf:
1753 case clang::ARM::BI_InterlockedExchange64_nf:
1754 case clang::ARM::BI_InterlockedExchangePointer_nf:
1755 return MSVCIntrin::_InterlockedExchange_nf;
1756 case clang::ARM::BI_InterlockedCompareExchange8_acq:
1757 case clang::ARM::BI_InterlockedCompareExchange16_acq:
1758 case clang::ARM::BI_InterlockedCompareExchange_acq:
1759 case clang::ARM::BI_InterlockedCompareExchange64_acq:
1760 case clang::ARM::BI_InterlockedCompareExchangePointer_acq:
1761 return MSVCIntrin::_InterlockedCompareExchange_acq;
1762 case clang::ARM::BI_InterlockedCompareExchange8_rel:
1763 case clang::ARM::BI_InterlockedCompareExchange16_rel:
1764 case clang::ARM::BI_InterlockedCompareExchange_rel:
1765 case clang::ARM::BI_InterlockedCompareExchange64_rel:
1766 case clang::ARM::BI_InterlockedCompareExchangePointer_rel:
1767 return MSVCIntrin::_InterlockedCompareExchange_rel;
1768 case clang::ARM::BI_InterlockedCompareExchange8_nf:
1769 case clang::ARM::BI_InterlockedCompareExchange16_nf:
1770 case clang::ARM::BI_InterlockedCompareExchange_nf:
1771 case clang::ARM::BI_InterlockedCompareExchange64_nf:
1772 return MSVCIntrin::_InterlockedCompareExchange_nf;
1773 case clang::ARM::BI_InterlockedOr8_acq:
1774 case clang::ARM::BI_InterlockedOr16_acq:
1775 case clang::ARM::BI_InterlockedOr_acq:
1776 case clang::ARM::BI_InterlockedOr64_acq:
1777 return MSVCIntrin::_InterlockedOr_acq;
1778 case clang::ARM::BI_InterlockedOr8_rel:
1779 case clang::ARM::BI_InterlockedOr16_rel:
1780 case clang::ARM::BI_InterlockedOr_rel:
1781 case clang::ARM::BI_InterlockedOr64_rel:
1782 return MSVCIntrin::_InterlockedOr_rel;
1783 case clang::ARM::BI_InterlockedOr8_nf:
1784 case clang::ARM::BI_InterlockedOr16_nf:
1785 case clang::ARM::BI_InterlockedOr_nf:
1786 case clang::ARM::BI_InterlockedOr64_nf:
1787 return MSVCIntrin::_InterlockedOr_nf;
1788 case clang::ARM::BI_InterlockedXor8_acq:
1789 case clang::ARM::BI_InterlockedXor16_acq:
1790 case clang::ARM::BI_InterlockedXor_acq:
1791 case clang::ARM::BI_InterlockedXor64_acq:
1792 return MSVCIntrin::_InterlockedXor_acq;
1793 case clang::ARM::BI_InterlockedXor8_rel:
1794 case clang::ARM::BI_InterlockedXor16_rel:
1795 case clang::ARM::BI_InterlockedXor_rel:
1796 case clang::ARM::BI_InterlockedXor64_rel:
1797 return MSVCIntrin::_InterlockedXor_rel;
1798 case clang::ARM::BI_InterlockedXor8_nf:
1799 case clang::ARM::BI_InterlockedXor16_nf:
1800 case clang::ARM::BI_InterlockedXor_nf:
1801 case clang::ARM::BI_InterlockedXor64_nf:
1802 return MSVCIntrin::_InterlockedXor_nf;
1803 case clang::ARM::BI_InterlockedAnd8_acq:
1804 case clang::ARM::BI_InterlockedAnd16_acq:
1805 case clang::ARM::BI_InterlockedAnd_acq:
1806 case clang::ARM::BI_InterlockedAnd64_acq:
1807 return MSVCIntrin::_InterlockedAnd_acq;
1808 case clang::ARM::BI_InterlockedAnd8_rel:
1809 case clang::ARM::BI_InterlockedAnd16_rel:
1810 case clang::ARM::BI_InterlockedAnd_rel:
1811 case clang::ARM::BI_InterlockedAnd64_rel:
1812 return MSVCIntrin::_InterlockedAnd_rel;
1813 case clang::ARM::BI_InterlockedAnd8_nf:
1814 case clang::ARM::BI_InterlockedAnd16_nf:
1815 case clang::ARM::BI_InterlockedAnd_nf:
1816 case clang::ARM::BI_InterlockedAnd64_nf:
1817 return MSVCIntrin::_InterlockedAnd_nf;
1818 case clang::ARM::BI_InterlockedIncrement16_acq:
1819 case clang::ARM::BI_InterlockedIncrement_acq:
1820 case clang::ARM::BI_InterlockedIncrement64_acq:
1821 return MSVCIntrin::_InterlockedIncrement_acq;
1822 case clang::ARM::BI_InterlockedIncrement16_rel:
1823 case clang::ARM::BI_InterlockedIncrement_rel:
1824 case clang::ARM::BI_InterlockedIncrement64_rel:
1825 return MSVCIntrin::_InterlockedIncrement_rel;
1826 case clang::ARM::BI_InterlockedIncrement16_nf:
1827 case clang::ARM::BI_InterlockedIncrement_nf:
1828 case clang::ARM::BI_InterlockedIncrement64_nf:
1829 return MSVCIntrin::_InterlockedIncrement_nf;
1830 case clang::ARM::BI_InterlockedDecrement16_acq:
1831 case clang::ARM::BI_InterlockedDecrement_acq:
1832 case clang::ARM::BI_InterlockedDecrement64_acq:
1833 return MSVCIntrin::_InterlockedDecrement_acq;
1834 case clang::ARM::BI_InterlockedDecrement16_rel:
1835 case clang::ARM::BI_InterlockedDecrement_rel:
1836 case clang::ARM::BI_InterlockedDecrement64_rel:
1837 return MSVCIntrin::_InterlockedDecrement_rel;
1838 case clang::ARM::BI_InterlockedDecrement16_nf:
1839 case clang::ARM::BI_InterlockedDecrement_nf:
1840 case clang::ARM::BI_InterlockedDecrement64_nf:
1841 return MSVCIntrin::_InterlockedDecrement_nf;
1843 llvm_unreachable(
"must return from switch");
1846static std::optional<CodeGenFunction::MSVCIntrin>
1849 switch (BuiltinID) {
1851 return std::nullopt;
1852 case clang::AArch64::BI_BitScanForward:
1853 case clang::AArch64::BI_BitScanForward64:
1854 return MSVCIntrin::_BitScanForward;
1855 case clang::AArch64::BI_BitScanReverse:
1856 case clang::AArch64::BI_BitScanReverse64:
1857 return MSVCIntrin::_BitScanReverse;
1858 case clang::AArch64::BI_InterlockedAnd64:
1859 return MSVCIntrin::_InterlockedAnd;
1860 case clang::AArch64::BI_InterlockedExchange64:
1861 return MSVCIntrin::_InterlockedExchange;
1862 case clang::AArch64::BI_InterlockedExchangeAdd64:
1863 return MSVCIntrin::_InterlockedExchangeAdd;
1864 case clang::AArch64::BI_InterlockedExchangeSub64:
1865 return MSVCIntrin::_InterlockedExchangeSub;
1866 case clang::AArch64::BI_InterlockedOr64:
1867 return MSVCIntrin::_InterlockedOr;
1868 case clang::AArch64::BI_InterlockedXor64:
1869 return MSVCIntrin::_InterlockedXor;
1870 case clang::AArch64::BI_InterlockedDecrement64:
1871 return MSVCIntrin::_InterlockedDecrement;
1872 case clang::AArch64::BI_InterlockedIncrement64:
1873 return MSVCIntrin::_InterlockedIncrement;
1874 case clang::AArch64::BI_InterlockedExchangeAdd8_acq:
1875 case clang::AArch64::BI_InterlockedExchangeAdd16_acq:
1876 case clang::AArch64::BI_InterlockedExchangeAdd_acq:
1877 case clang::AArch64::BI_InterlockedExchangeAdd64_acq:
1878 return MSVCIntrin::_InterlockedExchangeAdd_acq;
1879 case clang::AArch64::BI_InterlockedExchangeAdd8_rel:
1880 case clang::AArch64::BI_InterlockedExchangeAdd16_rel:
1881 case clang::AArch64::BI_InterlockedExchangeAdd_rel:
1882 case clang::AArch64::BI_InterlockedExchangeAdd64_rel:
1883 return MSVCIntrin::_InterlockedExchangeAdd_rel;
1884 case clang::AArch64::BI_InterlockedExchangeAdd8_nf:
1885 case clang::AArch64::BI_InterlockedExchangeAdd16_nf:
1886 case clang::AArch64::BI_InterlockedExchangeAdd_nf:
1887 case clang::AArch64::BI_InterlockedExchangeAdd64_nf:
1888 return MSVCIntrin::_InterlockedExchangeAdd_nf;
1889 case clang::AArch64::BI_InterlockedExchange8_acq:
1890 case clang::AArch64::BI_InterlockedExchange16_acq:
1891 case clang::AArch64::BI_InterlockedExchange_acq:
1892 case clang::AArch64::BI_InterlockedExchange64_acq:
1893 case clang::AArch64::BI_InterlockedExchangePointer_acq:
1894 return MSVCIntrin::_InterlockedExchange_acq;
1895 case clang::AArch64::BI_InterlockedExchange8_rel:
1896 case clang::AArch64::BI_InterlockedExchange16_rel:
1897 case clang::AArch64::BI_InterlockedExchange_rel:
1898 case clang::AArch64::BI_InterlockedExchange64_rel:
1899 case clang::AArch64::BI_InterlockedExchangePointer_rel:
1900 return MSVCIntrin::_InterlockedExchange_rel;
1901 case clang::AArch64::BI_InterlockedExchange8_nf:
1902 case clang::AArch64::BI_InterlockedExchange16_nf:
1903 case clang::AArch64::BI_InterlockedExchange_nf:
1904 case clang::AArch64::BI_InterlockedExchange64_nf:
1905 case clang::AArch64::BI_InterlockedExchangePointer_nf:
1906 return MSVCIntrin::_InterlockedExchange_nf;
1907 case clang::AArch64::BI_InterlockedCompareExchange8_acq:
1908 case clang::AArch64::BI_InterlockedCompareExchange16_acq:
1909 case clang::AArch64::BI_InterlockedCompareExchange_acq:
1910 case clang::AArch64::BI_InterlockedCompareExchange64_acq:
1911 case clang::AArch64::BI_InterlockedCompareExchangePointer_acq:
1912 return MSVCIntrin::_InterlockedCompareExchange_acq;
1913 case clang::AArch64::BI_InterlockedCompareExchange8_rel:
1914 case clang::AArch64::BI_InterlockedCompareExchange16_rel:
1915 case clang::AArch64::BI_InterlockedCompareExchange_rel:
1916 case clang::AArch64::BI_InterlockedCompareExchange64_rel:
1917 case clang::AArch64::BI_InterlockedCompareExchangePointer_rel:
1918 return MSVCIntrin::_InterlockedCompareExchange_rel;
1919 case clang::AArch64::BI_InterlockedCompareExchange8_nf:
1920 case clang::AArch64::BI_InterlockedCompareExchange16_nf:
1921 case clang::AArch64::BI_InterlockedCompareExchange_nf:
1922 case clang::AArch64::BI_InterlockedCompareExchange64_nf:
1923 return MSVCIntrin::_InterlockedCompareExchange_nf;
1924 case clang::AArch64::BI_InterlockedCompareExchange128:
1925 return MSVCIntrin::_InterlockedCompareExchange128;
1926 case clang::AArch64::BI_InterlockedCompareExchange128_acq:
1927 return MSVCIntrin::_InterlockedCompareExchange128_acq;
1928 case clang::AArch64::BI_InterlockedCompareExchange128_nf:
1929 return MSVCIntrin::_InterlockedCompareExchange128_nf;
1930 case clang::AArch64::BI_InterlockedCompareExchange128_rel:
1931 return MSVCIntrin::_InterlockedCompareExchange128_rel;
1932 case clang::AArch64::BI_InterlockedOr8_acq:
1933 case clang::AArch64::BI_InterlockedOr16_acq:
1934 case clang::AArch64::BI_InterlockedOr_acq:
1935 case clang::AArch64::BI_InterlockedOr64_acq:
1936 return MSVCIntrin::_InterlockedOr_acq;
1937 case clang::AArch64::BI_InterlockedOr8_rel:
1938 case clang::AArch64::BI_InterlockedOr16_rel:
1939 case clang::AArch64::BI_InterlockedOr_rel:
1940 case clang::AArch64::BI_InterlockedOr64_rel:
1941 return MSVCIntrin::_InterlockedOr_rel;
1942 case clang::AArch64::BI_InterlockedOr8_nf:
1943 case clang::AArch64::BI_InterlockedOr16_nf:
1944 case clang::AArch64::BI_InterlockedOr_nf:
1945 case clang::AArch64::BI_InterlockedOr64_nf:
1946 return MSVCIntrin::_InterlockedOr_nf;
1947 case clang::AArch64::BI_InterlockedXor8_acq:
1948 case clang::AArch64::BI_InterlockedXor16_acq:
1949 case clang::AArch64::BI_InterlockedXor_acq:
1950 case clang::AArch64::BI_InterlockedXor64_acq:
1951 return MSVCIntrin::_InterlockedXor_acq;
1952 case clang::AArch64::BI_InterlockedXor8_rel:
1953 case clang::AArch64::BI_InterlockedXor16_rel:
1954 case clang::AArch64::BI_InterlockedXor_rel:
1955 case clang::AArch64::BI_InterlockedXor64_rel:
1956 return MSVCIntrin::_InterlockedXor_rel;
1957 case clang::AArch64::BI_InterlockedXor8_nf:
1958 case clang::AArch64::BI_InterlockedXor16_nf:
1959 case clang::AArch64::BI_InterlockedXor_nf:
1960 case clang::AArch64::BI_InterlockedXor64_nf:
1961 return MSVCIntrin::_InterlockedXor_nf;
1962 case clang::AArch64::BI_InterlockedAnd8_acq:
1963 case clang::AArch64::BI_InterlockedAnd16_acq:
1964 case clang::AArch64::BI_InterlockedAnd_acq:
1965 case clang::AArch64::BI_InterlockedAnd64_acq:
1966 return MSVCIntrin::_InterlockedAnd_acq;
1967 case clang::AArch64::BI_InterlockedAnd8_rel:
1968 case clang::AArch64::BI_InterlockedAnd16_rel:
1969 case clang::AArch64::BI_InterlockedAnd_rel:
1970 case clang::AArch64::BI_InterlockedAnd64_rel:
1971 return MSVCIntrin::_InterlockedAnd_rel;
1972 case clang::AArch64::BI_InterlockedAnd8_nf:
1973 case clang::AArch64::BI_InterlockedAnd16_nf:
1974 case clang::AArch64::BI_InterlockedAnd_nf:
1975 case clang::AArch64::BI_InterlockedAnd64_nf:
1976 return MSVCIntrin::_InterlockedAnd_nf;
1977 case clang::AArch64::BI_InterlockedIncrement16_acq:
1978 case clang::AArch64::BI_InterlockedIncrement_acq:
1979 case clang::AArch64::BI_InterlockedIncrement64_acq:
1980 return MSVCIntrin::_InterlockedIncrement_acq;
1981 case clang::AArch64::BI_InterlockedIncrement16_rel:
1982 case clang::AArch64::BI_InterlockedIncrement_rel:
1983 case clang::AArch64::BI_InterlockedIncrement64_rel:
1984 return MSVCIntrin::_InterlockedIncrement_rel;
1985 case clang::AArch64::BI_InterlockedIncrement16_nf:
1986 case clang::AArch64::BI_InterlockedIncrement_nf:
1987 case clang::AArch64::BI_InterlockedIncrement64_nf:
1988 return MSVCIntrin::_InterlockedIncrement_nf;
1989 case clang::AArch64::BI_InterlockedDecrement16_acq:
1990 case clang::AArch64::BI_InterlockedDecrement_acq:
1991 case clang::AArch64::BI_InterlockedDecrement64_acq:
1992 return MSVCIntrin::_InterlockedDecrement_acq;
1993 case clang::AArch64::BI_InterlockedDecrement16_rel:
1994 case clang::AArch64::BI_InterlockedDecrement_rel:
1995 case clang::AArch64::BI_InterlockedDecrement64_rel:
1996 return MSVCIntrin::_InterlockedDecrement_rel;
1997 case clang::AArch64::BI_InterlockedDecrement16_nf:
1998 case clang::AArch64::BI_InterlockedDecrement_nf:
1999 case clang::AArch64::BI_InterlockedDecrement64_nf:
2000 return MSVCIntrin::_InterlockedDecrement_nf;
2002 llvm_unreachable(
"must return from switch");
2005static std::optional<CodeGenFunction::MSVCIntrin>
2008 switch (BuiltinID) {
2010 return std::nullopt;
2011 case clang::X86::BI_BitScanForward:
2012 case clang::X86::BI_BitScanForward64:
2013 return MSVCIntrin::_BitScanForward;
2014 case clang::X86::BI_BitScanReverse:
2015 case clang::X86::BI_BitScanReverse64:
2016 return MSVCIntrin::_BitScanReverse;
2017 case clang::X86::BI_InterlockedAnd64:
2018 return MSVCIntrin::_InterlockedAnd;
2019 case clang::X86::BI_InterlockedCompareExchange128:
2020 return MSVCIntrin::_InterlockedCompareExchange128;
2021 case clang::X86::BI_InterlockedExchange64:
2022 return MSVCIntrin::_InterlockedExchange;
2023 case clang::X86::BI_InterlockedExchangeAdd64:
2024 return MSVCIntrin::_InterlockedExchangeAdd;
2025 case clang::X86::BI_InterlockedExchangeSub64:
2026 return MSVCIntrin::_InterlockedExchangeSub;
2027 case clang::X86::BI_InterlockedOr64:
2028 return MSVCIntrin::_InterlockedOr;
2029 case clang::X86::BI_InterlockedXor64:
2030 return MSVCIntrin::_InterlockedXor;
2031 case clang::X86::BI_InterlockedDecrement64:
2032 return MSVCIntrin::_InterlockedDecrement;
2033 case clang::X86::BI_InterlockedIncrement64:
2034 return MSVCIntrin::_InterlockedIncrement;
2036 llvm_unreachable(
"must return from switch");
2042 switch (BuiltinID) {
2043 case MSVCIntrin::_BitScanForward:
2044 case MSVCIntrin::_BitScanReverse: {
2048 llvm::Type *ArgType = ArgValue->
getType();
2049 llvm::Type *IndexType = IndexAddress.getElementType();
2052 Value *ArgZero = llvm::Constant::getNullValue(ArgType);
2053 Value *ResZero = llvm::Constant::getNullValue(ResultType);
2054 Value *ResOne = llvm::ConstantInt::get(ResultType, 1);
2059 PHINode *
Result =
Builder.CreatePHI(ResultType, 2,
"bitscan_result");
2062 Value *IsZero =
Builder.CreateICmpEQ(ArgValue, ArgZero);
2064 Builder.CreateCondBr(IsZero, End, NotZero);
2067 Builder.SetInsertPoint(NotZero);
2069 if (BuiltinID == MSVCIntrin::_BitScanForward) {
2072 ZeroCount =
Builder.CreateIntCast(ZeroCount, IndexType,
false);
2075 unsigned ArgWidth = cast<llvm::IntegerType>(ArgType)->getBitWidth();
2076 Value *ArgTypeLastIndex = llvm::ConstantInt::get(IndexType, ArgWidth - 1);
2080 ZeroCount =
Builder.CreateIntCast(ZeroCount, IndexType,
false);
2081 Value *Index =
Builder.CreateNSWSub(ArgTypeLastIndex, ZeroCount);
2085 Result->addIncoming(ResOne, NotZero);
2090 case MSVCIntrin::_InterlockedAnd:
2092 case MSVCIntrin::_InterlockedExchange:
2094 case MSVCIntrin::_InterlockedExchangeAdd:
2096 case MSVCIntrin::_InterlockedExchangeSub:
2098 case MSVCIntrin::_InterlockedOr:
2100 case MSVCIntrin::_InterlockedXor:
2102 case MSVCIntrin::_InterlockedExchangeAdd_acq:
2104 AtomicOrdering::Acquire);
2105 case MSVCIntrin::_InterlockedExchangeAdd_rel:
2107 AtomicOrdering::Release);
2108 case MSVCIntrin::_InterlockedExchangeAdd_nf:
2110 AtomicOrdering::Monotonic);
2111 case MSVCIntrin::_InterlockedExchange_acq:
2113 AtomicOrdering::Acquire);
2114 case MSVCIntrin::_InterlockedExchange_rel:
2116 AtomicOrdering::Release);
2117 case MSVCIntrin::_InterlockedExchange_nf:
2119 AtomicOrdering::Monotonic);
2120 case MSVCIntrin::_InterlockedCompareExchange:
2122 case MSVCIntrin::_InterlockedCompareExchange_acq:
2124 case MSVCIntrin::_InterlockedCompareExchange_rel:
2126 case MSVCIntrin::_InterlockedCompareExchange_nf:
2128 case MSVCIntrin::_InterlockedCompareExchange128:
2130 *
this,
E, AtomicOrdering::SequentiallyConsistent);
2131 case MSVCIntrin::_InterlockedCompareExchange128_acq:
2133 case MSVCIntrin::_InterlockedCompareExchange128_rel:
2135 case MSVCIntrin::_InterlockedCompareExchange128_nf:
2137 case MSVCIntrin::_InterlockedOr_acq:
2139 AtomicOrdering::Acquire);
2140 case MSVCIntrin::_InterlockedOr_rel:
2142 AtomicOrdering::Release);
2143 case MSVCIntrin::_InterlockedOr_nf:
2145 AtomicOrdering::Monotonic);
2146 case MSVCIntrin::_InterlockedXor_acq:
2148 AtomicOrdering::Acquire);
2149 case MSVCIntrin::_InterlockedXor_rel:
2151 AtomicOrdering::Release);
2152 case MSVCIntrin::_InterlockedXor_nf:
2154 AtomicOrdering::Monotonic);
2155 case MSVCIntrin::_InterlockedAnd_acq:
2157 AtomicOrdering::Acquire);
2158 case MSVCIntrin::_InterlockedAnd_rel:
2160 AtomicOrdering::Release);
2161 case MSVCIntrin::_InterlockedAnd_nf:
2163 AtomicOrdering::Monotonic);
2164 case MSVCIntrin::_InterlockedIncrement_acq:
2166 case MSVCIntrin::_InterlockedIncrement_rel:
2168 case MSVCIntrin::_InterlockedIncrement_nf:
2170 case MSVCIntrin::_InterlockedDecrement_acq:
2172 case MSVCIntrin::_InterlockedDecrement_rel:
2174 case MSVCIntrin::_InterlockedDecrement_nf:
2177 case MSVCIntrin::_InterlockedDecrement:
2179 case MSVCIntrin::_InterlockedIncrement:
2182 case MSVCIntrin::__fastfail: {
2187 StringRef
Asm, Constraints;
2192 case llvm::Triple::x86:
2193 case llvm::Triple::x86_64:
2195 Constraints =
"{cx}";
2197 case llvm::Triple::thumb:
2199 Constraints =
"{r0}";
2201 case llvm::Triple::aarch64:
2202 Asm =
"brk #0xF003";
2203 Constraints =
"{w0}";
2205 llvm::FunctionType *FTy = llvm::FunctionType::get(
VoidTy, {
Int32Ty},
false);
2206 llvm::InlineAsm *IA =
2207 llvm::InlineAsm::get(FTy,
Asm, Constraints,
true);
2208 llvm::AttributeList NoReturnAttr = llvm::AttributeList::get(
2210 llvm::Attribute::NoReturn);
2212 CI->setAttributes(NoReturnAttr);
2216 llvm_unreachable(
"Incorrect MSVC intrinsic!");
2222 CallObjCArcUse(llvm::Value *
object) : object(object) {}
2223 llvm::Value *object;
2232 BuiltinCheckKind Kind) {
2234 "Unsupported builtin check kind");
2240 SanitizerScope SanScope(
this);
2242 ArgValue, llvm::Constant::getNullValue(ArgValue->
getType()));
2243 EmitCheck(std::make_pair(Cond, SanitizerKind::SO_Builtin),
2244 SanitizerHandler::InvalidBuiltin,
2246 llvm::ConstantInt::get(
Builder.getInt8Ty(), Kind)},
2256 SanitizerScope SanScope(
this);
2258 std::make_pair(ArgValue, SanitizerKind::SO_Builtin),
2259 SanitizerHandler::InvalidBuiltin,
2267 return CGF.
Builder.CreateBinaryIntrinsic(
2268 Intrinsic::abs, ArgValue,
2269 ConstantInt::get(CGF.
Builder.getInt1Ty(), HasNSW));
2273 bool SanitizeOverflow) {
2277 if (
const auto *VCI = dyn_cast<llvm::ConstantInt>(ArgValue)) {
2278 if (!VCI->isMinSignedValue())
2279 return EmitAbs(CGF, ArgValue,
true);
2282 CodeGenFunction::SanitizerScope SanScope(&CGF);
2284 Constant *Zero = Constant::getNullValue(ArgValue->
getType());
2285 Value *ResultAndOverflow = CGF.
Builder.CreateBinaryIntrinsic(
2286 Intrinsic::ssub_with_overflow, Zero, ArgValue);
2289 CGF.
Builder.CreateExtractValue(ResultAndOverflow, 1));
2292 if (SanitizeOverflow) {
2293 CGF.
EmitCheck({{NotOverflow, SanitizerKind::SO_SignedIntegerOverflow}},
2294 SanitizerHandler::NegateOverflow,
2299 CGF.
EmitTrapCheck(NotOverflow, SanitizerHandler::SubOverflow);
2301 Value *CmpResult = CGF.
Builder.CreateICmpSLT(ArgValue, Zero,
"abscond");
2302 return CGF.
Builder.CreateSelect(CmpResult,
Result, ArgValue,
"abs");
2307 QualType UnsignedTy =
C.getIntTypeForBitwidth(Size * 8,
false);
2308 return C.getCanonicalType(UnsignedTy);
2318 raw_svector_ostream OS(Name);
2319 OS <<
"__os_log_helper";
2323 for (
const auto &Item : Layout.
Items)
2324 OS <<
"_" <<
int(Item.getSizeByte()) <<
"_"
2325 <<
int(Item.getDescriptorByte());
2328 if (llvm::Function *F =
CGM.
getModule().getFunction(Name))
2338 for (
unsigned int I = 0,
E = Layout.
Items.size(); I <
E; ++I) {
2339 char Size = Layout.
Items[I].getSizeByte();
2346 &Ctx.
Idents.
get(std::string(
"arg") + llvm::to_string(I)), ArgTy,
2348 ArgTys.emplace_back(ArgTy);
2359 llvm::Function *
Fn = llvm::Function::Create(
2360 FuncTy, llvm::GlobalValue::LinkOnceODRLinkage, Name, &
CGM.
getModule());
2361 Fn->setVisibility(llvm::GlobalValue::HiddenVisibility);
2364 Fn->setDoesNotThrow();
2368 Fn->addFnAttr(llvm::Attribute::NoInline);
2386 for (
const auto &Item : Layout.
Items) {
2388 Builder.getInt8(Item.getDescriptorByte()),
2391 Builder.getInt8(Item.getSizeByte()),
2395 if (!
Size.getQuantity())
2412 assert(
E.getNumArgs() >= 2 &&
2413 "__builtin_os_log_format takes at least 2 arguments");
2424 for (
const auto &Item : Layout.
Items) {
2425 int Size = Item.getSizeByte();
2429 llvm::Value *ArgVal;
2433 for (
unsigned I = 0,
E = Item.getMaskType().size(); I <
E; ++I)
2434 Val |= ((
uint64_t)Item.getMaskType()[I]) << I * 8;
2435 ArgVal = llvm::Constant::getIntegerValue(
Int64Ty, llvm::APInt(64, Val));
2436 }
else if (
const Expr *TheExpr = Item.getExpr()) {
2442 auto LifetimeExtendObject = [&](
const Expr *
E) {
2450 if (isa<CallExpr>(
E) || isa<ObjCMessageExpr>(
E))
2455 if (TheExpr->getType()->isObjCRetainableType() &&
2456 getLangOpts().ObjCAutoRefCount && LifetimeExtendObject(TheExpr)) {
2458 "Only scalar can be a ObjC retainable type");
2459 if (!isa<Constant>(ArgVal)) {
2473 pushCleanupAfterFullExpr<CallObjCArcUse>(Cleanup, ArgVal);
2477 ArgVal =
Builder.getInt32(Item.getConstValue().getQuantity());
2480 unsigned ArgValSize =
2484 ArgVal =
Builder.CreateBitOrPointerCast(ArgVal,
IntTy);
2500 unsigned BuiltinID, WidthAndSignedness Op1Info, WidthAndSignedness Op2Info,
2501 WidthAndSignedness ResultInfo) {
2502 return BuiltinID == Builtin::BI__builtin_mul_overflow &&
2503 Op1Info.Width == Op2Info.Width && Op2Info.Width == ResultInfo.Width &&
2504 !Op1Info.Signed && !Op2Info.Signed && ResultInfo.Signed;
2509 const clang::Expr *Op2, WidthAndSignedness Op2Info,
2511 WidthAndSignedness ResultInfo) {
2513 Builtin::BI__builtin_mul_overflow, Op1Info, Op2Info, ResultInfo) &&
2514 "Cannot specialize this multiply");
2519 llvm::Value *HasOverflow;
2521 CGF, llvm::Intrinsic::umul_with_overflow, V1, V2, HasOverflow);
2526 auto IntMax = llvm::APInt::getSignedMaxValue(ResultInfo.Width);
2527 llvm::Value *IntMaxValue = llvm::ConstantInt::get(
Result->getType(), IntMax);
2529 llvm::Value *IntMaxOverflow = CGF.
Builder.CreateICmpUGT(
Result, IntMaxValue);
2530 HasOverflow = CGF.
Builder.CreateOr(HasOverflow, IntMaxOverflow);
2542 WidthAndSignedness Op1Info,
2543 WidthAndSignedness Op2Info,
2544 WidthAndSignedness ResultInfo) {
2545 return BuiltinID == Builtin::BI__builtin_mul_overflow &&
2546 std::max(Op1Info.Width, Op2Info.Width) >= ResultInfo.Width &&
2547 Op1Info.Signed != Op2Info.Signed;
2554 WidthAndSignedness Op1Info,
const clang::Expr *Op2,
2555 WidthAndSignedness Op2Info,
2557 WidthAndSignedness ResultInfo) {
2559 Op2Info, ResultInfo) &&
2560 "Not a mixed-sign multipliction we can specialize");
2563 const clang::Expr *SignedOp = Op1Info.Signed ? Op1 : Op2;
2564 const clang::Expr *UnsignedOp = Op1Info.Signed ? Op2 : Op1;
2567 unsigned SignedOpWidth = Op1Info.Signed ? Op1Info.Width : Op2Info.Width;
2568 unsigned UnsignedOpWidth = Op1Info.Signed ? Op2Info.Width : Op1Info.Width;
2571 if (SignedOpWidth < UnsignedOpWidth)
2573 if (UnsignedOpWidth < SignedOpWidth)
2576 llvm::Type *OpTy =
Signed->getType();
2577 llvm::Value *Zero = llvm::Constant::getNullValue(OpTy);
2580 unsigned OpWidth = std::max(Op1Info.Width, Op2Info.Width);
2583 llvm::Value *IsNegative = CGF.
Builder.CreateICmpSLT(
Signed, Zero);
2584 llvm::Value *AbsOfNegative = CGF.
Builder.CreateSub(Zero,
Signed);
2585 llvm::Value *AbsSigned =
2586 CGF.
Builder.CreateSelect(IsNegative, AbsOfNegative,
Signed);
2589 llvm::Value *UnsignedOverflow;
2590 llvm::Value *UnsignedResult =
2594 llvm::Value *Overflow, *
Result;
2595 if (ResultInfo.Signed) {
2599 llvm::APInt::getSignedMaxValue(ResultInfo.Width).zext(OpWidth);
2600 llvm::Value *MaxResult =
2601 CGF.
Builder.CreateAdd(llvm::ConstantInt::get(OpTy, IntMax),
2602 CGF.
Builder.CreateZExt(IsNegative, OpTy));
2603 llvm::Value *SignedOverflow =
2604 CGF.
Builder.CreateICmpUGT(UnsignedResult, MaxResult);
2605 Overflow = CGF.
Builder.CreateOr(UnsignedOverflow, SignedOverflow);
2608 llvm::Value *NegativeResult = CGF.
Builder.CreateNeg(UnsignedResult);
2609 llvm::Value *SignedResult =
2610 CGF.
Builder.CreateSelect(IsNegative, NegativeResult, UnsignedResult);
2614 llvm::Value *Underflow = CGF.
Builder.CreateAnd(
2615 IsNegative, CGF.
Builder.CreateIsNotNull(UnsignedResult));
2616 Overflow = CGF.
Builder.CreateOr(UnsignedOverflow, Underflow);
2617 if (ResultInfo.Width < OpWidth) {
2619 llvm::APInt::getMaxValue(ResultInfo.Width).zext(OpWidth);
2620 llvm::Value *TruncOverflow = CGF.
Builder.CreateICmpUGT(
2621 UnsignedResult, llvm::ConstantInt::get(OpTy, IntMax));
2622 Overflow = CGF.
Builder.CreateOr(Overflow, TruncOverflow);
2627 IsNegative, CGF.
Builder.CreateNeg(UnsignedResult), UnsignedResult);
2631 assert(Overflow &&
Result &&
"Missing overflow or result");
2642 llvm::SmallPtrSetImpl<const Decl *> &Seen) {
2651 if (!Seen.insert(
Record).second)
2654 assert(
Record->hasDefinition() &&
2655 "Incomplete types should already be diagnosed");
2657 if (
Record->isDynamicClass())
2682 llvm::Type *Ty = Src->getType();
2683 ShiftAmt =
Builder.CreateIntCast(ShiftAmt, Ty,
false);
2686 unsigned IID = IsRotateRight ? Intrinsic::fshr : Intrinsic::fshl;
2693 switch (BuiltinID) {
2694#define MUTATE_LDBL(func) \
2695 case Builtin::BI__builtin_##func##l: \
2696 return Builtin::BI__builtin_##func##f128;
2765 if (CGF.
Builder.getIsFPConstrained() &&
2766 CGF.
Builder.getDefaultConstrainedExcept() != fp::ebIgnore) {
2778 auto UBF = CGF->
CGM.
getModule().getOrInsertFunction(Name, FnTy);
2781 for (
auto &&FormalTy : FnTy->params())
2782 Args.push_back(llvm::PoisonValue::get(FormalTy));
2791 "Should not codegen for consteval builtins");
2798 !
Result.hasSideEffects()) {
2802 if (
Result.Val.isFloat())
2811 if (
getTarget().getTriple().isPPC64() &&
2812 &
getTarget().getLongDoubleFormat() == &llvm::APFloat::IEEEquad())
2819 const unsigned BuiltinIDIfNoAsmLabel =
2820 FD->
hasAttr<AsmLabelAttr>() ? 0 : BuiltinID;
2822 std::optional<bool> ErrnoOverriden;
2826 if (
E->hasStoredFPFeatures()) {
2828 if (OP.hasMathErrnoOverride())
2829 ErrnoOverriden = OP.getMathErrnoOverride();
2838 bool ErrnoOverridenToFalseWithOpt =
2839 ErrnoOverriden.has_value() && !ErrnoOverriden.value() && !OptNone &&
2857 switch (BuiltinID) {
2858 case Builtin::BI__builtin_fma:
2859 case Builtin::BI__builtin_fmaf:
2860 case Builtin::BI__builtin_fmal:
2861 case Builtin::BI__builtin_fmaf16:
2862 case Builtin::BIfma:
2863 case Builtin::BIfmaf:
2864 case Builtin::BIfmal: {
2866 if (Trip.isGNUEnvironment() || Trip.isOSMSVCRT())
2874 bool ConstWithoutErrnoAndExceptions =
2876 bool ConstWithoutExceptions =
2894 bool ConstWithoutErrnoOrExceptions =
2895 ConstWithoutErrnoAndExceptions || ConstWithoutExceptions;
2896 bool GenerateIntrinsics =
2897 (ConstAlways && !OptNone) ||
2899 !(ErrnoOverriden.has_value() && ErrnoOverriden.value()) && !OptNone);
2900 if (!GenerateIntrinsics) {
2901 GenerateIntrinsics =
2902 ConstWithoutErrnoOrExceptions && !ConstWithoutErrnoAndExceptions;
2903 if (!GenerateIntrinsics)
2904 GenerateIntrinsics =
2905 ConstWithoutErrnoOrExceptions &&
2907 !(ErrnoOverriden.has_value() && ErrnoOverriden.value()) && !OptNone);
2908 if (!GenerateIntrinsics)
2909 GenerateIntrinsics =
2910 ConstWithoutErrnoOrExceptions && ErrnoOverridenToFalseWithOpt;
2912 if (GenerateIntrinsics) {
2913 switch (BuiltinIDIfNoAsmLabel) {
2914 case Builtin::BIacos:
2915 case Builtin::BIacosf:
2916 case Builtin::BIacosl:
2917 case Builtin::BI__builtin_acos:
2918 case Builtin::BI__builtin_acosf:
2919 case Builtin::BI__builtin_acosf16:
2920 case Builtin::BI__builtin_acosl:
2921 case Builtin::BI__builtin_acosf128:
2923 *
this,
E, Intrinsic::acos, Intrinsic::experimental_constrained_acos));
2925 case Builtin::BIasin:
2926 case Builtin::BIasinf:
2927 case Builtin::BIasinl:
2928 case Builtin::BI__builtin_asin:
2929 case Builtin::BI__builtin_asinf:
2930 case Builtin::BI__builtin_asinf16:
2931 case Builtin::BI__builtin_asinl:
2932 case Builtin::BI__builtin_asinf128:
2934 *
this,
E, Intrinsic::asin, Intrinsic::experimental_constrained_asin));
2936 case Builtin::BIatan:
2937 case Builtin::BIatanf:
2938 case Builtin::BIatanl:
2939 case Builtin::BI__builtin_atan:
2940 case Builtin::BI__builtin_atanf:
2941 case Builtin::BI__builtin_atanf16:
2942 case Builtin::BI__builtin_atanl:
2943 case Builtin::BI__builtin_atanf128:
2945 *
this,
E, Intrinsic::atan, Intrinsic::experimental_constrained_atan));
2947 case Builtin::BIatan2:
2948 case Builtin::BIatan2f:
2949 case Builtin::BIatan2l:
2950 case Builtin::BI__builtin_atan2:
2951 case Builtin::BI__builtin_atan2f:
2952 case Builtin::BI__builtin_atan2f16:
2953 case Builtin::BI__builtin_atan2l:
2954 case Builtin::BI__builtin_atan2f128:
2956 *
this,
E, Intrinsic::atan2,
2957 Intrinsic::experimental_constrained_atan2));
2959 case Builtin::BIceil:
2960 case Builtin::BIceilf:
2961 case Builtin::BIceill:
2962 case Builtin::BI__builtin_ceil:
2963 case Builtin::BI__builtin_ceilf:
2964 case Builtin::BI__builtin_ceilf16:
2965 case Builtin::BI__builtin_ceill:
2966 case Builtin::BI__builtin_ceilf128:
2969 Intrinsic::experimental_constrained_ceil));
2971 case Builtin::BIcopysign:
2972 case Builtin::BIcopysignf:
2973 case Builtin::BIcopysignl:
2974 case Builtin::BI__builtin_copysign:
2975 case Builtin::BI__builtin_copysignf:
2976 case Builtin::BI__builtin_copysignf16:
2977 case Builtin::BI__builtin_copysignl:
2978 case Builtin::BI__builtin_copysignf128:
2980 emitBuiltinWithOneOverloadedType<2>(*
this,
E, Intrinsic::copysign));
2982 case Builtin::BIcos:
2983 case Builtin::BIcosf:
2984 case Builtin::BIcosl:
2985 case Builtin::BI__builtin_cos:
2986 case Builtin::BI__builtin_cosf:
2987 case Builtin::BI__builtin_cosf16:
2988 case Builtin::BI__builtin_cosl:
2989 case Builtin::BI__builtin_cosf128:
2992 Intrinsic::experimental_constrained_cos));
2994 case Builtin::BIcosh:
2995 case Builtin::BIcoshf:
2996 case Builtin::BIcoshl:
2997 case Builtin::BI__builtin_cosh:
2998 case Builtin::BI__builtin_coshf:
2999 case Builtin::BI__builtin_coshf16:
3000 case Builtin::BI__builtin_coshl:
3001 case Builtin::BI__builtin_coshf128:
3003 *
this,
E, Intrinsic::cosh, Intrinsic::experimental_constrained_cosh));
3005 case Builtin::BIexp:
3006 case Builtin::BIexpf:
3007 case Builtin::BIexpl:
3008 case Builtin::BI__builtin_exp:
3009 case Builtin::BI__builtin_expf:
3010 case Builtin::BI__builtin_expf16:
3011 case Builtin::BI__builtin_expl:
3012 case Builtin::BI__builtin_expf128:
3015 Intrinsic::experimental_constrained_exp));
3017 case Builtin::BIexp2:
3018 case Builtin::BIexp2f:
3019 case Builtin::BIexp2l:
3020 case Builtin::BI__builtin_exp2:
3021 case Builtin::BI__builtin_exp2f:
3022 case Builtin::BI__builtin_exp2f16:
3023 case Builtin::BI__builtin_exp2l:
3024 case Builtin::BI__builtin_exp2f128:
3027 Intrinsic::experimental_constrained_exp2));
3028 case Builtin::BI__builtin_exp10:
3029 case Builtin::BI__builtin_exp10f:
3030 case Builtin::BI__builtin_exp10f16:
3031 case Builtin::BI__builtin_exp10l:
3032 case Builtin::BI__builtin_exp10f128: {
3034 if (
Builder.getIsFPConstrained())
3037 emitBuiltinWithOneOverloadedType<1>(*
this,
E, Intrinsic::exp10));
3039 case Builtin::BIfabs:
3040 case Builtin::BIfabsf:
3041 case Builtin::BIfabsl:
3042 case Builtin::BI__builtin_fabs:
3043 case Builtin::BI__builtin_fabsf:
3044 case Builtin::BI__builtin_fabsf16:
3045 case Builtin::BI__builtin_fabsl:
3046 case Builtin::BI__builtin_fabsf128:
3048 emitBuiltinWithOneOverloadedType<1>(*
this,
E, Intrinsic::fabs));
3050 case Builtin::BIfloor:
3051 case Builtin::BIfloorf:
3052 case Builtin::BIfloorl:
3053 case Builtin::BI__builtin_floor:
3054 case Builtin::BI__builtin_floorf:
3055 case Builtin::BI__builtin_floorf16:
3056 case Builtin::BI__builtin_floorl:
3057 case Builtin::BI__builtin_floorf128:
3060 Intrinsic::experimental_constrained_floor));
3062 case Builtin::BIfma:
3063 case Builtin::BIfmaf:
3064 case Builtin::BIfmal:
3065 case Builtin::BI__builtin_fma:
3066 case Builtin::BI__builtin_fmaf:
3067 case Builtin::BI__builtin_fmaf16:
3068 case Builtin::BI__builtin_fmal:
3069 case Builtin::BI__builtin_fmaf128:
3072 Intrinsic::experimental_constrained_fma));
3074 case Builtin::BIfmax:
3075 case Builtin::BIfmaxf:
3076 case Builtin::BIfmaxl:
3077 case Builtin::BI__builtin_fmax:
3078 case Builtin::BI__builtin_fmaxf:
3079 case Builtin::BI__builtin_fmaxf16:
3080 case Builtin::BI__builtin_fmaxl:
3081 case Builtin::BI__builtin_fmaxf128:
3084 Intrinsic::experimental_constrained_maxnum));
3086 case Builtin::BIfmin:
3087 case Builtin::BIfminf:
3088 case Builtin::BIfminl:
3089 case Builtin::BI__builtin_fmin:
3090 case Builtin::BI__builtin_fminf:
3091 case Builtin::BI__builtin_fminf16:
3092 case Builtin::BI__builtin_fminl:
3093 case Builtin::BI__builtin_fminf128:
3096 Intrinsic::experimental_constrained_minnum));
3098 case Builtin::BIfmaximum_num:
3099 case Builtin::BIfmaximum_numf:
3100 case Builtin::BIfmaximum_numl:
3101 case Builtin::BI__builtin_fmaximum_num:
3102 case Builtin::BI__builtin_fmaximum_numf:
3103 case Builtin::BI__builtin_fmaximum_numf16:
3104 case Builtin::BI__builtin_fmaximum_numl:
3105 case Builtin::BI__builtin_fmaximum_numf128:
3107 emitBuiltinWithOneOverloadedType<2>(*
this,
E, Intrinsic::maximumnum));
3109 case Builtin::BIfminimum_num:
3110 case Builtin::BIfminimum_numf:
3111 case Builtin::BIfminimum_numl:
3112 case Builtin::BI__builtin_fminimum_num:
3113 case Builtin::BI__builtin_fminimum_numf:
3114 case Builtin::BI__builtin_fminimum_numf16:
3115 case Builtin::BI__builtin_fminimum_numl:
3116 case Builtin::BI__builtin_fminimum_numf128:
3118 emitBuiltinWithOneOverloadedType<2>(*
this,
E, Intrinsic::minimumnum));
3122 case Builtin::BIfmod:
3123 case Builtin::BIfmodf:
3124 case Builtin::BIfmodl:
3125 case Builtin::BI__builtin_fmod:
3126 case Builtin::BI__builtin_fmodf:
3127 case Builtin::BI__builtin_fmodf16:
3128 case Builtin::BI__builtin_fmodl:
3129 case Builtin::BI__builtin_fmodf128:
3130 case Builtin::BI__builtin_elementwise_fmod: {
3131 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
3137 case Builtin::BIlog:
3138 case Builtin::BIlogf:
3139 case Builtin::BIlogl:
3140 case Builtin::BI__builtin_log:
3141 case Builtin::BI__builtin_logf:
3142 case Builtin::BI__builtin_logf16:
3143 case Builtin::BI__builtin_logl:
3144 case Builtin::BI__builtin_logf128:
3147 Intrinsic::experimental_constrained_log));
3149 case Builtin::BIlog10:
3150 case Builtin::BIlog10f:
3151 case Builtin::BIlog10l:
3152 case Builtin::BI__builtin_log10:
3153 case Builtin::BI__builtin_log10f:
3154 case Builtin::BI__builtin_log10f16:
3155 case Builtin::BI__builtin_log10l:
3156 case Builtin::BI__builtin_log10f128:
3159 Intrinsic::experimental_constrained_log10));
3161 case Builtin::BIlog2:
3162 case Builtin::BIlog2f:
3163 case Builtin::BIlog2l:
3164 case Builtin::BI__builtin_log2:
3165 case Builtin::BI__builtin_log2f:
3166 case Builtin::BI__builtin_log2f16:
3167 case Builtin::BI__builtin_log2l:
3168 case Builtin::BI__builtin_log2f128:
3171 Intrinsic::experimental_constrained_log2));
3173 case Builtin::BInearbyint:
3174 case Builtin::BInearbyintf:
3175 case Builtin::BInearbyintl:
3176 case Builtin::BI__builtin_nearbyint:
3177 case Builtin::BI__builtin_nearbyintf:
3178 case Builtin::BI__builtin_nearbyintl:
3179 case Builtin::BI__builtin_nearbyintf128:
3181 Intrinsic::nearbyint,
3182 Intrinsic::experimental_constrained_nearbyint));
3184 case Builtin::BIpow:
3185 case Builtin::BIpowf:
3186 case Builtin::BIpowl:
3187 case Builtin::BI__builtin_pow:
3188 case Builtin::BI__builtin_powf:
3189 case Builtin::BI__builtin_powf16:
3190 case Builtin::BI__builtin_powl:
3191 case Builtin::BI__builtin_powf128:
3194 Intrinsic::experimental_constrained_pow));
3196 case Builtin::BIrint:
3197 case Builtin::BIrintf:
3198 case Builtin::BIrintl:
3199 case Builtin::BI__builtin_rint:
3200 case Builtin::BI__builtin_rintf:
3201 case Builtin::BI__builtin_rintf16:
3202 case Builtin::BI__builtin_rintl:
3203 case Builtin::BI__builtin_rintf128:
3206 Intrinsic::experimental_constrained_rint));
3208 case Builtin::BIround:
3209 case Builtin::BIroundf:
3210 case Builtin::BIroundl:
3211 case Builtin::BI__builtin_round:
3212 case Builtin::BI__builtin_roundf:
3213 case Builtin::BI__builtin_roundf16:
3214 case Builtin::BI__builtin_roundl:
3215 case Builtin::BI__builtin_roundf128:
3218 Intrinsic::experimental_constrained_round));
3220 case Builtin::BIroundeven:
3221 case Builtin::BIroundevenf:
3222 case Builtin::BIroundevenl:
3223 case Builtin::BI__builtin_roundeven:
3224 case Builtin::BI__builtin_roundevenf:
3225 case Builtin::BI__builtin_roundevenf16:
3226 case Builtin::BI__builtin_roundevenl:
3227 case Builtin::BI__builtin_roundevenf128:
3229 Intrinsic::roundeven,
3230 Intrinsic::experimental_constrained_roundeven));
3232 case Builtin::BIsin:
3233 case Builtin::BIsinf:
3234 case Builtin::BIsinl:
3235 case Builtin::BI__builtin_sin:
3236 case Builtin::BI__builtin_sinf:
3237 case Builtin::BI__builtin_sinf16:
3238 case Builtin::BI__builtin_sinl:
3239 case Builtin::BI__builtin_sinf128:
3242 Intrinsic::experimental_constrained_sin));
3244 case Builtin::BIsinh:
3245 case Builtin::BIsinhf:
3246 case Builtin::BIsinhl:
3247 case Builtin::BI__builtin_sinh:
3248 case Builtin::BI__builtin_sinhf:
3249 case Builtin::BI__builtin_sinhf16:
3250 case Builtin::BI__builtin_sinhl:
3251 case Builtin::BI__builtin_sinhf128:
3253 *
this,
E, Intrinsic::sinh, Intrinsic::experimental_constrained_sinh));
3255 case Builtin::BI__builtin_sincos:
3256 case Builtin::BI__builtin_sincosf:
3257 case Builtin::BI__builtin_sincosf16:
3258 case Builtin::BI__builtin_sincosl:
3259 case Builtin::BI__builtin_sincosf128:
3263 case Builtin::BIsqrt:
3264 case Builtin::BIsqrtf:
3265 case Builtin::BIsqrtl:
3266 case Builtin::BI__builtin_sqrt:
3267 case Builtin::BI__builtin_sqrtf:
3268 case Builtin::BI__builtin_sqrtf16:
3269 case Builtin::BI__builtin_sqrtl:
3270 case Builtin::BI__builtin_sqrtf128:
3271 case Builtin::BI__builtin_elementwise_sqrt: {
3273 *
this,
E, Intrinsic::sqrt, Intrinsic::experimental_constrained_sqrt);
3278 case Builtin::BItan:
3279 case Builtin::BItanf:
3280 case Builtin::BItanl:
3281 case Builtin::BI__builtin_tan:
3282 case Builtin::BI__builtin_tanf:
3283 case Builtin::BI__builtin_tanf16:
3284 case Builtin::BI__builtin_tanl:
3285 case Builtin::BI__builtin_tanf128:
3287 *
this,
E, Intrinsic::tan, Intrinsic::experimental_constrained_tan));
3289 case Builtin::BItanh:
3290 case Builtin::BItanhf:
3291 case Builtin::BItanhl:
3292 case Builtin::BI__builtin_tanh:
3293 case Builtin::BI__builtin_tanhf:
3294 case Builtin::BI__builtin_tanhf16:
3295 case Builtin::BI__builtin_tanhl:
3296 case Builtin::BI__builtin_tanhf128:
3298 *
this,
E, Intrinsic::tanh, Intrinsic::experimental_constrained_tanh));
3300 case Builtin::BItrunc:
3301 case Builtin::BItruncf:
3302 case Builtin::BItruncl:
3303 case Builtin::BI__builtin_trunc:
3304 case Builtin::BI__builtin_truncf:
3305 case Builtin::BI__builtin_truncf16:
3306 case Builtin::BI__builtin_truncl:
3307 case Builtin::BI__builtin_truncf128:
3310 Intrinsic::experimental_constrained_trunc));
3312 case Builtin::BIlround:
3313 case Builtin::BIlroundf:
3314 case Builtin::BIlroundl:
3315 case Builtin::BI__builtin_lround:
3316 case Builtin::BI__builtin_lroundf:
3317 case Builtin::BI__builtin_lroundl:
3318 case Builtin::BI__builtin_lroundf128:
3320 *
this,
E, Intrinsic::lround,
3321 Intrinsic::experimental_constrained_lround));
3323 case Builtin::BIllround:
3324 case Builtin::BIllroundf:
3325 case Builtin::BIllroundl:
3326 case Builtin::BI__builtin_llround:
3327 case Builtin::BI__builtin_llroundf:
3328 case Builtin::BI__builtin_llroundl:
3329 case Builtin::BI__builtin_llroundf128:
3331 *
this,
E, Intrinsic::llround,
3332 Intrinsic::experimental_constrained_llround));
3334 case Builtin::BIlrint:
3335 case Builtin::BIlrintf:
3336 case Builtin::BIlrintl:
3337 case Builtin::BI__builtin_lrint:
3338 case Builtin::BI__builtin_lrintf:
3339 case Builtin::BI__builtin_lrintl:
3340 case Builtin::BI__builtin_lrintf128:
3342 *
this,
E, Intrinsic::lrint,
3343 Intrinsic::experimental_constrained_lrint));
3345 case Builtin::BIllrint:
3346 case Builtin::BIllrintf:
3347 case Builtin::BIllrintl:
3348 case Builtin::BI__builtin_llrint:
3349 case Builtin::BI__builtin_llrintf:
3350 case Builtin::BI__builtin_llrintl:
3351 case Builtin::BI__builtin_llrintf128:
3353 *
this,
E, Intrinsic::llrint,
3354 Intrinsic::experimental_constrained_llrint));
3355 case Builtin::BI__builtin_ldexp:
3356 case Builtin::BI__builtin_ldexpf:
3357 case Builtin::BI__builtin_ldexpl:
3358 case Builtin::BI__builtin_ldexpf16:
3359 case Builtin::BI__builtin_ldexpf128: {
3361 *
this,
E, Intrinsic::ldexp,
3362 Intrinsic::experimental_constrained_ldexp));
3372 Value *Val = A.emitRawPointer(*
this);
3378 SkippedChecks.
set(SanitizerKind::All);
3379 SkippedChecks.
clear(SanitizerKind::Alignment);
3382 if (
auto *CE = dyn_cast<ImplicitCastExpr>(Arg))
3383 if (CE->getCastKind() == CK_BitCast)
3384 Arg = CE->getSubExpr();
3390 switch (BuiltinIDIfNoAsmLabel) {
3392 case Builtin::BI__builtin___CFStringMakeConstantString:
3393 case Builtin::BI__builtin___NSStringMakeConstantString:
3395 case Builtin::BI__builtin_stdarg_start:
3396 case Builtin::BI__builtin_va_start:
3397 case Builtin::BI__va_start:
3398 case Builtin::BI__builtin_va_end:
3402 BuiltinID != Builtin::BI__builtin_va_end);
3404 case Builtin::BI__builtin_va_copy: {
3411 case Builtin::BIabs:
3412 case Builtin::BIlabs:
3413 case Builtin::BIllabs:
3414 case Builtin::BI__builtin_abs:
3415 case Builtin::BI__builtin_labs:
3416 case Builtin::BI__builtin_llabs: {
3417 bool SanitizeOverflow =
SanOpts.
has(SanitizerKind::SignedIntegerOverflow);
3420 switch (
getLangOpts().getSignedOverflowBehavior()) {
3425 if (!SanitizeOverflow) {
3437 case Builtin::BI__builtin_complex: {
3442 case Builtin::BI__builtin_conj:
3443 case Builtin::BI__builtin_conjf:
3444 case Builtin::BI__builtin_conjl:
3445 case Builtin::BIconj:
3446 case Builtin::BIconjf:
3447 case Builtin::BIconjl: {
3449 Value *Real = ComplexVal.first;
3450 Value *Imag = ComplexVal.second;
3451 Imag =
Builder.CreateFNeg(Imag,
"neg");
3454 case Builtin::BI__builtin_creal:
3455 case Builtin::BI__builtin_crealf:
3456 case Builtin::BI__builtin_creall:
3457 case Builtin::BIcreal:
3458 case Builtin::BIcrealf:
3459 case Builtin::BIcreall: {
3464 case Builtin::BI__builtin_preserve_access_index: {
3485 case Builtin::BI__builtin_cimag:
3486 case Builtin::BI__builtin_cimagf:
3487 case Builtin::BI__builtin_cimagl:
3488 case Builtin::BIcimag:
3489 case Builtin::BIcimagf:
3490 case Builtin::BIcimagl: {
3495 case Builtin::BI__builtin_clrsb:
3496 case Builtin::BI__builtin_clrsbl:
3497 case Builtin::BI__builtin_clrsbll: {
3501 llvm::Type *ArgType = ArgValue->
getType();
3505 Value *
Zero = llvm::Constant::getNullValue(ArgType);
3506 Value *IsNeg =
Builder.CreateICmpSLT(ArgValue, Zero,
"isneg");
3508 Value *Tmp =
Builder.CreateSelect(IsNeg, Inverse, ArgValue);
3515 case Builtin::BI__builtin_ctzs:
3516 case Builtin::BI__builtin_ctz:
3517 case Builtin::BI__builtin_ctzl:
3518 case Builtin::BI__builtin_ctzll:
3519 case Builtin::BI__builtin_ctzg: {
3520 bool HasFallback = BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_ctzg &&
3521 E->getNumArgs() > 1;
3527 llvm::Type *ArgType = ArgValue->
getType();
3534 if (
Result->getType() != ResultType)
3540 Value *
Zero = Constant::getNullValue(ArgType);
3541 Value *IsZero =
Builder.CreateICmpEQ(ArgValue, Zero,
"iszero");
3543 Value *ResultOrFallback =
3544 Builder.CreateSelect(IsZero, FallbackValue,
Result,
"ctzg");
3547 case Builtin::BI__builtin_clzs:
3548 case Builtin::BI__builtin_clz:
3549 case Builtin::BI__builtin_clzl:
3550 case Builtin::BI__builtin_clzll:
3551 case Builtin::BI__builtin_clzg: {
3552 bool HasFallback = BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_clzg &&
3553 E->getNumArgs() > 1;
3559 llvm::Type *ArgType = ArgValue->
getType();
3566 if (
Result->getType() != ResultType)
3572 Value *
Zero = Constant::getNullValue(ArgType);
3573 Value *IsZero =
Builder.CreateICmpEQ(ArgValue, Zero,
"iszero");
3575 Value *ResultOrFallback =
3576 Builder.CreateSelect(IsZero, FallbackValue,
Result,
"clzg");
3579 case Builtin::BI__builtin_ffs:
3580 case Builtin::BI__builtin_ffsl:
3581 case Builtin::BI__builtin_ffsll: {
3585 llvm::Type *ArgType = ArgValue->
getType();
3590 Builder.CreateAdd(
Builder.CreateCall(F, {ArgValue, Builder.getTrue()}),
3591 llvm::ConstantInt::get(ArgType, 1));
3592 Value *
Zero = llvm::Constant::getNullValue(ArgType);
3593 Value *IsZero =
Builder.CreateICmpEQ(ArgValue, Zero,
"iszero");
3595 if (
Result->getType() != ResultType)
3600 case Builtin::BI__builtin_parity:
3601 case Builtin::BI__builtin_parityl:
3602 case Builtin::BI__builtin_parityll: {
3606 llvm::Type *ArgType = ArgValue->
getType();
3612 if (
Result->getType() != ResultType)
3617 case Builtin::BI__lzcnt16:
3618 case Builtin::BI__lzcnt:
3619 case Builtin::BI__lzcnt64: {
3622 llvm::Type *ArgType = ArgValue->
getType();
3627 if (
Result->getType() != ResultType)
3632 case Builtin::BI__popcnt16:
3633 case Builtin::BI__popcnt:
3634 case Builtin::BI__popcnt64:
3635 case Builtin::BI__builtin_popcount:
3636 case Builtin::BI__builtin_popcountl:
3637 case Builtin::BI__builtin_popcountll:
3638 case Builtin::BI__builtin_popcountg: {
3641 llvm::Type *ArgType = ArgValue->
getType();
3646 if (
Result->getType() != ResultType)
3651 case Builtin::BI__builtin_unpredictable: {
3657 case Builtin::BI__builtin_expect: {
3659 llvm::Type *ArgType = ArgValue->
getType();
3670 Builder.CreateCall(FnExpect, {ArgValue, ExpectedValue},
"expval");
3673 case Builtin::BI__builtin_expect_with_probability: {
3675 llvm::Type *ArgType = ArgValue->
getType();
3678 llvm::APFloat Probability(0.0);
3679 const Expr *ProbArg =
E->getArg(2);
3681 assert(EvalSucceed &&
"probability should be able to evaluate as float");
3683 bool LoseInfo =
false;
3684 Probability.convert(llvm::APFloat::IEEEdouble(),
3685 llvm::RoundingMode::Dynamic, &LoseInfo);
3687 Constant *Confidence = ConstantFP::get(Ty, Probability);
3697 FnExpect, {ArgValue, ExpectedValue, Confidence},
"expval");
3700 case Builtin::BI__builtin_assume_aligned: {
3701 const Expr *Ptr =
E->getArg(0);
3703 Value *OffsetValue =
3707 ConstantInt *AlignmentCI = cast<ConstantInt>(AlignmentValue);
3708 if (AlignmentCI->getValue().ugt(llvm::Value::MaximumAlignment))
3709 AlignmentCI = ConstantInt::get(AlignmentCI->getIntegerType(),
3710 llvm::Value::MaximumAlignment);
3714 AlignmentCI, OffsetValue);
3717 case Builtin::BI__assume:
3718 case Builtin::BI__builtin_assume: {
3724 Builder.CreateCall(FnAssume, ArgValue);
3727 case Builtin::BI__builtin_assume_separate_storage: {
3728 const Expr *Arg0 =
E->getArg(0);
3729 const Expr *Arg1 =
E->getArg(1);
3734 Value *Values[] = {Value0, Value1};
3735 OperandBundleDefT<Value *> OBD(
"separate_storage", Values);
3739 case Builtin::BI__builtin_allow_runtime_check: {
3743 llvm::Value *Allow =
Builder.CreateCall(
3745 llvm::MetadataAsValue::get(Ctx, llvm::MDString::get(Ctx, Kind)));
3748 case Builtin::BI__arithmetic_fence: {
3751 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
3752 llvm::FastMathFlags FMF =
Builder.getFastMathFlags();
3753 bool isArithmeticFenceEnabled =
3754 FMF.allowReassoc() &&
3758 if (isArithmeticFenceEnabled) {
3761 Value *Real =
Builder.CreateArithmeticFence(ComplexVal.first,
3763 Value *Imag =
Builder.CreateArithmeticFence(ComplexVal.second,
3768 Value *Real = ComplexVal.first;
3769 Value *Imag = ComplexVal.second;
3773 if (isArithmeticFenceEnabled)
3778 case Builtin::BI__builtin_bswap16:
3779 case Builtin::BI__builtin_bswap32:
3780 case Builtin::BI__builtin_bswap64:
3781 case Builtin::BI_byteswap_ushort:
3782 case Builtin::BI_byteswap_ulong:
3783 case Builtin::BI_byteswap_uint64: {
3785 emitBuiltinWithOneOverloadedType<1>(*
this,
E, Intrinsic::bswap));
3787 case Builtin::BI__builtin_bitreverse8:
3788 case Builtin::BI__builtin_bitreverse16:
3789 case Builtin::BI__builtin_bitreverse32:
3790 case Builtin::BI__builtin_bitreverse64: {
3792 emitBuiltinWithOneOverloadedType<1>(*
this,
E, Intrinsic::bitreverse));
3794 case Builtin::BI__builtin_rotateleft8:
3795 case Builtin::BI__builtin_rotateleft16:
3796 case Builtin::BI__builtin_rotateleft32:
3797 case Builtin::BI__builtin_rotateleft64:
3798 case Builtin::BI_rotl8:
3799 case Builtin::BI_rotl16:
3800 case Builtin::BI_rotl:
3801 case Builtin::BI_lrotl:
3802 case Builtin::BI_rotl64:
3805 case Builtin::BI__builtin_rotateright8:
3806 case Builtin::BI__builtin_rotateright16:
3807 case Builtin::BI__builtin_rotateright32:
3808 case Builtin::BI__builtin_rotateright64:
3809 case Builtin::BI_rotr8:
3810 case Builtin::BI_rotr16:
3811 case Builtin::BI_rotr:
3812 case Builtin::BI_lrotr:
3813 case Builtin::BI_rotr64:
3816 case Builtin::BI__builtin_constant_p: {
3819 const Expr *Arg =
E->getArg(0);
3827 return RValue::get(ConstantInt::get(ResultType, 0));
3832 return RValue::get(ConstantInt::get(ResultType, 0));
3844 if (
Result->getType() != ResultType)
3848 case Builtin::BI__builtin_dynamic_object_size:
3849 case Builtin::BI__builtin_object_size: {
3856 bool IsDynamic = BuiltinID == Builtin::BI__builtin_dynamic_object_size;
3858 nullptr, IsDynamic));
3860 case Builtin::BI__builtin_counted_by_ref: {
3862 llvm::Value *
Result = llvm::ConstantPointerNull::get(
3867 if (
auto *UO = dyn_cast<UnaryOperator>(Arg);
3868 UO && UO->getOpcode() == UO_AddrOf) {
3871 if (
auto *ASE = dyn_cast<ArraySubscriptExpr>(Arg))
3875 if (
const MemberExpr *ME = dyn_cast_if_present<MemberExpr>(Arg)) {
3879 const auto *FAMDecl = cast<FieldDecl>(ME->getMemberDecl());
3883 llvm::report_fatal_error(
"Cannot find the counted_by 'count' field");
3889 case Builtin::BI__builtin_prefetch: {
3893 llvm::ConstantInt::get(
Int32Ty, 0);
3895 llvm::ConstantInt::get(
Int32Ty, 3);
3901 case Builtin::BI__builtin_readcyclecounter: {
3905 case Builtin::BI__builtin_readsteadycounter: {
3909 case Builtin::BI__builtin___clear_cache: {
3915 case Builtin::BI__builtin_trap:
3918 case Builtin::BI__builtin_verbose_trap: {
3919 llvm::DILocation *TrapLocation =
Builder.getCurrentDebugLocation();
3930 case Builtin::BI__debugbreak:
3933 case Builtin::BI__builtin_unreachable: {
3942 case Builtin::BI__builtin_powi:
3943 case Builtin::BI__builtin_powif:
3944 case Builtin::BI__builtin_powil: {
3948 if (
Builder.getIsFPConstrained()) {
3951 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
3958 { Src0->getType(), Src1->getType() });
3961 case Builtin::BI__builtin_frexpl: {
3965 if (&
getTarget().getLongDoubleFormat() == &llvm::APFloat::PPCDoubleDouble())
3969 case Builtin::BI__builtin_frexp:
3970 case Builtin::BI__builtin_frexpf:
3971 case Builtin::BI__builtin_frexpf128:
3972 case Builtin::BI__builtin_frexpf16:
3974 case Builtin::BI__builtin_isgreater:
3975 case Builtin::BI__builtin_isgreaterequal:
3976 case Builtin::BI__builtin_isless:
3977 case Builtin::BI__builtin_islessequal:
3978 case Builtin::BI__builtin_islessgreater:
3979 case Builtin::BI__builtin_isunordered: {
3982 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
3986 switch (BuiltinID) {
3987 default: llvm_unreachable(
"Unknown ordered comparison");
3988 case Builtin::BI__builtin_isgreater:
3989 LHS =
Builder.CreateFCmpOGT(LHS, RHS,
"cmp");
3991 case Builtin::BI__builtin_isgreaterequal:
3992 LHS =
Builder.CreateFCmpOGE(LHS, RHS,
"cmp");
3994 case Builtin::BI__builtin_isless:
3995 LHS =
Builder.CreateFCmpOLT(LHS, RHS,
"cmp");
3997 case Builtin::BI__builtin_islessequal:
3998 LHS =
Builder.CreateFCmpOLE(LHS, RHS,
"cmp");
4000 case Builtin::BI__builtin_islessgreater:
4001 LHS =
Builder.CreateFCmpONE(LHS, RHS,
"cmp");
4003 case Builtin::BI__builtin_isunordered:
4004 LHS =
Builder.CreateFCmpUNO(LHS, RHS,
"cmp");
4011 case Builtin::BI__builtin_isnan: {
4012 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
4021 case Builtin::BI__builtin_issignaling: {
4022 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
4029 case Builtin::BI__builtin_isinf: {
4030 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
4039 case Builtin::BIfinite:
4040 case Builtin::BI__finite:
4041 case Builtin::BIfinitef:
4042 case Builtin::BI__finitef:
4043 case Builtin::BIfinitel:
4044 case Builtin::BI__finitel:
4045 case Builtin::BI__builtin_isfinite: {
4046 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
4055 case Builtin::BI__builtin_isnormal: {
4056 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
4063 case Builtin::BI__builtin_issubnormal: {
4064 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
4067 Builder.CreateZExt(
Builder.createIsFPClass(
V, FPClassTest::fcSubnormal),
4071 case Builtin::BI__builtin_iszero: {
4072 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
4079 case Builtin::BI__builtin_isfpclass: {
4084 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
4090 case Builtin::BI__builtin_nondeterministic_value: {
4099 case Builtin::BI__builtin_elementwise_abs: {
4104 QT = VecTy->getElementType();
4108 Builder.getFalse(),
nullptr,
"elt.abs");
4110 Result = emitBuiltinWithOneOverloadedType<1>(
4111 *
this,
E, llvm::Intrinsic::fabs,
"elt.abs");
4115 case Builtin::BI__builtin_elementwise_acos:
4116 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4117 *
this,
E, llvm::Intrinsic::acos,
"elt.acos"));
4118 case Builtin::BI__builtin_elementwise_asin:
4119 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4120 *
this,
E, llvm::Intrinsic::asin,
"elt.asin"));
4121 case Builtin::BI__builtin_elementwise_atan:
4122 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4123 *
this,
E, llvm::Intrinsic::atan,
"elt.atan"));
4124 case Builtin::BI__builtin_elementwise_atan2:
4125 return RValue::get(emitBuiltinWithOneOverloadedType<2>(
4126 *
this,
E, llvm::Intrinsic::atan2,
"elt.atan2"));
4127 case Builtin::BI__builtin_elementwise_ceil:
4128 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4129 *
this,
E, llvm::Intrinsic::ceil,
"elt.ceil"));
4130 case Builtin::BI__builtin_elementwise_exp:
4131 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4132 *
this,
E, llvm::Intrinsic::exp,
"elt.exp"));
4133 case Builtin::BI__builtin_elementwise_exp2:
4134 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4135 *
this,
E, llvm::Intrinsic::exp2,
"elt.exp2"));
4136 case Builtin::BI__builtin_elementwise_log:
4137 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4138 *
this,
E, llvm::Intrinsic::log,
"elt.log"));
4139 case Builtin::BI__builtin_elementwise_log2:
4140 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4141 *
this,
E, llvm::Intrinsic::log2,
"elt.log2"));
4142 case Builtin::BI__builtin_elementwise_log10:
4143 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4144 *
this,
E, llvm::Intrinsic::log10,
"elt.log10"));
4145 case Builtin::BI__builtin_elementwise_pow: {
4147 emitBuiltinWithOneOverloadedType<2>(*
this,
E, llvm::Intrinsic::pow));
4149 case Builtin::BI__builtin_elementwise_bitreverse:
4150 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4151 *
this,
E, llvm::Intrinsic::bitreverse,
"elt.bitreverse"));
4152 case Builtin::BI__builtin_elementwise_cos:
4153 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4154 *
this,
E, llvm::Intrinsic::cos,
"elt.cos"));
4155 case Builtin::BI__builtin_elementwise_cosh:
4156 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4157 *
this,
E, llvm::Intrinsic::cosh,
"elt.cosh"));
4158 case Builtin::BI__builtin_elementwise_floor:
4159 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4160 *
this,
E, llvm::Intrinsic::floor,
"elt.floor"));
4161 case Builtin::BI__builtin_elementwise_popcount:
4162 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4163 *
this,
E, llvm::Intrinsic::ctpop,
"elt.ctpop"));
4164 case Builtin::BI__builtin_elementwise_roundeven:
4165 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4166 *
this,
E, llvm::Intrinsic::roundeven,
"elt.roundeven"));
4167 case Builtin::BI__builtin_elementwise_round:
4168 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4169 *
this,
E, llvm::Intrinsic::round,
"elt.round"));
4170 case Builtin::BI__builtin_elementwise_rint:
4171 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4172 *
this,
E, llvm::Intrinsic::rint,
"elt.rint"));
4173 case Builtin::BI__builtin_elementwise_nearbyint:
4174 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4175 *
this,
E, llvm::Intrinsic::nearbyint,
"elt.nearbyint"));
4176 case Builtin::BI__builtin_elementwise_sin:
4177 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4178 *
this,
E, llvm::Intrinsic::sin,
"elt.sin"));
4179 case Builtin::BI__builtin_elementwise_sinh:
4180 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4181 *
this,
E, llvm::Intrinsic::sinh,
"elt.sinh"));
4182 case Builtin::BI__builtin_elementwise_tan:
4183 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4184 *
this,
E, llvm::Intrinsic::tan,
"elt.tan"));
4185 case Builtin::BI__builtin_elementwise_tanh:
4186 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4187 *
this,
E, llvm::Intrinsic::tanh,
"elt.tanh"));
4188 case Builtin::BI__builtin_elementwise_trunc:
4189 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4190 *
this,
E, llvm::Intrinsic::trunc,
"elt.trunc"));
4191 case Builtin::BI__builtin_elementwise_canonicalize:
4192 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4193 *
this,
E, llvm::Intrinsic::canonicalize,
"elt.canonicalize"));
4194 case Builtin::BI__builtin_elementwise_copysign:
4195 return RValue::get(emitBuiltinWithOneOverloadedType<2>(
4196 *
this,
E, llvm::Intrinsic::copysign));
4197 case Builtin::BI__builtin_elementwise_fma:
4199 emitBuiltinWithOneOverloadedType<3>(*
this,
E, llvm::Intrinsic::fma));
4200 case Builtin::BI__builtin_elementwise_add_sat:
4201 case Builtin::BI__builtin_elementwise_sub_sat: {
4205 assert(Op0->
getType()->isIntOrIntVectorTy() &&
"integer type expected");
4208 Ty = VecTy->getElementType();
4211 if (BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_elementwise_add_sat)
4212 Opc = IsSigned ? llvm::Intrinsic::sadd_sat : llvm::Intrinsic::uadd_sat;
4214 Opc = IsSigned ? llvm::Intrinsic::ssub_sat : llvm::Intrinsic::usub_sat;
4215 Result =
Builder.CreateBinaryIntrinsic(Opc, Op0, Op1,
nullptr,
"elt.sat");
4219 case Builtin::BI__builtin_elementwise_max: {
4223 if (Op0->
getType()->isIntOrIntVectorTy()) {
4226 Ty = VecTy->getElementType();
4228 ? llvm::Intrinsic::smax
4229 : llvm::Intrinsic::umax,
4230 Op0, Op1,
nullptr,
"elt.max");
4235 case Builtin::BI__builtin_elementwise_min: {
4239 if (Op0->
getType()->isIntOrIntVectorTy()) {
4242 Ty = VecTy->getElementType();
4244 ? llvm::Intrinsic::smin
4245 : llvm::Intrinsic::umin,
4246 Op0, Op1,
nullptr,
"elt.min");
4252 case Builtin::BI__builtin_elementwise_maximum: {
4256 Op1,
nullptr,
"elt.maximum");
4260 case Builtin::BI__builtin_elementwise_minimum: {
4264 Op1,
nullptr,
"elt.minimum");
4268 case Builtin::BI__builtin_reduce_max: {
4269 auto GetIntrinsicID = [
this](
QualType QT) {
4271 QT = VecTy->getElementType();
4276 return llvm::Intrinsic::vector_reduce_smax;
4278 return llvm::Intrinsic::vector_reduce_umax;
4280 return llvm::Intrinsic::vector_reduce_fmax;
4282 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4283 *
this,
E, GetIntrinsicID(
E->getArg(0)->
getType()),
"rdx.min"));
4286 case Builtin::BI__builtin_reduce_min: {
4287 auto GetIntrinsicID = [
this](
QualType QT) {
4289 QT = VecTy->getElementType();
4294 return llvm::Intrinsic::vector_reduce_smin;
4296 return llvm::Intrinsic::vector_reduce_umin;
4298 return llvm::Intrinsic::vector_reduce_fmin;
4301 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4302 *
this,
E, GetIntrinsicID(
E->getArg(0)->
getType()),
"rdx.min"));
4305 case Builtin::BI__builtin_reduce_add:
4306 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4307 *
this,
E, llvm::Intrinsic::vector_reduce_add,
"rdx.add"));
4308 case Builtin::BI__builtin_reduce_mul:
4309 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4310 *
this,
E, llvm::Intrinsic::vector_reduce_mul,
"rdx.mul"));
4311 case Builtin::BI__builtin_reduce_xor:
4312 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4313 *
this,
E, llvm::Intrinsic::vector_reduce_xor,
"rdx.xor"));
4314 case Builtin::BI__builtin_reduce_or:
4315 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4316 *
this,
E, llvm::Intrinsic::vector_reduce_or,
"rdx.or"));
4317 case Builtin::BI__builtin_reduce_and:
4318 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4319 *
this,
E, llvm::Intrinsic::vector_reduce_and,
"rdx.and"));
4320 case Builtin::BI__builtin_reduce_maximum:
4321 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4322 *
this,
E, llvm::Intrinsic::vector_reduce_fmaximum,
"rdx.maximum"));
4323 case Builtin::BI__builtin_reduce_minimum:
4324 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4325 *
this,
E, llvm::Intrinsic::vector_reduce_fminimum,
"rdx.minimum"));
4327 case Builtin::BI__builtin_matrix_transpose: {
4331 Value *
Result = MB.CreateMatrixTranspose(MatValue, MatrixTy->getNumRows(),
4332 MatrixTy->getNumColumns());
4336 case Builtin::BI__builtin_matrix_column_major_load: {
4342 assert(PtrTy &&
"arg0 must be of pointer type");
4352 ResultTy->getNumRows(), ResultTy->getNumColumns(),
"matrix");
4356 case Builtin::BI__builtin_matrix_column_major_store: {
4364 assert(PtrTy &&
"arg1 must be of pointer type");
4373 MatrixTy->getNumRows(), MatrixTy->getNumColumns());
4377 case Builtin::BI__builtin_isinf_sign: {
4379 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
4384 AbsArg, ConstantFP::getInfinity(Arg->
getType()),
"isinf");
4390 Value *NegativeOne = ConstantInt::get(
IntTy, -1);
4391 Value *SignResult =
Builder.CreateSelect(IsNeg, NegativeOne, One);
4396 case Builtin::BI__builtin_flt_rounds: {
4401 if (
Result->getType() != ResultType)
4407 case Builtin::BI__builtin_set_flt_rounds: {
4415 case Builtin::BI__builtin_fpclassify: {
4416 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
4427 "fpclassify_result");
4431 Value *IsZero =
Builder.CreateFCmpOEQ(
V, Constant::getNullValue(Ty),
4435 Builder.CreateCondBr(IsZero, End, NotZero);
4439 Builder.SetInsertPoint(NotZero);
4443 Builder.CreateCondBr(IsNan, End, NotNan);
4444 Result->addIncoming(NanLiteral, NotZero);
4447 Builder.SetInsertPoint(NotNan);
4450 Builder.CreateFCmpOEQ(VAbs, ConstantFP::getInfinity(
V->getType()),
4454 Builder.CreateCondBr(IsInf, End, NotInf);
4455 Result->addIncoming(InfLiteral, NotNan);
4458 Builder.SetInsertPoint(NotInf);
4459 APFloat Smallest = APFloat::getSmallestNormalized(
4462 Builder.CreateFCmpUGE(VAbs, ConstantFP::get(
V->getContext(), Smallest),
4464 Value *NormalResult =
4468 Result->addIncoming(NormalResult, NotInf);
4481 case Builtin::BIalloca:
4482 case Builtin::BI_alloca:
4483 case Builtin::BI__builtin_alloca_uninitialized:
4484 case Builtin::BI__builtin_alloca: {
4488 const Align SuitableAlignmentInBytes =
4492 AllocaInst *AI =
Builder.CreateAlloca(
Builder.getInt8Ty(), Size);
4493 AI->setAlignment(SuitableAlignmentInBytes);
4494 if (BuiltinID != Builtin::BI__builtin_alloca_uninitialized)
4506 case Builtin::BI__builtin_alloca_with_align_uninitialized:
4507 case Builtin::BI__builtin_alloca_with_align: {
4510 auto *AlignmentInBitsCI = cast<ConstantInt>(AlignmentInBitsValue);
4511 unsigned AlignmentInBits = AlignmentInBitsCI->getZExtValue();
4512 const Align AlignmentInBytes =
4514 AllocaInst *AI =
Builder.CreateAlloca(
Builder.getInt8Ty(), Size);
4515 AI->setAlignment(AlignmentInBytes);
4516 if (BuiltinID != Builtin::BI__builtin_alloca_with_align_uninitialized)
4528 case Builtin::BIbzero:
4529 case Builtin::BI__builtin_bzero: {
4538 case Builtin::BIbcopy:
4539 case Builtin::BI__builtin_bcopy: {
4553 case Builtin::BImemcpy:
4554 case Builtin::BI__builtin_memcpy:
4555 case Builtin::BImempcpy:
4556 case Builtin::BI__builtin_mempcpy: {
4560 EmitArgCheck(
TCK_Store, Dest,
E->getArg(0), 0);
4561 EmitArgCheck(
TCK_Load, Src,
E->getArg(1), 1);
4563 if (BuiltinID == Builtin::BImempcpy ||
4564 BuiltinID == Builtin::BI__builtin_mempcpy)
4571 case Builtin::BI__builtin_memcpy_inline: {
4576 EmitArgCheck(
TCK_Store, Dest,
E->getArg(0), 0);
4577 EmitArgCheck(
TCK_Load, Src,
E->getArg(1), 1);
4582 case Builtin::BI__builtin_char_memchr:
4583 BuiltinID = Builtin::BI__builtin_memchr;
4586 case Builtin::BI__builtin___memcpy_chk: {
4593 llvm::APSInt DstSize = DstSizeResult.
Val.
getInt();
4594 if (
Size.ugt(DstSize))
4598 Value *SizeVal = llvm::ConstantInt::get(
Builder.getContext(), Size);
4603 case Builtin::BI__builtin_objc_memmove_collectable: {
4608 DestAddr, SrcAddr, SizeVal);
4612 case Builtin::BI__builtin___memmove_chk: {
4619 llvm::APSInt DstSize = DstSizeResult.
Val.
getInt();
4620 if (
Size.ugt(DstSize))
4624 Value *SizeVal = llvm::ConstantInt::get(
Builder.getContext(), Size);
4629 case Builtin::BImemmove:
4630 case Builtin::BI__builtin_memmove: {
4634 EmitArgCheck(
TCK_Store, Dest,
E->getArg(0), 0);
4635 EmitArgCheck(
TCK_Load, Src,
E->getArg(1), 1);
4639 case Builtin::BImemset:
4640 case Builtin::BI__builtin_memset: {
4650 case Builtin::BI__builtin_memset_inline: {
4662 case Builtin::BI__builtin___memset_chk: {
4669 llvm::APSInt DstSize = DstSizeResult.
Val.
getInt();
4670 if (
Size.ugt(DstSize))
4675 Value *SizeVal = llvm::ConstantInt::get(
Builder.getContext(), Size);
4679 case Builtin::BI__builtin_wmemchr: {
4682 if (!
getTarget().getTriple().isOSMSVCRT())
4690 BasicBlock *Entry =
Builder.GetInsertBlock();
4695 Builder.CreateCondBr(SizeEq0, Exit, CmpEq);
4699 StrPhi->addIncoming(Str, Entry);
4701 SizePhi->addIncoming(Size, Entry);
4705 Value *FoundChr =
Builder.CreateConstInBoundsGEP1_32(WCharTy, StrPhi, 0);
4707 Builder.CreateCondBr(StrEqChr, Exit, Next);
4710 Value *NextStr =
Builder.CreateConstInBoundsGEP1_32(WCharTy, StrPhi, 1);
4712 Value *NextSizeEq0 =
4713 Builder.CreateICmpEQ(NextSize, ConstantInt::get(
SizeTy, 0));
4714 Builder.CreateCondBr(NextSizeEq0, Exit, CmpEq);
4715 StrPhi->addIncoming(NextStr, Next);
4716 SizePhi->addIncoming(NextSize, Next);
4720 Ret->addIncoming(llvm::Constant::getNullValue(Str->
getType()), Entry);
4721 Ret->addIncoming(llvm::Constant::getNullValue(Str->
getType()), Next);
4722 Ret->addIncoming(FoundChr, CmpEq);
4725 case Builtin::BI__builtin_wmemcmp: {
4728 if (!
getTarget().getTriple().isOSMSVCRT())
4737 BasicBlock *Entry =
Builder.GetInsertBlock();
4743 Builder.CreateCondBr(SizeEq0, Exit, CmpGT);
4747 DstPhi->addIncoming(Dst, Entry);
4749 SrcPhi->addIncoming(Src, Entry);
4751 SizePhi->addIncoming(Size, Entry);
4757 Builder.CreateCondBr(DstGtSrc, Exit, CmpLT);
4761 Builder.CreateCondBr(DstLtSrc, Exit, Next);
4764 Value *NextDst =
Builder.CreateConstInBoundsGEP1_32(WCharTy, DstPhi, 1);
4765 Value *NextSrc =
Builder.CreateConstInBoundsGEP1_32(WCharTy, SrcPhi, 1);
4767 Value *NextSizeEq0 =
4768 Builder.CreateICmpEQ(NextSize, ConstantInt::get(
SizeTy, 0));
4769 Builder.CreateCondBr(NextSizeEq0, Exit, CmpGT);
4770 DstPhi->addIncoming(NextDst, Next);
4771 SrcPhi->addIncoming(NextSrc, Next);
4772 SizePhi->addIncoming(NextSize, Next);
4776 Ret->addIncoming(ConstantInt::get(
IntTy, 0), Entry);
4777 Ret->addIncoming(ConstantInt::get(
IntTy, 1), CmpGT);
4778 Ret->addIncoming(ConstantInt::get(
IntTy, -1), CmpLT);
4779 Ret->addIncoming(ConstantInt::get(
IntTy, 0), Next);
4782 case Builtin::BI__builtin_dwarf_cfa: {
4795 llvm::ConstantInt::get(
Int32Ty, Offset)));
4797 case Builtin::BI__builtin_return_address: {
4803 case Builtin::BI_ReturnAddress: {
4807 case Builtin::BI__builtin_frame_address: {
4813 case Builtin::BI__builtin_extract_return_addr: {
4818 case Builtin::BI__builtin_frob_return_addr: {
4823 case Builtin::BI__builtin_dwarf_sp_column: {
4824 llvm::IntegerType *Ty
4833 case Builtin::BI__builtin_init_dwarf_reg_size_table: {
4839 case Builtin::BI__builtin_eh_return: {
4843 llvm::IntegerType *
IntTy = cast<llvm::IntegerType>(
Int->getType());
4844 assert((
IntTy->getBitWidth() == 32 ||
IntTy->getBitWidth() == 64) &&
4845 "LLVM's __builtin_eh_return only supports 32- and 64-bit variants");
4848 : Intrinsic::eh_return_i64);
4857 case Builtin::BI__builtin_unwind_init: {
4862 case Builtin::BI__builtin_extend_pointer: {
4887 case Builtin::BI__builtin_setjmp: {
4891 if (
getTarget().getTriple().getArch() == llvm::Triple::systemz) {
4902 ConstantInt::get(
Int32Ty, 0));
4916 case Builtin::BI__builtin_longjmp: {
4930 case Builtin::BI__builtin_launder: {
4931 const Expr *Arg =
E->getArg(0);
4939 case Builtin::BI__sync_fetch_and_add:
4940 case Builtin::BI__sync_fetch_and_sub:
4941 case Builtin::BI__sync_fetch_and_or:
4942 case Builtin::BI__sync_fetch_and_and:
4943 case Builtin::BI__sync_fetch_and_xor:
4944 case Builtin::BI__sync_fetch_and_nand:
4945 case Builtin::BI__sync_add_and_fetch:
4946 case Builtin::BI__sync_sub_and_fetch:
4947 case Builtin::BI__sync_and_and_fetch:
4948 case Builtin::BI__sync_or_and_fetch:
4949 case Builtin::BI__sync_xor_and_fetch:
4950 case Builtin::BI__sync_nand_and_fetch:
4951 case Builtin::BI__sync_val_compare_and_swap:
4952 case Builtin::BI__sync_bool_compare_and_swap:
4953 case Builtin::BI__sync_lock_test_and_set:
4954 case Builtin::BI__sync_lock_release:
4955 case Builtin::BI__sync_swap:
4956 llvm_unreachable(
"Shouldn't make it through sema");
4957 case Builtin::BI__sync_fetch_and_add_1:
4958 case Builtin::BI__sync_fetch_and_add_2:
4959 case Builtin::BI__sync_fetch_and_add_4:
4960 case Builtin::BI__sync_fetch_and_add_8:
4961 case Builtin::BI__sync_fetch_and_add_16:
4963 case Builtin::BI__sync_fetch_and_sub_1:
4964 case Builtin::BI__sync_fetch_and_sub_2:
4965 case Builtin::BI__sync_fetch_and_sub_4:
4966 case Builtin::BI__sync_fetch_and_sub_8:
4967 case Builtin::BI__sync_fetch_and_sub_16:
4969 case Builtin::BI__sync_fetch_and_or_1:
4970 case Builtin::BI__sync_fetch_and_or_2:
4971 case Builtin::BI__sync_fetch_and_or_4:
4972 case Builtin::BI__sync_fetch_and_or_8:
4973 case Builtin::BI__sync_fetch_and_or_16:
4975 case Builtin::BI__sync_fetch_and_and_1:
4976 case Builtin::BI__sync_fetch_and_and_2:
4977 case Builtin::BI__sync_fetch_and_and_4:
4978 case Builtin::BI__sync_fetch_and_and_8:
4979 case Builtin::BI__sync_fetch_and_and_16:
4981 case Builtin::BI__sync_fetch_and_xor_1:
4982 case Builtin::BI__sync_fetch_and_xor_2:
4983 case Builtin::BI__sync_fetch_and_xor_4:
4984 case Builtin::BI__sync_fetch_and_xor_8:
4985 case Builtin::BI__sync_fetch_and_xor_16:
4987 case Builtin::BI__sync_fetch_and_nand_1:
4988 case Builtin::BI__sync_fetch_and_nand_2:
4989 case Builtin::BI__sync_fetch_and_nand_4:
4990 case Builtin::BI__sync_fetch_and_nand_8:
4991 case Builtin::BI__sync_fetch_and_nand_16:
4995 case Builtin::BI__sync_fetch_and_min:
4997 case Builtin::BI__sync_fetch_and_max:
4999 case Builtin::BI__sync_fetch_and_umin:
5001 case Builtin::BI__sync_fetch_and_umax:
5004 case Builtin::BI__sync_add_and_fetch_1:
5005 case Builtin::BI__sync_add_and_fetch_2:
5006 case Builtin::BI__sync_add_and_fetch_4:
5007 case Builtin::BI__sync_add_and_fetch_8:
5008 case Builtin::BI__sync_add_and_fetch_16:
5010 llvm::Instruction::Add);
5011 case Builtin::BI__sync_sub_and_fetch_1:
5012 case Builtin::BI__sync_sub_and_fetch_2:
5013 case Builtin::BI__sync_sub_and_fetch_4:
5014 case Builtin::BI__sync_sub_and_fetch_8:
5015 case Builtin::BI__sync_sub_and_fetch_16:
5017 llvm::Instruction::Sub);
5018 case Builtin::BI__sync_and_and_fetch_1:
5019 case Builtin::BI__sync_and_and_fetch_2:
5020 case Builtin::BI__sync_and_and_fetch_4:
5021 case Builtin::BI__sync_and_and_fetch_8:
5022 case Builtin::BI__sync_and_and_fetch_16:
5024 llvm::Instruction::And);
5025 case Builtin::BI__sync_or_and_fetch_1:
5026 case Builtin::BI__sync_or_and_fetch_2:
5027 case Builtin::BI__sync_or_and_fetch_4:
5028 case Builtin::BI__sync_or_and_fetch_8:
5029 case Builtin::BI__sync_or_and_fetch_16:
5031 llvm::Instruction::Or);
5032 case Builtin::BI__sync_xor_and_fetch_1:
5033 case Builtin::BI__sync_xor_and_fetch_2:
5034 case Builtin::BI__sync_xor_and_fetch_4:
5035 case Builtin::BI__sync_xor_and_fetch_8:
5036 case Builtin::BI__sync_xor_and_fetch_16:
5038 llvm::Instruction::Xor);
5039 case Builtin::BI__sync_nand_and_fetch_1:
5040 case Builtin::BI__sync_nand_and_fetch_2:
5041 case Builtin::BI__sync_nand_and_fetch_4:
5042 case Builtin::BI__sync_nand_and_fetch_8:
5043 case Builtin::BI__sync_nand_and_fetch_16:
5045 llvm::Instruction::And,
true);
5047 case Builtin::BI__sync_val_compare_and_swap_1:
5048 case Builtin::BI__sync_val_compare_and_swap_2:
5049 case Builtin::BI__sync_val_compare_and_swap_4:
5050 case Builtin::BI__sync_val_compare_and_swap_8:
5051 case Builtin::BI__sync_val_compare_and_swap_16:
5054 case Builtin::BI__sync_bool_compare_and_swap_1:
5055 case Builtin::BI__sync_bool_compare_and_swap_2:
5056 case Builtin::BI__sync_bool_compare_and_swap_4:
5057 case Builtin::BI__sync_bool_compare_and_swap_8:
5058 case Builtin::BI__sync_bool_compare_and_swap_16:
5061 case Builtin::BI__sync_swap_1:
5062 case Builtin::BI__sync_swap_2:
5063 case Builtin::BI__sync_swap_4:
5064 case Builtin::BI__sync_swap_8:
5065 case Builtin::BI__sync_swap_16:
5068 case Builtin::BI__sync_lock_test_and_set_1:
5069 case Builtin::BI__sync_lock_test_and_set_2:
5070 case Builtin::BI__sync_lock_test_and_set_4:
5071 case Builtin::BI__sync_lock_test_and_set_8:
5072 case Builtin::BI__sync_lock_test_and_set_16:
5075 case Builtin::BI__sync_lock_release_1:
5076 case Builtin::BI__sync_lock_release_2:
5077 case Builtin::BI__sync_lock_release_4:
5078 case Builtin::BI__sync_lock_release_8:
5079 case Builtin::BI__sync_lock_release_16: {
5085 llvm::StoreInst *
Store =
5087 Store->setAtomic(llvm::AtomicOrdering::Release);
5091 case Builtin::BI__sync_synchronize: {
5099 Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent);
5103 case Builtin::BI__builtin_nontemporal_load:
5105 case Builtin::BI__builtin_nontemporal_store:
5107 case Builtin::BI__c11_atomic_is_lock_free:
5108 case Builtin::BI__atomic_is_lock_free: {
5112 const char *LibCallName =
"__atomic_is_lock_free";
5116 if (BuiltinID == Builtin::BI__atomic_is_lock_free)
5130 case Builtin::BI__atomic_test_and_set: {
5142 if (isa<llvm::ConstantInt>(Order)) {
5143 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
5144 AtomicRMWInst *
Result =
nullptr;
5149 llvm::AtomicOrdering::Monotonic);
5154 llvm::AtomicOrdering::Acquire);
5158 llvm::AtomicOrdering::Release);
5163 llvm::AtomicOrdering::AcquireRelease);
5167 llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
5168 llvm::AtomicOrdering::SequentiallyConsistent);
5171 Result->setVolatile(Volatile);
5177 llvm::BasicBlock *BBs[5] = {
5184 llvm::AtomicOrdering Orders[5] = {
5185 llvm::AtomicOrdering::Monotonic, llvm::AtomicOrdering::Acquire,
5186 llvm::AtomicOrdering::Release, llvm::AtomicOrdering::AcquireRelease,
5187 llvm::AtomicOrdering::SequentiallyConsistent};
5189 Order =
Builder.CreateIntCast(Order,
Builder.getInt32Ty(),
false);
5190 llvm::SwitchInst *SI =
Builder.CreateSwitch(Order, BBs[0]);
5192 Builder.SetInsertPoint(ContBB);
5195 for (
unsigned i = 0; i < 5; ++i) {
5196 Builder.SetInsertPoint(BBs[i]);
5198 Ptr, NewVal, Orders[i]);
5199 RMW->setVolatile(Volatile);
5200 Result->addIncoming(RMW, BBs[i]);
5204 SI->addCase(
Builder.getInt32(0), BBs[0]);
5205 SI->addCase(
Builder.getInt32(1), BBs[1]);
5206 SI->addCase(
Builder.getInt32(2), BBs[1]);
5207 SI->addCase(
Builder.getInt32(3), BBs[2]);
5208 SI->addCase(
Builder.getInt32(4), BBs[3]);
5209 SI->addCase(
Builder.getInt32(5), BBs[4]);
5211 Builder.SetInsertPoint(ContBB);
5215 case Builtin::BI__atomic_clear: {
5224 if (isa<llvm::ConstantInt>(Order)) {
5225 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
5230 Store->setOrdering(llvm::AtomicOrdering::Monotonic);
5233 Store->setOrdering(llvm::AtomicOrdering::Release);
5236 Store->setOrdering(llvm::AtomicOrdering::SequentiallyConsistent);
5244 llvm::BasicBlock *BBs[3] = {
5249 llvm::AtomicOrdering Orders[3] = {
5250 llvm::AtomicOrdering::Monotonic, llvm::AtomicOrdering::Release,
5251 llvm::AtomicOrdering::SequentiallyConsistent};
5253 Order =
Builder.CreateIntCast(Order,
Builder.getInt32Ty(),
false);
5254 llvm::SwitchInst *SI =
Builder.CreateSwitch(Order, BBs[0]);
5256 for (
unsigned i = 0; i < 3; ++i) {
5257 Builder.SetInsertPoint(BBs[i]);
5259 Store->setOrdering(Orders[i]);
5263 SI->addCase(
Builder.getInt32(0), BBs[0]);
5264 SI->addCase(
Builder.getInt32(3), BBs[1]);
5265 SI->addCase(
Builder.getInt32(5), BBs[2]);
5267 Builder.SetInsertPoint(ContBB);
5271 case Builtin::BI__atomic_thread_fence:
5272 case Builtin::BI__atomic_signal_fence:
5273 case Builtin::BI__c11_atomic_thread_fence:
5274 case Builtin::BI__c11_atomic_signal_fence: {
5275 llvm::SyncScope::ID SSID;
5276 if (BuiltinID == Builtin::BI__atomic_signal_fence ||
5277 BuiltinID == Builtin::BI__c11_atomic_signal_fence)
5278 SSID = llvm::SyncScope::SingleThread;
5280 SSID = llvm::SyncScope::System;
5282 if (isa<llvm::ConstantInt>(Order)) {
5283 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
5290 Builder.CreateFence(llvm::AtomicOrdering::Acquire, SSID);
5293 Builder.CreateFence(llvm::AtomicOrdering::Release, SSID);
5296 Builder.CreateFence(llvm::AtomicOrdering::AcquireRelease, SSID);
5299 Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent, SSID);
5305 llvm::BasicBlock *AcquireBB, *ReleaseBB, *AcqRelBB, *SeqCstBB;
5312 Order =
Builder.CreateIntCast(Order,
Builder.getInt32Ty(),
false);
5313 llvm::SwitchInst *SI =
Builder.CreateSwitch(Order, ContBB);
5315 Builder.SetInsertPoint(AcquireBB);
5316 Builder.CreateFence(llvm::AtomicOrdering::Acquire, SSID);
5318 SI->addCase(
Builder.getInt32(1), AcquireBB);
5319 SI->addCase(
Builder.getInt32(2), AcquireBB);
5321 Builder.SetInsertPoint(ReleaseBB);
5322 Builder.CreateFence(llvm::AtomicOrdering::Release, SSID);
5324 SI->addCase(
Builder.getInt32(3), ReleaseBB);
5326 Builder.SetInsertPoint(AcqRelBB);
5327 Builder.CreateFence(llvm::AtomicOrdering::AcquireRelease, SSID);
5329 SI->addCase(
Builder.getInt32(4), AcqRelBB);
5331 Builder.SetInsertPoint(SeqCstBB);
5332 Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent, SSID);
5334 SI->addCase(
Builder.getInt32(5), SeqCstBB);
5336 Builder.SetInsertPoint(ContBB);
5339 case Builtin::BI__scoped_atomic_thread_fence: {
5344 auto Ord = dyn_cast<llvm::ConstantInt>(Order);
5345 auto Scp = dyn_cast<llvm::ConstantInt>(
Scope);
5347 SyncScope SS = ScopeModel->isValid(Scp->getZExtValue())
5348 ? ScopeModel->map(Scp->getZExtValue())
5349 : ScopeModel->map(ScopeModel->getFallBackValue());
5350 switch (Ord->getZExtValue()) {
5357 llvm::AtomicOrdering::Acquire,
5359 llvm::AtomicOrdering::Acquire,
5364 llvm::AtomicOrdering::Release,
5366 llvm::AtomicOrdering::Release,
5370 Builder.CreateFence(llvm::AtomicOrdering::AcquireRelease,
5373 llvm::AtomicOrdering::AcquireRelease,
5377 Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent,
5380 llvm::AtomicOrdering::SequentiallyConsistent,
5392 switch (Ord->getZExtValue()) {
5395 ContBB->eraseFromParent();
5399 OrderBBs.emplace_back(
Builder.GetInsertBlock(),
5400 llvm::AtomicOrdering::Acquire);
5403 OrderBBs.emplace_back(
Builder.GetInsertBlock(),
5404 llvm::AtomicOrdering::Release);
5407 OrderBBs.emplace_back(
Builder.GetInsertBlock(),
5408 llvm::AtomicOrdering::AcquireRelease);
5411 OrderBBs.emplace_back(
Builder.GetInsertBlock(),
5412 llvm::AtomicOrdering::SequentiallyConsistent);
5421 Order =
Builder.CreateIntCast(Order,
Builder.getInt32Ty(),
false);
5422 llvm::SwitchInst *SI =
Builder.CreateSwitch(Order, ContBB);
5423 SI->addCase(
Builder.getInt32(1), AcquireBB);
5424 SI->addCase(
Builder.getInt32(2), AcquireBB);
5425 SI->addCase(
Builder.getInt32(3), ReleaseBB);
5426 SI->addCase(
Builder.getInt32(4), AcqRelBB);
5427 SI->addCase(
Builder.getInt32(5), SeqCstBB);
5429 OrderBBs.emplace_back(AcquireBB, llvm::AtomicOrdering::Acquire);
5430 OrderBBs.emplace_back(ReleaseBB, llvm::AtomicOrdering::Release);
5431 OrderBBs.emplace_back(AcqRelBB, llvm::AtomicOrdering::AcquireRelease);
5432 OrderBBs.emplace_back(SeqCstBB,
5433 llvm::AtomicOrdering::SequentiallyConsistent);
5436 for (
auto &[OrderBB, Ordering] : OrderBBs) {
5437 Builder.SetInsertPoint(OrderBB);
5439 SyncScope SS = ScopeModel->isValid(Scp->getZExtValue())
5440 ? ScopeModel->map(Scp->getZExtValue())
5441 : ScopeModel->map(ScopeModel->getFallBackValue());
5447 llvm::DenseMap<unsigned, llvm::BasicBlock *> BBs;
5448 for (
unsigned Scp : ScopeModel->getRuntimeValues())
5452 llvm::SwitchInst *SI =
Builder.CreateSwitch(SC, ContBB);
5453 for (
unsigned Scp : ScopeModel->getRuntimeValues()) {
5455 SI->addCase(
Builder.getInt32(Scp), B);
5466 Builder.SetInsertPoint(ContBB);
5470 case Builtin::BI__builtin_signbit:
5471 case Builtin::BI__builtin_signbitf:
5472 case Builtin::BI__builtin_signbitl: {
5477 case Builtin::BI__warn_memset_zero_len:
5479 case Builtin::BI__annotation: {
5482 for (
const Expr *Arg :
E->arguments()) {
5484 assert(Str->getCharByteWidth() == 2);
5485 StringRef WideBytes = Str->getBytes();
5486 std::string StrUtf8;
5487 if (!convertUTF16ToUTF8String(
5488 ArrayRef(WideBytes.data(), WideBytes.size()), StrUtf8)) {
5492 Strings.push_back(llvm::MDString::get(
getLLVMContext(), StrUtf8));
5502 case Builtin::BI__builtin_annotation: {
5511 StringRef Str = cast<StringLiteral>(AnnotationStrExpr)->getString();
5515 case Builtin::BI__builtin_addcb:
5516 case Builtin::BI__builtin_addcs:
5517 case Builtin::BI__builtin_addc:
5518 case Builtin::BI__builtin_addcl:
5519 case Builtin::BI__builtin_addcll:
5520 case Builtin::BI__builtin_subcb:
5521 case Builtin::BI__builtin_subcs:
5522 case Builtin::BI__builtin_subc:
5523 case Builtin::BI__builtin_subcl:
5524 case Builtin::BI__builtin_subcll: {
5550 llvm::Intrinsic::ID IntrinsicId;
5551 switch (BuiltinID) {
5552 default: llvm_unreachable(
"Unknown multiprecision builtin id.");
5553 case Builtin::BI__builtin_addcb:
5554 case Builtin::BI__builtin_addcs:
5555 case Builtin::BI__builtin_addc:
5556 case Builtin::BI__builtin_addcl:
5557 case Builtin::BI__builtin_addcll:
5558 IntrinsicId = llvm::Intrinsic::uadd_with_overflow;
5560 case Builtin::BI__builtin_subcb:
5561 case Builtin::BI__builtin_subcs:
5562 case Builtin::BI__builtin_subc:
5563 case Builtin::BI__builtin_subcl:
5564 case Builtin::BI__builtin_subcll:
5565 IntrinsicId = llvm::Intrinsic::usub_with_overflow;
5570 llvm::Value *Carry1;
5573 llvm::Value *Carry2;
5575 Sum1, Carryin, Carry2);
5576 llvm::Value *CarryOut =
Builder.CreateZExt(
Builder.CreateOr(Carry1, Carry2),
5582 case Builtin::BI__builtin_add_overflow:
5583 case Builtin::BI__builtin_sub_overflow:
5584 case Builtin::BI__builtin_mul_overflow: {
5592 WidthAndSignedness LeftInfo =
5594 WidthAndSignedness RightInfo =
5596 WidthAndSignedness ResultInfo =
5603 RightInfo, ResultArg, ResultQTy,
5609 *
this, LeftArg, LeftInfo, RightArg, RightInfo, ResultArg, ResultQTy,
5612 WidthAndSignedness EncompassingInfo =
5615 llvm::Type *EncompassingLLVMTy =
5620 llvm::Intrinsic::ID IntrinsicId;
5621 switch (BuiltinID) {
5623 llvm_unreachable(
"Unknown overflow builtin id.");
5624 case Builtin::BI__builtin_add_overflow:
5625 IntrinsicId = EncompassingInfo.Signed
5626 ? llvm::Intrinsic::sadd_with_overflow
5627 : llvm::Intrinsic::uadd_with_overflow;
5629 case Builtin::BI__builtin_sub_overflow:
5630 IntrinsicId = EncompassingInfo.Signed
5631 ? llvm::Intrinsic::ssub_with_overflow
5632 : llvm::Intrinsic::usub_with_overflow;
5634 case Builtin::BI__builtin_mul_overflow:
5635 IntrinsicId = EncompassingInfo.Signed
5636 ? llvm::Intrinsic::smul_with_overflow
5637 : llvm::Intrinsic::umul_with_overflow;
5646 Left =
Builder.CreateIntCast(Left, EncompassingLLVMTy, LeftInfo.Signed);
5647 Right =
Builder.CreateIntCast(Right, EncompassingLLVMTy, RightInfo.Signed);
5650 llvm::Value *Overflow, *
Result;
5653 if (EncompassingInfo.Width > ResultInfo.Width) {
5656 llvm::Value *ResultTrunc =
Builder.CreateTrunc(
Result, ResultLLVMTy);
5660 llvm::Value *ResultTruncExt =
Builder.CreateIntCast(
5661 ResultTrunc, EncompassingLLVMTy, ResultInfo.Signed);
5662 llvm::Value *TruncationOverflow =
5665 Overflow =
Builder.CreateOr(Overflow, TruncationOverflow);
5677 case Builtin::BI__builtin_uadd_overflow:
5678 case Builtin::BI__builtin_uaddl_overflow:
5679 case Builtin::BI__builtin_uaddll_overflow:
5680 case Builtin::BI__builtin_usub_overflow:
5681 case Builtin::BI__builtin_usubl_overflow:
5682 case Builtin::BI__builtin_usubll_overflow:
5683 case Builtin::BI__builtin_umul_overflow:
5684 case Builtin::BI__builtin_umull_overflow:
5685 case Builtin::BI__builtin_umulll_overflow:
5686 case Builtin::BI__builtin_sadd_overflow:
5687 case Builtin::BI__builtin_saddl_overflow:
5688 case Builtin::BI__builtin_saddll_overflow:
5689 case Builtin::BI__builtin_ssub_overflow:
5690 case Builtin::BI__builtin_ssubl_overflow:
5691 case Builtin::BI__builtin_ssubll_overflow:
5692 case Builtin::BI__builtin_smul_overflow:
5693 case Builtin::BI__builtin_smull_overflow:
5694 case Builtin::BI__builtin_smulll_overflow: {
5704 llvm::Intrinsic::ID IntrinsicId;
5705 switch (BuiltinID) {
5706 default: llvm_unreachable(
"Unknown overflow builtin id.");
5707 case Builtin::BI__builtin_uadd_overflow:
5708 case Builtin::BI__builtin_uaddl_overflow:
5709 case Builtin::BI__builtin_uaddll_overflow:
5710 IntrinsicId = llvm::Intrinsic::uadd_with_overflow;
5712 case Builtin::BI__builtin_usub_overflow:
5713 case Builtin::BI__builtin_usubl_overflow:
5714 case Builtin::BI__builtin_usubll_overflow:
5715 IntrinsicId = llvm::Intrinsic::usub_with_overflow;
5717 case Builtin::BI__builtin_umul_overflow:
5718 case Builtin::BI__builtin_umull_overflow:
5719 case Builtin::BI__builtin_umulll_overflow:
5720 IntrinsicId = llvm::Intrinsic::umul_with_overflow;
5722 case Builtin::BI__builtin_sadd_overflow:
5723 case Builtin::BI__builtin_saddl_overflow:
5724 case Builtin::BI__builtin_saddll_overflow:
5725 IntrinsicId = llvm::Intrinsic::sadd_with_overflow;
5727 case Builtin::BI__builtin_ssub_overflow:
5728 case Builtin::BI__builtin_ssubl_overflow:
5729 case Builtin::BI__builtin_ssubll_overflow:
5730 IntrinsicId = llvm::Intrinsic::ssub_with_overflow;
5732 case Builtin::BI__builtin_smul_overflow:
5733 case Builtin::BI__builtin_smull_overflow:
5734 case Builtin::BI__builtin_smulll_overflow:
5735 IntrinsicId = llvm::Intrinsic::smul_with_overflow;
5746 case Builtin::BIaddressof:
5747 case Builtin::BI__addressof:
5748 case Builtin::BI__builtin_addressof:
5750 case Builtin::BI__builtin_function_start:
5753 case Builtin::BI__builtin_operator_new:
5756 case Builtin::BI__builtin_operator_delete:
5761 case Builtin::BI__builtin_is_aligned:
5763 case Builtin::BI__builtin_align_up:
5765 case Builtin::BI__builtin_align_down:
5768 case Builtin::BI__noop:
5771 case Builtin::BI__builtin_call_with_static_chain: {
5773 const Expr *Chain =
E->getArg(1);
5778 case Builtin::BI_InterlockedExchange8:
5779 case Builtin::BI_InterlockedExchange16:
5780 case Builtin::BI_InterlockedExchange:
5781 case Builtin::BI_InterlockedExchangePointer:
5784 case Builtin::BI_InterlockedCompareExchangePointer:
5787 case Builtin::BI_InterlockedCompareExchangePointer_nf:
5790 case Builtin::BI_InterlockedCompareExchange8:
5791 case Builtin::BI_InterlockedCompareExchange16:
5792 case Builtin::BI_InterlockedCompareExchange:
5793 case Builtin::BI_InterlockedCompareExchange64:
5795 case Builtin::BI_InterlockedIncrement16:
5796 case Builtin::BI_InterlockedIncrement:
5799 case Builtin::BI_InterlockedDecrement16:
5800 case Builtin::BI_InterlockedDecrement:
5803 case Builtin::BI_InterlockedAnd8:
5804 case Builtin::BI_InterlockedAnd16:
5805 case Builtin::BI_InterlockedAnd:
5807 case Builtin::BI_InterlockedExchangeAdd8:
5808 case Builtin::BI_InterlockedExchangeAdd16:
5809 case Builtin::BI_InterlockedExchangeAdd:
5812 case Builtin::BI_InterlockedExchangeSub8:
5813 case Builtin::BI_InterlockedExchangeSub16:
5814 case Builtin::BI_InterlockedExchangeSub:
5817 case Builtin::BI_InterlockedOr8:
5818 case Builtin::BI_InterlockedOr16:
5819 case Builtin::BI_InterlockedOr:
5821 case Builtin::BI_InterlockedXor8:
5822 case Builtin::BI_InterlockedXor16:
5823 case Builtin::BI_InterlockedXor:
5826 case Builtin::BI_bittest64:
5827 case Builtin::BI_bittest:
5828 case Builtin::BI_bittestandcomplement64:
5829 case Builtin::BI_bittestandcomplement:
5830 case Builtin::BI_bittestandreset64:
5831 case Builtin::BI_bittestandreset:
5832 case Builtin::BI_bittestandset64:
5833 case Builtin::BI_bittestandset:
5834 case Builtin::BI_interlockedbittestandreset:
5835 case Builtin::BI_interlockedbittestandreset64:
5836 case Builtin::BI_interlockedbittestandset64:
5837 case Builtin::BI_interlockedbittestandset:
5838 case Builtin::BI_interlockedbittestandset_acq:
5839 case Builtin::BI_interlockedbittestandset_rel:
5840 case Builtin::BI_interlockedbittestandset_nf:
5841 case Builtin::BI_interlockedbittestandreset_acq:
5842 case Builtin::BI_interlockedbittestandreset_rel:
5843 case Builtin::BI_interlockedbittestandreset_nf:
5848 case Builtin::BI__iso_volatile_load8:
5849 case Builtin::BI__iso_volatile_load16:
5850 case Builtin::BI__iso_volatile_load32:
5851 case Builtin::BI__iso_volatile_load64:
5853 case Builtin::BI__iso_volatile_store8:
5854 case Builtin::BI__iso_volatile_store16:
5855 case Builtin::BI__iso_volatile_store32:
5856 case Builtin::BI__iso_volatile_store64:
5859 case Builtin::BI__builtin_ptrauth_sign_constant:
5862 case Builtin::BI__builtin_ptrauth_auth:
5863 case Builtin::BI__builtin_ptrauth_auth_and_resign:
5864 case Builtin::BI__builtin_ptrauth_blend_discriminator:
5865 case Builtin::BI__builtin_ptrauth_sign_generic_data:
5866 case Builtin::BI__builtin_ptrauth_sign_unauthenticated:
5867 case Builtin::BI__builtin_ptrauth_strip: {
5870 for (
auto argExpr :
E->arguments())
5874 llvm::Type *OrigValueType = Args[0]->getType();
5875 if (OrigValueType->isPointerTy())
5878 switch (BuiltinID) {
5879 case Builtin::BI__builtin_ptrauth_auth_and_resign:
5880 if (Args[4]->getType()->isPointerTy())
5884 case Builtin::BI__builtin_ptrauth_auth:
5885 case Builtin::BI__builtin_ptrauth_sign_unauthenticated:
5886 if (Args[2]->getType()->isPointerTy())
5890 case Builtin::BI__builtin_ptrauth_sign_generic_data:
5891 if (Args[1]->getType()->isPointerTy())
5895 case Builtin::BI__builtin_ptrauth_blend_discriminator:
5896 case Builtin::BI__builtin_ptrauth_strip:
5901 auto IntrinsicID = [&]() ->
unsigned {
5902 switch (BuiltinID) {
5903 case Builtin::BI__builtin_ptrauth_auth:
5904 return llvm::Intrinsic::ptrauth_auth;
5905 case Builtin::BI__builtin_ptrauth_auth_and_resign:
5906 return llvm::Intrinsic::ptrauth_resign;
5907 case Builtin::BI__builtin_ptrauth_blend_discriminator:
5908 return llvm::Intrinsic::ptrauth_blend;
5909 case Builtin::BI__builtin_ptrauth_sign_generic_data:
5910 return llvm::Intrinsic::ptrauth_sign_generic;
5911 case Builtin::BI__builtin_ptrauth_sign_unauthenticated:
5912 return llvm::Intrinsic::ptrauth_sign;
5913 case Builtin::BI__builtin_ptrauth_strip:
5914 return llvm::Intrinsic::ptrauth_strip;
5916 llvm_unreachable(
"bad ptrauth intrinsic");
5921 if (BuiltinID != Builtin::BI__builtin_ptrauth_sign_generic_data &&
5922 BuiltinID != Builtin::BI__builtin_ptrauth_blend_discriminator &&
5923 OrigValueType->isPointerTy()) {
5929 case Builtin::BI__exception_code:
5930 case Builtin::BI_exception_code:
5932 case Builtin::BI__exception_info:
5933 case Builtin::BI_exception_info:
5935 case Builtin::BI__abnormal_termination:
5936 case Builtin::BI_abnormal_termination:
5938 case Builtin::BI_setjmpex:
5939 if (
getTarget().getTriple().isOSMSVCRT() &&
E->getNumArgs() == 1 &&
5943 case Builtin::BI_setjmp:
5944 if (
getTarget().getTriple().isOSMSVCRT() &&
E->getNumArgs() == 1 &&
5946 if (
getTarget().getTriple().getArch() == llvm::Triple::x86)
5948 else if (
getTarget().getTriple().getArch() == llvm::Triple::aarch64)
5955 case Builtin::BImove:
5956 case Builtin::BImove_if_noexcept:
5957 case Builtin::BIforward:
5958 case Builtin::BIforward_like:
5959 case Builtin::BIas_const:
5961 case Builtin::BI__GetExceptionInfo: {
5962 if (llvm::GlobalVariable *GV =
5968 case Builtin::BI__fastfail:
5971 case Builtin::BI__builtin_coro_id:
5973 case Builtin::BI__builtin_coro_promise:
5975 case Builtin::BI__builtin_coro_resume:
5978 case Builtin::BI__builtin_coro_frame:
5980 case Builtin::BI__builtin_coro_noop:
5982 case Builtin::BI__builtin_coro_free:
5984 case Builtin::BI__builtin_coro_destroy:
5987 case Builtin::BI__builtin_coro_done:
5989 case Builtin::BI__builtin_coro_alloc:
5991 case Builtin::BI__builtin_coro_begin:
5993 case Builtin::BI__builtin_coro_end:
5995 case Builtin::BI__builtin_coro_suspend:
5997 case Builtin::BI__builtin_coro_size:
5999 case Builtin::BI__builtin_coro_align:
6003 case Builtin::BIread_pipe:
6004 case Builtin::BIwrite_pipe: {
6008 Value *PacketSize = OpenCLRT.getPipeElemSize(
E->getArg(0));
6009 Value *PacketAlign = OpenCLRT.getPipeElemAlign(
E->getArg(0));
6012 unsigned GenericAS =
6014 llvm::Type *I8PTy = llvm::PointerType::get(
getLLVMContext(), GenericAS);
6017 if (2U ==
E->getNumArgs()) {
6018 const char *Name = (BuiltinID == Builtin::BIread_pipe) ?
"__read_pipe_2"
6023 llvm::FunctionType *FTy = llvm::FunctionType::get(
6028 {Arg0, ACast, PacketSize, PacketAlign}));
6030 assert(4 ==
E->getNumArgs() &&
6031 "Illegal number of parameters to pipe function");
6032 const char *Name = (BuiltinID == Builtin::BIread_pipe) ?
"__read_pipe_4"
6039 llvm::FunctionType *FTy = llvm::FunctionType::get(
6048 {Arg0, Arg1, Arg2, ACast, PacketSize, PacketAlign}));
6053 case Builtin::BIreserve_read_pipe:
6054 case Builtin::BIreserve_write_pipe:
6055 case Builtin::BIwork_group_reserve_read_pipe:
6056 case Builtin::BIwork_group_reserve_write_pipe:
6057 case Builtin::BIsub_group_reserve_read_pipe:
6058 case Builtin::BIsub_group_reserve_write_pipe: {
6061 if (BuiltinID == Builtin::BIreserve_read_pipe)
6062 Name =
"__reserve_read_pipe";
6063 else if (BuiltinID == Builtin::BIreserve_write_pipe)
6064 Name =
"__reserve_write_pipe";
6065 else if (BuiltinID == Builtin::BIwork_group_reserve_read_pipe)
6066 Name =
"__work_group_reserve_read_pipe";
6067 else if (BuiltinID == Builtin::BIwork_group_reserve_write_pipe)
6068 Name =
"__work_group_reserve_write_pipe";
6069 else if (BuiltinID == Builtin::BIsub_group_reserve_read_pipe)
6070 Name =
"__sub_group_reserve_read_pipe";
6072 Name =
"__sub_group_reserve_write_pipe";
6078 Value *PacketSize = OpenCLRT.getPipeElemSize(
E->getArg(0));
6079 Value *PacketAlign = OpenCLRT.getPipeElemAlign(
E->getArg(0));
6083 llvm::FunctionType *FTy = llvm::FunctionType::get(
6090 {Arg0, Arg1, PacketSize, PacketAlign}));
6094 case Builtin::BIcommit_read_pipe:
6095 case Builtin::BIcommit_write_pipe:
6096 case Builtin::BIwork_group_commit_read_pipe:
6097 case Builtin::BIwork_group_commit_write_pipe:
6098 case Builtin::BIsub_group_commit_read_pipe:
6099 case Builtin::BIsub_group_commit_write_pipe: {
6101 if (BuiltinID == Builtin::BIcommit_read_pipe)
6102 Name =
"__commit_read_pipe";
6103 else if (BuiltinID == Builtin::BIcommit_write_pipe)
6104 Name =
"__commit_write_pipe";
6105 else if (BuiltinID == Builtin::BIwork_group_commit_read_pipe)
6106 Name =
"__work_group_commit_read_pipe";
6107 else if (BuiltinID == Builtin::BIwork_group_commit_write_pipe)
6108 Name =
"__work_group_commit_write_pipe";
6109 else if (BuiltinID == Builtin::BIsub_group_commit_read_pipe)
6110 Name =
"__sub_group_commit_read_pipe";
6112 Name =
"__sub_group_commit_write_pipe";
6117 Value *PacketSize = OpenCLRT.getPipeElemSize(
E->getArg(0));
6118 Value *PacketAlign = OpenCLRT.getPipeElemAlign(
E->getArg(0));
6122 llvm::FunctionType *FTy =
6127 {Arg0, Arg1, PacketSize, PacketAlign}));
6130 case Builtin::BIget_pipe_num_packets:
6131 case Builtin::BIget_pipe_max_packets: {
6132 const char *BaseName;
6134 if (BuiltinID == Builtin::BIget_pipe_num_packets)
6135 BaseName =
"__get_pipe_num_packets";
6137 BaseName =
"__get_pipe_max_packets";
6138 std::string Name = std::string(BaseName) +
6139 std::string(PipeTy->isReadOnly() ?
"_ro" :
"_wo");
6144 Value *PacketSize = OpenCLRT.getPipeElemSize(
E->getArg(0));
6145 Value *PacketAlign = OpenCLRT.getPipeElemAlign(
E->getArg(0));
6147 llvm::FunctionType *FTy = llvm::FunctionType::get(
6151 {Arg0, PacketSize, PacketAlign}));
6155 case Builtin::BIto_global:
6156 case Builtin::BIto_local:
6157 case Builtin::BIto_private: {
6159 auto NewArgT = llvm::PointerType::get(
6162 auto NewRetT = llvm::PointerType::get(
6166 auto FTy = llvm::FunctionType::get(NewRetT, {NewArgT},
false);
6167 llvm::Value *NewArg;
6168 if (Arg0->
getType()->getPointerAddressSpace() !=
6169 NewArgT->getPointerAddressSpace())
6172 NewArg =
Builder.CreateBitOrPointerCast(Arg0, NewArgT);
6173 auto NewName = std::string(
"__") +
E->getDirectCallee()->getName().str();
6188 case Builtin::BIenqueue_kernel: {
6190 unsigned NumArgs =
E->getNumArgs();
6193 llvm::Type *GenericVoidPtrTy =
Builder.getPtrTy(
6205 Name =
"__enqueue_kernel_basic";
6206 llvm::Type *ArgTys[] = {QueueTy,
Int32Ty, RangeTy, GenericVoidPtrTy,
6208 llvm::FunctionType *FTy = llvm::FunctionType::get(
6214 Builder.CreatePointerCast(Info.KernelHandle, GenericVoidPtrTy);
6215 llvm::Value *
Block =
6216 Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
6219 {Queue, Flags, Range, Kernel, Block});
6222 assert(NumArgs >= 5 &&
"Invalid enqueue_kernel signature");
6226 auto CreateArrayForSizeVar = [=](
unsigned First)
6227 -> std::tuple<llvm::Value *, llvm::Value *, llvm::Value *> {
6228 llvm::APInt ArraySize(32, NumArgs -
First);
6230 getContext().getSizeType(), ArraySize,
nullptr,
6234 llvm::Value *TmpPtr = Tmp.getPointer();
6239 llvm::Value *Alloca = TmpPtr->stripPointerCasts();
6242 llvm::Value *ElemPtr;
6245 auto *
Zero = llvm::ConstantInt::get(
IntTy, 0);
6246 for (
unsigned I =
First; I < NumArgs; ++I) {
6247 auto *Index = llvm::ConstantInt::get(
IntTy, I -
First);
6259 return std::tie(ElemPtr, TmpSize, Alloca);
6265 Name =
"__enqueue_kernel_varargs";
6269 Builder.CreatePointerCast(Info.KernelHandle, GenericVoidPtrTy);
6270 auto *
Block =
Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
6271 llvm::Value *ElemPtr, *TmpSize, *TmpPtr;
6272 std::tie(ElemPtr, TmpSize, TmpPtr) = CreateArrayForSizeVar(4);
6276 llvm::Value *
const Args[] = {Queue, Flags,
6280 llvm::Type *
const ArgTys[] = {
6281 QueueTy,
IntTy, RangeTy, GenericVoidPtrTy,
6282 GenericVoidPtrTy,
IntTy, ElemPtr->getType()};
6284 llvm::FunctionType *FTy = llvm::FunctionType::get(
Int32Ty, ArgTys,
false);
6293 llvm::PointerType *PtrTy = llvm::PointerType::get(
6297 llvm::Value *NumEvents =
6303 llvm::Value *EventWaitList =
nullptr;
6306 EventWaitList = llvm::ConstantPointerNull::get(PtrTy);
6313 EventWaitList =
Builder.CreatePointerCast(EventWaitList, PtrTy);
6315 llvm::Value *EventRet =
nullptr;
6318 EventRet = llvm::ConstantPointerNull::get(PtrTy);
6327 Builder.CreatePointerCast(Info.KernelHandle, GenericVoidPtrTy);
6328 llvm::Value *
Block =
6329 Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
6331 std::vector<llvm::Type *> ArgTys = {
6333 PtrTy, PtrTy, GenericVoidPtrTy, GenericVoidPtrTy};
6335 std::vector<llvm::Value *> Args = {Queue, Flags,
Range,
6336 NumEvents, EventWaitList, EventRet,
6341 Name =
"__enqueue_kernel_basic_events";
6342 llvm::FunctionType *FTy = llvm::FunctionType::get(
6350 Args.push_back(ConstantInt::get(
Int32Ty, NumArgs - 7));
6352 Name =
"__enqueue_kernel_events_varargs";
6354 llvm::Value *ElemPtr, *TmpSize, *TmpPtr;
6355 std::tie(ElemPtr, TmpSize, TmpPtr) = CreateArrayForSizeVar(7);
6356 Args.push_back(ElemPtr);
6357 ArgTys.push_back(ElemPtr->getType());
6359 llvm::FunctionType *FTy = llvm::FunctionType::get(
6368 llvm_unreachable(
"Unexpected enqueue_kernel signature");
6372 case Builtin::BIget_kernel_work_group_size: {
6373 llvm::Type *GenericVoidPtrTy =
Builder.getPtrTy(
6378 Builder.CreatePointerCast(Info.KernelHandle, GenericVoidPtrTy);
6379 Value *Arg =
Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
6382 llvm::FunctionType::get(
IntTy, {GenericVoidPtrTy, GenericVoidPtrTy},
6384 "__get_kernel_work_group_size_impl"),
6387 case Builtin::BIget_kernel_preferred_work_group_size_multiple: {
6388 llvm::Type *GenericVoidPtrTy =
Builder.getPtrTy(
6393 Builder.CreatePointerCast(Info.KernelHandle, GenericVoidPtrTy);
6394 Value *Arg =
Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
6397 llvm::FunctionType::get(
IntTy, {GenericVoidPtrTy, GenericVoidPtrTy},
6399 "__get_kernel_preferred_work_group_size_multiple_impl"),
6402 case Builtin::BIget_kernel_max_sub_group_size_for_ndrange:
6403 case Builtin::BIget_kernel_sub_group_count_for_ndrange: {
6404 llvm::Type *GenericVoidPtrTy =
Builder.getPtrTy(
6411 Builder.CreatePointerCast(Info.KernelHandle, GenericVoidPtrTy);
6414 BuiltinID == Builtin::BIget_kernel_max_sub_group_size_for_ndrange
6415 ?
"__get_kernel_max_sub_group_size_for_ndrange_impl"
6416 :
"__get_kernel_sub_group_count_for_ndrange_impl";
6419 llvm::FunctionType::get(
6420 IntTy, {NDRange->getType(), GenericVoidPtrTy, GenericVoidPtrTy},
6423 {NDRange, Kernel, Block}));
6425 case Builtin::BI__builtin_store_half:
6426 case Builtin::BI__builtin_store_halff: {
6433 case Builtin::BI__builtin_load_half: {
6438 case Builtin::BI__builtin_load_halff: {
6443 case Builtin::BI__builtin_printf:
6444 case Builtin::BIprintf:
6445 if (
getTarget().getTriple().isNVPTX() ||
6448 getTarget().getTriple().getVendor() == Triple::VendorType::AMD)) {
6451 if ((
getTarget().getTriple().isAMDGCN() ||
6458 case Builtin::BI__builtin_canonicalize:
6459 case Builtin::BI__builtin_canonicalizef:
6460 case Builtin::BI__builtin_canonicalizef16:
6461 case Builtin::BI__builtin_canonicalizel:
6463 emitBuiltinWithOneOverloadedType<1>(*
this,
E, Intrinsic::canonicalize));
6465 case Builtin::BI__builtin_thread_pointer: {
6466 if (!
getContext().getTargetInfo().isTLSSupported())
6471 case Builtin::BI__builtin_os_log_format:
6474 case Builtin::BI__xray_customevent: {
6487 auto FTy = F->getFunctionType();
6488 auto Arg0 =
E->getArg(0);
6490 auto Arg0Ty = Arg0->
getType();
6491 auto PTy0 = FTy->getParamType(0);
6492 if (PTy0 != Arg0Val->getType()) {
6493 if (Arg0Ty->isArrayType())
6496 Arg0Val =
Builder.CreatePointerCast(Arg0Val, PTy0);
6499 auto PTy1 = FTy->getParamType(1);
6501 Arg1 =
Builder.CreateTruncOrBitCast(Arg1, PTy1);
6505 case Builtin::BI__xray_typedevent: {
6521 auto FTy = F->getFunctionType();
6523 auto PTy0 = FTy->getParamType(0);
6525 Arg0 =
Builder.CreateTruncOrBitCast(Arg0, PTy0);
6526 auto Arg1 =
E->getArg(1);
6528 auto Arg1Ty = Arg1->
getType();
6529 auto PTy1 = FTy->getParamType(1);
6530 if (PTy1 != Arg1Val->getType()) {
6531 if (Arg1Ty->isArrayType())
6534 Arg1Val =
Builder.CreatePointerCast(Arg1Val, PTy1);
6537 auto PTy2 = FTy->getParamType(2);
6539 Arg2 =
Builder.CreateTruncOrBitCast(Arg2, PTy2);
6543 case Builtin::BI__builtin_ms_va_start:
6544 case Builtin::BI__builtin_ms_va_end:
6547 BuiltinID == Builtin::BI__builtin_ms_va_start));
6549 case Builtin::BI__builtin_ms_va_copy: {
6566 case Builtin::BI__builtin_get_device_side_mangled_name: {
6594 LargestVectorWidth = std::max(LargestVectorWidth, VectorWidth);
6598 Intrinsic::ID IntrinsicID = Intrinsic::not_intrinsic;
6600 llvm::Triple::getArchTypePrefix(
getTarget().getTriple().getArch());
6601 if (!Prefix.empty()) {
6602 IntrinsicID = Intrinsic::getIntrinsicForClangBuiltin(Prefix.data(), Name);
6603 if (IntrinsicID == Intrinsic::not_intrinsic && Prefix ==
"spv" &&
6604 getTarget().getTriple().getOS() == llvm::Triple::OSType::AMDHSA)
6605 IntrinsicID = Intrinsic::getIntrinsicForClangBuiltin(
"amdgcn", Name);
6609 if (IntrinsicID == Intrinsic::not_intrinsic)
6610 IntrinsicID = Intrinsic::getIntrinsicForMSBuiltin(Prefix.data(), Name);
6613 if (IntrinsicID != Intrinsic::not_intrinsic) {
6618 unsigned ICEArguments = 0;
6624 llvm::FunctionType *FTy = F->getFunctionType();
6626 for (
unsigned i = 0, e =
E->getNumArgs(); i != e; ++i) {
6630 llvm::Type *PTy = FTy->getParamType(i);
6631 if (PTy != ArgValue->
getType()) {
6633 if (
auto *PtrTy = dyn_cast<llvm::PointerType>(PTy)) {
6634 if (PtrTy->getAddressSpace() !=
6635 ArgValue->
getType()->getPointerAddressSpace()) {
6638 PtrTy->getAddressSpace()));
6644 if (PTy->isX86_AMXTy())
6645 ArgValue =
Builder.CreateIntrinsic(Intrinsic::x86_cast_vector_to_tile,
6646 {ArgValue->
getType()}, {ArgValue});
6648 ArgValue =
Builder.CreateBitCast(ArgValue, PTy);
6651 Args.push_back(ArgValue);
6657 llvm::Type *RetTy =
VoidTy;
6661 if (RetTy !=
V->getType()) {
6663 if (
auto *PtrTy = dyn_cast<llvm::PointerType>(RetTy)) {
6664 if (PtrTy->getAddressSpace() !=
V->getType()->getPointerAddressSpace()) {
6667 PtrTy->getAddressSpace()));
6673 if (
V->getType()->isX86_AMXTy())
6674 V =
Builder.CreateIntrinsic(Intrinsic::x86_cast_tile_to_vector, {RetTy},
6680 if (RetTy->isVoidTy())
6700 if (
V->getType()->isVoidTy())
6707 llvm_unreachable(
"No current target builtin returns complex");
6709 llvm_unreachable(
"Bad evaluation kind in EmitBuiltinExpr");
6716 if (
V->getType()->isVoidTy())
6723 llvm_unreachable(
"No current hlsl builtin returns complex");
6725 llvm_unreachable(
"Bad evaluation kind in EmitBuiltinExpr");
6740 llvm::Triple::ArchType Arch) {
6752 case llvm::Triple::arm:
6753 case llvm::Triple::armeb:
6754 case llvm::Triple::thumb:
6755 case llvm::Triple::thumbeb:
6757 case llvm::Triple::aarch64:
6758 case llvm::Triple::aarch64_32:
6759 case llvm::Triple::aarch64_be:
6761 case llvm::Triple::bpfeb:
6762 case llvm::Triple::bpfel:
6764 case llvm::Triple::x86:
6765 case llvm::Triple::x86_64:
6767 case llvm::Triple::ppc:
6768 case llvm::Triple::ppcle:
6769 case llvm::Triple::ppc64:
6770 case llvm::Triple::ppc64le:
6772 case llvm::Triple::r600:
6773 case llvm::Triple::amdgcn:
6775 case llvm::Triple::systemz:
6777 case llvm::Triple::nvptx:
6778 case llvm::Triple::nvptx64:
6780 case llvm::Triple::wasm32:
6781 case llvm::Triple::wasm64:
6783 case llvm::Triple::hexagon:
6785 case llvm::Triple::riscv32:
6786 case llvm::Triple::riscv64:
6788 case llvm::Triple::spirv:
6790 case llvm::Triple::spirv64:
6803 assert(
getContext().getAuxTargetInfo() &&
"Missing aux target info");
6815 bool HasLegalHalfType =
true,
6817 bool AllowBFloatArgsAndRet =
true) {
6818 int IsQuad = TypeFlags.
isQuad();
6822 return llvm::FixedVectorType::get(CGF->
Int8Ty, V1Ty ? 1 : (8 << IsQuad));
6825 return llvm::FixedVectorType::get(CGF->
Int16Ty, V1Ty ? 1 : (4 << IsQuad));
6827 if (AllowBFloatArgsAndRet)
6828 return llvm::FixedVectorType::get(CGF->
BFloatTy, V1Ty ? 1 : (4 << IsQuad));
6830 return llvm::FixedVectorType::get(CGF->
Int16Ty, V1Ty ? 1 : (4 << IsQuad));
6832 if (HasLegalHalfType)
6833 return llvm::FixedVectorType::get(CGF->
HalfTy, V1Ty ? 1 : (4 << IsQuad));
6835 return llvm::FixedVectorType::get(CGF->
Int16Ty, V1Ty ? 1 : (4 << IsQuad));
6837 return llvm::FixedVectorType::get(CGF->
Int32Ty, V1Ty ? 1 : (2 << IsQuad));
6840 return llvm::FixedVectorType::get(CGF->
Int64Ty, V1Ty ? 1 : (1 << IsQuad));
6845 return llvm::FixedVectorType::get(CGF->
Int8Ty, 16);
6847 return llvm::FixedVectorType::get(CGF->
FloatTy, V1Ty ? 1 : (2 << IsQuad));
6849 return llvm::FixedVectorType::get(CGF->
DoubleTy, V1Ty ? 1 : (1 << IsQuad));
6851 llvm_unreachable(
"Unknown vector element type!");
6856 int IsQuad = IntTypeFlags.
isQuad();
6859 return llvm::FixedVectorType::get(CGF->
HalfTy, (4 << IsQuad));
6861 return llvm::FixedVectorType::get(CGF->
FloatTy, (2 << IsQuad));
6863 return llvm::FixedVectorType::get(CGF->
DoubleTy, (1 << IsQuad));
6865 llvm_unreachable(
"Type can't be converted to floating-point!");
6870 const ElementCount &Count) {
6871 Value *SV = llvm::ConstantVector::getSplat(Count,
C);
6872 return Builder.CreateShuffleVector(
V,
V, SV,
"lane");
6876 ElementCount EC = cast<llvm::VectorType>(
V->getType())->getElementCount();
6882 unsigned shift,
bool rightshift) {
6884 for (Function::const_arg_iterator ai = F->arg_begin(), ae = F->arg_end();
6885 ai != ae; ++ai, ++j) {
6886 if (F->isConstrainedFPIntrinsic())
6887 if (ai->getType()->isMetadataTy())
6889 if (shift > 0 && shift == j)
6892 Ops[j] =
Builder.CreateBitCast(Ops[j], ai->getType(), name);
6895 if (F->isConstrainedFPIntrinsic())
6896 return Builder.CreateConstrainedFPCall(F, Ops, name);
6898 return Builder.CreateCall(F, Ops, name);
6903 int SV = cast<ConstantInt>(
V)->getSExtValue();
6904 return ConstantInt::get(Ty, neg ? -SV : SV);
6909 llvm::Type *Ty,
bool usgn,
6911 llvm::VectorType *VTy = cast<llvm::VectorType>(Ty);
6913 int ShiftAmt = cast<ConstantInt>(Shift)->getSExtValue();
6914 int EltSize = VTy->getScalarSizeInBits();
6916 Vec =
Builder.CreateBitCast(Vec, Ty);
6920 if (ShiftAmt == EltSize) {
6923 return llvm::ConstantAggregateZero::get(VTy);
6928 Shift = ConstantInt::get(VTy->getElementType(), ShiftAmt);
6934 return Builder.CreateLShr(Vec, Shift, name);
6936 return Builder.CreateAShr(Vec, Shift, name);
6962struct ARMVectorIntrinsicInfo {
6963 const char *NameHint;
6965 unsigned LLVMIntrinsic;
6966 unsigned AltLLVMIntrinsic;
6969 bool operator<(
unsigned RHSBuiltinID)
const {
6970 return BuiltinID < RHSBuiltinID;
6972 bool operator<(
const ARMVectorIntrinsicInfo &TE)
const {
6973 return BuiltinID < TE.BuiltinID;
6978#define NEONMAP0(NameBase) \
6979 { #NameBase, NEON::BI__builtin_neon_ ## NameBase, 0, 0, 0 }
6981#define NEONMAP1(NameBase, LLVMIntrinsic, TypeModifier) \
6982 { #NameBase, NEON:: BI__builtin_neon_ ## NameBase, \
6983 Intrinsic::LLVMIntrinsic, 0, TypeModifier }
6985#define NEONMAP2(NameBase, LLVMIntrinsic, AltLLVMIntrinsic, TypeModifier) \
6986 { #NameBase, NEON:: BI__builtin_neon_ ## NameBase, \
6987 Intrinsic::LLVMIntrinsic, Intrinsic::AltLLVMIntrinsic, \
6991 NEONMAP1(__a32_vcvt_bf16_f32, arm_neon_vcvtfp2bf, 0),
6998 NEONMAP1(vabs_v, arm_neon_vabs, 0),
6999 NEONMAP1(vabsq_v, arm_neon_vabs, 0),
7003 NEONMAP1(vaesdq_u8, arm_neon_aesd, 0),
7004 NEONMAP1(vaeseq_u8, arm_neon_aese, 0),
7005 NEONMAP1(vaesimcq_u8, arm_neon_aesimc, 0),
7006 NEONMAP1(vaesmcq_u8, arm_neon_aesmc, 0),
7007 NEONMAP1(vbfdot_f32, arm_neon_bfdot, 0),
7008 NEONMAP1(vbfdotq_f32, arm_neon_bfdot, 0),
7009 NEONMAP1(vbfmlalbq_f32, arm_neon_bfmlalb, 0),
7010 NEONMAP1(vbfmlaltq_f32, arm_neon_bfmlalt, 0),
7011 NEONMAP1(vbfmmlaq_f32, arm_neon_bfmmla, 0),
7024 NEONMAP1(vcage_v, arm_neon_vacge, 0),
7025 NEONMAP1(vcageq_v, arm_neon_vacge, 0),
7026 NEONMAP1(vcagt_v, arm_neon_vacgt, 0),
7027 NEONMAP1(vcagtq_v, arm_neon_vacgt, 0),
7028 NEONMAP1(vcale_v, arm_neon_vacge, 0),
7029 NEONMAP1(vcaleq_v, arm_neon_vacge, 0),
7030 NEONMAP1(vcalt_v, arm_neon_vacgt, 0),
7031 NEONMAP1(vcaltq_v, arm_neon_vacgt, 0),
7048 NEONMAP1(vcvt_f16_f32, arm_neon_vcvtfp2hf, 0),
7051 NEONMAP1(vcvt_f32_f16, arm_neon_vcvthf2fp, 0),
7053 NEONMAP1(vcvt_n_f16_s16, arm_neon_vcvtfxs2fp, 0),
7054 NEONMAP1(vcvt_n_f16_u16, arm_neon_vcvtfxu2fp, 0),
7055 NEONMAP2(vcvt_n_f32_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0),
7056 NEONMAP1(vcvt_n_s16_f16, arm_neon_vcvtfp2fxs, 0),
7057 NEONMAP1(vcvt_n_s32_v, arm_neon_vcvtfp2fxs, 0),
7058 NEONMAP1(vcvt_n_s64_v, arm_neon_vcvtfp2fxs, 0),
7059 NEONMAP1(vcvt_n_u16_f16, arm_neon_vcvtfp2fxu, 0),
7060 NEONMAP1(vcvt_n_u32_v, arm_neon_vcvtfp2fxu, 0),
7061 NEONMAP1(vcvt_n_u64_v, arm_neon_vcvtfp2fxu, 0),
7068 NEONMAP1(vcvta_s16_f16, arm_neon_vcvtas, 0),
7069 NEONMAP1(vcvta_s32_v, arm_neon_vcvtas, 0),
7070 NEONMAP1(vcvta_s64_v, arm_neon_vcvtas, 0),
7071 NEONMAP1(vcvta_u16_f16, arm_neon_vcvtau, 0),
7072 NEONMAP1(vcvta_u32_v, arm_neon_vcvtau, 0),
7073 NEONMAP1(vcvta_u64_v, arm_neon_vcvtau, 0),
7074 NEONMAP1(vcvtaq_s16_f16, arm_neon_vcvtas, 0),
7075 NEONMAP1(vcvtaq_s32_v, arm_neon_vcvtas, 0),
7076 NEONMAP1(vcvtaq_s64_v, arm_neon_vcvtas, 0),
7077 NEONMAP1(vcvtaq_u16_f16, arm_neon_vcvtau, 0),
7078 NEONMAP1(vcvtaq_u32_v, arm_neon_vcvtau, 0),
7079 NEONMAP1(vcvtaq_u64_v, arm_neon_vcvtau, 0),
7080 NEONMAP1(vcvth_bf16_f32, arm_neon_vcvtbfp2bf, 0),
7081 NEONMAP1(vcvtm_s16_f16, arm_neon_vcvtms, 0),
7082 NEONMAP1(vcvtm_s32_v, arm_neon_vcvtms, 0),
7083 NEONMAP1(vcvtm_s64_v, arm_neon_vcvtms, 0),
7084 NEONMAP1(vcvtm_u16_f16, arm_neon_vcvtmu, 0),
7085 NEONMAP1(vcvtm_u32_v, arm_neon_vcvtmu, 0),
7086 NEONMAP1(vcvtm_u64_v, arm_neon_vcvtmu, 0),
7087 NEONMAP1(vcvtmq_s16_f16, arm_neon_vcvtms, 0),
7088 NEONMAP1(vcvtmq_s32_v, arm_neon_vcvtms, 0),
7089 NEONMAP1(vcvtmq_s64_v, arm_neon_vcvtms, 0),
7090 NEONMAP1(vcvtmq_u16_f16, arm_neon_vcvtmu, 0),
7091 NEONMAP1(vcvtmq_u32_v, arm_neon_vcvtmu, 0),
7092 NEONMAP1(vcvtmq_u64_v, arm_neon_vcvtmu, 0),
7093 NEONMAP1(vcvtn_s16_f16, arm_neon_vcvtns, 0),
7094 NEONMAP1(vcvtn_s32_v, arm_neon_vcvtns, 0),
7095 NEONMAP1(vcvtn_s64_v, arm_neon_vcvtns, 0),
7096 NEONMAP1(vcvtn_u16_f16, arm_neon_vcvtnu, 0),
7097 NEONMAP1(vcvtn_u32_v, arm_neon_vcvtnu, 0),
7098 NEONMAP1(vcvtn_u64_v, arm_neon_vcvtnu, 0),
7099 NEONMAP1(vcvtnq_s16_f16, arm_neon_vcvtns, 0),
7100 NEONMAP1(vcvtnq_s32_v, arm_neon_vcvtns, 0),
7101 NEONMAP1(vcvtnq_s64_v, arm_neon_vcvtns, 0),
7102 NEONMAP1(vcvtnq_u16_f16, arm_neon_vcvtnu, 0),
7103 NEONMAP1(vcvtnq_u32_v, arm_neon_vcvtnu, 0),
7104 NEONMAP1(vcvtnq_u64_v, arm_neon_vcvtnu, 0),
7105 NEONMAP1(vcvtp_s16_f16, arm_neon_vcvtps, 0),
7106 NEONMAP1(vcvtp_s32_v, arm_neon_vcvtps, 0),
7107 NEONMAP1(vcvtp_s64_v, arm_neon_vcvtps, 0),
7108 NEONMAP1(vcvtp_u16_f16, arm_neon_vcvtpu, 0),
7109 NEONMAP1(vcvtp_u32_v, arm_neon_vcvtpu, 0),
7110 NEONMAP1(vcvtp_u64_v, arm_neon_vcvtpu, 0),
7111 NEONMAP1(vcvtpq_s16_f16, arm_neon_vcvtps, 0),
7112 NEONMAP1(vcvtpq_s32_v, arm_neon_vcvtps, 0),
7113 NEONMAP1(vcvtpq_s64_v, arm_neon_vcvtps, 0),
7114 NEONMAP1(vcvtpq_u16_f16, arm_neon_vcvtpu, 0),
7115 NEONMAP1(vcvtpq_u32_v, arm_neon_vcvtpu, 0),
7116 NEONMAP1(vcvtpq_u64_v, arm_neon_vcvtpu, 0),
7120 NEONMAP1(vcvtq_n_f16_s16, arm_neon_vcvtfxs2fp, 0),
7121 NEONMAP1(vcvtq_n_f16_u16, arm_neon_vcvtfxu2fp, 0),
7122 NEONMAP2(vcvtq_n_f32_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0),
7123 NEONMAP1(vcvtq_n_s16_f16, arm_neon_vcvtfp2fxs, 0),
7124 NEONMAP1(vcvtq_n_s32_v, arm_neon_vcvtfp2fxs, 0),
7125 NEONMAP1(vcvtq_n_s64_v, arm_neon_vcvtfp2fxs, 0),
7126 NEONMAP1(vcvtq_n_u16_f16, arm_neon_vcvtfp2fxu, 0),
7127 NEONMAP1(vcvtq_n_u32_v, arm_neon_vcvtfp2fxu, 0),
7128 NEONMAP1(vcvtq_n_u64_v, arm_neon_vcvtfp2fxu, 0),
7135 NEONMAP1(vdot_s32, arm_neon_sdot, 0),
7136 NEONMAP1(vdot_u32, arm_neon_udot, 0),
7137 NEONMAP1(vdotq_s32, arm_neon_sdot, 0),
7138 NEONMAP1(vdotq_u32, arm_neon_udot, 0),
7148 NEONMAP1(vld1_v, arm_neon_vld1, 0),
7149 NEONMAP1(vld1_x2_v, arm_neon_vld1x2, 0),
7150 NEONMAP1(vld1_x3_v, arm_neon_vld1x3, 0),
7151 NEONMAP1(vld1_x4_v, arm_neon_vld1x4, 0),
7153 NEONMAP1(vld1q_v, arm_neon_vld1, 0),
7154 NEONMAP1(vld1q_x2_v, arm_neon_vld1x2, 0),
7155 NEONMAP1(vld1q_x3_v, arm_neon_vld1x3, 0),
7156 NEONMAP1(vld1q_x4_v, arm_neon_vld1x4, 0),
7157 NEONMAP1(vld2_dup_v, arm_neon_vld2dup, 0),
7158 NEONMAP1(vld2_lane_v, arm_neon_vld2lane, 0),
7159 NEONMAP1(vld2_v, arm_neon_vld2, 0),
7160 NEONMAP1(vld2q_dup_v, arm_neon_vld2dup, 0),
7161 NEONMAP1(vld2q_lane_v, arm_neon_vld2lane, 0),
7162 NEONMAP1(vld2q_v, arm_neon_vld2, 0),
7163 NEONMAP1(vld3_dup_v, arm_neon_vld3dup, 0),
7164 NEONMAP1(vld3_lane_v, arm_neon_vld3lane, 0),
7165 NEONMAP1(vld3_v, arm_neon_vld3, 0),
7166 NEONMAP1(vld3q_dup_v, arm_neon_vld3dup, 0),
7167 NEONMAP1(vld3q_lane_v, arm_neon_vld3lane, 0),
7168 NEONMAP1(vld3q_v, arm_neon_vld3, 0),
7169 NEONMAP1(vld4_dup_v, arm_neon_vld4dup, 0),
7170 NEONMAP1(vld4_lane_v, arm_neon_vld4lane, 0),
7171 NEONMAP1(vld4_v, arm_neon_vld4, 0),
7172 NEONMAP1(vld4q_dup_v, arm_neon_vld4dup, 0),
7173 NEONMAP1(vld4q_lane_v, arm_neon_vld4lane, 0),
7174 NEONMAP1(vld4q_v, arm_neon_vld4, 0),
7183 NEONMAP1(vmmlaq_s32, arm_neon_smmla, 0),
7184 NEONMAP1(vmmlaq_u32, arm_neon_ummla, 0),
7202 NEONMAP2(vqdmlal_v, arm_neon_vqdmull, sadd_sat, 0),
7203 NEONMAP2(vqdmlsl_v, arm_neon_vqdmull, ssub_sat, 0),
7227 NEONMAP1(vqshlu_n_v, arm_neon_vqshiftsu, 0),
7228 NEONMAP1(vqshluq_n_v, arm_neon_vqshiftsu, 0),
7232 NEONMAP2(vrecpe_v, arm_neon_vrecpe, arm_neon_vrecpe, 0),
7233 NEONMAP2(vrecpeq_v, arm_neon_vrecpe, arm_neon_vrecpe, 0),
7256 NEONMAP2(vrsqrte_v, arm_neon_vrsqrte, arm_neon_vrsqrte, 0),
7257 NEONMAP2(vrsqrteq_v, arm_neon_vrsqrte, arm_neon_vrsqrte, 0),
7261 NEONMAP1(vsha1su0q_u32, arm_neon_sha1su0, 0),
7262 NEONMAP1(vsha1su1q_u32, arm_neon_sha1su1, 0),
7263 NEONMAP1(vsha256h2q_u32, arm_neon_sha256h2, 0),
7264 NEONMAP1(vsha256hq_u32, arm_neon_sha256h, 0),
7265 NEONMAP1(vsha256su0q_u32, arm_neon_sha256su0, 0),
7266 NEONMAP1(vsha256su1q_u32, arm_neon_sha256su1, 0),
7275 NEONMAP1(vst1_v, arm_neon_vst1, 0),
7276 NEONMAP1(vst1_x2_v, arm_neon_vst1x2, 0),
7277 NEONMAP1(vst1_x3_v, arm_neon_vst1x3, 0),
7278 NEONMAP1(vst1_x4_v, arm_neon_vst1x4, 0),
7279 NEONMAP1(vst1q_v, arm_neon_vst1, 0),
7280 NEONMAP1(vst1q_x2_v, arm_neon_vst1x2, 0),
7281 NEONMAP1(vst1q_x3_v, arm_neon_vst1x3, 0),
7282 NEONMAP1(vst1q_x4_v, arm_neon_vst1x4, 0),
7283 NEONMAP1(vst2_lane_v, arm_neon_vst2lane, 0),
7284 NEONMAP1(vst2_v, arm_neon_vst2, 0),
7285 NEONMAP1(vst2q_lane_v, arm_neon_vst2lane, 0),
7286 NEONMAP1(vst2q_v, arm_neon_vst2, 0),
7287 NEONMAP1(vst3_lane_v, arm_neon_vst3lane, 0),
7288 NEONMAP1(vst3_v, arm_neon_vst3, 0),
7289 NEONMAP1(vst3q_lane_v, arm_neon_vst3lane, 0),
7290 NEONMAP1(vst3q_v, arm_neon_vst3, 0),
7291 NEONMAP1(vst4_lane_v, arm_neon_vst4lane, 0),
7292 NEONMAP1(vst4_v, arm_neon_vst4, 0),
7293 NEONMAP1(vst4q_lane_v, arm_neon_vst4lane, 0),
7294 NEONMAP1(vst4q_v, arm_neon_vst4, 0),
7300 NEONMAP1(vusdot_s32, arm_neon_usdot, 0),
7301 NEONMAP1(vusdotq_s32, arm_neon_usdot, 0),
7302 NEONMAP1(vusmmlaq_s32, arm_neon_usmmla, 0),
7310 NEONMAP1(__a64_vcvtq_low_bf16_f32, aarch64_neon_bfcvtn, 0),
7315 NEONMAP1(vabs_v, aarch64_neon_abs, 0),
7316 NEONMAP1(vabsq_v, aarch64_neon_abs, 0),
7321 NEONMAP1(vaesdq_u8, aarch64_crypto_aesd, 0),
7322 NEONMAP1(vaeseq_u8, aarch64_crypto_aese, 0),
7323 NEONMAP1(vaesimcq_u8, aarch64_crypto_aesimc, 0),
7324 NEONMAP1(vaesmcq_u8, aarch64_crypto_aesmc, 0),
7333 NEONMAP1(vbfdot_f32, aarch64_neon_bfdot, 0),
7334 NEONMAP1(vbfdotq_f32, aarch64_neon_bfdot, 0),
7335 NEONMAP1(vbfmlalbq_f32, aarch64_neon_bfmlalb, 0),
7336 NEONMAP1(vbfmlaltq_f32, aarch64_neon_bfmlalt, 0),
7337 NEONMAP1(vbfmmlaq_f32, aarch64_neon_bfmmla, 0),
7348 NEONMAP1(vcage_v, aarch64_neon_facge, 0),
7349 NEONMAP1(vcageq_v, aarch64_neon_facge, 0),
7350 NEONMAP1(vcagt_v, aarch64_neon_facgt, 0),
7351 NEONMAP1(vcagtq_v, aarch64_neon_facgt, 0),
7352 NEONMAP1(vcale_v, aarch64_neon_facge, 0),
7353 NEONMAP1(vcaleq_v, aarch64_neon_facge, 0),
7354 NEONMAP1(vcalt_v, aarch64_neon_facgt, 0),
7355 NEONMAP1(vcaltq_v, aarch64_neon_facgt, 0),
7392 NEONMAP1(vcvt_f16_f32, aarch64_neon_vcvtfp2hf, 0),
7395 NEONMAP1(vcvt_f32_f16, aarch64_neon_vcvthf2fp, 0),
7397 NEONMAP1(vcvt_n_f16_s16, aarch64_neon_vcvtfxs2fp, 0),
7398 NEONMAP1(vcvt_n_f16_u16, aarch64_neon_vcvtfxu2fp, 0),
7399 NEONMAP2(vcvt_n_f32_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
7400 NEONMAP2(vcvt_n_f64_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
7401 NEONMAP1(vcvt_n_s16_f16, aarch64_neon_vcvtfp2fxs, 0),
7402 NEONMAP1(vcvt_n_s32_v, aarch64_neon_vcvtfp2fxs, 0),
7403 NEONMAP1(vcvt_n_s64_v, aarch64_neon_vcvtfp2fxs, 0),
7404 NEONMAP1(vcvt_n_u16_f16, aarch64_neon_vcvtfp2fxu, 0),
7405 NEONMAP1(vcvt_n_u32_v, aarch64_neon_vcvtfp2fxu, 0),
7406 NEONMAP1(vcvt_n_u64_v, aarch64_neon_vcvtfp2fxu, 0),
7410 NEONMAP1(vcvtq_high_bf16_f32, aarch64_neon_bfcvtn2, 0),
7411 NEONMAP1(vcvtq_n_f16_s16, aarch64_neon_vcvtfxs2fp, 0),
7412 NEONMAP1(vcvtq_n_f16_u16, aarch64_neon_vcvtfxu2fp, 0),
7413 NEONMAP2(vcvtq_n_f32_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
7414 NEONMAP2(vcvtq_n_f64_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
7415 NEONMAP1(vcvtq_n_s16_f16, aarch64_neon_vcvtfp2fxs, 0),
7416 NEONMAP1(vcvtq_n_s32_v, aarch64_neon_vcvtfp2fxs, 0),
7417 NEONMAP1(vcvtq_n_s64_v, aarch64_neon_vcvtfp2fxs, 0),
7418 NEONMAP1(vcvtq_n_u16_f16, aarch64_neon_vcvtfp2fxu, 0),
7419 NEONMAP1(vcvtq_n_u32_v, aarch64_neon_vcvtfp2fxu, 0),
7420 NEONMAP1(vcvtq_n_u64_v, aarch64_neon_vcvtfp2fxu, 0),
7422 NEONMAP1(vdot_s32, aarch64_neon_sdot, 0),
7423 NEONMAP1(vdot_u32, aarch64_neon_udot, 0),
7424 NEONMAP1(vdotq_s32, aarch64_neon_sdot, 0),
7425 NEONMAP1(vdotq_u32, aarch64_neon_udot, 0),
7438 NEONMAP1(vfmlal_high_f16, aarch64_neon_fmlal2, 0),
7439 NEONMAP1(vfmlal_low_f16, aarch64_neon_fmlal, 0),
7440 NEONMAP1(vfmlalq_high_f16, aarch64_neon_fmlal2, 0),
7441 NEONMAP1(vfmlalq_low_f16, aarch64_neon_fmlal, 0),
7442 NEONMAP1(vfmlsl_high_f16, aarch64_neon_fmlsl2, 0),
7443 NEONMAP1(vfmlsl_low_f16, aarch64_neon_fmlsl, 0),
7444 NEONMAP1(vfmlslq_high_f16, aarch64_neon_fmlsl2, 0),
7445 NEONMAP1(vfmlslq_low_f16, aarch64_neon_fmlsl, 0),
7450 NEONMAP1(vld1_x2_v, aarch64_neon_ld1x2, 0),
7451 NEONMAP1(vld1_x3_v, aarch64_neon_ld1x3, 0),
7452 NEONMAP1(vld1_x4_v, aarch64_neon_ld1x4, 0),
7453 NEONMAP1(vld1q_x2_v, aarch64_neon_ld1x2, 0),
7454 NEONMAP1(vld1q_x3_v, aarch64_neon_ld1x3, 0),
7455 NEONMAP1(vld1q_x4_v, aarch64_neon_ld1x4, 0),
7456 NEONMAP1(vmmlaq_s32, aarch64_neon_smmla, 0),
7457 NEONMAP1(vmmlaq_u32, aarch64_neon_ummla, 0),
7470 NEONMAP2(vqdmlal_v, aarch64_neon_sqdmull, aarch64_neon_sqadd, 0),
7471 NEONMAP2(vqdmlsl_v, aarch64_neon_sqdmull, aarch64_neon_sqsub, 0),
7472 NEONMAP1(vqdmulh_lane_v, aarch64_neon_sqdmulh_lane, 0),
7473 NEONMAP1(vqdmulh_laneq_v, aarch64_neon_sqdmulh_laneq, 0),
7475 NEONMAP1(vqdmulhq_lane_v, aarch64_neon_sqdmulh_lane, 0),
7476 NEONMAP1(vqdmulhq_laneq_v, aarch64_neon_sqdmulh_laneq, 0),
7491 NEONMAP1(vqrdmulh_lane_v, aarch64_neon_sqrdmulh_lane, 0),
7492 NEONMAP1(vqrdmulh_laneq_v, aarch64_neon_sqrdmulh_laneq, 0),
7494 NEONMAP1(vqrdmulhq_lane_v, aarch64_neon_sqrdmulh_lane, 0),
7495 NEONMAP1(vqrdmulhq_laneq_v, aarch64_neon_sqrdmulh_laneq, 0),
7503 NEONMAP1(vqshlu_n_v, aarch64_neon_sqshlu, 0),
7504 NEONMAP1(vqshluq_n_v, aarch64_neon_sqshlu, 0),
7508 NEONMAP1(vrax1q_u64, aarch64_crypto_rax1, 0),
7509 NEONMAP2(vrecpe_v, aarch64_neon_frecpe, aarch64_neon_urecpe, 0),
7510 NEONMAP2(vrecpeq_v, aarch64_neon_frecpe, aarch64_neon_urecpe, 0),
7537 NEONMAP2(vrsqrte_v, aarch64_neon_frsqrte, aarch64_neon_ursqrte, 0),
7538 NEONMAP2(vrsqrteq_v, aarch64_neon_frsqrte, aarch64_neon_ursqrte, 0),
7542 NEONMAP1(vsha1su0q_u32, aarch64_crypto_sha1su0, 0),
7543 NEONMAP1(vsha1su1q_u32, aarch64_crypto_sha1su1, 0),
7544 NEONMAP1(vsha256h2q_u32, aarch64_crypto_sha256h2, 0),
7545 NEONMAP1(vsha256hq_u32, aarch64_crypto_sha256h, 0),
7546 NEONMAP1(vsha256su0q_u32, aarch64_crypto_sha256su0, 0),
7547 NEONMAP1(vsha256su1q_u32, aarch64_crypto_sha256su1, 0),
7548 NEONMAP1(vsha512h2q_u64, aarch64_crypto_sha512h2, 0),
7549 NEONMAP1(vsha512hq_u64, aarch64_crypto_sha512h, 0),
7550 NEONMAP1(vsha512su0q_u64, aarch64_crypto_sha512su0, 0),
7551 NEONMAP1(vsha512su1q_u64, aarch64_crypto_sha512su1, 0),
7560 NEONMAP1(vsm3partw1q_u32, aarch64_crypto_sm3partw1, 0),
7561 NEONMAP1(vsm3partw2q_u32, aarch64_crypto_sm3partw2, 0),
7562 NEONMAP1(vsm3ss1q_u32, aarch64_crypto_sm3ss1, 0),
7563 NEONMAP1(vsm3tt1aq_u32, aarch64_crypto_sm3tt1a, 0),
7564 NEONMAP1(vsm3tt1bq_u32, aarch64_crypto_sm3tt1b, 0),
7565 NEONMAP1(vsm3tt2aq_u32, aarch64_crypto_sm3tt2a, 0),
7566 NEONMAP1(vsm3tt2bq_u32, aarch64_crypto_sm3tt2b, 0),
7567 NEONMAP1(vsm4ekeyq_u32, aarch64_crypto_sm4ekey, 0),
7568 NEONMAP1(vsm4eq_u32, aarch64_crypto_sm4e, 0),
7569 NEONMAP1(vst1_x2_v, aarch64_neon_st1x2, 0),
7570 NEONMAP1(vst1_x3_v, aarch64_neon_st1x3, 0),
7571 NEONMAP1(vst1_x4_v, aarch64_neon_st1x4, 0),
7572 NEONMAP1(vst1q_x2_v, aarch64_neon_st1x2, 0),
7573 NEONMAP1(vst1q_x3_v, aarch64_neon_st1x3, 0),
7574 NEONMAP1(vst1q_x4_v, aarch64_neon_st1x4, 0),
7578 NEONMAP1(vusdot_s32, aarch64_neon_usdot, 0),
7579 NEONMAP1(vusdotq_s32, aarch64_neon_usdot, 0),
7580 NEONMAP1(vusmmlaq_s32, aarch64_neon_usmmla, 0),
7581 NEONMAP1(vxarq_u64, aarch64_crypto_xar, 0),
7619 NEONMAP1(vcvth_bf16_f32, aarch64_neon_bfcvt, 0),
7638 NEONMAP1(vcvtxd_f32_f64, aarch64_sisd_fcvtxn, 0),
7659 NEONMAP1(vmull_p64, aarch64_neon_pmull64, 0),
7687 NEONMAP1(vqdmulls_s32, aarch64_neon_sqdmulls_scalar, 0),
7768 NEONMAP1(vsha1cq_u32, aarch64_crypto_sha1c, 0),
7769 NEONMAP1(vsha1h_u32, aarch64_crypto_sha1h, 0),
7770 NEONMAP1(vsha1mq_u32, aarch64_crypto_sha1m, 0),
7771 NEONMAP1(vsha1pq_u32, aarch64_crypto_sha1p, 0),
7825 { NEON::BI__builtin_neon_splat_lane_bf16, NEON::BI__builtin_neon_splat_lane_v, },
7826 { NEON::BI__builtin_neon_splat_laneq_bf16, NEON::BI__builtin_neon_splat_laneq_v, },
7827 { NEON::BI__builtin_neon_splatq_lane_bf16, NEON::BI__builtin_neon_splatq_lane_v, },
7828 { NEON::BI__builtin_neon_splatq_laneq_bf16, NEON::BI__builtin_neon_splatq_laneq_v, },
7829 { NEON::BI__builtin_neon_vabd_f16, NEON::BI__builtin_neon_vabd_v, },
7830 { NEON::BI__builtin_neon_vabdq_f16, NEON::BI__builtin_neon_vabdq_v, },
7831 { NEON::BI__builtin_neon_vabs_f16, NEON::BI__builtin_neon_vabs_v, },
7832 { NEON::BI__builtin_neon_vabsq_f16, NEON::BI__builtin_neon_vabsq_v, },
7833 { NEON::BI__builtin_neon_vcage_f16, NEON::BI__builtin_neon_vcage_v, },
7834 { NEON::BI__builtin_neon_vcageq_f16, NEON::BI__builtin_neon_vcageq_v, },
7835 { NEON::BI__builtin_neon_vcagt_f16, NEON::BI__builtin_neon_vcagt_v, },
7836 { NEON::BI__builtin_neon_vcagtq_f16, NEON::BI__builtin_neon_vcagtq_v, },
7837 { NEON::BI__builtin_neon_vcale_f16, NEON::BI__builtin_neon_vcale_v, },
7838 { NEON::BI__builtin_neon_vcaleq_f16, NEON::BI__builtin_neon_vcaleq_v, },
7839 { NEON::BI__builtin_neon_vcalt_f16, NEON::BI__builtin_neon_vcalt_v, },
7840 { NEON::BI__builtin_neon_vcaltq_f16, NEON::BI__builtin_neon_vcaltq_v, },
7841 { NEON::BI__builtin_neon_vceqz_f16, NEON::BI__builtin_neon_vceqz_v, },
7842 { NEON::BI__builtin_neon_vceqzq_f16, NEON::BI__builtin_neon_vceqzq_v, },
7843 { NEON::BI__builtin_neon_vcgez_f16, NEON::BI__builtin_neon_vcgez_v, },
7844 { NEON::BI__builtin_neon_vcgezq_f16, NEON::BI__builtin_neon_vcgezq_v, },
7845 { NEON::BI__builtin_neon_vcgtz_f16, NEON::BI__builtin_neon_vcgtz_v, },
7846 { NEON::BI__builtin_neon_vcgtzq_f16, NEON::BI__builtin_neon_vcgtzq_v, },
7847 { NEON::BI__builtin_neon_vclez_f16, NEON::BI__builtin_neon_vclez_v, },
7848 { NEON::BI__builtin_neon_vclezq_f16, NEON::BI__builtin_neon_vclezq_v, },
7849 { NEON::BI__builtin_neon_vcltz_f16, NEON::BI__builtin_neon_vcltz_v, },
7850 { NEON::BI__builtin_neon_vcltzq_f16, NEON::BI__builtin_neon_vcltzq_v, },
7851 { NEON::BI__builtin_neon_vfma_f16, NEON::BI__builtin_neon_vfma_v, },
7852 { NEON::BI__builtin_neon_vfma_lane_f16, NEON::BI__builtin_neon_vfma_lane_v, },
7853 { NEON::BI__builtin_neon_vfma_laneq_f16, NEON::BI__builtin_neon_vfma_laneq_v, },
7854 { NEON::BI__builtin_neon_vfmaq_f16, NEON::BI__builtin_neon_vfmaq_v, },
7855 { NEON::BI__builtin_neon_vfmaq_lane_f16, NEON::BI__builtin_neon_vfmaq_lane_v, },
7856 { NEON::BI__builtin_neon_vfmaq_laneq_f16, NEON::BI__builtin_neon_vfmaq_laneq_v, },
7857 { NEON::BI__builtin_neon_vld1_bf16_x2, NEON::BI__builtin_neon_vld1_x2_v },
7858 { NEON::BI__builtin_neon_vld1_bf16_x3, NEON::BI__builtin_neon_vld1_x3_v },
7859 { NEON::BI__builtin_neon_vld1_bf16_x4, NEON::BI__builtin_neon_vld1_x4_v },
7860 { NEON::BI__builtin_neon_vld1_bf16, NEON::BI__builtin_neon_vld1_v },
7861 { NEON::BI__builtin_neon_vld1_dup_bf16, NEON::BI__builtin_neon_vld1_dup_v },
7862 { NEON::BI__builtin_neon_vld1_lane_bf16, NEON::BI__builtin_neon_vld1_lane_v },
7863 { NEON::BI__builtin_neon_vld1q_bf16_x2, NEON::BI__builtin_neon_vld1q_x2_v },
7864 { NEON::BI__builtin_neon_vld1q_bf16_x3, NEON::BI__builtin_neon_vld1q_x3_v },
7865 { NEON::BI__builtin_neon_vld1q_bf16_x4, NEON::BI__builtin_neon_vld1q_x4_v },
7866 { NEON::BI__builtin_neon_vld1q_bf16, NEON::BI__builtin_neon_vld1q_v },
7867 { NEON::BI__builtin_neon_vld1q_dup_bf16, NEON::BI__builtin_neon_vld1q_dup_v },
7868 { NEON::BI__builtin_neon_vld1q_lane_bf16, NEON::BI__builtin_neon_vld1q_lane_v },
7869 { NEON::BI__builtin_neon_vld2_bf16, NEON::BI__builtin_neon_vld2_v },
7870 { NEON::BI__builtin_neon_vld2_dup_bf16, NEON::BI__builtin_neon_vld2_dup_v },
7871 { NEON::BI__builtin_neon_vld2_lane_bf16, NEON::BI__builtin_neon_vld2_lane_v },
7872 { NEON::BI__builtin_neon_vld2q_bf16, NEON::BI__builtin_neon_vld2q_v },
7873 { NEON::BI__builtin_neon_vld2q_dup_bf16, NEON::BI__builtin_neon_vld2q_dup_v },
7874 { NEON::BI__builtin_neon_vld2q_lane_bf16, NEON::BI__builtin_neon_vld2q_lane_v },
7875 { NEON::BI__builtin_neon_vld3_bf16, NEON::BI__builtin_neon_vld3_v },
7876 { NEON::BI__builtin_neon_vld3_dup_bf16, NEON::BI__builtin_neon_vld3_dup_v },
7877 { NEON::BI__builtin_neon_vld3_lane_bf16, NEON::BI__builtin_neon_vld3_lane_v },
7878 { NEON::BI__builtin_neon_vld3q_bf16, NEON::BI__builtin_neon_vld3q_v },
7879 { NEON::BI__builtin_neon_vld3q_dup_bf16, NEON::BI__builtin_neon_vld3q_dup_v },
7880 { NEON::BI__builtin_neon_vld3q_lane_bf16, NEON::BI__builtin_neon_vld3q_lane_v },
7881 { NEON::BI__builtin_neon_vld4_bf16, NEON::BI__builtin_neon_vld4_v },
7882 { NEON::BI__builtin_neon_vld4_dup_bf16, NEON::BI__builtin_neon_vld4_dup_v },
7883 { NEON::BI__builtin_neon_vld4_lane_bf16, NEON::BI__builtin_neon_vld4_lane_v },
7884 { NEON::BI__builtin_neon_vld4q_bf16, NEON::BI__builtin_neon_vld4q_v },
7885 { NEON::BI__builtin_neon_vld4q_dup_bf16, NEON::BI__builtin_neon_vld4q_dup_v },
7886 { NEON::BI__builtin_neon_vld4q_lane_bf16, NEON::BI__builtin_neon_vld4q_lane_v },
7887 { NEON::BI__builtin_neon_vmax_f16, NEON::BI__builtin_neon_vmax_v, },
7888 { NEON::BI__builtin_neon_vmaxnm_f16, NEON::BI__builtin_neon_vmaxnm_v, },
7889 { NEON::BI__builtin_neon_vmaxnmq_f16, NEON::BI__builtin_neon_vmaxnmq_v, },
7890 { NEON::BI__builtin_neon_vmaxq_f16, NEON::BI__builtin_neon_vmaxq_v, },
7891 { NEON::BI__builtin_neon_vmin_f16, NEON::BI__builtin_neon_vmin_v, },
7892 { NEON::BI__builtin_neon_vminnm_f16, NEON::BI__builtin_neon_vminnm_v, },
7893 { NEON::BI__builtin_neon_vminnmq_f16, NEON::BI__builtin_neon_vminnmq_v, },
7894 { NEON::BI__builtin_neon_vminq_f16, NEON::BI__builtin_neon_vminq_v, },
7895 { NEON::BI__builtin_neon_vmulx_f16, NEON::BI__builtin_neon_vmulx_v, },
7896 { NEON::BI__builtin_neon_vmulxq_f16, NEON::BI__builtin_neon_vmulxq_v, },
7897 { NEON::BI__builtin_neon_vpadd_f16, NEON::BI__builtin_neon_vpadd_v, },
7898 { NEON::BI__builtin_neon_vpaddq_f16, NEON::BI__builtin_neon_vpaddq_v, },
7899 { NEON::BI__builtin_neon_vpmax_f16, NEON::BI__builtin_neon_vpmax_v, },
7900 { NEON::BI__builtin_neon_vpmaxnm_f16, NEON::BI__builtin_neon_vpmaxnm_v, },
7901 { NEON::BI__builtin_neon_vpmaxnmq_f16, NEON::BI__builtin_neon_vpmaxnmq_v, },
7902 { NEON::BI__builtin_neon_vpmaxq_f16, NEON::BI__builtin_neon_vpmaxq_v, },
7903 { NEON::BI__builtin_neon_vpmin_f16, NEON::BI__builtin_neon_vpmin_v, },
7904 { NEON::BI__builtin_neon_vpminnm_f16, NEON::BI__builtin_neon_vpminnm_v, },
7905 { NEON::BI__builtin_neon_vpminnmq_f16, NEON::BI__builtin_neon_vpminnmq_v, },
7906 { NEON::BI__builtin_neon_vpminq_f16, NEON::BI__builtin_neon_vpminq_v, },
7907 { NEON::BI__builtin_neon_vrecpe_f16, NEON::BI__builtin_neon_vrecpe_v, },
7908 { NEON::BI__builtin_neon_vrecpeq_f16, NEON::BI__builtin_neon_vrecpeq_v, },
7909 { NEON::BI__builtin_neon_vrecps_f16, NEON::BI__builtin_neon_vrecps_v, },
7910 { NEON::BI__builtin_neon_vrecpsq_f16, NEON::BI__builtin_neon_vrecpsq_v, },
7911 { NEON::BI__builtin_neon_vrnd_f16, NEON::BI__builtin_neon_vrnd_v, },
7912 { NEON::BI__builtin_neon_vrnda_f16, NEON::BI__builtin_neon_vrnda_v, },
7913 { NEON::BI__builtin_neon_vrndaq_f16, NEON::BI__builtin_neon_vrndaq_v, },
7914 { NEON::BI__builtin_neon_vrndi_f16, NEON::BI__builtin_neon_vrndi_v, },
7915 { NEON::BI__builtin_neon_vrndiq_f16, NEON::BI__builtin_neon_vrndiq_v, },
7916 { NEON::BI__builtin_neon_vrndm_f16, NEON::BI__builtin_neon_vrndm_v, },
7917 { NEON::BI__builtin_neon_vrndmq_f16, NEON::BI__builtin_neon_vrndmq_v, },
7918 { NEON::BI__builtin_neon_vrndn_f16, NEON::BI__builtin_neon_vrndn_v, },
7919 { NEON::BI__builtin_neon_vrndnq_f16, NEON::BI__builtin_neon_vrndnq_v, },
7920 { NEON::BI__builtin_neon_vrndp_f16, NEON::BI__builtin_neon_vrndp_v, },
7921 { NEON::BI__builtin_neon_vrndpq_f16, NEON::BI__builtin_neon_vrndpq_v, },
7922 { NEON::BI__builtin_neon_vrndq_f16, NEON::BI__builtin_neon_vrndq_v, },
7923 { NEON::BI__builtin_neon_vrndx_f16, NEON::BI__builtin_neon_vrndx_v, },
7924 { NEON::BI__builtin_neon_vrndxq_f16, NEON::BI__builtin_neon_vrndxq_v, },
7925 { NEON::BI__builtin_neon_vrsqrte_f16, NEON::BI__builtin_neon_vrsqrte_v, },
7926 { NEON::BI__builtin_neon_vrsqrteq_f16, NEON::BI__builtin_neon_vrsqrteq_v, },
7927 { NEON::BI__builtin_neon_vrsqrts_f16, NEON::BI__builtin_neon_vrsqrts_v, },
7928 { NEON::BI__builtin_neon_vrsqrtsq_f16, NEON::BI__builtin_neon_vrsqrtsq_v, },
7929 { NEON::BI__builtin_neon_vsqrt_f16, NEON::BI__builtin_neon_vsqrt_v, },
7930 { NEON::BI__builtin_neon_vsqrtq_f16, NEON::BI__builtin_neon_vsqrtq_v, },
7931 { NEON::BI__builtin_neon_vst1_bf16_x2, NEON::BI__builtin_neon_vst1_x2_v },
7932 { NEON::BI__builtin_neon_vst1_bf16_x3, NEON::BI__builtin_neon_vst1_x3_v },
7933 { NEON::BI__builtin_neon_vst1_bf16_x4, NEON::BI__builtin_neon_vst1_x4_v },
7934 { NEON::BI__builtin_neon_vst1_bf16, NEON::BI__builtin_neon_vst1_v },
7935 { NEON::BI__builtin_neon_vst1_lane_bf16, NEON::BI__builtin_neon_vst1_lane_v },
7936 { NEON::BI__builtin_neon_vst1q_bf16_x2, NEON::BI__builtin_neon_vst1q_x2_v },
7937 { NEON::BI__builtin_neon_vst1q_bf16_x3, NEON::BI__builtin_neon_vst1q_x3_v },
7938 { NEON::BI__builtin_neon_vst1q_bf16_x4, NEON::BI__builtin_neon_vst1q_x4_v },
7939 { NEON::BI__builtin_neon_vst1q_bf16, NEON::BI__builtin_neon_vst1q_v },
7940 { NEON::BI__builtin_neon_vst1q_lane_bf16, NEON::BI__builtin_neon_vst1q_lane_v },
7941 { NEON::BI__builtin_neon_vst2_bf16, NEON::BI__builtin_neon_vst2_v },
7942 { NEON::BI__builtin_neon_vst2_lane_bf16, NEON::BI__builtin_neon_vst2_lane_v },
7943 { NEON::BI__builtin_neon_vst2q_bf16, NEON::BI__builtin_neon_vst2q_v },
7944 { NEON::BI__builtin_neon_vst2q_lane_bf16, NEON::BI__builtin_neon_vst2q_lane_v },
7945 { NEON::BI__builtin_neon_vst3_bf16, NEON::BI__builtin_neon_vst3_v },
7946 { NEON::BI__builtin_neon_vst3_lane_bf16, NEON::BI__builtin_neon_vst3_lane_v },
7947 { NEON::BI__builtin_neon_vst3q_bf16, NEON::BI__builtin_neon_vst3q_v },
7948 { NEON::BI__builtin_neon_vst3q_lane_bf16, NEON::BI__builtin_neon_vst3q_lane_v },
7949 { NEON::BI__builtin_neon_vst4_bf16, NEON::BI__builtin_neon_vst4_v },
7950 { NEON::BI__builtin_neon_vst4_lane_bf16, NEON::BI__builtin_neon_vst4_lane_v },
7951 { NEON::BI__builtin_neon_vst4q_bf16, NEON::BI__builtin_neon_vst4q_v },
7952 { NEON::BI__builtin_neon_vst4q_lane_bf16, NEON::BI__builtin_neon_vst4q_lane_v },
7956 { NEON::BI__builtin_neon_vldap1_lane_u64, NEON::BI__builtin_neon_vldap1_lane_s64 },
7957 { NEON::BI__builtin_neon_vldap1_lane_f64, NEON::BI__builtin_neon_vldap1_lane_s64 },
7958 { NEON::BI__builtin_neon_vldap1_lane_p64, NEON::BI__builtin_neon_vldap1_lane_s64 },
7959 { NEON::BI__builtin_neon_vldap1q_lane_u64, NEON::BI__builtin_neon_vldap1q_lane_s64 },
7960 { NEON::BI__builtin_neon_vldap1q_lane_f64, NEON::BI__builtin_neon_vldap1q_lane_s64 },
7961 { NEON::BI__builtin_neon_vldap1q_lane_p64, NEON::BI__builtin_neon_vldap1q_lane_s64 },
7962 { NEON::BI__builtin_neon_vstl1_lane_u64, NEON::BI__builtin_neon_vstl1_lane_s64 },
7963 { NEON::BI__builtin_neon_vstl1_lane_f64, NEON::BI__builtin_neon_vstl1_lane_s64 },
7964 { NEON::BI__builtin_neon_vstl1_lane_p64, NEON::BI__builtin_neon_vstl1_lane_s64 },
7965 { NEON::BI__builtin_neon_vstl1q_lane_u64, NEON::BI__builtin_neon_vstl1q_lane_s64 },
7966 { NEON::BI__builtin_neon_vstl1q_lane_f64, NEON::BI__builtin_neon_vstl1q_lane_s64 },
7967 { NEON::BI__builtin_neon_vstl1q_lane_p64, NEON::BI__builtin_neon_vstl1q_lane_s64 },
7974#define SVEMAP1(NameBase, LLVMIntrinsic, TypeModifier) \
7976 #NameBase, SVE::BI__builtin_sve_##NameBase, Intrinsic::LLVMIntrinsic, 0, \
7980#define SVEMAP2(NameBase, TypeModifier) \
7981 { #NameBase, SVE::BI__builtin_sve_##NameBase, 0, 0, TypeModifier }
7983#define GET_SVE_LLVM_INTRINSIC_MAP
7984#include "clang/Basic/arm_sve_builtin_cg.inc"
7985#include "clang/Basic/BuiltinsAArch64NeonSVEBridge_cg.def"
7986#undef GET_SVE_LLVM_INTRINSIC_MAP
7992#define SMEMAP1(NameBase, LLVMIntrinsic, TypeModifier) \
7994 #NameBase, SME::BI__builtin_sme_##NameBase, Intrinsic::LLVMIntrinsic, 0, \
7998#define SMEMAP2(NameBase, TypeModifier) \
7999 { #NameBase, SME::BI__builtin_sme_##NameBase, 0, 0, TypeModifier }
8001#define GET_SME_LLVM_INTRINSIC_MAP
8002#include "clang/Basic/arm_sme_builtin_cg.inc"
8003#undef GET_SME_LLVM_INTRINSIC_MAP
8016static const ARMVectorIntrinsicInfo *
8018 unsigned BuiltinID,
bool &MapProvenSorted) {
8021 if (!MapProvenSorted) {
8022 assert(llvm::is_sorted(IntrinsicMap));
8023 MapProvenSorted =
true;
8027 const ARMVectorIntrinsicInfo *Builtin =
8028 llvm::lower_bound(IntrinsicMap, BuiltinID);
8030 if (Builtin != IntrinsicMap.end() && Builtin->BuiltinID == BuiltinID)
8038 llvm::Type *ArgType,
8051 Ty = llvm::FixedVectorType::get(
8052 Ty, VectorSize ? VectorSize / Ty->getPrimitiveSizeInBits() : 1);
8059 int Elts = VectorSize ? VectorSize / ArgType->getPrimitiveSizeInBits() : 1;
8060 ArgType = llvm::FixedVectorType::get(ArgType, Elts);
8064 Tys.push_back(ArgType);
8067 Tys.push_back(ArgType);
8078 unsigned BuiltinID = SISDInfo.BuiltinID;
8079 unsigned int Int = SISDInfo.LLVMIntrinsic;
8080 unsigned Modifier = SISDInfo.TypeModifier;
8081 const char *
s = SISDInfo.NameHint;
8083 switch (BuiltinID) {
8084 case NEON::BI__builtin_neon_vcled_s64:
8085 case NEON::BI__builtin_neon_vcled_u64:
8086 case NEON::BI__builtin_neon_vcles_f32:
8087 case NEON::BI__builtin_neon_vcled_f64:
8088 case NEON::BI__builtin_neon_vcltd_s64:
8089 case NEON::BI__builtin_neon_vcltd_u64:
8090 case NEON::BI__builtin_neon_vclts_f32:
8091 case NEON::BI__builtin_neon_vcltd_f64:
8092 case NEON::BI__builtin_neon_vcales_f32:
8093 case NEON::BI__builtin_neon_vcaled_f64:
8094 case NEON::BI__builtin_neon_vcalts_f32:
8095 case NEON::BI__builtin_neon_vcaltd_f64:
8099 std::swap(Ops[0], Ops[1]);
8103 assert(Int &&
"Generic code assumes a valid intrinsic");
8106 const Expr *Arg =
E->getArg(0);
8111 ConstantInt *C0 = ConstantInt::get(CGF.
SizeTy, 0);
8112 for (Function::const_arg_iterator ai = F->arg_begin(), ae = F->arg_end();
8113 ai != ae; ++ai, ++j) {
8114 llvm::Type *ArgTy = ai->getType();
8115 if (Ops[j]->getType()->getPrimitiveSizeInBits() ==
8116 ArgTy->getPrimitiveSizeInBits())
8119 assert(ArgTy->isVectorTy() && !Ops[j]->getType()->isVectorTy());
8122 Ops[j] = CGF.
Builder.CreateTruncOrBitCast(
8123 Ops[j], cast<llvm::VectorType>(ArgTy)->getElementType());
8125 CGF.
Builder.CreateInsertElement(PoisonValue::get(ArgTy), Ops[j], C0);
8130 if (ResultType->getPrimitiveSizeInBits().getFixedValue() <
8131 Result->getType()->getPrimitiveSizeInBits().getFixedValue())
8138 unsigned BuiltinID,
unsigned LLVMIntrinsic,
unsigned AltLLVMIntrinsic,
8139 const char *NameHint,
unsigned Modifier,
const CallExpr *
E,
8141 llvm::Triple::ArchType Arch) {
8143 const Expr *Arg =
E->getArg(
E->getNumArgs() - 1);
8144 std::optional<llvm::APSInt> NeonTypeConst =
8151 bool Usgn =
Type.isUnsigned();
8152 bool Quad =
Type.isQuad();
8154 const bool AllowBFloatArgsAndRet =
8157 llvm::FixedVectorType *VTy =
8158 GetNeonType(
this,
Type, HasLegalHalfType,
false, AllowBFloatArgsAndRet);
8159 llvm::Type *Ty = VTy;
8163 auto getAlignmentValue32 = [&](
Address addr) ->
Value* {
8164 return Builder.getInt32(addr.getAlignment().getQuantity());
8167 unsigned Int = LLVMIntrinsic;
8169 Int = AltLLVMIntrinsic;
8171 switch (BuiltinID) {
8173 case NEON::BI__builtin_neon_splat_lane_v:
8174 case NEON::BI__builtin_neon_splat_laneq_v:
8175 case NEON::BI__builtin_neon_splatq_lane_v:
8176 case NEON::BI__builtin_neon_splatq_laneq_v: {
8177 auto NumElements = VTy->getElementCount();
8178 if (BuiltinID == NEON::BI__builtin_neon_splatq_lane_v)
8179 NumElements = NumElements * 2;
8180 if (BuiltinID == NEON::BI__builtin_neon_splat_laneq_v)
8181 NumElements = NumElements.divideCoefficientBy(2);
8183 Ops[0] =
Builder.CreateBitCast(Ops[0], VTy);
8184 return EmitNeonSplat(Ops[0], cast<ConstantInt>(Ops[1]), NumElements);
8186 case NEON::BI__builtin_neon_vpadd_v:
8187 case NEON::BI__builtin_neon_vpaddq_v:
8189 if (VTy->getElementType()->isFloatingPointTy() &&
8190 Int == Intrinsic::aarch64_neon_addp)
8191 Int = Intrinsic::aarch64_neon_faddp;
8193 case NEON::BI__builtin_neon_vabs_v:
8194 case NEON::BI__builtin_neon_vabsq_v:
8195 if (VTy->getElementType()->isFloatingPointTy())
8198 case NEON::BI__builtin_neon_vadd_v:
8199 case NEON::BI__builtin_neon_vaddq_v: {
8200 llvm::Type *VTy = llvm::FixedVectorType::get(
Int8Ty, Quad ? 16 : 8);
8201 Ops[0] =
Builder.CreateBitCast(Ops[0], VTy);
8202 Ops[1] =
Builder.CreateBitCast(Ops[1], VTy);
8203 Ops[0] =
Builder.CreateXor(Ops[0], Ops[1]);
8204 return Builder.CreateBitCast(Ops[0], Ty);
8206 case NEON::BI__builtin_neon_vaddhn_v: {
8207 llvm::FixedVectorType *SrcTy =
8208 llvm::FixedVectorType::getExtendedElementVectorType(VTy);
8211 Ops[0] =
Builder.CreateBitCast(Ops[0], SrcTy);
8212 Ops[1] =
Builder.CreateBitCast(Ops[1], SrcTy);
8213 Ops[0] =
Builder.CreateAdd(Ops[0], Ops[1],
"vaddhn");
8216 Constant *ShiftAmt =
8217 ConstantInt::get(SrcTy, SrcTy->getScalarSizeInBits() / 2);
8218 Ops[0] =
Builder.CreateLShr(Ops[0], ShiftAmt,
"vaddhn");
8221 return Builder.CreateTrunc(Ops[0], VTy,
"vaddhn");
8223 case NEON::BI__builtin_neon_vcale_v:
8224 case NEON::BI__builtin_neon_vcaleq_v:
8225 case NEON::BI__builtin_neon_vcalt_v:
8226 case NEON::BI__builtin_neon_vcaltq_v:
8227 std::swap(Ops[0], Ops[1]);
8229 case NEON::BI__builtin_neon_vcage_v:
8230 case NEON::BI__builtin_neon_vcageq_v:
8231 case NEON::BI__builtin_neon_vcagt_v:
8232 case NEON::BI__builtin_neon_vcagtq_v: {
8234 switch (VTy->getScalarSizeInBits()) {
8235 default: llvm_unreachable(
"unexpected type");
8246 auto *VecFlt = llvm::FixedVectorType::get(Ty, VTy->getNumElements());
8247 llvm::Type *Tys[] = { VTy, VecFlt };
8251 case NEON::BI__builtin_neon_vceqz_v:
8252 case NEON::BI__builtin_neon_vceqzq_v:
8254 ICmpInst::ICMP_EQ,
"vceqz");
8255 case NEON::BI__builtin_neon_vcgez_v:
8256 case NEON::BI__builtin_neon_vcgezq_v:
8258 ICmpInst::ICMP_SGE,
"vcgez");
8259 case NEON::BI__builtin_neon_vclez_v:
8260 case NEON::BI__builtin_neon_vclezq_v:
8262 ICmpInst::ICMP_SLE,
"vclez");
8263 case NEON::BI__builtin_neon_vcgtz_v:
8264 case NEON::BI__builtin_neon_vcgtzq_v:
8266 ICmpInst::ICMP_SGT,
"vcgtz");
8267 case NEON::BI__builtin_neon_vcltz_v:
8268 case NEON::BI__builtin_neon_vcltzq_v:
8270 ICmpInst::ICMP_SLT,
"vcltz");
8271 case NEON::BI__builtin_neon_vclz_v:
8272 case NEON::BI__builtin_neon_vclzq_v:
8277 case NEON::BI__builtin_neon_vcvt_f32_v:
8278 case NEON::BI__builtin_neon_vcvtq_f32_v:
8279 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
8282 return Usgn ?
Builder.CreateUIToFP(Ops[0], Ty,
"vcvt")
8283 :
Builder.CreateSIToFP(Ops[0], Ty,
"vcvt");
8284 case NEON::BI__builtin_neon_vcvt_f16_s16:
8285 case NEON::BI__builtin_neon_vcvt_f16_u16:
8286 case NEON::BI__builtin_neon_vcvtq_f16_s16:
8287 case NEON::BI__builtin_neon_vcvtq_f16_u16:
8288 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
8291 return Usgn ?
Builder.CreateUIToFP(Ops[0], Ty,
"vcvt")
8292 :
Builder.CreateSIToFP(Ops[0], Ty,
"vcvt");
8293 case NEON::BI__builtin_neon_vcvt_n_f16_s16:
8294 case NEON::BI__builtin_neon_vcvt_n_f16_u16:
8295 case NEON::BI__builtin_neon_vcvtq_n_f16_s16:
8296 case NEON::BI__builtin_neon_vcvtq_n_f16_u16: {
8301 case NEON::BI__builtin_neon_vcvt_n_f32_v:
8302 case NEON::BI__builtin_neon_vcvt_n_f64_v:
8303 case NEON::BI__builtin_neon_vcvtq_n_f32_v:
8304 case NEON::BI__builtin_neon_vcvtq_n_f64_v: {
8306 Int = Usgn ? LLVMIntrinsic : AltLLVMIntrinsic;
8310 case NEON::BI__builtin_neon_vcvt_n_s16_f16:
8311 case NEON::BI__builtin_neon_vcvt_n_s32_v:
8312 case NEON::BI__builtin_neon_vcvt_n_u16_f16:
8313 case NEON::BI__builtin_neon_vcvt_n_u32_v:
8314 case NEON::BI__builtin_neon_vcvt_n_s64_v:
8315 case NEON::BI__builtin_neon_vcvt_n_u64_v:
8316 case NEON::BI__builtin_neon_vcvtq_n_s16_f16:
8317 case NEON::BI__builtin_neon_vcvtq_n_s32_v:
8318 case NEON::BI__builtin_neon_vcvtq_n_u16_f16:
8319 case NEON::BI__builtin_neon_vcvtq_n_u32_v:
8320 case NEON::BI__builtin_neon_vcvtq_n_s64_v:
8321 case NEON::BI__builtin_neon_vcvtq_n_u64_v: {
8326 case NEON::BI__builtin_neon_vcvt_s32_v:
8327 case NEON::BI__builtin_neon_vcvt_u32_v:
8328 case NEON::BI__builtin_neon_vcvt_s64_v:
8329 case NEON::BI__builtin_neon_vcvt_u64_v:
8330 case NEON::BI__builtin_neon_vcvt_s16_f16:
8331 case NEON::BI__builtin_neon_vcvt_u16_f16:
8332 case NEON::BI__builtin_neon_vcvtq_s32_v:
8333 case NEON::BI__builtin_neon_vcvtq_u32_v:
8334 case NEON::BI__builtin_neon_vcvtq_s64_v:
8335 case NEON::BI__builtin_neon_vcvtq_u64_v:
8336 case NEON::BI__builtin_neon_vcvtq_s16_f16:
8337 case NEON::BI__builtin_neon_vcvtq_u16_f16: {
8339 return Usgn ?
Builder.CreateFPToUI(Ops[0], Ty,
"vcvt")
8340 :
Builder.CreateFPToSI(Ops[0], Ty,
"vcvt");
8342 case NEON::BI__builtin_neon_vcvta_s16_f16:
8343 case NEON::BI__builtin_neon_vcvta_s32_v:
8344 case NEON::BI__builtin_neon_vcvta_s64_v:
8345 case NEON::BI__builtin_neon_vcvta_u16_f16:
8346 case NEON::BI__builtin_neon_vcvta_u32_v:
8347 case NEON::BI__builtin_neon_vcvta_u64_v:
8348 case NEON::BI__builtin_neon_vcvtaq_s16_f16:
8349 case NEON::BI__builtin_neon_vcvtaq_s32_v:
8350 case NEON::BI__builtin_neon_vcvtaq_s64_v:
8351 case NEON::BI__builtin_neon_vcvtaq_u16_f16:
8352 case NEON::BI__builtin_neon_vcvtaq_u32_v:
8353 case NEON::BI__builtin_neon_vcvtaq_u64_v:
8354 case NEON::BI__builtin_neon_vcvtn_s16_f16:
8355 case NEON::BI__builtin_neon_vcvtn_s32_v:
8356 case NEON::BI__builtin_neon_vcvtn_s64_v:
8357 case NEON::BI__builtin_neon_vcvtn_u16_f16:
8358 case NEON::BI__builtin_neon_vcvtn_u32_v:
8359 case NEON::BI__builtin_neon_vcvtn_u64_v:
8360 case NEON::BI__builtin_neon_vcvtnq_s16_f16:
8361 case NEON::BI__builtin_neon_vcvtnq_s32_v:
8362 case NEON::BI__builtin_neon_vcvtnq_s64_v:
8363 case NEON::BI__builtin_neon_vcvtnq_u16_f16:
8364 case NEON::BI__builtin_neon_vcvtnq_u32_v:
8365 case NEON::BI__builtin_neon_vcvtnq_u64_v:
8366 case NEON::BI__builtin_neon_vcvtp_s16_f16:
8367 case NEON::BI__builtin_neon_vcvtp_s32_v:
8368 case NEON::BI__builtin_neon_vcvtp_s64_v:
8369 case NEON::BI__builtin_neon_vcvtp_u16_f16:
8370 case NEON::BI__builtin_neon_vcvtp_u32_v:
8371 case NEON::BI__builtin_neon_vcvtp_u64_v:
8372 case NEON::BI__builtin_neon_vcvtpq_s16_f16:
8373 case NEON::BI__builtin_neon_vcvtpq_s32_v:
8374 case NEON::BI__builtin_neon_vcvtpq_s64_v:
8375 case NEON::BI__builtin_neon_vcvtpq_u16_f16:
8376 case NEON::BI__builtin_neon_vcvtpq_u32_v:
8377 case NEON::BI__builtin_neon_vcvtpq_u64_v:
8378 case NEON::BI__builtin_neon_vcvtm_s16_f16:
8379 case NEON::BI__builtin_neon_vcvtm_s32_v:
8380 case NEON::BI__builtin_neon_vcvtm_s64_v:
8381 case NEON::BI__builtin_neon_vcvtm_u16_f16:
8382 case NEON::BI__builtin_neon_vcvtm_u32_v:
8383 case NEON::BI__builtin_neon_vcvtm_u64_v:
8384 case NEON::BI__builtin_neon_vcvtmq_s16_f16:
8385 case NEON::BI__builtin_neon_vcvtmq_s32_v:
8386 case NEON::BI__builtin_neon_vcvtmq_s64_v:
8387 case NEON::BI__builtin_neon_vcvtmq_u16_f16:
8388 case NEON::BI__builtin_neon_vcvtmq_u32_v:
8389 case NEON::BI__builtin_neon_vcvtmq_u64_v: {
8393 case NEON::BI__builtin_neon_vcvtx_f32_v: {
8394 llvm::Type *Tys[2] = { VTy->getTruncatedElementVectorType(VTy), Ty};
8398 case NEON::BI__builtin_neon_vext_v:
8399 case NEON::BI__builtin_neon_vextq_v: {
8400 int CV = cast<ConstantInt>(Ops[2])->getSExtValue();
8402 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
8403 Indices.push_back(i+CV);
8405 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
8406 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
8407 return Builder.CreateShuffleVector(Ops[0], Ops[1], Indices,
"vext");
8409 case NEON::BI__builtin_neon_vfma_v:
8410 case NEON::BI__builtin_neon_vfmaq_v: {
8411 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
8412 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
8413 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
8417 *
this, Intrinsic::fma, Intrinsic::experimental_constrained_fma, Ty,
8418 {Ops[1], Ops[2], Ops[0]});
8420 case NEON::BI__builtin_neon_vld1_v:
8421 case NEON::BI__builtin_neon_vld1q_v: {
8423 Ops.push_back(getAlignmentValue32(PtrOp0));
8426 case NEON::BI__builtin_neon_vld1_x2_v:
8427 case NEON::BI__builtin_neon_vld1q_x2_v:
8428 case NEON::BI__builtin_neon_vld1_x3_v:
8429 case NEON::BI__builtin_neon_vld1q_x3_v:
8430 case NEON::BI__builtin_neon_vld1_x4_v:
8431 case NEON::BI__builtin_neon_vld1q_x4_v: {
8434 Ops[1] =
Builder.CreateCall(F, Ops[1],
"vld1xN");
8437 case NEON::BI__builtin_neon_vld2_v:
8438 case NEON::BI__builtin_neon_vld2q_v:
8439 case NEON::BI__builtin_neon_vld3_v:
8440 case NEON::BI__builtin_neon_vld3q_v:
8441 case NEON::BI__builtin_neon_vld4_v:
8442 case NEON::BI__builtin_neon_vld4q_v:
8443 case NEON::BI__builtin_neon_vld2_dup_v:
8444 case NEON::BI__builtin_neon_vld2q_dup_v:
8445 case NEON::BI__builtin_neon_vld3_dup_v:
8446 case NEON::BI__builtin_neon_vld3q_dup_v:
8447 case NEON::BI__builtin_neon_vld4_dup_v:
8448 case NEON::BI__builtin_neon_vld4q_dup_v: {
8451 Value *Align = getAlignmentValue32(PtrOp1);
8452 Ops[1] =
Builder.CreateCall(F, {Ops[1], Align}, NameHint);
8455 case NEON::BI__builtin_neon_vld1_dup_v:
8456 case NEON::BI__builtin_neon_vld1q_dup_v: {
8457 Value *
V = PoisonValue::get(Ty);
8460 llvm::Constant *CI = ConstantInt::get(
SizeTy, 0);
8461 Ops[0] =
Builder.CreateInsertElement(
V, Ld, CI);
8464 case NEON::BI__builtin_neon_vld2_lane_v:
8465 case NEON::BI__builtin_neon_vld2q_lane_v:
8466 case NEON::BI__builtin_neon_vld3_lane_v:
8467 case NEON::BI__builtin_neon_vld3q_lane_v:
8468 case NEON::BI__builtin_neon_vld4_lane_v:
8469 case NEON::BI__builtin_neon_vld4q_lane_v: {
8472 for (
unsigned I = 2; I < Ops.size() - 1; ++I)
8473 Ops[I] =
Builder.CreateBitCast(Ops[I], Ty);
8474 Ops.push_back(getAlignmentValue32(PtrOp1));
8478 case NEON::BI__builtin_neon_vmovl_v: {
8479 llvm::FixedVectorType *DTy =
8480 llvm::FixedVectorType::getTruncatedElementVectorType(VTy);
8481 Ops[0] =
Builder.CreateBitCast(Ops[0], DTy);
8483 return Builder.CreateZExt(Ops[0], Ty,
"vmovl");
8484 return Builder.CreateSExt(Ops[0], Ty,
"vmovl");
8486 case NEON::BI__builtin_neon_vmovn_v: {
8487 llvm::FixedVectorType *QTy =
8488 llvm::FixedVectorType::getExtendedElementVectorType(VTy);
8489 Ops[0] =
Builder.CreateBitCast(Ops[0], QTy);
8490 return Builder.CreateTrunc(Ops[0], Ty,
"vmovn");
8492 case NEON::BI__builtin_neon_vmull_v:
8498 Int = Usgn ? Intrinsic::arm_neon_vmullu : Intrinsic::arm_neon_vmulls;
8501 case NEON::BI__builtin_neon_vpadal_v:
8502 case NEON::BI__builtin_neon_vpadalq_v: {
8504 unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits();
8508 llvm::FixedVectorType::get(EltTy, VTy->getNumElements() * 2);
8509 llvm::Type *Tys[2] = { Ty, NarrowTy };
8512 case NEON::BI__builtin_neon_vpaddl_v:
8513 case NEON::BI__builtin_neon_vpaddlq_v: {
8515 unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits();
8516 llvm::Type *EltTy = llvm::IntegerType::get(
getLLVMContext(), EltBits / 2);
8518 llvm::FixedVectorType::get(EltTy, VTy->getNumElements() * 2);
8519 llvm::Type *Tys[2] = { Ty, NarrowTy };
8522 case NEON::BI__builtin_neon_vqdmlal_v:
8523 case NEON::BI__builtin_neon_vqdmlsl_v: {
8530 case NEON::BI__builtin_neon_vqdmulhq_lane_v:
8531 case NEON::BI__builtin_neon_vqdmulh_lane_v:
8532 case NEON::BI__builtin_neon_vqrdmulhq_lane_v:
8533 case NEON::BI__builtin_neon_vqrdmulh_lane_v: {
8534 auto *RTy = cast<llvm::FixedVectorType>(Ty);
8535 if (BuiltinID == NEON::BI__builtin_neon_vqdmulhq_lane_v ||
8536 BuiltinID == NEON::BI__builtin_neon_vqrdmulhq_lane_v)
8537 RTy = llvm::FixedVectorType::get(RTy->getElementType(),
8538 RTy->getNumElements() * 2);
8539 llvm::Type *Tys[2] = {
8544 case NEON::BI__builtin_neon_vqdmulhq_laneq_v:
8545 case NEON::BI__builtin_neon_vqdmulh_laneq_v:
8546 case NEON::BI__builtin_neon_vqrdmulhq_laneq_v:
8547 case NEON::BI__builtin_neon_vqrdmulh_laneq_v: {
8548 llvm::Type *Tys[2] = {
8553 case NEON::BI__builtin_neon_vqshl_n_v:
8554 case NEON::BI__builtin_neon_vqshlq_n_v:
8557 case NEON::BI__builtin_neon_vqshlu_n_v:
8558 case NEON::BI__builtin_neon_vqshluq_n_v:
8561 case NEON::BI__builtin_neon_vrecpe_v:
8562 case NEON::BI__builtin_neon_vrecpeq_v:
8563 case NEON::BI__builtin_neon_vrsqrte_v:
8564 case NEON::BI__builtin_neon_vrsqrteq_v:
8565 Int = Ty->isFPOrFPVectorTy() ? LLVMIntrinsic : AltLLVMIntrinsic;
8567 case NEON::BI__builtin_neon_vrndi_v:
8568 case NEON::BI__builtin_neon_vrndiq_v:
8570 ? Intrinsic::experimental_constrained_nearbyint
8571 : Intrinsic::nearbyint;
8573 case NEON::BI__builtin_neon_vrshr_n_v:
8574 case NEON::BI__builtin_neon_vrshrq_n_v:
8577 case NEON::BI__builtin_neon_vsha512hq_u64:
8578 case NEON::BI__builtin_neon_vsha512h2q_u64:
8579 case NEON::BI__builtin_neon_vsha512su0q_u64:
8580 case NEON::BI__builtin_neon_vsha512su1q_u64: {
8584 case NEON::BI__builtin_neon_vshl_n_v:
8585 case NEON::BI__builtin_neon_vshlq_n_v:
8587 return Builder.CreateShl(
Builder.CreateBitCast(Ops[0],Ty), Ops[1],
8589 case NEON::BI__builtin_neon_vshll_n_v: {
8590 llvm::FixedVectorType *SrcTy =
8591 llvm::FixedVectorType::getTruncatedElementVectorType(VTy);
8592 Ops[0] =
Builder.CreateBitCast(Ops[0], SrcTy);
8594 Ops[0] =
Builder.CreateZExt(Ops[0], VTy);
8596 Ops[0] =
Builder.CreateSExt(Ops[0], VTy);
8598 return Builder.CreateShl(Ops[0], Ops[1],
"vshll_n");
8600 case NEON::BI__builtin_neon_vshrn_n_v: {
8601 llvm::FixedVectorType *SrcTy =
8602 llvm::FixedVectorType::getExtendedElementVectorType(VTy);
8603 Ops[0] =
Builder.CreateBitCast(Ops[0], SrcTy);
8606 Ops[0] =
Builder.CreateLShr(Ops[0], Ops[1]);
8608 Ops[0] =
Builder.CreateAShr(Ops[0], Ops[1]);
8609 return Builder.CreateTrunc(Ops[0], Ty,
"vshrn_n");
8611 case NEON::BI__builtin_neon_vshr_n_v:
8612 case NEON::BI__builtin_neon_vshrq_n_v:
8614 case NEON::BI__builtin_neon_vst1_v:
8615 case NEON::BI__builtin_neon_vst1q_v:
8616 case NEON::BI__builtin_neon_vst2_v:
8617 case NEON::BI__builtin_neon_vst2q_v:
8618 case NEON::BI__builtin_neon_vst3_v:
8619 case NEON::BI__builtin_neon_vst3q_v:
8620 case NEON::BI__builtin_neon_vst4_v:
8621 case NEON::BI__builtin_neon_vst4q_v:
8622 case NEON::BI__builtin_neon_vst2_lane_v:
8623 case NEON::BI__builtin_neon_vst2q_lane_v:
8624 case NEON::BI__builtin_neon_vst3_lane_v:
8625 case NEON::BI__builtin_neon_vst3q_lane_v:
8626 case NEON::BI__builtin_neon_vst4_lane_v:
8627 case NEON::BI__builtin_neon_vst4q_lane_v: {
8629 Ops.push_back(getAlignmentValue32(PtrOp0));
8632 case NEON::BI__builtin_neon_vsm3partw1q_u32:
8633 case NEON::BI__builtin_neon_vsm3partw2q_u32:
8634 case NEON::BI__builtin_neon_vsm3ss1q_u32:
8635 case NEON::BI__builtin_neon_vsm4ekeyq_u32:
8636 case NEON::BI__builtin_neon_vsm4eq_u32: {
8640 case NEON::BI__builtin_neon_vsm3tt1aq_u32:
8641 case NEON::BI__builtin_neon_vsm3tt1bq_u32:
8642 case NEON::BI__builtin_neon_vsm3tt2aq_u32:
8643 case NEON::BI__builtin_neon_vsm3tt2bq_u32: {
8648 case NEON::BI__builtin_neon_vst1_x2_v:
8649 case NEON::BI__builtin_neon_vst1q_x2_v:
8650 case NEON::BI__builtin_neon_vst1_x3_v:
8651 case NEON::BI__builtin_neon_vst1q_x3_v:
8652 case NEON::BI__builtin_neon_vst1_x4_v:
8653 case NEON::BI__builtin_neon_vst1q_x4_v: {
8656 if (Arch == llvm::Triple::aarch64 || Arch == llvm::Triple::aarch64_be ||
8657 Arch == llvm::Triple::aarch64_32) {
8659 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
8665 case NEON::BI__builtin_neon_vsubhn_v: {
8666 llvm::FixedVectorType *SrcTy =
8667 llvm::FixedVectorType::getExtendedElementVectorType(VTy);
8670 Ops[0] =
Builder.CreateBitCast(Ops[0], SrcTy);
8671 Ops[1] =
Builder.CreateBitCast(Ops[1], SrcTy);
8672 Ops[0] =
Builder.CreateSub(Ops[0], Ops[1],
"vsubhn");
8675 Constant *ShiftAmt =
8676 ConstantInt::get(SrcTy, SrcTy->getScalarSizeInBits() / 2);
8677 Ops[0] =
Builder.CreateLShr(Ops[0], ShiftAmt,
"vsubhn");
8680 return Builder.CreateTrunc(Ops[0], VTy,
"vsubhn");
8682 case NEON::BI__builtin_neon_vtrn_v:
8683 case NEON::BI__builtin_neon_vtrnq_v: {
8684 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
8685 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
8686 Value *SV =
nullptr;
8688 for (
unsigned vi = 0; vi != 2; ++vi) {
8690 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
8691 Indices.push_back(i+vi);
8692 Indices.push_back(i+e+vi);
8694 Value *Addr =
Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
8695 SV =
Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vtrn");
8700 case NEON::BI__builtin_neon_vtst_v:
8701 case NEON::BI__builtin_neon_vtstq_v: {
8702 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
8703 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
8704 Ops[0] =
Builder.CreateAnd(Ops[0], Ops[1]);
8705 Ops[0] =
Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0],
8706 ConstantAggregateZero::get(Ty));
8707 return Builder.CreateSExt(Ops[0], Ty,
"vtst");
8709 case NEON::BI__builtin_neon_vuzp_v:
8710 case NEON::BI__builtin_neon_vuzpq_v: {
8711 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
8712 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
8713 Value *SV =
nullptr;
8715 for (
unsigned vi = 0; vi != 2; ++vi) {
8717 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
8718 Indices.push_back(2*i+vi);
8720 Value *Addr =
Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
8721 SV =
Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vuzp");
8726 case NEON::BI__builtin_neon_vxarq_u64: {
8731 case NEON::BI__builtin_neon_vzip_v:
8732 case NEON::BI__builtin_neon_vzipq_v: {
8733 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
8734 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
8735 Value *SV =
nullptr;
8737 for (
unsigned vi = 0; vi != 2; ++vi) {
8739 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
8740 Indices.push_back((i + vi*e) >> 1);
8741 Indices.push_back(((i + vi*e) >> 1)+e);
8743 Value *Addr =
Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
8744 SV =
Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vzip");
8749 case NEON::BI__builtin_neon_vdot_s32:
8750 case NEON::BI__builtin_neon_vdot_u32:
8751 case NEON::BI__builtin_neon_vdotq_s32:
8752 case NEON::BI__builtin_neon_vdotq_u32: {
8754 llvm::FixedVectorType::get(
Int8Ty, Ty->getPrimitiveSizeInBits() / 8);
8755 llvm::Type *Tys[2] = { Ty, InputTy };
8758 case NEON::BI__builtin_neon_vfmlal_low_f16:
8759 case NEON::BI__builtin_neon_vfmlalq_low_f16: {
8761 llvm::FixedVectorType::get(
HalfTy, Ty->getPrimitiveSizeInBits() / 16);
8762 llvm::Type *Tys[2] = { Ty, InputTy };
8765 case NEON::BI__builtin_neon_vfmlsl_low_f16:
8766 case NEON::BI__builtin_neon_vfmlslq_low_f16: {
8768 llvm::FixedVectorType::get(
HalfTy, Ty->getPrimitiveSizeInBits() / 16);
8769 llvm::Type *Tys[2] = { Ty, InputTy };
8772 case NEON::BI__builtin_neon_vfmlal_high_f16:
8773 case NEON::BI__builtin_neon_vfmlalq_high_f16: {
8775 llvm::FixedVectorType::get(
HalfTy, Ty->getPrimitiveSizeInBits() / 16);
8776 llvm::Type *Tys[2] = { Ty, InputTy };
8779 case NEON::BI__builtin_neon_vfmlsl_high_f16:
8780 case NEON::BI__builtin_neon_vfmlslq_high_f16: {
8782 llvm::FixedVectorType::get(
HalfTy, Ty->getPrimitiveSizeInBits() / 16);
8783 llvm::Type *Tys[2] = { Ty, InputTy };
8786 case NEON::BI__builtin_neon_vmmlaq_s32:
8787 case NEON::BI__builtin_neon_vmmlaq_u32: {
8789 llvm::FixedVectorType::get(
Int8Ty, Ty->getPrimitiveSizeInBits() / 8);
8790 llvm::Type *Tys[2] = { Ty, InputTy };
8793 case NEON::BI__builtin_neon_vusmmlaq_s32: {
8795 llvm::FixedVectorType::get(
Int8Ty, Ty->getPrimitiveSizeInBits() / 8);
8796 llvm::Type *Tys[2] = { Ty, InputTy };
8799 case NEON::BI__builtin_neon_vusdot_s32:
8800 case NEON::BI__builtin_neon_vusdotq_s32: {
8802 llvm::FixedVectorType::get(
Int8Ty, Ty->getPrimitiveSizeInBits() / 8);
8803 llvm::Type *Tys[2] = { Ty, InputTy };
8806 case NEON::BI__builtin_neon_vbfdot_f32:
8807 case NEON::BI__builtin_neon_vbfdotq_f32: {
8808 llvm::Type *InputTy =
8809 llvm::FixedVectorType::get(
BFloatTy, Ty->getPrimitiveSizeInBits() / 16);
8810 llvm::Type *Tys[2] = { Ty, InputTy };
8813 case NEON::BI__builtin_neon___a32_vcvt_bf16_f32: {
8814 llvm::Type *Tys[1] = { Ty };
8821 assert(Int &&
"Expected valid intrinsic number");
8834 Value *Op, llvm::Type *Ty,
const CmpInst::Predicate Fp,
8835 const CmpInst::Predicate Ip,
const Twine &Name) {
8836 llvm::Type *OTy = Op->
getType();
8842 if (BitCastInst *BI = dyn_cast<BitCastInst>(Op))
8843 OTy = BI->getOperand(0)->getType();
8845 Op =
Builder.CreateBitCast(Op, OTy);
8846 if (OTy->getScalarType()->isFloatingPointTy()) {
8847 if (Fp == CmpInst::FCMP_OEQ)
8848 Op =
Builder.CreateFCmp(Fp, Op, Constant::getNullValue(OTy));
8850 Op =
Builder.CreateFCmpS(Fp, Op, Constant::getNullValue(OTy));
8852 Op =
Builder.CreateICmp(Ip, Op, Constant::getNullValue(OTy));
8854 return Builder.CreateSExt(Op, Ty, Name);
8859 llvm::Type *ResTy,
unsigned IntID,
8863 TblOps.push_back(ExtOp);
8867 auto *TblTy = cast<llvm::FixedVectorType>(Ops[0]->getType());
8868 for (
unsigned i = 0, e = TblTy->getNumElements(); i != e; ++i) {
8869 Indices.push_back(2*i);
8870 Indices.push_back(2*i+1);
8873 int PairPos = 0, End = Ops.size() - 1;
8874 while (PairPos < End) {
8875 TblOps.push_back(CGF.
Builder.CreateShuffleVector(Ops[PairPos],
8876 Ops[PairPos+1], Indices,
8883 if (PairPos == End) {
8884 Value *ZeroTbl = ConstantAggregateZero::get(TblTy);
8885 TblOps.push_back(CGF.
Builder.CreateShuffleVector(Ops[PairPos],
8886 ZeroTbl, Indices, Name));
8890 TblOps.push_back(IndexOp);
8896Value *CodeGenFunction::GetValueForARMHint(
unsigned BuiltinID) {
8898 switch (BuiltinID) {
8901 case clang::ARM::BI__builtin_arm_nop:
8904 case clang::ARM::BI__builtin_arm_yield:
8905 case clang::ARM::BI__yield:
8908 case clang::ARM::BI__builtin_arm_wfe:
8909 case clang::ARM::BI__wfe:
8912 case clang::ARM::BI__builtin_arm_wfi:
8913 case clang::ARM::BI__wfi:
8916 case clang::ARM::BI__builtin_arm_sev:
8917 case clang::ARM::BI__sev:
8920 case clang::ARM::BI__builtin_arm_sevl:
8921 case clang::ARM::BI__sevl:
8940 llvm::Type *ValueType,
bool isExecHi) {
8945 llvm::Value *
Call = Builder.CreateCall(F, {Builder.getInt1(
true)});
8948 Value *Rt2 = Builder.CreateLShr(
Call, 32);
8949 Rt2 = Builder.CreateTrunc(Rt2, CGF.
Int32Ty);
8962 llvm::Type *ValueType,
8964 StringRef SysReg =
"") {
8968 "Unsupported size for register.");
8974 if (SysReg.empty()) {
8976 SysReg = cast<clang::StringLiteral>(SysRegStrExpr)->getString();
8979 llvm::Metadata *Ops[] = { llvm::MDString::get(Context, SysReg) };
8980 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
8981 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
8985 bool MixedTypes =
RegisterType->isIntegerTy(64) && ValueType->isIntegerTy(32);
8986 assert(!(
RegisterType->isIntegerTy(32) && ValueType->isIntegerTy(64))
8987 &&
"Can't fit 64-bit value in 32-bit register");
8989 if (AccessKind !=
Write) {
8992 AccessKind ==
VolatileRead ? llvm::Intrinsic::read_volatile_register
8993 : llvm::Intrinsic::read_register,
8995 llvm::Value *
Call = Builder.CreateCall(F, Metadata);
8999 return Builder.CreateTrunc(
Call, ValueType);
9001 if (ValueType->isPointerTy())
9003 return Builder.CreateIntToPtr(
Call, ValueType);
9008 llvm::Function *F = CGM.
getIntrinsic(llvm::Intrinsic::write_register, Types);
9013 return Builder.CreateCall(F, { Metadata, ArgValue });
9016 if (ValueType->isPointerTy()) {
9018 ArgValue = Builder.CreatePtrToInt(ArgValue,
RegisterType);
9019 return Builder.CreateCall(F, { Metadata, ArgValue });
9022 return Builder.CreateCall(F, { Metadata, ArgValue });
9028 switch (BuiltinID) {
9030 case NEON::BI__builtin_neon_vget_lane_i8:
9031 case NEON::BI__builtin_neon_vget_lane_i16:
9032 case NEON::BI__builtin_neon_vget_lane_bf16:
9033 case NEON::BI__builtin_neon_vget_lane_i32:
9034 case NEON::BI__builtin_neon_vget_lane_i64:
9035 case NEON::BI__builtin_neon_vget_lane_f32:
9036 case NEON::BI__builtin_neon_vgetq_lane_i8:
9037 case NEON::BI__builtin_neon_vgetq_lane_i16:
9038 case NEON::BI__builtin_neon_vgetq_lane_bf16:
9039 case NEON::BI__builtin_neon_vgetq_lane_i32:
9040 case NEON::BI__builtin_neon_vgetq_lane_i64:
9041 case NEON::BI__builtin_neon_vgetq_lane_f32:
9042 case NEON::BI__builtin_neon_vduph_lane_bf16:
9043 case NEON::BI__builtin_neon_vduph_laneq_bf16:
9044 case NEON::BI__builtin_neon_vset_lane_i8:
9045 case NEON::BI__builtin_neon_vset_lane_i16:
9046 case NEON::BI__builtin_neon_vset_lane_bf16:
9047 case NEON::BI__builtin_neon_vset_lane_i32:
9048 case NEON::BI__builtin_neon_vset_lane_i64:
9049 case NEON::BI__builtin_neon_vset_lane_f32:
9050 case NEON::BI__builtin_neon_vsetq_lane_i8:
9051 case NEON::BI__builtin_neon_vsetq_lane_i16:
9052 case NEON::BI__builtin_neon_vsetq_lane_bf16:
9053 case NEON::BI__builtin_neon_vsetq_lane_i32:
9054 case NEON::BI__builtin_neon_vsetq_lane_i64:
9055 case NEON::BI__builtin_neon_vsetq_lane_f32:
9056 case NEON::BI__builtin_neon_vsha1h_u32:
9057 case NEON::BI__builtin_neon_vsha1cq_u32:
9058 case NEON::BI__builtin_neon_vsha1pq_u32:
9059 case NEON::BI__builtin_neon_vsha1mq_u32:
9060 case NEON::BI__builtin_neon_vcvth_bf16_f32:
9061 case clang::ARM::BI_MoveToCoprocessor:
9062 case clang::ARM::BI_MoveToCoprocessor2:
9071 llvm::Triple::ArchType Arch) {
9072 if (
auto Hint = GetValueForARMHint(BuiltinID))
9075 if (BuiltinID == clang::ARM::BI__emit) {
9077 llvm::FunctionType *FTy =
9078 llvm::FunctionType::get(
VoidTy,
false);
9082 llvm_unreachable(
"Sema will ensure that the parameter is constant");
9085 uint64_t ZExtValue =
Value.zextOrTrunc(IsThumb ? 16 : 32).getZExtValue();
9087 llvm::InlineAsm *Emit =
9088 IsThumb ? InlineAsm::get(FTy,
".inst.n 0x" + utohexstr(ZExtValue),
"",
9090 : InlineAsm::get(FTy,
".inst 0x" + utohexstr(ZExtValue),
"",
9093 return Builder.CreateCall(Emit);
9096 if (BuiltinID == clang::ARM::BI__builtin_arm_dbg) {
9101 if (BuiltinID == clang::ARM::BI__builtin_arm_prefetch) {
9113 if (BuiltinID == clang::ARM::BI__builtin_arm_rbit) {
9116 CGM.
getIntrinsic(Intrinsic::bitreverse, Arg->getType()), Arg,
"rbit");
9119 if (BuiltinID == clang::ARM::BI__builtin_arm_clz ||
9120 BuiltinID == clang::ARM::BI__builtin_arm_clz64) {
9124 if (BuiltinID == clang::ARM::BI__builtin_arm_clz64)
9130 if (BuiltinID == clang::ARM::BI__builtin_arm_cls) {
9134 if (BuiltinID == clang::ARM::BI__builtin_arm_cls64) {
9140 if (BuiltinID == clang::ARM::BI__clear_cache) {
9141 assert(
E->getNumArgs() == 2 &&
"__clear_cache takes 2 arguments");
9144 for (
unsigned i = 0; i < 2; i++)
9147 llvm::FunctionType *FTy = cast<llvm::FunctionType>(Ty);
9148 StringRef Name = FD->
getName();
9152 if (BuiltinID == clang::ARM::BI__builtin_arm_mcrr ||
9153 BuiltinID == clang::ARM::BI__builtin_arm_mcrr2) {
9156 switch (BuiltinID) {
9157 default: llvm_unreachable(
"unexpected builtin");
9158 case clang::ARM::BI__builtin_arm_mcrr:
9161 case clang::ARM::BI__builtin_arm_mcrr2:
9183 return Builder.CreateCall(F, {Coproc, Opc1, Rt, Rt2, CRm});
9186 if (BuiltinID == clang::ARM::BI__builtin_arm_mrrc ||
9187 BuiltinID == clang::ARM::BI__builtin_arm_mrrc2) {
9190 switch (BuiltinID) {
9191 default: llvm_unreachable(
"unexpected builtin");
9192 case clang::ARM::BI__builtin_arm_mrrc:
9195 case clang::ARM::BI__builtin_arm_mrrc2:
9203 Value *RtAndRt2 =
Builder.CreateCall(F, {Coproc, Opc1, CRm});
9213 Value *ShiftCast = llvm::ConstantInt::get(
Int64Ty, 32);
9214 RtAndRt2 =
Builder.CreateShl(Rt, ShiftCast,
"shl",
true);
9215 RtAndRt2 =
Builder.CreateOr(RtAndRt2, Rt1);
9220 if (BuiltinID == clang::ARM::BI__builtin_arm_ldrexd ||
9221 ((BuiltinID == clang::ARM::BI__builtin_arm_ldrex ||
9222 BuiltinID == clang::ARM::BI__builtin_arm_ldaex) &&
9224 BuiltinID == clang::ARM::BI__ldrexd) {
9227 switch (BuiltinID) {
9228 default: llvm_unreachable(
"unexpected builtin");
9229 case clang::ARM::BI__builtin_arm_ldaex:
9232 case clang::ARM::BI__builtin_arm_ldrexd:
9233 case clang::ARM::BI__builtin_arm_ldrex:
9234 case clang::ARM::BI__ldrexd:
9248 Val =
Builder.CreateShl(Val0, ShiftCst,
"shl",
true );
9249 Val =
Builder.CreateOr(Val, Val1);
9253 if (BuiltinID == clang::ARM::BI__builtin_arm_ldrex ||
9254 BuiltinID == clang::ARM::BI__builtin_arm_ldaex) {
9263 BuiltinID == clang::ARM::BI__builtin_arm_ldaex ? Intrinsic::arm_ldaex
9264 : Intrinsic::arm_ldrex,
9266 CallInst *Val =
Builder.CreateCall(F, LoadAddr,
"ldrex");
9270 if (RealResTy->isPointerTy())
9271 return Builder.CreateIntToPtr(Val, RealResTy);
9273 llvm::Type *IntResTy = llvm::IntegerType::get(
9275 return Builder.CreateBitCast(
Builder.CreateTruncOrBitCast(Val, IntResTy),
9280 if (BuiltinID == clang::ARM::BI__builtin_arm_strexd ||
9281 ((BuiltinID == clang::ARM::BI__builtin_arm_stlex ||
9282 BuiltinID == clang::ARM::BI__builtin_arm_strex) &&
9285 BuiltinID == clang::ARM::BI__builtin_arm_stlex ? Intrinsic::arm_stlexd
9286 : Intrinsic::arm_strexd);
9299 return Builder.CreateCall(F, {Arg0, Arg1, StPtr},
"strexd");
9302 if (BuiltinID == clang::ARM::BI__builtin_arm_strex ||
9303 BuiltinID == clang::ARM::BI__builtin_arm_stlex) {
9308 llvm::Type *StoreTy =
9311 if (StoreVal->
getType()->isPointerTy())
9314 llvm::Type *
IntTy = llvm::IntegerType::get(
9322 BuiltinID == clang::ARM::BI__builtin_arm_stlex ? Intrinsic::arm_stlex
9323 : Intrinsic::arm_strex,
9326 CallInst *CI =
Builder.CreateCall(F, {StoreVal, StoreAddr},
"strex");
9328 1, Attribute::get(
getLLVMContext(), Attribute::ElementType, StoreTy));
9332 if (BuiltinID == clang::ARM::BI__builtin_arm_clrex) {
9338 Intrinsic::ID CRCIntrinsicID = Intrinsic::not_intrinsic;
9339 switch (BuiltinID) {
9340 case clang::ARM::BI__builtin_arm_crc32b:
9341 CRCIntrinsicID = Intrinsic::arm_crc32b;
break;
9342 case clang::ARM::BI__builtin_arm_crc32cb:
9343 CRCIntrinsicID = Intrinsic::arm_crc32cb;
break;
9344 case clang::ARM::BI__builtin_arm_crc32h:
9345 CRCIntrinsicID = Intrinsic::arm_crc32h;
break;
9346 case clang::ARM::BI__builtin_arm_crc32ch:
9347 CRCIntrinsicID = Intrinsic::arm_crc32ch;
break;
9348 case clang::ARM::BI__builtin_arm_crc32w:
9349 case clang::ARM::BI__builtin_arm_crc32d:
9350 CRCIntrinsicID = Intrinsic::arm_crc32w;
break;
9351 case clang::ARM::BI__builtin_arm_crc32cw:
9352 case clang::ARM::BI__builtin_arm_crc32cd:
9353 CRCIntrinsicID = Intrinsic::arm_crc32cw;
break;
9356 if (CRCIntrinsicID != Intrinsic::not_intrinsic) {
9362 if (BuiltinID == clang::ARM::BI__builtin_arm_crc32d ||
9363 BuiltinID == clang::ARM::BI__builtin_arm_crc32cd) {
9371 return Builder.CreateCall(F, {Res, Arg1b});
9376 return Builder.CreateCall(F, {Arg0, Arg1});
9380 if (BuiltinID == clang::ARM::BI__builtin_arm_rsr ||
9381 BuiltinID == clang::ARM::BI__builtin_arm_rsr64 ||
9382 BuiltinID == clang::ARM::BI__builtin_arm_rsrp ||
9383 BuiltinID == clang::ARM::BI__builtin_arm_wsr ||
9384 BuiltinID == clang::ARM::BI__builtin_arm_wsr64 ||
9385 BuiltinID == clang::ARM::BI__builtin_arm_wsrp) {
9388 if (BuiltinID == clang::ARM::BI__builtin_arm_rsr ||
9389 BuiltinID == clang::ARM::BI__builtin_arm_rsr64 ||
9390 BuiltinID == clang::ARM::BI__builtin_arm_rsrp)
9393 bool IsPointerBuiltin = BuiltinID == clang::ARM::BI__builtin_arm_rsrp ||
9394 BuiltinID == clang::ARM::BI__builtin_arm_wsrp;
9396 bool Is64Bit = BuiltinID == clang::ARM::BI__builtin_arm_rsr64 ||
9397 BuiltinID == clang::ARM::BI__builtin_arm_wsr64;
9399 llvm::Type *ValueType;
9401 if (IsPointerBuiltin) {
9404 }
else if (Is64Bit) {
9414 if (BuiltinID == ARM::BI__builtin_sponentry) {
9433 return P.first == BuiltinID;
9436 BuiltinID = It->second;
9440 unsigned ICEArguments = 0;
9445 auto getAlignmentValue32 = [&](
Address addr) ->
Value* {
9446 return Builder.getInt32(addr.getAlignment().getQuantity());
9453 unsigned NumArgs =
E->getNumArgs() - (HasExtraArg ? 1 : 0);
9454 for (
unsigned i = 0, e = NumArgs; i != e; i++) {
9456 switch (BuiltinID) {
9457 case NEON::BI__builtin_neon_vld1_v:
9458 case NEON::BI__builtin_neon_vld1q_v:
9459 case NEON::BI__builtin_neon_vld1q_lane_v:
9460 case NEON::BI__builtin_neon_vld1_lane_v:
9461 case NEON::BI__builtin_neon_vld1_dup_v:
9462 case NEON::BI__builtin_neon_vld1q_dup_v:
9463 case NEON::BI__builtin_neon_vst1_v:
9464 case NEON::BI__builtin_neon_vst1q_v:
9465 case NEON::BI__builtin_neon_vst1q_lane_v:
9466 case NEON::BI__builtin_neon_vst1_lane_v:
9467 case NEON::BI__builtin_neon_vst2_v:
9468 case NEON::BI__builtin_neon_vst2q_v:
9469 case NEON::BI__builtin_neon_vst2_lane_v:
9470 case NEON::BI__builtin_neon_vst2q_lane_v:
9471 case NEON::BI__builtin_neon_vst3_v:
9472 case NEON::BI__builtin_neon_vst3q_v:
9473 case NEON::BI__builtin_neon_vst3_lane_v:
9474 case NEON::BI__builtin_neon_vst3q_lane_v:
9475 case NEON::BI__builtin_neon_vst4_v:
9476 case NEON::BI__builtin_neon_vst4q_v:
9477 case NEON::BI__builtin_neon_vst4_lane_v:
9478 case NEON::BI__builtin_neon_vst4q_lane_v:
9487 switch (BuiltinID) {
9488 case NEON::BI__builtin_neon_vld2_v:
9489 case NEON::BI__builtin_neon_vld2q_v:
9490 case NEON::BI__builtin_neon_vld3_v:
9491 case NEON::BI__builtin_neon_vld3q_v:
9492 case NEON::BI__builtin_neon_vld4_v:
9493 case NEON::BI__builtin_neon_vld4q_v:
9494 case NEON::BI__builtin_neon_vld2_lane_v:
9495 case NEON::BI__builtin_neon_vld2q_lane_v:
9496 case NEON::BI__builtin_neon_vld3_lane_v:
9497 case NEON::BI__builtin_neon_vld3q_lane_v:
9498 case NEON::BI__builtin_neon_vld4_lane_v:
9499 case NEON::BI__builtin_neon_vld4q_lane_v:
9500 case NEON::BI__builtin_neon_vld2_dup_v:
9501 case NEON::BI__builtin_neon_vld2q_dup_v:
9502 case NEON::BI__builtin_neon_vld3_dup_v:
9503 case NEON::BI__builtin_neon_vld3q_dup_v:
9504 case NEON::BI__builtin_neon_vld4_dup_v:
9505 case NEON::BI__builtin_neon_vld4q_dup_v:
9517 switch (BuiltinID) {
9520 case NEON::BI__builtin_neon_vget_lane_i8:
9521 case NEON::BI__builtin_neon_vget_lane_i16:
9522 case NEON::BI__builtin_neon_vget_lane_i32:
9523 case NEON::BI__builtin_neon_vget_lane_i64:
9524 case NEON::BI__builtin_neon_vget_lane_bf16:
9525 case NEON::BI__builtin_neon_vget_lane_f32:
9526 case NEON::BI__builtin_neon_vgetq_lane_i8:
9527 case NEON::BI__builtin_neon_vgetq_lane_i16:
9528 case NEON::BI__builtin_neon_vgetq_lane_i32:
9529 case NEON::BI__builtin_neon_vgetq_lane_i64:
9530 case NEON::BI__builtin_neon_vgetq_lane_bf16:
9531 case NEON::BI__builtin_neon_vgetq_lane_f32:
9532 case NEON::BI__builtin_neon_vduph_lane_bf16:
9533 case NEON::BI__builtin_neon_vduph_laneq_bf16:
9534 return Builder.CreateExtractElement(Ops[0], Ops[1],
"vget_lane");
9536 case NEON::BI__builtin_neon_vrndns_f32: {
9538 llvm::Type *Tys[] = {Arg->
getType()};
9540 return Builder.CreateCall(F, {Arg},
"vrndn"); }
9542 case NEON::BI__builtin_neon_vset_lane_i8:
9543 case NEON::BI__builtin_neon_vset_lane_i16:
9544 case NEON::BI__builtin_neon_vset_lane_i32:
9545 case NEON::BI__builtin_neon_vset_lane_i64:
9546 case NEON::BI__builtin_neon_vset_lane_bf16:
9547 case NEON::BI__builtin_neon_vset_lane_f32:
9548 case NEON::BI__builtin_neon_vsetq_lane_i8:
9549 case NEON::BI__builtin_neon_vsetq_lane_i16:
9550 case NEON::BI__builtin_neon_vsetq_lane_i32:
9551 case NEON::BI__builtin_neon_vsetq_lane_i64:
9552 case NEON::BI__builtin_neon_vsetq_lane_bf16:
9553 case NEON::BI__builtin_neon_vsetq_lane_f32:
9554 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vset_lane");
9556 case NEON::BI__builtin_neon_vsha1h_u32:
9559 case NEON::BI__builtin_neon_vsha1cq_u32:
9562 case NEON::BI__builtin_neon_vsha1pq_u32:
9565 case NEON::BI__builtin_neon_vsha1mq_u32:
9569 case NEON::BI__builtin_neon_vcvth_bf16_f32: {
9576 case clang::ARM::BI_MoveToCoprocessor:
9577 case clang::ARM::BI_MoveToCoprocessor2: {
9579 ? Intrinsic::arm_mcr
9580 : Intrinsic::arm_mcr2);
9581 return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0],
9582 Ops[3], Ops[4], Ops[5]});
9587 assert(HasExtraArg);
9588 const Expr *Arg =
E->getArg(
E->getNumArgs()-1);
9589 std::optional<llvm::APSInt>
Result =
9594 if (BuiltinID == clang::ARM::BI__builtin_arm_vcvtr_f ||
9595 BuiltinID == clang::ARM::BI__builtin_arm_vcvtr_d) {
9598 if (BuiltinID == clang::ARM::BI__builtin_arm_vcvtr_f)
9604 bool usgn =
Result->getZExtValue() == 1;
9605 unsigned Int = usgn ? Intrinsic::arm_vcvtru : Intrinsic::arm_vcvtr;
9609 return Builder.CreateCall(F, Ops,
"vcvtr");
9614 bool usgn =
Type.isUnsigned();
9615 bool rightShift =
false;
9617 llvm::FixedVectorType *VTy =
9620 llvm::Type *Ty = VTy;
9631 Builtin->BuiltinID, Builtin->LLVMIntrinsic, Builtin->AltLLVMIntrinsic,
9632 Builtin->NameHint, Builtin->TypeModifier,
E, Ops, PtrOp0, PtrOp1, Arch);
9635 switch (BuiltinID) {
9636 default:
return nullptr;
9637 case NEON::BI__builtin_neon_vld1q_lane_v:
9640 if (VTy->getElementType()->isIntegerTy(64)) {
9642 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
9643 int Lane = cast<ConstantInt>(Ops[2])->getZExtValue();
9644 Value *SV = llvm::ConstantVector::get(ConstantInt::get(
Int32Ty, 1-Lane));
9645 Ops[1] =
Builder.CreateShuffleVector(Ops[1], Ops[1], SV);
9647 Ty = llvm::FixedVectorType::get(VTy->getElementType(), 1);
9650 Value *Align = getAlignmentValue32(PtrOp0);
9653 int Indices[] = {1 - Lane, Lane};
9654 return Builder.CreateShuffleVector(Ops[1], Ld, Indices,
"vld1q_lane");
9657 case NEON::BI__builtin_neon_vld1_lane_v: {
9658 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
9661 return Builder.CreateInsertElement(Ops[1], Ld, Ops[2],
"vld1_lane");
9663 case NEON::BI__builtin_neon_vqrshrn_n_v:
9665 usgn ? Intrinsic::arm_neon_vqrshiftnu : Intrinsic::arm_neon_vqrshiftns;
9668 case NEON::BI__builtin_neon_vqrshrun_n_v:
9670 Ops,
"vqrshrun_n", 1,
true);
9671 case NEON::BI__builtin_neon_vqshrn_n_v:
9672 Int = usgn ? Intrinsic::arm_neon_vqshiftnu : Intrinsic::arm_neon_vqshiftns;
9675 case NEON::BI__builtin_neon_vqshrun_n_v:
9677 Ops,
"vqshrun_n", 1,
true);
9678 case NEON::BI__builtin_neon_vrecpe_v:
9679 case NEON::BI__builtin_neon_vrecpeq_v:
9682 case NEON::BI__builtin_neon_vrshrn_n_v:
9684 Ops,
"vrshrn_n", 1,
true);
9685 case NEON::BI__builtin_neon_vrsra_n_v:
9686 case NEON::BI__builtin_neon_vrsraq_n_v:
9687 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
9688 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
9690 Int = usgn ? Intrinsic::arm_neon_vrshiftu : Intrinsic::arm_neon_vrshifts;
9692 return Builder.CreateAdd(Ops[0], Ops[1],
"vrsra_n");
9693 case NEON::BI__builtin_neon_vsri_n_v:
9694 case NEON::BI__builtin_neon_vsriq_n_v:
9697 case NEON::BI__builtin_neon_vsli_n_v:
9698 case NEON::BI__builtin_neon_vsliq_n_v:
9702 case NEON::BI__builtin_neon_vsra_n_v:
9703 case NEON::BI__builtin_neon_vsraq_n_v:
9704 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
9706 return Builder.CreateAdd(Ops[0], Ops[1]);
9707 case NEON::BI__builtin_neon_vst1q_lane_v:
9710 if (VTy->getElementType()->isIntegerTy(64)) {
9711 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
9712 Value *SV = llvm::ConstantVector::get(cast<llvm::Constant>(Ops[2]));
9713 Ops[1] =
Builder.CreateShuffleVector(Ops[1], Ops[1], SV);
9714 Ops[2] = getAlignmentValue32(PtrOp0);
9715 llvm::Type *Tys[] = {
Int8PtrTy, Ops[1]->getType()};
9720 case NEON::BI__builtin_neon_vst1_lane_v: {
9721 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
9722 Ops[1] =
Builder.CreateExtractElement(Ops[1], Ops[2]);
9726 case NEON::BI__builtin_neon_vtbl1_v:
9729 case NEON::BI__builtin_neon_vtbl2_v:
9732 case NEON::BI__builtin_neon_vtbl3_v:
9735 case NEON::BI__builtin_neon_vtbl4_v:
9738 case NEON::BI__builtin_neon_vtbx1_v:
9741 case NEON::BI__builtin_neon_vtbx2_v:
9744 case NEON::BI__builtin_neon_vtbx3_v:
9747 case NEON::BI__builtin_neon_vtbx4_v:
9753template<
typename Integer>
9762 return Unsigned ? Builder.CreateZExt(
V,
T) : Builder.CreateSExt(
V,
T);
9772 unsigned LaneBits = cast<llvm::VectorType>(
V->getType())
9774 ->getPrimitiveSizeInBits();
9775 if (Shift == LaneBits) {
9780 return llvm::Constant::getNullValue(
V->getType());
9784 return Unsigned ? Builder.CreateLShr(
V, Shift) : Builder.CreateAShr(
V, Shift);
9791 unsigned Elements = 128 /
V->getType()->getPrimitiveSizeInBits();
9792 return Builder.CreateVectorSplat(Elements,
V);
9798 llvm::Type *DestType) {
9811 V->getType()->getScalarSizeInBits() != DestType->getScalarSizeInBits()) {
9812 return Builder.CreateCall(
9814 {DestType, V->getType()}),
9817 return Builder.CreateBitCast(
V, DestType);
9825 unsigned InputElements =
9826 cast<llvm::FixedVectorType>(
V->getType())->getNumElements();
9827 for (
unsigned i = 0; i < InputElements; i += 2)
9828 Indices.push_back(i + Odd);
9829 return Builder.CreateShuffleVector(
V, Indices);
9835 assert(V0->getType() == V1->getType() &&
"Can't zip different vector types");
9837 unsigned InputElements =
9838 cast<llvm::FixedVectorType>(V0->getType())->getNumElements();
9839 for (
unsigned i = 0; i < InputElements; i++) {
9840 Indices.push_back(i);
9841 Indices.push_back(i + InputElements);
9843 return Builder.CreateShuffleVector(V0, V1, Indices);
9846template<
unsigned HighBit,
unsigned OtherBits>
9850 llvm::Type *
T = cast<llvm::VectorType>(VT)->getElementType();
9851 unsigned LaneBits =
T->getPrimitiveSizeInBits();
9852 uint32_t
Value = HighBit << (LaneBits - 1);
9854 Value |= (1UL << (LaneBits - 1)) - 1;
9855 llvm::Value *Lane = llvm::ConstantInt::get(
T,
Value);
9861 unsigned ReverseWidth) {
9865 unsigned LaneSize =
V->getType()->getScalarSizeInBits();
9866 unsigned Elements = 128 / LaneSize;
9867 unsigned Mask = ReverseWidth / LaneSize - 1;
9868 for (
unsigned i = 0; i < Elements; i++)
9869 Indices.push_back(i ^ Mask);
9870 return Builder.CreateShuffleVector(
V, Indices);
9876 llvm::Triple::ArchType Arch) {
9877 enum class CustomCodeGen { VLD24, VST24 } CustomCodeGenType;
9878 Intrinsic::ID IRIntr;
9879 unsigned NumVectors;
9882 switch (BuiltinID) {
9883 #include "clang/Basic/arm_mve_builtin_cg.inc"
9894 switch (CustomCodeGenType) {
9896 case CustomCodeGen::VLD24: {
9902 assert(MvecLType->isStructTy() &&
9903 "Return type for vld[24]q should be a struct");
9904 assert(MvecLType->getStructNumElements() == 1 &&
9905 "Return-type struct for vld[24]q should have one element");
9906 auto MvecLTypeInner = MvecLType->getStructElementType(0);
9907 assert(MvecLTypeInner->isArrayTy() &&
9908 "Return-type struct for vld[24]q should contain an array");
9909 assert(MvecLTypeInner->getArrayNumElements() == NumVectors &&
9910 "Array member of return-type struct vld[24]q has wrong length");
9911 auto VecLType = MvecLTypeInner->getArrayElementType();
9913 Tys.push_back(VecLType);
9915 auto Addr =
E->getArg(0);
9921 Value *MvecOut = PoisonValue::get(MvecLType);
9922 for (
unsigned i = 0; i < NumVectors; ++i) {
9923 Value *Vec =
Builder.CreateExtractValue(LoadResult, i);
9924 MvecOut =
Builder.CreateInsertValue(MvecOut, Vec, {0, i});
9933 case CustomCodeGen::VST24: {
9937 auto Addr =
E->getArg(0);
9941 auto MvecCType =
E->getArg(1)->
getType();
9943 assert(MvecLType->isStructTy() &&
"Data type for vst2q should be a struct");
9944 assert(MvecLType->getStructNumElements() == 1 &&
9945 "Data-type struct for vst2q should have one element");
9946 auto MvecLTypeInner = MvecLType->getStructElementType(0);
9947 assert(MvecLTypeInner->isArrayTy() &&
9948 "Data-type struct for vst2q should contain an array");
9949 assert(MvecLTypeInner->getArrayNumElements() == NumVectors &&
9950 "Array member of return-type struct vld[24]q has wrong length");
9951 auto VecLType = MvecLTypeInner->getArrayElementType();
9953 Tys.push_back(VecLType);
9958 for (
unsigned i = 0; i < NumVectors; i++)
9959 Ops.push_back(
Builder.CreateExtractValue(Mvec, {0, i}));
9962 Value *ToReturn =
nullptr;
9963 for (
unsigned i = 0; i < NumVectors; i++) {
9964 Ops.push_back(llvm::ConstantInt::get(
Int32Ty, i));
9965 ToReturn =
Builder.CreateCall(F, Ops);
9971 llvm_unreachable(
"unknown custom codegen type.");
9977 llvm::Triple::ArchType Arch) {
9978 switch (BuiltinID) {
9981#include "clang/Basic/arm_cde_builtin_cg.inc"
9988 llvm::Triple::ArchType Arch) {
9989 unsigned int Int = 0;
9990 const char *
s =
nullptr;
9992 switch (BuiltinID) {
9995 case NEON::BI__builtin_neon_vtbl1_v:
9996 case NEON::BI__builtin_neon_vqtbl1_v:
9997 case NEON::BI__builtin_neon_vqtbl1q_v:
9998 case NEON::BI__builtin_neon_vtbl2_v:
9999 case NEON::BI__builtin_neon_vqtbl2_v:
10000 case NEON::BI__builtin_neon_vqtbl2q_v:
10001 case NEON::BI__builtin_neon_vtbl3_v:
10002 case NEON::BI__builtin_neon_vqtbl3_v:
10003 case NEON::BI__builtin_neon_vqtbl3q_v:
10004 case NEON::BI__builtin_neon_vtbl4_v:
10005 case NEON::BI__builtin_neon_vqtbl4_v:
10006 case NEON::BI__builtin_neon_vqtbl4q_v:
10008 case NEON::BI__builtin_neon_vtbx1_v:
10009 case NEON::BI__builtin_neon_vqtbx1_v:
10010 case NEON::BI__builtin_neon_vqtbx1q_v:
10011 case NEON::BI__builtin_neon_vtbx2_v:
10012 case NEON::BI__builtin_neon_vqtbx2_v:
10013 case NEON::BI__builtin_neon_vqtbx2q_v:
10014 case NEON::BI__builtin_neon_vtbx3_v:
10015 case NEON::BI__builtin_neon_vqtbx3_v:
10016 case NEON::BI__builtin_neon_vqtbx3q_v:
10017 case NEON::BI__builtin_neon_vtbx4_v:
10018 case NEON::BI__builtin_neon_vqtbx4_v:
10019 case NEON::BI__builtin_neon_vqtbx4q_v:
10023 assert(
E->getNumArgs() >= 3);
10026 const Expr *Arg =
E->getArg(
E->getNumArgs() - 1);
10027 std::optional<llvm::APSInt>
Result =
10042 switch (BuiltinID) {
10043 case NEON::BI__builtin_neon_vtbl1_v: {
10045 Ty, Intrinsic::aarch64_neon_tbl1,
"vtbl1");
10047 case NEON::BI__builtin_neon_vtbl2_v: {
10049 Ty, Intrinsic::aarch64_neon_tbl1,
"vtbl1");
10051 case NEON::BI__builtin_neon_vtbl3_v: {
10053 Ty, Intrinsic::aarch64_neon_tbl2,
"vtbl2");
10055 case NEON::BI__builtin_neon_vtbl4_v: {
10057 Ty, Intrinsic::aarch64_neon_tbl2,
"vtbl2");
10059 case NEON::BI__builtin_neon_vtbx1_v: {
10062 Intrinsic::aarch64_neon_tbl1,
"vtbl1");
10064 llvm::Constant *EightV = ConstantInt::get(Ty, 8);
10065 Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[2], EightV);
10066 CmpRes = Builder.CreateSExt(CmpRes, Ty);
10068 Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]);
10069 Value *EltsFromTbl = Builder.CreateAnd(Builder.CreateNot(CmpRes), TblRes);
10070 return Builder.CreateOr(EltsFromInput, EltsFromTbl,
"vtbx");
10072 case NEON::BI__builtin_neon_vtbx2_v: {
10074 Ty, Intrinsic::aarch64_neon_tbx1,
"vtbx1");
10076 case NEON::BI__builtin_neon_vtbx3_v: {
10079 Intrinsic::aarch64_neon_tbl2,
"vtbl2");
10081 llvm::Constant *TwentyFourV = ConstantInt::get(Ty, 24);
10082 Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[4],
10084 CmpRes = Builder.CreateSExt(CmpRes, Ty);
10086 Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]);
10087 Value *EltsFromTbl = Builder.CreateAnd(Builder.CreateNot(CmpRes), TblRes);
10088 return Builder.CreateOr(EltsFromInput, EltsFromTbl,
"vtbx");
10090 case NEON::BI__builtin_neon_vtbx4_v: {
10092 Ty, Intrinsic::aarch64_neon_tbx2,
"vtbx2");
10094 case NEON::BI__builtin_neon_vqtbl1_v:
10095 case NEON::BI__builtin_neon_vqtbl1q_v:
10096 Int = Intrinsic::aarch64_neon_tbl1;
s =
"vtbl1";
break;
10097 case NEON::BI__builtin_neon_vqtbl2_v:
10098 case NEON::BI__builtin_neon_vqtbl2q_v: {
10099 Int = Intrinsic::aarch64_neon_tbl2;
s =
"vtbl2";
break;
10100 case NEON::BI__builtin_neon_vqtbl3_v:
10101 case NEON::BI__builtin_neon_vqtbl3q_v:
10102 Int = Intrinsic::aarch64_neon_tbl3;
s =
"vtbl3";
break;
10103 case NEON::BI__builtin_neon_vqtbl4_v:
10104 case NEON::BI__builtin_neon_vqtbl4q_v:
10105 Int = Intrinsic::aarch64_neon_tbl4;
s =
"vtbl4";
break;
10106 case NEON::BI__builtin_neon_vqtbx1_v:
10107 case NEON::BI__builtin_neon_vqtbx1q_v:
10108 Int = Intrinsic::aarch64_neon_tbx1;
s =
"vtbx1";
break;
10109 case NEON::BI__builtin_neon_vqtbx2_v:
10110 case NEON::BI__builtin_neon_vqtbx2q_v:
10111 Int = Intrinsic::aarch64_neon_tbx2;
s =
"vtbx2";
break;
10112 case NEON::BI__builtin_neon_vqtbx3_v:
10113 case NEON::BI__builtin_neon_vqtbx3q_v:
10114 Int = Intrinsic::aarch64_neon_tbx3;
s =
"vtbx3";
break;
10115 case NEON::BI__builtin_neon_vqtbx4_v:
10116 case NEON::BI__builtin_neon_vqtbx4q_v:
10117 Int = Intrinsic::aarch64_neon_tbx4;
s =
"vtbx4";
break;
10129 auto *VTy = llvm::FixedVectorType::get(
Int16Ty, 4);
10131 Value *
V = PoisonValue::get(VTy);
10132 llvm::Constant *CI = ConstantInt::get(
SizeTy, 0);
10133 Op =
Builder.CreateInsertElement(
V, Op, CI);
10142 case SVETypeFlags::MemEltTyDefault:
10144 case SVETypeFlags::MemEltTyInt8:
10146 case SVETypeFlags::MemEltTyInt16:
10148 case SVETypeFlags::MemEltTyInt32:
10150 case SVETypeFlags::MemEltTyInt64:
10153 llvm_unreachable(
"Unknown MemEltType");
10159 llvm_unreachable(
"Invalid SVETypeFlag!");
10161 case SVETypeFlags::EltTyInt8:
10163 case SVETypeFlags::EltTyInt16:
10165 case SVETypeFlags::EltTyInt32:
10167 case SVETypeFlags::EltTyInt64:
10169 case SVETypeFlags::EltTyInt128:
10170 return Builder.getInt128Ty();
10172 case SVETypeFlags::EltTyFloat16:
10174 case SVETypeFlags::EltTyFloat32:
10176 case SVETypeFlags::EltTyFloat64:
10177 return Builder.getDoubleTy();
10179 case SVETypeFlags::EltTyBFloat16:
10180 return Builder.getBFloatTy();
10182 case SVETypeFlags::EltTyBool8:
10183 case SVETypeFlags::EltTyBool16:
10184 case SVETypeFlags::EltTyBool32:
10185 case SVETypeFlags::EltTyBool64:
10192llvm::ScalableVectorType *
10195 default: llvm_unreachable(
"Unhandled SVETypeFlag!");
10197 case SVETypeFlags::EltTyInt8:
10198 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 16);
10199 case SVETypeFlags::EltTyInt16:
10200 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 8);
10201 case SVETypeFlags::EltTyInt32:
10202 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 4);
10203 case SVETypeFlags::EltTyInt64:
10204 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 2);
10206 case SVETypeFlags::EltTyBFloat16:
10207 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 8);
10208 case SVETypeFlags::EltTyFloat16:
10209 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 8);
10210 case SVETypeFlags::EltTyFloat32:
10211 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 4);
10212 case SVETypeFlags::EltTyFloat64:
10213 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 2);
10215 case SVETypeFlags::EltTyBool8:
10216 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 16);
10217 case SVETypeFlags::EltTyBool16:
10218 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 8);
10219 case SVETypeFlags::EltTyBool32:
10220 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 4);
10221 case SVETypeFlags::EltTyBool64:
10222 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 2);
10227llvm::ScalableVectorType *
10231 llvm_unreachable(
"Invalid SVETypeFlag!");
10233 case SVETypeFlags::EltTyInt8:
10234 return llvm::ScalableVectorType::get(
Builder.getInt8Ty(), 16);
10235 case SVETypeFlags::EltTyInt16:
10236 return llvm::ScalableVectorType::get(
Builder.getInt16Ty(), 8);
10237 case SVETypeFlags::EltTyInt32:
10238 return llvm::ScalableVectorType::get(
Builder.getInt32Ty(), 4);
10239 case SVETypeFlags::EltTyInt64:
10240 return llvm::ScalableVectorType::get(
Builder.getInt64Ty(), 2);
10242 case SVETypeFlags::EltTyMFloat8:
10243 return llvm::ScalableVectorType::get(
Builder.getInt8Ty(), 16);
10244 case SVETypeFlags::EltTyFloat16:
10245 return llvm::ScalableVectorType::get(
Builder.getHalfTy(), 8);
10246 case SVETypeFlags::EltTyBFloat16:
10247 return llvm::ScalableVectorType::get(
Builder.getBFloatTy(), 8);
10248 case SVETypeFlags::EltTyFloat32:
10249 return llvm::ScalableVectorType::get(
Builder.getFloatTy(), 4);
10250 case SVETypeFlags::EltTyFloat64:
10251 return llvm::ScalableVectorType::get(
Builder.getDoubleTy(), 2);
10253 case SVETypeFlags::EltTyBool8:
10254 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 16);
10255 case SVETypeFlags::EltTyBool16:
10256 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 8);
10257 case SVETypeFlags::EltTyBool32:
10258 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 4);
10259 case SVETypeFlags::EltTyBool64:
10260 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 2);
10275 return llvm::ScalableVectorType::get(EltTy, NumElts);
10281 llvm::ScalableVectorType *VTy) {
10283 if (isa<TargetExtType>(Pred->
getType()) &&
10284 cast<TargetExtType>(Pred->
getType())->getName() ==
"aarch64.svcount")
10287 auto *RTy = llvm::VectorType::get(IntegerType::get(
getLLVMContext(), 1), VTy);
10292 llvm::Type *IntrinsicTy;
10293 switch (VTy->getMinNumElements()) {
10295 llvm_unreachable(
"unsupported element count!");
10300 IntID = Intrinsic::aarch64_sve_convert_from_svbool;
10304 IntID = Intrinsic::aarch64_sve_convert_to_svbool;
10305 IntrinsicTy = Pred->
getType();
10311 assert(
C->getType() == RTy &&
"Unexpected return type!");
10316 llvm::StructType *Ty) {
10317 if (PredTuple->
getType() == Ty)
10320 Value *
Ret = llvm::PoisonValue::get(Ty);
10321 for (
unsigned I = 0; I < Ty->getNumElements(); ++I) {
10322 Value *Pred =
Builder.CreateExtractValue(PredTuple, I);
10324 Pred, cast<llvm::ScalableVectorType>(Ty->getTypeAtIndex(I)));
10325 Ret =
Builder.CreateInsertValue(Ret, Pred, I);
10335 auto *OverloadedTy =
10339 if (Ops[1]->getType()->isVectorTy())
10359 Ops[0], cast<llvm::ScalableVectorType>(F->getArg(0)->getType()));
10364 if (Ops.size() == 2) {
10365 assert(Ops[1]->getType()->isVectorTy() &&
"Scalar base requires an offset");
10366 Ops.push_back(ConstantInt::get(
Int64Ty, 0));
10371 if (!TypeFlags.
isByteIndexed() && Ops[1]->getType()->isVectorTy()) {
10372 unsigned BytesPerElt =
10373 OverloadedTy->getElementType()->getScalarSizeInBits() / 8;
10374 Ops[2] =
Builder.CreateShl(Ops[2], Log2_32(BytesPerElt));
10389 auto *OverloadedTy =
10394 Ops.insert(Ops.begin(), Ops.pop_back_val());
10397 if (Ops[2]->getType()->isVectorTy())
10412 if (Ops.size() == 3) {
10413 assert(Ops[1]->getType()->isVectorTy() &&
"Scalar base requires an offset");
10414 Ops.push_back(ConstantInt::get(
Int64Ty, 0));
10419 Ops[0] =
Builder.CreateTrunc(Ops[0], OverloadedTy);
10429 Ops[1], cast<llvm::ScalableVectorType>(F->getArg(1)->getType()));
10433 if (!TypeFlags.
isByteIndexed() && Ops[2]->getType()->isVectorTy()) {
10434 unsigned BytesPerElt =
10435 OverloadedTy->getElementType()->getScalarSizeInBits() / 8;
10436 Ops[3] =
Builder.CreateShl(Ops[3], Log2_32(BytesPerElt));
10439 return Builder.CreateCall(F, Ops);
10447 auto *OverloadedTy = dyn_cast<llvm::ScalableVectorType>(Ops[1]->getType());
10449 OverloadedTy = cast<llvm::ScalableVectorType>(Ops[2]->getType());
10455 if (Ops[1]->getType()->isVectorTy()) {
10456 if (Ops.size() == 3) {
10458 Ops.push_back(ConstantInt::get(
Int64Ty, 0));
10461 std::swap(Ops[2], Ops[3]);
10465 unsigned BytesPerElt = MemEltTy->getPrimitiveSizeInBits() / 8;
10466 if (BytesPerElt > 1)
10467 Ops[2] =
Builder.CreateShl(Ops[2], Log2_32(BytesPerElt));
10472 return Builder.CreateCall(F, Ops);
10478 llvm::ScalableVectorType *VTy =
getSVEType(TypeFlags);
10480 Value *BasePtr = Ops[1];
10483 if (Ops.size() > 2)
10487 return Builder.CreateCall(F, {Predicate, BasePtr});
10493 llvm::ScalableVectorType *VTy =
getSVEType(TypeFlags);
10497 case Intrinsic::aarch64_sve_st2:
10498 case Intrinsic::aarch64_sve_st1_pn_x2:
10499 case Intrinsic::aarch64_sve_stnt1_pn_x2:
10500 case Intrinsic::aarch64_sve_st2q:
10503 case Intrinsic::aarch64_sve_st3:
10504 case Intrinsic::aarch64_sve_st3q:
10507 case Intrinsic::aarch64_sve_st4:
10508 case Intrinsic::aarch64_sve_st1_pn_x4:
10509 case Intrinsic::aarch64_sve_stnt1_pn_x4:
10510 case Intrinsic::aarch64_sve_st4q:
10514 llvm_unreachable(
"unknown intrinsic!");
10518 Value *BasePtr = Ops[1];
10521 if (Ops.size() > (2 + N))
10527 for (
unsigned I = Ops.size() - N; I < Ops.size(); ++I)
10528 Operands.push_back(Ops[I]);
10529 Operands.append({Predicate, BasePtr});
10532 return Builder.CreateCall(F, Operands);
10540 unsigned BuiltinID) {
10552 llvm::ScalableVectorType *Ty =
getSVEType(TypeFlags);
10558 llvm::Type *OverloadedTy =
getSVEType(TypeFlags);
10565 unsigned BuiltinID) {
10568 auto *MemoryTy = llvm::ScalableVectorType::get(MemEltTy, VectorTy);
10571 Value *BasePtr = Ops[1];
10574 if (Ops.size() > 3)
10577 Value *PrfOp = Ops.back();
10580 return Builder.CreateCall(F, {Predicate, BasePtr, PrfOp});
10584 llvm::Type *ReturnTy,
10586 unsigned IntrinsicID,
10587 bool IsZExtReturn) {
10594 auto VectorTy = cast<llvm::ScalableVectorType>(ReturnTy);
10595 llvm::ScalableVectorType *MemoryTy =
nullptr;
10596 llvm::ScalableVectorType *PredTy =
nullptr;
10597 bool IsQuadLoad =
false;
10598 switch (IntrinsicID) {
10599 case Intrinsic::aarch64_sve_ld1uwq:
10600 case Intrinsic::aarch64_sve_ld1udq:
10601 MemoryTy = llvm::ScalableVectorType::get(MemEltTy, 1);
10602 PredTy = llvm::ScalableVectorType::get(
10607 MemoryTy = llvm::ScalableVectorType::get(MemEltTy, VectorTy);
10613 Value *BasePtr = Ops[1];
10616 if (Ops.size() > 2)
10621 cast<llvm::Instruction>(
Builder.CreateCall(F, {Predicate, BasePtr}));
10628 return IsZExtReturn ?
Builder.CreateZExt(Load, VectorTy)
10629 :
Builder.CreateSExt(Load, VectorTy);
10634 unsigned IntrinsicID) {
10641 auto VectorTy = cast<llvm::ScalableVectorType>(Ops.back()->getType());
10642 auto MemoryTy = llvm::ScalableVectorType::get(MemEltTy, VectorTy);
10644 auto PredTy = MemoryTy;
10645 auto AddrMemoryTy = MemoryTy;
10646 bool IsQuadStore =
false;
10648 switch (IntrinsicID) {
10649 case Intrinsic::aarch64_sve_st1wq:
10650 case Intrinsic::aarch64_sve_st1dq:
10651 AddrMemoryTy = llvm::ScalableVectorType::get(MemEltTy, 1);
10653 llvm::ScalableVectorType::get(IntegerType::get(
getLLVMContext(), 1), 1);
10654 IsQuadStore =
true;
10660 Value *BasePtr = Ops[1];
10663 if (Ops.size() == 4)
10668 IsQuadStore ? Ops.back() :
Builder.CreateTrunc(Ops.back(), MemoryTy);
10673 cast<llvm::Instruction>(
Builder.CreateCall(F, {Val, Predicate, BasePtr}));
10686 NewOps.push_back(Ops[2]);
10688 llvm::Value *BasePtr = Ops[3];
10689 llvm::Value *RealSlice = Ops[1];
10692 if (Ops.size() == 5) {
10695 llvm::Value *StreamingVectorLengthCall =
10696 Builder.CreateCall(StreamingVectorLength);
10697 llvm::Value *Mulvl =
10698 Builder.CreateMul(StreamingVectorLengthCall, Ops[4],
"mulvl");
10702 RealSlice =
Builder.CreateAdd(RealSlice, Ops[4]);
10705 NewOps.push_back(BasePtr);
10706 NewOps.push_back(Ops[0]);
10707 NewOps.push_back(RealSlice);
10709 return Builder.CreateCall(F, NewOps);
10721 return Builder.CreateCall(F, Ops);
10728 if (Ops.size() == 0)
10729 Ops.push_back(llvm::ConstantInt::get(
Int32Ty, 255));
10731 return Builder.CreateCall(F, Ops);
10737 if (Ops.size() == 2)
10738 Ops.push_back(
Builder.getInt32(0));
10742 return Builder.CreateCall(F, Ops);
10748 return Builder.CreateVectorSplat(
10749 cast<llvm::VectorType>(Ty)->getElementCount(), Scalar);
10753 if (
auto *Ty =
Scalar->getType(); Ty->isVectorTy()) {
10755 auto *VecTy = cast<llvm::VectorType>(Ty);
10756 ElementCount EC = VecTy->getElementCount();
10757 assert(EC.isScalar() && VecTy->getElementType() ==
Int8Ty &&
10758 "Only <1 x i8> expected");
10773 if (
auto *StructTy = dyn_cast<StructType>(Ty)) {
10774 Value *Tuple = llvm::PoisonValue::get(Ty);
10776 for (
unsigned I = 0; I < StructTy->getNumElements(); ++I) {
10778 Value *Out =
Builder.CreateBitCast(In, StructTy->getTypeAtIndex(I));
10779 Tuple =
Builder.CreateInsertValue(Tuple, Out, I);
10785 return Builder.CreateBitCast(Val, Ty);
10790 auto *SplatZero = Constant::getNullValue(Ty);
10791 Ops.insert(Ops.begin(), SplatZero);
10796 auto *SplatUndef = UndefValue::get(Ty);
10797 Ops.insert(Ops.begin(), SplatUndef);
10802 llvm::Type *ResultType,
10807 llvm::Type *DefaultType =
getSVEType(TypeFlags);
10810 return {DefaultType, Ops[1]->getType()};
10816 return {Ops[0]->getType(), Ops.back()->getType()};
10818 if (TypeFlags.
isReductionQV() && !ResultType->isScalableTy() &&
10819 ResultType->isVectorTy())
10820 return {ResultType, Ops[1]->getType()};
10823 return {DefaultType};
10829 "Expects TypleFlags.isTupleSet() or TypeFlags.isTupleGet()");
10830 unsigned Idx = cast<ConstantInt>(Ops[1])->getZExtValue();
10833 return Builder.CreateInsertValue(Ops[0], Ops[2], Idx);
10834 return Builder.CreateExtractValue(Ops[0], Idx);
10840 assert(TypeFlags.
isTupleCreate() &&
"Expects TypleFlag isTupleCreate");
10842 Value *Tuple = llvm::PoisonValue::get(Ty);
10843 for (
unsigned Idx = 0; Idx < Ops.size(); Idx++)
10844 Tuple =
Builder.CreateInsertValue(Tuple, Ops[Idx], Idx);
10853 unsigned ICEArguments = 0;
10862 for (
unsigned i = 0, e =
E->getNumArgs(); i != e; i++) {
10863 bool IsICE = ICEArguments & (1 << i);
10869 std::optional<llvm::APSInt>
Result =
10871 assert(
Result &&
"Expected argument to be a constant");
10881 if (isa<StructType>(Arg->getType()) && !IsTupleGetOrSet) {
10882 for (
unsigned I = 0; I < Arg->getType()->getStructNumElements(); ++I)
10883 Ops.push_back(
Builder.CreateExtractValue(Arg, I));
10888 Ops.push_back(Arg);
10895 if (BuiltinID >= SVE::BI__builtin_sve_reinterpret_s8_s8 &&
10896 BuiltinID <= SVE::BI__builtin_sve_reinterpret_f64_f64_x4) {
10911 else if (TypeFlags.
isStore())
10929 else if (TypeFlags.
isUndef())
10930 return UndefValue::get(Ty);
10931 else if (Builtin->LLVMIntrinsic != 0) {
10935 Ops.pop_back_val());
10936 if (TypeFlags.
getMergeType() == SVETypeFlags::MergeZeroExp)
10939 if (TypeFlags.
getMergeType() == SVETypeFlags::MergeAnyExp)
10945 Ops.push_back(
Builder.getInt32( 31));
10947 Ops.insert(&Ops[1],
Builder.getInt32( 31));
10950 for (
unsigned i = 0, e = Ops.size(); i != e; ++i)
10951 if (
auto PredTy = dyn_cast<llvm::VectorType>(Ops[i]->getType()))
10952 if (PredTy->getElementType()->isIntegerTy(1))
10962 std::swap(Ops[1], Ops[2]);
10964 std::swap(Ops[1], Ops[2]);
10967 std::swap(Ops[1], Ops[2]);
10970 std::swap(Ops[1], Ops[3]);
10973 if (TypeFlags.
getMergeType() == SVETypeFlags::MergeZero) {
10974 llvm::Type *OpndTy = Ops[1]->getType();
10975 auto *SplatZero = Constant::getNullValue(OpndTy);
10976 Ops[1] =
Builder.CreateSelect(Ops[0], Ops[1], SplatZero);
10983 if (
Call->getType() == Ty)
10987 if (
auto PredTy = dyn_cast<llvm::ScalableVectorType>(Ty))
10989 if (
auto PredTupleTy = dyn_cast<llvm::StructType>(Ty))
10992 llvm_unreachable(
"unsupported element count!");
10995 switch (BuiltinID) {
10999 case SVE::BI__builtin_sve_svreinterpret_b: {
11003 CGM.
getIntrinsic(Intrinsic::aarch64_sve_convert_to_svbool, SVCountTy);
11004 return Builder.CreateCall(CastFromSVCountF, Ops[0]);
11006 case SVE::BI__builtin_sve_svreinterpret_c: {
11010 CGM.
getIntrinsic(Intrinsic::aarch64_sve_convert_from_svbool, SVCountTy);
11011 return Builder.CreateCall(CastToSVCountF, Ops[0]);
11014 case SVE::BI__builtin_sve_svpsel_lane_b8:
11015 case SVE::BI__builtin_sve_svpsel_lane_b16:
11016 case SVE::BI__builtin_sve_svpsel_lane_b32:
11017 case SVE::BI__builtin_sve_svpsel_lane_b64:
11018 case SVE::BI__builtin_sve_svpsel_lane_c8:
11019 case SVE::BI__builtin_sve_svpsel_lane_c16:
11020 case SVE::BI__builtin_sve_svpsel_lane_c32:
11021 case SVE::BI__builtin_sve_svpsel_lane_c64: {
11022 bool IsSVCount = isa<TargetExtType>(Ops[0]->getType());
11023 assert(((!IsSVCount || cast<TargetExtType>(Ops[0]->getType())->
getName() ==
11024 "aarch64.svcount")) &&
11025 "Unexpected TargetExtType");
11029 CGM.
getIntrinsic(Intrinsic::aarch64_sve_convert_to_svbool, SVCountTy);
11031 CGM.
getIntrinsic(Intrinsic::aarch64_sve_convert_from_svbool, SVCountTy);
11035 llvm::Value *Ops0 =
11036 IsSVCount ?
Builder.CreateCall(CastFromSVCountF, Ops[0]) : Ops[0];
11038 llvm::Value *PSel =
Builder.CreateCall(F, {Ops0, Ops1, Ops[2]});
11039 return IsSVCount ?
Builder.CreateCall(CastToSVCountF, PSel) : PSel;
11041 case SVE::BI__builtin_sve_svmov_b_z: {
11044 llvm::Type* OverloadedTy =
getSVEType(TypeFlags);
11046 return Builder.CreateCall(F, {Ops[0], Ops[1], Ops[1]});
11049 case SVE::BI__builtin_sve_svnot_b_z: {
11052 llvm::Type* OverloadedTy =
getSVEType(TypeFlags);
11054 return Builder.CreateCall(F, {Ops[0], Ops[1], Ops[0]});
11057 case SVE::BI__builtin_sve_svmovlb_u16:
11058 case SVE::BI__builtin_sve_svmovlb_u32:
11059 case SVE::BI__builtin_sve_svmovlb_u64:
11060 return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_ushllb);
11062 case SVE::BI__builtin_sve_svmovlb_s16:
11063 case SVE::BI__builtin_sve_svmovlb_s32:
11064 case SVE::BI__builtin_sve_svmovlb_s64:
11065 return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_sshllb);
11067 case SVE::BI__builtin_sve_svmovlt_u16:
11068 case SVE::BI__builtin_sve_svmovlt_u32:
11069 case SVE::BI__builtin_sve_svmovlt_u64:
11070 return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_ushllt);
11072 case SVE::BI__builtin_sve_svmovlt_s16:
11073 case SVE::BI__builtin_sve_svmovlt_s32:
11074 case SVE::BI__builtin_sve_svmovlt_s64:
11075 return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_sshllt);
11077 case SVE::BI__builtin_sve_svpmullt_u16:
11078 case SVE::BI__builtin_sve_svpmullt_u64:
11079 case SVE::BI__builtin_sve_svpmullt_n_u16:
11080 case SVE::BI__builtin_sve_svpmullt_n_u64:
11081 return EmitSVEPMull(TypeFlags, Ops, Intrinsic::aarch64_sve_pmullt_pair);
11083 case SVE::BI__builtin_sve_svpmullb_u16:
11084 case SVE::BI__builtin_sve_svpmullb_u64:
11085 case SVE::BI__builtin_sve_svpmullb_n_u16:
11086 case SVE::BI__builtin_sve_svpmullb_n_u64:
11087 return EmitSVEPMull(TypeFlags, Ops, Intrinsic::aarch64_sve_pmullb_pair);
11089 case SVE::BI__builtin_sve_svdup_n_b8:
11090 case SVE::BI__builtin_sve_svdup_n_b16:
11091 case SVE::BI__builtin_sve_svdup_n_b32:
11092 case SVE::BI__builtin_sve_svdup_n_b64: {
11094 Builder.CreateICmpNE(Ops[0], Constant::getNullValue(Ops[0]->getType()));
11095 llvm::ScalableVectorType *OverloadedTy =
getSVEType(TypeFlags);
11100 case SVE::BI__builtin_sve_svdupq_n_b8:
11101 case SVE::BI__builtin_sve_svdupq_n_b16:
11102 case SVE::BI__builtin_sve_svdupq_n_b32:
11103 case SVE::BI__builtin_sve_svdupq_n_b64:
11104 case SVE::BI__builtin_sve_svdupq_n_u8:
11105 case SVE::BI__builtin_sve_svdupq_n_s8:
11106 case SVE::BI__builtin_sve_svdupq_n_u64:
11107 case SVE::BI__builtin_sve_svdupq_n_f64:
11108 case SVE::BI__builtin_sve_svdupq_n_s64:
11109 case SVE::BI__builtin_sve_svdupq_n_u16:
11110 case SVE::BI__builtin_sve_svdupq_n_f16:
11111 case SVE::BI__builtin_sve_svdupq_n_bf16:
11112 case SVE::BI__builtin_sve_svdupq_n_s16:
11113 case SVE::BI__builtin_sve_svdupq_n_u32:
11114 case SVE::BI__builtin_sve_svdupq_n_f32:
11115 case SVE::BI__builtin_sve_svdupq_n_s32: {
11118 unsigned NumOpnds = Ops.size();
11121 cast<llvm::VectorType>(Ty)->getElementType()->isIntegerTy(1);
11126 llvm::Type *EltTy = Ops[0]->getType();
11131 for (
unsigned I = 0; I < NumOpnds; ++I)
11132 VecOps.push_back(
Builder.CreateZExt(Ops[I], EltTy));
11137 OverloadedTy, PoisonValue::get(OverloadedTy), Vec,
Builder.getInt64(0));
11152 : Intrinsic::aarch64_sve_cmpne_wide,
11159 case SVE::BI__builtin_sve_svpfalse_b:
11160 return ConstantInt::getFalse(Ty);
11162 case SVE::BI__builtin_sve_svpfalse_c: {
11163 auto SVBoolTy = ScalableVectorType::get(
Builder.getInt1Ty(), 16);
11166 return Builder.CreateCall(CastToSVCountF, ConstantInt::getFalse(SVBoolTy));
11169 case SVE::BI__builtin_sve_svlen_bf16:
11170 case SVE::BI__builtin_sve_svlen_f16:
11171 case SVE::BI__builtin_sve_svlen_f32:
11172 case SVE::BI__builtin_sve_svlen_f64:
11173 case SVE::BI__builtin_sve_svlen_s8:
11174 case SVE::BI__builtin_sve_svlen_s16:
11175 case SVE::BI__builtin_sve_svlen_s32:
11176 case SVE::BI__builtin_sve_svlen_s64:
11177 case SVE::BI__builtin_sve_svlen_u8:
11178 case SVE::BI__builtin_sve_svlen_u16:
11179 case SVE::BI__builtin_sve_svlen_u32:
11180 case SVE::BI__builtin_sve_svlen_u64: {
11182 auto VTy = cast<llvm::VectorType>(
getSVEType(TF));
11184 llvm::ConstantInt::get(Ty, VTy->getElementCount().getKnownMinValue());
11190 case SVE::BI__builtin_sve_svtbl2_u8:
11191 case SVE::BI__builtin_sve_svtbl2_s8:
11192 case SVE::BI__builtin_sve_svtbl2_u16:
11193 case SVE::BI__builtin_sve_svtbl2_s16:
11194 case SVE::BI__builtin_sve_svtbl2_u32:
11195 case SVE::BI__builtin_sve_svtbl2_s32:
11196 case SVE::BI__builtin_sve_svtbl2_u64:
11197 case SVE::BI__builtin_sve_svtbl2_s64:
11198 case SVE::BI__builtin_sve_svtbl2_f16:
11199 case SVE::BI__builtin_sve_svtbl2_bf16:
11200 case SVE::BI__builtin_sve_svtbl2_f32:
11201 case SVE::BI__builtin_sve_svtbl2_f64: {
11203 auto VTy = cast<llvm::ScalableVectorType>(
getSVEType(TF));
11205 return Builder.CreateCall(F, Ops);
11208 case SVE::BI__builtin_sve_svset_neonq_s8:
11209 case SVE::BI__builtin_sve_svset_neonq_s16:
11210 case SVE::BI__builtin_sve_svset_neonq_s32:
11211 case SVE::BI__builtin_sve_svset_neonq_s64:
11212 case SVE::BI__builtin_sve_svset_neonq_u8:
11213 case SVE::BI__builtin_sve_svset_neonq_u16:
11214 case SVE::BI__builtin_sve_svset_neonq_u32:
11215 case SVE::BI__builtin_sve_svset_neonq_u64:
11216 case SVE::BI__builtin_sve_svset_neonq_f16:
11217 case SVE::BI__builtin_sve_svset_neonq_f32:
11218 case SVE::BI__builtin_sve_svset_neonq_f64:
11219 case SVE::BI__builtin_sve_svset_neonq_bf16: {
11220 return Builder.CreateInsertVector(Ty, Ops[0], Ops[1],
Builder.getInt64(0));
11223 case SVE::BI__builtin_sve_svget_neonq_s8:
11224 case SVE::BI__builtin_sve_svget_neonq_s16:
11225 case SVE::BI__builtin_sve_svget_neonq_s32:
11226 case SVE::BI__builtin_sve_svget_neonq_s64:
11227 case SVE::BI__builtin_sve_svget_neonq_u8:
11228 case SVE::BI__builtin_sve_svget_neonq_u16:
11229 case SVE::BI__builtin_sve_svget_neonq_u32:
11230 case SVE::BI__builtin_sve_svget_neonq_u64:
11231 case SVE::BI__builtin_sve_svget_neonq_f16:
11232 case SVE::BI__builtin_sve_svget_neonq_f32:
11233 case SVE::BI__builtin_sve_svget_neonq_f64:
11234 case SVE::BI__builtin_sve_svget_neonq_bf16: {
11235 return Builder.CreateExtractVector(Ty, Ops[0],
Builder.getInt64(0));
11238 case SVE::BI__builtin_sve_svdup_neonq_s8:
11239 case SVE::BI__builtin_sve_svdup_neonq_s16:
11240 case SVE::BI__builtin_sve_svdup_neonq_s32:
11241 case SVE::BI__builtin_sve_svdup_neonq_s64:
11242 case SVE::BI__builtin_sve_svdup_neonq_u8:
11243 case SVE::BI__builtin_sve_svdup_neonq_u16:
11244 case SVE::BI__builtin_sve_svdup_neonq_u32:
11245 case SVE::BI__builtin_sve_svdup_neonq_u64:
11246 case SVE::BI__builtin_sve_svdup_neonq_f16:
11247 case SVE::BI__builtin_sve_svdup_neonq_f32:
11248 case SVE::BI__builtin_sve_svdup_neonq_f64:
11249 case SVE::BI__builtin_sve_svdup_neonq_bf16: {
11252 return Builder.CreateIntrinsic(Intrinsic::aarch64_sve_dupq_lane, {Ty},
11264 switch (BuiltinID) {
11267 case SME::BI__builtin_sme_svsumla_za32_s8_vg4x1:
11270 case SME::BI__builtin_sme_svsumla_za32_s8_vg4x2:
11271 case SME::BI__builtin_sme_svsudot_za32_s8_vg1x2:
11274 case SME::BI__builtin_sme_svsudot_za32_s8_vg1x4:
11275 case SME::BI__builtin_sme_svsumla_za32_s8_vg4x4:
11281 for (
unsigned I = 0; I < MultiVec; ++I)
11282 std::swap(Ops[I + 1], Ops[I + 1 + MultiVec]);
11295 return EmitSMELd1St1(TypeFlags, Ops, Builtin->LLVMIntrinsic);
11298 else if (BuiltinID == SME::BI__builtin_sme_svzero_mask_za ||
11299 BuiltinID == SME::BI__builtin_sme_svzero_za)
11300 return EmitSMEZero(TypeFlags, Ops, Builtin->LLVMIntrinsic);
11301 else if (BuiltinID == SME::BI__builtin_sme_svldr_vnum_za ||
11302 BuiltinID == SME::BI__builtin_sme_svstr_vnum_za ||
11303 BuiltinID == SME::BI__builtin_sme_svldr_za ||
11304 BuiltinID == SME::BI__builtin_sme_svstr_za)
11305 return EmitSMELdrStr(TypeFlags, Ops, Builtin->LLVMIntrinsic);
11310 Ops.pop_back_val());
11315 if (Builtin->LLVMIntrinsic == 0)
11318 if (BuiltinID == SME::BI__builtin_sme___arm_in_streaming_mode) {
11321 const auto *FD = cast<FunctionDecl>(
CurFuncDecl);
11323 unsigned SMEAttrs = FPT->getAArch64SMEAttributes();
11326 return ConstantInt::getBool(
Builder.getContext(), IsStreaming);
11332 for (
unsigned i = 0, e = Ops.size(); i != e; ++i)
11333 if (
auto PredTy = dyn_cast<llvm::VectorType>(Ops[i]->getType()))
11334 if (PredTy->getElementType()->isIntegerTy(1))
11342 return Builder.CreateCall(F, Ops);
11347 llvm::Triple::ArchType Arch) {
11356 if (BuiltinID == Builtin::BI__builtin_cpu_supports)
11357 return EmitAArch64CpuSupports(
E);
11359 unsigned HintID =
static_cast<unsigned>(-1);
11360 switch (BuiltinID) {
11362 case clang::AArch64::BI__builtin_arm_nop:
11365 case clang::AArch64::BI__builtin_arm_yield:
11366 case clang::AArch64::BI__yield:
11369 case clang::AArch64::BI__builtin_arm_wfe:
11370 case clang::AArch64::BI__wfe:
11373 case clang::AArch64::BI__builtin_arm_wfi:
11374 case clang::AArch64::BI__wfi:
11377 case clang::AArch64::BI__builtin_arm_sev:
11378 case clang::AArch64::BI__sev:
11381 case clang::AArch64::BI__builtin_arm_sevl:
11382 case clang::AArch64::BI__sevl:
11387 if (HintID !=
static_cast<unsigned>(-1)) {
11389 return Builder.CreateCall(F, llvm::ConstantInt::get(
Int32Ty, HintID));
11392 if (BuiltinID == clang::AArch64::BI__builtin_arm_trap) {
11398 if (BuiltinID == clang::AArch64::BI__builtin_arm_get_sme_state) {
11403 "__arm_sme_state"));
11405 "aarch64_pstate_sm_compatible");
11406 CI->setAttributes(Attrs);
11407 CI->setCallingConv(
11408 llvm::CallingConv::
11409 AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2);
11416 if (BuiltinID == clang::AArch64::BI__builtin_arm_rbit) {
11418 "rbit of unusual size!");
11421 CGM.
getIntrinsic(Intrinsic::bitreverse, Arg->getType()), Arg,
"rbit");
11423 if (BuiltinID == clang::AArch64::BI__builtin_arm_rbit64) {
11425 "rbit of unusual size!");
11428 CGM.
getIntrinsic(Intrinsic::bitreverse, Arg->getType()), Arg,
"rbit");
11431 if (BuiltinID == clang::AArch64::BI__builtin_arm_clz ||
11432 BuiltinID == clang::AArch64::BI__builtin_arm_clz64) {
11436 if (BuiltinID == clang::AArch64::BI__builtin_arm_clz64)
11441 if (BuiltinID == clang::AArch64::BI__builtin_arm_cls) {
11446 if (BuiltinID == clang::AArch64::BI__builtin_arm_cls64) {
11452 if (BuiltinID == clang::AArch64::BI__builtin_arm_rint32zf ||
11453 BuiltinID == clang::AArch64::BI__builtin_arm_rint32z) {
11455 llvm::Type *Ty = Arg->getType();
11460 if (BuiltinID == clang::AArch64::BI__builtin_arm_rint64zf ||
11461 BuiltinID == clang::AArch64::BI__builtin_arm_rint64z) {
11463 llvm::Type *Ty = Arg->getType();
11468 if (BuiltinID == clang::AArch64::BI__builtin_arm_rint32xf ||
11469 BuiltinID == clang::AArch64::BI__builtin_arm_rint32x) {
11471 llvm::Type *Ty = Arg->getType();
11476 if (BuiltinID == clang::AArch64::BI__builtin_arm_rint64xf ||
11477 BuiltinID == clang::AArch64::BI__builtin_arm_rint64x) {
11479 llvm::Type *Ty = Arg->getType();
11484 if (BuiltinID == clang::AArch64::BI__builtin_arm_jcvt) {
11486 "__jcvt of unusual size!");
11492 if (BuiltinID == clang::AArch64::BI__builtin_arm_ld64b ||
11493 BuiltinID == clang::AArch64::BI__builtin_arm_st64b ||
11494 BuiltinID == clang::AArch64::BI__builtin_arm_st64bv ||
11495 BuiltinID == clang::AArch64::BI__builtin_arm_st64bv0) {
11499 if (BuiltinID == clang::AArch64::BI__builtin_arm_ld64b) {
11503 llvm::Value *Val =
Builder.CreateCall(F, MemAddr);
11504 llvm::Value *ToRet;
11505 for (
size_t i = 0; i < 8; i++) {
11506 llvm::Value *ValOffsetPtr =
11517 Args.push_back(MemAddr);
11518 for (
size_t i = 0; i < 8; i++) {
11519 llvm::Value *ValOffsetPtr =
11526 auto Intr = (BuiltinID == clang::AArch64::BI__builtin_arm_st64b
11527 ? Intrinsic::aarch64_st64b
11528 : BuiltinID == clang::AArch64::BI__builtin_arm_st64bv
11529 ? Intrinsic::aarch64_st64bv
11530 : Intrinsic::aarch64_st64bv0);
11532 return Builder.CreateCall(F, Args);
11536 if (BuiltinID == clang::AArch64::BI__builtin_arm_rndr ||
11537 BuiltinID == clang::AArch64::BI__builtin_arm_rndrrs) {
11539 auto Intr = (BuiltinID == clang::AArch64::BI__builtin_arm_rndr
11540 ? Intrinsic::aarch64_rndr
11541 : Intrinsic::aarch64_rndrrs);
11543 llvm::Value *Val =
Builder.CreateCall(F);
11544 Value *RandomValue =
Builder.CreateExtractValue(Val, 0);
11553 if (BuiltinID == clang::AArch64::BI__clear_cache) {
11554 assert(
E->getNumArgs() == 2 &&
"__clear_cache takes 2 arguments");
11557 for (
unsigned i = 0; i < 2; i++)
11560 llvm::FunctionType *FTy = cast<llvm::FunctionType>(Ty);
11561 StringRef Name = FD->
getName();
11565 if ((BuiltinID == clang::AArch64::BI__builtin_arm_ldrex ||
11566 BuiltinID == clang::AArch64::BI__builtin_arm_ldaex) &&
11570 ? Intrinsic::aarch64_ldaxp
11571 : Intrinsic::aarch64_ldxp);
11578 llvm::Type *Int128Ty = llvm::IntegerType::get(
getLLVMContext(), 128);
11579 Val0 =
Builder.CreateZExt(Val0, Int128Ty);
11580 Val1 =
Builder.CreateZExt(Val1, Int128Ty);
11582 Value *ShiftCst = llvm::ConstantInt::get(Int128Ty, 64);
11583 Val =
Builder.CreateShl(Val0, ShiftCst,
"shl",
true );
11584 Val =
Builder.CreateOr(Val, Val1);
11586 }
else if (BuiltinID == clang::AArch64::BI__builtin_arm_ldrex ||
11587 BuiltinID == clang::AArch64::BI__builtin_arm_ldaex) {
11592 llvm::Type *
IntTy =
11597 ? Intrinsic::aarch64_ldaxr
11598 : Intrinsic::aarch64_ldxr,
11600 CallInst *Val =
Builder.CreateCall(F, LoadAddr,
"ldxr");
11604 if (RealResTy->isPointerTy())
11605 return Builder.CreateIntToPtr(Val, RealResTy);
11607 llvm::Type *IntResTy = llvm::IntegerType::get(
11609 return Builder.CreateBitCast(
Builder.CreateTruncOrBitCast(Val, IntResTy),
11613 if ((BuiltinID == clang::AArch64::BI__builtin_arm_strex ||
11614 BuiltinID == clang::AArch64::BI__builtin_arm_stlex) &&
11618 ? Intrinsic::aarch64_stlxp
11619 : Intrinsic::aarch64_stxp);
11631 return Builder.CreateCall(F, {Arg0, Arg1, StPtr},
"stxp");
11634 if (BuiltinID == clang::AArch64::BI__builtin_arm_strex ||
11635 BuiltinID == clang::AArch64::BI__builtin_arm_stlex) {
11640 llvm::Type *StoreTy =
11643 if (StoreVal->
getType()->isPointerTy())
11646 llvm::Type *
IntTy = llvm::IntegerType::get(
11655 ? Intrinsic::aarch64_stlxr
11656 : Intrinsic::aarch64_stxr,
11658 CallInst *CI =
Builder.CreateCall(F, {StoreVal, StoreAddr},
"stxr");
11660 1, Attribute::get(
getLLVMContext(), Attribute::ElementType, StoreTy));
11664 if (BuiltinID == clang::AArch64::BI__getReg) {
11667 llvm_unreachable(
"Sema will ensure that the parameter is constant");
11673 llvm::Metadata *Ops[] = {llvm::MDString::get(Context, Reg)};
11674 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
11675 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
11677 llvm::Function *F =
11679 return Builder.CreateCall(F, Metadata);
11682 if (BuiltinID == clang::AArch64::BI__break) {
11685 llvm_unreachable(
"Sema will ensure that the parameter is constant");
11687 llvm::Function *F =
CGM.
getIntrinsic(llvm::Intrinsic::aarch64_break);
11691 if (BuiltinID == clang::AArch64::BI__builtin_arm_clrex) {
11693 return Builder.CreateCall(F);
11696 if (BuiltinID == clang::AArch64::BI_ReadWriteBarrier)
11697 return Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent,
11698 llvm::SyncScope::SingleThread);
11701 Intrinsic::ID CRCIntrinsicID = Intrinsic::not_intrinsic;
11702 switch (BuiltinID) {
11703 case clang::AArch64::BI__builtin_arm_crc32b:
11704 CRCIntrinsicID = Intrinsic::aarch64_crc32b;
break;
11705 case clang::AArch64::BI__builtin_arm_crc32cb:
11706 CRCIntrinsicID = Intrinsic::aarch64_crc32cb;
break;
11707 case clang::AArch64::BI__builtin_arm_crc32h:
11708 CRCIntrinsicID = Intrinsic::aarch64_crc32h;
break;
11709 case clang::AArch64::BI__builtin_arm_crc32ch:
11710 CRCIntrinsicID = Intrinsic::aarch64_crc32ch;
break;
11711 case clang::AArch64::BI__builtin_arm_crc32w:
11712 CRCIntrinsicID = Intrinsic::aarch64_crc32w;
break;
11713 case clang::AArch64::BI__builtin_arm_crc32cw:
11714 CRCIntrinsicID = Intrinsic::aarch64_crc32cw;
break;
11715 case clang::AArch64::BI__builtin_arm_crc32d:
11716 CRCIntrinsicID = Intrinsic::aarch64_crc32x;
break;
11717 case clang::AArch64::BI__builtin_arm_crc32cd:
11718 CRCIntrinsicID = Intrinsic::aarch64_crc32cx;
break;
11721 if (CRCIntrinsicID != Intrinsic::not_intrinsic) {
11726 llvm::Type *DataTy = F->getFunctionType()->getParamType(1);
11727 Arg1 =
Builder.CreateZExtOrBitCast(Arg1, DataTy);
11729 return Builder.CreateCall(F, {Arg0, Arg1});
11733 if (BuiltinID == AArch64::BI__builtin_arm_mops_memset_tag) {
11740 CGM.
getIntrinsic(Intrinsic::aarch64_mops_memset_tag), {Dst, Val, Size});
11744 Intrinsic::ID MTEIntrinsicID = Intrinsic::not_intrinsic;
11745 switch (BuiltinID) {
11746 case clang::AArch64::BI__builtin_arm_irg:
11747 MTEIntrinsicID = Intrinsic::aarch64_irg;
break;
11748 case clang::AArch64::BI__builtin_arm_addg:
11749 MTEIntrinsicID = Intrinsic::aarch64_addg;
break;
11750 case clang::AArch64::BI__builtin_arm_gmi:
11751 MTEIntrinsicID = Intrinsic::aarch64_gmi;
break;
11752 case clang::AArch64::BI__builtin_arm_ldg:
11753 MTEIntrinsicID = Intrinsic::aarch64_ldg;
break;
11754 case clang::AArch64::BI__builtin_arm_stg:
11755 MTEIntrinsicID = Intrinsic::aarch64_stg;
break;
11756 case clang::AArch64::BI__builtin_arm_subp:
11757 MTEIntrinsicID = Intrinsic::aarch64_subp;
break;
11760 if (MTEIntrinsicID != Intrinsic::not_intrinsic) {
11761 if (MTEIntrinsicID == Intrinsic::aarch64_irg) {
11769 if (MTEIntrinsicID == Intrinsic::aarch64_addg) {
11775 {Pointer, TagOffset});
11777 if (MTEIntrinsicID == Intrinsic::aarch64_gmi) {
11788 if (MTEIntrinsicID == Intrinsic::aarch64_ldg) {
11791 {TagAddress, TagAddress});
11796 if (MTEIntrinsicID == Intrinsic::aarch64_stg) {
11799 {TagAddress, TagAddress});
11801 if (MTEIntrinsicID == Intrinsic::aarch64_subp) {
11809 if (BuiltinID == clang::AArch64::BI__builtin_arm_rsr ||
11810 BuiltinID == clang::AArch64::BI__builtin_arm_rsr64 ||
11811 BuiltinID == clang::AArch64::BI__builtin_arm_rsr128 ||
11812 BuiltinID == clang::AArch64::BI__builtin_arm_rsrp ||
11813 BuiltinID == clang::AArch64::BI__builtin_arm_wsr ||
11814 BuiltinID == clang::AArch64::BI__builtin_arm_wsr64 ||
11815 BuiltinID == clang::AArch64::BI__builtin_arm_wsr128 ||
11816 BuiltinID == clang::AArch64::BI__builtin_arm_wsrp) {
11819 if (BuiltinID == clang::AArch64::BI__builtin_arm_rsr ||
11820 BuiltinID == clang::AArch64::BI__builtin_arm_rsr64 ||
11821 BuiltinID == clang::AArch64::BI__builtin_arm_rsr128 ||
11822 BuiltinID == clang::AArch64::BI__builtin_arm_rsrp)
11825 bool IsPointerBuiltin = BuiltinID == clang::AArch64::BI__builtin_arm_rsrp ||
11826 BuiltinID == clang::AArch64::BI__builtin_arm_wsrp;
11828 bool Is32Bit = BuiltinID == clang::AArch64::BI__builtin_arm_rsr ||
11829 BuiltinID == clang::AArch64::BI__builtin_arm_wsr;
11831 bool Is128Bit = BuiltinID == clang::AArch64::BI__builtin_arm_rsr128 ||
11832 BuiltinID == clang::AArch64::BI__builtin_arm_wsr128;
11834 llvm::Type *ValueType;
11838 }
else if (Is128Bit) {
11839 llvm::Type *Int128Ty =
11841 ValueType = Int128Ty;
11843 }
else if (IsPointerBuiltin) {
11853 if (BuiltinID == clang::AArch64::BI_ReadStatusReg ||
11854 BuiltinID == clang::AArch64::BI_WriteStatusReg) {
11860 std::string SysRegStr;
11861 llvm::raw_string_ostream(SysRegStr) <<
11862 ((1 << 1) | ((SysReg >> 14) & 1)) <<
":" <<
11863 ((SysReg >> 11) & 7) <<
":" <<
11864 ((SysReg >> 7) & 15) <<
":" <<
11865 ((SysReg >> 3) & 15) <<
":" <<
11868 llvm::Metadata *Ops[] = { llvm::MDString::get(Context, SysRegStr) };
11869 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
11870 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
11875 if (BuiltinID == clang::AArch64::BI_ReadStatusReg) {
11876 llvm::Function *F =
CGM.
getIntrinsic(llvm::Intrinsic::read_register, Types);
11878 return Builder.CreateCall(F, Metadata);
11881 llvm::Function *F =
CGM.
getIntrinsic(llvm::Intrinsic::write_register, Types);
11884 return Builder.CreateCall(F, { Metadata, ArgValue });
11887 if (BuiltinID == clang::AArch64::BI_AddressOfReturnAddress) {
11888 llvm::Function *F =
11890 return Builder.CreateCall(F);
11893 if (BuiltinID == clang::AArch64::BI__builtin_sponentry) {
11895 return Builder.CreateCall(F);
11898 if (BuiltinID == clang::AArch64::BI__mulh ||
11899 BuiltinID == clang::AArch64::BI__umulh) {
11901 llvm::Type *Int128Ty = llvm::IntegerType::get(
getLLVMContext(), 128);
11903 bool IsSigned = BuiltinID == clang::AArch64::BI__mulh;
11909 Value *MulResult, *HigherBits;
11911 MulResult =
Builder.CreateNSWMul(LHS, RHS);
11912 HigherBits =
Builder.CreateAShr(MulResult, 64);
11914 MulResult =
Builder.CreateNUWMul(LHS, RHS);
11915 HigherBits =
Builder.CreateLShr(MulResult, 64);
11917 HigherBits =
Builder.CreateIntCast(HigherBits, ResType, IsSigned);
11922 if (BuiltinID == AArch64::BI__writex18byte ||
11923 BuiltinID == AArch64::BI__writex18word ||
11924 BuiltinID == AArch64::BI__writex18dword ||
11925 BuiltinID == AArch64::BI__writex18qword) {
11941 if (BuiltinID == AArch64::BI__readx18byte ||
11942 BuiltinID == AArch64::BI__readx18word ||
11943 BuiltinID == AArch64::BI__readx18dword ||
11944 BuiltinID == AArch64::BI__readx18qword) {
11959 if (BuiltinID == AArch64::BI__addx18byte ||
11960 BuiltinID == AArch64::BI__addx18word ||
11961 BuiltinID == AArch64::BI__addx18dword ||
11962 BuiltinID == AArch64::BI__addx18qword ||
11963 BuiltinID == AArch64::BI__incx18byte ||
11964 BuiltinID == AArch64::BI__incx18word ||
11965 BuiltinID == AArch64::BI__incx18dword ||
11966 BuiltinID == AArch64::BI__incx18qword) {
11969 switch (BuiltinID) {
11970 case AArch64::BI__incx18byte:
11972 isIncrement =
true;
11974 case AArch64::BI__incx18word:
11976 isIncrement =
true;
11978 case AArch64::BI__incx18dword:
11980 isIncrement =
true;
11982 case AArch64::BI__incx18qword:
11984 isIncrement =
true;
11988 isIncrement =
false;
12013 if (BuiltinID == AArch64::BI_CopyDoubleFromInt64 ||
12014 BuiltinID == AArch64::BI_CopyFloatFromInt32 ||
12015 BuiltinID == AArch64::BI_CopyInt32FromFloat ||
12016 BuiltinID == AArch64::BI_CopyInt64FromDouble) {
12019 return Builder.CreateBitCast(Arg, RetTy);
12022 if (BuiltinID == AArch64::BI_CountLeadingOnes ||
12023 BuiltinID == AArch64::BI_CountLeadingOnes64 ||
12024 BuiltinID == AArch64::BI_CountLeadingZeros ||
12025 BuiltinID == AArch64::BI_CountLeadingZeros64) {
12027 llvm::Type *ArgType = Arg->
getType();
12029 if (BuiltinID == AArch64::BI_CountLeadingOnes ||
12030 BuiltinID == AArch64::BI_CountLeadingOnes64)
12031 Arg =
Builder.CreateXor(Arg, Constant::getAllOnesValue(ArgType));
12036 if (BuiltinID == AArch64::BI_CountLeadingOnes64 ||
12037 BuiltinID == AArch64::BI_CountLeadingZeros64)
12042 if (BuiltinID == AArch64::BI_CountLeadingSigns ||
12043 BuiltinID == AArch64::BI_CountLeadingSigns64) {
12046 Function *F = (BuiltinID == AArch64::BI_CountLeadingSigns)
12051 if (BuiltinID == AArch64::BI_CountLeadingSigns64)
12056 if (BuiltinID == AArch64::BI_CountOneBits ||
12057 BuiltinID == AArch64::BI_CountOneBits64) {
12059 llvm::Type *ArgType = ArgValue->
getType();
12063 if (BuiltinID == AArch64::BI_CountOneBits64)
12068 if (BuiltinID == AArch64::BI__prefetch) {
12077 if (BuiltinID == AArch64::BI__hlt) {
12083 return ConstantInt::get(
Builder.getInt32Ty(), 0);
12088 if (std::optional<MSVCIntrin> MsvcIntId =
12094 return P.first == BuiltinID;
12097 BuiltinID = It->second;
12101 unsigned ICEArguments = 0;
12108 for (
unsigned i = 0, e =
E->getNumArgs() - 1; i != e; i++) {
12110 switch (BuiltinID) {
12111 case NEON::BI__builtin_neon_vld1_v:
12112 case NEON::BI__builtin_neon_vld1q_v:
12113 case NEON::BI__builtin_neon_vld1_dup_v:
12114 case NEON::BI__builtin_neon_vld1q_dup_v:
12115 case NEON::BI__builtin_neon_vld1_lane_v:
12116 case NEON::BI__builtin_neon_vld1q_lane_v:
12117 case NEON::BI__builtin_neon_vst1_v:
12118 case NEON::BI__builtin_neon_vst1q_v:
12119 case NEON::BI__builtin_neon_vst1_lane_v:
12120 case NEON::BI__builtin_neon_vst1q_lane_v:
12121 case NEON::BI__builtin_neon_vldap1_lane_s64:
12122 case NEON::BI__builtin_neon_vldap1q_lane_s64:
12123 case NEON::BI__builtin_neon_vstl1_lane_s64:
12124 case NEON::BI__builtin_neon_vstl1q_lane_s64:
12142 assert(
Result &&
"SISD intrinsic should have been handled");
12146 const Expr *Arg =
E->getArg(
E->getNumArgs()-1);
12148 if (std::optional<llvm::APSInt>
Result =
12153 bool usgn =
Type.isUnsigned();
12154 bool quad =
Type.isQuad();
12157 switch (BuiltinID) {
12159 case NEON::BI__builtin_neon_vabsh_f16:
12162 case NEON::BI__builtin_neon_vaddq_p128: {
12165 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
12166 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
12167 Ops[0] =
Builder.CreateXor(Ops[0], Ops[1]);
12168 llvm::Type *Int128Ty = llvm::Type::getIntNTy(
getLLVMContext(), 128);
12169 return Builder.CreateBitCast(Ops[0], Int128Ty);
12171 case NEON::BI__builtin_neon_vldrq_p128: {
12172 llvm::Type *Int128Ty = llvm::Type::getIntNTy(
getLLVMContext(), 128);
12177 case NEON::BI__builtin_neon_vstrq_p128: {
12178 Value *Ptr = Ops[0];
12181 case NEON::BI__builtin_neon_vcvts_f32_u32:
12182 case NEON::BI__builtin_neon_vcvtd_f64_u64:
12185 case NEON::BI__builtin_neon_vcvts_f32_s32:
12186 case NEON::BI__builtin_neon_vcvtd_f64_s64: {
12188 bool Is64 = Ops[0]->getType()->getPrimitiveSizeInBits() == 64;
12191 Ops[0] =
Builder.CreateBitCast(Ops[0], InTy);
12193 return Builder.CreateUIToFP(Ops[0], FTy);
12194 return Builder.CreateSIToFP(Ops[0], FTy);
12196 case NEON::BI__builtin_neon_vcvth_f16_u16:
12197 case NEON::BI__builtin_neon_vcvth_f16_u32:
12198 case NEON::BI__builtin_neon_vcvth_f16_u64:
12201 case NEON::BI__builtin_neon_vcvth_f16_s16:
12202 case NEON::BI__builtin_neon_vcvth_f16_s32:
12203 case NEON::BI__builtin_neon_vcvth_f16_s64: {
12205 llvm::Type *FTy =
HalfTy;
12207 if (Ops[0]->getType()->getPrimitiveSizeInBits() == 64)
12209 else if (Ops[0]->getType()->getPrimitiveSizeInBits() == 32)
12213 Ops[0] =
Builder.CreateBitCast(Ops[0], InTy);
12215 return Builder.CreateUIToFP(Ops[0], FTy);
12216 return Builder.CreateSIToFP(Ops[0], FTy);
12218 case NEON::BI__builtin_neon_vcvtah_u16_f16:
12219 case NEON::BI__builtin_neon_vcvtmh_u16_f16:
12220 case NEON::BI__builtin_neon_vcvtnh_u16_f16:
12221 case NEON::BI__builtin_neon_vcvtph_u16_f16:
12222 case NEON::BI__builtin_neon_vcvth_u16_f16:
12223 case NEON::BI__builtin_neon_vcvtah_s16_f16:
12224 case NEON::BI__builtin_neon_vcvtmh_s16_f16:
12225 case NEON::BI__builtin_neon_vcvtnh_s16_f16:
12226 case NEON::BI__builtin_neon_vcvtph_s16_f16:
12227 case NEON::BI__builtin_neon_vcvth_s16_f16: {
12230 llvm::Type* FTy =
HalfTy;
12231 llvm::Type *Tys[2] = {InTy, FTy};
12233 switch (BuiltinID) {
12234 default: llvm_unreachable(
"missing builtin ID in switch!");
12235 case NEON::BI__builtin_neon_vcvtah_u16_f16:
12236 Int = Intrinsic::aarch64_neon_fcvtau;
break;
12237 case NEON::BI__builtin_neon_vcvtmh_u16_f16:
12238 Int = Intrinsic::aarch64_neon_fcvtmu;
break;
12239 case NEON::BI__builtin_neon_vcvtnh_u16_f16:
12240 Int = Intrinsic::aarch64_neon_fcvtnu;
break;
12241 case NEON::BI__builtin_neon_vcvtph_u16_f16:
12242 Int = Intrinsic::aarch64_neon_fcvtpu;
break;
12243 case NEON::BI__builtin_neon_vcvth_u16_f16:
12244 Int = Intrinsic::aarch64_neon_fcvtzu;
break;
12245 case NEON::BI__builtin_neon_vcvtah_s16_f16:
12246 Int = Intrinsic::aarch64_neon_fcvtas;
break;
12247 case NEON::BI__builtin_neon_vcvtmh_s16_f16:
12248 Int = Intrinsic::aarch64_neon_fcvtms;
break;
12249 case NEON::BI__builtin_neon_vcvtnh_s16_f16:
12250 Int = Intrinsic::aarch64_neon_fcvtns;
break;
12251 case NEON::BI__builtin_neon_vcvtph_s16_f16:
12252 Int = Intrinsic::aarch64_neon_fcvtps;
break;
12253 case NEON::BI__builtin_neon_vcvth_s16_f16:
12254 Int = Intrinsic::aarch64_neon_fcvtzs;
break;
12259 case NEON::BI__builtin_neon_vcaleh_f16:
12260 case NEON::BI__builtin_neon_vcalth_f16:
12261 case NEON::BI__builtin_neon_vcageh_f16:
12262 case NEON::BI__builtin_neon_vcagth_f16: {
12265 llvm::Type* FTy =
HalfTy;
12266 llvm::Type *Tys[2] = {InTy, FTy};
12268 switch (BuiltinID) {
12269 default: llvm_unreachable(
"missing builtin ID in switch!");
12270 case NEON::BI__builtin_neon_vcageh_f16:
12271 Int = Intrinsic::aarch64_neon_facge;
break;
12272 case NEON::BI__builtin_neon_vcagth_f16:
12273 Int = Intrinsic::aarch64_neon_facgt;
break;
12274 case NEON::BI__builtin_neon_vcaleh_f16:
12275 Int = Intrinsic::aarch64_neon_facge; std::swap(Ops[0], Ops[1]);
break;
12276 case NEON::BI__builtin_neon_vcalth_f16:
12277 Int = Intrinsic::aarch64_neon_facgt; std::swap(Ops[0], Ops[1]);
break;
12282 case NEON::BI__builtin_neon_vcvth_n_s16_f16:
12283 case NEON::BI__builtin_neon_vcvth_n_u16_f16: {
12286 llvm::Type* FTy =
HalfTy;
12287 llvm::Type *Tys[2] = {InTy, FTy};
12289 switch (BuiltinID) {
12290 default: llvm_unreachable(
"missing builtin ID in switch!");
12291 case NEON::BI__builtin_neon_vcvth_n_s16_f16:
12292 Int = Intrinsic::aarch64_neon_vcvtfp2fxs;
break;
12293 case NEON::BI__builtin_neon_vcvth_n_u16_f16:
12294 Int = Intrinsic::aarch64_neon_vcvtfp2fxu;
break;
12299 case NEON::BI__builtin_neon_vcvth_n_f16_s16:
12300 case NEON::BI__builtin_neon_vcvth_n_f16_u16: {
12302 llvm::Type* FTy =
HalfTy;
12304 llvm::Type *Tys[2] = {FTy, InTy};
12306 switch (BuiltinID) {
12307 default: llvm_unreachable(
"missing builtin ID in switch!");
12308 case NEON::BI__builtin_neon_vcvth_n_f16_s16:
12309 Int = Intrinsic::aarch64_neon_vcvtfxs2fp;
12310 Ops[0] =
Builder.CreateSExt(Ops[0], InTy,
"sext");
12312 case NEON::BI__builtin_neon_vcvth_n_f16_u16:
12313 Int = Intrinsic::aarch64_neon_vcvtfxu2fp;
12314 Ops[0] =
Builder.CreateZExt(Ops[0], InTy);
12319 case NEON::BI__builtin_neon_vpaddd_s64: {
12320 auto *Ty = llvm::FixedVectorType::get(
Int64Ty, 2);
12323 Vec =
Builder.CreateBitCast(Vec, Ty,
"v2i64");
12324 llvm::Value *Idx0 = llvm::ConstantInt::get(
SizeTy, 0);
12325 llvm::Value *Idx1 = llvm::ConstantInt::get(
SizeTy, 1);
12326 Value *Op0 =
Builder.CreateExtractElement(Vec, Idx0,
"lane0");
12327 Value *Op1 =
Builder.CreateExtractElement(Vec, Idx1,
"lane1");
12329 return Builder.CreateAdd(Op0, Op1,
"vpaddd");
12331 case NEON::BI__builtin_neon_vpaddd_f64: {
12332 auto *Ty = llvm::FixedVectorType::get(
DoubleTy, 2);
12335 Vec =
Builder.CreateBitCast(Vec, Ty,
"v2f64");
12336 llvm::Value *Idx0 = llvm::ConstantInt::get(
SizeTy, 0);
12337 llvm::Value *Idx1 = llvm::ConstantInt::get(
SizeTy, 1);
12338 Value *Op0 =
Builder.CreateExtractElement(Vec, Idx0,
"lane0");
12339 Value *Op1 =
Builder.CreateExtractElement(Vec, Idx1,
"lane1");
12341 return Builder.CreateFAdd(Op0, Op1,
"vpaddd");
12343 case NEON::BI__builtin_neon_vpadds_f32: {
12344 auto *Ty = llvm::FixedVectorType::get(
FloatTy, 2);
12347 Vec =
Builder.CreateBitCast(Vec, Ty,
"v2f32");
12348 llvm::Value *Idx0 = llvm::ConstantInt::get(
SizeTy, 0);
12349 llvm::Value *Idx1 = llvm::ConstantInt::get(
SizeTy, 1);
12350 Value *Op0 =
Builder.CreateExtractElement(Vec, Idx0,
"lane0");
12351 Value *Op1 =
Builder.CreateExtractElement(Vec, Idx1,
"lane1");
12353 return Builder.CreateFAdd(Op0, Op1,
"vpaddd");
12355 case NEON::BI__builtin_neon_vceqzd_s64:
12356 case NEON::BI__builtin_neon_vceqzd_f64:
12357 case NEON::BI__builtin_neon_vceqzs_f32:
12358 case NEON::BI__builtin_neon_vceqzh_f16:
12362 ICmpInst::FCMP_OEQ, ICmpInst::ICMP_EQ,
"vceqz");
12363 case NEON::BI__builtin_neon_vcgezd_s64:
12364 case NEON::BI__builtin_neon_vcgezd_f64:
12365 case NEON::BI__builtin_neon_vcgezs_f32:
12366 case NEON::BI__builtin_neon_vcgezh_f16:
12370 ICmpInst::FCMP_OGE, ICmpInst::ICMP_SGE,
"vcgez");
12371 case NEON::BI__builtin_neon_vclezd_s64:
12372 case NEON::BI__builtin_neon_vclezd_f64:
12373 case NEON::BI__builtin_neon_vclezs_f32:
12374 case NEON::BI__builtin_neon_vclezh_f16:
12378 ICmpInst::FCMP_OLE, ICmpInst::ICMP_SLE,
"vclez");
12379 case NEON::BI__builtin_neon_vcgtzd_s64:
12380 case NEON::BI__builtin_neon_vcgtzd_f64:
12381 case NEON::BI__builtin_neon_vcgtzs_f32:
12382 case NEON::BI__builtin_neon_vcgtzh_f16:
12386 ICmpInst::FCMP_OGT, ICmpInst::ICMP_SGT,
"vcgtz");
12387 case NEON::BI__builtin_neon_vcltzd_s64:
12388 case NEON::BI__builtin_neon_vcltzd_f64:
12389 case NEON::BI__builtin_neon_vcltzs_f32:
12390 case NEON::BI__builtin_neon_vcltzh_f16:
12394 ICmpInst::FCMP_OLT, ICmpInst::ICMP_SLT,
"vcltz");
12396 case NEON::BI__builtin_neon_vceqzd_u64: {
12400 Builder.CreateICmpEQ(Ops[0], llvm::Constant::getNullValue(
Int64Ty));
12403 case NEON::BI__builtin_neon_vceqd_f64:
12404 case NEON::BI__builtin_neon_vcled_f64:
12405 case NEON::BI__builtin_neon_vcltd_f64:
12406 case NEON::BI__builtin_neon_vcged_f64:
12407 case NEON::BI__builtin_neon_vcgtd_f64: {
12408 llvm::CmpInst::Predicate
P;
12409 switch (BuiltinID) {
12410 default: llvm_unreachable(
"missing builtin ID in switch!");
12411 case NEON::BI__builtin_neon_vceqd_f64:
P = llvm::FCmpInst::FCMP_OEQ;
break;
12412 case NEON::BI__builtin_neon_vcled_f64:
P = llvm::FCmpInst::FCMP_OLE;
break;
12413 case NEON::BI__builtin_neon_vcltd_f64:
P = llvm::FCmpInst::FCMP_OLT;
break;
12414 case NEON::BI__builtin_neon_vcged_f64:
P = llvm::FCmpInst::FCMP_OGE;
break;
12415 case NEON::BI__builtin_neon_vcgtd_f64:
P = llvm::FCmpInst::FCMP_OGT;
break;
12420 if (
P == llvm::FCmpInst::FCMP_OEQ)
12421 Ops[0] =
Builder.CreateFCmp(
P, Ops[0], Ops[1]);
12423 Ops[0] =
Builder.CreateFCmpS(
P, Ops[0], Ops[1]);
12426 case NEON::BI__builtin_neon_vceqs_f32:
12427 case NEON::BI__builtin_neon_vcles_f32:
12428 case NEON::BI__builtin_neon_vclts_f32:
12429 case NEON::BI__builtin_neon_vcges_f32:
12430 case NEON::BI__builtin_neon_vcgts_f32: {
12431 llvm::CmpInst::Predicate
P;
12432 switch (BuiltinID) {
12433 default: llvm_unreachable(
"missing builtin ID in switch!");
12434 case NEON::BI__builtin_neon_vceqs_f32:
P = llvm::FCmpInst::FCMP_OEQ;
break;
12435 case NEON::BI__builtin_neon_vcles_f32:
P = llvm::FCmpInst::FCMP_OLE;
break;
12436 case NEON::BI__builtin_neon_vclts_f32:
P = llvm::FCmpInst::FCMP_OLT;
break;
12437 case NEON::BI__builtin_neon_vcges_f32:
P = llvm::FCmpInst::FCMP_OGE;
break;
12438 case NEON::BI__builtin_neon_vcgts_f32:
P = llvm::FCmpInst::FCMP_OGT;
break;
12443 if (
P == llvm::FCmpInst::FCMP_OEQ)
12444 Ops[0] =
Builder.CreateFCmp(
P, Ops[0], Ops[1]);
12446 Ops[0] =
Builder.CreateFCmpS(
P, Ops[0], Ops[1]);
12449 case NEON::BI__builtin_neon_vceqh_f16:
12450 case NEON::BI__builtin_neon_vcleh_f16:
12451 case NEON::BI__builtin_neon_vclth_f16:
12452 case NEON::BI__builtin_neon_vcgeh_f16:
12453 case NEON::BI__builtin_neon_vcgth_f16: {
12454 llvm::CmpInst::Predicate
P;
12455 switch (BuiltinID) {
12456 default: llvm_unreachable(
"missing builtin ID in switch!");
12457 case NEON::BI__builtin_neon_vceqh_f16:
P = llvm::FCmpInst::FCMP_OEQ;
break;
12458 case NEON::BI__builtin_neon_vcleh_f16:
P = llvm::FCmpInst::FCMP_OLE;
break;
12459 case NEON::BI__builtin_neon_vclth_f16:
P = llvm::FCmpInst::FCMP_OLT;
break;
12460 case NEON::BI__builtin_neon_vcgeh_f16:
P = llvm::FCmpInst::FCMP_OGE;
break;
12461 case NEON::BI__builtin_neon_vcgth_f16:
P = llvm::FCmpInst::FCMP_OGT;
break;
12466 if (
P == llvm::FCmpInst::FCMP_OEQ)
12467 Ops[0] =
Builder.CreateFCmp(
P, Ops[0], Ops[1]);
12469 Ops[0] =
Builder.CreateFCmpS(
P, Ops[0], Ops[1]);
12472 case NEON::BI__builtin_neon_vceqd_s64:
12473 case NEON::BI__builtin_neon_vceqd_u64:
12474 case NEON::BI__builtin_neon_vcgtd_s64:
12475 case NEON::BI__builtin_neon_vcgtd_u64:
12476 case NEON::BI__builtin_neon_vcltd_s64:
12477 case NEON::BI__builtin_neon_vcltd_u64:
12478 case NEON::BI__builtin_neon_vcged_u64:
12479 case NEON::BI__builtin_neon_vcged_s64:
12480 case NEON::BI__builtin_neon_vcled_u64:
12481 case NEON::BI__builtin_neon_vcled_s64: {
12482 llvm::CmpInst::Predicate
P;
12483 switch (BuiltinID) {
12484 default: llvm_unreachable(
"missing builtin ID in switch!");
12485 case NEON::BI__builtin_neon_vceqd_s64:
12486 case NEON::BI__builtin_neon_vceqd_u64:
P = llvm::ICmpInst::ICMP_EQ;
break;
12487 case NEON::BI__builtin_neon_vcgtd_s64:
P = llvm::ICmpInst::ICMP_SGT;
break;
12488 case NEON::BI__builtin_neon_vcgtd_u64:
P = llvm::ICmpInst::ICMP_UGT;
break;
12489 case NEON::BI__builtin_neon_vcltd_s64:
P = llvm::ICmpInst::ICMP_SLT;
break;
12490 case NEON::BI__builtin_neon_vcltd_u64:
P = llvm::ICmpInst::ICMP_ULT;
break;
12491 case NEON::BI__builtin_neon_vcged_u64:
P = llvm::ICmpInst::ICMP_UGE;
break;
12492 case NEON::BI__builtin_neon_vcged_s64:
P = llvm::ICmpInst::ICMP_SGE;
break;
12493 case NEON::BI__builtin_neon_vcled_u64:
P = llvm::ICmpInst::ICMP_ULE;
break;
12494 case NEON::BI__builtin_neon_vcled_s64:
P = llvm::ICmpInst::ICMP_SLE;
break;
12499 Ops[0] =
Builder.CreateICmp(
P, Ops[0], Ops[1]);
12502 case NEON::BI__builtin_neon_vtstd_s64:
12503 case NEON::BI__builtin_neon_vtstd_u64: {
12507 Ops[0] =
Builder.CreateAnd(Ops[0], Ops[1]);
12508 Ops[0] =
Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0],
12509 llvm::Constant::getNullValue(
Int64Ty));
12512 case NEON::BI__builtin_neon_vset_lane_i8:
12513 case NEON::BI__builtin_neon_vset_lane_i16:
12514 case NEON::BI__builtin_neon_vset_lane_i32:
12515 case NEON::BI__builtin_neon_vset_lane_i64:
12516 case NEON::BI__builtin_neon_vset_lane_bf16:
12517 case NEON::BI__builtin_neon_vset_lane_f32:
12518 case NEON::BI__builtin_neon_vsetq_lane_i8:
12519 case NEON::BI__builtin_neon_vsetq_lane_i16:
12520 case NEON::BI__builtin_neon_vsetq_lane_i32:
12521 case NEON::BI__builtin_neon_vsetq_lane_i64:
12522 case NEON::BI__builtin_neon_vsetq_lane_bf16:
12523 case NEON::BI__builtin_neon_vsetq_lane_f32:
12525 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vset_lane");
12526 case NEON::BI__builtin_neon_vset_lane_f64:
12529 Builder.CreateBitCast(Ops[1], llvm::FixedVectorType::get(
DoubleTy, 1));
12531 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vset_lane");
12532 case NEON::BI__builtin_neon_vsetq_lane_f64:
12535 Builder.CreateBitCast(Ops[1], llvm::FixedVectorType::get(
DoubleTy, 2));
12537 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vset_lane");
12539 case NEON::BI__builtin_neon_vget_lane_i8:
12540 case NEON::BI__builtin_neon_vdupb_lane_i8:
12542 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
Int8Ty, 8));
12545 case NEON::BI__builtin_neon_vgetq_lane_i8:
12546 case NEON::BI__builtin_neon_vdupb_laneq_i8:
12548 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
Int8Ty, 16));
12551 case NEON::BI__builtin_neon_vget_lane_i16:
12552 case NEON::BI__builtin_neon_vduph_lane_i16:
12554 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
Int16Ty, 4));
12557 case NEON::BI__builtin_neon_vgetq_lane_i16:
12558 case NEON::BI__builtin_neon_vduph_laneq_i16:
12560 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
Int16Ty, 8));
12563 case NEON::BI__builtin_neon_vget_lane_i32:
12564 case NEON::BI__builtin_neon_vdups_lane_i32:
12566 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
Int32Ty, 2));
12569 case NEON::BI__builtin_neon_vdups_lane_f32:
12571 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
FloatTy, 2));
12574 case NEON::BI__builtin_neon_vgetq_lane_i32:
12575 case NEON::BI__builtin_neon_vdups_laneq_i32:
12577 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
Int32Ty, 4));
12580 case NEON::BI__builtin_neon_vget_lane_i64:
12581 case NEON::BI__builtin_neon_vdupd_lane_i64:
12583 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
Int64Ty, 1));
12586 case NEON::BI__builtin_neon_vdupd_lane_f64:
12588 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
DoubleTy, 1));
12591 case NEON::BI__builtin_neon_vgetq_lane_i64:
12592 case NEON::BI__builtin_neon_vdupd_laneq_i64:
12594 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
Int64Ty, 2));
12597 case NEON::BI__builtin_neon_vget_lane_f32:
12599 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
FloatTy, 2));
12602 case NEON::BI__builtin_neon_vget_lane_f64:
12604 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
DoubleTy, 1));
12607 case NEON::BI__builtin_neon_vgetq_lane_f32:
12608 case NEON::BI__builtin_neon_vdups_laneq_f32:
12610 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
FloatTy, 4));
12613 case NEON::BI__builtin_neon_vgetq_lane_f64:
12614 case NEON::BI__builtin_neon_vdupd_laneq_f64:
12616 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
DoubleTy, 2));
12619 case NEON::BI__builtin_neon_vaddh_f16:
12621 return Builder.CreateFAdd(Ops[0], Ops[1],
"vaddh");
12622 case NEON::BI__builtin_neon_vsubh_f16:
12624 return Builder.CreateFSub(Ops[0], Ops[1],
"vsubh");
12625 case NEON::BI__builtin_neon_vmulh_f16:
12627 return Builder.CreateFMul(Ops[0], Ops[1],
"vmulh");
12628 case NEON::BI__builtin_neon_vdivh_f16:
12630 return Builder.CreateFDiv(Ops[0], Ops[1],
"vdivh");
12631 case NEON::BI__builtin_neon_vfmah_f16:
12634 *
this, Intrinsic::fma, Intrinsic::experimental_constrained_fma,
HalfTy,
12636 case NEON::BI__builtin_neon_vfmsh_f16: {
12641 *
this, Intrinsic::fma, Intrinsic::experimental_constrained_fma,
HalfTy,
12644 case NEON::BI__builtin_neon_vaddd_s64:
12645 case NEON::BI__builtin_neon_vaddd_u64:
12647 case NEON::BI__builtin_neon_vsubd_s64:
12648 case NEON::BI__builtin_neon_vsubd_u64:
12650 case NEON::BI__builtin_neon_vqdmlalh_s16:
12651 case NEON::BI__builtin_neon_vqdmlslh_s16: {
12655 auto *VTy = llvm::FixedVectorType::get(
Int32Ty, 4);
12657 ProductOps,
"vqdmlXl");
12658 Constant *CI = ConstantInt::get(
SizeTy, 0);
12659 Ops[1] =
Builder.CreateExtractElement(Ops[1], CI,
"lane0");
12661 unsigned AccumInt = BuiltinID == NEON::BI__builtin_neon_vqdmlalh_s16
12662 ? Intrinsic::aarch64_neon_sqadd
12663 : Intrinsic::aarch64_neon_sqsub;
12666 case NEON::BI__builtin_neon_vqshlud_n_s64: {
12672 case NEON::BI__builtin_neon_vqshld_n_u64:
12673 case NEON::BI__builtin_neon_vqshld_n_s64: {
12674 unsigned Int = BuiltinID == NEON::BI__builtin_neon_vqshld_n_u64
12675 ? Intrinsic::aarch64_neon_uqshl
12676 : Intrinsic::aarch64_neon_sqshl;
12681 case NEON::BI__builtin_neon_vrshrd_n_u64:
12682 case NEON::BI__builtin_neon_vrshrd_n_s64: {
12683 unsigned Int = BuiltinID == NEON::BI__builtin_neon_vrshrd_n_u64
12684 ? Intrinsic::aarch64_neon_urshl
12685 : Intrinsic::aarch64_neon_srshl;
12687 int SV = cast<ConstantInt>(Ops[1])->getSExtValue();
12688 Ops[1] = ConstantInt::get(
Int64Ty, -SV);
12691 case NEON::BI__builtin_neon_vrsrad_n_u64:
12692 case NEON::BI__builtin_neon_vrsrad_n_s64: {
12693 unsigned Int = BuiltinID == NEON::BI__builtin_neon_vrsrad_n_u64
12694 ? Intrinsic::aarch64_neon_urshl
12695 : Intrinsic::aarch64_neon_srshl;
12699 {Ops[1], Builder.CreateSExt(Ops[2], Int64Ty)});
12702 case NEON::BI__builtin_neon_vshld_n_s64:
12703 case NEON::BI__builtin_neon_vshld_n_u64: {
12704 llvm::ConstantInt *Amt = cast<ConstantInt>(
EmitScalarExpr(
E->getArg(1)));
12706 Ops[0], ConstantInt::get(
Int64Ty, Amt->getZExtValue()),
"shld_n");
12708 case NEON::BI__builtin_neon_vshrd_n_s64: {
12709 llvm::ConstantInt *Amt = cast<ConstantInt>(
EmitScalarExpr(
E->getArg(1)));
12711 Ops[0], ConstantInt::get(
Int64Ty, std::min(
static_cast<uint64_t>(63),
12712 Amt->getZExtValue())),
12715 case NEON::BI__builtin_neon_vshrd_n_u64: {
12716 llvm::ConstantInt *Amt = cast<ConstantInt>(
EmitScalarExpr(
E->getArg(1)));
12717 uint64_t ShiftAmt = Amt->getZExtValue();
12719 if (ShiftAmt == 64)
12720 return ConstantInt::get(
Int64Ty, 0);
12721 return Builder.CreateLShr(Ops[0], ConstantInt::get(
Int64Ty, ShiftAmt),
12724 case NEON::BI__builtin_neon_vsrad_n_s64: {
12725 llvm::ConstantInt *Amt = cast<ConstantInt>(
EmitScalarExpr(
E->getArg(2)));
12727 Ops[1], ConstantInt::get(
Int64Ty, std::min(
static_cast<uint64_t>(63),
12728 Amt->getZExtValue())),
12730 return Builder.CreateAdd(Ops[0], Ops[1]);
12732 case NEON::BI__builtin_neon_vsrad_n_u64: {
12733 llvm::ConstantInt *Amt = cast<ConstantInt>(
EmitScalarExpr(
E->getArg(2)));
12734 uint64_t ShiftAmt = Amt->getZExtValue();
12737 if (ShiftAmt == 64)
12739 Ops[1] =
Builder.CreateLShr(Ops[1], ConstantInt::get(
Int64Ty, ShiftAmt),
12741 return Builder.CreateAdd(Ops[0], Ops[1]);
12743 case NEON::BI__builtin_neon_vqdmlalh_lane_s16:
12744 case NEON::BI__builtin_neon_vqdmlalh_laneq_s16:
12745 case NEON::BI__builtin_neon_vqdmlslh_lane_s16:
12746 case NEON::BI__builtin_neon_vqdmlslh_laneq_s16: {
12752 auto *VTy = llvm::FixedVectorType::get(
Int32Ty, 4);
12754 ProductOps,
"vqdmlXl");
12755 Constant *CI = ConstantInt::get(
SizeTy, 0);
12756 Ops[1] =
Builder.CreateExtractElement(Ops[1], CI,
"lane0");
12759 unsigned AccInt = (BuiltinID == NEON::BI__builtin_neon_vqdmlalh_lane_s16 ||
12760 BuiltinID == NEON::BI__builtin_neon_vqdmlalh_laneq_s16)
12761 ? Intrinsic::aarch64_neon_sqadd
12762 : Intrinsic::aarch64_neon_sqsub;
12765 case NEON::BI__builtin_neon_vqdmlals_s32:
12766 case NEON::BI__builtin_neon_vqdmlsls_s32: {
12768 ProductOps.push_back(Ops[1]);
12772 ProductOps,
"vqdmlXl");
12774 unsigned AccumInt = BuiltinID == NEON::BI__builtin_neon_vqdmlals_s32
12775 ? Intrinsic::aarch64_neon_sqadd
12776 : Intrinsic::aarch64_neon_sqsub;
12779 case NEON::BI__builtin_neon_vqdmlals_lane_s32:
12780 case NEON::BI__builtin_neon_vqdmlals_laneq_s32:
12781 case NEON::BI__builtin_neon_vqdmlsls_lane_s32:
12782 case NEON::BI__builtin_neon_vqdmlsls_laneq_s32: {
12786 ProductOps.push_back(Ops[1]);
12787 ProductOps.push_back(Ops[2]);
12790 ProductOps,
"vqdmlXl");
12793 unsigned AccInt = (BuiltinID == NEON::BI__builtin_neon_vqdmlals_lane_s32 ||
12794 BuiltinID == NEON::BI__builtin_neon_vqdmlals_laneq_s32)
12795 ? Intrinsic::aarch64_neon_sqadd
12796 : Intrinsic::aarch64_neon_sqsub;
12799 case NEON::BI__builtin_neon_vget_lane_bf16:
12800 case NEON::BI__builtin_neon_vduph_lane_bf16:
12801 case NEON::BI__builtin_neon_vduph_lane_f16: {
12805 case NEON::BI__builtin_neon_vgetq_lane_bf16:
12806 case NEON::BI__builtin_neon_vduph_laneq_bf16:
12807 case NEON::BI__builtin_neon_vduph_laneq_f16: {
12812 case clang::AArch64::BI_InterlockedAdd:
12813 case clang::AArch64::BI_InterlockedAdd64: {
12816 AtomicRMWInst *RMWI =
12818 llvm::AtomicOrdering::SequentiallyConsistent);
12819 return Builder.CreateAdd(RMWI, Val);
12824 llvm::Type *Ty = VTy;
12835 Builtin->BuiltinID, Builtin->LLVMIntrinsic, Builtin->AltLLVMIntrinsic,
12836 Builtin->NameHint, Builtin->TypeModifier,
E, Ops,
12843 switch (BuiltinID) {
12844 default:
return nullptr;
12845 case NEON::BI__builtin_neon_vbsl_v:
12846 case NEON::BI__builtin_neon_vbslq_v: {
12847 llvm::Type *BitTy = llvm::VectorType::getInteger(VTy);
12848 Ops[0] =
Builder.CreateBitCast(Ops[0], BitTy,
"vbsl");
12849 Ops[1] =
Builder.CreateBitCast(Ops[1], BitTy,
"vbsl");
12850 Ops[2] =
Builder.CreateBitCast(Ops[2], BitTy,
"vbsl");
12852 Ops[1] =
Builder.CreateAnd(Ops[0], Ops[1],
"vbsl");
12853 Ops[2] =
Builder.CreateAnd(
Builder.CreateNot(Ops[0]), Ops[2],
"vbsl");
12854 Ops[0] =
Builder.CreateOr(Ops[1], Ops[2],
"vbsl");
12855 return Builder.CreateBitCast(Ops[0], Ty);
12857 case NEON::BI__builtin_neon_vfma_lane_v:
12858 case NEON::BI__builtin_neon_vfmaq_lane_v: {
12861 Value *Addend = Ops[0];
12862 Value *Multiplicand = Ops[1];
12863 Value *LaneSource = Ops[2];
12864 Ops[0] = Multiplicand;
12865 Ops[1] = LaneSource;
12869 auto *SourceTy = BuiltinID == NEON::BI__builtin_neon_vfmaq_lane_v
12870 ? llvm::FixedVectorType::get(VTy->getElementType(),
12871 VTy->getNumElements() / 2)
12873 llvm::Constant *cst = cast<Constant>(Ops[3]);
12874 Value *SV = llvm::ConstantVector::getSplat(VTy->getElementCount(), cst);
12875 Ops[1] =
Builder.CreateBitCast(Ops[1], SourceTy);
12876 Ops[1] =
Builder.CreateShuffleVector(Ops[1], Ops[1], SV,
"lane");
12879 Int =
Builder.getIsFPConstrained() ? Intrinsic::experimental_constrained_fma
12883 case NEON::BI__builtin_neon_vfma_laneq_v: {
12884 auto *VTy = cast<llvm::FixedVectorType>(Ty);
12886 if (VTy && VTy->getElementType() ==
DoubleTy) {
12889 llvm::FixedVectorType *VTy =
12891 Ops[2] =
Builder.CreateBitCast(Ops[2], VTy);
12892 Ops[2] =
Builder.CreateExtractElement(Ops[2], Ops[3],
"extract");
12895 *
this, Intrinsic::fma, Intrinsic::experimental_constrained_fma,
12896 DoubleTy, {Ops[1], Ops[2], Ops[0]});
12899 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
12900 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
12902 auto *STy = llvm::FixedVectorType::get(VTy->getElementType(),
12903 VTy->getNumElements() * 2);
12904 Ops[2] =
Builder.CreateBitCast(Ops[2], STy);
12905 Value *SV = llvm::ConstantVector::getSplat(VTy->getElementCount(),
12906 cast<ConstantInt>(Ops[3]));
12907 Ops[2] =
Builder.CreateShuffleVector(Ops[2], Ops[2], SV,
"lane");
12910 *
this, Intrinsic::fma, Intrinsic::experimental_constrained_fma, Ty,
12911 {Ops[2], Ops[1], Ops[0]});
12913 case NEON::BI__builtin_neon_vfmaq_laneq_v: {
12914 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
12915 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
12917 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
12920 *
this, Intrinsic::fma, Intrinsic::experimental_constrained_fma, Ty,
12921 {Ops[2], Ops[1], Ops[0]});
12923 case NEON::BI__builtin_neon_vfmah_lane_f16:
12924 case NEON::BI__builtin_neon_vfmas_lane_f32:
12925 case NEON::BI__builtin_neon_vfmah_laneq_f16:
12926 case NEON::BI__builtin_neon_vfmas_laneq_f32:
12927 case NEON::BI__builtin_neon_vfmad_lane_f64:
12928 case NEON::BI__builtin_neon_vfmad_laneq_f64: {
12931 Ops[2] =
Builder.CreateExtractElement(Ops[2], Ops[3],
"extract");
12933 *
this, Intrinsic::fma, Intrinsic::experimental_constrained_fma, Ty,
12934 {Ops[1], Ops[2], Ops[0]});
12936 case NEON::BI__builtin_neon_vmull_v:
12938 Int = usgn ? Intrinsic::aarch64_neon_umull : Intrinsic::aarch64_neon_smull;
12939 if (
Type.isPoly())
Int = Intrinsic::aarch64_neon_pmull;
12941 case NEON::BI__builtin_neon_vmax_v:
12942 case NEON::BI__builtin_neon_vmaxq_v:
12944 Int = usgn ? Intrinsic::aarch64_neon_umax : Intrinsic::aarch64_neon_smax;
12945 if (Ty->isFPOrFPVectorTy())
Int = Intrinsic::aarch64_neon_fmax;
12947 case NEON::BI__builtin_neon_vmaxh_f16: {
12949 Int = Intrinsic::aarch64_neon_fmax;
12952 case NEON::BI__builtin_neon_vmin_v:
12953 case NEON::BI__builtin_neon_vminq_v:
12955 Int = usgn ? Intrinsic::aarch64_neon_umin : Intrinsic::aarch64_neon_smin;
12956 if (Ty->isFPOrFPVectorTy())
Int = Intrinsic::aarch64_neon_fmin;
12958 case NEON::BI__builtin_neon_vminh_f16: {
12960 Int = Intrinsic::aarch64_neon_fmin;
12963 case NEON::BI__builtin_neon_vabd_v:
12964 case NEON::BI__builtin_neon_vabdq_v:
12966 Int = usgn ? Intrinsic::aarch64_neon_uabd : Intrinsic::aarch64_neon_sabd;
12967 if (Ty->isFPOrFPVectorTy())
Int = Intrinsic::aarch64_neon_fabd;
12969 case NEON::BI__builtin_neon_vpadal_v:
12970 case NEON::BI__builtin_neon_vpadalq_v: {
12971 unsigned ArgElts = VTy->getNumElements();
12972 llvm::IntegerType *EltTy = cast<IntegerType>(VTy->getElementType());
12973 unsigned BitWidth = EltTy->getBitWidth();
12974 auto *ArgTy = llvm::FixedVectorType::get(
12975 llvm::IntegerType::get(
getLLVMContext(), BitWidth / 2), 2 * ArgElts);
12976 llvm::Type* Tys[2] = { VTy, ArgTy };
12977 Int = usgn ? Intrinsic::aarch64_neon_uaddlp : Intrinsic::aarch64_neon_saddlp;
12979 TmpOps.push_back(Ops[1]);
12982 llvm::Value *addend =
Builder.CreateBitCast(Ops[0], tmp->getType());
12983 return Builder.CreateAdd(tmp, addend);
12985 case NEON::BI__builtin_neon_vpmin_v:
12986 case NEON::BI__builtin_neon_vpminq_v:
12988 Int = usgn ? Intrinsic::aarch64_neon_uminp : Intrinsic::aarch64_neon_sminp;
12989 if (Ty->isFPOrFPVectorTy())
Int = Intrinsic::aarch64_neon_fminp;
12991 case NEON::BI__builtin_neon_vpmax_v:
12992 case NEON::BI__builtin_neon_vpmaxq_v:
12994 Int = usgn ? Intrinsic::aarch64_neon_umaxp : Intrinsic::aarch64_neon_smaxp;
12995 if (Ty->isFPOrFPVectorTy())
Int = Intrinsic::aarch64_neon_fmaxp;
12997 case NEON::BI__builtin_neon_vminnm_v:
12998 case NEON::BI__builtin_neon_vminnmq_v:
12999 Int = Intrinsic::aarch64_neon_fminnm;
13001 case NEON::BI__builtin_neon_vminnmh_f16:
13003 Int = Intrinsic::aarch64_neon_fminnm;
13005 case NEON::BI__builtin_neon_vmaxnm_v:
13006 case NEON::BI__builtin_neon_vmaxnmq_v:
13007 Int = Intrinsic::aarch64_neon_fmaxnm;
13009 case NEON::BI__builtin_neon_vmaxnmh_f16:
13011 Int = Intrinsic::aarch64_neon_fmaxnm;
13013 case NEON::BI__builtin_neon_vrecpss_f32: {
13018 case NEON::BI__builtin_neon_vrecpsd_f64:
13022 case NEON::BI__builtin_neon_vrecpsh_f16:
13026 case NEON::BI__builtin_neon_vqshrun_n_v:
13027 Int = Intrinsic::aarch64_neon_sqshrun;
13029 case NEON::BI__builtin_neon_vqrshrun_n_v:
13030 Int = Intrinsic::aarch64_neon_sqrshrun;
13032 case NEON::BI__builtin_neon_vqshrn_n_v:
13033 Int = usgn ? Intrinsic::aarch64_neon_uqshrn : Intrinsic::aarch64_neon_sqshrn;
13035 case NEON::BI__builtin_neon_vrshrn_n_v:
13036 Int = Intrinsic::aarch64_neon_rshrn;
13038 case NEON::BI__builtin_neon_vqrshrn_n_v:
13039 Int = usgn ? Intrinsic::aarch64_neon_uqrshrn : Intrinsic::aarch64_neon_sqrshrn;
13041 case NEON::BI__builtin_neon_vrndah_f16: {
13044 ? Intrinsic::experimental_constrained_round
13045 : Intrinsic::round;
13048 case NEON::BI__builtin_neon_vrnda_v:
13049 case NEON::BI__builtin_neon_vrndaq_v: {
13051 ? Intrinsic::experimental_constrained_round
13052 : Intrinsic::round;
13055 case NEON::BI__builtin_neon_vrndih_f16: {
13058 ? Intrinsic::experimental_constrained_nearbyint
13059 : Intrinsic::nearbyint;
13062 case NEON::BI__builtin_neon_vrndmh_f16: {
13065 ? Intrinsic::experimental_constrained_floor
13066 : Intrinsic::floor;
13069 case NEON::BI__builtin_neon_vrndm_v:
13070 case NEON::BI__builtin_neon_vrndmq_v: {
13072 ? Intrinsic::experimental_constrained_floor
13073 : Intrinsic::floor;
13076 case NEON::BI__builtin_neon_vrndnh_f16: {
13079 ? Intrinsic::experimental_constrained_roundeven
13080 : Intrinsic::roundeven;
13083 case NEON::BI__builtin_neon_vrndn_v:
13084 case NEON::BI__builtin_neon_vrndnq_v: {
13086 ? Intrinsic::experimental_constrained_roundeven
13087 : Intrinsic::roundeven;
13090 case NEON::BI__builtin_neon_vrndns_f32: {
13093 ? Intrinsic::experimental_constrained_roundeven
13094 : Intrinsic::roundeven;
13097 case NEON::BI__builtin_neon_vrndph_f16: {
13100 ? Intrinsic::experimental_constrained_ceil
13104 case NEON::BI__builtin_neon_vrndp_v:
13105 case NEON::BI__builtin_neon_vrndpq_v: {
13107 ? Intrinsic::experimental_constrained_ceil
13111 case NEON::BI__builtin_neon_vrndxh_f16: {
13114 ? Intrinsic::experimental_constrained_rint
13118 case NEON::BI__builtin_neon_vrndx_v:
13119 case NEON::BI__builtin_neon_vrndxq_v: {
13121 ? Intrinsic::experimental_constrained_rint
13125 case NEON::BI__builtin_neon_vrndh_f16: {
13128 ? Intrinsic::experimental_constrained_trunc
13129 : Intrinsic::trunc;
13132 case NEON::BI__builtin_neon_vrnd32x_f32:
13133 case NEON::BI__builtin_neon_vrnd32xq_f32:
13134 case NEON::BI__builtin_neon_vrnd32x_f64:
13135 case NEON::BI__builtin_neon_vrnd32xq_f64: {
13137 Int = Intrinsic::aarch64_neon_frint32x;
13140 case NEON::BI__builtin_neon_vrnd32z_f32:
13141 case NEON::BI__builtin_neon_vrnd32zq_f32:
13142 case NEON::BI__builtin_neon_vrnd32z_f64:
13143 case NEON::BI__builtin_neon_vrnd32zq_f64: {
13145 Int = Intrinsic::aarch64_neon_frint32z;
13148 case NEON::BI__builtin_neon_vrnd64x_f32:
13149 case NEON::BI__builtin_neon_vrnd64xq_f32:
13150 case NEON::BI__builtin_neon_vrnd64x_f64:
13151 case NEON::BI__builtin_neon_vrnd64xq_f64: {
13153 Int = Intrinsic::aarch64_neon_frint64x;
13156 case NEON::BI__builtin_neon_vrnd64z_f32:
13157 case NEON::BI__builtin_neon_vrnd64zq_f32:
13158 case NEON::BI__builtin_neon_vrnd64z_f64:
13159 case NEON::BI__builtin_neon_vrnd64zq_f64: {
13161 Int = Intrinsic::aarch64_neon_frint64z;
13164 case NEON::BI__builtin_neon_vrnd_v:
13165 case NEON::BI__builtin_neon_vrndq_v: {
13167 ? Intrinsic::experimental_constrained_trunc
13168 : Intrinsic::trunc;
13171 case NEON::BI__builtin_neon_vcvt_f64_v:
13172 case NEON::BI__builtin_neon_vcvtq_f64_v:
13173 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
13175 return usgn ?
Builder.CreateUIToFP(Ops[0], Ty,
"vcvt")
13176 :
Builder.CreateSIToFP(Ops[0], Ty,
"vcvt");
13177 case NEON::BI__builtin_neon_vcvt_f64_f32: {
13179 "unexpected vcvt_f64_f32 builtin");
13183 return Builder.CreateFPExt(Ops[0], Ty,
"vcvt");
13185 case NEON::BI__builtin_neon_vcvt_f32_f64: {
13187 "unexpected vcvt_f32_f64 builtin");
13191 return Builder.CreateFPTrunc(Ops[0], Ty,
"vcvt");
13193 case NEON::BI__builtin_neon_vcvt_s32_v:
13194 case NEON::BI__builtin_neon_vcvt_u32_v:
13195 case NEON::BI__builtin_neon_vcvt_s64_v:
13196 case NEON::BI__builtin_neon_vcvt_u64_v:
13197 case NEON::BI__builtin_neon_vcvt_s16_f16:
13198 case NEON::BI__builtin_neon_vcvt_u16_f16:
13199 case NEON::BI__builtin_neon_vcvtq_s32_v:
13200 case NEON::BI__builtin_neon_vcvtq_u32_v:
13201 case NEON::BI__builtin_neon_vcvtq_s64_v:
13202 case NEON::BI__builtin_neon_vcvtq_u64_v:
13203 case NEON::BI__builtin_neon_vcvtq_s16_f16:
13204 case NEON::BI__builtin_neon_vcvtq_u16_f16: {
13206 usgn ? Intrinsic::aarch64_neon_fcvtzu : Intrinsic::aarch64_neon_fcvtzs;
13210 case NEON::BI__builtin_neon_vcvta_s16_f16:
13211 case NEON::BI__builtin_neon_vcvta_u16_f16:
13212 case NEON::BI__builtin_neon_vcvta_s32_v:
13213 case NEON::BI__builtin_neon_vcvtaq_s16_f16:
13214 case NEON::BI__builtin_neon_vcvtaq_s32_v:
13215 case NEON::BI__builtin_neon_vcvta_u32_v:
13216 case NEON::BI__builtin_neon_vcvtaq_u16_f16:
13217 case NEON::BI__builtin_neon_vcvtaq_u32_v:
13218 case NEON::BI__builtin_neon_vcvta_s64_v:
13219 case NEON::BI__builtin_neon_vcvtaq_s64_v:
13220 case NEON::BI__builtin_neon_vcvta_u64_v:
13221 case NEON::BI__builtin_neon_vcvtaq_u64_v: {
13222 Int = usgn ? Intrinsic::aarch64_neon_fcvtau : Intrinsic::aarch64_neon_fcvtas;
13226 case NEON::BI__builtin_neon_vcvtm_s16_f16:
13227 case NEON::BI__builtin_neon_vcvtm_s32_v:
13228 case NEON::BI__builtin_neon_vcvtmq_s16_f16:
13229 case NEON::BI__builtin_neon_vcvtmq_s32_v:
13230 case NEON::BI__builtin_neon_vcvtm_u16_f16:
13231 case NEON::BI__builtin_neon_vcvtm_u32_v:
13232 case NEON::BI__builtin_neon_vcvtmq_u16_f16:
13233 case NEON::BI__builtin_neon_vcvtmq_u32_v:
13234 case NEON::BI__builtin_neon_vcvtm_s64_v:
13235 case NEON::BI__builtin_neon_vcvtmq_s64_v:
13236 case NEON::BI__builtin_neon_vcvtm_u64_v:
13237 case NEON::BI__builtin_neon_vcvtmq_u64_v: {
13238 Int = usgn ? Intrinsic::aarch64_neon_fcvtmu : Intrinsic::aarch64_neon_fcvtms;
13242 case NEON::BI__builtin_neon_vcvtn_s16_f16:
13243 case NEON::BI__builtin_neon_vcvtn_s32_v:
13244 case NEON::BI__builtin_neon_vcvtnq_s16_f16:
13245 case NEON::BI__builtin_neon_vcvtnq_s32_v:
13246 case NEON::BI__builtin_neon_vcvtn_u16_f16:
13247 case NEON::BI__builtin_neon_vcvtn_u32_v:
13248 case NEON::BI__builtin_neon_vcvtnq_u16_f16:
13249 case NEON::BI__builtin_neon_vcvtnq_u32_v:
13250 case NEON::BI__builtin_neon_vcvtn_s64_v:
13251 case NEON::BI__builtin_neon_vcvtnq_s64_v:
13252 case NEON::BI__builtin_neon_vcvtn_u64_v:
13253 case NEON::BI__builtin_neon_vcvtnq_u64_v: {
13254 Int = usgn ? Intrinsic::aarch64_neon_fcvtnu : Intrinsic::aarch64_neon_fcvtns;
13258 case NEON::BI__builtin_neon_vcvtp_s16_f16:
13259 case NEON::BI__builtin_neon_vcvtp_s32_v:
13260 case NEON::BI__builtin_neon_vcvtpq_s16_f16:
13261 case NEON::BI__builtin_neon_vcvtpq_s32_v:
13262 case NEON::BI__builtin_neon_vcvtp_u16_f16:
13263 case NEON::BI__builtin_neon_vcvtp_u32_v:
13264 case NEON::BI__builtin_neon_vcvtpq_u16_f16:
13265 case NEON::BI__builtin_neon_vcvtpq_u32_v:
13266 case NEON::BI__builtin_neon_vcvtp_s64_v:
13267 case NEON::BI__builtin_neon_vcvtpq_s64_v:
13268 case NEON::BI__builtin_neon_vcvtp_u64_v:
13269 case NEON::BI__builtin_neon_vcvtpq_u64_v: {
13270 Int = usgn ? Intrinsic::aarch64_neon_fcvtpu : Intrinsic::aarch64_neon_fcvtps;
13274 case NEON::BI__builtin_neon_vmulx_v:
13275 case NEON::BI__builtin_neon_vmulxq_v: {
13276 Int = Intrinsic::aarch64_neon_fmulx;
13279 case NEON::BI__builtin_neon_vmulxh_lane_f16:
13280 case NEON::BI__builtin_neon_vmulxh_laneq_f16: {
13284 Ops[1] =
Builder.CreateExtractElement(Ops[1], Ops[2],
"extract");
13286 Int = Intrinsic::aarch64_neon_fmulx;
13289 case NEON::BI__builtin_neon_vmul_lane_v:
13290 case NEON::BI__builtin_neon_vmul_laneq_v: {
13293 if (BuiltinID == NEON::BI__builtin_neon_vmul_laneq_v)
13296 llvm::FixedVectorType *VTy =
13298 Ops[1] =
Builder.CreateBitCast(Ops[1], VTy);
13299 Ops[1] =
Builder.CreateExtractElement(Ops[1], Ops[2],
"extract");
13303 case NEON::BI__builtin_neon_vnegd_s64:
13305 case NEON::BI__builtin_neon_vnegh_f16:
13307 case NEON::BI__builtin_neon_vpmaxnm_v:
13308 case NEON::BI__builtin_neon_vpmaxnmq_v: {
13309 Int = Intrinsic::aarch64_neon_fmaxnmp;
13312 case NEON::BI__builtin_neon_vpminnm_v:
13313 case NEON::BI__builtin_neon_vpminnmq_v: {
13314 Int = Intrinsic::aarch64_neon_fminnmp;
13317 case NEON::BI__builtin_neon_vsqrth_f16: {
13320 ? Intrinsic::experimental_constrained_sqrt
13324 case NEON::BI__builtin_neon_vsqrt_v:
13325 case NEON::BI__builtin_neon_vsqrtq_v: {
13327 ? Intrinsic::experimental_constrained_sqrt
13329 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
13332 case NEON::BI__builtin_neon_vrbit_v:
13333 case NEON::BI__builtin_neon_vrbitq_v: {
13334 Int = Intrinsic::bitreverse;
13337 case NEON::BI__builtin_neon_vaddv_u8:
13341 case NEON::BI__builtin_neon_vaddv_s8: {
13342 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
13344 VTy = llvm::FixedVectorType::get(
Int8Ty, 8);
13345 llvm::Type *Tys[2] = { Ty, VTy };
13350 case NEON::BI__builtin_neon_vaddv_u16:
13353 case NEON::BI__builtin_neon_vaddv_s16: {
13354 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
13356 VTy = llvm::FixedVectorType::get(
Int16Ty, 4);
13357 llvm::Type *Tys[2] = { Ty, VTy };
13362 case NEON::BI__builtin_neon_vaddvq_u8:
13365 case NEON::BI__builtin_neon_vaddvq_s8: {
13366 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
13368 VTy = llvm::FixedVectorType::get(
Int8Ty, 16);
13369 llvm::Type *Tys[2] = { Ty, VTy };
13374 case NEON::BI__builtin_neon_vaddvq_u16:
13377 case NEON::BI__builtin_neon_vaddvq_s16: {
13378 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
13380 VTy = llvm::FixedVectorType::get(
Int16Ty, 8);
13381 llvm::Type *Tys[2] = { Ty, VTy };
13386 case NEON::BI__builtin_neon_vmaxv_u8: {
13387 Int = Intrinsic::aarch64_neon_umaxv;
13389 VTy = llvm::FixedVectorType::get(
Int8Ty, 8);
13390 llvm::Type *Tys[2] = { Ty, VTy };
13395 case NEON::BI__builtin_neon_vmaxv_u16: {
13396 Int = Intrinsic::aarch64_neon_umaxv;
13398 VTy = llvm::FixedVectorType::get(
Int16Ty, 4);
13399 llvm::Type *Tys[2] = { Ty, VTy };
13404 case NEON::BI__builtin_neon_vmaxvq_u8: {
13405 Int = Intrinsic::aarch64_neon_umaxv;
13407 VTy = llvm::FixedVectorType::get(
Int8Ty, 16);
13408 llvm::Type *Tys[2] = { Ty, VTy };
13413 case NEON::BI__builtin_neon_vmaxvq_u16: {
13414 Int = Intrinsic::aarch64_neon_umaxv;
13416 VTy = llvm::FixedVectorType::get(
Int16Ty, 8);
13417 llvm::Type *Tys[2] = { Ty, VTy };
13422 case NEON::BI__builtin_neon_vmaxv_s8: {
13423 Int = Intrinsic::aarch64_neon_smaxv;
13425 VTy = llvm::FixedVectorType::get(
Int8Ty, 8);
13426 llvm::Type *Tys[2] = { Ty, VTy };
13431 case NEON::BI__builtin_neon_vmaxv_s16: {
13432 Int = Intrinsic::aarch64_neon_smaxv;
13434 VTy = llvm::FixedVectorType::get(
Int16Ty, 4);
13435 llvm::Type *Tys[2] = { Ty, VTy };
13440 case NEON::BI__builtin_neon_vmaxvq_s8: {
13441 Int = Intrinsic::aarch64_neon_smaxv;
13443 VTy = llvm::FixedVectorType::get(
Int8Ty, 16);
13444 llvm::Type *Tys[2] = { Ty, VTy };
13449 case NEON::BI__builtin_neon_vmaxvq_s16: {
13450 Int = Intrinsic::aarch64_neon_smaxv;
13452 VTy = llvm::FixedVectorType::get(
Int16Ty, 8);
13453 llvm::Type *Tys[2] = { Ty, VTy };
13458 case NEON::BI__builtin_neon_vmaxv_f16: {
13459 Int = Intrinsic::aarch64_neon_fmaxv;
13461 VTy = llvm::FixedVectorType::get(
HalfTy, 4);
13462 llvm::Type *Tys[2] = { Ty, VTy };
13467 case NEON::BI__builtin_neon_vmaxvq_f16: {
13468 Int = Intrinsic::aarch64_neon_fmaxv;
13470 VTy = llvm::FixedVectorType::get(
HalfTy, 8);
13471 llvm::Type *Tys[2] = { Ty, VTy };
13476 case NEON::BI__builtin_neon_vminv_u8: {
13477 Int = Intrinsic::aarch64_neon_uminv;
13479 VTy = llvm::FixedVectorType::get(
Int8Ty, 8);
13480 llvm::Type *Tys[2] = { Ty, VTy };
13485 case NEON::BI__builtin_neon_vminv_u16: {
13486 Int = Intrinsic::aarch64_neon_uminv;
13488 VTy = llvm::FixedVectorType::get(
Int16Ty, 4);
13489 llvm::Type *Tys[2] = { Ty, VTy };
13494 case NEON::BI__builtin_neon_vminvq_u8: {
13495 Int = Intrinsic::aarch64_neon_uminv;
13497 VTy = llvm::FixedVectorType::get(
Int8Ty, 16);
13498 llvm::Type *Tys[2] = { Ty, VTy };
13503 case NEON::BI__builtin_neon_vminvq_u16: {
13504 Int = Intrinsic::aarch64_neon_uminv;
13506 VTy = llvm::FixedVectorType::get(
Int16Ty, 8);
13507 llvm::Type *Tys[2] = { Ty, VTy };
13512 case NEON::BI__builtin_neon_vminv_s8: {
13513 Int = Intrinsic::aarch64_neon_sminv;
13515 VTy = llvm::FixedVectorType::get(
Int8Ty, 8);
13516 llvm::Type *Tys[2] = { Ty, VTy };
13521 case NEON::BI__builtin_neon_vminv_s16: {
13522 Int = Intrinsic::aarch64_neon_sminv;
13524 VTy = llvm::FixedVectorType::get(
Int16Ty, 4);
13525 llvm::Type *Tys[2] = { Ty, VTy };
13530 case NEON::BI__builtin_neon_vminvq_s8: {
13531 Int = Intrinsic::aarch64_neon_sminv;
13533 VTy = llvm::FixedVectorType::get(
Int8Ty, 16);
13534 llvm::Type *Tys[2] = { Ty, VTy };
13539 case NEON::BI__builtin_neon_vminvq_s16: {
13540 Int = Intrinsic::aarch64_neon_sminv;
13542 VTy = llvm::FixedVectorType::get(
Int16Ty, 8);
13543 llvm::Type *Tys[2] = { Ty, VTy };
13548 case NEON::BI__builtin_neon_vminv_f16: {
13549 Int = Intrinsic::aarch64_neon_fminv;
13551 VTy = llvm::FixedVectorType::get(
HalfTy, 4);
13552 llvm::Type *Tys[2] = { Ty, VTy };
13557 case NEON::BI__builtin_neon_vminvq_f16: {
13558 Int = Intrinsic::aarch64_neon_fminv;
13560 VTy = llvm::FixedVectorType::get(
HalfTy, 8);
13561 llvm::Type *Tys[2] = { Ty, VTy };
13566 case NEON::BI__builtin_neon_vmaxnmv_f16: {
13567 Int = Intrinsic::aarch64_neon_fmaxnmv;
13569 VTy = llvm::FixedVectorType::get(
HalfTy, 4);
13570 llvm::Type *Tys[2] = { Ty, VTy };
13575 case NEON::BI__builtin_neon_vmaxnmvq_f16: {
13576 Int = Intrinsic::aarch64_neon_fmaxnmv;
13578 VTy = llvm::FixedVectorType::get(
HalfTy, 8);
13579 llvm::Type *Tys[2] = { Ty, VTy };
13584 case NEON::BI__builtin_neon_vminnmv_f16: {
13585 Int = Intrinsic::aarch64_neon_fminnmv;
13587 VTy = llvm::FixedVectorType::get(
HalfTy, 4);
13588 llvm::Type *Tys[2] = { Ty, VTy };
13593 case NEON::BI__builtin_neon_vminnmvq_f16: {
13594 Int = Intrinsic::aarch64_neon_fminnmv;
13596 VTy = llvm::FixedVectorType::get(
HalfTy, 8);
13597 llvm::Type *Tys[2] = { Ty, VTy };
13602 case NEON::BI__builtin_neon_vmul_n_f64: {
13605 return Builder.CreateFMul(Ops[0], RHS);
13607 case NEON::BI__builtin_neon_vaddlv_u8: {
13608 Int = Intrinsic::aarch64_neon_uaddlv;
13610 VTy = llvm::FixedVectorType::get(
Int8Ty, 8);
13611 llvm::Type *Tys[2] = { Ty, VTy };
13616 case NEON::BI__builtin_neon_vaddlv_u16: {
13617 Int = Intrinsic::aarch64_neon_uaddlv;
13619 VTy = llvm::FixedVectorType::get(
Int16Ty, 4);
13620 llvm::Type *Tys[2] = { Ty, VTy };
13624 case NEON::BI__builtin_neon_vaddlvq_u8: {
13625 Int = Intrinsic::aarch64_neon_uaddlv;
13627 VTy = llvm::FixedVectorType::get(
Int8Ty, 16);
13628 llvm::Type *Tys[2] = { Ty, VTy };
13633 case NEON::BI__builtin_neon_vaddlvq_u16: {
13634 Int = Intrinsic::aarch64_neon_uaddlv;
13636 VTy = llvm::FixedVectorType::get(
Int16Ty, 8);
13637 llvm::Type *Tys[2] = { Ty, VTy };
13641 case NEON::BI__builtin_neon_vaddlv_s8: {
13642 Int = Intrinsic::aarch64_neon_saddlv;
13644 VTy = llvm::FixedVectorType::get(
Int8Ty, 8);
13645 llvm::Type *Tys[2] = { Ty, VTy };
13650 case NEON::BI__builtin_neon_vaddlv_s16: {
13651 Int = Intrinsic::aarch64_neon_saddlv;
13653 VTy = llvm::FixedVectorType::get(
Int16Ty, 4);
13654 llvm::Type *Tys[2] = { Ty, VTy };
13658 case NEON::BI__builtin_neon_vaddlvq_s8: {
13659 Int = Intrinsic::aarch64_neon_saddlv;
13661 VTy = llvm::FixedVectorType::get(
Int8Ty, 16);
13662 llvm::Type *Tys[2] = { Ty, VTy };
13667 case NEON::BI__builtin_neon_vaddlvq_s16: {
13668 Int = Intrinsic::aarch64_neon_saddlv;
13670 VTy = llvm::FixedVectorType::get(
Int16Ty, 8);
13671 llvm::Type *Tys[2] = { Ty, VTy };
13675 case NEON::BI__builtin_neon_vsri_n_v:
13676 case NEON::BI__builtin_neon_vsriq_n_v: {
13677 Int = Intrinsic::aarch64_neon_vsri;
13681 case NEON::BI__builtin_neon_vsli_n_v:
13682 case NEON::BI__builtin_neon_vsliq_n_v: {
13683 Int = Intrinsic::aarch64_neon_vsli;
13687 case NEON::BI__builtin_neon_vsra_n_v:
13688 case NEON::BI__builtin_neon_vsraq_n_v:
13689 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
13691 return Builder.CreateAdd(Ops[0], Ops[1]);
13692 case NEON::BI__builtin_neon_vrsra_n_v:
13693 case NEON::BI__builtin_neon_vrsraq_n_v: {
13694 Int = usgn ? Intrinsic::aarch64_neon_urshl : Intrinsic::aarch64_neon_srshl;
13696 TmpOps.push_back(Ops[1]);
13697 TmpOps.push_back(Ops[2]);
13699 llvm::Value *tmp =
EmitNeonCall(F, TmpOps,
"vrshr_n", 1,
true);
13700 Ops[0] =
Builder.CreateBitCast(Ops[0], VTy);
13701 return Builder.CreateAdd(Ops[0], tmp);
13703 case NEON::BI__builtin_neon_vld1_v:
13704 case NEON::BI__builtin_neon_vld1q_v: {
13707 case NEON::BI__builtin_neon_vst1_v:
13708 case NEON::BI__builtin_neon_vst1q_v:
13709 Ops[1] =
Builder.CreateBitCast(Ops[1], VTy);
13711 case NEON::BI__builtin_neon_vld1_lane_v:
13712 case NEON::BI__builtin_neon_vld1q_lane_v: {
13713 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
13716 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vld1_lane");
13718 case NEON::BI__builtin_neon_vldap1_lane_s64:
13719 case NEON::BI__builtin_neon_vldap1q_lane_s64: {
13720 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
13722 VTy->getElementType(), Ops[0], PtrOp0.
getAlignment());
13723 LI->setAtomic(llvm::AtomicOrdering::Acquire);
13725 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vldap1_lane");
13727 case NEON::BI__builtin_neon_vld1_dup_v:
13728 case NEON::BI__builtin_neon_vld1q_dup_v: {
13729 Value *
V = PoisonValue::get(Ty);
13732 llvm::Constant *CI = ConstantInt::get(
Int32Ty, 0);
13733 Ops[0] =
Builder.CreateInsertElement(
V, Ops[0], CI);
13736 case NEON::BI__builtin_neon_vst1_lane_v:
13737 case NEON::BI__builtin_neon_vst1q_lane_v:
13738 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
13739 Ops[1] =
Builder.CreateExtractElement(Ops[1], Ops[2]);
13741 case NEON::BI__builtin_neon_vstl1_lane_s64:
13742 case NEON::BI__builtin_neon_vstl1q_lane_s64: {
13743 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
13744 Ops[1] =
Builder.CreateExtractElement(Ops[1], Ops[2]);
13745 llvm::StoreInst *SI =
13747 SI->setAtomic(llvm::AtomicOrdering::Release);
13750 case NEON::BI__builtin_neon_vld2_v:
13751 case NEON::BI__builtin_neon_vld2q_v: {
13754 Ops[1] =
Builder.CreateCall(F, Ops[1],
"vld2");
13757 case NEON::BI__builtin_neon_vld3_v:
13758 case NEON::BI__builtin_neon_vld3q_v: {
13761 Ops[1] =
Builder.CreateCall(F, Ops[1],
"vld3");
13764 case NEON::BI__builtin_neon_vld4_v:
13765 case NEON::BI__builtin_neon_vld4q_v: {
13768 Ops[1] =
Builder.CreateCall(F, Ops[1],
"vld4");
13771 case NEON::BI__builtin_neon_vld2_dup_v:
13772 case NEON::BI__builtin_neon_vld2q_dup_v: {
13775 Ops[1] =
Builder.CreateCall(F, Ops[1],
"vld2");
13778 case NEON::BI__builtin_neon_vld3_dup_v:
13779 case NEON::BI__builtin_neon_vld3q_dup_v: {
13782 Ops[1] =
Builder.CreateCall(F, Ops[1],
"vld3");
13785 case NEON::BI__builtin_neon_vld4_dup_v:
13786 case NEON::BI__builtin_neon_vld4q_dup_v: {
13789 Ops[1] =
Builder.CreateCall(F, Ops[1],
"vld4");
13792 case NEON::BI__builtin_neon_vld2_lane_v:
13793 case NEON::BI__builtin_neon_vld2q_lane_v: {
13794 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
13796 std::rotate(Ops.begin() + 1, Ops.begin() + 2, Ops.end());
13797 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
13798 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
13803 case NEON::BI__builtin_neon_vld3_lane_v:
13804 case NEON::BI__builtin_neon_vld3q_lane_v: {
13805 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
13807 std::rotate(Ops.begin() + 1, Ops.begin() + 2, Ops.end());
13808 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
13809 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
13810 Ops[3] =
Builder.CreateBitCast(Ops[3], Ty);
13815 case NEON::BI__builtin_neon_vld4_lane_v:
13816 case NEON::BI__builtin_neon_vld4q_lane_v: {
13817 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
13819 std::rotate(Ops.begin() + 1, Ops.begin() + 2, Ops.end());
13820 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
13821 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
13822 Ops[3] =
Builder.CreateBitCast(Ops[3], Ty);
13823 Ops[4] =
Builder.CreateBitCast(Ops[4], Ty);
13828 case NEON::BI__builtin_neon_vst2_v:
13829 case NEON::BI__builtin_neon_vst2q_v: {
13830 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
13831 llvm::Type *Tys[2] = { VTy, Ops[2]->getType() };
13835 case NEON::BI__builtin_neon_vst2_lane_v:
13836 case NEON::BI__builtin_neon_vst2q_lane_v: {
13837 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
13839 llvm::Type *Tys[2] = { VTy, Ops[3]->getType() };
13843 case NEON::BI__builtin_neon_vst3_v:
13844 case NEON::BI__builtin_neon_vst3q_v: {
13845 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
13846 llvm::Type *Tys[2] = { VTy, Ops[3]->getType() };
13850 case NEON::BI__builtin_neon_vst3_lane_v:
13851 case NEON::BI__builtin_neon_vst3q_lane_v: {
13852 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
13854 llvm::Type *Tys[2] = { VTy, Ops[4]->getType() };
13858 case NEON::BI__builtin_neon_vst4_v:
13859 case NEON::BI__builtin_neon_vst4q_v: {
13860 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
13861 llvm::Type *Tys[2] = { VTy, Ops[4]->getType() };
13865 case NEON::BI__builtin_neon_vst4_lane_v:
13866 case NEON::BI__builtin_neon_vst4q_lane_v: {
13867 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
13869 llvm::Type *Tys[2] = { VTy, Ops[5]->getType() };
13873 case NEON::BI__builtin_neon_vtrn_v:
13874 case NEON::BI__builtin_neon_vtrnq_v: {
13875 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
13876 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
13877 Value *SV =
nullptr;
13879 for (
unsigned vi = 0; vi != 2; ++vi) {
13881 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
13882 Indices.push_back(i+vi);
13883 Indices.push_back(i+e+vi);
13885 Value *Addr =
Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
13886 SV =
Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vtrn");
13891 case NEON::BI__builtin_neon_vuzp_v:
13892 case NEON::BI__builtin_neon_vuzpq_v: {
13893 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
13894 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
13895 Value *SV =
nullptr;
13897 for (
unsigned vi = 0; vi != 2; ++vi) {
13899 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
13900 Indices.push_back(2*i+vi);
13902 Value *Addr =
Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
13903 SV =
Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vuzp");
13908 case NEON::BI__builtin_neon_vzip_v:
13909 case NEON::BI__builtin_neon_vzipq_v: {
13910 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
13911 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
13912 Value *SV =
nullptr;
13914 for (
unsigned vi = 0; vi != 2; ++vi) {
13916 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
13917 Indices.push_back((i + vi*e) >> 1);
13918 Indices.push_back(((i + vi*e) >> 1)+e);
13920 Value *Addr =
Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
13921 SV =
Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vzip");
13926 case NEON::BI__builtin_neon_vqtbl1q_v: {
13930 case NEON::BI__builtin_neon_vqtbl2q_v: {
13934 case NEON::BI__builtin_neon_vqtbl3q_v: {
13938 case NEON::BI__builtin_neon_vqtbl4q_v: {
13942 case NEON::BI__builtin_neon_vqtbx1q_v: {
13946 case NEON::BI__builtin_neon_vqtbx2q_v: {
13950 case NEON::BI__builtin_neon_vqtbx3q_v: {
13954 case NEON::BI__builtin_neon_vqtbx4q_v: {
13958 case NEON::BI__builtin_neon_vsqadd_v:
13959 case NEON::BI__builtin_neon_vsqaddq_v: {
13960 Int = Intrinsic::aarch64_neon_usqadd;
13963 case NEON::BI__builtin_neon_vuqadd_v:
13964 case NEON::BI__builtin_neon_vuqaddq_v: {
13965 Int = Intrinsic::aarch64_neon_suqadd;
13969 case NEON::BI__builtin_neon_vluti2_laneq_bf16:
13970 case NEON::BI__builtin_neon_vluti2_laneq_f16:
13971 case NEON::BI__builtin_neon_vluti2_laneq_p16:
13972 case NEON::BI__builtin_neon_vluti2_laneq_p8:
13973 case NEON::BI__builtin_neon_vluti2_laneq_s16:
13974 case NEON::BI__builtin_neon_vluti2_laneq_s8:
13975 case NEON::BI__builtin_neon_vluti2_laneq_u16:
13976 case NEON::BI__builtin_neon_vluti2_laneq_u8: {
13977 Int = Intrinsic::aarch64_neon_vluti2_laneq;
13978 llvm::Type *Tys[2];
13984 case NEON::BI__builtin_neon_vluti2q_laneq_bf16:
13985 case NEON::BI__builtin_neon_vluti2q_laneq_f16:
13986 case NEON::BI__builtin_neon_vluti2q_laneq_p16:
13987 case NEON::BI__builtin_neon_vluti2q_laneq_p8:
13988 case NEON::BI__builtin_neon_vluti2q_laneq_s16:
13989 case NEON::BI__builtin_neon_vluti2q_laneq_s8:
13990 case NEON::BI__builtin_neon_vluti2q_laneq_u16:
13991 case NEON::BI__builtin_neon_vluti2q_laneq_u8: {
13992 Int = Intrinsic::aarch64_neon_vluti2_laneq;
13993 llvm::Type *Tys[2];
13999 case NEON::BI__builtin_neon_vluti2_lane_bf16:
14000 case NEON::BI__builtin_neon_vluti2_lane_f16:
14001 case NEON::BI__builtin_neon_vluti2_lane_p16:
14002 case NEON::BI__builtin_neon_vluti2_lane_p8:
14003 case NEON::BI__builtin_neon_vluti2_lane_s16:
14004 case NEON::BI__builtin_neon_vluti2_lane_s8:
14005 case NEON::BI__builtin_neon_vluti2_lane_u16:
14006 case NEON::BI__builtin_neon_vluti2_lane_u8: {
14007 Int = Intrinsic::aarch64_neon_vluti2_lane;
14008 llvm::Type *Tys[2];
14014 case NEON::BI__builtin_neon_vluti2q_lane_bf16:
14015 case NEON::BI__builtin_neon_vluti2q_lane_f16:
14016 case NEON::BI__builtin_neon_vluti2q_lane_p16:
14017 case NEON::BI__builtin_neon_vluti2q_lane_p8:
14018 case NEON::BI__builtin_neon_vluti2q_lane_s16:
14019 case NEON::BI__builtin_neon_vluti2q_lane_s8:
14020 case NEON::BI__builtin_neon_vluti2q_lane_u16:
14021 case NEON::BI__builtin_neon_vluti2q_lane_u8: {
14022 Int = Intrinsic::aarch64_neon_vluti2_lane;
14023 llvm::Type *Tys[2];
14029 case NEON::BI__builtin_neon_vluti4q_lane_p8:
14030 case NEON::BI__builtin_neon_vluti4q_lane_s8:
14031 case NEON::BI__builtin_neon_vluti4q_lane_u8: {
14032 Int = Intrinsic::aarch64_neon_vluti4q_lane;
14035 case NEON::BI__builtin_neon_vluti4q_laneq_p8:
14036 case NEON::BI__builtin_neon_vluti4q_laneq_s8:
14037 case NEON::BI__builtin_neon_vluti4q_laneq_u8: {
14038 Int = Intrinsic::aarch64_neon_vluti4q_laneq;
14041 case NEON::BI__builtin_neon_vluti4q_lane_bf16_x2:
14042 case NEON::BI__builtin_neon_vluti4q_lane_f16_x2:
14043 case NEON::BI__builtin_neon_vluti4q_lane_p16_x2:
14044 case NEON::BI__builtin_neon_vluti4q_lane_s16_x2:
14045 case NEON::BI__builtin_neon_vluti4q_lane_u16_x2: {
14046 Int = Intrinsic::aarch64_neon_vluti4q_lane_x2;
14049 case NEON::BI__builtin_neon_vluti4q_laneq_bf16_x2:
14050 case NEON::BI__builtin_neon_vluti4q_laneq_f16_x2:
14051 case NEON::BI__builtin_neon_vluti4q_laneq_p16_x2:
14052 case NEON::BI__builtin_neon_vluti4q_laneq_s16_x2:
14053 case NEON::BI__builtin_neon_vluti4q_laneq_u16_x2: {
14054 Int = Intrinsic::aarch64_neon_vluti4q_laneq_x2;
14058 case NEON::BI__builtin_neon_vamin_f16:
14059 case NEON::BI__builtin_neon_vaminq_f16:
14060 case NEON::BI__builtin_neon_vamin_f32:
14061 case NEON::BI__builtin_neon_vaminq_f32:
14062 case NEON::BI__builtin_neon_vaminq_f64: {
14063 Int = Intrinsic::aarch64_neon_famin;
14066 case NEON::BI__builtin_neon_vamax_f16:
14067 case NEON::BI__builtin_neon_vamaxq_f16:
14068 case NEON::BI__builtin_neon_vamax_f32:
14069 case NEON::BI__builtin_neon_vamaxq_f32:
14070 case NEON::BI__builtin_neon_vamaxq_f64: {
14071 Int = Intrinsic::aarch64_neon_famax;
14074 case NEON::BI__builtin_neon_vscale_f16:
14075 case NEON::BI__builtin_neon_vscaleq_f16:
14076 case NEON::BI__builtin_neon_vscale_f32:
14077 case NEON::BI__builtin_neon_vscaleq_f32:
14078 case NEON::BI__builtin_neon_vscaleq_f64: {
14079 Int = Intrinsic::aarch64_neon_fp8_fscale;
14087 assert((BuiltinID == BPF::BI__builtin_preserve_field_info ||
14088 BuiltinID == BPF::BI__builtin_btf_type_id ||
14089 BuiltinID == BPF::BI__builtin_preserve_type_info ||
14090 BuiltinID == BPF::BI__builtin_preserve_enum_value) &&
14091 "unexpected BPF builtin");
14098 switch (BuiltinID) {
14100 llvm_unreachable(
"Unexpected BPF builtin");
14101 case BPF::BI__builtin_preserve_field_info: {
14102 const Expr *Arg =
E->getArg(0);
14107 "using __builtin_preserve_field_info() without -g");
14120 Value *InfoKind = ConstantInt::get(
Int64Ty,
C->getSExtValue());
14123 llvm::Function *FnGetFieldInfo = llvm::Intrinsic::getOrInsertDeclaration(
14124 &
CGM.
getModule(), llvm::Intrinsic::bpf_preserve_field_info,
14125 {FieldAddr->getType()});
14126 return Builder.CreateCall(FnGetFieldInfo, {FieldAddr, InfoKind});
14128 case BPF::BI__builtin_btf_type_id:
14129 case BPF::BI__builtin_preserve_type_info: {
14135 const Expr *Arg0 =
E->getArg(0);
14140 Value *FlagValue = ConstantInt::get(
Int64Ty, Flag->getSExtValue());
14141 Value *SeqNumVal = ConstantInt::get(
Int32Ty, BuiltinSeqNum++);
14143 llvm::Function *FnDecl;
14144 if (BuiltinID == BPF::BI__builtin_btf_type_id)
14145 FnDecl = llvm::Intrinsic::getOrInsertDeclaration(
14146 &
CGM.
getModule(), llvm::Intrinsic::bpf_btf_type_id, {});
14148 FnDecl = llvm::Intrinsic::getOrInsertDeclaration(
14149 &
CGM.
getModule(), llvm::Intrinsic::bpf_preserve_type_info, {});
14150 CallInst *
Fn =
Builder.CreateCall(FnDecl, {SeqNumVal, FlagValue});
14151 Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
14154 case BPF::BI__builtin_preserve_enum_value: {
14160 const Expr *Arg0 =
E->getArg(0);
14165 const auto *UO = cast<UnaryOperator>(Arg0->
IgnoreParens());
14166 const auto *CE = cast<CStyleCastExpr>(UO->getSubExpr());
14167 const auto *DR = cast<DeclRefExpr>(CE->getSubExpr());
14168 const auto *Enumerator = cast<EnumConstantDecl>(DR->getDecl());
14170 auto InitVal = Enumerator->getInitVal();
14171 std::string InitValStr;
14172 if (InitVal.isNegative() || InitVal >
uint64_t(INT64_MAX))
14173 InitValStr = std::to_string(InitVal.getSExtValue());
14175 InitValStr = std::to_string(InitVal.getZExtValue());
14176 std::string EnumStr = Enumerator->getNameAsString() +
":" + InitValStr;
14177 Value *EnumStrVal =
Builder.CreateGlobalString(EnumStr);
14180 Value *FlagValue = ConstantInt::get(
Int64Ty, Flag->getSExtValue());
14181 Value *SeqNumVal = ConstantInt::get(
Int32Ty, BuiltinSeqNum++);
14183 llvm::Function *IntrinsicFn = llvm::Intrinsic::getOrInsertDeclaration(
14184 &
CGM.
getModule(), llvm::Intrinsic::bpf_preserve_enum_value, {});
14186 Builder.CreateCall(IntrinsicFn, {SeqNumVal, EnumStrVal, FlagValue});
14187 Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
14195 assert((Ops.size() & (Ops.size() - 1)) == 0 &&
14196 "Not a power-of-two sized vector!");
14197 bool AllConstants =
true;
14198 for (
unsigned i = 0, e = Ops.size(); i != e && AllConstants; ++i)
14199 AllConstants &= isa<Constant>(Ops[i]);
14202 if (AllConstants) {
14204 for (
unsigned i = 0, e = Ops.size(); i != e; ++i)
14205 CstOps.push_back(cast<Constant>(Ops[i]));
14206 return llvm::ConstantVector::get(CstOps);
14211 llvm::FixedVectorType::get(Ops[0]->getType(), Ops.size()));
14213 for (
unsigned i = 0, e = Ops.size(); i != e; ++i)
14221 unsigned NumElts) {
14223 auto *MaskTy = llvm::FixedVectorType::get(
14225 cast<IntegerType>(Mask->
getType())->getBitWidth());
14226 Value *MaskVec = CGF.
Builder.CreateBitCast(Mask, MaskTy);
14232 for (
unsigned i = 0; i != NumElts; ++i)
14234 MaskVec = CGF.
Builder.CreateShuffleVector(
14235 MaskVec, MaskVec,
ArrayRef(Indices, NumElts),
"extract");
14242 Value *Ptr = Ops[0];
14246 cast<llvm::FixedVectorType>(Ops[1]->getType())->getNumElements());
14248 return CGF.
Builder.CreateMaskedStore(Ops[1], Ptr, Alignment, MaskVec);
14253 llvm::Type *Ty = Ops[1]->getType();
14254 Value *Ptr = Ops[0];
14257 CGF, Ops[2], cast<llvm::FixedVectorType>(Ty)->getNumElements());
14259 return CGF.
Builder.CreateMaskedLoad(Ty, Ptr, Alignment, MaskVec, Ops[1]);
14264 auto *ResultTy = cast<llvm::VectorType>(Ops[1]->getType());
14265 Value *Ptr = Ops[0];
14268 CGF, Ops[2], cast<FixedVectorType>(ResultTy)->getNumElements());
14270 llvm::Function *F = CGF.
CGM.
getIntrinsic(Intrinsic::masked_expandload,
14272 return CGF.
Builder.CreateCall(F, { Ptr, MaskVec, Ops[1] });
14278 auto *ResultTy = cast<llvm::FixedVectorType>(Ops[1]->getType());
14282 Intrinsic::ID IID = IsCompress ? Intrinsic::x86_avx512_mask_compress
14283 : Intrinsic::x86_avx512_mask_expand;
14285 return CGF.
Builder.CreateCall(F, { Ops[0], Ops[1], MaskVec });
14290 auto *ResultTy = cast<llvm::FixedVectorType>(Ops[1]->getType());
14291 Value *Ptr = Ops[0];
14295 llvm::Function *F = CGF.
CGM.
getIntrinsic(Intrinsic::masked_compressstore,
14297 return CGF.
Builder.CreateCall(F, { Ops[1], Ptr, MaskVec });
14302 bool InvertLHS =
false) {
14303 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
14308 LHS = CGF.
Builder.CreateNot(LHS);
14310 return CGF.
Builder.CreateBitCast(CGF.
Builder.CreateBinOp(Opc, LHS, RHS),
14311 Ops[0]->getType());
14315 Value *Amt,
bool IsRight) {
14316 llvm::Type *Ty = Op0->
getType();
14322 unsigned NumElts = cast<llvm::FixedVectorType>(Ty)->getNumElements();
14323 Amt = CGF.
Builder.CreateIntCast(Amt, Ty->getScalarType(),
false);
14324 Amt = CGF.
Builder.CreateVectorSplat(NumElts, Amt);
14327 unsigned IID = IsRight ? Intrinsic::fshr : Intrinsic::fshl;
14329 return CGF.
Builder.CreateCall(F, {Op0, Op1, Amt});
14334 Value *Op0 = Ops[0];
14335 Value *Op1 = Ops[1];
14336 llvm::Type *Ty = Op0->
getType();
14337 uint64_t Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x7;
14339 CmpInst::Predicate Pred;
14342 Pred = IsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
14345 Pred = IsSigned ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
14348 Pred = IsSigned ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
14351 Pred = IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
14354 Pred = ICmpInst::ICMP_EQ;
14357 Pred = ICmpInst::ICMP_NE;
14360 return llvm::Constant::getNullValue(Ty);
14362 return llvm::Constant::getAllOnesValue(Ty);
14364 llvm_unreachable(
"Unexpected XOP vpcom/vpcomu predicate");
14376 if (
const auto *
C = dyn_cast<Constant>(Mask))
14377 if (
C->isAllOnesValue())
14381 CGF, Mask, cast<llvm::FixedVectorType>(Op0->
getType())->getNumElements());
14383 return CGF.
Builder.CreateSelect(Mask, Op0, Op1);
14389 if (
const auto *
C = dyn_cast<Constant>(Mask))
14390 if (
C->isAllOnesValue())
14393 auto *MaskTy = llvm::FixedVectorType::get(
14394 CGF.
Builder.getInt1Ty(), Mask->
getType()->getIntegerBitWidth());
14395 Mask = CGF.
Builder.CreateBitCast(Mask, MaskTy);
14396 Mask = CGF.
Builder.CreateExtractElement(Mask, (uint64_t)0);
14397 return CGF.
Builder.CreateSelect(Mask, Op0, Op1);
14401 unsigned NumElts,
Value *MaskIn) {
14403 const auto *
C = dyn_cast<Constant>(MaskIn);
14404 if (!
C || !
C->isAllOnesValue())
14410 for (
unsigned i = 0; i != NumElts; ++i)
14412 for (
unsigned i = NumElts; i != 8; ++i)
14413 Indices[i] = i % NumElts + NumElts;
14414 Cmp = CGF.
Builder.CreateShuffleVector(
14415 Cmp, llvm::Constant::getNullValue(Cmp->getType()), Indices);
14418 return CGF.
Builder.CreateBitCast(Cmp,
14420 std::max(NumElts, 8U)));
14425 assert((Ops.size() == 2 || Ops.size() == 4) &&
14426 "Unexpected number of arguments");
14428 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
14432 Cmp = Constant::getNullValue(
14433 llvm::FixedVectorType::get(CGF.
Builder.getInt1Ty(), NumElts));
14434 }
else if (CC == 7) {
14435 Cmp = Constant::getAllOnesValue(
14436 llvm::FixedVectorType::get(CGF.
Builder.getInt1Ty(), NumElts));
14438 ICmpInst::Predicate Pred;
14440 default: llvm_unreachable(
"Unknown condition code");
14441 case 0: Pred = ICmpInst::ICMP_EQ;
break;
14442 case 1: Pred =
Signed ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
break;
14443 case 2: Pred =
Signed ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
break;
14444 case 4: Pred = ICmpInst::ICMP_NE;
break;
14445 case 5: Pred =
Signed ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
break;
14446 case 6: Pred =
Signed ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
break;
14448 Cmp = CGF.
Builder.CreateICmp(Pred, Ops[0], Ops[1]);
14451 Value *MaskIn =
nullptr;
14452 if (Ops.size() == 4)
14459 Value *Zero = Constant::getNullValue(In->getType());
14465 unsigned Rnd = cast<llvm::ConstantInt>(Ops[3])->getZExtValue();
14466 llvm::Type *Ty = Ops[1]->getType();
14470 Intrinsic::ID IID = IsSigned ? Intrinsic::x86_avx512_sitofp_round
14471 : Intrinsic::x86_avx512_uitofp_round;
14473 Res = CGF.
Builder.CreateCall(F, { Ops[0], Ops[3] });
14475 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF,
E);
14476 Res = IsSigned ? CGF.
Builder.CreateSIToFP(Ops[0], Ty)
14477 : CGF.
Builder.CreateUIToFP(Ops[0], Ty);
14488 bool Subtract =
false;
14489 Intrinsic::ID IID = Intrinsic::not_intrinsic;
14490 switch (BuiltinID) {
14492 case clang::X86::BI__builtin_ia32_vfmsubph512_mask3:
14495 case clang::X86::BI__builtin_ia32_vfmaddph512_mask:
14496 case clang::X86::BI__builtin_ia32_vfmaddph512_maskz:
14497 case clang::X86::BI__builtin_ia32_vfmaddph512_mask3:
14498 IID = llvm::Intrinsic::x86_avx512fp16_vfmadd_ph_512;
14500 case clang::X86::BI__builtin_ia32_vfmsubaddph512_mask3:
14503 case clang::X86::BI__builtin_ia32_vfmaddsubph512_mask:
14504 case clang::X86::BI__builtin_ia32_vfmaddsubph512_maskz:
14505 case clang::X86::BI__builtin_ia32_vfmaddsubph512_mask3:
14506 IID = llvm::Intrinsic::x86_avx512fp16_vfmaddsub_ph_512;
14508 case clang::X86::BI__builtin_ia32_vfmsubps512_mask3:
14511 case clang::X86::BI__builtin_ia32_vfmaddps512_mask:
14512 case clang::X86::BI__builtin_ia32_vfmaddps512_maskz:
14513 case clang::X86::BI__builtin_ia32_vfmaddps512_mask3:
14514 IID = llvm::Intrinsic::x86_avx512_vfmadd_ps_512;
break;
14515 case clang::X86::BI__builtin_ia32_vfmsubpd512_mask3:
14518 case clang::X86::BI__builtin_ia32_vfmaddpd512_mask:
14519 case clang::X86::BI__builtin_ia32_vfmaddpd512_maskz:
14520 case clang::X86::BI__builtin_ia32_vfmaddpd512_mask3:
14521 IID = llvm::Intrinsic::x86_avx512_vfmadd_pd_512;
break;
14522 case clang::X86::BI__builtin_ia32_vfmsubaddps512_mask3:
14525 case clang::X86::BI__builtin_ia32_vfmaddsubps512_mask:
14526 case clang::X86::BI__builtin_ia32_vfmaddsubps512_maskz:
14527 case clang::X86::BI__builtin_ia32_vfmaddsubps512_mask3:
14528 IID = llvm::Intrinsic::x86_avx512_vfmaddsub_ps_512;
14530 case clang::X86::BI__builtin_ia32_vfmsubaddpd512_mask3:
14533 case clang::X86::BI__builtin_ia32_vfmaddsubpd512_mask:
14534 case clang::X86::BI__builtin_ia32_vfmaddsubpd512_maskz:
14535 case clang::X86::BI__builtin_ia32_vfmaddsubpd512_mask3:
14536 IID = llvm::Intrinsic::x86_avx512_vfmaddsub_pd_512;
14538 case clang::X86::BI__builtin_ia32_vfmsubph256_round_mask3:
14541 case clang::X86::BI__builtin_ia32_vfmaddph256_round_mask:
14542 case clang::X86::BI__builtin_ia32_vfmaddph256_round_maskz:
14543 case clang::X86::BI__builtin_ia32_vfmaddph256_round_mask3:
14544 IID = llvm::Intrinsic::x86_avx10_vfmaddph256;
14546 case clang::X86::BI__builtin_ia32_vfmsubaddph256_round_mask3:
14549 case clang::X86::BI__builtin_ia32_vfmaddsubph256_round_mask:
14550 case clang::X86::BI__builtin_ia32_vfmaddsubph256_round_maskz:
14551 case clang::X86::BI__builtin_ia32_vfmaddsubph256_round_mask3:
14552 IID = llvm::Intrinsic::x86_avx10_vfmaddsubph256;
14554 case clang::X86::BI__builtin_ia32_vfmsubps256_round_mask3:
14557 case clang::X86::BI__builtin_ia32_vfmaddps256_round_mask:
14558 case clang::X86::BI__builtin_ia32_vfmaddps256_round_maskz:
14559 case clang::X86::BI__builtin_ia32_vfmaddps256_round_mask3:
14560 IID = llvm::Intrinsic::x86_avx10_vfmaddps256;
14562 case clang::X86::BI__builtin_ia32_vfmsubpd256_round_mask3:
14565 case clang::X86::BI__builtin_ia32_vfmaddpd256_round_mask:
14566 case clang::X86::BI__builtin_ia32_vfmaddpd256_round_maskz:
14567 case clang::X86::BI__builtin_ia32_vfmaddpd256_round_mask3:
14568 IID = llvm::Intrinsic::x86_avx10_vfmaddpd256;
14570 case clang::X86::BI__builtin_ia32_vfmsubaddps256_round_mask3:
14573 case clang::X86::BI__builtin_ia32_vfmaddsubps256_round_mask:
14574 case clang::X86::BI__builtin_ia32_vfmaddsubps256_round_maskz:
14575 case clang::X86::BI__builtin_ia32_vfmaddsubps256_round_mask3:
14576 IID = llvm::Intrinsic::x86_avx10_vfmaddsubps256;
14578 case clang::X86::BI__builtin_ia32_vfmsubaddpd256_round_mask3:
14581 case clang::X86::BI__builtin_ia32_vfmaddsubpd256_round_mask:
14582 case clang::X86::BI__builtin_ia32_vfmaddsubpd256_round_maskz:
14583 case clang::X86::BI__builtin_ia32_vfmaddsubpd256_round_mask3:
14584 IID = llvm::Intrinsic::x86_avx10_vfmaddsubpd256;
14598 if (IID != Intrinsic::not_intrinsic &&
14599 (cast<llvm::ConstantInt>(Ops.back())->getZExtValue() != (uint64_t)4 ||
14602 Res = CGF.
Builder.CreateCall(Intr, {A, B,
C, Ops.back() });
14604 llvm::Type *Ty = A->
getType();
14606 if (CGF.
Builder.getIsFPConstrained()) {
14607 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF,
E);
14608 FMA = CGF.
CGM.
getIntrinsic(Intrinsic::experimental_constrained_fma, Ty);
14609 Res = CGF.
Builder.CreateConstrainedFPCall(FMA, {A, B,
C});
14612 Res = CGF.
Builder.CreateCall(FMA, {A, B,
C});
14617 Value *MaskFalseVal =
nullptr;
14618 switch (BuiltinID) {
14619 case clang::X86::BI__builtin_ia32_vfmaddph512_mask:
14620 case clang::X86::BI__builtin_ia32_vfmaddps512_mask:
14621 case clang::X86::BI__builtin_ia32_vfmaddpd512_mask:
14622 case clang::X86::BI__builtin_ia32_vfmaddsubph512_mask:
14623 case clang::X86::BI__builtin_ia32_vfmaddsubps512_mask:
14624 case clang::X86::BI__builtin_ia32_vfmaddsubpd512_mask:
14625 case clang::X86::BI__builtin_ia32_vfmaddph256_round_mask:
14626 case clang::X86::BI__builtin_ia32_vfmaddps256_round_mask:
14627 case clang::X86::BI__builtin_ia32_vfmaddpd256_round_mask:
14628 case clang::X86::BI__builtin_ia32_vfmaddsubph256_round_mask:
14629 case clang::X86::BI__builtin_ia32_vfmaddsubps256_round_mask:
14630 case clang::X86::BI__builtin_ia32_vfmaddsubpd256_round_mask:
14631 MaskFalseVal = Ops[0];
14633 case clang::X86::BI__builtin_ia32_vfmaddph512_maskz:
14634 case clang::X86::BI__builtin_ia32_vfmaddps512_maskz:
14635 case clang::X86::BI__builtin_ia32_vfmaddpd512_maskz:
14636 case clang::X86::BI__builtin_ia32_vfmaddsubph512_maskz:
14637 case clang::X86::BI__builtin_ia32_vfmaddsubps512_maskz:
14638 case clang::X86::BI__builtin_ia32_vfmaddsubpd512_maskz:
14639 case clang::X86::BI__builtin_ia32_vfmaddph256_round_maskz:
14640 case clang::X86::BI__builtin_ia32_vfmaddps256_round_maskz:
14641 case clang::X86::BI__builtin_ia32_vfmaddpd256_round_maskz:
14642 case clang::X86::BI__builtin_ia32_vfmaddsubph256_round_maskz:
14643 case clang::X86::BI__builtin_ia32_vfmaddsubps256_round_maskz:
14644 case clang::X86::BI__builtin_ia32_vfmaddsubpd256_round_maskz:
14645 MaskFalseVal = Constant::getNullValue(Ops[0]->getType());
14647 case clang::X86::BI__builtin_ia32_vfmsubph512_mask3:
14648 case clang::X86::BI__builtin_ia32_vfmaddph512_mask3:
14649 case clang::X86::BI__builtin_ia32_vfmsubps512_mask3:
14650 case clang::X86::BI__builtin_ia32_vfmaddps512_mask3:
14651 case clang::X86::BI__builtin_ia32_vfmsubpd512_mask3:
14652 case clang::X86::BI__builtin_ia32_vfmaddpd512_mask3:
14653 case clang::X86::BI__builtin_ia32_vfmsubaddph512_mask3:
14654 case clang::X86::BI__builtin_ia32_vfmaddsubph512_mask3:
14655 case clang::X86::BI__builtin_ia32_vfmsubaddps512_mask3:
14656 case clang::X86::BI__builtin_ia32_vfmaddsubps512_mask3:
14657 case clang::X86::BI__builtin_ia32_vfmsubaddpd512_mask3:
14658 case clang::X86::BI__builtin_ia32_vfmaddsubpd512_mask3:
14659 case clang::X86::BI__builtin_ia32_vfmsubph256_round_mask3:
14660 case clang::X86::BI__builtin_ia32_vfmaddph256_round_mask3:
14661 case clang::X86::BI__builtin_ia32_vfmsubps256_round_mask3:
14662 case clang::X86::BI__builtin_ia32_vfmaddps256_round_mask3:
14663 case clang::X86::BI__builtin_ia32_vfmsubpd256_round_mask3:
14664 case clang::X86::BI__builtin_ia32_vfmaddpd256_round_mask3:
14665 case clang::X86::BI__builtin_ia32_vfmsubaddph256_round_mask3:
14666 case clang::X86::BI__builtin_ia32_vfmaddsubph256_round_mask3:
14667 case clang::X86::BI__builtin_ia32_vfmsubaddps256_round_mask3:
14668 case clang::X86::BI__builtin_ia32_vfmaddsubps256_round_mask3:
14669 case clang::X86::BI__builtin_ia32_vfmsubaddpd256_round_mask3:
14670 case clang::X86::BI__builtin_ia32_vfmaddsubpd256_round_mask3:
14671 MaskFalseVal = Ops[2];
14683 bool ZeroMask =
false,
unsigned PTIdx = 0,
14684 bool NegAcc =
false) {
14686 if (Ops.size() > 4)
14687 Rnd = cast<llvm::ConstantInt>(Ops[4])->getZExtValue();
14690 Ops[2] = CGF.
Builder.CreateFNeg(Ops[2]);
14692 Ops[0] = CGF.
Builder.CreateExtractElement(Ops[0], (uint64_t)0);
14693 Ops[1] = CGF.
Builder.CreateExtractElement(Ops[1], (uint64_t)0);
14694 Ops[2] = CGF.
Builder.CreateExtractElement(Ops[2], (uint64_t)0);
14699 switch (Ops[0]->getType()->getPrimitiveSizeInBits()) {
14701 IID = Intrinsic::x86_avx512fp16_vfmadd_f16;
14704 IID = Intrinsic::x86_avx512_vfmadd_f32;
14707 IID = Intrinsic::x86_avx512_vfmadd_f64;
14710 llvm_unreachable(
"Unexpected size");
14713 {Ops[0], Ops[1], Ops[2], Ops[4]});
14714 }
else if (CGF.
Builder.getIsFPConstrained()) {
14715 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF,
E);
14717 Intrinsic::experimental_constrained_fma, Ops[0]->getType());
14718 Res = CGF.
Builder.CreateConstrainedFPCall(FMA, Ops.slice(0, 3));
14721 Res = CGF.
Builder.CreateCall(FMA, Ops.slice(0, 3));
14724 if (Ops.size() > 3) {
14725 Value *PassThru = ZeroMask ? Constant::getNullValue(Res->
getType())
14731 if (NegAcc && PTIdx == 2)
14732 PassThru = CGF.
Builder.CreateExtractElement(Upper, (uint64_t)0);
14736 return CGF.
Builder.CreateInsertElement(Upper, Res, (uint64_t)0);
14741 llvm::Type *Ty = Ops[0]->getType();
14743 Ty = llvm::FixedVectorType::get(CGF.
Int64Ty,
14744 Ty->getPrimitiveSizeInBits() / 64);
14750 Constant *ShiftAmt = ConstantInt::get(Ty, 32);
14751 LHS = CGF.
Builder.CreateShl(LHS, ShiftAmt);
14752 LHS = CGF.
Builder.CreateAShr(LHS, ShiftAmt);
14753 RHS = CGF.
Builder.CreateShl(RHS, ShiftAmt);
14754 RHS = CGF.
Builder.CreateAShr(RHS, ShiftAmt);
14757 Constant *Mask = ConstantInt::get(Ty, 0xffffffff);
14758 LHS = CGF.
Builder.CreateAnd(LHS, Mask);
14759 RHS = CGF.
Builder.CreateAnd(RHS, Mask);
14762 return CGF.
Builder.CreateMul(LHS, RHS);
14770 llvm::Type *Ty = Ops[0]->getType();
14772 unsigned VecWidth = Ty->getPrimitiveSizeInBits();
14773 unsigned EltWidth = Ty->getScalarSizeInBits();
14775 if (VecWidth == 128 && EltWidth == 32)
14776 IID = Intrinsic::x86_avx512_pternlog_d_128;
14777 else if (VecWidth == 256 && EltWidth == 32)
14778 IID = Intrinsic::x86_avx512_pternlog_d_256;
14779 else if (VecWidth == 512 && EltWidth == 32)
14780 IID = Intrinsic::x86_avx512_pternlog_d_512;
14781 else if (VecWidth == 128 && EltWidth == 64)
14782 IID = Intrinsic::x86_avx512_pternlog_q_128;
14783 else if (VecWidth == 256 && EltWidth == 64)
14784 IID = Intrinsic::x86_avx512_pternlog_q_256;
14785 else if (VecWidth == 512 && EltWidth == 64)
14786 IID = Intrinsic::x86_avx512_pternlog_q_512;
14788 llvm_unreachable(
"Unexpected intrinsic");
14792 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(Ty) : Ops[0];
14797 llvm::Type *DstTy) {
14798 unsigned NumberOfElements =
14799 cast<llvm::FixedVectorType>(DstTy)->getNumElements();
14801 return CGF.
Builder.CreateSExt(Mask, DstTy,
"vpmovm2");
14806 StringRef CPUStr = cast<clang::StringLiteral>(CPUExpr)->getString();
14807 return EmitX86CpuIs(CPUStr);
14813 llvm::Type *DstTy) {
14814 assert((Ops.size() == 1 || Ops.size() == 3 || Ops.size() == 4) &&
14815 "Unknown cvtph2ps intrinsic");
14818 if (Ops.size() == 4 && cast<llvm::ConstantInt>(Ops[3])->getZExtValue() != 4) {
14821 return CGF.
Builder.CreateCall(F, {Ops[0], Ops[1], Ops[2], Ops[3]});
14824 unsigned NumDstElts = cast<llvm::FixedVectorType>(DstTy)->getNumElements();
14825 Value *Src = Ops[0];
14829 cast<llvm::FixedVectorType>(Src->
getType())->getNumElements()) {
14830 assert(NumDstElts == 4 &&
"Unexpected vector size");
14835 auto *HalfTy = llvm::FixedVectorType::get(
14837 Src = CGF.
Builder.CreateBitCast(Src, HalfTy);
14840 Value *Res = CGF.
Builder.CreateFPExt(Src, DstTy,
"cvtph2ps");
14842 if (Ops.size() >= 3)
14847Value *CodeGenFunction::EmitX86CpuIs(StringRef CPUStr) {
14858 llvm::ArrayType::get(
Int32Ty, 1));
14862 cast<llvm::GlobalValue>(CpuModel)->setDSOLocal(
true);
14868 std::tie(Index,
Value) = StringSwitch<std::pair<unsigned, unsigned>>(CPUStr)
14870 .Case(STRING, {0u,
static_cast<unsigned>(llvm::X86::ENUM)})
14872 .Case(
ALIAS, {1u,
static_cast<unsigned>(llvm::X86::ENUM)})
14874 .Case(STR, {1u,
static_cast<unsigned>(llvm::X86::ENUM)})
14876 .Case(
ALIAS, {2u,
static_cast<unsigned>(llvm::X86::ENUM)})
14878 .Case(STR, {2u,
static_cast<unsigned>(llvm::X86::ENUM)})
14879#include
"llvm/TargetParser/X86TargetParser.def"
14881 assert(
Value != 0 &&
"Invalid CPUStr passed to CpuIs");
14884 llvm::Value *Idxs[] = {ConstantInt::get(
Int32Ty, 0),
14885 ConstantInt::get(
Int32Ty, Index)};
14891 return Builder.CreateICmpEQ(CpuValue,
14897 StringRef FeatureStr = cast<StringLiteral>(FeatureExpr)->getString();
14898 if (!
getContext().getTargetInfo().validateCpuSupports(FeatureStr))
14900 return EmitX86CpuSupports(FeatureStr);
14904 return EmitX86CpuSupports(llvm::X86::getCpuSupportsMask(FeatureStrs));
14908CodeGenFunction::EmitX86CpuSupports(std::array<uint32_t, 4> FeatureMask) {
14910 if (FeatureMask[0] != 0) {
14918 llvm::ArrayType::get(
Int32Ty, 1));
14922 cast<llvm::GlobalValue>(CpuModel)->setDSOLocal(
true);
14939 llvm::Type *ATy = llvm::ArrayType::get(
Int32Ty, 3);
14940 llvm::Constant *CpuFeatures2 =
14942 cast<llvm::GlobalValue>(CpuFeatures2)->setDSOLocal(
true);
14943 for (
int i = 1; i != 4; ++i) {
14944 const uint32_t M = FeatureMask[i];
14961Value *CodeGenFunction::EmitAArch64CpuInit() {
14962 llvm::FunctionType *FTy = llvm::FunctionType::get(
VoidTy,
false);
14963 llvm::FunctionCallee
Func =
14965 cast<llvm::GlobalValue>(
Func.getCallee())->setDSOLocal(
true);
14966 cast<llvm::GlobalValue>(
Func.getCallee())
14967 ->setDLLStorageClass(llvm::GlobalValue::DefaultStorageClass);
14972 llvm::FunctionType *FTy = llvm::FunctionType::get(
VoidTy, {
VoidPtrTy},
false);
14973 llvm::FunctionCallee
Func =
14975 auto *CalleeGV = cast<llvm::GlobalValue>(
Func.getCallee());
14976 CalleeGV->setDSOLocal(
true);
14977 CalleeGV->setDLLStorageClass(llvm::GlobalValue::DefaultStorageClass);
14981Value *CodeGenFunction::EmitX86CpuInit() {
14982 llvm::FunctionType *FTy = llvm::FunctionType::get(
VoidTy,
14984 llvm::FunctionCallee
Func =
14986 cast<llvm::GlobalValue>(
Func.getCallee())->setDSOLocal(
true);
14987 cast<llvm::GlobalValue>(
Func.getCallee())
14988 ->setDLLStorageClass(llvm::GlobalValue::DefaultStorageClass);
14992Value *CodeGenFunction::EmitAArch64CpuSupports(
const CallExpr *
E) {
14994 StringRef ArgStr = cast<StringLiteral>(ArgExpr)->getString();
14996 ArgStr.split(Features,
"+");
14997 for (
auto &Feature : Features) {
14998 Feature = Feature.trim();
14999 if (!llvm::AArch64::parseFMVExtension(Feature))
15001 if (Feature !=
"default")
15002 Features.push_back(Feature);
15004 return EmitAArch64CpuSupports(Features);
15009 uint64_t FeaturesMask = llvm::AArch64::getCpuSupportsMask(FeaturesStrs);
15011 if (FeaturesMask != 0) {
15016 llvm::Type *STy = llvm::StructType::get(
Int64Ty);
15017 llvm::Constant *AArch64CPUFeatures =
15019 cast<llvm::GlobalValue>(AArch64CPUFeatures)->setDSOLocal(
true);
15021 STy, AArch64CPUFeatures,
15036 StringRef FeatureStr = cast<StringLiteral>(FeatureExpr)->getString();
15037 if (!
getContext().getTargetInfo().validateCpuSupports(FeatureStr))
15045 llvm::Type *Int32Ty = Builder.getInt32Ty();
15046 llvm::Type *Int64Ty = Builder.getInt64Ty();
15047 llvm::ArrayType *ArrayOfInt64Ty =
15048 llvm::ArrayType::get(Int64Ty, llvm::RISCVISAInfo::FeatureBitSize);
15049 llvm::Type *StructTy = llvm::StructType::get(Int32Ty, ArrayOfInt64Ty);
15050 llvm::Constant *RISCVFeaturesBits =
15052 cast<llvm::GlobalValue>(RISCVFeaturesBits)->setDSOLocal(
true);
15053 Value *IndexVal = llvm::ConstantInt::get(Int32Ty, Index);
15054 llvm::Value *GEPIndices[] = {Builder.getInt32(0), Builder.getInt32(1),
15057 Builder.CreateInBoundsGEP(StructTy, RISCVFeaturesBits, GEPIndices);
15058 Value *FeaturesBit =
15060 return FeaturesBit;
15064 const unsigned RISCVFeatureLength = llvm::RISCVISAInfo::FeatureBitSize;
15065 uint64_t RequireBitMasks[RISCVFeatureLength] = {0};
15067 for (
auto Feat : FeaturesStrs) {
15068 auto [GroupID, BitPos] = RISCVISAInfo::getRISCVFeaturesBitsInfo(Feat);
15075 RequireBitMasks[GroupID] |= (1ULL << BitPos);
15079 for (
unsigned Idx = 0; Idx < RISCVFeatureLength; Idx++) {
15080 if (RequireBitMasks[Idx] == 0)
15090 assert(
Result &&
"Should have value here.");
15097 if (BuiltinID == Builtin::BI__builtin_cpu_is)
15098 return EmitX86CpuIs(
E);
15099 if (BuiltinID == Builtin::BI__builtin_cpu_supports)
15100 return EmitX86CpuSupports(
E);
15101 if (BuiltinID == Builtin::BI__builtin_cpu_init)
15102 return EmitX86CpuInit();
15110 bool IsMaskFCmp =
false;
15111 bool IsConjFMA =
false;
15114 unsigned ICEArguments = 0;
15119 for (
unsigned i = 0, e =
E->getNumArgs(); i != e; i++) {
15129 auto getCmpIntrinsicCall = [
this, &Ops](Intrinsic::ID
ID,
unsigned Imm) {
15130 Ops.push_back(llvm::ConstantInt::get(
Int8Ty, Imm));
15132 return Builder.CreateCall(F, Ops);
15140 auto getVectorFCmpIR = [
this, &Ops,
E](CmpInst::Predicate Pred,
15141 bool IsSignaling) {
15142 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
15145 Cmp =
Builder.CreateFCmpS(Pred, Ops[0], Ops[1]);
15147 Cmp =
Builder.CreateFCmp(Pred, Ops[0], Ops[1]);
15148 llvm::VectorType *FPVecTy = cast<llvm::VectorType>(Ops[0]->getType());
15149 llvm::VectorType *IntVecTy = llvm::VectorType::getInteger(FPVecTy);
15151 return Builder.CreateBitCast(Sext, FPVecTy);
15154 switch (BuiltinID) {
15155 default:
return nullptr;
15156 case X86::BI_mm_prefetch: {
15158 ConstantInt *
C = cast<ConstantInt>(Ops[1]);
15159 Value *RW = ConstantInt::get(
Int32Ty, (
C->getZExtValue() >> 2) & 0x1);
15160 Value *Locality = ConstantInt::get(
Int32Ty,
C->getZExtValue() & 0x3);
15165 case X86::BI_mm_clflush: {
15169 case X86::BI_mm_lfence: {
15172 case X86::BI_mm_mfence: {
15175 case X86::BI_mm_sfence: {
15178 case X86::BI_mm_pause: {
15181 case X86::BI__rdtsc: {
15184 case X86::BI__builtin_ia32_rdtscp: {
15190 case X86::BI__builtin_ia32_lzcnt_u16:
15191 case X86::BI__builtin_ia32_lzcnt_u32:
15192 case X86::BI__builtin_ia32_lzcnt_u64: {
15196 case X86::BI__builtin_ia32_tzcnt_u16:
15197 case X86::BI__builtin_ia32_tzcnt_u32:
15198 case X86::BI__builtin_ia32_tzcnt_u64: {
15202 case X86::BI__builtin_ia32_undef128:
15203 case X86::BI__builtin_ia32_undef256:
15204 case X86::BI__builtin_ia32_undef512:
15211 case X86::BI__builtin_ia32_vec_ext_v4hi:
15212 case X86::BI__builtin_ia32_vec_ext_v16qi:
15213 case X86::BI__builtin_ia32_vec_ext_v8hi:
15214 case X86::BI__builtin_ia32_vec_ext_v4si:
15215 case X86::BI__builtin_ia32_vec_ext_v4sf:
15216 case X86::BI__builtin_ia32_vec_ext_v2di:
15217 case X86::BI__builtin_ia32_vec_ext_v32qi:
15218 case X86::BI__builtin_ia32_vec_ext_v16hi:
15219 case X86::BI__builtin_ia32_vec_ext_v8si:
15220 case X86::BI__builtin_ia32_vec_ext_v4di: {
15222 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
15223 uint64_t Index = cast<ConstantInt>(Ops[1])->getZExtValue();
15224 Index &= NumElts - 1;
15227 return Builder.CreateExtractElement(Ops[0], Index);
15229 case X86::BI__builtin_ia32_vec_set_v4hi:
15230 case X86::BI__builtin_ia32_vec_set_v16qi:
15231 case X86::BI__builtin_ia32_vec_set_v8hi:
15232 case X86::BI__builtin_ia32_vec_set_v4si:
15233 case X86::BI__builtin_ia32_vec_set_v2di:
15234 case X86::BI__builtin_ia32_vec_set_v32qi:
15235 case X86::BI__builtin_ia32_vec_set_v16hi:
15236 case X86::BI__builtin_ia32_vec_set_v8si:
15237 case X86::BI__builtin_ia32_vec_set_v4di: {
15239 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
15240 unsigned Index = cast<ConstantInt>(Ops[2])->getZExtValue();
15241 Index &= NumElts - 1;
15244 return Builder.CreateInsertElement(Ops[0], Ops[1], Index);
15246 case X86::BI_mm_setcsr:
15247 case X86::BI__builtin_ia32_ldmxcsr: {
15253 case X86::BI_mm_getcsr:
15254 case X86::BI__builtin_ia32_stmxcsr: {
15260 case X86::BI__builtin_ia32_xsave:
15261 case X86::BI__builtin_ia32_xsave64:
15262 case X86::BI__builtin_ia32_xrstor:
15263 case X86::BI__builtin_ia32_xrstor64:
15264 case X86::BI__builtin_ia32_xsaveopt:
15265 case X86::BI__builtin_ia32_xsaveopt64:
15266 case X86::BI__builtin_ia32_xrstors:
15267 case X86::BI__builtin_ia32_xrstors64:
15268 case X86::BI__builtin_ia32_xsavec:
15269 case X86::BI__builtin_ia32_xsavec64:
15270 case X86::BI__builtin_ia32_xsaves:
15271 case X86::BI__builtin_ia32_xsaves64:
15272 case X86::BI__builtin_ia32_xsetbv:
15273 case X86::BI_xsetbv: {
15275#define INTRINSIC_X86_XSAVE_ID(NAME) \
15276 case X86::BI__builtin_ia32_##NAME: \
15277 ID = Intrinsic::x86_##NAME; \
15279 switch (BuiltinID) {
15280 default: llvm_unreachable(
"Unsupported intrinsic!");
15294 case X86::BI_xsetbv:
15295 ID = Intrinsic::x86_xsetbv;
15298#undef INTRINSIC_X86_XSAVE_ID
15303 Ops.push_back(Mlo);
15306 case X86::BI__builtin_ia32_xgetbv:
15307 case X86::BI_xgetbv:
15309 case X86::BI__builtin_ia32_storedqudi128_mask:
15310 case X86::BI__builtin_ia32_storedqusi128_mask:
15311 case X86::BI__builtin_ia32_storedquhi128_mask:
15312 case X86::BI__builtin_ia32_storedquqi128_mask:
15313 case X86::BI__builtin_ia32_storeupd128_mask:
15314 case X86::BI__builtin_ia32_storeups128_mask:
15315 case X86::BI__builtin_ia32_storedqudi256_mask:
15316 case X86::BI__builtin_ia32_storedqusi256_mask:
15317 case X86::BI__builtin_ia32_storedquhi256_mask:
15318 case X86::BI__builtin_ia32_storedquqi256_mask:
15319 case X86::BI__builtin_ia32_storeupd256_mask:
15320 case X86::BI__builtin_ia32_storeups256_mask:
15321 case X86::BI__builtin_ia32_storedqudi512_mask:
15322 case X86::BI__builtin_ia32_storedqusi512_mask:
15323 case X86::BI__builtin_ia32_storedquhi512_mask:
15324 case X86::BI__builtin_ia32_storedquqi512_mask:
15325 case X86::BI__builtin_ia32_storeupd512_mask:
15326 case X86::BI__builtin_ia32_storeups512_mask:
15329 case X86::BI__builtin_ia32_storesbf16128_mask:
15330 case X86::BI__builtin_ia32_storesh128_mask:
15331 case X86::BI__builtin_ia32_storess128_mask:
15332 case X86::BI__builtin_ia32_storesd128_mask:
15335 case X86::BI__builtin_ia32_cvtmask2b128:
15336 case X86::BI__builtin_ia32_cvtmask2b256:
15337 case X86::BI__builtin_ia32_cvtmask2b512:
15338 case X86::BI__builtin_ia32_cvtmask2w128:
15339 case X86::BI__builtin_ia32_cvtmask2w256:
15340 case X86::BI__builtin_ia32_cvtmask2w512:
15341 case X86::BI__builtin_ia32_cvtmask2d128:
15342 case X86::BI__builtin_ia32_cvtmask2d256:
15343 case X86::BI__builtin_ia32_cvtmask2d512:
15344 case X86::BI__builtin_ia32_cvtmask2q128:
15345 case X86::BI__builtin_ia32_cvtmask2q256:
15346 case X86::BI__builtin_ia32_cvtmask2q512:
15349 case X86::BI__builtin_ia32_cvtb2mask128:
15350 case X86::BI__builtin_ia32_cvtb2mask256:
15351 case X86::BI__builtin_ia32_cvtb2mask512:
15352 case X86::BI__builtin_ia32_cvtw2mask128:
15353 case X86::BI__builtin_ia32_cvtw2mask256:
15354 case X86::BI__builtin_ia32_cvtw2mask512:
15355 case X86::BI__builtin_ia32_cvtd2mask128:
15356 case X86::BI__builtin_ia32_cvtd2mask256:
15357 case X86::BI__builtin_ia32_cvtd2mask512:
15358 case X86::BI__builtin_ia32_cvtq2mask128:
15359 case X86::BI__builtin_ia32_cvtq2mask256:
15360 case X86::BI__builtin_ia32_cvtq2mask512:
15363 case X86::BI__builtin_ia32_cvtdq2ps512_mask:
15364 case X86::BI__builtin_ia32_cvtqq2ps512_mask:
15365 case X86::BI__builtin_ia32_cvtqq2pd512_mask:
15366 case X86::BI__builtin_ia32_vcvtw2ph512_mask:
15367 case X86::BI__builtin_ia32_vcvtdq2ph512_mask:
15368 case X86::BI__builtin_ia32_vcvtqq2ph512_mask:
15369 case X86::BI__builtin_ia32_vcvtdq2ph256_round_mask:
15370 case X86::BI__builtin_ia32_vcvtdq2ps256_round_mask:
15371 case X86::BI__builtin_ia32_vcvtqq2pd256_round_mask:
15372 case X86::BI__builtin_ia32_vcvtqq2ph256_round_mask:
15373 case X86::BI__builtin_ia32_vcvtqq2ps256_round_mask:
15374 case X86::BI__builtin_ia32_vcvtw2ph256_round_mask:
15376 case X86::BI__builtin_ia32_cvtudq2ps512_mask:
15377 case X86::BI__builtin_ia32_cvtuqq2ps512_mask:
15378 case X86::BI__builtin_ia32_cvtuqq2pd512_mask:
15379 case X86::BI__builtin_ia32_vcvtuw2ph512_mask:
15380 case X86::BI__builtin_ia32_vcvtudq2ph512_mask:
15381 case X86::BI__builtin_ia32_vcvtuqq2ph512_mask:
15382 case X86::BI__builtin_ia32_vcvtudq2ph256_round_mask:
15383 case X86::BI__builtin_ia32_vcvtudq2ps256_round_mask:
15384 case X86::BI__builtin_ia32_vcvtuqq2pd256_round_mask:
15385 case X86::BI__builtin_ia32_vcvtuqq2ph256_round_mask:
15386 case X86::BI__builtin_ia32_vcvtuqq2ps256_round_mask:
15387 case X86::BI__builtin_ia32_vcvtuw2ph256_round_mask:
15390 case X86::BI__builtin_ia32_vfmaddss3:
15391 case X86::BI__builtin_ia32_vfmaddsd3:
15392 case X86::BI__builtin_ia32_vfmaddsh3_mask:
15393 case X86::BI__builtin_ia32_vfmaddss3_mask:
15394 case X86::BI__builtin_ia32_vfmaddsd3_mask:
15396 case X86::BI__builtin_ia32_vfmaddss:
15397 case X86::BI__builtin_ia32_vfmaddsd:
15399 Constant::getNullValue(Ops[0]->getType()));
15400 case X86::BI__builtin_ia32_vfmaddsh3_maskz:
15401 case X86::BI__builtin_ia32_vfmaddss3_maskz:
15402 case X86::BI__builtin_ia32_vfmaddsd3_maskz:
15404 case X86::BI__builtin_ia32_vfmaddsh3_mask3:
15405 case X86::BI__builtin_ia32_vfmaddss3_mask3:
15406 case X86::BI__builtin_ia32_vfmaddsd3_mask3:
15408 case X86::BI__builtin_ia32_vfmsubsh3_mask3:
15409 case X86::BI__builtin_ia32_vfmsubss3_mask3:
15410 case X86::BI__builtin_ia32_vfmsubsd3_mask3:
15413 case X86::BI__builtin_ia32_vfmaddph:
15414 case X86::BI__builtin_ia32_vfmaddps:
15415 case X86::BI__builtin_ia32_vfmaddpd:
15416 case X86::BI__builtin_ia32_vfmaddph256:
15417 case X86::BI__builtin_ia32_vfmaddps256:
15418 case X86::BI__builtin_ia32_vfmaddpd256:
15419 case X86::BI__builtin_ia32_vfmaddph512_mask:
15420 case X86::BI__builtin_ia32_vfmaddph512_maskz:
15421 case X86::BI__builtin_ia32_vfmaddph512_mask3:
15422 case X86::BI__builtin_ia32_vfmaddnepbh128:
15423 case X86::BI__builtin_ia32_vfmaddnepbh256:
15424 case X86::BI__builtin_ia32_vfmaddnepbh512:
15425 case X86::BI__builtin_ia32_vfmaddps512_mask:
15426 case X86::BI__builtin_ia32_vfmaddps512_maskz:
15427 case X86::BI__builtin_ia32_vfmaddps512_mask3:
15428 case X86::BI__builtin_ia32_vfmsubps512_mask3:
15429 case X86::BI__builtin_ia32_vfmaddpd512_mask:
15430 case X86::BI__builtin_ia32_vfmaddpd512_maskz:
15431 case X86::BI__builtin_ia32_vfmaddpd512_mask3:
15432 case X86::BI__builtin_ia32_vfmsubpd512_mask3:
15433 case X86::BI__builtin_ia32_vfmsubph512_mask3:
15434 case X86::BI__builtin_ia32_vfmaddph256_round_mask:
15435 case X86::BI__builtin_ia32_vfmaddph256_round_maskz:
15436 case X86::BI__builtin_ia32_vfmaddph256_round_mask3:
15437 case X86::BI__builtin_ia32_vfmaddps256_round_mask:
15438 case X86::BI__builtin_ia32_vfmaddps256_round_maskz:
15439 case X86::BI__builtin_ia32_vfmaddps256_round_mask3:
15440 case X86::BI__builtin_ia32_vfmsubps256_round_mask3:
15441 case X86::BI__builtin_ia32_vfmaddpd256_round_mask:
15442 case X86::BI__builtin_ia32_vfmaddpd256_round_maskz:
15443 case X86::BI__builtin_ia32_vfmaddpd256_round_mask3:
15444 case X86::BI__builtin_ia32_vfmsubpd256_round_mask3:
15445 case X86::BI__builtin_ia32_vfmsubph256_round_mask3:
15447 case X86::BI__builtin_ia32_vfmaddsubph512_mask:
15448 case X86::BI__builtin_ia32_vfmaddsubph512_maskz:
15449 case X86::BI__builtin_ia32_vfmaddsubph512_mask3:
15450 case X86::BI__builtin_ia32_vfmsubaddph512_mask3:
15451 case X86::BI__builtin_ia32_vfmaddsubps512_mask:
15452 case X86::BI__builtin_ia32_vfmaddsubps512_maskz:
15453 case X86::BI__builtin_ia32_vfmaddsubps512_mask3:
15454 case X86::BI__builtin_ia32_vfmsubaddps512_mask3:
15455 case X86::BI__builtin_ia32_vfmaddsubpd512_mask:
15456 case X86::BI__builtin_ia32_vfmaddsubpd512_maskz:
15457 case X86::BI__builtin_ia32_vfmaddsubpd512_mask3:
15458 case X86::BI__builtin_ia32_vfmsubaddpd512_mask3:
15459 case X86::BI__builtin_ia32_vfmaddsubph256_round_mask:
15460 case X86::BI__builtin_ia32_vfmaddsubph256_round_maskz:
15461 case X86::BI__builtin_ia32_vfmaddsubph256_round_mask3:
15462 case X86::BI__builtin_ia32_vfmsubaddph256_round_mask3:
15463 case X86::BI__builtin_ia32_vfmaddsubps256_round_mask:
15464 case X86::BI__builtin_ia32_vfmaddsubps256_round_maskz:
15465 case X86::BI__builtin_ia32_vfmaddsubps256_round_mask3:
15466 case X86::BI__builtin_ia32_vfmsubaddps256_round_mask3:
15467 case X86::BI__builtin_ia32_vfmaddsubpd256_round_mask:
15468 case X86::BI__builtin_ia32_vfmaddsubpd256_round_maskz:
15469 case X86::BI__builtin_ia32_vfmaddsubpd256_round_mask3:
15470 case X86::BI__builtin_ia32_vfmsubaddpd256_round_mask3:
15473 case X86::BI__builtin_ia32_movdqa32store128_mask:
15474 case X86::BI__builtin_ia32_movdqa64store128_mask:
15475 case X86::BI__builtin_ia32_storeaps128_mask:
15476 case X86::BI__builtin_ia32_storeapd128_mask:
15477 case X86::BI__builtin_ia32_movdqa32store256_mask:
15478 case X86::BI__builtin_ia32_movdqa64store256_mask:
15479 case X86::BI__builtin_ia32_storeaps256_mask:
15480 case X86::BI__builtin_ia32_storeapd256_mask:
15481 case X86::BI__builtin_ia32_movdqa32store512_mask:
15482 case X86::BI__builtin_ia32_movdqa64store512_mask:
15483 case X86::BI__builtin_ia32_storeaps512_mask:
15484 case X86::BI__builtin_ia32_storeapd512_mask:
15489 case X86::BI__builtin_ia32_loadups128_mask:
15490 case X86::BI__builtin_ia32_loadups256_mask:
15491 case X86::BI__builtin_ia32_loadups512_mask:
15492 case X86::BI__builtin_ia32_loadupd128_mask:
15493 case X86::BI__builtin_ia32_loadupd256_mask:
15494 case X86::BI__builtin_ia32_loadupd512_mask:
15495 case X86::BI__builtin_ia32_loaddquqi128_mask:
15496 case X86::BI__builtin_ia32_loaddquqi256_mask:
15497 case X86::BI__builtin_ia32_loaddquqi512_mask:
15498 case X86::BI__builtin_ia32_loaddquhi128_mask:
15499 case X86::BI__builtin_ia32_loaddquhi256_mask:
15500 case X86::BI__builtin_ia32_loaddquhi512_mask:
15501 case X86::BI__builtin_ia32_loaddqusi128_mask:
15502 case X86::BI__builtin_ia32_loaddqusi256_mask:
15503 case X86::BI__builtin_ia32_loaddqusi512_mask:
15504 case X86::BI__builtin_ia32_loaddqudi128_mask:
15505 case X86::BI__builtin_ia32_loaddqudi256_mask:
15506 case X86::BI__builtin_ia32_loaddqudi512_mask:
15509 case X86::BI__builtin_ia32_loadsbf16128_mask:
15510 case X86::BI__builtin_ia32_loadsh128_mask:
15511 case X86::BI__builtin_ia32_loadss128_mask:
15512 case X86::BI__builtin_ia32_loadsd128_mask:
15515 case X86::BI__builtin_ia32_loadaps128_mask:
15516 case X86::BI__builtin_ia32_loadaps256_mask:
15517 case X86::BI__builtin_ia32_loadaps512_mask:
15518 case X86::BI__builtin_ia32_loadapd128_mask:
15519 case X86::BI__builtin_ia32_loadapd256_mask:
15520 case X86::BI__builtin_ia32_loadapd512_mask:
15521 case X86::BI__builtin_ia32_movdqa32load128_mask:
15522 case X86::BI__builtin_ia32_movdqa32load256_mask:
15523 case X86::BI__builtin_ia32_movdqa32load512_mask:
15524 case X86::BI__builtin_ia32_movdqa64load128_mask:
15525 case X86::BI__builtin_ia32_movdqa64load256_mask:
15526 case X86::BI__builtin_ia32_movdqa64load512_mask:
15531 case X86::BI__builtin_ia32_expandloaddf128_mask:
15532 case X86::BI__builtin_ia32_expandloaddf256_mask:
15533 case X86::BI__builtin_ia32_expandloaddf512_mask:
15534 case X86::BI__builtin_ia32_expandloadsf128_mask:
15535 case X86::BI__builtin_ia32_expandloadsf256_mask:
15536 case X86::BI__builtin_ia32_expandloadsf512_mask:
15537 case X86::BI__builtin_ia32_expandloaddi128_mask:
15538 case X86::BI__builtin_ia32_expandloaddi256_mask:
15539 case X86::BI__builtin_ia32_expandloaddi512_mask:
15540 case X86::BI__builtin_ia32_expandloadsi128_mask:
15541 case X86::BI__builtin_ia32_expandloadsi256_mask:
15542 case X86::BI__builtin_ia32_expandloadsi512_mask:
15543 case X86::BI__builtin_ia32_expandloadhi128_mask:
15544 case X86::BI__builtin_ia32_expandloadhi256_mask:
15545 case X86::BI__builtin_ia32_expandloadhi512_mask:
15546 case X86::BI__builtin_ia32_expandloadqi128_mask:
15547 case X86::BI__builtin_ia32_expandloadqi256_mask:
15548 case X86::BI__builtin_ia32_expandloadqi512_mask:
15551 case X86::BI__builtin_ia32_compressstoredf128_mask:
15552 case X86::BI__builtin_ia32_compressstoredf256_mask:
15553 case X86::BI__builtin_ia32_compressstoredf512_mask:
15554 case X86::BI__builtin_ia32_compressstoresf128_mask:
15555 case X86::BI__builtin_ia32_compressstoresf256_mask:
15556 case X86::BI__builtin_ia32_compressstoresf512_mask:
15557 case X86::BI__builtin_ia32_compressstoredi128_mask:
15558 case X86::BI__builtin_ia32_compressstoredi256_mask:
15559 case X86::BI__builtin_ia32_compressstoredi512_mask:
15560 case X86::BI__builtin_ia32_compressstoresi128_mask:
15561 case X86::BI__builtin_ia32_compressstoresi256_mask:
15562 case X86::BI__builtin_ia32_compressstoresi512_mask:
15563 case X86::BI__builtin_ia32_compressstorehi128_mask:
15564 case X86::BI__builtin_ia32_compressstorehi256_mask:
15565 case X86::BI__builtin_ia32_compressstorehi512_mask:
15566 case X86::BI__builtin_ia32_compressstoreqi128_mask:
15567 case X86::BI__builtin_ia32_compressstoreqi256_mask:
15568 case X86::BI__builtin_ia32_compressstoreqi512_mask:
15571 case X86::BI__builtin_ia32_expanddf128_mask:
15572 case X86::BI__builtin_ia32_expanddf256_mask:
15573 case X86::BI__builtin_ia32_expanddf512_mask:
15574 case X86::BI__builtin_ia32_expandsf128_mask:
15575 case X86::BI__builtin_ia32_expandsf256_mask:
15576 case X86::BI__builtin_ia32_expandsf512_mask:
15577 case X86::BI__builtin_ia32_expanddi128_mask:
15578 case X86::BI__builtin_ia32_expanddi256_mask:
15579 case X86::BI__builtin_ia32_expanddi512_mask:
15580 case X86::BI__builtin_ia32_expandsi128_mask:
15581 case X86::BI__builtin_ia32_expandsi256_mask:
15582 case X86::BI__builtin_ia32_expandsi512_mask:
15583 case X86::BI__builtin_ia32_expandhi128_mask:
15584 case X86::BI__builtin_ia32_expandhi256_mask:
15585 case X86::BI__builtin_ia32_expandhi512_mask:
15586 case X86::BI__builtin_ia32_expandqi128_mask:
15587 case X86::BI__builtin_ia32_expandqi256_mask:
15588 case X86::BI__builtin_ia32_expandqi512_mask:
15591 case X86::BI__builtin_ia32_compressdf128_mask:
15592 case X86::BI__builtin_ia32_compressdf256_mask:
15593 case X86::BI__builtin_ia32_compressdf512_mask:
15594 case X86::BI__builtin_ia32_compresssf128_mask:
15595 case X86::BI__builtin_ia32_compresssf256_mask:
15596 case X86::BI__builtin_ia32_compresssf512_mask:
15597 case X86::BI__builtin_ia32_compressdi128_mask:
15598 case X86::BI__builtin_ia32_compressdi256_mask:
15599 case X86::BI__builtin_ia32_compressdi512_mask:
15600 case X86::BI__builtin_ia32_compresssi128_mask:
15601 case X86::BI__builtin_ia32_compresssi256_mask:
15602 case X86::BI__builtin_ia32_compresssi512_mask:
15603 case X86::BI__builtin_ia32_compresshi128_mask:
15604 case X86::BI__builtin_ia32_compresshi256_mask:
15605 case X86::BI__builtin_ia32_compresshi512_mask:
15606 case X86::BI__builtin_ia32_compressqi128_mask:
15607 case X86::BI__builtin_ia32_compressqi256_mask:
15608 case X86::BI__builtin_ia32_compressqi512_mask:
15611 case X86::BI__builtin_ia32_gather3div2df:
15612 case X86::BI__builtin_ia32_gather3div2di:
15613 case X86::BI__builtin_ia32_gather3div4df:
15614 case X86::BI__builtin_ia32_gather3div4di:
15615 case X86::BI__builtin_ia32_gather3div4sf:
15616 case X86::BI__builtin_ia32_gather3div4si:
15617 case X86::BI__builtin_ia32_gather3div8sf:
15618 case X86::BI__builtin_ia32_gather3div8si:
15619 case X86::BI__builtin_ia32_gather3siv2df:
15620 case X86::BI__builtin_ia32_gather3siv2di:
15621 case X86::BI__builtin_ia32_gather3siv4df:
15622 case X86::BI__builtin_ia32_gather3siv4di:
15623 case X86::BI__builtin_ia32_gather3siv4sf:
15624 case X86::BI__builtin_ia32_gather3siv4si:
15625 case X86::BI__builtin_ia32_gather3siv8sf:
15626 case X86::BI__builtin_ia32_gather3siv8si:
15627 case X86::BI__builtin_ia32_gathersiv8df:
15628 case X86::BI__builtin_ia32_gathersiv16sf:
15629 case X86::BI__builtin_ia32_gatherdiv8df:
15630 case X86::BI__builtin_ia32_gatherdiv16sf:
15631 case X86::BI__builtin_ia32_gathersiv8di:
15632 case X86::BI__builtin_ia32_gathersiv16si:
15633 case X86::BI__builtin_ia32_gatherdiv8di:
15634 case X86::BI__builtin_ia32_gatherdiv16si: {
15636 switch (BuiltinID) {
15637 default: llvm_unreachable(
"Unexpected builtin");
15638 case X86::BI__builtin_ia32_gather3div2df:
15639 IID = Intrinsic::x86_avx512_mask_gather3div2_df;
15641 case X86::BI__builtin_ia32_gather3div2di:
15642 IID = Intrinsic::x86_avx512_mask_gather3div2_di;
15644 case X86::BI__builtin_ia32_gather3div4df:
15645 IID = Intrinsic::x86_avx512_mask_gather3div4_df;
15647 case X86::BI__builtin_ia32_gather3div4di:
15648 IID = Intrinsic::x86_avx512_mask_gather3div4_di;
15650 case X86::BI__builtin_ia32_gather3div4sf:
15651 IID = Intrinsic::x86_avx512_mask_gather3div4_sf;
15653 case X86::BI__builtin_ia32_gather3div4si:
15654 IID = Intrinsic::x86_avx512_mask_gather3div4_si;
15656 case X86::BI__builtin_ia32_gather3div8sf:
15657 IID = Intrinsic::x86_avx512_mask_gather3div8_sf;
15659 case X86::BI__builtin_ia32_gather3div8si:
15660 IID = Intrinsic::x86_avx512_mask_gather3div8_si;
15662 case X86::BI__builtin_ia32_gather3siv2df:
15663 IID = Intrinsic::x86_avx512_mask_gather3siv2_df;
15665 case X86::BI__builtin_ia32_gather3siv2di:
15666 IID = Intrinsic::x86_avx512_mask_gather3siv2_di;
15668 case X86::BI__builtin_ia32_gather3siv4df:
15669 IID = Intrinsic::x86_avx512_mask_gather3siv4_df;
15671 case X86::BI__builtin_ia32_gather3siv4di:
15672 IID = Intrinsic::x86_avx512_mask_gather3siv4_di;
15674 case X86::BI__builtin_ia32_gather3siv4sf:
15675 IID = Intrinsic::x86_avx512_mask_gather3siv4_sf;
15677 case X86::BI__builtin_ia32_gather3siv4si:
15678 IID = Intrinsic::x86_avx512_mask_gather3siv4_si;
15680 case X86::BI__builtin_ia32_gather3siv8sf:
15681 IID = Intrinsic::x86_avx512_mask_gather3siv8_sf;
15683 case X86::BI__builtin_ia32_gather3siv8si:
15684 IID = Intrinsic::x86_avx512_mask_gather3siv8_si;
15686 case X86::BI__builtin_ia32_gathersiv8df:
15687 IID = Intrinsic::x86_avx512_mask_gather_dpd_512;
15689 case X86::BI__builtin_ia32_gathersiv16sf:
15690 IID = Intrinsic::x86_avx512_mask_gather_dps_512;
15692 case X86::BI__builtin_ia32_gatherdiv8df:
15693 IID = Intrinsic::x86_avx512_mask_gather_qpd_512;
15695 case X86::BI__builtin_ia32_gatherdiv16sf:
15696 IID = Intrinsic::x86_avx512_mask_gather_qps_512;
15698 case X86::BI__builtin_ia32_gathersiv8di:
15699 IID = Intrinsic::x86_avx512_mask_gather_dpq_512;
15701 case X86::BI__builtin_ia32_gathersiv16si:
15702 IID = Intrinsic::x86_avx512_mask_gather_dpi_512;
15704 case X86::BI__builtin_ia32_gatherdiv8di:
15705 IID = Intrinsic::x86_avx512_mask_gather_qpq_512;
15707 case X86::BI__builtin_ia32_gatherdiv16si:
15708 IID = Intrinsic::x86_avx512_mask_gather_qpi_512;
15712 unsigned MinElts = std::min(
15713 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements(),
15714 cast<llvm::FixedVectorType>(Ops[2]->getType())->getNumElements());
15717 return Builder.CreateCall(Intr, Ops);
15720 case X86::BI__builtin_ia32_scattersiv8df:
15721 case X86::BI__builtin_ia32_scattersiv16sf:
15722 case X86::BI__builtin_ia32_scatterdiv8df:
15723 case X86::BI__builtin_ia32_scatterdiv16sf:
15724 case X86::BI__builtin_ia32_scattersiv8di:
15725 case X86::BI__builtin_ia32_scattersiv16si:
15726 case X86::BI__builtin_ia32_scatterdiv8di:
15727 case X86::BI__builtin_ia32_scatterdiv16si:
15728 case X86::BI__builtin_ia32_scatterdiv2df:
15729 case X86::BI__builtin_ia32_scatterdiv2di:
15730 case X86::BI__builtin_ia32_scatterdiv4df:
15731 case X86::BI__builtin_ia32_scatterdiv4di:
15732 case X86::BI__builtin_ia32_scatterdiv4sf:
15733 case X86::BI__builtin_ia32_scatterdiv4si:
15734 case X86::BI__builtin_ia32_scatterdiv8sf:
15735 case X86::BI__builtin_ia32_scatterdiv8si:
15736 case X86::BI__builtin_ia32_scattersiv2df:
15737 case X86::BI__builtin_ia32_scattersiv2di:
15738 case X86::BI__builtin_ia32_scattersiv4df:
15739 case X86::BI__builtin_ia32_scattersiv4di:
15740 case X86::BI__builtin_ia32_scattersiv4sf:
15741 case X86::BI__builtin_ia32_scattersiv4si:
15742 case X86::BI__builtin_ia32_scattersiv8sf:
15743 case X86::BI__builtin_ia32_scattersiv8si: {
15745 switch (BuiltinID) {
15746 default: llvm_unreachable(
"Unexpected builtin");
15747 case X86::BI__builtin_ia32_scattersiv8df:
15748 IID = Intrinsic::x86_avx512_mask_scatter_dpd_512;
15750 case X86::BI__builtin_ia32_scattersiv16sf:
15751 IID = Intrinsic::x86_avx512_mask_scatter_dps_512;
15753 case X86::BI__builtin_ia32_scatterdiv8df:
15754 IID = Intrinsic::x86_avx512_mask_scatter_qpd_512;
15756 case X86::BI__builtin_ia32_scatterdiv16sf:
15757 IID = Intrinsic::x86_avx512_mask_scatter_qps_512;
15759 case X86::BI__builtin_ia32_scattersiv8di:
15760 IID = Intrinsic::x86_avx512_mask_scatter_dpq_512;
15762 case X86::BI__builtin_ia32_scattersiv16si:
15763 IID = Intrinsic::x86_avx512_mask_scatter_dpi_512;
15765 case X86::BI__builtin_ia32_scatterdiv8di:
15766 IID = Intrinsic::x86_avx512_mask_scatter_qpq_512;
15768 case X86::BI__builtin_ia32_scatterdiv16si:
15769 IID = Intrinsic::x86_avx512_mask_scatter_qpi_512;
15771 case X86::BI__builtin_ia32_scatterdiv2df:
15772 IID = Intrinsic::x86_avx512_mask_scatterdiv2_df;
15774 case X86::BI__builtin_ia32_scatterdiv2di:
15775 IID = Intrinsic::x86_avx512_mask_scatterdiv2_di;
15777 case X86::BI__builtin_ia32_scatterdiv4df:
15778 IID = Intrinsic::x86_avx512_mask_scatterdiv4_df;
15780 case X86::BI__builtin_ia32_scatterdiv4di:
15781 IID = Intrinsic::x86_avx512_mask_scatterdiv4_di;
15783 case X86::BI__builtin_ia32_scatterdiv4sf:
15784 IID = Intrinsic::x86_avx512_mask_scatterdiv4_sf;
15786 case X86::BI__builtin_ia32_scatterdiv4si:
15787 IID = Intrinsic::x86_avx512_mask_scatterdiv4_si;
15789 case X86::BI__builtin_ia32_scatterdiv8sf:
15790 IID = Intrinsic::x86_avx512_mask_scatterdiv8_sf;
15792 case X86::BI__builtin_ia32_scatterdiv8si:
15793 IID = Intrinsic::x86_avx512_mask_scatterdiv8_si;
15795 case X86::BI__builtin_ia32_scattersiv2df:
15796 IID = Intrinsic::x86_avx512_mask_scattersiv2_df;
15798 case X86::BI__builtin_ia32_scattersiv2di:
15799 IID = Intrinsic::x86_avx512_mask_scattersiv2_di;
15801 case X86::BI__builtin_ia32_scattersiv4df:
15802 IID = Intrinsic::x86_avx512_mask_scattersiv4_df;
15804 case X86::BI__builtin_ia32_scattersiv4di:
15805 IID = Intrinsic::x86_avx512_mask_scattersiv4_di;
15807 case X86::BI__builtin_ia32_scattersiv4sf:
15808 IID = Intrinsic::x86_avx512_mask_scattersiv4_sf;
15810 case X86::BI__builtin_ia32_scattersiv4si:
15811 IID = Intrinsic::x86_avx512_mask_scattersiv4_si;
15813 case X86::BI__builtin_ia32_scattersiv8sf:
15814 IID = Intrinsic::x86_avx512_mask_scattersiv8_sf;
15816 case X86::BI__builtin_ia32_scattersiv8si:
15817 IID = Intrinsic::x86_avx512_mask_scattersiv8_si;
15821 unsigned MinElts = std::min(
15822 cast<llvm::FixedVectorType>(Ops[2]->getType())->getNumElements(),
15823 cast<llvm::FixedVectorType>(Ops[3]->getType())->getNumElements());
15826 return Builder.CreateCall(Intr, Ops);
15829 case X86::BI__builtin_ia32_vextractf128_pd256:
15830 case X86::BI__builtin_ia32_vextractf128_ps256:
15831 case X86::BI__builtin_ia32_vextractf128_si256:
15832 case X86::BI__builtin_ia32_extract128i256:
15833 case X86::BI__builtin_ia32_extractf64x4_mask:
15834 case X86::BI__builtin_ia32_extractf32x4_mask:
15835 case X86::BI__builtin_ia32_extracti64x4_mask:
15836 case X86::BI__builtin_ia32_extracti32x4_mask:
15837 case X86::BI__builtin_ia32_extractf32x8_mask:
15838 case X86::BI__builtin_ia32_extracti32x8_mask:
15839 case X86::BI__builtin_ia32_extractf32x4_256_mask:
15840 case X86::BI__builtin_ia32_extracti32x4_256_mask:
15841 case X86::BI__builtin_ia32_extractf64x2_256_mask:
15842 case X86::BI__builtin_ia32_extracti64x2_256_mask:
15843 case X86::BI__builtin_ia32_extractf64x2_512_mask:
15844 case X86::BI__builtin_ia32_extracti64x2_512_mask: {
15846 unsigned NumElts = DstTy->getNumElements();
15847 unsigned SrcNumElts =
15848 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
15849 unsigned SubVectors = SrcNumElts / NumElts;
15850 unsigned Index = cast<ConstantInt>(Ops[1])->getZExtValue();
15851 assert(llvm::isPowerOf2_32(SubVectors) &&
"Expected power of 2 subvectors");
15852 Index &= SubVectors - 1;
15856 for (
unsigned i = 0; i != NumElts; ++i)
15857 Indices[i] = i + Index;
15862 if (Ops.size() == 4)
15867 case X86::BI__builtin_ia32_vinsertf128_pd256:
15868 case X86::BI__builtin_ia32_vinsertf128_ps256:
15869 case X86::BI__builtin_ia32_vinsertf128_si256:
15870 case X86::BI__builtin_ia32_insert128i256:
15871 case X86::BI__builtin_ia32_insertf64x4:
15872 case X86::BI__builtin_ia32_insertf32x4:
15873 case X86::BI__builtin_ia32_inserti64x4:
15874 case X86::BI__builtin_ia32_inserti32x4:
15875 case X86::BI__builtin_ia32_insertf32x8:
15876 case X86::BI__builtin_ia32_inserti32x8:
15877 case X86::BI__builtin_ia32_insertf32x4_256:
15878 case X86::BI__builtin_ia32_inserti32x4_256:
15879 case X86::BI__builtin_ia32_insertf64x2_256:
15880 case X86::BI__builtin_ia32_inserti64x2_256:
15881 case X86::BI__builtin_ia32_insertf64x2_512:
15882 case X86::BI__builtin_ia32_inserti64x2_512: {
15883 unsigned DstNumElts =
15884 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
15885 unsigned SrcNumElts =
15886 cast<llvm::FixedVectorType>(Ops[1]->getType())->getNumElements();
15887 unsigned SubVectors = DstNumElts / SrcNumElts;
15888 unsigned Index = cast<ConstantInt>(Ops[2])->getZExtValue();
15889 assert(llvm::isPowerOf2_32(SubVectors) &&
"Expected power of 2 subvectors");
15890 Index &= SubVectors - 1;
15891 Index *= SrcNumElts;
15894 for (
unsigned i = 0; i != DstNumElts; ++i)
15895 Indices[i] = (i >= SrcNumElts) ? SrcNumElts + (i % SrcNumElts) : i;
15898 Ops[1],
ArrayRef(Indices, DstNumElts),
"widen");
15900 for (
unsigned i = 0; i != DstNumElts; ++i) {
15901 if (i >= Index && i < (Index + SrcNumElts))
15902 Indices[i] = (i - Index) + DstNumElts;
15907 return Builder.CreateShuffleVector(Ops[0], Op1,
15908 ArrayRef(Indices, DstNumElts),
"insert");
15910 case X86::BI__builtin_ia32_pmovqd512_mask:
15911 case X86::BI__builtin_ia32_pmovwb512_mask: {
15912 Value *Res =
Builder.CreateTrunc(Ops[0], Ops[1]->getType());
15915 case X86::BI__builtin_ia32_pmovdb512_mask:
15916 case X86::BI__builtin_ia32_pmovdw512_mask:
15917 case X86::BI__builtin_ia32_pmovqw512_mask: {
15918 if (
const auto *
C = dyn_cast<Constant>(Ops[2]))
15919 if (
C->isAllOnesValue())
15920 return Builder.CreateTrunc(Ops[0], Ops[1]->getType());
15923 switch (BuiltinID) {
15924 default: llvm_unreachable(
"Unsupported intrinsic!");
15925 case X86::BI__builtin_ia32_pmovdb512_mask:
15926 IID = Intrinsic::x86_avx512_mask_pmov_db_512;
15928 case X86::BI__builtin_ia32_pmovdw512_mask:
15929 IID = Intrinsic::x86_avx512_mask_pmov_dw_512;
15931 case X86::BI__builtin_ia32_pmovqw512_mask:
15932 IID = Intrinsic::x86_avx512_mask_pmov_qw_512;
15937 return Builder.CreateCall(Intr, Ops);
15939 case X86::BI__builtin_ia32_pblendw128:
15940 case X86::BI__builtin_ia32_blendpd:
15941 case X86::BI__builtin_ia32_blendps:
15942 case X86::BI__builtin_ia32_blendpd256:
15943 case X86::BI__builtin_ia32_blendps256:
15944 case X86::BI__builtin_ia32_pblendw256:
15945 case X86::BI__builtin_ia32_pblendd128:
15946 case X86::BI__builtin_ia32_pblendd256: {
15948 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
15949 unsigned Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
15954 for (
unsigned i = 0; i != NumElts; ++i)
15955 Indices[i] = ((Imm >> (i % 8)) & 0x1) ? NumElts + i : i;
15957 return Builder.CreateShuffleVector(Ops[0], Ops[1],
15958 ArrayRef(Indices, NumElts),
"blend");
15960 case X86::BI__builtin_ia32_pshuflw:
15961 case X86::BI__builtin_ia32_pshuflw256:
15962 case X86::BI__builtin_ia32_pshuflw512: {
15963 uint32_t Imm = cast<llvm::ConstantInt>(Ops[1])->getZExtValue();
15964 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType());
15965 unsigned NumElts = Ty->getNumElements();
15968 Imm = (Imm & 0xff) * 0x01010101;
15971 for (
unsigned l = 0; l != NumElts; l += 8) {
15972 for (
unsigned i = 0; i != 4; ++i) {
15973 Indices[l + i] = l + (Imm & 3);
15976 for (
unsigned i = 4; i != 8; ++i)
15977 Indices[l + i] = l + i;
15980 return Builder.CreateShuffleVector(Ops[0],
ArrayRef(Indices, NumElts),
15983 case X86::BI__builtin_ia32_pshufhw:
15984 case X86::BI__builtin_ia32_pshufhw256:
15985 case X86::BI__builtin_ia32_pshufhw512: {
15986 uint32_t Imm = cast<llvm::ConstantInt>(Ops[1])->getZExtValue();
15987 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType());
15988 unsigned NumElts = Ty->getNumElements();
15991 Imm = (Imm & 0xff) * 0x01010101;
15994 for (
unsigned l = 0; l != NumElts; l += 8) {
15995 for (
unsigned i = 0; i != 4; ++i)
15996 Indices[l + i] = l + i;
15997 for (
unsigned i = 4; i != 8; ++i) {
15998 Indices[l + i] = l + 4 + (Imm & 3);
16003 return Builder.CreateShuffleVector(Ops[0],
ArrayRef(Indices, NumElts),
16006 case X86::BI__builtin_ia32_pshufd:
16007 case X86::BI__builtin_ia32_pshufd256:
16008 case X86::BI__builtin_ia32_pshufd512:
16009 case X86::BI__builtin_ia32_vpermilpd:
16010 case X86::BI__builtin_ia32_vpermilps:
16011 case X86::BI__builtin_ia32_vpermilpd256:
16012 case X86::BI__builtin_ia32_vpermilps256:
16013 case X86::BI__builtin_ia32_vpermilpd512:
16014 case X86::BI__builtin_ia32_vpermilps512: {
16015 uint32_t Imm = cast<llvm::ConstantInt>(Ops[1])->getZExtValue();
16016 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType());
16017 unsigned NumElts = Ty->getNumElements();
16018 unsigned NumLanes = Ty->getPrimitiveSizeInBits() / 128;
16019 unsigned NumLaneElts = NumElts / NumLanes;
16022 Imm = (Imm & 0xff) * 0x01010101;
16025 for (
unsigned l = 0; l != NumElts; l += NumLaneElts) {
16026 for (
unsigned i = 0; i != NumLaneElts; ++i) {
16027 Indices[i + l] = (Imm % NumLaneElts) + l;
16028 Imm /= NumLaneElts;
16032 return Builder.CreateShuffleVector(Ops[0],
ArrayRef(Indices, NumElts),
16035 case X86::BI__builtin_ia32_shufpd:
16036 case X86::BI__builtin_ia32_shufpd256:
16037 case X86::BI__builtin_ia32_shufpd512:
16038 case X86::BI__builtin_ia32_shufps:
16039 case X86::BI__builtin_ia32_shufps256:
16040 case X86::BI__builtin_ia32_shufps512: {
16041 uint32_t Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
16042 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType());
16043 unsigned NumElts = Ty->getNumElements();
16044 unsigned NumLanes = Ty->getPrimitiveSizeInBits() / 128;
16045 unsigned NumLaneElts = NumElts / NumLanes;
16048 Imm = (Imm & 0xff) * 0x01010101;
16051 for (
unsigned l = 0; l != NumElts; l += NumLaneElts) {
16052 for (
unsigned i = 0; i != NumLaneElts; ++i) {
16053 unsigned Index = Imm % NumLaneElts;
16054 Imm /= NumLaneElts;
16055 if (i >= (NumLaneElts / 2))
16057 Indices[l + i] = l + Index;
16061 return Builder.CreateShuffleVector(Ops[0], Ops[1],
16062 ArrayRef(Indices, NumElts),
"shufp");
16064 case X86::BI__builtin_ia32_permdi256:
16065 case X86::BI__builtin_ia32_permdf256:
16066 case X86::BI__builtin_ia32_permdi512:
16067 case X86::BI__builtin_ia32_permdf512: {
16068 unsigned Imm = cast<llvm::ConstantInt>(Ops[1])->getZExtValue();
16069 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType());
16070 unsigned NumElts = Ty->getNumElements();
16074 for (
unsigned l = 0; l != NumElts; l += 4)
16075 for (
unsigned i = 0; i != 4; ++i)
16076 Indices[l + i] = l + ((Imm >> (2 * i)) & 0x3);
16078 return Builder.CreateShuffleVector(Ops[0],
ArrayRef(Indices, NumElts),
16081 case X86::BI__builtin_ia32_palignr128:
16082 case X86::BI__builtin_ia32_palignr256:
16083 case X86::BI__builtin_ia32_palignr512: {
16084 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0xff;
16087 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
16088 assert(NumElts % 16 == 0);
16092 if (ShiftVal >= 32)
16097 if (ShiftVal > 16) {
16100 Ops[0] = llvm::Constant::getNullValue(Ops[0]->getType());
16105 for (
unsigned l = 0; l != NumElts; l += 16) {
16106 for (
unsigned i = 0; i != 16; ++i) {
16107 unsigned Idx = ShiftVal + i;
16109 Idx += NumElts - 16;
16110 Indices[l + i] = Idx + l;
16114 return Builder.CreateShuffleVector(Ops[1], Ops[0],
16115 ArrayRef(Indices, NumElts),
"palignr");
16117 case X86::BI__builtin_ia32_alignd128:
16118 case X86::BI__builtin_ia32_alignd256:
16119 case X86::BI__builtin_ia32_alignd512:
16120 case X86::BI__builtin_ia32_alignq128:
16121 case X86::BI__builtin_ia32_alignq256:
16122 case X86::BI__builtin_ia32_alignq512: {
16124 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
16125 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0xff;
16128 ShiftVal &= NumElts - 1;
16131 for (
unsigned i = 0; i != NumElts; ++i)
16132 Indices[i] = i + ShiftVal;
16134 return Builder.CreateShuffleVector(Ops[1], Ops[0],
16135 ArrayRef(Indices, NumElts),
"valign");
16137 case X86::BI__builtin_ia32_shuf_f32x4_256:
16138 case X86::BI__builtin_ia32_shuf_f64x2_256:
16139 case X86::BI__builtin_ia32_shuf_i32x4_256:
16140 case X86::BI__builtin_ia32_shuf_i64x2_256:
16141 case X86::BI__builtin_ia32_shuf_f32x4:
16142 case X86::BI__builtin_ia32_shuf_f64x2:
16143 case X86::BI__builtin_ia32_shuf_i32x4:
16144 case X86::BI__builtin_ia32_shuf_i64x2: {
16145 unsigned Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
16146 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType());
16147 unsigned NumElts = Ty->getNumElements();
16148 unsigned NumLanes = Ty->getPrimitiveSizeInBits() == 512 ? 4 : 2;
16149 unsigned NumLaneElts = NumElts / NumLanes;
16152 for (
unsigned l = 0; l != NumElts; l += NumLaneElts) {
16153 unsigned Index = (Imm % NumLanes) * NumLaneElts;
16155 if (l >= (NumElts / 2))
16157 for (
unsigned i = 0; i != NumLaneElts; ++i) {
16158 Indices[l + i] = Index + i;
16162 return Builder.CreateShuffleVector(Ops[0], Ops[1],
16163 ArrayRef(Indices, NumElts),
"shuf");
16166 case X86::BI__builtin_ia32_vperm2f128_pd256:
16167 case X86::BI__builtin_ia32_vperm2f128_ps256:
16168 case X86::BI__builtin_ia32_vperm2f128_si256:
16169 case X86::BI__builtin_ia32_permti256: {
16170 unsigned Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
16172 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
16181 for (
unsigned l = 0; l != 2; ++l) {
16183 if (Imm & (1 << ((l * 4) + 3)))
16184 OutOps[l] = llvm::ConstantAggregateZero::get(Ops[0]->getType());
16185 else if (Imm & (1 << ((l * 4) + 1)))
16186 OutOps[l] = Ops[1];
16188 OutOps[l] = Ops[0];
16190 for (
unsigned i = 0; i != NumElts/2; ++i) {
16192 unsigned Idx = (l * NumElts) + i;
16195 if (Imm & (1 << (l * 4)))
16197 Indices[(l * (NumElts/2)) + i] = Idx;
16201 return Builder.CreateShuffleVector(OutOps[0], OutOps[1],
16202 ArrayRef(Indices, NumElts),
"vperm");
16205 case X86::BI__builtin_ia32_pslldqi128_byteshift:
16206 case X86::BI__builtin_ia32_pslldqi256_byteshift:
16207 case X86::BI__builtin_ia32_pslldqi512_byteshift: {
16208 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() & 0xff;
16209 auto *ResultType = cast<llvm::FixedVectorType>(Ops[0]->getType());
16211 unsigned NumElts = ResultType->getNumElements() * 8;
16214 if (ShiftVal >= 16)
16215 return llvm::Constant::getNullValue(ResultType);
16219 for (
unsigned l = 0; l != NumElts; l += 16) {
16220 for (
unsigned i = 0; i != 16; ++i) {
16221 unsigned Idx = NumElts + i - ShiftVal;
16222 if (Idx < NumElts) Idx -= NumElts - 16;
16223 Indices[l + i] = Idx + l;
16227 auto *VecTy = llvm::FixedVectorType::get(
Int8Ty, NumElts);
16229 Value *
Zero = llvm::Constant::getNullValue(VecTy);
16231 Zero, Cast,
ArrayRef(Indices, NumElts),
"pslldq");
16232 return Builder.CreateBitCast(SV, Ops[0]->getType(),
"cast");
16234 case X86::BI__builtin_ia32_psrldqi128_byteshift:
16235 case X86::BI__builtin_ia32_psrldqi256_byteshift:
16236 case X86::BI__builtin_ia32_psrldqi512_byteshift: {
16237 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() & 0xff;
16238 auto *ResultType = cast<llvm::FixedVectorType>(Ops[0]->getType());
16240 unsigned NumElts = ResultType->getNumElements() * 8;
16243 if (ShiftVal >= 16)
16244 return llvm::Constant::getNullValue(ResultType);
16248 for (
unsigned l = 0; l != NumElts; l += 16) {
16249 for (
unsigned i = 0; i != 16; ++i) {
16250 unsigned Idx = i + ShiftVal;
16251 if (Idx >= 16) Idx += NumElts - 16;
16252 Indices[l + i] = Idx + l;
16256 auto *VecTy = llvm::FixedVectorType::get(
Int8Ty, NumElts);
16258 Value *
Zero = llvm::Constant::getNullValue(VecTy);
16260 Cast, Zero,
ArrayRef(Indices, NumElts),
"psrldq");
16261 return Builder.CreateBitCast(SV, ResultType,
"cast");
16263 case X86::BI__builtin_ia32_kshiftliqi:
16264 case X86::BI__builtin_ia32_kshiftlihi:
16265 case X86::BI__builtin_ia32_kshiftlisi:
16266 case X86::BI__builtin_ia32_kshiftlidi: {
16267 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() & 0xff;
16268 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
16270 if (ShiftVal >= NumElts)
16271 return llvm::Constant::getNullValue(Ops[0]->getType());
16276 for (
unsigned i = 0; i != NumElts; ++i)
16277 Indices[i] = NumElts + i - ShiftVal;
16279 Value *
Zero = llvm::Constant::getNullValue(
In->getType());
16281 Zero, In,
ArrayRef(Indices, NumElts),
"kshiftl");
16282 return Builder.CreateBitCast(SV, Ops[0]->getType());
16284 case X86::BI__builtin_ia32_kshiftriqi:
16285 case X86::BI__builtin_ia32_kshiftrihi:
16286 case X86::BI__builtin_ia32_kshiftrisi:
16287 case X86::BI__builtin_ia32_kshiftridi: {
16288 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() & 0xff;
16289 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
16291 if (ShiftVal >= NumElts)
16292 return llvm::Constant::getNullValue(Ops[0]->getType());
16297 for (
unsigned i = 0; i != NumElts; ++i)
16298 Indices[i] = i + ShiftVal;
16300 Value *
Zero = llvm::Constant::getNullValue(
In->getType());
16302 In, Zero,
ArrayRef(Indices, NumElts),
"kshiftr");
16303 return Builder.CreateBitCast(SV, Ops[0]->getType());
16305 case X86::BI__builtin_ia32_movnti:
16306 case X86::BI__builtin_ia32_movnti64:
16307 case X86::BI__builtin_ia32_movntsd:
16308 case X86::BI__builtin_ia32_movntss: {
16309 llvm::MDNode *
Node = llvm::MDNode::get(
16312 Value *Ptr = Ops[0];
16313 Value *Src = Ops[1];
16316 if (BuiltinID == X86::BI__builtin_ia32_movntsd ||
16317 BuiltinID == X86::BI__builtin_ia32_movntss)
16318 Src =
Builder.CreateExtractElement(Src, (uint64_t)0,
"extract");
16322 SI->setMetadata(llvm::LLVMContext::MD_nontemporal,
Node);
16323 SI->setAlignment(llvm::Align(1));
16327 case X86::BI__builtin_ia32_vprotb:
16328 case X86::BI__builtin_ia32_vprotw:
16329 case X86::BI__builtin_ia32_vprotd:
16330 case X86::BI__builtin_ia32_vprotq:
16331 case X86::BI__builtin_ia32_vprotbi:
16332 case X86::BI__builtin_ia32_vprotwi:
16333 case X86::BI__builtin_ia32_vprotdi:
16334 case X86::BI__builtin_ia32_vprotqi:
16335 case X86::BI__builtin_ia32_prold128:
16336 case X86::BI__builtin_ia32_prold256:
16337 case X86::BI__builtin_ia32_prold512:
16338 case X86::BI__builtin_ia32_prolq128:
16339 case X86::BI__builtin_ia32_prolq256:
16340 case X86::BI__builtin_ia32_prolq512:
16341 case X86::BI__builtin_ia32_prolvd128:
16342 case X86::BI__builtin_ia32_prolvd256:
16343 case X86::BI__builtin_ia32_prolvd512:
16344 case X86::BI__builtin_ia32_prolvq128:
16345 case X86::BI__builtin_ia32_prolvq256:
16346 case X86::BI__builtin_ia32_prolvq512:
16348 case X86::BI__builtin_ia32_prord128:
16349 case X86::BI__builtin_ia32_prord256:
16350 case X86::BI__builtin_ia32_prord512:
16351 case X86::BI__builtin_ia32_prorq128:
16352 case X86::BI__builtin_ia32_prorq256:
16353 case X86::BI__builtin_ia32_prorq512:
16354 case X86::BI__builtin_ia32_prorvd128:
16355 case X86::BI__builtin_ia32_prorvd256:
16356 case X86::BI__builtin_ia32_prorvd512:
16357 case X86::BI__builtin_ia32_prorvq128:
16358 case X86::BI__builtin_ia32_prorvq256:
16359 case X86::BI__builtin_ia32_prorvq512:
16361 case X86::BI__builtin_ia32_selectb_128:
16362 case X86::BI__builtin_ia32_selectb_256:
16363 case X86::BI__builtin_ia32_selectb_512:
16364 case X86::BI__builtin_ia32_selectw_128:
16365 case X86::BI__builtin_ia32_selectw_256:
16366 case X86::BI__builtin_ia32_selectw_512:
16367 case X86::BI__builtin_ia32_selectd_128:
16368 case X86::BI__builtin_ia32_selectd_256:
16369 case X86::BI__builtin_ia32_selectd_512:
16370 case X86::BI__builtin_ia32_selectq_128:
16371 case X86::BI__builtin_ia32_selectq_256:
16372 case X86::BI__builtin_ia32_selectq_512:
16373 case X86::BI__builtin_ia32_selectph_128:
16374 case X86::BI__builtin_ia32_selectph_256:
16375 case X86::BI__builtin_ia32_selectph_512:
16376 case X86::BI__builtin_ia32_selectpbf_128:
16377 case X86::BI__builtin_ia32_selectpbf_256:
16378 case X86::BI__builtin_ia32_selectpbf_512:
16379 case X86::BI__builtin_ia32_selectps_128:
16380 case X86::BI__builtin_ia32_selectps_256:
16381 case X86::BI__builtin_ia32_selectps_512:
16382 case X86::BI__builtin_ia32_selectpd_128:
16383 case X86::BI__builtin_ia32_selectpd_256:
16384 case X86::BI__builtin_ia32_selectpd_512:
16386 case X86::BI__builtin_ia32_selectsh_128:
16387 case X86::BI__builtin_ia32_selectsbf_128:
16388 case X86::BI__builtin_ia32_selectss_128:
16389 case X86::BI__builtin_ia32_selectsd_128: {
16390 Value *A =
Builder.CreateExtractElement(Ops[1], (uint64_t)0);
16391 Value *B =
Builder.CreateExtractElement(Ops[2], (uint64_t)0);
16393 return Builder.CreateInsertElement(Ops[1], A, (uint64_t)0);
16395 case X86::BI__builtin_ia32_cmpb128_mask:
16396 case X86::BI__builtin_ia32_cmpb256_mask:
16397 case X86::BI__builtin_ia32_cmpb512_mask:
16398 case X86::BI__builtin_ia32_cmpw128_mask:
16399 case X86::BI__builtin_ia32_cmpw256_mask:
16400 case X86::BI__builtin_ia32_cmpw512_mask:
16401 case X86::BI__builtin_ia32_cmpd128_mask:
16402 case X86::BI__builtin_ia32_cmpd256_mask:
16403 case X86::BI__builtin_ia32_cmpd512_mask:
16404 case X86::BI__builtin_ia32_cmpq128_mask:
16405 case X86::BI__builtin_ia32_cmpq256_mask:
16406 case X86::BI__builtin_ia32_cmpq512_mask: {
16407 unsigned CC = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x7;
16410 case X86::BI__builtin_ia32_ucmpb128_mask:
16411 case X86::BI__builtin_ia32_ucmpb256_mask:
16412 case X86::BI__builtin_ia32_ucmpb512_mask:
16413 case X86::BI__builtin_ia32_ucmpw128_mask:
16414 case X86::BI__builtin_ia32_ucmpw256_mask:
16415 case X86::BI__builtin_ia32_ucmpw512_mask:
16416 case X86::BI__builtin_ia32_ucmpd128_mask:
16417 case X86::BI__builtin_ia32_ucmpd256_mask:
16418 case X86::BI__builtin_ia32_ucmpd512_mask:
16419 case X86::BI__builtin_ia32_ucmpq128_mask:
16420 case X86::BI__builtin_ia32_ucmpq256_mask:
16421 case X86::BI__builtin_ia32_ucmpq512_mask: {
16422 unsigned CC = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x7;
16425 case X86::BI__builtin_ia32_vpcomb:
16426 case X86::BI__builtin_ia32_vpcomw:
16427 case X86::BI__builtin_ia32_vpcomd:
16428 case X86::BI__builtin_ia32_vpcomq:
16430 case X86::BI__builtin_ia32_vpcomub:
16431 case X86::BI__builtin_ia32_vpcomuw:
16432 case X86::BI__builtin_ia32_vpcomud:
16433 case X86::BI__builtin_ia32_vpcomuq:
16436 case X86::BI__builtin_ia32_kortestcqi:
16437 case X86::BI__builtin_ia32_kortestchi:
16438 case X86::BI__builtin_ia32_kortestcsi:
16439 case X86::BI__builtin_ia32_kortestcdi: {
16441 Value *
C = llvm::Constant::getAllOnesValue(Ops[0]->getType());
16445 case X86::BI__builtin_ia32_kortestzqi:
16446 case X86::BI__builtin_ia32_kortestzhi:
16447 case X86::BI__builtin_ia32_kortestzsi:
16448 case X86::BI__builtin_ia32_kortestzdi: {
16450 Value *
C = llvm::Constant::getNullValue(Ops[0]->getType());
16455 case X86::BI__builtin_ia32_ktestcqi:
16456 case X86::BI__builtin_ia32_ktestzqi:
16457 case X86::BI__builtin_ia32_ktestchi:
16458 case X86::BI__builtin_ia32_ktestzhi:
16459 case X86::BI__builtin_ia32_ktestcsi:
16460 case X86::BI__builtin_ia32_ktestzsi:
16461 case X86::BI__builtin_ia32_ktestcdi:
16462 case X86::BI__builtin_ia32_ktestzdi: {
16464 switch (BuiltinID) {
16465 default: llvm_unreachable(
"Unsupported intrinsic!");
16466 case X86::BI__builtin_ia32_ktestcqi:
16467 IID = Intrinsic::x86_avx512_ktestc_b;
16469 case X86::BI__builtin_ia32_ktestzqi:
16470 IID = Intrinsic::x86_avx512_ktestz_b;
16472 case X86::BI__builtin_ia32_ktestchi:
16473 IID = Intrinsic::x86_avx512_ktestc_w;
16475 case X86::BI__builtin_ia32_ktestzhi:
16476 IID = Intrinsic::x86_avx512_ktestz_w;
16478 case X86::BI__builtin_ia32_ktestcsi:
16479 IID = Intrinsic::x86_avx512_ktestc_d;
16481 case X86::BI__builtin_ia32_ktestzsi:
16482 IID = Intrinsic::x86_avx512_ktestz_d;
16484 case X86::BI__builtin_ia32_ktestcdi:
16485 IID = Intrinsic::x86_avx512_ktestc_q;
16487 case X86::BI__builtin_ia32_ktestzdi:
16488 IID = Intrinsic::x86_avx512_ktestz_q;
16492 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
16496 return Builder.CreateCall(Intr, {LHS, RHS});
16499 case X86::BI__builtin_ia32_kaddqi:
16500 case X86::BI__builtin_ia32_kaddhi:
16501 case X86::BI__builtin_ia32_kaddsi:
16502 case X86::BI__builtin_ia32_kadddi: {
16504 switch (BuiltinID) {
16505 default: llvm_unreachable(
"Unsupported intrinsic!");
16506 case X86::BI__builtin_ia32_kaddqi:
16507 IID = Intrinsic::x86_avx512_kadd_b;
16509 case X86::BI__builtin_ia32_kaddhi:
16510 IID = Intrinsic::x86_avx512_kadd_w;
16512 case X86::BI__builtin_ia32_kaddsi:
16513 IID = Intrinsic::x86_avx512_kadd_d;
16515 case X86::BI__builtin_ia32_kadddi:
16516 IID = Intrinsic::x86_avx512_kadd_q;
16520 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
16525 return Builder.CreateBitCast(Res, Ops[0]->getType());
16527 case X86::BI__builtin_ia32_kandqi:
16528 case X86::BI__builtin_ia32_kandhi:
16529 case X86::BI__builtin_ia32_kandsi:
16530 case X86::BI__builtin_ia32_kanddi:
16532 case X86::BI__builtin_ia32_kandnqi:
16533 case X86::BI__builtin_ia32_kandnhi:
16534 case X86::BI__builtin_ia32_kandnsi:
16535 case X86::BI__builtin_ia32_kandndi:
16537 case X86::BI__builtin_ia32_korqi:
16538 case X86::BI__builtin_ia32_korhi:
16539 case X86::BI__builtin_ia32_korsi:
16540 case X86::BI__builtin_ia32_kordi:
16542 case X86::BI__builtin_ia32_kxnorqi:
16543 case X86::BI__builtin_ia32_kxnorhi:
16544 case X86::BI__builtin_ia32_kxnorsi:
16545 case X86::BI__builtin_ia32_kxnordi:
16547 case X86::BI__builtin_ia32_kxorqi:
16548 case X86::BI__builtin_ia32_kxorhi:
16549 case X86::BI__builtin_ia32_kxorsi:
16550 case X86::BI__builtin_ia32_kxordi:
16552 case X86::BI__builtin_ia32_knotqi:
16553 case X86::BI__builtin_ia32_knothi:
16554 case X86::BI__builtin_ia32_knotsi:
16555 case X86::BI__builtin_ia32_knotdi: {
16556 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
16559 Ops[0]->getType());
16561 case X86::BI__builtin_ia32_kmovb:
16562 case X86::BI__builtin_ia32_kmovw:
16563 case X86::BI__builtin_ia32_kmovd:
16564 case X86::BI__builtin_ia32_kmovq: {
16568 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
16570 return Builder.CreateBitCast(Res, Ops[0]->getType());
16573 case X86::BI__builtin_ia32_kunpckdi:
16574 case X86::BI__builtin_ia32_kunpcksi:
16575 case X86::BI__builtin_ia32_kunpckhi: {
16576 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
16580 for (
unsigned i = 0; i != NumElts; ++i)
16585 LHS =
Builder.CreateShuffleVector(LHS, LHS,
ArrayRef(Indices, NumElts / 2));
16586 RHS =
Builder.CreateShuffleVector(RHS, RHS,
ArrayRef(Indices, NumElts / 2));
16591 return Builder.CreateBitCast(Res, Ops[0]->getType());
16594 case X86::BI__builtin_ia32_vplzcntd_128:
16595 case X86::BI__builtin_ia32_vplzcntd_256:
16596 case X86::BI__builtin_ia32_vplzcntd_512:
16597 case X86::BI__builtin_ia32_vplzcntq_128:
16598 case X86::BI__builtin_ia32_vplzcntq_256:
16599 case X86::BI__builtin_ia32_vplzcntq_512: {
16603 case X86::BI__builtin_ia32_sqrtss:
16604 case X86::BI__builtin_ia32_sqrtsd: {
16605 Value *A =
Builder.CreateExtractElement(Ops[0], (uint64_t)0);
16607 if (
Builder.getIsFPConstrained()) {
16608 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
16611 A =
Builder.CreateConstrainedFPCall(F, {A});
16614 A =
Builder.CreateCall(F, {A});
16616 return Builder.CreateInsertElement(Ops[0], A, (uint64_t)0);
16618 case X86::BI__builtin_ia32_sqrtsh_round_mask:
16619 case X86::BI__builtin_ia32_sqrtsd_round_mask:
16620 case X86::BI__builtin_ia32_sqrtss_round_mask: {
16621 unsigned CC = cast<llvm::ConstantInt>(Ops[4])->getZExtValue();
16627 switch (BuiltinID) {
16629 llvm_unreachable(
"Unsupported intrinsic!");
16630 case X86::BI__builtin_ia32_sqrtsh_round_mask:
16631 IID = Intrinsic::x86_avx512fp16_mask_sqrt_sh;
16633 case X86::BI__builtin_ia32_sqrtsd_round_mask:
16634 IID = Intrinsic::x86_avx512_mask_sqrt_sd;
16636 case X86::BI__builtin_ia32_sqrtss_round_mask:
16637 IID = Intrinsic::x86_avx512_mask_sqrt_ss;
16642 Value *A =
Builder.CreateExtractElement(Ops[1], (uint64_t)0);
16644 if (
Builder.getIsFPConstrained()) {
16645 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
16648 A =
Builder.CreateConstrainedFPCall(F, A);
16651 A =
Builder.CreateCall(F, A);
16653 Value *Src =
Builder.CreateExtractElement(Ops[2], (uint64_t)0);
16655 return Builder.CreateInsertElement(Ops[0], A, (uint64_t)0);
16657 case X86::BI__builtin_ia32_sqrtpd256:
16658 case X86::BI__builtin_ia32_sqrtpd:
16659 case X86::BI__builtin_ia32_sqrtps256:
16660 case X86::BI__builtin_ia32_sqrtps:
16661 case X86::BI__builtin_ia32_sqrtph256:
16662 case X86::BI__builtin_ia32_sqrtph:
16663 case X86::BI__builtin_ia32_sqrtph512:
16664 case X86::BI__builtin_ia32_vsqrtnepbf16256:
16665 case X86::BI__builtin_ia32_vsqrtnepbf16:
16666 case X86::BI__builtin_ia32_vsqrtnepbf16512:
16667 case X86::BI__builtin_ia32_sqrtps512:
16668 case X86::BI__builtin_ia32_sqrtpd512: {
16669 if (Ops.size() == 2) {
16670 unsigned CC = cast<llvm::ConstantInt>(Ops[1])->getZExtValue();
16676 switch (BuiltinID) {
16678 llvm_unreachable(
"Unsupported intrinsic!");
16679 case X86::BI__builtin_ia32_sqrtph512:
16680 IID = Intrinsic::x86_avx512fp16_sqrt_ph_512;
16682 case X86::BI__builtin_ia32_sqrtps512:
16683 IID = Intrinsic::x86_avx512_sqrt_ps_512;
16685 case X86::BI__builtin_ia32_sqrtpd512:
16686 IID = Intrinsic::x86_avx512_sqrt_pd_512;
16692 if (
Builder.getIsFPConstrained()) {
16693 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
16695 Ops[0]->getType());
16696 return Builder.CreateConstrainedFPCall(F, Ops[0]);
16699 return Builder.CreateCall(F, Ops[0]);
16703 case X86::BI__builtin_ia32_pmuludq128:
16704 case X86::BI__builtin_ia32_pmuludq256:
16705 case X86::BI__builtin_ia32_pmuludq512:
16708 case X86::BI__builtin_ia32_pmuldq128:
16709 case X86::BI__builtin_ia32_pmuldq256:
16710 case X86::BI__builtin_ia32_pmuldq512:
16713 case X86::BI__builtin_ia32_pternlogd512_mask:
16714 case X86::BI__builtin_ia32_pternlogq512_mask:
16715 case X86::BI__builtin_ia32_pternlogd128_mask:
16716 case X86::BI__builtin_ia32_pternlogd256_mask:
16717 case X86::BI__builtin_ia32_pternlogq128_mask:
16718 case X86::BI__builtin_ia32_pternlogq256_mask:
16721 case X86::BI__builtin_ia32_pternlogd512_maskz:
16722 case X86::BI__builtin_ia32_pternlogq512_maskz:
16723 case X86::BI__builtin_ia32_pternlogd128_maskz:
16724 case X86::BI__builtin_ia32_pternlogd256_maskz:
16725 case X86::BI__builtin_ia32_pternlogq128_maskz:
16726 case X86::BI__builtin_ia32_pternlogq256_maskz:
16729 case X86::BI__builtin_ia32_vpshldd128:
16730 case X86::BI__builtin_ia32_vpshldd256:
16731 case X86::BI__builtin_ia32_vpshldd512:
16732 case X86::BI__builtin_ia32_vpshldq128:
16733 case X86::BI__builtin_ia32_vpshldq256:
16734 case X86::BI__builtin_ia32_vpshldq512:
16735 case X86::BI__builtin_ia32_vpshldw128:
16736 case X86::BI__builtin_ia32_vpshldw256:
16737 case X86::BI__builtin_ia32_vpshldw512:
16740 case X86::BI__builtin_ia32_vpshrdd128:
16741 case X86::BI__builtin_ia32_vpshrdd256:
16742 case X86::BI__builtin_ia32_vpshrdd512:
16743 case X86::BI__builtin_ia32_vpshrdq128:
16744 case X86::BI__builtin_ia32_vpshrdq256:
16745 case X86::BI__builtin_ia32_vpshrdq512:
16746 case X86::BI__builtin_ia32_vpshrdw128:
16747 case X86::BI__builtin_ia32_vpshrdw256:
16748 case X86::BI__builtin_ia32_vpshrdw512:
16752 case X86::BI__builtin_ia32_vpshldvd128:
16753 case X86::BI__builtin_ia32_vpshldvd256:
16754 case X86::BI__builtin_ia32_vpshldvd512:
16755 case X86::BI__builtin_ia32_vpshldvq128:
16756 case X86::BI__builtin_ia32_vpshldvq256:
16757 case X86::BI__builtin_ia32_vpshldvq512:
16758 case X86::BI__builtin_ia32_vpshldvw128:
16759 case X86::BI__builtin_ia32_vpshldvw256:
16760 case X86::BI__builtin_ia32_vpshldvw512:
16763 case X86::BI__builtin_ia32_vpshrdvd128:
16764 case X86::BI__builtin_ia32_vpshrdvd256:
16765 case X86::BI__builtin_ia32_vpshrdvd512:
16766 case X86::BI__builtin_ia32_vpshrdvq128:
16767 case X86::BI__builtin_ia32_vpshrdvq256:
16768 case X86::BI__builtin_ia32_vpshrdvq512:
16769 case X86::BI__builtin_ia32_vpshrdvw128:
16770 case X86::BI__builtin_ia32_vpshrdvw256:
16771 case X86::BI__builtin_ia32_vpshrdvw512:
16776 case X86::BI__builtin_ia32_reduce_fadd_pd512:
16777 case X86::BI__builtin_ia32_reduce_fadd_ps512:
16778 case X86::BI__builtin_ia32_reduce_fadd_ph512:
16779 case X86::BI__builtin_ia32_reduce_fadd_ph256:
16780 case X86::BI__builtin_ia32_reduce_fadd_ph128: {
16783 IRBuilder<>::FastMathFlagGuard FMFGuard(
Builder);
16784 Builder.getFastMathFlags().setAllowReassoc();
16785 return Builder.CreateCall(F, {Ops[0], Ops[1]});
16787 case X86::BI__builtin_ia32_reduce_fmul_pd512:
16788 case X86::BI__builtin_ia32_reduce_fmul_ps512:
16789 case X86::BI__builtin_ia32_reduce_fmul_ph512:
16790 case X86::BI__builtin_ia32_reduce_fmul_ph256:
16791 case X86::BI__builtin_ia32_reduce_fmul_ph128: {
16794 IRBuilder<>::FastMathFlagGuard FMFGuard(
Builder);
16795 Builder.getFastMathFlags().setAllowReassoc();
16796 return Builder.CreateCall(F, {Ops[0], Ops[1]});
16798 case X86::BI__builtin_ia32_reduce_fmax_pd512:
16799 case X86::BI__builtin_ia32_reduce_fmax_ps512:
16800 case X86::BI__builtin_ia32_reduce_fmax_ph512:
16801 case X86::BI__builtin_ia32_reduce_fmax_ph256:
16802 case X86::BI__builtin_ia32_reduce_fmax_ph128: {
16805 IRBuilder<>::FastMathFlagGuard FMFGuard(
Builder);
16806 Builder.getFastMathFlags().setNoNaNs();
16807 return Builder.CreateCall(F, {Ops[0]});
16809 case X86::BI__builtin_ia32_reduce_fmin_pd512:
16810 case X86::BI__builtin_ia32_reduce_fmin_ps512:
16811 case X86::BI__builtin_ia32_reduce_fmin_ph512:
16812 case X86::BI__builtin_ia32_reduce_fmin_ph256:
16813 case X86::BI__builtin_ia32_reduce_fmin_ph128: {
16816 IRBuilder<>::FastMathFlagGuard FMFGuard(
Builder);
16817 Builder.getFastMathFlags().setNoNaNs();
16818 return Builder.CreateCall(F, {Ops[0]});
16821 case X86::BI__builtin_ia32_rdrand16_step:
16822 case X86::BI__builtin_ia32_rdrand32_step:
16823 case X86::BI__builtin_ia32_rdrand64_step:
16824 case X86::BI__builtin_ia32_rdseed16_step:
16825 case X86::BI__builtin_ia32_rdseed32_step:
16826 case X86::BI__builtin_ia32_rdseed64_step: {
16828 switch (BuiltinID) {
16829 default: llvm_unreachable(
"Unsupported intrinsic!");
16830 case X86::BI__builtin_ia32_rdrand16_step:
16831 ID = Intrinsic::x86_rdrand_16;
16833 case X86::BI__builtin_ia32_rdrand32_step:
16834 ID = Intrinsic::x86_rdrand_32;
16836 case X86::BI__builtin_ia32_rdrand64_step:
16837 ID = Intrinsic::x86_rdrand_64;
16839 case X86::BI__builtin_ia32_rdseed16_step:
16840 ID = Intrinsic::x86_rdseed_16;
16842 case X86::BI__builtin_ia32_rdseed32_step:
16843 ID = Intrinsic::x86_rdseed_32;
16845 case X86::BI__builtin_ia32_rdseed64_step:
16846 ID = Intrinsic::x86_rdseed_64;
16855 case X86::BI__builtin_ia32_addcarryx_u32:
16856 case X86::BI__builtin_ia32_addcarryx_u64:
16857 case X86::BI__builtin_ia32_subborrow_u32:
16858 case X86::BI__builtin_ia32_subborrow_u64: {
16860 switch (BuiltinID) {
16861 default: llvm_unreachable(
"Unsupported intrinsic!");
16862 case X86::BI__builtin_ia32_addcarryx_u32:
16863 IID = Intrinsic::x86_addcarry_32;
16865 case X86::BI__builtin_ia32_addcarryx_u64:
16866 IID = Intrinsic::x86_addcarry_64;
16868 case X86::BI__builtin_ia32_subborrow_u32:
16869 IID = Intrinsic::x86_subborrow_32;
16871 case X86::BI__builtin_ia32_subborrow_u64:
16872 IID = Intrinsic::x86_subborrow_64;
16877 { Ops[0], Ops[1], Ops[2] });
16883 case X86::BI__builtin_ia32_fpclassps128_mask:
16884 case X86::BI__builtin_ia32_fpclassps256_mask:
16885 case X86::BI__builtin_ia32_fpclassps512_mask:
16886 case X86::BI__builtin_ia32_vfpclasspbf16128_mask:
16887 case X86::BI__builtin_ia32_vfpclasspbf16256_mask:
16888 case X86::BI__builtin_ia32_vfpclasspbf16512_mask:
16889 case X86::BI__builtin_ia32_fpclassph128_mask:
16890 case X86::BI__builtin_ia32_fpclassph256_mask:
16891 case X86::BI__builtin_ia32_fpclassph512_mask:
16892 case X86::BI__builtin_ia32_fpclasspd128_mask:
16893 case X86::BI__builtin_ia32_fpclasspd256_mask:
16894 case X86::BI__builtin_ia32_fpclasspd512_mask: {
16896 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
16897 Value *MaskIn = Ops[2];
16898 Ops.erase(&Ops[2]);
16901 switch (BuiltinID) {
16902 default: llvm_unreachable(
"Unsupported intrinsic!");
16903 case X86::BI__builtin_ia32_vfpclasspbf16128_mask:
16904 ID = Intrinsic::x86_avx10_fpclass_nepbf16_128;
16906 case X86::BI__builtin_ia32_vfpclasspbf16256_mask:
16907 ID = Intrinsic::x86_avx10_fpclass_nepbf16_256;
16909 case X86::BI__builtin_ia32_vfpclasspbf16512_mask:
16910 ID = Intrinsic::x86_avx10_fpclass_nepbf16_512;
16912 case X86::BI__builtin_ia32_fpclassph128_mask:
16913 ID = Intrinsic::x86_avx512fp16_fpclass_ph_128;
16915 case X86::BI__builtin_ia32_fpclassph256_mask:
16916 ID = Intrinsic::x86_avx512fp16_fpclass_ph_256;
16918 case X86::BI__builtin_ia32_fpclassph512_mask:
16919 ID = Intrinsic::x86_avx512fp16_fpclass_ph_512;
16921 case X86::BI__builtin_ia32_fpclassps128_mask:
16922 ID = Intrinsic::x86_avx512_fpclass_ps_128;
16924 case X86::BI__builtin_ia32_fpclassps256_mask:
16925 ID = Intrinsic::x86_avx512_fpclass_ps_256;
16927 case X86::BI__builtin_ia32_fpclassps512_mask:
16928 ID = Intrinsic::x86_avx512_fpclass_ps_512;
16930 case X86::BI__builtin_ia32_fpclasspd128_mask:
16931 ID = Intrinsic::x86_avx512_fpclass_pd_128;
16933 case X86::BI__builtin_ia32_fpclasspd256_mask:
16934 ID = Intrinsic::x86_avx512_fpclass_pd_256;
16936 case X86::BI__builtin_ia32_fpclasspd512_mask:
16937 ID = Intrinsic::x86_avx512_fpclass_pd_512;
16945 case X86::BI__builtin_ia32_vp2intersect_q_512:
16946 case X86::BI__builtin_ia32_vp2intersect_q_256:
16947 case X86::BI__builtin_ia32_vp2intersect_q_128:
16948 case X86::BI__builtin_ia32_vp2intersect_d_512:
16949 case X86::BI__builtin_ia32_vp2intersect_d_256:
16950 case X86::BI__builtin_ia32_vp2intersect_d_128: {
16952 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
16955 switch (BuiltinID) {
16956 default: llvm_unreachable(
"Unsupported intrinsic!");
16957 case X86::BI__builtin_ia32_vp2intersect_q_512:
16958 ID = Intrinsic::x86_avx512_vp2intersect_q_512;
16960 case X86::BI__builtin_ia32_vp2intersect_q_256:
16961 ID = Intrinsic::x86_avx512_vp2intersect_q_256;
16963 case X86::BI__builtin_ia32_vp2intersect_q_128:
16964 ID = Intrinsic::x86_avx512_vp2intersect_q_128;
16966 case X86::BI__builtin_ia32_vp2intersect_d_512:
16967 ID = Intrinsic::x86_avx512_vp2intersect_d_512;
16969 case X86::BI__builtin_ia32_vp2intersect_d_256:
16970 ID = Intrinsic::x86_avx512_vp2intersect_d_256;
16972 case X86::BI__builtin_ia32_vp2intersect_d_128:
16973 ID = Intrinsic::x86_avx512_vp2intersect_d_128;
16987 case X86::BI__builtin_ia32_vpmultishiftqb128:
16988 case X86::BI__builtin_ia32_vpmultishiftqb256:
16989 case X86::BI__builtin_ia32_vpmultishiftqb512: {
16991 switch (BuiltinID) {
16992 default: llvm_unreachable(
"Unsupported intrinsic!");
16993 case X86::BI__builtin_ia32_vpmultishiftqb128:
16994 ID = Intrinsic::x86_avx512_pmultishift_qb_128;
16996 case X86::BI__builtin_ia32_vpmultishiftqb256:
16997 ID = Intrinsic::x86_avx512_pmultishift_qb_256;
16999 case X86::BI__builtin_ia32_vpmultishiftqb512:
17000 ID = Intrinsic::x86_avx512_pmultishift_qb_512;
17007 case X86::BI__builtin_ia32_vpshufbitqmb128_mask:
17008 case X86::BI__builtin_ia32_vpshufbitqmb256_mask:
17009 case X86::BI__builtin_ia32_vpshufbitqmb512_mask: {
17011 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
17012 Value *MaskIn = Ops[2];
17013 Ops.erase(&Ops[2]);
17016 switch (BuiltinID) {
17017 default: llvm_unreachable(
"Unsupported intrinsic!");
17018 case X86::BI__builtin_ia32_vpshufbitqmb128_mask:
17019 ID = Intrinsic::x86_avx512_vpshufbitqmb_128;
17021 case X86::BI__builtin_ia32_vpshufbitqmb256_mask:
17022 ID = Intrinsic::x86_avx512_vpshufbitqmb_256;
17024 case X86::BI__builtin_ia32_vpshufbitqmb512_mask:
17025 ID = Intrinsic::x86_avx512_vpshufbitqmb_512;
17034 case X86::BI__builtin_ia32_cmpeqps:
17035 case X86::BI__builtin_ia32_cmpeqpd:
17036 return getVectorFCmpIR(CmpInst::FCMP_OEQ,
false);
17037 case X86::BI__builtin_ia32_cmpltps:
17038 case X86::BI__builtin_ia32_cmpltpd:
17039 return getVectorFCmpIR(CmpInst::FCMP_OLT,
true);
17040 case X86::BI__builtin_ia32_cmpleps:
17041 case X86::BI__builtin_ia32_cmplepd:
17042 return getVectorFCmpIR(CmpInst::FCMP_OLE,
true);
17043 case X86::BI__builtin_ia32_cmpunordps:
17044 case X86::BI__builtin_ia32_cmpunordpd:
17045 return getVectorFCmpIR(CmpInst::FCMP_UNO,
false);
17046 case X86::BI__builtin_ia32_cmpneqps:
17047 case X86::BI__builtin_ia32_cmpneqpd:
17048 return getVectorFCmpIR(CmpInst::FCMP_UNE,
false);
17049 case X86::BI__builtin_ia32_cmpnltps:
17050 case X86::BI__builtin_ia32_cmpnltpd:
17051 return getVectorFCmpIR(CmpInst::FCMP_UGE,
true);
17052 case X86::BI__builtin_ia32_cmpnleps:
17053 case X86::BI__builtin_ia32_cmpnlepd:
17054 return getVectorFCmpIR(CmpInst::FCMP_UGT,
true);
17055 case X86::BI__builtin_ia32_cmpordps:
17056 case X86::BI__builtin_ia32_cmpordpd:
17057 return getVectorFCmpIR(CmpInst::FCMP_ORD,
false);
17058 case X86::BI__builtin_ia32_cmpph128_mask:
17059 case X86::BI__builtin_ia32_cmpph256_mask:
17060 case X86::BI__builtin_ia32_cmpph512_mask:
17061 case X86::BI__builtin_ia32_cmpps128_mask:
17062 case X86::BI__builtin_ia32_cmpps256_mask:
17063 case X86::BI__builtin_ia32_cmpps512_mask:
17064 case X86::BI__builtin_ia32_cmppd128_mask:
17065 case X86::BI__builtin_ia32_cmppd256_mask:
17066 case X86::BI__builtin_ia32_cmppd512_mask:
17067 case X86::BI__builtin_ia32_vcmppd256_round_mask:
17068 case X86::BI__builtin_ia32_vcmpps256_round_mask:
17069 case X86::BI__builtin_ia32_vcmpph256_round_mask:
17070 case X86::BI__builtin_ia32_vcmppbf16512_mask:
17071 case X86::BI__builtin_ia32_vcmppbf16256_mask:
17072 case X86::BI__builtin_ia32_vcmppbf16128_mask:
17075 case X86::BI__builtin_ia32_cmpps:
17076 case X86::BI__builtin_ia32_cmpps256:
17077 case X86::BI__builtin_ia32_cmppd:
17078 case X86::BI__builtin_ia32_cmppd256: {
17086 unsigned CC = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x1f;
17091 FCmpInst::Predicate Pred;
17095 switch (CC & 0xf) {
17096 case 0x00: Pred = FCmpInst::FCMP_OEQ; IsSignaling =
false;
break;
17097 case 0x01: Pred = FCmpInst::FCMP_OLT; IsSignaling =
true;
break;
17098 case 0x02: Pred = FCmpInst::FCMP_OLE; IsSignaling =
true;
break;
17099 case 0x03: Pred = FCmpInst::FCMP_UNO; IsSignaling =
false;
break;
17100 case 0x04: Pred = FCmpInst::FCMP_UNE; IsSignaling =
false;
break;
17101 case 0x05: Pred = FCmpInst::FCMP_UGE; IsSignaling =
true;
break;
17102 case 0x06: Pred = FCmpInst::FCMP_UGT; IsSignaling =
true;
break;
17103 case 0x07: Pred = FCmpInst::FCMP_ORD; IsSignaling =
false;
break;
17104 case 0x08: Pred = FCmpInst::FCMP_UEQ; IsSignaling =
false;
break;
17105 case 0x09: Pred = FCmpInst::FCMP_ULT; IsSignaling =
true;
break;
17106 case 0x0a: Pred = FCmpInst::FCMP_ULE; IsSignaling =
true;
break;
17107 case 0x0b: Pred = FCmpInst::FCMP_FALSE; IsSignaling =
false;
break;
17108 case 0x0c: Pred = FCmpInst::FCMP_ONE; IsSignaling =
false;
break;
17109 case 0x0d: Pred = FCmpInst::FCMP_OGE; IsSignaling =
true;
break;
17110 case 0x0e: Pred = FCmpInst::FCMP_OGT; IsSignaling =
true;
break;
17111 case 0x0f: Pred = FCmpInst::FCMP_TRUE; IsSignaling =
false;
break;
17112 default: llvm_unreachable(
"Unhandled CC");
17117 IsSignaling = !IsSignaling;
17124 if (
Builder.getIsFPConstrained() &&
17125 (Pred == FCmpInst::FCMP_TRUE || Pred == FCmpInst::FCMP_FALSE ||
17129 switch (BuiltinID) {
17130 default: llvm_unreachable(
"Unexpected builtin");
17131 case X86::BI__builtin_ia32_cmpps:
17132 IID = Intrinsic::x86_sse_cmp_ps;
17134 case X86::BI__builtin_ia32_cmpps256:
17135 IID = Intrinsic::x86_avx_cmp_ps_256;
17137 case X86::BI__builtin_ia32_cmppd:
17138 IID = Intrinsic::x86_sse2_cmp_pd;
17140 case X86::BI__builtin_ia32_cmppd256:
17141 IID = Intrinsic::x86_avx_cmp_pd_256;
17143 case X86::BI__builtin_ia32_cmpph128_mask:
17144 IID = Intrinsic::x86_avx512fp16_mask_cmp_ph_128;
17146 case X86::BI__builtin_ia32_cmpph256_mask:
17147 IID = Intrinsic::x86_avx512fp16_mask_cmp_ph_256;
17149 case X86::BI__builtin_ia32_cmpph512_mask:
17150 IID = Intrinsic::x86_avx512fp16_mask_cmp_ph_512;
17152 case X86::BI__builtin_ia32_cmpps512_mask:
17153 IID = Intrinsic::x86_avx512_mask_cmp_ps_512;
17155 case X86::BI__builtin_ia32_cmppd512_mask:
17156 IID = Intrinsic::x86_avx512_mask_cmp_pd_512;
17158 case X86::BI__builtin_ia32_cmpps128_mask:
17159 IID = Intrinsic::x86_avx512_mask_cmp_ps_128;
17161 case X86::BI__builtin_ia32_cmpps256_mask:
17162 IID = Intrinsic::x86_avx512_mask_cmp_ps_256;
17164 case X86::BI__builtin_ia32_cmppd128_mask:
17165 IID = Intrinsic::x86_avx512_mask_cmp_pd_128;
17167 case X86::BI__builtin_ia32_cmppd256_mask:
17168 IID = Intrinsic::x86_avx512_mask_cmp_pd_256;
17175 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
17181 return Builder.CreateCall(Intr, Ops);
17192 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
17195 Cmp =
Builder.CreateFCmpS(Pred, Ops[0], Ops[1]);
17197 Cmp =
Builder.CreateFCmp(Pred, Ops[0], Ops[1]);
17201 return getVectorFCmpIR(Pred, IsSignaling);
17205 case X86::BI__builtin_ia32_cmpeqss:
17206 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 0);
17207 case X86::BI__builtin_ia32_cmpltss:
17208 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 1);
17209 case X86::BI__builtin_ia32_cmpless:
17210 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 2);
17211 case X86::BI__builtin_ia32_cmpunordss:
17212 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 3);
17213 case X86::BI__builtin_ia32_cmpneqss:
17214 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 4);
17215 case X86::BI__builtin_ia32_cmpnltss:
17216 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 5);
17217 case X86::BI__builtin_ia32_cmpnless:
17218 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 6);
17219 case X86::BI__builtin_ia32_cmpordss:
17220 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 7);
17221 case X86::BI__builtin_ia32_cmpeqsd:
17222 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 0);
17223 case X86::BI__builtin_ia32_cmpltsd:
17224 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 1);
17225 case X86::BI__builtin_ia32_cmplesd:
17226 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 2);
17227 case X86::BI__builtin_ia32_cmpunordsd:
17228 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 3);
17229 case X86::BI__builtin_ia32_cmpneqsd:
17230 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 4);
17231 case X86::BI__builtin_ia32_cmpnltsd:
17232 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 5);
17233 case X86::BI__builtin_ia32_cmpnlesd:
17234 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 6);
17235 case X86::BI__builtin_ia32_cmpordsd:
17236 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 7);
17239 case X86::BI__builtin_ia32_vcvtph2ps:
17240 case X86::BI__builtin_ia32_vcvtph2ps256:
17241 case X86::BI__builtin_ia32_vcvtph2ps_mask:
17242 case X86::BI__builtin_ia32_vcvtph2ps256_mask:
17243 case X86::BI__builtin_ia32_vcvtph2ps512_mask: {
17244 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
17249 case X86::BI__builtin_ia32_cvtneps2bf16_128_mask: {
17252 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements());
17253 Intrinsic::ID IID = Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128;
17256 case X86::BI__builtin_ia32_cvtsbf162ss_32:
17259 case X86::BI__builtin_ia32_cvtneps2bf16_256_mask:
17260 case X86::BI__builtin_ia32_cvtneps2bf16_512_mask: {
17262 switch (BuiltinID) {
17263 default: llvm_unreachable(
"Unsupported intrinsic!");
17264 case X86::BI__builtin_ia32_cvtneps2bf16_256_mask:
17265 IID = Intrinsic::x86_avx512bf16_cvtneps2bf16_256;
17267 case X86::BI__builtin_ia32_cvtneps2bf16_512_mask:
17268 IID = Intrinsic::x86_avx512bf16_cvtneps2bf16_512;
17275 case X86::BI__cpuid:
17276 case X86::BI__cpuidex: {
17278 Value *SubFuncId = BuiltinID == X86::BI__cpuidex
17282 llvm::StructType *CpuidRetTy =
17284 llvm::FunctionType *FTy =
17287 StringRef
Asm, Constraints;
17288 if (
getTarget().getTriple().getArch() == llvm::Triple::x86) {
17290 Constraints =
"={ax},={bx},={cx},={dx},{ax},{cx}";
17293 Asm =
"xchgq %rbx, ${1:q}\n"
17295 "xchgq %rbx, ${1:q}";
17296 Constraints =
"={ax},=r,={cx},={dx},0,2";
17299 llvm::InlineAsm *IA = llvm::InlineAsm::get(FTy,
Asm, Constraints,
17301 Value *IACall =
Builder.CreateCall(IA, {FuncId, SubFuncId});
17304 for (
unsigned i = 0; i < 4; i++) {
17305 Value *Extracted =
Builder.CreateExtractValue(IACall, i);
17315 case X86::BI__emul:
17316 case X86::BI__emulu: {
17318 bool isSigned = (BuiltinID == X86::BI__emul);
17321 return Builder.CreateMul(LHS, RHS,
"", !isSigned, isSigned);
17323 case X86::BI__mulh:
17324 case X86::BI__umulh:
17325 case X86::BI_mul128:
17326 case X86::BI_umul128: {
17328 llvm::Type *Int128Ty = llvm::IntegerType::get(
getLLVMContext(), 128);
17330 bool IsSigned = (BuiltinID == X86::BI__mulh || BuiltinID == X86::BI_mul128);
17331 Value *LHS =
Builder.CreateIntCast(Ops[0], Int128Ty, IsSigned);
17332 Value *RHS =
Builder.CreateIntCast(Ops[1], Int128Ty, IsSigned);
17334 Value *MulResult, *HigherBits;
17336 MulResult =
Builder.CreateNSWMul(LHS, RHS);
17337 HigherBits =
Builder.CreateAShr(MulResult, 64);
17339 MulResult =
Builder.CreateNUWMul(LHS, RHS);
17340 HigherBits =
Builder.CreateLShr(MulResult, 64);
17342 HigherBits =
Builder.CreateIntCast(HigherBits, ResType, IsSigned);
17344 if (BuiltinID == X86::BI__mulh || BuiltinID == X86::BI__umulh)
17349 return Builder.CreateIntCast(MulResult, ResType, IsSigned);
17352 case X86::BI__faststorefence: {
17353 return Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent,
17354 llvm::SyncScope::System);
17356 case X86::BI__shiftleft128:
17357 case X86::BI__shiftright128: {
17359 BuiltinID == X86::BI__shiftleft128 ? Intrinsic::fshl : Intrinsic::fshr,
17364 std::swap(Ops[0], Ops[1]);
17366 return Builder.CreateCall(F, Ops);
17368 case X86::BI_ReadWriteBarrier:
17369 case X86::BI_ReadBarrier:
17370 case X86::BI_WriteBarrier: {
17371 return Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent,
17372 llvm::SyncScope::SingleThread);
17375 case X86::BI_AddressOfReturnAddress: {
17378 return Builder.CreateCall(F);
17380 case X86::BI__stosb: {
17386 case X86::BI__builtin_ia32_t2rpntlvwz0_internal:
17387 case X86::BI__builtin_ia32_t2rpntlvwz0rs_internal:
17388 case X86::BI__builtin_ia32_t2rpntlvwz0t1_internal:
17389 case X86::BI__builtin_ia32_t2rpntlvwz0rst1_internal:
17390 case X86::BI__builtin_ia32_t2rpntlvwz1_internal:
17391 case X86::BI__builtin_ia32_t2rpntlvwz1rs_internal:
17392 case X86::BI__builtin_ia32_t2rpntlvwz1t1_internal:
17393 case X86::BI__builtin_ia32_t2rpntlvwz1rst1_internal: {
17395 switch (BuiltinID) {
17397 llvm_unreachable(
"Unsupported intrinsic!");
17398 case X86::BI__builtin_ia32_t2rpntlvwz0_internal:
17399 IID = Intrinsic::x86_t2rpntlvwz0_internal;
17401 case X86::BI__builtin_ia32_t2rpntlvwz0rs_internal:
17402 IID = Intrinsic::x86_t2rpntlvwz0rs_internal;
17404 case X86::BI__builtin_ia32_t2rpntlvwz0t1_internal:
17405 IID = Intrinsic::x86_t2rpntlvwz0t1_internal;
17407 case X86::BI__builtin_ia32_t2rpntlvwz0rst1_internal:
17408 IID = Intrinsic::x86_t2rpntlvwz0rst1_internal;
17410 case X86::BI__builtin_ia32_t2rpntlvwz1_internal:
17411 IID = Intrinsic::x86_t2rpntlvwz1_internal;
17413 case X86::BI__builtin_ia32_t2rpntlvwz1rs_internal:
17414 IID = Intrinsic::x86_t2rpntlvwz1rs_internal;
17416 case X86::BI__builtin_ia32_t2rpntlvwz1t1_internal:
17417 IID = Intrinsic::x86_t2rpntlvwz1t1_internal;
17419 case X86::BI__builtin_ia32_t2rpntlvwz1rst1_internal:
17420 IID = Intrinsic::x86_t2rpntlvwz1rst1_internal;
17426 {Ops[0], Ops[1], Ops[2], Ops[5], Ops[6]});
17429 assert(PtrTy &&
"arg3 must be of pointer type");
17436 Value *VecT0 =
Builder.CreateIntrinsic(Intrinsic::x86_cast_tile_to_vector,
17442 Value *VecT1 =
Builder.CreateIntrinsic(Intrinsic::x86_cast_tile_to_vector,
17456 case X86::BI__int2c: {
17458 llvm::FunctionType *FTy = llvm::FunctionType::get(
VoidTy,
false);
17459 llvm::InlineAsm *IA =
17460 llvm::InlineAsm::get(FTy,
"int $$0x2c",
"",
true);
17461 llvm::AttributeList NoReturnAttr = llvm::AttributeList::get(
17463 llvm::Attribute::NoReturn);
17464 llvm::CallInst *CI =
Builder.CreateCall(IA);
17465 CI->setAttributes(NoReturnAttr);
17468 case X86::BI__readfsbyte:
17469 case X86::BI__readfsword:
17470 case X86::BI__readfsdword:
17471 case X86::BI__readfsqword: {
17477 Load->setVolatile(
true);
17480 case X86::BI__readgsbyte:
17481 case X86::BI__readgsword:
17482 case X86::BI__readgsdword:
17483 case X86::BI__readgsqword: {
17489 Load->setVolatile(
true);
17492 case X86::BI__builtin_ia32_encodekey128_u32: {
17493 Intrinsic::ID IID = Intrinsic::x86_encodekey128;
17497 for (
int i = 0; i < 3; ++i) {
17505 case X86::BI__builtin_ia32_encodekey256_u32: {
17506 Intrinsic::ID IID = Intrinsic::x86_encodekey256;
17511 for (
int i = 0; i < 4; ++i) {
17519 case X86::BI__builtin_ia32_aesenc128kl_u8:
17520 case X86::BI__builtin_ia32_aesdec128kl_u8:
17521 case X86::BI__builtin_ia32_aesenc256kl_u8:
17522 case X86::BI__builtin_ia32_aesdec256kl_u8: {
17524 StringRef BlockName;
17525 switch (BuiltinID) {
17527 llvm_unreachable(
"Unexpected builtin");
17528 case X86::BI__builtin_ia32_aesenc128kl_u8:
17529 IID = Intrinsic::x86_aesenc128kl;
17530 BlockName =
"aesenc128kl";
17532 case X86::BI__builtin_ia32_aesdec128kl_u8:
17533 IID = Intrinsic::x86_aesdec128kl;
17534 BlockName =
"aesdec128kl";
17536 case X86::BI__builtin_ia32_aesenc256kl_u8:
17537 IID = Intrinsic::x86_aesenc256kl;
17538 BlockName =
"aesenc256kl";
17540 case X86::BI__builtin_ia32_aesdec256kl_u8:
17541 IID = Intrinsic::x86_aesdec256kl;
17542 BlockName =
"aesdec256kl";
17548 BasicBlock *NoError =
17556 Builder.CreateCondBr(Succ, NoError, Error);
17558 Builder.SetInsertPoint(NoError);
17562 Builder.SetInsertPoint(Error);
17563 Constant *
Zero = llvm::Constant::getNullValue(Out->getType());
17570 case X86::BI__builtin_ia32_aesencwide128kl_u8:
17571 case X86::BI__builtin_ia32_aesdecwide128kl_u8:
17572 case X86::BI__builtin_ia32_aesencwide256kl_u8:
17573 case X86::BI__builtin_ia32_aesdecwide256kl_u8: {
17575 StringRef BlockName;
17576 switch (BuiltinID) {
17577 case X86::BI__builtin_ia32_aesencwide128kl_u8:
17578 IID = Intrinsic::x86_aesencwide128kl;
17579 BlockName =
"aesencwide128kl";
17581 case X86::BI__builtin_ia32_aesdecwide128kl_u8:
17582 IID = Intrinsic::x86_aesdecwide128kl;
17583 BlockName =
"aesdecwide128kl";
17585 case X86::BI__builtin_ia32_aesencwide256kl_u8:
17586 IID = Intrinsic::x86_aesencwide256kl;
17587 BlockName =
"aesencwide256kl";
17589 case X86::BI__builtin_ia32_aesdecwide256kl_u8:
17590 IID = Intrinsic::x86_aesdecwide256kl;
17591 BlockName =
"aesdecwide256kl";
17595 llvm::Type *Ty = FixedVectorType::get(
Builder.getInt64Ty(), 2);
17598 for (
int i = 0; i != 8; ++i) {
17599 Value *Ptr =
Builder.CreateConstGEP1_32(Ty, Ops[1], i);
17605 BasicBlock *NoError =
17612 Builder.CreateCondBr(Succ, NoError, Error);
17614 Builder.SetInsertPoint(NoError);
17615 for (
int i = 0; i != 8; ++i) {
17622 Builder.SetInsertPoint(Error);
17623 for (
int i = 0; i != 8; ++i) {
17625 Constant *
Zero = llvm::Constant::getNullValue(Out->getType());
17626 Value *Ptr =
Builder.CreateConstGEP1_32(Out->getType(), Ops[0], i);
17634 case X86::BI__builtin_ia32_vfcmaddcph512_mask:
17637 case X86::BI__builtin_ia32_vfmaddcph512_mask: {
17638 Intrinsic::ID IID = IsConjFMA
17639 ? Intrinsic::x86_avx512fp16_mask_vfcmadd_cph_512
17640 : Intrinsic::x86_avx512fp16_mask_vfmadd_cph_512;
17644 case X86::BI__builtin_ia32_vfcmaddcph256_round_mask:
17647 case X86::BI__builtin_ia32_vfmaddcph256_round_mask: {
17648 Intrinsic::ID IID = IsConjFMA ? Intrinsic::x86_avx10_mask_vfcmaddcph256
17649 : Intrinsic::x86_avx10_mask_vfmaddcph256;
17653 case X86::BI__builtin_ia32_vfcmaddcsh_round_mask:
17656 case X86::BI__builtin_ia32_vfmaddcsh_round_mask: {
17657 Intrinsic::ID IID = IsConjFMA ? Intrinsic::x86_avx512fp16_mask_vfcmadd_csh
17658 : Intrinsic::x86_avx512fp16_mask_vfmadd_csh;
17663 case X86::BI__builtin_ia32_vfcmaddcsh_round_mask3:
17666 case X86::BI__builtin_ia32_vfmaddcsh_round_mask3: {
17667 Intrinsic::ID IID = IsConjFMA ? Intrinsic::x86_avx512fp16_mask_vfcmadd_csh
17668 : Intrinsic::x86_avx512fp16_mask_vfmadd_csh;
17670 static constexpr int Mask[] = {0, 5, 6, 7};
17671 return Builder.CreateShuffleVector(
Call, Ops[2], Mask);
17673 case X86::BI__builtin_ia32_prefetchi:
17676 {Ops[0], llvm::ConstantInt::get(Int32Ty, 0), Ops[1],
17677 llvm::ConstantInt::get(Int32Ty, 0)});
17695 Intrinsic::ID
ID = Intrinsic::not_intrinsic;
17697#include "llvm/TargetParser/PPCTargetParser.def"
17698 auto GenAIXPPCBuiltinCpuExpr = [&](
unsigned SupportMethod,
unsigned FieldIdx,
17699 unsigned Mask, CmpInst::Predicate CompOp,
17700 unsigned OpValue) ->
Value * {
17701 if (SupportMethod == BUILTIN_PPC_FALSE)
17704 if (SupportMethod == BUILTIN_PPC_TRUE)
17707 assert(SupportMethod <= SYS_CALL &&
"Invalid value for SupportMethod.");
17709 llvm::Value *FieldValue =
nullptr;
17710 if (SupportMethod == USE_SYS_CONF) {
17711 llvm::Type *STy = llvm::StructType::get(PPC_SYSTEMCONFIG_TYPE);
17712 llvm::Constant *SysConf =
17716 llvm::Value *Idxs[] = {ConstantInt::get(
Int32Ty, 0),
17717 ConstantInt::get(
Int32Ty, FieldIdx)};
17722 }
else if (SupportMethod == SYS_CALL) {
17723 llvm::FunctionType *FTy =
17725 llvm::FunctionCallee
Func =
17731 assert(FieldValue &&
17732 "SupportMethod value is not defined in PPCTargetParser.def.");
17735 FieldValue =
Builder.CreateAnd(FieldValue, Mask);
17737 llvm::Type *ValueType = FieldValue->getType();
17738 bool IsValueType64Bit = ValueType->isIntegerTy(64);
17740 (IsValueType64Bit || ValueType->isIntegerTy(32)) &&
17741 "Only 32/64-bit integers are supported in GenAIXPPCBuiltinCpuExpr().");
17744 CompOp, FieldValue,
17745 ConstantInt::get(IsValueType64Bit ?
Int64Ty :
Int32Ty, OpValue));
17748 switch (BuiltinID) {
17749 default:
return nullptr;
17751 case Builtin::BI__builtin_cpu_is: {
17753 StringRef CPUStr = cast<clang::StringLiteral>(CPUExpr)->getString();
17756 unsigned LinuxSupportMethod, LinuxIDValue, AIXSupportMethod, AIXIDValue;
17757 typedef std::tuple<unsigned, unsigned, unsigned, unsigned>
CPUInfo;
17759 std::tie(LinuxSupportMethod, LinuxIDValue, AIXSupportMethod, AIXIDValue) =
17760 static_cast<CPUInfo>(StringSwitch<CPUInfo>(CPUStr)
17761#define PPC_CPU(NAME, Linux_SUPPORT_METHOD, LinuxID, AIX_SUPPORT_METHOD, \
17763 .Case(NAME, {Linux_SUPPORT_METHOD, LinuxID, AIX_SUPPORT_METHOD, AIXID})
17764#include "llvm/TargetParser/PPCTargetParser.def"
17765 .Default({BUILTIN_PPC_UNSUPPORTED, 0,
17766 BUILTIN_PPC_UNSUPPORTED, 0}));
17768 if (Triple.isOSAIX()) {
17769 assert((AIXSupportMethod != BUILTIN_PPC_UNSUPPORTED) &&
17770 "Invalid CPU name. Missed by SemaChecking?");
17771 return GenAIXPPCBuiltinCpuExpr(AIXSupportMethod, AIX_SYSCON_IMPL_IDX, 0,
17772 ICmpInst::ICMP_EQ, AIXIDValue);
17775 assert(Triple.isOSLinux() &&
17776 "__builtin_cpu_is() is only supported for AIX and Linux.");
17778 assert((LinuxSupportMethod != BUILTIN_PPC_UNSUPPORTED) &&
17779 "Invalid CPU name. Missed by SemaChecking?");
17781 if (LinuxSupportMethod == BUILTIN_PPC_FALSE)
17784 Value *Op0 = llvm::ConstantInt::get(
Int32Ty, PPC_FAWORD_CPUID);
17786 Value *TheCall =
Builder.CreateCall(F, {Op0},
"cpu_is");
17787 return Builder.CreateICmpEQ(TheCall,
17788 llvm::ConstantInt::get(
Int32Ty, LinuxIDValue));
17790 case Builtin::BI__builtin_cpu_supports: {
17793 StringRef CPUStr = cast<clang::StringLiteral>(CPUExpr)->getString();
17794 if (Triple.isOSAIX()) {
17795 unsigned SupportMethod, FieldIdx, Mask,
Value;
17796 CmpInst::Predicate CompOp;
17800 std::tie(SupportMethod, FieldIdx, Mask, CompOp,
Value) =
17801 static_cast<CPUSupportType
>(StringSwitch<CPUSupportType>(CPUStr)
17802#define PPC_AIX_FEATURE(NAME, DESC, SUPPORT_METHOD, INDEX, MASK, COMP_OP, \
17804 .Case(NAME, {SUPPORT_METHOD, INDEX, MASK, COMP_OP, VALUE})
17805#include "llvm/TargetParser/PPCTargetParser.def"
17806 .Default({BUILTIN_PPC_FALSE, 0, 0,
17807 CmpInst::Predicate(), 0}));
17808 return GenAIXPPCBuiltinCpuExpr(SupportMethod, FieldIdx, Mask, CompOp,
17812 assert(Triple.isOSLinux() &&
17813 "__builtin_cpu_supports() is only supported for AIX and Linux.");
17814 unsigned FeatureWord;
17816 std::tie(FeatureWord, BitMask) =
17817 StringSwitch<std::pair<unsigned, unsigned>>(CPUStr)
17818#define
PPC_LNX_FEATURE(Name, Description, EnumName, Bitmask, FA_WORD) \
17819 .Case(Name, {FA_WORD, Bitmask})
17820#include
"llvm/TargetParser/PPCTargetParser.def"
17824 Value *Op0 = llvm::ConstantInt::get(
Int32Ty, FeatureWord);
17826 Value *TheCall =
Builder.CreateCall(F, {Op0},
"cpu_supports");
17828 Builder.CreateAnd(TheCall, llvm::ConstantInt::get(
Int32Ty, BitMask));
17829 return Builder.CreateICmpNE(Mask, llvm::Constant::getNullValue(
Int32Ty));
17830#undef PPC_FAWORD_HWCAP
17831#undef PPC_FAWORD_HWCAP2
17832#undef PPC_FAWORD_CPUID
17837 case PPC::BI__builtin_ppc_get_timebase:
17841 case PPC::BI__builtin_altivec_lvx:
17842 case PPC::BI__builtin_altivec_lvxl:
17843 case PPC::BI__builtin_altivec_lvebx:
17844 case PPC::BI__builtin_altivec_lvehx:
17845 case PPC::BI__builtin_altivec_lvewx:
17846 case PPC::BI__builtin_altivec_lvsl:
17847 case PPC::BI__builtin_altivec_lvsr:
17848 case PPC::BI__builtin_vsx_lxvd2x:
17849 case PPC::BI__builtin_vsx_lxvw4x:
17850 case PPC::BI__builtin_vsx_lxvd2x_be:
17851 case PPC::BI__builtin_vsx_lxvw4x_be:
17852 case PPC::BI__builtin_vsx_lxvl:
17853 case PPC::BI__builtin_vsx_lxvll:
17858 if (!(BuiltinID == PPC::BI__builtin_vsx_lxvl ||
17859 BuiltinID == PPC::BI__builtin_vsx_lxvll)) {
17864 switch (BuiltinID) {
17865 default: llvm_unreachable(
"Unsupported ld/lvsl/lvsr intrinsic!");
17866 case PPC::BI__builtin_altivec_lvx:
17867 ID = Intrinsic::ppc_altivec_lvx;
17869 case PPC::BI__builtin_altivec_lvxl:
17870 ID = Intrinsic::ppc_altivec_lvxl;
17872 case PPC::BI__builtin_altivec_lvebx:
17873 ID = Intrinsic::ppc_altivec_lvebx;
17875 case PPC::BI__builtin_altivec_lvehx:
17876 ID = Intrinsic::ppc_altivec_lvehx;
17878 case PPC::BI__builtin_altivec_lvewx:
17879 ID = Intrinsic::ppc_altivec_lvewx;
17881 case PPC::BI__builtin_altivec_lvsl:
17882 ID = Intrinsic::ppc_altivec_lvsl;
17884 case PPC::BI__builtin_altivec_lvsr:
17885 ID = Intrinsic::ppc_altivec_lvsr;
17887 case PPC::BI__builtin_vsx_lxvd2x:
17888 ID = Intrinsic::ppc_vsx_lxvd2x;
17890 case PPC::BI__builtin_vsx_lxvw4x:
17891 ID = Intrinsic::ppc_vsx_lxvw4x;
17893 case PPC::BI__builtin_vsx_lxvd2x_be:
17894 ID = Intrinsic::ppc_vsx_lxvd2x_be;
17896 case PPC::BI__builtin_vsx_lxvw4x_be:
17897 ID = Intrinsic::ppc_vsx_lxvw4x_be;
17899 case PPC::BI__builtin_vsx_lxvl:
17900 ID = Intrinsic::ppc_vsx_lxvl;
17902 case PPC::BI__builtin_vsx_lxvll:
17903 ID = Intrinsic::ppc_vsx_lxvll;
17907 return Builder.CreateCall(F, Ops,
"");
17911 case PPC::BI__builtin_altivec_stvx:
17912 case PPC::BI__builtin_altivec_stvxl:
17913 case PPC::BI__builtin_altivec_stvebx:
17914 case PPC::BI__builtin_altivec_stvehx:
17915 case PPC::BI__builtin_altivec_stvewx:
17916 case PPC::BI__builtin_vsx_stxvd2x:
17917 case PPC::BI__builtin_vsx_stxvw4x:
17918 case PPC::BI__builtin_vsx_stxvd2x_be:
17919 case PPC::BI__builtin_vsx_stxvw4x_be:
17920 case PPC::BI__builtin_vsx_stxvl:
17921 case PPC::BI__builtin_vsx_stxvll:
17927 if (!(BuiltinID == PPC::BI__builtin_vsx_stxvl ||
17928 BuiltinID == PPC::BI__builtin_vsx_stxvll)) {
17933 switch (BuiltinID) {
17934 default: llvm_unreachable(
"Unsupported st intrinsic!");
17935 case PPC::BI__builtin_altivec_stvx:
17936 ID = Intrinsic::ppc_altivec_stvx;
17938 case PPC::BI__builtin_altivec_stvxl:
17939 ID = Intrinsic::ppc_altivec_stvxl;
17941 case PPC::BI__builtin_altivec_stvebx:
17942 ID = Intrinsic::ppc_altivec_stvebx;
17944 case PPC::BI__builtin_altivec_stvehx:
17945 ID = Intrinsic::ppc_altivec_stvehx;
17947 case PPC::BI__builtin_altivec_stvewx:
17948 ID = Intrinsic::ppc_altivec_stvewx;
17950 case PPC::BI__builtin_vsx_stxvd2x:
17951 ID = Intrinsic::ppc_vsx_stxvd2x;
17953 case PPC::BI__builtin_vsx_stxvw4x:
17954 ID = Intrinsic::ppc_vsx_stxvw4x;
17956 case PPC::BI__builtin_vsx_stxvd2x_be:
17957 ID = Intrinsic::ppc_vsx_stxvd2x_be;
17959 case PPC::BI__builtin_vsx_stxvw4x_be:
17960 ID = Intrinsic::ppc_vsx_stxvw4x_be;
17962 case PPC::BI__builtin_vsx_stxvl:
17963 ID = Intrinsic::ppc_vsx_stxvl;
17965 case PPC::BI__builtin_vsx_stxvll:
17966 ID = Intrinsic::ppc_vsx_stxvll;
17970 return Builder.CreateCall(F, Ops,
"");
17972 case PPC::BI__builtin_vsx_ldrmb: {
17978 int64_t NumBytes = cast<ConstantInt>(Op1)->getZExtValue();
17983 if (NumBytes == 16) {
17991 for (
int Idx = 0; Idx < 16; Idx++)
17992 RevMask.push_back(15 - Idx);
17993 return Builder.CreateShuffleVector(LD, LD, RevMask);
17997 llvm::Function *Lvs =
CGM.
getIntrinsic(IsLE ? Intrinsic::ppc_altivec_lvsr
17998 : Intrinsic::ppc_altivec_lvsl);
17999 llvm::Function *Vperm =
CGM.
getIntrinsic(Intrinsic::ppc_altivec_vperm);
18001 Int8Ty, Op0, ConstantInt::get(Op1->
getType(), NumBytes - 1));
18003 Value *HiLd =
Builder.CreateCall(Lvx, HiMem,
"ld.hi");
18006 Op0 = IsLE ? HiLd : LoLd;
18007 Op1 = IsLE ? LoLd : HiLd;
18008 Value *AllElts =
Builder.CreateCall(Vperm, {Op0, Op1, Mask1},
"shuffle1");
18009 Constant *
Zero = llvm::Constant::getNullValue(IsLE ? ResTy : AllElts->
getType());
18013 for (
int Idx = 0; Idx < 16; Idx++) {
18014 int Val = (NumBytes - Idx - 1 >= 0) ? (NumBytes - Idx - 1)
18015 : 16 - (NumBytes - Idx);
18016 Consts.push_back(Val);
18018 return Builder.CreateShuffleVector(
Builder.CreateBitCast(AllElts, ResTy),
18022 for (
int Idx = 0; Idx < 16; Idx++)
18023 Consts.push_back(
Builder.getInt8(NumBytes + Idx));
18024 Value *Mask2 = ConstantVector::get(Consts);
18025 return Builder.CreateBitCast(
18026 Builder.CreateCall(Vperm, {Zero, AllElts, Mask2},
"shuffle2"), ResTy);
18028 case PPC::BI__builtin_vsx_strmb: {
18032 int64_t NumBytes = cast<ConstantInt>(Op1)->getZExtValue();
18034 auto StoreSubVec = [&](
unsigned Width,
unsigned Offset,
unsigned EltNo) {
18038 Value *StVec = Op2;
18041 for (
int Idx = 0; Idx < 16; Idx++)
18042 RevMask.push_back(15 - Idx);
18043 StVec =
Builder.CreateShuffleVector(Op2, Op2, RevMask);
18049 unsigned NumElts = 0;
18052 llvm_unreachable(
"width for stores must be a power of 2");
18071 Op2, llvm::FixedVectorType::get(ConvTy, NumElts));
18074 Value *Elt =
Builder.CreateExtractElement(Vec, EltNo);
18075 if (IsLE && Width > 1) {
18077 Elt =
Builder.CreateCall(F, Elt);
18082 unsigned Stored = 0;
18083 unsigned RemainingBytes = NumBytes;
18085 if (NumBytes == 16)
18086 return StoreSubVec(16, 0, 0);
18087 if (NumBytes >= 8) {
18088 Result = StoreSubVec(8, NumBytes - 8, IsLE ? 0 : 1);
18089 RemainingBytes -= 8;
18092 if (RemainingBytes >= 4) {
18093 Result = StoreSubVec(4, NumBytes - Stored - 4,
18094 IsLE ? (Stored >> 2) : 3 - (Stored >> 2));
18095 RemainingBytes -= 4;
18098 if (RemainingBytes >= 2) {
18099 Result = StoreSubVec(2, NumBytes - Stored - 2,
18100 IsLE ? (Stored >> 1) : 7 - (Stored >> 1));
18101 RemainingBytes -= 2;
18104 if (RemainingBytes)
18106 StoreSubVec(1, NumBytes - Stored - 1, IsLE ? Stored : 15 - Stored);
18110 case PPC::BI__builtin_vsx_xvsqrtsp:
18111 case PPC::BI__builtin_vsx_xvsqrtdp: {
18114 if (
Builder.getIsFPConstrained()) {
18116 Intrinsic::experimental_constrained_sqrt, ResultType);
18117 return Builder.CreateConstrainedFPCall(F,
X);
18124 case PPC::BI__builtin_altivec_vclzb:
18125 case PPC::BI__builtin_altivec_vclzh:
18126 case PPC::BI__builtin_altivec_vclzw:
18127 case PPC::BI__builtin_altivec_vclzd: {
18130 Value *Undef = ConstantInt::get(
Builder.getInt1Ty(),
false);
18132 return Builder.CreateCall(F, {
X, Undef});
18134 case PPC::BI__builtin_altivec_vctzb:
18135 case PPC::BI__builtin_altivec_vctzh:
18136 case PPC::BI__builtin_altivec_vctzw:
18137 case PPC::BI__builtin_altivec_vctzd: {
18140 Value *Undef = ConstantInt::get(
Builder.getInt1Ty(),
false);
18142 return Builder.CreateCall(F, {
X, Undef});
18144 case PPC::BI__builtin_altivec_vinsd:
18145 case PPC::BI__builtin_altivec_vinsw:
18146 case PPC::BI__builtin_altivec_vinsd_elt:
18147 case PPC::BI__builtin_altivec_vinsw_elt: {
18153 bool IsUnaligned = (BuiltinID == PPC::BI__builtin_altivec_vinsw ||
18154 BuiltinID == PPC::BI__builtin_altivec_vinsd);
18156 bool Is32bit = (BuiltinID == PPC::BI__builtin_altivec_vinsw ||
18157 BuiltinID == PPC::BI__builtin_altivec_vinsw_elt);
18160 ConstantInt *ArgCI = dyn_cast<ConstantInt>(Op2);
18162 "Third Arg to vinsw/vinsd intrinsic must be a constant integer!");
18166 int ValidMaxValue = 0;
18168 ValidMaxValue = (Is32bit) ? 12 : 8;
18170 ValidMaxValue = (Is32bit) ? 3 : 1;
18173 int64_t ConstArg = ArgCI->getSExtValue();
18176 std::string RangeErrMsg = IsUnaligned ?
"byte" :
"element";
18177 RangeErrMsg +=
" number " + llvm::to_string(ConstArg);
18178 RangeErrMsg +=
" is outside of the valid range [0, ";
18179 RangeErrMsg += llvm::to_string(ValidMaxValue) +
"]";
18182 if (ConstArg < 0 || ConstArg > ValidMaxValue)
18186 if (!IsUnaligned) {
18187 ConstArg *= Is32bit ? 4 : 8;
18190 ConstArg = (Is32bit ? 12 : 8) - ConstArg;
18193 ID = Is32bit ? Intrinsic::ppc_altivec_vinsw : Intrinsic::ppc_altivec_vinsd;
18194 Op2 = ConstantInt::getSigned(
Int32Ty, ConstArg);
18198 ?
Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(
Int32Ty, 4))
18200 llvm::FixedVectorType::get(
Int64Ty, 2));
18201 return Builder.CreateBitCast(
18204 case PPC::BI__builtin_altivec_vadduqm:
18205 case PPC::BI__builtin_altivec_vsubuqm: {
18208 llvm::Type *Int128Ty = llvm::IntegerType::get(
getLLVMContext(), 128);
18209 Op0 =
Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(Int128Ty, 1));
18210 Op1 =
Builder.CreateBitCast(Op1, llvm::FixedVectorType::get(Int128Ty, 1));
18211 if (BuiltinID == PPC::BI__builtin_altivec_vadduqm)
18212 return Builder.CreateAdd(Op0, Op1,
"vadduqm");
18214 return Builder.CreateSub(Op0, Op1,
"vsubuqm");
18216 case PPC::BI__builtin_altivec_vaddcuq_c:
18217 case PPC::BI__builtin_altivec_vsubcuq_c: {
18221 llvm::Type *V1I128Ty = llvm::FixedVectorType::get(
18223 Ops.push_back(
Builder.CreateBitCast(Op0, V1I128Ty));
18224 Ops.push_back(
Builder.CreateBitCast(Op1, V1I128Ty));
18225 ID = (BuiltinID == PPC::BI__builtin_altivec_vaddcuq_c)
18226 ? Intrinsic::ppc_altivec_vaddcuq
18227 : Intrinsic::ppc_altivec_vsubcuq;
18230 case PPC::BI__builtin_altivec_vaddeuqm_c:
18231 case PPC::BI__builtin_altivec_vaddecuq_c:
18232 case PPC::BI__builtin_altivec_vsubeuqm_c:
18233 case PPC::BI__builtin_altivec_vsubecuq_c: {
18238 llvm::Type *V1I128Ty = llvm::FixedVectorType::get(
18240 Ops.push_back(
Builder.CreateBitCast(Op0, V1I128Ty));
18241 Ops.push_back(
Builder.CreateBitCast(Op1, V1I128Ty));
18242 Ops.push_back(
Builder.CreateBitCast(Op2, V1I128Ty));
18243 switch (BuiltinID) {
18245 llvm_unreachable(
"Unsupported intrinsic!");
18246 case PPC::BI__builtin_altivec_vaddeuqm_c:
18247 ID = Intrinsic::ppc_altivec_vaddeuqm;
18249 case PPC::BI__builtin_altivec_vaddecuq_c:
18250 ID = Intrinsic::ppc_altivec_vaddecuq;
18252 case PPC::BI__builtin_altivec_vsubeuqm_c:
18253 ID = Intrinsic::ppc_altivec_vsubeuqm;
18255 case PPC::BI__builtin_altivec_vsubecuq_c:
18256 ID = Intrinsic::ppc_altivec_vsubecuq;
18261 case PPC::BI__builtin_ppc_rldimi:
18262 case PPC::BI__builtin_ppc_rlwimi: {
18269 if (BuiltinID == PPC::BI__builtin_ppc_rldimi &&
18279 ? Intrinsic::ppc_rldimi
18280 : Intrinsic::ppc_rlwimi),
18281 {Op0, Op1, Op2, Op3});
18283 case PPC::BI__builtin_ppc_rlwnm: {
18290 case PPC::BI__builtin_ppc_poppar4:
18291 case PPC::BI__builtin_ppc_poppar8: {
18293 llvm::Type *ArgType = Op0->
getType();
18299 if (
Result->getType() != ResultType)
18304 case PPC::BI__builtin_ppc_cmpb: {
18307 if (
getTarget().getTriple().isPPC64()) {
18310 return Builder.CreateCall(F, {Op0, Op1},
"cmpb");
18330 Constant *ShiftAmt = ConstantInt::get(
Int64Ty, 32);
18339 Value *ResHi =
Builder.CreateShl(ResHiShift, ShiftAmt);
18340 return Builder.CreateOr(ResLo, ResHi);
18343 case PPC::BI__builtin_vsx_xvcpsgnsp:
18344 case PPC::BI__builtin_vsx_xvcpsgndp: {
18348 ID = Intrinsic::copysign;
18350 return Builder.CreateCall(F, {
X, Y});
18353 case PPC::BI__builtin_vsx_xvrspip:
18354 case PPC::BI__builtin_vsx_xvrdpip:
18355 case PPC::BI__builtin_vsx_xvrdpim:
18356 case PPC::BI__builtin_vsx_xvrspim:
18357 case PPC::BI__builtin_vsx_xvrdpi:
18358 case PPC::BI__builtin_vsx_xvrspi:
18359 case PPC::BI__builtin_vsx_xvrdpic:
18360 case PPC::BI__builtin_vsx_xvrspic:
18361 case PPC::BI__builtin_vsx_xvrdpiz:
18362 case PPC::BI__builtin_vsx_xvrspiz: {
18365 if (BuiltinID == PPC::BI__builtin_vsx_xvrdpim ||
18366 BuiltinID == PPC::BI__builtin_vsx_xvrspim)
18368 ? Intrinsic::experimental_constrained_floor
18369 : Intrinsic::floor;
18370 else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpi ||
18371 BuiltinID == PPC::BI__builtin_vsx_xvrspi)
18373 ? Intrinsic::experimental_constrained_round
18374 : Intrinsic::round;
18375 else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpic ||
18376 BuiltinID == PPC::BI__builtin_vsx_xvrspic)
18378 ? Intrinsic::experimental_constrained_rint
18380 else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpip ||
18381 BuiltinID == PPC::BI__builtin_vsx_xvrspip)
18383 ? Intrinsic::experimental_constrained_ceil
18385 else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpiz ||
18386 BuiltinID == PPC::BI__builtin_vsx_xvrspiz)
18388 ? Intrinsic::experimental_constrained_trunc
18389 : Intrinsic::trunc;
18391 return Builder.getIsFPConstrained() ?
Builder.CreateConstrainedFPCall(F,
X)
18396 case PPC::BI__builtin_vsx_xvabsdp:
18397 case PPC::BI__builtin_vsx_xvabssp: {
18405 case PPC::BI__builtin_ppc_recipdivf:
18406 case PPC::BI__builtin_ppc_recipdivd:
18407 case PPC::BI__builtin_ppc_rsqrtf:
18408 case PPC::BI__builtin_ppc_rsqrtd: {
18409 FastMathFlags FMF =
Builder.getFastMathFlags();
18410 Builder.getFastMathFlags().setFast();
18414 if (BuiltinID == PPC::BI__builtin_ppc_recipdivf ||
18415 BuiltinID == PPC::BI__builtin_ppc_recipdivd) {
18418 Builder.getFastMathFlags() &= (FMF);
18421 auto *One = ConstantFP::get(ResultType, 1.0);
18424 Builder.getFastMathFlags() &= (FMF);
18427 case PPC::BI__builtin_ppc_alignx: {
18430 ConstantInt *AlignmentCI = cast<ConstantInt>(Op0);
18431 if (AlignmentCI->getValue().ugt(llvm::Value::MaximumAlignment))
18432 AlignmentCI = ConstantInt::get(AlignmentCI->getIntegerType(),
18433 llvm::Value::MaximumAlignment);
18437 AlignmentCI,
nullptr);
18440 case PPC::BI__builtin_ppc_rdlam: {
18444 llvm::Type *Ty = Op0->
getType();
18445 Value *ShiftAmt =
Builder.CreateIntCast(Op1, Ty,
false);
18447 Value *Rotate =
Builder.CreateCall(F, {Op0, Op0, ShiftAmt});
18448 return Builder.CreateAnd(Rotate, Op2);
18450 case PPC::BI__builtin_ppc_load2r: {
18457 case PPC::BI__builtin_ppc_fnmsub:
18458 case PPC::BI__builtin_ppc_fnmsubs:
18459 case PPC::BI__builtin_vsx_xvmaddadp:
18460 case PPC::BI__builtin_vsx_xvmaddasp:
18461 case PPC::BI__builtin_vsx_xvnmaddadp:
18462 case PPC::BI__builtin_vsx_xvnmaddasp:
18463 case PPC::BI__builtin_vsx_xvmsubadp:
18464 case PPC::BI__builtin_vsx_xvmsubasp:
18465 case PPC::BI__builtin_vsx_xvnmsubadp:
18466 case PPC::BI__builtin_vsx_xvnmsubasp: {
18472 if (
Builder.getIsFPConstrained())
18473 F =
CGM.
getIntrinsic(Intrinsic::experimental_constrained_fma, ResultType);
18476 switch (BuiltinID) {
18477 case PPC::BI__builtin_vsx_xvmaddadp:
18478 case PPC::BI__builtin_vsx_xvmaddasp:
18479 if (
Builder.getIsFPConstrained())
18480 return Builder.CreateConstrainedFPCall(F, {
X, Y, Z});
18482 return Builder.CreateCall(F, {
X, Y, Z});
18483 case PPC::BI__builtin_vsx_xvnmaddadp:
18484 case PPC::BI__builtin_vsx_xvnmaddasp:
18485 if (
Builder.getIsFPConstrained())
18487 Builder.CreateConstrainedFPCall(F, {X, Y, Z}),
"neg");
18489 return Builder.CreateFNeg(
Builder.CreateCall(F, {X, Y, Z}),
"neg");
18490 case PPC::BI__builtin_vsx_xvmsubadp:
18491 case PPC::BI__builtin_vsx_xvmsubasp:
18492 if (
Builder.getIsFPConstrained())
18493 return Builder.CreateConstrainedFPCall(
18494 F, {
X, Y,
Builder.CreateFNeg(Z,
"neg")});
18497 case PPC::BI__builtin_ppc_fnmsub:
18498 case PPC::BI__builtin_ppc_fnmsubs:
18499 case PPC::BI__builtin_vsx_xvnmsubadp:
18500 case PPC::BI__builtin_vsx_xvnmsubasp:
18501 if (
Builder.getIsFPConstrained())
18503 Builder.CreateConstrainedFPCall(
18504 F, {X, Y, Builder.CreateFNeg(Z,
"neg")}),
18510 llvm_unreachable(
"Unknown FMA operation");
18514 case PPC::BI__builtin_vsx_insertword: {
18522 ConstantInt *ArgCI = dyn_cast<ConstantInt>(Op2);
18524 "Third arg to xxinsertw intrinsic must be constant integer");
18526 int64_t Index = std::clamp(ArgCI->getSExtValue(), (int64_t)0, MaxIndex);
18533 std::swap(Op0, Op1);
18537 Op1 =
Builder.CreateBitCast(Op1, llvm::FixedVectorType::get(
Int64Ty, 2));
18541 Op0 =
Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(
Int64Ty, 2));
18545 Index = MaxIndex - Index;
18549 Op0 =
Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(
Int32Ty, 4));
18550 Op2 = ConstantInt::getSigned(
Int32Ty, Index);
18551 return Builder.CreateCall(F, {Op0, Op1, Op2});
18554 case PPC::BI__builtin_vsx_extractuword: {
18557 llvm::Function *F =
CGM.
getIntrinsic(Intrinsic::ppc_vsx_xxextractuw);
18560 Op0 =
Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(
Int64Ty, 2));
18564 ConstantInt *ArgCI = dyn_cast<ConstantInt>(Op1);
18566 "Second Arg to xxextractuw intrinsic must be a constant integer!");
18568 int64_t Index = std::clamp(ArgCI->getSExtValue(), (int64_t)0, MaxIndex);
18572 Index = MaxIndex - Index;
18573 Op1 = ConstantInt::getSigned(
Int32Ty, Index);
18578 Value *ShuffleCall =
18580 return ShuffleCall;
18582 Op1 = ConstantInt::getSigned(
Int32Ty, Index);
18583 return Builder.CreateCall(F, {Op0, Op1});
18587 case PPC::BI__builtin_vsx_xxpermdi: {
18591 ConstantInt *ArgCI = dyn_cast<ConstantInt>(Op2);
18592 assert(ArgCI &&
"Third arg must be constant integer!");
18594 unsigned Index = ArgCI->getZExtValue();
18595 Op0 =
Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(
Int64Ty, 2));
18596 Op1 =
Builder.CreateBitCast(Op1, llvm::FixedVectorType::get(
Int64Ty, 2));
18601 int ElemIdx0 = (Index & 2) >> 1;
18602 int ElemIdx1 = 2 + (Index & 1);
18604 int ShuffleElts[2] = {ElemIdx0, ElemIdx1};
18605 Value *ShuffleCall =
Builder.CreateShuffleVector(Op0, Op1, ShuffleElts);
18608 return Builder.CreateBitCast(ShuffleCall, RetTy);
18611 case PPC::BI__builtin_vsx_xxsldwi: {
18615 ConstantInt *ArgCI = dyn_cast<ConstantInt>(Op2);
18616 assert(ArgCI &&
"Third argument must be a compile time constant");
18617 unsigned Index = ArgCI->getZExtValue() & 0x3;
18618 Op0 =
Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(
Int32Ty, 4));
18619 Op1 =
Builder.CreateBitCast(Op1, llvm::FixedVectorType::get(
Int32Ty, 4));
18630 ElemIdx0 = (8 - Index) % 8;
18631 ElemIdx1 = (9 - Index) % 8;
18632 ElemIdx2 = (10 - Index) % 8;
18633 ElemIdx3 = (11 - Index) % 8;
18637 ElemIdx1 = Index + 1;
18638 ElemIdx2 = Index + 2;
18639 ElemIdx3 = Index + 3;
18642 int ShuffleElts[4] = {ElemIdx0, ElemIdx1, ElemIdx2, ElemIdx3};
18643 Value *ShuffleCall =
Builder.CreateShuffleVector(Op0, Op1, ShuffleElts);
18646 return Builder.CreateBitCast(ShuffleCall, RetTy);
18649 case PPC::BI__builtin_pack_vector_int128: {
18653 Value *PoisonValue =
18654 llvm::PoisonValue::get(llvm::FixedVectorType::get(Op0->
getType(), 2));
18656 PoisonValue, Op0, (uint64_t)(isLittleEndian ? 1 : 0));
18657 Res =
Builder.CreateInsertElement(Res, Op1,
18658 (uint64_t)(isLittleEndian ? 0 : 1));
18662 case PPC::BI__builtin_unpack_vector_int128: {
18665 ConstantInt *Index = cast<ConstantInt>(Op1);
18671 ConstantInt::get(Index->getIntegerType(), 1 - Index->getZExtValue());
18673 return Builder.CreateExtractElement(Unpacked, Index);
18676 case PPC::BI__builtin_ppc_sthcx: {
18680 return Builder.CreateCall(F, {Op0, Op1});
18689#define CUSTOM_BUILTIN(Name, Intr, Types, Accumulate, Feature) \
18690 case PPC::BI__builtin_##Name:
18691#include "clang/Basic/BuiltinsPPC.def"
18694 for (
unsigned i = 0, e =
E->getNumArgs(); i != e; i++)
18704 if (BuiltinID == PPC::BI__builtin_mma_disassemble_acc ||
18705 BuiltinID == PPC::BI__builtin_vsx_disassemble_pair ||
18706 BuiltinID == PPC::BI__builtin_mma_disassemble_pair) {
18707 unsigned NumVecs = 2;
18708 auto Intrinsic = Intrinsic::ppc_vsx_disassemble_pair;
18709 if (BuiltinID == PPC::BI__builtin_mma_disassemble_acc) {
18711 Intrinsic = Intrinsic::ppc_mma_disassemble_acc;
18717 llvm::Type *VTy = llvm::FixedVectorType::get(
Int8Ty, 16);
18718 Value *Ptr = Ops[0];
18719 for (
unsigned i=0; i<NumVecs; i++) {
18721 llvm::ConstantInt* Index = llvm::ConstantInt::get(
IntTy, i);
18727 if (BuiltinID == PPC::BI__builtin_vsx_build_pair ||
18728 BuiltinID == PPC::BI__builtin_mma_build_acc) {
18736 std::reverse(Ops.begin() + 1, Ops.end());
18739 switch (BuiltinID) {
18740 #define CUSTOM_BUILTIN(Name, Intr, Types, Acc, Feature) \
18741 case PPC::BI__builtin_##Name: \
18742 ID = Intrinsic::ppc_##Intr; \
18743 Accumulate = Acc; \
18745 #include "clang/Basic/BuiltinsPPC.def"
18747 if (BuiltinID == PPC::BI__builtin_vsx_lxvp ||
18748 BuiltinID == PPC::BI__builtin_vsx_stxvp ||
18749 BuiltinID == PPC::BI__builtin_mma_lxvp ||
18750 BuiltinID == PPC::BI__builtin_mma_stxvp) {
18751 if (BuiltinID == PPC::BI__builtin_vsx_lxvp ||
18752 BuiltinID == PPC::BI__builtin_mma_lxvp) {
18759 return Builder.CreateCall(F, Ops,
"");
18765 CallOps.push_back(Acc);
18767 for (
unsigned i=1; i<Ops.size(); i++)
18768 CallOps.push_back(Ops[i]);
18774 case PPC::BI__builtin_ppc_compare_and_swap:
18775 case PPC::BI__builtin_ppc_compare_and_swaplp: {
18784 llvm::AtomicOrdering::Monotonic, llvm::AtomicOrdering::Monotonic,
true);
18792 Value *LoadedVal = Pair.first.getScalarVal();
18796 case PPC::BI__builtin_ppc_fetch_and_add:
18797 case PPC::BI__builtin_ppc_fetch_and_addlp: {
18799 llvm::AtomicOrdering::Monotonic);
18801 case PPC::BI__builtin_ppc_fetch_and_and:
18802 case PPC::BI__builtin_ppc_fetch_and_andlp: {
18804 llvm::AtomicOrdering::Monotonic);
18807 case PPC::BI__builtin_ppc_fetch_and_or:
18808 case PPC::BI__builtin_ppc_fetch_and_orlp: {
18810 llvm::AtomicOrdering::Monotonic);
18812 case PPC::BI__builtin_ppc_fetch_and_swap:
18813 case PPC::BI__builtin_ppc_fetch_and_swaplp: {
18815 llvm::AtomicOrdering::Monotonic);
18817 case PPC::BI__builtin_ppc_ldarx:
18818 case PPC::BI__builtin_ppc_lwarx:
18819 case PPC::BI__builtin_ppc_lharx:
18820 case PPC::BI__builtin_ppc_lbarx:
18822 case PPC::BI__builtin_ppc_mfspr: {
18828 return Builder.CreateCall(F, {Op0});
18830 case PPC::BI__builtin_ppc_mtspr: {
18837 return Builder.CreateCall(F, {Op0, Op1});
18839 case PPC::BI__builtin_ppc_popcntb: {
18841 llvm::Type *ArgType = ArgValue->
getType();
18843 return Builder.CreateCall(F, {ArgValue},
"popcntb");
18845 case PPC::BI__builtin_ppc_mtfsf: {
18855 case PPC::BI__builtin_ppc_swdiv_nochk:
18856 case PPC::BI__builtin_ppc_swdivs_nochk: {
18859 FastMathFlags FMF =
Builder.getFastMathFlags();
18860 Builder.getFastMathFlags().setFast();
18861 Value *FDiv =
Builder.CreateFDiv(Op0, Op1,
"swdiv_nochk");
18862 Builder.getFastMathFlags() &= (FMF);
18865 case PPC::BI__builtin_ppc_fric:
18867 *
this,
E, Intrinsic::rint,
18868 Intrinsic::experimental_constrained_rint))
18870 case PPC::BI__builtin_ppc_frim:
18871 case PPC::BI__builtin_ppc_frims:
18873 *
this,
E, Intrinsic::floor,
18874 Intrinsic::experimental_constrained_floor))
18876 case PPC::BI__builtin_ppc_frin:
18877 case PPC::BI__builtin_ppc_frins:
18879 *
this,
E, Intrinsic::round,
18880 Intrinsic::experimental_constrained_round))
18882 case PPC::BI__builtin_ppc_frip:
18883 case PPC::BI__builtin_ppc_frips:
18885 *
this,
E, Intrinsic::ceil,
18886 Intrinsic::experimental_constrained_ceil))
18888 case PPC::BI__builtin_ppc_friz:
18889 case PPC::BI__builtin_ppc_frizs:
18891 *
this,
E, Intrinsic::trunc,
18892 Intrinsic::experimental_constrained_trunc))
18894 case PPC::BI__builtin_ppc_fsqrt:
18895 case PPC::BI__builtin_ppc_fsqrts:
18897 *
this,
E, Intrinsic::sqrt,
18898 Intrinsic::experimental_constrained_sqrt))
18900 case PPC::BI__builtin_ppc_test_data_class: {
18905 {Op0, Op1},
"test_data_class");
18907 case PPC::BI__builtin_ppc_maxfe: {
18913 {Op0, Op1, Op2, Op3});
18915 case PPC::BI__builtin_ppc_maxfl: {
18921 {Op0, Op1, Op2, Op3});
18923 case PPC::BI__builtin_ppc_maxfs: {
18929 {Op0, Op1, Op2, Op3});
18931 case PPC::BI__builtin_ppc_minfe: {
18937 {Op0, Op1, Op2, Op3});
18939 case PPC::BI__builtin_ppc_minfl: {
18945 {Op0, Op1, Op2, Op3});
18947 case PPC::BI__builtin_ppc_minfs: {
18953 {Op0, Op1, Op2, Op3});
18955 case PPC::BI__builtin_ppc_swdiv:
18956 case PPC::BI__builtin_ppc_swdivs: {
18959 return Builder.CreateFDiv(Op0, Op1,
"swdiv");
18961 case PPC::BI__builtin_ppc_set_fpscr_rn:
18963 {EmitScalarExpr(E->getArg(0))});
18964 case PPC::BI__builtin_ppc_mffs:
18977 Attribute::getWithDereferenceableBytes(
Call->getContext(), 64));
18978 Call->addRetAttr(Attribute::getWithAlignment(
Call->getContext(), Align(4)));
18982 auto *RetTy = cast<llvm::PointerType>(CGF.
ConvertType(BuiltinRetType));
18983 if (RetTy ==
Call->getType())
18992 Attribute::getWithDereferenceableBytes(
Call->getContext(), 256));
18993 Call->addRetAttr(Attribute::getWithAlignment(
Call->getContext(), Align(8)));
19008 llvm::LoadInst *LD;
19012 if (Cov == CodeObjectVersionKind::COV_None) {
19013 StringRef Name =
"__oclc_ABI_version";
19014 auto *ABIVersionC = CGF.
CGM.
getModule().getNamedGlobal(Name);
19016 ABIVersionC =
new llvm::GlobalVariable(
19018 llvm::GlobalValue::ExternalLinkage,
nullptr, Name,
nullptr,
19019 llvm::GlobalVariable::NotThreadLocal,
19030 llvm::ConstantInt::get(CGF.
Int32Ty, CodeObjectVersionKind::COV_5));
19034 CGF.
Int8Ty, EmitAMDGPUImplicitArgPtr(CGF), 12 + Index * 2);
19038 CGF.
Int8Ty, EmitAMDGPUDispatchPtr(CGF), 4 + Index * 2);
19040 auto Result = CGF.
Builder.CreateSelect(IsCOV5, ImplicitGEP, DispatchGEP);
19044 Value *GEP =
nullptr;
19045 if (Cov >= CodeObjectVersionKind::COV_5) {
19047 GEP = CGF.
Builder.CreateConstGEP1_32(
19048 CGF.
Int8Ty, EmitAMDGPUImplicitArgPtr(CGF), 12 + Index * 2);
19051 GEP = CGF.
Builder.CreateConstGEP1_32(
19052 CGF.
Int8Ty, EmitAMDGPUDispatchPtr(CGF), 4 + Index * 2);
19059 llvm::MDNode *RNode = MDHelper.createRange(
APInt(16, 1),
19061 LD->setMetadata(llvm::LLVMContext::MD_range, RNode);
19062 LD->setMetadata(llvm::LLVMContext::MD_noundef,
19064 LD->setMetadata(llvm::LLVMContext::MD_invariant_load,
19071 const unsigned XOffset = 12;
19072 auto *DP = EmitAMDGPUDispatchPtr(CGF);
19074 auto *Offset = llvm::ConstantInt::get(CGF.
Int32Ty, XOffset + Index * 4);
19082 LD->setMetadata(llvm::LLVMContext::MD_range,
19083 MDB.createRange(
APInt(32, 1), APInt::getZero(32)));
19084 LD->setMetadata(llvm::LLVMContext::MD_invariant_load,
19097 llvm::AtomicOrdering &AO,
19098 llvm::SyncScope::ID &SSID) {
19099 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
19102 assert(llvm::isValidAtomicOrderingCABI(ord));
19103 switch (
static_cast<llvm::AtomicOrderingCABI
>(ord)) {
19104 case llvm::AtomicOrderingCABI::acquire:
19105 case llvm::AtomicOrderingCABI::consume:
19106 AO = llvm::AtomicOrdering::Acquire;
19108 case llvm::AtomicOrderingCABI::release:
19109 AO = llvm::AtomicOrdering::Release;
19111 case llvm::AtomicOrderingCABI::acq_rel:
19112 AO = llvm::AtomicOrdering::AcquireRelease;
19114 case llvm::AtomicOrderingCABI::seq_cst:
19115 AO = llvm::AtomicOrdering::SequentiallyConsistent;
19117 case llvm::AtomicOrderingCABI::relaxed:
19118 AO = llvm::AtomicOrdering::Monotonic;
19124 if (llvm::getConstantStringInfo(
Scope, scp)) {
19130 int scope = cast<llvm::ConstantInt>(
Scope)->getZExtValue();
19133 SSID = llvm::SyncScope::System;
19145 SSID = llvm::SyncScope::SingleThread;
19148 SSID = llvm::SyncScope::System;
19156 llvm::Value *Arg =
nullptr;
19157 if ((ICEArguments & (1 << Idx)) == 0) {
19162 std::optional<llvm::APSInt>
Result =
19164 assert(
Result &&
"Expected argument to be a constant");
19173 return RT.getFDotIntrinsic();
19175 return RT.getSDotIntrinsic();
19177 return RT.getUDotIntrinsic();
19182 return RT.getFirstBitSHighIntrinsic();
19186 return RT.getFirstBitUHighIntrinsic();
19193 case llvm::Triple::spirv:
19194 return llvm::Intrinsic::spv_wave_reduce_sum;
19195 case llvm::Triple::dxil: {
19197 return llvm::Intrinsic::dx_wave_reduce_usum;
19198 return llvm::Intrinsic::dx_wave_reduce_sum;
19201 llvm_unreachable(
"Intrinsic WaveActiveSum"
19202 " not supported by target architecture");
19212 switch (BuiltinID) {
19213 case Builtin::BI__builtin_hlsl_resource_getpointer: {
19218 llvm::Type *RetTy = llvm::PointerType::getUnqual(
getLLVMContext());
19220 return Builder.CreateIntrinsic(
19224 case Builtin::BI__builtin_hlsl_all: {
19226 return Builder.CreateIntrinsic(
19231 case Builtin::BI__builtin_hlsl_any: {
19233 return Builder.CreateIntrinsic(
19238 case Builtin::BI__builtin_hlsl_asdouble:
19240 case Builtin::BI__builtin_hlsl_elementwise_clamp: {
19247 Ty = VecTy->getElementType();
19249 Intrinsic::ID Intr;
19258 return Builder.CreateIntrinsic(
19262 case Builtin::BI__builtin_hlsl_cross: {
19267 "cross operands must have a float representation");
19272 "input vectors must have 3 elements each");
19273 return Builder.CreateIntrinsic(
19277 case Builtin::BI__builtin_hlsl_dot: {
19280 llvm::Type *T0 = Op0->
getType();
19281 llvm::Type *T1 = Op1->
getType();
19284 if (!T0->isVectorTy() && !T1->isVectorTy()) {
19285 if (T0->isFloatingPointTy())
19286 return Builder.CreateFMul(Op0, Op1,
"hlsl.dot");
19288 if (T0->isIntegerTy())
19289 return Builder.CreateMul(Op0, Op1,
"hlsl.dot");
19292 "Scalar dot product is only supported on ints and floats.");
19297 assert(T0->isVectorTy() && T1->isVectorTy() &&
19298 "Dot product of vector and scalar is not supported.");
19301 [[maybe_unused]]
auto *VecTy1 =
19305 "Dot product of vectors need the same element types.");
19308 "Dot product requires vectors to be of the same size.");
19310 return Builder.CreateIntrinsic(
19311 T0->getScalarType(),
19315 case Builtin::BI__builtin_hlsl_dot4add_i8packed: {
19321 return Builder.CreateIntrinsic(
19323 "hlsl.dot4add.i8packed");
19325 case Builtin::BI__builtin_hlsl_dot4add_u8packed: {
19331 return Builder.CreateIntrinsic(
19333 "hlsl.dot4add.u8packed");
19335 case Builtin::BI__builtin_hlsl_elementwise_firstbithigh: {
19338 return Builder.CreateIntrinsic(
19343 case Builtin::BI__builtin_hlsl_elementwise_firstbitlow: {
19346 return Builder.CreateIntrinsic(
19349 nullptr,
"hlsl.firstbitlow");
19351 case Builtin::BI__builtin_hlsl_lerp: {
19356 llvm_unreachable(
"lerp operand must have a float representation");
19357 return Builder.CreateIntrinsic(
19361 case Builtin::BI__builtin_hlsl_normalize: {
19365 "normalize operand must have a float representation");
19367 return Builder.CreateIntrinsic(
19370 nullptr,
"hlsl.normalize");
19372 case Builtin::BI__builtin_hlsl_elementwise_degrees: {
19376 "degree operand must have a float representation");
19378 return Builder.CreateIntrinsic(
19382 case Builtin::BI__builtin_hlsl_elementwise_frac: {
19385 llvm_unreachable(
"frac operand must have a float representation");
19386 return Builder.CreateIntrinsic(
19390case Builtin::BI__builtin_hlsl_elementwise_isinf: {
19392 llvm::Type *Xty = Op0->
getType();
19393 llvm::Type *retType = llvm::Type::getInt1Ty(this->
getLLVMContext());
19394 if (Xty->isVectorTy()) {
19396 retType = llvm::VectorType::get(
19397 retType, ElementCount::getFixed(XVecTy->getNumElements()));
19400 llvm_unreachable(
"isinf operand must have a float representation");
19401 return Builder.CreateIntrinsic(retType, Intrinsic::dx_isinf,
19404 case Builtin::BI__builtin_hlsl_mad: {
19409 return Builder.CreateIntrinsic(
19410 M->
getType(), Intrinsic::fmuladd,
19415 return Builder.CreateIntrinsic(
19416 M->
getType(), Intrinsic::dx_imad,
19420 return Builder.CreateNSWAdd(Mul, B);
19424 return Builder.CreateIntrinsic(
19425 M->
getType(), Intrinsic::dx_umad,
19429 return Builder.CreateNUWAdd(Mul, B);
19431 case Builtin::BI__builtin_hlsl_elementwise_rcp: {
19434 llvm_unreachable(
"rcp operand must have a float representation");
19435 llvm::Type *Ty = Op0->
getType();
19436 llvm::Type *EltTy = Ty->getScalarType();
19437 Constant *One = Ty->isVectorTy()
19438 ? ConstantVector::getSplat(
19439 ElementCount::getFixed(
19440 cast<FixedVectorType>(Ty)->getNumElements()),
19441 ConstantFP::get(EltTy, 1.0))
19442 : ConstantFP::get(EltTy, 1.0);
19443 return Builder.CreateFDiv(One, Op0,
"hlsl.rcp");
19445 case Builtin::BI__builtin_hlsl_elementwise_rsqrt: {
19448 llvm_unreachable(
"rsqrt operand must have a float representation");
19449 return Builder.CreateIntrinsic(
19453 case Builtin::BI__builtin_hlsl_elementwise_saturate: {
19456 "saturate operand must have a float representation");
19457 return Builder.CreateIntrinsic(
19460 nullptr,
"hlsl.saturate");
19462 case Builtin::BI__builtin_hlsl_select: {
19476 Builder.CreateSelect(OpCond, OpTrue, OpFalse,
"hlsl.select");
19483 case Builtin::BI__builtin_hlsl_step: {
19488 "step operands must have a float representation");
19489 return Builder.CreateIntrinsic(
19493 case Builtin::BI__builtin_hlsl_wave_active_all_true: {
19495 assert(Op->
getType()->isIntegerTy(1) &&
19496 "Intrinsic WaveActiveAllTrue operand must be a bool");
19500 Intrinsic::getOrInsertDeclaration(&
CGM.
getModule(), ID), {Op});
19502 case Builtin::BI__builtin_hlsl_wave_active_any_true: {
19504 assert(Op->
getType()->isIntegerTy(1) &&
19505 "Intrinsic WaveActiveAnyTrue operand must be a bool");
19509 Intrinsic::getOrInsertDeclaration(&
CGM.
getModule(), ID), {Op});
19511 case Builtin::BI__builtin_hlsl_wave_active_count_bits: {
19515 Intrinsic::getOrInsertDeclaration(&
CGM.
getModule(), ID),
19518 case Builtin::BI__builtin_hlsl_wave_active_sum: {
19521 llvm::FunctionType *FT = llvm::FunctionType::get(
19533 ArrayRef{OpExpr},
"hlsl.wave.active.sum");
19535 case Builtin::BI__builtin_hlsl_wave_get_lane_index: {
19540 case llvm::Triple::dxil:
19543 case llvm::Triple::spirv:
19545 llvm::FunctionType::get(
IntTy, {},
false),
19546 "__hlsl_wave_get_lane_index", {},
false,
true));
19549 "Intrinsic WaveGetLaneIndex not supported by target architecture");
19552 case Builtin::BI__builtin_hlsl_wave_is_first_lane: {
19555 Intrinsic::getOrInsertDeclaration(&
CGM.
getModule(), ID));
19557 case Builtin::BI__builtin_hlsl_wave_read_lane_at: {
19562 llvm::FunctionType *FT = llvm::FunctionType::get(
19573 ArrayRef{OpExpr, OpIndex},
"hlsl.wave.readlane");
19575 case Builtin::BI__builtin_hlsl_elementwise_sign: {
19576 auto *Arg0 =
E->getArg(0);
19578 llvm::Type *Xty = Op0->
getType();
19579 llvm::Type *retType = llvm::Type::getInt32Ty(this->
getLLVMContext());
19580 if (Xty->isVectorTy()) {
19582 retType = llvm::VectorType::get(
19583 retType, ElementCount::getFixed(XVecTy->getNumElements()));
19587 "sign operand must have a float or int representation");
19590 Value *Cmp =
Builder.CreateICmpEQ(Op0, ConstantInt::get(Xty, 0));
19591 return Builder.CreateSelect(Cmp, ConstantInt::get(retType, 0),
19592 ConstantInt::get(retType, 1),
"hlsl.sign");
19595 return Builder.CreateIntrinsic(
19599 case Builtin::BI__builtin_hlsl_elementwise_radians: {
19602 "radians operand must have a float representation");
19603 return Builder.CreateIntrinsic(
19606 nullptr,
"hlsl.radians");
19608 case Builtin::BI__builtin_hlsl_buffer_update_counter: {
19612 return Builder.CreateIntrinsic(
19617 case Builtin::BI__builtin_hlsl_elementwise_splitdouble: {
19622 "asuint operands types mismatch");
19625 case Builtin::BI__builtin_hlsl_elementwise_clip:
19627 "clip operands types mismatch");
19629 case Builtin::BI__builtin_hlsl_group_memory_barrier_with_group_sync: {
19633 Intrinsic::getOrInsertDeclaration(&
CGM.
getModule(), ID));
19641 constexpr const char *
Tag =
"amdgpu-as";
19643 LLVMContext &Ctx = Inst->getContext();
19645 for (
unsigned K = 2; K <
E->getNumArgs(); ++K) {
19648 if (llvm::getConstantStringInfo(
V, AS)) {
19649 MMRAs.push_back({
Tag, AS});
19654 "expected an address space name as a string literal");
19658 MMRAs.erase(llvm::unique(MMRAs), MMRAs.end());
19659 Inst->setMetadata(LLVMContext::MD_mmra, MMRAMetadata::getMD(Ctx, MMRAs));
19664 llvm::AtomicOrdering AO = llvm::AtomicOrdering::SequentiallyConsistent;
19665 llvm::SyncScope::ID SSID;
19666 switch (BuiltinID) {
19667 case AMDGPU::BI__builtin_amdgcn_div_scale:
19668 case AMDGPU::BI__builtin_amdgcn_div_scalef: {
19681 llvm::Value *Tmp =
Builder.CreateCall(Callee, {
X, Y, Z});
19684 llvm::Value *Flag =
Builder.CreateExtractValue(Tmp, 1);
19688 llvm::Value *FlagExt =
Builder.CreateZExt(Flag, RealFlagType);
19692 case AMDGPU::BI__builtin_amdgcn_div_fmas:
19693 case AMDGPU::BI__builtin_amdgcn_div_fmasf: {
19701 llvm::Value *Src3ToBool =
Builder.CreateIsNotNull(Src3);
19702 return Builder.CreateCall(F, {Src0, Src1, Src2, Src3ToBool});
19705 case AMDGPU::BI__builtin_amdgcn_ds_swizzle:
19706 return emitBuiltinWithOneOverloadedType<2>(*
this,
E,
19707 Intrinsic::amdgcn_ds_swizzle);
19708 case AMDGPU::BI__builtin_amdgcn_mov_dpp8:
19709 case AMDGPU::BI__builtin_amdgcn_mov_dpp:
19710 case AMDGPU::BI__builtin_amdgcn_update_dpp: {
19714 unsigned ICEArguments = 0;
19719 unsigned Size = DataTy->getPrimitiveSizeInBits();
19720 llvm::Type *
IntTy =
19721 llvm::IntegerType::get(
Builder.getContext(), std::max(Size, 32u));
19724 ? Intrinsic::amdgcn_mov_dpp8
19725 : Intrinsic::amdgcn_update_dpp,
19727 assert(
E->getNumArgs() == 5 ||
E->getNumArgs() == 6 ||
19728 E->getNumArgs() == 2);
19729 bool InsertOld = BuiltinID == AMDGPU::BI__builtin_amdgcn_mov_dpp;
19731 Args.push_back(llvm::PoisonValue::get(
IntTy));
19732 for (
unsigned I = 0; I !=
E->getNumArgs(); ++I) {
19734 if (I < (BuiltinID == AMDGPU::BI__builtin_amdgcn_update_dpp ? 2u : 1u) &&
19736 if (!DataTy->isIntegerTy())
19738 V, llvm::IntegerType::get(
Builder.getContext(), Size));
19741 llvm::Type *ExpTy =
19742 F->getFunctionType()->getFunctionParamType(I + InsertOld);
19743 Args.push_back(
Builder.CreateTruncOrBitCast(
V, ExpTy));
19746 if (Size < 32 && !DataTy->isIntegerTy())
19748 V, llvm::IntegerType::get(
Builder.getContext(), Size));
19749 return Builder.CreateTruncOrBitCast(
V, DataTy);
19751 case AMDGPU::BI__builtin_amdgcn_permlane16:
19752 case AMDGPU::BI__builtin_amdgcn_permlanex16:
19753 return emitBuiltinWithOneOverloadedType<6>(
19755 BuiltinID == AMDGPU::BI__builtin_amdgcn_permlane16
19756 ? Intrinsic::amdgcn_permlane16
19757 : Intrinsic::amdgcn_permlanex16);
19758 case AMDGPU::BI__builtin_amdgcn_permlane64:
19759 return emitBuiltinWithOneOverloadedType<1>(*
this,
E,
19760 Intrinsic::amdgcn_permlane64);
19761 case AMDGPU::BI__builtin_amdgcn_readlane:
19762 return emitBuiltinWithOneOverloadedType<2>(*
this,
E,
19763 Intrinsic::amdgcn_readlane);
19764 case AMDGPU::BI__builtin_amdgcn_readfirstlane:
19765 return emitBuiltinWithOneOverloadedType<1>(*
this,
E,
19766 Intrinsic::amdgcn_readfirstlane);
19767 case AMDGPU::BI__builtin_amdgcn_div_fixup:
19768 case AMDGPU::BI__builtin_amdgcn_div_fixupf:
19769 case AMDGPU::BI__builtin_amdgcn_div_fixuph:
19770 return emitBuiltinWithOneOverloadedType<3>(*
this,
E,
19771 Intrinsic::amdgcn_div_fixup);
19772 case AMDGPU::BI__builtin_amdgcn_trig_preop:
19773 case AMDGPU::BI__builtin_amdgcn_trig_preopf:
19775 case AMDGPU::BI__builtin_amdgcn_rcp:
19776 case AMDGPU::BI__builtin_amdgcn_rcpf:
19777 case AMDGPU::BI__builtin_amdgcn_rcph:
19778 return emitBuiltinWithOneOverloadedType<1>(*
this,
E, Intrinsic::amdgcn_rcp);
19779 case AMDGPU::BI__builtin_amdgcn_sqrt:
19780 case AMDGPU::BI__builtin_amdgcn_sqrtf:
19781 case AMDGPU::BI__builtin_amdgcn_sqrth:
19782 return emitBuiltinWithOneOverloadedType<1>(*
this,
E,
19783 Intrinsic::amdgcn_sqrt);
19784 case AMDGPU::BI__builtin_amdgcn_rsq:
19785 case AMDGPU::BI__builtin_amdgcn_rsqf:
19786 case AMDGPU::BI__builtin_amdgcn_rsqh:
19787 return emitBuiltinWithOneOverloadedType<1>(*
this,
E, Intrinsic::amdgcn_rsq);
19788 case AMDGPU::BI__builtin_amdgcn_rsq_clamp:
19789 case AMDGPU::BI__builtin_amdgcn_rsq_clampf:
19790 return emitBuiltinWithOneOverloadedType<1>(*
this,
E,
19791 Intrinsic::amdgcn_rsq_clamp);
19792 case AMDGPU::BI__builtin_amdgcn_sinf:
19793 case AMDGPU::BI__builtin_amdgcn_sinh:
19794 return emitBuiltinWithOneOverloadedType<1>(*
this,
E, Intrinsic::amdgcn_sin);
19795 case AMDGPU::BI__builtin_amdgcn_cosf:
19796 case AMDGPU::BI__builtin_amdgcn_cosh:
19797 return emitBuiltinWithOneOverloadedType<1>(*
this,
E, Intrinsic::amdgcn_cos);
19798 case AMDGPU::BI__builtin_amdgcn_dispatch_ptr:
19799 return EmitAMDGPUDispatchPtr(*
this,
E);
19800 case AMDGPU::BI__builtin_amdgcn_logf:
19801 return emitBuiltinWithOneOverloadedType<1>(*
this,
E, Intrinsic::amdgcn_log);
19802 case AMDGPU::BI__builtin_amdgcn_exp2f:
19803 return emitBuiltinWithOneOverloadedType<1>(*
this,
E,
19804 Intrinsic::amdgcn_exp2);
19805 case AMDGPU::BI__builtin_amdgcn_log_clampf:
19806 return emitBuiltinWithOneOverloadedType<1>(*
this,
E,
19807 Intrinsic::amdgcn_log_clamp);
19808 case AMDGPU::BI__builtin_amdgcn_ldexp:
19809 case AMDGPU::BI__builtin_amdgcn_ldexpf: {
19812 llvm::Function *F =
19813 CGM.
getIntrinsic(Intrinsic::ldexp, {Src0->getType(), Src1->getType()});
19814 return Builder.CreateCall(F, {Src0, Src1});
19816 case AMDGPU::BI__builtin_amdgcn_ldexph: {
19821 llvm::Function *F =
19825 case AMDGPU::BI__builtin_amdgcn_frexp_mant:
19826 case AMDGPU::BI__builtin_amdgcn_frexp_mantf:
19827 case AMDGPU::BI__builtin_amdgcn_frexp_manth:
19828 return emitBuiltinWithOneOverloadedType<1>(*
this,
E,
19829 Intrinsic::amdgcn_frexp_mant);
19830 case AMDGPU::BI__builtin_amdgcn_frexp_exp:
19831 case AMDGPU::BI__builtin_amdgcn_frexp_expf: {
19835 return Builder.CreateCall(F, Src0);
19837 case AMDGPU::BI__builtin_amdgcn_frexp_exph: {
19841 return Builder.CreateCall(F, Src0);
19843 case AMDGPU::BI__builtin_amdgcn_fract:
19844 case AMDGPU::BI__builtin_amdgcn_fractf:
19845 case AMDGPU::BI__builtin_amdgcn_fracth:
19846 return emitBuiltinWithOneOverloadedType<1>(*
this,
E,
19847 Intrinsic::amdgcn_fract);
19848 case AMDGPU::BI__builtin_amdgcn_lerp:
19849 return emitBuiltinWithOneOverloadedType<3>(*
this,
E,
19850 Intrinsic::amdgcn_lerp);
19851 case AMDGPU::BI__builtin_amdgcn_ubfe:
19852 return emitBuiltinWithOneOverloadedType<3>(*
this,
E,
19853 Intrinsic::amdgcn_ubfe);
19854 case AMDGPU::BI__builtin_amdgcn_sbfe:
19855 return emitBuiltinWithOneOverloadedType<3>(*
this,
E,
19856 Intrinsic::amdgcn_sbfe);
19857 case AMDGPU::BI__builtin_amdgcn_ballot_w32:
19858 case AMDGPU::BI__builtin_amdgcn_ballot_w64: {
19862 return Builder.CreateCall(F, { Src });
19864 case AMDGPU::BI__builtin_amdgcn_uicmp:
19865 case AMDGPU::BI__builtin_amdgcn_uicmpl:
19866 case AMDGPU::BI__builtin_amdgcn_sicmp:
19867 case AMDGPU::BI__builtin_amdgcn_sicmpl: {
19874 {
Builder.getInt64Ty(), Src0->getType() });
19875 return Builder.CreateCall(F, { Src0, Src1, Src2 });
19877 case AMDGPU::BI__builtin_amdgcn_fcmp:
19878 case AMDGPU::BI__builtin_amdgcn_fcmpf: {
19885 {
Builder.getInt64Ty(), Src0->getType() });
19886 return Builder.CreateCall(F, { Src0, Src1, Src2 });
19888 case AMDGPU::BI__builtin_amdgcn_class:
19889 case AMDGPU::BI__builtin_amdgcn_classf:
19890 case AMDGPU::BI__builtin_amdgcn_classh:
19892 case AMDGPU::BI__builtin_amdgcn_fmed3f:
19893 case AMDGPU::BI__builtin_amdgcn_fmed3h:
19894 return emitBuiltinWithOneOverloadedType<3>(*
this,
E,
19895 Intrinsic::amdgcn_fmed3);
19896 case AMDGPU::BI__builtin_amdgcn_ds_append:
19897 case AMDGPU::BI__builtin_amdgcn_ds_consume: {
19898 Intrinsic::ID Intrin = BuiltinID == AMDGPU::BI__builtin_amdgcn_ds_append ?
19899 Intrinsic::amdgcn_ds_append : Intrinsic::amdgcn_ds_consume;
19904 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b64_i32:
19905 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b64_v2i32:
19906 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4i16:
19907 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4f16:
19908 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4bf16:
19909 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8i16:
19910 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8f16:
19911 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8bf16:
19912 case AMDGPU::BI__builtin_amdgcn_ds_read_tr4_b64_v2i32:
19913 case AMDGPU::BI__builtin_amdgcn_ds_read_tr8_b64_v2i32:
19914 case AMDGPU::BI__builtin_amdgcn_ds_read_tr6_b96_v3i32:
19915 case AMDGPU::BI__builtin_amdgcn_ds_read_tr16_b64_v4f16:
19916 case AMDGPU::BI__builtin_amdgcn_ds_read_tr16_b64_v4bf16:
19917 case AMDGPU::BI__builtin_amdgcn_ds_read_tr16_b64_v4i16: {
19919 switch (BuiltinID) {
19920 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b64_i32:
19921 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b64_v2i32:
19922 IID = Intrinsic::amdgcn_global_load_tr_b64;
19924 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4i16:
19925 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4f16:
19926 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4bf16:
19927 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8i16:
19928 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8f16:
19929 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8bf16:
19930 IID = Intrinsic::amdgcn_global_load_tr_b128;
19932 case AMDGPU::BI__builtin_amdgcn_ds_read_tr4_b64_v2i32:
19933 IID = Intrinsic::amdgcn_ds_read_tr4_b64;
19935 case AMDGPU::BI__builtin_amdgcn_ds_read_tr8_b64_v2i32:
19936 IID = Intrinsic::amdgcn_ds_read_tr8_b64;
19938 case AMDGPU::BI__builtin_amdgcn_ds_read_tr6_b96_v3i32:
19939 IID = Intrinsic::amdgcn_ds_read_tr6_b96;
19941 case AMDGPU::BI__builtin_amdgcn_ds_read_tr16_b64_v4i16:
19942 case AMDGPU::BI__builtin_amdgcn_ds_read_tr16_b64_v4f16:
19943 case AMDGPU::BI__builtin_amdgcn_ds_read_tr16_b64_v4bf16:
19944 IID = Intrinsic::amdgcn_ds_read_tr16_b64;
19950 return Builder.CreateCall(F, {Addr});
19952 case AMDGPU::BI__builtin_amdgcn_get_fpenv: {
19955 return Builder.CreateCall(F);
19957 case AMDGPU::BI__builtin_amdgcn_set_fpenv: {
19963 case AMDGPU::BI__builtin_amdgcn_read_exec:
19965 case AMDGPU::BI__builtin_amdgcn_read_exec_lo:
19967 case AMDGPU::BI__builtin_amdgcn_read_exec_hi:
19969 case AMDGPU::BI__builtin_amdgcn_image_bvh_intersect_ray:
19970 case AMDGPU::BI__builtin_amdgcn_image_bvh_intersect_ray_h:
19971 case AMDGPU::BI__builtin_amdgcn_image_bvh_intersect_ray_l:
19972 case AMDGPU::BI__builtin_amdgcn_image_bvh_intersect_ray_lh: {
19982 RayOrigin =
Builder.CreateShuffleVector(RayOrigin, RayOrigin,
19986 RayInverseDir =
Builder.CreateShuffleVector(RayInverseDir, RayInverseDir,
19990 {NodePtr->getType(), RayDir->getType()});
19991 return Builder.CreateCall(F, {NodePtr, RayExtent, RayOrigin, RayDir,
19992 RayInverseDir, TextureDescr});
19995 case AMDGPU::BI__builtin_amdgcn_ds_bvh_stack_rtn: {
19997 for (
int i = 0, e =
E->getNumArgs(); i != e; ++i)
20005 Value *I0 =
Builder.CreateInsertElement(PoisonValue::get(RetTy), Rtn,
20007 return Builder.CreateInsertElement(I0, A, 1);
20009 case AMDGPU::BI__builtin_amdgcn_mfma_scale_f32_16x16x128_f8f6f4:
20010 case AMDGPU::BI__builtin_amdgcn_mfma_scale_f32_32x32x64_f8f6f4: {
20011 llvm::FixedVectorType *VT = FixedVectorType::get(
Builder.getInt32Ty(), 8);
20013 BuiltinID == AMDGPU::BI__builtin_amdgcn_mfma_scale_f32_32x32x64_f8f6f4
20014 ? Intrinsic::amdgcn_mfma_scale_f32_32x32x64_f8f6f4
20015 : Intrinsic::amdgcn_mfma_scale_f32_16x16x128_f8f6f4,
20019 for (
unsigned I = 0, N =
E->getNumArgs(); I != N; ++I)
20021 return Builder.CreateCall(F, Args);
20023 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w32:
20024 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_tied_w32:
20025 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w64:
20026 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_tied_w64:
20027 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w32:
20028 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_tied_w32:
20029 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w64:
20030 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_tied_w64:
20031 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w32:
20032 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w64:
20033 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w32:
20034 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w64:
20035 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w32:
20036 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w64:
20037 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w32:
20038 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w64:
20039 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w32_gfx12:
20040 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w64_gfx12:
20041 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w32_gfx12:
20042 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w64_gfx12:
20043 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w32_gfx12:
20044 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w64_gfx12:
20045 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w32_gfx12:
20046 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w64_gfx12:
20047 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w32_gfx12:
20048 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w64_gfx12:
20049 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w32_gfx12:
20050 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w64_gfx12:
20051 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_fp8_w32_gfx12:
20052 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_fp8_w64_gfx12:
20053 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_bf8_w32_gfx12:
20054 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_bf8_w64_gfx12:
20055 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_fp8_w32_gfx12:
20056 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_fp8_w64_gfx12:
20057 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_bf8_w32_gfx12:
20058 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_bf8_w64_gfx12:
20059 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x32_iu4_w32_gfx12:
20060 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x32_iu4_w64_gfx12:
20061 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_f16_w32:
20062 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_f16_w64:
20063 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf16_w32:
20064 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf16_w64:
20065 case AMDGPU::BI__builtin_amdgcn_swmmac_f16_16x16x32_f16_w32:
20066 case AMDGPU::BI__builtin_amdgcn_swmmac_f16_16x16x32_f16_w64:
20067 case AMDGPU::BI__builtin_amdgcn_swmmac_bf16_16x16x32_bf16_w32:
20068 case AMDGPU::BI__builtin_amdgcn_swmmac_bf16_16x16x32_bf16_w64:
20069 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu8_w32:
20070 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu8_w64:
20071 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu4_w32:
20072 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu4_w64:
20073 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x64_iu4_w32:
20074 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x64_iu4_w64:
20075 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_fp8_w32:
20076 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_fp8_w64:
20077 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_bf8_w32:
20078 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_bf8_w64:
20079 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_fp8_w32:
20080 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_fp8_w64:
20081 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_bf8_w32:
20082 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_bf8_w64: {
20095 bool AppendFalseForOpselArg =
false;
20096 unsigned BuiltinWMMAOp;
20098 switch (BuiltinID) {
20099 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w32:
20100 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w64:
20101 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w32_gfx12:
20102 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w64_gfx12:
20103 ArgsForMatchingMatrixTypes = {2, 0};
20104 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x16_f16;
20106 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w32:
20107 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w64:
20108 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w32_gfx12:
20109 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w64_gfx12:
20110 ArgsForMatchingMatrixTypes = {2, 0};
20111 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x16_bf16;
20113 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w32_gfx12:
20114 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w64_gfx12:
20115 AppendFalseForOpselArg =
true;
20117 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w32:
20118 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w64:
20119 ArgsForMatchingMatrixTypes = {2, 0};
20120 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f16_16x16x16_f16;
20122 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w32_gfx12:
20123 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w64_gfx12:
20124 AppendFalseForOpselArg =
true;
20126 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w32:
20127 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w64:
20128 ArgsForMatchingMatrixTypes = {2, 0};
20129 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_bf16_16x16x16_bf16;
20131 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_tied_w32:
20132 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_tied_w64:
20133 ArgsForMatchingMatrixTypes = {2, 0};
20134 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f16_16x16x16_f16_tied;
20136 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_tied_w32:
20137 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_tied_w64:
20138 ArgsForMatchingMatrixTypes = {2, 0};
20139 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_bf16_16x16x16_bf16_tied;
20141 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w32:
20142 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w64:
20143 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w32_gfx12:
20144 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w64_gfx12:
20145 ArgsForMatchingMatrixTypes = {4, 1};
20146 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_i32_16x16x16_iu8;
20148 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w32:
20149 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w64:
20150 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w32_gfx12:
20151 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w64_gfx12:
20152 ArgsForMatchingMatrixTypes = {4, 1};
20153 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_i32_16x16x16_iu4;
20155 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_fp8_w32_gfx12:
20156 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_fp8_w64_gfx12:
20157 ArgsForMatchingMatrixTypes = {2, 0};
20158 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x16_fp8_fp8;
20160 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_bf8_w32_gfx12:
20161 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_bf8_w64_gfx12:
20162 ArgsForMatchingMatrixTypes = {2, 0};
20163 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x16_fp8_bf8;
20165 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_fp8_w32_gfx12:
20166 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_fp8_w64_gfx12:
20167 ArgsForMatchingMatrixTypes = {2, 0};
20168 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x16_bf8_fp8;
20170 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_bf8_w32_gfx12:
20171 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_bf8_w64_gfx12:
20172 ArgsForMatchingMatrixTypes = {2, 0};
20173 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x16_bf8_bf8;
20175 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x32_iu4_w32_gfx12:
20176 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x32_iu4_w64_gfx12:
20177 ArgsForMatchingMatrixTypes = {4, 1};
20178 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_i32_16x16x32_iu4;
20180 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_f16_w32:
20181 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_f16_w64:
20182 ArgsForMatchingMatrixTypes = {2, 0, 1, 3};
20183 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f32_16x16x32_f16;
20185 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf16_w32:
20186 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf16_w64:
20187 ArgsForMatchingMatrixTypes = {2, 0, 1, 3};
20188 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f32_16x16x32_bf16;
20190 case AMDGPU::BI__builtin_amdgcn_swmmac_f16_16x16x32_f16_w32:
20191 case AMDGPU::BI__builtin_amdgcn_swmmac_f16_16x16x32_f16_w64:
20192 ArgsForMatchingMatrixTypes = {2, 0, 1, 3};
20193 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f16_16x16x32_f16;
20195 case AMDGPU::BI__builtin_amdgcn_swmmac_bf16_16x16x32_bf16_w32:
20196 case AMDGPU::BI__builtin_amdgcn_swmmac_bf16_16x16x32_bf16_w64:
20197 ArgsForMatchingMatrixTypes = {2, 0, 1, 3};
20198 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_bf16_16x16x32_bf16;
20200 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu8_w32:
20201 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu8_w64:
20202 ArgsForMatchingMatrixTypes = {4, 1, 3, 5};
20203 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_i32_16x16x32_iu8;
20205 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu4_w32:
20206 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu4_w64:
20207 ArgsForMatchingMatrixTypes = {4, 1, 3, 5};
20208 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_i32_16x16x32_iu4;
20210 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x64_iu4_w32:
20211 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x64_iu4_w64:
20212 ArgsForMatchingMatrixTypes = {4, 1, 3, 5};
20213 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_i32_16x16x64_iu4;
20215 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_fp8_w32:
20216 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_fp8_w64:
20217 ArgsForMatchingMatrixTypes = {2, 0, 1, 3};
20218 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f32_16x16x32_fp8_fp8;
20220 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_bf8_w32:
20221 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_bf8_w64:
20222 ArgsForMatchingMatrixTypes = {2, 0, 1, 3};
20223 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f32_16x16x32_fp8_bf8;
20225 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_fp8_w32:
20226 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_fp8_w64:
20227 ArgsForMatchingMatrixTypes = {2, 0, 1, 3};
20228 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f32_16x16x32_bf8_fp8;
20230 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_bf8_w32:
20231 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_bf8_w64:
20232 ArgsForMatchingMatrixTypes = {2, 0, 1, 3};
20233 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f32_16x16x32_bf8_bf8;
20238 for (
int i = 0, e =
E->getNumArgs(); i != e; ++i)
20240 if (AppendFalseForOpselArg)
20241 Args.push_back(
Builder.getFalse());
20244 for (
auto ArgIdx : ArgsForMatchingMatrixTypes)
20245 ArgTypes.push_back(Args[ArgIdx]->getType());
20248 return Builder.CreateCall(F, Args);
20252 case AMDGPU::BI__builtin_amdgcn_workitem_id_x:
20254 case AMDGPU::BI__builtin_amdgcn_workitem_id_y:
20256 case AMDGPU::BI__builtin_amdgcn_workitem_id_z:
20260 case AMDGPU::BI__builtin_amdgcn_workgroup_size_x:
20261 return EmitAMDGPUWorkGroupSize(*
this, 0);
20262 case AMDGPU::BI__builtin_amdgcn_workgroup_size_y:
20263 return EmitAMDGPUWorkGroupSize(*
this, 1);
20264 case AMDGPU::BI__builtin_amdgcn_workgroup_size_z:
20265 return EmitAMDGPUWorkGroupSize(*
this, 2);
20268 case AMDGPU::BI__builtin_amdgcn_grid_size_x:
20269 return EmitAMDGPUGridSize(*
this, 0);
20270 case AMDGPU::BI__builtin_amdgcn_grid_size_y:
20271 return EmitAMDGPUGridSize(*
this, 1);
20272 case AMDGPU::BI__builtin_amdgcn_grid_size_z:
20273 return EmitAMDGPUGridSize(*
this, 2);
20276 case AMDGPU::BI__builtin_r600_recipsqrt_ieee:
20277 case AMDGPU::BI__builtin_r600_recipsqrt_ieeef:
20278 return emitBuiltinWithOneOverloadedType<1>(*
this,
E,
20279 Intrinsic::r600_recipsqrt_ieee);
20280 case AMDGPU::BI__builtin_r600_read_tidig_x:
20282 case AMDGPU::BI__builtin_r600_read_tidig_y:
20284 case AMDGPU::BI__builtin_r600_read_tidig_z:
20286 case AMDGPU::BI__builtin_amdgcn_alignbit: {
20291 return Builder.CreateCall(F, { Src0, Src1, Src2 });
20293 case AMDGPU::BI__builtin_amdgcn_fence: {
20296 FenceInst *Fence =
Builder.CreateFence(AO, SSID);
20297 if (
E->getNumArgs() > 2)
20301 case AMDGPU::BI__builtin_amdgcn_atomic_inc32:
20302 case AMDGPU::BI__builtin_amdgcn_atomic_inc64:
20303 case AMDGPU::BI__builtin_amdgcn_atomic_dec32:
20304 case AMDGPU::BI__builtin_amdgcn_atomic_dec64:
20305 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_f64:
20306 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_f32:
20307 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_v2f16:
20308 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_v2bf16:
20309 case AMDGPU::BI__builtin_amdgcn_ds_faddf:
20310 case AMDGPU::BI__builtin_amdgcn_ds_fminf:
20311 case AMDGPU::BI__builtin_amdgcn_ds_fmaxf:
20312 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_f32:
20313 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_f64:
20314 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_v2f16:
20315 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_v2f16:
20316 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_f32:
20317 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_f64:
20318 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_v2bf16:
20319 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_v2bf16:
20320 case AMDGPU::BI__builtin_amdgcn_global_atomic_fmin_f64:
20321 case AMDGPU::BI__builtin_amdgcn_global_atomic_fmax_f64:
20322 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fmin_f64:
20323 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fmax_f64: {
20324 llvm::AtomicRMWInst::BinOp BinOp;
20325 switch (BuiltinID) {
20326 case AMDGPU::BI__builtin_amdgcn_atomic_inc32:
20327 case AMDGPU::BI__builtin_amdgcn_atomic_inc64:
20328 BinOp = llvm::AtomicRMWInst::UIncWrap;
20330 case AMDGPU::BI__builtin_amdgcn_atomic_dec32:
20331 case AMDGPU::BI__builtin_amdgcn_atomic_dec64:
20332 BinOp = llvm::AtomicRMWInst::UDecWrap;
20334 case AMDGPU::BI__builtin_amdgcn_ds_faddf:
20335 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_f64:
20336 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_f32:
20337 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_v2f16:
20338 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_v2bf16:
20339 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_f32:
20340 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_f64:
20341 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_v2f16:
20342 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_v2f16:
20343 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_f32:
20344 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_f64:
20345 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_v2bf16:
20346 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_v2bf16:
20347 BinOp = llvm::AtomicRMWInst::FAdd;
20349 case AMDGPU::BI__builtin_amdgcn_ds_fminf:
20350 case AMDGPU::BI__builtin_amdgcn_global_atomic_fmin_f64:
20351 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fmin_f64:
20352 BinOp = llvm::AtomicRMWInst::FMin;
20354 case AMDGPU::BI__builtin_amdgcn_global_atomic_fmax_f64:
20355 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fmax_f64:
20356 case AMDGPU::BI__builtin_amdgcn_ds_fmaxf:
20357 BinOp = llvm::AtomicRMWInst::FMax;
20363 llvm::Type *OrigTy = Val->
getType();
20368 if (BuiltinID == AMDGPU::BI__builtin_amdgcn_ds_faddf ||
20369 BuiltinID == AMDGPU::BI__builtin_amdgcn_ds_fminf ||
20370 BuiltinID == AMDGPU::BI__builtin_amdgcn_ds_fmaxf) {
20380 if (
E->getNumArgs() >= 4) {
20392 AO = AtomicOrdering::Monotonic;
20395 if (BuiltinID == AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_v2bf16 ||
20396 BuiltinID == AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_v2bf16 ||
20397 BuiltinID == AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_v2bf16) {
20398 llvm::Type *V2BF16Ty = FixedVectorType::get(
20399 llvm::Type::getBFloatTy(
Builder.getContext()), 2);
20400 Val =
Builder.CreateBitCast(Val, V2BF16Ty);
20404 llvm::AtomicRMWInst *RMW =
20407 RMW->setVolatile(
true);
20409 unsigned AddrSpace = Ptr.
getType()->getAddressSpace();
20410 if (AddrSpace != llvm::AMDGPUAS::LOCAL_ADDRESS) {
20414 RMW->setMetadata(
"amdgpu.no.fine.grained.memory", EmptyMD);
20418 if (BinOp == llvm::AtomicRMWInst::FAdd && Val->
getType()->isFloatTy())
20419 RMW->setMetadata(
"amdgpu.ignore.denormal.mode", EmptyMD);
20422 return Builder.CreateBitCast(RMW, OrigTy);
20424 case AMDGPU::BI__builtin_amdgcn_s_sendmsg_rtn:
20425 case AMDGPU::BI__builtin_amdgcn_s_sendmsg_rtnl: {
20431 return Builder.CreateCall(F, {Arg});
20433 case AMDGPU::BI__builtin_amdgcn_permlane16_swap:
20434 case AMDGPU::BI__builtin_amdgcn_permlane32_swap: {
20442 CGM.
getIntrinsic(BuiltinID == AMDGPU::BI__builtin_amdgcn_permlane16_swap
20443 ? Intrinsic::amdgcn_permlane16_swap
20444 : Intrinsic::amdgcn_permlane32_swap);
20445 llvm::CallInst *
Call =
20446 Builder.CreateCall(F, {VDstOld, VSrcOld, FI, BoundCtrl});
20448 llvm::Value *Elt0 =
Builder.CreateExtractValue(
Call, 0);
20449 llvm::Value *Elt1 =
Builder.CreateExtractValue(
Call, 1);
20453 llvm::Value *Insert0 =
Builder.CreateInsertElement(
20454 llvm::PoisonValue::get(ResultType), Elt0, UINT64_C(0));
20455 llvm::Value *AsVector =
20456 Builder.CreateInsertElement(Insert0, Elt1, UINT64_C(1));
20459 case AMDGPU::BI__builtin_amdgcn_bitop3_b32:
20460 case AMDGPU::BI__builtin_amdgcn_bitop3_b16:
20461 return emitBuiltinWithOneOverloadedType<4>(*
this,
E,
20462 Intrinsic::amdgcn_bitop3);
20463 case AMDGPU::BI__builtin_amdgcn_make_buffer_rsrc:
20464 return emitBuiltinWithOneOverloadedType<4>(
20465 *
this,
E, Intrinsic::amdgcn_make_buffer_rsrc);
20466 case AMDGPU::BI__builtin_amdgcn_raw_buffer_store_b8:
20467 case AMDGPU::BI__builtin_amdgcn_raw_buffer_store_b16:
20468 case AMDGPU::BI__builtin_amdgcn_raw_buffer_store_b32:
20469 case AMDGPU::BI__builtin_amdgcn_raw_buffer_store_b64:
20470 case AMDGPU::BI__builtin_amdgcn_raw_buffer_store_b96:
20471 case AMDGPU::BI__builtin_amdgcn_raw_buffer_store_b128:
20472 return emitBuiltinWithOneOverloadedType<5>(
20473 *
this,
E, Intrinsic::amdgcn_raw_ptr_buffer_store);
20474 case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b8:
20475 case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b16:
20476 case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b32:
20477 case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b64:
20478 case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b96:
20479 case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b128: {
20480 llvm::Type *RetTy =
nullptr;
20481 switch (BuiltinID) {
20482 case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b8:
20485 case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b16:
20488 case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b32:
20491 case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b64:
20492 RetTy = llvm::FixedVectorType::get(
Int32Ty, 2);
20494 case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b96:
20495 RetTy = llvm::FixedVectorType::get(
Int32Ty, 3);
20497 case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b128:
20498 RetTy = llvm::FixedVectorType::get(
Int32Ty, 4);
20507 case AMDGPU::BI__builtin_amdgcn_s_prefetch_data:
20508 return emitBuiltinWithOneOverloadedType<2>(
20509 *
this,
E, Intrinsic::amdgcn_s_prefetch_data);
20517 switch (BuiltinID) {
20518 case SPIRV::BI__builtin_spirv_distance: {
20523 "Distance operands must have a float representation");
20526 "Distance operands must be a vector");
20527 return Builder.CreateIntrinsic(
20528 X->getType()->getScalarType(), Intrinsic::spv_distance,
20539 unsigned IntrinsicID,
20541 unsigned NumArgs =
E->getNumArgs() - 1;
20543 for (
unsigned I = 0; I < NumArgs; ++I)
20555 switch (BuiltinID) {
20556 case SystemZ::BI__builtin_tbegin: {
20558 Value *Control = llvm::ConstantInt::get(
Int32Ty, 0xff0c);
20560 return Builder.CreateCall(F, {TDB, Control});
20562 case SystemZ::BI__builtin_tbegin_nofloat: {
20564 Value *Control = llvm::ConstantInt::get(
Int32Ty, 0xff0c);
20566 return Builder.CreateCall(F, {TDB, Control});
20568 case SystemZ::BI__builtin_tbeginc: {
20570 Value *Control = llvm::ConstantInt::get(
Int32Ty, 0xff08);
20572 return Builder.CreateCall(F, {TDB, Control});
20574 case SystemZ::BI__builtin_tabort: {
20579 case SystemZ::BI__builtin_non_tx_store: {
20591 case SystemZ::BI__builtin_s390_vclzb:
20592 case SystemZ::BI__builtin_s390_vclzh:
20593 case SystemZ::BI__builtin_s390_vclzf:
20594 case SystemZ::BI__builtin_s390_vclzg: {
20597 Value *Undef = ConstantInt::get(
Builder.getInt1Ty(),
false);
20599 return Builder.CreateCall(F, {
X, Undef});
20602 case SystemZ::BI__builtin_s390_vctzb:
20603 case SystemZ::BI__builtin_s390_vctzh:
20604 case SystemZ::BI__builtin_s390_vctzf:
20605 case SystemZ::BI__builtin_s390_vctzg: {
20608 Value *Undef = ConstantInt::get(
Builder.getInt1Ty(),
false);
20610 return Builder.CreateCall(F, {
X, Undef});
20613 case SystemZ::BI__builtin_s390_verllb:
20614 case SystemZ::BI__builtin_s390_verllh:
20615 case SystemZ::BI__builtin_s390_verllf:
20616 case SystemZ::BI__builtin_s390_verllg: {
20621 unsigned NumElts = cast<llvm::FixedVectorType>(ResultType)->getNumElements();
20622 Amt =
Builder.CreateIntCast(Amt, ResultType->getScalarType(),
false);
20623 Amt =
Builder.CreateVectorSplat(NumElts, Amt);
20625 return Builder.CreateCall(F, { Src, Src, Amt });
20628 case SystemZ::BI__builtin_s390_verllvb:
20629 case SystemZ::BI__builtin_s390_verllvh:
20630 case SystemZ::BI__builtin_s390_verllvf:
20631 case SystemZ::BI__builtin_s390_verllvg: {
20636 return Builder.CreateCall(F, { Src, Src, Amt });
20639 case SystemZ::BI__builtin_s390_vfsqsb:
20640 case SystemZ::BI__builtin_s390_vfsqdb: {
20643 if (
Builder.getIsFPConstrained()) {
20645 return Builder.CreateConstrainedFPCall(F, {
X });
20651 case SystemZ::BI__builtin_s390_vfmasb:
20652 case SystemZ::BI__builtin_s390_vfmadb: {
20657 if (
Builder.getIsFPConstrained()) {
20659 return Builder.CreateConstrainedFPCall(F, {
X, Y, Z});
20662 return Builder.CreateCall(F, {
X, Y, Z});
20665 case SystemZ::BI__builtin_s390_vfmssb:
20666 case SystemZ::BI__builtin_s390_vfmsdb: {
20671 if (
Builder.getIsFPConstrained()) {
20673 return Builder.CreateConstrainedFPCall(F, {
X, Y,
Builder.CreateFNeg(Z,
"neg")});
20679 case SystemZ::BI__builtin_s390_vfnmasb:
20680 case SystemZ::BI__builtin_s390_vfnmadb: {
20685 if (
Builder.getIsFPConstrained()) {
20687 return Builder.CreateFNeg(
Builder.CreateConstrainedFPCall(F, {X, Y, Z}),
"neg");
20690 return Builder.CreateFNeg(
Builder.CreateCall(F, {X, Y, Z}),
"neg");
20693 case SystemZ::BI__builtin_s390_vfnmssb:
20694 case SystemZ::BI__builtin_s390_vfnmsdb: {
20699 if (
Builder.getIsFPConstrained()) {
20702 return Builder.CreateFNeg(
Builder.CreateConstrainedFPCall(F, {X, Y, NegZ}));
20709 case SystemZ::BI__builtin_s390_vflpsb:
20710 case SystemZ::BI__builtin_s390_vflpdb: {
20716 case SystemZ::BI__builtin_s390_vflnsb:
20717 case SystemZ::BI__builtin_s390_vflndb: {
20723 case SystemZ::BI__builtin_s390_vfisb:
20724 case SystemZ::BI__builtin_s390_vfidb: {
20732 Intrinsic::ID
ID = Intrinsic::not_intrinsic;
20734 switch (M4.getZExtValue()) {
20737 switch (M5.getZExtValue()) {
20739 case 0:
ID = Intrinsic::rint;
20740 CI = Intrinsic::experimental_constrained_rint;
break;
20744 switch (M5.getZExtValue()) {
20746 case 0:
ID = Intrinsic::nearbyint;
20747 CI = Intrinsic::experimental_constrained_nearbyint;
break;
20748 case 1:
ID = Intrinsic::round;
20749 CI = Intrinsic::experimental_constrained_round;
break;
20750 case 5:
ID = Intrinsic::trunc;
20751 CI = Intrinsic::experimental_constrained_trunc;
break;
20752 case 6:
ID = Intrinsic::ceil;
20753 CI = Intrinsic::experimental_constrained_ceil;
break;
20754 case 7:
ID = Intrinsic::floor;
20755 CI = Intrinsic::experimental_constrained_floor;
break;
20759 if (ID != Intrinsic::not_intrinsic) {
20760 if (
Builder.getIsFPConstrained()) {
20762 return Builder.CreateConstrainedFPCall(F,
X);
20768 switch (BuiltinID) {
20769 case SystemZ::BI__builtin_s390_vfisb:
ID = Intrinsic::s390_vfisb;
break;
20770 case SystemZ::BI__builtin_s390_vfidb:
ID = Intrinsic::s390_vfidb;
break;
20771 default: llvm_unreachable(
"Unknown BuiltinID");
20776 return Builder.CreateCall(F, {
X, M4Value, M5Value});
20778 case SystemZ::BI__builtin_s390_vfmaxsb:
20779 case SystemZ::BI__builtin_s390_vfmaxdb: {
20787 Intrinsic::ID
ID = Intrinsic::not_intrinsic;
20789 switch (M4.getZExtValue()) {
20791 case 4:
ID = Intrinsic::maxnum;
20792 CI = Intrinsic::experimental_constrained_maxnum;
break;
20794 if (ID != Intrinsic::not_intrinsic) {
20795 if (
Builder.getIsFPConstrained()) {
20797 return Builder.CreateConstrainedFPCall(F, {
X, Y});
20800 return Builder.CreateCall(F, {
X, Y});
20803 switch (BuiltinID) {
20804 case SystemZ::BI__builtin_s390_vfmaxsb:
ID = Intrinsic::s390_vfmaxsb;
break;
20805 case SystemZ::BI__builtin_s390_vfmaxdb:
ID = Intrinsic::s390_vfmaxdb;
break;
20806 default: llvm_unreachable(
"Unknown BuiltinID");
20810 return Builder.CreateCall(F, {
X, Y, M4Value});
20812 case SystemZ::BI__builtin_s390_vfminsb:
20813 case SystemZ::BI__builtin_s390_vfmindb: {
20821 Intrinsic::ID
ID = Intrinsic::not_intrinsic;
20823 switch (M4.getZExtValue()) {
20825 case 4:
ID = Intrinsic::minnum;
20826 CI = Intrinsic::experimental_constrained_minnum;
break;
20828 if (ID != Intrinsic::not_intrinsic) {
20829 if (
Builder.getIsFPConstrained()) {
20831 return Builder.CreateConstrainedFPCall(F, {
X, Y});
20834 return Builder.CreateCall(F, {
X, Y});
20837 switch (BuiltinID) {
20838 case SystemZ::BI__builtin_s390_vfminsb:
ID = Intrinsic::s390_vfminsb;
break;
20839 case SystemZ::BI__builtin_s390_vfmindb:
ID = Intrinsic::s390_vfmindb;
break;
20840 default: llvm_unreachable(
"Unknown BuiltinID");
20844 return Builder.CreateCall(F, {
X, Y, M4Value});
20847 case SystemZ::BI__builtin_s390_vlbrh:
20848 case SystemZ::BI__builtin_s390_vlbrf:
20849 case SystemZ::BI__builtin_s390_vlbrg: {
20858#define INTRINSIC_WITH_CC(NAME) \
20859 case SystemZ::BI__builtin_##NAME: \
20860 return EmitSystemZIntrinsicWithCC(*this, Intrinsic::NAME, E)
20939#undef INTRINSIC_WITH_CC
20948struct NVPTXMmaLdstInfo {
20949 unsigned NumResults;
20955#define MMA_INTR(geom_op_type, layout) \
20956 Intrinsic::nvvm_wmma_##geom_op_type##_##layout##_stride
20957#define MMA_LDST(n, geom_op_type) \
20958 { n, MMA_INTR(geom_op_type, col), MMA_INTR(geom_op_type, row) }
20960static NVPTXMmaLdstInfo getNVPTXMmaLdstInfo(
unsigned BuiltinID) {
20961 switch (BuiltinID) {
20963 case NVPTX::BI__hmma_m16n16k16_ld_a:
20964 return MMA_LDST(8, m16n16k16_load_a_f16);
20965 case NVPTX::BI__hmma_m16n16k16_ld_b:
20966 return MMA_LDST(8, m16n16k16_load_b_f16);
20967 case NVPTX::BI__hmma_m16n16k16_ld_c_f16:
20968 return MMA_LDST(4, m16n16k16_load_c_f16);
20969 case NVPTX::BI__hmma_m16n16k16_ld_c_f32:
20970 return MMA_LDST(8, m16n16k16_load_c_f32);
20971 case NVPTX::BI__hmma_m32n8k16_ld_a:
20972 return MMA_LDST(8, m32n8k16_load_a_f16);
20973 case NVPTX::BI__hmma_m32n8k16_ld_b:
20974 return MMA_LDST(8, m32n8k16_load_b_f16);
20975 case NVPTX::BI__hmma_m32n8k16_ld_c_f16:
20976 return MMA_LDST(4, m32n8k16_load_c_f16);
20977 case NVPTX::BI__hmma_m32n8k16_ld_c_f32:
20978 return MMA_LDST(8, m32n8k16_load_c_f32);
20979 case NVPTX::BI__hmma_m8n32k16_ld_a:
20980 return MMA_LDST(8, m8n32k16_load_a_f16);
20981 case NVPTX::BI__hmma_m8n32k16_ld_b:
20982 return MMA_LDST(8, m8n32k16_load_b_f16);
20983 case NVPTX::BI__hmma_m8n32k16_ld_c_f16:
20984 return MMA_LDST(4, m8n32k16_load_c_f16);
20985 case NVPTX::BI__hmma_m8n32k16_ld_c_f32:
20986 return MMA_LDST(8, m8n32k16_load_c_f32);
20989 case NVPTX::BI__imma_m16n16k16_ld_a_s8:
20990 return MMA_LDST(2, m16n16k16_load_a_s8);
20991 case NVPTX::BI__imma_m16n16k16_ld_a_u8:
20992 return MMA_LDST(2, m16n16k16_load_a_u8);
20993 case NVPTX::BI__imma_m16n16k16_ld_b_s8:
20994 return MMA_LDST(2, m16n16k16_load_b_s8);
20995 case NVPTX::BI__imma_m16n16k16_ld_b_u8:
20996 return MMA_LDST(2, m16n16k16_load_b_u8);
20997 case NVPTX::BI__imma_m16n16k16_ld_c:
20998 return MMA_LDST(8, m16n16k16_load_c_s32);
20999 case NVPTX::BI__imma_m32n8k16_ld_a_s8:
21000 return MMA_LDST(4, m32n8k16_load_a_s8);
21001 case NVPTX::BI__imma_m32n8k16_ld_a_u8:
21002 return MMA_LDST(4, m32n8k16_load_a_u8);
21003 case NVPTX::BI__imma_m32n8k16_ld_b_s8:
21004 return MMA_LDST(1, m32n8k16_load_b_s8);
21005 case NVPTX::BI__imma_m32n8k16_ld_b_u8:
21006 return MMA_LDST(1, m32n8k16_load_b_u8);
21007 case NVPTX::BI__imma_m32n8k16_ld_c:
21008 return MMA_LDST(8, m32n8k16_load_c_s32);
21009 case NVPTX::BI__imma_m8n32k16_ld_a_s8:
21010 return MMA_LDST(1, m8n32k16_load_a_s8);
21011 case NVPTX::BI__imma_m8n32k16_ld_a_u8:
21012 return MMA_LDST(1, m8n32k16_load_a_u8);
21013 case NVPTX::BI__imma_m8n32k16_ld_b_s8:
21014 return MMA_LDST(4, m8n32k16_load_b_s8);
21015 case NVPTX::BI__imma_m8n32k16_ld_b_u8:
21016 return MMA_LDST(4, m8n32k16_load_b_u8);
21017 case NVPTX::BI__imma_m8n32k16_ld_c:
21018 return MMA_LDST(8, m8n32k16_load_c_s32);
21022 case NVPTX::BI__imma_m8n8k32_ld_a_s4:
21023 return {1, 0,
MMA_INTR(m8n8k32_load_a_s4, row)};
21024 case NVPTX::BI__imma_m8n8k32_ld_a_u4:
21025 return {1, 0,
MMA_INTR(m8n8k32_load_a_u4, row)};
21026 case NVPTX::BI__imma_m8n8k32_ld_b_s4:
21027 return {1,
MMA_INTR(m8n8k32_load_b_s4, col), 0};
21028 case NVPTX::BI__imma_m8n8k32_ld_b_u4:
21029 return {1,
MMA_INTR(m8n8k32_load_b_u4, col), 0};
21030 case NVPTX::BI__imma_m8n8k32_ld_c:
21031 return MMA_LDST(2, m8n8k32_load_c_s32);
21032 case NVPTX::BI__bmma_m8n8k128_ld_a_b1:
21033 return {1, 0,
MMA_INTR(m8n8k128_load_a_b1, row)};
21034 case NVPTX::BI__bmma_m8n8k128_ld_b_b1:
21035 return {1,
MMA_INTR(m8n8k128_load_b_b1, col), 0};
21036 case NVPTX::BI__bmma_m8n8k128_ld_c:
21037 return MMA_LDST(2, m8n8k128_load_c_s32);
21040 case NVPTX::BI__dmma_m8n8k4_ld_a:
21041 return MMA_LDST(1, m8n8k4_load_a_f64);
21042 case NVPTX::BI__dmma_m8n8k4_ld_b:
21043 return MMA_LDST(1, m8n8k4_load_b_f64);
21044 case NVPTX::BI__dmma_m8n8k4_ld_c:
21045 return MMA_LDST(2, m8n8k4_load_c_f64);
21048 case NVPTX::BI__mma_bf16_m16n16k16_ld_a:
21049 return MMA_LDST(4, m16n16k16_load_a_bf16);
21050 case NVPTX::BI__mma_bf16_m16n16k16_ld_b:
21051 return MMA_LDST(4, m16n16k16_load_b_bf16);
21052 case NVPTX::BI__mma_bf16_m8n32k16_ld_a:
21053 return MMA_LDST(2, m8n32k16_load_a_bf16);
21054 case NVPTX::BI__mma_bf16_m8n32k16_ld_b:
21055 return MMA_LDST(8, m8n32k16_load_b_bf16);
21056 case NVPTX::BI__mma_bf16_m32n8k16_ld_a:
21057 return MMA_LDST(8, m32n8k16_load_a_bf16);
21058 case NVPTX::BI__mma_bf16_m32n8k16_ld_b:
21059 return MMA_LDST(2, m32n8k16_load_b_bf16);
21060 case NVPTX::BI__mma_tf32_m16n16k8_ld_a:
21061 return MMA_LDST(4, m16n16k8_load_a_tf32);
21062 case NVPTX::BI__mma_tf32_m16n16k8_ld_b:
21063 return MMA_LDST(4, m16n16k8_load_b_tf32);
21064 case NVPTX::BI__mma_tf32_m16n16k8_ld_c:
21065 return MMA_LDST(8, m16n16k8_load_c_f32);
21071 case NVPTX::BI__hmma_m16n16k16_st_c_f16:
21072 return MMA_LDST(4, m16n16k16_store_d_f16);
21073 case NVPTX::BI__hmma_m16n16k16_st_c_f32:
21074 return MMA_LDST(8, m16n16k16_store_d_f32);
21075 case NVPTX::BI__hmma_m32n8k16_st_c_f16:
21076 return MMA_LDST(4, m32n8k16_store_d_f16);
21077 case NVPTX::BI__hmma_m32n8k16_st_c_f32:
21078 return MMA_LDST(8, m32n8k16_store_d_f32);
21079 case NVPTX::BI__hmma_m8n32k16_st_c_f16:
21080 return MMA_LDST(4, m8n32k16_store_d_f16);
21081 case NVPTX::BI__hmma_m8n32k16_st_c_f32:
21082 return MMA_LDST(8, m8n32k16_store_d_f32);
21087 case NVPTX::BI__imma_m16n16k16_st_c_i32:
21088 return MMA_LDST(8, m16n16k16_store_d_s32);
21089 case NVPTX::BI__imma_m32n8k16_st_c_i32:
21090 return MMA_LDST(8, m32n8k16_store_d_s32);
21091 case NVPTX::BI__imma_m8n32k16_st_c_i32:
21092 return MMA_LDST(8, m8n32k16_store_d_s32);
21093 case NVPTX::BI__imma_m8n8k32_st_c_i32:
21094 return MMA_LDST(2, m8n8k32_store_d_s32);
21095 case NVPTX::BI__bmma_m8n8k128_st_c_i32:
21096 return MMA_LDST(2, m8n8k128_store_d_s32);
21099 case NVPTX::BI__dmma_m8n8k4_st_c_f64:
21100 return MMA_LDST(2, m8n8k4_store_d_f64);
21103 case NVPTX::BI__mma_m16n16k8_st_c_f32:
21104 return MMA_LDST(8, m16n16k8_store_d_f32);
21107 llvm_unreachable(
"Unknown MMA builtin");
21114struct NVPTXMmaInfo {
21123 std::array<unsigned, 8> Variants;
21125 unsigned getMMAIntrinsic(
int Layout,
bool Satf) {
21126 unsigned Index = Layout + 4 * Satf;
21127 if (Index >= Variants.size())
21129 return Variants[Index];
21135static NVPTXMmaInfo getNVPTXMmaInfo(
unsigned BuiltinID) {
21137#define MMA_VARIANTS(geom, type) \
21138 Intrinsic::nvvm_wmma_##geom##_mma_row_row_##type, \
21139 Intrinsic::nvvm_wmma_##geom##_mma_row_col_##type, \
21140 Intrinsic::nvvm_wmma_##geom##_mma_col_row_##type, \
21141 Intrinsic::nvvm_wmma_##geom##_mma_col_col_##type
21142#define MMA_SATF_VARIANTS(geom, type) \
21143 MMA_VARIANTS(geom, type), \
21144 Intrinsic::nvvm_wmma_##geom##_mma_row_row_##type##_satfinite, \
21145 Intrinsic::nvvm_wmma_##geom##_mma_row_col_##type##_satfinite, \
21146 Intrinsic::nvvm_wmma_##geom##_mma_col_row_##type##_satfinite, \
21147 Intrinsic::nvvm_wmma_##geom##_mma_col_col_##type##_satfinite
21149#define MMA_VARIANTS_I4(geom, type) \
21151 Intrinsic::nvvm_wmma_##geom##_mma_row_col_##type, \
21155 Intrinsic::nvvm_wmma_##geom##_mma_row_col_##type##_satfinite, \
21159#define MMA_VARIANTS_B1_XOR(geom, type) \
21161 Intrinsic::nvvm_wmma_##geom##_mma_xor_popc_row_col_##type, \
21168#define MMA_VARIANTS_B1_AND(geom, type) \
21170 Intrinsic::nvvm_wmma_##geom##_mma_and_popc_row_col_##type, \
21178 switch (BuiltinID) {
21182 case NVPTX::BI__hmma_m16n16k16_mma_f16f16:
21184 case NVPTX::BI__hmma_m16n16k16_mma_f32f16:
21186 case NVPTX::BI__hmma_m16n16k16_mma_f16f32:
21188 case NVPTX::BI__hmma_m16n16k16_mma_f32f32:
21190 case NVPTX::BI__hmma_m32n8k16_mma_f16f16:
21192 case NVPTX::BI__hmma_m32n8k16_mma_f32f16:
21194 case NVPTX::BI__hmma_m32n8k16_mma_f16f32:
21196 case NVPTX::BI__hmma_m32n8k16_mma_f32f32:
21198 case NVPTX::BI__hmma_m8n32k16_mma_f16f16:
21200 case NVPTX::BI__hmma_m8n32k16_mma_f32f16:
21202 case NVPTX::BI__hmma_m8n32k16_mma_f16f32:
21204 case NVPTX::BI__hmma_m8n32k16_mma_f32f32:
21208 case NVPTX::BI__imma_m16n16k16_mma_s8:
21210 case NVPTX::BI__imma_m16n16k16_mma_u8:
21212 case NVPTX::BI__imma_m32n8k16_mma_s8:
21214 case NVPTX::BI__imma_m32n8k16_mma_u8:
21216 case NVPTX::BI__imma_m8n32k16_mma_s8:
21218 case NVPTX::BI__imma_m8n32k16_mma_u8:
21222 case NVPTX::BI__imma_m8n8k32_mma_s4:
21224 case NVPTX::BI__imma_m8n8k32_mma_u4:
21226 case NVPTX::BI__bmma_m8n8k128_mma_xor_popc_b1:
21228 case NVPTX::BI__bmma_m8n8k128_mma_and_popc_b1:
21232 case NVPTX::BI__dmma_m8n8k4_mma_f64:
21236 case NVPTX::BI__mma_bf16_m16n16k16_mma_f32:
21237 return {4, 4, 8, 8, {{
MMA_VARIANTS(m16n16k16, bf16)}}};
21238 case NVPTX::BI__mma_bf16_m8n32k16_mma_f32:
21239 return {2, 8, 8, 8, {{
MMA_VARIANTS(m8n32k16, bf16)}}};
21240 case NVPTX::BI__mma_bf16_m32n8k16_mma_f32:
21241 return {8, 2, 8, 8, {{
MMA_VARIANTS(m32n8k16, bf16)}}};
21242 case NVPTX::BI__mma_tf32_m16n16k8_mma_f32:
21243 return {4, 4, 8, 8, {{
MMA_VARIANTS(m16n16k8, tf32)}}};
21245 llvm_unreachable(
"Unexpected builtin ID.");
21248#undef MMA_SATF_VARIANTS
21249#undef MMA_VARIANTS_I4
21250#undef MMA_VARIANTS_B1_AND
21251#undef MMA_VARIANTS_B1_XOR
21260 return CGF.
Builder.CreateCall(
21262 {Ptr, ConstantInt::get(CGF.Builder.getInt32Ty(), Align.getQuantity())});
21274 MDNode *MD = MDNode::get(CGF.
Builder.getContext(), {});
21275 LD->setMetadata(LLVMContext::MD_invariant_load, MD);
21283 llvm::Type *ElemTy =
21285 return CGF.
Builder.CreateCall(
21287 {Ptr, CGF.EmitScalarExpr(E->getArg(1))});
21290static Value *MakeCpAsync(
unsigned IntrinsicID,
unsigned IntrinsicIDS,
21293 return E->getNumArgs() == 3
21295 {CGF.EmitScalarExpr(E->getArg(0)),
21296 CGF.EmitScalarExpr(E->getArg(1)),
21297 CGF.EmitScalarExpr(E->getArg(2))})
21299 {CGF.EmitScalarExpr(E->getArg(0)),
21300 CGF.EmitScalarExpr(E->getArg(1))});
21303static Value *MakeHalfType(
unsigned IntrinsicID,
unsigned BuiltinID,
21306 if (!(
C.getLangOpts().NativeHalfType ||
21307 !
C.getTargetInfo().useFP16ConversionIntrinsics())) {
21309 " requires native half type support.");
21313 if (BuiltinID == NVPTX::BI__nvvm_ldg_h || BuiltinID == NVPTX::BI__nvvm_ldg_h2)
21314 return MakeLdg(CGF,
E);
21316 if (IntrinsicID == Intrinsic::nvvm_ldu_global_f)
21317 return MakeLdu(IntrinsicID, CGF,
E);
21321 auto *FTy = F->getFunctionType();
21322 unsigned ICEArguments = 0;
21324 C.GetBuiltinType(BuiltinID, Error, &ICEArguments);
21326 for (
unsigned i = 0, e =
E->getNumArgs(); i != e; ++i) {
21327 assert((ICEArguments & (1 << i)) == 0);
21329 auto *PTy = FTy->getParamType(i);
21330 if (PTy != ArgValue->
getType())
21331 ArgValue = CGF.
Builder.CreateBitCast(ArgValue, PTy);
21332 Args.push_back(ArgValue);
21335 return CGF.
Builder.CreateCall(F, Args);
21341 switch (BuiltinID) {
21342 case NVPTX::BI__nvvm_atom_add_gen_i:
21343 case NVPTX::BI__nvvm_atom_add_gen_l:
21344 case NVPTX::BI__nvvm_atom_add_gen_ll:
21347 case NVPTX::BI__nvvm_atom_sub_gen_i:
21348 case NVPTX::BI__nvvm_atom_sub_gen_l:
21349 case NVPTX::BI__nvvm_atom_sub_gen_ll:
21352 case NVPTX::BI__nvvm_atom_and_gen_i:
21353 case NVPTX::BI__nvvm_atom_and_gen_l:
21354 case NVPTX::BI__nvvm_atom_and_gen_ll:
21357 case NVPTX::BI__nvvm_atom_or_gen_i:
21358 case NVPTX::BI__nvvm_atom_or_gen_l:
21359 case NVPTX::BI__nvvm_atom_or_gen_ll:
21362 case NVPTX::BI__nvvm_atom_xor_gen_i:
21363 case NVPTX::BI__nvvm_atom_xor_gen_l:
21364 case NVPTX::BI__nvvm_atom_xor_gen_ll:
21367 case NVPTX::BI__nvvm_atom_xchg_gen_i:
21368 case NVPTX::BI__nvvm_atom_xchg_gen_l:
21369 case NVPTX::BI__nvvm_atom_xchg_gen_ll:
21372 case NVPTX::BI__nvvm_atom_max_gen_i:
21373 case NVPTX::BI__nvvm_atom_max_gen_l:
21374 case NVPTX::BI__nvvm_atom_max_gen_ll:
21377 case NVPTX::BI__nvvm_atom_max_gen_ui:
21378 case NVPTX::BI__nvvm_atom_max_gen_ul:
21379 case NVPTX::BI__nvvm_atom_max_gen_ull:
21382 case NVPTX::BI__nvvm_atom_min_gen_i:
21383 case NVPTX::BI__nvvm_atom_min_gen_l:
21384 case NVPTX::BI__nvvm_atom_min_gen_ll:
21387 case NVPTX::BI__nvvm_atom_min_gen_ui:
21388 case NVPTX::BI__nvvm_atom_min_gen_ul:
21389 case NVPTX::BI__nvvm_atom_min_gen_ull:
21392 case NVPTX::BI__nvvm_atom_cas_gen_us:
21393 case NVPTX::BI__nvvm_atom_cas_gen_i:
21394 case NVPTX::BI__nvvm_atom_cas_gen_l:
21395 case NVPTX::BI__nvvm_atom_cas_gen_ll:
21400 case NVPTX::BI__nvvm_atom_add_gen_f:
21401 case NVPTX::BI__nvvm_atom_add_gen_d: {
21406 AtomicOrdering::SequentiallyConsistent);
21409 case NVPTX::BI__nvvm_atom_inc_gen_ui: {
21414 return Builder.CreateCall(FnALI32, {Ptr, Val});
21417 case NVPTX::BI__nvvm_atom_dec_gen_ui: {
21422 return Builder.CreateCall(FnALD32, {Ptr, Val});
21425 case NVPTX::BI__nvvm_ldg_c:
21426 case NVPTX::BI__nvvm_ldg_sc:
21427 case NVPTX::BI__nvvm_ldg_c2:
21428 case NVPTX::BI__nvvm_ldg_sc2:
21429 case NVPTX::BI__nvvm_ldg_c4:
21430 case NVPTX::BI__nvvm_ldg_sc4:
21431 case NVPTX::BI__nvvm_ldg_s:
21432 case NVPTX::BI__nvvm_ldg_s2:
21433 case NVPTX::BI__nvvm_ldg_s4:
21434 case NVPTX::BI__nvvm_ldg_i:
21435 case NVPTX::BI__nvvm_ldg_i2:
21436 case NVPTX::BI__nvvm_ldg_i4:
21437 case NVPTX::BI__nvvm_ldg_l:
21438 case NVPTX::BI__nvvm_ldg_l2:
21439 case NVPTX::BI__nvvm_ldg_ll:
21440 case NVPTX::BI__nvvm_ldg_ll2:
21441 case NVPTX::BI__nvvm_ldg_uc:
21442 case NVPTX::BI__nvvm_ldg_uc2:
21443 case NVPTX::BI__nvvm_ldg_uc4:
21444 case NVPTX::BI__nvvm_ldg_us:
21445 case NVPTX::BI__nvvm_ldg_us2:
21446 case NVPTX::BI__nvvm_ldg_us4:
21447 case NVPTX::BI__nvvm_ldg_ui:
21448 case NVPTX::BI__nvvm_ldg_ui2:
21449 case NVPTX::BI__nvvm_ldg_ui4:
21450 case NVPTX::BI__nvvm_ldg_ul:
21451 case NVPTX::BI__nvvm_ldg_ul2:
21452 case NVPTX::BI__nvvm_ldg_ull:
21453 case NVPTX::BI__nvvm_ldg_ull2:
21454 case NVPTX::BI__nvvm_ldg_f:
21455 case NVPTX::BI__nvvm_ldg_f2:
21456 case NVPTX::BI__nvvm_ldg_f4:
21457 case NVPTX::BI__nvvm_ldg_d:
21458 case NVPTX::BI__nvvm_ldg_d2:
21462 return MakeLdg(*
this,
E);
21464 case NVPTX::BI__nvvm_ldu_c:
21465 case NVPTX::BI__nvvm_ldu_sc:
21466 case NVPTX::BI__nvvm_ldu_c2:
21467 case NVPTX::BI__nvvm_ldu_sc2:
21468 case NVPTX::BI__nvvm_ldu_c4:
21469 case NVPTX::BI__nvvm_ldu_sc4:
21470 case NVPTX::BI__nvvm_ldu_s:
21471 case NVPTX::BI__nvvm_ldu_s2:
21472 case NVPTX::BI__nvvm_ldu_s4:
21473 case NVPTX::BI__nvvm_ldu_i:
21474 case NVPTX::BI__nvvm_ldu_i2:
21475 case NVPTX::BI__nvvm_ldu_i4:
21476 case NVPTX::BI__nvvm_ldu_l:
21477 case NVPTX::BI__nvvm_ldu_l2:
21478 case NVPTX::BI__nvvm_ldu_ll:
21479 case NVPTX::BI__nvvm_ldu_ll2:
21480 case NVPTX::BI__nvvm_ldu_uc:
21481 case NVPTX::BI__nvvm_ldu_uc2:
21482 case NVPTX::BI__nvvm_ldu_uc4:
21483 case NVPTX::BI__nvvm_ldu_us:
21484 case NVPTX::BI__nvvm_ldu_us2:
21485 case NVPTX::BI__nvvm_ldu_us4:
21486 case NVPTX::BI__nvvm_ldu_ui:
21487 case NVPTX::BI__nvvm_ldu_ui2:
21488 case NVPTX::BI__nvvm_ldu_ui4:
21489 case NVPTX::BI__nvvm_ldu_ul:
21490 case NVPTX::BI__nvvm_ldu_ul2:
21491 case NVPTX::BI__nvvm_ldu_ull:
21492 case NVPTX::BI__nvvm_ldu_ull2:
21493 return MakeLdu(Intrinsic::nvvm_ldu_global_i, *
this,
E);
21494 case NVPTX::BI__nvvm_ldu_f:
21495 case NVPTX::BI__nvvm_ldu_f2:
21496 case NVPTX::BI__nvvm_ldu_f4:
21497 case NVPTX::BI__nvvm_ldu_d:
21498 case NVPTX::BI__nvvm_ldu_d2:
21499 return MakeLdu(Intrinsic::nvvm_ldu_global_f, *
this,
E);
21501 case NVPTX::BI__nvvm_atom_cta_add_gen_i:
21502 case NVPTX::BI__nvvm_atom_cta_add_gen_l:
21503 case NVPTX::BI__nvvm_atom_cta_add_gen_ll:
21504 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_i_cta, *
this,
E);
21505 case NVPTX::BI__nvvm_atom_sys_add_gen_i:
21506 case NVPTX::BI__nvvm_atom_sys_add_gen_l:
21507 case NVPTX::BI__nvvm_atom_sys_add_gen_ll:
21508 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_i_sys, *
this,
E);
21509 case NVPTX::BI__nvvm_atom_cta_add_gen_f:
21510 case NVPTX::BI__nvvm_atom_cta_add_gen_d:
21511 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_f_cta, *
this,
E);
21512 case NVPTX::BI__nvvm_atom_sys_add_gen_f:
21513 case NVPTX::BI__nvvm_atom_sys_add_gen_d:
21514 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_f_sys, *
this,
E);
21515 case NVPTX::BI__nvvm_atom_cta_xchg_gen_i:
21516 case NVPTX::BI__nvvm_atom_cta_xchg_gen_l:
21517 case NVPTX::BI__nvvm_atom_cta_xchg_gen_ll:
21518 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_gen_i_cta, *
this,
E);
21519 case NVPTX::BI__nvvm_atom_sys_xchg_gen_i:
21520 case NVPTX::BI__nvvm_atom_sys_xchg_gen_l:
21521 case NVPTX::BI__nvvm_atom_sys_xchg_gen_ll:
21522 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_gen_i_sys, *
this,
E);
21523 case NVPTX::BI__nvvm_atom_cta_max_gen_i:
21524 case NVPTX::BI__nvvm_atom_cta_max_gen_ui:
21525 case NVPTX::BI__nvvm_atom_cta_max_gen_l:
21526 case NVPTX::BI__nvvm_atom_cta_max_gen_ul:
21527 case NVPTX::BI__nvvm_atom_cta_max_gen_ll:
21528 case NVPTX::BI__nvvm_atom_cta_max_gen_ull:
21529 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_gen_i_cta, *
this,
E);
21530 case NVPTX::BI__nvvm_atom_sys_max_gen_i:
21531 case NVPTX::BI__nvvm_atom_sys_max_gen_ui:
21532 case NVPTX::BI__nvvm_atom_sys_max_gen_l:
21533 case NVPTX::BI__nvvm_atom_sys_max_gen_ul:
21534 case NVPTX::BI__nvvm_atom_sys_max_gen_ll:
21535 case NVPTX::BI__nvvm_atom_sys_max_gen_ull:
21536 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_gen_i_sys, *
this,
E);
21537 case NVPTX::BI__nvvm_atom_cta_min_gen_i:
21538 case NVPTX::BI__nvvm_atom_cta_min_gen_ui:
21539 case NVPTX::BI__nvvm_atom_cta_min_gen_l:
21540 case NVPTX::BI__nvvm_atom_cta_min_gen_ul:
21541 case NVPTX::BI__nvvm_atom_cta_min_gen_ll:
21542 case NVPTX::BI__nvvm_atom_cta_min_gen_ull:
21543 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_gen_i_cta, *
this,
E);
21544 case NVPTX::BI__nvvm_atom_sys_min_gen_i:
21545 case NVPTX::BI__nvvm_atom_sys_min_gen_ui:
21546 case NVPTX::BI__nvvm_atom_sys_min_gen_l:
21547 case NVPTX::BI__nvvm_atom_sys_min_gen_ul:
21548 case NVPTX::BI__nvvm_atom_sys_min_gen_ll:
21549 case NVPTX::BI__nvvm_atom_sys_min_gen_ull:
21550 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_gen_i_sys, *
this,
E);
21551 case NVPTX::BI__nvvm_atom_cta_inc_gen_ui:
21552 return MakeScopedAtomic(Intrinsic::nvvm_atomic_inc_gen_i_cta, *
this,
E);
21553 case NVPTX::BI__nvvm_atom_cta_dec_gen_ui:
21554 return MakeScopedAtomic(Intrinsic::nvvm_atomic_dec_gen_i_cta, *
this,
E);
21555 case NVPTX::BI__nvvm_atom_sys_inc_gen_ui:
21556 return MakeScopedAtomic(Intrinsic::nvvm_atomic_inc_gen_i_sys, *
this,
E);
21557 case NVPTX::BI__nvvm_atom_sys_dec_gen_ui:
21558 return MakeScopedAtomic(Intrinsic::nvvm_atomic_dec_gen_i_sys, *
this,
E);
21559 case NVPTX::BI__nvvm_atom_cta_and_gen_i:
21560 case NVPTX::BI__nvvm_atom_cta_and_gen_l:
21561 case NVPTX::BI__nvvm_atom_cta_and_gen_ll:
21562 return MakeScopedAtomic(Intrinsic::nvvm_atomic_and_gen_i_cta, *
this,
E);
21563 case NVPTX::BI__nvvm_atom_sys_and_gen_i:
21564 case NVPTX::BI__nvvm_atom_sys_and_gen_l:
21565 case NVPTX::BI__nvvm_atom_sys_and_gen_ll:
21566 return MakeScopedAtomic(Intrinsic::nvvm_atomic_and_gen_i_sys, *
this,
E);
21567 case NVPTX::BI__nvvm_atom_cta_or_gen_i:
21568 case NVPTX::BI__nvvm_atom_cta_or_gen_l:
21569 case NVPTX::BI__nvvm_atom_cta_or_gen_ll:
21570 return MakeScopedAtomic(Intrinsic::nvvm_atomic_or_gen_i_cta, *
this,
E);
21571 case NVPTX::BI__nvvm_atom_sys_or_gen_i:
21572 case NVPTX::BI__nvvm_atom_sys_or_gen_l:
21573 case NVPTX::BI__nvvm_atom_sys_or_gen_ll:
21574 return MakeScopedAtomic(Intrinsic::nvvm_atomic_or_gen_i_sys, *
this,
E);
21575 case NVPTX::BI__nvvm_atom_cta_xor_gen_i:
21576 case NVPTX::BI__nvvm_atom_cta_xor_gen_l:
21577 case NVPTX::BI__nvvm_atom_cta_xor_gen_ll:
21578 return MakeScopedAtomic(Intrinsic::nvvm_atomic_xor_gen_i_cta, *
this,
E);
21579 case NVPTX::BI__nvvm_atom_sys_xor_gen_i:
21580 case NVPTX::BI__nvvm_atom_sys_xor_gen_l:
21581 case NVPTX::BI__nvvm_atom_sys_xor_gen_ll:
21582 return MakeScopedAtomic(Intrinsic::nvvm_atomic_xor_gen_i_sys, *
this,
E);
21583 case NVPTX::BI__nvvm_atom_cta_cas_gen_us:
21584 case NVPTX::BI__nvvm_atom_cta_cas_gen_i:
21585 case NVPTX::BI__nvvm_atom_cta_cas_gen_l:
21586 case NVPTX::BI__nvvm_atom_cta_cas_gen_ll: {
21588 llvm::Type *ElemTy =
21592 Intrinsic::nvvm_atomic_cas_gen_i_cta, {ElemTy, Ptr->getType()}),
21593 {Ptr, EmitScalarExpr(E->getArg(1)), EmitScalarExpr(E->getArg(2))});
21595 case NVPTX::BI__nvvm_atom_sys_cas_gen_us:
21596 case NVPTX::BI__nvvm_atom_sys_cas_gen_i:
21597 case NVPTX::BI__nvvm_atom_sys_cas_gen_l:
21598 case NVPTX::BI__nvvm_atom_sys_cas_gen_ll: {
21600 llvm::Type *ElemTy =
21604 Intrinsic::nvvm_atomic_cas_gen_i_sys, {ElemTy, Ptr->getType()}),
21605 {Ptr, EmitScalarExpr(E->getArg(1)), EmitScalarExpr(E->getArg(2))});
21607 case NVPTX::BI__nvvm_match_all_sync_i32p:
21608 case NVPTX::BI__nvvm_match_all_sync_i64p: {
21614 ? Intrinsic::nvvm_match_all_sync_i32p
21615 : Intrinsic::nvvm_match_all_sync_i64p),
21620 return Builder.CreateExtractValue(ResultPair, 0);
21624 case NVPTX::BI__hmma_m16n16k16_ld_a:
21625 case NVPTX::BI__hmma_m16n16k16_ld_b:
21626 case NVPTX::BI__hmma_m16n16k16_ld_c_f16:
21627 case NVPTX::BI__hmma_m16n16k16_ld_c_f32:
21628 case NVPTX::BI__hmma_m32n8k16_ld_a:
21629 case NVPTX::BI__hmma_m32n8k16_ld_b:
21630 case NVPTX::BI__hmma_m32n8k16_ld_c_f16:
21631 case NVPTX::BI__hmma_m32n8k16_ld_c_f32:
21632 case NVPTX::BI__hmma_m8n32k16_ld_a:
21633 case NVPTX::BI__hmma_m8n32k16_ld_b:
21634 case NVPTX::BI__hmma_m8n32k16_ld_c_f16:
21635 case NVPTX::BI__hmma_m8n32k16_ld_c_f32:
21637 case NVPTX::BI__imma_m16n16k16_ld_a_s8:
21638 case NVPTX::BI__imma_m16n16k16_ld_a_u8:
21639 case NVPTX::BI__imma_m16n16k16_ld_b_s8:
21640 case NVPTX::BI__imma_m16n16k16_ld_b_u8:
21641 case NVPTX::BI__imma_m16n16k16_ld_c:
21642 case NVPTX::BI__imma_m32n8k16_ld_a_s8:
21643 case NVPTX::BI__imma_m32n8k16_ld_a_u8:
21644 case NVPTX::BI__imma_m32n8k16_ld_b_s8:
21645 case NVPTX::BI__imma_m32n8k16_ld_b_u8:
21646 case NVPTX::BI__imma_m32n8k16_ld_c:
21647 case NVPTX::BI__imma_m8n32k16_ld_a_s8:
21648 case NVPTX::BI__imma_m8n32k16_ld_a_u8:
21649 case NVPTX::BI__imma_m8n32k16_ld_b_s8:
21650 case NVPTX::BI__imma_m8n32k16_ld_b_u8:
21651 case NVPTX::BI__imma_m8n32k16_ld_c:
21653 case NVPTX::BI__imma_m8n8k32_ld_a_s4:
21654 case NVPTX::BI__imma_m8n8k32_ld_a_u4:
21655 case NVPTX::BI__imma_m8n8k32_ld_b_s4:
21656 case NVPTX::BI__imma_m8n8k32_ld_b_u4:
21657 case NVPTX::BI__imma_m8n8k32_ld_c:
21658 case NVPTX::BI__bmma_m8n8k128_ld_a_b1:
21659 case NVPTX::BI__bmma_m8n8k128_ld_b_b1:
21660 case NVPTX::BI__bmma_m8n8k128_ld_c:
21662 case NVPTX::BI__dmma_m8n8k4_ld_a:
21663 case NVPTX::BI__dmma_m8n8k4_ld_b:
21664 case NVPTX::BI__dmma_m8n8k4_ld_c:
21666 case NVPTX::BI__mma_bf16_m16n16k16_ld_a:
21667 case NVPTX::BI__mma_bf16_m16n16k16_ld_b:
21668 case NVPTX::BI__mma_bf16_m8n32k16_ld_a:
21669 case NVPTX::BI__mma_bf16_m8n32k16_ld_b:
21670 case NVPTX::BI__mma_bf16_m32n8k16_ld_a:
21671 case NVPTX::BI__mma_bf16_m32n8k16_ld_b:
21672 case NVPTX::BI__mma_tf32_m16n16k8_ld_a:
21673 case NVPTX::BI__mma_tf32_m16n16k8_ld_b:
21674 case NVPTX::BI__mma_tf32_m16n16k8_ld_c: {
21678 std::optional<llvm::APSInt> isColMajorArg =
21680 if (!isColMajorArg)
21682 bool isColMajor = isColMajorArg->getSExtValue();
21683 NVPTXMmaLdstInfo II = getNVPTXMmaLdstInfo(BuiltinID);
21684 unsigned IID = isColMajor ? II.IID_col : II.IID_row;
21692 assert(II.NumResults);
21693 if (II.NumResults == 1) {
21697 for (
unsigned i = 0; i < II.NumResults; ++i) {
21702 llvm::ConstantInt::get(
IntTy, i)),
21709 case NVPTX::BI__hmma_m16n16k16_st_c_f16:
21710 case NVPTX::BI__hmma_m16n16k16_st_c_f32:
21711 case NVPTX::BI__hmma_m32n8k16_st_c_f16:
21712 case NVPTX::BI__hmma_m32n8k16_st_c_f32:
21713 case NVPTX::BI__hmma_m8n32k16_st_c_f16:
21714 case NVPTX::BI__hmma_m8n32k16_st_c_f32:
21715 case NVPTX::BI__imma_m16n16k16_st_c_i32:
21716 case NVPTX::BI__imma_m32n8k16_st_c_i32:
21717 case NVPTX::BI__imma_m8n32k16_st_c_i32:
21718 case NVPTX::BI__imma_m8n8k32_st_c_i32:
21719 case NVPTX::BI__bmma_m8n8k128_st_c_i32:
21720 case NVPTX::BI__dmma_m8n8k4_st_c_f64:
21721 case NVPTX::BI__mma_m16n16k8_st_c_f32: {
21725 std::optional<llvm::APSInt> isColMajorArg =
21727 if (!isColMajorArg)
21729 bool isColMajor = isColMajorArg->getSExtValue();
21730 NVPTXMmaLdstInfo II = getNVPTXMmaLdstInfo(BuiltinID);
21731 unsigned IID = isColMajor ? II.IID_col : II.IID_row;
21736 llvm::Type *ParamType = Intrinsic->getFunctionType()->getParamType(1);
21738 for (
unsigned i = 0; i < II.NumResults; ++i) {
21742 llvm::ConstantInt::get(
IntTy, i)),
21744 Values.push_back(
Builder.CreateBitCast(
V, ParamType));
21746 Values.push_back(Ldm);
21753 case NVPTX::BI__hmma_m16n16k16_mma_f16f16:
21754 case NVPTX::BI__hmma_m16n16k16_mma_f32f16:
21755 case NVPTX::BI__hmma_m16n16k16_mma_f32f32:
21756 case NVPTX::BI__hmma_m16n16k16_mma_f16f32:
21757 case NVPTX::BI__hmma_m32n8k16_mma_f16f16:
21758 case NVPTX::BI__hmma_m32n8k16_mma_f32f16:
21759 case NVPTX::BI__hmma_m32n8k16_mma_f32f32:
21760 case NVPTX::BI__hmma_m32n8k16_mma_f16f32:
21761 case NVPTX::BI__hmma_m8n32k16_mma_f16f16:
21762 case NVPTX::BI__hmma_m8n32k16_mma_f32f16:
21763 case NVPTX::BI__hmma_m8n32k16_mma_f32f32:
21764 case NVPTX::BI__hmma_m8n32k16_mma_f16f32:
21765 case NVPTX::BI__imma_m16n16k16_mma_s8:
21766 case NVPTX::BI__imma_m16n16k16_mma_u8:
21767 case NVPTX::BI__imma_m32n8k16_mma_s8:
21768 case NVPTX::BI__imma_m32n8k16_mma_u8:
21769 case NVPTX::BI__imma_m8n32k16_mma_s8:
21770 case NVPTX::BI__imma_m8n32k16_mma_u8:
21771 case NVPTX::BI__imma_m8n8k32_mma_s4:
21772 case NVPTX::BI__imma_m8n8k32_mma_u4:
21773 case NVPTX::BI__bmma_m8n8k128_mma_xor_popc_b1:
21774 case NVPTX::BI__bmma_m8n8k128_mma_and_popc_b1:
21775 case NVPTX::BI__dmma_m8n8k4_mma_f64:
21776 case NVPTX::BI__mma_bf16_m16n16k16_mma_f32:
21777 case NVPTX::BI__mma_bf16_m8n32k16_mma_f32:
21778 case NVPTX::BI__mma_bf16_m32n8k16_mma_f32:
21779 case NVPTX::BI__mma_tf32_m16n16k8_mma_f32: {
21784 std::optional<llvm::APSInt> LayoutArg =
21788 int Layout = LayoutArg->getSExtValue();
21789 if (Layout < 0 || Layout > 3)
21791 llvm::APSInt SatfArg;
21792 if (BuiltinID == NVPTX::BI__bmma_m8n8k128_mma_xor_popc_b1 ||
21793 BuiltinID == NVPTX::BI__bmma_m8n8k128_mma_and_popc_b1)
21795 else if (std::optional<llvm::APSInt> OptSatfArg =
21797 SatfArg = *OptSatfArg;
21800 bool Satf = SatfArg.getSExtValue();
21801 NVPTXMmaInfo MI = getNVPTXMmaInfo(BuiltinID);
21802 unsigned IID = MI.getMMAIntrinsic(Layout, Satf);
21808 llvm::Type *AType = Intrinsic->getFunctionType()->getParamType(0);
21810 for (
unsigned i = 0; i < MI.NumEltsA; ++i) {
21814 llvm::ConstantInt::get(
IntTy, i)),
21816 Values.push_back(
Builder.CreateBitCast(
V, AType));
21819 llvm::Type *BType = Intrinsic->getFunctionType()->getParamType(MI.NumEltsA);
21820 for (
unsigned i = 0; i < MI.NumEltsB; ++i) {
21824 llvm::ConstantInt::get(
IntTy, i)),
21826 Values.push_back(
Builder.CreateBitCast(
V, BType));
21829 llvm::Type *CType =
21830 Intrinsic->getFunctionType()->getParamType(MI.NumEltsA + MI.NumEltsB);
21831 for (
unsigned i = 0; i < MI.NumEltsC; ++i) {
21835 llvm::ConstantInt::get(
IntTy, i)),
21837 Values.push_back(
Builder.CreateBitCast(
V, CType));
21841 for (
unsigned i = 0; i < MI.NumEltsD; ++i)
21845 llvm::ConstantInt::get(
IntTy, i)),
21850 case NVPTX::BI__nvvm_ex2_approx_f16:
21851 return MakeHalfType(Intrinsic::nvvm_ex2_approx_f16, BuiltinID,
E, *
this);
21852 case NVPTX::BI__nvvm_ex2_approx_f16x2:
21853 return MakeHalfType(Intrinsic::nvvm_ex2_approx_f16x2, BuiltinID,
E, *
this);
21854 case NVPTX::BI__nvvm_ff2f16x2_rn:
21855 return MakeHalfType(Intrinsic::nvvm_ff2f16x2_rn, BuiltinID,
E, *
this);
21856 case NVPTX::BI__nvvm_ff2f16x2_rn_relu:
21857 return MakeHalfType(Intrinsic::nvvm_ff2f16x2_rn_relu, BuiltinID,
E, *
this);
21858 case NVPTX::BI__nvvm_ff2f16x2_rz:
21859 return MakeHalfType(Intrinsic::nvvm_ff2f16x2_rz, BuiltinID,
E, *
this);
21860 case NVPTX::BI__nvvm_ff2f16x2_rz_relu:
21861 return MakeHalfType(Intrinsic::nvvm_ff2f16x2_rz_relu, BuiltinID,
E, *
this);
21862 case NVPTX::BI__nvvm_fma_rn_f16:
21863 return MakeHalfType(Intrinsic::nvvm_fma_rn_f16, BuiltinID,
E, *
this);
21864 case NVPTX::BI__nvvm_fma_rn_f16x2:
21865 return MakeHalfType(Intrinsic::nvvm_fma_rn_f16x2, BuiltinID,
E, *
this);
21866 case NVPTX::BI__nvvm_fma_rn_ftz_f16:
21867 return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_f16, BuiltinID,
E, *
this);
21868 case NVPTX::BI__nvvm_fma_rn_ftz_f16x2:
21869 return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_f16x2, BuiltinID,
E, *
this);
21870 case NVPTX::BI__nvvm_fma_rn_ftz_relu_f16:
21871 return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_relu_f16, BuiltinID,
E,
21873 case NVPTX::BI__nvvm_fma_rn_ftz_relu_f16x2:
21874 return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_relu_f16x2, BuiltinID,
E,
21876 case NVPTX::BI__nvvm_fma_rn_ftz_sat_f16:
21877 return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_sat_f16, BuiltinID,
E,
21879 case NVPTX::BI__nvvm_fma_rn_ftz_sat_f16x2:
21880 return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_sat_f16x2, BuiltinID,
E,
21882 case NVPTX::BI__nvvm_fma_rn_relu_f16:
21883 return MakeHalfType(Intrinsic::nvvm_fma_rn_relu_f16, BuiltinID,
E, *
this);
21884 case NVPTX::BI__nvvm_fma_rn_relu_f16x2:
21885 return MakeHalfType(Intrinsic::nvvm_fma_rn_relu_f16x2, BuiltinID,
E, *
this);
21886 case NVPTX::BI__nvvm_fma_rn_sat_f16:
21887 return MakeHalfType(Intrinsic::nvvm_fma_rn_sat_f16, BuiltinID,
E, *
this);
21888 case NVPTX::BI__nvvm_fma_rn_sat_f16x2:
21889 return MakeHalfType(Intrinsic::nvvm_fma_rn_sat_f16x2, BuiltinID,
E, *
this);
21890 case NVPTX::BI__nvvm_fmax_f16:
21891 return MakeHalfType(Intrinsic::nvvm_fmax_f16, BuiltinID,
E, *
this);
21892 case NVPTX::BI__nvvm_fmax_f16x2:
21893 return MakeHalfType(Intrinsic::nvvm_fmax_f16x2, BuiltinID,
E, *
this);
21894 case NVPTX::BI__nvvm_fmax_ftz_f16:
21895 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_f16, BuiltinID,
E, *
this);
21896 case NVPTX::BI__nvvm_fmax_ftz_f16x2:
21897 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_f16x2, BuiltinID,
E, *
this);
21898 case NVPTX::BI__nvvm_fmax_ftz_nan_f16:
21899 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_nan_f16, BuiltinID,
E, *
this);
21900 case NVPTX::BI__nvvm_fmax_ftz_nan_f16x2:
21901 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_nan_f16x2, BuiltinID,
E,
21903 case NVPTX::BI__nvvm_fmax_ftz_nan_xorsign_abs_f16:
21904 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_f16, BuiltinID,
21906 case NVPTX::BI__nvvm_fmax_ftz_nan_xorsign_abs_f16x2:
21907 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_f16x2,
21908 BuiltinID,
E, *
this);
21909 case NVPTX::BI__nvvm_fmax_ftz_xorsign_abs_f16:
21910 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_xorsign_abs_f16, BuiltinID,
E,
21912 case NVPTX::BI__nvvm_fmax_ftz_xorsign_abs_f16x2:
21913 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_xorsign_abs_f16x2, BuiltinID,
21915 case NVPTX::BI__nvvm_fmax_nan_f16:
21916 return MakeHalfType(Intrinsic::nvvm_fmax_nan_f16, BuiltinID,
E, *
this);
21917 case NVPTX::BI__nvvm_fmax_nan_f16x2:
21918 return MakeHalfType(Intrinsic::nvvm_fmax_nan_f16x2, BuiltinID,
E, *
this);
21919 case NVPTX::BI__nvvm_fmax_nan_xorsign_abs_f16:
21920 return MakeHalfType(Intrinsic::nvvm_fmax_nan_xorsign_abs_f16, BuiltinID,
E,
21922 case NVPTX::BI__nvvm_fmax_nan_xorsign_abs_f16x2:
21923 return MakeHalfType(Intrinsic::nvvm_fmax_nan_xorsign_abs_f16x2, BuiltinID,
21925 case NVPTX::BI__nvvm_fmax_xorsign_abs_f16:
21926 return MakeHalfType(Intrinsic::nvvm_fmax_xorsign_abs_f16, BuiltinID,
E,
21928 case NVPTX::BI__nvvm_fmax_xorsign_abs_f16x2:
21929 return MakeHalfType(Intrinsic::nvvm_fmax_xorsign_abs_f16x2, BuiltinID,
E,
21931 case NVPTX::BI__nvvm_fmin_f16:
21932 return MakeHalfType(Intrinsic::nvvm_fmin_f16, BuiltinID,
E, *
this);
21933 case NVPTX::BI__nvvm_fmin_f16x2:
21934 return MakeHalfType(Intrinsic::nvvm_fmin_f16x2, BuiltinID,
E, *
this);
21935 case NVPTX::BI__nvvm_fmin_ftz_f16:
21936 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_f16, BuiltinID,
E, *
this);
21937 case NVPTX::BI__nvvm_fmin_ftz_f16x2:
21938 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_f16x2, BuiltinID,
E, *
this);
21939 case NVPTX::BI__nvvm_fmin_ftz_nan_f16:
21940 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_nan_f16, BuiltinID,
E, *
this);
21941 case NVPTX::BI__nvvm_fmin_ftz_nan_f16x2:
21942 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_nan_f16x2, BuiltinID,
E,
21944 case NVPTX::BI__nvvm_fmin_ftz_nan_xorsign_abs_f16:
21945 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_f16, BuiltinID,
21947 case NVPTX::BI__nvvm_fmin_ftz_nan_xorsign_abs_f16x2:
21948 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_f16x2,
21949 BuiltinID,
E, *
this);
21950 case NVPTX::BI__nvvm_fmin_ftz_xorsign_abs_f16:
21951 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_xorsign_abs_f16, BuiltinID,
E,
21953 case NVPTX::BI__nvvm_fmin_ftz_xorsign_abs_f16x2:
21954 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_xorsign_abs_f16x2, BuiltinID,
21956 case NVPTX::BI__nvvm_fmin_nan_f16:
21957 return MakeHalfType(Intrinsic::nvvm_fmin_nan_f16, BuiltinID,
E, *
this);
21958 case NVPTX::BI__nvvm_fmin_nan_f16x2:
21959 return MakeHalfType(Intrinsic::nvvm_fmin_nan_f16x2, BuiltinID,
E, *
this);
21960 case NVPTX::BI__nvvm_fmin_nan_xorsign_abs_f16:
21961 return MakeHalfType(Intrinsic::nvvm_fmin_nan_xorsign_abs_f16, BuiltinID,
E,
21963 case NVPTX::BI__nvvm_fmin_nan_xorsign_abs_f16x2:
21964 return MakeHalfType(Intrinsic::nvvm_fmin_nan_xorsign_abs_f16x2, BuiltinID,
21966 case NVPTX::BI__nvvm_fmin_xorsign_abs_f16:
21967 return MakeHalfType(Intrinsic::nvvm_fmin_xorsign_abs_f16, BuiltinID,
E,
21969 case NVPTX::BI__nvvm_fmin_xorsign_abs_f16x2:
21970 return MakeHalfType(Intrinsic::nvvm_fmin_xorsign_abs_f16x2, BuiltinID,
E,
21972 case NVPTX::BI__nvvm_ldg_h:
21973 case NVPTX::BI__nvvm_ldg_h2:
21974 return MakeHalfType(Intrinsic::not_intrinsic, BuiltinID,
E, *
this);
21975 case NVPTX::BI__nvvm_ldu_h:
21976 case NVPTX::BI__nvvm_ldu_h2:
21977 return MakeHalfType(Intrinsic::nvvm_ldu_global_f, BuiltinID,
E, *
this);
21978 case NVPTX::BI__nvvm_cp_async_ca_shared_global_4:
21979 return MakeCpAsync(Intrinsic::nvvm_cp_async_ca_shared_global_4,
21980 Intrinsic::nvvm_cp_async_ca_shared_global_4_s, *
this,
E,
21982 case NVPTX::BI__nvvm_cp_async_ca_shared_global_8:
21983 return MakeCpAsync(Intrinsic::nvvm_cp_async_ca_shared_global_8,
21984 Intrinsic::nvvm_cp_async_ca_shared_global_8_s, *
this,
E,
21986 case NVPTX::BI__nvvm_cp_async_ca_shared_global_16:
21987 return MakeCpAsync(Intrinsic::nvvm_cp_async_ca_shared_global_16,
21988 Intrinsic::nvvm_cp_async_ca_shared_global_16_s, *
this,
E,
21990 case NVPTX::BI__nvvm_cp_async_cg_shared_global_16:
21991 return MakeCpAsync(Intrinsic::nvvm_cp_async_cg_shared_global_16,
21992 Intrinsic::nvvm_cp_async_cg_shared_global_16_s, *
this,
E,
21994 case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_x:
21997 case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_y:
22000 case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_z:
22003 case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_w:
22006 case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_x:
22009 case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_y:
22012 case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_z:
22015 case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_w:
22018 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_x:
22021 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_y:
22024 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_z:
22027 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_w:
22030 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_x:
22033 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_y:
22036 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_z:
22039 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_w:
22042 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctarank:
22045 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctarank:
22048 case NVPTX::BI__nvvm_is_explicit_cluster:
22051 case NVPTX::BI__nvvm_isspacep_shared_cluster:
22055 case NVPTX::BI__nvvm_mapa:
22058 {EmitScalarExpr(E->getArg(0)), EmitScalarExpr(E->getArg(1))});
22059 case NVPTX::BI__nvvm_mapa_shared_cluster:
22062 {EmitScalarExpr(E->getArg(0)), EmitScalarExpr(E->getArg(1))});
22063 case NVPTX::BI__nvvm_getctarank:
22067 case NVPTX::BI__nvvm_getctarank_shared_cluster:
22071 case NVPTX::BI__nvvm_barrier_cluster_arrive:
22074 case NVPTX::BI__nvvm_barrier_cluster_arrive_relaxed:
22077 case NVPTX::BI__nvvm_barrier_cluster_wait:
22080 case NVPTX::BI__nvvm_fence_sc_cluster:
22089struct BuiltinAlignArgs {
22090 llvm::Value *Src =
nullptr;
22091 llvm::Type *SrcType =
nullptr;
22092 llvm::Value *Alignment =
nullptr;
22093 llvm::Value *Mask =
nullptr;
22094 llvm::IntegerType *IntType =
nullptr;
22102 SrcType = Src->getType();
22103 if (SrcType->isPointerTy()) {
22104 IntType = IntegerType::get(
22108 assert(SrcType->isIntegerTy());
22109 IntType = cast<llvm::IntegerType>(SrcType);
22112 Alignment = CGF.
Builder.CreateZExtOrTrunc(Alignment, IntType,
"alignment");
22113 auto *One = llvm::ConstantInt::get(IntType, 1);
22114 Mask = CGF.
Builder.CreateSub(Alignment, One,
"mask");
22121 BuiltinAlignArgs Args(
E, *
this);
22122 llvm::Value *SrcAddress = Args.Src;
22123 if (Args.SrcType->isPointerTy())
22125 Builder.CreateBitOrPointerCast(Args.Src, Args.IntType,
"src_addr");
22127 Builder.CreateAnd(SrcAddress, Args.Mask,
"set_bits"),
22128 llvm::Constant::getNullValue(Args.IntType),
"is_aligned"));
22135 BuiltinAlignArgs Args(
E, *
this);
22136 llvm::Value *SrcForMask = Args.Src;
22142 if (Args.Src->getType()->isPointerTy()) {
22152 SrcForMask =
Builder.CreateAdd(SrcForMask, Args.Mask,
"over_boundary");
22156 llvm::Value *InvertedMask =
Builder.CreateNot(Args.Mask,
"inverted_mask");
22157 llvm::Value *
Result =
nullptr;
22158 if (Args.Src->getType()->isPointerTy()) {
22160 Intrinsic::ptrmask, {Args.SrcType, Args.IntType},
22161 {SrcForMask, InvertedMask},
nullptr,
"aligned_result");
22163 Result =
Builder.CreateAnd(SrcForMask, InvertedMask,
"aligned_result");
22165 assert(
Result->getType() == Args.SrcType);
22171 switch (BuiltinID) {
22172 case WebAssembly::BI__builtin_wasm_memory_size: {
22177 return Builder.CreateCall(Callee, I);
22179 case WebAssembly::BI__builtin_wasm_memory_grow: {
22185 return Builder.CreateCall(Callee, Args);
22187 case WebAssembly::BI__builtin_wasm_tls_size: {
22190 return Builder.CreateCall(Callee);
22192 case WebAssembly::BI__builtin_wasm_tls_align: {
22195 return Builder.CreateCall(Callee);
22197 case WebAssembly::BI__builtin_wasm_tls_base: {
22199 return Builder.CreateCall(Callee);
22201 case WebAssembly::BI__builtin_wasm_throw: {
22205 return Builder.CreateCall(Callee, {
Tag, Obj});
22207 case WebAssembly::BI__builtin_wasm_rethrow: {
22209 return Builder.CreateCall(Callee);
22211 case WebAssembly::BI__builtin_wasm_memory_atomic_wait32: {
22218 case WebAssembly::BI__builtin_wasm_memory_atomic_wait64: {
22225 case WebAssembly::BI__builtin_wasm_memory_atomic_notify: {
22229 return Builder.CreateCall(Callee, {Addr, Count});
22231 case WebAssembly::BI__builtin_wasm_trunc_s_i32_f32:
22232 case WebAssembly::BI__builtin_wasm_trunc_s_i32_f64:
22233 case WebAssembly::BI__builtin_wasm_trunc_s_i64_f32:
22234 case WebAssembly::BI__builtin_wasm_trunc_s_i64_f64: {
22239 return Builder.CreateCall(Callee, {Src});
22241 case WebAssembly::BI__builtin_wasm_trunc_u_i32_f32:
22242 case WebAssembly::BI__builtin_wasm_trunc_u_i32_f64:
22243 case WebAssembly::BI__builtin_wasm_trunc_u_i64_f32:
22244 case WebAssembly::BI__builtin_wasm_trunc_u_i64_f64: {
22249 return Builder.CreateCall(Callee, {Src});
22251 case WebAssembly::BI__builtin_wasm_trunc_saturate_s_i32_f32:
22252 case WebAssembly::BI__builtin_wasm_trunc_saturate_s_i32_f64:
22253 case WebAssembly::BI__builtin_wasm_trunc_saturate_s_i64_f32:
22254 case WebAssembly::BI__builtin_wasm_trunc_saturate_s_i64_f64:
22255 case WebAssembly::BI__builtin_wasm_trunc_saturate_s_i16x8_f16x8:
22256 case WebAssembly::BI__builtin_wasm_trunc_saturate_s_i32x4_f32x4: {
22261 return Builder.CreateCall(Callee, {Src});
22263 case WebAssembly::BI__builtin_wasm_trunc_saturate_u_i32_f32:
22264 case WebAssembly::BI__builtin_wasm_trunc_saturate_u_i32_f64:
22265 case WebAssembly::BI__builtin_wasm_trunc_saturate_u_i64_f32:
22266 case WebAssembly::BI__builtin_wasm_trunc_saturate_u_i64_f64:
22267 case WebAssembly::BI__builtin_wasm_trunc_saturate_u_i16x8_f16x8:
22268 case WebAssembly::BI__builtin_wasm_trunc_saturate_u_i32x4_f32x4: {
22273 return Builder.CreateCall(Callee, {Src});
22275 case WebAssembly::BI__builtin_wasm_min_f32:
22276 case WebAssembly::BI__builtin_wasm_min_f64:
22277 case WebAssembly::BI__builtin_wasm_min_f16x8:
22278 case WebAssembly::BI__builtin_wasm_min_f32x4:
22279 case WebAssembly::BI__builtin_wasm_min_f64x2: {
22284 return Builder.CreateCall(Callee, {LHS, RHS});
22286 case WebAssembly::BI__builtin_wasm_max_f32:
22287 case WebAssembly::BI__builtin_wasm_max_f64:
22288 case WebAssembly::BI__builtin_wasm_max_f16x8:
22289 case WebAssembly::BI__builtin_wasm_max_f32x4:
22290 case WebAssembly::BI__builtin_wasm_max_f64x2: {
22295 return Builder.CreateCall(Callee, {LHS, RHS});
22297 case WebAssembly::BI__builtin_wasm_pmin_f16x8:
22298 case WebAssembly::BI__builtin_wasm_pmin_f32x4:
22299 case WebAssembly::BI__builtin_wasm_pmin_f64x2: {
22304 return Builder.CreateCall(Callee, {LHS, RHS});
22306 case WebAssembly::BI__builtin_wasm_pmax_f16x8:
22307 case WebAssembly::BI__builtin_wasm_pmax_f32x4:
22308 case WebAssembly::BI__builtin_wasm_pmax_f64x2: {
22313 return Builder.CreateCall(Callee, {LHS, RHS});
22315 case WebAssembly::BI__builtin_wasm_ceil_f16x8:
22316 case WebAssembly::BI__builtin_wasm_floor_f16x8:
22317 case WebAssembly::BI__builtin_wasm_trunc_f16x8:
22318 case WebAssembly::BI__builtin_wasm_nearest_f16x8:
22319 case WebAssembly::BI__builtin_wasm_ceil_f32x4:
22320 case WebAssembly::BI__builtin_wasm_floor_f32x4:
22321 case WebAssembly::BI__builtin_wasm_trunc_f32x4:
22322 case WebAssembly::BI__builtin_wasm_nearest_f32x4:
22323 case WebAssembly::BI__builtin_wasm_ceil_f64x2:
22324 case WebAssembly::BI__builtin_wasm_floor_f64x2:
22325 case WebAssembly::BI__builtin_wasm_trunc_f64x2:
22326 case WebAssembly::BI__builtin_wasm_nearest_f64x2: {
22328 switch (BuiltinID) {
22329 case WebAssembly::BI__builtin_wasm_ceil_f16x8:
22330 case WebAssembly::BI__builtin_wasm_ceil_f32x4:
22331 case WebAssembly::BI__builtin_wasm_ceil_f64x2:
22332 IntNo = Intrinsic::ceil;
22334 case WebAssembly::BI__builtin_wasm_floor_f16x8:
22335 case WebAssembly::BI__builtin_wasm_floor_f32x4:
22336 case WebAssembly::BI__builtin_wasm_floor_f64x2:
22337 IntNo = Intrinsic::floor;
22339 case WebAssembly::BI__builtin_wasm_trunc_f16x8:
22340 case WebAssembly::BI__builtin_wasm_trunc_f32x4:
22341 case WebAssembly::BI__builtin_wasm_trunc_f64x2:
22342 IntNo = Intrinsic::trunc;
22344 case WebAssembly::BI__builtin_wasm_nearest_f16x8:
22345 case WebAssembly::BI__builtin_wasm_nearest_f32x4:
22346 case WebAssembly::BI__builtin_wasm_nearest_f64x2:
22347 IntNo = Intrinsic::nearbyint;
22350 llvm_unreachable(
"unexpected builtin ID");
22356 case WebAssembly::BI__builtin_wasm_ref_null_extern: {
22358 return Builder.CreateCall(Callee);
22360 case WebAssembly::BI__builtin_wasm_ref_null_func: {
22362 return Builder.CreateCall(Callee);
22364 case WebAssembly::BI__builtin_wasm_swizzle_i8x16: {
22368 return Builder.CreateCall(Callee, {Src, Indices});
22370 case WebAssembly::BI__builtin_wasm_abs_i8x16:
22371 case WebAssembly::BI__builtin_wasm_abs_i16x8:
22372 case WebAssembly::BI__builtin_wasm_abs_i32x4:
22373 case WebAssembly::BI__builtin_wasm_abs_i64x2: {
22376 Constant *
Zero = llvm::Constant::getNullValue(Vec->
getType());
22377 Value *ICmp =
Builder.CreateICmpSLT(Vec, Zero,
"abscond");
22378 return Builder.CreateSelect(ICmp, Neg, Vec,
"abs");
22380 case WebAssembly::BI__builtin_wasm_avgr_u_i8x16:
22381 case WebAssembly::BI__builtin_wasm_avgr_u_i16x8: {
22386 return Builder.CreateCall(Callee, {LHS, RHS});
22388 case WebAssembly::BI__builtin_wasm_q15mulr_sat_s_i16x8: {
22392 return Builder.CreateCall(Callee, {LHS, RHS});
22394 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i8x16_s_i16x8:
22395 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i8x16_u_i16x8:
22396 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i16x8_s_i32x4:
22397 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i16x8_u_i32x4: {
22400 switch (BuiltinID) {
22401 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i8x16_s_i16x8:
22402 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i16x8_s_i32x4:
22403 IntNo = Intrinsic::wasm_extadd_pairwise_signed;
22405 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i8x16_u_i16x8:
22406 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i16x8_u_i32x4:
22407 IntNo = Intrinsic::wasm_extadd_pairwise_unsigned;
22410 llvm_unreachable(
"unexpected builtin ID");
22414 return Builder.CreateCall(Callee, Vec);
22416 case WebAssembly::BI__builtin_wasm_bitselect: {
22422 return Builder.CreateCall(Callee, {V1, V2,
C});
22424 case WebAssembly::BI__builtin_wasm_dot_s_i32x4_i16x8: {
22428 return Builder.CreateCall(Callee, {LHS, RHS});
22430 case WebAssembly::BI__builtin_wasm_any_true_v128:
22431 case WebAssembly::BI__builtin_wasm_all_true_i8x16:
22432 case WebAssembly::BI__builtin_wasm_all_true_i16x8:
22433 case WebAssembly::BI__builtin_wasm_all_true_i32x4:
22434 case WebAssembly::BI__builtin_wasm_all_true_i64x2: {
22436 switch (BuiltinID) {
22437 case WebAssembly::BI__builtin_wasm_any_true_v128:
22438 IntNo = Intrinsic::wasm_anytrue;
22440 case WebAssembly::BI__builtin_wasm_all_true_i8x16:
22441 case WebAssembly::BI__builtin_wasm_all_true_i16x8:
22442 case WebAssembly::BI__builtin_wasm_all_true_i32x4:
22443 case WebAssembly::BI__builtin_wasm_all_true_i64x2:
22444 IntNo = Intrinsic::wasm_alltrue;
22447 llvm_unreachable(
"unexpected builtin ID");
22451 return Builder.CreateCall(Callee, {Vec});
22453 case WebAssembly::BI__builtin_wasm_bitmask_i8x16:
22454 case WebAssembly::BI__builtin_wasm_bitmask_i16x8:
22455 case WebAssembly::BI__builtin_wasm_bitmask_i32x4:
22456 case WebAssembly::BI__builtin_wasm_bitmask_i64x2: {
22460 return Builder.CreateCall(Callee, {Vec});
22462 case WebAssembly::BI__builtin_wasm_abs_f16x8:
22463 case WebAssembly::BI__builtin_wasm_abs_f32x4:
22464 case WebAssembly::BI__builtin_wasm_abs_f64x2: {
22467 return Builder.CreateCall(Callee, {Vec});
22469 case WebAssembly::BI__builtin_wasm_sqrt_f16x8:
22470 case WebAssembly::BI__builtin_wasm_sqrt_f32x4:
22471 case WebAssembly::BI__builtin_wasm_sqrt_f64x2: {
22474 return Builder.CreateCall(Callee, {Vec});
22476 case WebAssembly::BI__builtin_wasm_narrow_s_i8x16_i16x8:
22477 case WebAssembly::BI__builtin_wasm_narrow_u_i8x16_i16x8:
22478 case WebAssembly::BI__builtin_wasm_narrow_s_i16x8_i32x4:
22479 case WebAssembly::BI__builtin_wasm_narrow_u_i16x8_i32x4: {
22483 switch (BuiltinID) {
22484 case WebAssembly::BI__builtin_wasm_narrow_s_i8x16_i16x8:
22485 case WebAssembly::BI__builtin_wasm_narrow_s_i16x8_i32x4:
22486 IntNo = Intrinsic::wasm_narrow_signed;
22488 case WebAssembly::BI__builtin_wasm_narrow_u_i8x16_i16x8:
22489 case WebAssembly::BI__builtin_wasm_narrow_u_i16x8_i32x4:
22490 IntNo = Intrinsic::wasm_narrow_unsigned;
22493 llvm_unreachable(
"unexpected builtin ID");
22497 return Builder.CreateCall(Callee, {Low, High});
22499 case WebAssembly::BI__builtin_wasm_trunc_sat_s_zero_f64x2_i32x4:
22500 case WebAssembly::BI__builtin_wasm_trunc_sat_u_zero_f64x2_i32x4: {
22503 switch (BuiltinID) {
22504 case WebAssembly::BI__builtin_wasm_trunc_sat_s_zero_f64x2_i32x4:
22505 IntNo = Intrinsic::fptosi_sat;
22507 case WebAssembly::BI__builtin_wasm_trunc_sat_u_zero_f64x2_i32x4:
22508 IntNo = Intrinsic::fptoui_sat;
22511 llvm_unreachable(
"unexpected builtin ID");
22513 llvm::Type *SrcT = Vec->
getType();
22514 llvm::Type *TruncT = SrcT->getWithNewType(
Builder.getInt32Ty());
22517 Value *Splat = Constant::getNullValue(TruncT);
22520 case WebAssembly::BI__builtin_wasm_shuffle_i8x16: {
22525 while (OpIdx < 18) {
22526 std::optional<llvm::APSInt> LaneConst =
22528 assert(LaneConst &&
"Constant arg isn't actually constant?");
22529 Ops[OpIdx++] = llvm::ConstantInt::get(
getLLVMContext(), *LaneConst);
22532 return Builder.CreateCall(Callee, Ops);
22534 case WebAssembly::BI__builtin_wasm_relaxed_madd_f16x8:
22535 case WebAssembly::BI__builtin_wasm_relaxed_nmadd_f16x8:
22536 case WebAssembly::BI__builtin_wasm_relaxed_madd_f32x4:
22537 case WebAssembly::BI__builtin_wasm_relaxed_nmadd_f32x4:
22538 case WebAssembly::BI__builtin_wasm_relaxed_madd_f64x2:
22539 case WebAssembly::BI__builtin_wasm_relaxed_nmadd_f64x2: {
22544 switch (BuiltinID) {
22545 case WebAssembly::BI__builtin_wasm_relaxed_madd_f16x8:
22546 case WebAssembly::BI__builtin_wasm_relaxed_madd_f32x4:
22547 case WebAssembly::BI__builtin_wasm_relaxed_madd_f64x2:
22548 IntNo = Intrinsic::wasm_relaxed_madd;
22550 case WebAssembly::BI__builtin_wasm_relaxed_nmadd_f16x8:
22551 case WebAssembly::BI__builtin_wasm_relaxed_nmadd_f32x4:
22552 case WebAssembly::BI__builtin_wasm_relaxed_nmadd_f64x2:
22553 IntNo = Intrinsic::wasm_relaxed_nmadd;
22556 llvm_unreachable(
"unexpected builtin ID");
22559 return Builder.CreateCall(Callee, {A, B,
C});
22561 case WebAssembly::BI__builtin_wasm_relaxed_laneselect_i8x16:
22562 case WebAssembly::BI__builtin_wasm_relaxed_laneselect_i16x8:
22563 case WebAssembly::BI__builtin_wasm_relaxed_laneselect_i32x4:
22564 case WebAssembly::BI__builtin_wasm_relaxed_laneselect_i64x2: {
22570 return Builder.CreateCall(Callee, {A, B,
C});
22572 case WebAssembly::BI__builtin_wasm_relaxed_swizzle_i8x16: {
22576 return Builder.CreateCall(Callee, {Src, Indices});
22578 case WebAssembly::BI__builtin_wasm_relaxed_min_f32x4:
22579 case WebAssembly::BI__builtin_wasm_relaxed_max_f32x4:
22580 case WebAssembly::BI__builtin_wasm_relaxed_min_f64x2:
22581 case WebAssembly::BI__builtin_wasm_relaxed_max_f64x2: {
22585 switch (BuiltinID) {
22586 case WebAssembly::BI__builtin_wasm_relaxed_min_f32x4:
22587 case WebAssembly::BI__builtin_wasm_relaxed_min_f64x2:
22588 IntNo = Intrinsic::wasm_relaxed_min;
22590 case WebAssembly::BI__builtin_wasm_relaxed_max_f32x4:
22591 case WebAssembly::BI__builtin_wasm_relaxed_max_f64x2:
22592 IntNo = Intrinsic::wasm_relaxed_max;
22595 llvm_unreachable(
"unexpected builtin ID");
22598 return Builder.CreateCall(Callee, {LHS, RHS});
22600 case WebAssembly::BI__builtin_wasm_relaxed_trunc_s_i32x4_f32x4:
22601 case WebAssembly::BI__builtin_wasm_relaxed_trunc_u_i32x4_f32x4:
22602 case WebAssembly::BI__builtin_wasm_relaxed_trunc_s_zero_i32x4_f64x2:
22603 case WebAssembly::BI__builtin_wasm_relaxed_trunc_u_zero_i32x4_f64x2: {
22606 switch (BuiltinID) {
22607 case WebAssembly::BI__builtin_wasm_relaxed_trunc_s_i32x4_f32x4:
22608 IntNo = Intrinsic::wasm_relaxed_trunc_signed;
22610 case WebAssembly::BI__builtin_wasm_relaxed_trunc_u_i32x4_f32x4:
22611 IntNo = Intrinsic::wasm_relaxed_trunc_unsigned;
22613 case WebAssembly::BI__builtin_wasm_relaxed_trunc_s_zero_i32x4_f64x2:
22614 IntNo = Intrinsic::wasm_relaxed_trunc_signed_zero;
22616 case WebAssembly::BI__builtin_wasm_relaxed_trunc_u_zero_i32x4_f64x2:
22617 IntNo = Intrinsic::wasm_relaxed_trunc_unsigned_zero;
22620 llvm_unreachable(
"unexpected builtin ID");
22623 return Builder.CreateCall(Callee, {Vec});
22625 case WebAssembly::BI__builtin_wasm_relaxed_q15mulr_s_i16x8: {
22629 return Builder.CreateCall(Callee, {LHS, RHS});
22631 case WebAssembly::BI__builtin_wasm_relaxed_dot_i8x16_i7x16_s_i16x8: {
22636 return Builder.CreateCall(Callee, {LHS, RHS});
22638 case WebAssembly::BI__builtin_wasm_relaxed_dot_i8x16_i7x16_add_s_i32x4: {
22643 CGM.
getIntrinsic(Intrinsic::wasm_relaxed_dot_i8x16_i7x16_add_signed);
22644 return Builder.CreateCall(Callee, {LHS, RHS, Acc});
22646 case WebAssembly::BI__builtin_wasm_relaxed_dot_bf16x8_add_f32_f32x4: {
22652 return Builder.CreateCall(Callee, {LHS, RHS, Acc});
22654 case WebAssembly::BI__builtin_wasm_loadf16_f32: {
22657 return Builder.CreateCall(Callee, {Addr});
22659 case WebAssembly::BI__builtin_wasm_storef16_f32: {
22663 return Builder.CreateCall(Callee, {Val, Addr});
22665 case WebAssembly::BI__builtin_wasm_splat_f16x8: {
22668 return Builder.CreateCall(Callee, {Val});
22670 case WebAssembly::BI__builtin_wasm_extract_lane_f16x8: {
22676 case WebAssembly::BI__builtin_wasm_replace_lane_f16x8: {
22683 case WebAssembly::BI__builtin_wasm_table_get: {
22694 "Unexpected reference type for __builtin_wasm_table_get");
22695 return Builder.CreateCall(Callee, {Table, Index});
22697 case WebAssembly::BI__builtin_wasm_table_set: {
22709 "Unexpected reference type for __builtin_wasm_table_set");
22710 return Builder.CreateCall(Callee, {Table, Index, Val});
22712 case WebAssembly::BI__builtin_wasm_table_size: {
22718 case WebAssembly::BI__builtin_wasm_table_grow: {
22731 "Unexpected reference type for __builtin_wasm_table_grow");
22733 return Builder.CreateCall(Callee, {Table, Val, NElems});
22735 case WebAssembly::BI__builtin_wasm_table_fill: {
22749 "Unexpected reference type for __builtin_wasm_table_fill");
22751 return Builder.CreateCall(Callee, {Table, Index, Val, NElems});
22753 case WebAssembly::BI__builtin_wasm_table_copy: {
22763 return Builder.CreateCall(Callee, {TableX, TableY, SrcIdx, DstIdx, NElems});
22770static std::pair<Intrinsic::ID, unsigned>
22773 unsigned BuiltinID;
22774 Intrinsic::ID IntrinsicID;
22777 static Info Infos[] = {
22778#define CUSTOM_BUILTIN_MAPPING(x,s) \
22779 { Hexagon::BI__builtin_HEXAGON_##x, Intrinsic::hexagon_##x, s },
22811#include "clang/Basic/BuiltinsHexagonMapCustomDep.def"
22812#undef CUSTOM_BUILTIN_MAPPING
22815 auto CmpInfo = [] (Info A, Info B) {
return A.BuiltinID < B.BuiltinID; };
22816 static const bool SortOnce = (llvm::sort(Infos, CmpInfo),
true);
22819 const Info *F = llvm::lower_bound(Infos, Info{BuiltinID, 0, 0}, CmpInfo);
22820 if (F == std::end(Infos) || F->BuiltinID != BuiltinID)
22821 return {Intrinsic::not_intrinsic, 0};
22823 return {F->IntrinsicID, F->VecLen};
22832 auto MakeCircOp = [
this,
E](
unsigned IntID,
bool IsLoad) {
22846 for (
unsigned i = 1, e =
E->getNumArgs(); i != e; ++i)
22852 llvm::Value *NewBase = IsLoad ?
Builder.CreateExtractValue(
Result, 1)
22856 llvm::Value *RetVal =
22866 auto MakeBrevLd = [
this,
E](
unsigned IntID, llvm::Type *DestTy) {
22883 CGM.
getIntrinsic(IntID), {BaseAddress, EmitScalarExpr(E->getArg(2))});
22886 llvm::Value *DestVal =
Builder.CreateExtractValue(
Result, 0);
22891 DestVal =
Builder.CreateTrunc(DestVal, DestTy);
22898 auto V2Q = [
this, VecLen] (llvm::Value *Vec) {
22899 Intrinsic::ID
ID = VecLen == 128 ? Intrinsic::hexagon_V6_vandvrt_128B
22900 : Intrinsic::hexagon_V6_vandvrt;
22902 {Vec,
Builder.getInt32(-1)});
22904 auto Q2V = [
this, VecLen] (llvm::Value *Pred) {
22905 Intrinsic::ID
ID = VecLen == 128 ? Intrinsic::hexagon_V6_vandqrt_128B
22906 : Intrinsic::hexagon_V6_vandqrt;
22908 {Pred,
Builder.getInt32(-1)});
22911 switch (BuiltinID) {
22915 case Hexagon::BI__builtin_HEXAGON_V6_vaddcarry:
22916 case Hexagon::BI__builtin_HEXAGON_V6_vaddcarry_128B:
22917 case Hexagon::BI__builtin_HEXAGON_V6_vsubcarry:
22918 case Hexagon::BI__builtin_HEXAGON_V6_vsubcarry_128B: {
22925 {EmitScalarExpr(E->getArg(0)), EmitScalarExpr(E->getArg(1)), PredIn});
22927 llvm::Value *PredOut =
Builder.CreateExtractValue(
Result, 1);
22935 case Hexagon::BI__builtin_HEXAGON_V6_vaddcarryo:
22936 case Hexagon::BI__builtin_HEXAGON_V6_vaddcarryo_128B:
22937 case Hexagon::BI__builtin_HEXAGON_V6_vsubcarryo:
22938 case Hexagon::BI__builtin_HEXAGON_V6_vsubcarryo_128B: {
22944 {EmitScalarExpr(E->getArg(0)), EmitScalarExpr(E->getArg(1))});
22946 llvm::Value *PredOut =
Builder.CreateExtractValue(
Result, 1);
22952 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstoreq:
22953 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstorenq:
22954 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstorentq:
22955 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstorentnq:
22956 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstoreq_128B:
22957 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstorenq_128B:
22958 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstorentq_128B:
22959 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstorentnq_128B: {
22961 const Expr *PredOp =
E->getArg(0);
22963 if (
auto *Cast = dyn_cast<ImplicitCastExpr>(PredOp)) {
22964 if (
Cast->getCastKind() == CK_BitCast)
22965 PredOp =
Cast->getSubExpr();
22968 for (
int i = 1, e =
E->getNumArgs(); i != e; ++i)
22973 case Hexagon::BI__builtin_HEXAGON_L2_loadrub_pci:
22974 case Hexagon::BI__builtin_HEXAGON_L2_loadrb_pci:
22975 case Hexagon::BI__builtin_HEXAGON_L2_loadruh_pci:
22976 case Hexagon::BI__builtin_HEXAGON_L2_loadrh_pci:
22977 case Hexagon::BI__builtin_HEXAGON_L2_loadri_pci:
22978 case Hexagon::BI__builtin_HEXAGON_L2_loadrd_pci:
22979 case Hexagon::BI__builtin_HEXAGON_L2_loadrub_pcr:
22980 case Hexagon::BI__builtin_HEXAGON_L2_loadrb_pcr:
22981 case Hexagon::BI__builtin_HEXAGON_L2_loadruh_pcr:
22982 case Hexagon::BI__builtin_HEXAGON_L2_loadrh_pcr:
22983 case Hexagon::BI__builtin_HEXAGON_L2_loadri_pcr:
22984 case Hexagon::BI__builtin_HEXAGON_L2_loadrd_pcr:
22985 return MakeCircOp(ID,
true);
22986 case Hexagon::BI__builtin_HEXAGON_S2_storerb_pci:
22987 case Hexagon::BI__builtin_HEXAGON_S2_storerh_pci:
22988 case Hexagon::BI__builtin_HEXAGON_S2_storerf_pci:
22989 case Hexagon::BI__builtin_HEXAGON_S2_storeri_pci:
22990 case Hexagon::BI__builtin_HEXAGON_S2_storerd_pci:
22991 case Hexagon::BI__builtin_HEXAGON_S2_storerb_pcr:
22992 case Hexagon::BI__builtin_HEXAGON_S2_storerh_pcr:
22993 case Hexagon::BI__builtin_HEXAGON_S2_storerf_pcr:
22994 case Hexagon::BI__builtin_HEXAGON_S2_storeri_pcr:
22995 case Hexagon::BI__builtin_HEXAGON_S2_storerd_pcr:
22996 return MakeCircOp(ID,
false);
22997 case Hexagon::BI__builtin_brev_ldub:
22998 return MakeBrevLd(Intrinsic::hexagon_L2_loadrub_pbr,
Int8Ty);
22999 case Hexagon::BI__builtin_brev_ldb:
23000 return MakeBrevLd(Intrinsic::hexagon_L2_loadrb_pbr,
Int8Ty);
23001 case Hexagon::BI__builtin_brev_lduh:
23002 return MakeBrevLd(Intrinsic::hexagon_L2_loadruh_pbr,
Int16Ty);
23003 case Hexagon::BI__builtin_brev_ldh:
23004 return MakeBrevLd(Intrinsic::hexagon_L2_loadrh_pbr,
Int16Ty);
23005 case Hexagon::BI__builtin_brev_ldw:
23006 return MakeBrevLd(Intrinsic::hexagon_L2_loadri_pbr,
Int32Ty);
23007 case Hexagon::BI__builtin_brev_ldd:
23008 return MakeBrevLd(Intrinsic::hexagon_L2_loadrd_pbr,
Int64Ty);
23016 StringRef CPUStr = cast<clang::StringLiteral>(CPUExpr)->getString();
23024 llvm::Constant *RISCVCPUModel =
23026 cast<llvm::GlobalValue>(RISCVCPUModel)->setDSOLocal(
true);
23028 auto loadRISCVCPUID = [&](
unsigned Index) {
23031 Ptr, llvm::MaybeAlign());
23035 const llvm::RISCV::CPUModel Model = llvm::RISCV::getCPUModel(CPUStr);
23038 Value *VendorID = loadRISCVCPUID(0);
23040 Builder.CreateICmpEQ(VendorID,
Builder.getInt32(Model.MVendorID));
23043 Value *ArchID = loadRISCVCPUID(1);
23048 Value *ImpID = loadRISCVCPUID(2);
23059 if (BuiltinID == Builtin::BI__builtin_cpu_supports)
23061 if (BuiltinID == Builtin::BI__builtin_cpu_init)
23063 if (BuiltinID == Builtin::BI__builtin_cpu_is)
23070 unsigned ICEArguments = 0;
23078 if (BuiltinID == RISCVVector::BI__builtin_rvv_vget_v ||
23079 BuiltinID == RISCVVector::BI__builtin_rvv_vset_v)
23080 ICEArguments = 1 << 1;
23085 if (BuiltinID == RISCV::BI__builtin_riscv_ntl_load)
23086 ICEArguments |= (1 << 1);
23087 if (BuiltinID == RISCV::BI__builtin_riscv_ntl_store)
23088 ICEArguments |= (1 << 2);
23090 for (
unsigned i = 0, e =
E->getNumArgs(); i != e; i++) {
23095 Ops.push_back(AggValue);
23101 Intrinsic::ID
ID = Intrinsic::not_intrinsic;
23104 constexpr unsigned RVV_VTA = 0x1;
23105 constexpr unsigned RVV_VMA = 0x2;
23106 int PolicyAttrs = 0;
23107 bool IsMasked =
false;
23109 unsigned SegInstSEW = 8;
23113 switch (BuiltinID) {
23114 default: llvm_unreachable(
"unexpected builtin ID");
23115 case RISCV::BI__builtin_riscv_orc_b_32:
23116 case RISCV::BI__builtin_riscv_orc_b_64:
23117 case RISCV::BI__builtin_riscv_clmul_32:
23118 case RISCV::BI__builtin_riscv_clmul_64:
23119 case RISCV::BI__builtin_riscv_clmulh_32:
23120 case RISCV::BI__builtin_riscv_clmulh_64:
23121 case RISCV::BI__builtin_riscv_clmulr_32:
23122 case RISCV::BI__builtin_riscv_clmulr_64:
23123 case RISCV::BI__builtin_riscv_xperm4_32:
23124 case RISCV::BI__builtin_riscv_xperm4_64:
23125 case RISCV::BI__builtin_riscv_xperm8_32:
23126 case RISCV::BI__builtin_riscv_xperm8_64:
23127 case RISCV::BI__builtin_riscv_brev8_32:
23128 case RISCV::BI__builtin_riscv_brev8_64:
23129 case RISCV::BI__builtin_riscv_zip_32:
23130 case RISCV::BI__builtin_riscv_unzip_32: {
23131 switch (BuiltinID) {
23132 default: llvm_unreachable(
"unexpected builtin ID");
23134 case RISCV::BI__builtin_riscv_orc_b_32:
23135 case RISCV::BI__builtin_riscv_orc_b_64:
23136 ID = Intrinsic::riscv_orc_b;
23140 case RISCV::BI__builtin_riscv_clmul_32:
23141 case RISCV::BI__builtin_riscv_clmul_64:
23142 ID = Intrinsic::riscv_clmul;
23144 case RISCV::BI__builtin_riscv_clmulh_32:
23145 case RISCV::BI__builtin_riscv_clmulh_64:
23146 ID = Intrinsic::riscv_clmulh;
23148 case RISCV::BI__builtin_riscv_clmulr_32:
23149 case RISCV::BI__builtin_riscv_clmulr_64:
23150 ID = Intrinsic::riscv_clmulr;
23154 case RISCV::BI__builtin_riscv_xperm8_32:
23155 case RISCV::BI__builtin_riscv_xperm8_64:
23156 ID = Intrinsic::riscv_xperm8;
23158 case RISCV::BI__builtin_riscv_xperm4_32:
23159 case RISCV::BI__builtin_riscv_xperm4_64:
23160 ID = Intrinsic::riscv_xperm4;
23164 case RISCV::BI__builtin_riscv_brev8_32:
23165 case RISCV::BI__builtin_riscv_brev8_64:
23166 ID = Intrinsic::riscv_brev8;
23168 case RISCV::BI__builtin_riscv_zip_32:
23169 ID = Intrinsic::riscv_zip;
23171 case RISCV::BI__builtin_riscv_unzip_32:
23172 ID = Intrinsic::riscv_unzip;
23176 IntrinsicTypes = {ResultType};
23183 case RISCV::BI__builtin_riscv_sha256sig0:
23184 ID = Intrinsic::riscv_sha256sig0;
23186 case RISCV::BI__builtin_riscv_sha256sig1:
23187 ID = Intrinsic::riscv_sha256sig1;
23189 case RISCV::BI__builtin_riscv_sha256sum0:
23190 ID = Intrinsic::riscv_sha256sum0;
23192 case RISCV::BI__builtin_riscv_sha256sum1:
23193 ID = Intrinsic::riscv_sha256sum1;
23197 case RISCV::BI__builtin_riscv_sm4ks:
23198 ID = Intrinsic::riscv_sm4ks;
23200 case RISCV::BI__builtin_riscv_sm4ed:
23201 ID = Intrinsic::riscv_sm4ed;
23205 case RISCV::BI__builtin_riscv_sm3p0:
23206 ID = Intrinsic::riscv_sm3p0;
23208 case RISCV::BI__builtin_riscv_sm3p1:
23209 ID = Intrinsic::riscv_sm3p1;
23212 case RISCV::BI__builtin_riscv_clz_32:
23213 case RISCV::BI__builtin_riscv_clz_64: {
23216 if (
Result->getType() != ResultType)
23221 case RISCV::BI__builtin_riscv_ctz_32:
23222 case RISCV::BI__builtin_riscv_ctz_64: {
23225 if (
Result->getType() != ResultType)
23232 case RISCV::BI__builtin_riscv_ntl_load: {
23234 unsigned DomainVal = 5;
23235 if (Ops.size() == 2)
23236 DomainVal = cast<ConstantInt>(Ops[1])->getZExtValue();
23238 llvm::MDNode *RISCVDomainNode = llvm::MDNode::get(
23240 llvm::ConstantAsMetadata::get(
Builder.getInt32(DomainVal)));
23241 llvm::MDNode *NontemporalNode = llvm::MDNode::get(
23245 if(ResTy->isScalableTy()) {
23246 const ScalableVectorType *SVTy = cast<ScalableVectorType>(ResTy);
23247 llvm::Type *ScalarTy = ResTy->getScalarType();
23248 Width = ScalarTy->getPrimitiveSizeInBits() *
23249 SVTy->getElementCount().getKnownMinValue();
23251 Width = ResTy->getPrimitiveSizeInBits();
23255 Load->setMetadata(llvm::LLVMContext::MD_nontemporal, NontemporalNode);
23256 Load->setMetadata(
CGM.
getModule().getMDKindID(
"riscv-nontemporal-domain"),
23261 case RISCV::BI__builtin_riscv_ntl_store: {
23262 unsigned DomainVal = 5;
23263 if (Ops.size() == 3)
23264 DomainVal = cast<ConstantInt>(Ops[2])->getZExtValue();
23266 llvm::MDNode *RISCVDomainNode = llvm::MDNode::get(
23268 llvm::ConstantAsMetadata::get(
Builder.getInt32(DomainVal)));
23269 llvm::MDNode *NontemporalNode = llvm::MDNode::get(
23273 Store->setMetadata(llvm::LLVMContext::MD_nontemporal, NontemporalNode);
23280 case RISCV::BI__builtin_riscv_cv_alu_addN:
23281 ID = Intrinsic::riscv_cv_alu_addN;
23283 case RISCV::BI__builtin_riscv_cv_alu_addRN:
23284 ID = Intrinsic::riscv_cv_alu_addRN;
23286 case RISCV::BI__builtin_riscv_cv_alu_adduN:
23287 ID = Intrinsic::riscv_cv_alu_adduN;
23289 case RISCV::BI__builtin_riscv_cv_alu_adduRN:
23290 ID = Intrinsic::riscv_cv_alu_adduRN;
23292 case RISCV::BI__builtin_riscv_cv_alu_clip:
23293 ID = Intrinsic::riscv_cv_alu_clip;
23295 case RISCV::BI__builtin_riscv_cv_alu_clipu:
23296 ID = Intrinsic::riscv_cv_alu_clipu;
23298 case RISCV::BI__builtin_riscv_cv_alu_extbs:
23301 case RISCV::BI__builtin_riscv_cv_alu_extbz:
23304 case RISCV::BI__builtin_riscv_cv_alu_exths:
23307 case RISCV::BI__builtin_riscv_cv_alu_exthz:
23310 case RISCV::BI__builtin_riscv_cv_alu_slet:
23313 case RISCV::BI__builtin_riscv_cv_alu_sletu:
23316 case RISCV::BI__builtin_riscv_cv_alu_subN:
23317 ID = Intrinsic::riscv_cv_alu_subN;
23319 case RISCV::BI__builtin_riscv_cv_alu_subRN:
23320 ID = Intrinsic::riscv_cv_alu_subRN;
23322 case RISCV::BI__builtin_riscv_cv_alu_subuN:
23323 ID = Intrinsic::riscv_cv_alu_subuN;
23325 case RISCV::BI__builtin_riscv_cv_alu_subuRN:
23326 ID = Intrinsic::riscv_cv_alu_subuRN;
23330#include "clang/Basic/riscv_vector_builtin_cg.inc"
23333#include "clang/Basic/riscv_sifive_vector_builtin_cg.inc"
23336 assert(ID != Intrinsic::not_intrinsic);
23339 return Builder.CreateCall(F, Ops,
"");
Defines the clang::ASTContext interface.
#define PPC_LNX_FEATURE(NAME, DESC, ENUMNAME, ENUMVAL, HWCAPN)
static constexpr SparcCPUInfo CPUInfo[]
#define X86_CPU_SUBTYPE(ENUM, STR)
#define X86_CPU_SUBTYPE_ALIAS(ENUM, ALIAS)
#define X86_VENDOR(ENUM, STRING)
#define X86_CPU_TYPE_ALIAS(ENUM, ALIAS)
#define X86_CPU_TYPE(ENUM, STR)
static constexpr Builtin::Info BuiltinInfo[]
static void Accumulate(SMap &SM, CFGBlock *B)
static Value * EmitSpecialRegisterBuiltin(CodeGenFunction &CGF, const CallExpr *E, llvm::Type *RegisterType, llvm::Type *ValueType, SpecialRegisterAccessKind AccessKind, StringRef SysReg="")
static llvm::Value * ARMMVEVectorReinterpret(CGBuilderTy &Builder, CodeGenFunction *CGF, llvm::Value *V, llvm::Type *DestType)
static Value * MakeBinaryAtomicValue(CodeGenFunction &CGF, llvm::AtomicRMWInst::BinOp Kind, const CallExpr *E, AtomicOrdering Ordering=AtomicOrdering::SequentiallyConsistent)
Utility to insert an atomic instruction based on Intrinsic::ID and the expression node.
static char bitActionToX86BTCode(BitTest::ActionKind A)
#define NEONMAP2(NameBase, LLVMIntrinsic, AltLLVMIntrinsic, TypeModifier)
static Intrinsic::ID getWaveActiveSumIntrinsic(llvm::Triple::ArchType Arch, CGHLSLRuntime &RT, QualType QT)
static Value * EmitAtomicCmpXchg128ForMSIntrin(CodeGenFunction &CGF, const CallExpr *E, AtomicOrdering SuccessOrdering)
#define INTRINSIC_X86_XSAVE_ID(NAME)
static CanQualType getOSLogArgType(ASTContext &C, int Size)
Get the argument type for arguments to os_log_helper.
static Value * EmitOverflowCheckedAbs(CodeGenFunction &CGF, const CallExpr *E, bool SanitizeOverflow)
static llvm::VectorType * GetFloatNeonType(CodeGenFunction *CGF, NeonTypeFlags IntTypeFlags)
static Value * tryUseTestFPKind(CodeGenFunction &CGF, unsigned BuiltinID, Value *V)
static llvm::Value * MVEImmediateShr(CGBuilderTy &Builder, llvm::Value *V, uint32_t Shift, bool Unsigned)
static bool areBOSTypesCompatible(int From, int To)
Checks if using the result of __builtin_object_size(p, From) in place of __builtin_object_size(p,...
static llvm::Value * SignOrZeroExtend(CGBuilderTy &Builder, llvm::Value *V, llvm::Type *T, bool Unsigned)
static void InsertExplicitZeroOperand(CGBuilderTy &Builder, llvm::Type *Ty, SmallVectorImpl< Value * > &Ops)
static const ARMVectorIntrinsicInfo AArch64SMEIntrinsicMap[]
static Value * EmitAArch64TblBuiltinExpr(CodeGenFunction &CGF, unsigned BuiltinID, const CallExpr *E, SmallVectorImpl< Value * > &Ops, llvm::Triple::ArchType Arch)
#define MMA_VARIANTS_B1_AND(geom, type)
static void swapCommutativeSMEOperands(unsigned BuiltinID, SmallVectorImpl< Value * > &Ops)
static bool AArch64SISDIntrinsicsProvenSorted
static Value * EmitX86CompressExpand(CodeGenFunction &CGF, ArrayRef< Value * > Ops, bool IsCompress)
static const ARMVectorIntrinsicInfo AArch64SVEIntrinsicMap[]
static bool HasExtraNeonArgument(unsigned BuiltinID)
Return true if BuiltinID is an overloaded Neon intrinsic with an extra argument that specifies the ve...
static bool TypeRequiresBuiltinLaunderImp(const ASTContext &Ctx, QualType Ty, llvm::SmallPtrSetImpl< const Decl * > &Seen)
static Value * EmitAtomicIncrementValue(CodeGenFunction &CGF, const CallExpr *E, AtomicOrdering Ordering=AtomicOrdering::SequentiallyConsistent)
static std::pair< Intrinsic::ID, unsigned > getIntrinsicForHexagonNonClangBuiltin(unsigned BuiltinID)
static Value * emitRangedBuiltin(CodeGenFunction &CGF, unsigned IntrinsicID, int low, int high)
#define MMA_INTR(geom_op_type, layout)
static Value * EmitX86MaskedCompare(CodeGenFunction &CGF, unsigned CC, bool Signed, ArrayRef< Value * > Ops)
static Value * emitUnaryMaybeConstrainedFPBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID, unsigned ConstrainedIntrinsicID)
static Intrinsic::ID getDotProductIntrinsic(CGHLSLRuntime &RT, QualType QT)
#define NEONMAP1(NameBase, LLVMIntrinsic, TypeModifier)
static bool AArch64SVEIntrinsicsProvenSorted
static RValue EmitMSVCRTSetJmp(CodeGenFunction &CGF, MSVCSetJmpKind SJKind, const CallExpr *E)
MSVC handles setjmp a bit differently on different platforms.
static const ARMVectorIntrinsicInfo * findARMVectorIntrinsicInMap(ArrayRef< ARMVectorIntrinsicInfo > IntrinsicMap, unsigned BuiltinID, bool &MapProvenSorted)
static Value * EmitScalarFMAExpr(CodeGenFunction &CGF, const CallExpr *E, MutableArrayRef< Value * > Ops, Value *Upper, bool ZeroMask=false, unsigned PTIdx=0, bool NegAcc=false)
static Value * loadRISCVFeatureBits(unsigned Index, CGBuilderTy &Builder, CodeGenModule &CGM)
#define MUTATE_LDBL(func)
static Value * EmitX86ExpandLoad(CodeGenFunction &CGF, ArrayRef< Value * > Ops)
static unsigned CountCountedByAttrs(const RecordDecl *RD)
static Value * emitMaybeConstrainedFPToIntRoundBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID, unsigned ConstrainedIntrinsicID)
static Value * EmitX86MaskedLoad(CodeGenFunction &CGF, ArrayRef< Value * > Ops, Align Alignment)
static bool TypeRequiresBuiltinLaunder(CodeGenModule &CGM, QualType Ty)
Determine if the specified type requires laundering by checking if it is a dynamic class type or cont...
static void InsertExplicitUndefOperand(CGBuilderTy &Builder, llvm::Type *Ty, SmallVectorImpl< Value * > &Ops)
static Value * EmitISOVolatileLoad(CodeGenFunction &CGF, const CallExpr *E)
static struct WidthAndSignedness EncompassingIntegerType(ArrayRef< struct WidthAndSignedness > Types)
static Integer GetIntegerConstantValue(const Expr *E, ASTContext &Context)
#define MMA_VARIANTS(geom, type)
static bool AArch64SMEIntrinsicsProvenSorted
static llvm::Value * VectorZip(CGBuilderTy &Builder, llvm::Value *V0, llvm::Value *V1)
static Value * EmitTargetArchBuiltinExpr(CodeGenFunction *CGF, unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue, llvm::Triple::ArchType Arch)
constexpr unsigned SVEBitsPerBlock
static std::optional< CodeGenFunction::MSVCIntrin > translateX86ToMsvcIntrin(unsigned BuiltinID)
static const std::pair< unsigned, unsigned > NEONEquivalentIntrinsicMap[]
#define NEONMAP0(NameBase)
static Value * EmitX86MaskedStore(CodeGenFunction &CGF, ArrayRef< Value * > Ops, Align Alignment)
static Value * handleHlslSplitdouble(const CallExpr *E, CodeGenFunction *CGF)
static Value * emitBinaryMaybeConstrainedFPBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID, unsigned ConstrainedIntrinsicID)
static RValue EmitBinaryAtomicPost(CodeGenFunction &CGF, llvm::AtomicRMWInst::BinOp Kind, const CallExpr *E, Instruction::BinaryOps Op, bool Invert=false)
Utility to insert an atomic instruction based Intrinsic::ID and the expression node,...
static bool HasNoIndirectArgumentsOrResults(CGFunctionInfo const &FnInfo)
Checks no arguments or results are passed indirectly in the ABI (i.e.
static Value * EmitX86Muldq(CodeGenFunction &CGF, bool IsSigned, ArrayRef< Value * > Ops)
Value * readX18AsPtr(CodeGenFunction &CGF)
Helper for the read/write/add/inc X18 builtins: read the X18 register and return it as an i8 pointer.
static llvm::Value * ARMMVEVectorElementReverse(CGBuilderTy &Builder, llvm::Value *V, unsigned ReverseWidth)
#define MMA_SATF_VARIANTS(geom, type)
static std::optional< CodeGenFunction::MSVCIntrin > translateAarch64ToMsvcIntrin(unsigned BuiltinID)
static std::optional< CodeGenFunction::MSVCIntrin > translateArmToMsvcIntrin(unsigned BuiltinID)
static llvm::Value * EmitBitTestIntrinsic(CodeGenFunction &CGF, unsigned BuiltinID, const CallExpr *E)
Emit a _bittest* intrinsic.
static Value * emitBuiltinWithOneOverloadedType(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID, llvm::StringRef Name="")
static const ARMVectorIntrinsicInfo ARMSIMDIntrinsicMap[]
static Value * EmitSignBit(CodeGenFunction &CGF, Value *V)
Emit the computation of the sign bit for a floating point value.
static Value * EmitFAbs(CodeGenFunction &CGF, Value *V)
EmitFAbs - Emit a call to @llvm.fabs().
#define CUSTOM_BUILTIN_MAPPING(x, s)
static Value * EmitX86CvtF16ToFloatExpr(CodeGenFunction &CGF, ArrayRef< Value * > Ops, llvm::Type *DstTy)
static bool isSpecialUnsignedMultiplySignedResult(unsigned BuiltinID, WidthAndSignedness Op1Info, WidthAndSignedness Op2Info, WidthAndSignedness ResultInfo)
static llvm::Value * getDefaultBuiltinObjectSizeResult(unsigned Type, llvm::IntegerType *ResType)
static llvm::Value * emitPPCLoadReserveIntrinsic(CodeGenFunction &CGF, unsigned BuiltinID, const CallExpr *E)
static llvm::Value * VectorUnzip(CGBuilderTy &Builder, llvm::Value *V, bool Odd)
static Value * EmitX86SExtMask(CodeGenFunction &CGF, Value *Op, llvm::Type *DstTy)
static Value * emitTernaryMaybeConstrainedFPBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID, unsigned ConstrainedIntrinsicID)
static WidthAndSignedness getIntegerWidthAndSignedness(const clang::ASTContext &context, const clang::QualType Type)
static Value * EmitX86FunnelShift(CodeGenFunction &CGF, Value *Op0, Value *Op1, Value *Amt, bool IsRight)
static RValue EmitCheckedMixedSignMultiply(CodeGenFunction &CGF, const clang::Expr *Op1, WidthAndSignedness Op1Info, const clang::Expr *Op2, WidthAndSignedness Op2Info, const clang::Expr *ResultArg, QualType ResultQTy, WidthAndSignedness ResultInfo)
Emit a checked mixed-sign multiply.
static llvm::ScalableVectorType * getSVEVectorForElementType(llvm::Type *EltTy)
static unsigned mutateLongDoubleBuiltin(unsigned BuiltinID)
#define INTRINSIC_WITH_CC(NAME)
static llvm::FixedVectorType * GetNeonType(CodeGenFunction *CGF, NeonTypeFlags TypeFlags, bool HasLegalHalfType=true, bool V1Ty=false, bool AllowBFloatArgsAndRet=true)
static RValue EmitBinaryAtomic(CodeGenFunction &CGF, llvm::AtomicRMWInst::BinOp Kind, const CallExpr *E)
static llvm::Value * ARMMVEConstantSplat(CGBuilderTy &Builder, llvm::Type *VT)
static Value * EmitX86MaskLogic(CodeGenFunction &CGF, Instruction::BinaryOps Opc, ArrayRef< Value * > Ops, bool InvertLHS=false)
static Value * EmitFromInt(CodeGenFunction &CGF, llvm::Value *V, QualType T, llvm::Type *ResultType)
static Value * EmitAMDGCNBallotForExec(CodeGenFunction &CGF, const CallExpr *E, llvm::Type *RegisterType, llvm::Type *ValueType, bool isExecHi)
static void emitSincosBuiltin(CodeGenFunction &CGF, const CallExpr *E, llvm::Intrinsic::ID IntrinsicID)
static void initializeAlloca(CodeGenFunction &CGF, AllocaInst *AI, Value *Size, Align AlignmentInBytes)
static Value * EmitX86Select(CodeGenFunction &CGF, Value *Mask, Value *Op0, Value *Op1)
SpecialRegisterAccessKind
static Value * EmitAtomicCmpXchgForMSIntrin(CodeGenFunction &CGF, const CallExpr *E, AtomicOrdering SuccessOrdering=AtomicOrdering::SequentiallyConsistent)
This function should be invoked to emit atomic cmpxchg for Microsoft's _InterlockedCompareExchange* i...
static Address CheckAtomicAlignment(CodeGenFunction &CGF, const CallExpr *E)
static Value * EmitX86FMAExpr(CodeGenFunction &CGF, const CallExpr *E, ArrayRef< Value * > Ops, unsigned BuiltinID, bool IsAddSub)
static Value * getMaskVecValue(CodeGenFunction &CGF, Value *Mask, unsigned NumElts)
static bool isSpecialMixedSignMultiply(unsigned BuiltinID, WidthAndSignedness Op1Info, WidthAndSignedness Op2Info, WidthAndSignedness ResultInfo)
Determine if a binop is a checked mixed-sign multiply we can specialize.
static Value * MakeAtomicCmpXchgValue(CodeGenFunction &CGF, const CallExpr *E, bool ReturnBool)
Utility to insert an atomic cmpxchg instruction.
static Value * emitBinaryExpMaybeConstrainedFPBuiltin(CodeGenFunction &CGF, const CallExpr *E, llvm::Intrinsic::ID IntrinsicID, llvm::Intrinsic::ID ConstrainedIntrinsicID)
static Value * EmitToInt(CodeGenFunction &CGF, llvm::Value *V, QualType T, llvm::IntegerType *IntType)
Emit the conversions required to turn the given value into an integer of the given size.
static llvm::Value * ARMMVEVectorSplat(CGBuilderTy &Builder, llvm::Value *V)
static Value * EmitX86MaskedCompareResult(CodeGenFunction &CGF, Value *Cmp, unsigned NumElts, Value *MaskIn)
static Value * EmitX86CompressStore(CodeGenFunction &CGF, ArrayRef< Value * > Ops)
static bool NEONSIMDIntrinsicsProvenSorted
static const ARMVectorIntrinsicInfo AArch64SIMDIntrinsicMap[]
static Value * EmitNontemporalStore(CodeGenFunction &CGF, const CallExpr *E)
static Intrinsic::ID getFirstBitHighIntrinsic(CGHLSLRuntime &RT, QualType QT)
static llvm::Value * EmitOverflowIntrinsic(CodeGenFunction &CGF, const llvm::Intrinsic::ID IntrinsicID, llvm::Value *X, llvm::Value *Y, llvm::Value *&Carry)
Emit a call to llvm.
static Value * EmitCommonNeonSISDBuiltinExpr(CodeGenFunction &CGF, const ARMVectorIntrinsicInfo &SISDInfo, SmallVectorImpl< Value * > &Ops, const CallExpr *E)
static Value * emitFPIntBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID)
#define MMA_LDST(n, geom_op_type)
static Value * EmitX86vpcom(CodeGenFunction &CGF, ArrayRef< Value * > Ops, bool IsSigned)
static Value * emitFrexpBuiltin(CodeGenFunction &CGF, const CallExpr *E, llvm::Intrinsic::ID IntrinsicID)
@ _InterlockedExchangeAdd_rel
@ _InterlockedIncrement_acq
@ _InterlockedExchange_nf
@ _InterlockedIncrement_nf
@ _InterlockedExchange_acq
@ _InterlockedCompareExchange128_rel
@ _InterlockedCompareExchange128_acq
@ _InterlockedCompareExchange_acq
@ _InterlockedExchangeAdd_nf
@ _InterlockedCompareExchange_nf
@ _InterlockedDecrement_rel
@ _InterlockedExchangeSub
@ _InterlockedExchangeAdd_acq
@ _InterlockedIncrement_rel
@ _InterlockedCompareExchange128_nf
@ _InterlockedCompareExchange128
@ _InterlockedExchange_rel
@ _InterlockedCompareExchange
@ _InterlockedDecrement_nf
@ _InterlockedExchangeAdd
@ _InterlockedDecrement_acq
@ _InterlockedCompareExchange_rel
static Value * EmitX86ConvertToMask(CodeGenFunction &CGF, Value *In)
static Value * EmitISOVolatileStore(CodeGenFunction &CGF, const CallExpr *E)
static Value * EmitSystemZIntrinsicWithCC(CodeGenFunction &CGF, unsigned IntrinsicID, const CallExpr *E)
Handle a SystemZ function in which the final argument is a pointer to an int that receives the post-i...
static RValue EmitHipStdParUnsupportedBuiltin(CodeGenFunction *CGF, const FunctionDecl *FD)
static llvm::Value * EmitX86BitTestIntrinsic(CodeGenFunction &CGF, BitTest BT, const CallExpr *E, Value *BitBase, Value *BitPos)
static RValue EmitCheckedUnsignedMultiplySignedResult(CodeGenFunction &CGF, const clang::Expr *Op1, WidthAndSignedness Op1Info, const clang::Expr *Op2, WidthAndSignedness Op2Info, const clang::Expr *ResultArg, QualType ResultQTy, WidthAndSignedness ResultInfo)
static Value * emitCallMaybeConstrainedFPBuiltin(CodeGenFunction &CGF, unsigned IntrinsicID, unsigned ConstrainedIntrinsicID, llvm::Type *Ty, ArrayRef< Value * > Args)
static RValue emitLibraryCall(CodeGenFunction &CGF, const FunctionDecl *FD, const CallExpr *E, llvm::Constant *calleeValue)
static Value * handleAsDoubleBuiltin(CodeGenFunction &CGF, const CallExpr *E)
static Value * EmitNontemporalLoad(CodeGenFunction &CGF, const CallExpr *E)
static Value * EmitX86Ternlog(CodeGenFunction &CGF, bool ZeroMask, ArrayRef< Value * > Ops)
static llvm::AtomicOrdering getBitTestAtomicOrdering(BitTest::InterlockingKind I)
#define MMA_VARIANTS_B1_XOR(geom, type)
#define MMA_VARIANTS_I4(geom, type)
static Value * EmitX86ConvertIntToFp(CodeGenFunction &CGF, const CallExpr *E, ArrayRef< Value * > Ops, bool IsSigned)
static Value * packTBLDVectorList(CodeGenFunction &CGF, ArrayRef< Value * > Ops, Value *ExtOp, Value *IndexOp, llvm::Type *ResTy, unsigned IntID, const char *Name)
static Value * EmitAbs(CodeGenFunction &CGF, Value *ArgValue, bool HasNSW)
static Value * EmitX86ScalarSelect(CodeGenFunction &CGF, Value *Mask, Value *Op0, Value *Op1)
static Value * EmitAtomicDecrementValue(CodeGenFunction &CGF, const CallExpr *E, AtomicOrdering Ordering=AtomicOrdering::SequentiallyConsistent)
static Value * handleHlslClip(const CallExpr *E, CodeGenFunction *CGF)
static bool AArch64SIMDIntrinsicsProvenSorted
static const ARMVectorIntrinsicInfo AArch64SISDIntrinsicMap[]
CodeGenFunction::ComplexPairTy ComplexPairTy
#define ALIAS(NAME, TOK, FLAGS)
llvm::MachO::Record Record
static std::string getName(const CallEvent &Call)
static std::string toString(const clang::SanitizerSet &Sanitizers)
Produce a string containing comma-separated names of sanitizers in Sanitizers set.
HLSLResourceBindingAttr::RegisterType RegisterType
static QualType getPointeeType(const MemRegion *R)
Enumerates target-specific builtins in their own namespaces within namespace clang.
Defines the clang::TargetOptions class.
C Language Family Type Representation.
__DEVICE__ float modf(float __x, float *__iptr)
__DEVICE__ double nan(const char *)
__device__ __2f16 float __ockl_bool s
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
CharUnits getTypeAlignInChars(QualType T) const
Return the ABI-specified alignment of a (complete) type T, in characters.
unsigned getIntWidth(QualType T) const
const ASTRecordLayout & getASTRecordLayout(const RecordDecl *D) const
Get or compute information about the layout of the specified record (struct/union/class) D,...
Builtin::Context & BuiltinInfo
QualType getConstantArrayType(QualType EltTy, const llvm::APInt &ArySize, const Expr *SizeExpr, ArraySizeModifier ASM, unsigned IndexTypeQuals) const
Return the unique reference to the type for a constant array of the specified element type.
QualType getBaseElementType(const ArrayType *VAT) const
Return the innermost element type of an array type.
QualType getObjCIdType() const
Represents the Objective-CC id type.
bool hasSameUnqualifiedType(QualType T1, QualType T2) const
Determine whether the given types are equivalent after cvr-qualifiers have been removed.
const ArrayType * getAsArrayType(QualType T) const
Type Query functions.
uint64_t getTypeSize(QualType T) const
Return the size of the specified (complete) type T, in bits.
CharUnits getTypeSizeInChars(QualType T) const
Return the size of the specified (complete) type T, in characters.
QualType GetBuiltinType(unsigned ID, GetBuiltinTypeError &Error, unsigned *IntegerConstantArgs=nullptr) const
Return the type for the specified builtin.
const TargetInfo & getTargetInfo() const
CharUnits toCharUnitsFromBits(int64_t BitSize) const
Convert a size in bits to a size in characters.
unsigned getTargetAddressSpace(LangAS AS) const
@ GE_Missing_type
Missing a type.
ASTRecordLayout - This class contains layout information for one RecordDecl, which is a struct/union/...
uint64_t getFieldOffset(unsigned FieldNo) const
getFieldOffset - Get the offset of the given field index, in bits.
Represents an array type, per C99 6.7.5.2 - Array Declarators.
QualType getElementType() const
static std::unique_ptr< AtomicScopeModel > create(AtomicScopeModelKind K)
Create an atomic scope model by AtomicScopeModelKind.
bool isLibFunction(unsigned ID) const
Return true if this is a builtin for a libc/libm function, with a "__builtin_" prefix (e....
llvm::StringRef getName(unsigned ID) const
Return the identifier name for the specified builtin, e.g.
bool isConstWithoutErrnoAndExceptions(unsigned ID) const
Return true if this function has no side effects and doesn't read memory, except for possibly errno o...
bool isConstWithoutExceptions(unsigned ID) const
bool isConst(unsigned ID) const
Return true if this function has no side effects and doesn't read memory.
CallExpr - Represents a function call (C99 6.5.2.2, C++ [expr.call]).
CharUnits - This is an opaque type for sizes expressed in character units.
llvm::Align getAsAlign() const
getAsAlign - Returns Quantity as a valid llvm::Align, Beware llvm::Align assumes power of two 8-bit b...
QuantityType getQuantity() const
getQuantity - Get the raw integer representation of this quantity.
static CharUnits One()
One - Construct a CharUnits quantity of one.
static CharUnits fromQuantity(QuantityType Quantity)
fromQuantity - Construct a CharUnits quantity from a raw integer type.
XRayInstrSet XRayInstrumentationBundle
Set of XRay instrumentation kinds to emit.
ABIArgInfo - Helper class to encapsulate information about how a specific C type should be passed to ...
Like RawAddress, an abstract representation of an aligned address, but the pointer contained in this ...
llvm::Value * emitRawPointer(CodeGenFunction &CGF) const
Return the pointer contained in this class after authenticating it and adding offset to it if necessa...
CharUnits getAlignment() const
llvm::Type * getElementType() const
Return the type of the values stored in this address.
Address withElementType(llvm::Type *ElemTy) const
Return address with different element type, but same pointer and alignment.
Address withAlignment(CharUnits NewAlignment) const
Return address with different alignment, but same pointer and element type.
llvm::PointerType * getType() const
Return the type of the pointer value.
Address getAddress() const
A scoped helper to set the current debug location to the specified location or preferred location of ...
static ApplyDebugLocation CreateArtificial(CodeGenFunction &CGF)
Apply TemporaryLocation if it is valid.
static ApplyDebugLocation CreateEmpty(CodeGenFunction &CGF)
Set the IRBuilder to not attach debug locations.
llvm::StoreInst * CreateStore(llvm::Value *Val, Address Addr, bool IsVolatile=false)
llvm::StoreInst * CreateAlignedStore(llvm::Value *Val, llvm::Value *Addr, CharUnits Align, bool IsVolatile=false)
Address CreateGEP(CodeGenFunction &CGF, Address Addr, llvm::Value *Index, const llvm::Twine &Name="")
llvm::CallInst * CreateMemMove(Address Dest, Address Src, llvm::Value *Size, bool IsVolatile=false)
llvm::CallInst * CreateMemCpyInline(Address Dest, Address Src, uint64_t Size)
llvm::AtomicRMWInst * CreateAtomicRMW(llvm::AtomicRMWInst::BinOp Op, Address Addr, llvm::Value *Val, llvm::AtomicOrdering Ordering, llvm::SyncScope::ID SSID=llvm::SyncScope::System)
llvm::CallInst * CreateMemSetInline(Address Dest, llvm::Value *Value, uint64_t Size)
llvm::StoreInst * CreateDefaultAlignedStore(llvm::Value *Val, llvm::Value *Addr, bool IsVolatile=false)
llvm::CallInst * CreateMemSet(Address Dest, llvm::Value *Value, llvm::Value *Size, bool IsVolatile=false)
Address CreateStructGEP(Address Addr, unsigned Index, const llvm::Twine &Name="")
llvm::AtomicCmpXchgInst * CreateAtomicCmpXchg(Address Addr, llvm::Value *Cmp, llvm::Value *New, llvm::AtomicOrdering SuccessOrdering, llvm::AtomicOrdering FailureOrdering, llvm::SyncScope::ID SSID=llvm::SyncScope::System)
llvm::LoadInst * CreateLoad(Address Addr, const llvm::Twine &Name="")
Address CreateConstByteGEP(Address Addr, CharUnits Offset, const llvm::Twine &Name="")
Address CreateLaunderInvariantGroup(Address Addr)
llvm::CallInst * CreateMemCpy(Address Dest, Address Src, llvm::Value *Size, bool IsVolatile=false)
llvm::LoadInst * CreateAlignedLoad(llvm::Type *Ty, llvm::Value *Addr, CharUnits Align, const llvm::Twine &Name="")
Address CreateAddrSpaceCast(Address Addr, llvm::Type *Ty, llvm::Type *ElementTy, const llvm::Twine &Name="")
Address CreateConstInBoundsGEP(Address Addr, uint64_t Index, const llvm::Twine &Name="")
Given addr = T* ... produce name = getelementptr inbounds addr, i64 index where i64 is actually the t...
Address CreateInBoundsGEP(Address Addr, ArrayRef< llvm::Value * > IdxList, llvm::Type *ElementType, CharUnits Align, const Twine &Name="")
virtual std::string getDeviceSideName(const NamedDecl *ND)=0
Returns function or variable name on device side even if the current compilation is for host.
virtual llvm::GlobalVariable * getThrowInfo(QualType T)
All available information about a concrete callee.
static CGCallee forDirect(llvm::Constant *functionPtr, const CGCalleeInfo &abstractInfo=CGCalleeInfo())
llvm::DIType * getOrCreateStandaloneType(QualType Ty, SourceLocation Loc)
Emit standalone debug info for a type.
llvm::DILocation * CreateTrapFailureMessageFor(llvm::DebugLoc TrapLocation, StringRef Category, StringRef FailureMsg)
Create a debug location from TrapLocation that adds an artificial inline frame where the frame name i...
CGFunctionInfo - Class to encapsulate the information about a function definition.
ABIArgInfo & getReturnInfo()
MutableArrayRef< ArgInfo > arguments()
virtual void EmitGCMemmoveCollectable(CodeGen::CodeGenFunction &CGF, Address DestPtr, Address SrcPtr, llvm::Value *Size)=0
EnqueuedBlockInfo emitOpenCLEnqueuedBlock(CodeGenFunction &CGF, const Expr *E)
CallArgList - Type for representing both the value and type of arguments in a call.
void add(RValue rvalue, QualType type)
CodeGenFunction - This class organizes the per-function state that is used while generating LLVM code...
llvm::Value * EmitAArch64BuiltinExpr(unsigned BuiltinID, const CallExpr *E, llvm::Triple::ArchType Arch)
llvm::Value * EmitFromMemory(llvm::Value *Value, QualType Ty)
EmitFromMemory - Change a scalar value from its memory representation to its value representation.
llvm::Value * EmitMSVCBuiltinExpr(MSVCIntrin BuiltinID, const CallExpr *E)
void FinishFunction(SourceLocation EndLoc=SourceLocation())
FinishFunction - Complete IR generation of the current function.
llvm::Value * EmitLifetimeStart(llvm::TypeSize Size, llvm::Value *Addr)
std::pair< RValue, llvm::Value * > EmitAtomicCompareExchange(LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc, llvm::AtomicOrdering Success=llvm::AtomicOrdering::SequentiallyConsistent, llvm::AtomicOrdering Failure=llvm::AtomicOrdering::SequentiallyConsistent, bool IsWeak=false, AggValueSlot Slot=AggValueSlot::ignored())
static TypeEvaluationKind getEvaluationKind(QualType T)
getEvaluationKind - Return the TypeEvaluationKind of QualType T.
llvm::Value * EmitSVEPredicateCast(llvm::Value *Pred, llvm::ScalableVectorType *VTy)
llvm::CallInst * EmitTrapCall(llvm::Intrinsic::ID IntrID)
Emit a call to trap or debugtrap and attach function attribute "trap-func-name" if specified.
SanitizerSet SanOpts
Sanitizers enabled for this function.
RValue EmitBuiltinIsAligned(const CallExpr *E)
Emit IR for __builtin_is_aligned.
LValue EmitAggExprToLValue(const Expr *E)
EmitAggExprToLValue - Emit the computation of the specified expression of aggregate type into a tempo...
void EmitNonNullArgCheck(RValue RV, QualType ArgType, SourceLocation ArgLoc, AbstractCallee AC, unsigned ParmNum)
Create a check for a function parameter that may potentially be declared as non-null.
llvm::Value * EmitHLSLBuiltinExpr(unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue)
llvm::Value * EmitHexagonBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
void GetAArch64SVEProcessedOperands(unsigned BuiltinID, const CallExpr *E, SmallVectorImpl< llvm::Value * > &Ops, SVETypeFlags TypeFlags)
llvm::Value * EmitAMDGPUBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
void EmitLifetimeEnd(llvm::Value *Size, llvm::Value *Addr)
void pushLifetimeExtendedDestroy(CleanupKind kind, Address addr, QualType type, Destroyer *destroyer, bool useEHCleanupForArray)
void EmitTrapCheck(llvm::Value *Checked, SanitizerHandler CheckHandlerID, bool NoMerge=false)
Create a basic block that will call the trap intrinsic, and emit a conditional branch to it,...
llvm::Value * EmitCheckedArgForBuiltin(const Expr *E, BuiltinCheckKind Kind)
Emits an argument for a call to a builtin.
llvm::Value * EmitSVEGatherLoad(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
LValue EmitHLSLOutArgExpr(const HLSLOutArgExpr *E, CallArgList &Args, QualType Ty)
CleanupKind getARCCleanupKind()
Retrieves the default cleanup kind for an ARC cleanup.
llvm::Value * EmitRISCVCpuSupports(const CallExpr *E)
llvm::Value * EmitNVPTXBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
llvm::Value * EmitAArch64SVEBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
RValue emitBuiltinOSLogFormat(const CallExpr &E)
Emit IR for __builtin_os_log_format.
LValue EmitLValue(const Expr *E, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
EmitLValue - Emit code to compute a designator that specifies the location of the expression.
llvm::Value * EmitVAStartEnd(llvm::Value *ArgValue, bool IsStart)
Emits a call to an LLVM variable-argument intrinsic, either llvm.va_start or llvm....
llvm::Value * EmitSVEMaskedStore(const CallExpr *, SmallVectorImpl< llvm::Value * > &Ops, unsigned BuiltinID)
llvm::Value * EmitSVEReinterpret(llvm::Value *Val, llvm::Type *Ty)
llvm::BasicBlock * createBasicBlock(const Twine &name="", llvm::Function *parent=nullptr, llvm::BasicBlock *before=nullptr)
createBasicBlock - Create an LLVM basic block.
llvm::Value * EmitSEHExceptionInfo()
RValue EmitBuiltinAlignTo(const CallExpr *E, bool AlignUp)
Emit IR for __builtin_align_up/__builtin_align_down.
const LangOptions & getLangOpts() const
llvm::Value * EmitLoadOfCountedByField(const Expr *Base, const FieldDecl *FAMDecl, const FieldDecl *CountDecl)
Build an expression accessing the "counted_by" field.
void ProcessOrderScopeAMDGCN(llvm::Value *Order, llvm::Value *Scope, llvm::AtomicOrdering &AO, llvm::SyncScope::ID &SSID)
llvm::Constant * EmitCheckTypeDescriptor(QualType T)
Emit a description of a type in a format suitable for passing to a runtime sanitizer handler.
void EmitBlock(llvm::BasicBlock *BB, bool IsFinished=false)
EmitBlock - Emit the given block.
void EmitUnreachable(SourceLocation Loc)
Emit a reached-unreachable diagnostic if Loc is valid and runtime checking is enabled.
llvm::Value * EmitSVETupleCreate(const SVETypeFlags &TypeFlags, llvm::Type *ReturnType, ArrayRef< llvm::Value * > Ops)
Address makeNaturalAddressForPointer(llvm::Value *Ptr, QualType T, CharUnits Alignment=CharUnits::Zero(), bool ForPointeeType=false, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
Construct an address with the natural alignment of T.
llvm::Value * EmitCheckedArgForAssume(const Expr *E)
Emits an argument for a call to a __builtin_assume.
ComplexPairTy EmitComplexExpr(const Expr *E, bool IgnoreReal=false, bool IgnoreImag=false)
EmitComplexExpr - Emit the computation of the specified expression of complex type,...
void EmitAnyExprToMem(const Expr *E, Address Location, Qualifiers Quals, bool IsInitializer)
EmitAnyExprToMem - Emits the code necessary to evaluate an arbitrary expression into the given memory...
TypeCheckKind
Situations in which we might emit a check for the suitability of a pointer or glvalue.
@ TCK_Store
Checking the destination of a store. Must be suitably sized and aligned.
@ TCK_Load
Checking the operand of a load. Must be suitably sized and aligned.
llvm::Value * EmitSMELdrStr(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
llvm::Value * EmitNeonSplat(llvm::Value *V, llvm::Constant *Idx, const llvm::ElementCount &Count)
llvm::Type * ConvertTypeForMem(QualType T)
llvm::Value * EmitSVEMaskedLoad(const CallExpr *, llvm::Type *ReturnTy, SmallVectorImpl< llvm::Value * > &Ops, unsigned BuiltinID, bool IsZExtReturn)
bool AlwaysEmitXRayCustomEvents() const
AlwaysEmitXRayCustomEvents - Return true if we must unconditionally emit XRay custom event handling c...
llvm::Value * EmitSVEDupX(llvm::Value *Scalar)
RawAddress CreateMemTemp(QualType T, const Twine &Name="tmp", RawAddress *Alloca=nullptr)
CreateMemTemp - Create a temporary memory object of the given type, with appropriate alignmen and cas...
@ Default
! No language constraints on evaluation order.
const TargetInfo & getTarget() const
llvm::Value * vectorWrapScalar16(llvm::Value *Op)
llvm::Function * LookupNeonLLVMIntrinsic(unsigned IntrinsicID, unsigned Modifier, llvm::Type *ArgTy, const CallExpr *E)
llvm::Value * getTypeSize(QualType Ty)
Returns calculated size of the specified type.
llvm::Value * EmitSEHExceptionCode()
Address EmitPointerWithAlignment(const Expr *Addr, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
EmitPointerWithAlignment - Given an expression with a pointer type, emit the value and compute our be...
llvm::Value * EmitTargetBuiltinExpr(unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue)
EmitTargetBuiltinExpr - Emit the given builtin call.
RValue EmitAnyExprToTemp(const Expr *E)
EmitAnyExprToTemp - Similarly to EmitAnyExpr(), however, the result will always be accessible even if...
RValue EmitCoroutineIntrinsic(const CallExpr *E, unsigned int IID)
llvm::Value * EmitAArch64SMEBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
RValue EmitAMDGPUDevicePrintfCallExpr(const CallExpr *E)
Address EmitArrayToPointerDecay(const Expr *Array, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr)
RValue EmitBuiltinNewDeleteCall(const FunctionProtoType *Type, const CallExpr *TheCallExpr, bool IsDelete)
llvm::Value * EmitRISCVCpuInit()
void EmitCheck(ArrayRef< std::pair< llvm::Value *, SanitizerKind::SanitizerOrdinal > > Checked, SanitizerHandler Check, ArrayRef< llvm::Constant * > StaticArgs, ArrayRef< llvm::Value * > DynamicArgs)
Create a basic block that will either trap or call a handler function in the UBSan runtime with the p...
RValue EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue)
RValue EmitAnyExpr(const Expr *E, AggValueSlot aggSlot=AggValueSlot::ignored(), bool ignoreResult=false)
EmitAnyExpr - Emit code to compute the specified expression which can have any type.
llvm::Value * EmitSVEPrefetchLoad(const SVETypeFlags &TypeFlags, SmallVectorImpl< llvm::Value * > &Ops, unsigned BuiltinID)
llvm::Value * EmitAArch64CompareBuiltinExpr(llvm::Value *Op, llvm::Type *Ty, const llvm::CmpInst::Predicate Fp, const llvm::CmpInst::Predicate Ip, const llvm::Twine &Name="")
void StartFunction(GlobalDecl GD, QualType RetTy, llvm::Function *Fn, const CGFunctionInfo &FnInfo, const FunctionArgList &Args, SourceLocation Loc=SourceLocation(), SourceLocation StartLoc=SourceLocation())
Emit code for the start of a function.
AggValueSlot CreateAggTemp(QualType T, const Twine &Name="tmp", RawAddress *Alloca=nullptr)
CreateAggTemp - Create a temporary memory object for the given aggregate type.
llvm::ScalableVectorType * getSVEType(const SVETypeFlags &TypeFlags)
RValue emitRotate(const CallExpr *E, bool IsRotateRight)
llvm::Constant * EmitCheckSourceLocation(SourceLocation Loc)
Emit a description of a source location in a format suitable for passing to a runtime sanitizer handl...
void ErrorUnsupported(const Stmt *S, const char *Type)
ErrorUnsupported - Print out an error that codegen doesn't support the specified stmt yet.
CGDebugInfo * getDebugInfo()
const FieldDecl * FindFlexibleArrayMemberFieldAndOffset(ASTContext &Ctx, const RecordDecl *RD, const FieldDecl *FAMDecl, uint64_t &Offset)
llvm::Value * EmitRISCVCpuIs(const CallExpr *E)
Address EmitVAListRef(const Expr *E)
llvm::Value * EmitNeonShiftVector(llvm::Value *V, llvm::Type *Ty, bool negateForRightShift)
llvm::Value * EmitSVEMovl(const SVETypeFlags &TypeFlags, llvm::ArrayRef< llvm::Value * > Ops, unsigned BuiltinID)
void emitAlignmentAssumption(llvm::Value *PtrValue, QualType Ty, SourceLocation Loc, SourceLocation AssumptionLoc, llvm::Value *Alignment, llvm::Value *OffsetValue=nullptr)
const TargetCodeGenInfo & getTargetHooks() const
llvm::Value * EmitARMBuiltinExpr(unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue, llvm::Triple::ArchType Arch)
llvm::Type * getEltType(const SVETypeFlags &TypeFlags)
void EmitAggExpr(const Expr *E, AggValueSlot AS)
EmitAggExpr - Emit the computation of the specified expression of aggregate type.
bool ShouldXRayInstrumentFunction() const
ShouldXRayInstrument - Return true if the current function should be instrumented with XRay nop sleds...
llvm::Value * EmitSVEPMull(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned BuiltinID)
llvm::Value * EmitToMemory(llvm::Value *Value, QualType Ty)
EmitToMemory - Change a scalar value from its value representation to its in-memory representation.
bool IsInPreservedAIRegion
True if CodeGen currently emits code inside presereved access index region.
llvm::Value * EmitARCRetain(QualType type, llvm::Value *value)
bool AlwaysEmitXRayTypedEvents() const
AlwaysEmitXRayTypedEvents - Return true if clang must unconditionally emit XRay typed event handling ...
void SetSqrtFPAccuracy(llvm::Value *Val)
Set the minimum required accuracy of the given sqrt operation based on CodeGenOpts.
RValue EmitCall(const CGFunctionInfo &CallInfo, const CGCallee &Callee, ReturnValueSlot ReturnValue, const CallArgList &Args, llvm::CallBase **CallOrInvoke, bool IsMustTail, SourceLocation Loc, bool IsVirtualFunctionPointerThunk=false)
EmitCall - Generate a call of the given function, expecting the given result type,...
llvm::Value * EmitSVEScatterStore(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
llvm::CallInst * EmitNounwindRuntimeCall(llvm::FunctionCallee callee, const Twine &name="")
ASTContext & getContext() const
llvm::Function * generateBuiltinOSLogHelperFunction(const analyze_os_log::OSLogBufferLayout &Layout, CharUnits BufferAlignment)
llvm::Value * EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty, SourceLocation Loc, AlignmentSource Source=AlignmentSource::Type, bool isNontemporal=false)
EmitLoadOfScalar - Load a scalar value from an address, taking care to appropriately convert from the...
CGCallee EmitCallee(const Expr *E)
const Decl * CurFuncDecl
CurFuncDecl - Holds the Decl for the current outermost non-closure context.
llvm::Value * EmitScalarOrConstFoldImmArg(unsigned ICEArguments, unsigned Idx, const CallExpr *E)
void checkTargetFeatures(const CallExpr *E, const FunctionDecl *TargetDecl)
llvm::Value * BuildVector(ArrayRef< llvm::Value * > Ops)
llvm::Value * EmitWebAssemblyBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
llvm::Value * EmitARMCDEBuiltinExpr(unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue, llvm::Triple::ArchType Arch)
llvm::Value * GetCountedByFieldExprGEP(const Expr *Base, const FieldDecl *FAMDecl, const FieldDecl *CountDecl)
llvm::CallInst * EmitRuntimeCall(llvm::FunctionCallee callee, const Twine &name="")
llvm::Value * EmitSVEPredicateTupleCast(llvm::Value *PredTuple, llvm::StructType *Ty)
llvm::Type * ConvertType(QualType T)
void EmitWritebacks(const CallArgList &Args)
EmitWriteback - Emit callbacks for function.
llvm::CallBase * EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee, ArrayRef< llvm::Value * > args, const Twine &name="")
llvm::Value * EmitSystemZBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
llvm::Value * EmitSMEReadWrite(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
void EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, LValue LV, QualType Type, SanitizerSet SkippedChecks=SanitizerSet(), llvm::Value *ArraySize=nullptr)
llvm::Value * EmitSMELd1St1(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
llvm::Value * EmitPPCBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
static Destroyer destroyARCStrongPrecise
void EmitARCIntrinsicUse(ArrayRef< llvm::Value * > values)
RValue EmitNVPTXDevicePrintfCallExpr(const CallExpr *E)
llvm::Value * EvaluateExprAsBool(const Expr *E)
EvaluateExprAsBool - Perform the usual unary conversions on the specified expression and compare the ...
llvm::Value * EmitSVEStructLoad(const SVETypeFlags &TypeFlags, SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
Address EmitMSVAListRef(const Expr *E)
Emit a "reference" to a __builtin_ms_va_list; this is always the value of the expression,...
llvm::Value * EmitCheckedInBoundsGEP(llvm::Type *ElemTy, llvm::Value *Ptr, ArrayRef< llvm::Value * > IdxList, bool SignedIndices, bool IsSubtraction, SourceLocation Loc, const Twine &Name="")
Same as IRBuilder::CreateInBoundsGEP, but additionally emits a check to detect undefined behavior whe...
llvm::Value * EmitNeonRShiftImm(llvm::Value *Vec, llvm::Value *Amt, llvm::Type *Ty, bool usgn, const char *name)
SmallVector< llvm::Type *, 2 > getSVEOverloadTypes(const SVETypeFlags &TypeFlags, llvm::Type *ReturnType, ArrayRef< llvm::Value * > Ops)
static bool hasAggregateEvaluationKind(QualType T)
LValue MakeAddrLValue(Address Addr, QualType T, AlignmentSource Source=AlignmentSource::Type)
llvm::Value * EmitARMMVEBuiltinExpr(unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue, llvm::Triple::ArchType Arch)
LValue MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
llvm::Value * EmitSVEStructStore(const SVETypeFlags &TypeFlags, SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
Address GetAddrOfLocalVar(const VarDecl *VD)
GetAddrOfLocalVar - Return the address of a local variable.
llvm::Value * EmitSEHAbnormalTermination()
llvm::Value * EmitX86BuiltinExpr(unsigned BuiltinID, const CallExpr *E)
Address ReturnValue
ReturnValue - The temporary alloca to hold the return value.
llvm::Value * EmitSVEAllTruePred(const SVETypeFlags &TypeFlags)
RValue GetUndefRValue(QualType Ty)
GetUndefRValue - Get an appropriate 'undef' rvalue for the given type.
llvm::Type * SVEBuiltinMemEltTy(const SVETypeFlags &TypeFlags)
SVEBuiltinMemEltTy - Returns the memory element type for this memory access builtin.
llvm::LLVMContext & getLLVMContext()
llvm::Value * EmitScalarExpr(const Expr *E, bool IgnoreResultAssign=false)
EmitScalarExpr - Emit the computation of the specified expression of LLVM scalar type,...
void AddAMDGPUFenceAddressSpaceMMRA(llvm::Instruction *Inst, const CallExpr *E)
llvm::Value * EmitSMEZero(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
llvm::Value * EmitRISCVBuiltinExpr(unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue)
llvm::Value * EmitCommonNeonBuiltinExpr(unsigned BuiltinID, unsigned LLVMIntrinsic, unsigned AltLLVMIntrinsic, const char *NameHint, unsigned Modifier, const CallExpr *E, SmallVectorImpl< llvm::Value * > &Ops, Address PtrOp0, Address PtrOp1, llvm::Triple::ArchType Arch)
llvm::Value * EmitNeonCall(llvm::Function *F, SmallVectorImpl< llvm::Value * > &O, const char *name, unsigned shift=0, bool rightshift=false)
llvm::Value * EmitAnnotationCall(llvm::Function *AnnotationFn, llvm::Value *AnnotatedVal, StringRef AnnotationStr, SourceLocation Location, const AnnotateAttr *Attr)
Emit an annotation call (intrinsic).
llvm::ScalableVectorType * getSVEPredType(const SVETypeFlags &TypeFlags)
llvm::Value * EmitSVEGatherPrefetch(const SVETypeFlags &TypeFlags, SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
void EmitStoreOfScalar(llvm::Value *Value, Address Addr, bool Volatile, QualType Ty, AlignmentSource Source=AlignmentSource::Type, bool isInit=false, bool isNontemporal=false)
EmitStoreOfScalar - Store a scalar value to an address, taking care to appropriately convert from the...
llvm::Value * EmitSPIRVBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
llvm::Value * EmitBPFBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
llvm::Value * EmitSVETupleSetOrGet(const SVETypeFlags &TypeFlags, ArrayRef< llvm::Value * > Ops)
This class organizes the cross-function state that is used while generating LLVM code.
CGHLSLRuntime & getHLSLRuntime()
Return a reference to the configured HLSL runtime.
llvm::Module & getModule() const
llvm::FunctionCallee CreateRuntimeFunction(llvm::FunctionType *Ty, StringRef Name, llvm::AttributeList ExtraAttrs=llvm::AttributeList(), bool Local=false, bool AssumeConvergent=false)
Create or return a runtime function declaration with the specified type and name.
llvm::Constant * getRawFunctionPointer(GlobalDecl GD, llvm::Type *Ty=nullptr)
Return a function pointer for a reference to the given function.
llvm::Constant * getBuiltinLibFunction(const FunctionDecl *FD, unsigned BuiltinID)
Given a builtin id for a function like "__builtin_fabsf", return a Function* for "fabsf".
DiagnosticsEngine & getDiags() const
void ErrorUnsupported(const Stmt *S, const char *Type)
Print out an error that codegen doesn't support the specified stmt yet.
const LangOptions & getLangOpts() const
CGCUDARuntime & getCUDARuntime()
Return a reference to the configured CUDA runtime.
CodeGenTypes & getTypes()
CGOpenCLRuntime & getOpenCLRuntime()
Return a reference to the configured OpenCL runtime.
const TargetInfo & getTarget() const
const llvm::DataLayout & getDataLayout() const
void Error(SourceLocation loc, StringRef error)
Emit a general error that something can't be done.
CGCXXABI & getCXXABI() const
llvm::Constant * GetFunctionStart(const ValueDecl *Decl)
const llvm::Triple & getTriple() const
void DecorateInstructionWithTBAA(llvm::Instruction *Inst, TBAAAccessInfo TBAAInfo)
DecorateInstructionWithTBAA - Decorate the instruction with a TBAA tag.
llvm::Constant * CreateRuntimeVariable(llvm::Type *Ty, StringRef Name)
Create a new runtime global variable with the specified type and name.
TBAAAccessInfo getTBAAAccessInfo(QualType AccessType)
getTBAAAccessInfo - Get TBAA information that describes an access to an object of the given type.
ASTContext & getContext() const
const CodeGenOptions & getCodeGenOpts() const
StringRef getMangledName(GlobalDecl GD)
CharUnits getNaturalPointeeTypeAlignment(QualType T, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr)
llvm::LLVMContext & getLLVMContext()
llvm::Function * getIntrinsic(unsigned IID, ArrayRef< llvm::Type * > Tys={})
CGObjCRuntime & getObjCRuntime()
Return a reference to the configured Objective-C runtime.
void SetLLVMFunctionAttributes(GlobalDecl GD, const CGFunctionInfo &Info, llvm::Function *F, bool IsThunk)
Set the LLVM function attributes (sext, zext, etc).
void SetLLVMFunctionAttributesForDefinition(const Decl *D, llvm::Function *F)
Set the LLVM function attributes which only apply to a function definition.
ConstantAddress GetAddrOfConstantCString(const std::string &Str, const char *GlobalName=nullptr)
Returns a pointer to a character array containing the literal and a terminating '\0' character.
llvm::Type * ConvertType(QualType T)
ConvertType - Convert type T into a llvm::Type.
llvm::FunctionType * GetFunctionType(const CGFunctionInfo &Info)
GetFunctionType - Get the LLVM function type for.
const CGFunctionInfo & arrangeBuiltinFunctionDeclaration(QualType resultType, const FunctionArgList &args)
A builtin function is a freestanding function using the default C conventions.
const CGFunctionInfo & arrangeBuiltinFunctionCall(QualType resultType, const CallArgList &args)
llvm::Constant * emitAbstract(const Expr *E, QualType T)
Emit the result of the given expression as an abstract constant, asserting that it succeeded.
Information for lazily generating a cleanup.
FunctionArgList - Type for representing both the decl and type of parameters to a function.
LValue - This represents an lvalue references.
llvm::Value * getRawBitFieldPointer(CodeGenFunction &CGF) const
llvm::Value * getPointer(CodeGenFunction &CGF) const
Address getAddress() const
void setNontemporal(bool Value)
llvm::Value * emitRawPointer(CodeGenFunction &CGF) const
RValue - This trivial value class is used to represent the result of an expression that is evaluated.
llvm::Value * getAggregatePointer(QualType PointeeType, CodeGenFunction &CGF) const
static RValue getIgnored()
static RValue get(llvm::Value *V)
static RValue getAggregate(Address addr, bool isVolatile=false)
Convert an Address to an RValue.
static RValue getComplex(llvm::Value *V1, llvm::Value *V2)
llvm::Value * getScalarVal() const
getScalarVal() - Return the Value* of this scalar value.
An abstract representation of an aligned address.
llvm::Value * getPointer() const
static RawAddress invalid()
ReturnValueSlot - Contains the address where the return value of a function can be stored,...
virtual llvm::Value * encodeReturnAddress(CodeGen::CodeGenFunction &CGF, llvm::Value *Address) const
Performs the code-generation required to convert the address of an instruction into a return address ...
virtual llvm::Value * decodeReturnAddress(CodeGen::CodeGenFunction &CGF, llvm::Value *Address) const
Performs the code-generation required to convert a return address as stored by the system into the ac...
const T & getABIInfo() const
virtual int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const
Determines the DWARF register number for the stack pointer, for exception-handling purposes.
virtual llvm::Value * testFPKind(llvm::Value *V, unsigned BuiltinID, CGBuilderTy &Builder, CodeGenModule &CGM) const
Performs a target specific test of a floating point value for things like IsNaN, Infinity,...
Complex values, per C99 6.2.5p11.
Represents a concrete matrix type with constant number of rows and columns.
Represents a sugar type with __counted_by or __sized_by annotations, including their _or_null variant...
DynamicCountPointerKind getKind() const
RecordDecl * getOuterLexicalRecordContext()
Retrieve the outermost lexically enclosing record context.
bool isImplicit() const
isImplicit - Indicates whether the declaration was implicitly generated by the implementation.
FunctionDecl * getAsFunction() LLVM_READONLY
Returns the function itself, or the templated function if this is a function template.
DeclContext * getDeclContext()
static bool isFlexibleArrayMemberLike(ASTContext &Context, const Decl *D, QualType Ty, LangOptions::StrictFlexArraysLevelKind StrictFlexArraysLevel, bool IgnoreTemplateOrMacroSubstitution)
Whether it resembles a flexible array member.
Concrete class used by the front-end to report problems and issues.
DiagnosticBuilder Report(SourceLocation Loc, unsigned DiagID)
Issue the message to the client.
This represents one expression.
bool EvaluateAsInt(EvalResult &Result, const ASTContext &Ctx, SideEffectsKind AllowSideEffects=SE_NoSideEffects, bool InConstantContext=false) const
EvaluateAsInt - Return true if this is a constant which we can fold and convert to an integer,...
Expr * IgnoreParenCasts() LLVM_READONLY
Skip past any parentheses and casts which might surround this expression until reaching a fixed point...
llvm::APSInt EvaluateKnownConstInt(const ASTContext &Ctx, SmallVectorImpl< PartialDiagnosticAt > *Diag=nullptr) const
EvaluateKnownConstInt - Call EvaluateAsRValue and return the folded integer.
Expr * IgnoreParenImpCasts() LLVM_READONLY
Skip past any parentheses and implicit casts which might surround this expression until reaching a fi...
bool EvaluateAsFloat(llvm::APFloat &Result, const ASTContext &Ctx, SideEffectsKind AllowSideEffects=SE_NoSideEffects, bool InConstantContext=false) const
EvaluateAsFloat - Return true if this is a constant which we can fold and convert to a floating point...
Expr * IgnoreParens() LLVM_READONLY
Skip past any parentheses which might surround this expression until reaching a fixed point.
@ NPC_ValueDependentIsNotNull
Specifies that a value-dependent expression should be considered to never be a null pointer constant.
ExprObjectKind getObjectKind() const
getObjectKind - The object kind that this expression produces.
bool EvaluateAsRValue(EvalResult &Result, const ASTContext &Ctx, bool InConstantContext=false) const
EvaluateAsRValue - Return true if this is a constant which we can fold to an rvalue using any crazy t...
bool HasSideEffects(const ASTContext &Ctx, bool IncludePossibleEffects=true) const
HasSideEffects - This routine returns true for all those expressions which have any effect other than...
std::optional< std::string > tryEvaluateString(ASTContext &Ctx) const
If the current Expr can be evaluated to a pointer to a null-terminated constant string,...
Expr * IgnoreImpCasts() LLVM_READONLY
Skip past any implicit casts which might surround this expression until reaching a fixed point.
NullPointerConstantKind isNullPointerConstant(ASTContext &Ctx, NullPointerConstantValueDependence NPC) const
isNullPointerConstant - C99 6.3.2.3p3 - Test if this reduces down to a Null pointer constant.
SourceLocation getExprLoc() const LLVM_READONLY
getExprLoc - Return the preferred location for the arrow when diagnosing a problem with a generic exp...
std::optional< llvm::APSInt > getIntegerConstantExpr(const ASTContext &Ctx, SourceLocation *Loc=nullptr) const
isIntegerConstantExpr - Return the value if this expression is a valid integer constant expression.
bool tryEvaluateObjectSize(uint64_t &Result, ASTContext &Ctx, unsigned Type) const
If the current Expr is a pointer, this will try to statically determine the number of bytes available...
const ValueDecl * getAsBuiltinConstantDeclRef(const ASTContext &Context) const
If this expression is an unambiguous reference to a single declaration, in the style of __builtin_fun...
Represents difference between two FPOptions values.
Represents a member of a struct/union/class.
const FieldDecl * findCountedByField() const
Find the FieldDecl specified in a FAM's "counted_by" attribute.
Represents a function declaration or definition.
const ParmVarDecl * getParamDecl(unsigned i) const
unsigned getBuiltinID(bool ConsiderWrapperFunctions=false) const
Returns a value indicating whether this function corresponds to a builtin function.
Represents a prototype with parameter type info, e.g.
@ SME_PStateSMEnabledMask
@ SME_PStateSMCompatibleMask
GlobalDecl - represents a global declaration.
const Decl * getDecl() const
IdentifierInfo & get(StringRef Name)
Return the identifier token info for the specified named identifier.
static ImplicitParamDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation IdLoc, IdentifierInfo *Id, QualType T, ImplicitParamKind ParamKind)
Create implicit parameter.
StrictFlexArraysLevelKind
MemberExpr - [C99 6.5.2.3] Structure and Union Members.
StringRef getName() const
Get the name of identifier for this declaration as a StringRef.
std::string getNameAsString() const
Get a human-readable name for the declaration, even if it is one of the special kinds of names (C++ c...
Flags to identify the types for overloaded Neon builtins.
EltType getEltType() const
PointerType - C99 6.7.5.1 - Pointer Declarators.
QualType getPointeeType() const
A (possibly-)qualified type.
bool isVolatileQualified() const
Determine whether this type is volatile-qualified.
bool isWebAssemblyFuncrefType() const
Returns true if it is a WebAssembly Funcref Type.
LangAS getAddressSpace() const
Return the address space of this type.
bool isWebAssemblyExternrefType() const
Returns true if it is a WebAssembly Externref Type.
The collection of all-type qualifiers we support.
Represents a struct/union/class.
field_range fields() const
Flags to identify the types for overloaded SVE builtins.
bool isZExtReturn() const
bool isReverseUSDOT() const
bool isOverloadNone() const
MemEltType getMemEltType() const
bool isGatherLoad() const
bool isOverloadCvt() const
EltType getEltType() const
bool isOverloadDefault() const
bool isOverloadWhileRW() const
bool isReverseMergeAnyAccOp() const
bool isReductionQV() const
bool isInsertOp1SVALL() const
bool isAppendSVALL() const
bool isReverseMergeAnyBinOp() const
bool isStructStore() const
bool isTupleCreate() const
bool isGatherPrefetch() const
bool hasSplatOperand() const
MergeType getMergeType() const
bool isByteIndexed() const
bool isStructLoad() const
bool isOverloadWhileOrMultiVecCvt() const
unsigned getSplatOperand() const
bool isScatterStore() const
bool isReverseCompare() const
Scope - A scope is a transient data structure that is used while parsing the program.
Encodes a location in the source.
SourceLocation getBeginLoc() const LLVM_READONLY
bool areArgsDestroyedLeftToRightInCallee() const
Are arguments to a call destroyed left to right in the callee? This is a fundamental language change,...
Exposes information about the current target.
TargetOptions & getTargetOpts() const
Retrieve the target options.
virtual bool hasLegalHalfType() const
Determine whether _Float16 is supported on this target.
const llvm::Triple & getTriple() const
Returns the target triple of the primary target.
bool isLittleEndian() const
unsigned getMaxOpenCLWorkGroupSize() const
TargetCXXABI getCXXABI() const
Get the C++ ABI currently in use.
virtual bool checkArithmeticFenceSupported() const
Controls if __arithmetic_fence is supported in the targeted backend.
unsigned getSuitableAlign() const
Return the alignment that is the largest alignment ever used for any scalar/SIMD data type on the tar...
virtual std::string_view getClobbers() const =0
Returns a string of target-specific clobbers, in LLVM format.
llvm::CodeObjectVersionKind CodeObjectVersion
Code object version for AMDGPU.
The base class of the type hierarchy.
CXXRecordDecl * getAsCXXRecordDecl() const
Retrieves the CXXRecordDecl that this type refers to, either because the type is a RecordType or beca...
bool isBlockPointerType() const
bool isSignedIntegerType() const
Return true if this is an integer type that is signed, according to C99 6.2.5p4 [char,...
bool isComplexType() const
isComplexType() does not include complex integers (a GCC extension).
bool hasIntegerRepresentation() const
Determine whether this type has an integer representation of some sort, e.g., it is an integer type o...
bool isCountAttributedType() const
bool isPointerType() const
bool isIntegerType() const
isIntegerType() does not include complex integers (a GCC extension).
const T * castAs() const
Member-template castAs<specific type>.
QualType getPointeeType() const
If this is a pointer, ObjC object pointer, or block pointer, this returns the respective pointee.
bool isIntegralOrEnumerationType() const
Determine whether this type is an integral or enumeration type.
bool hasUnsignedIntegerRepresentation() const
Determine whether this type has an unsigned integer representation of some sort, e....
bool hasSignedIntegerRepresentation() const
Determine whether this type has an signed integer representation of some sort, e.g....
bool isObjCObjectPointerType() const
bool hasFloatingRepresentation() const
Determine whether this type has a floating-point representation of some sort, e.g....
bool isVectorType() const
bool isFloatingType() const
bool isUnsignedIntegerType() const
Return true if this is an integer type that is unsigned, according to C99 6.2.5p6 [which returns true...
const T * getAs() const
Member-template getAs<specific type>'.
bool isRecordType() const
bool isSizelessVectorType() const
Returns true for all scalable vector types.
QualType getSizelessVectorEltType(const ASTContext &Ctx) const
Returns the representative type for the element of a sizeless vector builtin type.
RecordDecl * getAsRecordDecl() const
Retrieves the RecordDecl this type refers to.
Represent the declaration of a variable (in which case it is an lvalue) a function (in which case it ...
Represents a GCC generic vector type.
unsigned getNumElements() const
QualType getElementType() const
SmallVector< OSLogBufferItem, 4 > Items
unsigned char getNumArgsByte() const
unsigned char getSummaryByte() const
Defines the clang::TargetInfo interface.
@ Type
The l-value was considered opaque, so the alignment was determined from a type.
llvm::Constant * initializationPatternFor(CodeGenModule &, llvm::Type *)
TypeEvaluationKind
The kind of evaluation to perform on values of a particular type.
@ EHCleanup
Denotes a cleanup that should run when a scope is exited using exceptional control flow (a throw stat...
constexpr XRayInstrMask Typed
constexpr XRayInstrMask Custom
bool computeOSLogBufferLayout(clang::ASTContext &Ctx, const clang::CallExpr *E, OSLogBufferLayout &layout)
const void * Store
Store - This opaque type encapsulates an immutable mapping from locations to values.
bool Dup(InterpState &S, CodePtr OpPC)
bool Zero(InterpState &S, CodePtr OpPC)
bool Mul(InterpState &S, CodePtr OpPC)
bool Neg(InterpState &S, CodePtr OpPC)
bool Load(InterpState &S, CodePtr OpPC)
bool Cast(InterpState &S, CodePtr OpPC)
bool Ret(InterpState &S, CodePtr &PC)
The JSON file list parser is used to communicate input to InstallAPI.
@ OK_BitField
A bitfield object is a bitfield on a C or C++ record.
@ Vector
'vector' clause, allowed on 'loop', Combined, and 'routine' directives.
@ DType
'dtype' clause, an alias for 'device_type', stored separately for diagnostic purposes.
bool operator<(DeclarationName LHS, DeclarationName RHS)
Ordering on two declaration names.
@ Asm
Assembly: we accept this only so that we can preprocess it.
@ Result
The result type of a method or function.
LangAS
Defines the address space values used by the address space qualifier of QualType.
const FunctionProtoType * T
SyncScope
Defines synch scope values used internally by clang.
llvm::StringRef getAsString(SyncScope S)
@ Success
Template argument deduction was successful.
@ Other
Other implicit parameter.
Diagnostic wrappers for TextAPI types for error reporting.
llvm::PointerType * VoidPtrTy
llvm::IntegerType * Int64Ty
llvm::PointerType * ConstGlobalsPtrTy
void* in the address space for constant globals
CharUnits getIntAlign() const
llvm::IntegerType * Int8Ty
i8, i16, i32, and i64
llvm::Type * HalfTy
half, bfloat, float, double
llvm::IntegerType * SizeTy
llvm::IntegerType * Int32Ty
llvm::IntegerType * IntPtrTy
llvm::IntegerType * IntTy
int
llvm::IntegerType * Int16Ty
llvm::PointerType * Int8PtrTy
llvm::PointerType * UnqualPtrTy
llvm::PointerType * AllocaInt8PtrTy
LangAS getASTAllocaAddressSpace() const
EvalResult is a struct with detailed info about an evaluated expression.
APValue Val
Val - This is the value the expression can be folded to.
void clear(SanitizerMask K=SanitizerKind::All)
Disable the sanitizers specified in K.
void set(SanitizerMask K, bool Value)
Enable or disable a certain (single) sanitizer.
bool has(SanitizerMask K) const
Check if a certain (single) sanitizer is enabled.
bool has(XRayInstrMask K) const
#define scalbln(__x, __y)
#define copysign(__x, __y)
#define remquo(__x, __y, __z)
#define nextafter(__x, __y)
#define nexttoward(__x, __y)
#define remainder(__x, __y)
#define fma(__x, __y, __z)