//===---- CGBuiltin.cpp - Emit LLVM Code for builtins ---------------------===//

//

// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.

// See https://llvm.org/LICENSE.txt for license information.

// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

//

//===----------------------------------------------------------------------===//

//

// This contains code to emit Builtin calls as LLVM code.

//

//===----------------------------------------------------------------------===//

#include "CGCUDARuntime.h"

#include "CGCXXABI.h"

#include "CGObjCRuntime.h"

#include "CGOpenCLRuntime.h"

#include "CGRecordLayout.h"

#include "CodeGenFunction.h"

#include "CodeGenModule.h"

#include "ConstantEmitter.h"

#include "PatternInit.h"

#include "TargetInfo.h"

#include "clang/AST/ASTContext.h"

#include "clang/AST/Attr.h"

#include "clang/AST/Decl.h"

#include "clang/AST/OSLog.h"

#include "clang/Basic/TargetBuiltins.h"

#include "clang/Basic/TargetInfo.h"

#include "clang/CodeGen/CGFunctionInfo.h"

#include "llvm/ADT/APFloat.h"

#include "llvm/ADT/APInt.h"

#include "llvm/ADT/SmallPtrSet.h"

#include "llvm/ADT/StringExtras.h"

#include "llvm/Analysis/ValueTracking.h"

#include "llvm/IR/DataLayout.h"

#include "llvm/IR/InlineAsm.h"

#include "llvm/IR/Intrinsics.h"

#include "llvm/IR/IntrinsicsAArch64.h"

#include "llvm/IR/IntrinsicsAMDGPU.h"

#include "llvm/IR/IntrinsicsARM.h"

#include "llvm/IR/IntrinsicsBPF.h"

#include "llvm/IR/IntrinsicsHexagon.h"

#include "llvm/IR/IntrinsicsNVPTX.h"

#include "llvm/IR/IntrinsicsPowerPC.h"

#include "llvm/IR/IntrinsicsR600.h"

#include "llvm/IR/IntrinsicsRISCV.h"

#include "llvm/IR/IntrinsicsS390.h"

#include "llvm/IR/IntrinsicsWebAssembly.h"

#include "llvm/IR/IntrinsicsX86.h"

#include "llvm/IR/MDBuilder.h"

#include "llvm/IR/MatrixBuilder.h"

#include "llvm/Support/ConvertUTF.h"

#include "llvm/Support/ScopedPrinter.h"

#include "llvm/Support/X86TargetParser.h"

#include <sstream>

using namespace clang;

using namespace CodeGen;

using namespace llvm;

static

int64_t clamp(int64_t Value, int64_t Low, int64_t High) {

  return std::min(High, std::max(Low, Value));

}

static void initializeAlloca(CodeGenFunction &CGF, AllocaInst *AI, Value *Size,

                             Align AlignmentInBytes) {

  ConstantInt *Byte;

  switch (CGF.getLangOpts().getTrivialAutoVarInit()) {

  case LangOptions::TrivialAutoVarInitKind::Uninitialized:

    // Nothing to initialize.

    return;

  case LangOptions::TrivialAutoVarInitKind::Zero:

    Byte = CGF.Builder.getInt8(0x00);

    break;

  case LangOptions::TrivialAutoVarInitKind::Pattern: {

    llvm::Type *Int8 = llvm::IntegerType::getInt8Ty(CGF.CGM.getLLVMContext());

    Byte = llvm::dyn_cast<llvm::ConstantInt>(

        initializationPatternFor(CGF.CGM, Int8));

    break;

  }

  }

  if (CGF.CGM.stopAutoInit())

    return;

  auto *I = CGF.Builder.CreateMemSet(AI, Byte, Size, AlignmentInBytes);

  I->addAnnotationMetadata("auto-init");

}

/// getBuiltinLibFunction - Given a builtin id for a function like

/// "__builtin_fabsf", return a Function* for "fabsf".

llvm::Constant *CodeGenModule::getBuiltinLibFunction(const FunctionDecl *FD,

                                                     unsigned BuiltinID) {

  assert(Context.BuiltinInfo.isLibFunction(BuiltinID));

  // Get the name, skip over the __builtin_ prefix (if necessary).

  StringRef Name;

  GlobalDecl D(FD);

  // If the builtin has been declared explicitly with an assembler label,

  // use the mangled name. This differs from the plain label on platforms

  // that prefix labels.

  if (FD->hasAttr<AsmLabelAttr>())

    Name = getMangledName(D);

  else

    Name = Context.BuiltinInfo.getName(BuiltinID) + 10;

  llvm::FunctionType *Ty =

    cast<llvm::FunctionType>(getTypes().ConvertType(FD->getType()));

  return GetOrCreateLLVMFunction(Name, Ty, D, /*ForVTable=*/false);

}

/// Emit the conversions required to turn the given value into an

/// integer of the given size.

static Value *EmitToInt(CodeGenFunction &CGF, llvm::Value *V,

                        QualType T, llvm::IntegerType *IntType) {

  V = CGF.EmitToMemory(V, T);

  if (V->getType()->isPointerTy())

    return CGF.Builder.CreatePtrToInt(V, IntType);

  assert(V->getType() == IntType);

  return V;

}

static Value *EmitFromInt(CodeGenFunction &CGF, llvm::Value *V,

                          QualType T, llvm::Type *ResultType) {

  V = CGF.EmitFromMemory(V, T);

  if (ResultType->isPointerTy())

    return CGF.Builder.CreateIntToPtr(V, ResultType);

  assert(V->getType() == ResultType);

  return V;

}

/// Utility to insert an atomic instruction based on Intrinsic::ID

/// and the expression node.

static Value *MakeBinaryAtomicValue(

    CodeGenFunction &CGF, llvm::AtomicRMWInst::BinOp Kind, const CallExpr *E,

    AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent) {

  QualType T = E->getType();

  assert(E->getArg(0)->getType()->isPointerType());

  assert(CGF.getContext().hasSameUnqualifiedType(T,

                                  E->getArg(0)->getType()->getPointeeType()));

  assert(CGF.getContext().hasSameUnqualifiedType(T, E->getArg(1)->getType()));

  llvm::Value *DestPtr = CGF.EmitScalarExpr(E->getArg(0));

  unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace();

  llvm::IntegerType *IntType =

    llvm::IntegerType::get(CGF.getLLVMContext(),

                           CGF.getContext().getTypeSize(T));

  llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace);

  llvm::Value *Args[2];

  Args[0] = CGF.Builder.CreateBitCast(DestPtr, IntPtrType);

  Args[1] = CGF.EmitScalarExpr(E->getArg(1));

  llvm::Type *ValueType = Args[1]->getType();

  Args[1] = EmitToInt(CGF, Args[1], T, IntType);

  llvm::Value *Result = CGF.Builder.CreateAtomicRMW(

      Kind, Args[0], Args[1], Ordering);

  return EmitFromInt(CGF, Result, T, ValueType);

}

static Value *EmitNontemporalStore(CodeGenFunction &CGF, const CallExpr *E) {

  Value *Val = CGF.EmitScalarExpr(E->getArg(0));

  Value *Address = CGF.EmitScalarExpr(E->getArg(1));

  // Convert the type of the pointer to a pointer to the stored type.

  Val = CGF.EmitToMemory(Val, E->getArg(0)->getType());

  Value *BC = CGF.Builder.CreateBitCast(

      Address, llvm::PointerType::getUnqual(Val->getType()), "cast");

  LValue LV = CGF.MakeNaturalAlignAddrLValue(BC, E->getArg(0)->getType());

  LV.setNontemporal(true);

  CGF.EmitStoreOfScalar(Val, LV, false);

  return nullptr;

}

static Value *EmitNontemporalLoad(CodeGenFunction &CGF, const CallExpr *E) {

  Value *Address = CGF.EmitScalarExpr(E->getArg(0));

  LValue LV = CGF.MakeNaturalAlignAddrLValue(Address, E->getType());

  LV.setNontemporal(true);

  return CGF.EmitLoadOfScalar(LV, E->getExprLoc());

}

static RValue EmitBinaryAtomic(CodeGenFunction &CGF,

                               llvm::AtomicRMWInst::BinOp Kind,

                               const CallExpr *E) {

  return RValue::get(MakeBinaryAtomicValue(CGF, Kind, E));

}

/// Utility to insert an atomic instruction based Intrinsic::ID and

/// the expression node, where the return value is the result of the

/// operation.

static RValue EmitBinaryAtomicPost(CodeGenFunction &CGF,

                                   llvm::AtomicRMWInst::BinOp Kind,

                                   const CallExpr *E,

                                   Instruction::BinaryOps Op,

                                   bool Invert = false) {

  QualType T = E->getType();

  assert(E->getArg(0)->getType()->isPointerType());

  assert(CGF.getContext().hasSameUnqualifiedType(T,

                                  E->getArg(0)->getType()->getPointeeType()));

  assert(CGF.getContext().hasSameUnqualifiedType(T, E->getArg(1)->getType()));

  llvm::Value *DestPtr = CGF.EmitScalarExpr(E->getArg(0));

  unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace();

  llvm::IntegerType *IntType =

    llvm::IntegerType::get(CGF.getLLVMContext(),

                           CGF.getContext().getTypeSize(T));

  llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace);

  llvm::Value *Args[2];

  Args[1] = CGF.EmitScalarExpr(E->getArg(1));

  llvm::Type *ValueType = Args[1]->getType();

  Args[1] = EmitToInt(CGF, Args[1], T, IntType);

  Args[0] = CGF.Builder.CreateBitCast(DestPtr, IntPtrType);

  llvm::Value *Result = CGF.Builder.CreateAtomicRMW(

      Kind, Args[0], Args[1], llvm::AtomicOrdering::SequentiallyConsistent);

  Result = CGF.Builder.CreateBinOp(Op, Result, Args[1]);

  if (Invert)

    Result =

        CGF.Builder.CreateBinOp(llvm::Instruction::Xor, Result,

                                llvm::ConstantInt::getAllOnesValue(IntType));

  Result = EmitFromInt(CGF, Result, T, ValueType);

  return RValue::get(Result);

}

/// Utility to insert an atomic cmpxchg instruction.

///

/// @param CGF The current codegen function.

/// @param E   Builtin call expression to convert to cmpxchg.

///            arg0 - address to operate on

///            arg1 - value to compare with

///            arg2 - new value

/// @param ReturnBool Specifies whether to return success flag of

///                   cmpxchg result or the old value.

///

/// @returns result of cmpxchg, according to ReturnBool

///

/// Note: In order to lower Microsoft's _InterlockedCompareExchange* intrinsics

/// invoke the function EmitAtomicCmpXchgForMSIntrin.

static Value *MakeAtomicCmpXchgValue(CodeGenFunction &CGF, const CallExpr *E,

                                     bool ReturnBool) {

  QualType T = ReturnBool ? 
E->getArg(1)->getType()12
 : 
E->getType()31
;

  llvm::Value *DestPtr = CGF.EmitScalarExpr(E->getArg(0));

  unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace();

  llvm::IntegerType *IntType = llvm::IntegerType::get(

      CGF.getLLVMContext(), CGF.getContext().getTypeSize(T));

  llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace);

  Value *Args[3];

  Args[0] = CGF.Builder.CreateBitCast(DestPtr, IntPtrType);

  Args[1] = CGF.EmitScalarExpr(E->getArg(1));

  llvm::Type *ValueType = Args[1]->getType();

  Args[1] = EmitToInt(CGF, Args[1], T, IntType);

  Args[2] = EmitToInt(CGF, CGF.EmitScalarExpr(E->getArg(2)), T, IntType);

  Value *Pair = CGF.Builder.CreateAtomicCmpXchg(

      Args[0], Args[1], Args[2], llvm::AtomicOrdering::SequentiallyConsistent,

      llvm::AtomicOrdering::SequentiallyConsistent);

  if (ReturnBool)

    // Extract boolean success flag and zext it to int.

    return CGF.Builder.CreateZExt(CGF.Builder.CreateExtractValue(Pair, 1),

                                  CGF.ConvertType(E->getType()));

  else

    // Extract old value and emit it using the same type as compare value.

    return EmitFromInt(CGF, CGF.Builder.CreateExtractValue(Pair, 0), T,

                       ValueType);

}

/// This function should be invoked to emit atomic cmpxchg for Microsoft's

/// _InterlockedCompareExchange* intrinsics which have the following signature:

/// T _InterlockedCompareExchange(T volatile *Destination,

///                               T Exchange,

///                               T Comparand);

///

/// Whereas the llvm 'cmpxchg' instruction has the following syntax:

/// cmpxchg *Destination, Comparand, Exchange.

/// So we need to swap Comparand and Exchange when invoking

/// CreateAtomicCmpXchg. That is the reason we could not use the above utility

/// function MakeAtomicCmpXchgValue since it expects the arguments to be

/// already swapped.

static

Value *EmitAtomicCmpXchgForMSIntrin(CodeGenFunction &CGF, const CallExpr *E,

    AtomicOrdering SuccessOrdering = AtomicOrdering::SequentiallyConsistent) {

  assert(E->getArg(0)->getType()->isPointerType());

  assert(CGF.getContext().hasSameUnqualifiedType(

      E->getType(), E->getArg(0)->getType()->getPointeeType()));

  assert(CGF.getContext().hasSameUnqualifiedType(E->getType(),

                                                 E->getArg(1)->getType()));

  assert(CGF.getContext().hasSameUnqualifiedType(E->getType(),

                                                 E->getArg(2)->getType()));

  auto *Destination = CGF.EmitScalarExpr(E->getArg(0));

  auto *Comparand = CGF.EmitScalarExpr(E->getArg(2));

  auto *Exchange = CGF.EmitScalarExpr(E->getArg(1));

  // For Release ordering, the failure ordering should be Monotonic.

  auto FailureOrdering = SuccessOrdering == AtomicOrdering::Release ?

                         AtomicOrdering::Monotonic :

                         
SuccessOrdering43
;

  // The atomic instruction is marked volatile for consistency with MSVC. This

  // blocks the few atomics optimizations that LLVM has. If we want to optimize

  // _Interlocked* operations in the future, we will have to remove the volatile

  // marker.

  auto *Result = CGF.Builder.CreateAtomicCmpXchg(

                   Destination, Comparand, Exchange,

                   SuccessOrdering, FailureOrdering);

  Result->setVolatile(true);

  return CGF.Builder.CreateExtractValue(Result, 0);

}

// 64-bit Microsoft platforms support 128 bit cmpxchg operations. They are

// prototyped like this:

//

// unsigned char _InterlockedCompareExchange128...(

//     __int64 volatile * _Destination,

//     __int64 _ExchangeHigh,

//     __int64 _ExchangeLow,

//     __int64 * _ComparandResult);

static Value *EmitAtomicCmpXchg128ForMSIntrin(CodeGenFunction &CGF,

                                              const CallExpr *E,

                                              AtomicOrdering SuccessOrdering) {

  assert(E->getNumArgs() == 4);

  llvm::Value *Destination = CGF.EmitScalarExpr(E->getArg(0));

  llvm::Value *ExchangeHigh = CGF.EmitScalarExpr(E->getArg(1));

  llvm::Value *ExchangeLow = CGF.EmitScalarExpr(E->getArg(2));

  llvm::Value *ComparandPtr = CGF.EmitScalarExpr(E->getArg(3));

  assert(Destination->getType()->isPointerTy());

  assert(!ExchangeHigh->getType()->isPointerTy());

  assert(!ExchangeLow->getType()->isPointerTy());

  assert(ComparandPtr->getType()->isPointerTy());

  // For Release ordering, the failure ordering should be Monotonic.

  auto FailureOrdering = SuccessOrdering == AtomicOrdering::Release

                             ? 
AtomicOrdering::Monotonic1

                             : 
SuccessOrdering4
;

  // Convert to i128 pointers and values.

  llvm::Type *Int128Ty = llvm::IntegerType::get(CGF.getLLVMContext(), 128);

  llvm::Type *Int128PtrTy = Int128Ty->getPointerTo();

  Destination = CGF.Builder.CreateBitCast(Destination, Int128PtrTy);

  Address ComparandResult(CGF.Builder.CreateBitCast(ComparandPtr, Int128PtrTy),

                          CGF.getContext().toCharUnitsFromBits(128));

  // (((i128)hi) << 64) | ((i128)lo)

  ExchangeHigh = CGF.Builder.CreateZExt(ExchangeHigh, Int128Ty);

  ExchangeLow = CGF.Builder.CreateZExt(ExchangeLow, Int128Ty);

  ExchangeHigh =

      CGF.Builder.CreateShl(ExchangeHigh, llvm::ConstantInt::get(Int128Ty, 64));

  llvm::Value *Exchange = CGF.Builder.CreateOr(ExchangeHigh, ExchangeLow);

  // Load the comparand for the instruction.

  llvm::Value *Comparand = CGF.Builder.CreateLoad(ComparandResult);

  auto *CXI = CGF.Builder.CreateAtomicCmpXchg(Destination, Comparand, Exchange,

                                              SuccessOrdering, FailureOrdering);

  // The atomic instruction is marked volatile for consistency with MSVC. This

  // blocks the few atomics optimizations that LLVM has. If we want to optimize

  // _Interlocked* operations in the future, we will have to remove the volatile

  // marker.

  CXI->setVolatile(true);

  // Store the result as an outparameter.

  CGF.Builder.CreateStore(CGF.Builder.CreateExtractValue(CXI, 0),

                          ComparandResult);

  // Get the success boolean and zero extend it to i8.

  Value *Success = CGF.Builder.CreateExtractValue(CXI, 1);

  return CGF.Builder.CreateZExt(Success, CGF.Int8Ty);

}

static Value *EmitAtomicIncrementValue(CodeGenFunction &CGF, const CallExpr *E,

    AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent) {

  assert(E->getArg(0)->getType()->isPointerType());

  auto *IntTy = CGF.ConvertType(E->getType());

  auto *Result = CGF.Builder.CreateAtomicRMW(

                   AtomicRMWInst::Add,

                   CGF.EmitScalarExpr(E->getArg(0)),

                   ConstantInt::get(IntTy, 1),

                   Ordering);

  return CGF.Builder.CreateAdd(Result, ConstantInt::get(IntTy, 1));

}

static Value *EmitAtomicDecrementValue(CodeGenFunction &CGF, const CallExpr *E,

    AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent) {

  assert(E->getArg(0)->getType()->isPointerType());

  auto *IntTy = CGF.ConvertType(E->getType());

  auto *Result = CGF.Builder.CreateAtomicRMW(

                   AtomicRMWInst::Sub,

                   CGF.EmitScalarExpr(E->getArg(0)),

                   ConstantInt::get(IntTy, 1),

                   Ordering);

  return CGF.Builder.CreateSub(Result, ConstantInt::get(IntTy, 1));

}

// Build a plain volatile load.

static Value *EmitISOVolatileLoad(CodeGenFunction &CGF, const CallExpr *E) {

  Value *Ptr = CGF.EmitScalarExpr(E->getArg(0));

  QualType ElTy = E->getArg(0)->getType()->getPointeeType();

  CharUnits LoadSize = CGF.getContext().getTypeSizeInChars(ElTy);

  llvm::Type *ITy =

      llvm::IntegerType::get(CGF.getLLVMContext(), LoadSize.getQuantity() * 8);

  Ptr = CGF.Builder.CreateBitCast(Ptr, ITy->getPointerTo());

  llvm::LoadInst *Load = CGF.Builder.CreateAlignedLoad(ITy, Ptr, LoadSize);

  Load->setVolatile(true);

  return Load;

}

// Build a plain volatile store.

static Value *EmitISOVolatileStore(CodeGenFunction &CGF, const CallExpr *E) {

  Value *Ptr = CGF.EmitScalarExpr(E->getArg(0));

  Value *Value = CGF.EmitScalarExpr(E->getArg(1));

  QualType ElTy = E->getArg(0)->getType()->getPointeeType();

  CharUnits StoreSize = CGF.getContext().getTypeSizeInChars(ElTy);

  llvm::Type *ITy =

      llvm::IntegerType::get(CGF.getLLVMContext(), StoreSize.getQuantity() * 8);

  Ptr = CGF.Builder.CreateBitCast(Ptr, ITy->getPointerTo());

  llvm::StoreInst *Store =

      CGF.Builder.CreateAlignedStore(Value, Ptr, StoreSize);

  Store->setVolatile(true);

  return Store;

}

// Emit a simple mangled intrinsic that has 1 argument and a return type

// matching the argument type. Depending on mode, this may be a constrained

// floating-point intrinsic.

static Value *emitUnaryMaybeConstrainedFPBuiltin(CodeGenFunction &CGF,

                                const CallExpr *E, unsigned IntrinsicID,

                                unsigned ConstrainedIntrinsicID) {

  llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));

  if (CGF.Builder.getIsFPConstrained()) {

    CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);

    Function *F = CGF.CGM.getIntrinsic(ConstrainedIntrinsicID, Src0->getType());

    return CGF.Builder.CreateConstrainedFPCall(F, { Src0 });

  } else {

    Function *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType());

    return CGF.Builder.CreateCall(F, Src0);

  }

}

// Emit an intrinsic that has 2 operands of the same type as its result.

// Depending on mode, this may be a constrained floating-point intrinsic.

static Value *emitBinaryMaybeConstrainedFPBuiltin(CodeGenFunction &CGF,

                                const CallExpr *E, unsigned IntrinsicID,

                                unsigned ConstrainedIntrinsicID) {

  llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));

  llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1));

  if (CGF.Builder.getIsFPConstrained()) {

    CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);

    Function *F = CGF.CGM.getIntrinsic(ConstrainedIntrinsicID, Src0->getType());

    return CGF.Builder.CreateConstrainedFPCall(F, { Src0, Src1 });

  } else {

    Function *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType());

    return CGF.Builder.CreateCall(F, { Src0, Src1 });

  }

}

// Emit an intrinsic that has 3 operands of the same type as its result.

// Depending on mode, this may be a constrained floating-point intrinsic.

static Value *emitTernaryMaybeConstrainedFPBuiltin(CodeGenFunction &CGF,

                                 const CallExpr *E, unsigned IntrinsicID,

                                 unsigned ConstrainedIntrinsicID) {

  llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));

  llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1));

  llvm::Value *Src2 = CGF.EmitScalarExpr(E->getArg(2));

  if (CGF.Builder.getIsFPConstrained()) {

    CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);

    Function *F = CGF.CGM.getIntrinsic(ConstrainedIntrinsicID, Src0->getType());

    return CGF.Builder.CreateConstrainedFPCall(F, { Src0, Src1, Src2 });

  } else {

    Function *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType());

    return CGF.Builder.CreateCall(F, { Src0, Src1, Src2 });

  }

}

// Emit an intrinsic where all operands are of the same type as the result.

// Depending on mode, this may be a constrained floating-point intrinsic.

static Value *emitCallMaybeConstrainedFPBuiltin(CodeGenFunction &CGF,

                                                unsigned IntrinsicID,

                                                unsigned ConstrainedIntrinsicID,

                                                llvm::Type *Ty,

                                                ArrayRef<Value *> Args) {

  Function *F;

  if (CGF.Builder.getIsFPConstrained())

    F = CGF.CGM.getIntrinsic(ConstrainedIntrinsicID, Ty);

  else

    F = CGF.CGM.getIntrinsic(IntrinsicID, Ty);

  if (CGF.Builder.getIsFPConstrained())

    return CGF.Builder.CreateConstrainedFPCall(F, Args);

  else

    return CGF.Builder.CreateCall(F, Args);

}

// Emit a simple mangled intrinsic that has 1 argument and a return type

// matching the argument type.

static Value *emitUnaryBuiltin(CodeGenFunction &CGF,

                               const CallExpr *E,

                               unsigned IntrinsicID) {

  llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));

  Function *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType());

  return CGF.Builder.CreateCall(F, Src0);

}

// Emit an intrinsic that has 2 operands of the same type as its result.

static Value *emitBinaryBuiltin(CodeGenFunction &CGF,

                                const CallExpr *E,

                                unsigned IntrinsicID) {

  llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));

  llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1));

  Function *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType());

  return CGF.Builder.CreateCall(F, { Src0, Src1 });

}

// Emit an intrinsic that has 3 operands of the same type as its result.

static Value *emitTernaryBuiltin(CodeGenFunction &CGF,

                                 const CallExpr *E,

                                 unsigned IntrinsicID) {

  llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));

  llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1));

  llvm::Value *Src2 = CGF.EmitScalarExpr(E->getArg(2));

  Function *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType());

  return CGF.Builder.CreateCall(F, { Src0, Src1, Src2 });

}

// Emit an intrinsic that has 1 float or double operand, and 1 integer.

static Value *emitFPIntBuiltin(CodeGenFunction &CGF,

                               const CallExpr *E,

                               unsigned IntrinsicID) {

  llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));

  llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1));

  Function *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType());

  return CGF.Builder.CreateCall(F, {Src0, Src1});

}

// Emit an intrinsic that has overloaded integer result and fp operand.

static Value *

emitMaybeConstrainedFPToIntRoundBuiltin(CodeGenFunction &CGF, const CallExpr *E,

                                        unsigned IntrinsicID,

                                        unsigned ConstrainedIntrinsicID) {

  llvm::Type *ResultType = CGF.ConvertType(E->getType());

  llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));

  if (CGF.Builder.getIsFPConstrained()) {

    CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);

    Function *F = CGF.CGM.getIntrinsic(ConstrainedIntrinsicID,

                                       {ResultType, Src0->getType()});

    return CGF.Builder.CreateConstrainedFPCall(F, {Src0});

  } else {

    Function *F =

        CGF.CGM.getIntrinsic(IntrinsicID, {ResultType, Src0->getType()});

    return CGF.Builder.CreateCall(F, Src0);

  }

}

/// EmitFAbs - Emit a call to @llvm.fabs().

static Value *EmitFAbs(CodeGenFunction &CGF, Value *V) {

  Function *F = CGF.CGM.getIntrinsic(Intrinsic::fabs, V->getType());

  llvm::CallInst *Call = CGF.Builder.CreateCall(F, V);

  Call->setDoesNotAccessMemory();

  return Call;

}

/// Emit the computation of the sign bit for a floating point value. Returns

/// the i1 sign bit value.

static Value *EmitSignBit(CodeGenFunction &CGF, Value *V) {

  LLVMContext &C = CGF.CGM.getLLVMContext();

  llvm::Type *Ty = V->getType();

  int Width = Ty->getPrimitiveSizeInBits();

  llvm::Type *IntTy = llvm::IntegerType::get(C, Width);

  V = CGF.Builder.CreateBitCast(V, IntTy);

  if (Ty->isPPC_FP128Ty()) {

    // We want the sign bit of the higher-order double. The bitcast we just

    // did works as if the double-double was stored to memory and then

    // read as an i128. The "store" will put the higher-order double in the

    // lower address in both little- and big-Endian modes, but the "load"

    // will treat those bits as a different part of the i128: the low bits in

    // little-Endian, the high bits in big-Endian. Therefore, on big-Endian

    // we need to shift the high bits down to the low before truncating.

    Width >>= 1;

    if (CGF.getTarget().isBigEndian()) {

      Value *ShiftCst = llvm::ConstantInt::get(IntTy, Width);

      V = CGF.Builder.CreateLShr(V, ShiftCst);

    }

    // We are truncating value in order to extract the higher-order

    // double, which we will be using to extract the sign from.

    IntTy = llvm::IntegerType::get(C, Width);

    V = CGF.Builder.CreateTrunc(V, IntTy);

  }

  Value *Zero = llvm::Constant::getNullValue(IntTy);

  return CGF.Builder.CreateICmpSLT(V, Zero);

}

static RValue emitLibraryCall(CodeGenFunction &CGF, const FunctionDecl *FD,

                              const CallExpr *E, llvm::Constant *calleeValue) {

  CGCallee callee = CGCallee::forDirect(calleeValue, GlobalDecl(FD));

  return CGF.EmitCall(E->getCallee()->getType(), callee, E, ReturnValueSlot());

}

/// Emit a call to llvm.{sadd,uadd,ssub,usub,smul,umul}.with.overflow.*

/// depending on IntrinsicID.

///

/// \arg CGF The current codegen function.

/// \arg IntrinsicID The ID for the Intrinsic we wish to generate.

/// \arg X The first argument to the llvm.*.with.overflow.*.

/// \arg Y The second argument to the llvm.*.with.overflow.*.

/// \arg Carry The carry returned by the llvm.*.with.overflow.*.

/// \returns The result (i.e. sum/product) returned by the intrinsic.

static llvm::Value *EmitOverflowIntrinsic(CodeGenFunction &CGF,

                                          const llvm::Intrinsic::ID IntrinsicID,

                                          llvm::Value *X, llvm::Value *Y,

                                          llvm::Value *&Carry) {

  // Make sure we have integers of the same width.

  assert(X->getType() == Y->getType() &&

         "Arguments must be the same type. (Did you forget to make sure both "

         "arguments have the same integer width?)");

  Function *Callee = CGF.CGM.getIntrinsic(IntrinsicID, X->getType());

  llvm::Value *Tmp = CGF.Builder.CreateCall(Callee, {X, Y});

  Carry = CGF.Builder.CreateExtractValue(Tmp, 1);

  return CGF.Builder.CreateExtractValue(Tmp, 0);

}

static Value *emitRangedBuiltin(CodeGenFunction &CGF,

                                unsigned IntrinsicID,

                                int low, int high) {

    llvm::MDBuilder MDHelper(CGF.getLLVMContext());

    llvm::MDNode *RNode = MDHelper.createRange(APInt(32, low), APInt(32, high));

    Function *F = CGF.CGM.getIntrinsic(IntrinsicID, {});

    llvm::Instruction *Call = CGF.Builder.CreateCall(F);

    Call->setMetadata(llvm::LLVMContext::MD_range, RNode);

    return Call;

}

namespace {

  struct WidthAndSignedness {

    unsigned Width;

    bool Signed;

  };

}

static WidthAndSignedness

getIntegerWidthAndSignedness(const clang::ASTContext &context,

                             const clang::QualType Type) {

  assert(Type->isIntegerType() && "Given type is not an integer.");

  unsigned Width = Type->isBooleanType()  ? 
118

                   : 
Type->isExtIntType()252
 ? 
context.getIntWidth(Type)45

                                          : 
context.getTypeInfo(Type).Width207
;

  bool Signed = Type->isSignedIntegerType();

  return {Width, Signed};

}

// Given one or more integer types, this function produces an integer type that

// encompasses them: any value in one of the given types could be expressed in

// the encompassing type.

static struct WidthAndSignedness

EncompassingIntegerType(ArrayRef<struct WidthAndSignedness> Types) {

  assert(Types.size() > 0 && "Empty list of types.");

  // If any of the given types is signed, we must return a signed type.

  bool Signed = false;

  for (const auto &Type : Types) {

    Signed |= Type.Signed;

  }

  // The encompassing type must have a width greater than or equal to the width

  // of the specified types.  Additionally, if the encompassing type is signed,

  // its width must be strictly greater than the width of any unsigned types

  // given.

  unsigned Width = 0;

  for (const auto &Type : Types) {

    unsigned MinWidth = Type.Width + (Signed && 
!Type.Signed108
);

    if (Width < MinWidth) {

      Width = MinWidth;

    }

  }

  return {Width, Signed};

}

Value *CodeGenFunction::EmitVAStartEnd(Value *ArgValue, bool IsStart) {

  llvm::Type *DestType = Int8PtrTy;

  if (ArgValue->getType() != DestType)

    ArgValue =

        Builder.CreateBitCast(ArgValue, DestType, ArgValue->getName().data());

  Intrinsic::ID inst = IsStart ? 
Intrinsic::vastart246
 : 
Intrinsic::vaend223
;

  return Builder.CreateCall(CGM.getIntrinsic(inst), ArgValue);

}

/// Checks if using the result of __builtin_object_size(p, @p From) in place of

/// __builtin_object_size(p, @p To) is correct

static bool areBOSTypesCompatible(int From, int To) {

  // Note: Our __builtin_object_size implementation currently treats Type=0 and

  // Type=2 identically. Encoding this implementation detail here may make

  // improving __builtin_object_size difficult in the future, so it's omitted.

  return From == To || 
(12
From == 012
 && 
To == 13
) || 
(11
From == 311
 && 
To == 23
);

}

static llvm::Value *

getDefaultBuiltinObjectSizeResult(unsigned Type, llvm::IntegerType *ResType) {

  return ConstantInt::get(ResType, (Type & 2) ? 0 : 
-10
, /*isSigned=*/true);

}

llvm::Value *

CodeGenFunction::evaluateOrEmitBuiltinObjectSize(const Expr *E, unsigned Type,

                                                 llvm::IntegerType *ResType,

                                                 llvm::Value *EmittedE,

                                                 bool IsDynamic) {

  uint64_t ObjectSize;

  if (!E->tryEvaluateObjectSize(ObjectSize, getContext(), Type))

    return emitBuiltinObjectSize(E, Type, ResType, EmittedE, IsDynamic);

  return ConstantInt::get(ResType, ObjectSize, /*isSigned=*/true);

}

/// Returns a Value corresponding to the size of the given expression.

/// This Value may be either of the following:

///   - A llvm::Argument (if E is a param with the pass_object_size attribute on

///     it)

///   - A call to the @llvm.objectsize intrinsic

///

/// EmittedE is the result of emitting `E` as a scalar expr. If it's non-null

/// and we wouldn't otherwise try to reference a pass_object_size parameter,

/// we'll call @llvm.objectsize on EmittedE, rather than emitting E.

llvm::Value *

CodeGenFunction::emitBuiltinObjectSize(const Expr *E, unsigned Type,

                                       llvm::IntegerType *ResType,

                                       llvm::Value *EmittedE, bool IsDynamic) {

  // We need to reference an argument if the pointer is a parameter with the

  // pass_object_size attribute.

  if (auto *D = dyn_cast<DeclRefExpr>(E->IgnoreParenImpCasts())) {

    auto *Param = dyn_cast<ParmVarDecl>(D->getDecl());

    auto *PS = D->getDecl()->getAttr<PassObjectSizeAttr>();

    if (Param != nullptr && 
PS != nullptr69
 &&

        
areBOSTypesCompatible(PS->getType(), Type)38
) {

      auto Iter = SizeArguments.find(Param);

      assert(Iter != SizeArguments.end());

      const ImplicitParamDecl *D = Iter->second;

      auto DIter = LocalDeclMap.find(D);

      assert(DIter != LocalDeclMap.end());

      return EmitLoadOfScalar(DIter->second, /*Volatile=*/false,

                              getContext().getSizeType(), E->getBeginLoc());

    }

  }

  // LLVM can't handle Type=3 appropriately, and __builtin_object_size shouldn't

  // evaluate E for side-effects. In either case, we shouldn't lower to

  // @llvm.objectsize.

  if (Type == 3 || 
(210
!EmittedE210
 && 
E->HasSideEffects(getContext())170
))

    return getDefaultBuiltinObjectSizeResult(Type, ResType);

  Value *Ptr = EmittedE ? 
EmittedE40
 : 
EmitScalarExpr(E)170
;

  assert(Ptr->getType()->isPointerTy() &&

         "Non-pointer passed to __builtin_object_size?");

  Function *F =

      CGM.getIntrinsic(Intrinsic::objectsize, {ResType, Ptr->getType()});

  // LLVM only supports 0 and 2, make sure that we pass along that as a boolean.

  Value *Min = Builder.getInt1((Type & 2) != 0);

  // For GCC compatibility, __builtin_object_size treat NULL as unknown size.

  Value *NullIsUnknown = Builder.getTrue();

  Value *Dynamic = Builder.getInt1(IsDynamic);

  return Builder.CreateCall(F, {Ptr, Min, NullIsUnknown, Dynamic});

}

namespace {

/// A struct to generically describe a bit test intrinsic.

struct BitTest {

  enum ActionKind : uint8_t { TestOnly, Complement, Reset, Set };

  enum InterlockingKind : uint8_t {

    Unlocked,

    Sequential,

    Acquire,

    Release,

    NoFence

  };

  ActionKind Action;

  InterlockingKind Interlocking;

  bool Is64Bit;

  static BitTest decodeBitTestBuiltin(unsigned BuiltinID);

};

} // namespace

BitTest BitTest::decodeBitTestBuiltin(unsigned BuiltinID) {

  switch (BuiltinID) {

    // Main portable variants.

  case Builtin::BI_bittest:

    return {TestOnly, Unlocked, false};

  case Builtin::BI_bittestandcomplement:

    return {Complement, Unlocked, false};

  case Builtin::BI_bittestandreset:

    return {Reset, Unlocked, false};

  case Builtin::BI_bittestandset:

    return {Set, Unlocked, false};

  case Builtin::BI_interlockedbittestandreset:

    return {Reset, Sequential, false};

  case Builtin::BI_interlockedbittestandset:

    return {Set, Sequential, false};

    // X86-specific 64-bit variants.

  case Builtin::BI_bittest64:

    return {TestOnly, Unlocked, true};

  case Builtin::BI_bittestandcomplement64:

    return {Complement, Unlocked, true};

  case Builtin::BI_bittestandreset64:

    return {Reset, Unlocked, true};

  case Builtin::BI_bittestandset64:

    return {Set, Unlocked, true};

  case Builtin::BI_interlockedbittestandreset64:

    return {Reset, Sequential, true};

  case Builtin::BI_interlockedbittestandset64:

    return {Set, Sequential, true};

    // ARM/AArch64-specific ordering variants.

  case Builtin::BI_interlockedbittestandset_acq:

    return {Set, Acquire, false};

  case Builtin::BI_interlockedbittestandset_rel:

    return {Set, Release, false};

  case Builtin::BI_interlockedbittestandset_nf:

    return {Set, NoFence, false};

  case Builtin::BI_interlockedbittestandreset_acq:

    return {Reset, Acquire, false};

  case Builtin::BI_interlockedbittestandreset_rel:

    return {Reset, Release, false};

  case Builtin::BI_interlockedbittestandreset_nf:

    return {Reset, NoFence, false};

  }

  llvm_unreachable("expected only bittest intrinsics");

}

static char bitActionToX86BTCode(BitTest::ActionKind A) {

  switch (A) {

  case BitTest::TestOnly:   return '\0';

  case BitTest::Complement: return 'c';

  case BitTest::Reset:      return 'r';

  case BitTest::Set:        return 's';

  }

  llvm_unreachable("invalid action");

}

static llvm::Value *EmitX86BitTestIntrinsic(CodeGenFunction &CGF,

                                            BitTest BT,

                                            const CallExpr *E, Value *BitBase,

                                            Value *BitPos) {

  char Action = bitActionToX86BTCode(BT.Action);

  char SizeSuffix = BT.Is64Bit ? 
'q'6
 : 
'l'7
;

  // Build the assembly.

  SmallString<64> Asm;

  raw_svector_ostream AsmOS(Asm);

  if (BT.Interlocking != BitTest::Unlocked)

    AsmOS << "lock ";

  AsmOS << "bt";

  if (Action)

    AsmOS << Action;

  AsmOS << SizeSuffix << " $2, ($1)";

  // Build the constraints. FIXME: We should support immediates when possible.

  std::string Constraints = "={@ccc},r,r,~{cc},~{memory}";

  std::string MachineClobbers = CGF.getTarget().getClobbers();

  if (!MachineClobbers.empty()) {

    Constraints += ',';

    Constraints += MachineClobbers;

  }

  llvm::IntegerType *IntType = llvm::IntegerType::get(

      CGF.getLLVMContext(),

      CGF.getContext().getTypeSize(E->getArg(1)->getType()));

  llvm::Type *IntPtrType = IntType->getPointerTo();

  llvm::FunctionType *FTy =

      llvm::FunctionType::get(CGF.Int8Ty, {IntPtrType, IntType}, false);

  llvm::InlineAsm *IA =

      llvm::InlineAsm::get(FTy, Asm, Constraints, /*hasSideEffects=*/true);

  return CGF.Builder.CreateCall(IA, {BitBase, BitPos});

}

static llvm::AtomicOrdering

getBitTestAtomicOrdering(BitTest::InterlockingKind I) {

  switch (I) {

  case BitTest::Unlocked:   return llvm::AtomicOrdering::NotAtomic;

  case BitTest::Sequential: return llvm::AtomicOrdering::SequentiallyConsistent;

  case BitTest::Acquire:    return llvm::AtomicOrdering::Acquire;

  case BitTest::Release:    return llvm::AtomicOrdering::Release;

  case BitTest::NoFence:    return llvm::AtomicOrdering::Monotonic;

  }

  llvm_unreachable("invalid interlocking");

}

/// Emit a _bittest* intrinsic. These intrinsics take a pointer to an array of

/// bits and a bit position and read and optionally modify the bit at that

/// position. The position index can be arbitrarily large, i.e. it can be larger

/// than 31 or 63, so we need an indexed load in the general case.

static llvm::Value *EmitBitTestIntrinsic(CodeGenFunction &CGF,

                                         unsigned BuiltinID,

                                         const CallExpr *E) {

  Value *BitBase = CGF.EmitScalarExpr(E->getArg(0));

  Value *BitPos = CGF.EmitScalarExpr(E->getArg(1));

  BitTest BT = BitTest::decodeBitTestBuiltin(BuiltinID);

  // X86 has special BT, BTC, BTR, and BTS instructions that handle the array

  // indexing operation internally. Use them if possible.

  if (CGF.getTarget().getTriple().isX86())

    return EmitX86BitTestIntrinsic(CGF, BT, E, BitBase, BitPos);

  // Otherwise, use generic code to load one byte and test the bit. Use all but

  // the bottom three bits as the array index, and the bottom three bits to form

  // a mask.

  // Bit = BitBaseI8[BitPos >> 3] & (1 << (BitPos & 0x7)) != 0;

  Value *ByteIndex = CGF.Builder.CreateAShr(

      BitPos, llvm::ConstantInt::get(BitPos->getType(), 3), "bittest.byteidx");

  Value *BitBaseI8 = CGF.Builder.CreatePointerCast(BitBase, CGF.Int8PtrTy);

  Address ByteAddr(CGF.Builder.CreateInBoundsGEP(CGF.Int8Ty, BitBaseI8,

                                                 ByteIndex, "bittest.byteaddr"),

                   CharUnits::One());

  Value *PosLow =

      CGF.Builder.CreateAnd(CGF.Builder.CreateTrunc(BitPos, CGF.Int8Ty),

                            llvm::ConstantInt::get(CGF.Int8Ty, 0x7));

  // The updating instructions will need a mask.

  Value *Mask = nullptr;

  if (BT.Action != BitTest::TestOnly) {

    Mask = CGF.Builder.CreateShl(llvm::ConstantInt::get(CGF.Int8Ty, 1), PosLow,

                                 "bittest.mask");

  }

  // Check the action and ordering of the interlocked intrinsics.

  llvm::AtomicOrdering Ordering = getBitTestAtomicOrdering(BT.Interlocking);

  Value *OldByte = nullptr;

  if (Ordering != llvm::AtomicOrdering::NotAtomic) {

    // Emit a combined atomicrmw load/store operation for the interlocked

    // intrinsics.

    llvm::AtomicRMWInst::BinOp RMWOp = llvm::AtomicRMWInst::Or;

    if (BT.Action == BitTest::Reset) {

      Mask = CGF.Builder.CreateNot(Mask);

      RMWOp = llvm::AtomicRMWInst::And;

    }

    OldByte = CGF.Builder.CreateAtomicRMW(RMWOp, ByteAddr.getPointer(), Mask,

                                          Ordering);

  } else {

    // Emit a plain load for the non-interlocked intrinsics.

    OldByte = CGF.Builder.CreateLoad(ByteAddr, "bittest.byte");

    Value *NewByte = nullptr;

    switch (BT.Action) {

    case BitTest::TestOnly:

      // Don't store anything.

      break;

    case BitTest::Complement:

      NewByte = CGF.Builder.CreateXor(OldByte, Mask);

      break;

    case BitTest::Reset:

      NewByte = CGF.Builder.CreateAnd(OldByte, CGF.Builder.CreateNot(Mask));

      break;

    case BitTest::Set:

      NewByte = CGF.Builder.CreateOr(OldByte, Mask);

      break;

    }

    if (NewByte)

      CGF.Builder.CreateStore(NewByte, ByteAddr);

  }

  // However we loaded the old byte, either by plain load or atomicrmw, shift

  // the bit into the low position and mask it to 0 or 1.

  Value *ShiftedByte = CGF.Builder.CreateLShr(OldByte, PosLow, "bittest.shr");

  return CGF.Builder.CreateAnd(

      ShiftedByte, llvm::ConstantInt::get(CGF.Int8Ty, 1), "bittest.res");

}

static llvm::Value *emitPPCLoadReserveIntrinsic(CodeGenFunction &CGF,

                                                unsigned BuiltinID,

                                                const CallExpr *E) {

  Value *Addr = CGF.EmitScalarExpr(E->getArg(0));

  SmallString<64> Asm;

  raw_svector_ostream AsmOS(Asm);

  llvm::IntegerType *RetType = CGF.Int32Ty;

  switch (BuiltinID) {

  case clang::PPC::BI__builtin_ppc_ldarx:

    AsmOS << "ldarx ";

    RetType = CGF.Int64Ty;

    break;

  case clang::PPC::BI__builtin_ppc_lwarx:

    AsmOS << "lwarx ";

    RetType = CGF.Int32Ty;

    break;

  case clang::PPC::BI__builtin_ppc_lharx:

    AsmOS << "lharx ";

    RetType = CGF.Int16Ty;

    break;

  case clang::PPC::BI__builtin_ppc_lbarx:

    AsmOS << "lbarx ";

    RetType = CGF.Int8Ty;

    break;

  default:

    llvm_unreachable("Expected only PowerPC load reserve intrinsics");

  }

  AsmOS << "$0, ${1:y}";

  std::string Constraints = "=r,*Z,~{memory}";

  std::string MachineClobbers = CGF.getTarget().getClobbers();

  if (!MachineClobbers.empty()) {

    Constraints += ',';

    Constraints += MachineClobbers;

  }

  llvm::Type *IntPtrType = RetType->getPointerTo();

  llvm::FunctionType *FTy =

      llvm::FunctionType::get(RetType, {IntPtrType}, false);

  llvm::InlineAsm *IA =

      llvm::InlineAsm::get(FTy, Asm, Constraints, /*hasSideEffects=*/true);

  return CGF.Builder.CreateCall(IA, {Addr});