//===- Intrinsics.td - Defines all LLVM intrinsics ---------*- tablegen -*-===//

//

// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.

// See https://llvm.org/LICENSE.txt for license information.

// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

//

//===----------------------------------------------------------------------===//

//

// This file defines properties of all LLVM intrinsics.

//

//===----------------------------------------------------------------------===//

include "llvm/CodeGen/ValueTypes.td"

include "llvm/CodeGen/SDNodeProperties.td"

//===----------------------------------------------------------------------===//

// Properties we keep track of for intrinsics.

//===----------------------------------------------------------------------===//

class IntrinsicProperty<bit is_default = false> {

bit IsDefault = is_default;

}

// Intr*Mem - Memory properties. If no property is set, the worst case

// is assumed (it may read and write any memory it can get access to and it may

// have other side effects).

// IntrNoMem - The intrinsic does not access memory or have any other side

// effects. It may be CSE'd deleted if dead, etc.

def IntrNoMem : IntrinsicProperty;

// IntrReadMem - This intrinsic only reads from memory. It does not write to

// memory and has no other side effects. Therefore, it cannot be moved across

// potentially aliasing stores. However, it can be reordered otherwise and can

// be deleted if dead.

def IntrReadMem : IntrinsicProperty;

// IntrWriteMem - This intrinsic only writes to memory, but does not read from

// memory, and has no other side effects. This means dead stores before calls

// to this intrinsics may be removed.

def IntrWriteMem : IntrinsicProperty;

// IntrArgMemOnly - This intrinsic only accesses memory that its pointer-typed

// argument(s) points to, but may access an unspecified amount. Other than

// reads from and (possibly volatile) writes to memory, it has no side effects.

def IntrArgMemOnly : IntrinsicProperty;

// IntrInaccessibleMemOnly -- This intrinsic only accesses memory that is not

// accessible by the module being compiled. This is a weaker form of IntrNoMem.

def IntrInaccessibleMemOnly : IntrinsicProperty;

// IntrInaccessibleMemOrArgMemOnly -- This intrinsic only accesses memory that

// its pointer-typed arguments point to or memory that is not accessible

// by the module being compiled. This is a weaker form of IntrArgMemOnly.

def IntrInaccessibleMemOrArgMemOnly : IntrinsicProperty;

// Tablegen representation of IRMemLocation.

class IntrinsicMemoryLocation;

// TODO: Populate with all IRMemLocation enum values and update

// getValueAsIRMemLocation accordingly.

def InaccessibleMem : IntrinsicMemoryLocation;

def TargetMem0 : IntrinsicMemoryLocation;

def TargetMem1 : IntrinsicMemoryLocation;

// The list of IRMemoryLocations that are read from.

class IntrRead<list<IntrinsicMemoryLocation> idx> : IntrinsicProperty {

list<IntrinsicMemoryLocation> MemLoc=idx;

}

// The list of IRMemoryLocations that are write to.

class IntrWrite<list<IntrinsicMemoryLocation> idx> : IntrinsicProperty {

list<IntrinsicMemoryLocation> MemLoc=idx;

}

// Commutative - This intrinsic is commutative: X op Y == Y op X.

def Commutative : IntrinsicProperty;

// Throws - This intrinsic can throw.

def Throws : IntrinsicProperty;

// Attribute index needs to match `AttrIndex` defined `Attributes.h`.

class AttrIndex<int idx> {

int Value = idx;

}

def RetIndex : AttrIndex<0>;

class ArgIndex<int argNo> : AttrIndex<!add(argNo, 1)>;

// Note: Properties that are applicable either to arguments or return values

// use AttrIndex. Properties applicable only to arguments use ArgIndex. Please

// refer to Attributes.td.

// NoCapture - The specified argument pointer is not captured by the intrinsic.

class NoCapture<ArgIndex idx> : IntrinsicProperty {

int ArgNo = idx.Value;

}

// NoAlias - The return value or the specified argument pointer is not aliasing

// other "noalias" pointer arguments of the intrinsic wrt. the intrinsic scope.

class NoAlias<AttrIndex idx> : IntrinsicProperty {

int ArgNo = idx.Value;

}

// NoUndef - The return value or specified argument is neither undef nor poison.

class NoUndef<AttrIndex idx> : IntrinsicProperty {

int ArgNo = idx.Value;

}

// NonNull - The return value or specified argument is not null.

class NonNull<AttrIndex idx> : IntrinsicProperty {

int ArgNo = idx.Value;

}

// Align - Alignment for return value or the specified argument.

class Align<AttrIndex idx, int align> : IntrinsicProperty {

int ArgNo = idx.Value;

int Align = align;

}

// Dereferenceable -- Return value or the specified argument is dereferenceable

// upto `bytes` bytes in size.

class Dereferenceable<AttrIndex idx, int bytes> : IntrinsicProperty {

int ArgNo = idx.Value;

int Bytes = bytes;

}

// Returned - The specified argument is always the return value of the

// intrinsic.

class Returned<ArgIndex idx> : IntrinsicProperty {

int ArgNo = idx.Value;

}

// ImmArg - The specified argument must be an immediate.

class ImmArg<ArgIndex idx> : IntrinsicProperty {

int ArgNo = idx.Value;

}

// ReadOnly - The specified argument pointer is not written to through the

// pointer by the intrinsic.

class ReadOnly<ArgIndex idx> : IntrinsicProperty {

int ArgNo = idx.Value;

}

// WriteOnly - The intrinsic does not read memory through the specified

// argument pointer.

class WriteOnly<ArgIndex idx> : IntrinsicProperty {

int ArgNo = idx.Value;

}

// ReadNone - The specified argument pointer is not dereferenced by the

// intrinsic.

class ReadNone<ArgIndex idx> : IntrinsicProperty {

int ArgNo = idx.Value;

}

// The return value or argument is in the range [lower, upper),

// where lower and upper are interpreted as signed integers.

class Range<AttrIndex idx, int lower, int upper> : IntrinsicProperty {

int ArgNo = idx.Value;

int Lower = lower;

int Upper = upper;

}

// ArgProperty - Base class for argument properties that can be specified in ArgInfo.

class ArgProperty;

// ArgName - Specifies the name of an argument for pretty-printing.

class ArgName<string name> : ArgProperty {

string Name = name;

}

// ImmArgPrinter - Specifies a custom printer function for immediate arguments.

class ImmArgPrinter<string funcname> : ArgProperty {

string FuncName = funcname;

}

// ArgInfo - The specified argument has properties defined by a list of ArgProperty objects.

class ArgInfo<ArgIndex idx, list<ArgProperty> arg_properties> : IntrinsicProperty {

int ArgNo = idx.Value;

list<ArgProperty> Properties = arg_properties;

}

def IntrNoReturn : IntrinsicProperty;

// Applied by default.

def IntrNoCallback : IntrinsicProperty<1>;

// IntrNoSync - Threads executing the intrinsic will not synchronize using

// memory or other means. Applied by default.

def IntrNoSync : IntrinsicProperty<1>;

// Applied by default.

def IntrNoFree : IntrinsicProperty<1>;

// Applied by default.

def IntrWillReturn : IntrinsicProperty<1>;

// IntrCold - Calls to this intrinsic are cold.

// Parallels the cold attribute on LLVM IR functions.

def IntrCold : IntrinsicProperty;

// IntrNoDuplicate - Calls to this intrinsic cannot be duplicated.

// Parallels the noduplicate attribute on LLVM IR functions.

def IntrNoDuplicate : IntrinsicProperty;

// IntrNoMerge - Calls to this intrinsic cannot be merged

// Parallels the nomerge attribute on LLVM IR functions.

def IntrNoMerge : IntrinsicProperty;

// IntrConvergent - Calls to this intrinsic are convergent and may not be made

// control-dependent on any additional values.

// Parallels the convergent attribute on LLVM IR functions.

def IntrConvergent : IntrinsicProperty;

// This property indicates that the intrinsic is safe to speculate.

def IntrSpeculatable : IntrinsicProperty;

// This property can be used to override the 'has no other side effects'

// language of the IntrNoMem, IntrReadMem, IntrWriteMem, and IntrArgMemOnly

// intrinsic properties. By default, intrinsics are assumed to have side

// effects, so this property is only necessary if you have defined one of

// the memory properties listed above.

// For this property, 'side effects' has the same meaning as 'side effects'

// defined by the hasSideEffects property of the TableGen Instruction class.

def IntrHasSideEffects : IntrinsicProperty;

// Result will not be undef or poison if all arguments are not undef and not

// poison.

def IntrNoCreateUndefOrPoison : IntrinsicProperty;

// This property indicates that the intrinsic is trivially scalarizable.

def IntrTriviallyScalarizable : IntrinsicProperty;

//===----------------------------------------------------------------------===//

// IIT constants and utils

//===----------------------------------------------------------------------===//

// llvm::Intrinsic::IITDescriptor::AnyKind::AK_%

def AnyKind {

int Any = 0;

int AnyInteger = 1;

int AnyFloat = 2;

int AnyVector = 3;

int AnyPointer = 4;

int MatchType = 7;

}

// Placeholder to encode the overload index of the current type. We encode bit

// 8 = 1 to indicate that this entry needs to be patched up with the overload

// index (to prevent conflict with any valid not-to-be-patched IIT enccoding

// byte, whose value will be <= 255). The AnyKind itself is in the lower bits.

// Note that this is just a transient representation till its gets processed

// by `DoPatchOverloadIndex` below, so this is *not* the encoding of the

// final type signature.

class OverloadIndexPlaceholder <int AnyKindVal> {

int ID = 0x100;

int ret = !or(ID, AnyKindVal);

}

// This class defines how the overload index and the arg kind are actually

// packed into a single byte for the final IIT table encoding. Overload index is

// packed in low 5 bits, argument kind is packed in upper 3 bits. This enables

// us to use the same packing for llvm_any* types, which use a argument kind

// and for partially dependent types like `LLVMVectorOfAnyPointersToElt` which

// do not use the argument kind and expect the overload index in the lower bits.

class PackOverloadIndex<int OverloadIndex, int AnyKindVal> {

assert !lt(OverloadIndex, 32), "Cannot support more than 32 overload types";

assert !lt(AnyKindVal, 8), "Cannot support more than 8 argument kinds";

int ret = !or(!shl(AnyKindVal, 5), OverloadIndex);

}

// This class handles the actual patching of the overload index into a component

// value `Sig` in the type signature. If the value is > 255, it's a placeholder

// value generated by OverloadIndexPlaceholder and patching is needed, else the

// value is left unchanged.

class PatchOverloadIndex<int Sig, int OverloadIndex> {

int AnyKindVal = !and(Sig, 0x7);

int ret = !cond(

// If the value is > 255, it indicates that patching is needed.

!gt(Sig, 255) : PackOverloadIndex<OverloadIndex, AnyKindVal>.ret,

true: Sig);

}

//===----------------------------------------------------------------------===//

// IIT_Info

//===----------------------------------------------------------------------===//

class IIT_Base<int num> {

int Number = num;

list<ValueType> VTs = ?;

}

class IIT_VT<ValueType vt, int num> : IIT_Base<num> {

let VTs = [vt];

}

class IIT_Int<int size, int num> : IIT_Base<num> {

let VTs = !filter(vti, ValueTypes,

!and(vti.isInteger, !eq(vti.Size, size)));

}

class IIT_Vec<int nelem, int num> : IIT_Base<num> {

let VTs = !filter(vti, ValueTypes,

!and(vti.isVector, !eq(vti.nElem, nelem)));

}

defset list<IIT_Base> IIT_all = {

def IIT_Done : IIT_Base< 0>;

def IIT_I1 : IIT_Int<1, 1>;

def IIT_I8 : IIT_Int<8, 2>;

def IIT_I16 : IIT_Int<16, 3>;

def IIT_I32 : IIT_Int<32, 4>;

def IIT_I64 : IIT_Int<64, 5>;

def IIT_F16 : IIT_VT<f16, 6>;

def IIT_F32 : IIT_VT<f32, 7>;

def IIT_F64 : IIT_VT<f64, 8>;

def IIT_V2 : IIT_Vec<2, 9>;

def IIT_V4 : IIT_Vec<4, 10>;

def IIT_V8 : IIT_Vec<8, 11>;

def IIT_V16 : IIT_Vec<16, 12>;

def IIT_V32 : IIT_Vec<32, 13>;

def IIT_PTR : IIT_Base< 14>; // pointer with address space 0.

def IIT_ANY : IIT_Base< 15>;

def IIT_V64 : IIT_Vec<64, 16>;

def IIT_MMX : IIT_VT<x86mmx, 17>;

def IIT_TOKEN : IIT_VT<token, 18>;

def IIT_METADATA : IIT_VT<MetadataVT, 19>;

// Note: Unused IIT code 20.

def IIT_STRUCT : IIT_Base<21>;

def IIT_EXTEND_ARG : IIT_Base<22>;

def IIT_TRUNC_ARG : IIT_Base<23>;

def IIT_PTR_AS : IIT_Base<24>; // Pointer with address space.

def IIT_V1 : IIT_Vec<1, 25>;

def IIT_VARARG : IIT_VT<isVoid, 26>;

def IIT_ONE_NTH_ELTS_VEC_ARG : IIT_Base<27>;

def IIT_SAME_VEC_WIDTH_ARG : IIT_Base<28>;

def IIT_VEC_OF_ANYPTRS_TO_ELT : IIT_Base<29>;

def IIT_I128 : IIT_Int<128, 30>;

def IIT_V512 : IIT_Vec<512, 31>;

def IIT_V1024 : IIT_Vec<1024, 32>;

def IIT_F128 : IIT_VT<f128, 33>;

def IIT_VEC_ELEMENT : IIT_Base<34>;

def IIT_SCALABLE_VEC : IIT_Base<35>;

def IIT_SUBDIVIDE2_ARG : IIT_Base<36>;

def IIT_SUBDIVIDE4_ARG : IIT_Base<37>;

def IIT_VEC_OF_BITCASTS_TO_INT : IIT_Base<38>;

def IIT_V128 : IIT_Vec<128, 39>;

def IIT_BF16 : IIT_VT<bf16, 40>;

def IIT_V256 : IIT_Vec<256, 41>;

def IIT_AMX : IIT_VT<x86amx, 42>;

def IIT_PPCF128 : IIT_VT<ppcf128, 43>;

def IIT_V3 : IIT_Vec<3, 44>;

def IIT_EXTERNREF : IIT_VT<externref, 45>;

def IIT_FUNCREF : IIT_VT<funcref, 46>;

def IIT_I2 : IIT_Int<2, 47>;

def IIT_I4 : IIT_Int<4, 48>;

def IIT_AARCH64_SVCOUNT : IIT_VT<aarch64svcount, 49>;

def IIT_V6 : IIT_Vec<6, 50>;

def IIT_V10 : IIT_Vec<10, 51>;

def IIT_V2048 : IIT_Vec<2048, 52>;

def IIT_V4096 : IIT_Vec<4096, 53>;

}

defvar IIT_all_FixedTypes = !filter(iit, IIT_all,

!or(!isa<IIT_VT>(iit), !isa<IIT_Int>(iit)));

defvar IIT_all_VectorTypes = !filter(iit, IIT_all,

!isa<IIT_Vec>(iit));

//===----------------------------------------------------------------------===//

// Types used by intrinsics.

//===----------------------------------------------------------------------===//

class LLVMType<ValueType vt> {

ValueType VT = vt;

list<IIT_Base> IITs = !filter(iit, IIT_all_FixedTypes,

!not(!empty(!filter(iit_vt, iit.VTs,

!eq(iit_vt, !if(vt.isVector, vt.ElementType, vt))))));

assert !le(!size(IITs), 1), "Duplicate type";

list<IIT_Base> IIT_Vecs = !if(vt.isVector,

!filter(iit, IIT_all_VectorTypes,

!not(!empty(!filter(iit_vt, iit.VTs, !and(

!eq(iit_vt.ElementType, vt.ElementType),

!eq(iit_vt.nElem, vt.nElem)))))),

[]);

assert !le(!size(IIT_Vecs), 1), "Duplicate type";

// For vector types, assert that the IIT_Vecs list is not empty.

assert !or(!not(vt.isVector), !not(!empty(IIT_Vecs))),

"Invalid IIT encoding for vector type v" # vt.nElem # vt.ElementType;

list<int> Sig = !listconcat(

!foreach(iit, IIT_Vecs, iit.Number),

!if(vt.isScalable, [IIT_SCALABLE_VEC.Number], []),

!foreach(iit, IITs, iit.Number));

}

class LLVMAnyType<ValueType vt> : LLVMType<vt> {

int ArgCode = !cond(

!eq(vt, Any) : AnyKind.Any,

!eq(vt, iAny) : AnyKind.AnyInteger,

!eq(vt, fAny) : AnyKind.AnyFloat,

!eq(vt, vAny) : AnyKind.AnyVector,

!eq(vt, pAny) : AnyKind.AnyPointer,

);

let Sig = [

IIT_ANY.Number,

OverloadIndexPlaceholder <ArgCode>.ret,

];

assert VT.isOverloaded, "LLVMAnyType.VT should have isOverloaded";

}

class LLVMQualPointerType<int addrspace>

: LLVMType<iPTR> {

assert !and(!le(0, addrspace), !le(addrspace, 255)),

"Address space exceeds 255";

let Sig =

!if(addrspace, [

IIT_PTR_AS.Number,

addrspace,

], [

IIT_PTR.Number,

]);

}

// Note: CodeGenIntrinsics.cpp seems to check this class to check pointers.

class LLVMAnyPointerType : LLVMAnyType<pAny>;

// Dependent types: These are types that depend on another LLVMAnyType overload

// type. There are 2 subclasses of dependent types:

// 1. Fully dependent types: dependent type can be completely derived from

// another overload type.

// 2. Partially dependent types: dependent type is constrained by another

// overload type, but cannot be fully derived from it. Such types get

// assigned an overload index and are a part of the overloaded types for an

// intrinsics.

class LLVMDependentType<int oidx> : LLVMType<OtherVT> {

// Overload index of the overload type that this dependent type is dependent

// on.

int OverloadIndex = oidx;

}

class LLVMFullyDependentType<int oidx, IIT_Base IIT_Info>

: LLVMDependentType<oidx> {

// For fully dependent overload types, the type signature is just the IIT code

// followed by the overload index of the overload type it depends on.

let Sig = [

IIT_Info.Number,

PackOverloadIndex<OverloadIndex, AnyKind.MatchType>.ret

];

}

class LLVMPartiallyDependentType<int oidx, IIT_Base IIT_Info>

: LLVMDependentType<oidx> {

// For partially dependent type, the type signature is the IIT code, followed

// by this type's oveerload index, followed by the overload index of the

// overload type its depends on.

let Sig = [

IIT_Info.Number,

// This types overload index, arg kind ignored.

OverloadIndexPlaceholder <0>.ret,

// Overload index of the reference overload type, arg kind ignored.

PackOverloadIndex<OverloadIndex, 0>.ret,

];

}

// ----------------------------------------------------------------------------

// Various sub-classes of fully dependent types.

// Match the type of another intrinsic parameter. `oidx` is the overload index

// of the overloaded type that this type is dependent on. Overload types are

// either LLVMAnyType or LLVMPartiallyDependentType.

//

// Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_anyfloat_ty, LLVMMatchType<0>]>

//

// has one overloaded type, the 2nd argument. LLVMMatchType<0> refers to this

// first overloaded type, which is the 2nd argument. As another example,

//

// Intrinsic<[llvm_anyint_ty],

// [llvm_anyfloat_ty, LLVMMatchType<0>, llvm_anyfloat_ty, LLVMMatchType<2>]>

//

// has 3 overloaded types:

// overload index 0 = return type

// overload index 1 = first argument

// overload index 2 = third argument

//

// LLVMMatchType<0> therefore will match the return type, and

// LLVMMatchType<2> will match the 3rd argument.

class LLVMMatchType<int oidx> : LLVMFullyDependentType<oidx, IIT_ANY>;

// Match the type of another intrinsic parameter that is expected to be based on

// an integral type (i.e. either iN or <N x iM>), but change the scalar size to

// be twice as wide or half as wide as the other type. This is only useful when

// the intrinsic is overloaded, so the matched type should be declared as iAny.

class LLVMExtendedType<int oidx> : LLVMFullyDependentType<oidx, IIT_EXTEND_ARG>;

class LLVMTruncatedType<int oidx> : LLVMFullyDependentType<oidx, IIT_TRUNC_ARG>;

// Match the scalar/vector of another intrinsic parameter but with a different

// element type. Either both are scalars or both are vectors with the same

// number of elements.

class LLVMScalarOrSameVectorWidth<int oidx, LLVMType elty>

: LLVMFullyDependentType<oidx, IIT_SAME_VEC_WIDTH_ARG> {

let Sig = !listconcat([

IIT_SAME_VEC_WIDTH_ARG.Number,

// Overload index of the reference overload type, arg kind ignored.

PackOverloadIndex<OverloadIndex, 0>.ret,

], elty.Sig);

}

class LLVMVectorElementType<int oidx>

: LLVMFullyDependentType<oidx, IIT_VEC_ELEMENT>;

// Match the type of another intrinsic parameter that is expected to be a

// vector type, but change the element count to be 1/n of it.

class LLVMOneNthElementsVectorType<int oidx, int n>

: LLVMFullyDependentType<oidx, IIT_ONE_NTH_ELTS_VEC_ARG> {

let Sig = [

IIT_ONE_NTH_ELTS_VEC_ARG.Number,

PackOverloadIndex<OverloadIndex, 0>.ret,

n,

];

}

// Match the type of another intrinsic parameter that is expected to be a

// vector type (i.e. <N x iM>) but with each element subdivided to

// form a vector with more elements that are smaller than the original.

class LLVMSubdivide2VectorType<int oidx>

: LLVMFullyDependentType<oidx, IIT_SUBDIVIDE2_ARG>;

class LLVMSubdivide4VectorType<int oidx>

: LLVMFullyDependentType<oidx, IIT_SUBDIVIDE4_ARG>;

// Match the element count and bit width of another intrinsic parameter, but

// change the element type to an integer.

class LLVMVectorOfBitcastsToInt<int oidx>

: LLVMFullyDependentType<oidx, IIT_VEC_OF_BITCASTS_TO_INT>;

// ----------------------------------------------------------------------------

// Various sub-classes of partially dependent types.

class LLVMVectorOfAnyPointersToElt<int oidx>

: LLVMPartiallyDependentType<oidx, IIT_VEC_OF_ANYPTRS_TO_ELT>;

def llvm_void_ty : LLVMType<isVoid>;

def llvm_any_ty : LLVMAnyType<Any>;

def llvm_anyint_ty : LLVMAnyType<iAny>;

def llvm_anyfloat_ty : LLVMAnyType<fAny>;

def llvm_anyvector_ty : LLVMAnyType<vAny>;

def llvm_anyptr_ty : LLVMAnyPointerType; // ptr addrspace(N)

def llvm_i1_ty : LLVMType<i1>;

def llvm_i8_ty : LLVMType<i8>;

def llvm_i16_ty : LLVMType<i16>;

def llvm_i32_ty : LLVMType<i32>;

def llvm_i64_ty : LLVMType<i64>;

def llvm_i128_ty : LLVMType<i128>;

def llvm_half_ty : LLVMType<f16>;

def llvm_bfloat_ty : LLVMType<bf16>;

def llvm_float_ty : LLVMType<f32>;

def llvm_double_ty : LLVMType<f64>;

def llvm_f80_ty : LLVMType<f80>;

def llvm_f128_ty : LLVMType<f128>;

def llvm_ppcf128_ty : LLVMType<ppcf128>;

def llvm_ptr_ty : LLVMQualPointerType<0>; // ptr

def llvm_empty_ty : LLVMType<OtherVT>; // { }

def llvm_metadata_ty : LLVMType<MetadataVT>; // !{...}

def llvm_token_ty : LLVMType<token>; // token

def llvm_x86mmx_ty : LLVMType<x86mmx>;

def llvm_aarch64_svcount_ty : LLVMType<aarch64svcount>;

def llvm_x86amx_ty : LLVMType<x86amx>;

def llvm_v2i1_ty : LLVMType<v2i1>; // 2 x i1

def llvm_v4i1_ty : LLVMType<v4i1>; // 4 x i1

def llvm_v8i1_ty : LLVMType<v8i1>; // 8 x i1

def llvm_v16i1_ty : LLVMType<v16i1>; // 16 x i1

def llvm_v32i1_ty : LLVMType<v32i1>; // 32 x i1

def llvm_v64i1_ty : LLVMType<v64i1>; // 64 x i1

def llvm_v128i1_ty : LLVMType<v128i1>; // 128 x i1

def llvm_v256i1_ty : LLVMType<v256i1>; // 256 x i1

def llvm_v512i1_ty : LLVMType<v512i1>; // 512 x i1

def llvm_v1024i1_ty : LLVMType<v1024i1>; //1024 x i1

def llvm_v2048i1_ty : LLVMType<v2048i1>; //2048 x i1

def llvm_v4096i1_ty : LLVMType<v4096i1>; //4096 x i1

def llvm_v1i8_ty : LLVMType<v1i8>; // 1 x i8

def llvm_v2i8_ty : LLVMType<v2i8>; // 2 x i8

def llvm_v3i8_ty : LLVMType<v3i8>; // 3 x i8

def llvm_v4i8_ty : LLVMType<v4i8>; // 4 x i8

def llvm_v8i8_ty : LLVMType<v8i8>; // 8 x i8

def llvm_v16i8_ty : LLVMType<v16i8>; // 16 x i8

def llvm_v32i8_ty : LLVMType<v32i8>; // 32 x i8

def llvm_v64i8_ty : LLVMType<v64i8>; // 64 x i8

def llvm_v128i8_ty : LLVMType<v128i8>; //128 x i8

def llvm_v256i8_ty : LLVMType<v256i8>; //256 x i8

def llvm_v1i16_ty : LLVMType<v1i16>; // 1 x i16

def llvm_v2i16_ty : LLVMType<v2i16>; // 2 x i16

def llvm_v4i16_ty : LLVMType<v4i16>; // 4 x i16

def llvm_v8i16_ty : LLVMType<v8i16>; // 8 x i16

def llvm_v16i16_ty : LLVMType<v16i16>; // 16 x i16

def llvm_v32i16_ty : LLVMType<v32i16>; // 32 x i16

def llvm_v64i16_ty : LLVMType<v64i16>; // 64 x i16

def llvm_v128i16_ty : LLVMType<v128i16>; // 128 x i16

def llvm_v4096i16_ty : LLVMType<v4096i16>; // 4096 x i16

def llvm_v1i32_ty : LLVMType<v1i32>; // 1 x i32

def llvm_v2i32_ty : LLVMType<v2i32>; // 2 x i32

def llvm_v3i32_ty : LLVMType<v3i32>; // 3 x i32

def llvm_v4i32_ty : LLVMType<v4i32>; // 4 x i32

def llvm_v6i32_ty : LLVMType<v6i32>; // 6 x i32

def llvm_v8i32_ty : LLVMType<v8i32>; // 8 x i32

def llvm_v10i32_ty : LLVMType<v10i32>; // 10 x i32

def llvm_v16i32_ty : LLVMType<v16i32>; // 16 x i32

def llvm_v32i32_ty : LLVMType<v32i32>; // 32 x i32

def llvm_v64i32_ty : LLVMType<v64i32>; // 64 x i32

def llvm_v128i32_ty : LLVMType<v128i32>; // 128 x i32

def llvm_v256i32_ty : LLVMType<v256i32>; // 256 x i32

def llvm_v2048i32_ty : LLVMType<v2048i32>; // 2048 x i32

def llvm_v4096i32_ty : LLVMType<v4096i32>; // 4096 x i32

def llvm_v1i64_ty : LLVMType<v1i64>; // 1 x i64

def llvm_v2i64_ty : LLVMType<v2i64>; // 2 x i64

def llvm_v4i64_ty : LLVMType<v4i64>; // 4 x i64

def llvm_v8i64_ty : LLVMType<v8i64>; // 8 x i64

def llvm_v16i64_ty : LLVMType<v16i64>; // 16 x i64

def llvm_v32i64_ty : LLVMType<v32i64>; // 32 x i64

def llvm_v1i128_ty : LLVMType<v1i128>; // 1 x i128

def llvm_v2f16_ty : LLVMType<v2f16>; // 2 x half (__fp16)

def llvm_v4f16_ty : LLVMType<v4f16>; // 4 x half (__fp16)

def llvm_v8f16_ty : LLVMType<v8f16>; // 8 x half (__fp16)

def llvm_v16f16_ty : LLVMType<v16f16>; // 16 x half (__fp16)

def llvm_v32f16_ty : LLVMType<v32f16>; // 32 x half (__fp16)

def llvm_v4096f16_ty : LLVMType<v4096f16>; // 4096 x half (__fp16)

def llvm_v2bf16_ty : LLVMType<v2bf16>; // 2 x bfloat (__bf16)

def llvm_v4bf16_ty : LLVMType<v4bf16>; // 4 x bfloat (__bf16)

def llvm_v8bf16_ty : LLVMType<v8bf16>; // 8 x bfloat (__bf16)

def llvm_v16bf16_ty : LLVMType<v16bf16>; // 16 x bfloat (__bf16)

def llvm_v32bf16_ty : LLVMType<v32bf16>; // 32 x bfloat (__bf16)

def llvm_v4096bf16_ty : LLVMType<v4096bf16>; // 4096 x bfloat (__bf16)

def llvm_v1f32_ty : LLVMType<v1f32>; // 1 x float

def llvm_v2f32_ty : LLVMType<v2f32>; // 2 x float

def llvm_v3f32_ty : LLVMType<v3f32>; // 3 x float

def llvm_v4f32_ty : LLVMType<v4f32>; // 4 x float

def llvm_v8f32_ty : LLVMType<v8f32>; // 8 x float

def llvm_v10f32_ty : LLVMType<v10f32>; // 10 x float

def llvm_v16f32_ty : LLVMType<v16f32>; // 16 x float

def llvm_v32f32_ty : LLVMType<v32f32>; // 32 x float

def llvm_v2048f32_ty : LLVMType<v2048f32>; // 2048 x float

def llvm_v1f64_ty : LLVMType<v1f64>; // 1 x double

def llvm_v2f64_ty : LLVMType<v2f64>; // 2 x double

def llvm_v4f64_ty : LLVMType<v4f64>; // 4 x double

def llvm_v8f64_ty : LLVMType<v8f64>; // 8 x double

def llvm_v16f64_ty : LLVMType<v16f64>; // 16 x double

def llvm_vararg_ty : LLVMType<isVoid>; // this means vararg here

def llvm_externref_ty : LLVMType<externref>;

def llvm_funcref_ty : LLVMType<funcref>;

def llvm_exnref_ty : LLVMType<exnref>;

//===----------------------------------------------------------------------===//

// Computes the overload index for overloaded types used by an intrinsic.

// Input `IsOverloadedType` is a list of booleans, one for each type in the

// intrinsic's type signature. A 0 value indicate that that type is not an

// overloaded type, and 1 indicates that its an overloaded type.

//

// Assigns overload index by essentially computing a prefix sum on this list.

// The output list `ret` has value ? for non-overload types and the overload

// index for overloaded types.

class AssignOverloadIndex<list<bit> IsOverloadedType> {

list<int> PrefixSum = !foreach(i, !range(IsOverloadedType),

!if(IsOverloadedType[i],

!foldl(0, !range(0, i), m, j, !add(m, IsOverloadedType[j])),

-1));

// Assign ? if the entry was -1, else its the expected prefix sum.

list<int> ret = !foreach(a, PrefixSum, !if(!ge(a, 0), a, ?));

}

class TypeInfoGen<list<LLVMType> RetTypes, list<LLVMType> ParamTypes> {

list<LLVMType> AllTypes = !listconcat(RetTypes, ParamTypes);

// Assign overload index for all overloaded types

// (LLVMAnyType or LLVMPartiallyDependentType).

list<int> OverloadIdxs = AssignOverloadIndex<

!foreach(ty, AllTypes,

!or(!isa<LLVMAnyType>(ty), !isa<LLVMPartiallyDependentType>(ty)))>.ret;

// List of all overloaded types, in their overload-index order.

list<LLVMType> OverloadTypes = !filter(ty, AllTypes,

!or(!isa<LLVMAnyType>(ty), !isa<LLVMPartiallyDependentType>(ty)));

bit isOverloaded = !not(!empty(OverloadTypes));

list<int> TypeSig = !listflatten(!listconcat(

[!cond(

!eq(!size(RetTypes), 0): [IIT_Done.Number],

!eq(!size(RetTypes), 1): []<int>,

true: [IIT_STRUCT.Number, !sub(!size(RetTypes), 2)])],

!foreach(i, !range(AllTypes),

!foreach(a, AllTypes[i].Sig,

PatchOverloadIndex<a, OverloadIdxs[i]>.ret))));

}

//===----------------------------------------------------------------------===//

// Intrinsic Definitions.

//===----------------------------------------------------------------------===//

// Intrinsic class - This is used to define one LLVM intrinsic. The name of the

// intrinsic definition should start with "int_", then match the LLVM intrinsic

// name with the "llvm." prefix removed, and all "."s turned into "_"s. For

// example, llvm.bswap.i16 -> int_bswap_i16.

//

// * RetTypes is a list containing the return types expected for the

// intrinsic.

// * ParamTypes is a list containing the parameter types expected for the

// intrinsic.

// * Properties can be set to describe the behavior of the intrinsic.

//

class Intrinsic<list<LLVMType> ret_types,

list<LLVMType> param_types = [],

list<IntrinsicProperty> intr_properties = [],

string name = "",

list<SDNodeProperty> sd_properties = [],

bit disable_default_attributes = true> : SDPatternOperator {

string LLVMName = name;

string TargetPrefix = ""; // Set to a prefix for target-specific intrinsics.

list<LLVMType> RetTypes = ret_types;

list<LLVMType> ParamTypes = param_types;

list<IntrinsicProperty> IntrProperties = intr_properties;

let Properties = sd_properties;

// Disable applying IntrinsicProperties that are marked default with

// IntrinsicProperty<1>

bit DisableDefaultAttributes = disable_default_attributes;

TypeInfoGen TypeInfo = TypeInfoGen<RetTypes, ParamTypes>;

}

// Intrinsic with default attributes (disable_default_attributes = false).

class DefaultAttrsIntrinsic<list<LLVMType> ret_types,

list<LLVMType> param_types = [],

list<IntrinsicProperty> intr_properties = [],

string name = "",

list<SDNodeProperty> sd_properties = []>

: Intrinsic<ret_types, param_types,

intr_properties, name,

sd_properties, /*disable_default_attributes*/ 0> {}

/// ClangBuiltin - If this intrinsic exactly corresponds to a Clang builtin, this

/// specifies the name of the builtin. This provides automatic CBE and CFE

/// support.

class ClangBuiltin<string name> {

string ClangBuiltinName = name;

}

class MSBuiltin<string name> {

string MSBuiltinName = name;

}

/// Utility class for intrinsics that

/// 1. Don't touch memory or any hidden state

/// 2. Can be freely speculated, and

/// 3. Will not create undef or poison on defined inputs.

class PureIntrinsic<list<LLVMType> ret_types,

list<LLVMType> param_types = [],

list<IntrinsicProperty> intr_properties = [],

string name = "",

list<SDNodeProperty> sd_properties = []>

: DefaultAttrsIntrinsic<ret_types, param_types,

intr_properties # [IntrNoMem, IntrSpeculatable, IntrNoCreateUndefOrPoison],

name, sd_properties>;

#ifndef TEST_INTRINSICS_SUPPRESS_DEFS

//===--------------- Variable Argument Handling Intrinsics ----------------===//

//

def int_vastart : DefaultAttrsIntrinsic<[],

[llvm_anyptr_ty], [], "llvm.va_start">;

def int_vacopy : DefaultAttrsIntrinsic<[],

[llvm_anyptr_ty, LLVMMatchType<0>], [],

"llvm.va_copy">;

def int_vaend : DefaultAttrsIntrinsic<[],

[llvm_anyptr_ty], [], "llvm.va_end">;

//===------------------- Garbage Collection Intrinsics --------------------===//

//

def int_gcroot : Intrinsic<[],

[llvm_ptr_ty, llvm_ptr_ty]>;

def int_gcread : Intrinsic<[llvm_ptr_ty],

[llvm_ptr_ty, llvm_ptr_ty],

[IntrReadMem, IntrArgMemOnly]>;

def int_gcwrite : Intrinsic<[],

[llvm_ptr_ty, llvm_ptr_ty, llvm_ptr_ty],

[IntrArgMemOnly, NoCapture<ArgIndex<1>>,

NoCapture<ArgIndex<2>>]>;

//===------------------- ObjC ARC runtime Intrinsics --------------------===//

//

// Note these are to support the Objective-C ARC optimizer which wants to

// eliminate retain and releases where possible.

def int_objc_autorelease : Intrinsic<[llvm_ptr_ty],

[llvm_ptr_ty],

[Returned<ArgIndex<0>>]>;

def int_objc_autoreleasePoolPop : Intrinsic<[], [llvm_ptr_ty]>;

def int_objc_autoreleasePoolPush : Intrinsic<[llvm_ptr_ty], []>;

def int_objc_autoreleaseReturnValue : Intrinsic<[llvm_ptr_ty],

[llvm_ptr_ty],

[Returned<ArgIndex<0>>]>;

def int_objc_copyWeak : Intrinsic<[],

[llvm_ptr_ty,

llvm_ptr_ty]>;

def int_objc_destroyWeak : Intrinsic<[], [llvm_ptr_ty]>;

def int_objc_initWeak : Intrinsic<[llvm_ptr_ty],

[llvm_ptr_ty,

llvm_ptr_ty]>;

def int_objc_loadWeak : Intrinsic<[llvm_ptr_ty],

[llvm_ptr_ty]>;

def int_objc_loadWeakRetained : Intrinsic<[llvm_ptr_ty],

[llvm_ptr_ty]>;

def int_objc_moveWeak : Intrinsic<[],

[llvm_ptr_ty,

llvm_ptr_ty]>;

def int_objc_release : Intrinsic<[], [llvm_ptr_ty]>;

def int_objc_retain : Intrinsic<[llvm_ptr_ty],

[llvm_ptr_ty],

[Returned<ArgIndex<0>>]>;

def int_objc_retainAutorelease : Intrinsic<[llvm_ptr_ty],

[llvm_ptr_ty],

[Returned<ArgIndex<0>>]>;

def int_objc_retainAutoreleaseReturnValue : Intrinsic<[llvm_ptr_ty],

[llvm_ptr_ty],

[Returned<ArgIndex<0>>]>;

def int_objc_retainAutoreleasedReturnValue : Intrinsic<[llvm_ptr_ty],

[llvm_ptr_ty]>;

def int_objc_unsafeClaimAutoreleasedReturnValue : Intrinsic<[llvm_ptr_ty],

[llvm_ptr_ty]>;

def int_objc_claimAutoreleasedReturnValue : Intrinsic<[llvm_ptr_ty],

[llvm_ptr_ty]>;

def int_objc_retainBlock : Intrinsic<[llvm_ptr_ty],

[llvm_ptr_ty]>;

def int_objc_storeStrong : Intrinsic<[],

[llvm_ptr_ty,

llvm_ptr_ty]>;

def int_objc_storeWeak : Intrinsic<[llvm_ptr_ty],

[llvm_ptr_ty,

llvm_ptr_ty]>;

def int_objc_clang_arc_use : Intrinsic<[],

[llvm_vararg_ty]>;

def int_objc_clang_arc_noop_use : DefaultAttrsIntrinsic<[],

[llvm_vararg_ty],

[IntrInaccessibleMemOnly]>;

def int_objc_retainedObject : Intrinsic<[llvm_ptr_ty],

[llvm_ptr_ty]>;

def int_objc_unretainedObject : Intrinsic<[llvm_ptr_ty],

[llvm_ptr_ty]>;

def int_objc_unretainedPointer : Intrinsic<[llvm_ptr_ty],

[llvm_ptr_ty]>;

def int_objc_retain_autorelease : Intrinsic<[llvm_ptr_ty],

[llvm_ptr_ty],

[Returned<ArgIndex<0>>]>;

def int_objc_sync_enter : Intrinsic<[llvm_i32_ty],

[llvm_ptr_ty]>;

def int_objc_sync_exit : Intrinsic<[llvm_i32_ty],

[llvm_ptr_ty]>;

def int_objc_arc_annotation_topdown_bbstart : Intrinsic<[],

[llvm_ptr_ty,

llvm_ptr_ty]>;

def int_objc_arc_annotation_topdown_bbend : Intrinsic<[],

[llvm_ptr_ty,

llvm_ptr_ty]>;

def int_objc_arc_annotation_bottomup_bbstart : Intrinsic<[],

[llvm_ptr_ty,

llvm_ptr_ty]>;

def int_objc_arc_annotation_bottomup_bbend : Intrinsic<[],

[llvm_ptr_ty,

llvm_ptr_ty]>;

//===--------------- Swift asynchronous context intrinsics ----------------===//

// Returns the location of the Swift asynchronous context (usually stored just

// before the frame pointer), and triggers the creation of a null context if it

// would otherwise be unneeded.

def int_swift_async_context_addr : Intrinsic<[llvm_ptr_ty], [], []>;

//===--------------------- Code Generator Intrinsics ----------------------===//

//

def int_returnaddress : DefaultAttrsIntrinsic<[llvm_anyptr_ty], [llvm_i32_ty],

[IntrNoMem, ImmArg<ArgIndex<0>>]>;

def int_addressofreturnaddress : DefaultAttrsIntrinsic<[llvm_anyptr_ty], [], [IntrNoMem]>;

def int_frameaddress : DefaultAttrsIntrinsic<[llvm_anyptr_ty], [llvm_i32_ty],

[IntrNoMem, ImmArg<ArgIndex<0>>]>;

def int_sponentry : DefaultAttrsIntrinsic<[llvm_anyptr_ty], [], [IntrNoMem]>;

def int_stackaddress : DefaultAttrsIntrinsic<[llvm_anyptr_ty], [], []>;

def int_read_register : DefaultAttrsIntrinsic<[llvm_anyint_ty], [llvm_metadata_ty],

[IntrReadMem], "llvm.read_register">;

def int_write_register : Intrinsic<[], [llvm_metadata_ty, llvm_anyint_ty],

[IntrNoCallback], "llvm.write_register">;

def int_read_volatile_register : Intrinsic<[llvm_anyint_ty], [llvm_metadata_ty],

[IntrHasSideEffects],

"llvm.read_volatile_register">;

// Gets the address of the local variable area. This is typically a copy of the

// stack, frame, or base pointer depending on the type of prologue.

def int_localaddress : DefaultAttrsIntrinsic<[llvm_ptr_ty], [], [IntrNoMem]>;

// Escapes local variables to allow access from other functions.

def int_localescape : DefaultAttrsIntrinsic<[], [llvm_vararg_ty]>;

// Given a function and the localaddress of a parent frame, returns a pointer

// to an escaped allocation indicated by the index.

def int_localrecover : DefaultAttrsIntrinsic<[llvm_ptr_ty],

[llvm_ptr_ty, llvm_ptr_ty, llvm_i32_ty],

[IntrNoMem, ImmArg<ArgIndex<2>>]>;

// Given the frame pointer passed into an SEH filter function, returns a

// pointer to the local variable area suitable for use with llvm.localrecover.

def int_eh_recoverfp : DefaultAttrsIntrinsic<[llvm_ptr_ty],

[llvm_ptr_ty, llvm_ptr_ty],

[IntrNoMem]>;

// To mark the beginning/end of a try-scope for Windows SEH -EHa

// calls/invokes to these intrinsics are placed to model control flows

// caused by HW exceptions under option -EHa.

// calls/invokes to these intrinsics will be discarded during a codegen pass

// after EH tables are generated

def int_seh_try_begin : Intrinsic<[], [], [IntrWriteMem, IntrWillReturn]>;

def int_seh_try_end : Intrinsic<[], [], [IntrWriteMem, IntrWillReturn]>;

def int_seh_scope_begin : Intrinsic<[], [], [IntrNoMem]>;

def int_seh_scope_end : Intrinsic<[], [], [IntrNoMem]>;

// Note: we treat stacksave/stackrestore as writemem because we don't otherwise

// model their dependencies on allocas.

def int_stacksave : DefaultAttrsIntrinsic<[llvm_anyptr_ty]>,

ClangBuiltin<"__builtin_stack_save">;

def int_stackrestore : DefaultAttrsIntrinsic<[], [llvm_anyptr_ty]>,

ClangBuiltin<"__builtin_stack_restore">;

def int_get_dynamic_area_offset : DefaultAttrsIntrinsic<[llvm_anyint_ty]>;

def int_thread_pointer : DefaultAttrsIntrinsic<[llvm_anyptr_ty], [], [IntrNoMem]>,

ClangBuiltin<"__builtin_thread_pointer">;

// IntrInaccessibleMemOrArgMemOnly is a little more pessimistic than strictly

// necessary for prefetch, however it does conveniently prevent the prefetch

// from being reordered overly much with respect to nearby access to the same

// memory while not impeding optimization.

def int_prefetch

: DefaultAttrsIntrinsic<[], [ llvm_anyptr_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty ],

[IntrInaccessibleMemOrArgMemOnly,

ReadOnly<ArgIndex<0>>, NoCapture<ArgIndex<0>>,

ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<2>>, ImmArg<ArgIndex<3>>]>;

def int_pcmarker : DefaultAttrsIntrinsic<[], [llvm_i32_ty]>;

def int_readcyclecounter : DefaultAttrsIntrinsic<[llvm_i64_ty]>;

def int_readsteadycounter : DefaultAttrsIntrinsic<[llvm_i64_ty]>;

// The assume intrinsic is marked InaccessibleMemOnly so that proper control

// dependencies will be maintained.

def int_assume : DefaultAttrsIntrinsic<

[], [llvm_i1_ty], [IntrWriteMem, IntrInaccessibleMemOnly, NoUndef<ArgIndex<0>>]>;

// 'llvm.experimental.noalias.scope.decl' intrinsic: Inserted at the location of

// noalias scope declaration. Makes it possible to identify that a noalias scope

// is only valid inside the body of a loop.

//

// Purpose of the different arguments:

// - arg0: id.scope: metadata representing the scope declaration.

def int_experimental_noalias_scope_decl

: DefaultAttrsIntrinsic<[], [llvm_metadata_ty],

[IntrInaccessibleMemOnly]>; // blocks LICM and some more

// Stack Protector Intrinsic - The stackprotector intrinsic writes the stack

// guard to the correct place on the stack frame.

def int_stackprotector : DefaultAttrsIntrinsic<[], [llvm_ptr_ty, llvm_ptr_ty], []>;

def int_stackguard : DefaultAttrsIntrinsic<[llvm_ptr_ty], [], []>;

// A cover for instrumentation based profiling.

def int_instrprof_cover : Intrinsic<[], [llvm_ptr_ty, llvm_i64_ty,

llvm_i32_ty, llvm_i32_ty]>;

// A counter increment for instrumentation based profiling.