Mercurial > hg > Members > tobaru > cbc > CbC_llvm
view lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp @ 83:60c9769439b8
LLVM 3.7
author | Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> |
---|---|
date | Wed, 18 Feb 2015 14:55:36 +0900 |
parents | 54457678186b |
children | afa8332a0e37 |
line wrap: on
line source
//===-- NVPTXISelDAGToDAG.cpp - A dag to dag inst selector for NVPTX ------===// // // The LLVM Compiler Infrastructure // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// // // This file defines an instruction selector for the NVPTX target. // //===----------------------------------------------------------------------===// #include "NVPTXISelDAGToDAG.h" #include "llvm/IR/GlobalValue.h" #include "llvm/IR/Instructions.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Target/TargetIntrinsicInfo.h" using namespace llvm; #define DEBUG_TYPE "nvptx-isel" static cl::opt<int> UsePrecDivF32( "nvptx-prec-divf32", cl::ZeroOrMore, cl::Hidden, cl::desc("NVPTX Specifies: 0 use div.approx, 1 use div.full, 2 use" " IEEE Compliant F32 div.rnd if available."), cl::init(2)); static cl::opt<bool> UsePrecSqrtF32("nvptx-prec-sqrtf32", cl::Hidden, cl::desc("NVPTX Specific: 0 use sqrt.approx, 1 use sqrt.rn."), cl::init(true)); static cl::opt<bool> FtzEnabled("nvptx-f32ftz", cl::ZeroOrMore, cl::Hidden, cl::desc("NVPTX Specific: Flush f32 subnormals to sign-preserving zero."), cl::init(false)); /// createNVPTXISelDag - This pass converts a legalized DAG into a /// NVPTX-specific DAG, ready for instruction scheduling. FunctionPass *llvm::createNVPTXISelDag(NVPTXTargetMachine &TM, llvm::CodeGenOpt::Level OptLevel) { return new NVPTXDAGToDAGISel(TM, OptLevel); } NVPTXDAGToDAGISel::NVPTXDAGToDAGISel(NVPTXTargetMachine &tm, CodeGenOpt::Level OptLevel) : SelectionDAGISel(tm, OptLevel) { doMulWide = (OptLevel > 0); } bool NVPTXDAGToDAGISel::runOnMachineFunction(MachineFunction &MF) { Subtarget = &static_cast<const NVPTXSubtarget &>(MF.getSubtarget()); return SelectionDAGISel::runOnMachineFunction(MF); } int NVPTXDAGToDAGISel::getDivF32Level() const { if (UsePrecDivF32.getNumOccurrences() > 0) { // If nvptx-prec-div32=N is used on the command-line, always honor it return UsePrecDivF32; } else { // Otherwise, use div.approx if fast math is enabled if (TM.Options.UnsafeFPMath) return 0; else return 2; } } bool NVPTXDAGToDAGISel::usePrecSqrtF32() const { if (UsePrecSqrtF32.getNumOccurrences() > 0) { // If nvptx-prec-sqrtf32 is used on the command-line, always honor it return UsePrecSqrtF32; } else { // Otherwise, use sqrt.approx if fast math is enabled if (TM.Options.UnsafeFPMath) return false; else return true; } } bool NVPTXDAGToDAGISel::useF32FTZ() const { if (FtzEnabled.getNumOccurrences() > 0) { // If nvptx-f32ftz is used on the command-line, always honor it return FtzEnabled; } else { const Function *F = MF->getFunction(); // Otherwise, check for an nvptx-f32ftz attribute on the function if (F->hasFnAttribute("nvptx-f32ftz")) return F->getFnAttribute("nvptx-f32ftz").getValueAsString() == "true"; else return false; } } bool NVPTXDAGToDAGISel::allowFMA() const { const NVPTXTargetLowering *TL = Subtarget->getTargetLowering(); return TL->allowFMA(*MF, OptLevel); } /// Select - Select instructions not customized! Used for /// expanded, promoted and normal instructions. SDNode *NVPTXDAGToDAGISel::Select(SDNode *N) { if (N->isMachineOpcode()) { N->setNodeId(-1); return nullptr; // Already selected. } SDNode *ResNode = nullptr; switch (N->getOpcode()) { case ISD::LOAD: ResNode = SelectLoad(N); break; case ISD::STORE: ResNode = SelectStore(N); break; case NVPTXISD::LoadV2: case NVPTXISD::LoadV4: ResNode = SelectLoadVector(N); break; case NVPTXISD::LDGV2: case NVPTXISD::LDGV4: case NVPTXISD::LDUV2: case NVPTXISD::LDUV4: ResNode = SelectLDGLDU(N); break; case NVPTXISD::StoreV2: case NVPTXISD::StoreV4: ResNode = SelectStoreVector(N); break; case NVPTXISD::LoadParam: case NVPTXISD::LoadParamV2: case NVPTXISD::LoadParamV4: ResNode = SelectLoadParam(N); break; case NVPTXISD::StoreRetval: case NVPTXISD::StoreRetvalV2: case NVPTXISD::StoreRetvalV4: ResNode = SelectStoreRetval(N); break; case NVPTXISD::StoreParam: case NVPTXISD::StoreParamV2: case NVPTXISD::StoreParamV4: case NVPTXISD::StoreParamS32: case NVPTXISD::StoreParamU32: ResNode = SelectStoreParam(N); break; case ISD::INTRINSIC_WO_CHAIN: ResNode = SelectIntrinsicNoChain(N); break; case ISD::INTRINSIC_W_CHAIN: ResNode = SelectIntrinsicChain(N); break; case NVPTXISD::Tex1DFloatS32: case NVPTXISD::Tex1DFloatFloat: case NVPTXISD::Tex1DFloatFloatLevel: case NVPTXISD::Tex1DFloatFloatGrad: case NVPTXISD::Tex1DS32S32: case NVPTXISD::Tex1DS32Float: case NVPTXISD::Tex1DS32FloatLevel: case NVPTXISD::Tex1DS32FloatGrad: case NVPTXISD::Tex1DU32S32: case NVPTXISD::Tex1DU32Float: case NVPTXISD::Tex1DU32FloatLevel: case NVPTXISD::Tex1DU32FloatGrad: case NVPTXISD::Tex1DArrayFloatS32: case NVPTXISD::Tex1DArrayFloatFloat: case NVPTXISD::Tex1DArrayFloatFloatLevel: case NVPTXISD::Tex1DArrayFloatFloatGrad: case NVPTXISD::Tex1DArrayS32S32: case NVPTXISD::Tex1DArrayS32Float: case NVPTXISD::Tex1DArrayS32FloatLevel: case NVPTXISD::Tex1DArrayS32FloatGrad: case NVPTXISD::Tex1DArrayU32S32: case NVPTXISD::Tex1DArrayU32Float: case NVPTXISD::Tex1DArrayU32FloatLevel: case NVPTXISD::Tex1DArrayU32FloatGrad: case NVPTXISD::Tex2DFloatS32: case NVPTXISD::Tex2DFloatFloat: case NVPTXISD::Tex2DFloatFloatLevel: case NVPTXISD::Tex2DFloatFloatGrad: case NVPTXISD::Tex2DS32S32: case NVPTXISD::Tex2DS32Float: case NVPTXISD::Tex2DS32FloatLevel: case NVPTXISD::Tex2DS32FloatGrad: case NVPTXISD::Tex2DU32S32: case NVPTXISD::Tex2DU32Float: case NVPTXISD::Tex2DU32FloatLevel: case NVPTXISD::Tex2DU32FloatGrad: case NVPTXISD::Tex2DArrayFloatS32: case NVPTXISD::Tex2DArrayFloatFloat: case NVPTXISD::Tex2DArrayFloatFloatLevel: case NVPTXISD::Tex2DArrayFloatFloatGrad: case NVPTXISD::Tex2DArrayS32S32: case NVPTXISD::Tex2DArrayS32Float: case NVPTXISD::Tex2DArrayS32FloatLevel: case NVPTXISD::Tex2DArrayS32FloatGrad: case NVPTXISD::Tex2DArrayU32S32: case NVPTXISD::Tex2DArrayU32Float: case NVPTXISD::Tex2DArrayU32FloatLevel: case NVPTXISD::Tex2DArrayU32FloatGrad: case NVPTXISD::Tex3DFloatS32: case NVPTXISD::Tex3DFloatFloat: case NVPTXISD::Tex3DFloatFloatLevel: case NVPTXISD::Tex3DFloatFloatGrad: case NVPTXISD::Tex3DS32S32: case NVPTXISD::Tex3DS32Float: case NVPTXISD::Tex3DS32FloatLevel: case NVPTXISD::Tex3DS32FloatGrad: case NVPTXISD::Tex3DU32S32: case NVPTXISD::Tex3DU32Float: case NVPTXISD::Tex3DU32FloatLevel: case NVPTXISD::Tex3DU32FloatGrad: case NVPTXISD::TexCubeFloatFloat: case NVPTXISD::TexCubeFloatFloatLevel: case NVPTXISD::TexCubeS32Float: case NVPTXISD::TexCubeS32FloatLevel: case NVPTXISD::TexCubeU32Float: case NVPTXISD::TexCubeU32FloatLevel: case NVPTXISD::TexCubeArrayFloatFloat: case NVPTXISD::TexCubeArrayFloatFloatLevel: case NVPTXISD::TexCubeArrayS32Float: case NVPTXISD::TexCubeArrayS32FloatLevel: case NVPTXISD::TexCubeArrayU32Float: case NVPTXISD::TexCubeArrayU32FloatLevel: case NVPTXISD::Tld4R2DFloatFloat: case NVPTXISD::Tld4G2DFloatFloat: case NVPTXISD::Tld4B2DFloatFloat: case NVPTXISD::Tld4A2DFloatFloat: case NVPTXISD::Tld4R2DS64Float: case NVPTXISD::Tld4G2DS64Float: case NVPTXISD::Tld4B2DS64Float: case NVPTXISD::Tld4A2DS64Float: case NVPTXISD::Tld4R2DU64Float: case NVPTXISD::Tld4G2DU64Float: case NVPTXISD::Tld4B2DU64Float: case NVPTXISD::Tld4A2DU64Float: case NVPTXISD::TexUnified1DFloatS32: case NVPTXISD::TexUnified1DFloatFloat: case NVPTXISD::TexUnified1DFloatFloatLevel: case NVPTXISD::TexUnified1DFloatFloatGrad: case NVPTXISD::TexUnified1DS32S32: case NVPTXISD::TexUnified1DS32Float: case NVPTXISD::TexUnified1DS32FloatLevel: case NVPTXISD::TexUnified1DS32FloatGrad: case NVPTXISD::TexUnified1DU32S32: case NVPTXISD::TexUnified1DU32Float: case NVPTXISD::TexUnified1DU32FloatLevel: case NVPTXISD::TexUnified1DU32FloatGrad: case NVPTXISD::TexUnified1DArrayFloatS32: case NVPTXISD::TexUnified1DArrayFloatFloat: case NVPTXISD::TexUnified1DArrayFloatFloatLevel: case NVPTXISD::TexUnified1DArrayFloatFloatGrad: case NVPTXISD::TexUnified1DArrayS32S32: case NVPTXISD::TexUnified1DArrayS32Float: case NVPTXISD::TexUnified1DArrayS32FloatLevel: case NVPTXISD::TexUnified1DArrayS32FloatGrad: case NVPTXISD::TexUnified1DArrayU32S32: case NVPTXISD::TexUnified1DArrayU32Float: case NVPTXISD::TexUnified1DArrayU32FloatLevel: case NVPTXISD::TexUnified1DArrayU32FloatGrad: case NVPTXISD::TexUnified2DFloatS32: case NVPTXISD::TexUnified2DFloatFloat: case NVPTXISD::TexUnified2DFloatFloatLevel: case NVPTXISD::TexUnified2DFloatFloatGrad: case NVPTXISD::TexUnified2DS32S32: case NVPTXISD::TexUnified2DS32Float: case NVPTXISD::TexUnified2DS32FloatLevel: case NVPTXISD::TexUnified2DS32FloatGrad: case NVPTXISD::TexUnified2DU32S32: case NVPTXISD::TexUnified2DU32Float: case NVPTXISD::TexUnified2DU32FloatLevel: case NVPTXISD::TexUnified2DU32FloatGrad: case NVPTXISD::TexUnified2DArrayFloatS32: case NVPTXISD::TexUnified2DArrayFloatFloat: case NVPTXISD::TexUnified2DArrayFloatFloatLevel: case NVPTXISD::TexUnified2DArrayFloatFloatGrad: case NVPTXISD::TexUnified2DArrayS32S32: case NVPTXISD::TexUnified2DArrayS32Float: case NVPTXISD::TexUnified2DArrayS32FloatLevel: case NVPTXISD::TexUnified2DArrayS32FloatGrad: case NVPTXISD::TexUnified2DArrayU32S32: case NVPTXISD::TexUnified2DArrayU32Float: case NVPTXISD::TexUnified2DArrayU32FloatLevel: case NVPTXISD::TexUnified2DArrayU32FloatGrad: case NVPTXISD::TexUnified3DFloatS32: case NVPTXISD::TexUnified3DFloatFloat: case NVPTXISD::TexUnified3DFloatFloatLevel: case NVPTXISD::TexUnified3DFloatFloatGrad: case NVPTXISD::TexUnified3DS32S32: case NVPTXISD::TexUnified3DS32Float: case NVPTXISD::TexUnified3DS32FloatLevel: case NVPTXISD::TexUnified3DS32FloatGrad: case NVPTXISD::TexUnified3DU32S32: case NVPTXISD::TexUnified3DU32Float: case NVPTXISD::TexUnified3DU32FloatLevel: case NVPTXISD::TexUnified3DU32FloatGrad: case NVPTXISD::TexUnifiedCubeFloatFloat: case NVPTXISD::TexUnifiedCubeFloatFloatLevel: case NVPTXISD::TexUnifiedCubeS32Float: case NVPTXISD::TexUnifiedCubeS32FloatLevel: case NVPTXISD::TexUnifiedCubeU32Float: case NVPTXISD::TexUnifiedCubeU32FloatLevel: case NVPTXISD::TexUnifiedCubeArrayFloatFloat: case NVPTXISD::TexUnifiedCubeArrayFloatFloatLevel: case NVPTXISD::TexUnifiedCubeArrayS32Float: case NVPTXISD::TexUnifiedCubeArrayS32FloatLevel: case NVPTXISD::TexUnifiedCubeArrayU32Float: case NVPTXISD::TexUnifiedCubeArrayU32FloatLevel: case NVPTXISD::Tld4UnifiedR2DFloatFloat: case NVPTXISD::Tld4UnifiedG2DFloatFloat: case NVPTXISD::Tld4UnifiedB2DFloatFloat: case NVPTXISD::Tld4UnifiedA2DFloatFloat: case NVPTXISD::Tld4UnifiedR2DS64Float: case NVPTXISD::Tld4UnifiedG2DS64Float: case NVPTXISD::Tld4UnifiedB2DS64Float: case NVPTXISD::Tld4UnifiedA2DS64Float: case NVPTXISD::Tld4UnifiedR2DU64Float: case NVPTXISD::Tld4UnifiedG2DU64Float: case NVPTXISD::Tld4UnifiedB2DU64Float: case NVPTXISD::Tld4UnifiedA2DU64Float: ResNode = SelectTextureIntrinsic(N); break; case NVPTXISD::Suld1DI8Clamp: case NVPTXISD::Suld1DI16Clamp: case NVPTXISD::Suld1DI32Clamp: case NVPTXISD::Suld1DI64Clamp: case NVPTXISD::Suld1DV2I8Clamp: case NVPTXISD::Suld1DV2I16Clamp: case NVPTXISD::Suld1DV2I32Clamp: case NVPTXISD::Suld1DV2I64Clamp: case NVPTXISD::Suld1DV4I8Clamp: case NVPTXISD::Suld1DV4I16Clamp: case NVPTXISD::Suld1DV4I32Clamp: case NVPTXISD::Suld1DArrayI8Clamp: case NVPTXISD::Suld1DArrayI16Clamp: case NVPTXISD::Suld1DArrayI32Clamp: case NVPTXISD::Suld1DArrayI64Clamp: case NVPTXISD::Suld1DArrayV2I8Clamp: case NVPTXISD::Suld1DArrayV2I16Clamp: case NVPTXISD::Suld1DArrayV2I32Clamp: case NVPTXISD::Suld1DArrayV2I64Clamp: case NVPTXISD::Suld1DArrayV4I8Clamp: case NVPTXISD::Suld1DArrayV4I16Clamp: case NVPTXISD::Suld1DArrayV4I32Clamp: case NVPTXISD::Suld2DI8Clamp: case NVPTXISD::Suld2DI16Clamp: case NVPTXISD::Suld2DI32Clamp: case NVPTXISD::Suld2DI64Clamp: case NVPTXISD::Suld2DV2I8Clamp: case NVPTXISD::Suld2DV2I16Clamp: case NVPTXISD::Suld2DV2I32Clamp: case NVPTXISD::Suld2DV2I64Clamp: case NVPTXISD::Suld2DV4I8Clamp: case NVPTXISD::Suld2DV4I16Clamp: case NVPTXISD::Suld2DV4I32Clamp: case NVPTXISD::Suld2DArrayI8Clamp: case NVPTXISD::Suld2DArrayI16Clamp: case NVPTXISD::Suld2DArrayI32Clamp: case NVPTXISD::Suld2DArrayI64Clamp: case NVPTXISD::Suld2DArrayV2I8Clamp: case NVPTXISD::Suld2DArrayV2I16Clamp: case NVPTXISD::Suld2DArrayV2I32Clamp: case NVPTXISD::Suld2DArrayV2I64Clamp: case NVPTXISD::Suld2DArrayV4I8Clamp: case NVPTXISD::Suld2DArrayV4I16Clamp: case NVPTXISD::Suld2DArrayV4I32Clamp: case NVPTXISD::Suld3DI8Clamp: case NVPTXISD::Suld3DI16Clamp: case NVPTXISD::Suld3DI32Clamp: case NVPTXISD::Suld3DI64Clamp: case NVPTXISD::Suld3DV2I8Clamp: case NVPTXISD::Suld3DV2I16Clamp: case NVPTXISD::Suld3DV2I32Clamp: case NVPTXISD::Suld3DV2I64Clamp: case NVPTXISD::Suld3DV4I8Clamp: case NVPTXISD::Suld3DV4I16Clamp: case NVPTXISD::Suld3DV4I32Clamp: case NVPTXISD::Suld1DI8Trap: case NVPTXISD::Suld1DI16Trap: case NVPTXISD::Suld1DI32Trap: case NVPTXISD::Suld1DI64Trap: case NVPTXISD::Suld1DV2I8Trap: case NVPTXISD::Suld1DV2I16Trap: case NVPTXISD::Suld1DV2I32Trap: case NVPTXISD::Suld1DV2I64Trap: case NVPTXISD::Suld1DV4I8Trap: case NVPTXISD::Suld1DV4I16Trap: case NVPTXISD::Suld1DV4I32Trap: case NVPTXISD::Suld1DArrayI8Trap: case NVPTXISD::Suld1DArrayI16Trap: case NVPTXISD::Suld1DArrayI32Trap: case NVPTXISD::Suld1DArrayI64Trap: case NVPTXISD::Suld1DArrayV2I8Trap: case NVPTXISD::Suld1DArrayV2I16Trap: case NVPTXISD::Suld1DArrayV2I32Trap: case NVPTXISD::Suld1DArrayV2I64Trap: case NVPTXISD::Suld1DArrayV4I8Trap: case NVPTXISD::Suld1DArrayV4I16Trap: case NVPTXISD::Suld1DArrayV4I32Trap: case NVPTXISD::Suld2DI8Trap: case NVPTXISD::Suld2DI16Trap: case NVPTXISD::Suld2DI32Trap: case NVPTXISD::Suld2DI64Trap: case NVPTXISD::Suld2DV2I8Trap: case NVPTXISD::Suld2DV2I16Trap: case NVPTXISD::Suld2DV2I32Trap: case NVPTXISD::Suld2DV2I64Trap: case NVPTXISD::Suld2DV4I8Trap: case NVPTXISD::Suld2DV4I16Trap: case NVPTXISD::Suld2DV4I32Trap: case NVPTXISD::Suld2DArrayI8Trap: case NVPTXISD::Suld2DArrayI16Trap: case NVPTXISD::Suld2DArrayI32Trap: case NVPTXISD::Suld2DArrayI64Trap: case NVPTXISD::Suld2DArrayV2I8Trap: case NVPTXISD::Suld2DArrayV2I16Trap: case NVPTXISD::Suld2DArrayV2I32Trap: case NVPTXISD::Suld2DArrayV2I64Trap: case NVPTXISD::Suld2DArrayV4I8Trap: case NVPTXISD::Suld2DArrayV4I16Trap: case NVPTXISD::Suld2DArrayV4I32Trap: case NVPTXISD::Suld3DI8Trap: case NVPTXISD::Suld3DI16Trap: case NVPTXISD::Suld3DI32Trap: case NVPTXISD::Suld3DI64Trap: case NVPTXISD::Suld3DV2I8Trap: case NVPTXISD::Suld3DV2I16Trap: case NVPTXISD::Suld3DV2I32Trap: case NVPTXISD::Suld3DV2I64Trap: case NVPTXISD::Suld3DV4I8Trap: case NVPTXISD::Suld3DV4I16Trap: case NVPTXISD::Suld3DV4I32Trap: case NVPTXISD::Suld1DI8Zero: case NVPTXISD::Suld1DI16Zero: case NVPTXISD::Suld1DI32Zero: case NVPTXISD::Suld1DI64Zero: case NVPTXISD::Suld1DV2I8Zero: case NVPTXISD::Suld1DV2I16Zero: case NVPTXISD::Suld1DV2I32Zero: case NVPTXISD::Suld1DV2I64Zero: case NVPTXISD::Suld1DV4I8Zero: case NVPTXISD::Suld1DV4I16Zero: case NVPTXISD::Suld1DV4I32Zero: case NVPTXISD::Suld1DArrayI8Zero: case NVPTXISD::Suld1DArrayI16Zero: case NVPTXISD::Suld1DArrayI32Zero: case NVPTXISD::Suld1DArrayI64Zero: case NVPTXISD::Suld1DArrayV2I8Zero: case NVPTXISD::Suld1DArrayV2I16Zero: case NVPTXISD::Suld1DArrayV2I32Zero: case NVPTXISD::Suld1DArrayV2I64Zero: case NVPTXISD::Suld1DArrayV4I8Zero: case NVPTXISD::Suld1DArrayV4I16Zero: case NVPTXISD::Suld1DArrayV4I32Zero: case NVPTXISD::Suld2DI8Zero: case NVPTXISD::Suld2DI16Zero: case NVPTXISD::Suld2DI32Zero: case NVPTXISD::Suld2DI64Zero: case NVPTXISD::Suld2DV2I8Zero: case NVPTXISD::Suld2DV2I16Zero: case NVPTXISD::Suld2DV2I32Zero: case NVPTXISD::Suld2DV2I64Zero: case NVPTXISD::Suld2DV4I8Zero: case NVPTXISD::Suld2DV4I16Zero: case NVPTXISD::Suld2DV4I32Zero: case NVPTXISD::Suld2DArrayI8Zero: case NVPTXISD::Suld2DArrayI16Zero: case NVPTXISD::Suld2DArrayI32Zero: case NVPTXISD::Suld2DArrayI64Zero: case NVPTXISD::Suld2DArrayV2I8Zero: case NVPTXISD::Suld2DArrayV2I16Zero: case NVPTXISD::Suld2DArrayV2I32Zero: case NVPTXISD::Suld2DArrayV2I64Zero: case NVPTXISD::Suld2DArrayV4I8Zero: case NVPTXISD::Suld2DArrayV4I16Zero: case NVPTXISD::Suld2DArrayV4I32Zero: case NVPTXISD::Suld3DI8Zero: case NVPTXISD::Suld3DI16Zero: case NVPTXISD::Suld3DI32Zero: case NVPTXISD::Suld3DI64Zero: case NVPTXISD::Suld3DV2I8Zero: case NVPTXISD::Suld3DV2I16Zero: case NVPTXISD::Suld3DV2I32Zero: case NVPTXISD::Suld3DV2I64Zero: case NVPTXISD::Suld3DV4I8Zero: case NVPTXISD::Suld3DV4I16Zero: case NVPTXISD::Suld3DV4I32Zero: ResNode = SelectSurfaceIntrinsic(N); break; case ISD::AND: case ISD::SRA: case ISD::SRL: // Try to select BFE ResNode = SelectBFE(N); break; case ISD::ADDRSPACECAST: ResNode = SelectAddrSpaceCast(N); break; default: break; } if (ResNode) return ResNode; return SelectCode(N); } SDNode *NVPTXDAGToDAGISel::SelectIntrinsicChain(SDNode *N) { unsigned IID = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue(); switch (IID) { default: return NULL; case Intrinsic::nvvm_ldg_global_f: case Intrinsic::nvvm_ldg_global_i: case Intrinsic::nvvm_ldg_global_p: case Intrinsic::nvvm_ldu_global_f: case Intrinsic::nvvm_ldu_global_i: case Intrinsic::nvvm_ldu_global_p: return SelectLDGLDU(N); } } static unsigned int getCodeAddrSpace(MemSDNode *N) { const Value *Src = N->getMemOperand()->getValue(); if (!Src) return NVPTX::PTXLdStInstCode::GENERIC; if (const PointerType *PT = dyn_cast<PointerType>(Src->getType())) { switch (PT->getAddressSpace()) { case llvm::ADDRESS_SPACE_LOCAL: return NVPTX::PTXLdStInstCode::LOCAL; case llvm::ADDRESS_SPACE_GLOBAL: return NVPTX::PTXLdStInstCode::GLOBAL; case llvm::ADDRESS_SPACE_SHARED: return NVPTX::PTXLdStInstCode::SHARED; case llvm::ADDRESS_SPACE_GENERIC: return NVPTX::PTXLdStInstCode::GENERIC; case llvm::ADDRESS_SPACE_PARAM: return NVPTX::PTXLdStInstCode::PARAM; case llvm::ADDRESS_SPACE_CONST: return NVPTX::PTXLdStInstCode::CONSTANT; default: break; } } return NVPTX::PTXLdStInstCode::GENERIC; } SDNode *NVPTXDAGToDAGISel::SelectIntrinsicNoChain(SDNode *N) { unsigned IID = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue(); switch (IID) { default: return nullptr; case Intrinsic::nvvm_texsurf_handle_internal: return SelectTexSurfHandle(N); } } SDNode *NVPTXDAGToDAGISel::SelectTexSurfHandle(SDNode *N) { // Op 0 is the intrinsic ID SDValue Wrapper = N->getOperand(1); SDValue GlobalVal = Wrapper.getOperand(0); return CurDAG->getMachineNode(NVPTX::texsurf_handles, SDLoc(N), MVT::i64, GlobalVal); } SDNode *NVPTXDAGToDAGISel::SelectAddrSpaceCast(SDNode *N) { SDValue Src = N->getOperand(0); AddrSpaceCastSDNode *CastN = cast<AddrSpaceCastSDNode>(N); unsigned SrcAddrSpace = CastN->getSrcAddressSpace(); unsigned DstAddrSpace = CastN->getDestAddressSpace(); assert(SrcAddrSpace != DstAddrSpace && "addrspacecast must be between different address spaces"); if (DstAddrSpace == ADDRESS_SPACE_GENERIC) { // Specific to generic unsigned Opc; switch (SrcAddrSpace) { default: report_fatal_error("Bad address space in addrspacecast"); case ADDRESS_SPACE_GLOBAL: Opc = Subtarget->is64Bit() ? NVPTX::cvta_global_yes_64 : NVPTX::cvta_global_yes; break; case ADDRESS_SPACE_SHARED: Opc = Subtarget->is64Bit() ? NVPTX::cvta_shared_yes_64 : NVPTX::cvta_shared_yes; break; case ADDRESS_SPACE_CONST: Opc = Subtarget->is64Bit() ? NVPTX::cvta_const_yes_64 : NVPTX::cvta_const_yes; break; case ADDRESS_SPACE_LOCAL: Opc = Subtarget->is64Bit() ? NVPTX::cvta_local_yes_64 : NVPTX::cvta_local_yes; break; } return CurDAG->getMachineNode(Opc, SDLoc(N), N->getValueType(0), Src); } else { // Generic to specific if (SrcAddrSpace != 0) report_fatal_error("Cannot cast between two non-generic address spaces"); unsigned Opc; switch (DstAddrSpace) { default: report_fatal_error("Bad address space in addrspacecast"); case ADDRESS_SPACE_GLOBAL: Opc = Subtarget->is64Bit() ? NVPTX::cvta_to_global_yes_64 : NVPTX::cvta_to_global_yes; break; case ADDRESS_SPACE_SHARED: Opc = Subtarget->is64Bit() ? NVPTX::cvta_to_shared_yes_64 : NVPTX::cvta_to_shared_yes; break; case ADDRESS_SPACE_CONST: Opc = Subtarget->is64Bit() ? NVPTX::cvta_to_const_yes_64 : NVPTX::cvta_to_const_yes; break; case ADDRESS_SPACE_LOCAL: Opc = Subtarget->is64Bit() ? NVPTX::cvta_to_local_yes_64 : NVPTX::cvta_to_local_yes; break; } return CurDAG->getMachineNode(Opc, SDLoc(N), N->getValueType(0), Src); } } SDNode *NVPTXDAGToDAGISel::SelectLoad(SDNode *N) { SDLoc dl(N); LoadSDNode *LD = cast<LoadSDNode>(N); EVT LoadedVT = LD->getMemoryVT(); SDNode *NVPTXLD = nullptr; // do not support pre/post inc/dec if (LD->isIndexed()) return nullptr; if (!LoadedVT.isSimple()) return nullptr; // Address Space Setting unsigned int codeAddrSpace = getCodeAddrSpace(LD); // Volatile Setting // - .volatile is only availalble for .global and .shared bool isVolatile = LD->isVolatile(); if (codeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL && codeAddrSpace != NVPTX::PTXLdStInstCode::SHARED && codeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC) isVolatile = false; // Vector Setting MVT SimpleVT = LoadedVT.getSimpleVT(); unsigned vecType = NVPTX::PTXLdStInstCode::Scalar; if (SimpleVT.isVector()) { unsigned num = SimpleVT.getVectorNumElements(); if (num == 2) vecType = NVPTX::PTXLdStInstCode::V2; else if (num == 4) vecType = NVPTX::PTXLdStInstCode::V4; else return nullptr; } // Type Setting: fromType + fromTypeWidth // // Sign : ISD::SEXTLOAD // Unsign : ISD::ZEXTLOAD, ISD::NON_EXTLOAD or ISD::EXTLOAD and the // type is integer // Float : ISD::NON_EXTLOAD or ISD::EXTLOAD and the type is float MVT ScalarVT = SimpleVT.getScalarType(); // Read at least 8 bits (predicates are stored as 8-bit values) unsigned fromTypeWidth = std::max(8U, ScalarVT.getSizeInBits()); unsigned int fromType; if ((LD->getExtensionType() == ISD::SEXTLOAD)) fromType = NVPTX::PTXLdStInstCode::Signed; else if (ScalarVT.isFloatingPoint()) fromType = NVPTX::PTXLdStInstCode::Float; else fromType = NVPTX::PTXLdStInstCode::Unsigned; // Create the machine instruction DAG SDValue Chain = N->getOperand(0); SDValue N1 = N->getOperand(1); SDValue Addr; SDValue Offset, Base; unsigned Opcode; MVT::SimpleValueType TargetVT = LD->getSimpleValueType(0).SimpleTy; if (SelectDirectAddr(N1, Addr)) { switch (TargetVT) { case MVT::i8: Opcode = NVPTX::LD_i8_avar; break; case MVT::i16: Opcode = NVPTX::LD_i16_avar; break; case MVT::i32: Opcode = NVPTX::LD_i32_avar; break; case MVT::i64: Opcode = NVPTX::LD_i64_avar; break; case MVT::f32: Opcode = NVPTX::LD_f32_avar; break; case MVT::f64: Opcode = NVPTX::LD_f64_avar; break; default: return nullptr; } SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace), getI32Imm(vecType), getI32Imm(fromType), getI32Imm(fromTypeWidth), Addr, Chain }; NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops); } else if (Subtarget->is64Bit() ? SelectADDRsi64(N1.getNode(), N1, Base, Offset) : SelectADDRsi(N1.getNode(), N1, Base, Offset)) { switch (TargetVT) { case MVT::i8: Opcode = NVPTX::LD_i8_asi; break; case MVT::i16: Opcode = NVPTX::LD_i16_asi; break; case MVT::i32: Opcode = NVPTX::LD_i32_asi; break; case MVT::i64: Opcode = NVPTX::LD_i64_asi; break; case MVT::f32: Opcode = NVPTX::LD_f32_asi; break; case MVT::f64: Opcode = NVPTX::LD_f64_asi; break; default: return nullptr; } SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace), getI32Imm(vecType), getI32Imm(fromType), getI32Imm(fromTypeWidth), Base, Offset, Chain }; NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops); } else if (Subtarget->is64Bit() ? SelectADDRri64(N1.getNode(), N1, Base, Offset) : SelectADDRri(N1.getNode(), N1, Base, Offset)) { if (Subtarget->is64Bit()) { switch (TargetVT) { case MVT::i8: Opcode = NVPTX::LD_i8_ari_64; break; case MVT::i16: Opcode = NVPTX::LD_i16_ari_64; break; case MVT::i32: Opcode = NVPTX::LD_i32_ari_64; break; case MVT::i64: Opcode = NVPTX::LD_i64_ari_64; break; case MVT::f32: Opcode = NVPTX::LD_f32_ari_64; break; case MVT::f64: Opcode = NVPTX::LD_f64_ari_64; break; default: return nullptr; } } else { switch (TargetVT) { case MVT::i8: Opcode = NVPTX::LD_i8_ari; break; case MVT::i16: Opcode = NVPTX::LD_i16_ari; break; case MVT::i32: Opcode = NVPTX::LD_i32_ari; break; case MVT::i64: Opcode = NVPTX::LD_i64_ari; break; case MVT::f32: Opcode = NVPTX::LD_f32_ari; break; case MVT::f64: Opcode = NVPTX::LD_f64_ari; break; default: return nullptr; } } SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace), getI32Imm(vecType), getI32Imm(fromType), getI32Imm(fromTypeWidth), Base, Offset, Chain }; NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops); } else { if (Subtarget->is64Bit()) { switch (TargetVT) { case MVT::i8: Opcode = NVPTX::LD_i8_areg_64; break; case MVT::i16: Opcode = NVPTX::LD_i16_areg_64; break; case MVT::i32: Opcode = NVPTX::LD_i32_areg_64; break; case MVT::i64: Opcode = NVPTX::LD_i64_areg_64; break; case MVT::f32: Opcode = NVPTX::LD_f32_areg_64; break; case MVT::f64: Opcode = NVPTX::LD_f64_areg_64; break; default: return nullptr; } } else { switch (TargetVT) { case MVT::i8: Opcode = NVPTX::LD_i8_areg; break; case MVT::i16: Opcode = NVPTX::LD_i16_areg; break; case MVT::i32: Opcode = NVPTX::LD_i32_areg; break; case MVT::i64: Opcode = NVPTX::LD_i64_areg; break; case MVT::f32: Opcode = NVPTX::LD_f32_areg; break; case MVT::f64: Opcode = NVPTX::LD_f64_areg; break; default: return nullptr; } } SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace), getI32Imm(vecType), getI32Imm(fromType), getI32Imm(fromTypeWidth), N1, Chain }; NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops); } if (NVPTXLD) { MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1); MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand(); cast<MachineSDNode>(NVPTXLD)->setMemRefs(MemRefs0, MemRefs0 + 1); } return NVPTXLD; } SDNode *NVPTXDAGToDAGISel::SelectLoadVector(SDNode *N) { SDValue Chain = N->getOperand(0); SDValue Op1 = N->getOperand(1); SDValue Addr, Offset, Base; unsigned Opcode; SDLoc DL(N); SDNode *LD; MemSDNode *MemSD = cast<MemSDNode>(N); EVT LoadedVT = MemSD->getMemoryVT(); if (!LoadedVT.isSimple()) return nullptr; // Address Space Setting unsigned int CodeAddrSpace = getCodeAddrSpace(MemSD); // Volatile Setting // - .volatile is only availalble for .global and .shared bool IsVolatile = MemSD->isVolatile(); if (CodeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL && CodeAddrSpace != NVPTX::PTXLdStInstCode::SHARED && CodeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC) IsVolatile = false; // Vector Setting MVT SimpleVT = LoadedVT.getSimpleVT(); // Type Setting: fromType + fromTypeWidth // // Sign : ISD::SEXTLOAD // Unsign : ISD::ZEXTLOAD, ISD::NON_EXTLOAD or ISD::EXTLOAD and the // type is integer // Float : ISD::NON_EXTLOAD or ISD::EXTLOAD and the type is float MVT ScalarVT = SimpleVT.getScalarType(); // Read at least 8 bits (predicates are stored as 8-bit values) unsigned FromTypeWidth = std::max(8U, ScalarVT.getSizeInBits()); unsigned int FromType; // The last operand holds the original LoadSDNode::getExtensionType() value unsigned ExtensionType = cast<ConstantSDNode>( N->getOperand(N->getNumOperands() - 1))->getZExtValue(); if (ExtensionType == ISD::SEXTLOAD) FromType = NVPTX::PTXLdStInstCode::Signed; else if (ScalarVT.isFloatingPoint()) FromType = NVPTX::PTXLdStInstCode::Float; else FromType = NVPTX::PTXLdStInstCode::Unsigned; unsigned VecType; switch (N->getOpcode()) { case NVPTXISD::LoadV2: VecType = NVPTX::PTXLdStInstCode::V2; break; case NVPTXISD::LoadV4: VecType = NVPTX::PTXLdStInstCode::V4; break; default: return nullptr; } EVT EltVT = N->getValueType(0); if (SelectDirectAddr(Op1, Addr)) { switch (N->getOpcode()) { default: return nullptr; case NVPTXISD::LoadV2: switch (EltVT.getSimpleVT().SimpleTy) { default: return nullptr; case MVT::i8: Opcode = NVPTX::LDV_i8_v2_avar; break; case MVT::i16: Opcode = NVPTX::LDV_i16_v2_avar; break; case MVT::i32: Opcode = NVPTX::LDV_i32_v2_avar; break; case MVT::i64: Opcode = NVPTX::LDV_i64_v2_avar; break; case MVT::f32: Opcode = NVPTX::LDV_f32_v2_avar; break; case MVT::f64: Opcode = NVPTX::LDV_f64_v2_avar; break; } break; case NVPTXISD::LoadV4: switch (EltVT.getSimpleVT().SimpleTy) { default: return nullptr; case MVT::i8: Opcode = NVPTX::LDV_i8_v4_avar; break; case MVT::i16: Opcode = NVPTX::LDV_i16_v4_avar; break; case MVT::i32: Opcode = NVPTX::LDV_i32_v4_avar; break; case MVT::f32: Opcode = NVPTX::LDV_f32_v4_avar; break; } break; } SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace), getI32Imm(VecType), getI32Imm(FromType), getI32Imm(FromTypeWidth), Addr, Chain }; LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops); } else if (Subtarget->is64Bit() ? SelectADDRsi64(Op1.getNode(), Op1, Base, Offset) : SelectADDRsi(Op1.getNode(), Op1, Base, Offset)) { switch (N->getOpcode()) { default: return nullptr; case NVPTXISD::LoadV2: switch (EltVT.getSimpleVT().SimpleTy) { default: return nullptr; case MVT::i8: Opcode = NVPTX::LDV_i8_v2_asi; break; case MVT::i16: Opcode = NVPTX::LDV_i16_v2_asi; break; case MVT::i32: Opcode = NVPTX::LDV_i32_v2_asi; break; case MVT::i64: Opcode = NVPTX::LDV_i64_v2_asi; break; case MVT::f32: Opcode = NVPTX::LDV_f32_v2_asi; break; case MVT::f64: Opcode = NVPTX::LDV_f64_v2_asi; break; } break; case NVPTXISD::LoadV4: switch (EltVT.getSimpleVT().SimpleTy) { default: return nullptr; case MVT::i8: Opcode = NVPTX::LDV_i8_v4_asi; break; case MVT::i16: Opcode = NVPTX::LDV_i16_v4_asi; break; case MVT::i32: Opcode = NVPTX::LDV_i32_v4_asi; break; case MVT::f32: Opcode = NVPTX::LDV_f32_v4_asi; break; } break; } SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace), getI32Imm(VecType), getI32Imm(FromType), getI32Imm(FromTypeWidth), Base, Offset, Chain }; LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops); } else if (Subtarget->is64Bit() ? SelectADDRri64(Op1.getNode(), Op1, Base, Offset) : SelectADDRri(Op1.getNode(), Op1, Base, Offset)) { if (Subtarget->is64Bit()) { switch (N->getOpcode()) { default: return nullptr; case NVPTXISD::LoadV2: switch (EltVT.getSimpleVT().SimpleTy) { default: return nullptr; case MVT::i8: Opcode = NVPTX::LDV_i8_v2_ari_64; break; case MVT::i16: Opcode = NVPTX::LDV_i16_v2_ari_64; break; case MVT::i32: Opcode = NVPTX::LDV_i32_v2_ari_64; break; case MVT::i64: Opcode = NVPTX::LDV_i64_v2_ari_64; break; case MVT::f32: Opcode = NVPTX::LDV_f32_v2_ari_64; break; case MVT::f64: Opcode = NVPTX::LDV_f64_v2_ari_64; break; } break; case NVPTXISD::LoadV4: switch (EltVT.getSimpleVT().SimpleTy) { default: return nullptr; case MVT::i8: Opcode = NVPTX::LDV_i8_v4_ari_64; break; case MVT::i16: Opcode = NVPTX::LDV_i16_v4_ari_64; break; case MVT::i32: Opcode = NVPTX::LDV_i32_v4_ari_64; break; case MVT::f32: Opcode = NVPTX::LDV_f32_v4_ari_64; break; } break; } } else { switch (N->getOpcode()) { default: return nullptr; case NVPTXISD::LoadV2: switch (EltVT.getSimpleVT().SimpleTy) { default: return nullptr; case MVT::i8: Opcode = NVPTX::LDV_i8_v2_ari; break; case MVT::i16: Opcode = NVPTX::LDV_i16_v2_ari; break; case MVT::i32: Opcode = NVPTX::LDV_i32_v2_ari; break; case MVT::i64: Opcode = NVPTX::LDV_i64_v2_ari; break; case MVT::f32: Opcode = NVPTX::LDV_f32_v2_ari; break; case MVT::f64: Opcode = NVPTX::LDV_f64_v2_ari; break; } break; case NVPTXISD::LoadV4: switch (EltVT.getSimpleVT().SimpleTy) { default: return nullptr; case MVT::i8: Opcode = NVPTX::LDV_i8_v4_ari; break; case MVT::i16: Opcode = NVPTX::LDV_i16_v4_ari; break; case MVT::i32: Opcode = NVPTX::LDV_i32_v4_ari; break; case MVT::f32: Opcode = NVPTX::LDV_f32_v4_ari; break; } break; } } SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace), getI32Imm(VecType), getI32Imm(FromType), getI32Imm(FromTypeWidth), Base, Offset, Chain }; LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops); } else { if (Subtarget->is64Bit()) { switch (N->getOpcode()) { default: return nullptr; case NVPTXISD::LoadV2: switch (EltVT.getSimpleVT().SimpleTy) { default: return nullptr; case MVT::i8: Opcode = NVPTX::LDV_i8_v2_areg_64; break; case MVT::i16: Opcode = NVPTX::LDV_i16_v2_areg_64; break; case MVT::i32: Opcode = NVPTX::LDV_i32_v2_areg_64; break; case MVT::i64: Opcode = NVPTX::LDV_i64_v2_areg_64; break; case MVT::f32: Opcode = NVPTX::LDV_f32_v2_areg_64; break; case MVT::f64: Opcode = NVPTX::LDV_f64_v2_areg_64; break; } break; case NVPTXISD::LoadV4: switch (EltVT.getSimpleVT().SimpleTy) { default: return nullptr; case MVT::i8: Opcode = NVPTX::LDV_i8_v4_areg_64; break; case MVT::i16: Opcode = NVPTX::LDV_i16_v4_areg_64; break; case MVT::i32: Opcode = NVPTX::LDV_i32_v4_areg_64; break; case MVT::f32: Opcode = NVPTX::LDV_f32_v4_areg_64; break; } break; } } else { switch (N->getOpcode()) { default: return nullptr; case NVPTXISD::LoadV2: switch (EltVT.getSimpleVT().SimpleTy) { default: return nullptr; case MVT::i8: Opcode = NVPTX::LDV_i8_v2_areg; break; case MVT::i16: Opcode = NVPTX::LDV_i16_v2_areg; break; case MVT::i32: Opcode = NVPTX::LDV_i32_v2_areg; break; case MVT::i64: Opcode = NVPTX::LDV_i64_v2_areg; break; case MVT::f32: Opcode = NVPTX::LDV_f32_v2_areg; break; case MVT::f64: Opcode = NVPTX::LDV_f64_v2_areg; break; } break; case NVPTXISD::LoadV4: switch (EltVT.getSimpleVT().SimpleTy) { default: return nullptr; case MVT::i8: Opcode = NVPTX::LDV_i8_v4_areg; break; case MVT::i16: Opcode = NVPTX::LDV_i16_v4_areg; break; case MVT::i32: Opcode = NVPTX::LDV_i32_v4_areg; break; case MVT::f32: Opcode = NVPTX::LDV_f32_v4_areg; break; } break; } } SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace), getI32Imm(VecType), getI32Imm(FromType), getI32Imm(FromTypeWidth), Op1, Chain }; LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops); } MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1); MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand(); cast<MachineSDNode>(LD)->setMemRefs(MemRefs0, MemRefs0 + 1); return LD; } SDNode *NVPTXDAGToDAGISel::SelectLDGLDU(SDNode *N) { SDValue Chain = N->getOperand(0); SDValue Op1; MemSDNode *Mem; bool IsLDG = true; // If this is an LDG intrinsic, the address is the third operand. Its its an // LDG/LDU SD node (from custom vector handling), then its the second operand if (N->getOpcode() == ISD::INTRINSIC_W_CHAIN) { Op1 = N->getOperand(2); Mem = cast<MemIntrinsicSDNode>(N); unsigned IID = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue(); switch (IID) { default: return NULL; case Intrinsic::nvvm_ldg_global_f: case Intrinsic::nvvm_ldg_global_i: case Intrinsic::nvvm_ldg_global_p: IsLDG = true; break; case Intrinsic::nvvm_ldu_global_f: case Intrinsic::nvvm_ldu_global_i: case Intrinsic::nvvm_ldu_global_p: IsLDG = false; break; } } else { Op1 = N->getOperand(1); Mem = cast<MemSDNode>(N); } unsigned Opcode; SDLoc DL(N); SDNode *LD; SDValue Base, Offset, Addr; EVT EltVT = Mem->getMemoryVT(); if (EltVT.isVector()) { EltVT = EltVT.getVectorElementType(); } if (SelectDirectAddr(Op1, Addr)) { switch (N->getOpcode()) { default: return nullptr; case ISD::INTRINSIC_W_CHAIN: if (IsLDG) { switch (EltVT.getSimpleVT().SimpleTy) { default: return nullptr; case MVT::i8: Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8avar; break; case MVT::i16: Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16avar; break; case MVT::i32: Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32avar; break; case MVT::i64: Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64avar; break; case MVT::f32: Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32avar; break; case MVT::f64: Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64avar; break; } } else { switch (EltVT.getSimpleVT().SimpleTy) { default: return nullptr; case MVT::i8: Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8avar; break; case MVT::i16: Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16avar; break; case MVT::i32: Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32avar; break; case MVT::i64: Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64avar; break; case MVT::f32: Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32avar; break; case MVT::f64: Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64avar; break; } } break; case NVPTXISD::LDGV2: switch (EltVT.getSimpleVT().SimpleTy) { default: return nullptr; case MVT::i8: Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_avar; break; case MVT::i16: Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_avar; break; case MVT::i32: Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_avar; break; case MVT::i64: Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_avar; break; case MVT::f32: Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_avar; break; case MVT::f64: Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_avar; break; } break; case NVPTXISD::LDUV2: switch (EltVT.getSimpleVT().SimpleTy) { default: return nullptr; case MVT::i8: Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_avar; break; case MVT::i16: Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_avar; break; case MVT::i32: Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_avar; break; case MVT::i64: Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_avar; break; case MVT::f32: Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_avar; break; case MVT::f64: Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_avar; break; } break; case NVPTXISD::LDGV4: switch (EltVT.getSimpleVT().SimpleTy) { default: return nullptr; case MVT::i8: Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_avar; break; case MVT::i16: Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_avar; break; case MVT::i32: Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_avar; break; case MVT::f32: Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_avar; break; } break; case NVPTXISD::LDUV4: switch (EltVT.getSimpleVT().SimpleTy) { default: return nullptr; case MVT::i8: Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_avar; break; case MVT::i16: Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_avar; break; case MVT::i32: Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_avar; break; case MVT::f32: Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_avar; break; } break; } SDValue Ops[] = { Addr, Chain }; LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops); } else if (Subtarget->is64Bit() ? SelectADDRri64(Op1.getNode(), Op1, Base, Offset) : SelectADDRri(Op1.getNode(), Op1, Base, Offset)) { if (Subtarget->is64Bit()) { switch (N->getOpcode()) { default: return nullptr; case ISD::INTRINSIC_W_CHAIN: if (IsLDG) { switch (EltVT.getSimpleVT().SimpleTy) { default: return nullptr; case MVT::i8: Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8ari64; break; case MVT::i16: Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16ari64; break; case MVT::i32: Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32ari64; break; case MVT::i64: Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64ari64; break; case MVT::f32: Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32ari64; break; case MVT::f64: Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64ari64; break; } } else { switch (EltVT.getSimpleVT().SimpleTy) { default: return nullptr; case MVT::i8: Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8ari64; break; case MVT::i16: Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16ari64; break; case MVT::i32: Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32ari64; break; case MVT::i64: Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64ari64; break; case MVT::f32: Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32ari64; break; case MVT::f64: Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64ari64; break; } } break; case NVPTXISD::LDGV2: switch (EltVT.getSimpleVT().SimpleTy) { default: return nullptr; case MVT::i8: Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_ari64; break; case MVT::i16: Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_ari64; break; case MVT::i32: Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_ari64; break; case MVT::i64: Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_ari64; break; case MVT::f32: Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_ari64; break; case MVT::f64: Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_ari64; break; } break; case NVPTXISD::LDUV2: switch (EltVT.getSimpleVT().SimpleTy) { default: return nullptr; case MVT::i8: Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_ari64; break; case MVT::i16: Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_ari64; break; case MVT::i32: Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_ari64; break; case MVT::i64: Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_ari64; break; case MVT::f32: Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_ari64; break; case MVT::f64: Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_ari64; break; } break; case NVPTXISD::LDGV4: switch (EltVT.getSimpleVT().SimpleTy) { default: return nullptr; case MVT::i8: Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_ari64; break; case MVT::i16: Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_ari64; break; case MVT::i32: Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_ari64; break; case MVT::f32: Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_ari64; break; } break; case NVPTXISD::LDUV4: switch (EltVT.getSimpleVT().SimpleTy) { default: return nullptr; case MVT::i8: Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_ari64; break; case MVT::i16: Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_ari64; break; case MVT::i32: Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_ari64; break; case MVT::f32: Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_ari64; break; } break; } } else { switch (N->getOpcode()) { default: return nullptr; case ISD::INTRINSIC_W_CHAIN: if (IsLDG) { switch (EltVT.getSimpleVT().SimpleTy) { default: return nullptr; case MVT::i8: Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8ari; break; case MVT::i16: Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16ari; break; case MVT::i32: Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32ari; break; case MVT::i64: Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64ari; break; case MVT::f32: Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32ari; break; case MVT::f64: Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64ari; break; } } else { switch (EltVT.getSimpleVT().SimpleTy) { default: return nullptr; case MVT::i8: Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8ari; break; case MVT::i16: Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16ari; break; case MVT::i32: Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32ari; break; case MVT::i64: Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64ari; break; case MVT::f32: Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32ari; break; case MVT::f64: Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64ari; break; } } break; case NVPTXISD::LDGV2: switch (EltVT.getSimpleVT().SimpleTy) { default: return nullptr; case MVT::i8: Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_ari32; break; case MVT::i16: Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_ari32; break; case MVT::i32: Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_ari32; break; case MVT::i64: Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_ari32; break; case MVT::f32: Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_ari32; break; case MVT::f64: Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_ari32; break; } break; case NVPTXISD::LDUV2: switch (EltVT.getSimpleVT().SimpleTy) { default: return nullptr; case MVT::i8: Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_ari32; break; case MVT::i16: Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_ari32; break; case MVT::i32: Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_ari32; break; case MVT::i64: Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_ari32; break; case MVT::f32: Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_ari32; break; case MVT::f64: Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_ari32; break; } break; case NVPTXISD::LDGV4: switch (EltVT.getSimpleVT().SimpleTy) { default: return nullptr; case MVT::i8: Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_ari32; break; case MVT::i16: Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_ari32; break; case MVT::i32: Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_ari32; break; case MVT::f32: Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_ari32; break; } break; case NVPTXISD::LDUV4: switch (EltVT.getSimpleVT().SimpleTy) { default: return nullptr; case MVT::i8: Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_ari32; break; case MVT::i16: Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_ari32; break; case MVT::i32: Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_ari32; break; case MVT::f32: Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_ari32; break; } break; } } SDValue Ops[] = { Base, Offset, Chain }; LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops); } else { if (Subtarget->is64Bit()) { switch (N->getOpcode()) { default: return nullptr; case ISD::INTRINSIC_W_CHAIN: if (IsLDG) { switch (EltVT.getSimpleVT().SimpleTy) { default: return nullptr; case MVT::i8: Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8areg64; break; case MVT::i16: Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16areg64; break; case MVT::i32: Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32areg64; break; case MVT::i64: Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64areg64; break; case MVT::f32: Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32areg64; break; case MVT::f64: Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64areg64; break; } } else { switch (EltVT.getSimpleVT().SimpleTy) { default: return nullptr; case MVT::i8: Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8areg64; break; case MVT::i16: Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16areg64; break; case MVT::i32: Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32areg64; break; case MVT::i64: Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64areg64; break; case MVT::f32: Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32areg64; break; case MVT::f64: Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64areg64; break; } } break; case NVPTXISD::LDGV2: switch (EltVT.getSimpleVT().SimpleTy) { default: return nullptr; case MVT::i8: Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_areg64; break; case MVT::i16: Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_areg64; break; case MVT::i32: Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_areg64; break; case MVT::i64: Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_areg64; break; case MVT::f32: Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_areg64; break; case MVT::f64: Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_areg64; break; } break; case NVPTXISD::LDUV2: switch (EltVT.getSimpleVT().SimpleTy) { default: return nullptr; case MVT::i8: Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_areg64; break; case MVT::i16: Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_areg64; break; case MVT::i32: Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_areg64; break; case MVT::i64: Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_areg64; break; case MVT::f32: Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_areg64; break; case MVT::f64: Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_areg64; break; } break; case NVPTXISD::LDGV4: switch (EltVT.getSimpleVT().SimpleTy) { default: return nullptr; case MVT::i8: Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_areg64; break; case MVT::i16: Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_areg64; break; case MVT::i32: Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_areg64; break; case MVT::f32: Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_areg64; break; } break; case NVPTXISD::LDUV4: switch (EltVT.getSimpleVT().SimpleTy) { default: return nullptr; case MVT::i8: Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_areg64; break; case MVT::i16: Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_areg64; break; case MVT::i32: Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_areg64; break; case MVT::f32: Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_areg64; break; } break; } } else { switch (N->getOpcode()) { default: return nullptr; case ISD::INTRINSIC_W_CHAIN: if (IsLDG) { switch (EltVT.getSimpleVT().SimpleTy) { default: return nullptr; case MVT::i8: Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8areg; break; case MVT::i16: Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16areg; break; case MVT::i32: Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32areg; break; case MVT::i64: Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64areg; break; case MVT::f32: Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32areg; break; case MVT::f64: Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64areg; break; } } else { switch (EltVT.getSimpleVT().SimpleTy) { default: return nullptr; case MVT::i8: Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8areg; break; case MVT::i16: Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16areg; break; case MVT::i32: Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32areg; break; case MVT::i64: Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64areg; break; case MVT::f32: Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32areg; break; case MVT::f64: Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64areg; break; } } break; case NVPTXISD::LDGV2: switch (EltVT.getSimpleVT().SimpleTy) { default: return nullptr; case MVT::i8: Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_areg32; break; case MVT::i16: Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_areg32; break; case MVT::i32: Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_areg32; break; case MVT::i64: Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_areg32; break; case MVT::f32: Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_areg32; break; case MVT::f64: Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_areg32; break; } break; case NVPTXISD::LDUV2: switch (EltVT.getSimpleVT().SimpleTy) { default: return nullptr; case MVT::i8: Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_areg32; break; case MVT::i16: Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_areg32; break; case MVT::i32: Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_areg32; break; case MVT::i64: Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_areg32; break; case MVT::f32: Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_areg32; break; case MVT::f64: Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_areg32; break; } break; case NVPTXISD::LDGV4: switch (EltVT.getSimpleVT().SimpleTy) { default: return nullptr; case MVT::i8: Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_areg32; break; case MVT::i16: Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_areg32; break; case MVT::i32: Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_areg32; break; case MVT::f32: Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_areg32; break; } break; case NVPTXISD::LDUV4: switch (EltVT.getSimpleVT().SimpleTy) { default: return nullptr; case MVT::i8: Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_areg32; break; case MVT::i16: Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_areg32; break; case MVT::i32: Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_areg32; break; case MVT::f32: Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_areg32; break; } break; } } SDValue Ops[] = { Op1, Chain }; LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops); } MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1); MemRefs0[0] = Mem->getMemOperand(); cast<MachineSDNode>(LD)->setMemRefs(MemRefs0, MemRefs0 + 1); return LD; } SDNode *NVPTXDAGToDAGISel::SelectStore(SDNode *N) { SDLoc dl(N); StoreSDNode *ST = cast<StoreSDNode>(N); EVT StoreVT = ST->getMemoryVT(); SDNode *NVPTXST = nullptr; // do not support pre/post inc/dec if (ST->isIndexed()) return nullptr; if (!StoreVT.isSimple()) return nullptr; // Address Space Setting unsigned int codeAddrSpace = getCodeAddrSpace(ST); // Volatile Setting // - .volatile is only availalble for .global and .shared bool isVolatile = ST->isVolatile(); if (codeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL && codeAddrSpace != NVPTX::PTXLdStInstCode::SHARED && codeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC) isVolatile = false; // Vector Setting MVT SimpleVT = StoreVT.getSimpleVT(); unsigned vecType = NVPTX::PTXLdStInstCode::Scalar; if (SimpleVT.isVector()) { unsigned num = SimpleVT.getVectorNumElements(); if (num == 2) vecType = NVPTX::PTXLdStInstCode::V2; else if (num == 4) vecType = NVPTX::PTXLdStInstCode::V4; else return nullptr; } // Type Setting: toType + toTypeWidth // - for integer type, always use 'u' // MVT ScalarVT = SimpleVT.getScalarType(); unsigned toTypeWidth = ScalarVT.getSizeInBits(); unsigned int toType; if (ScalarVT.isFloatingPoint()) toType = NVPTX::PTXLdStInstCode::Float; else toType = NVPTX::PTXLdStInstCode::Unsigned; // Create the machine instruction DAG SDValue Chain = N->getOperand(0); SDValue N1 = N->getOperand(1); SDValue N2 = N->getOperand(2); SDValue Addr; SDValue Offset, Base; unsigned Opcode; MVT::SimpleValueType SourceVT = N1.getNode()->getSimpleValueType(0).SimpleTy; if (SelectDirectAddr(N2, Addr)) { switch (SourceVT) { case MVT::i8: Opcode = NVPTX::ST_i8_avar; break; case MVT::i16: Opcode = NVPTX::ST_i16_avar; break; case MVT::i32: Opcode = NVPTX::ST_i32_avar; break; case MVT::i64: Opcode = NVPTX::ST_i64_avar; break; case MVT::f32: Opcode = NVPTX::ST_f32_avar; break; case MVT::f64: Opcode = NVPTX::ST_f64_avar; break; default: return nullptr; } SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace), getI32Imm(vecType), getI32Imm(toType), getI32Imm(toTypeWidth), Addr, Chain }; NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops); } else if (Subtarget->is64Bit() ? SelectADDRsi64(N2.getNode(), N2, Base, Offset) : SelectADDRsi(N2.getNode(), N2, Base, Offset)) { switch (SourceVT) { case MVT::i8: Opcode = NVPTX::ST_i8_asi; break; case MVT::i16: Opcode = NVPTX::ST_i16_asi; break; case MVT::i32: Opcode = NVPTX::ST_i32_asi; break; case MVT::i64: Opcode = NVPTX::ST_i64_asi; break; case MVT::f32: Opcode = NVPTX::ST_f32_asi; break; case MVT::f64: Opcode = NVPTX::ST_f64_asi; break; default: return nullptr; } SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace), getI32Imm(vecType), getI32Imm(toType), getI32Imm(toTypeWidth), Base, Offset, Chain }; NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops); } else if (Subtarget->is64Bit() ? SelectADDRri64(N2.getNode(), N2, Base, Offset) : SelectADDRri(N2.getNode(), N2, Base, Offset)) { if (Subtarget->is64Bit()) { switch (SourceVT) { case MVT::i8: Opcode = NVPTX::ST_i8_ari_64; break; case MVT::i16: Opcode = NVPTX::ST_i16_ari_64; break; case MVT::i32: Opcode = NVPTX::ST_i32_ari_64; break; case MVT::i64: Opcode = NVPTX::ST_i64_ari_64; break; case MVT::f32: Opcode = NVPTX::ST_f32_ari_64; break; case MVT::f64: Opcode = NVPTX::ST_f64_ari_64; break; default: return nullptr; } } else { switch (SourceVT) { case MVT::i8: Opcode = NVPTX::ST_i8_ari; break; case MVT::i16: Opcode = NVPTX::ST_i16_ari; break; case MVT::i32: Opcode = NVPTX::ST_i32_ari; break; case MVT::i64: Opcode = NVPTX::ST_i64_ari; break; case MVT::f32: Opcode = NVPTX::ST_f32_ari; break; case MVT::f64: Opcode = NVPTX::ST_f64_ari; break; default: return nullptr; } } SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace), getI32Imm(vecType), getI32Imm(toType), getI32Imm(toTypeWidth), Base, Offset, Chain }; NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops); } else { if (Subtarget->is64Bit()) { switch (SourceVT) { case MVT::i8: Opcode = NVPTX::ST_i8_areg_64; break; case MVT::i16: Opcode = NVPTX::ST_i16_areg_64; break; case MVT::i32: Opcode = NVPTX::ST_i32_areg_64; break; case MVT::i64: Opcode = NVPTX::ST_i64_areg_64; break; case MVT::f32: Opcode = NVPTX::ST_f32_areg_64; break; case MVT::f64: Opcode = NVPTX::ST_f64_areg_64; break; default: return nullptr; } } else { switch (SourceVT) { case MVT::i8: Opcode = NVPTX::ST_i8_areg; break; case MVT::i16: Opcode = NVPTX::ST_i16_areg; break; case MVT::i32: Opcode = NVPTX::ST_i32_areg; break; case MVT::i64: Opcode = NVPTX::ST_i64_areg; break; case MVT::f32: Opcode = NVPTX::ST_f32_areg; break; case MVT::f64: Opcode = NVPTX::ST_f64_areg; break; default: return nullptr; } } SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace), getI32Imm(vecType), getI32Imm(toType), getI32Imm(toTypeWidth), N2, Chain }; NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops); } if (NVPTXST) { MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1); MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand(); cast<MachineSDNode>(NVPTXST)->setMemRefs(MemRefs0, MemRefs0 + 1); } return NVPTXST; } SDNode *NVPTXDAGToDAGISel::SelectStoreVector(SDNode *N) { SDValue Chain = N->getOperand(0); SDValue Op1 = N->getOperand(1); SDValue Addr, Offset, Base; unsigned Opcode; SDLoc DL(N); SDNode *ST; EVT EltVT = Op1.getValueType(); MemSDNode *MemSD = cast<MemSDNode>(N); EVT StoreVT = MemSD->getMemoryVT(); // Address Space Setting unsigned CodeAddrSpace = getCodeAddrSpace(MemSD); if (CodeAddrSpace == NVPTX::PTXLdStInstCode::CONSTANT) { report_fatal_error("Cannot store to pointer that points to constant " "memory space"); } // Volatile Setting // - .volatile is only availalble for .global and .shared bool IsVolatile = MemSD->isVolatile(); if (CodeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL && CodeAddrSpace != NVPTX::PTXLdStInstCode::SHARED && CodeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC) IsVolatile = false; // Type Setting: toType + toTypeWidth // - for integer type, always use 'u' assert(StoreVT.isSimple() && "Store value is not simple"); MVT ScalarVT = StoreVT.getSimpleVT().getScalarType(); unsigned ToTypeWidth = ScalarVT.getSizeInBits(); unsigned ToType; if (ScalarVT.isFloatingPoint()) ToType = NVPTX::PTXLdStInstCode::Float; else ToType = NVPTX::PTXLdStInstCode::Unsigned; SmallVector<SDValue, 12> StOps; SDValue N2; unsigned VecType; switch (N->getOpcode()) { case NVPTXISD::StoreV2: VecType = NVPTX::PTXLdStInstCode::V2; StOps.push_back(N->getOperand(1)); StOps.push_back(N->getOperand(2)); N2 = N->getOperand(3); break; case NVPTXISD::StoreV4: VecType = NVPTX::PTXLdStInstCode::V4; StOps.push_back(N->getOperand(1)); StOps.push_back(N->getOperand(2)); StOps.push_back(N->getOperand(3)); StOps.push_back(N->getOperand(4)); N2 = N->getOperand(5); break; default: return nullptr; } StOps.push_back(getI32Imm(IsVolatile)); StOps.push_back(getI32Imm(CodeAddrSpace)); StOps.push_back(getI32Imm(VecType)); StOps.push_back(getI32Imm(ToType)); StOps.push_back(getI32Imm(ToTypeWidth)); if (SelectDirectAddr(N2, Addr)) { switch (N->getOpcode()) { default: return nullptr; case NVPTXISD::StoreV2: switch (EltVT.getSimpleVT().SimpleTy) { default: return nullptr; case MVT::i8: Opcode = NVPTX::STV_i8_v2_avar; break; case MVT::i16: Opcode = NVPTX::STV_i16_v2_avar; break; case MVT::i32: Opcode = NVPTX::STV_i32_v2_avar; break; case MVT::i64: Opcode = NVPTX::STV_i64_v2_avar; break; case MVT::f32: Opcode = NVPTX::STV_f32_v2_avar; break; case MVT::f64: Opcode = NVPTX::STV_f64_v2_avar; break; } break; case NVPTXISD::StoreV4: switch (EltVT.getSimpleVT().SimpleTy) { default: return nullptr; case MVT::i8: Opcode = NVPTX::STV_i8_v4_avar; break; case MVT::i16: Opcode = NVPTX::STV_i16_v4_avar; break; case MVT::i32: Opcode = NVPTX::STV_i32_v4_avar; break; case MVT::f32: Opcode = NVPTX::STV_f32_v4_avar; break; } break; } StOps.push_back(Addr); } else if (Subtarget->is64Bit() ? SelectADDRsi64(N2.getNode(), N2, Base, Offset) : SelectADDRsi(N2.getNode(), N2, Base, Offset)) { switch (N->getOpcode()) { default: return nullptr; case NVPTXISD::StoreV2: switch (EltVT.getSimpleVT().SimpleTy) { default: return nullptr; case MVT::i8: Opcode = NVPTX::STV_i8_v2_asi; break; case MVT::i16: Opcode = NVPTX::STV_i16_v2_asi; break; case MVT::i32: Opcode = NVPTX::STV_i32_v2_asi; break; case MVT::i64: Opcode = NVPTX::STV_i64_v2_asi; break; case MVT::f32: Opcode = NVPTX::STV_f32_v2_asi; break; case MVT::f64: Opcode = NVPTX::STV_f64_v2_asi; break; } break; case NVPTXISD::StoreV4: switch (EltVT.getSimpleVT().SimpleTy) { default: return nullptr; case MVT::i8: Opcode = NVPTX::STV_i8_v4_asi; break; case MVT::i16: Opcode = NVPTX::STV_i16_v4_asi; break; case MVT::i32: Opcode = NVPTX::STV_i32_v4_asi; break; case MVT::f32: Opcode = NVPTX::STV_f32_v4_asi; break; } break; } StOps.push_back(Base); StOps.push_back(Offset); } else if (Subtarget->is64Bit() ? SelectADDRri64(N2.getNode(), N2, Base, Offset) : SelectADDRri(N2.getNode(), N2, Base, Offset)) { if (Subtarget->is64Bit()) { switch (N->getOpcode()) { default: return nullptr; case NVPTXISD::StoreV2: switch (EltVT.getSimpleVT().SimpleTy) { default: return nullptr; case MVT::i8: Opcode = NVPTX::STV_i8_v2_ari_64; break; case MVT::i16: Opcode = NVPTX::STV_i16_v2_ari_64; break; case MVT::i32: Opcode = NVPTX::STV_i32_v2_ari_64; break; case MVT::i64: Opcode = NVPTX::STV_i64_v2_ari_64; break; case MVT::f32: Opcode = NVPTX::STV_f32_v2_ari_64; break; case MVT::f64: Opcode = NVPTX::STV_f64_v2_ari_64; break; } break; case NVPTXISD::StoreV4: switch (EltVT.getSimpleVT().SimpleTy) { default: return nullptr; case MVT::i8: Opcode = NVPTX::STV_i8_v4_ari_64; break; case MVT::i16: Opcode = NVPTX::STV_i16_v4_ari_64; break; case MVT::i32: Opcode = NVPTX::STV_i32_v4_ari_64; break; case MVT::f32: Opcode = NVPTX::STV_f32_v4_ari_64; break; } break; } } else { switch (N->getOpcode()) { default: return nullptr; case NVPTXISD::StoreV2: switch (EltVT.getSimpleVT().SimpleTy) { default: return nullptr; case MVT::i8: Opcode = NVPTX::STV_i8_v2_ari; break; case MVT::i16: Opcode = NVPTX::STV_i16_v2_ari; break; case MVT::i32: Opcode = NVPTX::STV_i32_v2_ari; break; case MVT::i64: Opcode = NVPTX::STV_i64_v2_ari; break; case MVT::f32: Opcode = NVPTX::STV_f32_v2_ari; break; case MVT::f64: Opcode = NVPTX::STV_f64_v2_ari; break; } break; case NVPTXISD::StoreV4: switch (EltVT.getSimpleVT().SimpleTy) { default: return nullptr; case MVT::i8: Opcode = NVPTX::STV_i8_v4_ari; break; case MVT::i16: Opcode = NVPTX::STV_i16_v4_ari; break; case MVT::i32: Opcode = NVPTX::STV_i32_v4_ari; break; case MVT::f32: Opcode = NVPTX::STV_f32_v4_ari; break; } break; } } StOps.push_back(Base); StOps.push_back(Offset); } else { if (Subtarget->is64Bit()) { switch (N->getOpcode()) { default: return nullptr; case NVPTXISD::StoreV2: switch (EltVT.getSimpleVT().SimpleTy) { default: return nullptr; case MVT::i8: Opcode = NVPTX::STV_i8_v2_areg_64; break; case MVT::i16: Opcode = NVPTX::STV_i16_v2_areg_64; break; case MVT::i32: Opcode = NVPTX::STV_i32_v2_areg_64; break; case MVT::i64: Opcode = NVPTX::STV_i64_v2_areg_64; break; case MVT::f32: Opcode = NVPTX::STV_f32_v2_areg_64; break; case MVT::f64: Opcode = NVPTX::STV_f64_v2_areg_64; break; } break; case NVPTXISD::StoreV4: switch (EltVT.getSimpleVT().SimpleTy) { default: return nullptr; case MVT::i8: Opcode = NVPTX::STV_i8_v4_areg_64; break; case MVT::i16: Opcode = NVPTX::STV_i16_v4_areg_64; break; case MVT::i32: Opcode = NVPTX::STV_i32_v4_areg_64; break; case MVT::f32: Opcode = NVPTX::STV_f32_v4_areg_64; break; } break; } } else { switch (N->getOpcode()) { default: return nullptr; case NVPTXISD::StoreV2: switch (EltVT.getSimpleVT().SimpleTy) { default: return nullptr; case MVT::i8: Opcode = NVPTX::STV_i8_v2_areg; break; case MVT::i16: Opcode = NVPTX::STV_i16_v2_areg; break; case MVT::i32: Opcode = NVPTX::STV_i32_v2_areg; break; case MVT::i64: Opcode = NVPTX::STV_i64_v2_areg; break; case MVT::f32: Opcode = NVPTX::STV_f32_v2_areg; break; case MVT::f64: Opcode = NVPTX::STV_f64_v2_areg; break; } break; case NVPTXISD::StoreV4: switch (EltVT.getSimpleVT().SimpleTy) { default: return nullptr; case MVT::i8: Opcode = NVPTX::STV_i8_v4_areg; break; case MVT::i16: Opcode = NVPTX::STV_i16_v4_areg; break; case MVT::i32: Opcode = NVPTX::STV_i32_v4_areg; break; case MVT::f32: Opcode = NVPTX::STV_f32_v4_areg; break; } break; } } StOps.push_back(N2); } StOps.push_back(Chain); ST = CurDAG->getMachineNode(Opcode, DL, MVT::Other, StOps); MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1); MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand(); cast<MachineSDNode>(ST)->setMemRefs(MemRefs0, MemRefs0 + 1); return ST; } SDNode *NVPTXDAGToDAGISel::SelectLoadParam(SDNode *Node) { SDValue Chain = Node->getOperand(0); SDValue Offset = Node->getOperand(2); SDValue Flag = Node->getOperand(3); SDLoc DL(Node); MemSDNode *Mem = cast<MemSDNode>(Node); unsigned VecSize; switch (Node->getOpcode()) { default: return nullptr; case NVPTXISD::LoadParam: VecSize = 1; break; case NVPTXISD::LoadParamV2: VecSize = 2; break; case NVPTXISD::LoadParamV4: VecSize = 4; break; } EVT EltVT = Node->getValueType(0); EVT MemVT = Mem->getMemoryVT(); unsigned Opc = 0; switch (VecSize) { default: return nullptr; case 1: switch (MemVT.getSimpleVT().SimpleTy) { default: return nullptr; case MVT::i1: Opc = NVPTX::LoadParamMemI8; break; case MVT::i8: Opc = NVPTX::LoadParamMemI8; break; case MVT::i16: Opc = NVPTX::LoadParamMemI16; break; case MVT::i32: Opc = NVPTX::LoadParamMemI32; break; case MVT::i64: Opc = NVPTX::LoadParamMemI64; break; case MVT::f32: Opc = NVPTX::LoadParamMemF32; break; case MVT::f64: Opc = NVPTX::LoadParamMemF64; break; } break; case 2: switch (MemVT.getSimpleVT().SimpleTy) { default: return nullptr; case MVT::i1: Opc = NVPTX::LoadParamMemV2I8; break; case MVT::i8: Opc = NVPTX::LoadParamMemV2I8; break; case MVT::i16: Opc = NVPTX::LoadParamMemV2I16; break; case MVT::i32: Opc = NVPTX::LoadParamMemV2I32; break; case MVT::i64: Opc = NVPTX::LoadParamMemV2I64; break; case MVT::f32: Opc = NVPTX::LoadParamMemV2F32; break; case MVT::f64: Opc = NVPTX::LoadParamMemV2F64; break; } break; case 4: switch (MemVT.getSimpleVT().SimpleTy) { default: return nullptr; case MVT::i1: Opc = NVPTX::LoadParamMemV4I8; break; case MVT::i8: Opc = NVPTX::LoadParamMemV4I8; break; case MVT::i16: Opc = NVPTX::LoadParamMemV4I16; break; case MVT::i32: Opc = NVPTX::LoadParamMemV4I32; break; case MVT::f32: Opc = NVPTX::LoadParamMemV4F32; break; } break; } SDVTList VTs; if (VecSize == 1) { VTs = CurDAG->getVTList(EltVT, MVT::Other, MVT::Glue); } else if (VecSize == 2) { VTs = CurDAG->getVTList(EltVT, EltVT, MVT::Other, MVT::Glue); } else { EVT EVTs[] = { EltVT, EltVT, EltVT, EltVT, MVT::Other, MVT::Glue }; VTs = CurDAG->getVTList(EVTs); } unsigned OffsetVal = cast<ConstantSDNode>(Offset)->getZExtValue(); SmallVector<SDValue, 2> Ops; Ops.push_back(CurDAG->getTargetConstant(OffsetVal, MVT::i32)); Ops.push_back(Chain); Ops.push_back(Flag); SDNode *Ret = CurDAG->getMachineNode(Opc, DL, VTs, Ops); return Ret; } SDNode *NVPTXDAGToDAGISel::SelectStoreRetval(SDNode *N) { SDLoc DL(N); SDValue Chain = N->getOperand(0); SDValue Offset = N->getOperand(1); unsigned OffsetVal = cast<ConstantSDNode>(Offset)->getZExtValue(); MemSDNode *Mem = cast<MemSDNode>(N); // How many elements do we have? unsigned NumElts = 1; switch (N->getOpcode()) { default: return nullptr; case NVPTXISD::StoreRetval: NumElts = 1; break; case NVPTXISD::StoreRetvalV2: NumElts = 2; break; case NVPTXISD::StoreRetvalV4: NumElts = 4; break; } // Build vector of operands SmallVector<SDValue, 6> Ops; for (unsigned i = 0; i < NumElts; ++i) Ops.push_back(N->getOperand(i + 2)); Ops.push_back(CurDAG->getTargetConstant(OffsetVal, MVT::i32)); Ops.push_back(Chain); // Determine target opcode // If we have an i1, use an 8-bit store. The lowering code in // NVPTXISelLowering will have already emitted an upcast. unsigned Opcode = 0; switch (NumElts) { default: return nullptr; case 1: switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) { default: return nullptr; case MVT::i1: Opcode = NVPTX::StoreRetvalI8; break; case MVT::i8: Opcode = NVPTX::StoreRetvalI8; break; case MVT::i16: Opcode = NVPTX::StoreRetvalI16; break; case MVT::i32: Opcode = NVPTX::StoreRetvalI32; break; case MVT::i64: Opcode = NVPTX::StoreRetvalI64; break; case MVT::f32: Opcode = NVPTX::StoreRetvalF32; break; case MVT::f64: Opcode = NVPTX::StoreRetvalF64; break; } break; case 2: switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) { default: return nullptr; case MVT::i1: Opcode = NVPTX::StoreRetvalV2I8; break; case MVT::i8: Opcode = NVPTX::StoreRetvalV2I8; break; case MVT::i16: Opcode = NVPTX::StoreRetvalV2I16; break; case MVT::i32: Opcode = NVPTX::StoreRetvalV2I32; break; case MVT::i64: Opcode = NVPTX::StoreRetvalV2I64; break; case MVT::f32: Opcode = NVPTX::StoreRetvalV2F32; break; case MVT::f64: Opcode = NVPTX::StoreRetvalV2F64; break; } break; case 4: switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) { default: return nullptr; case MVT::i1: Opcode = NVPTX::StoreRetvalV4I8; break; case MVT::i8: Opcode = NVPTX::StoreRetvalV4I8; break; case MVT::i16: Opcode = NVPTX::StoreRetvalV4I16; break; case MVT::i32: Opcode = NVPTX::StoreRetvalV4I32; break; case MVT::f32: Opcode = NVPTX::StoreRetvalV4F32; break; } break; } SDNode *Ret = CurDAG->getMachineNode(Opcode, DL, MVT::Other, Ops); MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1); MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand(); cast<MachineSDNode>(Ret)->setMemRefs(MemRefs0, MemRefs0 + 1); return Ret; } SDNode *NVPTXDAGToDAGISel::SelectStoreParam(SDNode *N) { SDLoc DL(N); SDValue Chain = N->getOperand(0); SDValue Param = N->getOperand(1); unsigned ParamVal = cast<ConstantSDNode>(Param)->getZExtValue(); SDValue Offset = N->getOperand(2); unsigned OffsetVal = cast<ConstantSDNode>(Offset)->getZExtValue(); MemSDNode *Mem = cast<MemSDNode>(N); SDValue Flag = N->getOperand(N->getNumOperands() - 1); // How many elements do we have? unsigned NumElts = 1; switch (N->getOpcode()) { default: return nullptr; case NVPTXISD::StoreParamU32: case NVPTXISD::StoreParamS32: case NVPTXISD::StoreParam: NumElts = 1; break; case NVPTXISD::StoreParamV2: NumElts = 2; break; case NVPTXISD::StoreParamV4: NumElts = 4; break; } // Build vector of operands SmallVector<SDValue, 8> Ops; for (unsigned i = 0; i < NumElts; ++i) Ops.push_back(N->getOperand(i + 3)); Ops.push_back(CurDAG->getTargetConstant(ParamVal, MVT::i32)); Ops.push_back(CurDAG->getTargetConstant(OffsetVal, MVT::i32)); Ops.push_back(Chain); Ops.push_back(Flag); // Determine target opcode // If we have an i1, use an 8-bit store. The lowering code in // NVPTXISelLowering will have already emitted an upcast. unsigned Opcode = 0; switch (N->getOpcode()) { default: switch (NumElts) { default: return nullptr; case 1: switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) { default: return nullptr; case MVT::i1: Opcode = NVPTX::StoreParamI8; break; case MVT::i8: Opcode = NVPTX::StoreParamI8; break; case MVT::i16: Opcode = NVPTX::StoreParamI16; break; case MVT::i32: Opcode = NVPTX::StoreParamI32; break; case MVT::i64: Opcode = NVPTX::StoreParamI64; break; case MVT::f32: Opcode = NVPTX::StoreParamF32; break; case MVT::f64: Opcode = NVPTX::StoreParamF64; break; } break; case 2: switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) { default: return nullptr; case MVT::i1: Opcode = NVPTX::StoreParamV2I8; break; case MVT::i8: Opcode = NVPTX::StoreParamV2I8; break; case MVT::i16: Opcode = NVPTX::StoreParamV2I16; break; case MVT::i32: Opcode = NVPTX::StoreParamV2I32; break; case MVT::i64: Opcode = NVPTX::StoreParamV2I64; break; case MVT::f32: Opcode = NVPTX::StoreParamV2F32; break; case MVT::f64: Opcode = NVPTX::StoreParamV2F64; break; } break; case 4: switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) { default: return nullptr; case MVT::i1: Opcode = NVPTX::StoreParamV4I8; break; case MVT::i8: Opcode = NVPTX::StoreParamV4I8; break; case MVT::i16: Opcode = NVPTX::StoreParamV4I16; break; case MVT::i32: Opcode = NVPTX::StoreParamV4I32; break; case MVT::f32: Opcode = NVPTX::StoreParamV4F32; break; } break; } break; // Special case: if we have a sign-extend/zero-extend node, insert the // conversion instruction first, and use that as the value operand to // the selected StoreParam node. case NVPTXISD::StoreParamU32: { Opcode = NVPTX::StoreParamI32; SDValue CvtNone = CurDAG->getTargetConstant(NVPTX::PTXCvtMode::NONE, MVT::i32); SDNode *Cvt = CurDAG->getMachineNode(NVPTX::CVT_u32_u16, DL, MVT::i32, Ops[0], CvtNone); Ops[0] = SDValue(Cvt, 0); break; } case NVPTXISD::StoreParamS32: { Opcode = NVPTX::StoreParamI32; SDValue CvtNone = CurDAG->getTargetConstant(NVPTX::PTXCvtMode::NONE, MVT::i32); SDNode *Cvt = CurDAG->getMachineNode(NVPTX::CVT_s32_s16, DL, MVT::i32, Ops[0], CvtNone); Ops[0] = SDValue(Cvt, 0); break; } } SDVTList RetVTs = CurDAG->getVTList(MVT::Other, MVT::Glue); SDNode *Ret = CurDAG->getMachineNode(Opcode, DL, RetVTs, Ops); MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1); MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand(); cast<MachineSDNode>(Ret)->setMemRefs(MemRefs0, MemRefs0 + 1); return Ret; } SDNode *NVPTXDAGToDAGISel::SelectTextureIntrinsic(SDNode *N) { SDValue Chain = N->getOperand(0); SDNode *Ret = nullptr; unsigned Opc = 0; SmallVector<SDValue, 8> Ops; switch (N->getOpcode()) { default: return nullptr; case NVPTXISD::Tex1DFloatS32: Opc = NVPTX::TEX_1D_F32_S32; break; case NVPTXISD::Tex1DFloatFloat: Opc = NVPTX::TEX_1D_F32_F32; break; case NVPTXISD::Tex1DFloatFloatLevel: Opc = NVPTX::TEX_1D_F32_F32_LEVEL; break; case NVPTXISD::Tex1DFloatFloatGrad: Opc = NVPTX::TEX_1D_F32_F32_GRAD; break; case NVPTXISD::Tex1DS32S32: Opc = NVPTX::TEX_1D_S32_S32; break; case NVPTXISD::Tex1DS32Float: Opc = NVPTX::TEX_1D_S32_F32; break; case NVPTXISD::Tex1DS32FloatLevel: Opc = NVPTX::TEX_1D_S32_F32_LEVEL; break; case NVPTXISD::Tex1DS32FloatGrad: Opc = NVPTX::TEX_1D_S32_F32_GRAD; break; case NVPTXISD::Tex1DU32S32: Opc = NVPTX::TEX_1D_U32_S32; break; case NVPTXISD::Tex1DU32Float: Opc = NVPTX::TEX_1D_U32_F32; break; case NVPTXISD::Tex1DU32FloatLevel: Opc = NVPTX::TEX_1D_U32_F32_LEVEL; break; case NVPTXISD::Tex1DU32FloatGrad: Opc = NVPTX::TEX_1D_U32_F32_GRAD; break; case NVPTXISD::Tex1DArrayFloatS32: Opc = NVPTX::TEX_1D_ARRAY_F32_S32; break; case NVPTXISD::Tex1DArrayFloatFloat: Opc = NVPTX::TEX_1D_ARRAY_F32_F32; break; case NVPTXISD::Tex1DArrayFloatFloatLevel: Opc = NVPTX::TEX_1D_ARRAY_F32_F32_LEVEL; break; case NVPTXISD::Tex1DArrayFloatFloatGrad: Opc = NVPTX::TEX_1D_ARRAY_F32_F32_GRAD; break; case NVPTXISD::Tex1DArrayS32S32: Opc = NVPTX::TEX_1D_ARRAY_S32_S32; break; case NVPTXISD::Tex1DArrayS32Float: Opc = NVPTX::TEX_1D_ARRAY_S32_F32; break; case NVPTXISD::Tex1DArrayS32FloatLevel: Opc = NVPTX::TEX_1D_ARRAY_S32_F32_LEVEL; break; case NVPTXISD::Tex1DArrayS32FloatGrad: Opc = NVPTX::TEX_1D_ARRAY_S32_F32_GRAD; break; case NVPTXISD::Tex1DArrayU32S32: Opc = NVPTX::TEX_1D_ARRAY_U32_S32; break; case NVPTXISD::Tex1DArrayU32Float: Opc = NVPTX::TEX_1D_ARRAY_U32_F32; break; case NVPTXISD::Tex1DArrayU32FloatLevel: Opc = NVPTX::TEX_1D_ARRAY_U32_F32_LEVEL; break; case NVPTXISD::Tex1DArrayU32FloatGrad: Opc = NVPTX::TEX_1D_ARRAY_U32_F32_GRAD; break; case NVPTXISD::Tex2DFloatS32: Opc = NVPTX::TEX_2D_F32_S32; break; case NVPTXISD::Tex2DFloatFloat: Opc = NVPTX::TEX_2D_F32_F32; break; case NVPTXISD::Tex2DFloatFloatLevel: Opc = NVPTX::TEX_2D_F32_F32_LEVEL; break; case NVPTXISD::Tex2DFloatFloatGrad: Opc = NVPTX::TEX_2D_F32_F32_GRAD; break; case NVPTXISD::Tex2DS32S32: Opc = NVPTX::TEX_2D_S32_S32; break; case NVPTXISD::Tex2DS32Float: Opc = NVPTX::TEX_2D_S32_F32; break; case NVPTXISD::Tex2DS32FloatLevel: Opc = NVPTX::TEX_2D_S32_F32_LEVEL; break; case NVPTXISD::Tex2DS32FloatGrad: Opc = NVPTX::TEX_2D_S32_F32_GRAD; break; case NVPTXISD::Tex2DU32S32: Opc = NVPTX::TEX_2D_U32_S32; break; case NVPTXISD::Tex2DU32Float: Opc = NVPTX::TEX_2D_U32_F32; break; case NVPTXISD::Tex2DU32FloatLevel: Opc = NVPTX::TEX_2D_U32_F32_LEVEL; break; case NVPTXISD::Tex2DU32FloatGrad: Opc = NVPTX::TEX_2D_U32_F32_GRAD; break; case NVPTXISD::Tex2DArrayFloatS32: Opc = NVPTX::TEX_2D_ARRAY_F32_S32; break; case NVPTXISD::Tex2DArrayFloatFloat: Opc = NVPTX::TEX_2D_ARRAY_F32_F32; break; case NVPTXISD::Tex2DArrayFloatFloatLevel: Opc = NVPTX::TEX_2D_ARRAY_F32_F32_LEVEL; break; case NVPTXISD::Tex2DArrayFloatFloatGrad: Opc = NVPTX::TEX_2D_ARRAY_F32_F32_GRAD; break; case NVPTXISD::Tex2DArrayS32S32: Opc = NVPTX::TEX_2D_ARRAY_S32_S32; break; case NVPTXISD::Tex2DArrayS32Float: Opc = NVPTX::TEX_2D_ARRAY_S32_F32; break; case NVPTXISD::Tex2DArrayS32FloatLevel: Opc = NVPTX::TEX_2D_ARRAY_S32_F32_LEVEL; break; case NVPTXISD::Tex2DArrayS32FloatGrad: Opc = NVPTX::TEX_2D_ARRAY_S32_F32_GRAD; break; case NVPTXISD::Tex2DArrayU32S32: Opc = NVPTX::TEX_2D_ARRAY_U32_S32; break; case NVPTXISD::Tex2DArrayU32Float: Opc = NVPTX::TEX_2D_ARRAY_U32_F32; break; case NVPTXISD::Tex2DArrayU32FloatLevel: Opc = NVPTX::TEX_2D_ARRAY_U32_F32_LEVEL; break; case NVPTXISD::Tex2DArrayU32FloatGrad: Opc = NVPTX::TEX_2D_ARRAY_U32_F32_GRAD; break; case NVPTXISD::Tex3DFloatS32: Opc = NVPTX::TEX_3D_F32_S32; break; case NVPTXISD::Tex3DFloatFloat: Opc = NVPTX::TEX_3D_F32_F32; break; case NVPTXISD::Tex3DFloatFloatLevel: Opc = NVPTX::TEX_3D_F32_F32_LEVEL; break; case NVPTXISD::Tex3DFloatFloatGrad: Opc = NVPTX::TEX_3D_F32_F32_GRAD; break; case NVPTXISD::Tex3DS32S32: Opc = NVPTX::TEX_3D_S32_S32; break; case NVPTXISD::Tex3DS32Float: Opc = NVPTX::TEX_3D_S32_F32; break; case NVPTXISD::Tex3DS32FloatLevel: Opc = NVPTX::TEX_3D_S32_F32_LEVEL; break; case NVPTXISD::Tex3DS32FloatGrad: Opc = NVPTX::TEX_3D_S32_F32_GRAD; break; case NVPTXISD::Tex3DU32S32: Opc = NVPTX::TEX_3D_U32_S32; break; case NVPTXISD::Tex3DU32Float: Opc = NVPTX::TEX_3D_U32_F32; break; case NVPTXISD::Tex3DU32FloatLevel: Opc = NVPTX::TEX_3D_U32_F32_LEVEL; break; case NVPTXISD::Tex3DU32FloatGrad: Opc = NVPTX::TEX_3D_U32_F32_GRAD; break; case NVPTXISD::TexCubeFloatFloat: Opc = NVPTX::TEX_CUBE_F32_F32; break; case NVPTXISD::TexCubeFloatFloatLevel: Opc = NVPTX::TEX_CUBE_F32_F32_LEVEL; break; case NVPTXISD::TexCubeS32Float: Opc = NVPTX::TEX_CUBE_S32_F32; break; case NVPTXISD::TexCubeS32FloatLevel: Opc = NVPTX::TEX_CUBE_S32_F32_LEVEL; break; case NVPTXISD::TexCubeU32Float: Opc = NVPTX::TEX_CUBE_U32_F32; break; case NVPTXISD::TexCubeU32FloatLevel: Opc = NVPTX::TEX_CUBE_U32_F32_LEVEL; break; case NVPTXISD::TexCubeArrayFloatFloat: Opc = NVPTX::TEX_CUBE_ARRAY_F32_F32; break; case NVPTXISD::TexCubeArrayFloatFloatLevel: Opc = NVPTX::TEX_CUBE_ARRAY_F32_F32_LEVEL; break; case NVPTXISD::TexCubeArrayS32Float: Opc = NVPTX::TEX_CUBE_ARRAY_S32_F32; break; case NVPTXISD::TexCubeArrayS32FloatLevel: Opc = NVPTX::TEX_CUBE_ARRAY_S32_F32_LEVEL; break; case NVPTXISD::TexCubeArrayU32Float: Opc = NVPTX::TEX_CUBE_ARRAY_U32_F32; break; case NVPTXISD::TexCubeArrayU32FloatLevel: Opc = NVPTX::TEX_CUBE_ARRAY_U32_F32_LEVEL; break; case NVPTXISD::Tld4R2DFloatFloat: Opc = NVPTX::TLD4_R_2D_F32_F32; break; case NVPTXISD::Tld4G2DFloatFloat: Opc = NVPTX::TLD4_G_2D_F32_F32; break; case NVPTXISD::Tld4B2DFloatFloat: Opc = NVPTX::TLD4_B_2D_F32_F32; break; case NVPTXISD::Tld4A2DFloatFloat: Opc = NVPTX::TLD4_A_2D_F32_F32; break; case NVPTXISD::Tld4R2DS64Float: Opc = NVPTX::TLD4_R_2D_S32_F32; break; case NVPTXISD::Tld4G2DS64Float: Opc = NVPTX::TLD4_G_2D_S32_F32; break; case NVPTXISD::Tld4B2DS64Float: Opc = NVPTX::TLD4_B_2D_S32_F32; break; case NVPTXISD::Tld4A2DS64Float: Opc = NVPTX::TLD4_A_2D_S32_F32; break; case NVPTXISD::Tld4R2DU64Float: Opc = NVPTX::TLD4_R_2D_U32_F32; break; case NVPTXISD::Tld4G2DU64Float: Opc = NVPTX::TLD4_G_2D_U32_F32; break; case NVPTXISD::Tld4B2DU64Float: Opc = NVPTX::TLD4_B_2D_U32_F32; break; case NVPTXISD::Tld4A2DU64Float: Opc = NVPTX::TLD4_A_2D_U32_F32; break; case NVPTXISD::TexUnified1DFloatS32: Opc = NVPTX::TEX_UNIFIED_1D_F32_S32; break; case NVPTXISD::TexUnified1DFloatFloat: Opc = NVPTX::TEX_UNIFIED_1D_F32_F32; break; case NVPTXISD::TexUnified1DFloatFloatLevel: Opc = NVPTX::TEX_UNIFIED_1D_F32_F32_LEVEL; break; case NVPTXISD::TexUnified1DFloatFloatGrad: Opc = NVPTX::TEX_UNIFIED_1D_F32_F32_GRAD; break; case NVPTXISD::TexUnified1DS32S32: Opc = NVPTX::TEX_UNIFIED_1D_S32_S32; break; case NVPTXISD::TexUnified1DS32Float: Opc = NVPTX::TEX_UNIFIED_1D_S32_F32; break; case NVPTXISD::TexUnified1DS32FloatLevel: Opc = NVPTX::TEX_UNIFIED_1D_S32_F32_LEVEL; break; case NVPTXISD::TexUnified1DS32FloatGrad: Opc = NVPTX::TEX_UNIFIED_1D_S32_F32_GRAD; break; case NVPTXISD::TexUnified1DU32S32: Opc = NVPTX::TEX_UNIFIED_1D_U32_S32; break; case NVPTXISD::TexUnified1DU32Float: Opc = NVPTX::TEX_UNIFIED_1D_U32_F32; break; case NVPTXISD::TexUnified1DU32FloatLevel: Opc = NVPTX::TEX_UNIFIED_1D_U32_F32_LEVEL; break; case NVPTXISD::TexUnified1DU32FloatGrad: Opc = NVPTX::TEX_UNIFIED_1D_U32_F32_GRAD; break; case NVPTXISD::TexUnified1DArrayFloatS32: Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_F32_S32; break; case NVPTXISD::TexUnified1DArrayFloatFloat: Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_F32_F32; break; case NVPTXISD::TexUnified1DArrayFloatFloatLevel: Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_F32_F32_LEVEL; break; case NVPTXISD::TexUnified1DArrayFloatFloatGrad: Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_F32_F32_GRAD; break; case NVPTXISD::TexUnified1DArrayS32S32: Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_S32_S32; break; case NVPTXISD::TexUnified1DArrayS32Float: Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_S32_F32; break; case NVPTXISD::TexUnified1DArrayS32FloatLevel: Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_S32_F32_LEVEL; break; case NVPTXISD::TexUnified1DArrayS32FloatGrad: Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_S32_F32_GRAD; break; case NVPTXISD::TexUnified1DArrayU32S32: Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_U32_S32; break; case NVPTXISD::TexUnified1DArrayU32Float: Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_U32_F32; break; case NVPTXISD::TexUnified1DArrayU32FloatLevel: Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_U32_F32_LEVEL; break; case NVPTXISD::TexUnified1DArrayU32FloatGrad: Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_U32_F32_GRAD; break; case NVPTXISD::TexUnified2DFloatS32: Opc = NVPTX::TEX_UNIFIED_2D_F32_S32; break; case NVPTXISD::TexUnified2DFloatFloat: Opc = NVPTX::TEX_UNIFIED_2D_F32_F32; break; case NVPTXISD::TexUnified2DFloatFloatLevel: Opc = NVPTX::TEX_UNIFIED_2D_F32_F32_LEVEL; break; case NVPTXISD::TexUnified2DFloatFloatGrad: Opc = NVPTX::TEX_UNIFIED_2D_F32_F32_GRAD; break; case NVPTXISD::TexUnified2DS32S32: Opc = NVPTX::TEX_UNIFIED_2D_S32_S32; break; case NVPTXISD::TexUnified2DS32Float: Opc = NVPTX::TEX_UNIFIED_2D_S32_F32; break; case NVPTXISD::TexUnified2DS32FloatLevel: Opc = NVPTX::TEX_UNIFIED_2D_S32_F32_LEVEL; break; case NVPTXISD::TexUnified2DS32FloatGrad: Opc = NVPTX::TEX_UNIFIED_2D_S32_F32_GRAD; break; case NVPTXISD::TexUnified2DU32S32: Opc = NVPTX::TEX_UNIFIED_2D_U32_S32; break; case NVPTXISD::TexUnified2DU32Float: Opc = NVPTX::TEX_UNIFIED_2D_U32_F32; break; case NVPTXISD::TexUnified2DU32FloatLevel: Opc = NVPTX::TEX_UNIFIED_2D_U32_F32_LEVEL; break; case NVPTXISD::TexUnified2DU32FloatGrad: Opc = NVPTX::TEX_UNIFIED_2D_U32_F32_GRAD; break; case NVPTXISD::TexUnified2DArrayFloatS32: Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_F32_S32; break; case NVPTXISD::TexUnified2DArrayFloatFloat: Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_F32_F32; break; case NVPTXISD::TexUnified2DArrayFloatFloatLevel: Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_F32_F32_LEVEL; break; case NVPTXISD::TexUnified2DArrayFloatFloatGrad: Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_F32_F32_GRAD; break; case NVPTXISD::TexUnified2DArrayS32S32: Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_S32_S32; break; case NVPTXISD::TexUnified2DArrayS32Float: Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_S32_F32; break; case NVPTXISD::TexUnified2DArrayS32FloatLevel: Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_S32_F32_LEVEL; break; case NVPTXISD::TexUnified2DArrayS32FloatGrad: Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_S32_F32_GRAD; break; case NVPTXISD::TexUnified2DArrayU32S32: Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_U32_S32; break; case NVPTXISD::TexUnified2DArrayU32Float: Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_U32_F32; break; case NVPTXISD::TexUnified2DArrayU32FloatLevel: Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_U32_F32_LEVEL; break; case NVPTXISD::TexUnified2DArrayU32FloatGrad: Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_U32_F32_GRAD; break; case NVPTXISD::TexUnified3DFloatS32: Opc = NVPTX::TEX_UNIFIED_3D_F32_S32; break; case NVPTXISD::TexUnified3DFloatFloat: Opc = NVPTX::TEX_UNIFIED_3D_F32_F32; break; case NVPTXISD::TexUnified3DFloatFloatLevel: Opc = NVPTX::TEX_UNIFIED_3D_F32_F32_LEVEL; break; case NVPTXISD::TexUnified3DFloatFloatGrad: Opc = NVPTX::TEX_UNIFIED_3D_F32_F32_GRAD; break; case NVPTXISD::TexUnified3DS32S32: Opc = NVPTX::TEX_UNIFIED_3D_S32_S32; break; case NVPTXISD::TexUnified3DS32Float: Opc = NVPTX::TEX_UNIFIED_3D_S32_F32; break; case NVPTXISD::TexUnified3DS32FloatLevel: Opc = NVPTX::TEX_UNIFIED_3D_S32_F32_LEVEL; break; case NVPTXISD::TexUnified3DS32FloatGrad: Opc = NVPTX::TEX_UNIFIED_3D_S32_F32_GRAD; break; case NVPTXISD::TexUnified3DU32S32: Opc = NVPTX::TEX_UNIFIED_3D_U32_S32; break; case NVPTXISD::TexUnified3DU32Float: Opc = NVPTX::TEX_UNIFIED_3D_U32_F32; break; case NVPTXISD::TexUnified3DU32FloatLevel: Opc = NVPTX::TEX_UNIFIED_3D_U32_F32_LEVEL; break; case NVPTXISD::TexUnified3DU32FloatGrad: Opc = NVPTX::TEX_UNIFIED_3D_U32_F32_GRAD; break; case NVPTXISD::TexUnifiedCubeFloatFloat: Opc = NVPTX::TEX_UNIFIED_CUBE_F32_F32; break; case NVPTXISD::TexUnifiedCubeFloatFloatLevel: Opc = NVPTX::TEX_UNIFIED_CUBE_F32_F32_LEVEL; break; case NVPTXISD::TexUnifiedCubeS32Float: Opc = NVPTX::TEX_UNIFIED_CUBE_S32_F32; break; case NVPTXISD::TexUnifiedCubeS32FloatLevel: Opc = NVPTX::TEX_UNIFIED_CUBE_S32_F32_LEVEL; break; case NVPTXISD::TexUnifiedCubeU32Float: Opc = NVPTX::TEX_UNIFIED_CUBE_U32_F32; break; case NVPTXISD::TexUnifiedCubeU32FloatLevel: Opc = NVPTX::TEX_UNIFIED_CUBE_U32_F32_LEVEL; break; case NVPTXISD::TexUnifiedCubeArrayFloatFloat: Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_F32_F32; break; case NVPTXISD::TexUnifiedCubeArrayFloatFloatLevel: Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_F32_F32_LEVEL; break; case NVPTXISD::TexUnifiedCubeArrayS32Float: Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_S32_F32; break; case NVPTXISD::TexUnifiedCubeArrayS32FloatLevel: Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_S32_F32_LEVEL; break; case NVPTXISD::TexUnifiedCubeArrayU32Float: Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_U32_F32; break; case NVPTXISD::TexUnifiedCubeArrayU32FloatLevel: Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_U32_F32_LEVEL; break; case NVPTXISD::Tld4UnifiedR2DFloatFloat: Opc = NVPTX::TLD4_UNIFIED_R_2D_F32_F32; break; case NVPTXISD::Tld4UnifiedG2DFloatFloat: Opc = NVPTX::TLD4_UNIFIED_G_2D_F32_F32; break; case NVPTXISD::Tld4UnifiedB2DFloatFloat: Opc = NVPTX::TLD4_UNIFIED_B_2D_F32_F32; break; case NVPTXISD::Tld4UnifiedA2DFloatFloat: Opc = NVPTX::TLD4_UNIFIED_A_2D_F32_F32; break; case NVPTXISD::Tld4UnifiedR2DS64Float: Opc = NVPTX::TLD4_UNIFIED_R_2D_S32_F32; break; case NVPTXISD::Tld4UnifiedG2DS64Float: Opc = NVPTX::TLD4_UNIFIED_G_2D_S32_F32; break; case NVPTXISD::Tld4UnifiedB2DS64Float: Opc = NVPTX::TLD4_UNIFIED_B_2D_S32_F32; break; case NVPTXISD::Tld4UnifiedA2DS64Float: Opc = NVPTX::TLD4_UNIFIED_A_2D_S32_F32; break; case NVPTXISD::Tld4UnifiedR2DU64Float: Opc = NVPTX::TLD4_UNIFIED_R_2D_U32_F32; break; case NVPTXISD::Tld4UnifiedG2DU64Float: Opc = NVPTX::TLD4_UNIFIED_G_2D_U32_F32; break; case NVPTXISD::Tld4UnifiedB2DU64Float: Opc = NVPTX::TLD4_UNIFIED_B_2D_U32_F32; break; case NVPTXISD::Tld4UnifiedA2DU64Float: Opc = NVPTX::TLD4_UNIFIED_A_2D_U32_F32; break; } // Copy over operands for (unsigned i = 1; i < N->getNumOperands(); ++i) { Ops.push_back(N->getOperand(i)); } Ops.push_back(Chain); Ret = CurDAG->getMachineNode(Opc, SDLoc(N), N->getVTList(), Ops); return Ret; } SDNode *NVPTXDAGToDAGISel::SelectSurfaceIntrinsic(SDNode *N) { SDValue Chain = N->getOperand(0); SDValue TexHandle = N->getOperand(1); SDNode *Ret = nullptr; unsigned Opc = 0; SmallVector<SDValue, 8> Ops; switch (N->getOpcode()) { default: return nullptr; case NVPTXISD::Suld1DI8Clamp: Opc = NVPTX::SULD_1D_I8_CLAMP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(Chain); break; case NVPTXISD::Suld1DI16Clamp: Opc = NVPTX::SULD_1D_I16_CLAMP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(Chain); break; case NVPTXISD::Suld1DI32Clamp: Opc = NVPTX::SULD_1D_I32_CLAMP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(Chain); break; case NVPTXISD::Suld1DI64Clamp: Opc = NVPTX::SULD_1D_I64_CLAMP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(Chain); break; case NVPTXISD::Suld1DV2I8Clamp: Opc = NVPTX::SULD_1D_V2I8_CLAMP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(Chain); break; case NVPTXISD::Suld1DV2I16Clamp: Opc = NVPTX::SULD_1D_V2I16_CLAMP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(Chain); break; case NVPTXISD::Suld1DV2I32Clamp: Opc = NVPTX::SULD_1D_V2I32_CLAMP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(Chain); break; case NVPTXISD::Suld1DV2I64Clamp: Opc = NVPTX::SULD_1D_V2I64_CLAMP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(Chain); break; case NVPTXISD::Suld1DV4I8Clamp: Opc = NVPTX::SULD_1D_V4I8_CLAMP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(Chain); break; case NVPTXISD::Suld1DV4I16Clamp: Opc = NVPTX::SULD_1D_V4I16_CLAMP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(Chain); break; case NVPTXISD::Suld1DV4I32Clamp: Opc = NVPTX::SULD_1D_V4I32_CLAMP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(Chain); break; case NVPTXISD::Suld1DArrayI8Clamp: Opc = NVPTX::SULD_1D_ARRAY_I8_CLAMP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(Chain); break; case NVPTXISD::Suld1DArrayI16Clamp: Opc = NVPTX::SULD_1D_ARRAY_I16_CLAMP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(Chain); break; case NVPTXISD::Suld1DArrayI32Clamp: Opc = NVPTX::SULD_1D_ARRAY_I32_CLAMP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(Chain); break; case NVPTXISD::Suld1DArrayI64Clamp: Opc = NVPTX::SULD_1D_ARRAY_I64_CLAMP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(Chain); break; case NVPTXISD::Suld1DArrayV2I8Clamp: Opc = NVPTX::SULD_1D_ARRAY_V2I8_CLAMP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(Chain); break; case NVPTXISD::Suld1DArrayV2I16Clamp: Opc = NVPTX::SULD_1D_ARRAY_V2I16_CLAMP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(Chain); break; case NVPTXISD::Suld1DArrayV2I32Clamp: Opc = NVPTX::SULD_1D_ARRAY_V2I32_CLAMP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(Chain); break; case NVPTXISD::Suld1DArrayV2I64Clamp: Opc = NVPTX::SULD_1D_ARRAY_V2I64_CLAMP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(Chain); break; case NVPTXISD::Suld1DArrayV4I8Clamp: Opc = NVPTX::SULD_1D_ARRAY_V4I8_CLAMP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(Chain); break; case NVPTXISD::Suld1DArrayV4I16Clamp: Opc = NVPTX::SULD_1D_ARRAY_V4I16_CLAMP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(Chain); break; case NVPTXISD::Suld1DArrayV4I32Clamp: Opc = NVPTX::SULD_1D_ARRAY_V4I32_CLAMP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(Chain); break; case NVPTXISD::Suld2DI8Clamp: Opc = NVPTX::SULD_2D_I8_CLAMP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(Chain); break; case NVPTXISD::Suld2DI16Clamp: Opc = NVPTX::SULD_2D_I16_CLAMP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(Chain); break; case NVPTXISD::Suld2DI32Clamp: Opc = NVPTX::SULD_2D_I32_CLAMP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(Chain); break; case NVPTXISD::Suld2DI64Clamp: Opc = NVPTX::SULD_2D_I64_CLAMP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(Chain); break; case NVPTXISD::Suld2DV2I8Clamp: Opc = NVPTX::SULD_2D_V2I8_CLAMP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(Chain); break; case NVPTXISD::Suld2DV2I16Clamp: Opc = NVPTX::SULD_2D_V2I16_CLAMP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(Chain); break; case NVPTXISD::Suld2DV2I32Clamp: Opc = NVPTX::SULD_2D_V2I32_CLAMP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(Chain); break; case NVPTXISD::Suld2DV2I64Clamp: Opc = NVPTX::SULD_2D_V2I64_CLAMP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(Chain); break; case NVPTXISD::Suld2DV4I8Clamp: Opc = NVPTX::SULD_2D_V4I8_CLAMP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(Chain); break; case NVPTXISD::Suld2DV4I16Clamp: Opc = NVPTX::SULD_2D_V4I16_CLAMP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(Chain); break; case NVPTXISD::Suld2DV4I32Clamp: Opc = NVPTX::SULD_2D_V4I32_CLAMP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(Chain); break; case NVPTXISD::Suld2DArrayI8Clamp: Opc = NVPTX::SULD_2D_ARRAY_I8_CLAMP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(N->getOperand(4)); Ops.push_back(Chain); break; case NVPTXISD::Suld2DArrayI16Clamp: Opc = NVPTX::SULD_2D_ARRAY_I16_CLAMP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(N->getOperand(4)); Ops.push_back(Chain); break; case NVPTXISD::Suld2DArrayI32Clamp: Opc = NVPTX::SULD_2D_ARRAY_I32_CLAMP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(N->getOperand(4)); Ops.push_back(Chain); break; case NVPTXISD::Suld2DArrayI64Clamp: Opc = NVPTX::SULD_2D_ARRAY_I64_CLAMP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(N->getOperand(4)); Ops.push_back(Chain); break; case NVPTXISD::Suld2DArrayV2I8Clamp: Opc = NVPTX::SULD_2D_ARRAY_V2I8_CLAMP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(N->getOperand(4)); Ops.push_back(Chain); break; case NVPTXISD::Suld2DArrayV2I16Clamp: Opc = NVPTX::SULD_2D_ARRAY_V2I16_CLAMP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(N->getOperand(4)); Ops.push_back(Chain); break; case NVPTXISD::Suld2DArrayV2I32Clamp: Opc = NVPTX::SULD_2D_ARRAY_V2I32_CLAMP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(N->getOperand(4)); Ops.push_back(Chain); break; case NVPTXISD::Suld2DArrayV2I64Clamp: Opc = NVPTX::SULD_2D_ARRAY_V2I64_CLAMP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(N->getOperand(4)); Ops.push_back(Chain); break; case NVPTXISD::Suld2DArrayV4I8Clamp: Opc = NVPTX::SULD_2D_ARRAY_V4I8_CLAMP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(N->getOperand(4)); Ops.push_back(Chain); break; case NVPTXISD::Suld2DArrayV4I16Clamp: Opc = NVPTX::SULD_2D_ARRAY_V4I16_CLAMP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(N->getOperand(4)); Ops.push_back(Chain); break; case NVPTXISD::Suld2DArrayV4I32Clamp: Opc = NVPTX::SULD_2D_ARRAY_V4I32_CLAMP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(N->getOperand(4)); Ops.push_back(Chain); break; case NVPTXISD::Suld3DI8Clamp: Opc = NVPTX::SULD_3D_I8_CLAMP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(N->getOperand(4)); Ops.push_back(Chain); break; case NVPTXISD::Suld3DI16Clamp: Opc = NVPTX::SULD_3D_I16_CLAMP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(N->getOperand(4)); Ops.push_back(Chain); break; case NVPTXISD::Suld3DI32Clamp: Opc = NVPTX::SULD_3D_I32_CLAMP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(N->getOperand(4)); Ops.push_back(Chain); break; case NVPTXISD::Suld3DI64Clamp: Opc = NVPTX::SULD_3D_I64_CLAMP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(N->getOperand(4)); Ops.push_back(Chain); break; case NVPTXISD::Suld3DV2I8Clamp: Opc = NVPTX::SULD_3D_V2I8_CLAMP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(N->getOperand(4)); Ops.push_back(Chain); break; case NVPTXISD::Suld3DV2I16Clamp: Opc = NVPTX::SULD_3D_V2I16_CLAMP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(N->getOperand(4)); Ops.push_back(Chain); break; case NVPTXISD::Suld3DV2I32Clamp: Opc = NVPTX::SULD_3D_V2I32_CLAMP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(N->getOperand(4)); Ops.push_back(Chain); break; case NVPTXISD::Suld3DV2I64Clamp: Opc = NVPTX::SULD_3D_V2I64_CLAMP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(N->getOperand(4)); Ops.push_back(Chain); break; case NVPTXISD::Suld3DV4I8Clamp: Opc = NVPTX::SULD_3D_V4I8_CLAMP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(N->getOperand(4)); Ops.push_back(Chain); break; case NVPTXISD::Suld3DV4I16Clamp: Opc = NVPTX::SULD_3D_V4I16_CLAMP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(N->getOperand(4)); Ops.push_back(Chain); break; case NVPTXISD::Suld3DV4I32Clamp: Opc = NVPTX::SULD_3D_V4I32_CLAMP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(N->getOperand(4)); Ops.push_back(Chain); break; case NVPTXISD::Suld1DI8Trap: Opc = NVPTX::SULD_1D_I8_TRAP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(Chain); break; case NVPTXISD::Suld1DI16Trap: Opc = NVPTX::SULD_1D_I16_TRAP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(Chain); break; case NVPTXISD::Suld1DI32Trap: Opc = NVPTX::SULD_1D_I32_TRAP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(Chain); break; case NVPTXISD::Suld1DI64Trap: Opc = NVPTX::SULD_1D_I64_TRAP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(Chain); break; case NVPTXISD::Suld1DV2I8Trap: Opc = NVPTX::SULD_1D_V2I8_TRAP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(Chain); break; case NVPTXISD::Suld1DV2I16Trap: Opc = NVPTX::SULD_1D_V2I16_TRAP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(Chain); break; case NVPTXISD::Suld1DV2I32Trap: Opc = NVPTX::SULD_1D_V2I32_TRAP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(Chain); break; case NVPTXISD::Suld1DV2I64Trap: Opc = NVPTX::SULD_1D_V2I64_TRAP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(Chain); break; case NVPTXISD::Suld1DV4I8Trap: Opc = NVPTX::SULD_1D_V4I8_TRAP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(Chain); break; case NVPTXISD::Suld1DV4I16Trap: Opc = NVPTX::SULD_1D_V4I16_TRAP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(Chain); break; case NVPTXISD::Suld1DV4I32Trap: Opc = NVPTX::SULD_1D_V4I32_TRAP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(Chain); break; case NVPTXISD::Suld1DArrayI8Trap: Opc = NVPTX::SULD_1D_ARRAY_I8_TRAP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(Chain); break; case NVPTXISD::Suld1DArrayI16Trap: Opc = NVPTX::SULD_1D_ARRAY_I16_TRAP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(Chain); break; case NVPTXISD::Suld1DArrayI32Trap: Opc = NVPTX::SULD_1D_ARRAY_I32_TRAP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(Chain); break; case NVPTXISD::Suld1DArrayI64Trap: Opc = NVPTX::SULD_1D_ARRAY_I64_TRAP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(Chain); break; case NVPTXISD::Suld1DArrayV2I8Trap: Opc = NVPTX::SULD_1D_ARRAY_V2I8_TRAP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(Chain); break; case NVPTXISD::Suld1DArrayV2I16Trap: Opc = NVPTX::SULD_1D_ARRAY_V2I16_TRAP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(Chain); break; case NVPTXISD::Suld1DArrayV2I32Trap: Opc = NVPTX::SULD_1D_ARRAY_V2I32_TRAP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(Chain); break; case NVPTXISD::Suld1DArrayV2I64Trap: Opc = NVPTX::SULD_1D_ARRAY_V2I64_TRAP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(Chain); break; case NVPTXISD::Suld1DArrayV4I8Trap: Opc = NVPTX::SULD_1D_ARRAY_V4I8_TRAP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(Chain); break; case NVPTXISD::Suld1DArrayV4I16Trap: Opc = NVPTX::SULD_1D_ARRAY_V4I16_TRAP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(Chain); break; case NVPTXISD::Suld1DArrayV4I32Trap: Opc = NVPTX::SULD_1D_ARRAY_V4I32_TRAP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(Chain); break; case NVPTXISD::Suld2DI8Trap: Opc = NVPTX::SULD_2D_I8_TRAP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(Chain); break; case NVPTXISD::Suld2DI16Trap: Opc = NVPTX::SULD_2D_I16_TRAP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(Chain); break; case NVPTXISD::Suld2DI32Trap: Opc = NVPTX::SULD_2D_I32_TRAP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(Chain); break; case NVPTXISD::Suld2DI64Trap: Opc = NVPTX::SULD_2D_I64_TRAP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(Chain); break; case NVPTXISD::Suld2DV2I8Trap: Opc = NVPTX::SULD_2D_V2I8_TRAP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(Chain); break; case NVPTXISD::Suld2DV2I16Trap: Opc = NVPTX::SULD_2D_V2I16_TRAP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(Chain); break; case NVPTXISD::Suld2DV2I32Trap: Opc = NVPTX::SULD_2D_V2I32_TRAP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(Chain); break; case NVPTXISD::Suld2DV2I64Trap: Opc = NVPTX::SULD_2D_V2I64_TRAP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(Chain); break; case NVPTXISD::Suld2DV4I8Trap: Opc = NVPTX::SULD_2D_V4I8_TRAP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(Chain); break; case NVPTXISD::Suld2DV4I16Trap: Opc = NVPTX::SULD_2D_V4I16_TRAP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(Chain); break; case NVPTXISD::Suld2DV4I32Trap: Opc = NVPTX::SULD_2D_V4I32_TRAP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(Chain); break; case NVPTXISD::Suld2DArrayI8Trap: Opc = NVPTX::SULD_2D_ARRAY_I8_TRAP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(N->getOperand(4)); Ops.push_back(Chain); break; case NVPTXISD::Suld2DArrayI16Trap: Opc = NVPTX::SULD_2D_ARRAY_I16_TRAP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(N->getOperand(4)); Ops.push_back(Chain); break; case NVPTXISD::Suld2DArrayI32Trap: Opc = NVPTX::SULD_2D_ARRAY_I32_TRAP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(N->getOperand(4)); Ops.push_back(Chain); break; case NVPTXISD::Suld2DArrayI64Trap: Opc = NVPTX::SULD_2D_ARRAY_I64_TRAP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(N->getOperand(4)); Ops.push_back(Chain); break; case NVPTXISD::Suld2DArrayV2I8Trap: Opc = NVPTX::SULD_2D_ARRAY_V2I8_TRAP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(N->getOperand(4)); Ops.push_back(Chain); break; case NVPTXISD::Suld2DArrayV2I16Trap: Opc = NVPTX::SULD_2D_ARRAY_V2I16_TRAP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(N->getOperand(4)); Ops.push_back(Chain); break; case NVPTXISD::Suld2DArrayV2I32Trap: Opc = NVPTX::SULD_2D_ARRAY_V2I32_TRAP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(N->getOperand(4)); Ops.push_back(Chain); break; case NVPTXISD::Suld2DArrayV2I64Trap: Opc = NVPTX::SULD_2D_ARRAY_V2I64_TRAP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(N->getOperand(4)); Ops.push_back(Chain); break; case NVPTXISD::Suld2DArrayV4I8Trap: Opc = NVPTX::SULD_2D_ARRAY_V4I8_TRAP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(N->getOperand(4)); Ops.push_back(Chain); break; case NVPTXISD::Suld2DArrayV4I16Trap: Opc = NVPTX::SULD_2D_ARRAY_V4I16_TRAP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(N->getOperand(4)); Ops.push_back(Chain); break; case NVPTXISD::Suld2DArrayV4I32Trap: Opc = NVPTX::SULD_2D_ARRAY_V4I32_TRAP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(N->getOperand(4)); Ops.push_back(Chain); break; case NVPTXISD::Suld3DI8Trap: Opc = NVPTX::SULD_3D_I8_TRAP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(N->getOperand(4)); Ops.push_back(Chain); break; case NVPTXISD::Suld3DI16Trap: Opc = NVPTX::SULD_3D_I16_TRAP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(N->getOperand(4)); Ops.push_back(Chain); break; case NVPTXISD::Suld3DI32Trap: Opc = NVPTX::SULD_3D_I32_TRAP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(N->getOperand(4)); Ops.push_back(Chain); break; case NVPTXISD::Suld3DI64Trap: Opc = NVPTX::SULD_3D_I64_TRAP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(N->getOperand(4)); Ops.push_back(Chain); break; case NVPTXISD::Suld3DV2I8Trap: Opc = NVPTX::SULD_3D_V2I8_TRAP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(N->getOperand(4)); Ops.push_back(Chain); break; case NVPTXISD::Suld3DV2I16Trap: Opc = NVPTX::SULD_3D_V2I16_TRAP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(N->getOperand(4)); Ops.push_back(Chain); break; case NVPTXISD::Suld3DV2I32Trap: Opc = NVPTX::SULD_3D_V2I32_TRAP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(N->getOperand(4)); Ops.push_back(Chain); break; case NVPTXISD::Suld3DV2I64Trap: Opc = NVPTX::SULD_3D_V2I64_TRAP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(N->getOperand(4)); Ops.push_back(Chain); break; case NVPTXISD::Suld3DV4I8Trap: Opc = NVPTX::SULD_3D_V4I8_TRAP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(N->getOperand(4)); Ops.push_back(Chain); break; case NVPTXISD::Suld3DV4I16Trap: Opc = NVPTX::SULD_3D_V4I16_TRAP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(N->getOperand(4)); Ops.push_back(Chain); break; case NVPTXISD::Suld3DV4I32Trap: Opc = NVPTX::SULD_3D_V4I32_TRAP; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(N->getOperand(4)); Ops.push_back(Chain); break; case NVPTXISD::Suld1DI8Zero: Opc = NVPTX::SULD_1D_I8_ZERO; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(Chain); break; case NVPTXISD::Suld1DI16Zero: Opc = NVPTX::SULD_1D_I16_ZERO; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(Chain); break; case NVPTXISD::Suld1DI32Zero: Opc = NVPTX::SULD_1D_I32_ZERO; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(Chain); break; case NVPTXISD::Suld1DI64Zero: Opc = NVPTX::SULD_1D_I64_ZERO; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(Chain); break; case NVPTXISD::Suld1DV2I8Zero: Opc = NVPTX::SULD_1D_V2I8_ZERO; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(Chain); break; case NVPTXISD::Suld1DV2I16Zero: Opc = NVPTX::SULD_1D_V2I16_ZERO; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(Chain); break; case NVPTXISD::Suld1DV2I32Zero: Opc = NVPTX::SULD_1D_V2I32_ZERO; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(Chain); break; case NVPTXISD::Suld1DV2I64Zero: Opc = NVPTX::SULD_1D_V2I64_ZERO; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(Chain); break; case NVPTXISD::Suld1DV4I8Zero: Opc = NVPTX::SULD_1D_V4I8_ZERO; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(Chain); break; case NVPTXISD::Suld1DV4I16Zero: Opc = NVPTX::SULD_1D_V4I16_ZERO; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(Chain); break; case NVPTXISD::Suld1DV4I32Zero: Opc = NVPTX::SULD_1D_V4I32_ZERO; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(Chain); break; case NVPTXISD::Suld1DArrayI8Zero: Opc = NVPTX::SULD_1D_ARRAY_I8_ZERO; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(Chain); break; case NVPTXISD::Suld1DArrayI16Zero: Opc = NVPTX::SULD_1D_ARRAY_I16_ZERO; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(Chain); break; case NVPTXISD::Suld1DArrayI32Zero: Opc = NVPTX::SULD_1D_ARRAY_I32_ZERO; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(Chain); break; case NVPTXISD::Suld1DArrayI64Zero: Opc = NVPTX::SULD_1D_ARRAY_I64_ZERO; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(Chain); break; case NVPTXISD::Suld1DArrayV2I8Zero: Opc = NVPTX::SULD_1D_ARRAY_V2I8_ZERO; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(Chain); break; case NVPTXISD::Suld1DArrayV2I16Zero: Opc = NVPTX::SULD_1D_ARRAY_V2I16_ZERO; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(Chain); break; case NVPTXISD::Suld1DArrayV2I32Zero: Opc = NVPTX::SULD_1D_ARRAY_V2I32_ZERO; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(Chain); break; case NVPTXISD::Suld1DArrayV2I64Zero: Opc = NVPTX::SULD_1D_ARRAY_V2I64_ZERO; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(Chain); break; case NVPTXISD::Suld1DArrayV4I8Zero: Opc = NVPTX::SULD_1D_ARRAY_V4I8_ZERO; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(Chain); break; case NVPTXISD::Suld1DArrayV4I16Zero: Opc = NVPTX::SULD_1D_ARRAY_V4I16_ZERO; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(Chain); break; case NVPTXISD::Suld1DArrayV4I32Zero: Opc = NVPTX::SULD_1D_ARRAY_V4I32_ZERO; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(Chain); break; case NVPTXISD::Suld2DI8Zero: Opc = NVPTX::SULD_2D_I8_ZERO; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(Chain); break; case NVPTXISD::Suld2DI16Zero: Opc = NVPTX::SULD_2D_I16_ZERO; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(Chain); break; case NVPTXISD::Suld2DI32Zero: Opc = NVPTX::SULD_2D_I32_ZERO; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(Chain); break; case NVPTXISD::Suld2DI64Zero: Opc = NVPTX::SULD_2D_I64_ZERO; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(Chain); break; case NVPTXISD::Suld2DV2I8Zero: Opc = NVPTX::SULD_2D_V2I8_ZERO; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(Chain); break; case NVPTXISD::Suld2DV2I16Zero: Opc = NVPTX::SULD_2D_V2I16_ZERO; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(Chain); break; case NVPTXISD::Suld2DV2I32Zero: Opc = NVPTX::SULD_2D_V2I32_ZERO; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(Chain); break; case NVPTXISD::Suld2DV2I64Zero: Opc = NVPTX::SULD_2D_V2I64_ZERO; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(Chain); break; case NVPTXISD::Suld2DV4I8Zero: Opc = NVPTX::SULD_2D_V4I8_ZERO; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(Chain); break; case NVPTXISD::Suld2DV4I16Zero: Opc = NVPTX::SULD_2D_V4I16_ZERO; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(Chain); break; case NVPTXISD::Suld2DV4I32Zero: Opc = NVPTX::SULD_2D_V4I32_ZERO; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(Chain); break; case NVPTXISD::Suld2DArrayI8Zero: Opc = NVPTX::SULD_2D_ARRAY_I8_ZERO; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(N->getOperand(4)); Ops.push_back(Chain); break; case NVPTXISD::Suld2DArrayI16Zero: Opc = NVPTX::SULD_2D_ARRAY_I16_ZERO; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(N->getOperand(4)); Ops.push_back(Chain); break; case NVPTXISD::Suld2DArrayI32Zero: Opc = NVPTX::SULD_2D_ARRAY_I32_ZERO; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(N->getOperand(4)); Ops.push_back(Chain); break; case NVPTXISD::Suld2DArrayI64Zero: Opc = NVPTX::SULD_2D_ARRAY_I64_ZERO; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(N->getOperand(4)); Ops.push_back(Chain); break; case NVPTXISD::Suld2DArrayV2I8Zero: Opc = NVPTX::SULD_2D_ARRAY_V2I8_ZERO; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(N->getOperand(4)); Ops.push_back(Chain); break; case NVPTXISD::Suld2DArrayV2I16Zero: Opc = NVPTX::SULD_2D_ARRAY_V2I16_ZERO; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(N->getOperand(4)); Ops.push_back(Chain); break; case NVPTXISD::Suld2DArrayV2I32Zero: Opc = NVPTX::SULD_2D_ARRAY_V2I32_ZERO; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(N->getOperand(4)); Ops.push_back(Chain); break; case NVPTXISD::Suld2DArrayV2I64Zero: Opc = NVPTX::SULD_2D_ARRAY_V2I64_ZERO; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(N->getOperand(4)); Ops.push_back(Chain); break; case NVPTXISD::Suld2DArrayV4I8Zero: Opc = NVPTX::SULD_2D_ARRAY_V4I8_ZERO; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(N->getOperand(4)); Ops.push_back(Chain); break; case NVPTXISD::Suld2DArrayV4I16Zero: Opc = NVPTX::SULD_2D_ARRAY_V4I16_ZERO; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(N->getOperand(4)); Ops.push_back(Chain); break; case NVPTXISD::Suld2DArrayV4I32Zero: Opc = NVPTX::SULD_2D_ARRAY_V4I32_ZERO; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(N->getOperand(4)); Ops.push_back(Chain); break; case NVPTXISD::Suld3DI8Zero: Opc = NVPTX::SULD_3D_I8_ZERO; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(N->getOperand(4)); Ops.push_back(Chain); break; case NVPTXISD::Suld3DI16Zero: Opc = NVPTX::SULD_3D_I16_ZERO; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(N->getOperand(4)); Ops.push_back(Chain); break; case NVPTXISD::Suld3DI32Zero: Opc = NVPTX::SULD_3D_I32_ZERO; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(N->getOperand(4)); Ops.push_back(Chain); break; case NVPTXISD::Suld3DI64Zero: Opc = NVPTX::SULD_3D_I64_ZERO; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(N->getOperand(4)); Ops.push_back(Chain); break; case NVPTXISD::Suld3DV2I8Zero: Opc = NVPTX::SULD_3D_V2I8_ZERO; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(N->getOperand(4)); Ops.push_back(Chain); break; case NVPTXISD::Suld3DV2I16Zero: Opc = NVPTX::SULD_3D_V2I16_ZERO; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(N->getOperand(4)); Ops.push_back(Chain); break; case NVPTXISD::Suld3DV2I32Zero: Opc = NVPTX::SULD_3D_V2I32_ZERO; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(N->getOperand(4)); Ops.push_back(Chain); break; case NVPTXISD::Suld3DV2I64Zero: Opc = NVPTX::SULD_3D_V2I64_ZERO; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(N->getOperand(4)); Ops.push_back(Chain); break; case NVPTXISD::Suld3DV4I8Zero: Opc = NVPTX::SULD_3D_V4I8_ZERO; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(N->getOperand(4)); Ops.push_back(Chain); break; case NVPTXISD::Suld3DV4I16Zero: Opc = NVPTX::SULD_3D_V4I16_ZERO; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(N->getOperand(4)); Ops.push_back(Chain); break; case NVPTXISD::Suld3DV4I32Zero: Opc = NVPTX::SULD_3D_V4I32_ZERO; Ops.push_back(TexHandle); Ops.push_back(N->getOperand(2)); Ops.push_back(N->getOperand(3)); Ops.push_back(N->getOperand(4)); Ops.push_back(Chain); break; } Ret = CurDAG->getMachineNode(Opc, SDLoc(N), N->getVTList(), Ops); return Ret; } /// SelectBFE - Look for instruction sequences that can be made more efficient /// by using the 'bfe' (bit-field extract) PTX instruction SDNode *NVPTXDAGToDAGISel::SelectBFE(SDNode *N) { SDValue LHS = N->getOperand(0); SDValue RHS = N->getOperand(1); SDValue Len; SDValue Start; SDValue Val; bool IsSigned = false; if (N->getOpcode() == ISD::AND) { // Canonicalize the operands // We want 'and %val, %mask' if (isa<ConstantSDNode>(LHS) && !isa<ConstantSDNode>(RHS)) { std::swap(LHS, RHS); } ConstantSDNode *Mask = dyn_cast<ConstantSDNode>(RHS); if (!Mask) { // We need a constant mask on the RHS of the AND return NULL; } // Extract the mask bits uint64_t MaskVal = Mask->getZExtValue(); if (!isMask_64(MaskVal)) { // We *could* handle shifted masks here, but doing so would require an // 'and' operation to fix up the low-order bits so we would trade // shr+and for bfe+and, which has the same throughput return NULL; } // How many bits are in our mask? uint64_t NumBits = countTrailingOnes(MaskVal); Len = CurDAG->getTargetConstant(NumBits, MVT::i32); if (LHS.getOpcode() == ISD::SRL || LHS.getOpcode() == ISD::SRA) { // We have a 'srl/and' pair, extract the effective start bit and length Val = LHS.getNode()->getOperand(0); Start = LHS.getNode()->getOperand(1); ConstantSDNode *StartConst = dyn_cast<ConstantSDNode>(Start); if (StartConst) { uint64_t StartVal = StartConst->getZExtValue(); // How many "good" bits do we have left? "good" is defined here as bits // that exist in the original value, not shifted in. uint64_t GoodBits = Start.getValueType().getSizeInBits() - StartVal; if (NumBits > GoodBits) { // Do not handle the case where bits have been shifted in. In theory // we could handle this, but the cost is likely higher than just // emitting the srl/and pair. return NULL; } Start = CurDAG->getTargetConstant(StartVal, MVT::i32); } else { // Do not handle the case where the shift amount (can be zero if no srl // was found) is not constant. We could handle this case, but it would // require run-time logic that would be more expensive than just // emitting the srl/and pair. return NULL; } } else { // Do not handle the case where the LHS of the and is not a shift. While // it would be trivial to handle this case, it would just transform // 'and' -> 'bfe', but 'and' has higher-throughput. return NULL; } } else if (N->getOpcode() == ISD::SRL || N->getOpcode() == ISD::SRA) { if (LHS->getOpcode() == ISD::AND) { ConstantSDNode *ShiftCnst = dyn_cast<ConstantSDNode>(RHS); if (!ShiftCnst) { // Shift amount must be constant return NULL; } uint64_t ShiftAmt = ShiftCnst->getZExtValue(); SDValue AndLHS = LHS->getOperand(0); SDValue AndRHS = LHS->getOperand(1); // Canonicalize the AND to have the mask on the RHS if (isa<ConstantSDNode>(AndLHS)) { std::swap(AndLHS, AndRHS); } ConstantSDNode *MaskCnst = dyn_cast<ConstantSDNode>(AndRHS); if (!MaskCnst) { // Mask must be constant return NULL; } uint64_t MaskVal = MaskCnst->getZExtValue(); uint64_t NumZeros; uint64_t NumBits; if (isMask_64(MaskVal)) { NumZeros = 0; // The number of bits in the result bitfield will be the number of // trailing ones (the AND) minus the number of bits we shift off NumBits = countTrailingOnes(MaskVal) - ShiftAmt; } else if (isShiftedMask_64(MaskVal)) { NumZeros = countTrailingZeros(MaskVal); unsigned NumOnes = countTrailingOnes(MaskVal >> NumZeros); // The number of bits in the result bitfield will be the number of // trailing zeros plus the number of set bits in the mask minus the // number of bits we shift off NumBits = NumZeros + NumOnes - ShiftAmt; } else { // This is not a mask we can handle return NULL; } if (ShiftAmt < NumZeros) { // Handling this case would require extra logic that would make this // transformation non-profitable return NULL; } Val = AndLHS; Start = CurDAG->getTargetConstant(ShiftAmt, MVT::i32); Len = CurDAG->getTargetConstant(NumBits, MVT::i32); } else if (LHS->getOpcode() == ISD::SHL) { // Here, we have a pattern like: // // (sra (shl val, NN), MM) // or // (srl (shl val, NN), MM) // // If MM >= NN, we can efficiently optimize this with bfe Val = LHS->getOperand(0); SDValue ShlRHS = LHS->getOperand(1); ConstantSDNode *ShlCnst = dyn_cast<ConstantSDNode>(ShlRHS); if (!ShlCnst) { // Shift amount must be constant return NULL; } uint64_t InnerShiftAmt = ShlCnst->getZExtValue(); SDValue ShrRHS = RHS; ConstantSDNode *ShrCnst = dyn_cast<ConstantSDNode>(ShrRHS); if (!ShrCnst) { // Shift amount must be constant return NULL; } uint64_t OuterShiftAmt = ShrCnst->getZExtValue(); // To avoid extra codegen and be profitable, we need Outer >= Inner if (OuterShiftAmt < InnerShiftAmt) { return NULL; } // If the outer shift is more than the type size, we have no bitfield to // extract (since we also check that the inner shift is <= the outer shift // then this also implies that the inner shift is < the type size) if (OuterShiftAmt >= Val.getValueType().getSizeInBits()) { return NULL; } Start = CurDAG->getTargetConstant(OuterShiftAmt - InnerShiftAmt, MVT::i32); Len = CurDAG->getTargetConstant(Val.getValueType().getSizeInBits() - OuterShiftAmt, MVT::i32); if (N->getOpcode() == ISD::SRA) { // If we have a arithmetic right shift, we need to use the signed bfe // variant IsSigned = true; } } else { // No can do... return NULL; } } else { // No can do... return NULL; } unsigned Opc; // For the BFE operations we form here from "and" and "srl", always use the // unsigned variants. if (Val.getValueType() == MVT::i32) { if (IsSigned) { Opc = NVPTX::BFE_S32rii; } else { Opc = NVPTX::BFE_U32rii; } } else if (Val.getValueType() == MVT::i64) { if (IsSigned) { Opc = NVPTX::BFE_S64rii; } else { Opc = NVPTX::BFE_U64rii; } } else { // We cannot handle this type return NULL; } SDValue Ops[] = { Val, Start, Len }; SDNode *Ret = CurDAG->getMachineNode(Opc, SDLoc(N), N->getVTList(), Ops); return Ret; } // SelectDirectAddr - Match a direct address for DAG. // A direct address could be a globaladdress or externalsymbol. bool NVPTXDAGToDAGISel::SelectDirectAddr(SDValue N, SDValue &Address) { // Return true if TGA or ES. if (N.getOpcode() == ISD::TargetGlobalAddress || N.getOpcode() == ISD::TargetExternalSymbol) { Address = N; return true; } if (N.getOpcode() == NVPTXISD::Wrapper) { Address = N.getOperand(0); return true; } if (N.getOpcode() == ISD::INTRINSIC_WO_CHAIN) { unsigned IID = cast<ConstantSDNode>(N.getOperand(0))->getZExtValue(); if (IID == Intrinsic::nvvm_ptr_gen_to_param) if (N.getOperand(1).getOpcode() == NVPTXISD::MoveParam) return (SelectDirectAddr(N.getOperand(1).getOperand(0), Address)); } return false; } // symbol+offset bool NVPTXDAGToDAGISel::SelectADDRsi_imp( SDNode *OpNode, SDValue Addr, SDValue &Base, SDValue &Offset, MVT mvt) { if (Addr.getOpcode() == ISD::ADD) { if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Addr.getOperand(1))) { SDValue base = Addr.getOperand(0); if (SelectDirectAddr(base, Base)) { Offset = CurDAG->getTargetConstant(CN->getZExtValue(), mvt); return true; } } } return false; } // symbol+offset bool NVPTXDAGToDAGISel::SelectADDRsi(SDNode *OpNode, SDValue Addr, SDValue &Base, SDValue &Offset) { return SelectADDRsi_imp(OpNode, Addr, Base, Offset, MVT::i32); } // symbol+offset bool NVPTXDAGToDAGISel::SelectADDRsi64(SDNode *OpNode, SDValue Addr, SDValue &Base, SDValue &Offset) { return SelectADDRsi_imp(OpNode, Addr, Base, Offset, MVT::i64); } // register+offset bool NVPTXDAGToDAGISel::SelectADDRri_imp( SDNode *OpNode, SDValue Addr, SDValue &Base, SDValue &Offset, MVT mvt) { if (FrameIndexSDNode *FIN = dyn_cast<FrameIndexSDNode>(Addr)) { Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), mvt); Offset = CurDAG->getTargetConstant(0, mvt); return true; } if (Addr.getOpcode() == ISD::TargetExternalSymbol || Addr.getOpcode() == ISD::TargetGlobalAddress) return false; // direct calls. if (Addr.getOpcode() == ISD::ADD) { if (SelectDirectAddr(Addr.getOperand(0), Addr)) { return false; } if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Addr.getOperand(1))) { if (FrameIndexSDNode *FIN = dyn_cast<FrameIndexSDNode>(Addr.getOperand(0))) // Constant offset from frame ref. Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), mvt); else Base = Addr.getOperand(0); Offset = CurDAG->getTargetConstant(CN->getZExtValue(), mvt); return true; } } return false; } // register+offset bool NVPTXDAGToDAGISel::SelectADDRri(SDNode *OpNode, SDValue Addr, SDValue &Base, SDValue &Offset) { return SelectADDRri_imp(OpNode, Addr, Base, Offset, MVT::i32); } // register+offset bool NVPTXDAGToDAGISel::SelectADDRri64(SDNode *OpNode, SDValue Addr, SDValue &Base, SDValue &Offset) { return SelectADDRri_imp(OpNode, Addr, Base, Offset, MVT::i64); } bool NVPTXDAGToDAGISel::ChkMemSDNodeAddressSpace(SDNode *N, unsigned int spN) const { const Value *Src = nullptr; if (MemSDNode *mN = dyn_cast<MemSDNode>(N)) { if (spN == 0 && mN->getMemOperand()->getPseudoValue()) return true; Src = mN->getMemOperand()->getValue(); } if (!Src) return false; if (const PointerType *PT = dyn_cast<PointerType>(Src->getType())) return (PT->getAddressSpace() == spN); return false; } /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for /// inline asm expressions. bool NVPTXDAGToDAGISel::SelectInlineAsmMemoryOperand( const SDValue &Op, char ConstraintCode, std::vector<SDValue> &OutOps) { SDValue Op0, Op1; switch (ConstraintCode) { default: return true; case 'm': // memory if (SelectDirectAddr(Op, Op0)) { OutOps.push_back(Op0); OutOps.push_back(CurDAG->getTargetConstant(0, MVT::i32)); return false; } if (SelectADDRri(Op.getNode(), Op, Op0, Op1)) { OutOps.push_back(Op0); OutOps.push_back(Op1); return false; } break; } return true; }