Mercurial > hg > Members > tobaru > cbc > CbC_llvm

comparison lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp @ 77:54457678186b

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
LLVM 3.6
author Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp>
date Mon, 08 Sep 2014 22:06:00 +0900
parents e4204d083e25
children 60c9769439b8
comparison
equal deleted inserted replaced
34:e874dbf0ad9d 77:54457678186b
18 #include "llvm/Support/Debug.h" 18 #include "llvm/Support/Debug.h"
19 #include "llvm/Support/ErrorHandling.h" 19 #include "llvm/Support/ErrorHandling.h"
20 #include "llvm/Support/raw_ostream.h" 20 #include "llvm/Support/raw_ostream.h"
21 #include "llvm/Target/TargetIntrinsicInfo.h" 21 #include "llvm/Target/TargetIntrinsicInfo.h"
22 22
23 #undef DEBUG_TYPE 23 using namespace llvm;
24
24 #define DEBUG_TYPE "nvptx-isel" 25 #define DEBUG_TYPE "nvptx-isel"
25
26 using namespace llvm;
27
28 static cl::opt<int>
29 FMAContractLevel("nvptx-fma-level", cl::ZeroOrMore, cl::Hidden,
30 cl::desc("NVPTX Specific: FMA contraction (0: don't do it"
31 " 1: do it 2: do it aggressively"),
32 cl::init(2));
33 26
34 static cl::opt<int> UsePrecDivF32( 27 static cl::opt<int> UsePrecDivF32(
35 "nvptx-prec-divf32", cl::ZeroOrMore, cl::Hidden, 28 "nvptx-prec-divf32", cl::ZeroOrMore, cl::Hidden,
36 cl::desc("NVPTX Specifies: 0 use div.approx, 1 use div.full, 2 use" 29 cl::desc("NVPTX Specifies: 0 use div.approx, 1 use div.full, 2 use"
37 " IEEE Compliant F32 div.rnd if avaiable."), 30 " IEEE Compliant F32 div.rnd if available."),
38 cl::init(2)); 31 cl::init(2));
39 32
40 static cl::opt<bool> 33 static cl::opt<bool>
41 UsePrecSqrtF32("nvptx-prec-sqrtf32", cl::Hidden, 34 UsePrecSqrtF32("nvptx-prec-sqrtf32", cl::Hidden,
42 cl::desc("NVPTX Specific: 0 use sqrt.approx, 1 use sqrt.rn."), 35 cl::desc("NVPTX Specific: 0 use sqrt.approx, 1 use sqrt.rn."),
57 50
58 NVPTXDAGToDAGISel::NVPTXDAGToDAGISel(NVPTXTargetMachine &tm, 51 NVPTXDAGToDAGISel::NVPTXDAGToDAGISel(NVPTXTargetMachine &tm,
59 CodeGenOpt::Level OptLevel) 52 CodeGenOpt::Level OptLevel)
60 : SelectionDAGISel(tm, OptLevel), 53 : SelectionDAGISel(tm, OptLevel),
61 Subtarget(tm.getSubtarget<NVPTXSubtarget>()) { 54 Subtarget(tm.getSubtarget<NVPTXSubtarget>()) {
62
63 doFMAF32 = (OptLevel > 0) && Subtarget.hasFMAF32() && (FMAContractLevel >= 1);
64 doFMAF64 = (OptLevel > 0) && Subtarget.hasFMAF64() && (FMAContractLevel >= 1);
65 doFMAF32AGG =
66 (OptLevel > 0) && Subtarget.hasFMAF32() && (FMAContractLevel == 2);
67 doFMAF64AGG =
68 (OptLevel > 0) && Subtarget.hasFMAF64() && (FMAContractLevel == 2);
69
70 allowFMA = (FMAContractLevel >= 1);
71
72 doMulWide = (OptLevel > 0); 55 doMulWide = (OptLevel > 0);
73 } 56 }
74 57
75 int NVPTXDAGToDAGISel::getDivF32Level() const { 58 int NVPTXDAGToDAGISel::getDivF32Level() const {
76 if (UsePrecDivF32.getNumOccurrences() > 0) { 59 if (UsePrecDivF32.getNumOccurrences() > 0) {
112 else 95 else
113 return false; 96 return false;
114 } 97 }
115 } 98 }
116 99
100 bool NVPTXDAGToDAGISel::allowFMA() const {
101 const NVPTXTargetLowering *TL = Subtarget.getTargetLowering();
102 return TL->allowFMA(*MF, OptLevel);
103 }
104
117 /// Select - Select instructions not customized! Used for 105 /// Select - Select instructions not customized! Used for
118 /// expanded, promoted and normal instructions. 106 /// expanded, promoted and normal instructions.
119 SDNode *NVPTXDAGToDAGISel::Select(SDNode *N) { 107 SDNode *NVPTXDAGToDAGISel::Select(SDNode *N) {
120 108
121 if (N->isMachineOpcode()) { 109 if (N->isMachineOpcode()) {
122 N->setNodeId(-1); 110 N->setNodeId(-1);
123 return NULL; // Already selected. 111 return nullptr; // Already selected.
124 } 112 }
125 113
126 SDNode *ResNode = NULL; 114 SDNode *ResNode = nullptr;
127 switch (N->getOpcode()) { 115 switch (N->getOpcode()) {
128 case ISD::LOAD: 116 case ISD::LOAD:
129 ResNode = SelectLoad(N); 117 ResNode = SelectLoad(N);
130 break; 118 break;
131 case ISD::STORE: 119 case ISD::STORE:
137 break; 125 break;
138 case NVPTXISD::LDGV2: 126 case NVPTXISD::LDGV2:
139 case NVPTXISD::LDGV4: 127 case NVPTXISD::LDGV4:
140 case NVPTXISD::LDUV2: 128 case NVPTXISD::LDUV2:
141 case NVPTXISD::LDUV4: 129 case NVPTXISD::LDUV4:
142 ResNode = SelectLDGLDUVector(N); 130 ResNode = SelectLDGLDU(N);
143 break; 131 break;
144 case NVPTXISD::StoreV2: 132 case NVPTXISD::StoreV2:
145 case NVPTXISD::StoreV4: 133 case NVPTXISD::StoreV4:
146 ResNode = SelectStoreVector(N); 134 ResNode = SelectStoreVector(N);
147 break; 135 break;
160 case NVPTXISD::StoreParamV4: 148 case NVPTXISD::StoreParamV4:
161 case NVPTXISD::StoreParamS32: 149 case NVPTXISD::StoreParamS32:
162 case NVPTXISD::StoreParamU32: 150 case NVPTXISD::StoreParamU32:
163 ResNode = SelectStoreParam(N); 151 ResNode = SelectStoreParam(N);
164 break; 152 break;
153 case ISD::INTRINSIC_WO_CHAIN:
154 ResNode = SelectIntrinsicNoChain(N);
155 break;
156 case ISD::INTRINSIC_W_CHAIN:
157 ResNode = SelectIntrinsicChain(N);
158 break;
159 case NVPTXISD::Tex1DFloatS32:
160 case NVPTXISD::Tex1DFloatFloat:
161 case NVPTXISD::Tex1DFloatFloatLevel:
162 case NVPTXISD::Tex1DFloatFloatGrad:
163 case NVPTXISD::Tex1DS32S32:
164 case NVPTXISD::Tex1DS32Float:
165 case NVPTXISD::Tex1DS32FloatLevel:
166 case NVPTXISD::Tex1DS32FloatGrad:
167 case NVPTXISD::Tex1DU32S32:
168 case NVPTXISD::Tex1DU32Float:
169 case NVPTXISD::Tex1DU32FloatLevel:
170 case NVPTXISD::Tex1DU32FloatGrad:
171 case NVPTXISD::Tex1DArrayFloatS32:
172 case NVPTXISD::Tex1DArrayFloatFloat:
173 case NVPTXISD::Tex1DArrayFloatFloatLevel:
174 case NVPTXISD::Tex1DArrayFloatFloatGrad:
175 case NVPTXISD::Tex1DArrayS32S32:
176 case NVPTXISD::Tex1DArrayS32Float:
177 case NVPTXISD::Tex1DArrayS32FloatLevel:
178 case NVPTXISD::Tex1DArrayS32FloatGrad:
179 case NVPTXISD::Tex1DArrayU32S32:
180 case NVPTXISD::Tex1DArrayU32Float:
181 case NVPTXISD::Tex1DArrayU32FloatLevel:
182 case NVPTXISD::Tex1DArrayU32FloatGrad:
183 case NVPTXISD::Tex2DFloatS32:
184 case NVPTXISD::Tex2DFloatFloat:
185 case NVPTXISD::Tex2DFloatFloatLevel:
186 case NVPTXISD::Tex2DFloatFloatGrad:
187 case NVPTXISD::Tex2DS32S32:
188 case NVPTXISD::Tex2DS32Float:
189 case NVPTXISD::Tex2DS32FloatLevel:
190 case NVPTXISD::Tex2DS32FloatGrad:
191 case NVPTXISD::Tex2DU32S32:
192 case NVPTXISD::Tex2DU32Float:
193 case NVPTXISD::Tex2DU32FloatLevel:
194 case NVPTXISD::Tex2DU32FloatGrad:
195 case NVPTXISD::Tex2DArrayFloatS32:
196 case NVPTXISD::Tex2DArrayFloatFloat:
197 case NVPTXISD::Tex2DArrayFloatFloatLevel:
198 case NVPTXISD::Tex2DArrayFloatFloatGrad:
199 case NVPTXISD::Tex2DArrayS32S32:
200 case NVPTXISD::Tex2DArrayS32Float:
201 case NVPTXISD::Tex2DArrayS32FloatLevel:
202 case NVPTXISD::Tex2DArrayS32FloatGrad:
203 case NVPTXISD::Tex2DArrayU32S32:
204 case NVPTXISD::Tex2DArrayU32Float:
205 case NVPTXISD::Tex2DArrayU32FloatLevel:
206 case NVPTXISD::Tex2DArrayU32FloatGrad:
207 case NVPTXISD::Tex3DFloatS32:
208 case NVPTXISD::Tex3DFloatFloat:
209 case NVPTXISD::Tex3DFloatFloatLevel:
210 case NVPTXISD::Tex3DFloatFloatGrad:
211 case NVPTXISD::Tex3DS32S32:
212 case NVPTXISD::Tex3DS32Float:
213 case NVPTXISD::Tex3DS32FloatLevel:
214 case NVPTXISD::Tex3DS32FloatGrad:
215 case NVPTXISD::Tex3DU32S32:
216 case NVPTXISD::Tex3DU32Float:
217 case NVPTXISD::Tex3DU32FloatLevel:
218 case NVPTXISD::Tex3DU32FloatGrad:
219 case NVPTXISD::TexCubeFloatFloat:
220 case NVPTXISD::TexCubeFloatFloatLevel:
221 case NVPTXISD::TexCubeS32Float:
222 case NVPTXISD::TexCubeS32FloatLevel:
223 case NVPTXISD::TexCubeU32Float:
224 case NVPTXISD::TexCubeU32FloatLevel:
225 case NVPTXISD::TexCubeArrayFloatFloat:
226 case NVPTXISD::TexCubeArrayFloatFloatLevel:
227 case NVPTXISD::TexCubeArrayS32Float:
228 case NVPTXISD::TexCubeArrayS32FloatLevel:
229 case NVPTXISD::TexCubeArrayU32Float:
230 case NVPTXISD::TexCubeArrayU32FloatLevel:
231 case NVPTXISD::Tld4R2DFloatFloat:
232 case NVPTXISD::Tld4G2DFloatFloat:
233 case NVPTXISD::Tld4B2DFloatFloat:
234 case NVPTXISD::Tld4A2DFloatFloat:
235 case NVPTXISD::Tld4R2DS64Float:
236 case NVPTXISD::Tld4G2DS64Float:
237 case NVPTXISD::Tld4B2DS64Float:
238 case NVPTXISD::Tld4A2DS64Float:
239 case NVPTXISD::Tld4R2DU64Float:
240 case NVPTXISD::Tld4G2DU64Float:
241 case NVPTXISD::Tld4B2DU64Float:
242 case NVPTXISD::Tld4A2DU64Float:
243 case NVPTXISD::TexUnified1DFloatS32:
244 case NVPTXISD::TexUnified1DFloatFloat:
245 case NVPTXISD::TexUnified1DFloatFloatLevel:
246 case NVPTXISD::TexUnified1DFloatFloatGrad:
247 case NVPTXISD::TexUnified1DS32S32:
248 case NVPTXISD::TexUnified1DS32Float:
249 case NVPTXISD::TexUnified1DS32FloatLevel:
250 case NVPTXISD::TexUnified1DS32FloatGrad:
251 case NVPTXISD::TexUnified1DU32S32:
252 case NVPTXISD::TexUnified1DU32Float:
253 case NVPTXISD::TexUnified1DU32FloatLevel:
254 case NVPTXISD::TexUnified1DU32FloatGrad:
255 case NVPTXISD::TexUnified1DArrayFloatS32:
256 case NVPTXISD::TexUnified1DArrayFloatFloat:
257 case NVPTXISD::TexUnified1DArrayFloatFloatLevel:
258 case NVPTXISD::TexUnified1DArrayFloatFloatGrad:
259 case NVPTXISD::TexUnified1DArrayS32S32:
260 case NVPTXISD::TexUnified1DArrayS32Float:
261 case NVPTXISD::TexUnified1DArrayS32FloatLevel:
262 case NVPTXISD::TexUnified1DArrayS32FloatGrad:
263 case NVPTXISD::TexUnified1DArrayU32S32:
264 case NVPTXISD::TexUnified1DArrayU32Float:
265 case NVPTXISD::TexUnified1DArrayU32FloatLevel:
266 case NVPTXISD::TexUnified1DArrayU32FloatGrad:
267 case NVPTXISD::TexUnified2DFloatS32:
268 case NVPTXISD::TexUnified2DFloatFloat:
269 case NVPTXISD::TexUnified2DFloatFloatLevel:
270 case NVPTXISD::TexUnified2DFloatFloatGrad:
271 case NVPTXISD::TexUnified2DS32S32:
272 case NVPTXISD::TexUnified2DS32Float:
273 case NVPTXISD::TexUnified2DS32FloatLevel:
274 case NVPTXISD::TexUnified2DS32FloatGrad:
275 case NVPTXISD::TexUnified2DU32S32:
276 case NVPTXISD::TexUnified2DU32Float:
277 case NVPTXISD::TexUnified2DU32FloatLevel:
278 case NVPTXISD::TexUnified2DU32FloatGrad:
279 case NVPTXISD::TexUnified2DArrayFloatS32:
280 case NVPTXISD::TexUnified2DArrayFloatFloat:
281 case NVPTXISD::TexUnified2DArrayFloatFloatLevel:
282 case NVPTXISD::TexUnified2DArrayFloatFloatGrad:
283 case NVPTXISD::TexUnified2DArrayS32S32:
284 case NVPTXISD::TexUnified2DArrayS32Float:
285 case NVPTXISD::TexUnified2DArrayS32FloatLevel:
286 case NVPTXISD::TexUnified2DArrayS32FloatGrad:
287 case NVPTXISD::TexUnified2DArrayU32S32:
288 case NVPTXISD::TexUnified2DArrayU32Float:
289 case NVPTXISD::TexUnified2DArrayU32FloatLevel:
290 case NVPTXISD::TexUnified2DArrayU32FloatGrad:
291 case NVPTXISD::TexUnified3DFloatS32:
292 case NVPTXISD::TexUnified3DFloatFloat:
293 case NVPTXISD::TexUnified3DFloatFloatLevel:
294 case NVPTXISD::TexUnified3DFloatFloatGrad:
295 case NVPTXISD::TexUnified3DS32S32:
296 case NVPTXISD::TexUnified3DS32Float:
297 case NVPTXISD::TexUnified3DS32FloatLevel:
298 case NVPTXISD::TexUnified3DS32FloatGrad:
299 case NVPTXISD::TexUnified3DU32S32:
300 case NVPTXISD::TexUnified3DU32Float:
301 case NVPTXISD::TexUnified3DU32FloatLevel:
302 case NVPTXISD::TexUnified3DU32FloatGrad:
303 case NVPTXISD::TexUnifiedCubeFloatFloat:
304 case NVPTXISD::TexUnifiedCubeFloatFloatLevel:
305 case NVPTXISD::TexUnifiedCubeS32Float:
306 case NVPTXISD::TexUnifiedCubeS32FloatLevel:
307 case NVPTXISD::TexUnifiedCubeU32Float:
308 case NVPTXISD::TexUnifiedCubeU32FloatLevel:
309 case NVPTXISD::TexUnifiedCubeArrayFloatFloat:
310 case NVPTXISD::TexUnifiedCubeArrayFloatFloatLevel:
311 case NVPTXISD::TexUnifiedCubeArrayS32Float:
312 case NVPTXISD::TexUnifiedCubeArrayS32FloatLevel:
313 case NVPTXISD::TexUnifiedCubeArrayU32Float:
314 case NVPTXISD::TexUnifiedCubeArrayU32FloatLevel:
315 case NVPTXISD::Tld4UnifiedR2DFloatFloat:
316 case NVPTXISD::Tld4UnifiedG2DFloatFloat:
317 case NVPTXISD::Tld4UnifiedB2DFloatFloat:
318 case NVPTXISD::Tld4UnifiedA2DFloatFloat:
319 case NVPTXISD::Tld4UnifiedR2DS64Float:
320 case NVPTXISD::Tld4UnifiedG2DS64Float:
321 case NVPTXISD::Tld4UnifiedB2DS64Float:
322 case NVPTXISD::Tld4UnifiedA2DS64Float:
323 case NVPTXISD::Tld4UnifiedR2DU64Float:
324 case NVPTXISD::Tld4UnifiedG2DU64Float:
325 case NVPTXISD::Tld4UnifiedB2DU64Float:
326 case NVPTXISD::Tld4UnifiedA2DU64Float:
327 ResNode = SelectTextureIntrinsic(N);
328 break;
329 case NVPTXISD::Suld1DI8Clamp:
330 case NVPTXISD::Suld1DI16Clamp:
331 case NVPTXISD::Suld1DI32Clamp:
332 case NVPTXISD::Suld1DI64Clamp:
333 case NVPTXISD::Suld1DV2I8Clamp:
334 case NVPTXISD::Suld1DV2I16Clamp:
335 case NVPTXISD::Suld1DV2I32Clamp:
336 case NVPTXISD::Suld1DV2I64Clamp:
337 case NVPTXISD::Suld1DV4I8Clamp:
338 case NVPTXISD::Suld1DV4I16Clamp:
339 case NVPTXISD::Suld1DV4I32Clamp:
340 case NVPTXISD::Suld1DArrayI8Clamp:
341 case NVPTXISD::Suld1DArrayI16Clamp:
342 case NVPTXISD::Suld1DArrayI32Clamp:
343 case NVPTXISD::Suld1DArrayI64Clamp:
344 case NVPTXISD::Suld1DArrayV2I8Clamp:
345 case NVPTXISD::Suld1DArrayV2I16Clamp:
346 case NVPTXISD::Suld1DArrayV2I32Clamp:
347 case NVPTXISD::Suld1DArrayV2I64Clamp:
348 case NVPTXISD::Suld1DArrayV4I8Clamp:
349 case NVPTXISD::Suld1DArrayV4I16Clamp:
350 case NVPTXISD::Suld1DArrayV4I32Clamp:
351 case NVPTXISD::Suld2DI8Clamp:
352 case NVPTXISD::Suld2DI16Clamp:
353 case NVPTXISD::Suld2DI32Clamp:
354 case NVPTXISD::Suld2DI64Clamp:
355 case NVPTXISD::Suld2DV2I8Clamp:
356 case NVPTXISD::Suld2DV2I16Clamp:
357 case NVPTXISD::Suld2DV2I32Clamp:
358 case NVPTXISD::Suld2DV2I64Clamp:
359 case NVPTXISD::Suld2DV4I8Clamp:
360 case NVPTXISD::Suld2DV4I16Clamp:
361 case NVPTXISD::Suld2DV4I32Clamp:
362 case NVPTXISD::Suld2DArrayI8Clamp:
363 case NVPTXISD::Suld2DArrayI16Clamp:
364 case NVPTXISD::Suld2DArrayI32Clamp:
365 case NVPTXISD::Suld2DArrayI64Clamp:
366 case NVPTXISD::Suld2DArrayV2I8Clamp:
367 case NVPTXISD::Suld2DArrayV2I16Clamp:
368 case NVPTXISD::Suld2DArrayV2I32Clamp:
369 case NVPTXISD::Suld2DArrayV2I64Clamp:
370 case NVPTXISD::Suld2DArrayV4I8Clamp:
371 case NVPTXISD::Suld2DArrayV4I16Clamp:
372 case NVPTXISD::Suld2DArrayV4I32Clamp:
373 case NVPTXISD::Suld3DI8Clamp:
374 case NVPTXISD::Suld3DI16Clamp:
375 case NVPTXISD::Suld3DI32Clamp:
376 case NVPTXISD::Suld3DI64Clamp:
377 case NVPTXISD::Suld3DV2I8Clamp:
378 case NVPTXISD::Suld3DV2I16Clamp:
379 case NVPTXISD::Suld3DV2I32Clamp:
380 case NVPTXISD::Suld3DV2I64Clamp:
381 case NVPTXISD::Suld3DV4I8Clamp:
382 case NVPTXISD::Suld3DV4I16Clamp:
383 case NVPTXISD::Suld3DV4I32Clamp:
384 case NVPTXISD::Suld1DI8Trap:
385 case NVPTXISD::Suld1DI16Trap:
386 case NVPTXISD::Suld1DI32Trap:
387 case NVPTXISD::Suld1DI64Trap:
388 case NVPTXISD::Suld1DV2I8Trap:
389 case NVPTXISD::Suld1DV2I16Trap:
390 case NVPTXISD::Suld1DV2I32Trap:
391 case NVPTXISD::Suld1DV2I64Trap:
392 case NVPTXISD::Suld1DV4I8Trap:
393 case NVPTXISD::Suld1DV4I16Trap:
394 case NVPTXISD::Suld1DV4I32Trap:
395 case NVPTXISD::Suld1DArrayI8Trap:
396 case NVPTXISD::Suld1DArrayI16Trap:
397 case NVPTXISD::Suld1DArrayI32Trap:
398 case NVPTXISD::Suld1DArrayI64Trap:
399 case NVPTXISD::Suld1DArrayV2I8Trap:
400 case NVPTXISD::Suld1DArrayV2I16Trap:
401 case NVPTXISD::Suld1DArrayV2I32Trap:
402 case NVPTXISD::Suld1DArrayV2I64Trap:
403 case NVPTXISD::Suld1DArrayV4I8Trap:
404 case NVPTXISD::Suld1DArrayV4I16Trap:
405 case NVPTXISD::Suld1DArrayV4I32Trap:
406 case NVPTXISD::Suld2DI8Trap:
407 case NVPTXISD::Suld2DI16Trap:
408 case NVPTXISD::Suld2DI32Trap:
409 case NVPTXISD::Suld2DI64Trap:
410 case NVPTXISD::Suld2DV2I8Trap:
411 case NVPTXISD::Suld2DV2I16Trap:
412 case NVPTXISD::Suld2DV2I32Trap:
413 case NVPTXISD::Suld2DV2I64Trap:
414 case NVPTXISD::Suld2DV4I8Trap:
415 case NVPTXISD::Suld2DV4I16Trap:
416 case NVPTXISD::Suld2DV4I32Trap:
417 case NVPTXISD::Suld2DArrayI8Trap:
418 case NVPTXISD::Suld2DArrayI16Trap:
419 case NVPTXISD::Suld2DArrayI32Trap:
420 case NVPTXISD::Suld2DArrayI64Trap:
421 case NVPTXISD::Suld2DArrayV2I8Trap:
422 case NVPTXISD::Suld2DArrayV2I16Trap:
423 case NVPTXISD::Suld2DArrayV2I32Trap:
424 case NVPTXISD::Suld2DArrayV2I64Trap:
425 case NVPTXISD::Suld2DArrayV4I8Trap:
426 case NVPTXISD::Suld2DArrayV4I16Trap:
427 case NVPTXISD::Suld2DArrayV4I32Trap:
428 case NVPTXISD::Suld3DI8Trap:
429 case NVPTXISD::Suld3DI16Trap:
430 case NVPTXISD::Suld3DI32Trap:
431 case NVPTXISD::Suld3DI64Trap:
432 case NVPTXISD::Suld3DV2I8Trap:
433 case NVPTXISD::Suld3DV2I16Trap:
434 case NVPTXISD::Suld3DV2I32Trap:
435 case NVPTXISD::Suld3DV2I64Trap:
436 case NVPTXISD::Suld3DV4I8Trap:
437 case NVPTXISD::Suld3DV4I16Trap:
438 case NVPTXISD::Suld3DV4I32Trap:
439 case NVPTXISD::Suld1DI8Zero:
440 case NVPTXISD::Suld1DI16Zero:
441 case NVPTXISD::Suld1DI32Zero:
442 case NVPTXISD::Suld1DI64Zero:
443 case NVPTXISD::Suld1DV2I8Zero:
444 case NVPTXISD::Suld1DV2I16Zero:
445 case NVPTXISD::Suld1DV2I32Zero:
446 case NVPTXISD::Suld1DV2I64Zero:
447 case NVPTXISD::Suld1DV4I8Zero:
448 case NVPTXISD::Suld1DV4I16Zero:
449 case NVPTXISD::Suld1DV4I32Zero:
450 case NVPTXISD::Suld1DArrayI8Zero:
451 case NVPTXISD::Suld1DArrayI16Zero:
452 case NVPTXISD::Suld1DArrayI32Zero:
453 case NVPTXISD::Suld1DArrayI64Zero:
454 case NVPTXISD::Suld1DArrayV2I8Zero:
455 case NVPTXISD::Suld1DArrayV2I16Zero:
456 case NVPTXISD::Suld1DArrayV2I32Zero:
457 case NVPTXISD::Suld1DArrayV2I64Zero:
458 case NVPTXISD::Suld1DArrayV4I8Zero:
459 case NVPTXISD::Suld1DArrayV4I16Zero:
460 case NVPTXISD::Suld1DArrayV4I32Zero:
461 case NVPTXISD::Suld2DI8Zero:
462 case NVPTXISD::Suld2DI16Zero:
463 case NVPTXISD::Suld2DI32Zero:
464 case NVPTXISD::Suld2DI64Zero:
465 case NVPTXISD::Suld2DV2I8Zero:
466 case NVPTXISD::Suld2DV2I16Zero:
467 case NVPTXISD::Suld2DV2I32Zero:
468 case NVPTXISD::Suld2DV2I64Zero:
469 case NVPTXISD::Suld2DV4I8Zero:
470 case NVPTXISD::Suld2DV4I16Zero:
471 case NVPTXISD::Suld2DV4I32Zero:
472 case NVPTXISD::Suld2DArrayI8Zero:
473 case NVPTXISD::Suld2DArrayI16Zero:
474 case NVPTXISD::Suld2DArrayI32Zero:
475 case NVPTXISD::Suld2DArrayI64Zero:
476 case NVPTXISD::Suld2DArrayV2I8Zero:
477 case NVPTXISD::Suld2DArrayV2I16Zero:
478 case NVPTXISD::Suld2DArrayV2I32Zero:
479 case NVPTXISD::Suld2DArrayV2I64Zero:
480 case NVPTXISD::Suld2DArrayV4I8Zero:
481 case NVPTXISD::Suld2DArrayV4I16Zero:
482 case NVPTXISD::Suld2DArrayV4I32Zero:
483 case NVPTXISD::Suld3DI8Zero:
484 case NVPTXISD::Suld3DI16Zero:
485 case NVPTXISD::Suld3DI32Zero:
486 case NVPTXISD::Suld3DI64Zero:
487 case NVPTXISD::Suld3DV2I8Zero:
488 case NVPTXISD::Suld3DV2I16Zero:
489 case NVPTXISD::Suld3DV2I32Zero:
490 case NVPTXISD::Suld3DV2I64Zero:
491 case NVPTXISD::Suld3DV4I8Zero:
492 case NVPTXISD::Suld3DV4I16Zero:
493 case NVPTXISD::Suld3DV4I32Zero:
494 ResNode = SelectSurfaceIntrinsic(N);
495 break;
496 case ISD::AND:
497 case ISD::SRA:
498 case ISD::SRL:
499 // Try to select BFE
500 ResNode = SelectBFE(N);
501 break;
502 case ISD::ADDRSPACECAST:
503 ResNode = SelectAddrSpaceCast(N);
504 break;
165 default: 505 default:
166 break; 506 break;
167 } 507 }
168 if (ResNode) 508 if (ResNode)
169 return ResNode; 509 return ResNode;
170 return SelectCode(N); 510 return SelectCode(N);
171 } 511 }
172 512
513 SDNode *NVPTXDAGToDAGISel::SelectIntrinsicChain(SDNode *N) {
514 unsigned IID = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
515 switch (IID) {
516 default:
517 return NULL;
518 case Intrinsic::nvvm_ldg_global_f:
519 case Intrinsic::nvvm_ldg_global_i:
520 case Intrinsic::nvvm_ldg_global_p:
521 case Intrinsic::nvvm_ldu_global_f:
522 case Intrinsic::nvvm_ldu_global_i:
523 case Intrinsic::nvvm_ldu_global_p:
524 return SelectLDGLDU(N);
525 }
526 }
527
173 static unsigned int getCodeAddrSpace(MemSDNode *N, 528 static unsigned int getCodeAddrSpace(MemSDNode *N,
174 const NVPTXSubtarget &Subtarget) { 529 const NVPTXSubtarget &Subtarget) {
175 const Value *Src = N->getSrcValue(); 530 const Value *Src = N->getMemOperand()->getValue();
176 531
177 if (!Src) 532 if (!Src)
178 return NVPTX::PTXLdStInstCode::GENERIC; 533 return NVPTX::PTXLdStInstCode::GENERIC;
179 534
180 if (const PointerType *PT = dyn_cast<PointerType>(Src->getType())) { 535 if (const PointerType *PT = dyn_cast<PointerType>(Src->getType())) {
189 } 544 }
190 } 545 }
191 return NVPTX::PTXLdStInstCode::GENERIC; 546 return NVPTX::PTXLdStInstCode::GENERIC;
192 } 547 }
193 548
549 SDNode *NVPTXDAGToDAGISel::SelectIntrinsicNoChain(SDNode *N) {
550 unsigned IID = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
551 switch (IID) {
552 default:
553 return nullptr;
554 case Intrinsic::nvvm_texsurf_handle_internal:
555 return SelectTexSurfHandle(N);
556 }
557 }
558
559 SDNode *NVPTXDAGToDAGISel::SelectTexSurfHandle(SDNode *N) {
560 // Op 0 is the intrinsic ID
561 SDValue Wrapper = N->getOperand(1);
562 SDValue GlobalVal = Wrapper.getOperand(0);
563 return CurDAG->getMachineNode(NVPTX::texsurf_handles, SDLoc(N), MVT::i64,
564 GlobalVal);
565 }
566
567 SDNode *NVPTXDAGToDAGISel::SelectAddrSpaceCast(SDNode *N) {
568 SDValue Src = N->getOperand(0);
569 AddrSpaceCastSDNode *CastN = cast<AddrSpaceCastSDNode>(N);
570 unsigned SrcAddrSpace = CastN->getSrcAddressSpace();
571 unsigned DstAddrSpace = CastN->getDestAddressSpace();
572
573 assert(SrcAddrSpace != DstAddrSpace &&
574 "addrspacecast must be between different address spaces");
575
576 if (DstAddrSpace == ADDRESS_SPACE_GENERIC) {
577 // Specific to generic
578 unsigned Opc;
579 switch (SrcAddrSpace) {
580 default: report_fatal_error("Bad address space in addrspacecast");
581 case ADDRESS_SPACE_GLOBAL:
582 Opc = Subtarget.is64Bit() ? NVPTX::cvta_global_yes_64
583 : NVPTX::cvta_global_yes;
584 break;
585 case ADDRESS_SPACE_SHARED:
586 Opc = Subtarget.is64Bit() ? NVPTX::cvta_shared_yes_64
587 : NVPTX::cvta_shared_yes;
588 break;
589 case ADDRESS_SPACE_CONST:
590 Opc = Subtarget.is64Bit() ? NVPTX::cvta_const_yes_64
591 : NVPTX::cvta_const_yes;
592 break;
593 case ADDRESS_SPACE_LOCAL:
594 Opc = Subtarget.is64Bit() ? NVPTX::cvta_local_yes_64
595 : NVPTX::cvta_local_yes;
596 break;
597 }
598 return CurDAG->getMachineNode(Opc, SDLoc(N), N->getValueType(0), Src);
599 } else {
600 // Generic to specific
601 if (SrcAddrSpace != 0)
602 report_fatal_error("Cannot cast between two non-generic address spaces");
603 unsigned Opc;
604 switch (DstAddrSpace) {
605 default: report_fatal_error("Bad address space in addrspacecast");
606 case ADDRESS_SPACE_GLOBAL:
607 Opc = Subtarget.is64Bit() ? NVPTX::cvta_to_global_yes_64
608 : NVPTX::cvta_to_global_yes;
609 break;
610 case ADDRESS_SPACE_SHARED:
611 Opc = Subtarget.is64Bit() ? NVPTX::cvta_to_shared_yes_64
612 : NVPTX::cvta_to_shared_yes;
613 break;
614 case ADDRESS_SPACE_CONST:
615 Opc = Subtarget.is64Bit() ? NVPTX::cvta_to_const_yes_64
616 : NVPTX::cvta_to_const_yes;
617 break;
618 case ADDRESS_SPACE_LOCAL:
619 Opc = Subtarget.is64Bit() ? NVPTX::cvta_to_local_yes_64
620 : NVPTX::cvta_to_local_yes;
621 break;
622 }
623 return CurDAG->getMachineNode(Opc, SDLoc(N), N->getValueType(0), Src);
624 }
625 }
626
194 SDNode *NVPTXDAGToDAGISel::SelectLoad(SDNode *N) { 627 SDNode *NVPTXDAGToDAGISel::SelectLoad(SDNode *N) {
195 SDLoc dl(N); 628 SDLoc dl(N);
196 LoadSDNode *LD = cast<LoadSDNode>(N); 629 LoadSDNode *LD = cast<LoadSDNode>(N);
197 EVT LoadedVT = LD->getMemoryVT(); 630 EVT LoadedVT = LD->getMemoryVT();
198 SDNode *NVPTXLD = NULL; 631 SDNode *NVPTXLD = nullptr;
199 632
200 // do not support pre/post inc/dec 633 // do not support pre/post inc/dec
201 if (LD->isIndexed()) 634 if (LD->isIndexed())
202 return NULL; 635 return nullptr;
203 636
204 if (!LoadedVT.isSimple()) 637 if (!LoadedVT.isSimple())
205 return NULL; 638 return nullptr;
206 639
207 // Address Space Setting 640 // Address Space Setting
208 unsigned int codeAddrSpace = getCodeAddrSpace(LD, Subtarget); 641 unsigned int codeAddrSpace = getCodeAddrSpace(LD, Subtarget);
209 642
210 // Volatile Setting 643 // Volatile Setting
223 if (num == 2) 656 if (num == 2)
224 vecType = NVPTX::PTXLdStInstCode::V2; 657 vecType = NVPTX::PTXLdStInstCode::V2;
225 else if (num == 4) 658 else if (num == 4)
226 vecType = NVPTX::PTXLdStInstCode::V4; 659 vecType = NVPTX::PTXLdStInstCode::V4;
227 else 660 else
228 return NULL; 661 return nullptr;
229 } 662 }
230 663
231 // Type Setting: fromType + fromTypeWidth 664 // Type Setting: fromType + fromTypeWidth
232 // 665 //
233 // Sign : ISD::SEXTLOAD 666 // Sign : ISD::SEXTLOAD
272 break; 705 break;
273 case MVT::f64: 706 case MVT::f64:
274 Opcode = NVPTX::LD_f64_avar; 707 Opcode = NVPTX::LD_f64_avar;
275 break; 708 break;
276 default: 709 default:
277 return NULL; 710 return nullptr;
278 } 711 }
279 SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace), 712 SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
280 getI32Imm(vecType), getI32Imm(fromType), 713 getI32Imm(vecType), getI32Imm(fromType),
281 getI32Imm(fromTypeWidth), Addr, Chain }; 714 getI32Imm(fromTypeWidth), Addr, Chain };
282 NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops); 715 NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops);
301 break; 734 break;
302 case MVT::f64: 735 case MVT::f64:
303 Opcode = NVPTX::LD_f64_asi; 736 Opcode = NVPTX::LD_f64_asi;
304 break; 737 break;
305 default: 738 default:
306 return NULL; 739 return nullptr;
307 } 740 }
308 SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace), 741 SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
309 getI32Imm(vecType), getI32Imm(fromType), 742 getI32Imm(vecType), getI32Imm(fromType),
310 getI32Imm(fromTypeWidth), Base, Offset, Chain }; 743 getI32Imm(fromTypeWidth), Base, Offset, Chain };
311 NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops); 744 NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops);
331 break; 764 break;
332 case MVT::f64: 765 case MVT::f64:
333 Opcode = NVPTX::LD_f64_ari_64; 766 Opcode = NVPTX::LD_f64_ari_64;
334 break; 767 break;
335 default: 768 default:
336 return NULL; 769 return nullptr;
337 } 770 }
338 } else { 771 } else {
339 switch (TargetVT) { 772 switch (TargetVT) {
340 case MVT::i8: 773 case MVT::i8:
341 Opcode = NVPTX::LD_i8_ari; 774 Opcode = NVPTX::LD_i8_ari;
354 break; 787 break;
355 case MVT::f64: 788 case MVT::f64:
356 Opcode = NVPTX::LD_f64_ari; 789 Opcode = NVPTX::LD_f64_ari;
357 break; 790 break;
358 default: 791 default:
359 return NULL; 792 return nullptr;
360 } 793 }
361 } 794 }
362 SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace), 795 SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
363 getI32Imm(vecType), getI32Imm(fromType), 796 getI32Imm(vecType), getI32Imm(fromType),
364 getI32Imm(fromTypeWidth), Base, Offset, Chain }; 797 getI32Imm(fromTypeWidth), Base, Offset, Chain };
383 break; 816 break;
384 case MVT::f64: 817 case MVT::f64:
385 Opcode = NVPTX::LD_f64_areg_64; 818 Opcode = NVPTX::LD_f64_areg_64;
386 break; 819 break;
387 default: 820 default:
388 return NULL; 821 return nullptr;
389 } 822 }
390 } else { 823 } else {
391 switch (TargetVT) { 824 switch (TargetVT) {
392 case MVT::i8: 825 case MVT::i8:
393 Opcode = NVPTX::LD_i8_areg; 826 Opcode = NVPTX::LD_i8_areg;
406 break; 839 break;
407 case MVT::f64: 840 case MVT::f64:
408 Opcode = NVPTX::LD_f64_areg; 841 Opcode = NVPTX::LD_f64_areg;
409 break; 842 break;
410 default: 843 default:
411 return NULL; 844 return nullptr;
412 } 845 }
413 } 846 }
414 SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace), 847 SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
415 getI32Imm(vecType), getI32Imm(fromType), 848 getI32Imm(vecType), getI32Imm(fromType),
416 getI32Imm(fromTypeWidth), N1, Chain }; 849 getI32Imm(fromTypeWidth), N1, Chain };
417 NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops); 850 NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops);
418 } 851 }
419 852
420 if (NVPTXLD != NULL) { 853 if (NVPTXLD) {
421 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1); 854 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
422 MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand(); 855 MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
423 cast<MachineSDNode>(NVPTXLD)->setMemRefs(MemRefs0, MemRefs0 + 1); 856 cast<MachineSDNode>(NVPTXLD)->setMemRefs(MemRefs0, MemRefs0 + 1);
424 } 857 }
425 858
436 SDNode *LD; 869 SDNode *LD;
437 MemSDNode *MemSD = cast<MemSDNode>(N); 870 MemSDNode *MemSD = cast<MemSDNode>(N);
438 EVT LoadedVT = MemSD->getMemoryVT(); 871 EVT LoadedVT = MemSD->getMemoryVT();
439 872
440 if (!LoadedVT.isSimple()) 873 if (!LoadedVT.isSimple())
441 return NULL; 874 return nullptr;
442 875
443 // Address Space Setting 876 // Address Space Setting
444 unsigned int CodeAddrSpace = getCodeAddrSpace(MemSD, Subtarget); 877 unsigned int CodeAddrSpace = getCodeAddrSpace(MemSD, Subtarget);
445 878
446 // Volatile Setting 879 // Volatile Setting
482 break; 915 break;
483 case NVPTXISD::LoadV4: 916 case NVPTXISD::LoadV4:
484 VecType = NVPTX::PTXLdStInstCode::V4; 917 VecType = NVPTX::PTXLdStInstCode::V4;
485 break; 918 break;
486 default: 919 default:
487 return NULL; 920 return nullptr;
488 } 921 }
489 922
490 EVT EltVT = N->getValueType(0); 923 EVT EltVT = N->getValueType(0);
491 924
492 if (SelectDirectAddr(Op1, Addr)) { 925 if (SelectDirectAddr(Op1, Addr)) {
493 switch (N->getOpcode()) { 926 switch (N->getOpcode()) {
494 default: 927 default:
495 return NULL; 928 return nullptr;
496 case NVPTXISD::LoadV2: 929 case NVPTXISD::LoadV2:
497 switch (EltVT.getSimpleVT().SimpleTy) { 930 switch (EltVT.getSimpleVT().SimpleTy) {
498 default: 931 default:
499 return NULL; 932 return nullptr;
500 case MVT::i8: 933 case MVT::i8:
501 Opcode = NVPTX::LDV_i8_v2_avar; 934 Opcode = NVPTX::LDV_i8_v2_avar;
502 break; 935 break;
503 case MVT::i16: 936 case MVT::i16:
504 Opcode = NVPTX::LDV_i16_v2_avar; 937 Opcode = NVPTX::LDV_i16_v2_avar;
518 } 951 }
519 break; 952 break;
520 case NVPTXISD::LoadV4: 953 case NVPTXISD::LoadV4:
521 switch (EltVT.getSimpleVT().SimpleTy) { 954 switch (EltVT.getSimpleVT().SimpleTy) {
522 default: 955 default:
523 return NULL; 956 return nullptr;
524 case MVT::i8: 957 case MVT::i8:
525 Opcode = NVPTX::LDV_i8_v4_avar; 958 Opcode = NVPTX::LDV_i8_v4_avar;
526 break; 959 break;
527 case MVT::i16: 960 case MVT::i16:
528 Opcode = NVPTX::LDV_i16_v4_avar; 961 Opcode = NVPTX::LDV_i16_v4_avar;
544 } else if (Subtarget.is64Bit() 977 } else if (Subtarget.is64Bit()
545 ? SelectADDRsi64(Op1.getNode(), Op1, Base, Offset) 978 ? SelectADDRsi64(Op1.getNode(), Op1, Base, Offset)
546 : SelectADDRsi(Op1.getNode(), Op1, Base, Offset)) { 979 : SelectADDRsi(Op1.getNode(), Op1, Base, Offset)) {
547 switch (N->getOpcode()) { 980 switch (N->getOpcode()) {
548 default: 981 default:
549 return NULL; 982 return nullptr;
550 case NVPTXISD::LoadV2: 983 case NVPTXISD::LoadV2:
551 switch (EltVT.getSimpleVT().SimpleTy) { 984 switch (EltVT.getSimpleVT().SimpleTy) {
552 default: 985 default:
553 return NULL; 986 return nullptr;
554 case MVT::i8: 987 case MVT::i8:
555 Opcode = NVPTX::LDV_i8_v2_asi; 988 Opcode = NVPTX::LDV_i8_v2_asi;
556 break; 989 break;
557 case MVT::i16: 990 case MVT::i16:
558 Opcode = NVPTX::LDV_i16_v2_asi; 991 Opcode = NVPTX::LDV_i16_v2_asi;
572 } 1005 }
573 break; 1006 break;
574 case NVPTXISD::LoadV4: 1007 case NVPTXISD::LoadV4:
575 switch (EltVT.getSimpleVT().SimpleTy) { 1008 switch (EltVT.getSimpleVT().SimpleTy) {
576 default: 1009 default:
577 return NULL; 1010 return nullptr;
578 case MVT::i8: 1011 case MVT::i8:
579 Opcode = NVPTX::LDV_i8_v4_asi; 1012 Opcode = NVPTX::LDV_i8_v4_asi;
580 break; 1013 break;
581 case MVT::i16: 1014 case MVT::i16:
582 Opcode = NVPTX::LDV_i16_v4_asi; 1015 Opcode = NVPTX::LDV_i16_v4_asi;
599 ? SelectADDRri64(Op1.getNode(), Op1, Base, Offset) 1032 ? SelectADDRri64(Op1.getNode(), Op1, Base, Offset)
600 : SelectADDRri(Op1.getNode(), Op1, Base, Offset)) { 1033 : SelectADDRri(Op1.getNode(), Op1, Base, Offset)) {
601 if (Subtarget.is64Bit()) { 1034 if (Subtarget.is64Bit()) {
602 switch (N->getOpcode()) { 1035 switch (N->getOpcode()) {
603 default: 1036 default:
604 return NULL; 1037 return nullptr;
605 case NVPTXISD::LoadV2: 1038 case NVPTXISD::LoadV2:
606 switch (EltVT.getSimpleVT().SimpleTy) { 1039 switch (EltVT.getSimpleVT().SimpleTy) {
607 default: 1040 default:
608 return NULL; 1041 return nullptr;
609 case MVT::i8: 1042 case MVT::i8:
610 Opcode = NVPTX::LDV_i8_v2_ari_64; 1043 Opcode = NVPTX::LDV_i8_v2_ari_64;
611 break; 1044 break;
612 case MVT::i16: 1045 case MVT::i16:
613 Opcode = NVPTX::LDV_i16_v2_ari_64; 1046 Opcode = NVPTX::LDV_i16_v2_ari_64;
627 } 1060 }
628 break; 1061 break;
629 case NVPTXISD::LoadV4: 1062 case NVPTXISD::LoadV4:
630 switch (EltVT.getSimpleVT().SimpleTy) { 1063 switch (EltVT.getSimpleVT().SimpleTy) {
631 default: 1064 default:
632 return NULL; 1065 return nullptr;
633 case MVT::i8: 1066 case MVT::i8:
634 Opcode = NVPTX::LDV_i8_v4_ari_64; 1067 Opcode = NVPTX::LDV_i8_v4_ari_64;
635 break; 1068 break;
636 case MVT::i16: 1069 case MVT::i16:
637 Opcode = NVPTX::LDV_i16_v4_ari_64; 1070 Opcode = NVPTX::LDV_i16_v4_ari_64;
646 break; 1079 break;
647 } 1080 }
648 } else { 1081 } else {
649 switch (N->getOpcode()) { 1082 switch (N->getOpcode()) {
650 default: 1083 default:
651 return NULL; 1084 return nullptr;
652 case NVPTXISD::LoadV2: 1085 case NVPTXISD::LoadV2:
653 switch (EltVT.getSimpleVT().SimpleTy) { 1086 switch (EltVT.getSimpleVT().SimpleTy) {
654 default: 1087 default:
655 return NULL; 1088 return nullptr;
656 case MVT::i8: 1089 case MVT::i8:
657 Opcode = NVPTX::LDV_i8_v2_ari; 1090 Opcode = NVPTX::LDV_i8_v2_ari;
658 break; 1091 break;
659 case MVT::i16: 1092 case MVT::i16:
660 Opcode = NVPTX::LDV_i16_v2_ari; 1093 Opcode = NVPTX::LDV_i16_v2_ari;
674 } 1107 }
675 break; 1108 break;
676 case NVPTXISD::LoadV4: 1109 case NVPTXISD::LoadV4:
677 switch (EltVT.getSimpleVT().SimpleTy) { 1110 switch (EltVT.getSimpleVT().SimpleTy) {
678 default: 1111 default:
679 return NULL; 1112 return nullptr;
680 case MVT::i8: 1113 case MVT::i8:
681 Opcode = NVPTX::LDV_i8_v4_ari; 1114 Opcode = NVPTX::LDV_i8_v4_ari;
682 break; 1115 break;
683 case MVT::i16: 1116 case MVT::i16:
684 Opcode = NVPTX::LDV_i16_v4_ari; 1117 Opcode = NVPTX::LDV_i16_v4_ari;
701 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops); 1134 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
702 } else { 1135 } else {
703 if (Subtarget.is64Bit()) { 1136 if (Subtarget.is64Bit()) {
704 switch (N->getOpcode()) { 1137 switch (N->getOpcode()) {
705 default: 1138 default:
706 return NULL; 1139 return nullptr;
707 case NVPTXISD::LoadV2: 1140 case NVPTXISD::LoadV2:
708 switch (EltVT.getSimpleVT().SimpleTy) { 1141 switch (EltVT.getSimpleVT().SimpleTy) {
709 default: 1142 default:
710 return NULL; 1143 return nullptr;
711 case MVT::i8: 1144 case MVT::i8:
712 Opcode = NVPTX::LDV_i8_v2_areg_64; 1145 Opcode = NVPTX::LDV_i8_v2_areg_64;
713 break; 1146 break;
714 case MVT::i16: 1147 case MVT::i16:
715 Opcode = NVPTX::LDV_i16_v2_areg_64; 1148 Opcode = NVPTX::LDV_i16_v2_areg_64;
729 } 1162 }
730 break; 1163 break;
731 case NVPTXISD::LoadV4: 1164 case NVPTXISD::LoadV4:
732 switch (EltVT.getSimpleVT().SimpleTy) { 1165 switch (EltVT.getSimpleVT().SimpleTy) {
733 default: 1166 default:
734 return NULL; 1167 return nullptr;
735 case MVT::i8: 1168 case MVT::i8:
736 Opcode = NVPTX::LDV_i8_v4_areg_64; 1169 Opcode = NVPTX::LDV_i8_v4_areg_64;
737 break; 1170 break;
738 case MVT::i16: 1171 case MVT::i16:
739 Opcode = NVPTX::LDV_i16_v4_areg_64; 1172 Opcode = NVPTX::LDV_i16_v4_areg_64;
748 break; 1181 break;
749 } 1182 }
750 } else { 1183 } else {
751 switch (N->getOpcode()) { 1184 switch (N->getOpcode()) {
752 default: 1185 default:
753 return NULL; 1186 return nullptr;
754 case NVPTXISD::LoadV2: 1187 case NVPTXISD::LoadV2:
755 switch (EltVT.getSimpleVT().SimpleTy) { 1188 switch (EltVT.getSimpleVT().SimpleTy) {
756 default: 1189 default:
757 return NULL; 1190 return nullptr;
758 case MVT::i8: 1191 case MVT::i8:
759 Opcode = NVPTX::LDV_i8_v2_areg; 1192 Opcode = NVPTX::LDV_i8_v2_areg;
760 break; 1193 break;
761 case MVT::i16: 1194 case MVT::i16:
762 Opcode = NVPTX::LDV_i16_v2_areg; 1195 Opcode = NVPTX::LDV_i16_v2_areg;
776 } 1209 }
777 break; 1210 break;
778 case NVPTXISD::LoadV4: 1211 case NVPTXISD::LoadV4:
779 switch (EltVT.getSimpleVT().SimpleTy) { 1212 switch (EltVT.getSimpleVT().SimpleTy) {
780 default: 1213 default:
781 return NULL; 1214 return nullptr;
782 case MVT::i8: 1215 case MVT::i8:
783 Opcode = NVPTX::LDV_i8_v4_areg; 1216 Opcode = NVPTX::LDV_i8_v4_areg;
784 break; 1217 break;
785 case MVT::i16: 1218 case MVT::i16:
786 Opcode = NVPTX::LDV_i16_v4_areg; 1219 Opcode = NVPTX::LDV_i16_v4_areg;
807 cast<MachineSDNode>(LD)->setMemRefs(MemRefs0, MemRefs0 + 1); 1240 cast<MachineSDNode>(LD)->setMemRefs(MemRefs0, MemRefs0 + 1);
808 1241
809 return LD; 1242 return LD;
810 } 1243 }
811 1244
812 SDNode *NVPTXDAGToDAGISel::SelectLDGLDUVector(SDNode *N) { 1245 SDNode *NVPTXDAGToDAGISel::SelectLDGLDU(SDNode *N) {
813 1246
814 SDValue Chain = N->getOperand(0); 1247 SDValue Chain = N->getOperand(0);
815 SDValue Op1 = N->getOperand(1); 1248 SDValue Op1;
1249 MemSDNode *Mem;
1250 bool IsLDG = true;
1251
1252 // If this is an LDG intrinsic, the address is the third operand. Its its an
1253 // LDG/LDU SD node (from custom vector handling), then its the second operand
1254 if (N->getOpcode() == ISD::INTRINSIC_W_CHAIN) {
1255 Op1 = N->getOperand(2);
1256 Mem = cast<MemIntrinsicSDNode>(N);
1257 unsigned IID = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
1258 switch (IID) {
1259 default:
1260 return NULL;
1261 case Intrinsic::nvvm_ldg_global_f:
1262 case Intrinsic::nvvm_ldg_global_i:
1263 case Intrinsic::nvvm_ldg_global_p:
1264 IsLDG = true;
1265 break;
1266 case Intrinsic::nvvm_ldu_global_f:
1267 case Intrinsic::nvvm_ldu_global_i:
1268 case Intrinsic::nvvm_ldu_global_p:
1269 IsLDG = false;
1270 break;
1271 }
1272 } else {
1273 Op1 = N->getOperand(1);
1274 Mem = cast<MemSDNode>(N);
1275 }
1276
816 unsigned Opcode; 1277 unsigned Opcode;
817 SDLoc DL(N); 1278 SDLoc DL(N);
818 SDNode *LD; 1279 SDNode *LD;
819 MemSDNode *Mem = cast<MemSDNode>(N);
820 SDValue Base, Offset, Addr; 1280 SDValue Base, Offset, Addr;
821 1281
822 EVT EltVT = Mem->getMemoryVT().getVectorElementType(); 1282 EVT EltVT = Mem->getMemoryVT();
1283 if (EltVT.isVector()) {
1284 EltVT = EltVT.getVectorElementType();
1285 }
823 1286
824 if (SelectDirectAddr(Op1, Addr)) { 1287 if (SelectDirectAddr(Op1, Addr)) {
825 switch (N->getOpcode()) { 1288 switch (N->getOpcode()) {
826 default: 1289 default:
827 return NULL; 1290 return nullptr;
1291 case ISD::INTRINSIC_W_CHAIN:
1292 if (IsLDG) {
1293 switch (EltVT.getSimpleVT().SimpleTy) {
1294 default:
1295 return nullptr;
1296 case MVT::i8:
1297 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8avar;
1298 break;
1299 case MVT::i16:
1300 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16avar;
1301 break;
1302 case MVT::i32:
1303 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32avar;
1304 break;
1305 case MVT::i64:
1306 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64avar;
1307 break;
1308 case MVT::f32:
1309 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32avar;
1310 break;
1311 case MVT::f64:
1312 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64avar;
1313 break;
1314 }
1315 } else {
1316 switch (EltVT.getSimpleVT().SimpleTy) {
1317 default:
1318 return nullptr;
1319 case MVT::i8:
1320 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8avar;
1321 break;
1322 case MVT::i16:
1323 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16avar;
1324 break;
1325 case MVT::i32:
1326 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32avar;
1327 break;
1328 case MVT::i64:
1329 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64avar;
1330 break;
1331 case MVT::f32:
1332 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32avar;
1333 break;
1334 case MVT::f64:
1335 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64avar;
1336 break;
1337 }
1338 }
1339 break;
828 case NVPTXISD::LDGV2: 1340 case NVPTXISD::LDGV2:
829 switch (EltVT.getSimpleVT().SimpleTy) { 1341 switch (EltVT.getSimpleVT().SimpleTy) {
830 default: 1342 default:
831 return NULL; 1343 return nullptr;
832 case MVT::i8: 1344 case MVT::i8:
833 Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_avar; 1345 Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_avar;
834 break; 1346 break;
835 case MVT::i16: 1347 case MVT::i16:
836 Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_avar; 1348 Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_avar;
850 } 1362 }
851 break; 1363 break;
852 case NVPTXISD::LDUV2: 1364 case NVPTXISD::LDUV2:
853 switch (EltVT.getSimpleVT().SimpleTy) { 1365 switch (EltVT.getSimpleVT().SimpleTy) {
854 default: 1366 default:
855 return NULL; 1367 return nullptr;
856 case MVT::i8: 1368 case MVT::i8:
857 Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_avar; 1369 Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_avar;
858 break; 1370 break;
859 case MVT::i16: 1371 case MVT::i16:
860 Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_avar; 1372 Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_avar;
874 } 1386 }
875 break; 1387 break;
876 case NVPTXISD::LDGV4: 1388 case NVPTXISD::LDGV4:
877 switch (EltVT.getSimpleVT().SimpleTy) { 1389 switch (EltVT.getSimpleVT().SimpleTy) {
878 default: 1390 default:
879 return NULL; 1391 return nullptr;
880 case MVT::i8: 1392 case MVT::i8:
881 Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_avar; 1393 Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_avar;
882 break; 1394 break;
883 case MVT::i16: 1395 case MVT::i16:
884 Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_avar; 1396 Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_avar;
892 } 1404 }
893 break; 1405 break;
894 case NVPTXISD::LDUV4: 1406 case NVPTXISD::LDUV4:
895 switch (EltVT.getSimpleVT().SimpleTy) { 1407 switch (EltVT.getSimpleVT().SimpleTy) {
896 default: 1408 default:
897 return NULL; 1409 return nullptr;
898 case MVT::i8: 1410 case MVT::i8:
899 Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_avar; 1411 Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_avar;
900 break; 1412 break;
901 case MVT::i16: 1413 case MVT::i16:
902 Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_avar; 1414 Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_avar;
910 } 1422 }
911 break; 1423 break;
912 } 1424 }
913 1425
914 SDValue Ops[] = { Addr, Chain }; 1426 SDValue Ops[] = { Addr, Chain };
915 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), 1427 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
916 ArrayRef<SDValue>(Ops, 2));
917 } else if (Subtarget.is64Bit() 1428 } else if (Subtarget.is64Bit()
918 ? SelectADDRri64(Op1.getNode(), Op1, Base, Offset) 1429 ? SelectADDRri64(Op1.getNode(), Op1, Base, Offset)
919 : SelectADDRri(Op1.getNode(), Op1, Base, Offset)) { 1430 : SelectADDRri(Op1.getNode(), Op1, Base, Offset)) {
920 if (Subtarget.is64Bit()) { 1431 if (Subtarget.is64Bit()) {
921 switch (N->getOpcode()) { 1432 switch (N->getOpcode()) {
922 default: 1433 default:
923 return NULL; 1434 return nullptr;
1435 case ISD::INTRINSIC_W_CHAIN:
1436 if (IsLDG) {
1437 switch (EltVT.getSimpleVT().SimpleTy) {
1438 default:
1439 return nullptr;
1440 case MVT::i8:
1441 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8ari64;
1442 break;
1443 case MVT::i16:
1444 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16ari64;
1445 break;
1446 case MVT::i32:
1447 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32ari64;
1448 break;
1449 case MVT::i64:
1450 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64ari64;
1451 break;
1452 case MVT::f32:
1453 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32ari64;
1454 break;
1455 case MVT::f64:
1456 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64ari64;
1457 break;
1458 }
1459 } else {
1460 switch (EltVT.getSimpleVT().SimpleTy) {
1461 default:
1462 return nullptr;
1463 case MVT::i8:
1464 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8ari64;
1465 break;
1466 case MVT::i16:
1467 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16ari64;
1468 break;
1469 case MVT::i32:
1470 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32ari64;
1471 break;
1472 case MVT::i64:
1473 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64ari64;
1474 break;
1475 case MVT::f32:
1476 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32ari64;
1477 break;
1478 case MVT::f64:
1479 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64ari64;
1480 break;
1481 }
1482 }
1483 break;
924 case NVPTXISD::LDGV2: 1484 case NVPTXISD::LDGV2:
925 switch (EltVT.getSimpleVT().SimpleTy) { 1485 switch (EltVT.getSimpleVT().SimpleTy) {
926 default: 1486 default:
927 return NULL; 1487 return nullptr;
928 case MVT::i8: 1488 case MVT::i8:
929 Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_ari64; 1489 Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_ari64;
930 break; 1490 break;
931 case MVT::i16: 1491 case MVT::i16:
932 Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_ari64; 1492 Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_ari64;
946 } 1506 }
947 break; 1507 break;
948 case NVPTXISD::LDUV2: 1508 case NVPTXISD::LDUV2:
949 switch (EltVT.getSimpleVT().SimpleTy) { 1509 switch (EltVT.getSimpleVT().SimpleTy) {
950 default: 1510 default:
951 return NULL; 1511 return nullptr;
952 case MVT::i8: 1512 case MVT::i8:
953 Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_ari64; 1513 Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_ari64;
954 break; 1514 break;
955 case MVT::i16: 1515 case MVT::i16:
956 Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_ari64; 1516 Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_ari64;
970 } 1530 }
971 break; 1531 break;
972 case NVPTXISD::LDGV4: 1532 case NVPTXISD::LDGV4:
973 switch (EltVT.getSimpleVT().SimpleTy) { 1533 switch (EltVT.getSimpleVT().SimpleTy) {
974 default: 1534 default:
975 return NULL; 1535 return nullptr;
976 case MVT::i8: 1536 case MVT::i8:
977 Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_ari64; 1537 Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_ari64;
978 break; 1538 break;
979 case MVT::i16: 1539 case MVT::i16:
980 Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_ari64; 1540 Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_ari64;
988 } 1548 }
989 break; 1549 break;
990 case NVPTXISD::LDUV4: 1550 case NVPTXISD::LDUV4:
991 switch (EltVT.getSimpleVT().SimpleTy) { 1551 switch (EltVT.getSimpleVT().SimpleTy) {
992 default: 1552 default:
993 return NULL; 1553 return nullptr;
994 case MVT::i8: 1554 case MVT::i8:
995 Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_ari64; 1555 Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_ari64;
996 break; 1556 break;
997 case MVT::i16: 1557 case MVT::i16:
998 Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_ari64; 1558 Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_ari64;
1007 break; 1567 break;
1008 } 1568 }
1009 } else { 1569 } else {
1010 switch (N->getOpcode()) { 1570 switch (N->getOpcode()) {
1011 default: 1571 default:
1012 return NULL; 1572 return nullptr;
1573 case ISD::INTRINSIC_W_CHAIN:
1574 if (IsLDG) {
1575 switch (EltVT.getSimpleVT().SimpleTy) {
1576 default:
1577 return nullptr;
1578 case MVT::i8:
1579 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8ari;
1580 break;
1581 case MVT::i16:
1582 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16ari;
1583 break;
1584 case MVT::i32:
1585 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32ari;
1586 break;
1587 case MVT::i64:
1588 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64ari;
1589 break;
1590 case MVT::f32:
1591 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32ari;
1592 break;
1593 case MVT::f64:
1594 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64ari;
1595 break;
1596 }
1597 } else {
1598 switch (EltVT.getSimpleVT().SimpleTy) {
1599 default:
1600 return nullptr;
1601 case MVT::i8:
1602 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8ari;
1603 break;
1604 case MVT::i16:
1605 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16ari;
1606 break;
1607 case MVT::i32:
1608 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32ari;
1609 break;
1610 case MVT::i64:
1611 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64ari;
1612 break;
1613 case MVT::f32:
1614 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32ari;
1615 break;
1616 case MVT::f64:
1617 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64ari;
1618 break;
1619 }
1620 }
1621 break;
1013 case NVPTXISD::LDGV2: 1622 case NVPTXISD::LDGV2:
1014 switch (EltVT.getSimpleVT().SimpleTy) { 1623 switch (EltVT.getSimpleVT().SimpleTy) {
1015 default: 1624 default:
1016 return NULL; 1625 return nullptr;
1017 case MVT::i8: 1626 case MVT::i8:
1018 Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_ari32; 1627 Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_ari32;
1019 break; 1628 break;
1020 case MVT::i16: 1629 case MVT::i16:
1021 Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_ari32; 1630 Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_ari32;
1035 } 1644 }
1036 break; 1645 break;
1037 case NVPTXISD::LDUV2: 1646 case NVPTXISD::LDUV2:
1038 switch (EltVT.getSimpleVT().SimpleTy) { 1647 switch (EltVT.getSimpleVT().SimpleTy) {
1039 default: 1648 default:
1040 return NULL; 1649 return nullptr;
1041 case MVT::i8: 1650 case MVT::i8:
1042 Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_ari32; 1651 Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_ari32;
1043 break; 1652 break;
1044 case MVT::i16: 1653 case MVT::i16:
1045 Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_ari32; 1654 Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_ari32;
1059 } 1668 }
1060 break; 1669 break;
1061 case NVPTXISD::LDGV4: 1670 case NVPTXISD::LDGV4:
1062 switch (EltVT.getSimpleVT().SimpleTy) { 1671 switch (EltVT.getSimpleVT().SimpleTy) {
1063 default: 1672 default:
1064 return NULL; 1673 return nullptr;
1065 case MVT::i8: 1674 case MVT::i8:
1066 Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_ari32; 1675 Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_ari32;
1067 break; 1676 break;
1068 case MVT::i16: 1677 case MVT::i16:
1069 Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_ari32; 1678 Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_ari32;
1077 } 1686 }
1078 break; 1687 break;
1079 case NVPTXISD::LDUV4: 1688 case NVPTXISD::LDUV4:
1080 switch (EltVT.getSimpleVT().SimpleTy) { 1689 switch (EltVT.getSimpleVT().SimpleTy) {
1081 default: 1690 default:
1082 return NULL; 1691 return nullptr;
1083 case MVT::i8: 1692 case MVT::i8:
1084 Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_ari32; 1693 Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_ari32;
1085 break; 1694 break;
1086 case MVT::i16: 1695 case MVT::i16:
1087 Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_ari32; 1696 Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_ari32;
1097 } 1706 }
1098 } 1707 }
1099 1708
1100 SDValue Ops[] = { Base, Offset, Chain }; 1709 SDValue Ops[] = { Base, Offset, Chain };
1101 1710
1102 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), 1711 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
1103 ArrayRef<SDValue>(Ops, 3));
1104 } else { 1712 } else {
1105 if (Subtarget.is64Bit()) { 1713 if (Subtarget.is64Bit()) {
1106 switch (N->getOpcode()) { 1714 switch (N->getOpcode()) {
1107 default: 1715 default:
1108 return NULL; 1716 return nullptr;
1717 case ISD::INTRINSIC_W_CHAIN:
1718 if (IsLDG) {
1719 switch (EltVT.getSimpleVT().SimpleTy) {
1720 default:
1721 return nullptr;
1722 case MVT::i8:
1723 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8areg64;
1724 break;
1725 case MVT::i16:
1726 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16areg64;
1727 break;
1728 case MVT::i32:
1729 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32areg64;
1730 break;
1731 case MVT::i64:
1732 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64areg64;
1733 break;
1734 case MVT::f32:
1735 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32areg64;
1736 break;
1737 case MVT::f64:
1738 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64areg64;
1739 break;
1740 }
1741 } else {
1742 switch (EltVT.getSimpleVT().SimpleTy) {
1743 default:
1744 return nullptr;
1745 case MVT::i8:
1746 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8areg64;
1747 break;
1748 case MVT::i16:
1749 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16areg64;
1750 break;
1751 case MVT::i32:
1752 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32areg64;
1753 break;
1754 case MVT::i64:
1755 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64areg64;
1756 break;
1757 case MVT::f32:
1758 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32areg64;
1759 break;
1760 case MVT::f64:
1761 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64areg64;
1762 break;
1763 }
1764 }
1765 break;
1109 case NVPTXISD::LDGV2: 1766 case NVPTXISD::LDGV2:
1110 switch (EltVT.getSimpleVT().SimpleTy) { 1767 switch (EltVT.getSimpleVT().SimpleTy) {
1111 default: 1768 default:
1112 return NULL; 1769 return nullptr;
1113 case MVT::i8: 1770 case MVT::i8:
1114 Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_areg64; 1771 Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_areg64;
1115 break; 1772 break;
1116 case MVT::i16: 1773 case MVT::i16:
1117 Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_areg64; 1774 Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_areg64;
1131 } 1788 }
1132 break; 1789 break;
1133 case NVPTXISD::LDUV2: 1790 case NVPTXISD::LDUV2:
1134 switch (EltVT.getSimpleVT().SimpleTy) { 1791 switch (EltVT.getSimpleVT().SimpleTy) {
1135 default: 1792 default:
1136 return NULL; 1793 return nullptr;
1137 case MVT::i8: 1794 case MVT::i8:
1138 Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_areg64; 1795 Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_areg64;
1139 break; 1796 break;
1140 case MVT::i16: 1797 case MVT::i16:
1141 Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_areg64; 1798 Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_areg64;
1155 } 1812 }
1156 break; 1813 break;
1157 case NVPTXISD::LDGV4: 1814 case NVPTXISD::LDGV4:
1158 switch (EltVT.getSimpleVT().SimpleTy) { 1815 switch (EltVT.getSimpleVT().SimpleTy) {
1159 default: 1816 default:
1160 return NULL; 1817 return nullptr;
1161 case MVT::i8: 1818 case MVT::i8:
1162 Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_areg64; 1819 Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_areg64;
1163 break; 1820 break;
1164 case MVT::i16: 1821 case MVT::i16:
1165 Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_areg64; 1822 Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_areg64;
1173 } 1830 }
1174 break; 1831 break;
1175 case NVPTXISD::LDUV4: 1832 case NVPTXISD::LDUV4:
1176 switch (EltVT.getSimpleVT().SimpleTy) { 1833 switch (EltVT.getSimpleVT().SimpleTy) {
1177 default: 1834 default:
1178 return NULL; 1835 return nullptr;
1179 case MVT::i8: 1836 case MVT::i8:
1180 Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_areg64; 1837 Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_areg64;
1181 break; 1838 break;
1182 case MVT::i16: 1839 case MVT::i16:
1183 Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_areg64; 1840 Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_areg64;
1192 break; 1849 break;
1193 } 1850 }
1194 } else { 1851 } else {
1195 switch (N->getOpcode()) { 1852 switch (N->getOpcode()) {
1196 default: 1853 default:
1197 return NULL; 1854 return nullptr;
1855 case ISD::INTRINSIC_W_CHAIN:
1856 if (IsLDG) {
1857 switch (EltVT.getSimpleVT().SimpleTy) {
1858 default:
1859 return nullptr;
1860 case MVT::i8:
1861 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8areg;
1862 break;
1863 case MVT::i16:
1864 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16areg;
1865 break;
1866 case MVT::i32:
1867 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32areg;
1868 break;
1869 case MVT::i64:
1870 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64areg;
1871 break;
1872 case MVT::f32:
1873 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32areg;
1874 break;
1875 case MVT::f64:
1876 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64areg;
1877 break;
1878 }
1879 } else {
1880 switch (EltVT.getSimpleVT().SimpleTy) {
1881 default:
1882 return nullptr;
1883 case MVT::i8:
1884 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8areg;
1885 break;
1886 case MVT::i16:
1887 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16areg;
1888 break;
1889 case MVT::i32:
1890 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32areg;
1891 break;
1892 case MVT::i64:
1893 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64areg;
1894 break;
1895 case MVT::f32:
1896 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32areg;
1897 break;
1898 case MVT::f64:
1899 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64areg;
1900 break;
1901 }
1902 }
1903 break;
1198 case NVPTXISD::LDGV2: 1904 case NVPTXISD::LDGV2:
1199 switch (EltVT.getSimpleVT().SimpleTy) { 1905 switch (EltVT.getSimpleVT().SimpleTy) {
1200 default: 1906 default:
1201 return NULL; 1907 return nullptr;
1202 case MVT::i8: 1908 case MVT::i8:
1203 Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_areg32; 1909 Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_areg32;
1204 break; 1910 break;
1205 case MVT::i16: 1911 case MVT::i16:
1206 Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_areg32; 1912 Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_areg32;
1220 } 1926 }
1221 break; 1927 break;
1222 case NVPTXISD::LDUV2: 1928 case NVPTXISD::LDUV2:
1223 switch (EltVT.getSimpleVT().SimpleTy) { 1929 switch (EltVT.getSimpleVT().SimpleTy) {
1224 default: 1930 default:
1225 return NULL; 1931 return nullptr;
1226 case MVT::i8: 1932 case MVT::i8:
1227 Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_areg32; 1933 Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_areg32;
1228 break; 1934 break;
1229 case MVT::i16: 1935 case MVT::i16:
1230 Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_areg32; 1936 Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_areg32;
1244 } 1950 }
1245 break; 1951 break;
1246 case NVPTXISD::LDGV4: 1952 case NVPTXISD::LDGV4:
1247 switch (EltVT.getSimpleVT().SimpleTy) { 1953 switch (EltVT.getSimpleVT().SimpleTy) {
1248 default: 1954 default:
1249 return NULL; 1955 return nullptr;
1250 case MVT::i8: 1956 case MVT::i8:
1251 Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_areg32; 1957 Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_areg32;
1252 break; 1958 break;
1253 case MVT::i16: 1959 case MVT::i16:
1254 Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_areg32; 1960 Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_areg32;
1262 } 1968 }
1263 break; 1969 break;
1264 case NVPTXISD::LDUV4: 1970 case NVPTXISD::LDUV4:
1265 switch (EltVT.getSimpleVT().SimpleTy) { 1971 switch (EltVT.getSimpleVT().SimpleTy) {
1266 default: 1972 default:
1267 return NULL; 1973 return nullptr;
1268 case MVT::i8: 1974 case MVT::i8:
1269 Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_areg32; 1975 Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_areg32;
1270 break; 1976 break;
1271 case MVT::i16: 1977 case MVT::i16:
1272 Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_areg32; 1978 Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_areg32;
1281 break; 1987 break;
1282 } 1988 }
1283 } 1989 }
1284 1990
1285 SDValue Ops[] = { Op1, Chain }; 1991 SDValue Ops[] = { Op1, Chain };
1286 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), 1992 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
1287 ArrayRef<SDValue>(Ops, 2));
1288 } 1993 }
1289 1994
1290 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1); 1995 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
1291 MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand(); 1996 MemRefs0[0] = Mem->getMemOperand();
1292 cast<MachineSDNode>(LD)->setMemRefs(MemRefs0, MemRefs0 + 1); 1997 cast<MachineSDNode>(LD)->setMemRefs(MemRefs0, MemRefs0 + 1);
1293 1998
1294 return LD; 1999 return LD;
1295 } 2000 }
1296 2001
1297 SDNode *NVPTXDAGToDAGISel::SelectStore(SDNode *N) { 2002 SDNode *NVPTXDAGToDAGISel::SelectStore(SDNode *N) {
1298 SDLoc dl(N); 2003 SDLoc dl(N);
1299 StoreSDNode *ST = cast<StoreSDNode>(N); 2004 StoreSDNode *ST = cast<StoreSDNode>(N);
1300 EVT StoreVT = ST->getMemoryVT(); 2005 EVT StoreVT = ST->getMemoryVT();
1301 SDNode *NVPTXST = NULL; 2006 SDNode *NVPTXST = nullptr;
1302 2007
1303 // do not support pre/post inc/dec 2008 // do not support pre/post inc/dec
1304 if (ST->isIndexed()) 2009 if (ST->isIndexed())
1305 return NULL; 2010 return nullptr;
1306 2011
1307 if (!StoreVT.isSimple()) 2012 if (!StoreVT.isSimple())
1308 return NULL; 2013 return nullptr;
1309 2014
1310 // Address Space Setting 2015 // Address Space Setting
1311 unsigned int codeAddrSpace = getCodeAddrSpace(ST, Subtarget); 2016 unsigned int codeAddrSpace = getCodeAddrSpace(ST, Subtarget);
1312 2017
1313 // Volatile Setting 2018 // Volatile Setting
1326 if (num == 2) 2031 if (num == 2)
1327 vecType = NVPTX::PTXLdStInstCode::V2; 2032 vecType = NVPTX::PTXLdStInstCode::V2;
1328 else if (num == 4) 2033 else if (num == 4)
1329 vecType = NVPTX::PTXLdStInstCode::V4; 2034 vecType = NVPTX::PTXLdStInstCode::V4;
1330 else 2035 else
1331 return NULL; 2036 return nullptr;
1332 } 2037 }
1333 2038
1334 // Type Setting: toType + toTypeWidth 2039 // Type Setting: toType + toTypeWidth
1335 // - for integer type, always use 'u' 2040 // - for integer type, always use 'u'
1336 // 2041 //
1370 break; 2075 break;
1371 case MVT::f64: 2076 case MVT::f64:
1372 Opcode = NVPTX::ST_f64_avar; 2077 Opcode = NVPTX::ST_f64_avar;
1373 break; 2078 break;
1374 default: 2079 default:
1375 return NULL; 2080 return nullptr;
1376 } 2081 }
1377 SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace), 2082 SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
1378 getI32Imm(vecType), getI32Imm(toType), 2083 getI32Imm(vecType), getI32Imm(toType),
1379 getI32Imm(toTypeWidth), Addr, Chain }; 2084 getI32Imm(toTypeWidth), Addr, Chain };
1380 NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops); 2085 NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops);
1399 break; 2104 break;
1400 case MVT::f64: 2105 case MVT::f64:
1401 Opcode = NVPTX::ST_f64_asi; 2106 Opcode = NVPTX::ST_f64_asi;
1402 break; 2107 break;
1403 default: 2108 default:
1404 return NULL; 2109 return nullptr;
1405 } 2110 }
1406 SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace), 2111 SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
1407 getI32Imm(vecType), getI32Imm(toType), 2112 getI32Imm(vecType), getI32Imm(toType),
1408 getI32Imm(toTypeWidth), Base, Offset, Chain }; 2113 getI32Imm(toTypeWidth), Base, Offset, Chain };
1409 NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops); 2114 NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops);
1429 break; 2134 break;
1430 case MVT::f64: 2135 case MVT::f64:
1431 Opcode = NVPTX::ST_f64_ari_64; 2136 Opcode = NVPTX::ST_f64_ari_64;
1432 break; 2137 break;
1433 default: 2138 default:
1434 return NULL; 2139 return nullptr;
1435 } 2140 }
1436 } else { 2141 } else {
1437 switch (SourceVT) { 2142 switch (SourceVT) {
1438 case MVT::i8: 2143 case MVT::i8:
1439 Opcode = NVPTX::ST_i8_ari; 2144 Opcode = NVPTX::ST_i8_ari;
1452 break; 2157 break;
1453 case MVT::f64: 2158 case MVT::f64:
1454 Opcode = NVPTX::ST_f64_ari; 2159 Opcode = NVPTX::ST_f64_ari;
1455 break; 2160 break;
1456 default: 2161 default:
1457 return NULL; 2162 return nullptr;
1458 } 2163 }
1459 } 2164 }
1460 SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace), 2165 SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
1461 getI32Imm(vecType), getI32Imm(toType), 2166 getI32Imm(vecType), getI32Imm(toType),
1462 getI32Imm(toTypeWidth), Base, Offset, Chain }; 2167 getI32Imm(toTypeWidth), Base, Offset, Chain };
1481 break; 2186 break;
1482 case MVT::f64: 2187 case MVT::f64:
1483 Opcode = NVPTX::ST_f64_areg_64; 2188 Opcode = NVPTX::ST_f64_areg_64;
1484 break; 2189 break;
1485 default: 2190 default:
1486 return NULL; 2191 return nullptr;
1487 } 2192 }
1488 } else { 2193 } else {
1489 switch (SourceVT) { 2194 switch (SourceVT) {
1490 case MVT::i8: 2195 case MVT::i8:
1491 Opcode = NVPTX::ST_i8_areg; 2196 Opcode = NVPTX::ST_i8_areg;
1504 break; 2209 break;
1505 case MVT::f64: 2210 case MVT::f64:
1506 Opcode = NVPTX::ST_f64_areg; 2211 Opcode = NVPTX::ST_f64_areg;
1507 break; 2212 break;
1508 default: 2213 default:
1509 return NULL; 2214 return nullptr;
1510 } 2215 }
1511 } 2216 }
1512 SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace), 2217 SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
1513 getI32Imm(vecType), getI32Imm(toType), 2218 getI32Imm(vecType), getI32Imm(toType),
1514 getI32Imm(toTypeWidth), N2, Chain }; 2219 getI32Imm(toTypeWidth), N2, Chain };
1515 NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops); 2220 NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops);
1516 } 2221 }
1517 2222
1518 if (NVPTXST != NULL) { 2223 if (NVPTXST) {
1519 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1); 2224 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
1520 MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand(); 2225 MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
1521 cast<MachineSDNode>(NVPTXST)->setMemRefs(MemRefs0, MemRefs0 + 1); 2226 cast<MachineSDNode>(NVPTXST)->setMemRefs(MemRefs0, MemRefs0 + 1);
1522 } 2227 }
1523 2228
1580 StOps.push_back(N->getOperand(3)); 2285 StOps.push_back(N->getOperand(3));
1581 StOps.push_back(N->getOperand(4)); 2286 StOps.push_back(N->getOperand(4));
1582 N2 = N->getOperand(5); 2287 N2 = N->getOperand(5);
1583 break; 2288 break;
1584 default: 2289 default:
1585 return NULL; 2290 return nullptr;
1586 } 2291 }
1587 2292
1588 StOps.push_back(getI32Imm(IsVolatile)); 2293 StOps.push_back(getI32Imm(IsVolatile));
1589 StOps.push_back(getI32Imm(CodeAddrSpace)); 2294 StOps.push_back(getI32Imm(CodeAddrSpace));
1590 StOps.push_back(getI32Imm(VecType)); 2295 StOps.push_back(getI32Imm(VecType));
1592 StOps.push_back(getI32Imm(ToTypeWidth)); 2297 StOps.push_back(getI32Imm(ToTypeWidth));
1593 2298
1594 if (SelectDirectAddr(N2, Addr)) { 2299 if (SelectDirectAddr(N2, Addr)) {
1595 switch (N->getOpcode()) { 2300 switch (N->getOpcode()) {
1596 default: 2301 default:
1597 return NULL; 2302 return nullptr;
1598 case NVPTXISD::StoreV2: 2303 case NVPTXISD::StoreV2:
1599 switch (EltVT.getSimpleVT().SimpleTy) { 2304 switch (EltVT.getSimpleVT().SimpleTy) {
1600 default: 2305 default:
1601 return NULL; 2306 return nullptr;
1602 case MVT::i8: 2307 case MVT::i8:
1603 Opcode = NVPTX::STV_i8_v2_avar; 2308 Opcode = NVPTX::STV_i8_v2_avar;
1604 break; 2309 break;
1605 case MVT::i16: 2310 case MVT::i16:
1606 Opcode = NVPTX::STV_i16_v2_avar; 2311 Opcode = NVPTX::STV_i16_v2_avar;
1620 } 2325 }
1621 break; 2326 break;
1622 case NVPTXISD::StoreV4: 2327 case NVPTXISD::StoreV4:
1623 switch (EltVT.getSimpleVT().SimpleTy) { 2328 switch (EltVT.getSimpleVT().SimpleTy) {
1624 default: 2329 default:
1625 return NULL; 2330 return nullptr;
1626 case MVT::i8: 2331 case MVT::i8:
1627 Opcode = NVPTX::STV_i8_v4_avar; 2332 Opcode = NVPTX::STV_i8_v4_avar;
1628 break; 2333 break;
1629 case MVT::i16: 2334 case MVT::i16:
1630 Opcode = NVPTX::STV_i16_v4_avar; 2335 Opcode = NVPTX::STV_i16_v4_avar;
1642 } else if (Subtarget.is64Bit() 2347 } else if (Subtarget.is64Bit()
1643 ? SelectADDRsi64(N2.getNode(), N2, Base, Offset) 2348 ? SelectADDRsi64(N2.getNode(), N2, Base, Offset)
1644 : SelectADDRsi(N2.getNode(), N2, Base, Offset)) { 2349 : SelectADDRsi(N2.getNode(), N2, Base, Offset)) {
1645 switch (N->getOpcode()) { 2350 switch (N->getOpcode()) {
1646 default: 2351 default:
1647 return NULL; 2352 return nullptr;
1648 case NVPTXISD::StoreV2: 2353 case NVPTXISD::StoreV2:
1649 switch (EltVT.getSimpleVT().SimpleTy) { 2354 switch (EltVT.getSimpleVT().SimpleTy) {
1650 default: 2355 default:
1651 return NULL; 2356 return nullptr;
1652 case MVT::i8: 2357 case MVT::i8:
1653 Opcode = NVPTX::STV_i8_v2_asi; 2358 Opcode = NVPTX::STV_i8_v2_asi;
1654 break; 2359 break;
1655 case MVT::i16: 2360 case MVT::i16:
1656 Opcode = NVPTX::STV_i16_v2_asi; 2361 Opcode = NVPTX::STV_i16_v2_asi;
1670 } 2375 }
1671 break; 2376 break;
1672 case NVPTXISD::StoreV4: 2377 case NVPTXISD::StoreV4:
1673 switch (EltVT.getSimpleVT().SimpleTy) { 2378 switch (EltVT.getSimpleVT().SimpleTy) {
1674 default: 2379 default:
1675 return NULL; 2380 return nullptr;
1676 case MVT::i8: 2381 case MVT::i8:
1677 Opcode = NVPTX::STV_i8_v4_asi; 2382 Opcode = NVPTX::STV_i8_v4_asi;
1678 break; 2383 break;
1679 case MVT::i16: 2384 case MVT::i16:
1680 Opcode = NVPTX::STV_i16_v4_asi; 2385 Opcode = NVPTX::STV_i16_v4_asi;
1694 ? SelectADDRri64(N2.getNode(), N2, Base, Offset) 2399 ? SelectADDRri64(N2.getNode(), N2, Base, Offset)
1695 : SelectADDRri(N2.getNode(), N2, Base, Offset)) { 2400 : SelectADDRri(N2.getNode(), N2, Base, Offset)) {
1696 if (Subtarget.is64Bit()) { 2401 if (Subtarget.is64Bit()) {
1697 switch (N->getOpcode()) { 2402 switch (N->getOpcode()) {
1698 default: 2403 default:
1699 return NULL; 2404 return nullptr;
1700 case NVPTXISD::StoreV2: 2405 case NVPTXISD::StoreV2:
1701 switch (EltVT.getSimpleVT().SimpleTy) { 2406 switch (EltVT.getSimpleVT().SimpleTy) {
1702 default: 2407 default:
1703 return NULL; 2408 return nullptr;
1704 case MVT::i8: 2409 case MVT::i8:
1705 Opcode = NVPTX::STV_i8_v2_ari_64; 2410 Opcode = NVPTX::STV_i8_v2_ari_64;
1706 break; 2411 break;
1707 case MVT::i16: 2412 case MVT::i16:
1708 Opcode = NVPTX::STV_i16_v2_ari_64; 2413 Opcode = NVPTX::STV_i16_v2_ari_64;
1722 } 2427 }
1723 break; 2428 break;
1724 case NVPTXISD::StoreV4: 2429 case NVPTXISD::StoreV4:
1725 switch (EltVT.getSimpleVT().SimpleTy) { 2430 switch (EltVT.getSimpleVT().SimpleTy) {
1726 default: 2431 default:
1727 return NULL; 2432 return nullptr;
1728 case MVT::i8: 2433 case MVT::i8:
1729 Opcode = NVPTX::STV_i8_v4_ari_64; 2434 Opcode = NVPTX::STV_i8_v4_ari_64;
1730 break; 2435 break;
1731 case MVT::i16: 2436 case MVT::i16:
1732 Opcode = NVPTX::STV_i16_v4_ari_64; 2437 Opcode = NVPTX::STV_i16_v4_ari_64;
1741 break; 2446 break;
1742 } 2447 }
1743 } else { 2448 } else {
1744 switch (N->getOpcode()) { 2449 switch (N->getOpcode()) {
1745 default: 2450 default:
1746 return NULL; 2451 return nullptr;
1747 case NVPTXISD::StoreV2: 2452 case NVPTXISD::StoreV2:
1748 switch (EltVT.getSimpleVT().SimpleTy) { 2453 switch (EltVT.getSimpleVT().SimpleTy) {
1749 default: 2454 default:
1750 return NULL; 2455 return nullptr;
1751 case MVT::i8: 2456 case MVT::i8:
1752 Opcode = NVPTX::STV_i8_v2_ari; 2457 Opcode = NVPTX::STV_i8_v2_ari;
1753 break; 2458 break;
1754 case MVT::i16: 2459 case MVT::i16:
1755 Opcode = NVPTX::STV_i16_v2_ari; 2460 Opcode = NVPTX::STV_i16_v2_ari;
1769 } 2474 }
1770 break; 2475 break;
1771 case NVPTXISD::StoreV4: 2476 case NVPTXISD::StoreV4:
1772 switch (EltVT.getSimpleVT().SimpleTy) { 2477 switch (EltVT.getSimpleVT().SimpleTy) {
1773 default: 2478 default:
1774 return NULL; 2479 return nullptr;
1775 case MVT::i8: 2480 case MVT::i8:
1776 Opcode = NVPTX::STV_i8_v4_ari; 2481 Opcode = NVPTX::STV_i8_v4_ari;
1777 break; 2482 break;
1778 case MVT::i16: 2483 case MVT::i16:
1779 Opcode = NVPTX::STV_i16_v4_ari; 2484 Opcode = NVPTX::STV_i16_v4_ari;
1792 StOps.push_back(Offset); 2497 StOps.push_back(Offset);
1793 } else { 2498 } else {
1794 if (Subtarget.is64Bit()) { 2499 if (Subtarget.is64Bit()) {
1795 switch (N->getOpcode()) { 2500 switch (N->getOpcode()) {
1796 default: 2501 default:
1797 return NULL; 2502 return nullptr;
1798 case NVPTXISD::StoreV2: 2503 case NVPTXISD::StoreV2:
1799 switch (EltVT.getSimpleVT().SimpleTy) { 2504 switch (EltVT.getSimpleVT().SimpleTy) {
1800 default: 2505 default:
1801 return NULL; 2506 return nullptr;
1802 case MVT::i8: 2507 case MVT::i8:
1803 Opcode = NVPTX::STV_i8_v2_areg_64; 2508 Opcode = NVPTX::STV_i8_v2_areg_64;
1804 break; 2509 break;
1805 case MVT::i16: 2510 case MVT::i16:
1806 Opcode = NVPTX::STV_i16_v2_areg_64; 2511 Opcode = NVPTX::STV_i16_v2_areg_64;
1820 } 2525 }
1821 break; 2526 break;
1822 case NVPTXISD::StoreV4: 2527 case NVPTXISD::StoreV4:
1823 switch (EltVT.getSimpleVT().SimpleTy) { 2528 switch (EltVT.getSimpleVT().SimpleTy) {
1824 default: 2529 default:
1825 return NULL; 2530 return nullptr;
1826 case MVT::i8: 2531 case MVT::i8:
1827 Opcode = NVPTX::STV_i8_v4_areg_64; 2532 Opcode = NVPTX::STV_i8_v4_areg_64;
1828 break; 2533 break;
1829 case MVT::i16: 2534 case MVT::i16:
1830 Opcode = NVPTX::STV_i16_v4_areg_64; 2535 Opcode = NVPTX::STV_i16_v4_areg_64;
1839 break; 2544 break;
1840 } 2545 }
1841 } else { 2546 } else {
1842 switch (N->getOpcode()) { 2547 switch (N->getOpcode()) {
1843 default: 2548 default:
1844 return NULL; 2549 return nullptr;
1845 case NVPTXISD::StoreV2: 2550 case NVPTXISD::StoreV2:
1846 switch (EltVT.getSimpleVT().SimpleTy) { 2551 switch (EltVT.getSimpleVT().SimpleTy) {
1847 default: 2552 default:
1848 return NULL; 2553 return nullptr;
1849 case MVT::i8: 2554 case MVT::i8:
1850 Opcode = NVPTX::STV_i8_v2_areg; 2555 Opcode = NVPTX::STV_i8_v2_areg;
1851 break; 2556 break;
1852 case MVT::i16: 2557 case MVT::i16:
1853 Opcode = NVPTX::STV_i16_v2_areg; 2558 Opcode = NVPTX::STV_i16_v2_areg;
1867 } 2572 }
1868 break; 2573 break;
1869 case NVPTXISD::StoreV4: 2574 case NVPTXISD::StoreV4:
1870 switch (EltVT.getSimpleVT().SimpleTy) { 2575 switch (EltVT.getSimpleVT().SimpleTy) {
1871 default: 2576 default:
1872 return NULL; 2577 return nullptr;
1873 case MVT::i8: 2578 case MVT::i8:
1874 Opcode = NVPTX::STV_i8_v4_areg; 2579 Opcode = NVPTX::STV_i8_v4_areg;
1875 break; 2580 break;
1876 case MVT::i16: 2581 case MVT::i16:
1877 Opcode = NVPTX::STV_i16_v4_areg; 2582 Opcode = NVPTX::STV_i16_v4_areg;
1908 MemSDNode *Mem = cast<MemSDNode>(Node); 2613 MemSDNode *Mem = cast<MemSDNode>(Node);
1909 2614
1910 unsigned VecSize; 2615 unsigned VecSize;
1911 switch (Node->getOpcode()) { 2616 switch (Node->getOpcode()) {
1912 default: 2617 default:
1913 return NULL; 2618 return nullptr;
1914 case NVPTXISD::LoadParam: 2619 case NVPTXISD::LoadParam:
1915 VecSize = 1; 2620 VecSize = 1;
1916 break; 2621 break;
1917 case NVPTXISD::LoadParamV2: 2622 case NVPTXISD::LoadParamV2:
1918 VecSize = 2; 2623 VecSize = 2;
1927 2632
1928 unsigned Opc = 0; 2633 unsigned Opc = 0;
1929 2634
1930 switch (VecSize) { 2635 switch (VecSize) {
1931 default: 2636 default:
1932 return NULL; 2637 return nullptr;
1933 case 1: 2638 case 1:
1934 switch (MemVT.getSimpleVT().SimpleTy) { 2639 switch (MemVT.getSimpleVT().SimpleTy) {
1935 default: 2640 default:
1936 return NULL; 2641 return nullptr;
1937 case MVT::i1: 2642 case MVT::i1:
1938 Opc = NVPTX::LoadParamMemI8; 2643 Opc = NVPTX::LoadParamMemI8;
1939 break; 2644 break;
1940 case MVT::i8: 2645 case MVT::i8:
1941 Opc = NVPTX::LoadParamMemI8; 2646 Opc = NVPTX::LoadParamMemI8;
1958 } 2663 }
1959 break; 2664 break;
1960 case 2: 2665 case 2:
1961 switch (MemVT.getSimpleVT().SimpleTy) { 2666 switch (MemVT.getSimpleVT().SimpleTy) {
1962 default: 2667 default:
1963 return NULL; 2668 return nullptr;
1964 case MVT::i1: 2669 case MVT::i1:
1965 Opc = NVPTX::LoadParamMemV2I8; 2670 Opc = NVPTX::LoadParamMemV2I8;
1966 break; 2671 break;
1967 case MVT::i8: 2672 case MVT::i8:
1968 Opc = NVPTX::LoadParamMemV2I8; 2673 Opc = NVPTX::LoadParamMemV2I8;
1985 } 2690 }
1986 break; 2691 break;
1987 case 4: 2692 case 4:
1988 switch (MemVT.getSimpleVT().SimpleTy) { 2693 switch (MemVT.getSimpleVT().SimpleTy) {
1989 default: 2694 default:
1990 return NULL; 2695 return nullptr;
1991 case MVT::i1: 2696 case MVT::i1:
1992 Opc = NVPTX::LoadParamMemV4I8; 2697 Opc = NVPTX::LoadParamMemV4I8;
1993 break; 2698 break;
1994 case MVT::i8: 2699 case MVT::i8:
1995 Opc = NVPTX::LoadParamMemV4I8; 2700 Opc = NVPTX::LoadParamMemV4I8;
2012 VTs = CurDAG->getVTList(EltVT, MVT::Other, MVT::Glue); 2717 VTs = CurDAG->getVTList(EltVT, MVT::Other, MVT::Glue);
2013 } else if (VecSize == 2) { 2718 } else if (VecSize == 2) {
2014 VTs = CurDAG->getVTList(EltVT, EltVT, MVT::Other, MVT::Glue); 2719 VTs = CurDAG->getVTList(EltVT, EltVT, MVT::Other, MVT::Glue);
2015 } else { 2720 } else {
2016 EVT EVTs[] = { EltVT, EltVT, EltVT, EltVT, MVT::Other, MVT::Glue }; 2721 EVT EVTs[] = { EltVT, EltVT, EltVT, EltVT, MVT::Other, MVT::Glue };
2017 VTs = CurDAG->getVTList(&EVTs[0], array_lengthof(EVTs)); 2722 VTs = CurDAG->getVTList(EVTs);
2018 } 2723 }
2019 2724
2020 unsigned OffsetVal = cast<ConstantSDNode>(Offset)->getZExtValue(); 2725 unsigned OffsetVal = cast<ConstantSDNode>(Offset)->getZExtValue();
2021 2726
2022 SmallVector<SDValue, 2> Ops; 2727 SmallVector<SDValue, 2> Ops;
2038 2743
2039 // How many elements do we have? 2744 // How many elements do we have?
2040 unsigned NumElts = 1; 2745 unsigned NumElts = 1;
2041 switch (N->getOpcode()) { 2746 switch (N->getOpcode()) {
2042 default: 2747 default:
2043 return NULL; 2748 return nullptr;
2044 case NVPTXISD::StoreRetval: 2749 case NVPTXISD::StoreRetval:
2045 NumElts = 1; 2750 NumElts = 1;
2046 break; 2751 break;
2047 case NVPTXISD::StoreRetvalV2: 2752 case NVPTXISD::StoreRetvalV2:
2048 NumElts = 2; 2753 NumElts = 2;
2063 // If we have an i1, use an 8-bit store. The lowering code in 2768 // If we have an i1, use an 8-bit store. The lowering code in
2064 // NVPTXISelLowering will have already emitted an upcast. 2769 // NVPTXISelLowering will have already emitted an upcast.
2065 unsigned Opcode = 0; 2770 unsigned Opcode = 0;
2066 switch (NumElts) { 2771 switch (NumElts) {
2067 default: 2772 default:
2068 return NULL; 2773 return nullptr;
2069 case 1: 2774 case 1:
2070 switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) { 2775 switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
2071 default: 2776 default:
2072 return NULL; 2777 return nullptr;
2073 case MVT::i1: 2778 case MVT::i1:
2074 Opcode = NVPTX::StoreRetvalI8; 2779 Opcode = NVPTX::StoreRetvalI8;
2075 break; 2780 break;
2076 case MVT::i8: 2781 case MVT::i8:
2077 Opcode = NVPTX::StoreRetvalI8; 2782 Opcode = NVPTX::StoreRetvalI8;
2094 } 2799 }
2095 break; 2800 break;
2096 case 2: 2801 case 2:
2097 switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) { 2802 switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
2098 default: 2803 default:
2099 return NULL; 2804 return nullptr;
2100 case MVT::i1: 2805 case MVT::i1:
2101 Opcode = NVPTX::StoreRetvalV2I8; 2806 Opcode = NVPTX::StoreRetvalV2I8;
2102 break; 2807 break;
2103 case MVT::i8: 2808 case MVT::i8:
2104 Opcode = NVPTX::StoreRetvalV2I8; 2809 Opcode = NVPTX::StoreRetvalV2I8;
2121 } 2826 }
2122 break; 2827 break;
2123 case 4: 2828 case 4:
2124 switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) { 2829 switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
2125 default: 2830 default:
2126 return NULL; 2831 return nullptr;
2127 case MVT::i1: 2832 case MVT::i1:
2128 Opcode = NVPTX::StoreRetvalV4I8; 2833 Opcode = NVPTX::StoreRetvalV4I8;
2129 break; 2834 break;
2130 case MVT::i8: 2835 case MVT::i8:
2131 Opcode = NVPTX::StoreRetvalV4I8; 2836 Opcode = NVPTX::StoreRetvalV4I8;
2164 2869
2165 // How many elements do we have? 2870 // How many elements do we have?
2166 unsigned NumElts = 1; 2871 unsigned NumElts = 1;
2167 switch (N->getOpcode()) { 2872 switch (N->getOpcode()) {
2168 default: 2873 default:
2169 return NULL; 2874 return nullptr;
2170 case NVPTXISD::StoreParamU32: 2875 case NVPTXISD::StoreParamU32:
2171 case NVPTXISD::StoreParamS32: 2876 case NVPTXISD::StoreParamS32:
2172 case NVPTXISD::StoreParam: 2877 case NVPTXISD::StoreParam:
2173 NumElts = 1; 2878 NumElts = 1;
2174 break; 2879 break;
2195 unsigned Opcode = 0; 2900 unsigned Opcode = 0;
2196 switch (N->getOpcode()) { 2901 switch (N->getOpcode()) {
2197 default: 2902 default:
2198 switch (NumElts) { 2903 switch (NumElts) {
2199 default: 2904 default:
2200 return NULL; 2905 return nullptr;
2201 case 1: 2906 case 1:
2202 switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) { 2907 switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
2203 default: 2908 default:
2204 return NULL; 2909 return nullptr;
2205 case MVT::i1: 2910 case MVT::i1:
2206 Opcode = NVPTX::StoreParamI8; 2911 Opcode = NVPTX::StoreParamI8;
2207 break; 2912 break;
2208 case MVT::i8: 2913 case MVT::i8:
2209 Opcode = NVPTX::StoreParamI8; 2914 Opcode = NVPTX::StoreParamI8;
2226 } 2931 }
2227 break; 2932 break;
2228 case 2: 2933 case 2:
2229 switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) { 2934 switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
2230 default: 2935 default:
2231 return NULL; 2936 return nullptr;
2232 case MVT::i1: 2937 case MVT::i1:
2233 Opcode = NVPTX::StoreParamV2I8; 2938 Opcode = NVPTX::StoreParamV2I8;
2234 break; 2939 break;
2235 case MVT::i8: 2940 case MVT::i8:
2236 Opcode = NVPTX::StoreParamV2I8; 2941 Opcode = NVPTX::StoreParamV2I8;
2253 } 2958 }
2254 break; 2959 break;
2255 case 4: 2960 case 4:
2256 switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) { 2961 switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
2257 default: 2962 default:
2258 return NULL; 2963 return nullptr;
2259 case MVT::i1: 2964 case MVT::i1:
2260 Opcode = NVPTX::StoreParamV4I8; 2965 Opcode = NVPTX::StoreParamV4I8;
2261 break; 2966 break;
2262 case MVT::i8: 2967 case MVT::i8:
2263 Opcode = NVPTX::StoreParamV4I8; 2968 Opcode = NVPTX::StoreParamV4I8;
2302 SDNode *Ret = 3007 SDNode *Ret =
2303 CurDAG->getMachineNode(Opcode, DL, RetVTs, Ops); 3008 CurDAG->getMachineNode(Opcode, DL, RetVTs, Ops);
2304 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1); 3009 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
2305 MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand(); 3010 MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
2306 cast<MachineSDNode>(Ret)->setMemRefs(MemRefs0, MemRefs0 + 1); 3011 cast<MachineSDNode>(Ret)->setMemRefs(MemRefs0, MemRefs0 + 1);
3012
3013 return Ret;
3014 }
3015
3016 SDNode *NVPTXDAGToDAGISel::SelectTextureIntrinsic(SDNode *N) {
3017 SDValue Chain = N->getOperand(0);
3018 SDNode *Ret = nullptr;
3019 unsigned Opc = 0;
3020 SmallVector<SDValue, 8> Ops;
3021
3022 switch (N->getOpcode()) {
3023 default: return nullptr;
3024 case NVPTXISD::Tex1DFloatS32:
3025 Opc = NVPTX::TEX_1D_F32_S32;
3026 break;
3027 case NVPTXISD::Tex1DFloatFloat:
3028 Opc = NVPTX::TEX_1D_F32_F32;
3029 break;
3030 case NVPTXISD::Tex1DFloatFloatLevel:
3031 Opc = NVPTX::TEX_1D_F32_F32_LEVEL;
3032 break;
3033 case NVPTXISD::Tex1DFloatFloatGrad:
3034 Opc = NVPTX::TEX_1D_F32_F32_GRAD;
3035 break;
3036 case NVPTXISD::Tex1DS32S32:
3037 Opc = NVPTX::TEX_1D_S32_S32;
3038 break;
3039 case NVPTXISD::Tex1DS32Float:
3040 Opc = NVPTX::TEX_1D_S32_F32;
3041 break;
3042 case NVPTXISD::Tex1DS32FloatLevel:
3043 Opc = NVPTX::TEX_1D_S32_F32_LEVEL;
3044 break;
3045 case NVPTXISD::Tex1DS32FloatGrad:
3046 Opc = NVPTX::TEX_1D_S32_F32_GRAD;
3047 break;
3048 case NVPTXISD::Tex1DU32S32:
3049 Opc = NVPTX::TEX_1D_U32_S32;
3050 break;
3051 case NVPTXISD::Tex1DU32Float:
3052 Opc = NVPTX::TEX_1D_U32_F32;
3053 break;
3054 case NVPTXISD::Tex1DU32FloatLevel:
3055 Opc = NVPTX::TEX_1D_U32_F32_LEVEL;
3056 break;
3057 case NVPTXISD::Tex1DU32FloatGrad:
3058 Opc = NVPTX::TEX_1D_U32_F32_GRAD;
3059 break;
3060 case NVPTXISD::Tex1DArrayFloatS32:
3061 Opc = NVPTX::TEX_1D_ARRAY_F32_S32;
3062 break;
3063 case NVPTXISD::Tex1DArrayFloatFloat:
3064 Opc = NVPTX::TEX_1D_ARRAY_F32_F32;
3065 break;
3066 case NVPTXISD::Tex1DArrayFloatFloatLevel:
3067 Opc = NVPTX::TEX_1D_ARRAY_F32_F32_LEVEL;
3068 break;
3069 case NVPTXISD::Tex1DArrayFloatFloatGrad:
3070 Opc = NVPTX::TEX_1D_ARRAY_F32_F32_GRAD;
3071 break;
3072 case NVPTXISD::Tex1DArrayS32S32:
3073 Opc = NVPTX::TEX_1D_ARRAY_S32_S32;
3074 break;
3075 case NVPTXISD::Tex1DArrayS32Float:
3076 Opc = NVPTX::TEX_1D_ARRAY_S32_F32;
3077 break;
3078 case NVPTXISD::Tex1DArrayS32FloatLevel:
3079 Opc = NVPTX::TEX_1D_ARRAY_S32_F32_LEVEL;
3080 break;
3081 case NVPTXISD::Tex1DArrayS32FloatGrad:
3082 Opc = NVPTX::TEX_1D_ARRAY_S32_F32_GRAD;
3083 break;
3084 case NVPTXISD::Tex1DArrayU32S32:
3085 Opc = NVPTX::TEX_1D_ARRAY_U32_S32;
3086 break;
3087 case NVPTXISD::Tex1DArrayU32Float:
3088 Opc = NVPTX::TEX_1D_ARRAY_U32_F32;
3089 break;
3090 case NVPTXISD::Tex1DArrayU32FloatLevel:
3091 Opc = NVPTX::TEX_1D_ARRAY_U32_F32_LEVEL;
3092 break;
3093 case NVPTXISD::Tex1DArrayU32FloatGrad:
3094 Opc = NVPTX::TEX_1D_ARRAY_U32_F32_GRAD;
3095 break;
3096 case NVPTXISD::Tex2DFloatS32:
3097 Opc = NVPTX::TEX_2D_F32_S32;
3098 break;
3099 case NVPTXISD::Tex2DFloatFloat:
3100 Opc = NVPTX::TEX_2D_F32_F32;
3101 break;
3102 case NVPTXISD::Tex2DFloatFloatLevel:
3103 Opc = NVPTX::TEX_2D_F32_F32_LEVEL;
3104 break;
3105 case NVPTXISD::Tex2DFloatFloatGrad:
3106 Opc = NVPTX::TEX_2D_F32_F32_GRAD;
3107 break;
3108 case NVPTXISD::Tex2DS32S32:
3109 Opc = NVPTX::TEX_2D_S32_S32;
3110 break;
3111 case NVPTXISD::Tex2DS32Float:
3112 Opc = NVPTX::TEX_2D_S32_F32;
3113 break;
3114 case NVPTXISD::Tex2DS32FloatLevel:
3115 Opc = NVPTX::TEX_2D_S32_F32_LEVEL;
3116 break;
3117 case NVPTXISD::Tex2DS32FloatGrad:
3118 Opc = NVPTX::TEX_2D_S32_F32_GRAD;
3119 break;
3120 case NVPTXISD::Tex2DU32S32:
3121 Opc = NVPTX::TEX_2D_U32_S32;
3122 break;
3123 case NVPTXISD::Tex2DU32Float:
3124 Opc = NVPTX::TEX_2D_U32_F32;
3125 break;
3126 case NVPTXISD::Tex2DU32FloatLevel:
3127 Opc = NVPTX::TEX_2D_U32_F32_LEVEL;
3128 break;
3129 case NVPTXISD::Tex2DU32FloatGrad:
3130 Opc = NVPTX::TEX_2D_U32_F32_GRAD;
3131 break;
3132 case NVPTXISD::Tex2DArrayFloatS32:
3133 Opc = NVPTX::TEX_2D_ARRAY_F32_S32;
3134 break;
3135 case NVPTXISD::Tex2DArrayFloatFloat:
3136 Opc = NVPTX::TEX_2D_ARRAY_F32_F32;
3137 break;
3138 case NVPTXISD::Tex2DArrayFloatFloatLevel:
3139 Opc = NVPTX::TEX_2D_ARRAY_F32_F32_LEVEL;
3140 break;
3141 case NVPTXISD::Tex2DArrayFloatFloatGrad:
3142 Opc = NVPTX::TEX_2D_ARRAY_F32_F32_GRAD;
3143 break;
3144 case NVPTXISD::Tex2DArrayS32S32:
3145 Opc = NVPTX::TEX_2D_ARRAY_S32_S32;
3146 break;
3147 case NVPTXISD::Tex2DArrayS32Float:
3148 Opc = NVPTX::TEX_2D_ARRAY_S32_F32;
3149 break;
3150 case NVPTXISD::Tex2DArrayS32FloatLevel:
3151 Opc = NVPTX::TEX_2D_ARRAY_S32_F32_LEVEL;
3152 break;
3153 case NVPTXISD::Tex2DArrayS32FloatGrad:
3154 Opc = NVPTX::TEX_2D_ARRAY_S32_F32_GRAD;
3155 break;
3156 case NVPTXISD::Tex2DArrayU32S32:
3157 Opc = NVPTX::TEX_2D_ARRAY_U32_S32;
3158 break;
3159 case NVPTXISD::Tex2DArrayU32Float:
3160 Opc = NVPTX::TEX_2D_ARRAY_U32_F32;
3161 break;
3162 case NVPTXISD::Tex2DArrayU32FloatLevel:
3163 Opc = NVPTX::TEX_2D_ARRAY_U32_F32_LEVEL;
3164 break;
3165 case NVPTXISD::Tex2DArrayU32FloatGrad:
3166 Opc = NVPTX::TEX_2D_ARRAY_U32_F32_GRAD;
3167 break;
3168 case NVPTXISD::Tex3DFloatS32:
3169 Opc = NVPTX::TEX_3D_F32_S32;
3170 break;
3171 case NVPTXISD::Tex3DFloatFloat:
3172 Opc = NVPTX::TEX_3D_F32_F32;
3173 break;
3174 case NVPTXISD::Tex3DFloatFloatLevel:
3175 Opc = NVPTX::TEX_3D_F32_F32_LEVEL;
3176 break;
3177 case NVPTXISD::Tex3DFloatFloatGrad:
3178 Opc = NVPTX::TEX_3D_F32_F32_GRAD;
3179 break;
3180 case NVPTXISD::Tex3DS32S32:
3181 Opc = NVPTX::TEX_3D_S32_S32;
3182 break;
3183 case NVPTXISD::Tex3DS32Float:
3184 Opc = NVPTX::TEX_3D_S32_F32;
3185 break;
3186 case NVPTXISD::Tex3DS32FloatLevel:
3187 Opc = NVPTX::TEX_3D_S32_F32_LEVEL;
3188 break;
3189 case NVPTXISD::Tex3DS32FloatGrad:
3190 Opc = NVPTX::TEX_3D_S32_F32_GRAD;
3191 break;
3192 case NVPTXISD::Tex3DU32S32:
3193 Opc = NVPTX::TEX_3D_U32_S32;
3194 break;
3195 case NVPTXISD::Tex3DU32Float:
3196 Opc = NVPTX::TEX_3D_U32_F32;
3197 break;
3198 case NVPTXISD::Tex3DU32FloatLevel:
3199 Opc = NVPTX::TEX_3D_U32_F32_LEVEL;
3200 break;
3201 case NVPTXISD::Tex3DU32FloatGrad:
3202 Opc = NVPTX::TEX_3D_U32_F32_GRAD;
3203 break;
3204 case NVPTXISD::TexCubeFloatFloat:
3205 Opc = NVPTX::TEX_CUBE_F32_F32;
3206 break;
3207 case NVPTXISD::TexCubeFloatFloatLevel:
3208 Opc = NVPTX::TEX_CUBE_F32_F32_LEVEL;
3209 break;
3210 case NVPTXISD::TexCubeS32Float:
3211 Opc = NVPTX::TEX_CUBE_S32_F32;
3212 break;
3213 case NVPTXISD::TexCubeS32FloatLevel:
3214 Opc = NVPTX::TEX_CUBE_S32_F32_LEVEL;
3215 break;
3216 case NVPTXISD::TexCubeU32Float:
3217 Opc = NVPTX::TEX_CUBE_U32_F32;
3218 break;
3219 case NVPTXISD::TexCubeU32FloatLevel:
3220 Opc = NVPTX::TEX_CUBE_U32_F32_LEVEL;
3221 break;
3222 case NVPTXISD::TexCubeArrayFloatFloat:
3223 Opc = NVPTX::TEX_CUBE_ARRAY_F32_F32;
3224 break;
3225 case NVPTXISD::TexCubeArrayFloatFloatLevel:
3226 Opc = NVPTX::TEX_CUBE_ARRAY_F32_F32_LEVEL;
3227 break;
3228 case NVPTXISD::TexCubeArrayS32Float:
3229 Opc = NVPTX::TEX_CUBE_ARRAY_S32_F32;
3230 break;
3231 case NVPTXISD::TexCubeArrayS32FloatLevel:
3232 Opc = NVPTX::TEX_CUBE_ARRAY_S32_F32_LEVEL;
3233 break;
3234 case NVPTXISD::TexCubeArrayU32Float:
3235 Opc = NVPTX::TEX_CUBE_ARRAY_U32_F32;
3236 break;
3237 case NVPTXISD::TexCubeArrayU32FloatLevel:
3238 Opc = NVPTX::TEX_CUBE_ARRAY_U32_F32_LEVEL;
3239 break;
3240 case NVPTXISD::Tld4R2DFloatFloat:
3241 Opc = NVPTX::TLD4_R_2D_F32_F32;
3242 break;
3243 case NVPTXISD::Tld4G2DFloatFloat:
3244 Opc = NVPTX::TLD4_G_2D_F32_F32;
3245 break;
3246 case NVPTXISD::Tld4B2DFloatFloat:
3247 Opc = NVPTX::TLD4_B_2D_F32_F32;
3248 break;
3249 case NVPTXISD::Tld4A2DFloatFloat:
3250 Opc = NVPTX::TLD4_A_2D_F32_F32;
3251 break;
3252 case NVPTXISD::Tld4R2DS64Float:
3253 Opc = NVPTX::TLD4_R_2D_S32_F32;
3254 break;
3255 case NVPTXISD::Tld4G2DS64Float:
3256 Opc = NVPTX::TLD4_G_2D_S32_F32;
3257 break;
3258 case NVPTXISD::Tld4B2DS64Float:
3259 Opc = NVPTX::TLD4_B_2D_S32_F32;
3260 break;
3261 case NVPTXISD::Tld4A2DS64Float:
3262 Opc = NVPTX::TLD4_A_2D_S32_F32;
3263 break;
3264 case NVPTXISD::Tld4R2DU64Float:
3265 Opc = NVPTX::TLD4_R_2D_U32_F32;
3266 break;
3267 case NVPTXISD::Tld4G2DU64Float:
3268 Opc = NVPTX::TLD4_G_2D_U32_F32;
3269 break;
3270 case NVPTXISD::Tld4B2DU64Float:
3271 Opc = NVPTX::TLD4_B_2D_U32_F32;
3272 break;
3273 case NVPTXISD::Tld4A2DU64Float:
3274 Opc = NVPTX::TLD4_A_2D_U32_F32;
3275 break;
3276 case NVPTXISD::TexUnified1DFloatS32:
3277 Opc = NVPTX::TEX_UNIFIED_1D_F32_S32;
3278 break;
3279 case NVPTXISD::TexUnified1DFloatFloat:
3280 Opc = NVPTX::TEX_UNIFIED_1D_F32_F32;
3281 break;
3282 case NVPTXISD::TexUnified1DFloatFloatLevel:
3283 Opc = NVPTX::TEX_UNIFIED_1D_F32_F32_LEVEL;
3284 break;
3285 case NVPTXISD::TexUnified1DFloatFloatGrad:
3286 Opc = NVPTX::TEX_UNIFIED_1D_F32_F32_GRAD;
3287 break;
3288 case NVPTXISD::TexUnified1DS32S32:
3289 Opc = NVPTX::TEX_UNIFIED_1D_S32_S32;
3290 break;
3291 case NVPTXISD::TexUnified1DS32Float:
3292 Opc = NVPTX::TEX_UNIFIED_1D_S32_F32;
3293 break;
3294 case NVPTXISD::TexUnified1DS32FloatLevel:
3295 Opc = NVPTX::TEX_UNIFIED_1D_S32_F32_LEVEL;
3296 break;
3297 case NVPTXISD::TexUnified1DS32FloatGrad:
3298 Opc = NVPTX::TEX_UNIFIED_1D_S32_F32_GRAD;
3299 break;
3300 case NVPTXISD::TexUnified1DU32S32:
3301 Opc = NVPTX::TEX_UNIFIED_1D_U32_S32;
3302 break;
3303 case NVPTXISD::TexUnified1DU32Float:
3304 Opc = NVPTX::TEX_UNIFIED_1D_U32_F32;
3305 break;
3306 case NVPTXISD::TexUnified1DU32FloatLevel:
3307 Opc = NVPTX::TEX_UNIFIED_1D_U32_F32_LEVEL;
3308 break;
3309 case NVPTXISD::TexUnified1DU32FloatGrad:
3310 Opc = NVPTX::TEX_UNIFIED_1D_U32_F32_GRAD;
3311 break;
3312 case NVPTXISD::TexUnified1DArrayFloatS32:
3313 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_F32_S32;
3314 break;
3315 case NVPTXISD::TexUnified1DArrayFloatFloat:
3316 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_F32_F32;
3317 break;
3318 case NVPTXISD::TexUnified1DArrayFloatFloatLevel:
3319 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_F32_F32_LEVEL;
3320 break;
3321 case NVPTXISD::TexUnified1DArrayFloatFloatGrad:
3322 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_F32_F32_GRAD;
3323 break;
3324 case NVPTXISD::TexUnified1DArrayS32S32:
3325 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_S32_S32;
3326 break;
3327 case NVPTXISD::TexUnified1DArrayS32Float:
3328 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_S32_F32;
3329 break;
3330 case NVPTXISD::TexUnified1DArrayS32FloatLevel:
3331 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_S32_F32_LEVEL;
3332 break;
3333 case NVPTXISD::TexUnified1DArrayS32FloatGrad:
3334 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_S32_F32_GRAD;
3335 break;
3336 case NVPTXISD::TexUnified1DArrayU32S32:
3337 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_U32_S32;
3338 break;
3339 case NVPTXISD::TexUnified1DArrayU32Float:
3340 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_U32_F32;
3341 break;
3342 case NVPTXISD::TexUnified1DArrayU32FloatLevel:
3343 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_U32_F32_LEVEL;
3344 break;
3345 case NVPTXISD::TexUnified1DArrayU32FloatGrad:
3346 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_U32_F32_GRAD;
3347 break;
3348 case NVPTXISD::TexUnified2DFloatS32:
3349 Opc = NVPTX::TEX_UNIFIED_2D_F32_S32;
3350 break;
3351 case NVPTXISD::TexUnified2DFloatFloat:
3352 Opc = NVPTX::TEX_UNIFIED_2D_F32_F32;
3353 break;
3354 case NVPTXISD::TexUnified2DFloatFloatLevel:
3355 Opc = NVPTX::TEX_UNIFIED_2D_F32_F32_LEVEL;
3356 break;
3357 case NVPTXISD::TexUnified2DFloatFloatGrad:
3358 Opc = NVPTX::TEX_UNIFIED_2D_F32_F32_GRAD;
3359 break;
3360 case NVPTXISD::TexUnified2DS32S32:
3361 Opc = NVPTX::TEX_UNIFIED_2D_S32_S32;
3362 break;
3363 case NVPTXISD::TexUnified2DS32Float:
3364 Opc = NVPTX::TEX_UNIFIED_2D_S32_F32;
3365 break;
3366 case NVPTXISD::TexUnified2DS32FloatLevel:
3367 Opc = NVPTX::TEX_UNIFIED_2D_S32_F32_LEVEL;
3368 break;
3369 case NVPTXISD::TexUnified2DS32FloatGrad:
3370 Opc = NVPTX::TEX_UNIFIED_2D_S32_F32_GRAD;
3371 break;
3372 case NVPTXISD::TexUnified2DU32S32:
3373 Opc = NVPTX::TEX_UNIFIED_2D_U32_S32;
3374 break;
3375 case NVPTXISD::TexUnified2DU32Float:
3376 Opc = NVPTX::TEX_UNIFIED_2D_U32_F32;
3377 break;
3378 case NVPTXISD::TexUnified2DU32FloatLevel:
3379 Opc = NVPTX::TEX_UNIFIED_2D_U32_F32_LEVEL;
3380 break;
3381 case NVPTXISD::TexUnified2DU32FloatGrad:
3382 Opc = NVPTX::TEX_UNIFIED_2D_U32_F32_GRAD;
3383 break;
3384 case NVPTXISD::TexUnified2DArrayFloatS32:
3385 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_F32_S32;
3386 break;
3387 case NVPTXISD::TexUnified2DArrayFloatFloat:
3388 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_F32_F32;
3389 break;
3390 case NVPTXISD::TexUnified2DArrayFloatFloatLevel:
3391 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_F32_F32_LEVEL;
3392 break;
3393 case NVPTXISD::TexUnified2DArrayFloatFloatGrad:
3394 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_F32_F32_GRAD;
3395 break;
3396 case NVPTXISD::TexUnified2DArrayS32S32:
3397 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_S32_S32;
3398 break;
3399 case NVPTXISD::TexUnified2DArrayS32Float:
3400 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_S32_F32;
3401 break;
3402 case NVPTXISD::TexUnified2DArrayS32FloatLevel:
3403 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_S32_F32_LEVEL;
3404 break;
3405 case NVPTXISD::TexUnified2DArrayS32FloatGrad:
3406 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_S32_F32_GRAD;
3407 break;
3408 case NVPTXISD::TexUnified2DArrayU32S32:
3409 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_U32_S32;
3410 break;
3411 case NVPTXISD::TexUnified2DArrayU32Float:
3412 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_U32_F32;
3413 break;
3414 case NVPTXISD::TexUnified2DArrayU32FloatLevel:
3415 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_U32_F32_LEVEL;
3416 break;
3417 case NVPTXISD::TexUnified2DArrayU32FloatGrad:
3418 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_U32_F32_GRAD;
3419 break;
3420 case NVPTXISD::TexUnified3DFloatS32:
3421 Opc = NVPTX::TEX_UNIFIED_3D_F32_S32;
3422 break;
3423 case NVPTXISD::TexUnified3DFloatFloat:
3424 Opc = NVPTX::TEX_UNIFIED_3D_F32_F32;
3425 break;
3426 case NVPTXISD::TexUnified3DFloatFloatLevel:
3427 Opc = NVPTX::TEX_UNIFIED_3D_F32_F32_LEVEL;
3428 break;
3429 case NVPTXISD::TexUnified3DFloatFloatGrad:
3430 Opc = NVPTX::TEX_UNIFIED_3D_F32_F32_GRAD;
3431 break;
3432 case NVPTXISD::TexUnified3DS32S32:
3433 Opc = NVPTX::TEX_UNIFIED_3D_S32_S32;
3434 break;
3435 case NVPTXISD::TexUnified3DS32Float:
3436 Opc = NVPTX::TEX_UNIFIED_3D_S32_F32;
3437 break;
3438 case NVPTXISD::TexUnified3DS32FloatLevel:
3439 Opc = NVPTX::TEX_UNIFIED_3D_S32_F32_LEVEL;
3440 break;
3441 case NVPTXISD::TexUnified3DS32FloatGrad:
3442 Opc = NVPTX::TEX_UNIFIED_3D_S32_F32_GRAD;
3443 break;
3444 case NVPTXISD::TexUnified3DU32S32:
3445 Opc = NVPTX::TEX_UNIFIED_3D_U32_S32;
3446 break;
3447 case NVPTXISD::TexUnified3DU32Float:
3448 Opc = NVPTX::TEX_UNIFIED_3D_U32_F32;
3449 break;
3450 case NVPTXISD::TexUnified3DU32FloatLevel:
3451 Opc = NVPTX::TEX_UNIFIED_3D_U32_F32_LEVEL;
3452 break;
3453 case NVPTXISD::TexUnified3DU32FloatGrad:
3454 Opc = NVPTX::TEX_UNIFIED_3D_U32_F32_GRAD;
3455 break;
3456 case NVPTXISD::TexUnifiedCubeFloatFloat:
3457 Opc = NVPTX::TEX_UNIFIED_CUBE_F32_F32;
3458 break;
3459 case NVPTXISD::TexUnifiedCubeFloatFloatLevel:
3460 Opc = NVPTX::TEX_UNIFIED_CUBE_F32_F32_LEVEL;
3461 break;
3462 case NVPTXISD::TexUnifiedCubeS32Float:
3463 Opc = NVPTX::TEX_UNIFIED_CUBE_S32_F32;
3464 break;
3465 case NVPTXISD::TexUnifiedCubeS32FloatLevel:
3466 Opc = NVPTX::TEX_UNIFIED_CUBE_S32_F32_LEVEL;
3467 break;
3468 case NVPTXISD::TexUnifiedCubeU32Float:
3469 Opc = NVPTX::TEX_UNIFIED_CUBE_U32_F32;
3470 break;
3471 case NVPTXISD::TexUnifiedCubeU32FloatLevel:
3472 Opc = NVPTX::TEX_UNIFIED_CUBE_U32_F32_LEVEL;
3473 break;
3474 case NVPTXISD::TexUnifiedCubeArrayFloatFloat:
3475 Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_F32_F32;
3476 break;
3477 case NVPTXISD::TexUnifiedCubeArrayFloatFloatLevel:
3478 Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_F32_F32_LEVEL;
3479 break;
3480 case NVPTXISD::TexUnifiedCubeArrayS32Float:
3481 Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_S32_F32;
3482 break;
3483 case NVPTXISD::TexUnifiedCubeArrayS32FloatLevel:
3484 Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_S32_F32_LEVEL;
3485 break;
3486 case NVPTXISD::TexUnifiedCubeArrayU32Float:
3487 Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_U32_F32;
3488 break;
3489 case NVPTXISD::TexUnifiedCubeArrayU32FloatLevel:
3490 Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_U32_F32_LEVEL;
3491 break;
3492 case NVPTXISD::Tld4UnifiedR2DFloatFloat:
3493 Opc = NVPTX::TLD4_UNIFIED_R_2D_F32_F32;
3494 break;
3495 case NVPTXISD::Tld4UnifiedG2DFloatFloat:
3496 Opc = NVPTX::TLD4_UNIFIED_G_2D_F32_F32;
3497 break;
3498 case NVPTXISD::Tld4UnifiedB2DFloatFloat:
3499 Opc = NVPTX::TLD4_UNIFIED_B_2D_F32_F32;
3500 break;
3501 case NVPTXISD::Tld4UnifiedA2DFloatFloat:
3502 Opc = NVPTX::TLD4_UNIFIED_A_2D_F32_F32;
3503 break;
3504 case NVPTXISD::Tld4UnifiedR2DS64Float:
3505 Opc = NVPTX::TLD4_UNIFIED_R_2D_S32_F32;
3506 break;
3507 case NVPTXISD::Tld4UnifiedG2DS64Float:
3508 Opc = NVPTX::TLD4_UNIFIED_G_2D_S32_F32;
3509 break;
3510 case NVPTXISD::Tld4UnifiedB2DS64Float:
3511 Opc = NVPTX::TLD4_UNIFIED_B_2D_S32_F32;
3512 break;
3513 case NVPTXISD::Tld4UnifiedA2DS64Float:
3514 Opc = NVPTX::TLD4_UNIFIED_A_2D_S32_F32;
3515 break;
3516 case NVPTXISD::Tld4UnifiedR2DU64Float:
3517 Opc = NVPTX::TLD4_UNIFIED_R_2D_U32_F32;
3518 break;
3519 case NVPTXISD::Tld4UnifiedG2DU64Float:
3520 Opc = NVPTX::TLD4_UNIFIED_G_2D_U32_F32;
3521 break;
3522 case NVPTXISD::Tld4UnifiedB2DU64Float:
3523 Opc = NVPTX::TLD4_UNIFIED_B_2D_U32_F32;
3524 break;
3525 case NVPTXISD::Tld4UnifiedA2DU64Float:
3526 Opc = NVPTX::TLD4_UNIFIED_A_2D_U32_F32;
3527 break;
3528 }
3529
3530 // Copy over operands
3531 for (unsigned i = 1; i < N->getNumOperands(); ++i) {
3532 Ops.push_back(N->getOperand(i));
3533 }
3534
3535 Ops.push_back(Chain);
3536 Ret = CurDAG->getMachineNode(Opc, SDLoc(N), N->getVTList(), Ops);
3537 return Ret;
3538 }
3539
3540 SDNode *NVPTXDAGToDAGISel::SelectSurfaceIntrinsic(SDNode *N) {
3541 SDValue Chain = N->getOperand(0);
3542 SDValue TexHandle = N->getOperand(1);
3543 SDNode *Ret = nullptr;
3544 unsigned Opc = 0;
3545 SmallVector<SDValue, 8> Ops;
3546 switch (N->getOpcode()) {
3547 default: return nullptr;
3548 case NVPTXISD::Suld1DI8Clamp:
3549 Opc = NVPTX::SULD_1D_I8_CLAMP;
3550 Ops.push_back(TexHandle);
3551 Ops.push_back(N->getOperand(2));
3552 Ops.push_back(Chain);
3553 break;
3554 case NVPTXISD::Suld1DI16Clamp:
3555 Opc = NVPTX::SULD_1D_I16_CLAMP;
3556 Ops.push_back(TexHandle);
3557 Ops.push_back(N->getOperand(2));
3558 Ops.push_back(Chain);
3559 break;
3560 case NVPTXISD::Suld1DI32Clamp:
3561 Opc = NVPTX::SULD_1D_I32_CLAMP;
3562 Ops.push_back(TexHandle);
3563 Ops.push_back(N->getOperand(2));
3564 Ops.push_back(Chain);
3565 break;
3566 case NVPTXISD::Suld1DI64Clamp:
3567 Opc = NVPTX::SULD_1D_I64_CLAMP;
3568 Ops.push_back(TexHandle);
3569 Ops.push_back(N->getOperand(2));
3570 Ops.push_back(Chain);
3571 break;
3572 case NVPTXISD::Suld1DV2I8Clamp:
3573 Opc = NVPTX::SULD_1D_V2I8_CLAMP;
3574 Ops.push_back(TexHandle);
3575 Ops.push_back(N->getOperand(2));
3576 Ops.push_back(Chain);
3577 break;
3578 case NVPTXISD::Suld1DV2I16Clamp:
3579 Opc = NVPTX::SULD_1D_V2I16_CLAMP;
3580 Ops.push_back(TexHandle);
3581 Ops.push_back(N->getOperand(2));
3582 Ops.push_back(Chain);
3583 break;
3584 case NVPTXISD::Suld1DV2I32Clamp:
3585 Opc = NVPTX::SULD_1D_V2I32_CLAMP;
3586 Ops.push_back(TexHandle);
3587 Ops.push_back(N->getOperand(2));
3588 Ops.push_back(Chain);
3589 break;
3590 case NVPTXISD::Suld1DV2I64Clamp:
3591 Opc = NVPTX::SULD_1D_V2I64_CLAMP;
3592 Ops.push_back(TexHandle);
3593 Ops.push_back(N->getOperand(2));
3594 Ops.push_back(Chain);
3595 break;
3596 case NVPTXISD::Suld1DV4I8Clamp:
3597 Opc = NVPTX::SULD_1D_V4I8_CLAMP;
3598 Ops.push_back(TexHandle);
3599 Ops.push_back(N->getOperand(2));
3600 Ops.push_back(Chain);
3601 break;
3602 case NVPTXISD::Suld1DV4I16Clamp:
3603 Opc = NVPTX::SULD_1D_V4I16_CLAMP;
3604 Ops.push_back(TexHandle);
3605 Ops.push_back(N->getOperand(2));
3606 Ops.push_back(Chain);
3607 break;
3608 case NVPTXISD::Suld1DV4I32Clamp:
3609 Opc = NVPTX::SULD_1D_V4I32_CLAMP;
3610 Ops.push_back(TexHandle);
3611 Ops.push_back(N->getOperand(2));
3612 Ops.push_back(Chain);
3613 break;
3614 case NVPTXISD::Suld1DArrayI8Clamp:
3615 Opc = NVPTX::SULD_1D_ARRAY_I8_CLAMP;
3616 Ops.push_back(TexHandle);
3617 Ops.push_back(N->getOperand(2));
3618 Ops.push_back(N->getOperand(3));
3619 Ops.push_back(Chain);
3620 break;
3621 case NVPTXISD::Suld1DArrayI16Clamp:
3622 Opc = NVPTX::SULD_1D_ARRAY_I16_CLAMP;
3623 Ops.push_back(TexHandle);
3624 Ops.push_back(N->getOperand(2));
3625 Ops.push_back(N->getOperand(3));
3626 Ops.push_back(Chain);
3627 break;
3628 case NVPTXISD::Suld1DArrayI32Clamp:
3629 Opc = NVPTX::SULD_1D_ARRAY_I32_CLAMP;
3630 Ops.push_back(TexHandle);
3631 Ops.push_back(N->getOperand(2));
3632 Ops.push_back(N->getOperand(3));
3633 Ops.push_back(Chain);
3634 break;
3635 case NVPTXISD::Suld1DArrayI64Clamp:
3636 Opc = NVPTX::SULD_1D_ARRAY_I64_CLAMP;
3637 Ops.push_back(TexHandle);
3638 Ops.push_back(N->getOperand(2));
3639 Ops.push_back(N->getOperand(3));
3640 Ops.push_back(Chain);
3641 break;
3642 case NVPTXISD::Suld1DArrayV2I8Clamp:
3643 Opc = NVPTX::SULD_1D_ARRAY_V2I8_CLAMP;
3644 Ops.push_back(TexHandle);
3645 Ops.push_back(N->getOperand(2));
3646 Ops.push_back(N->getOperand(3));
3647 Ops.push_back(Chain);
3648 break;
3649 case NVPTXISD::Suld1DArrayV2I16Clamp:
3650 Opc = NVPTX::SULD_1D_ARRAY_V2I16_CLAMP;
3651 Ops.push_back(TexHandle);
3652 Ops.push_back(N->getOperand(2));
3653 Ops.push_back(N->getOperand(3));
3654 Ops.push_back(Chain);
3655 break;
3656 case NVPTXISD::Suld1DArrayV2I32Clamp:
3657 Opc = NVPTX::SULD_1D_ARRAY_V2I32_CLAMP;
3658 Ops.push_back(TexHandle);
3659 Ops.push_back(N->getOperand(2));
3660 Ops.push_back(N->getOperand(3));
3661 Ops.push_back(Chain);
3662 break;
3663 case NVPTXISD::Suld1DArrayV2I64Clamp:
3664 Opc = NVPTX::SULD_1D_ARRAY_V2I64_CLAMP;
3665 Ops.push_back(TexHandle);
3666 Ops.push_back(N->getOperand(2));
3667 Ops.push_back(N->getOperand(3));
3668 Ops.push_back(Chain);
3669 break;
3670 case NVPTXISD::Suld1DArrayV4I8Clamp:
3671 Opc = NVPTX::SULD_1D_ARRAY_V4I8_CLAMP;
3672 Ops.push_back(TexHandle);
3673 Ops.push_back(N->getOperand(2));
3674 Ops.push_back(N->getOperand(3));
3675 Ops.push_back(Chain);
3676 break;
3677 case NVPTXISD::Suld1DArrayV4I16Clamp:
3678 Opc = NVPTX::SULD_1D_ARRAY_V4I16_CLAMP;
3679 Ops.push_back(TexHandle);
3680 Ops.push_back(N->getOperand(2));
3681 Ops.push_back(N->getOperand(3));
3682 Ops.push_back(Chain);
3683 break;
3684 case NVPTXISD::Suld1DArrayV4I32Clamp:
3685 Opc = NVPTX::SULD_1D_ARRAY_V4I32_CLAMP;
3686 Ops.push_back(TexHandle);
3687 Ops.push_back(N->getOperand(2));
3688 Ops.push_back(N->getOperand(3));
3689 Ops.push_back(Chain);
3690 break;
3691 case NVPTXISD::Suld2DI8Clamp:
3692 Opc = NVPTX::SULD_2D_I8_CLAMP;
3693 Ops.push_back(TexHandle);
3694 Ops.push_back(N->getOperand(2));
3695 Ops.push_back(N->getOperand(3));
3696 Ops.push_back(Chain);
3697 break;
3698 case NVPTXISD::Suld2DI16Clamp:
3699 Opc = NVPTX::SULD_2D_I16_CLAMP;
3700 Ops.push_back(TexHandle);
3701 Ops.push_back(N->getOperand(2));
3702 Ops.push_back(N->getOperand(3));
3703 Ops.push_back(Chain);
3704 break;
3705 case NVPTXISD::Suld2DI32Clamp:
3706 Opc = NVPTX::SULD_2D_I32_CLAMP;
3707 Ops.push_back(TexHandle);
3708 Ops.push_back(N->getOperand(2));
3709 Ops.push_back(N->getOperand(3));
3710 Ops.push_back(Chain);
3711 break;
3712 case NVPTXISD::Suld2DI64Clamp:
3713 Opc = NVPTX::SULD_2D_I64_CLAMP;
3714 Ops.push_back(TexHandle);
3715 Ops.push_back(N->getOperand(2));
3716 Ops.push_back(N->getOperand(3));
3717 Ops.push_back(Chain);
3718 break;
3719 case NVPTXISD::Suld2DV2I8Clamp:
3720 Opc = NVPTX::SULD_2D_V2I8_CLAMP;
3721 Ops.push_back(TexHandle);
3722 Ops.push_back(N->getOperand(2));
3723 Ops.push_back(N->getOperand(3));
3724 Ops.push_back(Chain);
3725 break;
3726 case NVPTXISD::Suld2DV2I16Clamp:
3727 Opc = NVPTX::SULD_2D_V2I16_CLAMP;
3728 Ops.push_back(TexHandle);
3729 Ops.push_back(N->getOperand(2));
3730 Ops.push_back(N->getOperand(3));
3731 Ops.push_back(Chain);
3732 break;
3733 case NVPTXISD::Suld2DV2I32Clamp:
3734 Opc = NVPTX::SULD_2D_V2I32_CLAMP;
3735 Ops.push_back(TexHandle);
3736 Ops.push_back(N->getOperand(2));
3737 Ops.push_back(N->getOperand(3));
3738 Ops.push_back(Chain);
3739 break;
3740 case NVPTXISD::Suld2DV2I64Clamp:
3741 Opc = NVPTX::SULD_2D_V2I64_CLAMP;
3742 Ops.push_back(TexHandle);
3743 Ops.push_back(N->getOperand(2));
3744 Ops.push_back(N->getOperand(3));
3745 Ops.push_back(Chain);
3746 break;
3747 case NVPTXISD::Suld2DV4I8Clamp:
3748 Opc = NVPTX::SULD_2D_V4I8_CLAMP;
3749 Ops.push_back(TexHandle);
3750 Ops.push_back(N->getOperand(2));
3751 Ops.push_back(N->getOperand(3));
3752 Ops.push_back(Chain);
3753 break;
3754 case NVPTXISD::Suld2DV4I16Clamp:
3755 Opc = NVPTX::SULD_2D_V4I16_CLAMP;
3756 Ops.push_back(TexHandle);
3757 Ops.push_back(N->getOperand(2));
3758 Ops.push_back(N->getOperand(3));
3759 Ops.push_back(Chain);
3760 break;
3761 case NVPTXISD::Suld2DV4I32Clamp:
3762 Opc = NVPTX::SULD_2D_V4I32_CLAMP;
3763 Ops.push_back(TexHandle);
3764 Ops.push_back(N->getOperand(2));
3765 Ops.push_back(N->getOperand(3));
3766 Ops.push_back(Chain);
3767 break;
3768 case NVPTXISD::Suld2DArrayI8Clamp:
3769 Opc = NVPTX::SULD_2D_ARRAY_I8_CLAMP;
3770 Ops.push_back(TexHandle);
3771 Ops.push_back(N->getOperand(2));
3772 Ops.push_back(N->getOperand(3));
3773 Ops.push_back(N->getOperand(4));
3774 Ops.push_back(Chain);
3775 break;
3776 case NVPTXISD::Suld2DArrayI16Clamp:
3777 Opc = NVPTX::SULD_2D_ARRAY_I16_CLAMP;
3778 Ops.push_back(TexHandle);
3779 Ops.push_back(N->getOperand(2));
3780 Ops.push_back(N->getOperand(3));
3781 Ops.push_back(N->getOperand(4));
3782 Ops.push_back(Chain);
3783 break;
3784 case NVPTXISD::Suld2DArrayI32Clamp:
3785 Opc = NVPTX::SULD_2D_ARRAY_I32_CLAMP;
3786 Ops.push_back(TexHandle);
3787 Ops.push_back(N->getOperand(2));
3788 Ops.push_back(N->getOperand(3));
3789 Ops.push_back(N->getOperand(4));
3790 Ops.push_back(Chain);
3791 break;
3792 case NVPTXISD::Suld2DArrayI64Clamp:
3793 Opc = NVPTX::SULD_2D_ARRAY_I64_CLAMP;
3794 Ops.push_back(TexHandle);
3795 Ops.push_back(N->getOperand(2));
3796 Ops.push_back(N->getOperand(3));
3797 Ops.push_back(N->getOperand(4));
3798 Ops.push_back(Chain);
3799 break;
3800 case NVPTXISD::Suld2DArrayV2I8Clamp:
3801 Opc = NVPTX::SULD_2D_ARRAY_V2I8_CLAMP;
3802 Ops.push_back(TexHandle);
3803 Ops.push_back(N->getOperand(2));
3804 Ops.push_back(N->getOperand(3));
3805 Ops.push_back(N->getOperand(4));
3806 Ops.push_back(Chain);
3807 break;
3808 case NVPTXISD::Suld2DArrayV2I16Clamp:
3809 Opc = NVPTX::SULD_2D_ARRAY_V2I16_CLAMP;
3810 Ops.push_back(TexHandle);
3811 Ops.push_back(N->getOperand(2));
3812 Ops.push_back(N->getOperand(3));
3813 Ops.push_back(N->getOperand(4));
3814 Ops.push_back(Chain);
3815 break;
3816 case NVPTXISD::Suld2DArrayV2I32Clamp:
3817 Opc = NVPTX::SULD_2D_ARRAY_V2I32_CLAMP;
3818 Ops.push_back(TexHandle);
3819 Ops.push_back(N->getOperand(2));
3820 Ops.push_back(N->getOperand(3));
3821 Ops.push_back(N->getOperand(4));
3822 Ops.push_back(Chain);
3823 break;
3824 case NVPTXISD::Suld2DArrayV2I64Clamp:
3825 Opc = NVPTX::SULD_2D_ARRAY_V2I64_CLAMP;
3826 Ops.push_back(TexHandle);
3827 Ops.push_back(N->getOperand(2));
3828 Ops.push_back(N->getOperand(3));
3829 Ops.push_back(N->getOperand(4));
3830 Ops.push_back(Chain);
3831 break;
3832 case NVPTXISD::Suld2DArrayV4I8Clamp:
3833 Opc = NVPTX::SULD_2D_ARRAY_V4I8_CLAMP;
3834 Ops.push_back(TexHandle);
3835 Ops.push_back(N->getOperand(2));
3836 Ops.push_back(N->getOperand(3));
3837 Ops.push_back(N->getOperand(4));
3838 Ops.push_back(Chain);
3839 break;
3840 case NVPTXISD::Suld2DArrayV4I16Clamp:
3841 Opc = NVPTX::SULD_2D_ARRAY_V4I16_CLAMP;
3842 Ops.push_back(TexHandle);
3843 Ops.push_back(N->getOperand(2));
3844 Ops.push_back(N->getOperand(3));
3845 Ops.push_back(N->getOperand(4));
3846 Ops.push_back(Chain);
3847 break;
3848 case NVPTXISD::Suld2DArrayV4I32Clamp:
3849 Opc = NVPTX::SULD_2D_ARRAY_V4I32_CLAMP;
3850 Ops.push_back(TexHandle);
3851 Ops.push_back(N->getOperand(2));
3852 Ops.push_back(N->getOperand(3));
3853 Ops.push_back(N->getOperand(4));
3854 Ops.push_back(Chain);
3855 break;
3856 case NVPTXISD::Suld3DI8Clamp:
3857 Opc = NVPTX::SULD_3D_I8_CLAMP;
3858 Ops.push_back(TexHandle);
3859 Ops.push_back(N->getOperand(2));
3860 Ops.push_back(N->getOperand(3));
3861 Ops.push_back(N->getOperand(4));
3862 Ops.push_back(Chain);
3863 break;
3864 case NVPTXISD::Suld3DI16Clamp:
3865 Opc = NVPTX::SULD_3D_I16_CLAMP;
3866 Ops.push_back(TexHandle);
3867 Ops.push_back(N->getOperand(2));
3868 Ops.push_back(N->getOperand(3));
3869 Ops.push_back(N->getOperand(4));
3870 Ops.push_back(Chain);
3871 break;
3872 case NVPTXISD::Suld3DI32Clamp:
3873 Opc = NVPTX::SULD_3D_I32_CLAMP;
3874 Ops.push_back(TexHandle);
3875 Ops.push_back(N->getOperand(2));
3876 Ops.push_back(N->getOperand(3));
3877 Ops.push_back(N->getOperand(4));
3878 Ops.push_back(Chain);
3879 break;
3880 case NVPTXISD::Suld3DI64Clamp:
3881 Opc = NVPTX::SULD_3D_I64_CLAMP;
3882 Ops.push_back(TexHandle);
3883 Ops.push_back(N->getOperand(2));
3884 Ops.push_back(N->getOperand(3));
3885 Ops.push_back(N->getOperand(4));
3886 Ops.push_back(Chain);
3887 break;
3888 case NVPTXISD::Suld3DV2I8Clamp:
3889 Opc = NVPTX::SULD_3D_V2I8_CLAMP;
3890 Ops.push_back(TexHandle);
3891 Ops.push_back(N->getOperand(2));
3892 Ops.push_back(N->getOperand(3));
3893 Ops.push_back(N->getOperand(4));
3894 Ops.push_back(Chain);
3895 break;
3896 case NVPTXISD::Suld3DV2I16Clamp:
3897 Opc = NVPTX::SULD_3D_V2I16_CLAMP;
3898 Ops.push_back(TexHandle);
3899 Ops.push_back(N->getOperand(2));
3900 Ops.push_back(N->getOperand(3));
3901 Ops.push_back(N->getOperand(4));
3902 Ops.push_back(Chain);
3903 break;
3904 case NVPTXISD::Suld3DV2I32Clamp:
3905 Opc = NVPTX::SULD_3D_V2I32_CLAMP;
3906 Ops.push_back(TexHandle);
3907 Ops.push_back(N->getOperand(2));
3908 Ops.push_back(N->getOperand(3));
3909 Ops.push_back(N->getOperand(4));
3910 Ops.push_back(Chain);
3911 break;
3912 case NVPTXISD::Suld3DV2I64Clamp:
3913 Opc = NVPTX::SULD_3D_V2I64_CLAMP;
3914 Ops.push_back(TexHandle);
3915 Ops.push_back(N->getOperand(2));
3916 Ops.push_back(N->getOperand(3));
3917 Ops.push_back(N->getOperand(4));
3918 Ops.push_back(Chain);
3919 break;
3920 case NVPTXISD::Suld3DV4I8Clamp:
3921 Opc = NVPTX::SULD_3D_V4I8_CLAMP;
3922 Ops.push_back(TexHandle);
3923 Ops.push_back(N->getOperand(2));
3924 Ops.push_back(N->getOperand(3));
3925 Ops.push_back(N->getOperand(4));
3926 Ops.push_back(Chain);
3927 break;
3928 case NVPTXISD::Suld3DV4I16Clamp:
3929 Opc = NVPTX::SULD_3D_V4I16_CLAMP;
3930 Ops.push_back(TexHandle);
3931 Ops.push_back(N->getOperand(2));
3932 Ops.push_back(N->getOperand(3));
3933 Ops.push_back(N->getOperand(4));
3934 Ops.push_back(Chain);
3935 break;
3936 case NVPTXISD::Suld3DV4I32Clamp:
3937 Opc = NVPTX::SULD_3D_V4I32_CLAMP;
3938 Ops.push_back(TexHandle);
3939 Ops.push_back(N->getOperand(2));
3940 Ops.push_back(N->getOperand(3));
3941 Ops.push_back(N->getOperand(4));
3942 Ops.push_back(Chain);
3943 break;
3944 case NVPTXISD::Suld1DI8Trap:
3945 Opc = NVPTX::SULD_1D_I8_TRAP;
3946 Ops.push_back(TexHandle);
3947 Ops.push_back(N->getOperand(2));
3948 Ops.push_back(Chain);
3949 break;
3950 case NVPTXISD::Suld1DI16Trap:
3951 Opc = NVPTX::SULD_1D_I16_TRAP;
3952 Ops.push_back(TexHandle);
3953 Ops.push_back(N->getOperand(2));
3954 Ops.push_back(Chain);
3955 break;
3956 case NVPTXISD::Suld1DI32Trap:
3957 Opc = NVPTX::SULD_1D_I32_TRAP;
3958 Ops.push_back(TexHandle);
3959 Ops.push_back(N->getOperand(2));
3960 Ops.push_back(Chain);
3961 break;
3962 case NVPTXISD::Suld1DI64Trap:
3963 Opc = NVPTX::SULD_1D_I64_TRAP;
3964 Ops.push_back(TexHandle);
3965 Ops.push_back(N->getOperand(2));
3966 Ops.push_back(Chain);
3967 break;
3968 case NVPTXISD::Suld1DV2I8Trap:
3969 Opc = NVPTX::SULD_1D_V2I8_TRAP;
3970 Ops.push_back(TexHandle);
3971 Ops.push_back(N->getOperand(2));
3972 Ops.push_back(Chain);
3973 break;
3974 case NVPTXISD::Suld1DV2I16Trap:
3975 Opc = NVPTX::SULD_1D_V2I16_TRAP;
3976 Ops.push_back(TexHandle);
3977 Ops.push_back(N->getOperand(2));
3978 Ops.push_back(Chain);
3979 break;
3980 case NVPTXISD::Suld1DV2I32Trap:
3981 Opc = NVPTX::SULD_1D_V2I32_TRAP;
3982 Ops.push_back(TexHandle);
3983 Ops.push_back(N->getOperand(2));
3984 Ops.push_back(Chain);
3985 break;
3986 case NVPTXISD::Suld1DV2I64Trap:
3987 Opc = NVPTX::SULD_1D_V2I64_TRAP;
3988 Ops.push_back(TexHandle);
3989 Ops.push_back(N->getOperand(2));
3990 Ops.push_back(Chain);
3991 break;
3992 case NVPTXISD::Suld1DV4I8Trap:
3993 Opc = NVPTX::SULD_1D_V4I8_TRAP;
3994 Ops.push_back(TexHandle);
3995 Ops.push_back(N->getOperand(2));
3996 Ops.push_back(Chain);
3997 break;
3998 case NVPTXISD::Suld1DV4I16Trap:
3999 Opc = NVPTX::SULD_1D_V4I16_TRAP;
4000 Ops.push_back(TexHandle);
4001 Ops.push_back(N->getOperand(2));
4002 Ops.push_back(Chain);
4003 break;
4004 case NVPTXISD::Suld1DV4I32Trap:
4005 Opc = NVPTX::SULD_1D_V4I32_TRAP;
4006 Ops.push_back(TexHandle);
4007 Ops.push_back(N->getOperand(2));
4008 Ops.push_back(Chain);
4009 break;
4010 case NVPTXISD::Suld1DArrayI8Trap:
4011 Opc = NVPTX::SULD_1D_ARRAY_I8_TRAP;
4012 Ops.push_back(TexHandle);
4013 Ops.push_back(N->getOperand(2));
4014 Ops.push_back(N->getOperand(3));
4015 Ops.push_back(Chain);
4016 break;
4017 case NVPTXISD::Suld1DArrayI16Trap:
4018 Opc = NVPTX::SULD_1D_ARRAY_I16_TRAP;
4019 Ops.push_back(TexHandle);
4020 Ops.push_back(N->getOperand(2));
4021 Ops.push_back(N->getOperand(3));
4022 Ops.push_back(Chain);
4023 break;
4024 case NVPTXISD::Suld1DArrayI32Trap:
4025 Opc = NVPTX::SULD_1D_ARRAY_I32_TRAP;
4026 Ops.push_back(TexHandle);
4027 Ops.push_back(N->getOperand(2));
4028 Ops.push_back(N->getOperand(3));
4029 Ops.push_back(Chain);
4030 break;
4031 case NVPTXISD::Suld1DArrayI64Trap:
4032 Opc = NVPTX::SULD_1D_ARRAY_I64_TRAP;
4033 Ops.push_back(TexHandle);
4034 Ops.push_back(N->getOperand(2));
4035 Ops.push_back(N->getOperand(3));
4036 Ops.push_back(Chain);
4037 break;
4038 case NVPTXISD::Suld1DArrayV2I8Trap:
4039 Opc = NVPTX::SULD_1D_ARRAY_V2I8_TRAP;
4040 Ops.push_back(TexHandle);
4041 Ops.push_back(N->getOperand(2));
4042 Ops.push_back(N->getOperand(3));
4043 Ops.push_back(Chain);
4044 break;
4045 case NVPTXISD::Suld1DArrayV2I16Trap:
4046 Opc = NVPTX::SULD_1D_ARRAY_V2I16_TRAP;
4047 Ops.push_back(TexHandle);
4048 Ops.push_back(N->getOperand(2));
4049 Ops.push_back(N->getOperand(3));
4050 Ops.push_back(Chain);
4051 break;
4052 case NVPTXISD::Suld1DArrayV2I32Trap:
4053 Opc = NVPTX::SULD_1D_ARRAY_V2I32_TRAP;
4054 Ops.push_back(TexHandle);
4055 Ops.push_back(N->getOperand(2));
4056 Ops.push_back(N->getOperand(3));
4057 Ops.push_back(Chain);
4058 break;
4059 case NVPTXISD::Suld1DArrayV2I64Trap:
4060 Opc = NVPTX::SULD_1D_ARRAY_V2I64_TRAP;
4061 Ops.push_back(TexHandle);
4062 Ops.push_back(N->getOperand(2));
4063 Ops.push_back(N->getOperand(3));
4064 Ops.push_back(Chain);
4065 break;
4066 case NVPTXISD::Suld1DArrayV4I8Trap:
4067 Opc = NVPTX::SULD_1D_ARRAY_V4I8_TRAP;
4068 Ops.push_back(TexHandle);
4069 Ops.push_back(N->getOperand(2));
4070 Ops.push_back(N->getOperand(3));
4071 Ops.push_back(Chain);
4072 break;
4073 case NVPTXISD::Suld1DArrayV4I16Trap:
4074 Opc = NVPTX::SULD_1D_ARRAY_V4I16_TRAP;
4075 Ops.push_back(TexHandle);
4076 Ops.push_back(N->getOperand(2));
4077 Ops.push_back(N->getOperand(3));
4078 Ops.push_back(Chain);
4079 break;
4080 case NVPTXISD::Suld1DArrayV4I32Trap:
4081 Opc = NVPTX::SULD_1D_ARRAY_V4I32_TRAP;
4082 Ops.push_back(TexHandle);
4083 Ops.push_back(N->getOperand(2));
4084 Ops.push_back(N->getOperand(3));
4085 Ops.push_back(Chain);
4086 break;
4087 case NVPTXISD::Suld2DI8Trap:
4088 Opc = NVPTX::SULD_2D_I8_TRAP;
4089 Ops.push_back(TexHandle);
4090 Ops.push_back(N->getOperand(2));
4091 Ops.push_back(N->getOperand(3));
4092 Ops.push_back(Chain);
4093 break;
4094 case NVPTXISD::Suld2DI16Trap:
4095 Opc = NVPTX::SULD_2D_I16_TRAP;
4096 Ops.push_back(TexHandle);
4097 Ops.push_back(N->getOperand(2));
4098 Ops.push_back(N->getOperand(3));
4099 Ops.push_back(Chain);
4100 break;
4101 case NVPTXISD::Suld2DI32Trap:
4102 Opc = NVPTX::SULD_2D_I32_TRAP;
4103 Ops.push_back(TexHandle);
4104 Ops.push_back(N->getOperand(2));
4105 Ops.push_back(N->getOperand(3));
4106 Ops.push_back(Chain);
4107 break;
4108 case NVPTXISD::Suld2DI64Trap:
4109 Opc = NVPTX::SULD_2D_I64_TRAP;
4110 Ops.push_back(TexHandle);
4111 Ops.push_back(N->getOperand(2));
4112 Ops.push_back(N->getOperand(3));
4113 Ops.push_back(Chain);
4114 break;
4115 case NVPTXISD::Suld2DV2I8Trap:
4116 Opc = NVPTX::SULD_2D_V2I8_TRAP;
4117 Ops.push_back(TexHandle);
4118 Ops.push_back(N->getOperand(2));
4119 Ops.push_back(N->getOperand(3));
4120 Ops.push_back(Chain);
4121 break;
4122 case NVPTXISD::Suld2DV2I16Trap:
4123 Opc = NVPTX::SULD_2D_V2I16_TRAP;
4124 Ops.push_back(TexHandle);
4125 Ops.push_back(N->getOperand(2));
4126 Ops.push_back(N->getOperand(3));
4127 Ops.push_back(Chain);
4128 break;
4129 case NVPTXISD::Suld2DV2I32Trap:
4130 Opc = NVPTX::SULD_2D_V2I32_TRAP;
4131 Ops.push_back(TexHandle);
4132 Ops.push_back(N->getOperand(2));
4133 Ops.push_back(N->getOperand(3));
4134 Ops.push_back(Chain);
4135 break;
4136 case NVPTXISD::Suld2DV2I64Trap:
4137 Opc = NVPTX::SULD_2D_V2I64_TRAP;
4138 Ops.push_back(TexHandle);
4139 Ops.push_back(N->getOperand(2));
4140 Ops.push_back(N->getOperand(3));
4141 Ops.push_back(Chain);
4142 break;
4143 case NVPTXISD::Suld2DV4I8Trap:
4144 Opc = NVPTX::SULD_2D_V4I8_TRAP;
4145 Ops.push_back(TexHandle);
4146 Ops.push_back(N->getOperand(2));
4147 Ops.push_back(N->getOperand(3));
4148 Ops.push_back(Chain);
4149 break;
4150 case NVPTXISD::Suld2DV4I16Trap:
4151 Opc = NVPTX::SULD_2D_V4I16_TRAP;
4152 Ops.push_back(TexHandle);
4153 Ops.push_back(N->getOperand(2));
4154 Ops.push_back(N->getOperand(3));
4155 Ops.push_back(Chain);
4156 break;
4157 case NVPTXISD::Suld2DV4I32Trap:
4158 Opc = NVPTX::SULD_2D_V4I32_TRAP;
4159 Ops.push_back(TexHandle);
4160 Ops.push_back(N->getOperand(2));
4161 Ops.push_back(N->getOperand(3));
4162 Ops.push_back(Chain);
4163 break;
4164 case NVPTXISD::Suld2DArrayI8Trap:
4165 Opc = NVPTX::SULD_2D_ARRAY_I8_TRAP;
4166 Ops.push_back(TexHandle);
4167 Ops.push_back(N->getOperand(2));
4168 Ops.push_back(N->getOperand(3));
4169 Ops.push_back(N->getOperand(4));
4170 Ops.push_back(Chain);
4171 break;
4172 case NVPTXISD::Suld2DArrayI16Trap:
4173 Opc = NVPTX::SULD_2D_ARRAY_I16_TRAP;
4174 Ops.push_back(TexHandle);
4175 Ops.push_back(N->getOperand(2));
4176 Ops.push_back(N->getOperand(3));
4177 Ops.push_back(N->getOperand(4));
4178 Ops.push_back(Chain);
4179 break;
4180 case NVPTXISD::Suld2DArrayI32Trap:
4181 Opc = NVPTX::SULD_2D_ARRAY_I32_TRAP;
4182 Ops.push_back(TexHandle);
4183 Ops.push_back(N->getOperand(2));
4184 Ops.push_back(N->getOperand(3));
4185 Ops.push_back(N->getOperand(4));
4186 Ops.push_back(Chain);
4187 break;
4188 case NVPTXISD::Suld2DArrayI64Trap:
4189 Opc = NVPTX::SULD_2D_ARRAY_I64_TRAP;
4190 Ops.push_back(TexHandle);
4191 Ops.push_back(N->getOperand(2));
4192 Ops.push_back(N->getOperand(3));
4193 Ops.push_back(N->getOperand(4));
4194 Ops.push_back(Chain);
4195 break;
4196 case NVPTXISD::Suld2DArrayV2I8Trap:
4197 Opc = NVPTX::SULD_2D_ARRAY_V2I8_TRAP;
4198 Ops.push_back(TexHandle);
4199 Ops.push_back(N->getOperand(2));
4200 Ops.push_back(N->getOperand(3));
4201 Ops.push_back(N->getOperand(4));
4202 Ops.push_back(Chain);
4203 break;
4204 case NVPTXISD::Suld2DArrayV2I16Trap:
4205 Opc = NVPTX::SULD_2D_ARRAY_V2I16_TRAP;
4206 Ops.push_back(TexHandle);
4207 Ops.push_back(N->getOperand(2));
4208 Ops.push_back(N->getOperand(3));
4209 Ops.push_back(N->getOperand(4));
4210 Ops.push_back(Chain);
4211 break;
4212 case NVPTXISD::Suld2DArrayV2I32Trap:
4213 Opc = NVPTX::SULD_2D_ARRAY_V2I32_TRAP;
4214 Ops.push_back(TexHandle);
4215 Ops.push_back(N->getOperand(2));
4216 Ops.push_back(N->getOperand(3));
4217 Ops.push_back(N->getOperand(4));
4218 Ops.push_back(Chain);
4219 break;
4220 case NVPTXISD::Suld2DArrayV2I64Trap:
4221 Opc = NVPTX::SULD_2D_ARRAY_V2I64_TRAP;
4222 Ops.push_back(TexHandle);
4223 Ops.push_back(N->getOperand(2));
4224 Ops.push_back(N->getOperand(3));
4225 Ops.push_back(N->getOperand(4));
4226 Ops.push_back(Chain);
4227 break;
4228 case NVPTXISD::Suld2DArrayV4I8Trap:
4229 Opc = NVPTX::SULD_2D_ARRAY_V4I8_TRAP;
4230 Ops.push_back(TexHandle);
4231 Ops.push_back(N->getOperand(2));
4232 Ops.push_back(N->getOperand(3));
4233 Ops.push_back(N->getOperand(4));
4234 Ops.push_back(Chain);
4235 break;
4236 case NVPTXISD::Suld2DArrayV4I16Trap:
4237 Opc = NVPTX::SULD_2D_ARRAY_V4I16_TRAP;
4238 Ops.push_back(TexHandle);
4239 Ops.push_back(N->getOperand(2));
4240 Ops.push_back(N->getOperand(3));
4241 Ops.push_back(N->getOperand(4));
4242 Ops.push_back(Chain);
4243 break;
4244 case NVPTXISD::Suld2DArrayV4I32Trap:
4245 Opc = NVPTX::SULD_2D_ARRAY_V4I32_TRAP;
4246 Ops.push_back(TexHandle);
4247 Ops.push_back(N->getOperand(2));
4248 Ops.push_back(N->getOperand(3));
4249 Ops.push_back(N->getOperand(4));
4250 Ops.push_back(Chain);
4251 break;
4252 case NVPTXISD::Suld3DI8Trap:
4253 Opc = NVPTX::SULD_3D_I8_TRAP;
4254 Ops.push_back(TexHandle);
4255 Ops.push_back(N->getOperand(2));
4256 Ops.push_back(N->getOperand(3));
4257 Ops.push_back(N->getOperand(4));
4258 Ops.push_back(Chain);
4259 break;
4260 case NVPTXISD::Suld3DI16Trap:
4261 Opc = NVPTX::SULD_3D_I16_TRAP;
4262 Ops.push_back(TexHandle);
4263 Ops.push_back(N->getOperand(2));
4264 Ops.push_back(N->getOperand(3));
4265 Ops.push_back(N->getOperand(4));
4266 Ops.push_back(Chain);
4267 break;
4268 case NVPTXISD::Suld3DI32Trap:
4269 Opc = NVPTX::SULD_3D_I32_TRAP;
4270 Ops.push_back(TexHandle);
4271 Ops.push_back(N->getOperand(2));
4272 Ops.push_back(N->getOperand(3));
4273 Ops.push_back(N->getOperand(4));
4274 Ops.push_back(Chain);
4275 break;
4276 case NVPTXISD::Suld3DI64Trap:
4277 Opc = NVPTX::SULD_3D_I64_TRAP;
4278 Ops.push_back(TexHandle);
4279 Ops.push_back(N->getOperand(2));
4280 Ops.push_back(N->getOperand(3));
4281 Ops.push_back(N->getOperand(4));
4282 Ops.push_back(Chain);
4283 break;
4284 case NVPTXISD::Suld3DV2I8Trap:
4285 Opc = NVPTX::SULD_3D_V2I8_TRAP;
4286 Ops.push_back(TexHandle);
4287 Ops.push_back(N->getOperand(2));
4288 Ops.push_back(N->getOperand(3));
4289 Ops.push_back(N->getOperand(4));
4290 Ops.push_back(Chain);
4291 break;
4292 case NVPTXISD::Suld3DV2I16Trap:
4293 Opc = NVPTX::SULD_3D_V2I16_TRAP;
4294 Ops.push_back(TexHandle);
4295 Ops.push_back(N->getOperand(2));
4296 Ops.push_back(N->getOperand(3));
4297 Ops.push_back(N->getOperand(4));
4298 Ops.push_back(Chain);
4299 break;
4300 case NVPTXISD::Suld3DV2I32Trap:
4301 Opc = NVPTX::SULD_3D_V2I32_TRAP;
4302 Ops.push_back(TexHandle);
4303 Ops.push_back(N->getOperand(2));
4304 Ops.push_back(N->getOperand(3));
4305 Ops.push_back(N->getOperand(4));
4306 Ops.push_back(Chain);
4307 break;
4308 case NVPTXISD::Suld3DV2I64Trap:
4309 Opc = NVPTX::SULD_3D_V2I64_TRAP;
4310 Ops.push_back(TexHandle);
4311 Ops.push_back(N->getOperand(2));
4312 Ops.push_back(N->getOperand(3));
4313 Ops.push_back(N->getOperand(4));
4314 Ops.push_back(Chain);
4315 break;
4316 case NVPTXISD::Suld3DV4I8Trap:
4317 Opc = NVPTX::SULD_3D_V4I8_TRAP;
4318 Ops.push_back(TexHandle);
4319 Ops.push_back(N->getOperand(2));
4320 Ops.push_back(N->getOperand(3));
4321 Ops.push_back(N->getOperand(4));
4322 Ops.push_back(Chain);
4323 break;
4324 case NVPTXISD::Suld3DV4I16Trap:
4325 Opc = NVPTX::SULD_3D_V4I16_TRAP;
4326 Ops.push_back(TexHandle);
4327 Ops.push_back(N->getOperand(2));
4328 Ops.push_back(N->getOperand(3));
4329 Ops.push_back(N->getOperand(4));
4330 Ops.push_back(Chain);
4331 break;
4332 case NVPTXISD::Suld3DV4I32Trap:
4333 Opc = NVPTX::SULD_3D_V4I32_TRAP;
4334 Ops.push_back(TexHandle);
4335 Ops.push_back(N->getOperand(2));
4336 Ops.push_back(N->getOperand(3));
4337 Ops.push_back(N->getOperand(4));
4338 Ops.push_back(Chain);
4339 break;
4340 case NVPTXISD::Suld1DI8Zero:
4341 Opc = NVPTX::SULD_1D_I8_ZERO;
4342 Ops.push_back(TexHandle);
4343 Ops.push_back(N->getOperand(2));
4344 Ops.push_back(Chain);
4345 break;
4346 case NVPTXISD::Suld1DI16Zero:
4347 Opc = NVPTX::SULD_1D_I16_ZERO;
4348 Ops.push_back(TexHandle);
4349 Ops.push_back(N->getOperand(2));
4350 Ops.push_back(Chain);
4351 break;
4352 case NVPTXISD::Suld1DI32Zero:
4353 Opc = NVPTX::SULD_1D_I32_ZERO;
4354 Ops.push_back(TexHandle);
4355 Ops.push_back(N->getOperand(2));
4356 Ops.push_back(Chain);
4357 break;
4358 case NVPTXISD::Suld1DI64Zero:
4359 Opc = NVPTX::SULD_1D_I64_ZERO;
4360 Ops.push_back(TexHandle);
4361 Ops.push_back(N->getOperand(2));
4362 Ops.push_back(Chain);
4363 break;
4364 case NVPTXISD::Suld1DV2I8Zero:
4365 Opc = NVPTX::SULD_1D_V2I8_ZERO;
4366 Ops.push_back(TexHandle);
4367 Ops.push_back(N->getOperand(2));
4368 Ops.push_back(Chain);
4369 break;
4370 case NVPTXISD::Suld1DV2I16Zero:
4371 Opc = NVPTX::SULD_1D_V2I16_ZERO;
4372 Ops.push_back(TexHandle);
4373 Ops.push_back(N->getOperand(2));
4374 Ops.push_back(Chain);
4375 break;
4376 case NVPTXISD::Suld1DV2I32Zero:
4377 Opc = NVPTX::SULD_1D_V2I32_ZERO;
4378 Ops.push_back(TexHandle);
4379 Ops.push_back(N->getOperand(2));
4380 Ops.push_back(Chain);
4381 break;
4382 case NVPTXISD::Suld1DV2I64Zero:
4383 Opc = NVPTX::SULD_1D_V2I64_ZERO;
4384 Ops.push_back(TexHandle);
4385 Ops.push_back(N->getOperand(2));
4386 Ops.push_back(Chain);
4387 break;
4388 case NVPTXISD::Suld1DV4I8Zero:
4389 Opc = NVPTX::SULD_1D_V4I8_ZERO;
4390 Ops.push_back(TexHandle);
4391 Ops.push_back(N->getOperand(2));
4392 Ops.push_back(Chain);
4393 break;
4394 case NVPTXISD::Suld1DV4I16Zero:
4395 Opc = NVPTX::SULD_1D_V4I16_ZERO;
4396 Ops.push_back(TexHandle);
4397 Ops.push_back(N->getOperand(2));
4398 Ops.push_back(Chain);
4399 break;
4400 case NVPTXISD::Suld1DV4I32Zero:
4401 Opc = NVPTX::SULD_1D_V4I32_ZERO;
4402 Ops.push_back(TexHandle);
4403 Ops.push_back(N->getOperand(2));
4404 Ops.push_back(Chain);
4405 break;
4406 case NVPTXISD::Suld1DArrayI8Zero:
4407 Opc = NVPTX::SULD_1D_ARRAY_I8_ZERO;
4408 Ops.push_back(TexHandle);
4409 Ops.push_back(N->getOperand(2));
4410 Ops.push_back(N->getOperand(3));
4411 Ops.push_back(Chain);
4412 break;
4413 case NVPTXISD::Suld1DArrayI16Zero:
4414 Opc = NVPTX::SULD_1D_ARRAY_I16_ZERO;
4415 Ops.push_back(TexHandle);
4416 Ops.push_back(N->getOperand(2));
4417 Ops.push_back(N->getOperand(3));
4418 Ops.push_back(Chain);
4419 break;
4420 case NVPTXISD::Suld1DArrayI32Zero:
4421 Opc = NVPTX::SULD_1D_ARRAY_I32_ZERO;
4422 Ops.push_back(TexHandle);
4423 Ops.push_back(N->getOperand(2));
4424 Ops.push_back(N->getOperand(3));
4425 Ops.push_back(Chain);
4426 break;
4427 case NVPTXISD::Suld1DArrayI64Zero:
4428 Opc = NVPTX::SULD_1D_ARRAY_I64_ZERO;
4429 Ops.push_back(TexHandle);
4430 Ops.push_back(N->getOperand(2));
4431 Ops.push_back(N->getOperand(3));
4432 Ops.push_back(Chain);
4433 break;
4434 case NVPTXISD::Suld1DArrayV2I8Zero:
4435 Opc = NVPTX::SULD_1D_ARRAY_V2I8_ZERO;
4436 Ops.push_back(TexHandle);
4437 Ops.push_back(N->getOperand(2));
4438 Ops.push_back(N->getOperand(3));
4439 Ops.push_back(Chain);
4440 break;
4441 case NVPTXISD::Suld1DArrayV2I16Zero:
4442 Opc = NVPTX::SULD_1D_ARRAY_V2I16_ZERO;
4443 Ops.push_back(TexHandle);
4444 Ops.push_back(N->getOperand(2));
4445 Ops.push_back(N->getOperand(3));
4446 Ops.push_back(Chain);
4447 break;
4448 case NVPTXISD::Suld1DArrayV2I32Zero:
4449 Opc = NVPTX::SULD_1D_ARRAY_V2I32_ZERO;
4450 Ops.push_back(TexHandle);
4451 Ops.push_back(N->getOperand(2));
4452 Ops.push_back(N->getOperand(3));
4453 Ops.push_back(Chain);
4454 break;
4455 case NVPTXISD::Suld1DArrayV2I64Zero:
4456 Opc = NVPTX::SULD_1D_ARRAY_V2I64_ZERO;
4457 Ops.push_back(TexHandle);
4458 Ops.push_back(N->getOperand(2));
4459 Ops.push_back(N->getOperand(3));
4460 Ops.push_back(Chain);
4461 break;
4462 case NVPTXISD::Suld1DArrayV4I8Zero:
4463 Opc = NVPTX::SULD_1D_ARRAY_V4I8_ZERO;
4464 Ops.push_back(TexHandle);
4465 Ops.push_back(N->getOperand(2));
4466 Ops.push_back(N->getOperand(3));
4467 Ops.push_back(Chain);
4468 break;
4469 case NVPTXISD::Suld1DArrayV4I16Zero:
4470 Opc = NVPTX::SULD_1D_ARRAY_V4I16_ZERO;
4471 Ops.push_back(TexHandle);
4472 Ops.push_back(N->getOperand(2));
4473 Ops.push_back(N->getOperand(3));
4474 Ops.push_back(Chain);
4475 break;
4476 case NVPTXISD::Suld1DArrayV4I32Zero:
4477 Opc = NVPTX::SULD_1D_ARRAY_V4I32_ZERO;
4478 Ops.push_back(TexHandle);
4479 Ops.push_back(N->getOperand(2));
4480 Ops.push_back(N->getOperand(3));
4481 Ops.push_back(Chain);
4482 break;
4483 case NVPTXISD::Suld2DI8Zero:
4484 Opc = NVPTX::SULD_2D_I8_ZERO;
4485 Ops.push_back(TexHandle);
4486 Ops.push_back(N->getOperand(2));
4487 Ops.push_back(N->getOperand(3));
4488 Ops.push_back(Chain);
4489 break;
4490 case NVPTXISD::Suld2DI16Zero:
4491 Opc = NVPTX::SULD_2D_I16_ZERO;
4492 Ops.push_back(TexHandle);
4493 Ops.push_back(N->getOperand(2));
4494 Ops.push_back(N->getOperand(3));
4495 Ops.push_back(Chain);
4496 break;
4497 case NVPTXISD::Suld2DI32Zero:
4498 Opc = NVPTX::SULD_2D_I32_ZERO;
4499 Ops.push_back(TexHandle);
4500 Ops.push_back(N->getOperand(2));
4501 Ops.push_back(N->getOperand(3));
4502 Ops.push_back(Chain);
4503 break;
4504 case NVPTXISD::Suld2DI64Zero:
4505 Opc = NVPTX::SULD_2D_I64_ZERO;
4506 Ops.push_back(TexHandle);
4507 Ops.push_back(N->getOperand(2));
4508 Ops.push_back(N->getOperand(3));
4509 Ops.push_back(Chain);
4510 break;
4511 case NVPTXISD::Suld2DV2I8Zero:
4512 Opc = NVPTX::SULD_2D_V2I8_ZERO;
4513 Ops.push_back(TexHandle);
4514 Ops.push_back(N->getOperand(2));
4515 Ops.push_back(N->getOperand(3));
4516 Ops.push_back(Chain);
4517 break;
4518 case NVPTXISD::Suld2DV2I16Zero:
4519 Opc = NVPTX::SULD_2D_V2I16_ZERO;
4520 Ops.push_back(TexHandle);
4521 Ops.push_back(N->getOperand(2));
4522 Ops.push_back(N->getOperand(3));
4523 Ops.push_back(Chain);
4524 break;
4525 case NVPTXISD::Suld2DV2I32Zero:
4526 Opc = NVPTX::SULD_2D_V2I32_ZERO;
4527 Ops.push_back(TexHandle);
4528 Ops.push_back(N->getOperand(2));
4529 Ops.push_back(N->getOperand(3));
4530 Ops.push_back(Chain);
4531 break;
4532 case NVPTXISD::Suld2DV2I64Zero:
4533 Opc = NVPTX::SULD_2D_V2I64_ZERO;
4534 Ops.push_back(TexHandle);
4535 Ops.push_back(N->getOperand(2));
4536 Ops.push_back(N->getOperand(3));
4537 Ops.push_back(Chain);
4538 break;
4539 case NVPTXISD::Suld2DV4I8Zero:
4540 Opc = NVPTX::SULD_2D_V4I8_ZERO;
4541 Ops.push_back(TexHandle);
4542 Ops.push_back(N->getOperand(2));
4543 Ops.push_back(N->getOperand(3));
4544 Ops.push_back(Chain);
4545 break;
4546 case NVPTXISD::Suld2DV4I16Zero:
4547 Opc = NVPTX::SULD_2D_V4I16_ZERO;
4548 Ops.push_back(TexHandle);
4549 Ops.push_back(N->getOperand(2));
4550 Ops.push_back(N->getOperand(3));
4551 Ops.push_back(Chain);
4552 break;
4553 case NVPTXISD::Suld2DV4I32Zero:
4554 Opc = NVPTX::SULD_2D_V4I32_ZERO;
4555 Ops.push_back(TexHandle);
4556 Ops.push_back(N->getOperand(2));
4557 Ops.push_back(N->getOperand(3));
4558 Ops.push_back(Chain);
4559 break;
4560 case NVPTXISD::Suld2DArrayI8Zero:
4561 Opc = NVPTX::SULD_2D_ARRAY_I8_ZERO;
4562 Ops.push_back(TexHandle);
4563 Ops.push_back(N->getOperand(2));
4564 Ops.push_back(N->getOperand(3));
4565 Ops.push_back(N->getOperand(4));
4566 Ops.push_back(Chain);
4567 break;
4568 case NVPTXISD::Suld2DArrayI16Zero:
4569 Opc = NVPTX::SULD_2D_ARRAY_I16_ZERO;
4570 Ops.push_back(TexHandle);
4571 Ops.push_back(N->getOperand(2));
4572 Ops.push_back(N->getOperand(3));
4573 Ops.push_back(N->getOperand(4));
4574 Ops.push_back(Chain);
4575 break;
4576 case NVPTXISD::Suld2DArrayI32Zero:
4577 Opc = NVPTX::SULD_2D_ARRAY_I32_ZERO;
4578 Ops.push_back(TexHandle);
4579 Ops.push_back(N->getOperand(2));
4580 Ops.push_back(N->getOperand(3));
4581 Ops.push_back(N->getOperand(4));
4582 Ops.push_back(Chain);
4583 break;
4584 case NVPTXISD::Suld2DArrayI64Zero:
4585 Opc = NVPTX::SULD_2D_ARRAY_I64_ZERO;
4586 Ops.push_back(TexHandle);
4587 Ops.push_back(N->getOperand(2));
4588 Ops.push_back(N->getOperand(3));
4589 Ops.push_back(N->getOperand(4));
4590 Ops.push_back(Chain);
4591 break;
4592 case NVPTXISD::Suld2DArrayV2I8Zero:
4593 Opc = NVPTX::SULD_2D_ARRAY_V2I8_ZERO;
4594 Ops.push_back(TexHandle);
4595 Ops.push_back(N->getOperand(2));
4596 Ops.push_back(N->getOperand(3));
4597 Ops.push_back(N->getOperand(4));
4598 Ops.push_back(Chain);
4599 break;
4600 case NVPTXISD::Suld2DArrayV2I16Zero:
4601 Opc = NVPTX::SULD_2D_ARRAY_V2I16_ZERO;
4602 Ops.push_back(TexHandle);
4603 Ops.push_back(N->getOperand(2));
4604 Ops.push_back(N->getOperand(3));
4605 Ops.push_back(N->getOperand(4));
4606 Ops.push_back(Chain);
4607 break;
4608 case NVPTXISD::Suld2DArrayV2I32Zero:
4609 Opc = NVPTX::SULD_2D_ARRAY_V2I32_ZERO;
4610 Ops.push_back(TexHandle);
4611 Ops.push_back(N->getOperand(2));
4612 Ops.push_back(N->getOperand(3));
4613 Ops.push_back(N->getOperand(4));
4614 Ops.push_back(Chain);
4615 break;
4616 case NVPTXISD::Suld2DArrayV2I64Zero:
4617 Opc = NVPTX::SULD_2D_ARRAY_V2I64_ZERO;
4618 Ops.push_back(TexHandle);
4619 Ops.push_back(N->getOperand(2));
4620 Ops.push_back(N->getOperand(3));
4621 Ops.push_back(N->getOperand(4));
4622 Ops.push_back(Chain);
4623 break;
4624 case NVPTXISD::Suld2DArrayV4I8Zero:
4625 Opc = NVPTX::SULD_2D_ARRAY_V4I8_ZERO;
4626 Ops.push_back(TexHandle);
4627 Ops.push_back(N->getOperand(2));
4628 Ops.push_back(N->getOperand(3));
4629 Ops.push_back(N->getOperand(4));
4630 Ops.push_back(Chain);
4631 break;
4632 case NVPTXISD::Suld2DArrayV4I16Zero:
4633 Opc = NVPTX::SULD_2D_ARRAY_V4I16_ZERO;
4634 Ops.push_back(TexHandle);
4635 Ops.push_back(N->getOperand(2));
4636 Ops.push_back(N->getOperand(3));
4637 Ops.push_back(N->getOperand(4));
4638 Ops.push_back(Chain);
4639 break;
4640 case NVPTXISD::Suld2DArrayV4I32Zero:
4641 Opc = NVPTX::SULD_2D_ARRAY_V4I32_ZERO;
4642 Ops.push_back(TexHandle);
4643 Ops.push_back(N->getOperand(2));
4644 Ops.push_back(N->getOperand(3));
4645 Ops.push_back(N->getOperand(4));
4646 Ops.push_back(Chain);
4647 break;
4648 case NVPTXISD::Suld3DI8Zero:
4649 Opc = NVPTX::SULD_3D_I8_ZERO;
4650 Ops.push_back(TexHandle);
4651 Ops.push_back(N->getOperand(2));
4652 Ops.push_back(N->getOperand(3));
4653 Ops.push_back(N->getOperand(4));
4654 Ops.push_back(Chain);
4655 break;
4656 case NVPTXISD::Suld3DI16Zero:
4657 Opc = NVPTX::SULD_3D_I16_ZERO;
4658 Ops.push_back(TexHandle);
4659 Ops.push_back(N->getOperand(2));
4660 Ops.push_back(N->getOperand(3));
4661 Ops.push_back(N->getOperand(4));
4662 Ops.push_back(Chain);
4663 break;
4664 case NVPTXISD::Suld3DI32Zero:
4665 Opc = NVPTX::SULD_3D_I32_ZERO;
4666 Ops.push_back(TexHandle);
4667 Ops.push_back(N->getOperand(2));
4668 Ops.push_back(N->getOperand(3));
4669 Ops.push_back(N->getOperand(4));
4670 Ops.push_back(Chain);
4671 break;
4672 case NVPTXISD::Suld3DI64Zero:
4673 Opc = NVPTX::SULD_3D_I64_ZERO;
4674 Ops.push_back(TexHandle);
4675 Ops.push_back(N->getOperand(2));
4676 Ops.push_back(N->getOperand(3));
4677 Ops.push_back(N->getOperand(4));
4678 Ops.push_back(Chain);
4679 break;
4680 case NVPTXISD::Suld3DV2I8Zero:
4681 Opc = NVPTX::SULD_3D_V2I8_ZERO;
4682 Ops.push_back(TexHandle);
4683 Ops.push_back(N->getOperand(2));
4684 Ops.push_back(N->getOperand(3));
4685 Ops.push_back(N->getOperand(4));
4686 Ops.push_back(Chain);
4687 break;
4688 case NVPTXISD::Suld3DV2I16Zero:
4689 Opc = NVPTX::SULD_3D_V2I16_ZERO;
4690 Ops.push_back(TexHandle);
4691 Ops.push_back(N->getOperand(2));
4692 Ops.push_back(N->getOperand(3));
4693 Ops.push_back(N->getOperand(4));
4694 Ops.push_back(Chain);
4695 break;
4696 case NVPTXISD::Suld3DV2I32Zero:
4697 Opc = NVPTX::SULD_3D_V2I32_ZERO;
4698 Ops.push_back(TexHandle);
4699 Ops.push_back(N->getOperand(2));
4700 Ops.push_back(N->getOperand(3));
4701 Ops.push_back(N->getOperand(4));
4702 Ops.push_back(Chain);
4703 break;
4704 case NVPTXISD::Suld3DV2I64Zero:
4705 Opc = NVPTX::SULD_3D_V2I64_ZERO;
4706 Ops.push_back(TexHandle);
4707 Ops.push_back(N->getOperand(2));
4708 Ops.push_back(N->getOperand(3));
4709 Ops.push_back(N->getOperand(4));
4710 Ops.push_back(Chain);
4711 break;
4712 case NVPTXISD::Suld3DV4I8Zero:
4713 Opc = NVPTX::SULD_3D_V4I8_ZERO;
4714 Ops.push_back(TexHandle);
4715 Ops.push_back(N->getOperand(2));
4716 Ops.push_back(N->getOperand(3));
4717 Ops.push_back(N->getOperand(4));
4718 Ops.push_back(Chain);
4719 break;
4720 case NVPTXISD::Suld3DV4I16Zero:
4721 Opc = NVPTX::SULD_3D_V4I16_ZERO;
4722 Ops.push_back(TexHandle);
4723 Ops.push_back(N->getOperand(2));
4724 Ops.push_back(N->getOperand(3));
4725 Ops.push_back(N->getOperand(4));
4726 Ops.push_back(Chain);
4727 break;
4728 case NVPTXISD::Suld3DV4I32Zero:
4729 Opc = NVPTX::SULD_3D_V4I32_ZERO;
4730 Ops.push_back(TexHandle);
4731 Ops.push_back(N->getOperand(2));
4732 Ops.push_back(N->getOperand(3));
4733 Ops.push_back(N->getOperand(4));
4734 Ops.push_back(Chain);
4735 break;
4736 }
4737 Ret = CurDAG->getMachineNode(Opc, SDLoc(N), N->getVTList(), Ops);
4738 return Ret;
4739 }
4740
4741
4742 /// SelectBFE - Look for instruction sequences that can be made more efficient
4743 /// by using the 'bfe' (bit-field extract) PTX instruction
4744 SDNode *NVPTXDAGToDAGISel::SelectBFE(SDNode *N) {
4745 SDValue LHS = N->getOperand(0);
4746 SDValue RHS = N->getOperand(1);
4747 SDValue Len;
4748 SDValue Start;
4749 SDValue Val;
4750 bool IsSigned = false;
4751
4752 if (N->getOpcode() == ISD::AND) {
4753 // Canonicalize the operands
4754 // We want 'and %val, %mask'
4755 if (isa<ConstantSDNode>(LHS) && !isa<ConstantSDNode>(RHS)) {
4756 std::swap(LHS, RHS);
4757 }
4758
4759 ConstantSDNode *Mask = dyn_cast<ConstantSDNode>(RHS);
4760 if (!Mask) {
4761 // We need a constant mask on the RHS of the AND
4762 return NULL;
4763 }
4764
4765 // Extract the mask bits
4766 uint64_t MaskVal = Mask->getZExtValue();
4767 if (!isMask_64(MaskVal)) {
4768 // We *could* handle shifted masks here, but doing so would require an
4769 // 'and' operation to fix up the low-order bits so we would trade
4770 // shr+and for bfe+and, which has the same throughput
4771 return NULL;
4772 }
4773
4774 // How many bits are in our mask?
4775 uint64_t NumBits = CountTrailingOnes_64(MaskVal);
4776 Len = CurDAG->getTargetConstant(NumBits, MVT::i32);
4777
4778 if (LHS.getOpcode() == ISD::SRL || LHS.getOpcode() == ISD::SRA) {
4779 // We have a 'srl/and' pair, extract the effective start bit and length
4780 Val = LHS.getNode()->getOperand(0);
4781 Start = LHS.getNode()->getOperand(1);
4782 ConstantSDNode *StartConst = dyn_cast<ConstantSDNode>(Start);
4783 if (StartConst) {
4784 uint64_t StartVal = StartConst->getZExtValue();
4785 // How many "good" bits do we have left? "good" is defined here as bits
4786 // that exist in the original value, not shifted in.
4787 uint64_t GoodBits = Start.getValueType().getSizeInBits() - StartVal;
4788 if (NumBits > GoodBits) {
4789 // Do not handle the case where bits have been shifted in. In theory
4790 // we could handle this, but the cost is likely higher than just
4791 // emitting the srl/and pair.
4792 return NULL;
4793 }
4794 Start = CurDAG->getTargetConstant(StartVal, MVT::i32);
4795 } else {
4796 // Do not handle the case where the shift amount (can be zero if no srl
4797 // was found) is not constant. We could handle this case, but it would
4798 // require run-time logic that would be more expensive than just
4799 // emitting the srl/and pair.
4800 return NULL;
4801 }
4802 } else {
4803 // Do not handle the case where the LHS of the and is not a shift. While
4804 // it would be trivial to handle this case, it would just transform
4805 // 'and' -> 'bfe', but 'and' has higher-throughput.
4806 return NULL;
4807 }
4808 } else if (N->getOpcode() == ISD::SRL || N->getOpcode() == ISD::SRA) {
4809 if (LHS->getOpcode() == ISD::AND) {
4810 ConstantSDNode *ShiftCnst = dyn_cast<ConstantSDNode>(RHS);
4811 if (!ShiftCnst) {
4812 // Shift amount must be constant
4813 return NULL;
4814 }
4815
4816 uint64_t ShiftAmt = ShiftCnst->getZExtValue();
4817
4818 SDValue AndLHS = LHS->getOperand(0);
4819 SDValue AndRHS = LHS->getOperand(1);
4820
4821 // Canonicalize the AND to have the mask on the RHS
4822 if (isa<ConstantSDNode>(AndLHS)) {
4823 std::swap(AndLHS, AndRHS);
4824 }
4825
4826 ConstantSDNode *MaskCnst = dyn_cast<ConstantSDNode>(AndRHS);
4827 if (!MaskCnst) {
4828 // Mask must be constant
4829 return NULL;
4830 }
4831
4832 uint64_t MaskVal = MaskCnst->getZExtValue();
4833 uint64_t NumZeros;
4834 uint64_t NumBits;
4835 if (isMask_64(MaskVal)) {
4836 NumZeros = 0;
4837 // The number of bits in the result bitfield will be the number of
4838 // trailing ones (the AND) minus the number of bits we shift off
4839 NumBits = CountTrailingOnes_64(MaskVal) - ShiftAmt;
4840 } else if (isShiftedMask_64(MaskVal)) {
4841 NumZeros = countTrailingZeros(MaskVal);
4842 unsigned NumOnes = CountTrailingOnes_64(MaskVal >> NumZeros);
4843 // The number of bits in the result bitfield will be the number of
4844 // trailing zeros plus the number of set bits in the mask minus the
4845 // number of bits we shift off
4846 NumBits = NumZeros + NumOnes - ShiftAmt;
4847 } else {
4848 // This is not a mask we can handle
4849 return NULL;
4850 }
4851
4852 if (ShiftAmt < NumZeros) {
4853 // Handling this case would require extra logic that would make this
4854 // transformation non-profitable
4855 return NULL;
4856 }
4857
4858 Val = AndLHS;
4859 Start = CurDAG->getTargetConstant(ShiftAmt, MVT::i32);
4860 Len = CurDAG->getTargetConstant(NumBits, MVT::i32);
4861 } else if (LHS->getOpcode() == ISD::SHL) {
4862 // Here, we have a pattern like:
4863 //
4864 // (sra (shl val, NN), MM)
4865 // or
4866 // (srl (shl val, NN), MM)
4867 //
4868 // If MM >= NN, we can efficiently optimize this with bfe
4869 Val = LHS->getOperand(0);
4870
4871 SDValue ShlRHS = LHS->getOperand(1);
4872 ConstantSDNode *ShlCnst = dyn_cast<ConstantSDNode>(ShlRHS);
4873 if (!ShlCnst) {
4874 // Shift amount must be constant
4875 return NULL;
4876 }
4877 uint64_t InnerShiftAmt = ShlCnst->getZExtValue();
4878
4879 SDValue ShrRHS = RHS;
4880 ConstantSDNode *ShrCnst = dyn_cast<ConstantSDNode>(ShrRHS);
4881 if (!ShrCnst) {
4882 // Shift amount must be constant
4883 return NULL;
4884 }
4885 uint64_t OuterShiftAmt = ShrCnst->getZExtValue();
4886
4887 // To avoid extra codegen and be profitable, we need Outer >= Inner
4888 if (OuterShiftAmt < InnerShiftAmt) {
4889 return NULL;
4890 }
4891
4892 // If the outer shift is more than the type size, we have no bitfield to
4893 // extract (since we also check that the inner shift is <= the outer shift
4894 // then this also implies that the inner shift is < the type size)
4895 if (OuterShiftAmt >= Val.getValueType().getSizeInBits()) {
4896 return NULL;
4897 }
4898
4899 Start =
4900 CurDAG->getTargetConstant(OuterShiftAmt - InnerShiftAmt, MVT::i32);
4901 Len =
4902 CurDAG->getTargetConstant(Val.getValueType().getSizeInBits() -
4903 OuterShiftAmt, MVT::i32);
4904
4905 if (N->getOpcode() == ISD::SRA) {
4906 // If we have a arithmetic right shift, we need to use the signed bfe
4907 // variant
4908 IsSigned = true;
4909 }
4910 } else {
4911 // No can do...
4912 return NULL;
4913 }
4914 } else {
4915 // No can do...
4916 return NULL;
4917 }
4918
4919
4920 unsigned Opc;
4921 // For the BFE operations we form here from "and" and "srl", always use the
4922 // unsigned variants.
4923 if (Val.getValueType() == MVT::i32) {
4924 if (IsSigned) {
4925 Opc = NVPTX::BFE_S32rii;
4926 } else {
4927 Opc = NVPTX::BFE_U32rii;
4928 }
4929 } else if (Val.getValueType() == MVT::i64) {
4930 if (IsSigned) {
4931 Opc = NVPTX::BFE_S64rii;
4932 } else {
4933 Opc = NVPTX::BFE_U64rii;
4934 }
4935 } else {
4936 // We cannot handle this type
4937 return NULL;
4938 }
4939
4940 SDValue Ops[] = {
4941 Val, Start, Len
4942 };
4943
4944 SDNode *Ret =
4945 CurDAG->getMachineNode(Opc, SDLoc(N), N->getVTList(), Ops);
2307 4946
2308 return Ret; 4947 return Ret;
2309 } 4948 }
2310 4949
2311 // SelectDirectAddr - Match a direct address for DAG. 4950 // SelectDirectAddr - Match a direct address for DAG.
2399 return SelectADDRri_imp(OpNode, Addr, Base, Offset, MVT::i64); 5038 return SelectADDRri_imp(OpNode, Addr, Base, Offset, MVT::i64);
2400 } 5039 }
2401 5040
2402 bool NVPTXDAGToDAGISel::ChkMemSDNodeAddressSpace(SDNode *N, 5041 bool NVPTXDAGToDAGISel::ChkMemSDNodeAddressSpace(SDNode *N,
2403 unsigned int spN) const { 5042 unsigned int spN) const {
2404 const Value *Src = NULL; 5043 const Value *Src = nullptr;
2405 // Even though MemIntrinsicSDNode is a subclas of MemSDNode,
2406 // the classof() for MemSDNode does not include MemIntrinsicSDNode
2407 // (See SelectionDAGNodes.h). So we need to check for both.
2408 if (MemSDNode *mN = dyn_cast<MemSDNode>(N)) { 5044 if (MemSDNode *mN = dyn_cast<MemSDNode>(N)) {
2409 Src = mN->getSrcValue(); 5045 if (spN == 0 && mN->getMemOperand()->getPseudoValue())
2410 } else if (MemSDNode *mN = dyn_cast<MemIntrinsicSDNode>(N)) { 5046 return true;
2411 Src = mN->getSrcValue(); 5047 Src = mN->getMemOperand()->getValue();
2412 } 5048 }
2413 if (!Src) 5049 if (!Src)
2414 return false; 5050 return false;
2415 if (const PointerType *PT = dyn_cast<PointerType>(Src->getType())) 5051 if (const PointerType *PT = dyn_cast<PointerType>(Src->getType()))
2416 return (PT->getAddressSpace() == spN); 5052 return (PT->getAddressSpace() == spN);
2438 } 5074 }
2439 break; 5075 break;
2440 } 5076 }
2441 return true; 5077 return true;
2442 } 5078 }
2443
2444 // Return true if N is a undef or a constant.
2445 // If N was undef, return a (i8imm 0) in Retval
2446 // If N was imm, convert it to i8imm and return in Retval
2447 // Note: The convert to i8imm is required, otherwise the
2448 // pattern matcher inserts a bunch of IMOVi8rr to convert
2449 // the imm to i8imm, and this causes instruction selection
2450 // to fail.
2451 bool NVPTXDAGToDAGISel::UndefOrImm(SDValue Op, SDValue N, SDValue &Retval) {
2452 if (!(N.getOpcode() == ISD::UNDEF) && !(N.getOpcode() == ISD::Constant))
2453 return false;
2454
2455 if (N.getOpcode() == ISD::UNDEF)
2456 Retval = CurDAG->getTargetConstant(0, MVT::i8);
2457 else {
2458 ConstantSDNode *cn = cast<ConstantSDNode>(N.getNode());
2459 unsigned retval = cn->getZExtValue();
2460 Retval = CurDAG->getTargetConstant(retval, MVT::i8);
2461 }
2462 return true;
2463 }