Mercurial > hg > Members > tobaru > cbc > CbC_llvm

comparison lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp @ 0:95c75e76d11b

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
LLVM 3.4
author Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp>
date Thu, 12 Dec 2013 13:56:28 +0900
parents
children e4204d083e25
comparison
equal deleted inserted replaced
-1:000000000000 0:95c75e76d11b
1 //===-- NVPTXISelDAGToDAG.cpp - A dag to dag inst selector for NVPTX ------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file defines an instruction selector for the NVPTX target.
11 //
12 //===----------------------------------------------------------------------===//
13
14 #include "NVPTXISelDAGToDAG.h"
15 #include "llvm/IR/GlobalValue.h"
16 #include "llvm/IR/Instructions.h"
17 #include "llvm/Support/CommandLine.h"
18 #include "llvm/Support/Debug.h"
19 #include "llvm/Support/ErrorHandling.h"
20 #include "llvm/Support/raw_ostream.h"
21 #include "llvm/Target/TargetIntrinsicInfo.h"
22
23 #undef DEBUG_TYPE
24 #define DEBUG_TYPE "nvptx-isel"
25
26 using namespace llvm;
27
28 static cl::opt<int>
29 FMAContractLevel("nvptx-fma-level", cl::ZeroOrMore, cl::Hidden,
30 cl::desc("NVPTX Specific: FMA contraction (0: don't do it"
31 " 1: do it 2: do it aggressively"),
32 cl::init(2));
33
34 static cl::opt<int> UsePrecDivF32(
35 "nvptx-prec-divf32", cl::ZeroOrMore, cl::Hidden,
36 cl::desc("NVPTX Specifies: 0 use div.approx, 1 use div.full, 2 use"
37 " IEEE Compliant F32 div.rnd if avaiable."),
38 cl::init(2));
39
40 static cl::opt<bool>
41 UsePrecSqrtF32("nvptx-prec-sqrtf32", cl::Hidden,
42 cl::desc("NVPTX Specific: 0 use sqrt.approx, 1 use sqrt.rn."),
43 cl::init(true));
44
45 static cl::opt<bool>
46 FtzEnabled("nvptx-f32ftz", cl::ZeroOrMore, cl::Hidden,
47 cl::desc("NVPTX Specific: Flush f32 subnormals to sign-preserving zero."),
48 cl::init(false));
49
50
51 /// createNVPTXISelDag - This pass converts a legalized DAG into a
52 /// NVPTX-specific DAG, ready for instruction scheduling.
53 FunctionPass *llvm::createNVPTXISelDag(NVPTXTargetMachine &TM,
54 llvm::CodeGenOpt::Level OptLevel) {
55 return new NVPTXDAGToDAGISel(TM, OptLevel);
56 }
57
58 NVPTXDAGToDAGISel::NVPTXDAGToDAGISel(NVPTXTargetMachine &tm,
59 CodeGenOpt::Level OptLevel)
60 : SelectionDAGISel(tm, OptLevel),
61 Subtarget(tm.getSubtarget<NVPTXSubtarget>()) {
62
63 doFMAF32 = (OptLevel > 0) && Subtarget.hasFMAF32() && (FMAContractLevel >= 1);
64 doFMAF64 = (OptLevel > 0) && Subtarget.hasFMAF64() && (FMAContractLevel >= 1);
65 doFMAF32AGG =
66 (OptLevel > 0) && Subtarget.hasFMAF32() && (FMAContractLevel == 2);
67 doFMAF64AGG =
68 (OptLevel > 0) && Subtarget.hasFMAF64() && (FMAContractLevel == 2);
69
70 allowFMA = (FMAContractLevel >= 1);
71
72 doMulWide = (OptLevel > 0);
73 }
74
75 int NVPTXDAGToDAGISel::getDivF32Level() const {
76 if (UsePrecDivF32.getNumOccurrences() > 0) {
77 // If nvptx-prec-div32=N is used on the command-line, always honor it
78 return UsePrecDivF32;
79 } else {
80 // Otherwise, use div.approx if fast math is enabled
81 if (TM.Options.UnsafeFPMath)
82 return 0;
83 else
84 return 2;
85 }
86 }
87
88 bool NVPTXDAGToDAGISel::usePrecSqrtF32() const {
89 if (UsePrecSqrtF32.getNumOccurrences() > 0) {
90 // If nvptx-prec-sqrtf32 is used on the command-line, always honor it
91 return UsePrecSqrtF32;
92 } else {
93 // Otherwise, use sqrt.approx if fast math is enabled
94 if (TM.Options.UnsafeFPMath)
95 return false;
96 else
97 return true;
98 }
99 }
100
101 bool NVPTXDAGToDAGISel::useF32FTZ() const {
102 if (FtzEnabled.getNumOccurrences() > 0) {
103 // If nvptx-f32ftz is used on the command-line, always honor it
104 return FtzEnabled;
105 } else {
106 const Function *F = MF->getFunction();
107 // Otherwise, check for an nvptx-f32ftz attribute on the function
108 if (F->hasFnAttribute("nvptx-f32ftz"))
109 return (F->getAttributes().getAttribute(AttributeSet::FunctionIndex,
110 "nvptx-f32ftz")
111 .getValueAsString() == "true");
112 else
113 return false;
114 }
115 }
116
117 /// Select - Select instructions not customized! Used for
118 /// expanded, promoted and normal instructions.
119 SDNode *NVPTXDAGToDAGISel::Select(SDNode *N) {
120
121 if (N->isMachineOpcode()) {
122 N->setNodeId(-1);
123 return NULL; // Already selected.
124 }
125
126 SDNode *ResNode = NULL;
127 switch (N->getOpcode()) {
128 case ISD::LOAD:
129 ResNode = SelectLoad(N);
130 break;
131 case ISD::STORE:
132 ResNode = SelectStore(N);
133 break;
134 case NVPTXISD::LoadV2:
135 case NVPTXISD::LoadV4:
136 ResNode = SelectLoadVector(N);
137 break;
138 case NVPTXISD::LDGV2:
139 case NVPTXISD::LDGV4:
140 case NVPTXISD::LDUV2:
141 case NVPTXISD::LDUV4:
142 ResNode = SelectLDGLDUVector(N);
143 break;
144 case NVPTXISD::StoreV2:
145 case NVPTXISD::StoreV4:
146 ResNode = SelectStoreVector(N);
147 break;
148 case NVPTXISD::LoadParam:
149 case NVPTXISD::LoadParamV2:
150 case NVPTXISD::LoadParamV4:
151 ResNode = SelectLoadParam(N);
152 break;
153 case NVPTXISD::StoreRetval:
154 case NVPTXISD::StoreRetvalV2:
155 case NVPTXISD::StoreRetvalV4:
156 ResNode = SelectStoreRetval(N);
157 break;
158 case NVPTXISD::StoreParam:
159 case NVPTXISD::StoreParamV2:
160 case NVPTXISD::StoreParamV4:
161 case NVPTXISD::StoreParamS32:
162 case NVPTXISD::StoreParamU32:
163 ResNode = SelectStoreParam(N);
164 break;
165 default:
166 break;
167 }
168 if (ResNode)
169 return ResNode;
170 return SelectCode(N);
171 }
172
173 static unsigned int getCodeAddrSpace(MemSDNode *N,
174 const NVPTXSubtarget &Subtarget) {
175 const Value *Src = N->getSrcValue();
176
177 if (!Src)
178 return NVPTX::PTXLdStInstCode::GENERIC;
179
180 if (const PointerType *PT = dyn_cast<PointerType>(Src->getType())) {
181 switch (PT->getAddressSpace()) {
182 case llvm::ADDRESS_SPACE_LOCAL: return NVPTX::PTXLdStInstCode::LOCAL;
183 case llvm::ADDRESS_SPACE_GLOBAL: return NVPTX::PTXLdStInstCode::GLOBAL;
184 case llvm::ADDRESS_SPACE_SHARED: return NVPTX::PTXLdStInstCode::SHARED;
185 case llvm::ADDRESS_SPACE_GENERIC: return NVPTX::PTXLdStInstCode::GENERIC;
186 case llvm::ADDRESS_SPACE_PARAM: return NVPTX::PTXLdStInstCode::PARAM;
187 case llvm::ADDRESS_SPACE_CONST: return NVPTX::PTXLdStInstCode::CONSTANT;
188 default: break;
189 }
190 }
191 return NVPTX::PTXLdStInstCode::GENERIC;
192 }
193
194 SDNode *NVPTXDAGToDAGISel::SelectLoad(SDNode *N) {
195 SDLoc dl(N);
196 LoadSDNode *LD = cast<LoadSDNode>(N);
197 EVT LoadedVT = LD->getMemoryVT();
198 SDNode *NVPTXLD = NULL;
199
200 // do not support pre/post inc/dec
201 if (LD->isIndexed())
202 return NULL;
203
204 if (!LoadedVT.isSimple())
205 return NULL;
206
207 // Address Space Setting
208 unsigned int codeAddrSpace = getCodeAddrSpace(LD, Subtarget);
209
210 // Volatile Setting
211 // - .volatile is only availalble for .global and .shared
212 bool isVolatile = LD->isVolatile();
213 if (codeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL &&
214 codeAddrSpace != NVPTX::PTXLdStInstCode::SHARED &&
215 codeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC)
216 isVolatile = false;
217
218 // Vector Setting
219 MVT SimpleVT = LoadedVT.getSimpleVT();
220 unsigned vecType = NVPTX::PTXLdStInstCode::Scalar;
221 if (SimpleVT.isVector()) {
222 unsigned num = SimpleVT.getVectorNumElements();
223 if (num == 2)
224 vecType = NVPTX::PTXLdStInstCode::V2;
225 else if (num == 4)
226 vecType = NVPTX::PTXLdStInstCode::V4;
227 else
228 return NULL;
229 }
230
231 // Type Setting: fromType + fromTypeWidth
232 //
233 // Sign : ISD::SEXTLOAD
234 // Unsign : ISD::ZEXTLOAD, ISD::NON_EXTLOAD or ISD::EXTLOAD and the
235 // type is integer
236 // Float : ISD::NON_EXTLOAD or ISD::EXTLOAD and the type is float
237 MVT ScalarVT = SimpleVT.getScalarType();
238 // Read at least 8 bits (predicates are stored as 8-bit values)
239 unsigned fromTypeWidth = std::max(8U, ScalarVT.getSizeInBits());
240 unsigned int fromType;
241 if ((LD->getExtensionType() == ISD::SEXTLOAD))
242 fromType = NVPTX::PTXLdStInstCode::Signed;
243 else if (ScalarVT.isFloatingPoint())
244 fromType = NVPTX::PTXLdStInstCode::Float;
245 else
246 fromType = NVPTX::PTXLdStInstCode::Unsigned;
247
248 // Create the machine instruction DAG
249 SDValue Chain = N->getOperand(0);
250 SDValue N1 = N->getOperand(1);
251 SDValue Addr;
252 SDValue Offset, Base;
253 unsigned Opcode;
254 MVT::SimpleValueType TargetVT = LD->getSimpleValueType(0).SimpleTy;
255
256 if (SelectDirectAddr(N1, Addr)) {
257 switch (TargetVT) {
258 case MVT::i8:
259 Opcode = NVPTX::LD_i8_avar;
260 break;
261 case MVT::i16:
262 Opcode = NVPTX::LD_i16_avar;
263 break;
264 case MVT::i32:
265 Opcode = NVPTX::LD_i32_avar;
266 break;
267 case MVT::i64:
268 Opcode = NVPTX::LD_i64_avar;
269 break;
270 case MVT::f32:
271 Opcode = NVPTX::LD_f32_avar;
272 break;
273 case MVT::f64:
274 Opcode = NVPTX::LD_f64_avar;
275 break;
276 default:
277 return NULL;
278 }
279 SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
280 getI32Imm(vecType), getI32Imm(fromType),
281 getI32Imm(fromTypeWidth), Addr, Chain };
282 NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops);
283 } else if (Subtarget.is64Bit()
284 ? SelectADDRsi64(N1.getNode(), N1, Base, Offset)
285 : SelectADDRsi(N1.getNode(), N1, Base, Offset)) {
286 switch (TargetVT) {
287 case MVT::i8:
288 Opcode = NVPTX::LD_i8_asi;
289 break;
290 case MVT::i16:
291 Opcode = NVPTX::LD_i16_asi;
292 break;
293 case MVT::i32:
294 Opcode = NVPTX::LD_i32_asi;
295 break;
296 case MVT::i64:
297 Opcode = NVPTX::LD_i64_asi;
298 break;
299 case MVT::f32:
300 Opcode = NVPTX::LD_f32_asi;
301 break;
302 case MVT::f64:
303 Opcode = NVPTX::LD_f64_asi;
304 break;
305 default:
306 return NULL;
307 }
308 SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
309 getI32Imm(vecType), getI32Imm(fromType),
310 getI32Imm(fromTypeWidth), Base, Offset, Chain };
311 NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops);
312 } else if (Subtarget.is64Bit()
313 ? SelectADDRri64(N1.getNode(), N1, Base, Offset)
314 : SelectADDRri(N1.getNode(), N1, Base, Offset)) {
315 if (Subtarget.is64Bit()) {
316 switch (TargetVT) {
317 case MVT::i8:
318 Opcode = NVPTX::LD_i8_ari_64;
319 break;
320 case MVT::i16:
321 Opcode = NVPTX::LD_i16_ari_64;
322 break;
323 case MVT::i32:
324 Opcode = NVPTX::LD_i32_ari_64;
325 break;
326 case MVT::i64:
327 Opcode = NVPTX::LD_i64_ari_64;
328 break;
329 case MVT::f32:
330 Opcode = NVPTX::LD_f32_ari_64;
331 break;
332 case MVT::f64:
333 Opcode = NVPTX::LD_f64_ari_64;
334 break;
335 default:
336 return NULL;
337 }
338 } else {
339 switch (TargetVT) {
340 case MVT::i8:
341 Opcode = NVPTX::LD_i8_ari;
342 break;
343 case MVT::i16:
344 Opcode = NVPTX::LD_i16_ari;
345 break;
346 case MVT::i32:
347 Opcode = NVPTX::LD_i32_ari;
348 break;
349 case MVT::i64:
350 Opcode = NVPTX::LD_i64_ari;
351 break;
352 case MVT::f32:
353 Opcode = NVPTX::LD_f32_ari;
354 break;
355 case MVT::f64:
356 Opcode = NVPTX::LD_f64_ari;
357 break;
358 default:
359 return NULL;
360 }
361 }
362 SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
363 getI32Imm(vecType), getI32Imm(fromType),
364 getI32Imm(fromTypeWidth), Base, Offset, Chain };
365 NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops);
366 } else {
367 if (Subtarget.is64Bit()) {
368 switch (TargetVT) {
369 case MVT::i8:
370 Opcode = NVPTX::LD_i8_areg_64;
371 break;
372 case MVT::i16:
373 Opcode = NVPTX::LD_i16_areg_64;
374 break;
375 case MVT::i32:
376 Opcode = NVPTX::LD_i32_areg_64;
377 break;
378 case MVT::i64:
379 Opcode = NVPTX::LD_i64_areg_64;
380 break;
381 case MVT::f32:
382 Opcode = NVPTX::LD_f32_areg_64;
383 break;
384 case MVT::f64:
385 Opcode = NVPTX::LD_f64_areg_64;
386 break;
387 default:
388 return NULL;
389 }
390 } else {
391 switch (TargetVT) {
392 case MVT::i8:
393 Opcode = NVPTX::LD_i8_areg;
394 break;
395 case MVT::i16:
396 Opcode = NVPTX::LD_i16_areg;
397 break;
398 case MVT::i32:
399 Opcode = NVPTX::LD_i32_areg;
400 break;
401 case MVT::i64:
402 Opcode = NVPTX::LD_i64_areg;
403 break;
404 case MVT::f32:
405 Opcode = NVPTX::LD_f32_areg;
406 break;
407 case MVT::f64:
408 Opcode = NVPTX::LD_f64_areg;
409 break;
410 default:
411 return NULL;
412 }
413 }
414 SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
415 getI32Imm(vecType), getI32Imm(fromType),
416 getI32Imm(fromTypeWidth), N1, Chain };
417 NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops);
418 }
419
420 if (NVPTXLD != NULL) {
421 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
422 MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
423 cast<MachineSDNode>(NVPTXLD)->setMemRefs(MemRefs0, MemRefs0 + 1);
424 }
425
426 return NVPTXLD;
427 }
428
429 SDNode *NVPTXDAGToDAGISel::SelectLoadVector(SDNode *N) {
430
431 SDValue Chain = N->getOperand(0);
432 SDValue Op1 = N->getOperand(1);
433 SDValue Addr, Offset, Base;
434 unsigned Opcode;
435 SDLoc DL(N);
436 SDNode *LD;
437 MemSDNode *MemSD = cast<MemSDNode>(N);
438 EVT LoadedVT = MemSD->getMemoryVT();
439
440 if (!LoadedVT.isSimple())
441 return NULL;
442
443 // Address Space Setting
444 unsigned int CodeAddrSpace = getCodeAddrSpace(MemSD, Subtarget);
445
446 // Volatile Setting
447 // - .volatile is only availalble for .global and .shared
448 bool IsVolatile = MemSD->isVolatile();
449 if (CodeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL &&
450 CodeAddrSpace != NVPTX::PTXLdStInstCode::SHARED &&
451 CodeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC)
452 IsVolatile = false;
453
454 // Vector Setting
455 MVT SimpleVT = LoadedVT.getSimpleVT();
456
457 // Type Setting: fromType + fromTypeWidth
458 //
459 // Sign : ISD::SEXTLOAD
460 // Unsign : ISD::ZEXTLOAD, ISD::NON_EXTLOAD or ISD::EXTLOAD and the
461 // type is integer
462 // Float : ISD::NON_EXTLOAD or ISD::EXTLOAD and the type is float
463 MVT ScalarVT = SimpleVT.getScalarType();
464 // Read at least 8 bits (predicates are stored as 8-bit values)
465 unsigned FromTypeWidth = std::max(8U, ScalarVT.getSizeInBits());
466 unsigned int FromType;
467 // The last operand holds the original LoadSDNode::getExtensionType() value
468 unsigned ExtensionType = cast<ConstantSDNode>(
469 N->getOperand(N->getNumOperands() - 1))->getZExtValue();
470 if (ExtensionType == ISD::SEXTLOAD)
471 FromType = NVPTX::PTXLdStInstCode::Signed;
472 else if (ScalarVT.isFloatingPoint())
473 FromType = NVPTX::PTXLdStInstCode::Float;
474 else
475 FromType = NVPTX::PTXLdStInstCode::Unsigned;
476
477 unsigned VecType;
478
479 switch (N->getOpcode()) {
480 case NVPTXISD::LoadV2:
481 VecType = NVPTX::PTXLdStInstCode::V2;
482 break;
483 case NVPTXISD::LoadV4:
484 VecType = NVPTX::PTXLdStInstCode::V4;
485 break;
486 default:
487 return NULL;
488 }
489
490 EVT EltVT = N->getValueType(0);
491
492 if (SelectDirectAddr(Op1, Addr)) {
493 switch (N->getOpcode()) {
494 default:
495 return NULL;
496 case NVPTXISD::LoadV2:
497 switch (EltVT.getSimpleVT().SimpleTy) {
498 default:
499 return NULL;
500 case MVT::i8:
501 Opcode = NVPTX::LDV_i8_v2_avar;
502 break;
503 case MVT::i16:
504 Opcode = NVPTX::LDV_i16_v2_avar;
505 break;
506 case MVT::i32:
507 Opcode = NVPTX::LDV_i32_v2_avar;
508 break;
509 case MVT::i64:
510 Opcode = NVPTX::LDV_i64_v2_avar;
511 break;
512 case MVT::f32:
513 Opcode = NVPTX::LDV_f32_v2_avar;
514 break;
515 case MVT::f64:
516 Opcode = NVPTX::LDV_f64_v2_avar;
517 break;
518 }
519 break;
520 case NVPTXISD::LoadV4:
521 switch (EltVT.getSimpleVT().SimpleTy) {
522 default:
523 return NULL;
524 case MVT::i8:
525 Opcode = NVPTX::LDV_i8_v4_avar;
526 break;
527 case MVT::i16:
528 Opcode = NVPTX::LDV_i16_v4_avar;
529 break;
530 case MVT::i32:
531 Opcode = NVPTX::LDV_i32_v4_avar;
532 break;
533 case MVT::f32:
534 Opcode = NVPTX::LDV_f32_v4_avar;
535 break;
536 }
537 break;
538 }
539
540 SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace),
541 getI32Imm(VecType), getI32Imm(FromType),
542 getI32Imm(FromTypeWidth), Addr, Chain };
543 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
544 } else if (Subtarget.is64Bit()
545 ? SelectADDRsi64(Op1.getNode(), Op1, Base, Offset)
546 : SelectADDRsi(Op1.getNode(), Op1, Base, Offset)) {
547 switch (N->getOpcode()) {
548 default:
549 return NULL;
550 case NVPTXISD::LoadV2:
551 switch (EltVT.getSimpleVT().SimpleTy) {
552 default:
553 return NULL;
554 case MVT::i8:
555 Opcode = NVPTX::LDV_i8_v2_asi;
556 break;
557 case MVT::i16:
558 Opcode = NVPTX::LDV_i16_v2_asi;
559 break;
560 case MVT::i32:
561 Opcode = NVPTX::LDV_i32_v2_asi;
562 break;
563 case MVT::i64:
564 Opcode = NVPTX::LDV_i64_v2_asi;
565 break;
566 case MVT::f32:
567 Opcode = NVPTX::LDV_f32_v2_asi;
568 break;
569 case MVT::f64:
570 Opcode = NVPTX::LDV_f64_v2_asi;
571 break;
572 }
573 break;
574 case NVPTXISD::LoadV4:
575 switch (EltVT.getSimpleVT().SimpleTy) {
576 default:
577 return NULL;
578 case MVT::i8:
579 Opcode = NVPTX::LDV_i8_v4_asi;
580 break;
581 case MVT::i16:
582 Opcode = NVPTX::LDV_i16_v4_asi;
583 break;
584 case MVT::i32:
585 Opcode = NVPTX::LDV_i32_v4_asi;
586 break;
587 case MVT::f32:
588 Opcode = NVPTX::LDV_f32_v4_asi;
589 break;
590 }
591 break;
592 }
593
594 SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace),
595 getI32Imm(VecType), getI32Imm(FromType),
596 getI32Imm(FromTypeWidth), Base, Offset, Chain };
597 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
598 } else if (Subtarget.is64Bit()
599 ? SelectADDRri64(Op1.getNode(), Op1, Base, Offset)
600 : SelectADDRri(Op1.getNode(), Op1, Base, Offset)) {
601 if (Subtarget.is64Bit()) {
602 switch (N->getOpcode()) {
603 default:
604 return NULL;
605 case NVPTXISD::LoadV2:
606 switch (EltVT.getSimpleVT().SimpleTy) {
607 default:
608 return NULL;
609 case MVT::i8:
610 Opcode = NVPTX::LDV_i8_v2_ari_64;
611 break;
612 case MVT::i16:
613 Opcode = NVPTX::LDV_i16_v2_ari_64;
614 break;
615 case MVT::i32:
616 Opcode = NVPTX::LDV_i32_v2_ari_64;
617 break;
618 case MVT::i64:
619 Opcode = NVPTX::LDV_i64_v2_ari_64;
620 break;
621 case MVT::f32:
622 Opcode = NVPTX::LDV_f32_v2_ari_64;
623 break;
624 case MVT::f64:
625 Opcode = NVPTX::LDV_f64_v2_ari_64;
626 break;
627 }
628 break;
629 case NVPTXISD::LoadV4:
630 switch (EltVT.getSimpleVT().SimpleTy) {
631 default:
632 return NULL;
633 case MVT::i8:
634 Opcode = NVPTX::LDV_i8_v4_ari_64;
635 break;
636 case MVT::i16:
637 Opcode = NVPTX::LDV_i16_v4_ari_64;
638 break;
639 case MVT::i32:
640 Opcode = NVPTX::LDV_i32_v4_ari_64;
641 break;
642 case MVT::f32:
643 Opcode = NVPTX::LDV_f32_v4_ari_64;
644 break;
645 }
646 break;
647 }
648 } else {
649 switch (N->getOpcode()) {
650 default:
651 return NULL;
652 case NVPTXISD::LoadV2:
653 switch (EltVT.getSimpleVT().SimpleTy) {
654 default:
655 return NULL;
656 case MVT::i8:
657 Opcode = NVPTX::LDV_i8_v2_ari;
658 break;
659 case MVT::i16:
660 Opcode = NVPTX::LDV_i16_v2_ari;
661 break;
662 case MVT::i32:
663 Opcode = NVPTX::LDV_i32_v2_ari;
664 break;
665 case MVT::i64:
666 Opcode = NVPTX::LDV_i64_v2_ari;
667 break;
668 case MVT::f32:
669 Opcode = NVPTX::LDV_f32_v2_ari;
670 break;
671 case MVT::f64:
672 Opcode = NVPTX::LDV_f64_v2_ari;
673 break;
674 }
675 break;
676 case NVPTXISD::LoadV4:
677 switch (EltVT.getSimpleVT().SimpleTy) {
678 default:
679 return NULL;
680 case MVT::i8:
681 Opcode = NVPTX::LDV_i8_v4_ari;
682 break;
683 case MVT::i16:
684 Opcode = NVPTX::LDV_i16_v4_ari;
685 break;
686 case MVT::i32:
687 Opcode = NVPTX::LDV_i32_v4_ari;
688 break;
689 case MVT::f32:
690 Opcode = NVPTX::LDV_f32_v4_ari;
691 break;
692 }
693 break;
694 }
695 }
696
697 SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace),
698 getI32Imm(VecType), getI32Imm(FromType),
699 getI32Imm(FromTypeWidth), Base, Offset, Chain };
700
701 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
702 } else {
703 if (Subtarget.is64Bit()) {
704 switch (N->getOpcode()) {
705 default:
706 return NULL;
707 case NVPTXISD::LoadV2:
708 switch (EltVT.getSimpleVT().SimpleTy) {
709 default:
710 return NULL;
711 case MVT::i8:
712 Opcode = NVPTX::LDV_i8_v2_areg_64;
713 break;
714 case MVT::i16:
715 Opcode = NVPTX::LDV_i16_v2_areg_64;
716 break;
717 case MVT::i32:
718 Opcode = NVPTX::LDV_i32_v2_areg_64;
719 break;
720 case MVT::i64:
721 Opcode = NVPTX::LDV_i64_v2_areg_64;
722 break;
723 case MVT::f32:
724 Opcode = NVPTX::LDV_f32_v2_areg_64;
725 break;
726 case MVT::f64:
727 Opcode = NVPTX::LDV_f64_v2_areg_64;
728 break;
729 }
730 break;
731 case NVPTXISD::LoadV4:
732 switch (EltVT.getSimpleVT().SimpleTy) {
733 default:
734 return NULL;
735 case MVT::i8:
736 Opcode = NVPTX::LDV_i8_v4_areg_64;
737 break;
738 case MVT::i16:
739 Opcode = NVPTX::LDV_i16_v4_areg_64;
740 break;
741 case MVT::i32:
742 Opcode = NVPTX::LDV_i32_v4_areg_64;
743 break;
744 case MVT::f32:
745 Opcode = NVPTX::LDV_f32_v4_areg_64;
746 break;
747 }
748 break;
749 }
750 } else {
751 switch (N->getOpcode()) {
752 default:
753 return NULL;
754 case NVPTXISD::LoadV2:
755 switch (EltVT.getSimpleVT().SimpleTy) {
756 default:
757 return NULL;
758 case MVT::i8:
759 Opcode = NVPTX::LDV_i8_v2_areg;
760 break;
761 case MVT::i16:
762 Opcode = NVPTX::LDV_i16_v2_areg;
763 break;
764 case MVT::i32:
765 Opcode = NVPTX::LDV_i32_v2_areg;
766 break;
767 case MVT::i64:
768 Opcode = NVPTX::LDV_i64_v2_areg;
769 break;
770 case MVT::f32:
771 Opcode = NVPTX::LDV_f32_v2_areg;
772 break;
773 case MVT::f64:
774 Opcode = NVPTX::LDV_f64_v2_areg;
775 break;
776 }
777 break;
778 case NVPTXISD::LoadV4:
779 switch (EltVT.getSimpleVT().SimpleTy) {
780 default:
781 return NULL;
782 case MVT::i8:
783 Opcode = NVPTX::LDV_i8_v4_areg;
784 break;
785 case MVT::i16:
786 Opcode = NVPTX::LDV_i16_v4_areg;
787 break;
788 case MVT::i32:
789 Opcode = NVPTX::LDV_i32_v4_areg;
790 break;
791 case MVT::f32:
792 Opcode = NVPTX::LDV_f32_v4_areg;
793 break;
794 }
795 break;
796 }
797 }
798
799 SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace),
800 getI32Imm(VecType), getI32Imm(FromType),
801 getI32Imm(FromTypeWidth), Op1, Chain };
802 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
803 }
804
805 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
806 MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
807 cast<MachineSDNode>(LD)->setMemRefs(MemRefs0, MemRefs0 + 1);
808
809 return LD;
810 }
811
812 SDNode *NVPTXDAGToDAGISel::SelectLDGLDUVector(SDNode *N) {
813
814 SDValue Chain = N->getOperand(0);
815 SDValue Op1 = N->getOperand(1);
816 unsigned Opcode;
817 SDLoc DL(N);
818 SDNode *LD;
819 MemSDNode *Mem = cast<MemSDNode>(N);
820 SDValue Base, Offset, Addr;
821
822 EVT EltVT = Mem->getMemoryVT().getVectorElementType();
823
824 if (SelectDirectAddr(Op1, Addr)) {
825 switch (N->getOpcode()) {
826 default:
827 return NULL;
828 case NVPTXISD::LDGV2:
829 switch (EltVT.getSimpleVT().SimpleTy) {
830 default:
831 return NULL;
832 case MVT::i8:
833 Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_avar;
834 break;
835 case MVT::i16:
836 Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_avar;
837 break;
838 case MVT::i32:
839 Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_avar;
840 break;
841 case MVT::i64:
842 Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_avar;
843 break;
844 case MVT::f32:
845 Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_avar;
846 break;
847 case MVT::f64:
848 Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_avar;
849 break;
850 }
851 break;
852 case NVPTXISD::LDUV2:
853 switch (EltVT.getSimpleVT().SimpleTy) {
854 default:
855 return NULL;
856 case MVT::i8:
857 Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_avar;
858 break;
859 case MVT::i16:
860 Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_avar;
861 break;
862 case MVT::i32:
863 Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_avar;
864 break;
865 case MVT::i64:
866 Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_avar;
867 break;
868 case MVT::f32:
869 Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_avar;
870 break;
871 case MVT::f64:
872 Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_avar;
873 break;
874 }
875 break;
876 case NVPTXISD::LDGV4:
877 switch (EltVT.getSimpleVT().SimpleTy) {
878 default:
879 return NULL;
880 case MVT::i8:
881 Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_avar;
882 break;
883 case MVT::i16:
884 Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_avar;
885 break;
886 case MVT::i32:
887 Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_avar;
888 break;
889 case MVT::f32:
890 Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_avar;
891 break;
892 }
893 break;
894 case NVPTXISD::LDUV4:
895 switch (EltVT.getSimpleVT().SimpleTy) {
896 default:
897 return NULL;
898 case MVT::i8:
899 Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_avar;
900 break;
901 case MVT::i16:
902 Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_avar;
903 break;
904 case MVT::i32:
905 Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_avar;
906 break;
907 case MVT::f32:
908 Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_avar;
909 break;
910 }
911 break;
912 }
913
914 SDValue Ops[] = { Addr, Chain };
915 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(),
916 ArrayRef<SDValue>(Ops, 2));
917 } else if (Subtarget.is64Bit()
918 ? SelectADDRri64(Op1.getNode(), Op1, Base, Offset)
919 : SelectADDRri(Op1.getNode(), Op1, Base, Offset)) {
920 if (Subtarget.is64Bit()) {
921 switch (N->getOpcode()) {
922 default:
923 return NULL;
924 case NVPTXISD::LDGV2:
925 switch (EltVT.getSimpleVT().SimpleTy) {
926 default:
927 return NULL;
928 case MVT::i8:
929 Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_ari64;
930 break;
931 case MVT::i16:
932 Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_ari64;
933 break;
934 case MVT::i32:
935 Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_ari64;
936 break;
937 case MVT::i64:
938 Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_ari64;
939 break;
940 case MVT::f32:
941 Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_ari64;
942 break;
943 case MVT::f64:
944 Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_ari64;
945 break;
946 }
947 break;
948 case NVPTXISD::LDUV2:
949 switch (EltVT.getSimpleVT().SimpleTy) {
950 default:
951 return NULL;
952 case MVT::i8:
953 Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_ari64;
954 break;
955 case MVT::i16:
956 Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_ari64;
957 break;
958 case MVT::i32:
959 Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_ari64;
960 break;
961 case MVT::i64:
962 Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_ari64;
963 break;
964 case MVT::f32:
965 Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_ari64;
966 break;
967 case MVT::f64:
968 Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_ari64;
969 break;
970 }
971 break;
972 case NVPTXISD::LDGV4:
973 switch (EltVT.getSimpleVT().SimpleTy) {
974 default:
975 return NULL;
976 case MVT::i8:
977 Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_ari64;
978 break;
979 case MVT::i16:
980 Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_ari64;
981 break;
982 case MVT::i32:
983 Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_ari64;
984 break;
985 case MVT::f32:
986 Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_ari64;
987 break;
988 }
989 break;
990 case NVPTXISD::LDUV4:
991 switch (EltVT.getSimpleVT().SimpleTy) {
992 default:
993 return NULL;
994 case MVT::i8:
995 Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_ari64;
996 break;
997 case MVT::i16:
998 Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_ari64;
999 break;
1000 case MVT::i32:
1001 Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_ari64;
1002 break;
1003 case MVT::f32:
1004 Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_ari64;
1005 break;
1006 }
1007 break;
1008 }
1009 } else {
1010 switch (N->getOpcode()) {
1011 default:
1012 return NULL;
1013 case NVPTXISD::LDGV2:
1014 switch (EltVT.getSimpleVT().SimpleTy) {
1015 default:
1016 return NULL;
1017 case MVT::i8:
1018 Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_ari32;
1019 break;
1020 case MVT::i16:
1021 Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_ari32;
1022 break;
1023 case MVT::i32:
1024 Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_ari32;
1025 break;
1026 case MVT::i64:
1027 Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_ari32;
1028 break;
1029 case MVT::f32:
1030 Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_ari32;
1031 break;
1032 case MVT::f64:
1033 Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_ari32;
1034 break;
1035 }
1036 break;
1037 case NVPTXISD::LDUV2:
1038 switch (EltVT.getSimpleVT().SimpleTy) {
1039 default:
1040 return NULL;
1041 case MVT::i8:
1042 Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_ari32;
1043 break;
1044 case MVT::i16:
1045 Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_ari32;
1046 break;
1047 case MVT::i32:
1048 Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_ari32;
1049 break;
1050 case MVT::i64:
1051 Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_ari32;
1052 break;
1053 case MVT::f32:
1054 Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_ari32;
1055 break;
1056 case MVT::f64:
1057 Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_ari32;
1058 break;
1059 }
1060 break;
1061 case NVPTXISD::LDGV4:
1062 switch (EltVT.getSimpleVT().SimpleTy) {
1063 default:
1064 return NULL;
1065 case MVT::i8:
1066 Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_ari32;
1067 break;
1068 case MVT::i16:
1069 Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_ari32;
1070 break;
1071 case MVT::i32:
1072 Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_ari32;
1073 break;
1074 case MVT::f32:
1075 Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_ari32;
1076 break;
1077 }
1078 break;
1079 case NVPTXISD::LDUV4:
1080 switch (EltVT.getSimpleVT().SimpleTy) {
1081 default:
1082 return NULL;
1083 case MVT::i8:
1084 Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_ari32;
1085 break;
1086 case MVT::i16:
1087 Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_ari32;
1088 break;
1089 case MVT::i32:
1090 Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_ari32;
1091 break;
1092 case MVT::f32:
1093 Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_ari32;
1094 break;
1095 }
1096 break;
1097 }
1098 }
1099
1100 SDValue Ops[] = { Base, Offset, Chain };
1101
1102 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(),
1103 ArrayRef<SDValue>(Ops, 3));
1104 } else {
1105 if (Subtarget.is64Bit()) {
1106 switch (N->getOpcode()) {
1107 default:
1108 return NULL;
1109 case NVPTXISD::LDGV2:
1110 switch (EltVT.getSimpleVT().SimpleTy) {
1111 default:
1112 return NULL;
1113 case MVT::i8:
1114 Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_areg64;
1115 break;
1116 case MVT::i16:
1117 Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_areg64;
1118 break;
1119 case MVT::i32:
1120 Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_areg64;
1121 break;
1122 case MVT::i64:
1123 Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_areg64;
1124 break;
1125 case MVT::f32:
1126 Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_areg64;
1127 break;
1128 case MVT::f64:
1129 Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_areg64;
1130 break;
1131 }
1132 break;
1133 case NVPTXISD::LDUV2:
1134 switch (EltVT.getSimpleVT().SimpleTy) {
1135 default:
1136 return NULL;
1137 case MVT::i8:
1138 Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_areg64;
1139 break;
1140 case MVT::i16:
1141 Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_areg64;
1142 break;
1143 case MVT::i32:
1144 Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_areg64;
1145 break;
1146 case MVT::i64:
1147 Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_areg64;
1148 break;
1149 case MVT::f32:
1150 Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_areg64;
1151 break;
1152 case MVT::f64:
1153 Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_areg64;
1154 break;
1155 }
1156 break;
1157 case NVPTXISD::LDGV4:
1158 switch (EltVT.getSimpleVT().SimpleTy) {
1159 default:
1160 return NULL;
1161 case MVT::i8:
1162 Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_areg64;
1163 break;
1164 case MVT::i16:
1165 Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_areg64;
1166 break;
1167 case MVT::i32:
1168 Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_areg64;
1169 break;
1170 case MVT::f32:
1171 Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_areg64;
1172 break;
1173 }
1174 break;
1175 case NVPTXISD::LDUV4:
1176 switch (EltVT.getSimpleVT().SimpleTy) {
1177 default:
1178 return NULL;
1179 case MVT::i8:
1180 Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_areg64;
1181 break;
1182 case MVT::i16:
1183 Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_areg64;
1184 break;
1185 case MVT::i32:
1186 Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_areg64;
1187 break;
1188 case MVT::f32:
1189 Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_areg64;
1190 break;
1191 }
1192 break;
1193 }
1194 } else {
1195 switch (N->getOpcode()) {
1196 default:
1197 return NULL;
1198 case NVPTXISD::LDGV2:
1199 switch (EltVT.getSimpleVT().SimpleTy) {
1200 default:
1201 return NULL;
1202 case MVT::i8:
1203 Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_areg32;
1204 break;
1205 case MVT::i16:
1206 Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_areg32;
1207 break;
1208 case MVT::i32:
1209 Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_areg32;
1210 break;
1211 case MVT::i64:
1212 Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_areg32;
1213 break;
1214 case MVT::f32:
1215 Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_areg32;
1216 break;
1217 case MVT::f64:
1218 Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_areg32;
1219 break;
1220 }
1221 break;
1222 case NVPTXISD::LDUV2:
1223 switch (EltVT.getSimpleVT().SimpleTy) {
1224 default:
1225 return NULL;
1226 case MVT::i8:
1227 Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_areg32;
1228 break;
1229 case MVT::i16:
1230 Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_areg32;
1231 break;
1232 case MVT::i32:
1233 Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_areg32;
1234 break;
1235 case MVT::i64:
1236 Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_areg32;
1237 break;
1238 case MVT::f32:
1239 Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_areg32;
1240 break;
1241 case MVT::f64:
1242 Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_areg32;
1243 break;
1244 }
1245 break;
1246 case NVPTXISD::LDGV4:
1247 switch (EltVT.getSimpleVT().SimpleTy) {
1248 default:
1249 return NULL;
1250 case MVT::i8:
1251 Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_areg32;
1252 break;
1253 case MVT::i16:
1254 Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_areg32;
1255 break;
1256 case MVT::i32:
1257 Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_areg32;
1258 break;
1259 case MVT::f32:
1260 Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_areg32;
1261 break;
1262 }
1263 break;
1264 case NVPTXISD::LDUV4:
1265 switch (EltVT.getSimpleVT().SimpleTy) {
1266 default:
1267 return NULL;
1268 case MVT::i8:
1269 Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_areg32;
1270 break;
1271 case MVT::i16:
1272 Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_areg32;
1273 break;
1274 case MVT::i32:
1275 Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_areg32;
1276 break;
1277 case MVT::f32:
1278 Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_areg32;
1279 break;
1280 }
1281 break;
1282 }
1283 }
1284
1285 SDValue Ops[] = { Op1, Chain };
1286 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(),
1287 ArrayRef<SDValue>(Ops, 2));
1288 }
1289
1290 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
1291 MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
1292 cast<MachineSDNode>(LD)->setMemRefs(MemRefs0, MemRefs0 + 1);
1293
1294 return LD;
1295 }
1296
1297 SDNode *NVPTXDAGToDAGISel::SelectStore(SDNode *N) {
1298 SDLoc dl(N);
1299 StoreSDNode *ST = cast<StoreSDNode>(N);
1300 EVT StoreVT = ST->getMemoryVT();
1301 SDNode *NVPTXST = NULL;
1302
1303 // do not support pre/post inc/dec
1304 if (ST->isIndexed())
1305 return NULL;
1306
1307 if (!StoreVT.isSimple())
1308 return NULL;
1309
1310 // Address Space Setting
1311 unsigned int codeAddrSpace = getCodeAddrSpace(ST, Subtarget);
1312
1313 // Volatile Setting
1314 // - .volatile is only availalble for .global and .shared
1315 bool isVolatile = ST->isVolatile();
1316 if (codeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL &&
1317 codeAddrSpace != NVPTX::PTXLdStInstCode::SHARED &&
1318 codeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC)
1319 isVolatile = false;
1320
1321 // Vector Setting
1322 MVT SimpleVT = StoreVT.getSimpleVT();
1323 unsigned vecType = NVPTX::PTXLdStInstCode::Scalar;
1324 if (SimpleVT.isVector()) {
1325 unsigned num = SimpleVT.getVectorNumElements();
1326 if (num == 2)
1327 vecType = NVPTX::PTXLdStInstCode::V2;
1328 else if (num == 4)
1329 vecType = NVPTX::PTXLdStInstCode::V4;
1330 else
1331 return NULL;
1332 }
1333
1334 // Type Setting: toType + toTypeWidth
1335 // - for integer type, always use 'u'
1336 //
1337 MVT ScalarVT = SimpleVT.getScalarType();
1338 unsigned toTypeWidth = ScalarVT.getSizeInBits();
1339 unsigned int toType;
1340 if (ScalarVT.isFloatingPoint())
1341 toType = NVPTX::PTXLdStInstCode::Float;
1342 else
1343 toType = NVPTX::PTXLdStInstCode::Unsigned;
1344
1345 // Create the machine instruction DAG
1346 SDValue Chain = N->getOperand(0);
1347 SDValue N1 = N->getOperand(1);
1348 SDValue N2 = N->getOperand(2);
1349 SDValue Addr;
1350 SDValue Offset, Base;
1351 unsigned Opcode;
1352 MVT::SimpleValueType SourceVT = N1.getNode()->getSimpleValueType(0).SimpleTy;
1353
1354 if (SelectDirectAddr(N2, Addr)) {
1355 switch (SourceVT) {
1356 case MVT::i8:
1357 Opcode = NVPTX::ST_i8_avar;
1358 break;
1359 case MVT::i16:
1360 Opcode = NVPTX::ST_i16_avar;
1361 break;
1362 case MVT::i32:
1363 Opcode = NVPTX::ST_i32_avar;
1364 break;
1365 case MVT::i64:
1366 Opcode = NVPTX::ST_i64_avar;
1367 break;
1368 case MVT::f32:
1369 Opcode = NVPTX::ST_f32_avar;
1370 break;
1371 case MVT::f64:
1372 Opcode = NVPTX::ST_f64_avar;
1373 break;
1374 default:
1375 return NULL;
1376 }
1377 SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
1378 getI32Imm(vecType), getI32Imm(toType),
1379 getI32Imm(toTypeWidth), Addr, Chain };
1380 NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops);
1381 } else if (Subtarget.is64Bit()
1382 ? SelectADDRsi64(N2.getNode(), N2, Base, Offset)
1383 : SelectADDRsi(N2.getNode(), N2, Base, Offset)) {
1384 switch (SourceVT) {
1385 case MVT::i8:
1386 Opcode = NVPTX::ST_i8_asi;
1387 break;
1388 case MVT::i16:
1389 Opcode = NVPTX::ST_i16_asi;
1390 break;
1391 case MVT::i32:
1392 Opcode = NVPTX::ST_i32_asi;
1393 break;
1394 case MVT::i64:
1395 Opcode = NVPTX::ST_i64_asi;
1396 break;
1397 case MVT::f32:
1398 Opcode = NVPTX::ST_f32_asi;
1399 break;
1400 case MVT::f64:
1401 Opcode = NVPTX::ST_f64_asi;
1402 break;
1403 default:
1404 return NULL;
1405 }
1406 SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
1407 getI32Imm(vecType), getI32Imm(toType),
1408 getI32Imm(toTypeWidth), Base, Offset, Chain };
1409 NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops);
1410 } else if (Subtarget.is64Bit()
1411 ? SelectADDRri64(N2.getNode(), N2, Base, Offset)
1412 : SelectADDRri(N2.getNode(), N2, Base, Offset)) {
1413 if (Subtarget.is64Bit()) {
1414 switch (SourceVT) {
1415 case MVT::i8:
1416 Opcode = NVPTX::ST_i8_ari_64;
1417 break;
1418 case MVT::i16:
1419 Opcode = NVPTX::ST_i16_ari_64;
1420 break;
1421 case MVT::i32:
1422 Opcode = NVPTX::ST_i32_ari_64;
1423 break;
1424 case MVT::i64:
1425 Opcode = NVPTX::ST_i64_ari_64;
1426 break;
1427 case MVT::f32:
1428 Opcode = NVPTX::ST_f32_ari_64;
1429 break;
1430 case MVT::f64:
1431 Opcode = NVPTX::ST_f64_ari_64;
1432 break;
1433 default:
1434 return NULL;
1435 }
1436 } else {
1437 switch (SourceVT) {
1438 case MVT::i8:
1439 Opcode = NVPTX::ST_i8_ari;
1440 break;
1441 case MVT::i16:
1442 Opcode = NVPTX::ST_i16_ari;
1443 break;
1444 case MVT::i32:
1445 Opcode = NVPTX::ST_i32_ari;
1446 break;
1447 case MVT::i64:
1448 Opcode = NVPTX::ST_i64_ari;
1449 break;
1450 case MVT::f32:
1451 Opcode = NVPTX::ST_f32_ari;
1452 break;
1453 case MVT::f64:
1454 Opcode = NVPTX::ST_f64_ari;
1455 break;
1456 default:
1457 return NULL;
1458 }
1459 }
1460 SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
1461 getI32Imm(vecType), getI32Imm(toType),
1462 getI32Imm(toTypeWidth), Base, Offset, Chain };
1463 NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops);
1464 } else {
1465 if (Subtarget.is64Bit()) {
1466 switch (SourceVT) {
1467 case MVT::i8:
1468 Opcode = NVPTX::ST_i8_areg_64;
1469 break;
1470 case MVT::i16:
1471 Opcode = NVPTX::ST_i16_areg_64;
1472 break;
1473 case MVT::i32:
1474 Opcode = NVPTX::ST_i32_areg_64;
1475 break;
1476 case MVT::i64:
1477 Opcode = NVPTX::ST_i64_areg_64;
1478 break;
1479 case MVT::f32:
1480 Opcode = NVPTX::ST_f32_areg_64;
1481 break;
1482 case MVT::f64:
1483 Opcode = NVPTX::ST_f64_areg_64;
1484 break;
1485 default:
1486 return NULL;
1487 }
1488 } else {
1489 switch (SourceVT) {
1490 case MVT::i8:
1491 Opcode = NVPTX::ST_i8_areg;
1492 break;
1493 case MVT::i16:
1494 Opcode = NVPTX::ST_i16_areg;
1495 break;
1496 case MVT::i32:
1497 Opcode = NVPTX::ST_i32_areg;
1498 break;
1499 case MVT::i64:
1500 Opcode = NVPTX::ST_i64_areg;
1501 break;
1502 case MVT::f32:
1503 Opcode = NVPTX::ST_f32_areg;
1504 break;
1505 case MVT::f64:
1506 Opcode = NVPTX::ST_f64_areg;
1507 break;
1508 default:
1509 return NULL;
1510 }
1511 }
1512 SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
1513 getI32Imm(vecType), getI32Imm(toType),
1514 getI32Imm(toTypeWidth), N2, Chain };
1515 NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops);
1516 }
1517
1518 if (NVPTXST != NULL) {
1519 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
1520 MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
1521 cast<MachineSDNode>(NVPTXST)->setMemRefs(MemRefs0, MemRefs0 + 1);
1522 }
1523
1524 return NVPTXST;
1525 }
1526
1527 SDNode *NVPTXDAGToDAGISel::SelectStoreVector(SDNode *N) {
1528 SDValue Chain = N->getOperand(0);
1529 SDValue Op1 = N->getOperand(1);
1530 SDValue Addr, Offset, Base;
1531 unsigned Opcode;
1532 SDLoc DL(N);
1533 SDNode *ST;
1534 EVT EltVT = Op1.getValueType();
1535 MemSDNode *MemSD = cast<MemSDNode>(N);
1536 EVT StoreVT = MemSD->getMemoryVT();
1537
1538 // Address Space Setting
1539 unsigned CodeAddrSpace = getCodeAddrSpace(MemSD, Subtarget);
1540
1541 if (CodeAddrSpace == NVPTX::PTXLdStInstCode::CONSTANT) {
1542 report_fatal_error("Cannot store to pointer that points to constant "
1543 "memory space");
1544 }
1545
1546 // Volatile Setting
1547 // - .volatile is only availalble for .global and .shared
1548 bool IsVolatile = MemSD->isVolatile();
1549 if (CodeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL &&
1550 CodeAddrSpace != NVPTX::PTXLdStInstCode::SHARED &&
1551 CodeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC)
1552 IsVolatile = false;
1553
1554 // Type Setting: toType + toTypeWidth
1555 // - for integer type, always use 'u'
1556 assert(StoreVT.isSimple() && "Store value is not simple");
1557 MVT ScalarVT = StoreVT.getSimpleVT().getScalarType();
1558 unsigned ToTypeWidth = ScalarVT.getSizeInBits();
1559 unsigned ToType;
1560 if (ScalarVT.isFloatingPoint())
1561 ToType = NVPTX::PTXLdStInstCode::Float;
1562 else
1563 ToType = NVPTX::PTXLdStInstCode::Unsigned;
1564
1565 SmallVector<SDValue, 12> StOps;
1566 SDValue N2;
1567 unsigned VecType;
1568
1569 switch (N->getOpcode()) {
1570 case NVPTXISD::StoreV2:
1571 VecType = NVPTX::PTXLdStInstCode::V2;
1572 StOps.push_back(N->getOperand(1));
1573 StOps.push_back(N->getOperand(2));
1574 N2 = N->getOperand(3);
1575 break;
1576 case NVPTXISD::StoreV4:
1577 VecType = NVPTX::PTXLdStInstCode::V4;
1578 StOps.push_back(N->getOperand(1));
1579 StOps.push_back(N->getOperand(2));
1580 StOps.push_back(N->getOperand(3));
1581 StOps.push_back(N->getOperand(4));
1582 N2 = N->getOperand(5);
1583 break;
1584 default:
1585 return NULL;
1586 }
1587
1588 StOps.push_back(getI32Imm(IsVolatile));
1589 StOps.push_back(getI32Imm(CodeAddrSpace));
1590 StOps.push_back(getI32Imm(VecType));
1591 StOps.push_back(getI32Imm(ToType));
1592 StOps.push_back(getI32Imm(ToTypeWidth));
1593
1594 if (SelectDirectAddr(N2, Addr)) {
1595 switch (N->getOpcode()) {
1596 default:
1597 return NULL;
1598 case NVPTXISD::StoreV2:
1599 switch (EltVT.getSimpleVT().SimpleTy) {
1600 default:
1601 return NULL;
1602 case MVT::i8:
1603 Opcode = NVPTX::STV_i8_v2_avar;
1604 break;
1605 case MVT::i16:
1606 Opcode = NVPTX::STV_i16_v2_avar;
1607 break;
1608 case MVT::i32:
1609 Opcode = NVPTX::STV_i32_v2_avar;
1610 break;
1611 case MVT::i64:
1612 Opcode = NVPTX::STV_i64_v2_avar;
1613 break;
1614 case MVT::f32:
1615 Opcode = NVPTX::STV_f32_v2_avar;
1616 break;
1617 case MVT::f64:
1618 Opcode = NVPTX::STV_f64_v2_avar;
1619 break;
1620 }
1621 break;
1622 case NVPTXISD::StoreV4:
1623 switch (EltVT.getSimpleVT().SimpleTy) {
1624 default:
1625 return NULL;
1626 case MVT::i8:
1627 Opcode = NVPTX::STV_i8_v4_avar;
1628 break;
1629 case MVT::i16:
1630 Opcode = NVPTX::STV_i16_v4_avar;
1631 break;
1632 case MVT::i32:
1633 Opcode = NVPTX::STV_i32_v4_avar;
1634 break;
1635 case MVT::f32:
1636 Opcode = NVPTX::STV_f32_v4_avar;
1637 break;
1638 }
1639 break;
1640 }
1641 StOps.push_back(Addr);
1642 } else if (Subtarget.is64Bit()
1643 ? SelectADDRsi64(N2.getNode(), N2, Base, Offset)
1644 : SelectADDRsi(N2.getNode(), N2, Base, Offset)) {
1645 switch (N->getOpcode()) {
1646 default:
1647 return NULL;
1648 case NVPTXISD::StoreV2:
1649 switch (EltVT.getSimpleVT().SimpleTy) {
1650 default:
1651 return NULL;
1652 case MVT::i8:
1653 Opcode = NVPTX::STV_i8_v2_asi;
1654 break;
1655 case MVT::i16:
1656 Opcode = NVPTX::STV_i16_v2_asi;
1657 break;
1658 case MVT::i32:
1659 Opcode = NVPTX::STV_i32_v2_asi;
1660 break;
1661 case MVT::i64:
1662 Opcode = NVPTX::STV_i64_v2_asi;
1663 break;
1664 case MVT::f32:
1665 Opcode = NVPTX::STV_f32_v2_asi;
1666 break;
1667 case MVT::f64:
1668 Opcode = NVPTX::STV_f64_v2_asi;
1669 break;
1670 }
1671 break;
1672 case NVPTXISD::StoreV4:
1673 switch (EltVT.getSimpleVT().SimpleTy) {
1674 default:
1675 return NULL;
1676 case MVT::i8:
1677 Opcode = NVPTX::STV_i8_v4_asi;
1678 break;
1679 case MVT::i16:
1680 Opcode = NVPTX::STV_i16_v4_asi;
1681 break;
1682 case MVT::i32:
1683 Opcode = NVPTX::STV_i32_v4_asi;
1684 break;
1685 case MVT::f32:
1686 Opcode = NVPTX::STV_f32_v4_asi;
1687 break;
1688 }
1689 break;
1690 }
1691 StOps.push_back(Base);
1692 StOps.push_back(Offset);
1693 } else if (Subtarget.is64Bit()
1694 ? SelectADDRri64(N2.getNode(), N2, Base, Offset)
1695 : SelectADDRri(N2.getNode(), N2, Base, Offset)) {
1696 if (Subtarget.is64Bit()) {
1697 switch (N->getOpcode()) {
1698 default:
1699 return NULL;
1700 case NVPTXISD::StoreV2:
1701 switch (EltVT.getSimpleVT().SimpleTy) {
1702 default:
1703 return NULL;
1704 case MVT::i8:
1705 Opcode = NVPTX::STV_i8_v2_ari_64;
1706 break;
1707 case MVT::i16:
1708 Opcode = NVPTX::STV_i16_v2_ari_64;
1709 break;
1710 case MVT::i32:
1711 Opcode = NVPTX::STV_i32_v2_ari_64;
1712 break;
1713 case MVT::i64:
1714 Opcode = NVPTX::STV_i64_v2_ari_64;
1715 break;
1716 case MVT::f32:
1717 Opcode = NVPTX::STV_f32_v2_ari_64;
1718 break;
1719 case MVT::f64:
1720 Opcode = NVPTX::STV_f64_v2_ari_64;
1721 break;
1722 }
1723 break;
1724 case NVPTXISD::StoreV4:
1725 switch (EltVT.getSimpleVT().SimpleTy) {
1726 default:
1727 return NULL;
1728 case MVT::i8:
1729 Opcode = NVPTX::STV_i8_v4_ari_64;
1730 break;
1731 case MVT::i16:
1732 Opcode = NVPTX::STV_i16_v4_ari_64;
1733 break;
1734 case MVT::i32:
1735 Opcode = NVPTX::STV_i32_v4_ari_64;
1736 break;
1737 case MVT::f32:
1738 Opcode = NVPTX::STV_f32_v4_ari_64;
1739 break;
1740 }
1741 break;
1742 }
1743 } else {
1744 switch (N->getOpcode()) {
1745 default:
1746 return NULL;
1747 case NVPTXISD::StoreV2:
1748 switch (EltVT.getSimpleVT().SimpleTy) {
1749 default:
1750 return NULL;
1751 case MVT::i8:
1752 Opcode = NVPTX::STV_i8_v2_ari;
1753 break;
1754 case MVT::i16:
1755 Opcode = NVPTX::STV_i16_v2_ari;
1756 break;
1757 case MVT::i32:
1758 Opcode = NVPTX::STV_i32_v2_ari;
1759 break;
1760 case MVT::i64:
1761 Opcode = NVPTX::STV_i64_v2_ari;
1762 break;
1763 case MVT::f32:
1764 Opcode = NVPTX::STV_f32_v2_ari;
1765 break;
1766 case MVT::f64:
1767 Opcode = NVPTX::STV_f64_v2_ari;
1768 break;
1769 }
1770 break;
1771 case NVPTXISD::StoreV4:
1772 switch (EltVT.getSimpleVT().SimpleTy) {
1773 default:
1774 return NULL;
1775 case MVT::i8:
1776 Opcode = NVPTX::STV_i8_v4_ari;
1777 break;
1778 case MVT::i16:
1779 Opcode = NVPTX::STV_i16_v4_ari;
1780 break;
1781 case MVT::i32:
1782 Opcode = NVPTX::STV_i32_v4_ari;
1783 break;
1784 case MVT::f32:
1785 Opcode = NVPTX::STV_f32_v4_ari;
1786 break;
1787 }
1788 break;
1789 }
1790 }
1791 StOps.push_back(Base);
1792 StOps.push_back(Offset);
1793 } else {
1794 if (Subtarget.is64Bit()) {
1795 switch (N->getOpcode()) {
1796 default:
1797 return NULL;
1798 case NVPTXISD::StoreV2:
1799 switch (EltVT.getSimpleVT().SimpleTy) {
1800 default:
1801 return NULL;
1802 case MVT::i8:
1803 Opcode = NVPTX::STV_i8_v2_areg_64;
1804 break;
1805 case MVT::i16:
1806 Opcode = NVPTX::STV_i16_v2_areg_64;
1807 break;
1808 case MVT::i32:
1809 Opcode = NVPTX::STV_i32_v2_areg_64;
1810 break;
1811 case MVT::i64:
1812 Opcode = NVPTX::STV_i64_v2_areg_64;
1813 break;
1814 case MVT::f32:
1815 Opcode = NVPTX::STV_f32_v2_areg_64;
1816 break;
1817 case MVT::f64:
1818 Opcode = NVPTX::STV_f64_v2_areg_64;
1819 break;
1820 }
1821 break;
1822 case NVPTXISD::StoreV4:
1823 switch (EltVT.getSimpleVT().SimpleTy) {
1824 default:
1825 return NULL;
1826 case MVT::i8:
1827 Opcode = NVPTX::STV_i8_v4_areg_64;
1828 break;
1829 case MVT::i16:
1830 Opcode = NVPTX::STV_i16_v4_areg_64;
1831 break;
1832 case MVT::i32:
1833 Opcode = NVPTX::STV_i32_v4_areg_64;
1834 break;
1835 case MVT::f32:
1836 Opcode = NVPTX::STV_f32_v4_areg_64;
1837 break;
1838 }
1839 break;
1840 }
1841 } else {
1842 switch (N->getOpcode()) {
1843 default:
1844 return NULL;
1845 case NVPTXISD::StoreV2:
1846 switch (EltVT.getSimpleVT().SimpleTy) {
1847 default:
1848 return NULL;
1849 case MVT::i8:
1850 Opcode = NVPTX::STV_i8_v2_areg;
1851 break;
1852 case MVT::i16:
1853 Opcode = NVPTX::STV_i16_v2_areg;
1854 break;
1855 case MVT::i32:
1856 Opcode = NVPTX::STV_i32_v2_areg;
1857 break;
1858 case MVT::i64:
1859 Opcode = NVPTX::STV_i64_v2_areg;
1860 break;
1861 case MVT::f32:
1862 Opcode = NVPTX::STV_f32_v2_areg;
1863 break;
1864 case MVT::f64:
1865 Opcode = NVPTX::STV_f64_v2_areg;
1866 break;
1867 }
1868 break;
1869 case NVPTXISD::StoreV4:
1870 switch (EltVT.getSimpleVT().SimpleTy) {
1871 default:
1872 return NULL;
1873 case MVT::i8:
1874 Opcode = NVPTX::STV_i8_v4_areg;
1875 break;
1876 case MVT::i16:
1877 Opcode = NVPTX::STV_i16_v4_areg;
1878 break;
1879 case MVT::i32:
1880 Opcode = NVPTX::STV_i32_v4_areg;
1881 break;
1882 case MVT::f32:
1883 Opcode = NVPTX::STV_f32_v4_areg;
1884 break;
1885 }
1886 break;
1887 }
1888 }
1889 StOps.push_back(N2);
1890 }
1891
1892 StOps.push_back(Chain);
1893
1894 ST = CurDAG->getMachineNode(Opcode, DL, MVT::Other, StOps);
1895
1896 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
1897 MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
1898 cast<MachineSDNode>(ST)->setMemRefs(MemRefs0, MemRefs0 + 1);
1899
1900 return ST;
1901 }
1902
1903 SDNode *NVPTXDAGToDAGISel::SelectLoadParam(SDNode *Node) {
1904 SDValue Chain = Node->getOperand(0);
1905 SDValue Offset = Node->getOperand(2);
1906 SDValue Flag = Node->getOperand(3);
1907 SDLoc DL(Node);
1908 MemSDNode *Mem = cast<MemSDNode>(Node);
1909
1910 unsigned VecSize;
1911 switch (Node->getOpcode()) {
1912 default:
1913 return NULL;
1914 case NVPTXISD::LoadParam:
1915 VecSize = 1;
1916 break;
1917 case NVPTXISD::LoadParamV2:
1918 VecSize = 2;
1919 break;
1920 case NVPTXISD::LoadParamV4:
1921 VecSize = 4;
1922 break;
1923 }
1924
1925 EVT EltVT = Node->getValueType(0);
1926 EVT MemVT = Mem->getMemoryVT();
1927
1928 unsigned Opc = 0;
1929
1930 switch (VecSize) {
1931 default:
1932 return NULL;
1933 case 1:
1934 switch (MemVT.getSimpleVT().SimpleTy) {
1935 default:
1936 return NULL;
1937 case MVT::i1:
1938 Opc = NVPTX::LoadParamMemI8;
1939 break;
1940 case MVT::i8:
1941 Opc = NVPTX::LoadParamMemI8;
1942 break;
1943 case MVT::i16:
1944 Opc = NVPTX::LoadParamMemI16;
1945 break;
1946 case MVT::i32:
1947 Opc = NVPTX::LoadParamMemI32;
1948 break;
1949 case MVT::i64:
1950 Opc = NVPTX::LoadParamMemI64;
1951 break;
1952 case MVT::f32:
1953 Opc = NVPTX::LoadParamMemF32;
1954 break;
1955 case MVT::f64:
1956 Opc = NVPTX::LoadParamMemF64;
1957 break;
1958 }
1959 break;
1960 case 2:
1961 switch (MemVT.getSimpleVT().SimpleTy) {
1962 default:
1963 return NULL;
1964 case MVT::i1:
1965 Opc = NVPTX::LoadParamMemV2I8;
1966 break;
1967 case MVT::i8:
1968 Opc = NVPTX::LoadParamMemV2I8;
1969 break;
1970 case MVT::i16:
1971 Opc = NVPTX::LoadParamMemV2I16;
1972 break;
1973 case MVT::i32:
1974 Opc = NVPTX::LoadParamMemV2I32;
1975 break;
1976 case MVT::i64:
1977 Opc = NVPTX::LoadParamMemV2I64;
1978 break;
1979 case MVT::f32:
1980 Opc = NVPTX::LoadParamMemV2F32;
1981 break;
1982 case MVT::f64:
1983 Opc = NVPTX::LoadParamMemV2F64;
1984 break;
1985 }
1986 break;
1987 case 4:
1988 switch (MemVT.getSimpleVT().SimpleTy) {
1989 default:
1990 return NULL;
1991 case MVT::i1:
1992 Opc = NVPTX::LoadParamMemV4I8;
1993 break;
1994 case MVT::i8:
1995 Opc = NVPTX::LoadParamMemV4I8;
1996 break;
1997 case MVT::i16:
1998 Opc = NVPTX::LoadParamMemV4I16;
1999 break;
2000 case MVT::i32:
2001 Opc = NVPTX::LoadParamMemV4I32;
2002 break;
2003 case MVT::f32:
2004 Opc = NVPTX::LoadParamMemV4F32;
2005 break;
2006 }
2007 break;
2008 }
2009
2010 SDVTList VTs;
2011 if (VecSize == 1) {
2012 VTs = CurDAG->getVTList(EltVT, MVT::Other, MVT::Glue);
2013 } else if (VecSize == 2) {
2014 VTs = CurDAG->getVTList(EltVT, EltVT, MVT::Other, MVT::Glue);
2015 } else {
2016 EVT EVTs[] = { EltVT, EltVT, EltVT, EltVT, MVT::Other, MVT::Glue };
2017 VTs = CurDAG->getVTList(&EVTs[0], 5);
2018 }
2019
2020 unsigned OffsetVal = cast<ConstantSDNode>(Offset)->getZExtValue();
2021
2022 SmallVector<SDValue, 2> Ops;
2023 Ops.push_back(CurDAG->getTargetConstant(OffsetVal, MVT::i32));
2024 Ops.push_back(Chain);
2025 Ops.push_back(Flag);
2026
2027 SDNode *Ret =
2028 CurDAG->getMachineNode(Opc, DL, VTs, Ops);
2029 return Ret;
2030 }
2031
2032 SDNode *NVPTXDAGToDAGISel::SelectStoreRetval(SDNode *N) {
2033 SDLoc DL(N);
2034 SDValue Chain = N->getOperand(0);
2035 SDValue Offset = N->getOperand(1);
2036 unsigned OffsetVal = cast<ConstantSDNode>(Offset)->getZExtValue();
2037 MemSDNode *Mem = cast<MemSDNode>(N);
2038
2039 // How many elements do we have?
2040 unsigned NumElts = 1;
2041 switch (N->getOpcode()) {
2042 default:
2043 return NULL;
2044 case NVPTXISD::StoreRetval:
2045 NumElts = 1;
2046 break;
2047 case NVPTXISD::StoreRetvalV2:
2048 NumElts = 2;
2049 break;
2050 case NVPTXISD::StoreRetvalV4:
2051 NumElts = 4;
2052 break;
2053 }
2054
2055 // Build vector of operands
2056 SmallVector<SDValue, 6> Ops;
2057 for (unsigned i = 0; i < NumElts; ++i)
2058 Ops.push_back(N->getOperand(i + 2));
2059 Ops.push_back(CurDAG->getTargetConstant(OffsetVal, MVT::i32));
2060 Ops.push_back(Chain);
2061
2062 // Determine target opcode
2063 // If we have an i1, use an 8-bit store. The lowering code in
2064 // NVPTXISelLowering will have already emitted an upcast.
2065 unsigned Opcode = 0;
2066 switch (NumElts) {
2067 default:
2068 return NULL;
2069 case 1:
2070 switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
2071 default:
2072 return NULL;
2073 case MVT::i1:
2074 Opcode = NVPTX::StoreRetvalI8;
2075 break;
2076 case MVT::i8:
2077 Opcode = NVPTX::StoreRetvalI8;
2078 break;
2079 case MVT::i16:
2080 Opcode = NVPTX::StoreRetvalI16;
2081 break;
2082 case MVT::i32:
2083 Opcode = NVPTX::StoreRetvalI32;
2084 break;
2085 case MVT::i64:
2086 Opcode = NVPTX::StoreRetvalI64;
2087 break;
2088 case MVT::f32:
2089 Opcode = NVPTX::StoreRetvalF32;
2090 break;
2091 case MVT::f64:
2092 Opcode = NVPTX::StoreRetvalF64;
2093 break;
2094 }
2095 break;
2096 case 2:
2097 switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
2098 default:
2099 return NULL;
2100 case MVT::i1:
2101 Opcode = NVPTX::StoreRetvalV2I8;
2102 break;
2103 case MVT::i8:
2104 Opcode = NVPTX::StoreRetvalV2I8;
2105 break;
2106 case MVT::i16:
2107 Opcode = NVPTX::StoreRetvalV2I16;
2108 break;
2109 case MVT::i32:
2110 Opcode = NVPTX::StoreRetvalV2I32;
2111 break;
2112 case MVT::i64:
2113 Opcode = NVPTX::StoreRetvalV2I64;
2114 break;
2115 case MVT::f32:
2116 Opcode = NVPTX::StoreRetvalV2F32;
2117 break;
2118 case MVT::f64:
2119 Opcode = NVPTX::StoreRetvalV2F64;
2120 break;
2121 }
2122 break;
2123 case 4:
2124 switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
2125 default:
2126 return NULL;
2127 case MVT::i1:
2128 Opcode = NVPTX::StoreRetvalV4I8;
2129 break;
2130 case MVT::i8:
2131 Opcode = NVPTX::StoreRetvalV4I8;
2132 break;
2133 case MVT::i16:
2134 Opcode = NVPTX::StoreRetvalV4I16;
2135 break;
2136 case MVT::i32:
2137 Opcode = NVPTX::StoreRetvalV4I32;
2138 break;
2139 case MVT::f32:
2140 Opcode = NVPTX::StoreRetvalV4F32;
2141 break;
2142 }
2143 break;
2144 }
2145
2146 SDNode *Ret =
2147 CurDAG->getMachineNode(Opcode, DL, MVT::Other, Ops);
2148 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
2149 MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
2150 cast<MachineSDNode>(Ret)->setMemRefs(MemRefs0, MemRefs0 + 1);
2151
2152 return Ret;
2153 }
2154
2155 SDNode *NVPTXDAGToDAGISel::SelectStoreParam(SDNode *N) {
2156 SDLoc DL(N);
2157 SDValue Chain = N->getOperand(0);
2158 SDValue Param = N->getOperand(1);
2159 unsigned ParamVal = cast<ConstantSDNode>(Param)->getZExtValue();
2160 SDValue Offset = N->getOperand(2);
2161 unsigned OffsetVal = cast<ConstantSDNode>(Offset)->getZExtValue();
2162 MemSDNode *Mem = cast<MemSDNode>(N);
2163 SDValue Flag = N->getOperand(N->getNumOperands() - 1);
2164
2165 // How many elements do we have?
2166 unsigned NumElts = 1;
2167 switch (N->getOpcode()) {
2168 default:
2169 return NULL;
2170 case NVPTXISD::StoreParamU32:
2171 case NVPTXISD::StoreParamS32:
2172 case NVPTXISD::StoreParam:
2173 NumElts = 1;
2174 break;
2175 case NVPTXISD::StoreParamV2:
2176 NumElts = 2;
2177 break;
2178 case NVPTXISD::StoreParamV4:
2179 NumElts = 4;
2180 break;
2181 }
2182
2183 // Build vector of operands
2184 SmallVector<SDValue, 8> Ops;
2185 for (unsigned i = 0; i < NumElts; ++i)
2186 Ops.push_back(N->getOperand(i + 3));
2187 Ops.push_back(CurDAG->getTargetConstant(ParamVal, MVT::i32));
2188 Ops.push_back(CurDAG->getTargetConstant(OffsetVal, MVT::i32));
2189 Ops.push_back(Chain);
2190 Ops.push_back(Flag);
2191
2192 // Determine target opcode
2193 // If we have an i1, use an 8-bit store. The lowering code in
2194 // NVPTXISelLowering will have already emitted an upcast.
2195 unsigned Opcode = 0;
2196 switch (N->getOpcode()) {
2197 default:
2198 switch (NumElts) {
2199 default:
2200 return NULL;
2201 case 1:
2202 switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
2203 default:
2204 return NULL;
2205 case MVT::i1:
2206 Opcode = NVPTX::StoreParamI8;
2207 break;
2208 case MVT::i8:
2209 Opcode = NVPTX::StoreParamI8;
2210 break;
2211 case MVT::i16:
2212 Opcode = NVPTX::StoreParamI16;
2213 break;
2214 case MVT::i32:
2215 Opcode = NVPTX::StoreParamI32;
2216 break;
2217 case MVT::i64:
2218 Opcode = NVPTX::StoreParamI64;
2219 break;
2220 case MVT::f32:
2221 Opcode = NVPTX::StoreParamF32;
2222 break;
2223 case MVT::f64:
2224 Opcode = NVPTX::StoreParamF64;
2225 break;
2226 }
2227 break;
2228 case 2:
2229 switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
2230 default:
2231 return NULL;
2232 case MVT::i1:
2233 Opcode = NVPTX::StoreParamV2I8;
2234 break;
2235 case MVT::i8:
2236 Opcode = NVPTX::StoreParamV2I8;
2237 break;
2238 case MVT::i16:
2239 Opcode = NVPTX::StoreParamV2I16;
2240 break;
2241 case MVT::i32:
2242 Opcode = NVPTX::StoreParamV2I32;
2243 break;
2244 case MVT::i64:
2245 Opcode = NVPTX::StoreParamV2I64;
2246 break;
2247 case MVT::f32:
2248 Opcode = NVPTX::StoreParamV2F32;
2249 break;
2250 case MVT::f64:
2251 Opcode = NVPTX::StoreParamV2F64;
2252 break;
2253 }
2254 break;
2255 case 4:
2256 switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
2257 default:
2258 return NULL;
2259 case MVT::i1:
2260 Opcode = NVPTX::StoreParamV4I8;
2261 break;
2262 case MVT::i8:
2263 Opcode = NVPTX::StoreParamV4I8;
2264 break;
2265 case MVT::i16:
2266 Opcode = NVPTX::StoreParamV4I16;
2267 break;
2268 case MVT::i32:
2269 Opcode = NVPTX::StoreParamV4I32;
2270 break;
2271 case MVT::f32:
2272 Opcode = NVPTX::StoreParamV4F32;
2273 break;
2274 }
2275 break;
2276 }
2277 break;
2278 // Special case: if we have a sign-extend/zero-extend node, insert the
2279 // conversion instruction first, and use that as the value operand to
2280 // the selected StoreParam node.
2281 case NVPTXISD::StoreParamU32: {
2282 Opcode = NVPTX::StoreParamI32;
2283 SDValue CvtNone = CurDAG->getTargetConstant(NVPTX::PTXCvtMode::NONE,
2284 MVT::i32);
2285 SDNode *Cvt = CurDAG->getMachineNode(NVPTX::CVT_u32_u16, DL,
2286 MVT::i32, Ops[0], CvtNone);
2287 Ops[0] = SDValue(Cvt, 0);
2288 break;
2289 }
2290 case NVPTXISD::StoreParamS32: {
2291 Opcode = NVPTX::StoreParamI32;
2292 SDValue CvtNone = CurDAG->getTargetConstant(NVPTX::PTXCvtMode::NONE,
2293 MVT::i32);
2294 SDNode *Cvt = CurDAG->getMachineNode(NVPTX::CVT_s32_s16, DL,
2295 MVT::i32, Ops[0], CvtNone);
2296 Ops[0] = SDValue(Cvt, 0);
2297 break;
2298 }
2299 }
2300
2301 SDVTList RetVTs = CurDAG->getVTList(MVT::Other, MVT::Glue);
2302 SDNode *Ret =
2303 CurDAG->getMachineNode(Opcode, DL, RetVTs, Ops);
2304 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
2305 MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
2306 cast<MachineSDNode>(Ret)->setMemRefs(MemRefs0, MemRefs0 + 1);
2307
2308 return Ret;
2309 }
2310
2311 // SelectDirectAddr - Match a direct address for DAG.
2312 // A direct address could be a globaladdress or externalsymbol.
2313 bool NVPTXDAGToDAGISel::SelectDirectAddr(SDValue N, SDValue &Address) {
2314 // Return true if TGA or ES.
2315 if (N.getOpcode() == ISD::TargetGlobalAddress ||
2316 N.getOpcode() == ISD::TargetExternalSymbol) {
2317 Address = N;
2318 return true;
2319 }
2320 if (N.getOpcode() == NVPTXISD::Wrapper) {
2321 Address = N.getOperand(0);
2322 return true;
2323 }
2324 if (N.getOpcode() == ISD::INTRINSIC_WO_CHAIN) {
2325 unsigned IID = cast<ConstantSDNode>(N.getOperand(0))->getZExtValue();
2326 if (IID == Intrinsic::nvvm_ptr_gen_to_param)
2327 if (N.getOperand(1).getOpcode() == NVPTXISD::MoveParam)
2328 return (SelectDirectAddr(N.getOperand(1).getOperand(0), Address));
2329 }
2330 return false;
2331 }
2332
2333 // symbol+offset
2334 bool NVPTXDAGToDAGISel::SelectADDRsi_imp(
2335 SDNode *OpNode, SDValue Addr, SDValue &Base, SDValue &Offset, MVT mvt) {
2336 if (Addr.getOpcode() == ISD::ADD) {
2337 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Addr.getOperand(1))) {
2338 SDValue base = Addr.getOperand(0);
2339 if (SelectDirectAddr(base, Base)) {
2340 Offset = CurDAG->getTargetConstant(CN->getZExtValue(), mvt);
2341 return true;
2342 }
2343 }
2344 }
2345 return false;
2346 }
2347
2348 // symbol+offset
2349 bool NVPTXDAGToDAGISel::SelectADDRsi(SDNode *OpNode, SDValue Addr,
2350 SDValue &Base, SDValue &Offset) {
2351 return SelectADDRsi_imp(OpNode, Addr, Base, Offset, MVT::i32);
2352 }
2353
2354 // symbol+offset
2355 bool NVPTXDAGToDAGISel::SelectADDRsi64(SDNode *OpNode, SDValue Addr,
2356 SDValue &Base, SDValue &Offset) {
2357 return SelectADDRsi_imp(OpNode, Addr, Base, Offset, MVT::i64);
2358 }
2359
2360 // register+offset
2361 bool NVPTXDAGToDAGISel::SelectADDRri_imp(
2362 SDNode *OpNode, SDValue Addr, SDValue &Base, SDValue &Offset, MVT mvt) {
2363 if (FrameIndexSDNode *FIN = dyn_cast<FrameIndexSDNode>(Addr)) {
2364 Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), mvt);
2365 Offset = CurDAG->getTargetConstant(0, mvt);
2366 return true;
2367 }
2368 if (Addr.getOpcode() == ISD::TargetExternalSymbol ||
2369 Addr.getOpcode() == ISD::TargetGlobalAddress)
2370 return false; // direct calls.
2371
2372 if (Addr.getOpcode() == ISD::ADD) {
2373 if (SelectDirectAddr(Addr.getOperand(0), Addr)) {
2374 return false;
2375 }
2376 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Addr.getOperand(1))) {
2377 if (FrameIndexSDNode *FIN =
2378 dyn_cast<FrameIndexSDNode>(Addr.getOperand(0)))
2379 // Constant offset from frame ref.
2380 Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), mvt);
2381 else
2382 Base = Addr.getOperand(0);
2383 Offset = CurDAG->getTargetConstant(CN->getZExtValue(), mvt);
2384 return true;
2385 }
2386 }
2387 return false;
2388 }
2389
2390 // register+offset
2391 bool NVPTXDAGToDAGISel::SelectADDRri(SDNode *OpNode, SDValue Addr,
2392 SDValue &Base, SDValue &Offset) {
2393 return SelectADDRri_imp(OpNode, Addr, Base, Offset, MVT::i32);
2394 }
2395
2396 // register+offset
2397 bool NVPTXDAGToDAGISel::SelectADDRri64(SDNode *OpNode, SDValue Addr,
2398 SDValue &Base, SDValue &Offset) {
2399 return SelectADDRri_imp(OpNode, Addr, Base, Offset, MVT::i64);
2400 }
2401
2402 bool NVPTXDAGToDAGISel::ChkMemSDNodeAddressSpace(SDNode *N,
2403 unsigned int spN) const {
2404 const Value *Src = NULL;
2405 // Even though MemIntrinsicSDNode is a subclas of MemSDNode,
2406 // the classof() for MemSDNode does not include MemIntrinsicSDNode
2407 // (See SelectionDAGNodes.h). So we need to check for both.
2408 if (MemSDNode *mN = dyn_cast<MemSDNode>(N)) {
2409 Src = mN->getSrcValue();
2410 } else if (MemSDNode *mN = dyn_cast<MemIntrinsicSDNode>(N)) {
2411 Src = mN->getSrcValue();
2412 }
2413 if (!Src)
2414 return false;
2415 if (const PointerType *PT = dyn_cast<PointerType>(Src->getType()))
2416 return (PT->getAddressSpace() == spN);
2417 return false;
2418 }
2419
2420 /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for
2421 /// inline asm expressions.
2422 bool NVPTXDAGToDAGISel::SelectInlineAsmMemoryOperand(
2423 const SDValue &Op, char ConstraintCode, std::vector<SDValue> &OutOps) {
2424 SDValue Op0, Op1;
2425 switch (ConstraintCode) {
2426 default:
2427 return true;
2428 case 'm': // memory
2429 if (SelectDirectAddr(Op, Op0)) {
2430 OutOps.push_back(Op0);
2431 OutOps.push_back(CurDAG->getTargetConstant(0, MVT::i32));
2432 return false;
2433 }
2434 if (SelectADDRri(Op.getNode(), Op, Op0, Op1)) {
2435 OutOps.push_back(Op0);
2436 OutOps.push_back(Op1);
2437 return false;
2438 }
2439 break;
2440 }
2441 return true;
2442 }
2443
2444 // Return true if N is a undef or a constant.
2445 // If N was undef, return a (i8imm 0) in Retval
2446 // If N was imm, convert it to i8imm and return in Retval
2447 // Note: The convert to i8imm is required, otherwise the
2448 // pattern matcher inserts a bunch of IMOVi8rr to convert
2449 // the imm to i8imm, and this causes instruction selection
2450 // to fail.
2451 bool NVPTXDAGToDAGISel::UndefOrImm(SDValue Op, SDValue N, SDValue &Retval) {
2452 if (!(N.getOpcode() == ISD::UNDEF) && !(N.getOpcode() == ISD::Constant))
2453 return false;
2454
2455 if (N.getOpcode() == ISD::UNDEF)
2456 Retval = CurDAG->getTargetConstant(0, MVT::i8);
2457 else {
2458 ConstantSDNode *cn = cast<ConstantSDNode>(N.getNode());
2459 unsigned retval = cn->getZExtValue();
2460 Retval = CurDAG->getTargetConstant(retval, MVT::i8);
2461 }
2462 return true;
2463 }