Members/tobaru/cbc/CbC_llvm: lib/CodeGen/AtomicExpandPass.cpp annotate

annotate lib/CodeGen/AtomicExpandPass.cpp @ 83:60c9769439b8

LLVM 3.7

author	Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp>
date	Wed, 18 Feb 2015 14:55:36 +0900
parents	54457678186b
children	afa8332a0e37

rev	line source
77 54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	1 //===-- AtomicExpandPass.cpp - Expand atomic instructions -------===//
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	2 //
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	3 // The LLVM Compiler Infrastructure
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	4 //
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	5 // This file is distributed under the University of Illinois Open Source
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	6 // License. See LICENSE.TXT for details.
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	7 //
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	8 //===----------------------------------------------------------------------===//
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	9 //
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	10 // This file contains a pass (at IR level) to replace atomic instructions with
83 60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	11 // either (intrinsic-based) load-linked/store-conditional loops or AtomicCmpXchg.
77 54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	12 //
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	13 //===----------------------------------------------------------------------===//
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	14
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	15 #include "llvm/CodeGen/Passes.h"
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	16 #include "llvm/IR/Function.h"
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	17 #include "llvm/IR/IRBuilder.h"
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	18 #include "llvm/IR/InstIterator.h"
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	19 #include "llvm/IR/Instructions.h"
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	20 #include "llvm/IR/Intrinsics.h"
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	21 #include "llvm/IR/Module.h"
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	22 #include "llvm/Support/Debug.h"
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	23 #include "llvm/Target/TargetLowering.h"
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	24 #include "llvm/Target/TargetMachine.h"
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	25 #include "llvm/Target/TargetSubtargetInfo.h"
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	26
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	27 using namespace llvm;
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	28
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	29 #define DEBUG_TYPE "atomic-expand"
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	30
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	31 namespace {
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	32 class AtomicExpand: public FunctionPass {
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	33 const TargetMachine *TM;
83 60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	34 const TargetLowering *TLI;
77 54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	35 public:
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	36 static char ID; // Pass identification, replacement for typeid
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	37 explicit AtomicExpand(const TargetMachine *TM = nullptr)
83 60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	38 : FunctionPass(ID), TM(TM), TLI(nullptr) {
77 54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	39 initializeAtomicExpandPass(*PassRegistry::getPassRegistry());
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	40 }
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	41
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	42 bool runOnFunction(Function &F) override;
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	43
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	44 private:
83 60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	45 bool bracketInstWithFences(Instruction *I, AtomicOrdering Order,
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	46 bool IsStore, bool IsLoad);
77 54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	47 bool expandAtomicLoad(LoadInst *LI);
83 60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	48 bool expandAtomicLoadToLL(LoadInst *LI);
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	49 bool expandAtomicLoadToCmpXchg(LoadInst *LI);
77 54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	50 bool expandAtomicStore(StoreInst *SI);
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	51 bool expandAtomicRMW(AtomicRMWInst *AI);
83 60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	52 bool expandAtomicRMWToLLSC(AtomicRMWInst *AI);
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	53 bool expandAtomicRMWToCmpXchg(AtomicRMWInst *AI);
77 54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	54 bool expandAtomicCmpXchg(AtomicCmpXchgInst *CI);
83 60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	55 bool isIdempotentRMW(AtomicRMWInst *AI);
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	56 bool simplifyIdempotentRMW(AtomicRMWInst *AI);
77 54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	57 };
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	58 }
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	59
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	60 char AtomicExpand::ID = 0;
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	61 char &llvm::AtomicExpandID = AtomicExpand::ID;
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	62 INITIALIZE_TM_PASS(AtomicExpand, "atomic-expand",
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	63 "Expand Atomic calls in terms of either load-linked & store-conditional or cmpxchg",
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	64 false, false)
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	65
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	66 FunctionPass llvm::createAtomicExpandPass(const TargetMachine TM) {
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	67 return new AtomicExpand(TM);
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	68 }
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	69
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	70 bool AtomicExpand::runOnFunction(Function &F) {
83 60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	71 if (!TM \|\| !TM->getSubtargetImpl(F)->enableAtomicExpand())
77 54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	72 return false;
83 60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	73 TLI = TM->getSubtargetImpl(F)->getTargetLowering();
77 54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	74
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	75 SmallVector<Instruction *, 1> AtomicInsts;
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	76
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	77 // Changing control-flow while iterating through it is a bad idea, so gather a
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	78 // list of all atomic instructions before we start.
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	79 for (inst_iterator I = inst_begin(F), E = inst_end(F); I != E; ++I) {
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	80 if (I->isAtomic())
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	81 AtomicInsts.push_back(&*I);
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	82 }
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	83
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	84 bool MadeChange = false;
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	85 for (auto I : AtomicInsts) {
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	86 auto LI = dyn_cast<LoadInst>(I);
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	87 auto SI = dyn_cast<StoreInst>(I);
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	88 auto RMWI = dyn_cast<AtomicRMWInst>(I);
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	89 auto CASI = dyn_cast<AtomicCmpXchgInst>(I);
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	90 assert((LI \|\| SI \|\| RMWI \|\| CASI \|\| isa<FenceInst>(I)) &&
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	91 "Unknown atomic instruction");
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	92
83 60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	93 auto FenceOrdering = Monotonic;
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	94 bool IsStore, IsLoad;
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	95 if (TLI->getInsertFencesForAtomic()) {
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	96 if (LI && isAtLeastAcquire(LI->getOrdering())) {
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	97 FenceOrdering = LI->getOrdering();
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	98 LI->setOrdering(Monotonic);
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	99 IsStore = false;
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	100 IsLoad = true;
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	101 } else if (SI && isAtLeastRelease(SI->getOrdering())) {
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	102 FenceOrdering = SI->getOrdering();
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	103 SI->setOrdering(Monotonic);
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	104 IsStore = true;
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	105 IsLoad = false;
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	106 } else if (RMWI && (isAtLeastRelease(RMWI->getOrdering()) \|\|
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	107 isAtLeastAcquire(RMWI->getOrdering()))) {
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	108 FenceOrdering = RMWI->getOrdering();
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	109 RMWI->setOrdering(Monotonic);
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	110 IsStore = IsLoad = true;
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	111 } else if (CASI && !TLI->hasLoadLinkedStoreConditional() &&
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	112 (isAtLeastRelease(CASI->getSuccessOrdering()) \|\|
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	113 isAtLeastAcquire(CASI->getSuccessOrdering()))) {
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	114 // If a compare and swap is lowered to LL/SC, we can do smarter fence
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	115 // insertion, with a stronger one on the success path than on the
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	116 // failure path. As a result, fence insertion is directly done by
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	117 // expandAtomicCmpXchg in that case.
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	118 FenceOrdering = CASI->getSuccessOrdering();
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	119 CASI->setSuccessOrdering(Monotonic);
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	120 CASI->setFailureOrdering(Monotonic);
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	121 IsStore = IsLoad = true;
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	122 }
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	123
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	124 if (FenceOrdering != Monotonic) {
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	125 MadeChange \|= bracketInstWithFences(I, FenceOrdering, IsStore, IsLoad);
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	126 }
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	127 }
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	128
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	129 if (LI && TLI->shouldExpandAtomicLoadInIR(LI)) {
77 54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	130 MadeChange \|= expandAtomicLoad(LI);
83 60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	131 } else if (SI && TLI->shouldExpandAtomicStoreInIR(SI)) {
77 54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	132 MadeChange \|= expandAtomicStore(SI);
83 60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	133 } else if (RMWI) {
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	134 // There are two different ways of expanding RMW instructions:
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	135 // - into a load if it is idempotent
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	136 // - into a Cmpxchg/LL-SC loop otherwise
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	137 // we try them in that order.
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	138 MadeChange \|=
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	139 (isIdempotentRMW(RMWI) && simplifyIdempotentRMW(RMWI)) \|\|
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	140 (TLI->shouldExpandAtomicRMWInIR(RMWI) && expandAtomicRMW(RMWI));
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	141 } else if (CASI && TLI->hasLoadLinkedStoreConditional()) {
77 54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	142 MadeChange \|= expandAtomicCmpXchg(CASI);
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	143 }
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	144 }
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	145 return MadeChange;
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	146 }
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	147
83 60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	148 bool AtomicExpand::bracketInstWithFences(Instruction *I, AtomicOrdering Order,
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	149 bool IsStore, bool IsLoad) {
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	150 IRBuilder<> Builder(I);
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	151
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	152 auto LeadingFence = TLI->emitLeadingFence(Builder, Order, IsStore, IsLoad);
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	153
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	154 auto TrailingFence = TLI->emitTrailingFence(Builder, Order, IsStore, IsLoad);
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	155 // The trailing fence is emitted before the instruction instead of after
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	156 // because there is no easy way of setting Builder insertion point after
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	157 // an instruction. So we must erase it from the BB, and insert it back
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	158 // in the right place.
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	159 // We have a guard here because not every atomic operation generates a
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	160 // trailing fence.
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	161 if (TrailingFence) {
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	162 TrailingFence->removeFromParent();
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	163 TrailingFence->insertAfter(I);
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	164 }
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	165
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	166 return (LeadingFence \|\| TrailingFence);
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	167 }
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	168
77 54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	169 bool AtomicExpand::expandAtomicLoad(LoadInst *LI) {
83 60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	170 if (TLI->hasLoadLinkedStoreConditional())
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	171 return expandAtomicLoadToLL(LI);
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	172 else
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	173 return expandAtomicLoadToCmpXchg(LI);
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	174 }
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	175
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	176 bool AtomicExpand::expandAtomicLoadToLL(LoadInst *LI) {
77 54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	177 IRBuilder<> Builder(LI);
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	178
83 60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	179 // On some architectures, load-linked instructions are atomic for larger
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	180 // sizes than normal loads. For example, the only 64-bit load guaranteed
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	181 // to be single-copy atomic by ARM is an ldrexd (A3.5.3).
77 54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	182 Value *Val =
83 60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	183 TLI->emitLoadLinked(Builder, LI->getPointerOperand(), LI->getOrdering());
77 54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	184
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	185 LI->replaceAllUsesWith(Val);
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	186 LI->eraseFromParent();
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	187
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	188 return true;
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	189 }
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	190
83 60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	191 bool AtomicExpand::expandAtomicLoadToCmpXchg(LoadInst *LI) {
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	192 IRBuilder<> Builder(LI);
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	193 AtomicOrdering Order = LI->getOrdering();
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	194 Value *Addr = LI->getPointerOperand();
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	195 Type *Ty = cast<PointerType>(Addr->getType())->getElementType();
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	196 Constant *DummyVal = Constant::getNullValue(Ty);
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	197
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	198 Value *Pair = Builder.CreateAtomicCmpXchg(
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	199 Addr, DummyVal, DummyVal, Order,
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	200 AtomicCmpXchgInst::getStrongestFailureOrdering(Order));
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	201 Value *Loaded = Builder.CreateExtractValue(Pair, 0, "loaded");
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	202
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	203 LI->replaceAllUsesWith(Loaded);
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	204 LI->eraseFromParent();
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	205
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	206 return true;
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	207 }
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	208
77 54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	209 bool AtomicExpand::expandAtomicStore(StoreInst *SI) {
83 60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	210 // This function is only called on atomic stores that are too large to be
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	211 // atomic if implemented as a native store. So we replace them by an
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	212 // atomic swap, that can be implemented for example as a ldrex/strex on ARM
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	213 // or lock cmpxchg8/16b on X86, as these are atomic for larger sizes.
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	214 // It is the responsibility of the target to only return true in
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	215 // shouldExpandAtomicRMW in cases where this is required and possible.
77 54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	216 IRBuilder<> Builder(SI);
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	217 AtomicRMWInst *AI =
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	218 Builder.CreateAtomicRMW(AtomicRMWInst::Xchg, SI->getPointerOperand(),
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	219 SI->getValueOperand(), SI->getOrdering());
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	220 SI->eraseFromParent();
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	221
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	222 // Now we have an appropriate swap instruction, lower it as usual.
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	223 return expandAtomicRMW(AI);
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	224 }
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	225
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	226 bool AtomicExpand::expandAtomicRMW(AtomicRMWInst *AI) {
83 60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	227 if (TLI->hasLoadLinkedStoreConditional())
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	228 return expandAtomicRMWToLLSC(AI);
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	229 else
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	230 return expandAtomicRMWToCmpXchg(AI);
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	231 }
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	232
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	233 /// Emit IR to implement the given atomicrmw operation on values in registers,
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	234 /// returning the new value.
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	235 static Value *performAtomicOp(AtomicRMWInst::BinOp Op, IRBuilder<> &Builder,
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	236 Value Loaded, Value Inc) {
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	237 Value *NewVal;
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	238 switch (Op) {
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	239 case AtomicRMWInst::Xchg:
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	240 return Inc;
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	241 case AtomicRMWInst::Add:
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	242 return Builder.CreateAdd(Loaded, Inc, "new");
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	243 case AtomicRMWInst::Sub:
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	244 return Builder.CreateSub(Loaded, Inc, "new");
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	245 case AtomicRMWInst::And:
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	246 return Builder.CreateAnd(Loaded, Inc, "new");
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	247 case AtomicRMWInst::Nand:
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	248 return Builder.CreateNot(Builder.CreateAnd(Loaded, Inc), "new");
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	249 case AtomicRMWInst::Or:
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	250 return Builder.CreateOr(Loaded, Inc, "new");
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	251 case AtomicRMWInst::Xor:
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	252 return Builder.CreateXor(Loaded, Inc, "new");
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	253 case AtomicRMWInst::Max:
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	254 NewVal = Builder.CreateICmpSGT(Loaded, Inc);
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	255 return Builder.CreateSelect(NewVal, Loaded, Inc, "new");
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	256 case AtomicRMWInst::Min:
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	257 NewVal = Builder.CreateICmpSLE(Loaded, Inc);
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	258 return Builder.CreateSelect(NewVal, Loaded, Inc, "new");
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	259 case AtomicRMWInst::UMax:
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	260 NewVal = Builder.CreateICmpUGT(Loaded, Inc);
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	261 return Builder.CreateSelect(NewVal, Loaded, Inc, "new");
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	262 case AtomicRMWInst::UMin:
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	263 NewVal = Builder.CreateICmpULE(Loaded, Inc);
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	264 return Builder.CreateSelect(NewVal, Loaded, Inc, "new");
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	265 default:
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	266 llvm_unreachable("Unknown atomic op");
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	267 }
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	268 }
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	269
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	270 bool AtomicExpand::expandAtomicRMWToLLSC(AtomicRMWInst *AI) {
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	271 AtomicOrdering MemOpOrder = AI->getOrdering();
77 54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	272 Value *Addr = AI->getPointerOperand();
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	273 BasicBlock *BB = AI->getParent();
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	274 Function *F = BB->getParent();
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	275 LLVMContext &Ctx = F->getContext();
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	276
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	277 // Given: atomicrmw some_op iN* %addr, iN %incr ordering
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	278 //
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	279 // The standard expansion we produce is:
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	280 // [...]
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	281 // fence?
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	282 // atomicrmw.start:
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	283 // %loaded = @load.linked(%addr)
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	284 // %new = some_op iN %loaded, %incr
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	285 // %stored = @store_conditional(%new, %addr)
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	286 // %try_again = icmp i32 ne %stored, 0
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	287 // br i1 %try_again, label %loop, label %atomicrmw.end
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	288 // atomicrmw.end:
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	289 // fence?
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	290 // [...]
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	291 BasicBlock *ExitBB = BB->splitBasicBlock(AI, "atomicrmw.end");
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	292 BasicBlock *LoopBB = BasicBlock::Create(Ctx, "atomicrmw.start", F, ExitBB);
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	293
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	294 // This grabs the DebugLoc from AI.
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	295 IRBuilder<> Builder(AI);
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	296
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	297 // The split call above "helpfully" added a branch at the end of BB (to the
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	298 // wrong place), but we might want a fence too. It's easiest to just remove
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	299 // the branch entirely.
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	300 std::prev(BB->end())->eraseFromParent();
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	301 Builder.SetInsertPoint(BB);
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	302 Builder.CreateBr(LoopBB);
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	303
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	304 // Start the main loop block now that we've taken care of the preliminaries.
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	305 Builder.SetInsertPoint(LoopBB);
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	306 Value *Loaded = TLI->emitLoadLinked(Builder, Addr, MemOpOrder);
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	307
83 60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	308 Value *NewVal =
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	309 performAtomicOp(AI->getOperation(), Builder, Loaded, AI->getValOperand());
77 54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	310
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	311 Value *StoreSuccess =
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	312 TLI->emitStoreConditional(Builder, NewVal, Addr, MemOpOrder);
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	313 Value *TryAgain = Builder.CreateICmpNE(
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	314 StoreSuccess, ConstantInt::get(IntegerType::get(Ctx, 32), 0), "tryagain");
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	315 Builder.CreateCondBr(TryAgain, LoopBB, ExitBB);
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	316
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	317 Builder.SetInsertPoint(ExitBB, ExitBB->begin());
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	318
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	319 AI->replaceAllUsesWith(Loaded);
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	320 AI->eraseFromParent();
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	321
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	322 return true;
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	323 }
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	324
83 60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	325 bool AtomicExpand::expandAtomicRMWToCmpXchg(AtomicRMWInst *AI) {
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	326 AtomicOrdering MemOpOrder =
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	327 AI->getOrdering() == Unordered ? Monotonic : AI->getOrdering();
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	328 Value *Addr = AI->getPointerOperand();
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	329 BasicBlock *BB = AI->getParent();
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	330 Function *F = BB->getParent();
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	331 LLVMContext &Ctx = F->getContext();
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	332
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	333 // Given: atomicrmw some_op iN* %addr, iN %incr ordering
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	334 //
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	335 // The standard expansion we produce is:
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	336 // [...]
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	337 // %init_loaded = load atomic iN* %addr
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	338 // br label %loop
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	339 // loop:
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	340 // %loaded = phi iN [ %init_loaded, %entry ], [ %new_loaded, %loop ]
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	341 // %new = some_op iN %loaded, %incr
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	342 // %pair = cmpxchg iN* %addr, iN %loaded, iN %new
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	343 // %new_loaded = extractvalue { iN, i1 } %pair, 0
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	344 // %success = extractvalue { iN, i1 } %pair, 1
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	345 // br i1 %success, label %atomicrmw.end, label %loop
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	346 // atomicrmw.end:
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	347 // [...]
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	348 BasicBlock *ExitBB = BB->splitBasicBlock(AI, "atomicrmw.end");
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	349 BasicBlock *LoopBB = BasicBlock::Create(Ctx, "atomicrmw.start", F, ExitBB);
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	350
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	351 // This grabs the DebugLoc from AI.
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	352 IRBuilder<> Builder(AI);
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	353
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	354 // The split call above "helpfully" added a branch at the end of BB (to the
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	355 // wrong place), but we want a load. It's easiest to just remove
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	356 // the branch entirely.
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	357 std::prev(BB->end())->eraseFromParent();
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	358 Builder.SetInsertPoint(BB);
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	359 LoadInst *InitLoaded = Builder.CreateLoad(Addr);
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	360 // Atomics require at least natural alignment.
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	361 InitLoaded->setAlignment(AI->getType()->getPrimitiveSizeInBits());
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	362 Builder.CreateBr(LoopBB);
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	363
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	364 // Start the main loop block now that we've taken care of the preliminaries.
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	365 Builder.SetInsertPoint(LoopBB);
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	366 PHINode *Loaded = Builder.CreatePHI(AI->getType(), 2, "loaded");
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	367 Loaded->addIncoming(InitLoaded, BB);
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	368
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	369 Value *NewVal =
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	370 performAtomicOp(AI->getOperation(), Builder, Loaded, AI->getValOperand());
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	371
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	372 Value *Pair = Builder.CreateAtomicCmpXchg(
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	373 Addr, Loaded, NewVal, MemOpOrder,
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	374 AtomicCmpXchgInst::getStrongestFailureOrdering(MemOpOrder));
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	375 Value *NewLoaded = Builder.CreateExtractValue(Pair, 0, "newloaded");
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	376 Loaded->addIncoming(NewLoaded, LoopBB);
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	377
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	378 Value *Success = Builder.CreateExtractValue(Pair, 1, "success");
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	379 Builder.CreateCondBr(Success, ExitBB, LoopBB);
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	380
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	381 Builder.SetInsertPoint(ExitBB, ExitBB->begin());
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	382
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	383 AI->replaceAllUsesWith(NewLoaded);
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	384 AI->eraseFromParent();
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	385
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	386 return true;
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	387 }
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	388
77 54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	389 bool AtomicExpand::expandAtomicCmpXchg(AtomicCmpXchgInst *CI) {
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	390 AtomicOrdering SuccessOrder = CI->getSuccessOrdering();
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	391 AtomicOrdering FailureOrder = CI->getFailureOrdering();
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	392 Value *Addr = CI->getPointerOperand();
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	393 BasicBlock *BB = CI->getParent();
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	394 Function *F = BB->getParent();
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	395 LLVMContext &Ctx = F->getContext();
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	396 // If getInsertFencesForAtomic() returns true, then the target does not want
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	397 // to deal with memory orders, and emitLeading/TrailingFence should take care
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	398 // of everything. Otherwise, emitLeading/TrailingFence are no-op and we
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	399 // should preserve the ordering.
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	400 AtomicOrdering MemOpOrder =
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	401 TLI->getInsertFencesForAtomic() ? Monotonic : SuccessOrder;
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	402
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	403 // Given: cmpxchg some_op iN* %addr, iN %desired, iN %new success_ord fail_ord
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	404 //
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	405 // The full expansion we produce is:
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	406 // [...]
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	407 // fence?
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	408 // cmpxchg.start:
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	409 // %loaded = @load.linked(%addr)
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	410 // %should_store = icmp eq %loaded, %desired
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	411 // br i1 %should_store, label %cmpxchg.trystore,
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	412 // label %cmpxchg.failure
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	413 // cmpxchg.trystore:
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	414 // %stored = @store_conditional(%new, %addr)
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	415 // %success = icmp eq i32 %stored, 0
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	416 // br i1 %success, label %cmpxchg.success, label %loop/%cmpxchg.failure
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	417 // cmpxchg.success:
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	418 // fence?
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	419 // br label %cmpxchg.end
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	420 // cmpxchg.failure:
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	421 // fence?
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	422 // br label %cmpxchg.end
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	423 // cmpxchg.end:
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	424 // %success = phi i1 [true, %cmpxchg.success], [false, %cmpxchg.failure]
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	425 // %restmp = insertvalue { iN, i1 } undef, iN %loaded, 0
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	426 // %res = insertvalue { iN, i1 } %restmp, i1 %success, 1
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	427 // [...]
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	428 BasicBlock *ExitBB = BB->splitBasicBlock(CI, "cmpxchg.end");
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	429 auto FailureBB = BasicBlock::Create(Ctx, "cmpxchg.failure", F, ExitBB);
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	430 auto SuccessBB = BasicBlock::Create(Ctx, "cmpxchg.success", F, FailureBB);
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	431 auto TryStoreBB = BasicBlock::Create(Ctx, "cmpxchg.trystore", F, SuccessBB);
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	432 auto LoopBB = BasicBlock::Create(Ctx, "cmpxchg.start", F, TryStoreBB);
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	433
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	434 // This grabs the DebugLoc from CI
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	435 IRBuilder<> Builder(CI);
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	436
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	437 // The split call above "helpfully" added a branch at the end of BB (to the
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	438 // wrong place), but we might want a fence too. It's easiest to just remove
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	439 // the branch entirely.
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	440 std::prev(BB->end())->eraseFromParent();
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	441 Builder.SetInsertPoint(BB);
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	442 TLI->emitLeadingFence(Builder, SuccessOrder, /IsStore=/true,
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	443 /IsLoad=/true);
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	444 Builder.CreateBr(LoopBB);
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	445
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	446 // Start the main loop block now that we've taken care of the preliminaries.
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	447 Builder.SetInsertPoint(LoopBB);
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	448 Value *Loaded = TLI->emitLoadLinked(Builder, Addr, MemOpOrder);
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	449 Value *ShouldStore =
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	450 Builder.CreateICmpEQ(Loaded, CI->getCompareOperand(), "should_store");
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	451
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	452 // If the the cmpxchg doesn't actually need any ordering when it fails, we can
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	453 // jump straight past that fence instruction (if it exists).
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	454 Builder.CreateCondBr(ShouldStore, TryStoreBB, FailureBB);
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	455
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	456 Builder.SetInsertPoint(TryStoreBB);
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	457 Value *StoreSuccess = TLI->emitStoreConditional(
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	458 Builder, CI->getNewValOperand(), Addr, MemOpOrder);
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	459 StoreSuccess = Builder.CreateICmpEQ(
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	460 StoreSuccess, ConstantInt::get(Type::getInt32Ty(Ctx), 0), "success");
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	461 Builder.CreateCondBr(StoreSuccess, SuccessBB,
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	462 CI->isWeak() ? FailureBB : LoopBB);
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	463
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	464 // Make sure later instructions don't get reordered with a fence if necessary.
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	465 Builder.SetInsertPoint(SuccessBB);
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	466 TLI->emitTrailingFence(Builder, SuccessOrder, /IsStore=/true,
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	467 /IsLoad=/true);
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	468 Builder.CreateBr(ExitBB);
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	469
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	470 Builder.SetInsertPoint(FailureBB);
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	471 TLI->emitTrailingFence(Builder, FailureOrder, /IsStore=/true,
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	472 /IsLoad=/true);
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	473 Builder.CreateBr(ExitBB);
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	474
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	475 // Finally, we have control-flow based knowledge of whether the cmpxchg
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	476 // succeeded or not. We expose this to later passes by converting any
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	477 // subsequent "icmp eq/ne %loaded, %oldval" into a use of an appropriate PHI.
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	478
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	479 // Setup the builder so we can create any PHIs we need.
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	480 Builder.SetInsertPoint(ExitBB, ExitBB->begin());
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	481 PHINode *Success = Builder.CreatePHI(Type::getInt1Ty(Ctx), 2);
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	482 Success->addIncoming(ConstantInt::getTrue(Ctx), SuccessBB);
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	483 Success->addIncoming(ConstantInt::getFalse(Ctx), FailureBB);
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	484
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	485 // Look for any users of the cmpxchg that are just comparing the loaded value
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	486 // against the desired one, and replace them with the CFG-derived version.
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	487 SmallVector<ExtractValueInst *, 2> PrunedInsts;
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	488 for (auto User : CI->users()) {
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	489 ExtractValueInst *EV = dyn_cast<ExtractValueInst>(User);
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	490 if (!EV)
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	491 continue;
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	492
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	493 assert(EV->getNumIndices() == 1 && EV->getIndices()[0] <= 1 &&
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	494 "weird extraction from { iN, i1 }");
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	495
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	496 if (EV->getIndices()[0] == 0)
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	497 EV->replaceAllUsesWith(Loaded);
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	498 else
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	499 EV->replaceAllUsesWith(Success);
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	500
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	501 PrunedInsts.push_back(EV);
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	502 }
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	503
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	504 // We can remove the instructions now we're no longer iterating through them.
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	505 for (auto EV : PrunedInsts)
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	506 EV->eraseFromParent();
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	507
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	508 if (!CI->use_empty()) {
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	509 // Some use of the full struct return that we don't understand has happened,
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	510 // so we've got to reconstruct it properly.
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	511 Value *Res;
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	512 Res = Builder.CreateInsertValue(UndefValue::get(CI->getType()), Loaded, 0);
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	513 Res = Builder.CreateInsertValue(Res, Success, 1);
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	514
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	515 CI->replaceAllUsesWith(Res);
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	516 }
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	517
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	518 CI->eraseFromParent();
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	519 return true;
54457678186b LLVM 3.6 Kaito Tokumori <e105711@ie.u-ryukyu.ac.jp> parents: diff changeset	520 }
83 60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	521
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	522 bool AtomicExpand::isIdempotentRMW(AtomicRMWInst* RMWI) {
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	523 auto C = dyn_cast<ConstantInt>(RMWI->getValOperand());
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	524 if(!C)
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	525 return false;
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	526
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	527 AtomicRMWInst::BinOp Op = RMWI->getOperation();
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	528 switch(Op) {
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	529 case AtomicRMWInst::Add:
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	530 case AtomicRMWInst::Sub:
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	531 case AtomicRMWInst::Or:
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	532 case AtomicRMWInst::Xor:
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	533 return C->isZero();
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	534 case AtomicRMWInst::And:
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	535 return C->isMinusOne();
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	536 // FIXME: we could also treat Min/Max/UMin/UMax by the INT_MIN/INT_MAX/...
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	537 default:
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	538 return false;
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	539 }
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	540 }
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	541
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	542 bool AtomicExpand::simplifyIdempotentRMW(AtomicRMWInst* RMWI) {
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	543 if (auto ResultingLoad = TLI->lowerIdempotentRMWIntoFencedLoad(RMWI)) {
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	544 if (TLI->shouldExpandAtomicLoadInIR(ResultingLoad))
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	545 expandAtomicLoad(ResultingLoad);
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	546 return true;
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	547 }
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	548 return false;
60c9769439b8 LLVM 3.7 Tatsuki IHA <e125716@ie.u-ryukyu.ac.jp> parents: 77 diff changeset	549 }

Mercurial > hg > Members > tobaru > cbc > CbC_llvm

annotate lib/CodeGen/AtomicExpandPass.cpp @ 83:60c9769439b8