CbC/CbC_llvm: lib/Target/Hexagon/HexagonISelDAGToDAGHVX.cpp comparison

comparison lib/Target/Hexagon/HexagonISelDAGToDAGHVX.cpp @ 148:63bd29f05246

merged

author	Shinji KONO <kono@ie.u-ryukyu.ac.jp>
date	Wed, 14 Aug 2019 19:46:37 +0900
parents	c2174574ed3a
children

comparison

equal deleted inserted replaced

-:3fc4d5c3e21e
+:63bd29f05246
 //===-- HexagonISelDAGToDAGHVX.cpp ----------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-//
+// See https://llvm.org/LICENSE.txt for license information.
-// This file is distributed under the University of Illinois Open Source
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-// License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 #include "Hexagon.h"
 #include "HexagonISelDAGToDAG.h"
 if (Color == ColorKind::None)
 return ColorKind::Red;
 return Color == ColorKind::Red ? ColorKind::Black : ColorKind::Red;
 }
-void dump() const;
+LLVM_DUMP_METHOD void dump() const;
 private:
 ArrayRef<Node> Order;
 MapType Colors;
 std::set<Node> Needed;
 }
 Colors[N] = ColorN;
 Colors[C] = ColorC;
 }
-// Explicitly assign "None" all all uncolored nodes.
+// Explicitly assign "None" to all uncolored nodes.
 for (unsigned I = 0; I != Order.size(); ++I)
 if (Colors.count(I) == 0)
 Colors[I] = ColorKind::None;
 return true;
 }
-LLVM_DUMP_METHOD
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 void Coloring::dump() const {
 dbgs() << "{ Order:   {";
 for (unsigned I = 0; I != Order.size(); ++I) {
 Node P = Order[I];
 if (P != Ignore)
 dbgs() << "  Colors: {\n";
 for (auto C : Colors)
 dbgs() << "    " << C.first << " -> " << ColorKindToName(C.second) << "\n";
 dbgs() << "  }\n}\n";
 }
+#endif
 namespace {
 // Base class of for reordering networks. They don't strictly need to be
 // permutations, as outputs with repeated occurrences of an input element
 // are allowed.
 Undef   = 0x80000000,
 Index   = 0x0FFFFFFF,  // Mask of the index value.
 IndexBits = 28,
 };
+LLVM_DUMP_METHOD
 void print(raw_ostream &OS, const SelectionDAG &G) const;
 private:
 OpRef(unsigned N) : OpN(N) {}
 };
 NodeTemplate() = default;
 unsigned Opc = 0;
 MVT Ty = MVT::Other;
 std::vector<OpRef> Ops;
-void print(raw_ostream &OS, const SelectionDAG &G) const;
+LLVM_DUMP_METHOD void print(raw_ostream &OS, const SelectionDAG &G) const;
 };
 struct ResultStack {
 ResultStack(SDNode *Inp)
 : InpNode(Inp), InpTy(Inp->getValueType(0).getSimpleVT()) {}
 BaseType::const_iterator begin() const { return List.begin(); }
 BaseType::const_iterator end() const   { return List.end(); }
 BaseType List;
+LLVM_DUMP_METHOD
 void print(raw_ostream &OS, const SelectionDAG &G) const;
 };
 } // namespace
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 void OpRef::print(raw_ostream &OS, const SelectionDAG &G) const {
 if (isValue()) {
 OpV.getNode()->print(OS, &G);
 return;
 }
 OS << '[' << I << "] ";
 List[I].print(OS, G);
 OS << '\n';
 }
 }
+#endif
 namespace {
 struct ShuffleMask {
 ShuffleMask(ArrayRef<int> M) : Mask(M) {
 for (unsigned I = 0, E = Mask.size(); I != E; ++I) {
 }
 ShuffleMask hi() const {
 size_t H = Mask.size()/2;
 return ShuffleMask(Mask.take_back(H));
 }
+void print(raw_ostream &OS) const {
+OS << "MinSrc:" << MinSrc << ", MaxSrc:" << MaxSrc << " {";
+for (int M : Mask)
+OS << ' ' << M;
+OS << " }";
+}
 };
 } // namespace
 // --------------------------------------------------------------------
 // The HvxSelector class.
 return MVT::getVectorVT(ElemTy, NumElems);
 }
 void selectShuffle(SDNode *N);
 void selectRor(SDNode *N);
+void selectVAlign(SDNode *N);
 private:
 void materialize(const ResultStack &Results);
 SDValue getVectorConstant(ArrayRef<uint8_t> Data, const SDLoc &dl);
 int N = Mask.size();
 return 2*Sum == N*(N-1);
 }
 bool HvxSelector::selectVectorConstants(SDNode *N) {
-// Constant vectors are generated as loads from constant pools or
+// Constant vectors are generated as loads from constant pools or as
-// as VSPLATs of a constant value.
+// splats of a constant value. Since they are generated during the
-// Since they are generated during the selection process, the main
+// selection process, the main selection algorithm is not aware of them.
-// selection algorithm is not aware of them. Select them directly
+// Select them directly here.
-// here.
 SmallVector<SDNode*,4> Nodes;
 SetVector<SDNode*> WorkQ;
+// The one-use test for VSPLATW's operand may fail due to dead nodes
+// left over in the DAG.
+DAG.RemoveDeadNodes();
 // The DAG can change (due to CSE) during selection, so cache all the
 // unselected nodes first to avoid traversing a mutating DAG.
 auto IsNodeToSelect = [] (SDNode *N) {
 if (N->isMachineOpcode())
 return false;
-unsigned Opc = N->getOpcode();
+switch (N->getOpcode()) {
-if (Opc == HexagonISD::VSPLAT || Opc == HexagonISD::VZERO)
+case HexagonISD::VZERO:
-return true;
+case HexagonISD::VSPLATW:
-if (Opc == ISD::BITCAST) {
-// Only select bitcasts of VSPLATs.
-if (N->getOperand(0).getOpcode() == HexagonISD::VSPLAT)
 return true;
-}
+case ISD::LOAD: {
-if (Opc == ISD::LOAD) {
+SDValue Addr = cast<LoadSDNode>(N)->getBasePtr();
-SDValue Addr = cast<LoadSDNode>(N)->getBasePtr();
+unsigned AddrOpc = Addr.getOpcode();
-unsigned AddrOpc = Addr.getOpcode();
+if (AddrOpc == HexagonISD::AT_PCREL || AddrOpc == HexagonISD::CP)
-if (AddrOpc == HexagonISD::AT_PCREL || AddrOpc == HexagonISD::CP)
+if (Addr.getOperand(0).getOpcode() == ISD::TargetConstantPool)
-if (Addr.getOperand(0).getOpcode() == ISD::TargetConstantPool)
+return true;
-return true;
+}
-}
+break;
-return false;
+}
+// Make sure to select the operand of VSPLATW.
+bool IsSplatOp = N->hasOneUse() &&
+N->use_begin()->getOpcode() == HexagonISD::VSPLATW;
+return IsSplatOp;
 };
 WorkQ.insert(N);
 for (unsigned i = 0; i != WorkQ.size(); ++i) {
 SDNode *W = WorkQ[i];
 return OpRef::fail();
 int VecLen = SM.Mask.size();
 MVT Ty = getSingleVT(MVT::i8);
+auto IsExtSubvector = [] (ShuffleMask M) {
+assert(M.MinSrc >= 0 && M.MaxSrc >= 0);
+for (int I = 0, E = M.Mask.size(); I != E; ++I) {
+if (M.Mask[I] >= 0 && M.Mask[I]-I != M.MinSrc)
+return false;
+}
+return true;
+};
 if (SM.MaxSrc - SM.MinSrc < int(HwLen)) {
+if (SM.MinSrc == 0 || SM.MinSrc == int(HwLen) || !IsExtSubvector(SM)) {
+// If the mask picks elements from only one of the operands, return
+// that operand, and update the mask to use index 0 to refer to the
+// first element of that operand.
+// If the mask extracts a subvector, it will be handled below, so
+// skip it here.
+if (SM.MaxSrc < int(HwLen)) {
+memcpy(NewMask.data(), SM.Mask.data(), sizeof(int)*VecLen);
+return Va;
+}
+if (SM.MinSrc >= int(HwLen)) {
+for (int I = 0; I != VecLen; ++I) {
+int M = SM.Mask[I];
+if (M != -1)
+M -= HwLen;
+NewMask[I] = M;
+}
+return Vb;
+}
+}
+int MinSrc = SM.MinSrc;
 if (SM.MaxSrc < int(HwLen)) {
-memcpy(NewMask.data(), SM.Mask.data(), sizeof(int)*VecLen);
+Vb = Va;
-return Va;
+} else if (SM.MinSrc > int(HwLen)) {
-}
+Va = Vb;
-if (SM.MinSrc >= int(HwLen)) {
+MinSrc = SM.MinSrc - HwLen;
-for (int I = 0; I != VecLen; ++I) {
-int M = SM.Mask[I];
-if (M != -1)
-M -= HwLen;
-NewMask[I] = M;
-}
-return Vb;
 }
 const SDLoc &dl(Results.InpNode);
-SDValue S = DAG.getTargetConstant(SM.MinSrc, dl, MVT::i32);
+if (isUInt<3>(MinSrc) || isUInt<3>(HwLen-MinSrc)) {
-if (isUInt<3>(SM.MinSrc)) {
+bool IsRight = isUInt<3>(MinSrc); // Right align.
-Results.push(Hexagon::V6_valignbi, Ty, {Vb, Va, S});
+SDValue S = DAG.getTargetConstant(IsRight ? MinSrc : HwLen-MinSrc,
+dl, MVT::i32);
+unsigned Opc = IsRight ? Hexagon::V6_valignbi
+: Hexagon::V6_vlalignbi;
+Results.push(Opc, Ty, {Vb, Va, S});
 } else {
+SDValue S = DAG.getTargetConstant(MinSrc, dl, MVT::i32);
 Results.push(Hexagon::A2_tfrsi, MVT::i32, {S});
 unsigned Top = Results.top();
 Results.push(Hexagon::V6_valignb, Ty, {Vb, Va, OpRef::res(Top)});
 }
 for (int I = 0; I != VecLen; ++I) {
 Bytes[I] = 0xFF;
 }
 return vmuxp(Bytes, L, R, Results);
 }
+namespace {
+struct Deleter : public SelectionDAG::DAGNodeDeletedListener {
+template <typename T>
+Deleter(SelectionDAG &D, T &C)
+: SelectionDAG::DAGNodeDeletedListener(D, [&C] (SDNode *N, SDNode *E) {
+C.erase(N);
+}) {}
+};
+template <typename T>
+struct NullifyingVector : public T {
+DenseMap<SDNode*, SDNode**> Refs;
+NullifyingVector(T &&V) : T(V) {
+for (unsigned i = 0, e = T::size(); i != e; ++i) {
+SDNode *&N = T::operator[](i);
+Refs[N] = &N;
+}
+}
+void erase(SDNode *N) {
+auto F = Refs.find(N);
+if (F != Refs.end())
+*F->second = nullptr;
+}
+};
+}
 bool HvxSelector::scalarizeShuffle(ArrayRef<int> Mask, const SDLoc &dl,
 MVT ResTy, SDValue Va, SDValue Vb,
 SDNode *N) {
 DEBUG_WITH_TYPE("isel", {dbgs() << __func__ << '\n';});
 MVT ElemTy = ResTy.getVectorElementType();
 assert(ElemTy == MVT::i8);
 unsigned VecLen = Mask.size();
 bool HavePairs = (2*HwLen == VecLen);
 MVT SingleTy = getSingleVT(MVT::i8);
+// The prior attempts to handle this shuffle may have left a bunch of
+// dead nodes in the DAG (such as constants). These nodes will be added
+// at the end of DAG's node list, which at that point had already been
+// sorted topologically. In the main selection loop, the node list is
+// traversed backwards from the root node, which means that any new
+// nodes (from the end of the list) will not be visited.
+// Scalarization will replace the shuffle node with the scalarized
+// expression, and if that expression reused any if the leftoever (dead)
+// nodes, these nodes would not be selected (since the "local" selection
+// only visits nodes that are not in AllNodes).
+// To avoid this issue, remove all dead nodes from the DAG now.
+DAG.RemoveDeadNodes();
+DenseSet<SDNode*> AllNodes;
+for (SDNode &S : DAG.allnodes())
+AllNodes.insert(&S);
+Deleter DUA(DAG, AllNodes);
 SmallVector<SDValue,128> Ops;
+LLVMContext &Ctx = *DAG.getContext();
+MVT LegalTy = Lower.getTypeToTransformTo(Ctx, ElemTy).getSimpleVT();
 for (int I : Mask) {
 if (I < 0) {
-Ops.push_back(ISel.selectUndef(dl, ElemTy));
+Ops.push_back(ISel.selectUndef(dl, LegalTy));
 continue;
 }
 SDValue Vec;
 unsigned M = I;
 if (M < VecLen) {
 Vec = DAG.getTargetExtractSubreg(Hexagon::vsub_hi, dl, SingleTy, Vec);
 M -= HwLen;
 }
 }
 SDValue Idx = DAG.getConstant(M, dl, MVT::i32);
-SDValue Ex = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ElemTy, {Vec, Idx});
+SDValue Ex = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, LegalTy, {Vec, Idx});
 SDValue L = Lower.LowerOperation(Ex, DAG);
 assert(L.getNode());
 Ops.push_back(L);
 }
 LV = Lower.LowerOperation(BV, DAG);
 }
 assert(!N->use_empty());
 ISel.ReplaceNode(N, LV.getNode());
-DAG.RemoveDeadNodes();
+if (AllNodes.count(LV.getNode())) {
-std::deque<SDNode*> SubNodes;
+DAG.RemoveDeadNodes();
-SubNodes.push_back(LV.getNode());
+return true;
+}
+// The lowered build-vector node will now need to be selected. It needs
+// to be done here because this node and its submodes are not included
+// in the main selection loop.
+// Implement essentially the same topological ordering algorithm as is
+// used in SelectionDAGISel.
+SetVector<SDNode*> SubNodes, TmpQ;
+std::map<SDNode*,unsigned> NumOps;
+SubNodes.insert(LV.getNode());
 for (unsigned I = 0; I != SubNodes.size(); ++I) {
-for (SDValue Op : SubNodes[I]->ops())
+unsigned OpN = 0;
-SubNodes.push_back(Op.getNode());
+SDNode *S = SubNodes[I];
-}
+for (SDValue Op : S->ops()) {
-while (!SubNodes.empty()) {
+if (AllNodes.count(Op.getNode()))
-SDNode *S = SubNodes.front();
+continue;
-SubNodes.pop_front();
+SubNodes.insert(Op.getNode());
-if (S->use_empty())
+++OpN;
-continue;
+}
-// This isn't great, but users need to be selected before any nodes that
+NumOps.insert({S, OpN});
-// they use. (The reason is to match larger patterns, and avoid nodes that
+if (OpN == 0)
-// cannot be matched on their own, e.g. ValueType, TokenFactor, etc.).
+TmpQ.insert(S);
-bool PendingUser = llvm::any_of(S->uses(), [&SubNodes](const SDNode *U) {
+}
-return llvm::any_of(SubNodes, [U](const SDNode *T) {
-return T == U;
+for (unsigned I = 0; I != TmpQ.size(); ++I) {
-});
+SDNode *S = TmpQ[I];
-});
+for (SDNode *U : S->uses()) {
-if (PendingUser)
+if (!SubNodes.count(U))
-SubNodes.push_back(S);
+continue;
-else
+auto F = NumOps.find(U);
+assert(F != NumOps.end());
+assert(F->second > 0);
+if (!--F->second)
+TmpQ.insert(F->first);
+}
+}
+assert(SubNodes.size() == TmpQ.size());
+NullifyingVector<decltype(TmpQ)::vector_type> Queue(TmpQ.takeVector());
+Deleter DUQ(DAG, Queue);
+for (SDNode *S : reverse(Queue))
+if (S != nullptr)
 ISel.Select(S);
-}
 DAG.RemoveDeadNodes();
 return true;
 }
 << HavePairs << '\n';
 });
 // If the mask is all -1's, generate "undef".
 if (!UseLeft && !UseRight) {
 ISel.ReplaceNode(N, ISel.selectUndef(SDLoc(SN), ResTy).getNode());
-DAG.RemoveDeadNode(N);
 return;
 }
 SDValue Vec0 = N->getOperand(0);
 SDValue Vec1 = N->getOperand(1);
 if (!NewN)
 NewN = DAG.getMachineNode(Hexagon::V6_vror, dl, Ty, {VecV, RotV});
 ISel.ReplaceNode(N, NewN);
+}
+void HvxSelector::selectVAlign(SDNode *N) {
+SDValue Vv = N->getOperand(0);
+SDValue Vu = N->getOperand(1);
+SDValue Rt = N->getOperand(2);
+SDNode *NewN = DAG.getMachineNode(Hexagon::V6_valignb, SDLoc(N),
+N->getValueType(0), {Vv, Vu, Rt});
+ISel.ReplaceNode(N, NewN);
 DAG.RemoveDeadNode(N);
 }
 void HexagonDAGToDAGISel::SelectHvxShuffle(SDNode *N) {
 HvxSelector(*this, *CurDAG).selectShuffle(N);
 }
 void HexagonDAGToDAGISel::SelectHvxRor(SDNode *N) {
 HvxSelector(*this, *CurDAG).selectRor(N);
+}
+void HexagonDAGToDAGISel::SelectHvxVAlign(SDNode *N) {
+HvxSelector(*this, *CurDAG).selectVAlign(N);
 }
 void HexagonDAGToDAGISel::SelectV65GatherPred(SDNode *N) {
 const SDLoc &dl(N);
 SDValue Chain = N->getOperand(0);
 SDVTList VTs = CurDAG->getVTList(MVT::Other);
 SDValue Ops[] = { Address, Predicate, Base, Modifier, Offset, Chain };
 SDNode *Result = CurDAG->getMachineNode(Opcode, dl, VTs, Ops);
-MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
+MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand();
-MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand();
+CurDAG->setNodeMemRefs(cast<MachineSDNode>(Result), {MemOp});
-cast<MachineSDNode>(Result)->setMemRefs(MemOp, MemOp + 1);
+ReplaceNode(N, Result);
-ReplaceUses(N, Result);
-CurDAG->RemoveDeadNode(N);
 }
 void HexagonDAGToDAGISel::SelectV65Gather(SDNode *N) {
 const SDLoc &dl(N);
 SDValue Chain = N->getOperand(0);
 SDVTList VTs = CurDAG->getVTList(MVT::Other);
 SDValue Ops[] = { Address, Base, Modifier, Offset, Chain };
 SDNode *Result = CurDAG->getMachineNode(Opcode, dl, VTs, Ops);
-MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
+MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand();
-MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand();
+CurDAG->setNodeMemRefs(cast<MachineSDNode>(Result), {MemOp});
-cast<MachineSDNode>(Result)->setMemRefs(MemOp, MemOp + 1);
+ReplaceNode(N, Result);
-ReplaceUses(N, Result);
-CurDAG->RemoveDeadNode(N);
 }
 void HexagonDAGToDAGISel::SelectHVXDualOutput(SDNode *N) {
 unsigned IID = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
 SDNode *Result;
 ReplaceUses(N, Result);
 ReplaceUses(SDValue(N, 0), SDValue(Result, 0));
 ReplaceUses(SDValue(N, 1), SDValue(Result, 1));
 CurDAG->RemoveDeadNode(N);
 }

Mercurial > hg > CbC > CbC_llvm

comparison lib/Target/Hexagon/HexagonISelDAGToDAGHVX.cpp @ 148:63bd29f05246