================
@@ -8781,8 +8961,445 @@ static bool combineCCMask(SDValue &CCReg, int &CCValid,
int &CCMask) {
return false;
}
-SDValue SystemZTargetLowering::combineBR_CCMASK(
- SDNode *N, DAGCombinerInfo &DCI) const {
+// Combine (select_cc_a (select_cc_b)), where select_cc_a has one of TrueVal
+// or FalseVal has nested select_cc_b(already been combined sequence)
+SDValue
+SystemZTargetLowering::combineSELECT_CC_CCIPMMask(SDNode *N,
+ DAGCombinerInfo &DCI) const {
+ SelectionDAG &DAG = DCI.DAG;
+ // Check if CCOp1 and CCOp2 refers to the same CC condition node.
+ const auto isSameCCIPMOp = [](SDValue &CCOp1, SDValue &CCOp2,
+ int &CCValidVal) {
+ // Already combined sequence.
+ if (CCValidVal != SystemZ::CCMASK_ANY)
+ return false;
+ SDNode *N1 = CCOp1.getNode(), *N2 = CCOp2.getNode();
+ return N1 && N2 && N1 == N2;
+ };
+
+ auto *OuterTrueVal = dyn_cast<ConstantSDNode>(N->getOperand(0));
+ auto *OuterFalseVal = dyn_cast<ConstantSDNode>(N->getOperand(1));
+
+ // Already handled the case both operands constant in combineCCMask.
+ // Not yet encountered the case where both operands are sub-expressions,
+ // that case can be handled by removing this condition.
+ if (!((OuterTrueVal != nullptr) ^ (OuterFalseVal != nullptr)))
+ return SDValue();
+
+ SDValue NestedCCOp = OuterTrueVal ? N->getOperand(1) : N->getOperand(0);
+ auto *NestedCCNode = NestedCCOp.getNode();
+ // check if nested select_cc_b has already been combined.
+ if (!NestedCCNode || NestedCCNode->getOpcode() != SystemZISD::SELECT_CCMASK)
+ return SDValue();
+
+ auto *NestedTrueVal = dyn_cast<ConstantSDNode>(NestedCCNode->getOperand(0));
+ auto *NestedFalseVal = dyn_cast<ConstantSDNode>(NestedCCNode->getOperand(1));
+ if (!NestedTrueVal || !NestedFalseVal)
+ return SDValue();
+ bool Invert = false;
+ // Check if outer select_cc_a and nested select_cc_b True/False matching
+ // or inverted.
+ if (OuterTrueVal) {
+ // OuterFalseVal points to already combined nested select_cc_b.
+ if (OuterTrueVal->getZExtValue() == NestedFalseVal->getZExtValue())
+ Invert = !Invert; // Inverted.
+ else if (OuterTrueVal->getZExtValue() != NestedTrueVal->getZExtValue())
+ return SDValue();
+ } else if (OuterFalseVal) {
+ // OuterTrueVal points to already combined nested select_cc_b.
+ if (OuterFalseVal->getZExtValue() == NestedTrueVal->getZExtValue())
+ Invert = !Invert; // Inverted.
+ else if (OuterFalseVal->getZExtValue() != NestedFalseVal->getZExtValue())
+ return SDValue();
+ }
+ auto *OuterCCValid = dyn_cast<ConstantSDNode>(N->getOperand(2));
+ auto *OuterCCMask = dyn_cast<ConstantSDNode>(N->getOperand(3));
+ auto *NestedCCValid = dyn_cast<ConstantSDNode>(NestedCCOp->getOperand(2));
+ auto *NestedCCMask = dyn_cast<ConstantSDNode>(NestedCCOp->getOperand(3));
+ if (!OuterCCValid || !OuterCCMask || !NestedCCValid || !NestedCCMask)
+ return SDValue();
+
+ int OuterCCValidVal = OuterCCValid->getZExtValue();
+ int OuterCCMaskVal = OuterCCMask->getZExtValue();
+ int NestedCCValidVal = NestedCCValid->getZExtValue();
+ int NestedCCMaskVal = NestedCCMask->getZExtValue();
+ int CCMask = OuterCCMaskVal;
+ SDValue OuterCCReg = N->getOperand(4);
+ SDValue NestedCCReg = NestedCCOp->getOperand(4);
+
+ // Combine two already combined (select_cc_a (select_cc_b)), where TrueVal
+ // of select_cc_a points to select_cc_b. We return select_cc with TrueVal
+ // and FalseVal from select_cc_b with combined CCMask.
+ // One of OuterTrueVal or OuterFalseVal has select_cc_b.
+ if ((OuterTrueVal != nullptr) ^ (OuterFalseVal != nullptr)) {
+ // Both OuterCCValidVal and NestedCCValidVal have already been combined.
+ if (OuterCCValidVal == SystemZ::CCMASK_ANY &&
+ // And both points to the same CC.
+ isSameCCIPMOp(OuterCCReg, NestedCCReg, NestedCCValidVal)) {
+ CCMask |= NestedCCMaskVal;
+ // NestedCCOp has both operands constants.
+ auto Op0 = NestedCCOp->getOperand(0);
+ auto Op1 = NestedCCOp->getOperand(1);
+ // Return combined select_cc.
+ return DAG.getNode(
+ SystemZISD::SELECT_CCMASK, SDLoc(N), N->getValueType(0), Op0, Op1,
+ DAG.getTargetConstant(OuterCCValidVal, SDLoc(N), MVT::i32),
+ DAG.getTargetConstant(CCMask, SDLoc(N), MVT::i32), NestedCCReg);
+ }
+ }
+ // Now handles the case where outer select_cc_a has not yet been combined.
+ // Combine outer select_cc and check if it corresponds to the same
+ // CC as nested CC.
+ // Outer select_cc has yet not been combined.
+ if (OuterCCValidVal != SystemZ::CCMASK_ICMP ||
+ // Try combining outer select_cc.
+ !combineCCMask(OuterCCReg, OuterCCValidVal, OuterCCMaskVal) ||
+ // Check nested select_cc has already been combined and points to
+ // same Condtiion code as outer select_cc.
+ !isSameCCIPMOp(OuterCCReg, NestedCCReg, NestedCCValidVal))
+ return SDValue();
+ // Check if nested select_cc original CCMask was CCMASK_CMP_EQ.
+ // Only one-bit is set in NestedCCMaskVal for CCMASK_CMP_EQ.
+ bool IsNestedCMP_EQ =
+ NestedCCMaskVal && !(NestedCCMaskVal & (NestedCCMaskVal - 1));
+
+ // Outer select_cc is inverted with respect to nested select_cc.
+ if (Invert)
+ OuterCCMaskVal ^= SystemZ::CCMASK_ANY;
+
+ // Intersection of masks.
+ if (CCMask != SystemZ::CCMASK_CMP_EQ && !IsNestedCMP_EQ)
+ OuterCCMaskVal &= NestedCCMaskVal;
+ else // Union the masks.
+ OuterCCMaskVal |= NestedCCMaskVal;
+
+ SDValue Op0 = NestedCCOp->getOperand(0);
+ SDValue Op1 = NestedCCOp->getOperand(1);
+
+ return DAG.getNode(
+ SystemZISD::SELECT_CCMASK, SDLoc(N), N->getValueType(0), Op0, Op1,
+ DAG.getTargetConstant(OuterCCValidVal, SDLoc(N), MVT::i32),
+ DAG.getTargetConstant(OuterCCMaskVal, SDLoc(N), MVT::i32), OuterCCReg);
+}
+
+// Merging versus split in multiple branches cost.
+TargetLoweringBase::CondMergingParams
+SystemZTargetLowering::getJumpConditionMergingParams(Instruction::BinaryOps
Opc,
+ const Value *Lhs,
+ const Value *Rhs) const {
+ const auto isFlagOutOpCC = [](const Value *V) {
+ using namespace llvm::PatternMatch;
+ const Value *RHSVal;
+ const APInt *RHSC;
+ if (const auto *I = dyn_cast<Instruction>(V)) {
+ // PatternMatch.h provides concise tree-based pattern match of llvm IR.
+ if (match(I->getOperand(0), m_And(m_Value(RHSVal), m_APInt(RHSC))) ||
+ match(I, m_Cmp(m_Value(RHSVal), m_APInt(RHSC)))) {
+ if (const auto *CB = dyn_cast<CallBase>(RHSVal)) {
+ if (CB->isInlineAsm()) {
+ const InlineAsm *IA = cast<InlineAsm>(CB->getCalledOperand());
+ return IA &&
+ IA->getConstraintString().find("{@cc}") !=
std::string::npos;
+ }
+ }
+ }
+ }
+ return false;
+ };
+ // Pattern (ICmp %asm) or (ICmp (And %asm)).
+ // Cost of longest dependency chain (ICmp, And) is 2. CostThreshold or
+ // BaseCost can be set >=2. If cost of instruction <= CostThreshold
+ // conditionals will be merged or else conditionals will be split.
+ if (isFlagOutOpCC(Lhs) && isFlagOutOpCC(Rhs))
+ return {3, 0, -1};
+ // Default.
+ return {-1, -1, -1};
+}
+
+SDValue SystemZTargetLowering::combineTM(SDNode *N,
+ DAGCombinerInfo &DCI) const {
+ SelectionDAG &DAG = DCI.DAG;
+ auto *TMOp0Node = N->getOperand(0).getNode();
+ auto *TMOp1Const = dyn_cast<ConstantSDNode>(N->getOperand(1));
+ auto *TMOp2Const = dyn_cast<ConstantSDNode>(N->getOperand(2));
+ if (!TMOp0Node || !TMOp1Const || !TMOp2Const ||
+ // Third operand of TM is false.
+ TMOp2Const->getZExtValue() != 0)
+ return SDValue();
+ auto TMOp1ConstVal = TMOp1Const->getZExtValue();
+ // Optimize (TM (IPM)).
+ if (TMOp0Node->getOpcode() == SystemZISD::IPM) {
+ int CCMask, CCValid;
+ if (TMOp1ConstVal == (1 << SystemZ::IPM_CC))
+ CCMask = SystemZ::CCMASK_CMP_GE;
+ else if (TMOp1ConstVal == (1 << (SystemZ::IPM_CC + 1)))
+ CCMask = SystemZ::CCMASK_CMP_LE;
+ else
+ return SDValue();
+ SDValue CCReg = TMOp0Node->getOperand(0);
+ CCValid = SystemZ::CCMASK_ANY;
+ // Return combined node.
+ return DAG.getNode(SystemZISD::SELECT_CCMASK, SDLoc(N), N->getValueType(0),
+ N->getOperand(0), N->getOperand(1),
+ DAG.getTargetConstant(CCValid, SDLoc(N), MVT::i32),
+ DAG.getTargetConstant(CCMask, SDLoc(N), MVT::i32),
+ CCReg);
+ }
+ // Optimize (TM (XOR (Op0 Op1))).
+ if (TMOp0Node->getOpcode() == ISD::XOR) {
+ auto *XorOp0 = TMOp0Node->getOperand(0).getNode();
+ // Op0. (SELECT_CCMASK (ICMP (SRL (IPM)))).
+ // Op1. (SRL (IPM (CC))).
+ if (XorOp0 && XorOp0->getOpcode() == SystemZISD::SELECT_CCMASK) {
+ auto *XorOp0CCValid = dyn_cast<ConstantSDNode>(XorOp0->getOperand(2));
+ auto *XorOp0CCMask = dyn_cast<ConstantSDNode>(XorOp0->getOperand(3));
+ if (!XorOp0CCValid || !XorOp0CCMask)
+ return SDValue();
+ SDValue XorOp0CCReg = XorOp0->getOperand(4);
+ int XorOp0CCMaskVal = XorOp0CCMask->getZExtValue();
+ int XorOp0CCValidVal = XorOp0CCValid->getZExtValue();
+ int CCMask = SystemZ::CCMASK_CMP_EQ, CCValid, TMCCMask;
+ SDValue CCReg = TMOp0Node->getOperand(1);
+ // (SELECT_CCMASK (ICMP (SRL (IPM)))).
+ if (!combineCCMask(XorOp0CCReg, XorOp0CCValidVal, XorOp0CCMaskVal) ||
+ // (SRL (IPM (CC))).
+ !combineCCMask(CCReg, CCValid, CCMask))
+ return SDValue();
+ auto *N0 = XorOp0CCReg.getNode(), *N1 = CCReg.getNode();
+ // Check if Op0 and Op1 point to the same CC.
+ if (!N0 || !N1 || N0 != N1)
+ return SDValue();
+ if (TMOp1ConstVal == 1)
+ TMCCMask = SystemZ::CCMASK_CMP_GE;
+ else
+ return SDValue();
+ // CCMask ^ XorOp0CCMaskVal = TMCCMask..
+ CCMask = XorOp0CCMaskVal ^ TMCCMask;
+ // Returned combined node with evaluated CCMask.
+ return DAG.getNode(
+ SystemZISD::SELECT_CCMASK, SDLoc(N), N->getValueType(0),
+ XorOp0->getOperand(0), XorOp0->getOperand(1),
+ DAG.getTargetConstant(XorOp0CCValidVal, SDLoc(N), MVT::i32),
+ DAG.getTargetConstant(CCMask, SDLoc(N), MVT::i32), CCReg);
+ }
+ }
+ return SDValue();
+}
+
+SDValue SystemZTargetLowering::combineAND(SDNode *N,
+ DAGCombinerInfo &DCI) const {
+ SelectionDAG &DAG = DCI.DAG;
+ auto *AndOp0 = N->getOperand(0).getNode();
+ auto *AndOp1 = N->getOperand(1).getNode();
+ if (!AndOp0 || !AndOp1)
+ return SDValue();
+ // Both Operands of ISD::AND are SystemZISD::SELECT_CCMASK.
+ // And CCMask of both operands and check if they points to the
+ // same CC and update CCReg. Return combined SDNode.
+ if (AndOp0->getOpcode() == SystemZISD::SELECT_CCMASK &&
+ AndOp1->getOpcode() == SystemZISD::SELECT_CCMASK) {
+ auto *Op0CCValid = dyn_cast<ConstantSDNode>(AndOp0->getOperand(2));
+ auto *Op0CCMask = dyn_cast<ConstantSDNode>(AndOp0->getOperand(3));
+ auto *Op1CCValid = dyn_cast<ConstantSDNode>(AndOp1->getOperand(2));
+ auto *Op1CCMask = dyn_cast<ConstantSDNode>(AndOp1->getOperand(3));
+ if (!Op0CCValid || !Op1CCValid || !Op0CCMask || !Op1CCMask)
+ return SDValue();
+ int Op0CCValidVal = Op0CCValid->getZExtValue();
+ int Op1CCValidVal = Op1CCValid->getZExtValue();
+ // Check if both AndOp0 and AndOp1 have aleady been combined.
+ if (Op0CCValidVal != SystemZ::CCMASK_ANY ||
+ Op1CCValidVal != SystemZ::CCMASK_ANY)
+ return SDValue();
+ SDValue Op0CCReg = AndOp0->getOperand(4), Op1CCReg = AndOp1->getOperand(4);
+ auto *CCNode0 = Op0CCReg.getNode(), *CCNode1 = Op1CCReg.getNode();
+ // Check if AndOp0 and AndOp1 refers to same CC.
+ if (!CCNode0 || !CCNode1 || CCNode0 != CCNode1)
+ return SDValue();
+ int Op0CCMaskVal = Op0CCMask->getZExtValue();
+ int Op1CCMaskVal = Op1CCMask->getZExtValue();
+ int CCMask = Op0CCMaskVal & Op1CCMaskVal;
+ return DAG.getNode(SystemZISD::SELECT_CCMASK, SDLoc(N), N->getValueType(0),
+ AndOp0->getOperand(0), AndOp0->getOperand(1),
+ DAG.getTargetConstant(Op0CCValidVal, SDLoc(N),
MVT::i32),
+ DAG.getTargetConstant(CCMask, SDLoc(N), MVT::i32),
+ Op0CCReg);
+ } else if (AndOp0->getOpcode() == SystemZISD::SELECT_CCMASK) {
+ // check AndOp1 for (SRL (IPM (CC))) pattern. Second operand is CC.
+ SDValue CCReg = N->getOperand(1);
+ int CCMask = SystemZ::CCMASK_CMP_EQ, CCValid;
+ if (!combineCCMask(CCReg, CCValid, CCMask))
+ return SDValue();
+ SDValue Op0CCReg = AndOp0->getOperand(4);
+ auto *CCNode0 = Op0CCReg.getNode(), *CCNode1 = CCReg.getNode();
+ // Check if AndOp0 and AndOp1 refers to same CC.
+ if (!CCNode0 || !CCNode1 || CCNode0 != CCNode1)
+ return SDValue();
+ auto *Op0CCValid = dyn_cast<ConstantSDNode>(AndOp0->getOperand(2));
+ auto *Op0CCMask = dyn_cast<ConstantSDNode>(AndOp0->getOperand(3));
+ int Op0CCValidVal = Op0CCValid->getZExtValue();
+ if (!Op0CCMask || !Op0CCValid || Op0CCValidVal != SystemZ::CCMASK_ANY)
+ return SDValue();
+ int Op0CCMaskVal = Op0CCMask->getZExtValue();
+ // Op0CCMaskVal & CCMask = 0. Invert Op0CCMaskVal.
+ int CCMaskVal = Op0CCMaskVal ^ 0xf;
+ assert(CCMaskVal < 4 && "CC out of range");
+ CCMask = 1 << (3 - CCMaskVal);
+ return DAG.getNode(SystemZISD::SELECT_CCMASK, SDLoc(N), N->getValueType(0),
+ AndOp0->getOperand(0), AndOp0->getOperand(1),
+ DAG.getTargetConstant(Op0CCValidVal, SDLoc(N),
MVT::i32),
+ DAG.getTargetConstant(CCMask, SDLoc(N), MVT::i32),
+ CCReg);
+ }
+ return SDValue();
+}
+
+SDValue SystemZTargetLowering::combineOR(SDNode *N,
+ DAGCombinerInfo &DCI) const {
+ SelectionDAG &DAG = DCI.DAG;
+ auto *OrOp0 = N->getOperand(0).getNode();
+ auto *OrOp1 = N->getOperand(1).getNode();
+ if (!OrOp0 || !OrOp1)
+ return SDValue();
+ // Both Operands of ISD::OR are SystemZISD::SELECT_CCMASK.
+ // And CCMask of both operands and check if they points to the
+ // same CC and update CCReg. Return combined SDNode.
+ if (OrOp0->getOpcode() == SystemZISD::SELECT_CCMASK &&
+ OrOp1->getOpcode() == SystemZISD::SELECT_CCMASK) {
+ auto *Op0CCValid = dyn_cast<ConstantSDNode>(OrOp0->getOperand(2));
+ auto *Op0CCMask = dyn_cast<ConstantSDNode>(OrOp0->getOperand(3));
+ auto *Op1CCValid = dyn_cast<ConstantSDNode>(OrOp1->getOperand(2));
+ auto *Op1CCMask = dyn_cast<ConstantSDNode>(OrOp1->getOperand(3));
+ if (!Op0CCValid || !Op1CCValid || !Op0CCMask || !Op1CCMask)
+ return SDValue();
+ int Op0CCValidVal = Op0CCValid->getZExtValue();
+ int Op1CCValidVal = Op1CCValid->getZExtValue();
+ // Check if both OrOp0 and OrOp1 have aleady been combined.
+ if (Op0CCValidVal != SystemZ::CCMASK_ANY ||
+ Op1CCValidVal != SystemZ::CCMASK_ANY)
+ return SDValue();
----------------
uweigand wrote:
Why is this check needed? The transformation is just as valid if CCValid is
not CCMASK_ANY. Of course, the two valid masks must be equal (which they
should always be as long as the CC nodes are the same, but it should still be
verified).
https://github.com/llvm/llvm-project/pull/125970
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
