================
@@ -8781,8 +8958,483 @@ static bool combineCCMask(SDValue &CCReg, int &CCValid,
int &CCMask) {
return false;
}
-SDValue SystemZTargetLowering::combineBR_CCMASK(
- SDNode *N, DAGCombinerInfo &DCI) const {
+// Combine (select_ccmask_a (select_ccmask_b)), where select_ccmask_a has one
+// of TrueVal or FalseVal has nested select_ccmask_b.
+SDValue
+SystemZTargetLowering::combineSELECT_CC_CCIPMMask(SDNode *N,
+ DAGCombinerInfo &DCI) const {
+ SelectionDAG &DAG = DCI.DAG;
+
+ // Verify CCOp1 and CCOp2 refers to the same CC condition node.
+ const auto isSameCCIPMOp = [](SDValue &CCOp1, SDValue &CCOp2) {
+ SDNode *N1 = CCOp1.getNode(), *N2 = CCOp2.getNode();
+ return N1 && N2 && N1 == N2;
+ };
+ auto *OuterTrueVal = dyn_cast<ConstantSDNode>(N->getOperand(0));
+ auto *OuterFalseVal = dyn_cast<ConstantSDNode>(N->getOperand(1));
+
+ // Already handled the case both operands constant in combineCCMask.
+ // Not yet encountered the case where both operands are not constants,
+ // that case can be handled by removing this condition.
+ if (!((OuterTrueVal != nullptr) ^ (OuterFalseVal != nullptr)))
+ return SDValue();
+
+ SDValue NestedCCOp = OuterTrueVal ? N->getOperand(1) : N->getOperand(0);
+ auto *NestedCCNode = NestedCCOp.getNode();
+ if (!NestedCCNode || NestedCCNode->getOpcode() != SystemZISD::SELECT_CCMASK)
+ return SDValue();
+
+ auto *NestedTrueVal = dyn_cast<ConstantSDNode>(NestedCCNode->getOperand(0));
+ auto *NestedFalseVal = dyn_cast<ConstantSDNode>(NestedCCNode->getOperand(1));
+ if (!NestedTrueVal || !NestedFalseVal)
+ return SDValue();
+ bool Invert = false;
+ // Check if outer select_ccmask and nested select_ccmask True/False swapped
+ // between the two.
+ if (OuterTrueVal) {
+ // OuterFalseVal has nested select_ccmask.
+ if (OuterTrueVal->getZExtValue() == NestedFalseVal->getZExtValue())
+ Invert = !Invert;
+ else if (OuterTrueVal->getZExtValue() != NestedTrueVal->getZExtValue())
+ return SDValue();
+ } else if (OuterFalseVal) {
+ // OuterTrueVal has nested select_ccmask.
+ if (OuterFalseVal->getZExtValue() == NestedTrueVal->getZExtValue())
+ Invert = !Invert;
+ else if (OuterFalseVal->getZExtValue() != NestedFalseVal->getZExtValue())
+ return SDValue();
+ }
+ auto *OuterCCValid = dyn_cast<ConstantSDNode>(N->getOperand(2));
+ auto *OuterCCMask = dyn_cast<ConstantSDNode>(N->getOperand(3));
+ auto *NestedCCValid = dyn_cast<ConstantSDNode>(NestedCCOp->getOperand(2));
+ auto *NestedCCMask = dyn_cast<ConstantSDNode>(NestedCCOp->getOperand(3));
+ if (!OuterCCValid || !OuterCCMask || !NestedCCValid || !NestedCCMask)
+ return SDValue();
+
+ int OuterCCValidVal = OuterCCValid->getZExtValue();
+ int OuterCCMaskVal = OuterCCMask->getZExtValue();
+ int NestedCCMaskVal = NestedCCMask->getZExtValue();
+ int CCMask = OuterCCMaskVal;
+ SDValue OuterCCReg = N->getOperand(4);
+ SDValue NestedCCReg = NestedCCOp->getOperand(4);
+
+ // Try combining outer select_ccmask, and updated OuterCCReg points to the
+ // same CC as NestedCCReg CC.
+ if (OuterCCValidVal != SystemZ::CCMASK_ICMP ||
+ !combineCCMask(OuterCCReg, OuterCCValidVal, OuterCCMaskVal) ||
+ !isSameCCIPMOp(OuterCCReg, NestedCCReg))
+ return SDValue();
+
+ // Try to decipher if nested select_ccmask original CCMask was CCMASK_CMP_EQ.
+ // Only one-bit sets in NestedCCMaskVal indicates CCMASK_CMP_EQ.
+ bool IsNestedCMP_EQ =
+ NestedCCMaskVal && !(NestedCCMaskVal & (NestedCCMaskVal - 1));
+
+ // Outer and nested TrueVal and FalseVal swapped between the two.
+ if (Invert)
+ OuterCCMaskVal ^= SystemZ::CCMASK_ANY;
+
+ // Compute the effective CC mask for select.
+ // Generalizing into two categories.
+ // 1. If either of CCMask for outer or nested select_ccmask is
+ // SystemZ::CCMASK_CMP_EQ, any combination will compute to union of masks.
+ // 2.a. Both are SystemZ::CCMASK_CMP_NE, !(!a | !b) => (a & b)
+ // 2.b. Combinations of CCMask - SystemZ::CCMASK_CMP_LT or
+ // SystemZ::CCMASK_CMP_GT with !IsNestedCMP_EQ, overlap masks.
+ if (CCMask == SystemZ::CCMASK_CMP_EQ || IsNestedCMP_EQ)
+ OuterCCMaskVal |= NestedCCMaskVal;
+ else
+ OuterCCMaskVal &= NestedCCMaskVal;
+
+ // Get operands from nested select_ccmask.
+ SDValue Op0 = NestedCCOp->getOperand(0);
+ SDValue Op1 = NestedCCOp->getOperand(1);
+
+ // Return combined select_ccmask.
+ return DAG.getNode(
+ SystemZISD::SELECT_CCMASK, SDLoc(N), N->getValueType(0), Op0, Op1,
+ DAG.getTargetConstant(OuterCCValidVal, SDLoc(N), MVT::i32),
+ DAG.getTargetConstant(OuterCCMaskVal, SDLoc(N), MVT::i32), OuterCCReg);
+}
+
+// Merging versus split in multiple branches cost.
+TargetLoweringBase::CondMergingParams
+SystemZTargetLowering::getJumpConditionMergingParams(Instruction::BinaryOps
Opc,
+ const Value *Lhs,
+ const Value *Rhs) const {
+ const auto isFlagOutOpCC = [](const Value *V) {
+ using namespace llvm::PatternMatch;
+ const Value *RHSVal;
+ const APInt *RHSC;
+ if (const auto *I = dyn_cast<Instruction>(V)) {
+ // PatternMatch.h provides concise tree-based pattern match of llvm IR.
+ if (match(I->getOperand(0), m_And(m_Value(RHSVal), m_APInt(RHSC))) ||
+ match(I, m_Cmp(m_Value(RHSVal), m_APInt(RHSC)))) {
+ if (const auto *CB = dyn_cast<CallBase>(RHSVal)) {
+ if (CB->isInlineAsm()) {
+ const InlineAsm *IA = cast<InlineAsm>(CB->getCalledOperand());
+ return IA &&
+ IA->getConstraintString().find("{@cc}") !=
std::string::npos;
+ }
+ }
+ }
+ }
+ return false;
+ };
+ // Pattern (ICmp %asm) or (ICmp (And %asm)).
+ // Cost of longest dependency chain (ICmp, And) is 2. CostThreshold or
+ // BaseCost can be set >=2. If cost of instruction <= CostThreshold
+ // conditionals will be merged or else conditionals will be split.
+ if (isFlagOutOpCC(Lhs) && isFlagOutOpCC(Rhs))
+ return {3, 0, -1};
+ // Default.
+ return {-1, -1, -1};
+}
+
+SDValue SystemZTargetLowering::combineTM(SDNode *N,
+ DAGCombinerInfo &DCI) const {
+ SelectionDAG &DAG = DCI.DAG;
+ auto *TMOp0Node = N->getOperand(0).getNode();
+ auto *TMOp1Const = dyn_cast<ConstantSDNode>(N->getOperand(1));
+ auto *TMOp2Const = dyn_cast<ConstantSDNode>(N->getOperand(2));
+ // Third operand of TM is false.
+ if (!TMOp0Node || !TMOp1Const || !TMOp2Const ||
+ TMOp2Const->getZExtValue() != 0)
+ return SDValue();
+
+ auto TMOp1ConstVal = TMOp1Const->getZExtValue();
+ // Optimize (TM (IPM)).
+ if (TMOp0Node->getOpcode() == SystemZISD::IPM) {
+ int CCMask, CCValid;
+
+ // Compute the effective CC mask for select.
+ if (TMOp1ConstVal == (1 << SystemZ::IPM_CC))
+ CCMask = SystemZ::CCMASK_CMP_GE;
+ else if (TMOp1ConstVal == (1 << (SystemZ::IPM_CC + 1)))
+ CCMask = SystemZ::CCMASK_CMP_LE;
+ else
+ return SDValue();
+
+ // Set CCReg.
+ SDValue CCReg = TMOp0Node->getOperand(0);
+ CCValid = SystemZ::CCMASK_ANY;
+ // Return combined node.
+ return DAG.getNode(SystemZISD::SELECT_CCMASK, SDLoc(N), N->getValueType(0),
+ N->getOperand(0), N->getOperand(1),
+ DAG.getTargetConstant(CCValid, SDLoc(N), MVT::i32),
+ DAG.getTargetConstant(CCMask, SDLoc(N), MVT::i32),
+ CCReg);
+ }
+ // Optimize (TM (XOR (Op0 Op1))).
+ if (TMOp0Node->getOpcode() == ISD::XOR) {
+ auto *XorOp0 = TMOp0Node->getOperand(0).getNode();
+ // Op0: (SELECT_CCMASK (ICMP (SRL (IPM)))).
+ // Op1: (SRL (IPM (CC))).
+ if (XorOp0 && XorOp0->getOpcode() == SystemZISD::SELECT_CCMASK) {
+ auto *XorOp0CCValid = dyn_cast<ConstantSDNode>(XorOp0->getOperand(2));
+ auto *XorOp0CCMask = dyn_cast<ConstantSDNode>(XorOp0->getOperand(3));
+ if (!XorOp0CCValid || !XorOp0CCMask)
+ return SDValue();
+
+ SDValue XorOp0CCReg = XorOp0->getOperand(4);
+ int XorOp0CCMaskVal = XorOp0CCMask->getZExtValue();
+ int XorOp0CCValidVal = XorOp0CCValid->getZExtValue();
+ int CCMask = SystemZ::CCMASK_CMP_EQ, CCValid, TMCCMask;
+ SDValue CCReg = TMOp0Node->getOperand(1);
+
+ // Combine (SELECT_CCMASK (ICMP (SRL (IPM)))) and get CC.
+ if (!combineCCMask(XorOp0CCReg, XorOp0CCValidVal, XorOp0CCMaskVal) ||
+ // (SRL (IPM (CC))).
+ !combineCCMask(CCReg, CCValid, CCMask))
+ return SDValue();
+
+ // Op0 and Op1 should point to the same CC.
+ auto *N0 = XorOp0CCReg.getNode(), *N1 = CCReg.getNode();
+ if (!N0 || !N1 || N0 != N1)
+ return SDValue();
+ // Compute the effective CC mask for select.
+ if (TMOp1ConstVal == 1)
+ TMCCMask = SystemZ::CCMASK_CMP_GE;
+ else
+ return SDValue();
+ // CCMask ^ XorOp0CCMaskVal = TMCCMask
+ CCMask = XorOp0CCMaskVal ^ TMCCMask;
+
+ // Return combined node with combined CCMask.
+ return DAG.getNode(
+ SystemZISD::SELECT_CCMASK, SDLoc(N), N->getValueType(0),
+ XorOp0->getOperand(0), XorOp0->getOperand(1),
+ DAG.getTargetConstant(XorOp0CCValidVal, SDLoc(N), MVT::i32),
+ DAG.getTargetConstant(CCMask, SDLoc(N), MVT::i32), CCReg);
+ }
+ }
+ return SDValue();
+}
+
+SDValue SystemZTargetLowering::combineAND(SDNode *N,
+ DAGCombinerInfo &DCI) const {
+ SelectionDAG &DAG = DCI.DAG;
+
+ auto *AndOp0 = N->getOperand(0).getNode();
+ auto *AndOp1 = N->getOperand(1).getNode();
+ if (!AndOp0 || !AndOp1)
+ return SDValue();
+
+ // Both Operands of ISD::AND are SystemZISD::SELECT_CCMASK.
+ // And CCMask of both operands and they should point to the
+ // same CC and update CCReg. Return combined SDNode.
+ if (AndOp0->getOpcode() == SystemZISD::SELECT_CCMASK &&
+ AndOp1->getOpcode() == SystemZISD::SELECT_CCMASK) {
+ auto *Op0CCValid = dyn_cast<ConstantSDNode>(AndOp0->getOperand(2));
+ auto *Op0CCMask = dyn_cast<ConstantSDNode>(AndOp0->getOperand(3));
+ auto *Op1CCValid = dyn_cast<ConstantSDNode>(AndOp1->getOperand(2));
+ auto *Op1CCMask = dyn_cast<ConstantSDNode>(AndOp1->getOperand(3));
+ if (!Op0CCValid || !Op1CCValid || !Op0CCMask || !Op1CCMask)
+ return SDValue();
+
+ int Op0CCValidVal = Op0CCValid->getZExtValue();
+ int Op1CCValidVal = Op1CCValid->getZExtValue();
+ if (Op0CCValidVal != Op1CCValidVal)
+ return SDValue();
+
+ // AndOp0 and AndOp1 should refer to same CC.
+ SDValue Op0CCReg = AndOp0->getOperand(4), Op1CCReg = AndOp1->getOperand(4);
+ auto *CCNode0 = Op0CCReg.getNode(), *CCNode1 = Op1CCReg.getNode();
+ if (!CCNode0 || !CCNode1 || CCNode0 != CCNode1)
+ return SDValue();
+
+ // Optimizing only the case where Op0TrueVal and Op1TrueVal are equal
+ // and at the same time Op0FalseVal and Op1FalseVal are also equal.
+ auto *Op0TrueVal = dyn_cast<ConstantSDNode>(AndOp0->getOperand(0));
+ auto *Op0FalseVal = dyn_cast<ConstantSDNode>(AndOp0->getOperand(1));
+ auto *Op1TrueVal = dyn_cast<ConstantSDNode>(AndOp1->getOperand(0));
+ auto *Op1FalseVal = dyn_cast<ConstantSDNode>(AndOp1->getOperand(1));
+ if (!Op0TrueVal || !Op0FalseVal || !Op1TrueVal || !Op1FalseVal)
+ return SDValue();
+ if (Op0TrueVal->getZExtValue() != Op1TrueVal->getZExtValue() ||
+ Op0FalseVal->getZExtValue() != Op1FalseVal->getZExtValue())
+ return SDValue();
+
+ // Compute the effective CC mask for select.
+ int Op0CCMaskVal = Op0CCMask->getZExtValue();
+ int Op1CCMaskVal = Op1CCMask->getZExtValue();
+ int CCMask = Op0CCMaskVal & Op1CCMaskVal;
----------------
uweigand wrote:
I'm not seeing any changes here, so this still looks incorrect.
https://github.com/llvm/llvm-project/pull/125970
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
