================
@@ -8781,8 +8981,499 @@ static bool combineCCMask(SDValue &CCReg, int &CCValid,
int &CCMask) {
return false;
}
-SDValue SystemZTargetLowering::combineBR_CCMASK(
- SDNode *N, DAGCombinerInfo &DCI) const {
+// Combine (select_ccmask_a (select_ccmask_b)), where select_ccmask_a has one
+// of TrueVal or FalseVal has nested select_ccmask_b.
+SDValue
+SystemZTargetLowering::combineSELECT_CC_CCIPMMask(SDNode *N,
+ DAGCombinerInfo &DCI) const {
+ SelectionDAG &DAG = DCI.DAG;
+
+ // Verify CCOp1 and CCOp2 refers to the same CC condition node.
+ const auto isSameCCIPMOp = [](SDValue &CCOp1, SDValue &CCOp2) {
+ SDNode *N1 = CCOp1.getNode(), *N2 = CCOp2.getNode();
+ return N1 && N2 && N1 == N2;
+ };
+ auto *OuterTrueVal = dyn_cast<ConstantSDNode>(N->getOperand(0));
+ auto *OuterFalseVal = dyn_cast<ConstantSDNode>(N->getOperand(1));
+
+ // Already handled the case both operands constant in combineCCMask.
+ // Not yet encountered the case where both operands are not constants,
+ // that case can be handled by removing this condition.
+ if (!((OuterTrueVal != nullptr) ^ (OuterFalseVal != nullptr)))
+ return SDValue();
+
+ SDValue NestedCCOp = OuterTrueVal ? N->getOperand(1) : N->getOperand(0);
+ auto *NestedCCNode = NestedCCOp.getNode();
+ if (!NestedCCNode || NestedCCNode->getOpcode() != SystemZISD::SELECT_CCMASK)
+ return SDValue();
+
+ auto *NestedTrueVal = dyn_cast<ConstantSDNode>(NestedCCNode->getOperand(0));
+ auto *NestedFalseVal = dyn_cast<ConstantSDNode>(NestedCCNode->getOperand(1));
+ if (!NestedTrueVal || !NestedFalseVal)
+ return SDValue();
+ bool Invert = false;
+ // Check if outer select_ccmask and nested select_ccmask True/False swapped
+ // between the two.
+ if (OuterTrueVal) {
+ // OuterFalseVal has nested select_ccmask.
+ if (OuterTrueVal->getZExtValue() == NestedFalseVal->getZExtValue())
+ Invert = !Invert;
+ else if (OuterTrueVal->getZExtValue() != NestedTrueVal->getZExtValue())
+ return SDValue();
+ } else if (OuterFalseVal) {
+ // OuterTrueVal has nested select_ccmask.
+ if (OuterFalseVal->getZExtValue() == NestedTrueVal->getZExtValue())
+ Invert = !Invert;
+ else if (OuterFalseVal->getZExtValue() != NestedFalseVal->getZExtValue())
+ return SDValue();
+ }
+ auto *OuterCCValid = dyn_cast<ConstantSDNode>(N->getOperand(2));
+ auto *OuterCCMask = dyn_cast<ConstantSDNode>(N->getOperand(3));
+ auto *NestedCCValid = dyn_cast<ConstantSDNode>(NestedCCOp->getOperand(2));
+ auto *NestedCCMask = dyn_cast<ConstantSDNode>(NestedCCOp->getOperand(3));
+ if (!OuterCCValid || !OuterCCMask || !NestedCCValid || !NestedCCMask)
+ return SDValue();
+
+ int OuterCCValidVal = OuterCCValid->getZExtValue();
+ int OuterCCMaskVal = OuterCCMask->getZExtValue();
+ int NestedCCMaskVal = NestedCCMask->getZExtValue();
+ int CCMask = OuterCCMaskVal;
+ SDValue OuterCCReg = N->getOperand(4);
+ SDValue NestedCCReg = NestedCCOp->getOperand(4);
+
+ // Try combining outer select_ccmask, and updated OuterCCReg points to the
+ // same CC as NestedCCReg CC.
+ if (OuterCCValidVal != SystemZ::CCMASK_ICMP ||
+ !combineCCMask(OuterCCReg, OuterCCValidVal, OuterCCMaskVal) ||
+ !isSameCCIPMOp(OuterCCReg, NestedCCReg))
+ return SDValue();
+
+ // Try to decipher if nested select_ccmask original CCMask was CCMASK_CMP_EQ.
+ // Only one-bit sets in NestedCCMaskVal indicates CCMASK_CMP_EQ.
+ bool IsNestedCMP_EQ =
+ NestedCCMaskVal && !(NestedCCMaskVal & (NestedCCMaskVal - 1));
+
+ // Outer and nested TrueVal and FalseVal swapped between the two.
+ if (Invert)
+ OuterCCMaskVal ^= SystemZ::CCMASK_ANY;
+
+ // Compute the effective CC mask for select.
+ // Generalizing into two categories.
+ // 1. If either of CCMask for outer or nested select_ccmask is
+ // SystemZ::CCMASK_CMP_EQ, any combination will compute to union of masks.
+ // 2.a. Both are SystemZ::CCMASK_CMP_NE, !(!a | !b) => (a & b)
+ // 2.b. Combinations of CCMask - SystemZ::CCMASK_CMP_LT or
+ // SystemZ::CCMASK_CMP_GT with !IsNestedCMP_EQ, overlap masks.
+ if (CCMask == SystemZ::CCMASK_CMP_EQ || IsNestedCMP_EQ)
+ OuterCCMaskVal |= NestedCCMaskVal;
+ else
+ OuterCCMaskVal &= NestedCCMaskVal;
+
+ // Get operands from nested select_ccmask.
+ SDValue Op0 = NestedCCOp->getOperand(0);
+ SDValue Op1 = NestedCCOp->getOperand(1);
+
+ // Return combined select_ccmask.
+ return DAG.getNode(
+ SystemZISD::SELECT_CCMASK, SDLoc(N), N->getValueType(0), Op0, Op1,
+ DAG.getTargetConstant(OuterCCValidVal, SDLoc(N), MVT::i32),
+ DAG.getTargetConstant(OuterCCMaskVal, SDLoc(N), MVT::i32), OuterCCReg);
+}
+
+// Merging versus split in multiple branches cost.
+TargetLoweringBase::CondMergingParams
+SystemZTargetLowering::getJumpConditionMergingParams(Instruction::BinaryOps
Opc,
+ const Value *Lhs,
+ const Value *Rhs) const {
+ const auto isFlagOutOpCC = [](const Value *V) {
+ using namespace llvm::PatternMatch;
+ const Value *RHSVal;
+ const APInt *RHSC;
+ if (const auto *I = dyn_cast<Instruction>(V)) {
+ // PatternMatch.h provides concise tree-based pattern match of llvm IR.
+ if (match(I->getOperand(0), m_And(m_Value(RHSVal), m_APInt(RHSC))) ||
+ match(I, m_Cmp(m_Value(RHSVal), m_APInt(RHSC)))) {
+ if (const auto *CB = dyn_cast<CallBase>(RHSVal)) {
+ if (CB->isInlineAsm()) {
+ const InlineAsm *IA = cast<InlineAsm>(CB->getCalledOperand());
+ return IA &&
+ IA->getConstraintString().find("{@cc}") !=
std::string::npos;
+ }
+ }
+ }
+ }
+ return false;
+ };
+ // Pattern (ICmp %asm) or (ICmp (And %asm)).
+ // Cost of longest dependency chain (ICmp, And) is 2. CostThreshold or
+ // BaseCost can be set >=2. If cost of instruction <= CostThreshold
+ // conditionals will be merged or else conditionals will be split.
+ if (isFlagOutOpCC(Lhs) && isFlagOutOpCC(Rhs))
+ return {3, 0, -1};
+ // Default.
+ return {-1, -1, -1};
+}
+
+SDValue SystemZTargetLowering::combineAND(SDNode *N,
+ DAGCombinerInfo &DCI) const {
+ SelectionDAG &DAG = DCI.DAG;
+
+ auto *AndOp0 = N->getOperand(0).getNode();
+ auto *AndOp1 = N->getOperand(1).getNode();
+ if (!AndOp0 || !AndOp1)
+ return SDValue();
+
+ // Both Operands of ISD::AND are SystemZISD::SELECT_CCMASK.
+ // And CCMask of both operands and they should point to the
+ // same CC and update CCReg. Return combined SDNode.
+ if (AndOp0->getOpcode() == SystemZISD::SELECT_CCMASK &&
+ AndOp1->getOpcode() == SystemZISD::SELECT_CCMASK) {
+ auto *Op0CCValid = dyn_cast<ConstantSDNode>(AndOp0->getOperand(2));
+ auto *Op0CCMask = dyn_cast<ConstantSDNode>(AndOp0->getOperand(3));
+ auto *Op1CCValid = dyn_cast<ConstantSDNode>(AndOp1->getOperand(2));
+ auto *Op1CCMask = dyn_cast<ConstantSDNode>(AndOp1->getOperand(3));
+ if (!Op0CCValid || !Op1CCValid || !Op0CCMask || !Op1CCMask)
+ return SDValue();
+
+ int Op0CCValidVal = Op0CCValid->getZExtValue();
+ int Op1CCValidVal = Op1CCValid->getZExtValue();
+ if (Op0CCValidVal != Op1CCValidVal)
+ return SDValue();
+
+ // AndOp0 and AndOp1 should refer to same CC.
+ SDValue Op0CCReg = AndOp0->getOperand(4), Op1CCReg = AndOp1->getOperand(4);
+ auto *CCNode0 = Op0CCReg.getNode(), *CCNode1 = Op1CCReg.getNode();
+ if (!CCNode0 || !CCNode1 || CCNode0 != CCNode1)
+ return SDValue();
+
+ // Optimizing only the case where Op0TrueVal and Op1TrueVal are equal
+ // and at the same time Op0FalseVal and Op1FalseVal are also equal.
+ auto *Op0TrueVal = dyn_cast<ConstantSDNode>(AndOp0->getOperand(0));
+ auto *Op0FalseVal = dyn_cast<ConstantSDNode>(AndOp0->getOperand(1));
+ auto *Op1TrueVal = dyn_cast<ConstantSDNode>(AndOp1->getOperand(0));
+ auto *Op1FalseVal = dyn_cast<ConstantSDNode>(AndOp1->getOperand(1));
+ if (!Op0TrueVal || !Op0FalseVal || !Op1TrueVal || !Op1FalseVal)
+ return SDValue();
+ if (Op0TrueVal->getZExtValue() != Op1TrueVal->getZExtValue() ||
+ Op0FalseVal->getZExtValue() != Op1FalseVal->getZExtValue())
+ return SDValue();
+ if (Op0TrueVal->getZExtValue() == Op1FalseVal->getZExtValue() ||
+ Op1TrueVal->getZExtValue() == Op0FalseVal->getZExtValue())
+ return SDValue();
+
+ // Compute the effective CC mask for select.
+ int Op0CCMaskVal = Op0CCMask->getZExtValue();
+ int Op1CCMaskVal = Op1CCMask->getZExtValue();
+ int CCMask = Op0CCMaskVal & Op1CCMaskVal;
+
+ // Check And's user's user if it it select_ccmask or br_ccmask, put it
+ // back to WorkList so that algortirm can start processing from outer
+ // select_ccmask/br_ccmask and combine with (icmp (select_ccmask)).
+ for (SDUse &IcmpUse : N->uses()) {
+ auto *Icmp = IcmpUse.getUser();
+ if (Icmp && Icmp->getOpcode() == SystemZISD::ICMP) {
+ for (SDUse &SelectBrUse : Icmp->uses()) {
+ auto *SelectBr = SelectBrUse.getUser();
+ if (SelectBr && (SelectBr->getOpcode() == SystemZISD::SELECT_CCMASK
||
+ SelectBr->getOpcode() == SystemZISD::BR_CCMASK))
+ DCI.AddToWorklist(SelectBr);
+ }
+ }
+ }
+ return DAG.getNode(SystemZISD::SELECT_CCMASK, SDLoc(N), N->getValueType(0),
+ AndOp0->getOperand(0), AndOp0->getOperand(1),
+ DAG.getTargetConstant(Op0CCValidVal, SDLoc(N),
MVT::i32),
+ DAG.getTargetConstant(CCMask, SDLoc(N), MVT::i32),
+ Op0CCReg);
+ } else if (AndOp0->getOpcode() == SystemZISD::SELECT_CCMASK) {
+ // AndOp1: (SRL (IPM)).
+ if (AndOp1->getOpcode() != ISD::SRL)
+ return SDValue();
+ auto *SRLCount = dyn_cast<ConstantSDNode>(AndOp1->getOperand(1));
+ if (!SRLCount || SRLCount->getZExtValue() != SystemZ::IPM_CC)
+ return SDValue();
+ auto *IPM = AndOp1->getOperand(0).getNode();
+ if (!IPM || IPM->getOpcode() != SystemZISD::IPM)
+ return SDValue();
+ SDValue CCReg = IPM->getOperand(0);
+
+ // AndOp0 and AndOp1 should refer to same CC.
+ SDValue Op0CCReg = AndOp0->getOperand(4);
+ auto *CCNode0 = Op0CCReg.getNode(), *CCNode1 = CCReg.getNode();
+ if (!CCNode0 || !CCNode1 || CCNode0 != CCNode1)
+ return SDValue();
+ auto *Op0TrueVal = dyn_cast<ConstantSDNode>(AndOp0->getOperand(0));
+ auto *Op0FalseVal = dyn_cast<ConstantSDNode>(AndOp0->getOperand(1));
+ auto *Op0CCValid = dyn_cast<ConstantSDNode>(AndOp0->getOperand(2));
+ auto *Op0CCMask = dyn_cast<ConstantSDNode>(AndOp0->getOperand(3));
+ int Op0CCValidVal = Op0CCValid->getZExtValue();
+ if (!Op0TrueVal || !Op0FalseVal || !Op0CCMask || !Op0CCValid)
+ return SDValue();
+ if (Op0TrueVal->getZExtValue() != 1 || Op0FalseVal->getZExtValue() != 0)
+ return SDValue();
+
+ // Compute the effective CC mask for select.
+ // Op0CCMaskVal & CCMask = 0. Invert Op0CCMaskVal.
+ int Op0CCMaskVal = Op0CCMask->getZExtValue();
+ int CCMaskVal = Op0CCMaskVal ^ 0xf;
+ assert(CCMaskVal < 4 && "CC out of range");
+ int CCMask = 1 << (3 - CCMaskVal);
----------------
uweigand wrote:
Ah, I see in the XOR case you actually handle this. Need similar handling here
as well.
https://github.com/llvm/llvm-project/pull/125970
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
