@@ -8279,8 +8279,8 @@ static SDValue ExpandHorizontalBinOp(const SDValue &V0, const SDValue &V1,
8279
8279
static bool isAddSubOrSubAdd(const BuildVectorSDNode *BV,
8280
8280
const X86Subtarget &Subtarget, SelectionDAG &DAG,
8281
8281
SDValue &Opnd0, SDValue &Opnd1,
8282
- unsigned &NumExtracts,
8283
- bool &IsSubAdd ) {
8282
+ unsigned &NumExtracts, bool &IsSubAdd,
8283
+ bool &HasAllowContract ) {
8284
8284
using namespace SDPatternMatch;
8285
8285
8286
8286
MVT VT = BV->getSimpleValueType(0);
@@ -8292,6 +8292,7 @@ static bool isAddSubOrSubAdd(const BuildVectorSDNode *BV,
8292
8292
SDValue InVec1 = DAG.getUNDEF(VT);
8293
8293
8294
8294
NumExtracts = 0;
8295
+ HasAllowContract = NumElts != 0;
8295
8296
8296
8297
// Odd-numbered elements in the input build vector are obtained from
8297
8298
// adding/subtracting two integer/float elements.
@@ -8350,6 +8351,7 @@ static bool isAddSubOrSubAdd(const BuildVectorSDNode *BV,
8350
8351
8351
8352
// Increment the number of extractions done.
8352
8353
++NumExtracts;
8354
+ HasAllowContract &= Op->getFlags().hasAllowContract();
8353
8355
}
8354
8356
8355
8357
// Ensure we have found an opcode for both parities and that they are
@@ -8393,9 +8395,10 @@ static bool isAddSubOrSubAdd(const BuildVectorSDNode *BV,
8393
8395
/// is illegal sometimes. E.g. 512-bit ADDSUB is not available, while 512-bit
8394
8396
/// FMADDSUB is.
8395
8397
static bool isFMAddSubOrFMSubAdd(const X86Subtarget &Subtarget,
8396
- SelectionDAG &DAG,
8397
- SDValue &Opnd0, SDValue &Opnd1, SDValue &Opnd2,
8398
- unsigned ExpectedUses) {
8398
+ SelectionDAG &DAG, SDValue &Opnd0,
8399
+ SDValue &Opnd1, SDValue &Opnd2,
8400
+ unsigned ExpectedUses,
8401
+ bool AllowSubAddOrAddSubContract) {
8399
8402
if (Opnd0.getOpcode() != ISD::FMUL ||
8400
8403
!Opnd0->hasNUsesOfValue(ExpectedUses, 0) || !Subtarget.hasAnyFMA())
8401
8404
return false;
@@ -8406,7 +8409,8 @@ static bool isFMAddSubOrFMSubAdd(const X86Subtarget &Subtarget,
8406
8409
// or MUL + ADDSUB to FMADDSUB.
8407
8410
const TargetOptions &Options = DAG.getTarget().Options;
8408
8411
bool AllowFusion =
8409
- (Options.AllowFPOpFusion == FPOpFusion::Fast || Options.UnsafeFPMath);
8412
+ Options.AllowFPOpFusion == FPOpFusion::Fast ||
8413
+ (AllowSubAddOrAddSubContract && Opnd0->getFlags().hasAllowContract());
8410
8414
if (!AllowFusion)
8411
8415
return false;
8412
8416
@@ -8427,15 +8431,17 @@ static SDValue lowerToAddSubOrFMAddSub(const BuildVectorSDNode *BV,
8427
8431
SDValue Opnd0, Opnd1;
8428
8432
unsigned NumExtracts;
8429
8433
bool IsSubAdd;
8430
- if (!isAddSubOrSubAdd(BV, Subtarget, DAG, Opnd0, Opnd1, NumExtracts,
8431
- IsSubAdd))
8434
+ bool HasAllowContract;
8435
+ if (!isAddSubOrSubAdd(BV, Subtarget, DAG, Opnd0, Opnd1, NumExtracts, IsSubAdd,
8436
+ HasAllowContract))
8432
8437
return SDValue();
8433
8438
8434
8439
MVT VT = BV->getSimpleValueType(0);
8435
8440
8436
8441
// Try to generate X86ISD::FMADDSUB node here.
8437
8442
SDValue Opnd2;
8438
- if (isFMAddSubOrFMSubAdd(Subtarget, DAG, Opnd0, Opnd1, Opnd2, NumExtracts)) {
8443
+ if (isFMAddSubOrFMSubAdd(Subtarget, DAG, Opnd0, Opnd1, Opnd2, NumExtracts,
8444
+ HasAllowContract)) {
8439
8445
unsigned Opc = IsSubAdd ? X86ISD::FMSUBADD : X86ISD::FMADDSUB;
8440
8446
return DAG.getNode(Opc, DL, VT, Opnd0, Opnd1, Opnd2);
8441
8447
}
@@ -43180,7 +43186,7 @@ static bool isAddSubOrSubAddMask(ArrayRef<int> Mask, bool &Op0Even) {
43180
43186
/// the fact that they're unused.
43181
43187
static bool isAddSubOrSubAdd(SDNode *N, const X86Subtarget &Subtarget,
43182
43188
SelectionDAG &DAG, SDValue &Opnd0, SDValue &Opnd1,
43183
- bool &IsSubAdd) {
43189
+ bool &IsSubAdd, bool &HasAllowContract ) {
43184
43190
43185
43191
EVT VT = N->getValueType(0);
43186
43192
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
@@ -43231,6 +43237,8 @@ static bool isAddSubOrSubAdd(SDNode *N, const X86Subtarget &Subtarget,
43231
43237
// It's a subadd if the vector in the even parity is an FADD.
43232
43238
IsSubAdd = Op0Even ? V1->getOpcode() == ISD::FADD
43233
43239
: V2->getOpcode() == ISD::FADD;
43240
+ HasAllowContract =
43241
+ V1->getFlags().hasAllowContract() && V2->getFlags().hasAllowContract();
43234
43242
43235
43243
Opnd0 = LHS;
43236
43244
Opnd1 = RHS;
@@ -43288,14 +43296,17 @@ static SDValue combineShuffleToAddSubOrFMAddSub(SDNode *N, const SDLoc &DL,
43288
43296
43289
43297
SDValue Opnd0, Opnd1;
43290
43298
bool IsSubAdd;
43291
- if (!isAddSubOrSubAdd(N, Subtarget, DAG, Opnd0, Opnd1, IsSubAdd))
43299
+ bool HasAllowContract;
43300
+ if (!isAddSubOrSubAdd(N, Subtarget, DAG, Opnd0, Opnd1, IsSubAdd,
43301
+ HasAllowContract))
43292
43302
return SDValue();
43293
43303
43294
43304
MVT VT = N->getSimpleValueType(0);
43295
43305
43296
43306
// Try to generate X86ISD::FMADDSUB node here.
43297
43307
SDValue Opnd2;
43298
- if (isFMAddSubOrFMSubAdd(Subtarget, DAG, Opnd0, Opnd1, Opnd2, 2)) {
43308
+ if (isFMAddSubOrFMSubAdd(Subtarget, DAG, Opnd0, Opnd1, Opnd2, 2,
43309
+ HasAllowContract)) {
43299
43310
unsigned Opc = IsSubAdd ? X86ISD::FMSUBADD : X86ISD::FMADDSUB;
43300
43311
return DAG.getNode(Opc, DL, VT, Opnd0, Opnd1, Opnd2);
43301
43312
}
0 commit comments