Coverage Report

Created: 2019-07-24 05:18

/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/lib/Target/X86/X86DomainReassignment.cpp
Line
Count
Source (jump to first uncovered line)
1
//===--- X86DomainReassignment.cpp - Selectively switch register classes---===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// This pass attempts to find instruction chains (closures) in one domain,
10
// and convert them to equivalent instructions in a different domain,
11
// if profitable.
12
//
13
//===----------------------------------------------------------------------===//
14
15
#include "X86.h"
16
#include "X86InstrInfo.h"
17
#include "X86Subtarget.h"
18
#include "llvm/ADT/DenseMap.h"
19
#include "llvm/ADT/DenseMapInfo.h"
20
#include "llvm/ADT/STLExtras.h"
21
#include "llvm/ADT/SmallVector.h"
22
#include "llvm/ADT/Statistic.h"
23
#include "llvm/CodeGen/MachineFunctionPass.h"
24
#include "llvm/CodeGen/MachineInstrBuilder.h"
25
#include "llvm/CodeGen/MachineRegisterInfo.h"
26
#include "llvm/CodeGen/TargetRegisterInfo.h"
27
#include "llvm/Support/Debug.h"
28
#include "llvm/Support/Printable.h"
29
#include <bitset>
30
31
using namespace llvm;
32
33
#define DEBUG_TYPE "x86-domain-reassignment"
34
35
STATISTIC(NumClosuresConverted, "Number of closures converted by the pass");
36
37
static cl::opt<bool> DisableX86DomainReassignment(
38
    "disable-x86-domain-reassignment", cl::Hidden,
39
    cl::desc("X86: Disable Virtual Register Reassignment."), cl::init(false));
40
41
namespace {
42
enum RegDomain { NoDomain = -1, GPRDomain, MaskDomain, OtherDomain, NumDomains };
43
44
228k
static bool isGPR(const TargetRegisterClass *RC) {
45
228k
  return X86::GR64RegClass.hasSubClassEq(RC) ||
46
228k
         
X86::GR32RegClass.hasSubClassEq(RC)197k
||
47
228k
         
X86::GR16RegClass.hasSubClassEq(RC)137k
||
48
228k
         
X86::GR8RegClass.hasSubClassEq(RC)130k
;
49
228k
}
50
51
static bool isMask(const TargetRegisterClass *RC,
52
16.4k
                   const TargetRegisterInfo *TRI) {
53
16.4k
  return X86::VK16RegClass.hasSubClassEq(RC);
54
16.4k
}
55
56
static RegDomain getDomain(const TargetRegisterClass *RC,
57
102k
                           const TargetRegisterInfo *TRI) {
58
102k
  if (isGPR(RC))
59
85.8k
    return GPRDomain;
60
16.4k
  if (isMask(RC, TRI))
61
11.8k
    return MaskDomain;
62
4.62k
  return OtherDomain;
63
4.62k
}
64
65
/// Return a register class equivalent to \p SrcRC, in \p Domain.
66
static const TargetRegisterClass *getDstRC(const TargetRegisterClass *SrcRC,
67
400
                                           RegDomain Domain) {
68
400
  assert(Domain == MaskDomain && "add domain");
69
400
  if (X86::GR8RegClass.hasSubClassEq(SrcRC))
70
70
    return &X86::VK8RegClass;
71
330
  if (X86::GR16RegClass.hasSubClassEq(SrcRC))
72
74
    return &X86::VK16RegClass;
73
256
  if (X86::GR32RegClass.hasSubClassEq(SrcRC))
74
244
    return &X86::VK32RegClass;
75
12
  if (X86::GR64RegClass.hasSubClassEq(SrcRC))
76
12
    return &X86::VK64RegClass;
77
0
  llvm_unreachable("add register class");
78
0
  return nullptr;
79
0
}
80
81
/// Abstract Instruction Converter class.
82
class InstrConverterBase {
83
protected:
84
  unsigned SrcOpcode;
85
86
public:
87
837k
  InstrConverterBase(unsigned SrcOpcode) : SrcOpcode(SrcOpcode) {}
88
89
837k
  virtual ~InstrConverterBase() {}
90
91
  /// \returns true if \p MI is legal to convert.
92
  virtual bool isLegal(const MachineInstr *MI,
93
22.3k
                       const TargetInstrInfo *TII) const {
94
22.3k
    assert(MI->getOpcode() == SrcOpcode &&
95
22.3k
           "Wrong instruction passed to converter");
96
22.3k
    return true;
97
22.3k
  }
98
99
  /// Applies conversion to \p MI.
100
  ///
101
  /// \returns true if \p MI is no longer need, and can be deleted.
102
  virtual bool convertInstr(MachineInstr *MI, const TargetInstrInfo *TII,
103
                            MachineRegisterInfo *MRI) const = 0;
104
105
  /// \returns the cost increment incurred by converting \p MI.
106
  virtual double getExtraCost(const MachineInstr *MI,
107
                              MachineRegisterInfo *MRI) const = 0;
108
};
109
110
/// An Instruction Converter which ignores the given instruction.
111
/// For example, PHI instructions can be safely ignored since only the registers
112
/// need to change.
113
class InstrIgnore : public InstrConverterBase {
114
public:
115
37.3k
  InstrIgnore(unsigned SrcOpcode) : InstrConverterBase(SrcOpcode) {}
116
117
  bool convertInstr(MachineInstr *MI, const TargetInstrInfo *TII,
118
88
                    MachineRegisterInfo *MRI) const override {
119
88
    assert(isLegal(MI, TII) && "Cannot convert instruction");
120
88
    return false;
121
88
  }
122
123
  double getExtraCost(const MachineInstr *MI,
124
1.80k
                      MachineRegisterInfo *MRI) const override {
125
1.80k
    return 0;
126
1.80k
  }
127
};
128
129
/// An Instruction Converter which replaces an instruction with another.
130
class InstrReplacer : public InstrConverterBase {
131
public:
132
  /// Opcode of the destination instruction.
133
  unsigned DstOpcode;
134
135
  InstrReplacer(unsigned SrcOpcode, unsigned DstOpcode)
136
667k
      : InstrConverterBase(SrcOpcode), DstOpcode(DstOpcode) {}
137
138
  bool isLegal(const MachineInstr *MI,
139
1.53k
               const TargetInstrInfo *TII) const override {
140
1.53k
    if (!InstrConverterBase::isLegal(MI, TII))
141
0
      return false;
142
1.53k
    // It's illegal to replace an instruction that implicitly defines a register
143
1.53k
    // with an instruction that doesn't, unless that register dead.
144
1.53k
    for (auto &MO : MI->implicit_operands())
145
282
      if (MO.isReg() && MO.isDef() && !MO.isDead() &&
146
282
          
!TII->get(DstOpcode).hasImplicitDefOfPhysReg(MO.getReg())25
)
147
25
        return false;
148
1.53k
    
return true1.51k
;
149
1.53k
  }
150
151
  bool convertInstr(MachineInstr *MI, const TargetInstrInfo *TII,
152
392
                    MachineRegisterInfo *MRI) const override {
153
392
    assert(isLegal(MI, TII) && "Cannot convert instruction");
154
392
    MachineInstrBuilder Bld =
155
392
        BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), TII->get(DstOpcode));
156
392
    // Transfer explicit operands from original instruction. Implicit operands
157
392
    // are handled by BuildMI.
158
392
    for (auto &Op : MI->explicit_operands())
159
1.45k
      Bld.add(Op);
160
392
    return true;
161
392
  }
162
163
  double getExtraCost(const MachineInstr *MI,
164
234
                      MachineRegisterInfo *MRI) const override {
165
234
    // Assuming instructions have the same cost.
166
234
    return 0;
167
234
  }
168
};
169
170
/// An Instruction Converter which replaces an instruction with another, and
171
/// adds a COPY from the new instruction's destination to the old one's.
172
class InstrReplacerDstCOPY : public InstrConverterBase {
173
public:
174
  unsigned DstOpcode;
175
176
  InstrReplacerDstCOPY(unsigned SrcOpcode, unsigned DstOpcode)
177
113k
      : InstrConverterBase(SrcOpcode), DstOpcode(DstOpcode) {}
178
179
  bool convertInstr(MachineInstr *MI, const TargetInstrInfo *TII,
180
5
                    MachineRegisterInfo *MRI) const override {
181
5
    assert(isLegal(MI, TII) && "Cannot convert instruction");
182
5
    MachineBasicBlock *MBB = MI->getParent();
183
5
    auto &DL = MI->getDebugLoc();
184
5
185
5
    unsigned Reg = MRI->createVirtualRegister(
186
5
        TII->getRegClass(TII->get(DstOpcode), 0, MRI->getTargetRegisterInfo(),
187
5
                         *MBB->getParent()));
188
5
    MachineInstrBuilder Bld = BuildMI(*MBB, MI, DL, TII->get(DstOpcode), Reg);
189
30
    for (unsigned Idx = 1, End = MI->getNumOperands(); Idx < End; 
++Idx25
)
190
25
      Bld.add(MI->getOperand(Idx));
191
5
192
5
    BuildMI(*MBB, MI, DL, TII->get(TargetOpcode::COPY))
193
5
        .add(MI->getOperand(0))
194
5
        .addReg(Reg);
195
5
196
5
    return true;
197
5
  }
198
199
  double getExtraCost(const MachineInstr *MI,
200
5
                      MachineRegisterInfo *MRI) const override {
201
5
    // Assuming instructions have the same cost, and that COPY is in the same
202
5
    // domain so it will be eliminated.
203
5
    return 0;
204
5
  }
205
};
206
207
/// An Instruction Converter for replacing COPY instructions.
208
class InstrCOPYReplacer : public InstrReplacer {
209
public:
210
  RegDomain DstDomain;
211
212
  InstrCOPYReplacer(unsigned SrcOpcode, RegDomain DstDomain, unsigned DstOpcode)
213
18.6k
      : InstrReplacer(SrcOpcode, DstOpcode), DstDomain(DstDomain) {}
214
215
  bool isLegal(const MachineInstr *MI,
216
16.6k
               const TargetInstrInfo *TII) const override {
217
16.6k
    if (!InstrConverterBase::isLegal(MI, TII))
218
0
      return false;
219
16.6k
220
16.6k
    // Don't allow copies to/flow GR8/GR16 physical registers.
221
16.6k
    // FIXME: Is there some better way to support this?
222
16.6k
    unsigned DstReg = MI->getOperand(0).getReg();
223
16.6k
    if (TargetRegisterInfo::isPhysicalRegister(DstReg) &&
224
16.6k
        
(1.36k
X86::GR8RegClass.contains(DstReg)1.36k
||
225
1.36k
         
X86::GR16RegClass.contains(DstReg)1.14k
))
226
487
      return false;
227
16.1k
    unsigned SrcReg = MI->getOperand(1).getReg();
228
16.1k
    if (TargetRegisterInfo::isPhysicalRegister(SrcReg) &&
229
16.1k
        
(8.22k
X86::GR8RegClass.contains(SrcReg)8.22k
||
230
8.22k
         
X86::GR16RegClass.contains(SrcReg)8.20k
))
231
39
      return false;
232
16.1k
233
16.1k
    return true;
234
16.1k
  }
235
236
  double getExtraCost(const MachineInstr *MI,
237
6.56k
                      MachineRegisterInfo *MRI) const override {
238
6.56k
    assert(MI->getOpcode() == TargetOpcode::COPY && "Expected a COPY");
239
6.56k
240
10.6k
    for (auto &MO : MI->operands()) {
241
10.6k
      // Physical registers will not be converted. Assume that converting the
242
10.6k
      // COPY to the destination domain will eventually result in a actual
243
10.6k
      // instruction.
244
10.6k
      if (TargetRegisterInfo::isPhysicalRegister(MO.getReg()))
245
2.33k
        return 1;
246
8.34k
247
8.34k
      RegDomain OpDomain = getDomain(MRI->getRegClass(MO.getReg()),
248
8.34k
                                     MRI->getTargetRegisterInfo());
249
8.34k
      // Converting a cross domain COPY to a same domain COPY should eliminate
250
8.34k
      // an insturction
251
8.34k
      if (OpDomain == DstDomain)
252
2.46k
        return -1;
253
8.34k
    }
254
6.56k
    
return 01.76k
;
255
6.56k
  }
256
};
257
258
/// An Instruction Converter which replaces an instruction with a COPY.
259
class InstrReplaceWithCopy : public InstrConverterBase {
260
public:
261
  // Source instruction operand Index, to be used as the COPY source.
262
  unsigned SrcOpIdx;
263
264
  InstrReplaceWithCopy(unsigned SrcOpcode, unsigned SrcOpIdx)
265
18.6k
      : InstrConverterBase(SrcOpcode), SrcOpIdx(SrcOpIdx) {}
266
267
  bool convertInstr(MachineInstr *MI, const TargetInstrInfo *TII,
268
71
                    MachineRegisterInfo *MRI) const override {
269
71
    assert(isLegal(MI, TII) && "Cannot convert instruction");
270
71
    BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
271
71
            TII->get(TargetOpcode::COPY))
272
71
        .add({MI->getOperand(0), MI->getOperand(SrcOpIdx)});
273
71
    return true;
274
71
  }
275
276
  double getExtraCost(const MachineInstr *MI,
277
1.79k
                      MachineRegisterInfo *MRI) const override {
278
1.79k
    return 0;
279
1.79k
  }
280
};
281
282
// Key type to be used by the Instruction Converters map.
283
// A converter is identified by <destination domain, source opcode>
284
typedef std::pair<int, unsigned> InstrConverterBaseKeyTy;
285
286
typedef DenseMap<InstrConverterBaseKeyTy, InstrConverterBase *>
287
    InstrConverterBaseMap;
288
289
/// A closure is a set of virtual register representing all of the edges in
290
/// the closure, as well as all of the instructions connected by those edges.
291
///
292
/// A closure may encompass virtual registers in the same register bank that
293
/// have different widths. For example, it may contain 32-bit GPRs as well as
294
/// 64-bit GPRs.
295
///
296
/// A closure that computes an address (i.e. defines a virtual register that is
297
/// used in a memory operand) excludes the instructions that contain memory
298
/// operands using the address. Such an instruction will be included in a
299
/// different closure that manipulates the loaded or stored value.
300
class Closure {
301
private:
302
  /// Virtual registers in the closure.
303
  DenseSet<unsigned> Edges;
304
305
  /// Instructions in the closure.
306
  SmallVector<MachineInstr *, 8> Instrs;
307
308
  /// Domains which this closure can legally be reassigned to.
309
  std::bitset<NumDomains> LegalDstDomains;
310
311
  /// An ID to uniquely identify this closure, even when it gets
312
  /// moved around
313
  unsigned ID;
314
315
public:
316
27.1k
  Closure(unsigned ID, std::initializer_list<RegDomain> LegalDstDomainList) : ID(ID) {
317
27.1k
    for (RegDomain D : LegalDstDomainList)
318
27.1k
      LegalDstDomains.set(D);
319
27.1k
  }
320
321
  /// Mark this closure as illegal for reassignment to all domains.
322
77.6k
  void setAllIllegal() { LegalDstDomains.reset(); }
323
324
  /// \returns true if this closure has domains which are legal to reassign to.
325
0
  bool hasLegalDstDomain() const { return LegalDstDomains.any(); }
326
327
  /// \returns true if is legal to reassign this closure to domain \p RD.
328
133k
  bool isLegal(RegDomain RD) const { return LegalDstDomains[RD]; }
329
330
  /// Mark this closure as illegal for reassignment to domain \p RD.
331
5.24k
  void setIllegal(RegDomain RD) { LegalDstDomains[RD] = false; }
332
333
27.1k
  bool empty() const { return Edges.empty(); }
334
335
79.9k
  bool insertEdge(unsigned Reg) {
336
79.9k
    return Edges.insert(Reg).second;
337
79.9k
  }
338
339
  using const_edge_iterator = DenseSet<unsigned>::const_iterator;
340
156
  iterator_range<const_edge_iterator> edges() const {
341
156
    return iterator_range<const_edge_iterator>(Edges.begin(), Edges.end());
342
156
  }
343
344
35.6k
  void addInstruction(MachineInstr *I) {
345
35.6k
    Instrs.push_back(I);
346
35.6k
  }
347
348
2.65k
  ArrayRef<MachineInstr *> instructions() const {
349
2.65k
    return Instrs;
350
2.65k
  }
351
352
0
  LLVM_DUMP_METHOD void dump(const MachineRegisterInfo *MRI) const {
353
0
    dbgs() << "Registers: ";
354
0
    bool First = true;
355
0
    for (unsigned Reg : Edges) {
356
0
      if (!First)
357
0
        dbgs() << ", ";
358
0
      First = false;
359
0
      dbgs() << printReg(Reg, MRI->getTargetRegisterInfo(), 0, MRI);
360
0
    }
361
0
    dbgs() << "\n" << "Instructions:";
362
0
    for (MachineInstr *MI : Instrs) {
363
0
      dbgs() << "\n  ";
364
0
      MI->print(dbgs());
365
0
    }
366
0
    dbgs() << "\n";
367
0
  }
368
369
111k
  unsigned getID() const {
370
111k
    return ID;
371
111k
  }
372
373
};
374
375
class X86DomainReassignment : public MachineFunctionPass {
376
  const X86Subtarget *STI;
377
  MachineRegisterInfo *MRI;
378
  const X86InstrInfo *TII;
379
380
  /// All edges that are included in some closure
381
  DenseSet<unsigned> EnclosedEdges;
382
383
  /// All instructions that are included in some closure.
384
  DenseMap<MachineInstr *, unsigned> EnclosedInstrs;
385
386
public:
387
  static char ID;
388
389
11.3k
  X86DomainReassignment() : MachineFunctionPass(ID) { }
390
391
  bool runOnMachineFunction(MachineFunction &MF) override;
392
393
11.3k
  void getAnalysisUsage(AnalysisUsage &AU) const override {
394
11.3k
    AU.setPreservesCFG();
395
11.3k
    MachineFunctionPass::getAnalysisUsage(AU);
396
11.3k
  }
397
398
146k
  StringRef getPassName() const override {
399
146k
    return "X86 Domain Reassignment Pass";
400
146k
  }
401
402
private:
403
  /// A map of available Instruction Converters.
404
  InstrConverterBaseMap Converters;
405
406
  /// Initialize Converters map.
407
  void initConverters();
408
409
  /// Starting from \Reg, expand the closure as much as possible.
410
  void buildClosure(Closure &, unsigned Reg);
411
412
  /// Enqueue \p Reg to be considered for addition to the closure.
413
  void visitRegister(Closure &, unsigned Reg, RegDomain &Domain,
414
                     SmallVectorImpl<unsigned> &Worklist);
415
416
  /// Reassign the closure to \p Domain.
417
  void reassign(const Closure &C, RegDomain Domain) const;
418
419
  /// Add \p MI to the closure.
420
  void encloseInstr(Closure &C, MachineInstr *MI);
421
422
  /// /returns true if it is profitable to reassign the closure to \p Domain.
423
  bool isReassignmentProfitable(const Closure &C, RegDomain Domain) const;
424
425
  /// Calculate the total cost of reassigning the closure to \p Domain.
426
  double calculateCost(const Closure &C, RegDomain Domain) const;
427
};
428
429
char X86DomainReassignment::ID = 0;
430
431
} // End anonymous namespace.
432
433
void X86DomainReassignment::visitRegister(Closure &C, unsigned Reg,
434
                                          RegDomain &Domain,
435
134k
                                          SmallVectorImpl<unsigned> &Worklist) {
436
134k
  if (EnclosedEdges.count(Reg))
437
29.4k
    return;
438
105k
439
105k
  if (!TargetRegisterInfo::isVirtualRegister(Reg))
440
10.9k
    return;
441
94.1k
442
94.1k
  if (!MRI->hasOneDef(Reg))
443
255
    return;
444
93.9k
445
93.9k
  RegDomain RD = getDomain(MRI->getRegClass(Reg), MRI->getTargetRegisterInfo());
446
93.9k
  // First edge in closure sets the domain.
447
93.9k
  if (Domain == NoDomain)
448
26.9k
    Domain = RD;
449
93.9k
450
93.9k
  if (Domain != RD)
451
13.9k
    return;
452
79.9k
453
79.9k
  Worklist.push_back(Reg);
454
79.9k
}
455
456
111k
void X86DomainReassignment::encloseInstr(Closure &C, MachineInstr *MI) {
457
111k
  auto I = EnclosedInstrs.find(MI);
458
111k
  if (I != EnclosedInstrs.end()) {
459
75.9k
    if (I->second != C.getID())
460
63.0k
      // Instruction already belongs to another closure, avoid conflicts between
461
63.0k
      // closure and mark this closure as illegal.
462
63.0k
      C.setAllIllegal();
463
75.9k
    return;
464
75.9k
  }
465
35.6k
466
35.6k
  EnclosedInstrs[MI] = C.getID();
467
35.6k
  C.addInstruction(MI);
468
35.6k
469
35.6k
  // Mark closure as illegal for reassignment to domains, if there is no
470
35.6k
  // converter for the instruction or if the converter cannot convert the
471
35.6k
  // instruction.
472
142k
  for (int i = 0; i != NumDomains; 
++i106k
) {
473
106k
    if (C.isLegal((RegDomain)i)) {
474
27.0k
      InstrConverterBase *IC = Converters.lookup({i, MI->getOpcode()});
475
27.0k
      if (!IC || 
!IC->isLegal(MI, TII)22.3k
)
476
5.24k
        C.setIllegal((RegDomain)i);
477
27.0k
    }
478
106k
  }
479
35.6k
}
480
481
double X86DomainReassignment::calculateCost(const Closure &C,
482
2.50k
                                            RegDomain DstDomain) const {
483
2.50k
  assert(C.isLegal(DstDomain) && "Cannot calculate cost for illegal closure");
484
2.50k
485
2.50k
  double Cost = 0.0;
486
2.50k
  for (auto *MI : C.instructions())
487
10.4k
    Cost +=
488
10.4k
        Converters.lookup({DstDomain, MI->getOpcode()})->getExtraCost(MI, MRI);
489
2.50k
  return Cost;
490
2.50k
}
491
492
bool X86DomainReassignment::isReassignmentProfitable(const Closure &C,
493
2.50k
                                                     RegDomain Domain) const {
494
2.50k
  return calculateCost(C, Domain) < 0.0;
495
2.50k
}
496
497
156
void X86DomainReassignment::reassign(const Closure &C, RegDomain Domain) const {
498
156
  assert(C.isLegal(Domain) && "Cannot convert illegal closure");
499
156
500
156
  // Iterate all instructions in the closure, convert each one using the
501
156
  // appropriate converter.
502
156
  SmallVector<MachineInstr *, 8> ToErase;
503
156
  for (auto *MI : C.instructions())
504
556
    if (Converters.lookup({Domain, MI->getOpcode()})
505
556
            ->convertInstr(MI, TII, MRI))
506
468
      ToErase.push_back(MI);
507
156
508
156
  // Iterate all registers in the closure, replace them with registers in the
509
156
  // destination domain.
510
400
  for (unsigned Reg : C.edges()) {
511
400
    MRI->setRegClass(Reg, getDstRC(MRI->getRegClass(Reg), Domain));
512
743
    for (auto &MO : MRI->use_operands(Reg)) {
513
743
      if (MO.isReg())
514
743
        // Remove all subregister references as they are not valid in the
515
743
        // destination domain.
516
743
        MO.setSubReg(0);
517
743
    }
518
400
  }
519
156
520
156
  for (auto MI : ToErase)
521
468
    MI->eraseFromParent();
522
156
}
523
524
/// \returns true when \p Reg is used as part of an address calculation in \p
525
/// MI.
526
static bool usedAsAddr(const MachineInstr &MI, unsigned Reg,
527
61.6k
                       const TargetInstrInfo *TII) {
528
61.6k
  if (!MI.mayLoadOrStore())
529
51.3k
    return false;
530
10.2k
531
10.2k
  const MCInstrDesc &Desc = TII->get(MI.getOpcode());
532
10.2k
  int MemOpStart = X86II::getMemoryOperandNo(Desc.TSFlags);
533
10.2k
  if (MemOpStart == -1)
534
24
    return false;
535
10.1k
536
10.1k
  MemOpStart += X86II::getOperandBias(Desc);
537
10.1k
  for (unsigned MemOpIdx = MemOpStart,
538
10.1k
                MemOpEnd = MemOpStart + X86::AddrNumOperands;
539
20.1k
       MemOpIdx < MemOpEnd; 
++MemOpIdx9.99k
) {
540
18.5k
    auto &Op = MI.getOperand(MemOpIdx);
541
18.5k
    if (Op.isReg() && 
Op.getReg() == Reg13.1k
)
542
8.51k
      return true;
543
18.5k
  }
544
10.1k
  
return false1.67k
;
545
10.1k
}
546
547
27.1k
void X86DomainReassignment::buildClosure(Closure &C, unsigned Reg) {
548
27.1k
  SmallVector<unsigned, 4> Worklist;
549
27.1k
  RegDomain Domain = NoDomain;
550
27.1k
  visitRegister(C, Reg, Domain, Worklist);
551
107k
  while (!Worklist.empty()) {
552
79.9k
    unsigned CurReg = Worklist.pop_back_val();
553
79.9k
554
79.9k
    // Register already in this closure.
555
79.9k
    if (!C.insertEdge(CurReg))
556
21.4k
      continue;
557
58.4k
    EnclosedEdges.insert(Reg);
558
58.4k
559
58.4k
    MachineInstr *DefMI = MRI->getVRegDef(CurReg);
560
58.4k
    encloseInstr(C, DefMI);
561
58.4k
562
58.4k
    // Add register used by the defining MI to the worklist.
563
58.4k
    // Do not add registers which are used in address calculation, they will be
564
58.4k
    // added to a different closure.
565
58.4k
    int OpEnd = DefMI->getNumOperands();
566
58.4k
    const MCInstrDesc &Desc = DefMI->getDesc();
567
58.4k
    int MemOp = X86II::getMemoryOperandNo(Desc.TSFlags);
568
58.4k
    if (MemOp != -1)
569
3.37k
      MemOp += X86II::getOperandBias(Desc);
570
211k
    for (int OpIdx = 0; OpIdx < OpEnd; 
++OpIdx153k
) {
571
153k
      if (OpIdx == MemOp) {
572
3.37k
        // skip address calculation.
573
3.37k
        OpIdx += (X86::AddrNumOperands - 1);
574
3.37k
        continue;
575
3.37k
      }
576
150k
      auto &Op = DefMI->getOperand(OpIdx);
577
150k
      if (!Op.isReg() || 
!Op.isUse()133k
)
578
87.8k
        continue;
579
62.1k
      visitRegister(C, Op.getReg(), Domain, Worklist);
580
62.1k
    }
581
58.4k
582
58.4k
    // Expand closure through register uses.
583
61.6k
    for (auto &UseMI : MRI->use_nodbg_instructions(CurReg)) {
584
61.6k
      // We would like to avoid converting closures which calculare addresses,
585
61.6k
      // as this should remain in GPRs.
586
61.6k
      if (usedAsAddr(UseMI, CurReg, TII)) {
587
8.51k
        C.setAllIllegal();
588
8.51k
        continue;
589
8.51k
      }
590
53.0k
      encloseInstr(C, &UseMI);
591
53.0k
592
53.0k
      for (auto &DefOp : UseMI.defs()) {
593
51.2k
        if (!DefOp.isReg())
594
0
          continue;
595
51.2k
596
51.2k
        unsigned DefReg = DefOp.getReg();
597
51.2k
        if (!TargetRegisterInfo::isVirtualRegister(DefReg)) {
598
6.02k
          C.setAllIllegal();
599
6.02k
          continue;
600
6.02k
        }
601
45.2k
        visitRegister(C, DefReg, Domain, Worklist);
602
45.2k
      }
603
53.0k
    }
604
58.4k
  }
605
27.1k
}
606
607
18.6k
void X86DomainReassignment::initConverters() {
608
18.6k
  Converters[{MaskDomain, TargetOpcode::PHI}] =
609
18.6k
      new InstrIgnore(TargetOpcode::PHI);
610
18.6k
611
18.6k
  Converters[{MaskDomain, TargetOpcode::IMPLICIT_DEF}] =
612
18.6k
      new InstrIgnore(TargetOpcode::IMPLICIT_DEF);
613
18.6k
614
18.6k
  Converters[{MaskDomain, TargetOpcode::INSERT_SUBREG}] =
615
18.6k
      new InstrReplaceWithCopy(TargetOpcode::INSERT_SUBREG, 2);
616
18.6k
617
18.6k
  Converters[{MaskDomain, TargetOpcode::COPY}] =
618
18.6k
      new InstrCOPYReplacer(TargetOpcode::COPY, MaskDomain, TargetOpcode::COPY);
619
18.6k
620
113k
  auto createReplacerDstCOPY = [&](unsigned From, unsigned To) {
621
113k
    Converters[{MaskDomain, From}] = new InstrReplacerDstCOPY(From, To);
622
113k
  };
623
18.6k
624
18.6k
  createReplacerDstCOPY(X86::MOVZX32rm16, X86::KMOVWkm);
625
18.6k
  createReplacerDstCOPY(X86::MOVZX64rm16, X86::KMOVWkm);
626
18.6k
627
18.6k
  createReplacerDstCOPY(X86::MOVZX32rr16, X86::KMOVWkk);
628
18.6k
  createReplacerDstCOPY(X86::MOVZX64rr16, X86::KMOVWkk);
629
18.6k
630
18.6k
  if (STI->hasDQI()) {
631
6.41k
    createReplacerDstCOPY(X86::MOVZX16rm8, X86::KMOVBkm);
632
6.41k
    createReplacerDstCOPY(X86::MOVZX32rm8, X86::KMOVBkm);
633
6.41k
    createReplacerDstCOPY(X86::MOVZX64rm8, X86::KMOVBkm);
634
6.41k
635
6.41k
    createReplacerDstCOPY(X86::MOVZX16rr8, X86::KMOVBkk);
636
6.41k
    createReplacerDstCOPY(X86::MOVZX32rr8, X86::KMOVBkk);
637
6.41k
    createReplacerDstCOPY(X86::MOVZX64rr8, X86::KMOVBkk);
638
6.41k
  }
639
18.6k
640
649k
  auto createReplacer = [&](unsigned From, unsigned To) {
641
649k
    Converters[{MaskDomain, From}] = new InstrReplacer(From, To);
642
649k
  };
643
18.6k
644
18.6k
  createReplacer(X86::MOV16rm, X86::KMOVWkm);
645
18.6k
  createReplacer(X86::MOV16mr, X86::KMOVWmk);
646
18.6k
  createReplacer(X86::MOV16rr, X86::KMOVWkk);
647
18.6k
  createReplacer(X86::SHR16ri, X86::KSHIFTRWri);
648
18.6k
  createReplacer(X86::SHL16ri, X86::KSHIFTLWri);
649
18.6k
  createReplacer(X86::NOT16r, X86::KNOTWrr);
650
18.6k
  createReplacer(X86::OR16rr, X86::KORWrr);
651
18.6k
  createReplacer(X86::AND16rr, X86::KANDWrr);
652
18.6k
  createReplacer(X86::XOR16rr, X86::KXORWrr);
653
18.6k
654
18.6k
  if (STI->hasBWI()) {
655
18.6k
    createReplacer(X86::MOV32rm, X86::KMOVDkm);
656
18.6k
    createReplacer(X86::MOV64rm, X86::KMOVQkm);
657
18.6k
658
18.6k
    createReplacer(X86::MOV32mr, X86::KMOVDmk);
659
18.6k
    createReplacer(X86::MOV64mr, X86::KMOVQmk);
660
18.6k
661
18.6k
    createReplacer(X86::MOV32rr, X86::KMOVDkk);
662
18.6k
    createReplacer(X86::MOV64rr, X86::KMOVQkk);
663
18.6k
664
18.6k
    createReplacer(X86::SHR32ri, X86::KSHIFTRDri);
665
18.6k
    createReplacer(X86::SHR64ri, X86::KSHIFTRQri);
666
18.6k
667
18.6k
    createReplacer(X86::SHL32ri, X86::KSHIFTLDri);
668
18.6k
    createReplacer(X86::SHL64ri, X86::KSHIFTLQri);
669
18.6k
670
18.6k
    createReplacer(X86::ADD32rr, X86::KADDDrr);
671
18.6k
    createReplacer(X86::ADD64rr, X86::KADDQrr);
672
18.6k
673
18.6k
    createReplacer(X86::NOT32r, X86::KNOTDrr);
674
18.6k
    createReplacer(X86::NOT64r, X86::KNOTQrr);
675
18.6k
676
18.6k
    createReplacer(X86::OR32rr, X86::KORDrr);
677
18.6k
    createReplacer(X86::OR64rr, X86::KORQrr);
678
18.6k
679
18.6k
    createReplacer(X86::AND32rr, X86::KANDDrr);
680
18.6k
    createReplacer(X86::AND64rr, X86::KANDQrr);
681
18.6k
682
18.6k
    createReplacer(X86::ANDN32rr, X86::KANDNDrr);
683
18.6k
    createReplacer(X86::ANDN64rr, X86::KANDNQrr);
684
18.6k
685
18.6k
    createReplacer(X86::XOR32rr, X86::KXORDrr);
686
18.6k
    createReplacer(X86::XOR64rr, X86::KXORQrr);
687
18.6k
688
18.6k
    // TODO: KTEST is not a replacement for TEST due to flag differences. Need
689
18.6k
    // to prove only Z flag is used.
690
18.6k
    //createReplacer(X86::TEST32rr, X86::KTESTDrr);
691
18.6k
    //createReplacer(X86::TEST64rr, X86::KTESTQrr);
692
18.6k
  }
693
18.6k
694
18.6k
  if (STI->hasDQI()) {
695
6.41k
    createReplacer(X86::ADD8rr, X86::KADDBrr);
696
6.41k
    createReplacer(X86::ADD16rr, X86::KADDWrr);
697
6.41k
698
6.41k
    createReplacer(X86::AND8rr, X86::KANDBrr);
699
6.41k
700
6.41k
    createReplacer(X86::MOV8rm, X86::KMOVBkm);
701
6.41k
    createReplacer(X86::MOV8mr, X86::KMOVBmk);
702
6.41k
    createReplacer(X86::MOV8rr, X86::KMOVBkk);
703
6.41k
704
6.41k
    createReplacer(X86::NOT8r, X86::KNOTBrr);
705
6.41k
706
6.41k
    createReplacer(X86::OR8rr, X86::KORBrr);
707
6.41k
708
6.41k
    createReplacer(X86::SHR8ri, X86::KSHIFTRBri);
709
6.41k
    createReplacer(X86::SHL8ri, X86::KSHIFTLBri);
710
6.41k
711
6.41k
    // TODO: KTEST is not a replacement for TEST due to flag differences. Need
712
6.41k
    // to prove only Z flag is used.
713
6.41k
    //createReplacer(X86::TEST8rr, X86::KTESTBrr);
714
6.41k
    //createReplacer(X86::TEST16rr, X86::KTESTWrr);
715
6.41k
716
6.41k
    createReplacer(X86::XOR8rr, X86::KXORBrr);
717
6.41k
  }
718
18.6k
}
719
720
135k
bool X86DomainReassignment::runOnMachineFunction(MachineFunction &MF) {
721
135k
  if (skipFunction(MF.getFunction()))
722
196
    return false;
723
135k
  if (DisableX86DomainReassignment)
724
0
    return false;
725
135k
726
135k
  LLVM_DEBUG(
727
135k
      dbgs() << "***** Machine Function before Domain Reassignment *****\n");
728
135k
  LLVM_DEBUG(MF.print(dbgs()));
729
135k
730
135k
  STI = &MF.getSubtarget<X86Subtarget>();
731
135k
  // GPR->K is the only transformation currently supported, bail out early if no
732
135k
  // AVX512.
733
135k
  // TODO: We're also bailing of AVX512BW isn't supported since we use VK32 and
734
135k
  // VK64 for GR32/GR64, but those aren't legal classes on KNL. If the register
735
135k
  // coalescer doesn't clean it up and we generate a spill we will crash.
736
135k
  if (!STI->hasAVX512() || 
!STI->hasBWI()41.5k
)
737
116k
    return false;
738
18.6k
739
18.6k
  MRI = &MF.getRegInfo();
740
18.6k
  assert(MRI->isSSA() && "Expected MIR to be in SSA form");
741
18.6k
742
18.6k
  TII = STI->getInstrInfo();
743
18.6k
  initConverters();
744
18.6k
  bool Changed = false;
745
18.6k
746
18.6k
  EnclosedEdges.clear();
747
18.6k
  EnclosedInstrs.clear();
748
18.6k
749
18.6k
  std::vector<Closure> Closures;
750
18.6k
751
18.6k
  // Go over all virtual registers and calculate a closure.
752
18.6k
  unsigned ClosureID = 0;
753
144k
  for (unsigned Idx = 0; Idx < MRI->getNumVirtRegs(); 
++Idx126k
) {
754
126k
    unsigned Reg = TargetRegisterInfo::index2VirtReg(Idx);
755
126k
756
126k
    // GPR only current source domain supported.
757
126k
    if (!isGPR(MRI->getRegClass(Reg)))
758
98.9k
      continue;
759
27.1k
760
27.1k
    // Register already in closure.
761
27.1k
    if (EnclosedEdges.count(Reg))
762
0
      continue;
763
27.1k
764
27.1k
    // Calculate closure starting with Reg.
765
27.1k
    Closure C(ClosureID++, {MaskDomain});
766
27.1k
    buildClosure(C, Reg);
767
27.1k
768
27.1k
    // Collect all closures that can potentially be converted.
769
27.1k
    if (!C.empty() && 
C.isLegal(MaskDomain)26.9k
)
770
2.50k
      Closures.push_back(std::move(C));
771
27.1k
  }
772
18.6k
773
18.6k
  for (Closure &C : Closures) {
774
2.50k
    LLVM_DEBUG(C.dump(MRI));
775
2.50k
    if (isReassignmentProfitable(C, MaskDomain)) {
776
156
      reassign(C, MaskDomain);
777
156
      ++NumClosuresConverted;
778
156
      Changed = true;
779
156
    }
780
2.50k
  }
781
18.6k
782
18.6k
  DeleteContainerSeconds(Converters);
783
18.6k
784
18.6k
  LLVM_DEBUG(
785
18.6k
      dbgs() << "***** Machine Function after Domain Reassignment *****\n");
786
18.6k
  LLVM_DEBUG(MF.print(dbgs()));
787
18.6k
788
18.6k
  return Changed;
789
18.6k
}
790
791
INITIALIZE_PASS(X86DomainReassignment, "x86-domain-reassignment",
792
                "X86 Domain Reassignment Pass", false, false)
793
794
/// Returns an instance of the Domain Reassignment pass.
795
11.3k
FunctionPass *llvm::createX86DomainReassignmentPass() {
796
11.3k
  return new X86DomainReassignment();
797
11.3k
}