Coverage Report

Created: 2017-10-03 07:32

/Users/buildslave/jenkins/sharedspace/clang-stage2-coverage-R@2/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
Line
Count
Source (jump to first uncovered line)
1
//===-- llvm/CodeGen/GlobalISel/LegalizerHelper.cpp -----------------------===//
2
//
3
//                     The LLVM Compiler Infrastructure
4
//
5
// This file is distributed under the University of Illinois Open Source
6
// License. See LICENSE.TXT for details.
7
//
8
//===----------------------------------------------------------------------===//
9
//
10
/// \file This file implements the LegalizerHelper class to legalize
11
/// individual instructions and the LegalizeMachineIR wrapper pass for the
12
/// primary legalization.
13
//
14
//===----------------------------------------------------------------------===//
15
16
#include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
17
#include "llvm/CodeGen/GlobalISel/CallLowering.h"
18
#include "llvm/CodeGen/GlobalISel/LegalizerInfo.h"
19
#include "llvm/CodeGen/MachineRegisterInfo.h"
20
#include "llvm/Support/Debug.h"
21
#include "llvm/Support/raw_ostream.h"
22
#include "llvm/Target/TargetLowering.h"
23
#include "llvm/Target/TargetSubtargetInfo.h"
24
25
#include <sstream>
26
27
#define DEBUG_TYPE "legalizer"
28
29
using namespace llvm;
30
31
LegalizerHelper::LegalizerHelper(MachineFunction &MF)
32
221k
    : MRI(MF.getRegInfo()), LI(*MF.getSubtarget().getLegalizerInfo()) {
33
221k
  MIRBuilder.setMF(MF);
34
221k
}
35
36
LegalizerHelper::LegalizeResult
37
7.71M
LegalizerHelper::legalizeInstrStep(MachineInstr &MI) {
38
7.71M
  DEBUG(dbgs() << "Legalizing: "; MI.print(dbgs()));
39
7.71M
40
7.71M
  auto Action = LI.getAction(MI, MRI);
41
7.71M
  switch (std::get<0>(Action)) {
42
6.90M
  case LegalizerInfo::Legal:
43
6.90M
    DEBUG(dbgs() << ".. Already legal\n");
44
6.90M
    return AlreadyLegal;
45
104
  case LegalizerInfo::Libcall:
46
104
    DEBUG(dbgs() << ".. Convert to libcall\n");
47
104
    return libcall(MI);
48
11.7k
  case LegalizerInfo::NarrowScalar:
49
11.7k
    DEBUG(dbgs() << ".. Narrow scalar\n");
50
11.7k
    return narrowScalar(MI, std::get<1>(Action), std::get<2>(Action));
51
756k
  case LegalizerInfo::WidenScalar:
52
756k
    DEBUG(dbgs() << ".. Widen scalar\n");
53
756k
    return widenScalar(MI, std::get<1>(Action), std::get<2>(Action));
54
2.59k
  case LegalizerInfo::Lower:
55
2.59k
    DEBUG(dbgs() << ".. Lower\n");
56
2.59k
    return lower(MI, std::get<1>(Action), std::get<2>(Action));
57
2.26k
  case LegalizerInfo::FewerElements:
58
2.26k
    DEBUG(dbgs() << ".. Reduce number of elements\n");
59
2.26k
    return fewerElementsVector(MI, std::get<1>(Action), std::get<2>(Action));
60
114
  case LegalizerInfo::Custom:
61
114
    DEBUG(dbgs() << ".. Custom legalization\n");
62
114
    return LI.legalizeCustom(MI, MRI, MIRBuilder) ? Legalized
63
0
                                                  : UnableToLegalize;
64
39.5k
  default:
65
39.5k
    DEBUG(dbgs() << ".. Unable to legalize\n");
66
39.5k
    return UnableToLegalize;
67
0
  }
68
0
}
69
70
void LegalizerHelper::extractParts(unsigned Reg, LLT Ty, int NumParts,
71
9.02k
                                   SmallVectorImpl<unsigned> &VRegs) {
72
27.5k
  for (int i = 0; 
i < NumParts27.5k
;
++i18.5k
)
73
18.5k
    VRegs.push_back(MRI.createGenericVirtualRegister(Ty));
74
9.02k
  MIRBuilder.buildUnmerge(VRegs, Reg);
75
9.02k
}
76
77
104
static RTLIB::Libcall getRTLibDesc(unsigned Opcode, unsigned Size) {
78
104
  switch (Opcode) {
79
12
  case TargetOpcode::G_SDIV:
80
12
    assert(Size == 32 && "Unsupported size");
81
12
    return RTLIB::SDIV_I32;
82
12
  case TargetOpcode::G_UDIV:
83
12
    assert(Size == 32 && "Unsupported size");
84
12
    return RTLIB::UDIV_I32;
85
6
  case TargetOpcode::G_SREM:
86
6
    assert(Size == 32 && "Unsupported size");
87
6
    return RTLIB::SREM_I32;
88
6
  case TargetOpcode::G_UREM:
89
6
    assert(Size == 32 && "Unsupported size");
90
6
    return RTLIB::UREM_I32;
91
8
  case TargetOpcode::G_FADD:
92
8
    assert((Size == 32 || Size == 64) && "Unsupported size");
93
8
    return Size == 64 ? 
RTLIB::ADD_F644
:
RTLIB::ADD_F324
;
94
14
  case TargetOpcode::G_FREM:
95
14
    return Size == 64 ? 
RTLIB::REM_F647
:
RTLIB::REM_F327
;
96
46
  case TargetOpcode::G_FPOW:
97
46
    return Size == 64 ? 
RTLIB::POW_F6439
:
RTLIB::POW_F327
;
98
0
  }
99
0
  
llvm_unreachable0
("Unknown libcall function");
100
0
}
101
102
LegalizerHelper::LegalizeResult
103
llvm::createLibcall(MachineIRBuilder &MIRBuilder, RTLIB::Libcall Libcall,
104
                    const CallLowering::ArgInfo &Result,
105
186
                    ArrayRef<CallLowering::ArgInfo> Args) {
106
186
  auto &CLI = *MIRBuilder.getMF().getSubtarget().getCallLowering();
107
186
  auto &TLI = *MIRBuilder.getMF().getSubtarget().getTargetLowering();
108
186
  const char *Name = TLI.getLibcallName(Libcall);
109
186
110
186
  MIRBuilder.getMF().getFrameInfo().setHasCalls(true);
111
186
  if (!CLI.lowerCall(MIRBuilder, TLI.getLibcallCallingConv(Libcall),
112
186
                     MachineOperand::CreateES(Name), Result, Args))
113
0
    return LegalizerHelper::UnableToLegalize;
114
186
115
186
  return LegalizerHelper::Legalized;
116
186
}
117
118
static LegalizerHelper::LegalizeResult
119
simpleLibcall(MachineInstr &MI, MachineIRBuilder &MIRBuilder, unsigned Size,
120
104
              Type *OpType) {
121
104
  auto Libcall = getRTLibDesc(MI.getOpcode(), Size);
122
104
  return createLibcall(MIRBuilder, Libcall, {MI.getOperand(0).getReg(), OpType},
123
104
                       {{MI.getOperand(1).getReg(), OpType},
124
104
                        {MI.getOperand(2).getReg(), OpType}});
125
104
}
126
127
LegalizerHelper::LegalizeResult
128
104
LegalizerHelper::libcall(MachineInstr &MI) {
129
104
  LLT LLTy = MRI.getType(MI.getOperand(0).getReg());
130
104
  unsigned Size = LLTy.getSizeInBits();
131
104
  auto &Ctx = MIRBuilder.getMF().getFunction()->getContext();
132
104
133
104
  MIRBuilder.setInstr(MI);
134
104
135
104
  switch (MI.getOpcode()) {
136
0
  default:
137
0
    return UnableToLegalize;
138
36
  case TargetOpcode::G_SDIV:
139
36
  case TargetOpcode::G_UDIV:
140
36
  case TargetOpcode::G_SREM:
141
36
  case TargetOpcode::G_UREM: {
142
36
    Type *HLTy = Type::getInt32Ty(Ctx);
143
36
    auto Status = simpleLibcall(MI, MIRBuilder, Size, HLTy);
144
36
    if (Status != Legalized)
145
0
      return Status;
146
36
    break;
147
36
  }
148
68
  case TargetOpcode::G_FADD:
149
68
  case TargetOpcode::G_FPOW:
150
68
  case TargetOpcode::G_FREM: {
151
68
    Type *HLTy = Size == 64 ? 
Type::getDoubleTy(Ctx)50
:
Type::getFloatTy(Ctx)18
;
152
68
    auto Status = simpleLibcall(MI, MIRBuilder, Size, HLTy);
153
68
    if (Status != Legalized)
154
0
      return Status;
155
68
    break;
156
68
  }
157
104
  }
158
104
159
104
  MI.eraseFromParent();
160
104
  return Legalized;
161
104
}
162
163
LegalizerHelper::LegalizeResult LegalizerHelper::narrowScalar(MachineInstr &MI,
164
                                                              unsigned TypeIdx,
165
11.7k
                                                              LLT NarrowTy) {
166
11.7k
  // FIXME: Don't know how to handle secondary types yet.
167
11.7k
  if (
TypeIdx != 0 && 11.7k
MI.getOpcode() != TargetOpcode::G_EXTRACT3.28k
)
168
0
    return UnableToLegalize;
169
11.7k
170
11.7k
  MIRBuilder.setInstr(MI);
171
11.7k
172
11.7k
  switch (MI.getOpcode()) {
173
0
  default:
174
0
    return UnableToLegalize;
175
2.83k
  case TargetOpcode::G_IMPLICIT_DEF: {
176
2.83k
    int NumParts = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits() /
177
2.83k
                   NarrowTy.getSizeInBits();
178
2.83k
179
2.83k
    SmallVector<unsigned, 2> DstRegs;
180
8.53k
    for (int i = 0; 
i < NumParts8.53k
;
++i5.69k
) {
181
5.69k
      unsigned Dst = MRI.createGenericVirtualRegister(NarrowTy);
182
5.69k
      MIRBuilder.buildUndef(Dst);
183
5.69k
      DstRegs.push_back(Dst);
184
5.69k
    }
185
2.83k
    MIRBuilder.buildMerge(MI.getOperand(0).getReg(), DstRegs);
186
2.83k
    MI.eraseFromParent();
187
2.83k
    return Legalized;
188
11.7k
  }
189
3
  case TargetOpcode::G_ADD: {
190
3
    // Expand in terms of carry-setting/consuming G_ADDE instructions.
191
3
    int NumParts = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits() /
192
3
                   NarrowTy.getSizeInBits();
193
3
194
3
    SmallVector<unsigned, 2> Src1Regs, Src2Regs, DstRegs;
195
3
    extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, Src1Regs);
196
3
    extractParts(MI.getOperand(2).getReg(), NarrowTy, NumParts, Src2Regs);
197
3
198
3
    unsigned CarryIn = MRI.createGenericVirtualRegister(LLT::scalar(1));
199
3
    MIRBuilder.buildConstant(CarryIn, 0);
200
3
201
9
    for (int i = 0; 
i < NumParts9
;
++i6
) {
202
6
      unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy);
203
6
      unsigned CarryOut = MRI.createGenericVirtualRegister(LLT::scalar(1));
204
6
205
6
      MIRBuilder.buildUAdde(DstReg, CarryOut, Src1Regs[i],
206
6
                            Src2Regs[i], CarryIn);
207
6
208
6
      DstRegs.push_back(DstReg);
209
6
      CarryIn = CarryOut;
210
6
    }
211
3
    unsigned DstReg = MI.getOperand(0).getReg();
212
3
    MIRBuilder.buildMerge(DstReg, DstRegs);
213
3
    MI.eraseFromParent();
214
3
    return Legalized;
215
11.7k
  }
216
3.28k
  case TargetOpcode::G_EXTRACT: {
217
3.28k
    if (TypeIdx != 1)
218
0
      return UnableToLegalize;
219
3.28k
220
3.28k
    int64_t NarrowSize = NarrowTy.getSizeInBits();
221
3.28k
    int NumParts =
222
3.28k
        MRI.getType(MI.getOperand(1).getReg()).getSizeInBits() / NarrowSize;
223
3.28k
224
3.28k
    SmallVector<unsigned, 2> SrcRegs, DstRegs;
225
3.28k
    SmallVector<uint64_t, 2> Indexes;
226
3.28k
    extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, SrcRegs);
227
3.28k
228
3.28k
    unsigned OpReg = MI.getOperand(0).getReg();
229
3.28k
    int64_t OpStart = MI.getOperand(2).getImm();
230
3.28k
    int64_t OpSize = MRI.getType(OpReg).getSizeInBits();
231
9.98k
    for (int i = 0; 
i < NumParts9.98k
;
++i6.70k
) {
232
6.70k
      unsigned SrcStart = i * NarrowSize;
233
6.70k
234
6.70k
      if (
SrcStart + NarrowSize <= OpStart || 6.70k
SrcStart >= OpStart + OpSize5.58k
) {
235
3.42k
        // No part of the extract uses this subregister, ignore it.
236
3.42k
        continue;
237
3.28k
      } else 
if (3.28k
SrcStart == OpStart && 3.28k
NarrowTy == MRI.getType(OpReg)3.27k
) {
238
480
        // The entire subregister is extracted, forward the value.
239
480
        DstRegs.push_back(SrcRegs[i]);
240
480
        continue;
241
480
      }
242
2.80k
243
2.80k
      // OpSegStart is where this destination segment would start in OpReg if it
244
2.80k
      // extended infinitely in both directions.
245
2.80k
      int64_t ExtractOffset, SegSize;
246
2.80k
      if (
OpStart < SrcStart2.80k
) {
247
1
        ExtractOffset = 0;
248
1
        SegSize = std::min(NarrowSize, OpStart + OpSize - SrcStart);
249
2.80k
      } else {
250
2.80k
        ExtractOffset = OpStart - SrcStart;
251
2.80k
        SegSize = std::min(SrcStart + NarrowSize - OpStart, OpSize);
252
2.80k
      }
253
2.80k
254
2.80k
      unsigned SegReg = SrcRegs[i];
255
2.80k
      if (
ExtractOffset != 0 || 2.80k
SegSize != NarrowSize2.79k
) {
256
900
        // A genuine extract is needed.
257
900
        SegReg = MRI.createGenericVirtualRegister(LLT::scalar(SegSize));
258
900
        MIRBuilder.buildExtract(SegReg, SrcRegs[i], ExtractOffset);
259
900
      }
260
6.70k
261
6.70k
      DstRegs.push_back(SegReg);
262
6.70k
    }
263
3.28k
264
3.28k
    MIRBuilder.buildMerge(MI.getOperand(0).getReg(), DstRegs);
265
3.28k
    MI.eraseFromParent();
266
3.28k
    return Legalized;
267
3.28k
  }
268
5.62k
  case TargetOpcode::G_INSERT: {
269
5.62k
    if (TypeIdx != 0)
270
0
      return UnableToLegalize;
271
5.62k
272
5.62k
    int64_t NarrowSize = NarrowTy.getSizeInBits();
273
5.62k
    int NumParts =
274
5.62k
        MRI.getType(MI.getOperand(0).getReg()).getSizeInBits() / NarrowSize;
275
5.62k
276
5.62k
    SmallVector<unsigned, 2> SrcRegs, DstRegs;
277
5.62k
    SmallVector<uint64_t, 2> Indexes;
278
5.62k
    extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, SrcRegs);
279
5.62k
280
5.62k
    unsigned OpReg = MI.getOperand(2).getReg();
281
5.62k
    int64_t OpStart = MI.getOperand(3).getImm();
282
5.62k
    int64_t OpSize = MRI.getType(OpReg).getSizeInBits();
283
16.9k
    for (int i = 0; 
i < NumParts16.9k
;
++i11.3k
) {
284
11.3k
      unsigned DstStart = i * NarrowSize;
285
11.3k
286
11.3k
      if (
DstStart + NarrowSize <= OpStart || 11.3k
DstStart >= OpStart + OpSize8.46k
) {
287
5.67k
        // No part of the insert affects this subregister, forward the original.
288
5.67k
        DstRegs.push_back(SrcRegs[i]);
289
5.67k
        continue;
290
5.62k
      } else 
if (5.62k
DstStart == OpStart && 5.62k
NarrowTy == MRI.getType(OpReg)5.62k
) {
291
239
        // The entire subregister is defined by this insert, forward the new
292
239
        // value.
293
239
        DstRegs.push_back(OpReg);
294
239
        continue;
295
239
      }
296
5.38k
297
5.38k
      // OpSegStart is where this destination segment would start in OpReg if it
298
5.38k
      // extended infinitely in both directions.
299
5.38k
      int64_t ExtractOffset, InsertOffset, SegSize;
300
5.38k
      if (
OpStart < DstStart5.38k
) {
301
1
        InsertOffset = 0;
302
1
        ExtractOffset = DstStart - OpStart;
303
1
        SegSize = std::min(NarrowSize, OpStart + OpSize - DstStart);
304
5.38k
      } else {
305
5.38k
        InsertOffset = OpStart - DstStart;
306
5.38k
        ExtractOffset = 0;
307
5.38k
        SegSize =
308
5.38k
            std::min(NarrowSize - InsertOffset, OpStart + OpSize - DstStart);
309
5.38k
      }
310
5.38k
311
5.38k
      unsigned SegReg = OpReg;
312
5.38k
      if (
ExtractOffset != 0 || 5.38k
SegSize != OpSize5.38k
) {
313
2
        // A genuine extract is needed.
314
2
        SegReg = MRI.createGenericVirtualRegister(LLT::scalar(SegSize));
315
2
        MIRBuilder.buildExtract(SegReg, OpReg, ExtractOffset);
316
2
      }
317
11.3k
318
11.3k
      unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy);
319
11.3k
      MIRBuilder.buildInsert(DstReg, SrcRegs[i], SegReg, InsertOffset);
320
11.3k
      DstRegs.push_back(DstReg);
321
11.3k
    }
322
5.62k
323
5.62k
    assert(DstRegs.size() == (unsigned)NumParts && "not all parts covered");
324
5.62k
    MIRBuilder.buildMerge(MI.getOperand(0).getReg(), DstRegs);
325
5.62k
    MI.eraseFromParent();
326
5.62k
    return Legalized;
327
5.62k
  }
328
8
  case TargetOpcode::G_LOAD: {
329
8
    unsigned NarrowSize = NarrowTy.getSizeInBits();
330
8
    int NumParts =
331
8
        MRI.getType(MI.getOperand(0).getReg()).getSizeInBits() / NarrowSize;
332
8
    LLT OffsetTy = LLT::scalar(
333
8
        MRI.getType(MI.getOperand(1).getReg()).getScalarSizeInBits());
334
8
335
8
    SmallVector<unsigned, 2> DstRegs;
336
24
    for (int i = 0; 
i < NumParts24
;
++i16
) {
337
16
      unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy);
338
16
      unsigned SrcReg = 0;
339
16
      unsigned Adjustment = i * NarrowSize / 8;
340
16
341
16
      MIRBuilder.materializeGEP(SrcReg, MI.getOperand(1).getReg(), OffsetTy,
342
16
                                Adjustment);
343
16
344
16
      // TODO: This is conservatively correct, but we probably want to split the
345
16
      // memory operands in the future.
346
16
      MIRBuilder.buildLoad(DstReg, SrcReg, **MI.memoperands_begin());
347
16
348
16
      DstRegs.push_back(DstReg);
349
16
    }
350
8
    unsigned DstReg = MI.getOperand(0).getReg();
351
8
    MIRBuilder.buildMerge(DstReg, DstRegs);
352
8
    MI.eraseFromParent();
353
8
    return Legalized;
354
5.62k
  }
355
6
  case TargetOpcode::G_STORE: {
356
6
    unsigned NarrowSize = NarrowTy.getSizeInBits();
357
6
    int NumParts =
358
6
        MRI.getType(MI.getOperand(0).getReg()).getSizeInBits() / NarrowSize;
359
6
    LLT OffsetTy = LLT::scalar(
360
6
        MRI.getType(MI.getOperand(1).getReg()).getScalarSizeInBits());
361
6
362
6
    SmallVector<unsigned, 2> SrcRegs;
363
6
    extractParts(MI.getOperand(0).getReg(), NarrowTy, NumParts, SrcRegs);
364
6
365
18
    for (int i = 0; 
i < NumParts18
;
++i12
) {
366
12
      unsigned DstReg = 0;
367
12
      unsigned Adjustment = i * NarrowSize / 8;
368
12
369
12
      MIRBuilder.materializeGEP(DstReg, MI.getOperand(1).getReg(), OffsetTy,
370
12
                                Adjustment);
371
12
372
12
      // TODO: This is conservatively correct, but we probably want to split the
373
12
      // memory operands in the future.
374
12
      MIRBuilder.buildStore(SrcRegs[i], DstReg, **MI.memoperands_begin());
375
12
    }
376
6
    MI.eraseFromParent();
377
6
    return Legalized;
378
5.62k
  }
379
2
  case TargetOpcode::G_CONSTANT: {
380
2
    unsigned NarrowSize = NarrowTy.getSizeInBits();
381
2
    int NumParts =
382
2
        MRI.getType(MI.getOperand(0).getReg()).getSizeInBits() / NarrowSize;
383
2
    const APInt &Cst = MI.getOperand(1).getCImm()->getValue();
384
2
    LLVMContext &Ctx = MIRBuilder.getMF().getFunction()->getContext();
385
2
386
2
    SmallVector<unsigned, 2> DstRegs;
387
6
    for (int i = 0; 
i < NumParts6
;
++i4
) {
388
4
      unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy);
389
4
      ConstantInt *CI =
390
4
          ConstantInt::get(Ctx, Cst.lshr(NarrowSize * i).trunc(NarrowSize));
391
4
      MIRBuilder.buildConstant(DstReg, *CI);
392
4
      DstRegs.push_back(DstReg);
393
4
    }
394
2
    unsigned DstReg = MI.getOperand(0).getReg();
395
2
    MIRBuilder.buildMerge(DstReg, DstRegs);
396
2
    MI.eraseFromParent();
397
2
    return Legalized;
398
0
  }
399
11.7k
  }
400
11.7k
}
401
402
LegalizerHelper::LegalizeResult
403
756k
LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
404
756k
  MIRBuilder.setInstr(MI);
405
756k
406
756k
  switch (MI.getOpcode()) {
407
0
  default:
408
0
    return UnableToLegalize;
409
39.7k
  case TargetOpcode::G_ADD:
410
39.7k
  case TargetOpcode::G_AND:
411
39.7k
  case TargetOpcode::G_MUL:
412
39.7k
  case TargetOpcode::G_OR:
413
39.7k
  case TargetOpcode::G_XOR:
414
39.7k
  case TargetOpcode::G_SUB:
415
39.7k
  case TargetOpcode::G_SHL: {
416
39.7k
    // Perform operation at larger width (any extension is fine here, high bits
417
39.7k
    // don't affect the result) and then truncate the result back to the
418
39.7k
    // original type.
419
39.7k
    unsigned Src1Ext = MRI.createGenericVirtualRegister(WideTy);
420
39.7k
    unsigned Src2Ext = MRI.createGenericVirtualRegister(WideTy);
421
39.7k
    MIRBuilder.buildAnyExt(Src1Ext, MI.getOperand(1).getReg());
422
39.7k
    MIRBuilder.buildAnyExt(Src2Ext, MI.getOperand(2).getReg());
423
39.7k
424
39.7k
    unsigned DstExt = MRI.createGenericVirtualRegister(WideTy);
425
39.7k
    MIRBuilder.buildInstr(MI.getOpcode())
426
39.7k
        .addDef(DstExt)
427
39.7k
        .addUse(Src1Ext)
428
39.7k
        .addUse(Src2Ext);
429
39.7k
430
39.7k
    MIRBuilder.buildTrunc(MI.getOperand(0).getReg(), DstExt);
431
39.7k
    MI.eraseFromParent();
432
39.7k
    return Legalized;
433
39.7k
  }
434
273
  case TargetOpcode::G_SDIV:
435
273
  case TargetOpcode::G_UDIV:
436
273
  case TargetOpcode::G_SREM:
437
273
  case TargetOpcode::G_UREM:
438
273
  case TargetOpcode::G_ASHR:
439
273
  case TargetOpcode::G_LSHR: {
440
273
    unsigned ExtOp = MI.getOpcode() == TargetOpcode::G_SDIV ||
441
252
                             MI.getOpcode() == TargetOpcode::G_SREM ||
442
236
                             MI.getOpcode() == TargetOpcode::G_ASHR
443
47
                         ? TargetOpcode::G_SEXT
444
226
                         : TargetOpcode::G_ZEXT;
445
273
446
273
    unsigned LHSExt = MRI.createGenericVirtualRegister(WideTy);
447
273
    MIRBuilder.buildInstr(ExtOp).addDef(LHSExt).addUse(
448
273
        MI.getOperand(1).getReg());
449
273
450
273
    unsigned RHSExt = MRI.createGenericVirtualRegister(WideTy);
451
273
    MIRBuilder.buildInstr(ExtOp).addDef(RHSExt).addUse(
452
273
        MI.getOperand(2).getReg());
453
273
454
273
    unsigned ResExt = MRI.createGenericVirtualRegister(WideTy);
455
273
    MIRBuilder.buildInstr(MI.getOpcode())
456
273
        .addDef(ResExt)
457
273
        .addUse(LHSExt)
458
273
        .addUse(RHSExt);
459
273
460
273
    MIRBuilder.buildTrunc(MI.getOperand(0).getReg(), ResExt);
461
273
    MI.eraseFromParent();
462
273
    return Legalized;
463
273
  }
464
4.27k
  case TargetOpcode::G_SELECT: {
465
4.27k
    if (TypeIdx != 0)
466
0
      return UnableToLegalize;
467
4.27k
468
4.27k
    // Perform operation at larger width (any extension is fine here, high bits
469
4.27k
    // don't affect the result) and then truncate the result back to the
470
4.27k
    // original type.
471
4.27k
    unsigned Src1Ext = MRI.createGenericVirtualRegister(WideTy);
472
4.27k
    unsigned Src2Ext = MRI.createGenericVirtualRegister(WideTy);
473
4.27k
    MIRBuilder.buildAnyExt(Src1Ext, MI.getOperand(2).getReg());
474
4.27k
    MIRBuilder.buildAnyExt(Src2Ext, MI.getOperand(3).getReg());
475
4.27k
476
4.27k
    unsigned DstExt = MRI.createGenericVirtualRegister(WideTy);
477
4.27k
    MIRBuilder.buildInstr(TargetOpcode::G_SELECT)
478
4.27k
        .addDef(DstExt)
479
4.27k
        .addReg(MI.getOperand(1).getReg())
480
4.27k
        .addUse(Src1Ext)
481
4.27k
        .addUse(Src2Ext);
482
4.27k
483
4.27k
    MIRBuilder.buildTrunc(MI.getOperand(0).getReg(), DstExt);
484
4.27k
    MI.eraseFromParent();
485
4.27k
    return Legalized;
486
4.27k
  }
487
36
  case TargetOpcode::G_FPTOSI:
488
36
  case TargetOpcode::G_FPTOUI: {
489
36
    if (TypeIdx != 0)
490
0
      return UnableToLegalize;
491
36
492
36
    unsigned DstExt = MRI.createGenericVirtualRegister(WideTy);
493
36
    MIRBuilder.buildInstr(MI.getOpcode())
494
36
        .addDef(DstExt)
495
36
        .addUse(MI.getOperand(1).getReg());
496
36
497
36
    MIRBuilder.buildTrunc(MI.getOperand(0).getReg(), DstExt);
498
36
    MI.eraseFromParent();
499
36
    return Legalized;
500
36
  }
501
50
  case TargetOpcode::G_SITOFP:
502
50
  case TargetOpcode::G_UITOFP: {
503
50
    if (TypeIdx != 1)
504
0
      return UnableToLegalize;
505
50
506
50
    unsigned Src = MI.getOperand(1).getReg();
507
50
    unsigned SrcExt = MRI.createGenericVirtualRegister(WideTy);
508
50
509
50
    if (
MI.getOpcode() == TargetOpcode::G_SITOFP50
) {
510
26
      MIRBuilder.buildSExt(SrcExt, Src);
511
50
    } else {
512
24
      assert(MI.getOpcode() == TargetOpcode::G_UITOFP && "Unexpected conv op");
513
24
      MIRBuilder.buildZExt(SrcExt, Src);
514
24
    }
515
50
516
50
    MIRBuilder.buildInstr(MI.getOpcode())
517
50
        .addDef(MI.getOperand(0).getReg())
518
50
        .addUse(SrcExt);
519
50
520
50
    MI.eraseFromParent();
521
50
    return Legalized;
522
50
  }
523
1
  case TargetOpcode::G_INSERT: {
524
1
    if (TypeIdx != 0)
525
0
      return UnableToLegalize;
526
1
527
1
    unsigned Src = MI.getOperand(1).getReg();
528
1
    unsigned SrcExt = MRI.createGenericVirtualRegister(WideTy);
529
1
    MIRBuilder.buildAnyExt(SrcExt, Src);
530
1
531
1
    unsigned DstExt = MRI.createGenericVirtualRegister(WideTy);
532
1
    auto MIB = MIRBuilder.buildInsert(DstExt, SrcExt, MI.getOperand(2).getReg(),
533
1
                                      MI.getOperand(3).getImm());
534
1
    for (unsigned OpNum = 4; 
OpNum < MI.getNumOperands()1
;
OpNum += 20
) {
535
0
      MIB.addReg(MI.getOperand(OpNum).getReg());
536
0
      MIB.addImm(MI.getOperand(OpNum + 1).getImm());
537
0
    }
538
1
539
1
    MIRBuilder.buildTrunc(MI.getOperand(0).getReg(), DstExt);
540
1
    MI.eraseFromParent();
541
1
    return Legalized;
542
1
  }
543
1.50k
  case TargetOpcode::G_LOAD: {
544
1.50k
    assert(alignTo(MRI.getType(MI.getOperand(0).getReg()).getSizeInBits(), 8) ==
545
1.50k
               WideTy.getSizeInBits() &&
546
1.50k
           "illegal to increase number of bytes loaded");
547
1.50k
548
1.50k
    unsigned DstExt = MRI.createGenericVirtualRegister(WideTy);
549
1.50k
    MIRBuilder.buildLoad(DstExt, MI.getOperand(1).getReg(),
550
1.50k
                         **MI.memoperands_begin());
551
1.50k
    MIRBuilder.buildTrunc(MI.getOperand(0).getReg(), DstExt);
552
1.50k
    MI.eraseFromParent();
553
1.50k
    return Legalized;
554
1
  }
555
6.57k
  case TargetOpcode::G_STORE: {
556
6.57k
    if (MRI.getType(MI.getOperand(0).getReg()) != LLT::scalar(1) ||
557
6.57k
        WideTy != LLT::scalar(8))
558
0
      return UnableToLegalize;
559
6.57k
560
6.57k
    auto &TLI = *MIRBuilder.getMF().getSubtarget().getTargetLowering();
561
6.57k
    auto Content = TLI.getBooleanContents(false, false);
562
6.57k
563
6.57k
    unsigned ExtOp = TargetOpcode::G_ANYEXT;
564
6.57k
    if (Content == TargetLoweringBase::ZeroOrOneBooleanContent)
565
6.57k
      ExtOp = TargetOpcode::G_ZEXT;
566
0
    else 
if (0
Content == TargetLoweringBase::ZeroOrNegativeOneBooleanContent0
)
567
0
      ExtOp = TargetOpcode::G_SEXT;
568
0
    else
569
0
      ExtOp = TargetOpcode::G_ANYEXT;
570
6.57k
571
6.57k
    unsigned SrcExt = MRI.createGenericVirtualRegister(WideTy);
572
6.57k
    MIRBuilder.buildInstr(ExtOp).addDef(SrcExt).addUse(
573
6.57k
        MI.getOperand(0).getReg());
574
6.57k
    MIRBuilder.buildStore(SrcExt, MI.getOperand(1).getReg(),
575
6.57k
                          **MI.memoperands_begin());
576
6.57k
    MI.eraseFromParent();
577
6.57k
    return Legalized;
578
6.57k
  }
579
181k
  case TargetOpcode::G_CONSTANT: {
580
181k
    unsigned DstExt = MRI.createGenericVirtualRegister(WideTy);
581
181k
    MIRBuilder.buildConstant(DstExt, *MI.getOperand(1).getCImm());
582
181k
    MIRBuilder.buildTrunc(MI.getOperand(0).getReg(), DstExt);
583
181k
    MI.eraseFromParent();
584
181k
    return Legalized;
585
6.57k
  }
586
1
  case TargetOpcode::G_FCONSTANT: {
587
1
    unsigned DstExt = MRI.createGenericVirtualRegister(WideTy);
588
1
    MIRBuilder.buildFConstant(DstExt, *MI.getOperand(1).getFPImm());
589
1
    MIRBuilder.buildFPTrunc(MI.getOperand(0).getReg(), DstExt);
590
1
    MI.eraseFromParent();
591
1
    return Legalized;
592
6.57k
  }
593
0
  case TargetOpcode::G_BRCOND: {
594
0
    unsigned TstExt = MRI.createGenericVirtualRegister(WideTy);
595
0
    MIRBuilder.buildAnyExt(TstExt, MI.getOperand(0).getReg());
596
0
    MIRBuilder.buildBrCond(TstExt, *MI.getOperand(1).getMBB());
597
0
    MI.eraseFromParent();
598
0
    return Legalized;
599
6.57k
  }
600
1.95k
  case TargetOpcode::G_FCMP: {
601
1.95k
    unsigned Op0Ext, Op1Ext, DstReg;
602
1.95k
    unsigned Cmp1 = MI.getOperand(2).getReg();
603
1.95k
    unsigned Cmp2 = MI.getOperand(3).getReg();
604
1.95k
    if (
TypeIdx == 01.95k
) {
605
1.95k
      Op0Ext = Cmp1;
606
1.95k
      Op1Ext = Cmp2;
607
1.95k
      DstReg = MRI.createGenericVirtualRegister(WideTy);
608
1.95k
    } else {
609
0
      Op0Ext = MRI.createGenericVirtualRegister(WideTy);
610
0
      Op1Ext = MRI.createGenericVirtualRegister(WideTy);
611
0
      DstReg = MI.getOperand(0).getReg();
612
0
      MIRBuilder.buildInstr(TargetOpcode::G_FPEXT, Op0Ext, Cmp1);
613
0
      MIRBuilder.buildInstr(TargetOpcode::G_FPEXT, Op1Ext, Cmp2);
614
0
    }
615
1.95k
    MIRBuilder.buildFCmp(
616
1.95k
        static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate()),
617
1.95k
        DstReg, Op0Ext, Op1Ext);
618
1.95k
    if (TypeIdx == 0)
619
1.95k
      MIRBuilder.buildInstr(TargetOpcode::G_TRUNC, MI.getOperand(0).getReg(),
620
1.95k
                            DstReg);
621
1.95k
    MI.eraseFromParent();
622
1.95k
    return Legalized;
623
6.57k
  }
624
511k
  case TargetOpcode::G_ICMP: {
625
511k
    bool IsSigned = CmpInst::isSigned(
626
511k
        static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate()));
627
511k
    unsigned Cmp1 = MI.getOperand(2).getReg();
628
511k
    unsigned Cmp2 = MI.getOperand(3).getReg();
629
511k
    unsigned Op0Ext, Op1Ext, DstReg;
630
511k
    if (
TypeIdx == 0511k
) {
631
477k
      Op0Ext = Cmp1;
632
477k
      Op1Ext = Cmp2;
633
477k
      DstReg = MRI.createGenericVirtualRegister(WideTy);
634
511k
    } else {
635
34.6k
      Op0Ext = MRI.createGenericVirtualRegister(WideTy);
636
34.6k
      Op1Ext = MRI.createGenericVirtualRegister(WideTy);
637
34.6k
      DstReg = MI.getOperand(0).getReg();
638
34.6k
      if (
IsSigned34.6k
) {
639
315
        MIRBuilder.buildSExt(Op0Ext, Cmp1);
640
315
        MIRBuilder.buildSExt(Op1Ext, Cmp2);
641
34.6k
      } else {
642
34.3k
        MIRBuilder.buildZExt(Op0Ext, Cmp1);
643
34.3k
        MIRBuilder.buildZExt(Op1Ext, Cmp2);
644
34.3k
      }
645
34.6k
    }
646
511k
    MIRBuilder.buildICmp(
647
511k
        static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate()),
648
511k
        DstReg, Op0Ext, Op1Ext);
649
511k
    if (TypeIdx == 0)
650
477k
      MIRBuilder.buildInstr(TargetOpcode::G_TRUNC, MI.getOperand(0).getReg(),
651
477k
                            DstReg);
652
511k
    MI.eraseFromParent();
653
511k
    return Legalized;
654
6.57k
  }
655
3
  case TargetOpcode::G_GEP: {
656
3
    assert(TypeIdx == 1 && "unable to legalize pointer of GEP");
657
3
    unsigned OffsetExt = MRI.createGenericVirtualRegister(WideTy);
658
3
    MIRBuilder.buildSExt(OffsetExt, MI.getOperand(2).getReg());
659
3
    MI.getOperand(2).setReg(OffsetExt);
660
3
    return Legalized;
661
6.57k
  }
662
9.07k
  case TargetOpcode::G_PHI: {
663
9.07k
    assert(TypeIdx == 0 && "Expecting only Idx 0");
664
20.1k
    auto getExtendedReg = [&](unsigned Reg, MachineBasicBlock &MBB) {
665
20.1k
      auto FirstTermIt = MBB.getFirstTerminator();
666
20.1k
      MIRBuilder.setInsertPt(MBB, FirstTermIt);
667
20.1k
      MachineInstr *DefMI = MRI.getVRegDef(Reg);
668
20.1k
      MachineInstrBuilder MIB;
669
20.1k
      if (DefMI->getOpcode() == TargetOpcode::G_TRUNC)
670
15.4k
        MIB = MIRBuilder.buildAnyExtOrTrunc(WideTy,
671
15.4k
                                            DefMI->getOperand(1).getReg());
672
20.1k
      else
673
4.66k
        MIB = MIRBuilder.buildAnyExt(WideTy, Reg);
674
20.1k
      return MIB->getOperand(0).getReg();
675
20.1k
    };
676
9.07k
    auto MIB = MIRBuilder.buildInstr(TargetOpcode::G_PHI, WideTy);
677
9.07k
    for (auto OpIt = MI.operands_begin() + 1, OpE = MI.operands_end();
678
29.2k
         
OpIt != OpE29.2k
;) {
679
20.1k
      unsigned Reg = OpIt++->getReg();
680
20.1k
      MachineBasicBlock *OpMBB = OpIt++->getMBB();
681
20.1k
      MIB.addReg(getExtendedReg(Reg, *OpMBB));
682
20.1k
      MIB.addMBB(OpMBB);
683
20.1k
    }
684
9.07k
    auto *MBB = MI.getParent();
685
9.07k
    MIRBuilder.setInsertPt(*MBB, MBB->getFirstNonPHI());
686
9.07k
    MIRBuilder.buildTrunc(MI.getOperand(0).getReg(),
687
9.07k
                          MIB->getOperand(0).getReg());
688
9.07k
    MI.eraseFromParent();
689
9.07k
    return Legalized;
690
0
  }
691
756k
  }
692
756k
}
693
694
LegalizerHelper::LegalizeResult
695
2.59k
LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT Ty) {
696
2.59k
  using namespace TargetOpcode;
697
2.59k
  MIRBuilder.setInstr(MI);
698
2.59k
699
2.59k
  switch(MI.getOpcode()) {
700
0
  default:
701
0
    return UnableToLegalize;
702
837
  case TargetOpcode::G_SREM:
703
837
  case TargetOpcode::G_UREM: {
704
837
    unsigned QuotReg = MRI.createGenericVirtualRegister(Ty);
705
837
    MIRBuilder.buildInstr(MI.getOpcode() == G_SREM ? 
G_SDIV301
:
G_UDIV536
)
706
837
        .addDef(QuotReg)
707
837
        .addUse(MI.getOperand(1).getReg())
708
837
        .addUse(MI.getOperand(2).getReg());
709
837
710
837
    unsigned ProdReg = MRI.createGenericVirtualRegister(Ty);
711
837
    MIRBuilder.buildMul(ProdReg, QuotReg, MI.getOperand(2).getReg());
712
837
    MIRBuilder.buildSub(MI.getOperand(0).getReg(), MI.getOperand(1).getReg(),
713
837
                        ProdReg);
714
837
    MI.eraseFromParent();
715
837
    return Legalized;
716
837
  }
717
141
  case TargetOpcode::G_SMULO:
718
141
  case TargetOpcode::G_UMULO: {
719
141
    // Generate G_UMULH/G_SMULH to check for overflow and a normal G_MUL for the
720
141
    // result.
721
141
    unsigned Res = MI.getOperand(0).getReg();
722
141
    unsigned Overflow = MI.getOperand(1).getReg();
723
141
    unsigned LHS = MI.getOperand(2).getReg();
724
141
    unsigned RHS = MI.getOperand(3).getReg();
725
141
726
141
    MIRBuilder.buildMul(Res, LHS, RHS);
727
141
728
141
    unsigned Opcode = MI.getOpcode() == TargetOpcode::G_SMULO
729
1
                          ? TargetOpcode::G_SMULH
730
140
                          : TargetOpcode::G_UMULH;
731
141
732
141
    unsigned HiPart = MRI.createGenericVirtualRegister(Ty);
733
141
    MIRBuilder.buildInstr(Opcode)
734
141
      .addDef(HiPart)
735
141
      .addUse(LHS)
736
141
      .addUse(RHS);
737
141
738
141
    unsigned Zero = MRI.createGenericVirtualRegister(Ty);
739
141
    MIRBuilder.buildConstant(Zero, 0);
740
141
    MIRBuilder.buildICmp(CmpInst::ICMP_NE, Overflow, HiPart, Zero);
741
141
    MI.eraseFromParent();
742
141
    return Legalized;
743
141
  }
744
1.61k
  case TargetOpcode::G_FNEG: {
745
1.61k
    // TODO: Handle vector types once we are able to
746
1.61k
    // represent them.
747
1.61k
    if (Ty.isVector())
748
0
      return UnableToLegalize;
749
1.61k
    unsigned Res = MI.getOperand(0).getReg();
750
1.61k
    Type *ZeroTy;
751
1.61k
    LLVMContext &Ctx = MIRBuilder.getMF().getFunction()->getContext();
752
1.61k
    switch (Ty.getSizeInBits()) {
753
0
    case 16:
754
0
      ZeroTy = Type::getHalfTy(Ctx);
755
0
      break;
756
29
    case 32:
757
29
      ZeroTy = Type::getFloatTy(Ctx);
758
29
      break;
759
1.58k
    case 64:
760
1.58k
      ZeroTy = Type::getDoubleTy(Ctx);
761
1.58k
      break;
762
0
    default:
763
0
      llvm_unreachable("unexpected floating-point type");
764
1.61k
    }
765
1.61k
    ConstantFP &ZeroForNegation =
766
1.61k
        *cast<ConstantFP>(ConstantFP::getZeroValueForNegation(ZeroTy));
767
1.61k
    unsigned Zero = MRI.createGenericVirtualRegister(Ty);
768
1.61k
    MIRBuilder.buildFConstant(Zero, ZeroForNegation);
769
1.61k
    MIRBuilder.buildInstr(TargetOpcode::G_FSUB)
770
1.61k
        .addDef(Res)
771
1.61k
        .addUse(Zero)
772
1.61k
        .addUse(MI.getOperand(1).getReg());
773
1.61k
    MI.eraseFromParent();
774
1.61k
    return Legalized;
775
1.61k
  }
776
0
  case TargetOpcode::G_FSUB: {
777
0
    // Lower (G_FSUB LHS, RHS) to (G_FADD LHS, (G_FNEG RHS)).
778
0
    // First, check if G_FNEG is marked as Lower. If so, we may
779
0
    // end up with an infinite loop as G_FSUB is used to legalize G_FNEG.
780
0
    if (LI.getAction({G_FNEG, Ty}).first == LegalizerInfo::Lower)
781
0
      return UnableToLegalize;
782
0
    unsigned Res = MI.getOperand(0).getReg();
783
0
    unsigned LHS = MI.getOperand(1).getReg();
784
0
    unsigned RHS = MI.getOperand(2).getReg();
785
0
    unsigned Neg = MRI.createGenericVirtualRegister(Ty);
786
0
    MIRBuilder.buildInstr(TargetOpcode::G_FNEG).addDef(Neg).addUse(RHS);
787
0
    MIRBuilder.buildInstr(TargetOpcode::G_FADD)
788
0
        .addDef(Res)
789
0
        .addUse(LHS)
790
0
        .addUse(Neg);
791
0
    MI.eraseFromParent();
792
0
    return Legalized;
793
0
  }
794
2.59k
  }
795
2.59k
}
796
797
LegalizerHelper::LegalizeResult
798
LegalizerHelper::fewerElementsVector(MachineInstr &MI, unsigned TypeIdx,
799
2.26k
                                     LLT NarrowTy) {
800
2.26k
  // FIXME: Don't know how to handle secondary types yet.
801
2.26k
  if (TypeIdx != 0)
802
0
    return UnableToLegalize;
803
2.26k
  switch (MI.getOpcode()) {
804
2.21k
  default:
805
2.21k
    return UnableToLegalize;
806
51
  case TargetOpcode::G_ADD: {
807
51
    unsigned NarrowSize = NarrowTy.getSizeInBits();
808
51
    unsigned DstReg = MI.getOperand(0).getReg();
809
51
    int NumParts = MRI.getType(DstReg).getSizeInBits() / NarrowSize;
810
51
811
51
    MIRBuilder.setInstr(MI);
812
51
813
51
    SmallVector<unsigned, 2> Src1Regs, Src2Regs, DstRegs;
814
51
    extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, Src1Regs);
815
51
    extractParts(MI.getOperand(2).getReg(), NarrowTy, NumParts, Src2Regs);
816
51
817
299
    for (int i = 0; 
i < NumParts299
;
++i248
) {
818
248
      unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy);
819
248
      MIRBuilder.buildAdd(DstReg, Src1Regs[i], Src2Regs[i]);
820
248
      DstRegs.push_back(DstReg);
821
248
    }
822
51
823
51
    MIRBuilder.buildMerge(DstReg, DstRegs);
824
51
    MI.eraseFromParent();
825
51
    return Legalized;
826
0
  }
827
0
  }
828
0
}