Coverage Report

Created: 2019-07-24 05:18

/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonShuffler.cpp
Line
Count
Source (jump to first uncovered line)
1
//===- HexagonShuffler.cpp - Instruction bundle shuffling -----------------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// This implements the shuffling of insns inside a bundle according to the
10
// packet formation rules of the Hexagon ISA.
11
//
12
//===----------------------------------------------------------------------===//
13
14
#define DEBUG_TYPE "hexagon-shuffle"
15
16
#include "MCTargetDesc/HexagonShuffler.h"
17
#include "MCTargetDesc/HexagonBaseInfo.h"
18
#include "MCTargetDesc/HexagonMCInstrInfo.h"
19
#include "MCTargetDesc/HexagonMCTargetDesc.h"
20
#include "llvm/ADT/SmallVector.h"
21
#include "llvm/ADT/Twine.h"
22
#include "llvm/MC/MCContext.h"
23
#include "llvm/MC/MCInst.h"
24
#include "llvm/MC/MCInstrDesc.h"
25
#include "llvm/MC/MCSubtargetInfo.h"
26
#include "llvm/Support/Compiler.h"
27
#include "llvm/Support/Debug.h"
28
#include "llvm/Support/MathExtras.h"
29
#include "llvm/Support/SourceMgr.h"
30
#include "llvm/Support/raw_ostream.h"
31
#include <algorithm>
32
#include <cassert>
33
#include <utility>
34
#include <vector>
35
36
using namespace llvm;
37
38
namespace {
39
40
// Insn shuffling priority.
41
class HexagonBid {
42
  // The priority is directly proportional to how restricted the insn is based
43
  // on its flexibility to run on the available slots.  So, the fewer slots it
44
  // may run on, the higher its priority.
45
  enum { MAX = 360360 }; // LCD of 1/2, 1/3, 1/4,... 1/15.
46
  unsigned Bid = 0;
47
48
public:
49
170k
  HexagonBid() = default;
50
183k
  HexagonBid(unsigned B) { Bid = B ? MAX / countPopulation(B) : 
00
; }
51
52
  // Check if the insn priority is overflowed.
53
183k
  bool isSold() const { return (Bid >= MAX); }
54
55
183k
  HexagonBid &operator+=(const HexagonBid &B) {
56
183k
    Bid += B.Bid;
57
183k
    return *this;
58
183k
  }
59
};
60
61
// Slot shuffling allocation.
62
class HexagonUnitAuction {
63
  HexagonBid Scores[HEXAGON_PACKET_SIZE];
64
  // Mask indicating which slot is unavailable.
65
  unsigned isSold : HEXAGON_PACKET_SIZE;
66
67
public:
68
42.6k
  HexagonUnitAuction(unsigned cs = 0) : isSold(cs) {}
69
70
  // Allocate slots.
71
93.8k
  bool bid(unsigned B) {
72
93.8k
    // Exclude already auctioned slots from the bid.
73
93.8k
    unsigned b = B & ~isSold;
74
93.8k
    if (b) {
75
469k
      for (unsigned i = 0; i < HEXAGON_PACKET_SIZE; 
++i375k
)
76
375k
        if (b & (1 << i)) {
77
183k
          // Request candidate slots.
78
183k
          Scores[i] += HexagonBid(b);
79
183k
          isSold |= Scores[i].isSold() << i;
80
183k
        }
81
93.8k
      return true;
82
93.8k
    } else
83
32
      // Error if the desired slots are already full.
84
32
      return false;
85
93.8k
  }
86
};
87
88
} // end anonymous namespace
89
90
667k
unsigned HexagonResource::setWeight(unsigned s) {
91
667k
  const unsigned SlotWeight = 8;
92
667k
  const unsigned MaskWeight = SlotWeight - 1;
93
667k
  unsigned Units = getUnits();
94
667k
  unsigned Key = ((1u << s) & Units) != 0;
95
667k
96
667k
  // Calculate relative weight of the insn for the given slot, weighing it the
97
667k
  // heavier the more restrictive the insn is and the lowest the slots that the
98
667k
  // insn may be executed in.
99
667k
  if (Key == 0 || 
Units == 0113k
||
(SlotWeight * s >= 32)113k
)
100
554k
    return Weight = 0;
101
113k
102
113k
  unsigned Ctpop = countPopulation(Units);
103
113k
  unsigned Cttz = countTrailingZeros(Units);
104
113k
  Weight = (1u << (SlotWeight * s)) * ((MaskWeight - Ctpop) << Cttz);
105
113k
  return Weight;
106
113k
}
107
108
100k
void HexagonCVIResource::SetupTUL(TypeUnitsAndLanes *TUL, StringRef CPU) {
109
100k
  (*TUL)[HexagonII::TypeCVI_VA] =
110
100k
      UnitsAndLanes(CVI_XLANE | CVI_SHIFT | CVI_MPY0 | CVI_MPY1, 1);
111
100k
  (*TUL)[HexagonII::TypeCVI_VA_DV] = UnitsAndLanes(CVI_XLANE | CVI_MPY0, 2);
112
100k
  (*TUL)[HexagonII::TypeCVI_VX] = UnitsAndLanes(CVI_MPY0 | CVI_MPY1, 1);
113
100k
  (*TUL)[HexagonII::TypeCVI_VX_LATE] = UnitsAndLanes(CVI_MPY0 | CVI_MPY1, 1);
114
100k
  (*TUL)[HexagonII::TypeCVI_VX_DV] = UnitsAndLanes(CVI_MPY0, 2);
115
100k
  (*TUL)[HexagonII::TypeCVI_VP] = UnitsAndLanes(CVI_XLANE, 1);
116
100k
  (*TUL)[HexagonII::TypeCVI_VP_VS] = UnitsAndLanes(CVI_XLANE, 2);
117
100k
  (*TUL)[HexagonII::TypeCVI_VS] = UnitsAndLanes(CVI_SHIFT, 1);
118
100k
  (*TUL)[HexagonII::TypeCVI_VS_VX] = UnitsAndLanes(CVI_XLANE | CVI_SHIFT, 1);
119
100k
  (*TUL)[HexagonII::TypeCVI_VINLANESAT] =
120
100k
      (CPU == "hexagonv60")
121
100k
          ? 
UnitsAndLanes(CVI_SHIFT, 1)51.5k
122
100k
          : 
UnitsAndLanes(CVI_XLANE | CVI_SHIFT | CVI_MPY0 | CVI_MPY1, 1)48.8k
;
123
100k
  (*TUL)[HexagonII::TypeCVI_VM_LD] =
124
100k
      UnitsAndLanes(CVI_XLANE | CVI_SHIFT | CVI_MPY0 | CVI_MPY1, 1);
125
100k
  (*TUL)[HexagonII::TypeCVI_VM_TMP_LD] = UnitsAndLanes(CVI_NONE, 0);
126
100k
  (*TUL)[HexagonII::TypeCVI_VM_VP_LDU] = UnitsAndLanes(CVI_XLANE, 1);
127
100k
  (*TUL)[HexagonII::TypeCVI_VM_ST] =
128
100k
      UnitsAndLanes(CVI_XLANE | CVI_SHIFT | CVI_MPY0 | CVI_MPY1, 1);
129
100k
  (*TUL)[HexagonII::TypeCVI_VM_NEW_ST] = UnitsAndLanes(CVI_NONE, 0);
130
100k
  (*TUL)[HexagonII::TypeCVI_VM_STU] = UnitsAndLanes(CVI_XLANE, 1);
131
100k
  (*TUL)[HexagonII::TypeCVI_HIST] = UnitsAndLanes(CVI_XLANE, 4);
132
100k
  (*TUL)[HexagonII::TypeCVI_GATHER] =
133
100k
      UnitsAndLanes(CVI_XLANE | CVI_SHIFT | CVI_MPY0 | CVI_MPY1, 1);
134
100k
  (*TUL)[HexagonII::TypeCVI_SCATTER] =
135
100k
      UnitsAndLanes(CVI_XLANE | CVI_SHIFT | CVI_MPY0 | CVI_MPY1, 1);
136
100k
  (*TUL)[HexagonII::TypeCVI_SCATTER_DV] =
137
100k
      UnitsAndLanes(CVI_XLANE | CVI_MPY0, 2);
138
100k
  (*TUL)[HexagonII::TypeCVI_SCATTER_NEW_ST] =
139
100k
      UnitsAndLanes(CVI_XLANE | CVI_SHIFT | CVI_MPY0 | CVI_MPY1, 1);
140
100k
  (*TUL)[HexagonII::TypeCVI_4SLOT_MPY] = UnitsAndLanes(CVI_XLANE, 4);
141
100k
  (*TUL)[HexagonII::TypeCVI_ZW] = UnitsAndLanes(CVI_ZW, 1);
142
100k
}
143
144
HexagonCVIResource::HexagonCVIResource(TypeUnitsAndLanes *TUL,
145
                                       MCInstrInfo const &MCII, unsigned s,
146
                                       MCInst const *id)
147
152k
    : HexagonResource(s) {
148
152k
  unsigned T = HexagonMCInstrInfo::getType(MCII, *id);
149
152k
150
152k
  if (TUL->count(T)) {
151
32.7k
    // For an HVX insn.
152
32.7k
    Valid = true;
153
32.7k
    setUnits((*TUL)[T].first);
154
32.7k
    setLanes((*TUL)[T].second);
155
32.7k
    setLoad(HexagonMCInstrInfo::getDesc(MCII, *id).mayLoad());
156
32.7k
    setStore(HexagonMCInstrInfo::getDesc(MCII, *id).mayStore());
157
119k
  } else {
158
119k
    // For core insns.
159
119k
    Valid = false;
160
119k
    setUnits(0);
161
119k
    setLanes(0);
162
119k
    setLoad(false);
163
119k
    setStore(false);
164
119k
  }
165
152k
}
166
167
struct CVIUnits {
168
  unsigned Units;
169
  unsigned Lanes;
170
};
171
using HVXInstsT = SmallVector<struct CVIUnits, 8>;
172
173
static unsigned makeAllBits(unsigned startBit, unsigned Lanes)
174
24.3k
{
175
28.7k
  for (unsigned i = 1; i < Lanes; 
++i4.35k
)
176
4.35k
    startBit = (startBit << 1) | startBit;
177
24.3k
  return startBit;
178
24.3k
}
179
180
static bool checkHVXPipes(const HVXInstsT &hvxInsts, unsigned startIdx,
181
34.4k
                          unsigned usedUnits) {
182
34.4k
  if (startIdx < hvxInsts.size()) {
183
20.0k
    if (!hvxInsts[startIdx].Units)
184
0
      return checkHVXPipes(hvxInsts, startIdx + 1, usedUnits);
185
32.8k
    
for (unsigned b = 0x1; 20.0k
b <= 0x8;
b <<= 112.7k
) {
186
32.8k
      if ((hvxInsts[startIdx].Units & b) == 0)
187
8.42k
        continue;
188
24.3k
      unsigned allBits = makeAllBits(b, hvxInsts[startIdx].Lanes);
189
24.3k
      if ((allBits & usedUnits) == 0) {
190
20.0k
        if (checkHVXPipes(hvxInsts, startIdx + 1, usedUnits | allBits))
191
20.0k
          return true;
192
20.0k
      }
193
24.3k
    }
194
20.0k
    
return false8
;
195
14.4k
  }
196
14.4k
  return true;
197
14.4k
}
198
199
HexagonShuffler::HexagonShuffler(MCContext &Context, bool ReportErrors,
200
                                 MCInstrInfo const &MCII,
201
                                 MCSubtargetInfo const &STI)
202
100k
    : Context(Context), MCII(MCII), STI(STI), ReportErrors(ReportErrors) {
203
100k
  reset();
204
100k
  HexagonCVIResource::SetupTUL(&TUL, STI.getCPU());
205
100k
}
206
207
100k
void HexagonShuffler::reset() {
208
100k
  Packet.clear();
209
100k
  BundleFlags = 0;
210
100k
}
211
212
void HexagonShuffler::append(MCInst const &ID, MCInst const *Extender,
213
152k
                             unsigned S) {
214
152k
  HexagonInstr PI(&TUL, MCII, &ID, Extender, S);
215
152k
216
152k
  Packet.push_back(PI);
217
152k
}
218
219
static struct {
220
  unsigned first;
221
  unsigned second;
222
} jumpSlots[] = {{8, 4}, {8, 2}, {8, 1}, {4, 2}, {4, 1}, {2, 1}};
223
25
#define MAX_JUMP_SLOTS (sizeof(jumpSlots) / sizeof(jumpSlots[0]))
224
225
42.9k
void HexagonShuffler::restrictSlot1AOK() {
226
42.9k
  bool HasRestrictSlot1AOK = false;
227
42.9k
  SMLoc RestrictLoc;
228
137k
  for (iterator ISJ = begin(); ISJ != end(); 
++ISJ94.5k
) {
229
94.5k
    MCInst const &Inst = ISJ->getDesc();
230
94.5k
    if (HexagonMCInstrInfo::isRestrictSlot1AOK(MCII, Inst)) {
231
8
      HasRestrictSlot1AOK = true;
232
8
      RestrictLoc = Inst.getLoc();
233
8
    }
234
94.5k
  }
235
42.9k
  if (HasRestrictSlot1AOK)
236
16
    
for (iterator ISJ = begin(); 8
ISJ != end();
++ISJ8
) {
237
8
      MCInst const &Inst = ISJ->getDesc();
238
8
      unsigned Type = HexagonMCInstrInfo::getType(MCII, Inst);
239
8
      if (Type != HexagonII::TypeALU32_2op &&
240
8
          Type != HexagonII::TypeALU32_3op &&
241
8
          Type != HexagonII::TypeALU32_ADDI) {
242
8
        unsigned Units = ISJ->Core.getUnits();
243
8
        if (Units & 2U) {
244
0
          AppliedRestrictions.push_back(std::make_pair(
245
0
              Inst.getLoc(),
246
0
              "Instruction was restricted from being in slot 1"));
247
0
          AppliedRestrictions.push_back(
248
0
              std::make_pair(RestrictLoc, "Instruction can only be combine "
249
0
                                          "with an ALU instruction in slot 1"));
250
0
          ISJ->Core.setUnits(Units & ~2U);
251
0
        }
252
8
      }
253
8
    }
254
42.9k
}
255
256
42.9k
void HexagonShuffler::restrictNoSlot1Store() {
257
42.9k
  bool HasRestrictNoSlot1Store = false;
258
42.9k
  SMLoc RestrictLoc;
259
137k
  for (iterator ISJ = begin(); ISJ != end(); 
++ISJ94.5k
) {
260
94.5k
    MCInst const &Inst = ISJ->getDesc();
261
94.5k
    if (HexagonMCInstrInfo::isRestrictNoSlot1Store(MCII, Inst)) {
262
8.29k
      HasRestrictNoSlot1Store = true;
263
8.29k
      RestrictLoc = Inst.getLoc();
264
8.29k
    }
265
94.5k
  }
266
42.9k
  if (HasRestrictNoSlot1Store) {
267
8.28k
    bool AppliedRestriction = false;
268
28.1k
    for (iterator ISJ = begin(); ISJ != end(); 
++ISJ19.8k
) {
269
19.8k
      MCInst const &Inst = ISJ->getDesc();
270
19.8k
      if (HexagonMCInstrInfo::getDesc(MCII, Inst).mayStore()) {
271
4.35k
        unsigned Units = ISJ->Core.getUnits();
272
4.35k
        if (Units & 2U) {
273
60
          AppliedRestriction = true;
274
60
          AppliedRestrictions.push_back(std::make_pair(
275
60
              Inst.getLoc(),
276
60
              "Instruction was restricted from being in slot 1"));
277
60
          ISJ->Core.setUnits(Units & ~2U);
278
60
        }
279
4.35k
      }
280
19.8k
    }
281
8.28k
    if (AppliedRestriction)
282
60
      AppliedRestrictions.push_back(std::make_pair(
283
60
          RestrictLoc, "Instruction does not allow a store in slot 1"));
284
8.28k
  }
285
42.9k
}
286
287
42.9k
void HexagonShuffler::applySlotRestrictions() {
288
42.9k
  restrictSlot1AOK();
289
42.9k
  restrictNoSlot1Store();
290
42.9k
}
291
292
/// Check that the packet is legal and enforce relative insn order.
293
42.9k
bool HexagonShuffler::check() {
294
42.9k
  // Descriptive slot masks.
295
42.9k
  const unsigned slotSingleLoad = 0x1, slotSingleStore = 0x1,
296
42.9k
                 slotThree = 0x8, // slotFirstJump = 0x8,
297
42.9k
                 slotFirstLoadStore = 0x2, slotLastLoadStore = 0x1;
298
42.9k
  // Highest slots for branches and stores used to keep their original order.
299
42.9k
  // unsigned slotJump = slotFirstJump;
300
42.9k
  unsigned slotLoadStore = slotFirstLoadStore;
301
42.9k
  // Number of memory operations, loads, solo loads, stores, solo stores, single
302
42.9k
  // stores.
303
42.9k
  unsigned memory = 0, loads = 0, load0 = 0, stores = 0, store0 = 0, store1 = 0;
304
42.9k
  unsigned NonZCVIloads = 0, AllCVIloads = 0, CVIstores = 0;
305
42.9k
  // Number of duplex insns
306
42.9k
  unsigned duplex = 0;
307
42.9k
  unsigned pSlot3Cnt = 0;
308
42.9k
  unsigned memops = 0;
309
42.9k
  iterator slot3ISJ = end();
310
42.9k
  std::vector<iterator> foundBranches;
311
42.9k
  unsigned reservedSlots = 0;
312
42.9k
313
42.9k
  // Collect information from the insns in the packet.
314
137k
  for (iterator ISJ = begin(); ISJ != end(); 
++ISJ94.5k
) {
315
94.5k
    MCInst const &ID = ISJ->getDesc();
316
94.5k
317
94.5k
    if (HexagonMCInstrInfo::prefersSlot3(MCII, ID)) {
318
7.91k
      ++pSlot3Cnt;
319
7.91k
      slot3ISJ = ISJ;
320
7.91k
    }
321
94.5k
    reservedSlots |= HexagonMCInstrInfo::getOtherReservedSlots(MCII, STI, ID);
322
94.5k
323
94.5k
    switch (HexagonMCInstrInfo::getType(MCII, ID)) {
324
94.5k
    case HexagonII::TypeS_2op:
325
9.04k
    case HexagonII::TypeS_3op:
326
9.04k
    case HexagonII::TypeALU64:
327
9.04k
      break;
328
14.3k
    case HexagonII::TypeJ:
329
14.3k
      foundBranches.push_back(ISJ);
330
14.3k
      break;
331
9.04k
    case HexagonII::TypeCVI_VM_VP_LDU:
332
3.83k
    case HexagonII::TypeCVI_VM_LD:
333
3.83k
    case HexagonII::TypeCVI_VM_TMP_LD:
334
3.83k
    case HexagonII::TypeCVI_GATHER:
335
3.83k
    case HexagonII::TypeCVI_GATHER_RST:
336
3.83k
      ++NonZCVIloads;
337
3.83k
      LLVM_FALLTHROUGH;
338
3.84k
    case HexagonII::TypeCVI_ZW:
339
3.84k
      ++AllCVIloads;
340
3.84k
      LLVM_FALLTHROUGH;
341
10.6k
    case HexagonII::TypeLD:
342
10.6k
      ++loads;
343
10.6k
      ++memory;
344
10.6k
      if (ISJ->Core.getUnits() == slotSingleLoad ||
345
10.6k
          
HexagonMCInstrInfo::getType(MCII, ID) == HexagonII::TypeCVI_VM_VP_LDU9.68k
)
346
984
        ++load0;
347
10.6k
      if (HexagonMCInstrInfo::getDesc(MCII, ID).isReturn())
348
400
        foundBranches.push_back(ISJ);
349
10.6k
      break;
350
3.84k
    case HexagonII::TypeCVI_VM_STU:
351
3.29k
    case HexagonII::TypeCVI_VM_ST:
352
3.29k
    case HexagonII::TypeCVI_VM_NEW_ST:
353
3.29k
    case HexagonII::TypeCVI_SCATTER:
354
3.29k
    case HexagonII::TypeCVI_SCATTER_DV:
355
3.29k
    case HexagonII::TypeCVI_SCATTER_RST:
356
3.29k
    case HexagonII::TypeCVI_SCATTER_NEW_RST:
357
3.29k
    case HexagonII::TypeCVI_SCATTER_NEW_ST:
358
3.29k
      ++CVIstores;
359
3.29k
      LLVM_FALLTHROUGH;
360
10.7k
    case HexagonII::TypeST:
361
10.7k
      ++stores;
362
10.7k
      ++memory;
363
10.7k
      if (ISJ->Core.getUnits() == slotSingleStore ||
364
10.7k
          
HexagonMCInstrInfo::getType(MCII, ID) == HexagonII::TypeCVI_VM_STU5.04k
)
365
5.65k
        ++store0;
366
10.7k
      break;
367
3.29k
    case HexagonII::TypeV4LDST:
368
1.30k
      ++loads;
369
1.30k
      ++stores;
370
1.30k
      ++store1;
371
1.30k
      ++memops;
372
1.30k
      ++memory;
373
1.30k
      break;
374
3.29k
    case HexagonII::TypeNCJ:
375
784
      ++memory; // NV insns are memory-like.
376
784
      foundBranches.push_back(ISJ);
377
784
      break;
378
3.29k
    case HexagonII::TypeV2LDST:
379
2.90k
      if (HexagonMCInstrInfo::getDesc(MCII, ID).mayLoad()) {
380
1.36k
        ++loads;
381
1.36k
        ++memory;
382
1.36k
        if (ISJ->Core.getUnits() == slotSingleLoad ||
383
1.36k
            HexagonMCInstrInfo::getType(MCII, ID) ==
384
1.36k
                HexagonII::TypeCVI_VM_VP_LDU)
385
0
          ++load0;
386
1.53k
      } else {
387
1.53k
        assert(HexagonMCInstrInfo::getDesc(MCII, ID).mayStore());
388
1.53k
        ++memory;
389
1.53k
        ++stores;
390
1.53k
      }
391
2.90k
      break;
392
3.29k
    case HexagonII::TypeCR:
393
1.84k
    // Legacy conditional branch predicated on a register.
394
1.84k
    case HexagonII::TypeCJ:
395
1.84k
      if (HexagonMCInstrInfo::getDesc(MCII, ID).isBranch())
396
506
        foundBranches.push_back(ISJ);
397
1.84k
      break;
398
1.84k
    case HexagonII::TypeDUPLEX: {
399
1.65k
      ++duplex;
400
1.65k
      MCInst const &Inst0 = *ID.getOperand(0).getInst();
401
1.65k
      MCInst const &Inst1 = *ID.getOperand(1).getInst();
402
1.65k
      if (HexagonMCInstrInfo::getDesc(MCII, Inst0).isBranch())
403
0
        foundBranches.push_back(ISJ);
404
1.65k
      if (HexagonMCInstrInfo::getDesc(MCII, Inst1).isBranch())
405
0
        foundBranches.push_back(ISJ);
406
1.65k
      if (HexagonMCInstrInfo::getDesc(MCII, Inst0).isReturn())
407
179
        foundBranches.push_back(ISJ);
408
1.65k
      if (HexagonMCInstrInfo::getDesc(MCII, Inst1).isReturn())
409
0
        foundBranches.push_back(ISJ);
410
1.65k
      break;
411
1.84k
    }
412
94.5k
    }
413
94.5k
  }
414
42.9k
  applySlotRestrictions();
415
42.9k
416
42.9k
  // Check if the packet is legal.
417
42.9k
  const unsigned ZCVIloads = AllCVIloads - NonZCVIloads;
418
42.9k
  const bool ValidHVXMem =
419
42.9k
      NonZCVIloads <= 1 && ZCVIloads <= 1 && CVIstores <= 1;
420
42.9k
  if ((load0 > 1 || store0 > 1 || !ValidHVXMem) ||
421
42.9k
      (duplex > 1 || (duplex && 
memory1.65k
))) {
422
304
    reportError(llvm::Twine("invalid instruction packet"));
423
304
    return false;
424
304
  }
425
42.6k
426
42.6k
  // Modify packet accordingly.
427
42.6k
  // TODO: need to reserve slots #0 and #1 for duplex insns.
428
42.6k
  bool bOnlySlot3 = false;
429
136k
  for (iterator ISJ = begin(); ISJ != end(); 
++ISJ93.8k
) {
430
93.8k
    MCInst const &ID = ISJ->getDesc();
431
93.8k
432
93.8k
    if (!ISJ->Core.getUnits()) {
433
0
      // Error if insn may not be executed in any slot.
434
0
      return false;
435
0
    }
436
93.8k
437
93.8k
    // A single load must use slot #0.
438
93.8k
    if (HexagonMCInstrInfo::getDesc(MCII, ID).mayLoad()) {
439
13.0k
      if (loads == 1 && 
loads == memory9.99k
&&
memops == 07.41k
)
440
6.12k
        // Pin the load to slot #0.
441
6.12k
        switch (ID.getOpcode()) {
442
6.12k
        case Hexagon::V6_vgathermw:
443
12
        case Hexagon::V6_vgathermh:
444
12
        case Hexagon::V6_vgathermhw:
445
12
        case Hexagon::V6_vgathermwq:
446
12
        case Hexagon::V6_vgathermhq:
447
12
        case Hexagon::V6_vgathermhwq:
448
12
          // Slot1 only loads
449
12
          break;
450
6.11k
        default:
451
6.11k
          ISJ->Core.setUnits(ISJ->Core.getUnits() & slotSingleLoad);
452
6.11k
          break;
453
6.89k
        }
454
6.89k
      else if (loads >= 1 && 
isMemReorderDisabled()6.86k
) { // }:mem_noshuf
455
4
        // Loads must keep the original order ONLY if
456
4
        // isMemReorderDisabled() == true
457
4
        if (slotLoadStore < slotLastLoadStore) {
458
0
          // Error if no more slots available for loads.
459
0
          reportError(
460
0
              llvm::Twine("invalid instruction packet: too many loads"));
461
0
          return false;
462
0
        }
463
4
        // Pin the load to the highest slot available to it.
464
4
        ISJ->Core.setUnits(ISJ->Core.getUnits() & slotLoadStore);
465
4
        // Update the next highest slot available to loads.
466
4
        slotLoadStore >>= 1;
467
4
      }
468
13.0k
    }
469
93.8k
470
93.8k
    // A single store must use slot #0.
471
93.8k
    if (HexagonMCInstrInfo::getDesc(MCII, ID).mayStore()) {
472
13.3k
      if (!store0) {
473
7.49k
        if (stores == 1 && 
(5.65k
loads == 05.65k
||
!isMemReorderDisabled()2.10k
))
474
5.65k
          // Pin the store to slot #0 only if isMemReorderDisabled() == false
475
5.65k
          ISJ->Core.setUnits(ISJ->Core.getUnits() & slotSingleStore);
476
1.84k
        else if (stores >= 1) {
477
1.83k
          if (slotLoadStore < slotLastLoadStore) {
478
0
            // Error if no more slots available for stores.
479
0
            reportError(Twine("invalid instruction packet: too many stores"));
480
0
            return false;
481
0
          }
482
1.83k
          // Pin the store to the highest slot available to it.
483
1.83k
          ISJ->Core.setUnits(ISJ->Core.getUnits() & slotLoadStore);
484
1.83k
          // Update the next highest slot available to stores.
485
1.83k
          slotLoadStore >>= 1;
486
1.83k
        }
487
7.49k
      }
488
13.3k
      if (store1 && 
stores > 11.30k
) {
489
0
        // Error if a single store with another store.
490
0
        reportError(Twine("invalid instruction packet: too many stores"));
491
0
        return false;
492
0
      }
493
93.8k
    }
494
93.8k
495
93.8k
    // flag if an instruction requires to be in slot 3
496
93.8k
    if (ISJ->Core.getUnits() == slotThree)
497
1.06k
      bOnlySlot3 = true;
498
93.8k
499
93.8k
    if (!ISJ->Core.getUnits()) {
500
0
      // Error if insn may not be executed in any slot.
501
0
      reportError(Twine("invalid instruction packet: out of slots"));
502
0
      return false;
503
0
    }
504
93.8k
  }
505
42.6k
506
42.6k
  // preserve branch order
507
42.6k
  bool validateSlots = true;
508
42.6k
  if (foundBranches.size() > 1) {
509
13
    if (foundBranches.size() > 2) {
510
0
      reportError(Twine("too many branches in packet"));
511
0
      return false;
512
0
    }
513
13
514
13
    // try all possible choices
515
25
    
for (unsigned int i = 0; 13
i < MAX_JUMP_SLOTS;
++i12
) {
516
24
      // validate first jump with this slot rule
517
24
      if (!(jumpSlots[i].first & foundBranches[0]->Core.getUnits()))
518
0
        continue;
519
24
520
24
      // validate second jump with this slot rule
521
24
      if (!(jumpSlots[i].second & foundBranches[1]->Core.getUnits()))
522
5
        continue;
523
19
524
19
      // both valid for this configuration, set new slot rules
525
19
      PacketSave = Packet;
526
19
      foundBranches[0]->Core.setUnits(jumpSlots[i].first);
527
19
      foundBranches[1]->Core.setUnits(jumpSlots[i].second);
528
19
529
19
      HexagonUnitAuction AuctionCore(reservedSlots);
530
19
      std::stable_sort(begin(), end(), HexagonInstr::lessCore);
531
19
532
19
      // see if things ok with that instruction being pinned to slot "slotJump"
533
19
      bool bFail = false;
534
76
      for (iterator I = begin(); I != end() && 
!bFail63
;
++I57
)
535
57
        if (!AuctionCore.bid(I->Core.getUnits()))
536
7
          bFail = true;
537
19
538
19
      // if yes, great, if not then restore original slot mask
539
19
      if (!bFail) {
540
12
        validateSlots = false; // all good, no need to re-do auction
541
12
        break;
542
12
      } else
543
7
        // restore original values
544
7
        Packet = PacketSave;
545
19
    }
546
13
    if (validateSlots) {
547
1
      reportError(Twine("invalid instruction packet: out of slots"));
548
1
      return false;
549
1
    }
550
42.6k
  }
551
42.6k
552
42.6k
  if (foundBranches.size() <= 1 && 
bOnlySlot3 == false42.6k
&&
pSlot3Cnt == 141.5k
&&
553
42.6k
      
slot3ISJ != end()6.28k
) {
554
6.28k
    validateSlots = true;
555
6.28k
    // save off slot mask of instruction marked with A_PREFER_SLOT3
556
6.28k
    // and then pin it to slot #3
557
6.28k
    unsigned saveUnits = slot3ISJ->Core.getUnits();
558
6.28k
    slot3ISJ->Core.setUnits(saveUnits & slotThree);
559
6.28k
560
6.28k
    HexagonUnitAuction AuctionCore(reservedSlots);
561
6.28k
    std::stable_sort(begin(), end(), HexagonInstr::lessCore);
562
6.28k
563
6.28k
    // see if things ok with that instruction being pinned to slot #3
564
6.28k
    bool bFail = false;
565
20.1k
    for (iterator I = begin(); I != end() && 
!bFail13.9k
;
++I13.9k
)
566
13.9k
      if (!AuctionCore.bid(I->Core.getUnits()))
567
25
        bFail = true;
568
6.28k
569
6.28k
    // if yes, great, if not then restore original slot mask
570
6.28k
    if (!bFail)
571
6.26k
      validateSlots = false; // all good, no need to re-do auction
572
25
    else
573
74
      
for (iterator ISJ = begin(); 25
ISJ != end();
++ISJ49
) {
574
49
        MCInst const &ID = ISJ->getDesc();
575
49
        if (HexagonMCInstrInfo::prefersSlot3(MCII, ID))
576
25
          ISJ->Core.setUnits(saveUnits);
577
49
      }
578
6.28k
  }
579
42.6k
580
42.6k
  // Check if any slot, core or CVI, is over-subscribed.
581
42.6k
  // Verify the core slot subscriptions.
582
42.6k
  if (validateSlots) {
583
36.3k
    HexagonUnitAuction AuctionCore(reservedSlots);
584
36.3k
585
36.3k
    std::stable_sort(begin(), end(), HexagonInstr::lessCore);
586
36.3k
587
116k
    for (iterator I = begin(); I != end(); 
++I79.9k
)
588
79.9k
      if (!AuctionCore.bid(I->Core.getUnits())) {
589
0
        reportError(Twine("invalid instruction packet: slot error"));
590
0
        return false;
591
0
      }
592
36.3k
  }
593
42.6k
  // Verify the CVI slot subscriptions.
594
42.6k
  std::stable_sort(begin(), end(), HexagonInstr::lessCVI);
595
42.6k
  // create vector of hvx instructions to check
596
42.6k
  HVXInstsT hvxInsts;
597
42.6k
  hvxInsts.clear();
598
136k
  for (iterator I = begin(); I != end(); 
++I93.8k
) {
599
93.8k
    struct CVIUnits inst;
600
93.8k
    inst.Units = I->CVI.getUnits();
601
93.8k
    inst.Lanes = I->CVI.getLanes();
602
93.8k
    if (inst.Units == 0)
603
73.8k
      continue; // not an hvx inst or an hvx inst that doesn't uses any pipes
604
20.0k
    hvxInsts.push_back(inst);
605
20.0k
  }
606
42.6k
  // if there are any hvx instructions in this packet, check pipe usage
607
42.6k
  if (hvxInsts.size() > 0) {
608
14.4k
    unsigned startIdx, usedUnits;
609
14.4k
    startIdx = usedUnits = 0x0;
610
14.4k
    if (!checkHVXPipes(hvxInsts, startIdx, usedUnits)) {
611
4
      // too many pipes used to be valid
612
4
      reportError(Twine("invalid instruction packet: slot error"));
613
4
      return false;
614
4
    }
615
42.6k
  }
616
42.6k
617
42.6k
  return true;
618
42.6k
}
619
620
93.2k
bool HexagonShuffler::shuffle() {
621
93.2k
  if (size() > HEXAGON_PACKET_SIZE) {
622
1
    // Ignore a packet with with more than what a packet can hold
623
1
    // or with compound or duplex insns for now.
624
1
    reportError(Twine("invalid instruction packet"));
625
1
    return false;
626
1
  }
627
93.2k
628
93.2k
  // Check and prepare packet.
629
93.2k
  bool Ok = true;
630
93.2k
  if (size() > 1 && 
(Ok = check())36.4k
)
631
36.1k
    // Reorder the handles for each slot.
632
180k
    
for (unsigned nSlot = 0, emptySlots = 0; 36.1k
nSlot < HEXAGON_PACKET_SIZE;
633
144k
         ++nSlot) {
634
144k
      iterator ISJ, ISK;
635
144k
      unsigned slotSkip, slotWeight;
636
144k
637
144k
      // Prioritize the handles considering their restrictions.
638
144k
      for (ISJ = ISK = Packet.begin(), slotSkip = slotWeight = 0;
639
487k
           ISK != Packet.end(); 
++ISK, ++slotSkip343k
)
640
343k
        if (slotSkip < nSlot - emptySlots)
641
151k
          // Note which handle to begin at.
642
151k
          ++ISJ;
643
191k
        else
644
191k
          // Calculate the weight of the slot.
645
191k
          slotWeight += ISK->Core.setWeight(HEXAGON_PACKET_SIZE - nSlot - 1);
646
144k
647
144k
      if (slotWeight)
648
85.7k
        // Sort the packet, favoring source order,
649
85.7k
        // beginning after the previous slot.
650
85.7k
        std::stable_sort(ISJ, Packet.end());
651
58.7k
      else
652
58.7k
        // Skip unused slot.
653
58.7k
        ++emptySlots;
654
144k
    }
655
93.2k
656
236k
  for (iterator ISJ = begin(); ISJ != end(); 
++ISJ143k
)
657
93.2k
    LLVM_DEBUG(dbgs().write_hex(ISJ->Core.getUnits()); if (ISJ->CVI.isValid()) {
658
93.2k
      dbgs() << '/';
659
93.2k
      dbgs().write_hex(ISJ->CVI.getUnits()) << '|';
660
93.2k
      dbgs() << ISJ->CVI.getLanes();
661
93.2k
    } dbgs() << ':'
662
93.2k
             << HexagonMCInstrInfo::getDesc(MCII, ISJ->getDesc()).getOpcode();
663
93.2k
               dbgs() << '\n');
664
93.2k
  LLVM_DEBUG(dbgs() << '\n');
665
93.2k
666
93.2k
  return Ok;
667
93.2k
}
668
669
310
void HexagonShuffler::reportError(Twine const &Msg) {
670
310
  if (ReportErrors) {
671
1
    for (auto const &I : AppliedRestrictions) {
672
0
      auto SM = Context.getSourceManager();
673
0
      if (SM)
674
0
        SM->PrintMessage(I.first, SourceMgr::DK_Note, I.second);
675
0
    }
676
1
    Context.reportError(Loc, Msg);
677
1
  }
678
310
}