Coverage Report

Created: 2019-07-24 05:18

/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/tools/clang/lib/CodeGen/SwiftCallingConv.cpp
Line
Count
Source (jump to first uncovered line)
1
//===--- SwiftCallingConv.cpp - Lowering for the Swift calling convention -===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// Implementation of the abstract lowering for the Swift calling convention.
10
//
11
//===----------------------------------------------------------------------===//
12
13
#include "clang/CodeGen/SwiftCallingConv.h"
14
#include "clang/Basic/TargetInfo.h"
15
#include "CodeGenModule.h"
16
#include "TargetInfo.h"
17
18
using namespace clang;
19
using namespace CodeGen;
20
using namespace swiftcall;
21
22
2.53k
static const SwiftABIInfo &getSwiftABIInfo(CodeGenModule &CGM) {
23
2.53k
  return cast<SwiftABIInfo>(CGM.getTargetCodeGenInfo().getABIInfo());
24
2.53k
}
25
26
10.2k
static bool isPowerOf2(unsigned n) {
27
10.2k
  return n == (n & -n);
28
10.2k
}
29
30
/// Given two types with the same size, try to find a common type.
31
32
static llvm::Type *getCommonType(llvm::Type *first, llvm::Type *second) {
32
32
  assert(first != second);
33
32
34
32
  // Allow pointers to merge with integers, but prefer the integer type.
35
32
  if (first->isIntegerTy()) {
36
32
    if (second->isPointerTy()) 
return first0
;
37
0
  } else if (first->isPointerTy()) {
38
0
    if (second->isIntegerTy()) return second;
39
0
    if (second->isPointerTy()) return first;
40
0
41
0
  // Allow two vectors to be merged (given that they have the same size).
42
0
  // This assumes that we never have two different vector register sets.
43
0
  } else if (auto firstVecTy = dyn_cast<llvm::VectorType>(first)) {
44
0
    if (auto secondVecTy = dyn_cast<llvm::VectorType>(second)) {
45
0
      if (auto commonTy = getCommonType(firstVecTy->getElementType(),
46
0
                                        secondVecTy->getElementType())) {
47
0
        return (commonTy == firstVecTy->getElementType() ? first : second);
48
0
      }
49
32
    }
50
0
  }
51
32
52
32
  return nullptr;
53
32
}
54
55
21.6k
static CharUnits getTypeStoreSize(CodeGenModule &CGM, llvm::Type *type) {
56
21.6k
  return CharUnits::fromQuantity(CGM.getDataLayout().getTypeStoreSize(type));
57
21.6k
}
58
59
1.91k
static CharUnits getTypeAllocSize(CodeGenModule &CGM, llvm::Type *type) {
60
1.91k
  return CharUnits::fromQuantity(CGM.getDataLayout().getTypeAllocSize(type));
61
1.91k
}
62
63
11.2k
void SwiftAggLowering::addTypedData(QualType type, CharUnits begin) {
64
11.2k
  // Deal with various aggregate types as special cases:
65
11.2k
66
11.2k
  // Record types.
67
11.2k
  if (auto recType = type->getAs<RecordType>()) {
68
28
    addTypedData(recType->getDecl(), begin);
69
28
70
28
  // Array types.
71
11.2k
  } else if (type->isArrayType()) {
72
34
    // Incomplete array types (flexible array members?) don't provide
73
34
    // data to lay out, and the other cases shouldn't be possible.
74
34
    auto arrayType = CGM.getContext().getAsConstantArrayType(type);
75
34
    if (!arrayType) 
return0
;
76
34
77
34
    QualType eltType = arrayType->getElementType();
78
34
    auto eltSize = CGM.getContext().getTypeSizeInChars(eltType);
79
8.32k
    for (uint64_t i = 0, e = arrayType->getSize().getZExtValue(); i != e; 
++i8.29k
) {
80
8.29k
      addTypedData(eltType, begin + i * eltSize);
81
8.29k
    }
82
34
83
34
  // Complex types.
84
11.1k
  } else if (auto complexType = type->getAs<ComplexType>()) {
85
0
    auto eltType = complexType->getElementType();
86
0
    auto eltSize = CGM.getContext().getTypeSizeInChars(eltType);
87
0
    auto eltLLVMType = CGM.getTypes().ConvertType(eltType);
88
0
    addTypedData(eltLLVMType, begin, begin + eltSize);
89
0
    addTypedData(eltLLVMType, begin + eltSize, begin + 2 * eltSize);
90
0
91
0
  // Member pointer types.
92
11.1k
  } else if (type->getAs<MemberPointerType>()) {
93
0
    // Just add it all as opaque.
94
0
    addOpaqueData(begin, begin + CGM.getContext().getTypeSizeInChars(type));
95
0
96
0
  // Everything else is scalar and should not convert as an LLVM aggregate.
97
11.1k
  } else {
98
11.1k
    // We intentionally convert as !ForMem because we want to preserve
99
11.1k
    // that a type was an i1.
100
11.1k
    auto llvmType = CGM.getTypes().ConvertType(type);
101
11.1k
    addTypedData(llvmType, begin);
102
11.1k
  }
103
11.2k
}
104
105
28
void SwiftAggLowering::addTypedData(const RecordDecl *record, CharUnits begin) {
106
28
  addTypedData(record, begin, CGM.getContext().getASTRecordLayout(record));
107
28
}
108
109
void SwiftAggLowering::addTypedData(const RecordDecl *record, CharUnits begin,
110
1.04k
                                    const ASTRecordLayout &layout) {
111
1.04k
  // Unions are a special case.
112
1.04k
  if (record->isUnion()) {
113
160
    for (auto field : record->fields()) {
114
160
      if (field->isBitField()) {
115
0
        addBitFieldData(field, begin, 0);
116
160
      } else {
117
160
        addTypedData(field->getType(), begin);
118
160
      }
119
160
    }
120
80
    return;
121
80
  }
122
965
123
965
  // Note that correctness does not rely on us adding things in
124
965
  // their actual order of layout; it's just somewhat more efficient
125
965
  // for the builder.
126
965
127
965
  // With that in mind, add "early" C++ data.
128
965
  auto cxxRecord = dyn_cast<CXXRecordDecl>(record);
129
965
  if (cxxRecord) {
130
9
    //   - a v-table pointer, if the class adds its own
131
9
    if (layout.hasOwnVFPtr()) {
132
0
      addTypedData(CGM.Int8PtrTy, begin);
133
0
    }
134
9
135
9
    //   - non-virtual bases
136
9
    for (auto &baseSpecifier : cxxRecord->bases()) {
137
0
      if (baseSpecifier.isVirtual()) continue;
138
0
139
0
      auto baseRecord = baseSpecifier.getType()->getAsCXXRecordDecl();
140
0
      addTypedData(baseRecord, begin + layout.getBaseClassOffset(baseRecord));
141
0
    }
142
9
143
9
    //   - a vbptr if the class adds its own
144
9
    if (layout.hasOwnVBPtr()) {
145
0
      addTypedData(CGM.Int8PtrTy, begin + layout.getVBPtrOffset());
146
0
    }
147
9
  }
148
965
149
965
  // Add fields.
150
2.75k
  for (auto field : record->fields()) {
151
2.75k
    auto fieldOffsetInBits = layout.getFieldOffset(field->getFieldIndex());
152
2.75k
    if (field->isBitField()) {
153
0
      addBitFieldData(field, begin, fieldOffsetInBits);
154
2.75k
    } else {
155
2.75k
      addTypedData(field->getType(),
156
2.75k
              begin + CGM.getContext().toCharUnitsFromBits(fieldOffsetInBits));
157
2.75k
    }
158
2.75k
  }
159
965
160
965
  // Add "late" C++ data:
161
965
  if (cxxRecord) {
162
9
    //   - virtual bases
163
9
    for (auto &vbaseSpecifier : cxxRecord->vbases()) {
164
0
      auto baseRecord = vbaseSpecifier.getType()->getAsCXXRecordDecl();
165
0
      addTypedData(baseRecord, begin + layout.getVBaseClassOffset(baseRecord));
166
0
    }
167
9
  }
168
965
}
169
170
void SwiftAggLowering::addBitFieldData(const FieldDecl *bitfield,
171
                                       CharUnits recordBegin,
172
0
                                       uint64_t bitfieldBitBegin) {
173
0
  assert(bitfield->isBitField());
174
0
  auto &ctx = CGM.getContext();
175
0
  auto width = bitfield->getBitWidthValue(ctx);
176
0
177
0
  // We can ignore zero-width bit-fields.
178
0
  if (width == 0) return;
179
0
180
0
  // toCharUnitsFromBits rounds down.
181
0
  CharUnits bitfieldByteBegin = ctx.toCharUnitsFromBits(bitfieldBitBegin);
182
0
183
0
  // Find the offset of the last byte that is partially occupied by the
184
0
  // bit-field; since we otherwise expect exclusive ends, the end is the
185
0
  // next byte.
186
0
  uint64_t bitfieldBitLast = bitfieldBitBegin + width - 1;
187
0
  CharUnits bitfieldByteEnd =
188
0
    ctx.toCharUnitsFromBits(bitfieldBitLast) + CharUnits::One();
189
0
  addOpaqueData(recordBegin + bitfieldByteBegin,
190
0
                recordBegin + bitfieldByteEnd);
191
0
}
192
193
11.1k
void SwiftAggLowering::addTypedData(llvm::Type *type, CharUnits begin) {
194
11.1k
  assert(type && "didn't provide type for typed data");
195
11.1k
  addTypedData(type, begin, begin + getTypeStoreSize(CGM, type));
196
11.1k
}
197
198
void SwiftAggLowering::addTypedData(llvm::Type *type,
199
11.1k
                                    CharUnits begin, CharUnits end) {
200
11.1k
  assert(type && "didn't provide type for typed data");
201
11.1k
  assert(getTypeStoreSize(CGM, type) == end - begin);
202
11.1k
203
11.1k
  // Legalize vector types.
204
11.1k
  if (auto vecTy = dyn_cast<llvm::VectorType>(type)) {
205
1.31k
    SmallVector<llvm::Type*, 4> componentTys;
206
1.31k
    legalizeVectorType(CGM, end - begin, vecTy, componentTys);
207
1.31k
    assert(componentTys.size() >= 1);
208
1.31k
209
1.31k
    // Walk the initial components.
210
1.47k
    for (size_t i = 0, e = componentTys.size(); i != e - 1; 
++i158
) {
211
158
      llvm::Type *componentTy = componentTys[i];
212
158
      auto componentSize = getTypeStoreSize(CGM, componentTy);
213
158
      assert(componentSize < end - begin);
214
158
      addLegalTypedData(componentTy, begin, begin + componentSize);
215
158
      begin += componentSize;
216
158
    }
217
1.31k
218
1.31k
    return addLegalTypedData(componentTys.back(), begin, end);
219
1.31k
  }
220
9.87k
221
9.87k
  // Legalize integer types.
222
9.87k
  if (auto intTy = dyn_cast<llvm::IntegerType>(type)) {
223
9.39k
    if (!isLegalIntegerType(CGM, intTy))
224
0
      return addOpaqueData(begin, end);
225
9.87k
  }
226
9.87k
227
9.87k
  // All other types should be legal.
228
9.87k
  return addLegalTypedData(type, begin, end);
229
9.87k
}
230
231
void SwiftAggLowering::addLegalTypedData(llvm::Type *type,
232
11.3k
                                         CharUnits begin, CharUnits end) {
233
11.3k
  // Require the type to be naturally aligned.
234
11.3k
  if (!begin.isZero() && 
!begin.isMultipleOf(getNaturalAlignment(CGM, type))10.2k
) {
235
32
236
32
    // Try splitting vector types.
237
32
    if (auto vecTy = dyn_cast<llvm::VectorType>(type)) {
238
16
      auto split = splitLegalVectorType(CGM, end - begin, vecTy);
239
16
      auto eltTy = split.first;
240
16
      auto numElts = split.second;
241
16
242
16
      auto eltSize = (end - begin) / numElts;
243
16
      assert(eltSize == getTypeStoreSize(CGM, eltTy));
244
54
      for (size_t i = 0, e = numElts; i != e; 
++i38
) {
245
38
        addLegalTypedData(eltTy, begin, begin + eltSize);
246
38
        begin += eltSize;
247
38
      }
248
16
      assert(begin == end);
249
16
      return;
250
16
    }
251
16
252
16
    return addOpaqueData(begin, end);
253
16
  }
254
11.3k
255
11.3k
  addEntry(type, begin, end);
256
11.3k
}
257
258
void SwiftAggLowering::addEntry(llvm::Type *type,
259
11.5k
                                CharUnits begin, CharUnits end) {
260
11.5k
  assert((!type ||
261
11.5k
          (!isa<llvm::StructType>(type) && !isa<llvm::ArrayType>(type))) &&
262
11.5k
         "cannot add aggregate-typed data");
263
11.5k
  assert(!type || begin.isMultipleOf(getNaturalAlignment(CGM, type)));
264
11.5k
265
11.5k
  // Fast path: we can just add entries to the end.
266
11.5k
  if (Entries.empty() || 
Entries.back().End <= begin10.4k
) {
267
11.3k
    Entries.push_back({begin, end, type});
268
11.3k
    return;
269
11.3k
  }
270
168
271
168
  // Find the first existing entry that ends after the start of the new data.
272
168
  // TODO: do a binary search if Entries is big enough for it to matter.
273
168
  size_t index = Entries.size() - 1;
274
208
  while (index != 0) {
275
88
    if (Entries[index - 1].End <= begin) 
break48
;
276
40
    --index;
277
40
  }
278
168
279
168
  // The entry ends after the start of the new data.
280
168
  // If the entry starts after the end of the new data, there's no conflict.
281
168
  if (Entries[index].Begin >= end) {
282
24
    // This insertion is potentially O(n), but the way we generally build
283
24
    // these layouts makes that unlikely to matter: we'd need a union of
284
24
    // several very large types.
285
24
    Entries.insert(Entries.begin() + index, {begin, end, type});
286
24
    return;
287
24
  }
288
168
289
168
  // Otherwise, the ranges overlap.  The new range might also overlap
290
168
  // with later ranges.
291
168
restartAfterSplit:
292
168
293
168
  // Simplest case: an exact overlap.
294
168
  if (Entries[index].Begin == begin && 
Entries[index].End == end144
) {
295
72
    // If the types match exactly, great.
296
72
    if (Entries[index].Type == type) 
return40
;
297
32
298
32
    // If either type is opaque, make the entry opaque and return.
299
32
    if (Entries[index].Type == nullptr) {
300
0
      return;
301
32
    } else if (type == nullptr) {
302
0
      Entries[index].Type = nullptr;
303
0
      return;
304
0
    }
305
32
306
32
    // If they disagree in an ABI-agnostic way, just resolve the conflict
307
32
    // arbitrarily.
308
32
    if (auto entryType = getCommonType(Entries[index].Type, type)) {
309
0
      Entries[index].Type = entryType;
310
0
      return;
311
0
    }
312
32
313
32
    // Otherwise, make the entry opaque.
314
32
    Entries[index].Type = nullptr;
315
32
    return;
316
32
  }
317
96
318
96
  // Okay, we have an overlapping conflict of some sort.
319
96
320
96
  // If we have a vector type, split it.
321
96
  if (auto vecTy = dyn_cast_or_null<llvm::VectorType>(type)) {
322
40
    auto eltTy = vecTy->getElementType();
323
40
    CharUnits eltSize = (end - begin) / vecTy->getNumElements();
324
40
    assert(eltSize == getTypeStoreSize(CGM, eltTy));
325
188
    for (unsigned i = 0, e = vecTy->getNumElements(); i != e; 
++i148
) {
326
148
      addEntry(eltTy, begin, begin + eltSize);
327
148
      begin += eltSize;
328
148
    }
329
40
    assert(begin == end);
330
40
    return;
331
40
  }
332
56
333
56
  // If the entry is a vector type, split it and try again.
334
56
  if (Entries[index].Type && Entries[index].Type->isVectorTy()) {
335
24
    splitVectorEntry(index);
336
24
    goto restartAfterSplit;
337
24
  }
338
32
339
32
  // Okay, we have no choice but to make the existing entry opaque.
340
32
341
32
  Entries[index].Type = nullptr;
342
32
343
32
  // Stretch the start of the entry to the beginning of the range.
344
32
  if (begin < Entries[index].Begin) {
345
16
    Entries[index].Begin = begin;
346
16
    assert(index == 0 || begin >= Entries[index - 1].End);
347
16
  }
348
32
349
32
  // Stretch the end of the entry to the end of the range; but if we run
350
32
  // into the start of the next entry, just leave the range there and repeat.
351
32
  while (end > Entries[index].End) {
352
16
    assert(Entries[index].Type == nullptr);
353
16
354
16
    // If the range doesn't overlap the next entry, we're done.
355
16
    if (index == Entries.size() - 1 || 
end <= Entries[index + 1].Begin0
) {
356
16
      Entries[index].End = end;
357
16
      break;
358
16
    }
359
0
360
0
    // Otherwise, stretch to the start of the next entry.
361
0
    Entries[index].End = Entries[index + 1].Begin;
362
0
363
0
    // Continue with the next entry.
364
0
    index++;
365
0
366
0
    // This entry needs to be made opaque if it is not already.
367
0
    if (Entries[index].Type == nullptr)
368
0
      continue;
369
0
370
0
    // Split vector entries unless we completely subsume them.
371
0
    if (Entries[index].Type->isVectorTy() &&
372
0
        end < Entries[index].End) {
373
0
      splitVectorEntry(index);
374
0
    }
375
0
376
0
    // Make the entry opaque.
377
0
    Entries[index].Type = nullptr;
378
0
  }
379
32
}
380
381
/// Replace the entry of vector type at offset 'index' with a sequence
382
/// of its component vectors.
383
24
void SwiftAggLowering::splitVectorEntry(unsigned index) {
384
24
  auto vecTy = cast<llvm::VectorType>(Entries[index].Type);
385
24
  auto split = splitLegalVectorType(CGM, Entries[index].getWidth(), vecTy);
386
24
387
24
  auto eltTy = split.first;
388
24
  CharUnits eltSize = getTypeStoreSize(CGM, eltTy);
389
24
  auto numElts = split.second;
390
24
  Entries.insert(Entries.begin() + index + 1, numElts - 1, StorageEntry());
391
24
392
24
  CharUnits begin = Entries[index].Begin;
393
88
  for (unsigned i = 0; i != numElts; 
++i64
) {
394
64
    Entries[index].Type = eltTy;
395
64
    Entries[index].Begin = begin;
396
64
    Entries[index].End = begin + eltSize;
397
64
    begin += eltSize;
398
64
  }
399
24
}
400
401
/// Given a power-of-two unit size, return the offset of the aligned unit
402
/// of that size which contains the given offset.
403
///
404
/// In other words, round down to the nearest multiple of the unit size.
405
28.6k
static CharUnits getOffsetAtStartOfUnit(CharUnits offset, CharUnits unitSize) {
406
28.6k
  assert(isPowerOf2(unitSize.getQuantity()));
407
28.6k
  auto unitMask = ~(unitSize.getQuantity() - 1);
408
28.6k
  return CharUnits::fromQuantity(offset.getQuantity() & unitMask);
409
28.6k
}
410
411
static bool areBytesInSameUnit(CharUnits first, CharUnits second,
412
10.3k
                               CharUnits chunkSize) {
413
10.3k
  return getOffsetAtStartOfUnit(first, chunkSize)
414
10.3k
      == getOffsetAtStartOfUnit(second, chunkSize);
415
10.3k
}
416
417
14.5k
static bool isMergeableEntryType(llvm::Type *type) {
418
14.5k
  // Opaquely-typed memory is always mergeable.
419
14.5k
  if (type == nullptr) 
return true5.59k
;
420
8.96k
421
8.96k
  // Pointers and integers are always mergeable.  In theory we should not
422
8.96k
  // merge pointers, but (1) it doesn't currently matter in practice because
423
8.96k
  // the chunk size is never greater than the size of a pointer and (2)
424
8.96k
  // Swift IRGen uses integer types for a lot of things that are "really"
425
8.96k
  // just storing pointers (like Optional<SomePointer>).  If we ever have a
426
8.96k
  // target that would otherwise combine pointers, we should put some effort
427
8.96k
  // into fixing those cases in Swift IRGen and then call out pointer types
428
8.96k
  // here.
429
8.96k
430
8.96k
  // Floating-point and vector types should never be merged.
431
8.96k
  // Most such types are too large and highly-aligned to ever trigger merging
432
8.96k
  // in practice, but it's important for the rule to cover at least 'half'
433
8.96k
  // and 'float', as well as things like small vectors of 'i1' or 'i8'.
434
8.96k
  return (!type->isFloatingPointTy() && 
!type->isVectorTy()8.89k
);
435
8.96k
}
436
437
bool SwiftAggLowering::shouldMergeEntries(const StorageEntry &first,
438
                                          const StorageEntry &second,
439
10.3k
                                          CharUnits chunkSize) {
440
10.3k
  // Only merge entries that overlap the same chunk.  We test this first
441
10.3k
  // despite being a bit more expensive because this is the condition that
442
10.3k
  // tends to prevent merging.
443
10.3k
  if (!areBytesInSameUnit(first.End - CharUnits::One(), second.Begin,
444
10.3k
                          chunkSize))
445
3.05k
    return false;
446
7.31k
447
7.31k
  return (isMergeableEntryType(first.Type) &&
448
7.31k
          
isMergeableEntryType(second.Type)7.24k
);
449
7.31k
}
450
451
1.06k
void SwiftAggLowering::finish() {
452
1.06k
  if (Entries.empty()) {
453
18
    Finished = true;
454
18
    return;
455
18
  }
456
1.04k
457
1.04k
  // We logically split the layout down into a series of chunks of this size,
458
1.04k
  // which is generally the size of a pointer.
459
1.04k
  const CharUnits chunkSize = getMaximumVoluntaryIntegerSize(CGM);
460
1.04k
461
1.04k
  // First pass: if two entries should be merged, make them both opaque
462
1.04k
  // and stretch one to meet the next.
463
1.04k
  // Also, remember if there are any opaque entries.
464
1.04k
  bool hasOpaqueEntries = (Entries[0].Type == nullptr);
465
11.4k
  for (size_t i = 1, e = Entries.size(); i != e; 
++i10.3k
) {
466
10.3k
    if (shouldMergeEntries(Entries[i - 1], Entries[i], chunkSize)) {
467
7.24k
      Entries[i - 1].Type = nullptr;
468
7.24k
      Entries[i].Type = nullptr;
469
7.24k
      Entries[i - 1].End = Entries[i].Begin;
470
7.24k
      hasOpaqueEntries = true;
471
7.24k
472
7.24k
    } else 
if (3.12k
Entries[i].Type == nullptr3.12k
) {
473
54
      hasOpaqueEntries = true;
474
54
    }
475
10.3k
  }
476
1.04k
477
1.04k
  // The rest of the algorithm leaves non-opaque entries alone, so if we
478
1.04k
  // have no opaque entries, we're done.
479
1.04k
  if (!hasOpaqueEntries) {
480
797
    Finished = true;
481
797
    return;
482
797
  }
483
250
484
250
  // Okay, move the entries to a temporary and rebuild Entries.
485
250
  auto orig = std::move(Entries);
486
250
  assert(Entries.empty());
487
250
488
678
  for (size_t i = 0, e = orig.size(); i != e; 
++i428
) {
489
428
    // Just copy over non-opaque entries.
490
428
    if (orig[i].Type != nullptr) {
491
126
      Entries.push_back(orig[i]);
492
126
      continue;
493
126
    }
494
302
495
302
    // Scan forward to determine the full extent of the next opaque range.
496
302
    // We know from the first pass that only contiguous ranges will overlap
497
302
    // the same aligned chunk.
498
302
    auto begin = orig[i].Begin;
499
302
    auto end = orig[i].End;
500
9.01k
    while (i + 1 != e &&
501
9.01k
           
orig[i + 1].Type == nullptr8.83k
&&
502
9.01k
           
end == orig[i + 1].Begin8.75k
) {
503
8.71k
      end = orig[i + 1].End;
504
8.71k
      i++;
505
8.71k
    }
506
302
507
302
    // Add an entry per intersected chunk.
508
1.77k
    do {
509
1.77k
      // Find the smallest aligned storage unit in the maximal aligned
510
1.77k
      // storage unit containing 'begin' that contains all the bytes in
511
1.77k
      // the intersection between the range and this chunk.
512
1.77k
      CharUnits localBegin = begin;
513
1.77k
      CharUnits chunkBegin = getOffsetAtStartOfUnit(localBegin, chunkSize);
514
1.77k
      CharUnits chunkEnd = chunkBegin + chunkSize;
515
1.77k
      CharUnits localEnd = std::min(end, chunkEnd);
516
1.77k
517
1.77k
      // Just do a simple loop over ever-increasing unit sizes.
518
1.77k
      CharUnits unitSize = CharUnits::One();
519
1.77k
      CharUnits unitBegin, unitEnd;
520
6.10k
      for (; ; 
unitSize *= 24.33k
) {
521
6.10k
        assert(unitSize <= chunkSize);
522
6.10k
        unitBegin = getOffsetAtStartOfUnit(localBegin, unitSize);
523
6.10k
        unitEnd = unitBegin + unitSize;
524
6.10k
        if (unitEnd >= localEnd) 
break1.77k
;
525
6.10k
      }
526
1.77k
527
1.77k
      // Add an entry for this unit.
528
1.77k
      auto entryTy =
529
1.77k
        llvm::IntegerType::get(CGM.getLLVMContext(),
530
1.77k
                               CGM.getContext().toBits(unitSize));
531
1.77k
      Entries.push_back({unitBegin, unitEnd, entryTy});
532
1.77k
533
1.77k
      // The next chunk starts where this chunk left off.
534
1.77k
      begin = localEnd;
535
1.77k
    } while (begin != end);
536
302
  }
537
250
538
250
  // Okay, finally finished.
539
250
  Finished = true;
540
250
}
541
542
0
void SwiftAggLowering::enumerateComponents(EnumerationCallback callback) const {
543
0
  assert(Finished && "haven't yet finished lowering");
544
0
545
0
  for (auto &entry : Entries) {
546
0
    callback(entry.Begin, entry.End, entry.Type);
547
0
  }
548
0
}
549
550
std::pair<llvm::StructType*, llvm::Type*>
551
877
SwiftAggLowering::getCoerceAndExpandTypes() const {
552
877
  assert(Finished && "haven't yet finished lowering");
553
877
554
877
  auto &ctx = CGM.getLLVMContext();
555
877
556
877
  if (Entries.empty()) {
557
0
    auto type = llvm::StructType::get(ctx);
558
0
    return { type, type };
559
0
  }
560
877
561
877
  SmallVector<llvm::Type*, 8> elts;
562
877
  CharUnits lastEnd = CharUnits::Zero();
563
877
  bool hasPadding = false;
564
877
  bool packed = false;
565
1.91k
  for (auto &entry : Entries) {
566
1.91k
    if (entry.Begin != lastEnd) {
567
14
      auto paddingSize = entry.Begin - lastEnd;
568
14
      assert(!paddingSize.isNegative());
569
14
570
14
      auto padding = llvm::ArrayType::get(llvm::Type::getInt8Ty(ctx),
571
14
                                          paddingSize.getQuantity());
572
14
      elts.push_back(padding);
573
14
      hasPadding = true;
574
14
    }
575
1.91k
576
1.91k
    if (!packed && !entry.Begin.isMultipleOf(
577
1.91k
          CharUnits::fromQuantity(
578
1.91k
            CGM.getDataLayout().getABITypeAlignment(entry.Type))))
579
0
      packed = true;
580
1.91k
581
1.91k
    elts.push_back(entry.Type);
582
1.91k
583
1.91k
    lastEnd = entry.Begin + getTypeAllocSize(CGM, entry.Type);
584
1.91k
    assert(entry.End <= lastEnd);
585
1.91k
  }
586
877
587
877
  // We don't need to adjust 'packed' to deal with possible tail padding
588
877
  // because we never do that kind of access through the coercion type.
589
877
  auto coercionType = llvm::StructType::get(ctx, elts, packed);
590
877
591
877
  llvm::Type *unpaddedType = coercionType;
592
877
  if (hasPadding) {
593
12
    elts.clear();
594
44
    for (auto &entry : Entries) {
595
44
      elts.push_back(entry.Type);
596
44
    }
597
12
    if (elts.size() == 1) {
598
0
      unpaddedType = elts[0];
599
12
    } else {
600
12
      unpaddedType = llvm::StructType::get(ctx, elts, /*packed*/ false);
601
12
    }
602
865
  } else if (Entries.size() == 1) {
603
313
    unpaddedType = Entries[0].Type;
604
313
  }
605
877
606
877
  return { coercionType, unpaddedType };
607
877
}
608
609
1.04k
bool SwiftAggLowering::shouldPassIndirectly(bool asReturnValue) const {
610
1.04k
  assert(Finished && "haven't yet finished lowering");
611
1.04k
612
1.04k
  // Empty types don't need to be passed indirectly.
613
1.04k
  if (Entries.empty()) 
return false0
;
614
1.04k
615
1.04k
  // Avoid copying the array of types when there's just a single element.
616
1.04k
  if (Entries.size() == 1) {
617
313
    return getSwiftABIInfo(CGM).shouldPassIndirectlyForSwift(
618
313
                                                           Entries.back().Type,
619
313
                                                             asReturnValue);
620
313
  }
621
734
622
734
  SmallVector<llvm::Type*, 8> componentTys;
623
734
  componentTys.reserve(Entries.size());
624
3.85k
  for (auto &entry : Entries) {
625
3.85k
    componentTys.push_back(entry.Type);
626
3.85k
  }
627
734
  return getSwiftABIInfo(CGM).shouldPassIndirectlyForSwift(componentTys,
628
734
                                                           asReturnValue);
629
734
}
630
631
bool swiftcall::shouldPassIndirectly(CodeGenModule &CGM,
632
                                     ArrayRef<llvm::Type*> componentTys,
633
0
                                     bool asReturnValue) {
634
0
  return getSwiftABIInfo(CGM).shouldPassIndirectlyForSwift(componentTys,
635
0
                                                           asReturnValue);
636
0
}
637
638
1.04k
CharUnits swiftcall::getMaximumVoluntaryIntegerSize(CodeGenModule &CGM) {
639
1.04k
  // Currently always the size of an ordinary pointer.
640
1.04k
  return CGM.getContext().toCharUnitsFromBits(
641
1.04k
           CGM.getContext().getTargetInfo().getPointerWidth(0));
642
1.04k
}
643
644
10.2k
CharUnits swiftcall::getNaturalAlignment(CodeGenModule &CGM, llvm::Type *type) {
645
10.2k
  // For Swift's purposes, this is always just the store size of the type
646
10.2k
  // rounded up to a power of 2.
647
10.2k
  auto size = (unsigned long long) getTypeStoreSize(CGM, type).getQuantity();
648
10.2k
  if (!isPowerOf2(size)) {
649
6
    size = 1ULL << (llvm::findLastSet(size, llvm::ZB_Undefined) + 1);
650
6
  }
651
10.2k
  assert(size >= CGM.getDataLayout().getABITypeAlignment(type));
652
10.2k
  return CharUnits::fromQuantity(size);
653
10.2k
}
654
655
bool swiftcall::isLegalIntegerType(CodeGenModule &CGM,
656
9.39k
                                   llvm::IntegerType *intTy) {
657
9.39k
  auto size = intTy->getBitWidth();
658
9.39k
  switch (size) {
659
9.39k
  case 1:
660
9.39k
  case 8:
661
9.39k
  case 16:
662
9.39k
  case 32:
663
9.39k
  case 64:
664
9.39k
    // Just assume that the above are always legal.
665
9.39k
    return true;
666
9.39k
667
9.39k
  case 128:
668
0
    return CGM.getContext().getTargetInfo().hasInt128Type();
669
9.39k
670
9.39k
  default:
671
0
    return false;
672
9.39k
  }
673
9.39k
}
674
675
bool swiftcall::isLegalVectorType(CodeGenModule &CGM, CharUnits vectorSize,
676
1.31k
                                  llvm::VectorType *vectorTy) {
677
1.31k
  return isLegalVectorType(CGM, vectorSize, vectorTy->getElementType(),
678
1.31k
                           vectorTy->getNumElements());
679
1.31k
}
680
681
bool swiftcall::isLegalVectorType(CodeGenModule &CGM, CharUnits vectorSize,
682
1.49k
                                  llvm::Type *eltTy, unsigned numElts) {
683
1.49k
  assert(numElts > 1 && "illegal vector length");
684
1.49k
  return getSwiftABIInfo(CGM)
685
1.49k
           .isLegalVectorTypeForSwift(vectorSize, eltTy, numElts);
686
1.49k
}
687
688
std::pair<llvm::Type*, unsigned>
689
swiftcall::splitLegalVectorType(CodeGenModule &CGM, CharUnits vectorSize,
690
40
                                llvm::VectorType *vectorTy) {
691
40
  auto numElts = vectorTy->getNumElements();
692
40
  auto eltTy = vectorTy->getElementType();
693
40
694
40
  // Try to split the vector type in half.
695
40
  if (numElts >= 4 && 
isPowerOf2(numElts)16
) {
696
16
    if (isLegalVectorType(CGM, vectorSize / 2, eltTy, numElts / 2))
697
8
      return {llvm::VectorType::get(eltTy, numElts / 2), 2};
698
32
  }
699
32
700
32
  return {eltTy, numElts};
701
32
}
702
703
void swiftcall::legalizeVectorType(CodeGenModule &CGM, CharUnits origVectorSize,
704
                                   llvm::VectorType *origVectorTy,
705
1.31k
                             llvm::SmallVectorImpl<llvm::Type*> &components) {
706
1.31k
  // If it's already a legal vector type, use it.
707
1.31k
  if (isLegalVectorType(CGM, origVectorSize, origVectorTy)) {
708
1.16k
    components.push_back(origVectorTy);
709
1.16k
    return;
710
1.16k
  }
711
158
712
158
  // Try to split the vector into legal subvectors.
713
158
  auto numElts = origVectorTy->getNumElements();
714
158
  auto eltTy = origVectorTy->getElementType();
715
158
  assert(numElts != 1);
716
158
717
158
  // The largest size that we're still considering making subvectors of.
718
158
  // Always a power of 2.
719
158
  unsigned logCandidateNumElts = llvm::findLastSet(numElts, llvm::ZB_Undefined);
720
158
  unsigned candidateNumElts = 1U << logCandidateNumElts;
721
158
  assert(candidateNumElts <= numElts && candidateNumElts * 2 > numElts);
722
158
723
158
  // Minor optimization: don't check the legality of this exact size twice.
724
158
  if (candidateNumElts == numElts) {
725
114
    logCandidateNumElts--;
726
114
    candidateNumElts >>= 1;
727
114
  }
728
158
729
158
  CharUnits eltSize = (origVectorSize / numElts);
730
158
  CharUnits candidateSize = eltSize * candidateNumElts;
731
158
732
158
  // The sensibility of this algorithm relies on the fact that we never
733
158
  // have a legal non-power-of-2 vector size without having the power of 2
734
158
  // also be legal.
735
202
  while (logCandidateNumElts > 0) {
736
158
    assert(candidateNumElts == 1U << logCandidateNumElts);
737
158
    assert(candidateNumElts <= numElts);
738
158
    assert(candidateSize == eltSize * candidateNumElts);
739
158
740
158
    // Skip illegal vector sizes.
741
158
    if (!isLegalVectorType(CGM, candidateSize, eltTy, candidateNumElts)) {
742
0
      logCandidateNumElts--;
743
0
      candidateNumElts /= 2;
744
0
      candidateSize /= 2;
745
0
      continue;
746
0
    }
747
158
748
158
    // Add the right number of vectors of this size.
749
158
    auto numVecs = numElts >> logCandidateNumElts;
750
158
    components.append(numVecs, llvm::VectorType::get(eltTy, candidateNumElts));
751
158
    numElts -= (numVecs << logCandidateNumElts);
752
158
753
158
    if (numElts == 0) 
return114
;
754
44
755
44
    // It's possible that the number of elements remaining will be legal.
756
44
    // This can happen with e.g. <7 x float> when <3 x float> is legal.
757
44
    // This only needs to be separately checked if it's not a power of 2.
758
44
    if (numElts > 2 && 
!isPowerOf2(numElts)0
&&
759
44
        
isLegalVectorType(CGM, eltSize * numElts, eltTy, numElts)0
) {
760
0
      components.push_back(llvm::VectorType::get(eltTy, numElts));
761
0
      return;
762
0
    }
763
44
764
44
    // Bring vecSize down to something no larger than numElts.
765
60
    
do 44
{
766
60
      logCandidateNumElts--;
767
60
      candidateNumElts /= 2;
768
60
      candidateSize /= 2;
769
60
    } while (candidateNumElts > numElts);
770
44
  }
771
158
772
158
  // Otherwise, just append a bunch of individual elements.
773
158
  components.append(numElts, eltTy);
774
44
}
775
776
bool swiftcall::mustPassRecordIndirectly(CodeGenModule &CGM,
777
1.01k
                                         const RecordDecl *record) {
778
1.01k
  // FIXME: should we not rely on the standard computation in Sema, just in
779
1.01k
  // case we want to diverge from the platform ABI (e.g. on targets where
780
1.01k
  // that uses the MSVC rule)?
781
1.01k
  return !record->canPassInRegisters();
782
1.01k
}
783
784
static ABIArgInfo classifyExpandedType(SwiftAggLowering &lowering,
785
                                       bool forReturn,
786
1.06k
                                       CharUnits alignmentForIndirect) {
787
1.06k
  if (lowering.empty()) {
788
18
    return ABIArgInfo::getIgnore();
789
1.04k
  } else if (lowering.shouldPassIndirectly(forReturn)) {
790
170
    return ABIArgInfo::getIndirect(alignmentForIndirect, /*byval*/ false);
791
877
  } else {
792
877
    auto types = lowering.getCoerceAndExpandTypes();
793
877
    return ABIArgInfo::getCoerceAndExpand(types.first, types.second);
794
877
  }
795
1.06k
}
796
797
static ABIArgInfo classifyType(CodeGenModule &CGM, CanQualType type,
798
1.76k
                               bool forReturn) {
799
1.76k
  if (auto recordType = dyn_cast<RecordType>(type)) {
800
1.01k
    auto record = recordType->getDecl();
801
1.01k
    auto &layout = CGM.getContext().getASTRecordLayout(record);
802
1.01k
803
1.01k
    if (mustPassRecordIndirectly(CGM, record))
804
2
      return ABIArgInfo::getIndirect(layout.getAlignment(), /*byval*/ false);
805
1.01k
806
1.01k
    SwiftAggLowering lowering(CGM);
807
1.01k
    lowering.addTypedData(recordType->getDecl(), CharUnits::Zero(), layout);
808
1.01k
    lowering.finish();
809
1.01k
810
1.01k
    return classifyExpandedType(lowering, forReturn, layout.getAlignment());
811
1.01k
  }
812
750
813
750
  // Just assume that all of our target ABIs can support returning at least
814
750
  // two integer or floating-point values.
815
750
  if (isa<ComplexType>(type)) {
816
0
    return (forReturn ? ABIArgInfo::getDirect() : ABIArgInfo::getExpand());
817
0
  }
818
750
819
750
  // Vector types may need to be legalized.
820
750
  if (isa<VectorType>(type)) {
821
48
    SwiftAggLowering lowering(CGM);
822
48
    lowering.addTypedData(type, CharUnits::Zero());
823
48
    lowering.finish();
824
48
825
48
    CharUnits alignment = CGM.getContext().getTypeAlignInChars(type);
826
48
    return classifyExpandedType(lowering, forReturn, alignment);
827
48
  }
828
702
829
702
  // Member pointer types need to be expanded, but it's a simple form of
830
702
  // expansion that 'Direct' can handle.  Note that CanBeFlattened should be
831
702
  // true for this to work.
832
702
833
702
  // 'void' needs to be ignored.
834
702
  if (type->isVoidType()) {
835
577
    return ABIArgInfo::getIgnore();
836
577
  }
837
125
838
125
  // Everything else can be passed directly.
839
125
  return ABIArgInfo::getDirect();
840
125
}
841
842
1.12k
ABIArgInfo swiftcall::classifyReturnType(CodeGenModule &CGM, CanQualType type) {
843
1.12k
  return classifyType(CGM, type, /*forReturn*/ true);
844
1.12k
}
845
846
ABIArgInfo swiftcall::classifyArgumentType(CodeGenModule &CGM,
847
646
                                           CanQualType type) {
848
646
  return classifyType(CGM, type, /*forReturn*/ false);
849
646
}
850
851
1.12k
void swiftcall::computeABIInfo(CodeGenModule &CGM, CGFunctionInfo &FI) {
852
1.12k
  auto &retInfo = FI.getReturnInfo();
853
1.12k
  retInfo = classifyReturnType(CGM, FI.getReturnType());
854
1.12k
855
1.76k
  for (unsigned i = 0, e = FI.arg_size(); i != e; 
++i646
) {
856
646
    auto &argInfo = FI.arg_begin()[i];
857
646
    argInfo.info = classifyArgumentType(CGM, argInfo.type);
858
646
  }
859
1.12k
}
860
861
// Is swifterror lowered to a register by the target ABI.
862
0
bool swiftcall::isSwiftErrorLoweredInRegister(CodeGenModule &CGM) {
863
0
  return getSwiftABIInfo(CGM).isSwiftErrorInRegister();
864
0
}