Coverage Report

Created: 2021-08-24 07:12

/Users/buildslave/jenkins/workspace/coverage/llvm-project/clang/lib/CodeGen/CGAtomic.cpp
Line
Count
Source (jump to first uncovered line)
1
//===--- CGAtomic.cpp - Emit LLVM IR for atomic operations ----------------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// This file contains the code for emitting atomic operations.
10
//
11
//===----------------------------------------------------------------------===//
12
13
#include "CGCall.h"
14
#include "CGRecordLayout.h"
15
#include "CodeGenFunction.h"
16
#include "CodeGenModule.h"
17
#include "TargetInfo.h"
18
#include "clang/AST/ASTContext.h"
19
#include "clang/CodeGen/CGFunctionInfo.h"
20
#include "clang/Frontend/FrontendDiagnostic.h"
21
#include "llvm/ADT/DenseMap.h"
22
#include "llvm/IR/DataLayout.h"
23
#include "llvm/IR/Intrinsics.h"
24
#include "llvm/IR/Operator.h"
25
26
using namespace clang;
27
using namespace CodeGen;
28
29
namespace {
30
  class AtomicInfo {
31
    CodeGenFunction &CGF;
32
    QualType AtomicTy;
33
    QualType ValueTy;
34
    uint64_t AtomicSizeInBits;
35
    uint64_t ValueSizeInBits;
36
    CharUnits AtomicAlign;
37
    CharUnits ValueAlign;
38
    TypeEvaluationKind EvaluationKind;
39
    bool UseLibcall;
40
    LValue LVal;
41
    CGBitFieldInfo BFI;
42
  public:
43
    AtomicInfo(CodeGenFunction &CGF, LValue &lvalue)
44
        : CGF(CGF), AtomicSizeInBits(0), ValueSizeInBits(0),
45
1.97k
          EvaluationKind(TEK_Scalar), UseLibcall(true) {
46
1.97k
      assert(!lvalue.isGlobalReg());
47
0
      ASTContext &C = CGF.getContext();
48
1.97k
      if (lvalue.isSimple()) {
49
1.85k
        AtomicTy = lvalue.getType();
50
1.85k
        if (auto *ATy = AtomicTy->getAs<AtomicType>())
51
872
          ValueTy = ATy->getValueType();
52
985
        else
53
985
          ValueTy = AtomicTy;
54
1.85k
        EvaluationKind = CGF.getEvaluationKind(ValueTy);
55
56
1.85k
        uint64_t ValueAlignInBits;
57
1.85k
        uint64_t AtomicAlignInBits;
58
1.85k
        TypeInfo ValueTI = C.getTypeInfo(ValueTy);
59
1.85k
        ValueSizeInBits = ValueTI.Width;
60
1.85k
        ValueAlignInBits = ValueTI.Align;
61
62
1.85k
        TypeInfo AtomicTI = C.getTypeInfo(AtomicTy);
63
1.85k
        AtomicSizeInBits = AtomicTI.Width;
64
1.85k
        AtomicAlignInBits = AtomicTI.Align;
65
66
1.85k
        assert(ValueSizeInBits <= AtomicSizeInBits);
67
0
        assert(ValueAlignInBits <= AtomicAlignInBits);
68
69
0
        AtomicAlign = C.toCharUnitsFromBits(AtomicAlignInBits);
70
1.85k
        ValueAlign = C.toCharUnitsFromBits(ValueAlignInBits);
71
1.85k
        if (lvalue.getAlignment().isZero())
72
0
          lvalue.setAlignment(AtomicAlign);
73
74
1.85k
        LVal = lvalue;
75
1.85k
      } else 
if (120
lvalue.isBitField()120
) {
76
100
        ValueTy = lvalue.getType();
77
100
        ValueSizeInBits = C.getTypeSize(ValueTy);
78
100
        auto &OrigBFI = lvalue.getBitFieldInfo();
79
100
        auto Offset = OrigBFI.Offset % C.toBits(lvalue.getAlignment());
80
100
        AtomicSizeInBits = C.toBits(
81
100
            C.toCharUnitsFromBits(Offset + OrigBFI.Size + C.getCharWidth() - 1)
82
100
                .alignTo(lvalue.getAlignment()));
83
100
        auto VoidPtrAddr = CGF.EmitCastToVoidPtr(lvalue.getBitFieldPointer());
84
100
        auto OffsetInChars =
85
100
            (C.toCharUnitsFromBits(OrigBFI.Offset) / lvalue.getAlignment()) *
86
100
            lvalue.getAlignment();
87
100
        VoidPtrAddr = CGF.Builder.CreateConstGEP1_64(
88
100
            CGF.Int8Ty, VoidPtrAddr, OffsetInChars.getQuantity());
89
100
        auto Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
90
100
            VoidPtrAddr,
91
100
            CGF.Builder.getIntNTy(AtomicSizeInBits)->getPointerTo(),
92
100
            "atomic_bitfield_base");
93
100
        BFI = OrigBFI;
94
100
        BFI.Offset = Offset;
95
100
        BFI.StorageSize = AtomicSizeInBits;
96
100
        BFI.StorageOffset += OffsetInChars;
97
100
        LVal = LValue::MakeBitfield(Address(Addr, lvalue.getAlignment()),
98
100
                                    BFI, lvalue.getType(), lvalue.getBaseInfo(),
99
100
                                    lvalue.getTBAAInfo());
100
100
        AtomicTy = C.getIntTypeForBitwidth(AtomicSizeInBits, OrigBFI.IsSigned);
101
100
        if (AtomicTy.isNull()) {
102
10
          llvm::APInt Size(
103
10
              /*numBits=*/32,
104
10
              C.toCharUnitsFromBits(AtomicSizeInBits).getQuantity());
105
10
          AtomicTy =
106
10
              C.getConstantArrayType(C.CharTy, Size, nullptr, ArrayType::Normal,
107
10
                                     /*IndexTypeQuals=*/0);
108
10
        }
109
100
        AtomicAlign = ValueAlign = lvalue.getAlignment();
110
100
      } else 
if (20
lvalue.isVectorElt()20
) {
111
10
        ValueTy = lvalue.getType()->castAs<VectorType>()->getElementType();
112
10
        ValueSizeInBits = C.getTypeSize(ValueTy);
113
10
        AtomicTy = lvalue.getType();
114
10
        AtomicSizeInBits = C.getTypeSize(AtomicTy);
115
10
        AtomicAlign = ValueAlign = lvalue.getAlignment();
116
10
        LVal = lvalue;
117
10
      } else {
118
10
        assert(lvalue.isExtVectorElt());
119
0
        ValueTy = lvalue.getType();
120
10
        ValueSizeInBits = C.getTypeSize(ValueTy);
121
10
        AtomicTy = ValueTy = CGF.getContext().getExtVectorType(
122
10
            lvalue.getType(), cast<llvm::FixedVectorType>(
123
10
                                  lvalue.getExtVectorAddress().getElementType())
124
10
                                  ->getNumElements());
125
10
        AtomicSizeInBits = C.getTypeSize(AtomicTy);
126
10
        AtomicAlign = ValueAlign = lvalue.getAlignment();
127
10
        LVal = lvalue;
128
10
      }
129
0
      UseLibcall = !C.getTargetInfo().hasBuiltinAtomic(
130
1.97k
          AtomicSizeInBits, C.toBits(lvalue.getAlignment()));
131
1.97k
    }
132
133
141
    QualType getAtomicType() const { return AtomicTy; }
134
321
    QualType getValueType() const { return ValueTy; }
135
3.03k
    CharUnits getAtomicAlignment() const { return AtomicAlign; }
136
233
    uint64_t getAtomicSizeInBits() const { return AtomicSizeInBits; }
137
263
    uint64_t getValueSizeInBits() const { return ValueSizeInBits; }
138
1.06k
    TypeEvaluationKind getEvaluationKind() const { return EvaluationKind; }
139
743
    bool shouldUseLibcall() const { return UseLibcall; }
140
539
    const LValue &getAtomicLValue() const { return LVal; }
141
1.95k
    llvm::Value *getAtomicPointer() const {
142
1.95k
      if (LVal.isSimple())
143
1.48k
        return LVal.getPointer(CGF);
144
468
      else if (LVal.isBitField())
145
404
        return LVal.getBitFieldPointer();
146
64
      else if (LVal.isVectorElt())
147
32
        return LVal.getVectorPointer();
148
32
      assert(LVal.isExtVectorElt());
149
0
      return LVal.getExtVectorPointer();
150
1.95k
    }
151
1.75k
    Address getAtomicAddress() const {
152
1.75k
      return Address(getAtomicPointer(), getAtomicAlignment());
153
1.75k
    }
154
155
721
    Address getAtomicAddressAsAtomicIntPointer() const {
156
721
      return emitCastToAtomicIntPointer(getAtomicAddress());
157
721
    }
158
159
    /// Is the atomic size larger than the underlying value type?
160
    ///
161
    /// Note that the absence of padding does not mean that atomic
162
    /// objects are completely interchangeable with non-atomic
163
    /// objects: we might have promoted the alignment of a type
164
    /// without making it bigger.
165
1.26k
    bool hasPadding() const {
166
1.26k
      return (ValueSizeInBits != AtomicSizeInBits);
167
1.26k
    }
168
169
    bool emitMemSetZeroIfNecessary() const;
170
171
201
    llvm::Value *getAtomicSizeValue() const {
172
201
      CharUnits size = CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits);
173
201
      return CGF.CGM.getSize(size);
174
201
    }
175
176
    /// Cast the given pointer to an integer pointer suitable for atomic
177
    /// operations if the source.
178
    Address emitCastToAtomicIntPointer(Address Addr) const;
179
180
    /// If Addr is compatible with the iN that will be used for an atomic
181
    /// operation, bitcast it. Otherwise, create a temporary that is suitable
182
    /// and copy the value across.
183
    Address convertToAtomicIntPointer(Address Addr) const;
184
185
    /// Turn an atomic-layout object into an r-value.
186
    RValue convertAtomicTempToRValue(Address addr, AggValueSlot resultSlot,
187
                                     SourceLocation loc, bool AsValue) const;
188
189
    /// Converts a rvalue to integer value.
190
    llvm::Value *convertRValueToInt(RValue RVal) const;
191
192
    RValue ConvertIntToValueOrAtomic(llvm::Value *IntVal,
193
                                     AggValueSlot ResultSlot,
194
                                     SourceLocation Loc, bool AsValue) const;
195
196
    /// Copy an atomic r-value into atomic-layout memory.
197
    void emitCopyIntoMemory(RValue rvalue) const;
198
199
    /// Project an l-value down to the value field.
200
190
    LValue projectValue() const {
201
190
      assert(LVal.isSimple());
202
0
      Address addr = getAtomicAddress();
203
190
      if (hasPadding())
204
2
        addr = CGF.Builder.CreateStructGEP(addr, 0);
205
206
190
      return LValue::MakeAddr(addr, getValueType(), CGF.getContext(),
207
190
                              LVal.getBaseInfo(), LVal.getTBAAInfo());
208
190
    }
209
210
    /// Emits atomic load.
211
    /// \returns Loaded value.
212
    RValue EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
213
                          bool AsValue, llvm::AtomicOrdering AO,
214
                          bool IsVolatile);
215
216
    /// Emits atomic compare-and-exchange sequence.
217
    /// \param Expected Expected value.
218
    /// \param Desired Desired value.
219
    /// \param Success Atomic ordering for success operation.
220
    /// \param Failure Atomic ordering for failed operation.
221
    /// \param IsWeak true if atomic operation is weak, false otherwise.
222
    /// \returns Pair of values: previous value from storage (value type) and
223
    /// boolean flag (i1 type) with true if success and false otherwise.
224
    std::pair<RValue, llvm::Value *>
225
    EmitAtomicCompareExchange(RValue Expected, RValue Desired,
226
                              llvm::AtomicOrdering Success =
227
                                  llvm::AtomicOrdering::SequentiallyConsistent,
228
                              llvm::AtomicOrdering Failure =
229
                                  llvm::AtomicOrdering::SequentiallyConsistent,
230
                              bool IsWeak = false);
231
232
    /// Emits atomic update.
233
    /// \param AO Atomic ordering.
234
    /// \param UpdateOp Update operation for the current lvalue.
235
    void EmitAtomicUpdate(llvm::AtomicOrdering AO,
236
                          const llvm::function_ref<RValue(RValue)> &UpdateOp,
237
                          bool IsVolatile);
238
    /// Emits atomic update.
239
    /// \param AO Atomic ordering.
240
    void EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
241
                          bool IsVolatile);
242
243
    /// Materialize an atomic r-value in atomic-layout memory.
244
    Address materializeRValue(RValue rvalue) const;
245
246
    /// Creates temp alloca for intermediate operations on atomic value.
247
    Address CreateTempAlloca() const;
248
  private:
249
    bool requiresMemSetZero(llvm::Type *type) const;
250
251
252
    /// Emits atomic load as a libcall.
253
    void EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
254
                               llvm::AtomicOrdering AO, bool IsVolatile);
255
    /// Emits atomic load as LLVM instruction.
256
    llvm::Value *EmitAtomicLoadOp(llvm::AtomicOrdering AO, bool IsVolatile);
257
    /// Emits atomic compare-and-exchange op as a libcall.
258
    llvm::Value *EmitAtomicCompareExchangeLibcall(
259
        llvm::Value *ExpectedAddr, llvm::Value *DesiredAddr,
260
        llvm::AtomicOrdering Success =
261
            llvm::AtomicOrdering::SequentiallyConsistent,
262
        llvm::AtomicOrdering Failure =
263
            llvm::AtomicOrdering::SequentiallyConsistent);
264
    /// Emits atomic compare-and-exchange op as LLVM instruction.
265
    std::pair<llvm::Value *, llvm::Value *> EmitAtomicCompareExchangeOp(
266
        llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
267
        llvm::AtomicOrdering Success =
268
            llvm::AtomicOrdering::SequentiallyConsistent,
269
        llvm::AtomicOrdering Failure =
270
            llvm::AtomicOrdering::SequentiallyConsistent,
271
        bool IsWeak = false);
272
    /// Emit atomic update as libcalls.
273
    void
274
    EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
275
                            const llvm::function_ref<RValue(RValue)> &UpdateOp,
276
                            bool IsVolatile);
277
    /// Emit atomic update as LLVM instructions.
278
    void EmitAtomicUpdateOp(llvm::AtomicOrdering AO,
279
                            const llvm::function_ref<RValue(RValue)> &UpdateOp,
280
                            bool IsVolatile);
281
    /// Emit atomic update as libcalls.
282
    void EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, RValue UpdateRVal,
283
                                 bool IsVolatile);
284
    /// Emit atomic update as LLVM instructions.
285
    void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRal,
286
                            bool IsVolatile);
287
  };
288
}
289
290
1.28k
Address AtomicInfo::CreateTempAlloca() const {
291
1.28k
  Address TempAlloca = CGF.CreateMemTemp(
292
1.28k
      (LVal.isBitField() && 
ValueSizeInBits > AtomicSizeInBits176
) ?
ValueTy72
293
1.28k
                                                                : 
AtomicTy1.21k
,
294
1.28k
      getAtomicAlignment(),
295
1.28k
      "atomic-temp");
296
  // Cast to pointer to value type for bitfields.
297
1.28k
  if (LVal.isBitField())
298
176
    return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
299
176
        TempAlloca, getAtomicAddress().getType());
300
1.11k
  return TempAlloca;
301
1.28k
}
302
303
static RValue emitAtomicLibcall(CodeGenFunction &CGF,
304
                                StringRef fnName,
305
                                QualType resultType,
306
384
                                CallArgList &args) {
307
384
  const CGFunctionInfo &fnInfo =
308
384
    CGF.CGM.getTypes().arrangeBuiltinFunctionCall(resultType, args);
309
384
  llvm::FunctionType *fnTy = CGF.CGM.getTypes().GetFunctionType(fnInfo);
310
384
  llvm::AttrBuilder fnAttrB;
311
384
  fnAttrB.addAttribute(llvm::Attribute::NoUnwind);
312
384
  fnAttrB.addAttribute(llvm::Attribute::WillReturn);
313
384
  llvm::AttributeList fnAttrs = llvm::AttributeList::get(
314
384
      CGF.getLLVMContext(), llvm::AttributeList::FunctionIndex, fnAttrB);
315
316
384
  llvm::FunctionCallee fn =
317
384
      CGF.CGM.CreateRuntimeFunction(fnTy, fnName, fnAttrs);
318
384
  auto callee = CGCallee::forDirect(fn);
319
384
  return CGF.EmitCall(fnInfo, callee, ReturnValueSlot(), args);
320
384
}
321
322
/// Does a store of the given IR type modify the full expected width?
323
static bool isFullSizeType(CodeGenModule &CGM, llvm::Type *type,
324
427
                           uint64_t expectedSize) {
325
427
  return (CGM.getDataLayout().getTypeStoreSize(type) * 8 == expectedSize);
326
427
}
327
328
/// Does the atomic type require memsetting to zero before initialization?
329
///
330
/// The IR type is provided as a way of making certain queries faster.
331
454
bool AtomicInfo::requiresMemSetZero(llvm::Type *type) const {
332
  // If the atomic type has size padding, we definitely need a memset.
333
454
  if (hasPadding()) 
return true18
;
334
335
  // Otherwise, do some simple heuristics to try to avoid it:
336
436
  switch (getEvaluationKind()) {
337
  // For scalars and complexes, check whether the store size of the
338
  // type uses the full size.
339
377
  case TEK_Scalar:
340
377
    return !isFullSizeType(CGF.CGM, type, AtomicSizeInBits);
341
50
  case TEK_Complex:
342
50
    return !isFullSizeType(CGF.CGM, type->getStructElementType(0),
343
50
                           AtomicSizeInBits / 2);
344
345
  // Padding in structs has an undefined bit pattern.  User beware.
346
9
  case TEK_Aggregate:
347
9
    return false;
348
436
  }
349
0
  llvm_unreachable("bad evaluation kind");
350
0
}
351
352
190
bool AtomicInfo::emitMemSetZeroIfNecessary() const {
353
190
  assert(LVal.isSimple());
354
0
  llvm::Value *addr = LVal.getPointer(CGF);
355
190
  if (!requiresMemSetZero(addr->getType()->getPointerElementType()))
356
156
    return false;
357
358
34
  CGF.Builder.CreateMemSet(
359
34
      addr, llvm::ConstantInt::get(CGF.Int8Ty, 0),
360
34
      CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits).getQuantity(),
361
34
      LVal.getAlignment().getAsAlign());
362
34
  return true;
363
190
}
364
365
static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak,
366
                              Address Dest, Address Ptr,
367
                              Address Val1, Address Val2,
368
                              uint64_t Size,
369
                              llvm::AtomicOrdering SuccessOrder,
370
                              llvm::AtomicOrdering FailureOrder,
371
168
                              llvm::SyncScope::ID Scope) {
372
  // Note that cmpxchg doesn't support weak cmpxchg, at least at the moment.
373
168
  llvm::Value *Expected = CGF.Builder.CreateLoad(Val1);
374
168
  llvm::Value *Desired = CGF.Builder.CreateLoad(Val2);
375
376
168
  llvm::AtomicCmpXchgInst *Pair = CGF.Builder.CreateAtomicCmpXchg(
377
168
      Ptr.getPointer(), Expected, Desired, SuccessOrder, FailureOrder,
378
168
      Scope);
379
168
  Pair->setVolatile(E->isVolatile());
380
168
  Pair->setWeak(IsWeak);
381
382
  // Cmp holds the result of the compare-exchange operation: true on success,
383
  // false on failure.
384
168
  llvm::Value *Old = CGF.Builder.CreateExtractValue(Pair, 0);
385
168
  llvm::Value *Cmp = CGF.Builder.CreateExtractValue(Pair, 1);
386
387
  // This basic block is used to hold the store instruction if the operation
388
  // failed.
389
168
  llvm::BasicBlock *StoreExpectedBB =
390
168
      CGF.createBasicBlock("cmpxchg.store_expected", CGF.CurFn);
391
392
  // This basic block is the exit point of the operation, we should end up
393
  // here regardless of whether or not the operation succeeded.
394
168
  llvm::BasicBlock *ContinueBB =
395
168
      CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
396
397
  // Update Expected if Expected isn't equal to Old, otherwise branch to the
398
  // exit point.
399
168
  CGF.Builder.CreateCondBr(Cmp, ContinueBB, StoreExpectedBB);
400
401
168
  CGF.Builder.SetInsertPoint(StoreExpectedBB);
402
  // Update the memory at Expected with Old's value.
403
168
  CGF.Builder.CreateStore(Old, Val1);
404
  // Finally, branch to the exit point.
405
168
  CGF.Builder.CreateBr(ContinueBB);
406
407
168
  CGF.Builder.SetInsertPoint(ContinueBB);
408
  // Update the memory at Dest with Cmp's value.
409
168
  CGF.EmitStoreOfScalar(Cmp, CGF.MakeAddrLValue(Dest, E->getType()));
410
168
}
411
412
/// Given an ordering required on success, emit all possible cmpxchg
413
/// instructions to cope with the provided (but possibly only dynamically known)
414
/// FailureOrder.
415
static void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E,
416
                                        bool IsWeak, Address Dest, Address Ptr,
417
                                        Address Val1, Address Val2,
418
                                        llvm::Value *FailureOrderVal,
419
                                        uint64_t Size,
420
                                        llvm::AtomicOrdering SuccessOrder,
421
84
                                        llvm::SyncScope::ID Scope) {
422
84
  llvm::AtomicOrdering FailureOrder;
423
84
  if (llvm::ConstantInt *FO = dyn_cast<llvm::ConstantInt>(FailureOrderVal)) {
424
42
    auto FOS = FO->getSExtValue();
425
42
    if (!llvm::isValidAtomicOrderingCABI(FOS))
426
2
      FailureOrder = llvm::AtomicOrdering::Monotonic;
427
40
    else
428
40
      switch ((llvm::AtomicOrderingCABI)FOS) {
429
7
      case llvm::AtomicOrderingCABI::relaxed:
430
      // 31.7.2.18: "The failure argument shall not be memory_order_release
431
      // nor memory_order_acq_rel". Fallback to monotonic.
432
7
      case llvm::AtomicOrderingCABI::release:
433
7
      case llvm::AtomicOrderingCABI::acq_rel:
434
7
        FailureOrder = llvm::AtomicOrdering::Monotonic;
435
7
        break;
436
0
      case llvm::AtomicOrderingCABI::consume:
437
24
      case llvm::AtomicOrderingCABI::acquire:
438
24
        FailureOrder = llvm::AtomicOrdering::Acquire;
439
24
        break;
440
9
      case llvm::AtomicOrderingCABI::seq_cst:
441
9
        FailureOrder = llvm::AtomicOrdering::SequentiallyConsistent;
442
9
        break;
443
40
      }
444
    // Prior to c++17, "the failure argument shall be no stronger than the
445
    // success argument". This condition has been lifted and the only
446
    // precondition is 31.7.2.18. Effectively treat this as a DR and skip
447
    // language version checks.
448
42
    emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
449
42
                      FailureOrder, Scope);
450
42
    return;
451
42
  }
452
453
  // Create all the relevant BB's
454
42
  auto *MonotonicBB = CGF.createBasicBlock("monotonic_fail", CGF.CurFn);
455
42
  auto *AcquireBB = CGF.createBasicBlock("acquire_fail", CGF.CurFn);
456
42
  auto *SeqCstBB = CGF.createBasicBlock("seqcst_fail", CGF.CurFn);
457
42
  auto *ContBB = CGF.createBasicBlock("atomic.continue", CGF.CurFn);
458
459
  // MonotonicBB is arbitrarily chosen as the default case; in practice, this
460
  // doesn't matter unless someone is crazy enough to use something that
461
  // doesn't fold to a constant for the ordering.
462
42
  llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(FailureOrderVal, MonotonicBB);
463
  // Implemented as acquire, since it's the closest in LLVM.
464
42
  SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::consume),
465
42
              AcquireBB);
466
42
  SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire),
467
42
              AcquireBB);
468
42
  SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst),
469
42
              SeqCstBB);
470
471
  // Emit all the different atomics
472
42
  CGF.Builder.SetInsertPoint(MonotonicBB);
473
42
  emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
474
42
                    Size, SuccessOrder, llvm::AtomicOrdering::Monotonic, Scope);
475
42
  CGF.Builder.CreateBr(ContBB);
476
477
42
  CGF.Builder.SetInsertPoint(AcquireBB);
478
42
  emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
479
42
                    llvm::AtomicOrdering::Acquire, Scope);
480
42
  CGF.Builder.CreateBr(ContBB);
481
482
42
  CGF.Builder.SetInsertPoint(SeqCstBB);
483
42
  emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
484
42
                    llvm::AtomicOrdering::SequentiallyConsistent, Scope);
485
42
  CGF.Builder.CreateBr(ContBB);
486
487
42
  CGF.Builder.SetInsertPoint(ContBB);
488
42
}
489
490
/// Duplicate the atomic min/max operation in conventional IR for the builtin
491
/// variants that return the new rather than the original value.
492
static llvm::Value *EmitPostAtomicMinMax(CGBuilderTy &Builder,
493
                                         AtomicExpr::AtomicOp Op,
494
                                         bool IsSigned,
495
                                         llvm::Value *OldVal,
496
14
                                         llvm::Value *RHS) {
497
14
  llvm::CmpInst::Predicate Pred;
498
14
  switch (Op) {
499
0
  default:
500
0
    llvm_unreachable("Unexpected min/max operation");
501
4
  case AtomicExpr::AO__atomic_max_fetch:
502
4
    Pred = IsSigned ? 
llvm::CmpInst::ICMP_SGT2
:
llvm::CmpInst::ICMP_UGT2
;
503
4
    break;
504
10
  case AtomicExpr::AO__atomic_min_fetch:
505
10
    Pred = IsSigned ? 
llvm::CmpInst::ICMP_SLT4
:
llvm::CmpInst::ICMP_ULT6
;
506
10
    break;
507
14
  }
508
14
  llvm::Value *Cmp = Builder.CreateICmp(Pred, OldVal, RHS, "tst");
509
14
  return Builder.CreateSelect(Cmp, OldVal, RHS, "newval");
510
14
}
511
512
static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest,
513
                         Address Ptr, Address Val1, Address Val2,
514
                         llvm::Value *IsWeak, llvm::Value *FailureOrder,
515
                         uint64_t Size, llvm::AtomicOrdering Order,
516
1.82k
                         llvm::SyncScope::ID Scope) {
517
1.82k
  llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add;
518
1.82k
  bool PostOpMinMax = false;
519
1.82k
  unsigned PostOp = 0;
520
521
1.82k
  switch (E->getOp()) {
522
0
  case AtomicExpr::AO__c11_atomic_init:
523
0
  case AtomicExpr::AO__opencl_atomic_init:
524
0
    llvm_unreachable("Already handled!");
525
526
21
  case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
527
35
  case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
528
35
    emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
529
35
                                FailureOrder, Size, Order, Scope);
530
35
    return;
531
2
  case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
532
4
  case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
533
4
    emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
534
4
                                FailureOrder, Size, Order, Scope);
535
4
    return;
536
21
  case AtomicExpr::AO__atomic_compare_exchange:
537
31
  case AtomicExpr::AO__atomic_compare_exchange_n: {
538
31
    if (llvm::ConstantInt *IsWeakC = dyn_cast<llvm::ConstantInt>(IsWeak)) {
539
17
      emitAtomicCmpXchgFailureSet(CGF, E, IsWeakC->getZExtValue(), Dest, Ptr,
540
17
                                  Val1, Val2, FailureOrder, Size, Order, Scope);
541
17
    } else {
542
      // Create all the relevant BB's
543
14
      llvm::BasicBlock *StrongBB =
544
14
          CGF.createBasicBlock("cmpxchg.strong", CGF.CurFn);
545
14
      llvm::BasicBlock *WeakBB = CGF.createBasicBlock("cmxchg.weak", CGF.CurFn);
546
14
      llvm::BasicBlock *ContBB =
547
14
          CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
548
549
14
      llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(IsWeak, WeakBB);
550
14
      SI->addCase(CGF.Builder.getInt1(false), StrongBB);
551
552
14
      CGF.Builder.SetInsertPoint(StrongBB);
553
14
      emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
554
14
                                  FailureOrder, Size, Order, Scope);
555
14
      CGF.Builder.CreateBr(ContBB);
556
557
14
      CGF.Builder.SetInsertPoint(WeakBB);
558
14
      emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
559
14
                                  FailureOrder, Size, Order, Scope);
560
14
      CGF.Builder.CreateBr(ContBB);
561
562
14
      CGF.Builder.SetInsertPoint(ContBB);
563
14
    }
564
31
    return;
565
21
  }
566
268
  case AtomicExpr::AO__c11_atomic_load:
567
312
  case AtomicExpr::AO__opencl_atomic_load:
568
331
  case AtomicExpr::AO__atomic_load_n:
569
382
  case AtomicExpr::AO__atomic_load: {
570
382
    llvm::LoadInst *Load = CGF.Builder.CreateLoad(Ptr);
571
382
    Load->setAtomic(Order, Scope);
572
382
    Load->setVolatile(E->isVolatile());
573
382
    CGF.Builder.CreateStore(Load, Dest);
574
382
    return;
575
331
  }
576
577
290
  case AtomicExpr::AO__c11_atomic_store:
578
300
  case AtomicExpr::AO__opencl_atomic_store:
579
349
  case AtomicExpr::AO__atomic_store:
580
353
  case AtomicExpr::AO__atomic_store_n: {
581
353
    llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1);
582
353
    llvm::StoreInst *Store = CGF.Builder.CreateStore(LoadVal1, Ptr);
583
353
    Store->setAtomic(Order, Scope);
584
353
    Store->setVolatile(E->isVolatile());
585
353
    return;
586
349
  }
587
588
13
  case AtomicExpr::AO__c11_atomic_exchange:
589
15
  case AtomicExpr::AO__opencl_atomic_exchange:
590
19
  case AtomicExpr::AO__atomic_exchange_n:
591
25
  case AtomicExpr::AO__atomic_exchange:
592
25
    Op = llvm::AtomicRMWInst::Xchg;
593
25
    break;
594
595
63
  case AtomicExpr::AO__atomic_add_fetch:
596
63
    PostOp = E->getValueType()->isFloatingType() ? 
llvm::Instruction::FAdd0
597
63
                                                 : llvm::Instruction::Add;
598
63
    LLVM_FALLTHROUGH;
599
472
  case AtomicExpr::AO__c11_atomic_fetch_add:
600
490
  case AtomicExpr::AO__opencl_atomic_fetch_add:
601
516
  case AtomicExpr::AO__atomic_fetch_add:
602
516
    Op = E->getValueType()->isFloatingType() ? 
llvm::AtomicRMWInst::FAdd32
603
516
                                             : 
llvm::AtomicRMWInst::Add484
;
604
516
    break;
605
606
0
  case AtomicExpr::AO__atomic_sub_fetch:
607
0
    PostOp = E->getValueType()->isFloatingType() ? llvm::Instruction::FSub
608
0
                                                 : llvm::Instruction::Sub;
609
0
    LLVM_FALLTHROUGH;
610
386
  case AtomicExpr::AO__c11_atomic_fetch_sub:
611
386
  case AtomicExpr::AO__opencl_atomic_fetch_sub:
612
414
  case AtomicExpr::AO__atomic_fetch_sub:
613
414
    Op = E->getValueType()->isFloatingType() ? 
llvm::AtomicRMWInst::FSub16
614
414
                                             : 
llvm::AtomicRMWInst::Sub398
;
615
414
    break;
616
617
8
  case AtomicExpr::AO__atomic_min_fetch:
618
8
    PostOpMinMax = true;
619
8
    LLVM_FALLTHROUGH;
620
18
  case AtomicExpr::AO__c11_atomic_fetch_min:
621
22
  case AtomicExpr::AO__opencl_atomic_fetch_min:
622
24
  case AtomicExpr::AO__atomic_fetch_min:
623
24
    Op = E->getValueType()->isSignedIntegerType() ? 
llvm::AtomicRMWInst::Min13
624
24
                                                  : 
llvm::AtomicRMWInst::UMin11
;
625
24
    break;
626
627
4
  case AtomicExpr::AO__atomic_max_fetch:
628
4
    PostOpMinMax = true;
629
4
    LLVM_FALLTHROUGH;
630
14
  case AtomicExpr::AO__c11_atomic_fetch_max:
631
18
  case AtomicExpr::AO__opencl_atomic_fetch_max:
632
20
  case AtomicExpr::AO__atomic_fetch_max:
633
20
    Op = E->getValueType()->isSignedIntegerType() ? 
llvm::AtomicRMWInst::Max11
634
20
                                                  : 
llvm::AtomicRMWInst::UMax9
;
635
20
    break;
636
637
0
  case AtomicExpr::AO__atomic_and_fetch:
638
0
    PostOp = llvm::Instruction::And;
639
0
    LLVM_FALLTHROUGH;
640
4
  case AtomicExpr::AO__c11_atomic_fetch_and:
641
6
  case AtomicExpr::AO__opencl_atomic_fetch_and:
642
6
  case AtomicExpr::AO__atomic_fetch_and:
643
6
    Op = llvm::AtomicRMWInst::And;
644
6
    break;
645
646
2
  case AtomicExpr::AO__atomic_or_fetch:
647
2
    PostOp = llvm::Instruction::Or;
648
2
    LLVM_FALLTHROUGH;
649
4
  case AtomicExpr::AO__c11_atomic_fetch_or:
650
4
  case AtomicExpr::AO__opencl_atomic_fetch_or:
651
7
  case AtomicExpr::AO__atomic_fetch_or:
652
7
    Op = llvm::AtomicRMWInst::Or;
653
7
    break;
654
655
0
  case AtomicExpr::AO__atomic_xor_fetch:
656
0
    PostOp = llvm::Instruction::Xor;
657
0
    LLVM_FALLTHROUGH;
658
0
  case AtomicExpr::AO__c11_atomic_fetch_xor:
659
0
  case AtomicExpr::AO__opencl_atomic_fetch_xor:
660
4
  case AtomicExpr::AO__atomic_fetch_xor:
661
4
    Op = llvm::AtomicRMWInst::Xor;
662
4
    break;
663
664
4
  case AtomicExpr::AO__atomic_nand_fetch:
665
4
    PostOp = llvm::Instruction::And; // the NOT is special cased below
666
4
    LLVM_FALLTHROUGH;
667
8
  case AtomicExpr::AO__atomic_fetch_nand:
668
8
    Op = llvm::AtomicRMWInst::Nand;
669
8
    break;
670
1.82k
  }
671
672
1.02k
  llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1);
673
1.02k
  llvm::AtomicRMWInst *RMWI =
674
1.02k
      CGF.Builder.CreateAtomicRMW(Op, Ptr.getPointer(), LoadVal1, Order, Scope);
675
1.02k
  RMWI->setVolatile(E->isVolatile());
676
677
  // For __atomic_*_fetch operations, perform the operation again to
678
  // determine the value which was written.
679
1.02k
  llvm::Value *Result = RMWI;
680
1.02k
  if (PostOpMinMax)
681
12
    Result = EmitPostAtomicMinMax(CGF.Builder, E->getOp(),
682
12
                                  E->getValueType()->isSignedIntegerType(),
683
12
                                  RMWI, LoadVal1);
684
1.01k
  else if (PostOp)
685
69
    Result = CGF.Builder.CreateBinOp((llvm::Instruction::BinaryOps)PostOp, RMWI,
686
69
                                     LoadVal1);
687
1.02k
  if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch)
688
4
    Result = CGF.Builder.CreateNot(Result);
689
1.02k
  CGF.Builder.CreateStore(Result, Dest);
690
1.02k
}
691
692
// This function emits any expression (scalar, complex, or aggregate)
693
// into a temporary alloca.
694
static Address
695
670
EmitValToTemp(CodeGenFunction &CGF, Expr *E) {
696
670
  Address DeclPtr = CGF.CreateMemTemp(E->getType(), ".atomictmp");
697
670
  CGF.EmitAnyExprToMem(E, DeclPtr, E->getType().getQualifiers(),
698
670
                       /*Init*/ true);
699
670
  return DeclPtr;
700
670
}
701
702
static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *Expr, Address Dest,
703
                         Address Ptr, Address Val1, Address Val2,
704
                         llvm::Value *IsWeak, llvm::Value *FailureOrder,
705
                         uint64_t Size, llvm::AtomicOrdering Order,
706
1.80k
                         llvm::Value *Scope) {
707
1.80k
  auto ScopeModel = Expr->getScopeModel();
708
709
  // LLVM atomic instructions always have synch scope. If clang atomic
710
  // expression has no scope operand, use default LLVM synch scope.
711
1.80k
  if (!ScopeModel) {
712
1.72k
    EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
713
1.72k
                 Order, CGF.CGM.getLLVMContext().getOrInsertSyncScopeID(""));
714
1.72k
    return;
715
1.72k
  }
716
717
  // Handle constant scope.
718
76
  if (auto SC = dyn_cast<llvm::ConstantInt>(Scope)) {
719
68
    auto SCID = CGF.getTargetHooks().getLLVMSyncScopeID(
720
68
        CGF.CGM.getLangOpts(), ScopeModel->map(SC->getZExtValue()),
721
68
        Order, CGF.CGM.getLLVMContext());
722
68
    EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
723
68
                 Order, SCID);
724
68
    return;
725
68
  }
726
727
  // Handle non-constant scope.
728
8
  auto &Builder = CGF.Builder;
729
8
  auto Scopes = ScopeModel->getRuntimeValues();
730
8
  llvm::DenseMap<unsigned, llvm::BasicBlock *> BB;
731
8
  for (auto S : Scopes)
732
32
    BB[S] = CGF.createBasicBlock(getAsString(ScopeModel->map(S)), CGF.CurFn);
733
734
8
  llvm::BasicBlock *ContBB =
735
8
      CGF.createBasicBlock("atomic.scope.continue", CGF.CurFn);
736
737
8
  auto *SC = Builder.CreateIntCast(Scope, Builder.getInt32Ty(), false);
738
  // If unsupported synch scope is encountered at run time, assume a fallback
739
  // synch scope value.
740
8
  auto FallBack = ScopeModel->getFallBackValue();
741
8
  llvm::SwitchInst *SI = Builder.CreateSwitch(SC, BB[FallBack]);
742
32
  for (auto S : Scopes) {
743
32
    auto *B = BB[S];
744
32
    if (S != FallBack)
745
24
      SI->addCase(Builder.getInt32(S), B);
746
747
32
    Builder.SetInsertPoint(B);
748
32
    EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
749
32
                 Order,
750
32
                 CGF.getTargetHooks().getLLVMSyncScopeID(CGF.CGM.getLangOpts(),
751
32
                                                         ScopeModel->map(S),
752
32
                                                         Order,
753
32
                                                         CGF.getLLVMContext()));
754
32
    Builder.CreateBr(ContBB);
755
32
  }
756
757
8
  Builder.SetInsertPoint(ContBB);
758
8
}
759
760
static void
761
AddDirectArgument(CodeGenFunction &CGF, CallArgList &Args,
762
                  bool UseOptimizedLibcall, llvm::Value *Val, QualType ValTy,
763
133
                  SourceLocation Loc, CharUnits SizeInChars) {
764
133
  if (UseOptimizedLibcall) {
765
    // Load value and pass it to the function directly.
766
81
    CharUnits Align = CGF.getContext().getTypeAlignInChars(ValTy);
767
81
    int64_t SizeInBits = CGF.getContext().toBits(SizeInChars);
768
81
    ValTy =
769
81
        CGF.getContext().getIntTypeForBitwidth(SizeInBits, /*Signed=*/false);
770
81
    llvm::Type *IPtrTy = llvm::IntegerType::get(CGF.getLLVMContext(),
771
81
                                                SizeInBits)->getPointerTo();
772
81
    Address Ptr = Address(CGF.Builder.CreateBitCast(Val, IPtrTy), Align);
773
81
    Val = CGF.EmitLoadOfScalar(Ptr, false,
774
81
                               CGF.getContext().getPointerType(ValTy),
775
81
                               Loc);
776
    // Coerce the value into an appropriately sized integer type.
777
81
    Args.add(RValue::get(Val), ValTy);
778
81
  } else {
779
    // Non-optimized functions always take a reference.
780
52
    Args.add(RValue::get(CGF.EmitCastToVoidPtr(Val)),
781
52
                         CGF.getContext().VoidPtrTy);
782
52
  }
783
133
}
784
785
1.05k
RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) {
786
1.05k
  QualType AtomicTy = E->getPtr()->getType()->getPointeeType();
787
1.05k
  QualType MemTy = AtomicTy;
788
1.05k
  if (const AtomicType *AT = AtomicTy->getAs<AtomicType>())
789
638
    MemTy = AT->getValueType();
790
1.05k
  llvm::Value *IsWeak = nullptr, *OrderFail = nullptr;
791
792
1.05k
  Address Val1 = Address::invalid();
793
1.05k
  Address Val2 = Address::invalid();
794
1.05k
  Address Dest = Address::invalid();
795
1.05k
  Address Ptr = EmitPointerWithAlignment(E->getPtr());
796
797
1.05k
  if (E->getOp() == AtomicExpr::AO__c11_atomic_init ||
798
1.05k
      
E->getOp() == AtomicExpr::AO__opencl_atomic_init1.04k
) {
799
16
    LValue lvalue = MakeAddrLValue(Ptr, AtomicTy);
800
16
    EmitAtomicInit(E->getVal1(), lvalue);
801
16
    return RValue::get(nullptr);
802
16
  }
803
804
1.04k
  auto TInfo = getContext().getTypeInfoInChars(AtomicTy);
805
1.04k
  uint64_t Size = TInfo.Width.getQuantity();
806
1.04k
  unsigned MaxInlineWidthInBits = getTarget().getMaxAtomicInlineWidth();
807
808
1.04k
  bool Oversized = getContext().toBits(TInfo.Width) > MaxInlineWidthInBits;
809
1.04k
  bool Misaligned = (Ptr.getAlignment() % TInfo.Width) != 0;
810
1.04k
  bool UseLibcall = Misaligned | Oversized;
811
1.04k
  bool ShouldCastToIntPtrTy = true;
812
813
1.04k
  CharUnits MaxInlineWidth =
814
1.04k
      getContext().toCharUnitsFromBits(MaxInlineWidthInBits);
815
816
1.04k
  DiagnosticsEngine &Diags = CGM.getDiags();
817
818
1.04k
  if (Misaligned) {
819
80
    Diags.Report(E->getBeginLoc(), diag::warn_atomic_op_misaligned)
820
80
        << (int)TInfo.Width.getQuantity()
821
80
        << (int)Ptr.getAlignment().getQuantity();
822
80
  }
823
824
1.04k
  if (Oversized) {
825
148
    Diags.Report(E->getBeginLoc(), diag::warn_atomic_op_oversized)
826
148
        << (int)TInfo.Width.getQuantity() << (int)MaxInlineWidth.getQuantity();
827
148
  }
828
829
1.04k
  llvm::Value *Order = EmitScalarExpr(E->getOrder());
830
1.04k
  llvm::Value *Scope =
831
1.04k
      E->getScopeModel() ? 
EmitScalarExpr(E->getScope())91
:
nullptr949
;
832
833
1.04k
  switch (E->getOp()) {
834
0
  case AtomicExpr::AO__c11_atomic_init:
835
0
  case AtomicExpr::AO__opencl_atomic_init:
836
0
    llvm_unreachable("Already handled above with EmitAtomicInit!");
837
838
119
  case AtomicExpr::AO__c11_atomic_load:
839
137
  case AtomicExpr::AO__opencl_atomic_load:
840
156
  case AtomicExpr::AO__atomic_load_n:
841
156
    break;
842
843
88
  case AtomicExpr::AO__atomic_load:
844
88
    Dest = EmitPointerWithAlignment(E->getVal1());
845
88
    break;
846
847
84
  case AtomicExpr::AO__atomic_store:
848
84
    Val1 = EmitPointerWithAlignment(E->getVal1());
849
84
    break;
850
851
14
  case AtomicExpr::AO__atomic_exchange:
852
14
    Val1 = EmitPointerWithAlignment(E->getVal1());
853
14
    Dest = EmitPointerWithAlignment(E->getVal2());
854
14
    break;
855
856
18
  case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
857
20
  case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
858
28
  case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
859
39
  case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
860
49
  case AtomicExpr::AO__atomic_compare_exchange_n:
861
71
  case AtomicExpr::AO__atomic_compare_exchange:
862
71
    Val1 = EmitPointerWithAlignment(E->getVal1());
863
71
    if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange)
864
22
      Val2 = EmitPointerWithAlignment(E->getVal2());
865
49
    else
866
49
      Val2 = EmitValToTemp(*this, E->getVal2());
867
71
    OrderFail = EmitScalarExpr(E->getOrderFail());
868
71
    if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange_n ||
869
71
        
E->getOp() == AtomicExpr::AO__atomic_compare_exchange61
)
870
32
      IsWeak = EmitScalarExpr(E->getWeak());
871
71
    break;
872
873
146
  case AtomicExpr::AO__c11_atomic_fetch_add:
874
226
  case AtomicExpr::AO__c11_atomic_fetch_sub:
875
246
  case AtomicExpr::AO__opencl_atomic_fetch_add:
876
246
  case AtomicExpr::AO__opencl_atomic_fetch_sub:
877
246
    if (MemTy->isPointerType()) {
878
      // For pointer arithmetic, we're required to do a bit of math:
879
      // adding 1 to an int* is not the same as adding 1 to a uintptr_t.
880
      // ... but only for the C11 builtins. The GNU builtins expect the
881
      // user to multiply by sizeof(T).
882
6
      QualType Val1Ty = E->getVal1()->getType();
883
6
      llvm::Value *Val1Scalar = EmitScalarExpr(E->getVal1());
884
6
      CharUnits PointeeIncAmt =
885
6
          getContext().getTypeSizeInChars(MemTy->getPointeeType());
886
6
      Val1Scalar = Builder.CreateMul(Val1Scalar, CGM.getSize(PointeeIncAmt));
887
6
      auto Temp = CreateMemTemp(Val1Ty, ".atomictmp");
888
6
      Val1 = Temp;
889
6
      EmitStoreOfScalar(Val1Scalar, MakeAddrLValue(Temp, Val1Ty));
890
6
      break;
891
6
    }
892
246
    
LLVM_FALLTHROUGH240
;240
893
271
  case AtomicExpr::AO__atomic_fetch_add:
894
305
  case AtomicExpr::AO__atomic_fetch_sub:
895
369
  case AtomicExpr::AO__atomic_add_fetch:
896
370
  case AtomicExpr::AO__atomic_sub_fetch:
897
370
    ShouldCastToIntPtrTy = !MemTy->isFloatingType();
898
370
    LLVM_FALLTHROUGH;
899
900
492
  case AtomicExpr::AO__c11_atomic_store:
901
510
  case AtomicExpr::AO__c11_atomic_exchange:
902
528
  case AtomicExpr::AO__opencl_atomic_store:
903
530
  case AtomicExpr::AO__opencl_atomic_exchange:
904
534
  case AtomicExpr::AO__atomic_store_n:
905
538
  case AtomicExpr::AO__atomic_exchange_n:
906
542
  case AtomicExpr::AO__c11_atomic_fetch_and:
907
544
  case AtomicExpr::AO__c11_atomic_fetch_or:
908
544
  case AtomicExpr::AO__c11_atomic_fetch_xor:
909
554
  case AtomicExpr::AO__c11_atomic_fetch_max:
910
564
  case AtomicExpr::AO__c11_atomic_fetch_min:
911
566
  case AtomicExpr::AO__opencl_atomic_fetch_and:
912
566
  case AtomicExpr::AO__opencl_atomic_fetch_or:
913
566
  case AtomicExpr::AO__opencl_atomic_fetch_xor:
914
574
  case AtomicExpr::AO__opencl_atomic_fetch_min:
915
578
  case AtomicExpr::AO__opencl_atomic_fetch_max:
916
579
  case AtomicExpr::AO__atomic_fetch_and:
917
583
  case AtomicExpr::AO__atomic_fetch_or:
918
588
  case AtomicExpr::AO__atomic_fetch_xor:
919
593
  case AtomicExpr::AO__atomic_fetch_nand:
920
594
  case AtomicExpr::AO__atomic_and_fetch:
921
597
  case AtomicExpr::AO__atomic_or_fetch:
922
598
  case AtomicExpr::AO__atomic_xor_fetch:
923
603
  case AtomicExpr::AO__atomic_nand_fetch:
924
607
  case AtomicExpr::AO__atomic_max_fetch:
925
617
  case AtomicExpr::AO__atomic_min_fetch:
926
619
  case AtomicExpr::AO__atomic_fetch_max:
927
621
  case AtomicExpr::AO__atomic_fetch_min:
928
621
    Val1 = EmitValToTemp(*this, E->getVal1());
929
621
    break;
930
1.04k
  }
931
932
1.04k
  QualType RValTy = E->getType().getUnqualifiedType();
933
934
  // The inlined atomics only function on iN types, where N is a power of 2. We
935
  // need to make sure (via temporaries if necessary) that all incoming values
936
  // are compatible.
937
1.04k
  LValue AtomicVal = MakeAddrLValue(Ptr, AtomicTy);
938
1.04k
  AtomicInfo Atomics(*this, AtomicVal);
939
940
1.04k
  if (ShouldCastToIntPtrTy) {
941
992
    Ptr = Atomics.emitCastToAtomicIntPointer(Ptr);
942
992
    if (Val1.isValid())
943
748
      Val1 = Atomics.convertToAtomicIntPointer(Val1);
944
992
    if (Val2.isValid())
945
71
      Val2 = Atomics.convertToAtomicIntPointer(Val2);
946
992
  }
947
1.04k
  if (Dest.isValid()) {
948
102
    if (ShouldCastToIntPtrTy)
949
102
      Dest = Atomics.emitCastToAtomicIntPointer(Dest);
950
938
  } else if (E->isCmpXChg())
951
71
    Dest = CreateMemTemp(RValTy, "cmpxchg.bool");
952
867
  else if (!RValTy->isVoidType()) {
953
639
    Dest = Atomics.CreateTempAlloca();
954
639
    if (ShouldCastToIntPtrTy)
955
591
      Dest = Atomics.emitCastToAtomicIntPointer(Dest);
956
639
  }
957
958
  // Use a library call.  See: http://gcc.gnu.org/wiki/Atomic/GCCMM/LIbrary .
959
1.04k
  if (UseLibcall) {
960
183
    bool UseOptimizedLibcall = false;
961
183
    switch (E->getOp()) {
962
0
    case AtomicExpr::AO__c11_atomic_init:
963
0
    case AtomicExpr::AO__opencl_atomic_init:
964
0
      llvm_unreachable("Already handled above with EmitAtomicInit!");
965
966
9
    case AtomicExpr::AO__c11_atomic_fetch_add:
967
11
    case AtomicExpr::AO__opencl_atomic_fetch_add:
968
16
    case AtomicExpr::AO__atomic_fetch_add:
969
16
    case AtomicExpr::AO__c11_atomic_fetch_and:
970
16
    case AtomicExpr::AO__opencl_atomic_fetch_and:
971
17
    case AtomicExpr::AO__atomic_fetch_and:
972
17
    case AtomicExpr::AO__c11_atomic_fetch_or:
973
17
    case AtomicExpr::AO__opencl_atomic_fetch_or:
974
18
    case AtomicExpr::AO__atomic_fetch_or:
975
19
    case AtomicExpr::AO__atomic_fetch_nand:
976
21
    case AtomicExpr::AO__c11_atomic_fetch_sub:
977
21
    case AtomicExpr::AO__opencl_atomic_fetch_sub:
978
27
    case AtomicExpr::AO__atomic_fetch_sub:
979
27
    case AtomicExpr::AO__c11_atomic_fetch_xor:
980
27
    case AtomicExpr::AO__opencl_atomic_fetch_xor:
981
31
    case AtomicExpr::AO__opencl_atomic_fetch_min:
982
31
    case AtomicExpr::AO__opencl_atomic_fetch_max:
983
32
    case AtomicExpr::AO__atomic_fetch_xor:
984
32
    case AtomicExpr::AO__c11_atomic_fetch_max:
985
32
    case AtomicExpr::AO__c11_atomic_fetch_min:
986
33
    case AtomicExpr::AO__atomic_add_fetch:
987
34
    case AtomicExpr::AO__atomic_and_fetch:
988
35
    case AtomicExpr::AO__atomic_nand_fetch:
989
36
    case AtomicExpr::AO__atomic_or_fetch:
990
37
    case AtomicExpr::AO__atomic_sub_fetch:
991
38
    case AtomicExpr::AO__atomic_xor_fetch:
992
38
    case AtomicExpr::AO__atomic_fetch_max:
993
38
    case AtomicExpr::AO__atomic_fetch_min:
994
38
    case AtomicExpr::AO__atomic_max_fetch:
995
40
    case AtomicExpr::AO__atomic_min_fetch:
996
      // For these, only library calls for certain sizes exist.
997
40
      UseOptimizedLibcall = true;
998
40
      break;
999
1000
37
    case AtomicExpr::AO__atomic_load:
1001
72
    case AtomicExpr::AO__atomic_store:
1002
80
    case AtomicExpr::AO__atomic_exchange:
1003
89
    case AtomicExpr::AO__atomic_compare_exchange:
1004
      // Use the generic version if we don't know that the operand will be
1005
      // suitably aligned for the optimized version.
1006
89
      if (Misaligned)
1007
63
        break;
1008
89
      
LLVM_FALLTHROUGH26
;26
1009
37
    case AtomicExpr::AO__c11_atomic_load:
1010
49
    case AtomicExpr::AO__c11_atomic_store:
1011
54
    case AtomicExpr::AO__c11_atomic_exchange:
1012
54
    case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
1013
59
    case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
1014
61
    case AtomicExpr::AO__opencl_atomic_load:
1015
69
    case AtomicExpr::AO__opencl_atomic_store:
1016
69
    case AtomicExpr::AO__opencl_atomic_exchange:
1017
78
    case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
1018
80
    case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
1019
80
    case AtomicExpr::AO__atomic_load_n:
1020
80
    case AtomicExpr::AO__atomic_store_n:
1021
80
    case AtomicExpr::AO__atomic_exchange_n:
1022
80
    case AtomicExpr::AO__atomic_compare_exchange_n:
1023
      // Only use optimized library calls for sizes for which they exist.
1024
      // FIXME: Size == 16 optimized library functions exist too.
1025
80
      if (Size == 1 || 
Size == 272
||
Size == 470
||
Size == 841
)
1026
63
        UseOptimizedLibcall = true;
1027
80
      break;
1028
183
    }
1029
1030
183
    CallArgList Args;
1031
183
    if (!UseOptimizedLibcall) {
1032
      // For non-optimized library calls, the size is the first parameter
1033
80
      Args.add(RValue::get(llvm::ConstantInt::get(SizeTy, Size)),
1034
80
               getContext().getSizeType());
1035
80
    }
1036
    // Atomic address is the first or second parameter
1037
    // The OpenCL atomic library functions only accept pointer arguments to
1038
    // generic address space.
1039
208
    auto CastToGenericAddrSpace = [&](llvm::Value *V, QualType PT) {
1040
208
      if (!E->isOpenCL())
1041
170
        return V;
1042
38
      auto AS = PT->castAs<PointerType>()->getPointeeType().getAddressSpace();
1043
38
      if (AS == LangAS::opencl_generic)
1044
21
        return V;
1045
17
      auto DestAS = getContext().getTargetAddressSpace(LangAS::opencl_generic);
1046
17
      auto T = V->getType();
1047
17
      auto *DestType = T->getPointerElementType()->getPointerTo(DestAS);
1048
1049
17
      return getTargetHooks().performAddrSpaceCast(
1050
17
          *this, V, AS, LangAS::opencl_generic, DestType, false);
1051
38
    };
1052
1053
183
    Args.add(RValue::get(CastToGenericAddrSpace(
1054
183
                 EmitCastToVoidPtr(Ptr.getPointer()), E->getPtr()->getType())),
1055
183
             getContext().VoidPtrTy);
1056
1057
183
    std::string LibCallName;
1058
183
    QualType LoweredMemTy =
1059
183
      MemTy->isPointerType() ? 
getContext().getIntPtrType()3
:
MemTy180
;
1060
183
    QualType RetTy;
1061
183
    bool HaveRetTy = false;
1062
183
    llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0;
1063
183
    bool PostOpMinMax = false;
1064
183
    switch (E->getOp()) {
1065
0
    case AtomicExpr::AO__c11_atomic_init:
1066
0
    case AtomicExpr::AO__opencl_atomic_init:
1067
0
      llvm_unreachable("Already handled!");
1068
1069
    // There is only one libcall for compare an exchange, because there is no
1070
    // optimisation benefit possible from a libcall version of a weak compare
1071
    // and exchange.
1072
    // bool __atomic_compare_exchange(size_t size, void *mem, void *expected,
1073
    //                                void *desired, int success, int failure)
1074
    // bool __atomic_compare_exchange_N(T *mem, T *expected, T desired,
1075
    //                                  int success, int failure)
1076
0
    case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
1077
5
    case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
1078
14
    case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
1079
16
    case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
1080
25
    case AtomicExpr::AO__atomic_compare_exchange:
1081
25
    case AtomicExpr::AO__atomic_compare_exchange_n:
1082
25
      LibCallName = "__atomic_compare_exchange";
1083
25
      RetTy = getContext().BoolTy;
1084
25
      HaveRetTy = true;
1085
25
      Args.add(
1086
25
          RValue::get(CastToGenericAddrSpace(
1087
25
              EmitCastToVoidPtr(Val1.getPointer()), E->getVal1()->getType())),
1088
25
          getContext().VoidPtrTy);
1089
25
      AddDirectArgument(*this, Args, UseOptimizedLibcall, Val2.getPointer(),
1090
25
                        MemTy, E->getExprLoc(), TInfo.Width);
1091
25
      Args.add(RValue::get(Order), getContext().IntTy);
1092
25
      Order = OrderFail;
1093
25
      break;
1094
    // void __atomic_exchange(size_t size, void *mem, void *val, void *return,
1095
    //                        int order)
1096
    // T __atomic_exchange_N(T *mem, T val, int order)
1097
5
    case AtomicExpr::AO__c11_atomic_exchange:
1098
5
    case AtomicExpr::AO__opencl_atomic_exchange:
1099
5
    case AtomicExpr::AO__atomic_exchange_n:
1100
13
    case AtomicExpr::AO__atomic_exchange:
1101
13
      LibCallName = "__atomic_exchange";
1102
13
      AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1103
13
                        MemTy, E->getExprLoc(), TInfo.Width);
1104
13
      break;
1105
    // void __atomic_store(size_t size, void *mem, void *val, int order)
1106
    // void __atomic_store_N(T *mem, T val, int order)
1107
12
    case AtomicExpr::AO__c11_atomic_store:
1108
20
    case AtomicExpr::AO__opencl_atomic_store:
1109
55
    case AtomicExpr::AO__atomic_store:
1110
55
    case AtomicExpr::AO__atomic_store_n:
1111
55
      LibCallName = "__atomic_store";
1112
55
      RetTy = getContext().VoidTy;
1113
55
      HaveRetTy = true;
1114
55
      AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1115
55
                        MemTy, E->getExprLoc(), TInfo.Width);
1116
55
      break;
1117
    // void __atomic_load(size_t size, void *mem, void *return, int order)
1118
    // T __atomic_load_N(T *mem, int order)
1119
11
    case AtomicExpr::AO__c11_atomic_load:
1120
13
    case AtomicExpr::AO__opencl_atomic_load:
1121
50
    case AtomicExpr::AO__atomic_load:
1122
50
    case AtomicExpr::AO__atomic_load_n:
1123
50
      LibCallName = "__atomic_load";
1124
50
      break;
1125
    // T __atomic_add_fetch_N(T *mem, T val, int order)
1126
    // T __atomic_fetch_add_N(T *mem, T val, int order)
1127
1
    case AtomicExpr::AO__atomic_add_fetch:
1128
1
      PostOp = llvm::Instruction::Add;
1129
1
      LLVM_FALLTHROUGH;
1130
10
    case AtomicExpr::AO__c11_atomic_fetch_add:
1131
12
    case AtomicExpr::AO__opencl_atomic_fetch_add:
1132
17
    case AtomicExpr::AO__atomic_fetch_add:
1133
17
      LibCallName = "__atomic_fetch_add";
1134
17
      AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1135
17
                        LoweredMemTy, E->getExprLoc(), TInfo.Width);
1136
17
      break;
1137
    // T __atomic_and_fetch_N(T *mem, T val, int order)
1138
    // T __atomic_fetch_and_N(T *mem, T val, int order)
1139
1
    case AtomicExpr::AO__atomic_and_fetch:
1140
1
      PostOp = llvm::Instruction::And;
1141
1
      LLVM_FALLTHROUGH;
1142
1
    case AtomicExpr::AO__c11_atomic_fetch_and:
1143
1
    case AtomicExpr::AO__opencl_atomic_fetch_and:
1144
2
    case AtomicExpr::AO__atomic_fetch_and:
1145
2
      LibCallName = "__atomic_fetch_and";
1146
2
      AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1147
2
                        MemTy, E->getExprLoc(), TInfo.Width);
1148
2
      break;
1149
    // T __atomic_or_fetch_N(T *mem, T val, int order)
1150
    // T __atomic_fetch_or_N(T *mem, T val, int order)
1151
1
    case AtomicExpr::AO__atomic_or_fetch:
1152
1
      PostOp = llvm::Instruction::Or;
1153
1
      LLVM_FALLTHROUGH;
1154
1
    case AtomicExpr::AO__c11_atomic_fetch_or:
1155
1
    case AtomicExpr::AO__opencl_atomic_fetch_or:
1156
2
    case AtomicExpr::AO__atomic_fetch_or:
1157
2
      LibCallName = "__atomic_fetch_or";
1158
2
      AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1159
2
                        MemTy, E->getExprLoc(), TInfo.Width);
1160
2
      break;
1161
    // T __atomic_sub_fetch_N(T *mem, T val, int order)
1162
    // T __atomic_fetch_sub_N(T *mem, T val, int order)
1163
1
    case AtomicExpr::AO__atomic_sub_fetch:
1164
1
      PostOp = llvm::Instruction::Sub;
1165
1
      LLVM_FALLTHROUGH;
1166
3
    case AtomicExpr::AO__c11_atomic_fetch_sub:
1167
3
    case AtomicExpr::AO__opencl_atomic_fetch_sub:
1168
9
    case AtomicExpr::AO__atomic_fetch_sub:
1169
9
      LibCallName = "__atomic_fetch_sub";
1170
9
      AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1171
9
                        LoweredMemTy, E->getExprLoc(), TInfo.Width);
1172
9
      break;
1173
    // T __atomic_xor_fetch_N(T *mem, T val, int order)
1174
    // T __atomic_fetch_xor_N(T *mem, T val, int order)
1175
1
    case AtomicExpr::AO__atomic_xor_fetch:
1176
1
      PostOp = llvm::Instruction::Xor;
1177
1
      LLVM_FALLTHROUGH;
1178
1
    case AtomicExpr::AO__c11_atomic_fetch_xor:
1179
1
    case AtomicExpr::AO__opencl_atomic_fetch_xor:
1180
2
    case AtomicExpr::AO__atomic_fetch_xor:
1181
2
      LibCallName = "__atomic_fetch_xor";
1182
2
      AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1183
2
                        MemTy, E->getExprLoc(), TInfo.Width);
1184
2
      break;
1185
2
    case AtomicExpr::AO__atomic_min_fetch:
1186
2
      PostOpMinMax = true;
1187
2
      LLVM_FALLTHROUGH;
1188
2
    case AtomicExpr::AO__c11_atomic_fetch_min:
1189
2
    case AtomicExpr::AO__atomic_fetch_min:
1190
6
    case AtomicExpr::AO__opencl_atomic_fetch_min:
1191
6
      LibCallName = E->getValueType()->isSignedIntegerType()
1192
6
                        ? 
"__atomic_fetch_min"2
1193
6
                        : 
"__atomic_fetch_umin"4
;
1194
6
      AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1195
6
                        LoweredMemTy, E->getExprLoc(), TInfo.Width);
1196
6
      break;
1197
0
    case AtomicExpr::AO__atomic_max_fetch:
1198
0
      PostOpMinMax = true;
1199
0
      LLVM_FALLTHROUGH;
1200
0
    case AtomicExpr::AO__c11_atomic_fetch_max:
1201
0
    case AtomicExpr::AO__atomic_fetch_max:
1202
0
    case AtomicExpr::AO__opencl_atomic_fetch_max:
1203
0
      LibCallName = E->getValueType()->isSignedIntegerType()
1204
0
                        ? "__atomic_fetch_max"
1205
0
                        : "__atomic_fetch_umax";
1206
0
      AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1207
0
                        LoweredMemTy, E->getExprLoc(), TInfo.Width);
1208
0
      break;
1209
    // T __atomic_nand_fetch_N(T *mem, T val, int order)
1210
    // T __atomic_fetch_nand_N(T *mem, T val, int order)
1211
1
    case AtomicExpr::AO__atomic_nand_fetch:
1212
1
      PostOp = llvm::Instruction::And; // the NOT is special cased below
1213
1
      LLVM_FALLTHROUGH;
1214
2
    case AtomicExpr::AO__atomic_fetch_nand:
1215
2
      LibCallName = "__atomic_fetch_nand";
1216
2
      AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1217
2
                        MemTy, E->getExprLoc(), TInfo.Width);
1218
2
      break;
1219
183
    }
1220
1221
183
    if (E->isOpenCL()) {
1222
27
      LibCallName = std::string("__opencl") +
1223
27
          StringRef(LibCallName).drop_front(1).str();
1224
1225
27
    }
1226
    // Optimized functions have the size in their name.
1227
183
    if (UseOptimizedLibcall)
1228
103
      LibCallName += "_" + llvm::utostr(Size);
1229
    // By default, assume we return a value of the atomic type.
1230
183
    if (!HaveRetTy) {
1231
103
      if (UseOptimizedLibcall) {
1232
        // Value is returned directly.
1233
        // The function returns an appropriately sized integer type.
1234
63
        RetTy = getContext().getIntTypeForBitwidth(
1235
63
            getContext().toBits(TInfo.Width), /*Signed=*/false);
1236
63
      } else {
1237
        // Value is returned through parameter before the order.
1238
40
        RetTy = getContext().VoidTy;
1239
40
        Args.add(RValue::get(EmitCastToVoidPtr(Dest.getPointer())),
1240
40
                 getContext().VoidPtrTy);
1241
40
      }
1242
103
    }
1243
    // order is always the last parameter
1244
183
    Args.add(RValue::get(Order),
1245
183
             getContext().IntTy);
1246
183
    if (E->isOpenCL())
1247
27
      Args.add(RValue::get(Scope), getContext().IntTy);
1248
1249
    // PostOp is only needed for the atomic_*_fetch operations, and
1250
    // thus is only needed for and implemented in the
1251
    // UseOptimizedLibcall codepath.
1252
183
    assert(UseOptimizedLibcall || (!PostOp && !PostOpMinMax));
1253
1254
0
    RValue Res = emitAtomicLibcall(*this, LibCallName, RetTy, Args);
1255
    // The value is returned directly from the libcall.
1256
183
    if (E->isCmpXChg())
1257
25
      return Res;
1258
1259
    // The value is returned directly for optimized libcalls but the expr
1260
    // provided an out-param.
1261
158
    if (UseOptimizedLibcall && 
Res.getScalarVal()91
) {
1262
63
      llvm::Value *ResVal = Res.getScalarVal();
1263
63
      if (PostOpMinMax) {
1264
2
        llvm::Value *LoadVal1 = Args[1].getRValue(*this).getScalarVal();
1265
2
        ResVal = EmitPostAtomicMinMax(Builder, E->getOp(),
1266
2
                                      E->getValueType()->isSignedIntegerType(),
1267
2
                                      ResVal, LoadVal1);
1268
61
      } else if (PostOp) {
1269
6
        llvm::Value *LoadVal1 = Args[1].getRValue(*this).getScalarVal();
1270
6
        ResVal = Builder.CreateBinOp(PostOp, ResVal, LoadVal1);
1271
6
      }
1272
63
      if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch)
1273
1
        ResVal = Builder.CreateNot(ResVal);
1274
1275
63
      Builder.CreateStore(
1276
63
          ResVal,
1277
63
          Builder.CreateBitCast(Dest, ResVal->getType()->getPointerTo()));
1278
63
    }
1279
1280
158
    if (RValTy->isVoidType())
1281
100
      return RValue::get(nullptr);
1282
1283
58
    return convertTempToRValue(
1284
58
        Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo()),
1285
58
        RValTy, E->getExprLoc());
1286
158
  }
1287
1288
857
  bool IsStore = E->getOp() == AtomicExpr::AO__c11_atomic_store ||
1289
857
                 
E->getOp() == AtomicExpr::AO__opencl_atomic_store747
||
1290
857
                 
E->getOp() == AtomicExpr::AO__atomic_store737
||
1291
857
                 
E->getOp() == AtomicExpr::AO__atomic_store_n688
;
1292
857
  bool IsLoad = E->getOp() == AtomicExpr::AO__c11_atomic_load ||
1293
857
                
E->getOp() == AtomicExpr::AO__opencl_atomic_load749
||
1294
857
                
E->getOp() == AtomicExpr::AO__atomic_load733
||
1295
857
                
E->getOp() == AtomicExpr::AO__atomic_load_n682
;
1296
1297
857
  if (isa<llvm::ConstantInt>(Order)) {
1298
534
    auto ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
1299
    // We should not ever get to a case where the ordering isn't a valid C ABI
1300
    // value, but it's hard to enforce that in general.
1301
534
    if (llvm::isValidAtomicOrderingCABI(ord))
1302
534
      switch ((llvm::AtomicOrderingCABI)ord) {
1303
104
      case llvm::AtomicOrderingCABI::relaxed:
1304
104
        EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1305
104
                     llvm::AtomicOrdering::Monotonic, Scope);
1306
104
        break;
1307
0
      case llvm::AtomicOrderingCABI::consume:
1308
42
      case llvm::AtomicOrderingCABI::acquire:
1309
42
        if (IsStore)
1310
0
          break; // Avoid crashing on code with undefined behavior
1311
42
        EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1312
42
                     llvm::AtomicOrdering::Acquire, Scope);
1313
42
        break;
1314
22
      case llvm::AtomicOrderingCABI::release:
1315
22
        if (IsLoad)
1316
0
          break; // Avoid crashing on code with undefined behavior
1317
22
        EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1318
22
                     llvm::AtomicOrdering::Release, Scope);
1319
22
        break;
1320
54
      case llvm::AtomicOrderingCABI::acq_rel:
1321
54
        if (IsLoad || IsStore)
1322
0
          break; // Avoid crashing on code with undefined behavior
1323
54
        EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1324
54
                     llvm::AtomicOrdering::AcquireRelease, Scope);
1325
54
        break;
1326
312
      case llvm::AtomicOrderingCABI::seq_cst:
1327
312
        EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1328
312
                     llvm::AtomicOrdering::SequentiallyConsistent, Scope);
1329
312
        break;
1330
534
      }
1331
534
    if (RValTy->isVoidType())
1332
140
      return RValue::get(nullptr);
1333
1334
394
    return convertTempToRValue(
1335
394
        Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo(
1336
394
                                        Dest.getAddressSpace())),
1337
394
        RValTy, E->getExprLoc());
1338
534
  }
1339
1340
  // Long case, when Order isn't obviously constant.
1341
1342
  // Create all the relevant BB's
1343
323
  llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr,
1344
323
                   *ReleaseBB = nullptr, *AcqRelBB = nullptr,
1345
323
                   *SeqCstBB = nullptr;
1346
323
  MonotonicBB = createBasicBlock("monotonic", CurFn);
1347
323
  if (!IsStore)
1348
233
    AcquireBB = createBasicBlock("acquire", CurFn);
1349
323
  if (!IsLoad)
1350
241
    ReleaseBB = createBasicBlock("release", CurFn);
1351
323
  if (!IsLoad && 
!IsStore241
)
1352
151
    AcqRelBB = createBasicBlock("acqrel", CurFn);
1353
323
  SeqCstBB = createBasicBlock("seqcst", CurFn);
1354
323
  llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn);
1355
1356
  // Create the switch for the split
1357
  // MonotonicBB is arbitrarily chosen as the default case; in practice, this
1358
  // doesn't matter unless someone is crazy enough to use something that
1359
  // doesn't fold to a constant for the ordering.
1360
323
  Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);
1361
323
  llvm::SwitchInst *SI = Builder.CreateSwitch(Order, MonotonicBB);
1362
1363
  // Emit all the different atomics
1364
323
  Builder.SetInsertPoint(MonotonicBB);
1365
323
  EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1366
323
               llvm::AtomicOrdering::Monotonic, Scope);
1367
323
  Builder.CreateBr(ContBB);
1368
323
  if (!IsStore) {
1369
233
    Builder.SetInsertPoint(AcquireBB);
1370
233
    EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1371
233
                 llvm::AtomicOrdering::Acquire, Scope);
1372
233
    Builder.CreateBr(ContBB);
1373
233
    SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::consume),
1374
233
                AcquireBB);
1375
233
    SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire),
1376
233
                AcquireBB);
1377
233
  }
1378
323
  if (!IsLoad) {
1379
241
    Builder.SetInsertPoint(ReleaseBB);
1380
241
    EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1381
241
                 llvm::AtomicOrdering::Release, Scope);
1382
241
    Builder.CreateBr(ContBB);
1383
241
    SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::release),
1384
241
                ReleaseBB);
1385
241
  }
1386
323
  if (!IsLoad && 
!IsStore241
) {
1387
151
    Builder.SetInsertPoint(AcqRelBB);
1388
151
    EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1389
151
                 llvm::AtomicOrdering::AcquireRelease, Scope);
1390
151
    Builder.CreateBr(ContBB);
1391
151
    SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acq_rel),
1392
151
                AcqRelBB);
1393
151
  }
1394
323
  Builder.SetInsertPoint(SeqCstBB);
1395
323
  EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1396
323
               llvm::AtomicOrdering::SequentiallyConsistent, Scope);
1397
323
  Builder.CreateBr(ContBB);
1398
323
  SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst),
1399
323
              SeqCstBB);
1400
1401
  // Cleanup and return
1402
323
  Builder.SetInsertPoint(ContBB);
1403
323
  if (RValTy->isVoidType())
1404
90
    return RValue::get(nullptr);
1405
1406
233
  assert(Atomics.getValueSizeInBits() <= Atomics.getAtomicSizeInBits());
1407
0
  return convertTempToRValue(
1408
233
      Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo(
1409
233
                                      Dest.getAddressSpace())),
1410
233
      RValTy, E->getExprLoc());
1411
323
}
1412
1413
3.71k
Address AtomicInfo::emitCastToAtomicIntPointer(Address addr) const {
1414
3.71k
  unsigned addrspace =
1415
3.71k
    cast<llvm::PointerType>(addr.getPointer()->getType())->getAddressSpace();
1416
3.71k
  llvm::IntegerType *ty =
1417
3.71k
    llvm::IntegerType::get(CGF.getLLVMContext(), AtomicSizeInBits);
1418
3.71k
  return CGF.Builder.CreateBitCast(addr, ty->getPointerTo(addrspace));
1419
3.71k
}
1420
1421
819
Address AtomicInfo::convertToAtomicIntPointer(Address Addr) const {
1422
819
  llvm::Type *Ty = Addr.getElementType();
1423
819
  uint64_t SourceSizeInBits = CGF.CGM.getDataLayout().getTypeSizeInBits(Ty);
1424
819
  if (SourceSizeInBits != AtomicSizeInBits) {
1425
9
    Address Tmp = CreateTempAlloca();
1426
9
    CGF.Builder.CreateMemCpy(Tmp, Addr,
1427
9
                             std::min(AtomicSizeInBits, SourceSizeInBits) / 8);
1428
9
    Addr = Tmp;
1429
9
  }
1430
1431
819
  return emitCastToAtomicIntPointer(Addr);
1432
819
}
1433
1434
RValue AtomicInfo::convertAtomicTempToRValue(Address addr,
1435
                                             AggValueSlot resultSlot,
1436
                                             SourceLocation loc,
1437
180
                                             bool asValue) const {
1438
180
  if (LVal.isSimple()) {
1439
144
    if (EvaluationKind == TEK_Aggregate)
1440
13
      return resultSlot.asRValue();
1441
1442
    // Drill into the padding structure if we have one.
1443
131
    if (hasPadding())
1444
0
      addr = CGF.Builder.CreateStructGEP(addr, 0);
1445
1446
    // Otherwise, just convert the temporary to an r-value using the
1447
    // normal conversion routine.
1448
131
    return CGF.convertTempToRValue(addr, getValueType(), loc);
1449
144
  }
1450
36
  if (!asValue)
1451
    // Get RValue from temp memory as atomic for non-simple lvalues
1452
12
    return RValue::get(CGF.Builder.CreateLoad(addr));
1453
24
  if (LVal.isBitField())
1454
20
    return CGF.EmitLoadOfBitfieldLValue(
1455
20
        LValue::MakeBitfield(addr, LVal.getBitFieldInfo(), LVal.getType(),
1456
20
                             LVal.getBaseInfo(), TBAAAccessInfo()), loc);
1457
4
  if (LVal.isVectorElt())
1458
2
    return CGF.EmitLoadOfLValue(
1459
2
        LValue::MakeVectorElt(addr, LVal.getVectorIdx(), LVal.getType(),
1460
2
                              LVal.getBaseInfo(), TBAAAccessInfo()), loc);
1461
2
  assert(LVal.isExtVectorElt());
1462
0
  return CGF.EmitLoadOfExtVectorElementLValue(LValue::MakeExtVectorElt(
1463
2
      addr, LVal.getExtVectorElts(), LVal.getType(),
1464
2
      LVal.getBaseInfo(), TBAAAccessInfo()));
1465
4
}
1466
1467
RValue AtomicInfo::ConvertIntToValueOrAtomic(llvm::Value *IntVal,
1468
                                             AggValueSlot ResultSlot,
1469
                                             SourceLocation Loc,
1470
418
                                             bool AsValue) const {
1471
  // Try not to in some easy cases.
1472
418
  assert(IntVal->getType()->isIntegerTy() && "Expected integer value");
1473
418
  if (getEvaluationKind() == TEK_Scalar &&
1474
418
      
(408
(408
(408
!LVal.isBitField()408
||
1475
408
         
LVal.getBitFieldInfo().Size == ValueSizeInBits64
) &&
1476
408
        
!hasPadding()344
) ||
1477
408
       
!AsValue80
)) {
1478
388
    auto *ValTy = AsValue
1479
388
                      ? 
CGF.ConvertTypeForMem(ValueTy)109
1480
388
                      : 
getAtomicAddress().getType()->getPointerElementType()279
;
1481
388
    if (ValTy->isIntegerTy()) {
1482
279
      assert(IntVal->getType() == ValTy && "Different integer types.");
1483
0
      return RValue::get(CGF.EmitFromMemory(IntVal, ValueTy));
1484
279
    } else 
if (109
ValTy->isPointerTy()109
)
1485
0
      return RValue::get(CGF.Builder.CreateIntToPtr(IntVal, ValTy));
1486
109
    else if (llvm::CastInst::isBitCastable(IntVal->getType(), ValTy))
1487
77
      return RValue::get(CGF.Builder.CreateBitCast(IntVal, ValTy));
1488
388
  }
1489
1490
  // Create a temporary.  This needs to be big enough to hold the
1491
  // atomic integer.
1492
62
  Address Temp = Address::invalid();
1493
62
  bool TempIsVolatile = false;
1494
62
  if (AsValue && 
getEvaluationKind() == TEK_Aggregate44
) {
1495
8
    assert(!ResultSlot.isIgnored());
1496
0
    Temp = ResultSlot.getAddress();
1497
8
    TempIsVolatile = ResultSlot.isVolatile();
1498
54
  } else {
1499
54
    Temp = CreateTempAlloca();
1500
54
  }
1501
1502
  // Slam the integer into the temporary.
1503
0
  Address CastTemp = emitCastToAtomicIntPointer(Temp);
1504
62
  CGF.Builder.CreateStore(IntVal, CastTemp)
1505
62
      ->setVolatile(TempIsVolatile);
1506
1507
62
  return convertAtomicTempToRValue(Temp, ResultSlot, Loc, AsValue);
1508
418
}
1509
1510
void AtomicInfo::EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
1511
105
                                       llvm::AtomicOrdering AO, bool) {
1512
  // void __atomic_load(size_t size, void *mem, void *return, int order);
1513
105
  CallArgList Args;
1514
105
  Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
1515
105
  Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())),
1516
105
           CGF.getContext().VoidPtrTy);
1517
105
  Args.add(RValue::get(CGF.EmitCastToVoidPtr(AddForLoaded)),
1518
105
           CGF.getContext().VoidPtrTy);
1519
105
  Args.add(
1520
105
      RValue::get(llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(AO))),
1521
105
      CGF.getContext().IntTy);
1522
105
  emitAtomicLibcall(CGF, "__atomic_load", CGF.getContext().VoidTy, Args);
1523
105
}
1524
1525
llvm::Value *AtomicInfo::EmitAtomicLoadOp(llvm::AtomicOrdering AO,
1526
422
                                          bool IsVolatile) {
1527
  // Okay, we're doing this natively.
1528
422
  Address Addr = getAtomicAddressAsAtomicIntPointer();
1529
422
  llvm::LoadInst *Load = CGF.Builder.CreateLoad(Addr, "atomic-load");
1530
422
  Load->setAtomic(AO);
1531
1532
  // Other decoration.
1533
422
  if (IsVolatile)
1534
12
    Load->setVolatile(true);
1535
422
  CGF.CGM.DecorateInstructionWithTBAA(Load, LVal.getTBAAInfo());
1536
422
  return Load;
1537
422
}
1538
1539
/// An LValue is a candidate for having its loads and stores be made atomic if
1540
/// we are operating under /volatile:ms *and* the LValue itself is volatile and
1541
/// performing such an operation can be performed without a libcall.
1542
1.78M
bool CodeGenFunction::LValueIsSuitableForInlineAtomic(LValue LV) {
1543
1.78M
  if (!CGM.getCodeGenOpts().MSVolatile) 
return false1.78M
;
1544
45
  AtomicInfo AI(*this, LV);
1545
45
  bool IsVolatile = LV.isVolatile() || 
hasVolatileMember(LV.getType())27
;
1546
  // An atomic is inline if we don't need to use a libcall.
1547
45
  bool AtomicIsInline = !AI.shouldUseLibcall();
1548
  // MSVC doesn't seem to do this for types wider than a pointer.
1549
45
  if (getContext().getTypeSize(LV.getType()) >
1550
45
      getContext().getTypeSize(getContext().getIntPtrType()))
1551
5
    return false;
1552
40
  return IsVolatile && 
AtomicIsInline15
;
1553
45
}
1554
1555
RValue CodeGenFunction::EmitAtomicLoad(LValue LV, SourceLocation SL,
1556
77
                                       AggValueSlot Slot) {
1557
77
  llvm::AtomicOrdering AO;
1558
77
  bool IsVolatile = LV.isVolatileQualified();
1559
77
  if (LV.getType()->isAtomicType()) {
1560
70
    AO = llvm::AtomicOrdering::SequentiallyConsistent;
1561
70
  } else {
1562
7
    AO = llvm::AtomicOrdering::Acquire;
1563
7
    IsVolatile = true;
1564
7
  }
1565
77
  return EmitAtomicLoad(LV, SL, AO, IsVolatile, Slot);
1566
77
}
1567
1568
RValue AtomicInfo::EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
1569
                                  bool AsValue, llvm::AtomicOrdering AO,
1570
183
                                  bool IsVolatile) {
1571
  // Check whether we should use a library call.
1572
183
  if (shouldUseLibcall()) {
1573
44
    Address TempAddr = Address::invalid();
1574
44
    if (LVal.isSimple() && 
!ResultSlot.isIgnored()40
) {
1575
5
      assert(getEvaluationKind() == TEK_Aggregate);
1576
0
      TempAddr = ResultSlot.getAddress();
1577
5
    } else
1578
39
      TempAddr = CreateTempAlloca();
1579
1580
0
    EmitAtomicLoadLibcall(TempAddr.getPointer(), AO, IsVolatile);
1581
1582
    // Okay, turn that back into the original value or whole atomic (for
1583
    // non-simple lvalues) type.
1584
44
    return convertAtomicTempToRValue(TempAddr, ResultSlot, Loc, AsValue);
1585
44
  }
1586
1587
  // Okay, we're doing this natively.
1588
139
  auto *Load = EmitAtomicLoadOp(AO, IsVolatile);
1589
1590
  // If we're ignoring an aggregate return, don't do anything.
1591
139
  if (getEvaluationKind() == TEK_Aggregate && 
ResultSlot.isIgnored()8
)
1592
0
    return RValue::getAggregate(Address::invalid(), false);
1593
1594
  // Okay, turn that back into the original value or atomic (for non-simple
1595
  // lvalues) type.
1596
139
  return ConvertIntToValueOrAtomic(Load, ResultSlot, Loc, AsValue);
1597
139
}
1598
1599
/// Emit a load from an l-value of atomic type.  Note that the r-value
1600
/// we produce is an r-value of the atomic *value* type.
1601
RValue CodeGenFunction::EmitAtomicLoad(LValue src, SourceLocation loc,
1602
                                       llvm::AtomicOrdering AO, bool IsVolatile,
1603
183
                                       AggValueSlot resultSlot) {
1604
183
  AtomicInfo Atomics(*this, src);
1605
183
  return Atomics.EmitAtomicLoad(resultSlot, loc, /*AsValue=*/true, AO,
1606
183
                                IsVolatile);
1607
183
}
1608
1609
/// Copy an r-value into memory as part of storing to an atomic type.
1610
/// This needs to create a bit-pattern suitable for atomic operations.
1611
185
void AtomicInfo::emitCopyIntoMemory(RValue rvalue) const {
1612
185
  assert(LVal.isSimple());
1613
  // If we have an r-value, the rvalue should be of the atomic type,
1614
  // which means that the caller is responsible for having zeroed
1615
  // any padding.  Just do an aggregate copy of that type.
1616
185
  if (rvalue.isAggregate()) {
1617
0
    LValue Dest = CGF.MakeAddrLValue(getAtomicAddress(), getAtomicType());
1618
0
    LValue Src = CGF.MakeAddrLValue(rvalue.getAggregateAddress(),
1619
0
                                    getAtomicType());
1620
0
    bool IsVolatile = rvalue.isVolatileQualified() ||
1621
0
                      LVal.isVolatileQualified();
1622
0
    CGF.EmitAggregateCopy(Dest, Src, getAtomicType(),
1623
0
                          AggValueSlot::DoesNotOverlap, IsVolatile);
1624
0
    return;
1625
0
  }
1626
1627
  // Okay, otherwise we're copying stuff.
1628
1629
  // Zero out the buffer if necessary.
1630
185
  emitMemSetZeroIfNecessary();
1631
1632
  // Drill past the padding if present.
1633
185
  LValue TempLVal = projectValue();
1634
1635
  // Okay, store the rvalue in.
1636
185
  if (rvalue.isScalar()) {
1637
167
    CGF.EmitStoreOfScalar(rvalue.getScalarVal(), TempLVal, /*init*/ true);
1638
167
  } else {
1639
18
    CGF.EmitStoreOfComplex(rvalue.getComplexVal(), TempLVal, /*init*/ true);
1640
18
  }
1641
185
}
1642
1643
1644
/// Materialize an r-value into memory for the purposes of storing it
1645
/// to an atomic type.
1646
153
Address AtomicInfo::materializeRValue(RValue rvalue) const {
1647
  // Aggregate r-values are already in memory, and EmitAtomicStore
1648
  // requires them to be values of the atomic type.
1649
153
  if (rvalue.isAggregate())
1650
12
    return rvalue.getAggregateAddress();
1651
1652
  // Otherwise, make a temporary and materialize into it.
1653
141
  LValue TempLV = CGF.MakeAddrLValue(CreateTempAlloca(), getAtomicType());
1654
141
  AtomicInfo Atomics(CGF, TempLV);
1655
141
  Atomics.emitCopyIntoMemory(rvalue);
1656
141
  return TempLV.getAddress(CGF);
1657
153
}
1658
1659
152
llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal) const {
1660
  // If we've got a scalar value of the right size, try to avoid going
1661
  // through memory.
1662
152
  if (RVal.isScalar() && 
(141
!hasPadding()141
||
!LVal.isSimple()0
)) {
1663
141
    llvm::Value *Value = RVal.getScalarVal();
1664
141
    if (isa<llvm::IntegerType>(Value->getType()))
1665
111
      return CGF.EmitToMemory(Value, ValueTy);
1666
30
    else {
1667
30
      llvm::IntegerType *InputIntTy = llvm::IntegerType::get(
1668
30
          CGF.getLLVMContext(),
1669
30
          LVal.isSimple() ? getValueSizeInBits() : 
getAtomicSizeInBits()0
);
1670
30
      if (isa<llvm::PointerType>(Value->getType()))
1671
0
        return CGF.Builder.CreatePtrToInt(Value, InputIntTy);
1672
30
      else if (llvm::BitCastInst::isBitCastable(Value->getType(), InputIntTy))
1673
12
        return CGF.Builder.CreateBitCast(Value, InputIntTy);
1674
30
    }
1675
141
  }
1676
  // Otherwise, we need to go through memory.
1677
  // Put the r-value in memory.
1678
29
  Address Addr = materializeRValue(RVal);
1679
1680
  // Cast the temporary to the atomic int type and pull a value out.
1681
29
  Addr = emitCastToAtomicIntPointer(Addr);
1682
29
  return CGF.Builder.CreateLoad(Addr);
1683
152
}
1684
1685
std::pair<llvm::Value *, llvm::Value *> AtomicInfo::EmitAtomicCompareExchangeOp(
1686
    llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
1687
299
    llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak) {
1688
  // Do the atomic store.
1689
299
  Address Addr = getAtomicAddressAsAtomicIntPointer();
1690
299
  auto *Inst = CGF.Builder.CreateAtomicCmpXchg(Addr.getPointer(),
1691
299
                                               ExpectedVal, DesiredVal,
1692
299
                                               Success, Failure);
1693
  // Other decoration.
1694
299
  Inst->setVolatile(LVal.isVolatileQualified());
1695
299
  Inst->setWeak(IsWeak);
1696
1697
  // Okay, turn that back into the original value type.
1698
299
  auto *PreviousVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/0);
1699
299
  auto *SuccessFailureVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/1);
1700
299
  return std::make_pair(PreviousVal, SuccessFailureVal);
1701
299
}
1702
1703
llvm::Value *
1704
AtomicInfo::EmitAtomicCompareExchangeLibcall(llvm::Value *ExpectedAddr,
1705
                                             llvm::Value *DesiredAddr,
1706
                                             llvm::AtomicOrdering Success,
1707
78
                                             llvm::AtomicOrdering Failure) {
1708
  // bool __atomic_compare_exchange(size_t size, void *obj, void *expected,
1709
  // void *desired, int success, int failure);
1710
78
  CallArgList Args;
1711
78
  Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
1712
78
  Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())),
1713
78
           CGF.getContext().VoidPtrTy);
1714
78
  Args.add(RValue::get(CGF.EmitCastToVoidPtr(ExpectedAddr)),
1715
78
           CGF.getContext().VoidPtrTy);
1716
78
  Args.add(RValue::get(CGF.EmitCastToVoidPtr(DesiredAddr)),
1717
78
           CGF.getContext().VoidPtrTy);
1718
78
  Args.add(RValue::get(
1719
78
               llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Success))),
1720
78
           CGF.getContext().IntTy);
1721
78
  Args.add(RValue::get(
1722
78
               llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Failure))),
1723
78
           CGF.getContext().IntTy);
1724
78
  auto SuccessFailureRVal = emitAtomicLibcall(CGF, "__atomic_compare_exchange",
1725
78
                                              CGF.getContext().BoolTy, Args);
1726
1727
78
  return SuccessFailureRVal.getScalarVal();
1728
78
}
1729
1730
std::pair<RValue, llvm::Value *> AtomicInfo::EmitAtomicCompareExchange(
1731
    RValue Expected, RValue Desired, llvm::AtomicOrdering Success,
1732
33
    llvm::AtomicOrdering Failure, bool IsWeak) {
1733
  // Check whether we should use a library call.
1734
33
  if (shouldUseLibcall()) {
1735
    // Produce a source address.
1736
17
    Address ExpectedAddr = materializeRValue(Expected);
1737
17
    Address DesiredAddr = materializeRValue(Desired);
1738
17
    auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1739
17
                                                 DesiredAddr.getPointer(),
1740
17
                                                 Success, Failure);
1741
17
    return std::make_pair(
1742
17
        convertAtomicTempToRValue(ExpectedAddr, AggValueSlot::ignored(),
1743
17
                                  SourceLocation(), /*AsValue=*/false),
1744
17
        Res);
1745
17
  }
1746
1747
  // If we've got a scalar value of the right size, try to avoid going
1748
  // through memory.
1749
16
  auto *ExpectedVal = convertRValueToInt(Expected);
1750
16
  auto *DesiredVal = convertRValueToInt(Desired);
1751
16
  auto Res = EmitAtomicCompareExchangeOp(ExpectedVal, DesiredVal, Success,
1752
16
                                         Failure, IsWeak);
1753
16
  return std::make_pair(
1754
16
      ConvertIntToValueOrAtomic(Res.first, AggValueSlot::ignored(),
1755
16
                                SourceLocation(), /*AsValue=*/false),
1756
16
      Res.second);
1757
33
}
1758
1759
static void
1760
EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, RValue OldRVal,
1761
                      const llvm::function_ref<RValue(RValue)> &UpdateOp,
1762
320
                      Address DesiredAddr) {
1763
320
  RValue UpRVal;
1764
320
  LValue AtomicLVal = Atomics.getAtomicLValue();
1765
320
  LValue DesiredLVal;
1766
320
  if (AtomicLVal.isSimple()) {
1767
248
    UpRVal = OldRVal;
1768
248
    DesiredLVal = CGF.MakeAddrLValue(DesiredAddr, AtomicLVal.getType());
1769
248
  } else {
1770
    // Build new lvalue for temp address.
1771
72
    Address Ptr = Atomics.materializeRValue(OldRVal);
1772
72
    LValue UpdateLVal;
1773
72
    if (AtomicLVal.isBitField()) {
1774
60
      UpdateLVal =
1775
60
          LValue::MakeBitfield(Ptr, AtomicLVal.getBitFieldInfo(),
1776
60
                               AtomicLVal.getType(),
1777
60
                               AtomicLVal.getBaseInfo(),
1778
60
                               AtomicLVal.getTBAAInfo());
1779
60
      DesiredLVal =
1780
60
          LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(),
1781
60
                               AtomicLVal.getType(), AtomicLVal.getBaseInfo(),
1782
60
                               AtomicLVal.getTBAAInfo());
1783
60
    } else 
if (12
AtomicLVal.isVectorElt()12
) {
1784
6
      UpdateLVal = LValue::MakeVectorElt(Ptr, AtomicLVal.getVectorIdx(),
1785
6
                                         AtomicLVal.getType(),
1786
6
                                         AtomicLVal.getBaseInfo(),
1787
6
                                         AtomicLVal.getTBAAInfo());
1788
6
      DesiredLVal = LValue::MakeVectorElt(
1789
6
          DesiredAddr, AtomicLVal.getVectorIdx(), AtomicLVal.getType(),
1790
6
          AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo());
1791
6
    } else {
1792
6
      assert(AtomicLVal.isExtVectorElt());
1793
0
      UpdateLVal = LValue::MakeExtVectorElt(Ptr, AtomicLVal.getExtVectorElts(),
1794
6
                                            AtomicLVal.getType(),
1795
6
                                            AtomicLVal.getBaseInfo(),
1796
6
                                            AtomicLVal.getTBAAInfo());
1797
6
      DesiredLVal = LValue::MakeExtVectorElt(
1798
6
          DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(),
1799
6
          AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo());
1800
6
    }
1801
0
    UpRVal = CGF.EmitLoadOfLValue(UpdateLVal, SourceLocation());
1802
72
  }
1803
  // Store new value in the corresponding memory area.
1804
0
  RValue NewRVal = UpdateOp(UpRVal);
1805
320
  if (NewRVal.isScalar()) {
1806
288
    CGF.EmitStoreThroughLValue(NewRVal, DesiredLVal);
1807
288
  } else {
1808
32
    assert(NewRVal.isComplex());
1809
0
    CGF.EmitStoreOfComplex(NewRVal.getComplexVal(), DesiredLVal,
1810
32
                           /*isInit=*/false);
1811
32
  }
1812
320
}
1813
1814
void AtomicInfo::EmitAtomicUpdateLibcall(
1815
    llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1816
57
    bool IsVolatile) {
1817
57
  auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1818
1819
57
  Address ExpectedAddr = CreateTempAlloca();
1820
1821
57
  EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile);
1822
57
  auto *ContBB = CGF.createBasicBlock("atomic_cont");
1823
57
  auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1824
57
  CGF.EmitBlock(ContBB);
1825
57
  Address DesiredAddr = CreateTempAlloca();
1826
57
  if ((LVal.isBitField() && 
BFI.Size != ValueSizeInBits12
) ||
1827
57
      
requiresMemSetZero(getAtomicAddress().getElementType())45
) {
1828
12
    auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr);
1829
12
    CGF.Builder.CreateStore(OldVal, DesiredAddr);
1830
12
  }
1831
57
  auto OldRVal = convertAtomicTempToRValue(ExpectedAddr,
1832
57
                                           AggValueSlot::ignored(),
1833
57
                                           SourceLocation(), /*AsValue=*/false);
1834
57
  EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, DesiredAddr);
1835
57
  auto *Res =
1836
57
      EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1837
57
                                       DesiredAddr.getPointer(),
1838
57
                                       AO, Failure);
1839
57
  CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
1840
57
  CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1841
57
}
1842
1843
void AtomicInfo::EmitAtomicUpdateOp(
1844
    llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1845
263
    bool IsVolatile) {
1846
263
  auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1847
1848
  // Do the atomic load.
1849
263
  auto *OldVal = EmitAtomicLoadOp(Failure, IsVolatile);
1850
  // For non-simple lvalues perform compare-and-swap procedure.
1851
263
  auto *ContBB = CGF.createBasicBlock("atomic_cont");
1852
263
  auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1853
263
  auto *CurBB = CGF.Builder.GetInsertBlock();
1854
263
  CGF.EmitBlock(ContBB);
1855
263
  llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(),
1856
263
                                             /*NumReservedValues=*/2);
1857
263
  PHI->addIncoming(OldVal, CurBB);
1858
263
  Address NewAtomicAddr = CreateTempAlloca();
1859
263
  Address NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr);
1860
263
  if ((LVal.isBitField() && 
BFI.Size != ValueSizeInBits48
) ||
1861
263
      
requiresMemSetZero(getAtomicAddress().getElementType())215
) {
1862
72
    CGF.Builder.CreateStore(PHI, NewAtomicIntAddr);
1863
72
  }
1864
263
  auto OldRVal = ConvertIntToValueOrAtomic(PHI, AggValueSlot::ignored(),
1865
263
                                           SourceLocation(), /*AsValue=*/false);
1866
263
  EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, NewAtomicAddr);
1867
263
  auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr);
1868
  // Try to write new value using cmpxchg operation.
1869
263
  auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure);
1870
263
  PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock());
1871
263
  CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB);
1872
263
  CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1873
263
}
1874
1875
static void EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics,
1876
24
                                  RValue UpdateRVal, Address DesiredAddr) {
1877
24
  LValue AtomicLVal = Atomics.getAtomicLValue();
1878
24
  LValue DesiredLVal;
1879
  // Build new lvalue for temp address.
1880
24
  if (AtomicLVal.isBitField()) {
1881
20
    DesiredLVal =
1882
20
        LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(),
1883
20
                             AtomicLVal.getType(), AtomicLVal.getBaseInfo(),
1884
20
                             AtomicLVal.getTBAAInfo());
1885
20
  } else 
if (4
AtomicLVal.isVectorElt()4
) {
1886
2
    DesiredLVal =
1887
2
        LValue::MakeVectorElt(DesiredAddr, AtomicLVal.getVectorIdx(),
1888
2
                              AtomicLVal.getType(), AtomicLVal.getBaseInfo(),
1889
2
                              AtomicLVal.getTBAAInfo());
1890
2
  } else {
1891
2
    assert(AtomicLVal.isExtVectorElt());
1892
0
    DesiredLVal = LValue::MakeExtVectorElt(
1893
2
        DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(),
1894
2
        AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo());
1895
2
  }
1896
  // Store new value in the corresponding memory area.
1897
0
  assert(UpdateRVal.isScalar());
1898
0
  CGF.EmitStoreThroughLValue(UpdateRVal, DesiredLVal);
1899
24
}
1900
1901
void AtomicInfo::EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
1902
4
                                         RValue UpdateRVal, bool IsVolatile) {
1903
4
  auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1904
1905
4
  Address ExpectedAddr = CreateTempAlloca();
1906
1907
4
  EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile);
1908
4
  auto *ContBB = CGF.createBasicBlock("atomic_cont");
1909
4
  auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1910
4
  CGF.EmitBlock(ContBB);
1911
4
  Address DesiredAddr = CreateTempAlloca();
1912
4
  if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1913
4
      
requiresMemSetZero(getAtomicAddress().getElementType())0
) {
1914
4
    auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr);
1915
4
    CGF.Builder.CreateStore(OldVal, DesiredAddr);
1916
4
  }
1917
4
  EmitAtomicUpdateValue(CGF, *this, UpdateRVal, DesiredAddr);
1918
4
  auto *Res =
1919
4
      EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1920
4
                                       DesiredAddr.getPointer(),
1921
4
                                       AO, Failure);
1922
4
  CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
1923
4
  CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1924
4
}
1925
1926
void AtomicInfo::EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRVal,
1927
20
                                    bool IsVolatile) {
1928
20
  auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1929
1930
  // Do the atomic load.
1931
20
  auto *OldVal = EmitAtomicLoadOp(Failure, IsVolatile);
1932
  // For non-simple lvalues perform compare-and-swap procedure.
1933
20
  auto *ContBB = CGF.createBasicBlock("atomic_cont");
1934
20
  auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1935
20
  auto *CurBB = CGF.Builder.GetInsertBlock();
1936
20
  CGF.EmitBlock(ContBB);
1937
20
  llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(),
1938
20
                                             /*NumReservedValues=*/2);
1939
20
  PHI->addIncoming(OldVal, CurBB);
1940
20
  Address NewAtomicAddr = CreateTempAlloca();
1941
20
  Address NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr);
1942
20
  if ((LVal.isBitField() && 
BFI.Size != ValueSizeInBits16
) ||
1943
20
      
requiresMemSetZero(getAtomicAddress().getElementType())4
) {
1944
20
    CGF.Builder.CreateStore(PHI, NewAtomicIntAddr);
1945
20
  }
1946
20
  EmitAtomicUpdateValue(CGF, *this, UpdateRVal, NewAtomicAddr);
1947
20
  auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr);
1948
  // Try to write new value using cmpxchg operation.
1949
20
  auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure);
1950
20
  PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock());
1951
20
  CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB);
1952
20
  CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1953
20
}
1954
1955
void AtomicInfo::EmitAtomicUpdate(
1956
    llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1957
320
    bool IsVolatile) {
1958
320
  if (shouldUseLibcall()) {
1959
57
    EmitAtomicUpdateLibcall(AO, UpdateOp, IsVolatile);
1960
263
  } else {
1961
263
    EmitAtomicUpdateOp(AO, UpdateOp, IsVolatile);
1962
263
  }
1963
320
}
1964
1965
void AtomicInfo::EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
1966
24
                                  bool IsVolatile) {
1967
24
  if (shouldUseLibcall()) {
1968
4
    EmitAtomicUpdateLibcall(AO, UpdateRVal, IsVolatile);
1969
20
  } else {
1970
20
    EmitAtomicUpdateOp(AO, UpdateRVal, IsVolatile);
1971
20
  }
1972
24
}
1973
1974
void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue lvalue,
1975
83
                                      bool isInit) {
1976
83
  bool IsVolatile = lvalue.isVolatileQualified();
1977
83
  llvm::AtomicOrdering AO;
1978
83
  if (lvalue.getType()->isAtomicType()) {
1979
76
    AO = llvm::AtomicOrdering::SequentiallyConsistent;
1980
76
  } else {
1981
7
    AO = llvm::AtomicOrdering::Release;
1982
7
    IsVolatile = true;
1983
7
  }
1984
83
  return EmitAtomicStore(rvalue, lvalue, AO, IsVolatile, isInit);
1985
83
}
1986
1987
/// Emit a store to an l-value of atomic type.
1988
///
1989
/// Note that the r-value is expected to be an r-value *of the atomic
1990
/// type*; this means that for aggregate r-values, it should include
1991
/// storage for any padding that was necessary.
1992
void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest,
1993
                                      llvm::AtomicOrdering AO, bool IsVolatile,
1994
195
                                      bool isInit) {
1995
  // If this is an aggregate r-value, it should agree in type except
1996
  // maybe for address-space qualification.
1997
195
  assert(!rvalue.isAggregate() ||
1998
195
         rvalue.getAggregateAddress().getElementType() ==
1999
195
             dest.getAddress(*this).getElementType());
2000
2001
0
  AtomicInfo atomics(*this, dest);
2002
195
  LValue LVal = atomics.getAtomicLValue();
2003
2004
  // If this is an initialization, just put the value there normally.
2005
195
  if (LVal.isSimple()) {
2006
171
    if (isInit) {
2007
33
      atomics.emitCopyIntoMemory(rvalue);
2008
33
      return;
2009
33
    }
2010
2011
    // Check whether we should use a library call.
2012
138
    if (atomics.shouldUseLibcall()) {
2013
      // Produce a source address.
2014
18
      Address srcAddr = atomics.materializeRValue(rvalue);
2015
2016
      // void __atomic_store(size_t size, void *mem, void *val, int order)
2017
18
      CallArgList args;
2018
18
      args.add(RValue::get(atomics.getAtomicSizeValue()),
2019
18
               getContext().getSizeType());
2020
18
      args.add(RValue::get(EmitCastToVoidPtr(atomics.getAtomicPointer())),
2021
18
               getContext().VoidPtrTy);
2022
18
      args.add(RValue::get(EmitCastToVoidPtr(srcAddr.getPointer())),
2023
18
               getContext().VoidPtrTy);
2024
18
      args.add(
2025
18
          RValue::get(llvm::ConstantInt::get(IntTy, (int)llvm::toCABI(AO))),
2026
18
          getContext().IntTy);
2027
18
      emitAtomicLibcall(*this, "__atomic_store", getContext().VoidTy, args);
2028
18
      return;
2029
18
    }
2030
2031
    // Okay, we're doing this natively.
2032
120
    llvm::Value *intValue = atomics.convertRValueToInt(rvalue);
2033
2034
    // Do the atomic store.
2035
120
    Address addr =
2036
120
        atomics.emitCastToAtomicIntPointer(atomics.getAtomicAddress());
2037
120
    intValue = Builder.CreateIntCast(
2038
120
        intValue, addr.getElementType(), /*isSigned=*/false);
2039
120
    llvm::StoreInst *store = Builder.CreateStore(intValue, addr);
2040
2041
120
    if (AO == llvm::AtomicOrdering::Acquire)
2042
0
      AO = llvm::AtomicOrdering::Monotonic;
2043
120
    else if (AO == llvm::AtomicOrdering::AcquireRelease)
2044
0
      AO = llvm::AtomicOrdering::Release;
2045
    // Initializations don't need to be atomic.
2046
120
    if (!isInit)
2047
120
      store->setAtomic(AO);
2048
2049
    // Other decoration.
2050
120
    if (IsVolatile)
2051
10
      store->setVolatile(true);
2052
120
    CGM.DecorateInstructionWithTBAA(store, dest.getTBAAInfo());
2053
120
    return;
2054
138
  }
2055
2056
  // Emit simple atomic update operation.
2057
24
  atomics.EmitAtomicUpdate(AO, rvalue, IsVolatile);
2058
24
}
2059
2060
/// Emit a compare-and-exchange op for atomic type.
2061
///
2062
std::pair<RValue, llvm::Value *> CodeGenFunction::EmitAtomicCompareExchange(
2063
    LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc,
2064
    llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak,
2065
33
    AggValueSlot Slot) {
2066
  // If this is an aggregate r-value, it should agree in type except
2067
  // maybe for address-space qualification.
2068
33
  assert(!Expected.isAggregate() ||
2069
33
         Expected.getAggregateAddress().getElementType() ==
2070
33
             Obj.getAddress(*this).getElementType());
2071
0
  assert(!Desired.isAggregate() ||
2072
33
         Desired.getAggregateAddress().getElementType() ==
2073
33
             Obj.getAddress(*this).getElementType());
2074
0
  AtomicInfo Atomics(*this, Obj);
2075
2076
33
  return Atomics.EmitAtomicCompareExchange(Expected, Desired, Success, Failure,
2077
33
                                           IsWeak);
2078
33
}
2079
2080
void CodeGenFunction::EmitAtomicUpdate(
2081
    LValue LVal, llvm::AtomicOrdering AO,
2082
320
    const llvm::function_ref<RValue(RValue)> &UpdateOp, bool IsVolatile) {
2083
320
  AtomicInfo Atomics(*this, LVal);
2084
320
  Atomics.EmitAtomicUpdate(AO, UpdateOp, IsVolatile);
2085
320
}
2086
2087
20
void CodeGenFunction::EmitAtomicInit(Expr *init, LValue dest) {
2088
20
  AtomicInfo atomics(*this, dest);
2089
2090
20
  switch (atomics.getEvaluationKind()) {
2091
9
  case TEK_Scalar: {
2092
9
    llvm::Value *value = EmitScalarExpr(init);
2093
9
    atomics.emitCopyIntoMemory(RValue::get(value));
2094
9
    return;
2095
0
  }
2096
2097
2
  case TEK_Complex: {
2098
2
    ComplexPairTy value = EmitComplexExpr(init);
2099
2
    atomics.emitCopyIntoMemory(RValue::getComplex(value));
2100
2
    return;
2101
0
  }
2102
2103
9
  case TEK_Aggregate: {
2104
    // Fix up the destination if the initializer isn't an expression
2105
    // of atomic type.
2106
9
    bool Zeroed = false;
2107
9
    if (!init->getType()->isAtomicType()) {
2108
5
      Zeroed = atomics.emitMemSetZeroIfNecessary();
2109
5
      dest = atomics.projectValue();
2110
5
    }
2111
2112
    // Evaluate the expression directly into the destination.
2113
9
    AggValueSlot slot = AggValueSlot::forLValue(
2114
9
        dest, *this, AggValueSlot::IsNotDestructed,
2115
9
        AggValueSlot::DoesNotNeedGCBarriers, AggValueSlot::IsNotAliased,
2116
9
        AggValueSlot::DoesNotOverlap,
2117
9
        Zeroed ? 
AggValueSlot::IsZeroed2
:
AggValueSlot::IsNotZeroed7
);
2118
2119
9
    EmitAggExpr(init, slot);
2120
9
    return;
2121
0
  }
2122
20
  }
2123
0
  llvm_unreachable("bad evaluation kind");
2124
0
}