Coverage Report

Created: 2022-07-16 07:03

/Users/buildslave/jenkins/workspace/coverage/llvm-project/clang/lib/CodeGen/CGAtomic.cpp
Line
Count
Source (jump to first uncovered line)
1
//===--- CGAtomic.cpp - Emit LLVM IR for atomic operations ----------------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// This file contains the code for emitting atomic operations.
10
//
11
//===----------------------------------------------------------------------===//
12
13
#include "CGCall.h"
14
#include "CGRecordLayout.h"
15
#include "CodeGenFunction.h"
16
#include "CodeGenModule.h"
17
#include "TargetInfo.h"
18
#include "clang/AST/ASTContext.h"
19
#include "clang/CodeGen/CGFunctionInfo.h"
20
#include "clang/Frontend/FrontendDiagnostic.h"
21
#include "llvm/ADT/DenseMap.h"
22
#include "llvm/IR/DataLayout.h"
23
#include "llvm/IR/Intrinsics.h"
24
#include "llvm/IR/Operator.h"
25
26
using namespace clang;
27
using namespace CodeGen;
28
29
namespace {
30
  class AtomicInfo {
31
    CodeGenFunction &CGF;
32
    QualType AtomicTy;
33
    QualType ValueTy;
34
    uint64_t AtomicSizeInBits;
35
    uint64_t ValueSizeInBits;
36
    CharUnits AtomicAlign;
37
    CharUnits ValueAlign;
38
    TypeEvaluationKind EvaluationKind;
39
    bool UseLibcall;
40
    LValue LVal;
41
    CGBitFieldInfo BFI;
42
  public:
43
    AtomicInfo(CodeGenFunction &CGF, LValue &lvalue)
44
        : CGF(CGF), AtomicSizeInBits(0), ValueSizeInBits(0),
45
2.08k
          EvaluationKind(TEK_Scalar), UseLibcall(true) {
46
2.08k
      assert(!lvalue.isGlobalReg());
47
0
      ASTContext &C = CGF.getContext();
48
2.08k
      if (lvalue.isSimple()) {
49
1.96k
        AtomicTy = lvalue.getType();
50
1.96k
        if (auto *ATy = AtomicTy->getAs<AtomicType>())
51
890
          ValueTy = ATy->getValueType();
52
1.07k
        else
53
1.07k
          ValueTy = AtomicTy;
54
1.96k
        EvaluationKind = CGF.getEvaluationKind(ValueTy);
55
56
1.96k
        uint64_t ValueAlignInBits;
57
1.96k
        uint64_t AtomicAlignInBits;
58
1.96k
        TypeInfo ValueTI = C.getTypeInfo(ValueTy);
59
1.96k
        ValueSizeInBits = ValueTI.Width;
60
1.96k
        ValueAlignInBits = ValueTI.Align;
61
62
1.96k
        TypeInfo AtomicTI = C.getTypeInfo(AtomicTy);
63
1.96k
        AtomicSizeInBits = AtomicTI.Width;
64
1.96k
        AtomicAlignInBits = AtomicTI.Align;
65
66
1.96k
        assert(ValueSizeInBits <= AtomicSizeInBits);
67
0
        assert(ValueAlignInBits <= AtomicAlignInBits);
68
69
0
        AtomicAlign = C.toCharUnitsFromBits(AtomicAlignInBits);
70
1.96k
        ValueAlign = C.toCharUnitsFromBits(ValueAlignInBits);
71
1.96k
        if (lvalue.getAlignment().isZero())
72
0
          lvalue.setAlignment(AtomicAlign);
73
74
1.96k
        LVal = lvalue;
75
1.96k
      } else 
if (120
lvalue.isBitField()120
) {
76
100
        ValueTy = lvalue.getType();
77
100
        ValueSizeInBits = C.getTypeSize(ValueTy);
78
100
        auto &OrigBFI = lvalue.getBitFieldInfo();
79
100
        auto Offset = OrigBFI.Offset % C.toBits(lvalue.getAlignment());
80
100
        AtomicSizeInBits = C.toBits(
81
100
            C.toCharUnitsFromBits(Offset + OrigBFI.Size + C.getCharWidth() - 1)
82
100
                .alignTo(lvalue.getAlignment()));
83
100
        auto VoidPtrAddr = CGF.EmitCastToVoidPtr(lvalue.getBitFieldPointer());
84
100
        auto OffsetInChars =
85
100
            (C.toCharUnitsFromBits(OrigBFI.Offset) / lvalue.getAlignment()) *
86
100
            lvalue.getAlignment();
87
100
        VoidPtrAddr = CGF.Builder.CreateConstGEP1_64(
88
100
            CGF.Int8Ty, VoidPtrAddr, OffsetInChars.getQuantity());
89
100
        llvm::Type *IntTy = CGF.Builder.getIntNTy(AtomicSizeInBits);
90
100
        auto Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
91
100
            VoidPtrAddr, IntTy->getPointerTo(), "atomic_bitfield_base");
92
100
        BFI = OrigBFI;
93
100
        BFI.Offset = Offset;
94
100
        BFI.StorageSize = AtomicSizeInBits;
95
100
        BFI.StorageOffset += OffsetInChars;
96
100
        LVal = LValue::MakeBitfield(Address(Addr, IntTy, lvalue.getAlignment()),
97
100
                                    BFI, lvalue.getType(), lvalue.getBaseInfo(),
98
100
                                    lvalue.getTBAAInfo());
99
100
        AtomicTy = C.getIntTypeForBitwidth(AtomicSizeInBits, OrigBFI.IsSigned);
100
100
        if (AtomicTy.isNull()) {
101
10
          llvm::APInt Size(
102
10
              /*numBits=*/32,
103
10
              C.toCharUnitsFromBits(AtomicSizeInBits).getQuantity());
104
10
          AtomicTy =
105
10
              C.getConstantArrayType(C.CharTy, Size, nullptr, ArrayType::Normal,
106
10
                                     /*IndexTypeQuals=*/0);
107
10
        }
108
100
        AtomicAlign = ValueAlign = lvalue.getAlignment();
109
100
      } else 
if (20
lvalue.isVectorElt()20
) {
110
10
        ValueTy = lvalue.getType()->castAs<VectorType>()->getElementType();
111
10
        ValueSizeInBits = C.getTypeSize(ValueTy);
112
10
        AtomicTy = lvalue.getType();
113
10
        AtomicSizeInBits = C.getTypeSize(AtomicTy);
114
10
        AtomicAlign = ValueAlign = lvalue.getAlignment();
115
10
        LVal = lvalue;
116
10
      } else {
117
10
        assert(lvalue.isExtVectorElt());
118
0
        ValueTy = lvalue.getType();
119
10
        ValueSizeInBits = C.getTypeSize(ValueTy);
120
10
        AtomicTy = ValueTy = CGF.getContext().getExtVectorType(
121
10
            lvalue.getType(), cast<llvm::FixedVectorType>(
122
10
                                  lvalue.getExtVectorAddress().getElementType())
123
10
                                  ->getNumElements());
124
10
        AtomicSizeInBits = C.getTypeSize(AtomicTy);
125
10
        AtomicAlign = ValueAlign = lvalue.getAlignment();
126
10
        LVal = lvalue;
127
10
      }
128
0
      UseLibcall = !C.getTargetInfo().hasBuiltinAtomic(
129
2.08k
          AtomicSizeInBits, C.toBits(lvalue.getAlignment()));
130
2.08k
    }
131
132
141
    QualType getAtomicType() const { return AtomicTy; }
133
321
    QualType getValueType() const { return ValueTy; }
134
3.16k
    CharUnits getAtomicAlignment() const { return AtomicAlign; }
135
232
    uint64_t getAtomicSizeInBits() const { return AtomicSizeInBits; }
136
262
    uint64_t getValueSizeInBits() const { return ValueSizeInBits; }
137
1.00k
    TypeEvaluationKind getEvaluationKind() const { return EvaluationKind; }
138
712
    bool shouldUseLibcall() const { return UseLibcall; }
139
512
    const LValue &getAtomicLValue() const { return LVal; }
140
2.01k
    llvm::Value *getAtomicPointer() const {
141
2.01k
      if (LVal.isSimple())
142
1.36k
        return LVal.getPointer(CGF);
143
644
      else if (LVal.isBitField())
144
580
        return LVal.getBitFieldPointer();
145
64
      else if (LVal.isVectorElt())
146
32
        return LVal.getVectorPointer();
147
32
      assert(LVal.isExtVectorElt());
148
0
      return LVal.getExtVectorPointer();
149
2.01k
    }
150
1.81k
    Address getAtomicAddress() const {
151
1.81k
      llvm::Type *ElTy;
152
1.81k
      if (LVal.isSimple())
153
1.20k
        ElTy = LVal.getAddress(CGF).getElementType();
154
608
      else if (LVal.isBitField())
155
544
        ElTy = LVal.getBitFieldAddress().getElementType();
156
64
      else if (LVal.isVectorElt())
157
32
        ElTy = LVal.getVectorAddress().getElementType();
158
32
      else
159
32
        ElTy = LVal.getExtVectorAddress().getElementType();
160
1.81k
      return Address(getAtomicPointer(), ElTy, getAtomicAlignment());
161
1.81k
    }
162
163
663
    Address getAtomicAddressAsAtomicIntPointer() const {
164
663
      return emitCastToAtomicIntPointer(getAtomicAddress());
165
663
    }
166
167
    /// Is the atomic size larger than the underlying value type?
168
    ///
169
    /// Note that the absence of padding does not mean that atomic
170
    /// objects are completely interchangeable with non-atomic
171
    /// objects: we might have promoted the alignment of a type
172
    /// without making it bigger.
173
1.19k
    bool hasPadding() const {
174
1.19k
      return (ValueSizeInBits != AtomicSizeInBits);
175
1.19k
    }
176
177
    bool emitMemSetZeroIfNecessary() const;
178
179
201
    llvm::Value *getAtomicSizeValue() const {
180
201
      CharUnits size = CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits);
181
201
      return CGF.CGM.getSize(size);
182
201
    }
183
184
    /// Cast the given pointer to an integer pointer suitable for atomic
185
    /// operations if the source.
186
    Address emitCastToAtomicIntPointer(Address Addr) const;
187
188
    /// If Addr is compatible with the iN that will be used for an atomic
189
    /// operation, bitcast it. Otherwise, create a temporary that is suitable
190
    /// and copy the value across.
191
    Address convertToAtomicIntPointer(Address Addr) const;
192
193
    /// Turn an atomic-layout object into an r-value.
194
    RValue convertAtomicTempToRValue(Address addr, AggValueSlot resultSlot,
195
                                     SourceLocation loc, bool AsValue) const;
196
197
    /// Converts a rvalue to integer value.
198
    llvm::Value *convertRValueToInt(RValue RVal) const;
199
200
    RValue ConvertIntToValueOrAtomic(llvm::Value *IntVal,
201
                                     AggValueSlot ResultSlot,
202
                                     SourceLocation Loc, bool AsValue) const;
203
204
    /// Copy an atomic r-value into atomic-layout memory.
205
    void emitCopyIntoMemory(RValue rvalue) const;
206
207
    /// Project an l-value down to the value field.
208
190
    LValue projectValue() const {
209
190
      assert(LVal.isSimple());
210
0
      Address addr = getAtomicAddress();
211
190
      if (hasPadding())
212
2
        addr = CGF.Builder.CreateStructGEP(addr, 0);
213
214
190
      return LValue::MakeAddr(addr, getValueType(), CGF.getContext(),
215
190
                              LVal.getBaseInfo(), LVal.getTBAAInfo());
216
190
    }
217
218
    /// Emits atomic load.
219
    /// \returns Loaded value.
220
    RValue EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
221
                          bool AsValue, llvm::AtomicOrdering AO,
222
                          bool IsVolatile);
223
224
    /// Emits atomic compare-and-exchange sequence.
225
    /// \param Expected Expected value.
226
    /// \param Desired Desired value.
227
    /// \param Success Atomic ordering for success operation.
228
    /// \param Failure Atomic ordering for failed operation.
229
    /// \param IsWeak true if atomic operation is weak, false otherwise.
230
    /// \returns Pair of values: previous value from storage (value type) and
231
    /// boolean flag (i1 type) with true if success and false otherwise.
232
    std::pair<RValue, llvm::Value *>
233
    EmitAtomicCompareExchange(RValue Expected, RValue Desired,
234
                              llvm::AtomicOrdering Success =
235
                                  llvm::AtomicOrdering::SequentiallyConsistent,
236
                              llvm::AtomicOrdering Failure =
237
                                  llvm::AtomicOrdering::SequentiallyConsistent,
238
                              bool IsWeak = false);
239
240
    /// Emits atomic update.
241
    /// \param AO Atomic ordering.
242
    /// \param UpdateOp Update operation for the current lvalue.
243
    void EmitAtomicUpdate(llvm::AtomicOrdering AO,
244
                          const llvm::function_ref<RValue(RValue)> &UpdateOp,
245
                          bool IsVolatile);
246
    /// Emits atomic update.
247
    /// \param AO Atomic ordering.
248
    void EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
249
                          bool IsVolatile);
250
251
    /// Materialize an atomic r-value in atomic-layout memory.
252
    Address materializeRValue(RValue rvalue) const;
253
254
    /// Creates temp alloca for intermediate operations on atomic value.
255
    Address CreateTempAlloca() const;
256
  private:
257
    bool requiresMemSetZero(llvm::Type *type) const;
258
259
260
    /// Emits atomic load as a libcall.
261
    void EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
262
                               llvm::AtomicOrdering AO, bool IsVolatile);
263
    /// Emits atomic load as LLVM instruction.
264
    llvm::Value *EmitAtomicLoadOp(llvm::AtomicOrdering AO, bool IsVolatile);
265
    /// Emits atomic compare-and-exchange op as a libcall.
266
    llvm::Value *EmitAtomicCompareExchangeLibcall(
267
        llvm::Value *ExpectedAddr, llvm::Value *DesiredAddr,
268
        llvm::AtomicOrdering Success =
269
            llvm::AtomicOrdering::SequentiallyConsistent,
270
        llvm::AtomicOrdering Failure =
271
            llvm::AtomicOrdering::SequentiallyConsistent);
272
    /// Emits atomic compare-and-exchange op as LLVM instruction.
273
    std::pair<llvm::Value *, llvm::Value *> EmitAtomicCompareExchangeOp(
274
        llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
275
        llvm::AtomicOrdering Success =
276
            llvm::AtomicOrdering::SequentiallyConsistent,
277
        llvm::AtomicOrdering Failure =
278
            llvm::AtomicOrdering::SequentiallyConsistent,
279
        bool IsWeak = false);
280
    /// Emit atomic update as libcalls.
281
    void
282
    EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
283
                            const llvm::function_ref<RValue(RValue)> &UpdateOp,
284
                            bool IsVolatile);
285
    /// Emit atomic update as LLVM instructions.
286
    void EmitAtomicUpdateOp(llvm::AtomicOrdering AO,
287
                            const llvm::function_ref<RValue(RValue)> &UpdateOp,
288
                            bool IsVolatile);
289
    /// Emit atomic update as libcalls.
290
    void EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, RValue UpdateRVal,
291
                                 bool IsVolatile);
292
    /// Emit atomic update as LLVM instructions.
293
    void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRal,
294
                            bool IsVolatile);
295
  };
296
}
297
298
1.35k
Address AtomicInfo::CreateTempAlloca() const {
299
1.35k
  Address TempAlloca = CGF.CreateMemTemp(
300
1.35k
      (LVal.isBitField() && 
ValueSizeInBits > AtomicSizeInBits176
) ?
ValueTy72
301
1.35k
                                                                : 
AtomicTy1.28k
,
302
1.35k
      getAtomicAlignment(),
303
1.35k
      "atomic-temp");
304
  // Cast to pointer to value type for bitfields.
305
1.35k
  if (LVal.isBitField())
306
176
    return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
307
176
        TempAlloca, getAtomicAddress().getType(),
308
176
        getAtomicAddress().getElementType());
309
1.18k
  return TempAlloca;
310
1.35k
}
311
312
static RValue emitAtomicLibcall(CodeGenFunction &CGF,
313
                                StringRef fnName,
314
                                QualType resultType,
315
396
                                CallArgList &args) {
316
396
  const CGFunctionInfo &fnInfo =
317
396
    CGF.CGM.getTypes().arrangeBuiltinFunctionCall(resultType, args);
318
396
  llvm::FunctionType *fnTy = CGF.CGM.getTypes().GetFunctionType(fnInfo);
319
396
  llvm::AttrBuilder fnAttrB(CGF.getLLVMContext());
320
396
  fnAttrB.addAttribute(llvm::Attribute::NoUnwind);
321
396
  fnAttrB.addAttribute(llvm::Attribute::WillReturn);
322
396
  llvm::AttributeList fnAttrs = llvm::AttributeList::get(
323
396
      CGF.getLLVMContext(), llvm::AttributeList::FunctionIndex, fnAttrB);
324
325
396
  llvm::FunctionCallee fn =
326
396
      CGF.CGM.CreateRuntimeFunction(fnTy, fnName, fnAttrs);
327
396
  auto callee = CGCallee::forDirect(fn);
328
396
  return CGF.EmitCall(fnInfo, callee, ReturnValueSlot(), args);
329
396
}
330
331
/// Does a store of the given IR type modify the full expected width?
332
static bool isFullSizeType(CodeGenModule &CGM, llvm::Type *type,
333
400
                           uint64_t expectedSize) {
334
400
  return (CGM.getDataLayout().getTypeStoreSize(type) * 8 == expectedSize);
335
400
}
336
337
/// Does the atomic type require memsetting to zero before initialization?
338
///
339
/// The IR type is provided as a way of making certain queries faster.
340
427
bool AtomicInfo::requiresMemSetZero(llvm::Type *type) const {
341
  // If the atomic type has size padding, we definitely need a memset.
342
427
  if (hasPadding()) 
return true18
;
343
344
  // Otherwise, do some simple heuristics to try to avoid it:
345
409
  switch (getEvaluationKind()) {
346
  // For scalars and complexes, check whether the store size of the
347
  // type uses the full size.
348
350
  case TEK_Scalar:
349
350
    return !isFullSizeType(CGF.CGM, type, AtomicSizeInBits);
350
50
  case TEK_Complex:
351
50
    return !isFullSizeType(CGF.CGM, type->getStructElementType(0),
352
50
                           AtomicSizeInBits / 2);
353
354
  // Padding in structs has an undefined bit pattern.  User beware.
355
9
  case TEK_Aggregate:
356
9
    return false;
357
409
  }
358
0
  llvm_unreachable("bad evaluation kind");
359
0
}
360
361
190
bool AtomicInfo::emitMemSetZeroIfNecessary() const {
362
190
  assert(LVal.isSimple());
363
0
  Address addr = LVal.getAddress(CGF);
364
190
  if (!requiresMemSetZero(addr.getElementType()))
365
156
    return false;
366
367
34
  CGF.Builder.CreateMemSet(
368
34
      addr.getPointer(), llvm::ConstantInt::get(CGF.Int8Ty, 0),
369
34
      CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits).getQuantity(),
370
34
      LVal.getAlignment().getAsAlign());
371
34
  return true;
372
190
}
373
374
static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak,
375
                              Address Dest, Address Ptr,
376
                              Address Val1, Address Val2,
377
                              uint64_t Size,
378
                              llvm::AtomicOrdering SuccessOrder,
379
                              llvm::AtomicOrdering FailureOrder,
380
190
                              llvm::SyncScope::ID Scope) {
381
  // Note that cmpxchg doesn't support weak cmpxchg, at least at the moment.
382
190
  llvm::Value *Expected = CGF.Builder.CreateLoad(Val1);
383
190
  llvm::Value *Desired = CGF.Builder.CreateLoad(Val2);
384
385
190
  llvm::AtomicCmpXchgInst *Pair = CGF.Builder.CreateAtomicCmpXchg(
386
190
      Ptr.getPointer(), Expected, Desired, SuccessOrder, FailureOrder,
387
190
      Scope);
388
190
  Pair->setVolatile(E->isVolatile());
389
190
  Pair->setWeak(IsWeak);
390
391
  // Cmp holds the result of the compare-exchange operation: true on success,
392
  // false on failure.
393
190
  llvm::Value *Old = CGF.Builder.CreateExtractValue(Pair, 0);
394
190
  llvm::Value *Cmp = CGF.Builder.CreateExtractValue(Pair, 1);
395
396
  // This basic block is used to hold the store instruction if the operation
397
  // failed.
398
190
  llvm::BasicBlock *StoreExpectedBB =
399
190
      CGF.createBasicBlock("cmpxchg.store_expected", CGF.CurFn);
400
401
  // This basic block is the exit point of the operation, we should end up
402
  // here regardless of whether or not the operation succeeded.
403
190
  llvm::BasicBlock *ContinueBB =
404
190
      CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
405
406
  // Update Expected if Expected isn't equal to Old, otherwise branch to the
407
  // exit point.
408
190
  CGF.Builder.CreateCondBr(Cmp, ContinueBB, StoreExpectedBB);
409
410
190
  CGF.Builder.SetInsertPoint(StoreExpectedBB);
411
  // Update the memory at Expected with Old's value.
412
190
  CGF.Builder.CreateStore(Old, Val1);
413
  // Finally, branch to the exit point.
414
190
  CGF.Builder.CreateBr(ContinueBB);
415
416
190
  CGF.Builder.SetInsertPoint(ContinueBB);
417
  // Update the memory at Dest with Cmp's value.
418
190
  CGF.EmitStoreOfScalar(Cmp, CGF.MakeAddrLValue(Dest, E->getType()));
419
190
}
420
421
/// Given an ordering required on success, emit all possible cmpxchg
422
/// instructions to cope with the provided (but possibly only dynamically known)
423
/// FailureOrder.
424
static void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E,
425
                                        bool IsWeak, Address Dest, Address Ptr,
426
                                        Address Val1, Address Val2,
427
                                        llvm::Value *FailureOrderVal,
428
                                        uint64_t Size,
429
                                        llvm::AtomicOrdering SuccessOrder,
430
106
                                        llvm::SyncScope::ID Scope) {
431
106
  llvm::AtomicOrdering FailureOrder;
432
106
  if (llvm::ConstantInt *FO = dyn_cast<llvm::ConstantInt>(FailureOrderVal)) {
433
64
    auto FOS = FO->getSExtValue();
434
64
    if (!llvm::isValidAtomicOrderingCABI(FOS))
435
2
      FailureOrder = llvm::AtomicOrdering::Monotonic;
436
62
    else
437
62
      switch ((llvm::AtomicOrderingCABI)FOS) {
438
29
      case llvm::AtomicOrderingCABI::relaxed:
439
      // 31.7.2.18: "The failure argument shall not be memory_order_release
440
      // nor memory_order_acq_rel". Fallback to monotonic.
441
29
      case llvm::AtomicOrderingCABI::release:
442
29
      case llvm::AtomicOrderingCABI::acq_rel:
443
29
        FailureOrder = llvm::AtomicOrdering::Monotonic;
444
29
        break;
445
0
      case llvm::AtomicOrderingCABI::consume:
446
24
      case llvm::AtomicOrderingCABI::acquire:
447
24
        FailureOrder = llvm::AtomicOrdering::Acquire;
448
24
        break;
449
9
      case llvm::AtomicOrderingCABI::seq_cst:
450
9
        FailureOrder = llvm::AtomicOrdering::SequentiallyConsistent;
451
9
        break;
452
62
      }
453
    // Prior to c++17, "the failure argument shall be no stronger than the
454
    // success argument". This condition has been lifted and the only
455
    // precondition is 31.7.2.18. Effectively treat this as a DR and skip
456
    // language version checks.
457
64
    emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
458
64
                      FailureOrder, Scope);
459
64
    return;
460
64
  }
461
462
  // Create all the relevant BB's
463
42
  auto *MonotonicBB = CGF.createBasicBlock("monotonic_fail", CGF.CurFn);
464
42
  auto *AcquireBB = CGF.createBasicBlock("acquire_fail", CGF.CurFn);
465
42
  auto *SeqCstBB = CGF.createBasicBlock("seqcst_fail", CGF.CurFn);
466
42
  auto *ContBB = CGF.createBasicBlock("atomic.continue", CGF.CurFn);
467
468
  // MonotonicBB is arbitrarily chosen as the default case; in practice, this
469
  // doesn't matter unless someone is crazy enough to use something that
470
  // doesn't fold to a constant for the ordering.
471
42
  llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(FailureOrderVal, MonotonicBB);
472
  // Implemented as acquire, since it's the closest in LLVM.
473
42
  SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::consume),
474
42
              AcquireBB);
475
42
  SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire),
476
42
              AcquireBB);
477
42
  SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst),
478
42
              SeqCstBB);
479
480
  // Emit all the different atomics
481
42
  CGF.Builder.SetInsertPoint(MonotonicBB);
482
42
  emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
483
42
                    Size, SuccessOrder, llvm::AtomicOrdering::Monotonic, Scope);
484
42
  CGF.Builder.CreateBr(ContBB);
485
486
42
  CGF.Builder.SetInsertPoint(AcquireBB);
487
42
  emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
488
42
                    llvm::AtomicOrdering::Acquire, Scope);
489
42
  CGF.Builder.CreateBr(ContBB);
490
491
42
  CGF.Builder.SetInsertPoint(SeqCstBB);
492
42
  emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
493
42
                    llvm::AtomicOrdering::SequentiallyConsistent, Scope);
494
42
  CGF.Builder.CreateBr(ContBB);
495
496
42
  CGF.Builder.SetInsertPoint(ContBB);
497
42
}
498
499
/// Duplicate the atomic min/max operation in conventional IR for the builtin
500
/// variants that return the new rather than the original value.
501
static llvm::Value *EmitPostAtomicMinMax(CGBuilderTy &Builder,
502
                                         AtomicExpr::AtomicOp Op,
503
                                         bool IsSigned,
504
                                         llvm::Value *OldVal,
505
14
                                         llvm::Value *RHS) {
506
14
  llvm::CmpInst::Predicate Pred;
507
14
  switch (Op) {
508
0
  default:
509
0
    llvm_unreachable("Unexpected min/max operation");
510
4
  case AtomicExpr::AO__atomic_max_fetch:
511
4
    Pred = IsSigned ? 
llvm::CmpInst::ICMP_SGT2
:
llvm::CmpInst::ICMP_UGT2
;
512
4
    break;
513
10
  case AtomicExpr::AO__atomic_min_fetch:
514
10
    Pred = IsSigned ? 
llvm::CmpInst::ICMP_SLT4
:
llvm::CmpInst::ICMP_ULT6
;
515
10
    break;
516
14
  }
517
14
  llvm::Value *Cmp = Builder.CreateICmp(Pred, OldVal, RHS, "tst");
518
14
  return Builder.CreateSelect(Cmp, OldVal, RHS, "newval");
519
14
}
520
521
static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest,
522
                         Address Ptr, Address Val1, Address Val2,
523
                         llvm::Value *IsWeak, llvm::Value *FailureOrder,
524
                         uint64_t Size, llvm::AtomicOrdering Order,
525
1.95k
                         llvm::SyncScope::ID Scope) {
526
1.95k
  llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add;
527
1.95k
  bool PostOpMinMax = false;
528
1.95k
  unsigned PostOp = 0;
529
530
1.95k
  switch (E->getOp()) {
531
0
  case AtomicExpr::AO__c11_atomic_init:
532
0
  case AtomicExpr::AO__opencl_atomic_init:
533
0
    llvm_unreachable("Already handled!");
534
535
22
  case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
536
32
  case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
537
46
  case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
538
46
    emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
539
46
                                FailureOrder, Size, Order, Scope);
540
46
    return;
541
3
  case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
542
5
  case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
543
15
  case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
544
15
    emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
545
15
                                FailureOrder, Size, Order, Scope);
546
15
    return;
547
21
  case AtomicExpr::AO__atomic_compare_exchange:
548
31
  case AtomicExpr::AO__atomic_compare_exchange_n: {
549
31
    if (llvm::ConstantInt *IsWeakC = dyn_cast<llvm::ConstantInt>(IsWeak)) {
550
17
      emitAtomicCmpXchgFailureSet(CGF, E, IsWeakC->getZExtValue(), Dest, Ptr,
551
17
                                  Val1, Val2, FailureOrder, Size, Order, Scope);
552
17
    } else {
553
      // Create all the relevant BB's
554
14
      llvm::BasicBlock *StrongBB =
555
14
          CGF.createBasicBlock("cmpxchg.strong", CGF.CurFn);
556
14
      llvm::BasicBlock *WeakBB = CGF.createBasicBlock("cmxchg.weak", CGF.CurFn);
557
14
      llvm::BasicBlock *ContBB =
558
14
          CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
559
560
14
      llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(IsWeak, WeakBB);
561
14
      SI->addCase(CGF.Builder.getInt1(false), StrongBB);
562
563
14
      CGF.Builder.SetInsertPoint(StrongBB);
564
14
      emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
565
14
                                  FailureOrder, Size, Order, Scope);
566
14
      CGF.Builder.CreateBr(ContBB);
567
568
14
      CGF.Builder.SetInsertPoint(WeakBB);
569
14
      emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
570
14
                                  FailureOrder, Size, Order, Scope);
571
14
      CGF.Builder.CreateBr(ContBB);
572
573
14
      CGF.Builder.SetInsertPoint(ContBB);
574
14
    }
575
31
    return;
576
21
  }
577
272
  case AtomicExpr::AO__c11_atomic_load:
578
316
  case AtomicExpr::AO__opencl_atomic_load:
579
324
  case AtomicExpr::AO__hip_atomic_load:
580
340
  case AtomicExpr::AO__atomic_load_n:
581
392
  case AtomicExpr::AO__atomic_load: {
582
392
    llvm::LoadInst *Load = CGF.Builder.CreateLoad(Ptr);
583
392
    Load->setAtomic(Order, Scope);
584
392
    Load->setVolatile(E->isVolatile());
585
392
    CGF.Builder.CreateStore(Load, Dest);
586
392
    return;
587
340
  }
588
589
294
  case AtomicExpr::AO__c11_atomic_store:
590
304
  case AtomicExpr::AO__opencl_atomic_store:
591
319
  case AtomicExpr::AO__hip_atomic_store:
592
368
  case AtomicExpr::AO__atomic_store:
593
372
  case AtomicExpr::AO__atomic_store_n: {
594
372
    llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1);
595
372
    llvm::StoreInst *Store = CGF.Builder.CreateStore(LoadVal1, Ptr);
596
372
    Store->setAtomic(Order, Scope);
597
372
    Store->setVolatile(E->isVolatile());
598
372
    return;
599
368
  }
600
601
14
  case AtomicExpr::AO__c11_atomic_exchange:
602
24
  case AtomicExpr::AO__hip_atomic_exchange:
603
26
  case AtomicExpr::AO__opencl_atomic_exchange:
604
30
  case AtomicExpr::AO__atomic_exchange_n:
605
36
  case AtomicExpr::AO__atomic_exchange:
606
36
    Op = llvm::AtomicRMWInst::Xchg;
607
36
    break;
608
609
56
  case AtomicExpr::AO__atomic_add_fetch:
610
56
    PostOp = E->getValueType()->isFloatingType() ? 
llvm::Instruction::FAdd0
611
56
                                                 : llvm::Instruction::Add;
612
56
    LLVM_FALLTHROUGH;
613
451
  case AtomicExpr::AO__c11_atomic_fetch_add:
614
461
  case AtomicExpr::AO__hip_atomic_fetch_add:
615
479
  case AtomicExpr::AO__opencl_atomic_fetch_add:
616
505
  case AtomicExpr::AO__atomic_fetch_add:
617
505
    Op = E->getValueType()->isFloatingType() ? 
llvm::AtomicRMWInst::FAdd32
618
505
                                             : 
llvm::AtomicRMWInst::Add473
;
619
505
    break;
620
621
0
  case AtomicExpr::AO__atomic_sub_fetch:
622
0
    PostOp = E->getValueType()->isFloatingType() ? llvm::Instruction::FSub
623
0
                                                 : llvm::Instruction::Sub;
624
0
    LLVM_FALLTHROUGH;
625
391
  case AtomicExpr::AO__c11_atomic_fetch_sub:
626
391
  case AtomicExpr::AO__opencl_atomic_fetch_sub:
627
419
  case AtomicExpr::AO__atomic_fetch_sub:
628
419
    Op = E->getValueType()->isFloatingType() ? 
llvm::AtomicRMWInst::FSub16
629
419
                                             : 
llvm::AtomicRMWInst::Sub403
;
630
419
    break;
631
632
8
  case AtomicExpr::AO__atomic_min_fetch:
633
8
    PostOpMinMax = true;
634
8
    LLVM_FALLTHROUGH;
635
18
  case AtomicExpr::AO__c11_atomic_fetch_min:
636
38
  case AtomicExpr::AO__hip_atomic_fetch_min:
637
42
  case AtomicExpr::AO__opencl_atomic_fetch_min:
638
44
  case AtomicExpr::AO__atomic_fetch_min:
639
44
    Op = E->getValueType()->isSignedIntegerType() ? 
llvm::AtomicRMWInst::Min23
640
44
                                                  : 
llvm::AtomicRMWInst::UMin21
;
641
44
    break;
642
643
4
  case AtomicExpr::AO__atomic_max_fetch:
644
4
    PostOpMinMax = true;
645
4
    LLVM_FALLTHROUGH;
646
14
  case AtomicExpr::AO__c11_atomic_fetch_max:
647
34
  case AtomicExpr::AO__hip_atomic_fetch_max:
648
38
  case AtomicExpr::AO__opencl_atomic_fetch_max:
649
40
  case AtomicExpr::AO__atomic_fetch_max:
650
40
    Op = E->getValueType()->isSignedIntegerType() ? 
llvm::AtomicRMWInst::Max21
651
40
                                                  : 
llvm::AtomicRMWInst::UMax19
;
652
40
    break;
653
654
0
  case AtomicExpr::AO__atomic_and_fetch:
655
0
    PostOp = llvm::Instruction::And;
656
0
    LLVM_FALLTHROUGH;
657
4
  case AtomicExpr::AO__c11_atomic_fetch_and:
658
14
  case AtomicExpr::AO__hip_atomic_fetch_and:
659
16
  case AtomicExpr::AO__opencl_atomic_fetch_and:
660
16
  case AtomicExpr::AO__atomic_fetch_and:
661
16
    Op = llvm::AtomicRMWInst::And;
662
16
    break;
663
664
2
  case AtomicExpr::AO__atomic_or_fetch:
665
2
    PostOp = llvm::Instruction::Or;
666
2
    LLVM_FALLTHROUGH;
667
4
  case AtomicExpr::AO__c11_atomic_fetch_or:
668
14
  case AtomicExpr::AO__hip_atomic_fetch_or:
669
14
  case AtomicExpr::AO__opencl_atomic_fetch_or:
670
17
  case AtomicExpr::AO__atomic_fetch_or:
671
17
    Op = llvm::AtomicRMWInst::Or;
672
17
    break;
673
674
0
  case AtomicExpr::AO__atomic_xor_fetch:
675
0
    PostOp = llvm::Instruction::Xor;
676
0
    LLVM_FALLTHROUGH;
677
0
  case AtomicExpr::AO__c11_atomic_fetch_xor:
678
10
  case AtomicExpr::AO__hip_atomic_fetch_xor:
679
10
  case AtomicExpr::AO__opencl_atomic_fetch_xor:
680
14
  case AtomicExpr::AO__atomic_fetch_xor:
681
14
    Op = llvm::AtomicRMWInst::Xor;
682
14
    break;
683
684
4
  case AtomicExpr::AO__atomic_nand_fetch:
685
4
    PostOp = llvm::Instruction::And; // the NOT is special cased below
686
4
    LLVM_FALLTHROUGH;
687
4
  case AtomicExpr::AO__c11_atomic_fetch_nand:
688
8
  case AtomicExpr::AO__atomic_fetch_nand:
689
8
    Op = llvm::AtomicRMWInst::Nand;
690
8
    break;
691
1.95k
  }
692
693
1.09k
  llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1);
694
1.09k
  llvm::AtomicRMWInst *RMWI =
695
1.09k
      CGF.Builder.CreateAtomicRMW(Op, Ptr.getPointer(), LoadVal1, Order, Scope);
696
1.09k
  RMWI->setVolatile(E->isVolatile());
697
698
  // For __atomic_*_fetch operations, perform the operation again to
699
  // determine the value which was written.
700
1.09k
  llvm::Value *Result = RMWI;
701
1.09k
  if (PostOpMinMax)
702
12
    Result = EmitPostAtomicMinMax(CGF.Builder, E->getOp(),
703
12
                                  E->getValueType()->isSignedIntegerType(),
704
12
                                  RMWI, LoadVal1);
705
1.08k
  else if (PostOp)
706
62
    Result = CGF.Builder.CreateBinOp((llvm::Instruction::BinaryOps)PostOp, RMWI,
707
62
                                     LoadVal1);
708
1.09k
  if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch)
709
4
    Result = CGF.Builder.CreateNot(Result);
710
1.09k
  CGF.Builder.CreateStore(Result, Dest);
711
1.09k
}
712
713
// This function emits any expression (scalar, complex, or aggregate)
714
// into a temporary alloca.
715
static Address
716
802
EmitValToTemp(CodeGenFunction &CGF, Expr *E) {
717
802
  Address DeclPtr = CGF.CreateMemTemp(E->getType(), ".atomictmp");
718
802
  CGF.EmitAnyExprToMem(E, DeclPtr, E->getType().getQualifiers(),
719
802
                       /*Init*/ true);
720
802
  return DeclPtr;
721
802
}
722
723
static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *Expr, Address Dest,
724
                         Address Ptr, Address Val1, Address Val2,
725
                         llvm::Value *IsWeak, llvm::Value *FailureOrder,
726
                         uint64_t Size, llvm::AtomicOrdering Order,
727
1.93k
                         llvm::Value *Scope) {
728
1.93k
  auto ScopeModel = Expr->getScopeModel();
729
730
  // LLVM atomic instructions always have synch scope. If clang atomic
731
  // expression has no scope operand, use default LLVM synch scope.
732
1.93k
  if (!ScopeModel) {
733
1.72k
    EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
734
1.72k
                 Order, CGF.CGM.getLLVMContext().getOrInsertSyncScopeID(""));
735
1.72k
    return;
736
1.72k
  }
737
738
  // Handle constant scope.
739
209
  if (auto SC = dyn_cast<llvm::ConstantInt>(Scope)) {
740
201
    auto SCID = CGF.getTargetHooks().getLLVMSyncScopeID(
741
201
        CGF.CGM.getLangOpts(), ScopeModel->map(SC->getZExtValue()),
742
201
        Order, CGF.CGM.getLLVMContext());
743
201
    EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
744
201
                 Order, SCID);
745
201
    return;
746
201
  }
747
748
  // Handle non-constant scope.
749
8
  auto &Builder = CGF.Builder;
750
8
  auto Scopes = ScopeModel->getRuntimeValues();
751
8
  llvm::DenseMap<unsigned, llvm::BasicBlock *> BB;
752
8
  for (auto S : Scopes)
753
32
    BB[S] = CGF.createBasicBlock(getAsString(ScopeModel->map(S)), CGF.CurFn);
754
755
8
  llvm::BasicBlock *ContBB =
756
8
      CGF.createBasicBlock("atomic.scope.continue", CGF.CurFn);
757
758
8
  auto *SC = Builder.CreateIntCast(Scope, Builder.getInt32Ty(), false);
759
  // If unsupported synch scope is encountered at run time, assume a fallback
760
  // synch scope value.
761
8
  auto FallBack = ScopeModel->getFallBackValue();
762
8
  llvm::SwitchInst *SI = Builder.CreateSwitch(SC, BB[FallBack]);
763
32
  for (auto S : Scopes) {
764
32
    auto *B = BB[S];
765
32
    if (S != FallBack)
766
24
      SI->addCase(Builder.getInt32(S), B);
767
768
32
    Builder.SetInsertPoint(B);
769
32
    EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
770
32
                 Order,
771
32
                 CGF.getTargetHooks().getLLVMSyncScopeID(CGF.CGM.getLangOpts(),
772
32
                                                         ScopeModel->map(S),
773
32
                                                         Order,
774
32
                                                         CGF.getLLVMContext()));
775
32
    Builder.CreateBr(ContBB);
776
32
  }
777
778
8
  Builder.SetInsertPoint(ContBB);
779
8
}
780
781
static void
782
AddDirectArgument(CodeGenFunction &CGF, CallArgList &Args,
783
                  bool UseOptimizedLibcall, llvm::Value *Val, QualType ValTy,
784
143
                  SourceLocation Loc, CharUnits SizeInChars) {
785
143
  if (UseOptimizedLibcall) {
786
    // Load value and pass it to the function directly.
787
83
    CharUnits Align = CGF.getContext().getTypeAlignInChars(ValTy);
788
83
    int64_t SizeInBits = CGF.getContext().toBits(SizeInChars);
789
83
    ValTy =
790
83
        CGF.getContext().getIntTypeForBitwidth(SizeInBits, /*Signed=*/false);
791
83
    llvm::Type *ITy = llvm::IntegerType::get(CGF.getLLVMContext(), SizeInBits);
792
83
    Address Ptr = Address(CGF.Builder.CreateBitCast(Val, ITy->getPointerTo()),
793
83
                          ITy, Align);
794
83
    Val = CGF.EmitLoadOfScalar(Ptr, false,
795
83
                               CGF.getContext().getPointerType(ValTy),
796
83
                               Loc);
797
    // Coerce the value into an appropriately sized integer type.
798
83
    Args.add(RValue::get(Val), ValTy);
799
83
  } else {
800
    // Non-optimized functions always take a reference.
801
60
    Args.add(RValue::get(CGF.EmitCastToVoidPtr(Val)),
802
60
                         CGF.getContext().VoidPtrTy);
803
60
  }
804
143
}
805
806
1.19k
RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) {
807
1.19k
  QualType AtomicTy = E->getPtr()->getType()->getPointeeType();
808
1.19k
  QualType MemTy = AtomicTy;
809
1.19k
  if (const AtomicType *AT = AtomicTy->getAs<AtomicType>())
810
656
    MemTy = AT->getValueType();
811
1.19k
  llvm::Value *IsWeak = nullptr, *OrderFail = nullptr;
812
813
1.19k
  Address Val1 = Address::invalid();
814
1.19k
  Address Val2 = Address::invalid();
815
1.19k
  Address Dest = Address::invalid();
816
1.19k
  Address Ptr = EmitPointerWithAlignment(E->getPtr());
817
818
1.19k
  if (E->getOp() == AtomicExpr::AO__c11_atomic_init ||
819
1.19k
      
E->getOp() == AtomicExpr::AO__opencl_atomic_init1.18k
) {
820
16
    LValue lvalue = MakeAddrLValue(Ptr, AtomicTy);
821
16
    EmitAtomicInit(E->getVal1(), lvalue);
822
16
    return RValue::get(nullptr);
823
16
  }
824
825
1.18k
  auto TInfo = getContext().getTypeInfoInChars(AtomicTy);
826
1.18k
  uint64_t Size = TInfo.Width.getQuantity();
827
1.18k
  unsigned MaxInlineWidthInBits = getTarget().getMaxAtomicInlineWidth();
828
829
1.18k
  bool Oversized = getContext().toBits(TInfo.Width) > MaxInlineWidthInBits;
830
1.18k
  bool Misaligned = (Ptr.getAlignment() % TInfo.Width) != 0;
831
1.18k
  bool UseLibcall = Misaligned | Oversized;
832
1.18k
  bool ShouldCastToIntPtrTy = true;
833
834
1.18k
  CharUnits MaxInlineWidth =
835
1.18k
      getContext().toCharUnitsFromBits(MaxInlineWidthInBits);
836
837
1.18k
  DiagnosticsEngine &Diags = CGM.getDiags();
838
839
1.18k
  if (Misaligned) {
840
80
    Diags.Report(E->getBeginLoc(), diag::warn_atomic_op_misaligned)
841
80
        << (int)TInfo.Width.getQuantity()
842
80
        << (int)Ptr.getAlignment().getQuantity();
843
80
  }
844
845
1.18k
  if (Oversized) {
846
160
    Diags.Report(E->getBeginLoc(), diag::warn_atomic_op_oversized)
847
160
        << (int)TInfo.Width.getQuantity() << (int)MaxInlineWidth.getQuantity();
848
160
  }
849
850
1.18k
  llvm::Value *Order = EmitScalarExpr(E->getOrder());
851
1.18k
  llvm::Value *Scope =
852
1.18k
      E->getScopeModel() ? 
EmitScalarExpr(E->getScope())224
:
nullptr958
;
853
854
1.18k
  switch (E->getOp()) {
855
0
  case AtomicExpr::AO__c11_atomic_init:
856
0
  case AtomicExpr::AO__opencl_atomic_init:
857
0
    llvm_unreachable("Already handled above with EmitAtomicInit!");
858
859
123
  case AtomicExpr::AO__c11_atomic_load:
860
141
  case AtomicExpr::AO__opencl_atomic_load:
861
149
  case AtomicExpr::AO__hip_atomic_load:
862
165
  case AtomicExpr::AO__atomic_load_n:
863
165
    break;
864
865
89
  case AtomicExpr::AO__atomic_load:
866
89
    Dest = EmitPointerWithAlignment(E->getVal1());
867
89
    break;
868
869
84
  case AtomicExpr::AO__atomic_store:
870
84
    Val1 = EmitPointerWithAlignment(E->getVal1());
871
84
    break;
872
873
14
  case AtomicExpr::AO__atomic_exchange:
874
14
    Val1 = EmitPointerWithAlignment(E->getVal1());
875
14
    Dest = EmitPointerWithAlignment(E->getVal2());
876
14
    break;
877
878
21
  case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
879
26
  case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
880
34
  case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
881
44
  case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
882
55
  case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
883
65
  case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
884
75
  case AtomicExpr::AO__atomic_compare_exchange_n:
885
97
  case AtomicExpr::AO__atomic_compare_exchange:
886
97
    Val1 = EmitPointerWithAlignment(E->getVal1());
887
97
    if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange)
888
22
      Val2 = EmitPointerWithAlignment(E->getVal2());
889
75
    else
890
75
      Val2 = EmitValToTemp(*this, E->getVal2());
891
97
    OrderFail = EmitScalarExpr(E->getOrderFail());
892
97
    if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange_n ||
893
97
        
E->getOp() == AtomicExpr::AO__atomic_compare_exchange87
)
894
32
      IsWeak = EmitScalarExpr(E->getWeak());
895
97
    break;
896
897
146
  case AtomicExpr::AO__c11_atomic_fetch_add:
898
227
  case AtomicExpr::AO__c11_atomic_fetch_sub:
899
237
  case AtomicExpr::AO__hip_atomic_fetch_add:
900
257
  case AtomicExpr::AO__opencl_atomic_fetch_add:
901
257
  case AtomicExpr::AO__opencl_atomic_fetch_sub:
902
257
    if (MemTy->isPointerType()) {
903
      // For pointer arithmetic, we're required to do a bit of math:
904
      // adding 1 to an int* is not the same as adding 1 to a uintptr_t.
905
      // ... but only for the C11 builtins. The GNU builtins expect the
906
      // user to multiply by sizeof(T).
907
6
      QualType Val1Ty = E->getVal1()->getType();
908
6
      llvm::Value *Val1Scalar = EmitScalarExpr(E->getVal1());
909
6
      CharUnits PointeeIncAmt =
910
6
          getContext().getTypeSizeInChars(MemTy->getPointeeType());
911
6
      Val1Scalar = Builder.CreateMul(Val1Scalar, CGM.getSize(PointeeIncAmt));
912
6
      auto Temp = CreateMemTemp(Val1Ty, ".atomictmp");
913
6
      Val1 = Temp;
914
6
      EmitStoreOfScalar(Val1Scalar, MakeAddrLValue(Temp, Val1Ty));
915
6
      break;
916
6
    }
917
257
    
LLVM_FALLTHROUGH251
;251
918
282
  case AtomicExpr::AO__atomic_fetch_add:
919
316
  case AtomicExpr::AO__atomic_fetch_sub:
920
373
  case AtomicExpr::AO__atomic_add_fetch:
921
374
  case AtomicExpr::AO__atomic_sub_fetch:
922
374
    ShouldCastToIntPtrTy = !MemTy->isFloatingType();
923
374
    LLVM_FALLTHROUGH;
924
925
500
  case AtomicExpr::AO__c11_atomic_store:
926
521
  case AtomicExpr::AO__c11_atomic_exchange:
927
539
  case AtomicExpr::AO__opencl_atomic_store:
928
554
  case AtomicExpr::AO__hip_atomic_store:
929
556
  case AtomicExpr::AO__opencl_atomic_exchange:
930
566
  case AtomicExpr::AO__hip_atomic_exchange:
931
570
  case AtomicExpr::AO__atomic_store_n:
932
574
  case AtomicExpr::AO__atomic_exchange_n:
933
578
  case AtomicExpr::AO__c11_atomic_fetch_and:
934
580
  case AtomicExpr::AO__c11_atomic_fetch_or:
935
580
  case AtomicExpr::AO__c11_atomic_fetch_xor:
936
580
  case AtomicExpr::AO__c11_atomic_fetch_nand:
937
590
  case AtomicExpr::AO__c11_atomic_fetch_max:
938
600
  case AtomicExpr::AO__c11_atomic_fetch_min:
939
602
  case AtomicExpr::AO__opencl_atomic_fetch_and:
940
602
  case AtomicExpr::AO__opencl_atomic_fetch_or:
941
602
  case AtomicExpr::AO__opencl_atomic_fetch_xor:
942
610
  case AtomicExpr::AO__opencl_atomic_fetch_min:
943
614
  case AtomicExpr::AO__opencl_atomic_fetch_max:
944
615
  case AtomicExpr::AO__atomic_fetch_and:
945
625
  case AtomicExpr::AO__hip_atomic_fetch_and:
946
629
  case AtomicExpr::AO__atomic_fetch_or:
947
639
  case AtomicExpr::AO__hip_atomic_fetch_or:
948
644
  case AtomicExpr::AO__atomic_fetch_xor:
949
654
  case AtomicExpr::AO__hip_atomic_fetch_xor:
950
659
  case AtomicExpr::AO__atomic_fetch_nand:
951
660
  case AtomicExpr::AO__atomic_and_fetch:
952
663
  case AtomicExpr::AO__atomic_or_fetch:
953
664
  case AtomicExpr::AO__atomic_xor_fetch:
954
669
  case AtomicExpr::AO__atomic_nand_fetch:
955
673
  case AtomicExpr::AO__atomic_max_fetch:
956
683
  case AtomicExpr::AO__atomic_min_fetch:
957
685
  case AtomicExpr::AO__atomic_fetch_max:
958
705
  case AtomicExpr::AO__hip_atomic_fetch_max:
959
707
  case AtomicExpr::AO__atomic_fetch_min:
960
727
  case AtomicExpr::AO__hip_atomic_fetch_min:
961
727
    Val1 = EmitValToTemp(*this, E->getVal1());
962
727
    break;
963
1.18k
  }
964
965
1.18k
  QualType RValTy = E->getType().getUnqualifiedType();
966
967
  // The inlined atomics only function on iN types, where N is a power of 2. We
968
  // need to make sure (via temporaries if necessary) that all incoming values
969
  // are compatible.
970
1.18k
  LValue AtomicVal = MakeAddrLValue(Ptr, AtomicTy);
971
1.18k
  AtomicInfo Atomics(*this, AtomicVal);
972
973
1.18k
  if (ShouldCastToIntPtrTy) {
974
1.13k
    Ptr = Atomics.emitCastToAtomicIntPointer(Ptr);
975
1.13k
    if (Val1.isValid())
976
880
      Val1 = Atomics.convertToAtomicIntPointer(Val1);
977
1.13k
    if (Val2.isValid())
978
97
      Val2 = Atomics.convertToAtomicIntPointer(Val2);
979
1.13k
  }
980
1.18k
  if (Dest.isValid()) {
981
103
    if (ShouldCastToIntPtrTy)
982
103
      Dest = Atomics.emitCastToAtomicIntPointer(Dest);
983
1.07k
  } else if (E->isCmpXChg())
984
97
    Dest = CreateMemTemp(RValTy, "cmpxchg.bool");
985
982
  else if (!RValTy->isVoidType()) {
986
735
    Dest = Atomics.CreateTempAlloca();
987
735
    if (ShouldCastToIntPtrTy)
988
687
      Dest = Atomics.emitCastToAtomicIntPointer(Dest);
989
735
  }
990
991
  // Use a library call.  See: http://gcc.gnu.org/wiki/Atomic/GCCMM/LIbrary .
992
1.18k
  if (UseLibcall) {
993
195
    bool UseOptimizedLibcall = false;
994
195
    switch (E->getOp()) {
995
0
    case AtomicExpr::AO__c11_atomic_init:
996
0
    case AtomicExpr::AO__opencl_atomic_init:
997
0
      llvm_unreachable("Already handled above with EmitAtomicInit!");
998
999
11
    case AtomicExpr::AO__c11_atomic_fetch_add:
1000
13
    case AtomicExpr::AO__opencl_atomic_fetch_add:
1001
18
    case AtomicExpr::AO__atomic_fetch_add:
1002
18
    case AtomicExpr::AO__hip_atomic_fetch_add:
1003
18
    case AtomicExpr::AO__c11_atomic_fetch_and:
1004
18
    case AtomicExpr::AO__opencl_atomic_fetch_and:
1005
18
    case AtomicExpr::AO__hip_atomic_fetch_and:
1006
19
    case AtomicExpr::AO__atomic_fetch_and:
1007
19
    case AtomicExpr::AO__c11_atomic_fetch_or:
1008
19
    case AtomicExpr::AO__opencl_atomic_fetch_or:
1009
19
    case AtomicExpr::AO__hip_atomic_fetch_or:
1010
20
    case AtomicExpr::AO__atomic_fetch_or:
1011
20
    case AtomicExpr::AO__c11_atomic_fetch_nand:
1012
21
    case AtomicExpr::AO__atomic_fetch_nand:
1013
23
    case AtomicExpr::AO__c11_atomic_fetch_sub:
1014
23
    case AtomicExpr::AO__opencl_atomic_fetch_sub:
1015
29
    case AtomicExpr::AO__atomic_fetch_sub:
1016
29
    case AtomicExpr::AO__c11_atomic_fetch_xor:
1017
29
    case AtomicExpr::AO__opencl_atomic_fetch_xor:
1018
33
    case AtomicExpr::AO__opencl_atomic_fetch_min:
1019
33
    case AtomicExpr::AO__opencl_atomic_fetch_max:
1020
34
    case AtomicExpr::AO__atomic_fetch_xor:
1021
34
    case AtomicExpr::AO__hip_atomic_fetch_xor:
1022
34
    case AtomicExpr::AO__c11_atomic_fetch_max:
1023
34
    case AtomicExpr::AO__c11_atomic_fetch_min:
1024
35
    case AtomicExpr::AO__atomic_add_fetch:
1025
36
    case AtomicExpr::AO__atomic_and_fetch:
1026
37
    case AtomicExpr::AO__atomic_nand_fetch:
1027
38
    case AtomicExpr::AO__atomic_or_fetch:
1028
39
    case AtomicExpr::AO__atomic_sub_fetch:
1029
40
    case AtomicExpr::AO__atomic_xor_fetch:
1030
40
    case AtomicExpr::AO__atomic_fetch_max:
1031
40
    case AtomicExpr::AO__hip_atomic_fetch_max:
1032
40
    case AtomicExpr::AO__atomic_fetch_min:
1033
40
    case AtomicExpr::AO__hip_atomic_fetch_min:
1034
40
    case AtomicExpr::AO__atomic_max_fetch:
1035
42
    case AtomicExpr::AO__atomic_min_fetch:
1036
      // For these, only library calls for certain sizes exist.
1037
42
      UseOptimizedLibcall = true;
1038
42
      break;
1039
1040
37
    case AtomicExpr::AO__atomic_load:
1041
72
    case AtomicExpr::AO__atomic_store:
1042
80
    case AtomicExpr::AO__atomic_exchange:
1043
89
    case AtomicExpr::AO__atomic_compare_exchange:
1044
      // Use the generic version if we don't know that the operand will be
1045
      // suitably aligned for the optimized version.
1046
89
      if (Misaligned)
1047
63
        break;
1048
89
      
LLVM_FALLTHROUGH26
;26
1049
39
    case AtomicExpr::AO__c11_atomic_load:
1050
53
    case AtomicExpr::AO__c11_atomic_store:
1051
60
    case AtomicExpr::AO__c11_atomic_exchange:
1052
62
    case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
1053
69
    case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
1054
69
    case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
1055
71
    case AtomicExpr::AO__opencl_atomic_load:
1056
71
    case AtomicExpr::AO__hip_atomic_load:
1057
79
    case AtomicExpr::AO__opencl_atomic_store:
1058
79
    case AtomicExpr::AO__hip_atomic_store:
1059
79
    case AtomicExpr::AO__opencl_atomic_exchange:
1060
79
    case AtomicExpr::AO__hip_atomic_exchange:
1061
88
    case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
1062
88
    case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
1063
90
    case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
1064
90
    case AtomicExpr::AO__atomic_load_n:
1065
90
    case AtomicExpr::AO__atomic_store_n:
1066
90
    case AtomicExpr::AO__atomic_exchange_n:
1067
90
    case AtomicExpr::AO__atomic_compare_exchange_n:
1068
      // Only use optimized library calls for sizes for which they exist.
1069
      // FIXME: Size == 16 optimized library functions exist too.
1070
90
      if (Size == 1 || 
Size == 282
||
Size == 480
||
Size == 851
)
1071
63
        UseOptimizedLibcall = true;
1072
90
      break;
1073
195
    }
1074
1075
195
    CallArgList Args;
1076
195
    if (!UseOptimizedLibcall) {
1077
      // For non-optimized library calls, the size is the first parameter
1078
90
      Args.add(RValue::get(llvm::ConstantInt::get(SizeTy, Size)),
1079
90
               getContext().getSizeType());
1080
90
    }
1081
    // Atomic address is the first or second parameter
1082
    // The OpenCL atomic library functions only accept pointer arguments to
1083
    // generic address space.
1084
224
    auto CastToGenericAddrSpace = [&](llvm::Value *V, QualType PT) {
1085
224
      if (!E->isOpenCL())
1086
186
        return V;
1087
38
      auto AS = PT->castAs<PointerType>()->getPointeeType().getAddressSpace();
1088
38
      if (AS == LangAS::opencl_generic)
1089
21
        return V;
1090
17
      auto DestAS = getContext().getTargetAddressSpace(LangAS::opencl_generic);
1091
17
      auto T = llvm::cast<llvm::PointerType>(V->getType());
1092
17
      auto *DestType = llvm::PointerType::getWithSamePointeeType(T, DestAS);
1093
1094
17
      return getTargetHooks().performAddrSpaceCast(
1095
17
          *this, V, AS, LangAS::opencl_generic, DestType, false);
1096
38
    };
1097
1098
195
    Args.add(RValue::get(CastToGenericAddrSpace(
1099
195
                 EmitCastToVoidPtr(Ptr.getPointer()), E->getPtr()->getType())),
1100
195
             getContext().VoidPtrTy);
1101
1102
195
    std::string LibCallName;
1103
195
    QualType LoweredMemTy =
1104
195
      MemTy->isPointerType() ? 
getContext().getIntPtrType()3
:
MemTy192
;
1105
195
    QualType RetTy;
1106
195
    bool HaveRetTy = false;
1107
195
    llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0;
1108
195
    bool PostOpMinMax = false;
1109
195
    switch (E->getOp()) {
1110
0
    case AtomicExpr::AO__c11_atomic_init:
1111
0
    case AtomicExpr::AO__opencl_atomic_init:
1112
0
      llvm_unreachable("Already handled!");
1113
1114
    // There is only one libcall for compare an exchange, because there is no
1115
    // optimisation benefit possible from a libcall version of a weak compare
1116
    // and exchange.
1117
    // bool __atomic_compare_exchange(size_t size, void *mem, void *expected,
1118
    //                                void *desired, int success, int failure)
1119
    // bool __atomic_compare_exchange_N(T *mem, T *expected, T desired,
1120
    //                                  int success, int failure)
1121
2
    case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
1122
9
    case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
1123
18
    case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
1124
18
    case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
1125
20
    case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
1126
20
    case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
1127
29
    case AtomicExpr::AO__atomic_compare_exchange:
1128
29
    case AtomicExpr::AO__atomic_compare_exchange_n:
1129
29
      LibCallName = "__atomic_compare_exchange";
1130
29
      RetTy = getContext().BoolTy;
1131
29
      HaveRetTy = true;
1132
29
      Args.add(
1133
29
          RValue::get(CastToGenericAddrSpace(
1134
29
              EmitCastToVoidPtr(Val1.getPointer()), E->getVal1()->getType())),
1135
29
          getContext().VoidPtrTy);
1136
29
      AddDirectArgument(*this, Args, UseOptimizedLibcall, Val2.getPointer(),
1137
29
                        MemTy, E->getExprLoc(), TInfo.Width);
1138
29
      Args.add(RValue::get(Order), getContext().IntTy);
1139
29
      Order = OrderFail;
1140
29
      break;
1141
    // void __atomic_exchange(size_t size, void *mem, void *val, void *return,
1142
    //                        int order)
1143
    // T __atomic_exchange_N(T *mem, T val, int order)
1144
7
    case AtomicExpr::AO__c11_atomic_exchange:
1145
7
    case AtomicExpr::AO__opencl_atomic_exchange:
1146
7
    case AtomicExpr::AO__atomic_exchange_n:
1147
15
    case AtomicExpr::AO__atomic_exchange:
1148
15
    case AtomicExpr::AO__hip_atomic_exchange:
1149
15
      LibCallName = "__atomic_exchange";
1150
15
      AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1151
15
                        MemTy, E->getExprLoc(), TInfo.Width);
1152
15
      break;
1153
    // void __atomic_store(size_t size, void *mem, void *val, int order)
1154
    // void __atomic_store_N(T *mem, T val, int order)
1155
14
    case AtomicExpr::AO__c11_atomic_store:
1156
22
    case AtomicExpr::AO__opencl_atomic_store:
1157
22
    case AtomicExpr::AO__hip_atomic_store:
1158
57
    case AtomicExpr::AO__atomic_store:
1159
57
    case AtomicExpr::AO__atomic_store_n:
1160
57
      LibCallName = "__atomic_store";
1161
57
      RetTy = getContext().VoidTy;
1162
57
      HaveRetTy = true;
1163
57
      AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1164
57
                        MemTy, E->getExprLoc(), TInfo.Width);
1165
57
      break;
1166
    // void __atomic_load(size_t size, void *mem, void *return, int order)
1167
    // T __atomic_load_N(T *mem, int order)
1168
13
    case AtomicExpr::AO__c11_atomic_load:
1169
15
    case AtomicExpr::AO__opencl_atomic_load:
1170
15
    case AtomicExpr::AO__hip_atomic_load:
1171
52
    case AtomicExpr::AO__atomic_load:
1172
52
    case AtomicExpr::AO__atomic_load_n:
1173
52
      LibCallName = "__atomic_load";
1174
52
      break;
1175
    // T __atomic_add_fetch_N(T *mem, T val, int order)
1176
    // T __atomic_fetch_add_N(T *mem, T val, int order)
1177
1
    case AtomicExpr::AO__atomic_add_fetch:
1178
1
      PostOp = llvm::Instruction::Add;
1179
1
      LLVM_FALLTHROUGH;
1180
12
    case AtomicExpr::AO__c11_atomic_fetch_add:
1181
14
    case AtomicExpr::AO__opencl_atomic_fetch_add:
1182
19
    case AtomicExpr::AO__atomic_fetch_add:
1183
19
    case AtomicExpr::AO__hip_atomic_fetch_add:
1184
19
      LibCallName = "__atomic_fetch_add";
1185
19
      AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1186
19
                        LoweredMemTy, E->getExprLoc(), TInfo.Width);
1187
19
      break;
1188
    // T __atomic_and_fetch_N(T *mem, T val, int order)
1189
    // T __atomic_fetch_and_N(T *mem, T val, int order)
1190
1
    case AtomicExpr::AO__atomic_and_fetch:
1191
1
      PostOp = llvm::Instruction::And;
1192
1
      LLVM_FALLTHROUGH;
1193
1
    case AtomicExpr::AO__c11_atomic_fetch_and:
1194
1
    case AtomicExpr::AO__opencl_atomic_fetch_and:
1195
1
    case AtomicExpr::AO__hip_atomic_fetch_and:
1196
2
    case AtomicExpr::AO__atomic_fetch_and:
1197
2
      LibCallName = "__atomic_fetch_and";
1198
2
      AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1199
2
                        MemTy, E->getExprLoc(), TInfo.Width);
1200
2
      break;
1201
    // T __atomic_or_fetch_N(T *mem, T val, int order)
1202
    // T __atomic_fetch_or_N(T *mem, T val, int order)
1203
1
    case AtomicExpr::AO__atomic_or_fetch:
1204
1
      PostOp = llvm::Instruction::Or;
1205
1
      LLVM_FALLTHROUGH;
1206
1
    case AtomicExpr::AO__c11_atomic_fetch_or:
1207
1
    case AtomicExpr::AO__opencl_atomic_fetch_or:
1208
1
    case AtomicExpr::AO__hip_atomic_fetch_or:
1209
2
    case AtomicExpr::AO__atomic_fetch_or:
1210
2
      LibCallName = "__atomic_fetch_or";
1211
2
      AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1212
2
                        MemTy, E->getExprLoc(), TInfo.Width);
1213
2
      break;
1214
    // T __atomic_sub_fetch_N(T *mem, T val, int order)
1215
    // T __atomic_fetch_sub_N(T *mem, T val, int order)
1216
1
    case AtomicExpr::AO__atomic_sub_fetch:
1217
1
      PostOp = llvm::Instruction::Sub;
1218
1
      LLVM_FALLTHROUGH;
1219
3
    case AtomicExpr::AO__c11_atomic_fetch_sub:
1220
3
    case AtomicExpr::AO__opencl_atomic_fetch_sub:
1221
9
    case AtomicExpr::AO__atomic_fetch_sub:
1222
9
      LibCallName = "__atomic_fetch_sub";
1223
9
      AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1224
9
                        LoweredMemTy, E->getExprLoc(), TInfo.Width);
1225
9
      break;
1226
    // T __atomic_xor_fetch_N(T *mem, T val, int order)
1227
    // T __atomic_fetch_xor_N(T *mem, T val, int order)
1228
1
    case AtomicExpr::AO__atomic_xor_fetch:
1229
1
      PostOp = llvm::Instruction::Xor;
1230
1
      LLVM_FALLTHROUGH;
1231
1
    case AtomicExpr::AO__c11_atomic_fetch_xor:
1232
1
    case AtomicExpr::AO__opencl_atomic_fetch_xor:
1233
1
    case AtomicExpr::AO__hip_atomic_fetch_xor:
1234
2
    case AtomicExpr::AO__atomic_fetch_xor:
1235
2
      LibCallName = "__atomic_fetch_xor";
1236
2
      AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1237
2
                        MemTy, E->getExprLoc(), TInfo.Width);
1238
2
      break;
1239
2
    case AtomicExpr::AO__atomic_min_fetch:
1240
2
      PostOpMinMax = true;
1241
2
      LLVM_FALLTHROUGH;
1242
2
    case AtomicExpr::AO__c11_atomic_fetch_min:
1243
2
    case AtomicExpr::AO__atomic_fetch_min:
1244
2
    case AtomicExpr::AO__hip_atomic_fetch_min:
1245
6
    case AtomicExpr::AO__opencl_atomic_fetch_min:
1246
6
      LibCallName = E->getValueType()->isSignedIntegerType()
1247
6
                        ? 
"__atomic_fetch_min"2
1248
6
                        : 
"__atomic_fetch_umin"4
;
1249
6
      AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1250
6
                        LoweredMemTy, E->getExprLoc(), TInfo.Width);
1251
6
      break;
1252
0
    case AtomicExpr::AO__atomic_max_fetch:
1253
0
      PostOpMinMax = true;
1254
0
      LLVM_FALLTHROUGH;
1255
0
    case AtomicExpr::AO__c11_atomic_fetch_max:
1256
0
    case AtomicExpr::AO__atomic_fetch_max:
1257
0
    case AtomicExpr::AO__hip_atomic_fetch_max:
1258
0
    case AtomicExpr::AO__opencl_atomic_fetch_max:
1259
0
      LibCallName = E->getValueType()->isSignedIntegerType()
1260
0
                        ? "__atomic_fetch_max"
1261
0
                        : "__atomic_fetch_umax";
1262
0
      AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1263
0
                        LoweredMemTy, E->getExprLoc(), TInfo.Width);
1264
0
      break;
1265
    // T __atomic_nand_fetch_N(T *mem, T val, int order)
1266
    // T __atomic_fetch_nand_N(T *mem, T val, int order)
1267
1
    case AtomicExpr::AO__atomic_nand_fetch:
1268
1
      PostOp = llvm::Instruction::And; // the NOT is special cased below
1269
1
      LLVM_FALLTHROUGH;
1270
1
    case AtomicExpr::AO__c11_atomic_fetch_nand:
1271
2
    case AtomicExpr::AO__atomic_fetch_nand:
1272
2
      LibCallName = "__atomic_fetch_nand";
1273
2
      AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1274
2
                        MemTy, E->getExprLoc(), TInfo.Width);
1275
2
      break;
1276
195
    }
1277
1278
195
    if (E->isOpenCL()) {
1279
27
      LibCallName = std::string("__opencl") +
1280
27
          StringRef(LibCallName).drop_front(1).str();
1281
1282
27
    }
1283
    // Optimized functions have the size in their name.
1284
195
    if (UseOptimizedLibcall)
1285
105
      LibCallName += "_" + llvm::utostr(Size);
1286
    // By default, assume we return a value of the atomic type.
1287
195
    if (!HaveRetTy) {
1288
109
      if (UseOptimizedLibcall) {
1289
        // Value is returned directly.
1290
        // The function returns an appropriately sized integer type.
1291
65
        RetTy = getContext().getIntTypeForBitwidth(
1292
65
            getContext().toBits(TInfo.Width), /*Signed=*/false);
1293
65
      } else {
1294
        // Value is returned through parameter before the order.
1295
44
        RetTy = getContext().VoidTy;
1296
44
        Args.add(RValue::get(EmitCastToVoidPtr(Dest.getPointer())),
1297
44
                 getContext().VoidPtrTy);
1298
44
      }
1299
109
    }
1300
    // order is always the last parameter
1301
195
    Args.add(RValue::get(Order),
1302
195
             getContext().IntTy);
1303
195
    if (E->isOpenCL())
1304
27
      Args.add(RValue::get(Scope), getContext().IntTy);
1305
1306
    // PostOp is only needed for the atomic_*_fetch operations, and
1307
    // thus is only needed for and implemented in the
1308
    // UseOptimizedLibcall codepath.
1309
195
    assert(UseOptimizedLibcall || (!PostOp && !PostOpMinMax));
1310
1311
0
    RValue Res = emitAtomicLibcall(*this, LibCallName, RetTy, Args);
1312
    // The value is returned directly from the libcall.
1313
195
    if (E->isCmpXChg())
1314
29
      return Res;
1315
1316
    // The value is returned directly for optimized libcalls but the expr
1317
    // provided an out-param.
1318
166
    if (UseOptimizedLibcall && 
Res.getScalarVal()93
) {
1319
65
      llvm::Value *ResVal = Res.getScalarVal();
1320
65
      if (PostOpMinMax) {
1321
2
        llvm::Value *LoadVal1 = Args[1].getRValue(*this).getScalarVal();
1322
2
        ResVal = EmitPostAtomicMinMax(Builder, E->getOp(),
1323
2
                                      E->getValueType()->isSignedIntegerType(),
1324
2
                                      ResVal, LoadVal1);
1325
63
      } else if (PostOp) {
1326
6
        llvm::Value *LoadVal1 = Args[1].getRValue(*this).getScalarVal();
1327
6
        ResVal = Builder.CreateBinOp(PostOp, ResVal, LoadVal1);
1328
6
      }
1329
65
      if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch)
1330
1
        ResVal = Builder.CreateNot(ResVal);
1331
1332
65
      Builder.CreateStore(
1333
65
          ResVal, Builder.CreateElementBitCast(Dest, ResVal->getType()));
1334
65
    }
1335
1336
166
    if (RValTy->isVoidType())
1337
102
      return RValue::get(nullptr);
1338
1339
64
    return convertTempToRValue(
1340
64
        Builder.CreateElementBitCast(Dest, ConvertTypeForMem(RValTy)),
1341
64
        RValTy, E->getExprLoc());
1342
166
  }
1343
1344
987
  bool IsStore = E->getOp() == AtomicExpr::AO__c11_atomic_store ||
1345
987
                 
E->getOp() == AtomicExpr::AO__opencl_atomic_store875
||
1346
987
                 
E->getOp() == AtomicExpr::AO__hip_atomic_store865
||
1347
987
                 
E->getOp() == AtomicExpr::AO__atomic_store850
||
1348
987
                 
E->getOp() == AtomicExpr::AO__atomic_store_n801
;
1349
987
  bool IsLoad = E->getOp() == AtomicExpr::AO__c11_atomic_load ||
1350
987
                
E->getOp() == AtomicExpr::AO__opencl_atomic_load877
||
1351
987
                
E->getOp() == AtomicExpr::AO__hip_atomic_load861
||
1352
987
                
E->getOp() == AtomicExpr::AO__atomic_load853
||
1353
987
                
E->getOp() == AtomicExpr::AO__atomic_load_n801
;
1354
1355
987
  if (isa<llvm::ConstantInt>(Order)) {
1356
664
    auto ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
1357
    // We should not ever get to a case where the ordering isn't a valid C ABI
1358
    // value, but it's hard to enforce that in general.
1359
664
    if (llvm::isValidAtomicOrderingCABI(ord))
1360
664
      switch ((llvm::AtomicOrderingCABI)ord) {
1361
233
      case llvm::AtomicOrderingCABI::relaxed:
1362
233
        EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1363
233
                     llvm::AtomicOrdering::Monotonic, Scope);
1364
233
        break;
1365
0
      case llvm::AtomicOrderingCABI::consume:
1366
43
      case llvm::AtomicOrderingCABI::acquire:
1367
43
        if (IsStore)
1368
0
          break; // Avoid crashing on code with undefined behavior
1369
43
        EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1370
43
                     llvm::AtomicOrdering::Acquire, Scope);
1371
43
        break;
1372
23
      case llvm::AtomicOrderingCABI::release:
1373
23
        if (IsLoad)
1374
0
          break; // Avoid crashing on code with undefined behavior
1375
23
        EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1376
23
                     llvm::AtomicOrdering::Release, Scope);
1377
23
        break;
1378
50
      case llvm::AtomicOrderingCABI::acq_rel:
1379
50
        if (IsLoad || IsStore)
1380
0
          break; // Avoid crashing on code with undefined behavior
1381
50
        EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1382
50
                     llvm::AtomicOrdering::AcquireRelease, Scope);
1383
50
        break;
1384
315
      case llvm::AtomicOrderingCABI::seq_cst:
1385
315
        EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1386
315
                     llvm::AtomicOrdering::SequentiallyConsistent, Scope);
1387
315
        break;
1388
664
      }
1389
664
    if (RValTy->isVoidType())
1390
157
      return RValue::get(nullptr);
1391
1392
507
    return convertTempToRValue(
1393
507
        Builder.CreateElementBitCast(Dest, ConvertTypeForMem(RValTy)),
1394
507
        RValTy, E->getExprLoc());
1395
664
  }
1396
1397
  // Long case, when Order isn't obviously constant.
1398
1399
  // Create all the relevant BB's
1400
323
  llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr,
1401
323
                   *ReleaseBB = nullptr, *AcqRelBB = nullptr,
1402
323
                   *SeqCstBB = nullptr;
1403
323
  MonotonicBB = createBasicBlock("monotonic", CurFn);
1404
323
  if (!IsStore)
1405
232
    AcquireBB = createBasicBlock("acquire", CurFn);
1406
323
  if (!IsLoad)
1407
240
    ReleaseBB = createBasicBlock("release", CurFn);
1408
323
  if (!IsLoad && 
!IsStore240
)
1409
149
    AcqRelBB = createBasicBlock("acqrel", CurFn);
1410
323
  SeqCstBB = createBasicBlock("seqcst", CurFn);
1411
323
  llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn);
1412
1413
  // Create the switch for the split
1414
  // MonotonicBB is arbitrarily chosen as the default case; in practice, this
1415
  // doesn't matter unless someone is crazy enough to use something that
1416
  // doesn't fold to a constant for the ordering.
1417
323
  Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);
1418
323
  llvm::SwitchInst *SI = Builder.CreateSwitch(Order, MonotonicBB);
1419
1420
  // Emit all the different atomics
1421
323
  Builder.SetInsertPoint(MonotonicBB);
1422
323
  EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1423
323
               llvm::AtomicOrdering::Monotonic, Scope);
1424
323
  Builder.CreateBr(ContBB);
1425
323
  if (!IsStore) {
1426
232
    Builder.SetInsertPoint(AcquireBB);
1427
232
    EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1428
232
                 llvm::AtomicOrdering::Acquire, Scope);
1429
232
    Builder.CreateBr(ContBB);
1430
232
    SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::consume),
1431
232
                AcquireBB);
1432
232
    SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire),
1433
232
                AcquireBB);
1434
232
  }
1435
323
  if (!IsLoad) {
1436
240
    Builder.SetInsertPoint(ReleaseBB);
1437
240
    EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1438
240
                 llvm::AtomicOrdering::Release, Scope);
1439
240
    Builder.CreateBr(ContBB);
1440
240
    SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::release),
1441
240
                ReleaseBB);
1442
240
  }
1443
323
  if (!IsLoad && 
!IsStore240
) {
1444
149
    Builder.SetInsertPoint(AcqRelBB);
1445
149
    EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1446
149
                 llvm::AtomicOrdering::AcquireRelease, Scope);
1447
149
    Builder.CreateBr(ContBB);
1448
149
    SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acq_rel),
1449
149
                AcqRelBB);
1450
149
  }
1451
323
  Builder.SetInsertPoint(SeqCstBB);
1452
323
  EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1453
323
               llvm::AtomicOrdering::SequentiallyConsistent, Scope);
1454
323
  Builder.CreateBr(ContBB);
1455
323
  SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst),
1456
323
              SeqCstBB);
1457
1458
  // Cleanup and return
1459
323
  Builder.SetInsertPoint(ContBB);
1460
323
  if (RValTy->isVoidType())
1461
91
    return RValue::get(nullptr);
1462
1463
232
  assert(Atomics.getValueSizeInBits() <= Atomics.getAtomicSizeInBits());
1464
0
  return convertTempToRValue(
1465
232
      Builder.CreateElementBitCast(Dest, ConvertTypeForMem(RValTy)),
1466
232
      RValTy, E->getExprLoc());
1467
323
}
1468
1469
4.03k
Address AtomicInfo::emitCastToAtomicIntPointer(Address addr) const {
1470
4.03k
  llvm::IntegerType *ty =
1471
4.03k
    llvm::IntegerType::get(CGF.getLLVMContext(), AtomicSizeInBits);
1472
4.03k
  return CGF.Builder.CreateElementBitCast(addr, ty);
1473
4.03k
}
1474
1475
977
Address AtomicInfo::convertToAtomicIntPointer(Address Addr) const {
1476
977
  llvm::Type *Ty = Addr.getElementType();
1477
977
  uint64_t SourceSizeInBits = CGF.CGM.getDataLayout().getTypeSizeInBits(Ty);
1478
977
  if (SourceSizeInBits != AtomicSizeInBits) {
1479
9
    Address Tmp = CreateTempAlloca();
1480
9
    CGF.Builder.CreateMemCpy(Tmp, Addr,
1481
9
                             std::min(AtomicSizeInBits, SourceSizeInBits) / 8);
1482
9
    Addr = Tmp;
1483
9
  }
1484
1485
977
  return emitCastToAtomicIntPointer(Addr);
1486
977
}
1487
1488
RValue AtomicInfo::convertAtomicTempToRValue(Address addr,
1489
                                             AggValueSlot resultSlot,
1490
                                             SourceLocation loc,
1491
180
                                             bool asValue) const {
1492
180
  if (LVal.isSimple()) {
1493
144
    if (EvaluationKind == TEK_Aggregate)
1494
13
      return resultSlot.asRValue();
1495
1496
    // Drill into the padding structure if we have one.
1497
131
    if (hasPadding())
1498
0
      addr = CGF.Builder.CreateStructGEP(addr, 0);
1499
1500
    // Otherwise, just convert the temporary to an r-value using the
1501
    // normal conversion routine.
1502
131
    return CGF.convertTempToRValue(addr, getValueType(), loc);
1503
144
  }
1504
36
  if (!asValue)
1505
    // Get RValue from temp memory as atomic for non-simple lvalues
1506
12
    return RValue::get(CGF.Builder.CreateLoad(addr));
1507
24
  if (LVal.isBitField())
1508
20
    return CGF.EmitLoadOfBitfieldLValue(
1509
20
        LValue::MakeBitfield(addr, LVal.getBitFieldInfo(), LVal.getType(),
1510
20
                             LVal.getBaseInfo(), TBAAAccessInfo()), loc);
1511
4
  if (LVal.isVectorElt())
1512
2
    return CGF.EmitLoadOfLValue(
1513
2
        LValue::MakeVectorElt(addr, LVal.getVectorIdx(), LVal.getType(),
1514
2
                              LVal.getBaseInfo(), TBAAAccessInfo()), loc);
1515
2
  assert(LVal.isExtVectorElt());
1516
0
  return CGF.EmitLoadOfExtVectorElementLValue(LValue::MakeExtVectorElt(
1517
2
      addr, LVal.getExtVectorElts(), LVal.getType(),
1518
2
      LVal.getBaseInfo(), TBAAAccessInfo()));
1519
4
}
1520
1521
RValue AtomicInfo::ConvertIntToValueOrAtomic(llvm::Value *IntVal,
1522
                                             AggValueSlot ResultSlot,
1523
                                             SourceLocation Loc,
1524
387
                                             bool AsValue) const {
1525
  // Try not to in some easy cases.
1526
387
  assert(IntVal->getType()->isIntegerTy() && "Expected integer value");
1527
387
  if (getEvaluationKind() == TEK_Scalar &&
1528
387
      
(377
(377
(377
!LVal.isBitField()377
||
1529
377
         
LVal.getBitFieldInfo().Size == ValueSizeInBits64
) &&
1530
377
        
!hasPadding()313
) ||
1531
377
       
!AsValue80
)) {
1532
357
    auto *ValTy = AsValue
1533
357
                      ? 
CGF.ConvertTypeForMem(ValueTy)109
1534
357
                      : 
getAtomicAddress().getElementType()248
;
1535
357
    if (ValTy->isIntegerTy()) {
1536
275
      assert(IntVal->getType() == ValTy && "Different integer types.");
1537
0
      return RValue::get(CGF.EmitFromMemory(IntVal, ValueTy));
1538
275
    } else 
if (82
ValTy->isPointerTy()82
)
1539
0
      return RValue::get(CGF.Builder.CreateIntToPtr(IntVal, ValTy));
1540
82
    else if (llvm::CastInst::isBitCastable(IntVal->getType(), ValTy))
1541
50
      return RValue::get(CGF.Builder.CreateBitCast(IntVal, ValTy));
1542
357
  }
1543
1544
  // Create a temporary.  This needs to be big enough to hold the
1545
  // atomic integer.
1546
62
  Address Temp = Address::invalid();
1547
62
  bool TempIsVolatile = false;
1548
62
  if (AsValue && 
getEvaluationKind() == TEK_Aggregate44
) {
1549
8
    assert(!ResultSlot.isIgnored());
1550
0
    Temp = ResultSlot.getAddress();
1551
8
    TempIsVolatile = ResultSlot.isVolatile();
1552
54
  } else {
1553
54
    Temp = CreateTempAlloca();
1554
54
  }
1555
1556
  // Slam the integer into the temporary.
1557
0
  Address CastTemp = emitCastToAtomicIntPointer(Temp);
1558
62
  CGF.Builder.CreateStore(IntVal, CastTemp)
1559
62
      ->setVolatile(TempIsVolatile);
1560
1561
62
  return convertAtomicTempToRValue(Temp, ResultSlot, Loc, AsValue);
1562
387
}
1563
1564
void AtomicInfo::EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
1565
105
                                       llvm::AtomicOrdering AO, bool) {
1566
  // void __atomic_load(size_t size, void *mem, void *return, int order);
1567
105
  CallArgList Args;
1568
105
  Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
1569
105
  Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())),
1570
105
           CGF.getContext().VoidPtrTy);
1571
105
  Args.add(RValue::get(CGF.EmitCastToVoidPtr(AddForLoaded)),
1572
105
           CGF.getContext().VoidPtrTy);
1573
105
  Args.add(
1574
105
      RValue::get(llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(AO))),
1575
105
      CGF.getContext().IntTy);
1576
105
  emitAtomicLibcall(CGF, "__atomic_load", CGF.getContext().VoidTy, Args);
1577
105
}
1578
1579
llvm::Value *AtomicInfo::EmitAtomicLoadOp(llvm::AtomicOrdering AO,
1580
395
                                          bool IsVolatile) {
1581
  // Okay, we're doing this natively.
1582
395
  Address Addr = getAtomicAddressAsAtomicIntPointer();
1583
395
  llvm::LoadInst *Load = CGF.Builder.CreateLoad(Addr, "atomic-load");
1584
395
  Load->setAtomic(AO);
1585
1586
  // Other decoration.
1587
395
  if (IsVolatile)
1588
12
    Load->setVolatile(true);
1589
395
  CGF.CGM.DecorateInstructionWithTBAA(Load, LVal.getTBAAInfo());
1590
395
  return Load;
1591
395
}
1592
1593
/// An LValue is a candidate for having its loads and stores be made atomic if
1594
/// we are operating under /volatile:ms *and* the LValue itself is volatile and
1595
/// performing such an operation can be performed without a libcall.
1596
1.48M
bool CodeGenFunction::LValueIsSuitableForInlineAtomic(LValue LV) {
1597
1.48M
  if (!CGM.getCodeGenOpts().MSVolatile) 
return false1.48M
;
1598
45
  AtomicInfo AI(*this, LV);
1599
45
  bool IsVolatile = LV.isVolatile() || 
hasVolatileMember(LV.getType())27
;
1600
  // An atomic is inline if we don't need to use a libcall.
1601
45
  bool AtomicIsInline = !AI.shouldUseLibcall();
1602
  // MSVC doesn't seem to do this for types wider than a pointer.
1603
45
  if (getContext().getTypeSize(LV.getType()) >
1604
45
      getContext().getTypeSize(getContext().getIntPtrType()))
1605
5
    return false;
1606
40
  return IsVolatile && 
AtomicIsInline15
;
1607
45
}
1608
1609
RValue CodeGenFunction::EmitAtomicLoad(LValue LV, SourceLocation SL,
1610
77
                                       AggValueSlot Slot) {
1611
77
  llvm::AtomicOrdering AO;
1612
77
  bool IsVolatile = LV.isVolatileQualified();
1613
77
  if (LV.getType()->isAtomicType()) {
1614
70
    AO = llvm::AtomicOrdering::SequentiallyConsistent;
1615
70
  } else {
1616
7
    AO = llvm::AtomicOrdering::Acquire;
1617
7
    IsVolatile = true;
1618
7
  }
1619
77
  return EmitAtomicLoad(LV, SL, AO, IsVolatile, Slot);
1620
77
}
1621
1622
RValue AtomicInfo::EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
1623
                                  bool AsValue, llvm::AtomicOrdering AO,
1624
183
                                  bool IsVolatile) {
1625
  // Check whether we should use a library call.
1626
183
  if (shouldUseLibcall()) {
1627
44
    Address TempAddr = Address::invalid();
1628
44
    if (LVal.isSimple() && 
!ResultSlot.isIgnored()40
) {
1629
5
      assert(getEvaluationKind() == TEK_Aggregate);
1630
0
      TempAddr = ResultSlot.getAddress();
1631
5
    } else
1632
39
      TempAddr = CreateTempAlloca();
1633
1634
0
    EmitAtomicLoadLibcall(TempAddr.getPointer(), AO, IsVolatile);
1635
1636
    // Okay, turn that back into the original value or whole atomic (for
1637
    // non-simple lvalues) type.
1638
44
    return convertAtomicTempToRValue(TempAddr, ResultSlot, Loc, AsValue);
1639
44
  }
1640
1641
  // Okay, we're doing this natively.
1642
139
  auto *Load = EmitAtomicLoadOp(AO, IsVolatile);
1643
1644
  // If we're ignoring an aggregate return, don't do anything.
1645
139
  if (getEvaluationKind() == TEK_Aggregate && 
ResultSlot.isIgnored()8
)
1646
0
    return RValue::getAggregate(Address::invalid(), false);
1647
1648
  // Okay, turn that back into the original value or atomic (for non-simple
1649
  // lvalues) type.
1650
139
  return ConvertIntToValueOrAtomic(Load, ResultSlot, Loc, AsValue);
1651
139
}
1652
1653
/// Emit a load from an l-value of atomic type.  Note that the r-value
1654
/// we produce is an r-value of the atomic *value* type.
1655
RValue CodeGenFunction::EmitAtomicLoad(LValue src, SourceLocation loc,
1656
                                       llvm::AtomicOrdering AO, bool IsVolatile,
1657
183
                                       AggValueSlot resultSlot) {
1658
183
  AtomicInfo Atomics(*this, src);
1659
183
  return Atomics.EmitAtomicLoad(resultSlot, loc, /*AsValue=*/true, AO,
1660
183
                                IsVolatile);
1661
183
}
1662
1663
/// Copy an r-value into memory as part of storing to an atomic type.
1664
/// This needs to create a bit-pattern suitable for atomic operations.
1665
185
void AtomicInfo::emitCopyIntoMemory(RValue rvalue) const {
1666
185
  assert(LVal.isSimple());
1667
  // If we have an r-value, the rvalue should be of the atomic type,
1668
  // which means that the caller is responsible for having zeroed
1669
  // any padding.  Just do an aggregate copy of that type.
1670
185
  if (rvalue.isAggregate()) {
1671
0
    LValue Dest = CGF.MakeAddrLValue(getAtomicAddress(), getAtomicType());
1672
0
    LValue Src = CGF.MakeAddrLValue(rvalue.getAggregateAddress(),
1673
0
                                    getAtomicType());
1674
0
    bool IsVolatile = rvalue.isVolatileQualified() ||
1675
0
                      LVal.isVolatileQualified();
1676
0
    CGF.EmitAggregateCopy(Dest, Src, getAtomicType(),
1677
0
                          AggValueSlot::DoesNotOverlap, IsVolatile);
1678
0
    return;
1679
0
  }
1680
1681
  // Okay, otherwise we're copying stuff.
1682
1683
  // Zero out the buffer if necessary.
1684
185
  emitMemSetZeroIfNecessary();
1685
1686
  // Drill past the padding if present.
1687
185
  LValue TempLVal = projectValue();
1688
1689
  // Okay, store the rvalue in.
1690
185
  if (rvalue.isScalar()) {
1691
167
    CGF.EmitStoreOfScalar(rvalue.getScalarVal(), TempLVal, /*init*/ true);
1692
167
  } else {
1693
18
    CGF.EmitStoreOfComplex(rvalue.getComplexVal(), TempLVal, /*init*/ true);
1694
18
  }
1695
185
}
1696
1697
1698
/// Materialize an r-value into memory for the purposes of storing it
1699
/// to an atomic type.
1700
153
Address AtomicInfo::materializeRValue(RValue rvalue) const {
1701
  // Aggregate r-values are already in memory, and EmitAtomicStore
1702
  // requires them to be values of the atomic type.
1703
153
  if (rvalue.isAggregate())
1704
12
    return rvalue.getAggregateAddress();
1705
1706
  // Otherwise, make a temporary and materialize into it.
1707
141
  LValue TempLV = CGF.MakeAddrLValue(CreateTempAlloca(), getAtomicType());
1708
141
  AtomicInfo Atomics(CGF, TempLV);
1709
141
  Atomics.emitCopyIntoMemory(rvalue);
1710
141
  return TempLV.getAddress(CGF);
1711
153
}
1712
1713
144
llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal) const {
1714
  // If we've got a scalar value of the right size, try to avoid going
1715
  // through memory.
1716
144
  if (RVal.isScalar() && 
(133
!hasPadding()133
||
!LVal.isSimple()0
)) {
1717
133
    llvm::Value *Value = RVal.getScalarVal();
1718
133
    if (isa<llvm::IntegerType>(Value->getType()))
1719
103
      return CGF.EmitToMemory(Value, ValueTy);
1720
30
    else {
1721
30
      llvm::IntegerType *InputIntTy = llvm::IntegerType::get(
1722
30
          CGF.getLLVMContext(),
1723
30
          LVal.isSimple() ? getValueSizeInBits() : 
getAtomicSizeInBits()0
);
1724
30
      if (isa<llvm::PointerType>(Value->getType()))
1725
0
        return CGF.Builder.CreatePtrToInt(Value, InputIntTy);
1726
30
      else if (llvm::BitCastInst::isBitCastable(Value->getType(), InputIntTy))
1727
12
        return CGF.Builder.CreateBitCast(Value, InputIntTy);
1728
30
    }
1729
133
  }
1730
  // Otherwise, we need to go through memory.
1731
  // Put the r-value in memory.
1732
29
  Address Addr = materializeRValue(RVal);
1733
1734
  // Cast the temporary to the atomic int type and pull a value out.
1735
29
  Addr = emitCastToAtomicIntPointer(Addr);
1736
29
  return CGF.Builder.CreateLoad(Addr);
1737
144
}
1738
1739
std::pair<llvm::Value *, llvm::Value *> AtomicInfo::EmitAtomicCompareExchangeOp(
1740
    llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
1741
268
    llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak) {
1742
  // Do the atomic store.
1743
268
  Address Addr = getAtomicAddressAsAtomicIntPointer();
1744
268
  auto *Inst = CGF.Builder.CreateAtomicCmpXchg(Addr.getPointer(),
1745
268
                                               ExpectedVal, DesiredVal,
1746
268
                                               Success, Failure);
1747
  // Other decoration.
1748
268
  Inst->setVolatile(LVal.isVolatileQualified());
1749
268
  Inst->setWeak(IsWeak);
1750
1751
  // Okay, turn that back into the original value type.
1752
268
  auto *PreviousVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/0);
1753
268
  auto *SuccessFailureVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/1);
1754
268
  return std::make_pair(PreviousVal, SuccessFailureVal);
1755
268
}
1756
1757
llvm::Value *
1758
AtomicInfo::EmitAtomicCompareExchangeLibcall(llvm::Value *ExpectedAddr,
1759
                                             llvm::Value *DesiredAddr,
1760
                                             llvm::AtomicOrdering Success,
1761
78
                                             llvm::AtomicOrdering Failure) {
1762
  // bool __atomic_compare_exchange(size_t size, void *obj, void *expected,
1763
  // void *desired, int success, int failure);
1764
78
  CallArgList Args;
1765
78
  Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
1766
78
  Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())),
1767
78
           CGF.getContext().VoidPtrTy);
1768
78
  Args.add(RValue::get(CGF.EmitCastToVoidPtr(ExpectedAddr)),
1769
78
           CGF.getContext().VoidPtrTy);
1770
78
  Args.add(RValue::get(CGF.EmitCastToVoidPtr(DesiredAddr)),
1771
78
           CGF.getContext().VoidPtrTy);
1772
78
  Args.add(RValue::get(
1773
78
               llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Success))),
1774
78
           CGF.getContext().IntTy);
1775
78
  Args.add(RValue::get(
1776
78
               llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Failure))),
1777
78
           CGF.getContext().IntTy);
1778
78
  auto SuccessFailureRVal = emitAtomicLibcall(CGF, "__atomic_compare_exchange",
1779
78
                                              CGF.getContext().BoolTy, Args);
1780
1781
78
  return SuccessFailureRVal.getScalarVal();
1782
78
}
1783
1784
std::pair<RValue, llvm::Value *> AtomicInfo::EmitAtomicCompareExchange(
1785
    RValue Expected, RValue Desired, llvm::AtomicOrdering Success,
1786
29
    llvm::AtomicOrdering Failure, bool IsWeak) {
1787
  // Check whether we should use a library call.
1788
29
  if (shouldUseLibcall()) {
1789
    // Produce a source address.
1790
17
    Address ExpectedAddr = materializeRValue(Expected);
1791
17
    Address DesiredAddr = materializeRValue(Desired);
1792
17
    auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1793
17
                                                 DesiredAddr.getPointer(),
1794
17
                                                 Success, Failure);
1795
17
    return std::make_pair(
1796
17
        convertAtomicTempToRValue(ExpectedAddr, AggValueSlot::ignored(),
1797
17
                                  SourceLocation(), /*AsValue=*/false),
1798
17
        Res);
1799
17
  }
1800
1801
  // If we've got a scalar value of the right size, try to avoid going
1802
  // through memory.
1803
12
  auto *ExpectedVal = convertRValueToInt(Expected);
1804
12
  auto *DesiredVal = convertRValueToInt(Desired);
1805
12
  auto Res = EmitAtomicCompareExchangeOp(ExpectedVal, DesiredVal, Success,
1806
12
                                         Failure, IsWeak);
1807
12
  return std::make_pair(
1808
12
      ConvertIntToValueOrAtomic(Res.first, AggValueSlot::ignored(),
1809
12
                                SourceLocation(), /*AsValue=*/false),
1810
12
      Res.second);
1811
29
}
1812
1813
static void
1814
EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, RValue OldRVal,
1815
                      const llvm::function_ref<RValue(RValue)> &UpdateOp,
1816
293
                      Address DesiredAddr) {
1817
293
  RValue UpRVal;
1818
293
  LValue AtomicLVal = Atomics.getAtomicLValue();
1819
293
  LValue DesiredLVal;
1820
293
  if (AtomicLVal.isSimple()) {
1821
221
    UpRVal = OldRVal;
1822
221
    DesiredLVal = CGF.MakeAddrLValue(DesiredAddr, AtomicLVal.getType());
1823
221
  } else {
1824
    // Build new lvalue for temp address.
1825
72
    Address Ptr = Atomics.materializeRValue(OldRVal);
1826
72
    LValue UpdateLVal;
1827
72
    if (AtomicLVal.isBitField()) {
1828
60
      UpdateLVal =
1829
60
          LValue::MakeBitfield(Ptr, AtomicLVal.getBitFieldInfo(),
1830
60
                               AtomicLVal.getType(),
1831
60
                               AtomicLVal.getBaseInfo(),
1832
60
                               AtomicLVal.getTBAAInfo());
1833
60
      DesiredLVal =
1834
60
          LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(),
1835
60
                               AtomicLVal.getType(), AtomicLVal.getBaseInfo(),
1836
60
                               AtomicLVal.getTBAAInfo());
1837
60
    } else 
if (12
AtomicLVal.isVectorElt()12
) {
1838
6
      UpdateLVal = LValue::MakeVectorElt(Ptr, AtomicLVal.getVectorIdx(),
1839
6
                                         AtomicLVal.getType(),
1840
6
                                         AtomicLVal.getBaseInfo(),
1841
6
                                         AtomicLVal.getTBAAInfo());
1842
6
      DesiredLVal = LValue::MakeVectorElt(
1843
6
          DesiredAddr, AtomicLVal.getVectorIdx(), AtomicLVal.getType(),
1844
6
          AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo());
1845
6
    } else {
1846
6
      assert(AtomicLVal.isExtVectorElt());
1847
0
      UpdateLVal = LValue::MakeExtVectorElt(Ptr, AtomicLVal.getExtVectorElts(),
1848
6
                                            AtomicLVal.getType(),
1849
6
                                            AtomicLVal.getBaseInfo(),
1850
6
                                            AtomicLVal.getTBAAInfo());
1851
6
      DesiredLVal = LValue::MakeExtVectorElt(
1852
6
          DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(),
1853
6
          AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo());
1854
6
    }
1855
0
    UpRVal = CGF.EmitLoadOfLValue(UpdateLVal, SourceLocation());
1856
72
  }
1857
  // Store new value in the corresponding memory area.
1858
0
  RValue NewRVal = UpdateOp(UpRVal);
1859
293
  if (NewRVal.isScalar()) {
1860
261
    CGF.EmitStoreThroughLValue(NewRVal, DesiredLVal);
1861
261
  } else {
1862
32
    assert(NewRVal.isComplex());
1863
0
    CGF.EmitStoreOfComplex(NewRVal.getComplexVal(), DesiredLVal,
1864
32
                           /*isInit=*/false);
1865
32
  }
1866
293
}
1867
1868
void AtomicInfo::EmitAtomicUpdateLibcall(
1869
    llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1870
57
    bool IsVolatile) {
1871
57
  auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1872
1873
57
  Address ExpectedAddr = CreateTempAlloca();
1874
1875
57
  EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile);
1876
57
  auto *ContBB = CGF.createBasicBlock("atomic_cont");
1877
57
  auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1878
57
  CGF.EmitBlock(ContBB);
1879
57
  Address DesiredAddr = CreateTempAlloca();
1880
57
  if ((LVal.isBitField() && 
BFI.Size != ValueSizeInBits12
) ||
1881
57
      
requiresMemSetZero(getAtomicAddress().getElementType())45
) {
1882
12
    auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr);
1883
12
    CGF.Builder.CreateStore(OldVal, DesiredAddr);
1884
12
  }
1885
57
  auto OldRVal = convertAtomicTempToRValue(ExpectedAddr,
1886
57
                                           AggValueSlot::ignored(),
1887
57
                                           SourceLocation(), /*AsValue=*/false);
1888
57
  EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, DesiredAddr);
1889
57
  auto *Res =
1890
57
      EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1891
57
                                       DesiredAddr.getPointer(),
1892
57
                                       AO, Failure);
1893
57
  CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
1894
57
  CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1895
57
}
1896
1897
void AtomicInfo::EmitAtomicUpdateOp(
1898
    llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1899
236
    bool IsVolatile) {
1900
236
  auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1901
1902
  // Do the atomic load.
1903
236
  auto *OldVal = EmitAtomicLoadOp(Failure, IsVolatile);
1904
  // For non-simple lvalues perform compare-and-swap procedure.
1905
236
  auto *ContBB = CGF.createBasicBlock("atomic_cont");
1906
236
  auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1907
236
  auto *CurBB = CGF.Builder.GetInsertBlock();
1908
236
  CGF.EmitBlock(ContBB);
1909
236
  llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(),
1910
236
                                             /*NumReservedValues=*/2);
1911
236
  PHI->addIncoming(OldVal, CurBB);
1912
236
  Address NewAtomicAddr = CreateTempAlloca();
1913
236
  Address NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr);
1914
236
  if ((LVal.isBitField() && 
BFI.Size != ValueSizeInBits48
) ||
1915
236
      
requiresMemSetZero(getAtomicAddress().getElementType())188
) {
1916
72
    CGF.Builder.CreateStore(PHI, NewAtomicIntAddr);
1917
72
  }
1918
236
  auto OldRVal = ConvertIntToValueOrAtomic(PHI, AggValueSlot::ignored(),
1919
236
                                           SourceLocation(), /*AsValue=*/false);
1920
236
  EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, NewAtomicAddr);
1921
236
  auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr);
1922
  // Try to write new value using cmpxchg operation.
1923
236
  auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure);
1924
236
  PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock());
1925
236
  CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB);
1926
236
  CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1927
236
}
1928
1929
static void EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics,
1930
24
                                  RValue UpdateRVal, Address DesiredAddr) {
1931
24
  LValue AtomicLVal = Atomics.getAtomicLValue();
1932
24
  LValue DesiredLVal;
1933
  // Build new lvalue for temp address.
1934
24
  if (AtomicLVal.isBitField()) {
1935
20
    DesiredLVal =
1936
20
        LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(),
1937
20
                             AtomicLVal.getType(), AtomicLVal.getBaseInfo(),
1938
20
                             AtomicLVal.getTBAAInfo());
1939
20
  } else 
if (4
AtomicLVal.isVectorElt()4
) {
1940
2
    DesiredLVal =
1941
2
        LValue::MakeVectorElt(DesiredAddr, AtomicLVal.getVectorIdx(),
1942
2
                              AtomicLVal.getType(), AtomicLVal.getBaseInfo(),
1943
2
                              AtomicLVal.getTBAAInfo());
1944
2
  } else {
1945
2
    assert(AtomicLVal.isExtVectorElt());
1946
0
    DesiredLVal = LValue::MakeExtVectorElt(
1947
2
        DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(),
1948
2
        AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo());
1949
2
  }
1950
  // Store new value in the corresponding memory area.
1951
0
  assert(UpdateRVal.isScalar());
1952
0
  CGF.EmitStoreThroughLValue(UpdateRVal, DesiredLVal);
1953
24
}
1954
1955
void AtomicInfo::EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
1956
4
                                         RValue UpdateRVal, bool IsVolatile) {
1957
4
  auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1958
1959
4
  Address ExpectedAddr = CreateTempAlloca();
1960
1961
4
  EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile);
1962
4
  auto *ContBB = CGF.createBasicBlock("atomic_cont");
1963
4
  auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1964
4
  CGF.EmitBlock(ContBB);
1965
4
  Address DesiredAddr = CreateTempAlloca();
1966
4
  if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1967
4
      
requiresMemSetZero(getAtomicAddress().getElementType())0
) {
1968
4
    auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr);
1969
4
    CGF.Builder.CreateStore(OldVal, DesiredAddr);
1970
4
  }
1971
4
  EmitAtomicUpdateValue(CGF, *this, UpdateRVal, DesiredAddr);
1972
4
  auto *Res =
1973
4
      EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1974
4
                                       DesiredAddr.getPointer(),
1975
4
                                       AO, Failure);
1976
4
  CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
1977
4
  CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1978
4
}
1979
1980
void AtomicInfo::EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRVal,
1981
20
                                    bool IsVolatile) {
1982
20
  auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1983
1984
  // Do the atomic load.
1985
20
  auto *OldVal = EmitAtomicLoadOp(Failure, IsVolatile);
1986
  // For non-simple lvalues perform compare-and-swap procedure.
1987
20
  auto *ContBB = CGF.createBasicBlock("atomic_cont");
1988
20
  auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1989
20
  auto *CurBB = CGF.Builder.GetInsertBlock();
1990
20
  CGF.EmitBlock(ContBB);
1991
20
  llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(),
1992
20
                                             /*NumReservedValues=*/2);
1993
20
  PHI->addIncoming(OldVal, CurBB);
1994
20
  Address NewAtomicAddr = CreateTempAlloca();
1995
20
  Address NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr);
1996
20
  if ((LVal.isBitField() && 
BFI.Size != ValueSizeInBits16
) ||
1997
20
      
requiresMemSetZero(getAtomicAddress().getElementType())4
) {
1998
20
    CGF.Builder.CreateStore(PHI, NewAtomicIntAddr);
1999
20
  }
2000
20
  EmitAtomicUpdateValue(CGF, *this, UpdateRVal, NewAtomicAddr);
2001
20
  auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr);
2002
  // Try to write new value using cmpxchg operation.
2003
20
  auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure);
2004
20
  PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock());
2005
20
  CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB);
2006
20
  CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
2007
20
}
2008
2009
void AtomicInfo::EmitAtomicUpdate(
2010
    llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
2011
293
    bool IsVolatile) {
2012
293
  if (shouldUseLibcall()) {
2013
57
    EmitAtomicUpdateLibcall(AO, UpdateOp, IsVolatile);
2014
236
  } else {
2015
236
    EmitAtomicUpdateOp(AO, UpdateOp, IsVolatile);
2016
236
  }
2017
293
}
2018
2019
void AtomicInfo::EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
2020
24
                                  bool IsVolatile) {
2021
24
  if (shouldUseLibcall()) {
2022
4
    EmitAtomicUpdateLibcall(AO, UpdateRVal, IsVolatile);
2023
20
  } else {
2024
20
    EmitAtomicUpdateOp(AO, UpdateRVal, IsVolatile);
2025
20
  }
2026
24
}
2027
2028
void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue lvalue,
2029
83
                                      bool isInit) {
2030
83
  bool IsVolatile = lvalue.isVolatileQualified();
2031
83
  llvm::AtomicOrdering AO;
2032
83
  if (lvalue.getType()->isAtomicType()) {
2033
76
    AO = llvm::AtomicOrdering::SequentiallyConsistent;
2034
76
  } else {
2035
7
    AO = llvm::AtomicOrdering::Release;
2036
7
    IsVolatile = true;
2037
7
  }
2038
83
  return EmitAtomicStore(rvalue, lvalue, AO, IsVolatile, isInit);
2039
83
}
2040
2041
/// Emit a store to an l-value of atomic type.
2042
///
2043
/// Note that the r-value is expected to be an r-value *of the atomic
2044
/// type*; this means that for aggregate r-values, it should include
2045
/// storage for any padding that was necessary.
2046
void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest,
2047
                                      llvm::AtomicOrdering AO, bool IsVolatile,
2048
195
                                      bool isInit) {
2049
  // If this is an aggregate r-value, it should agree in type except
2050
  // maybe for address-space qualification.
2051
195
  assert(!rvalue.isAggregate() ||
2052
195
         rvalue.getAggregateAddress().getElementType() ==
2053
195
             dest.getAddress(*this).getElementType());
2054
2055
0
  AtomicInfo atomics(*this, dest);
2056
195
  LValue LVal = atomics.getAtomicLValue();
2057
2058
  // If this is an initialization, just put the value there normally.
2059
195
  if (LVal.isSimple()) {
2060
171
    if (isInit) {
2061
33
      atomics.emitCopyIntoMemory(rvalue);
2062
33
      return;
2063
33
    }
2064
2065
    // Check whether we should use a library call.
2066
138
    if (atomics.shouldUseLibcall()) {
2067
      // Produce a source address.
2068
18
      Address srcAddr = atomics.materializeRValue(rvalue);
2069
2070
      // void __atomic_store(size_t size, void *mem, void *val, int order)
2071
18
      CallArgList args;
2072
18
      args.add(RValue::get(atomics.getAtomicSizeValue()),
2073
18
               getContext().getSizeType());
2074
18
      args.add(RValue::get(EmitCastToVoidPtr(atomics.getAtomicPointer())),
2075
18
               getContext().VoidPtrTy);
2076
18
      args.add(RValue::get(EmitCastToVoidPtr(srcAddr.getPointer())),
2077
18
               getContext().VoidPtrTy);
2078
18
      args.add(
2079
18
          RValue::get(llvm::ConstantInt::get(IntTy, (int)llvm::toCABI(AO))),
2080
18
          getContext().IntTy);
2081
18
      emitAtomicLibcall(*this, "__atomic_store", getContext().VoidTy, args);
2082
18
      return;
2083
18
    }
2084
2085
    // Okay, we're doing this natively.
2086
120
    llvm::Value *intValue = atomics.convertRValueToInt(rvalue);
2087
2088
    // Do the atomic store.
2089
120
    Address addr =
2090
120
        atomics.emitCastToAtomicIntPointer(atomics.getAtomicAddress());
2091
120
    intValue = Builder.CreateIntCast(
2092
120
        intValue, addr.getElementType(), /*isSigned=*/false);
2093
120
    llvm::StoreInst *store = Builder.CreateStore(intValue, addr);
2094
2095
120
    if (AO == llvm::AtomicOrdering::Acquire)
2096
0
      AO = llvm::AtomicOrdering::Monotonic;
2097
120
    else if (AO == llvm::AtomicOrdering::AcquireRelease)
2098
0
      AO = llvm::AtomicOrdering::Release;
2099
    // Initializations don't need to be atomic.
2100
120
    if (!isInit)
2101
120
      store->setAtomic(AO);
2102
2103
    // Other decoration.
2104
120
    if (IsVolatile)
2105
10
      store->setVolatile(true);
2106
120
    CGM.DecorateInstructionWithTBAA(store, dest.getTBAAInfo());
2107
120
    return;
2108
138
  }
2109
2110
  // Emit simple atomic update operation.
2111
24
  atomics.EmitAtomicUpdate(AO, rvalue, IsVolatile);
2112
24
}
2113
2114
/// Emit a compare-and-exchange op for atomic type.
2115
///
2116
std::pair<RValue, llvm::Value *> CodeGenFunction::EmitAtomicCompareExchange(
2117
    LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc,
2118
    llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak,
2119
29
    AggValueSlot Slot) {
2120
  // If this is an aggregate r-value, it should agree in type except
2121
  // maybe for address-space qualification.
2122
29
  assert(!Expected.isAggregate() ||
2123
29
         Expected.getAggregateAddress().getElementType() ==
2124
29
             Obj.getAddress(*this).getElementType());
2125
0
  assert(!Desired.isAggregate() ||
2126
29
         Desired.getAggregateAddress().getElementType() ==
2127
29
             Obj.getAddress(*this).getElementType());
2128
0
  AtomicInfo Atomics(*this, Obj);
2129
2130
29
  return Atomics.EmitAtomicCompareExchange(Expected, Desired, Success, Failure,
2131
29
                                           IsWeak);
2132
29
}
2133
2134
void CodeGenFunction::EmitAtomicUpdate(
2135
    LValue LVal, llvm::AtomicOrdering AO,
2136
293
    const llvm::function_ref<RValue(RValue)> &UpdateOp, bool IsVolatile) {
2137
293
  AtomicInfo Atomics(*this, LVal);
2138
293
  Atomics.EmitAtomicUpdate(AO, UpdateOp, IsVolatile);
2139
293
}
2140
2141
20
void CodeGenFunction::EmitAtomicInit(Expr *init, LValue dest) {
2142
20
  AtomicInfo atomics(*this, dest);
2143
2144
20
  switch (atomics.getEvaluationKind()) {
2145
9
  case TEK_Scalar: {
2146
9
    llvm::Value *value = EmitScalarExpr(init);
2147
9
    atomics.emitCopyIntoMemory(RValue::get(value));
2148
9
    return;
2149
0
  }
2150
2151
2
  case TEK_Complex: {
2152
2
    ComplexPairTy value = EmitComplexExpr(init);
2153
2
    atomics.emitCopyIntoMemory(RValue::getComplex(value));
2154
2
    return;
2155
0
  }
2156
2157
9
  case TEK_Aggregate: {
2158
    // Fix up the destination if the initializer isn't an expression
2159
    // of atomic type.
2160
9
    bool Zeroed = false;
2161
9
    if (!init->getType()->isAtomicType()) {
2162
5
      Zeroed = atomics.emitMemSetZeroIfNecessary();
2163
5
      dest = atomics.projectValue();
2164
5
    }
2165
2166
    // Evaluate the expression directly into the destination.
2167
9
    AggValueSlot slot = AggValueSlot::forLValue(
2168
9
        dest, *this, AggValueSlot::IsNotDestructed,
2169
9
        AggValueSlot::DoesNotNeedGCBarriers, AggValueSlot::IsNotAliased,
2170
9
        AggValueSlot::DoesNotOverlap,
2171
9
        Zeroed ? 
AggValueSlot::IsZeroed2
:
AggValueSlot::IsNotZeroed7
);
2172
2173
9
    EmitAggExpr(init, slot);
2174
9
    return;
2175
0
  }
2176
20
  }
2177
0
  llvm_unreachable("bad evaluation kind");
2178
0
}