Coverage Report

Created: 2022-01-18 06:27

/Users/buildslave/jenkins/workspace/coverage/llvm-project/clang/lib/CodeGen/CGAtomic.cpp
Line
Count
Source (jump to first uncovered line)
1
//===--- CGAtomic.cpp - Emit LLVM IR for atomic operations ----------------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// This file contains the code for emitting atomic operations.
10
//
11
//===----------------------------------------------------------------------===//
12
13
#include "CGCall.h"
14
#include "CGRecordLayout.h"
15
#include "CodeGenFunction.h"
16
#include "CodeGenModule.h"
17
#include "TargetInfo.h"
18
#include "clang/AST/ASTContext.h"
19
#include "clang/CodeGen/CGFunctionInfo.h"
20
#include "clang/Frontend/FrontendDiagnostic.h"
21
#include "llvm/ADT/DenseMap.h"
22
#include "llvm/IR/DataLayout.h"
23
#include "llvm/IR/Intrinsics.h"
24
#include "llvm/IR/Operator.h"
25
26
using namespace clang;
27
using namespace CodeGen;
28
29
namespace {
30
  class AtomicInfo {
31
    CodeGenFunction &CGF;
32
    QualType AtomicTy;
33
    QualType ValueTy;
34
    uint64_t AtomicSizeInBits;
35
    uint64_t ValueSizeInBits;
36
    CharUnits AtomicAlign;
37
    CharUnits ValueAlign;
38
    TypeEvaluationKind EvaluationKind;
39
    bool UseLibcall;
40
    LValue LVal;
41
    CGBitFieldInfo BFI;
42
  public:
43
    AtomicInfo(CodeGenFunction &CGF, LValue &lvalue)
44
        : CGF(CGF), AtomicSizeInBits(0), ValueSizeInBits(0),
45
2.09k
          EvaluationKind(TEK_Scalar), UseLibcall(true) {
46
2.09k
      assert(!lvalue.isGlobalReg());
47
0
      ASTContext &C = CGF.getContext();
48
2.09k
      if (lvalue.isSimple()) {
49
1.97k
        AtomicTy = lvalue.getType();
50
1.97k
        if (auto *ATy = AtomicTy->getAs<AtomicType>())
51
862
          ValueTy = ATy->getValueType();
52
1.10k
        else
53
1.10k
          ValueTy = AtomicTy;
54
1.97k
        EvaluationKind = CGF.getEvaluationKind(ValueTy);
55
56
1.97k
        uint64_t ValueAlignInBits;
57
1.97k
        uint64_t AtomicAlignInBits;
58
1.97k
        TypeInfo ValueTI = C.getTypeInfo(ValueTy);
59
1.97k
        ValueSizeInBits = ValueTI.Width;
60
1.97k
        ValueAlignInBits = ValueTI.Align;
61
62
1.97k
        TypeInfo AtomicTI = C.getTypeInfo(AtomicTy);
63
1.97k
        AtomicSizeInBits = AtomicTI.Width;
64
1.97k
        AtomicAlignInBits = AtomicTI.Align;
65
66
1.97k
        assert(ValueSizeInBits <= AtomicSizeInBits);
67
0
        assert(ValueAlignInBits <= AtomicAlignInBits);
68
69
0
        AtomicAlign = C.toCharUnitsFromBits(AtomicAlignInBits);
70
1.97k
        ValueAlign = C.toCharUnitsFromBits(ValueAlignInBits);
71
1.97k
        if (lvalue.getAlignment().isZero())
72
0
          lvalue.setAlignment(AtomicAlign);
73
74
1.97k
        LVal = lvalue;
75
1.97k
      } else 
if (120
lvalue.isBitField()120
) {
76
100
        ValueTy = lvalue.getType();
77
100
        ValueSizeInBits = C.getTypeSize(ValueTy);
78
100
        auto &OrigBFI = lvalue.getBitFieldInfo();
79
100
        auto Offset = OrigBFI.Offset % C.toBits(lvalue.getAlignment());
80
100
        AtomicSizeInBits = C.toBits(
81
100
            C.toCharUnitsFromBits(Offset + OrigBFI.Size + C.getCharWidth() - 1)
82
100
                .alignTo(lvalue.getAlignment()));
83
100
        auto VoidPtrAddr = CGF.EmitCastToVoidPtr(lvalue.getBitFieldPointer());
84
100
        auto OffsetInChars =
85
100
            (C.toCharUnitsFromBits(OrigBFI.Offset) / lvalue.getAlignment()) *
86
100
            lvalue.getAlignment();
87
100
        VoidPtrAddr = CGF.Builder.CreateConstGEP1_64(
88
100
            CGF.Int8Ty, VoidPtrAddr, OffsetInChars.getQuantity());
89
100
        auto Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
90
100
            VoidPtrAddr,
91
100
            CGF.Builder.getIntNTy(AtomicSizeInBits)->getPointerTo(),
92
100
            "atomic_bitfield_base");
93
100
        BFI = OrigBFI;
94
100
        BFI.Offset = Offset;
95
100
        BFI.StorageSize = AtomicSizeInBits;
96
100
        BFI.StorageOffset += OffsetInChars;
97
100
        LVal = LValue::MakeBitfield(Address(Addr, lvalue.getAlignment()),
98
100
                                    BFI, lvalue.getType(), lvalue.getBaseInfo(),
99
100
                                    lvalue.getTBAAInfo());
100
100
        AtomicTy = C.getIntTypeForBitwidth(AtomicSizeInBits, OrigBFI.IsSigned);
101
100
        if (AtomicTy.isNull()) {
102
10
          llvm::APInt Size(
103
10
              /*numBits=*/32,
104
10
              C.toCharUnitsFromBits(AtomicSizeInBits).getQuantity());
105
10
          AtomicTy =
106
10
              C.getConstantArrayType(C.CharTy, Size, nullptr, ArrayType::Normal,
107
10
                                     /*IndexTypeQuals=*/0);
108
10
        }
109
100
        AtomicAlign = ValueAlign = lvalue.getAlignment();
110
100
      } else 
if (20
lvalue.isVectorElt()20
) {
111
10
        ValueTy = lvalue.getType()->castAs<VectorType>()->getElementType();
112
10
        ValueSizeInBits = C.getTypeSize(ValueTy);
113
10
        AtomicTy = lvalue.getType();
114
10
        AtomicSizeInBits = C.getTypeSize(AtomicTy);
115
10
        AtomicAlign = ValueAlign = lvalue.getAlignment();
116
10
        LVal = lvalue;
117
10
      } else {
118
10
        assert(lvalue.isExtVectorElt());
119
0
        ValueTy = lvalue.getType();
120
10
        ValueSizeInBits = C.getTypeSize(ValueTy);
121
10
        AtomicTy = ValueTy = CGF.getContext().getExtVectorType(
122
10
            lvalue.getType(), cast<llvm::FixedVectorType>(
123
10
                                  lvalue.getExtVectorAddress().getElementType())
124
10
                                  ->getNumElements());
125
10
        AtomicSizeInBits = C.getTypeSize(AtomicTy);
126
10
        AtomicAlign = ValueAlign = lvalue.getAlignment();
127
10
        LVal = lvalue;
128
10
      }
129
0
      UseLibcall = !C.getTargetInfo().hasBuiltinAtomic(
130
2.09k
          AtomicSizeInBits, C.toBits(lvalue.getAlignment()));
131
2.09k
    }
132
133
141
    QualType getAtomicType() const { return AtomicTy; }
134
321
    QualType getValueType() const { return ValueTy; }
135
3.11k
    CharUnits getAtomicAlignment() const { return AtomicAlign; }
136
236
    uint64_t getAtomicSizeInBits() const { return AtomicSizeInBits; }
137
266
    uint64_t getValueSizeInBits() const { return ValueSizeInBits; }
138
1.05k
    TypeEvaluationKind getEvaluationKind() const { return EvaluationKind; }
139
739
    bool shouldUseLibcall() const { return UseLibcall; }
140
539
    const LValue &getAtomicLValue() const { return LVal; }
141
1.94k
    llvm::Value *getAtomicPointer() const {
142
1.94k
      if (LVal.isSimple())
143
1.47k
        return LVal.getPointer(CGF);
144
468
      else if (LVal.isBitField())
145
404
        return LVal.getBitFieldPointer();
146
64
      else if (LVal.isVectorElt())
147
32
        return LVal.getVectorPointer();
148
32
      assert(LVal.isExtVectorElt());
149
0
      return LVal.getExtVectorPointer();
150
1.94k
    }
151
1.74k
    Address getAtomicAddress() const {
152
1.74k
      return Address(getAtomicPointer(), getAtomicAlignment());
153
1.74k
    }
154
155
717
    Address getAtomicAddressAsAtomicIntPointer() const {
156
717
      return emitCastToAtomicIntPointer(getAtomicAddress());
157
717
    }
158
159
    /// Is the atomic size larger than the underlying value type?
160
    ///
161
    /// Note that the absence of padding does not mean that atomic
162
    /// objects are completely interchangeable with non-atomic
163
    /// objects: we might have promoted the alignment of a type
164
    /// without making it bigger.
165
1.24k
    bool hasPadding() const {
166
1.24k
      return (ValueSizeInBits != AtomicSizeInBits);
167
1.24k
    }
168
169
    bool emitMemSetZeroIfNecessary() const;
170
171
201
    llvm::Value *getAtomicSizeValue() const {
172
201
      CharUnits size = CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits);
173
201
      return CGF.CGM.getSize(size);
174
201
    }
175
176
    /// Cast the given pointer to an integer pointer suitable for atomic
177
    /// operations if the source.
178
    Address emitCastToAtomicIntPointer(Address Addr) const;
179
180
    /// If Addr is compatible with the iN that will be used for an atomic
181
    /// operation, bitcast it. Otherwise, create a temporary that is suitable
182
    /// and copy the value across.
183
    Address convertToAtomicIntPointer(Address Addr) const;
184
185
    /// Turn an atomic-layout object into an r-value.
186
    RValue convertAtomicTempToRValue(Address addr, AggValueSlot resultSlot,
187
                                     SourceLocation loc, bool AsValue) const;
188
189
    /// Converts a rvalue to integer value.
190
    llvm::Value *convertRValueToInt(RValue RVal) const;
191
192
    RValue ConvertIntToValueOrAtomic(llvm::Value *IntVal,
193
                                     AggValueSlot ResultSlot,
194
                                     SourceLocation Loc, bool AsValue) const;
195
196
    /// Copy an atomic r-value into atomic-layout memory.
197
    void emitCopyIntoMemory(RValue rvalue) const;
198
199
    /// Project an l-value down to the value field.
200
190
    LValue projectValue() const {
201
190
      assert(LVal.isSimple());
202
0
      Address addr = getAtomicAddress();
203
190
      if (hasPadding())
204
2
        addr = CGF.Builder.CreateStructGEP(addr, 0);
205
206
190
      return LValue::MakeAddr(addr, getValueType(), CGF.getContext(),
207
190
                              LVal.getBaseInfo(), LVal.getTBAAInfo());
208
190
    }
209
210
    /// Emits atomic load.
211
    /// \returns Loaded value.
212
    RValue EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
213
                          bool AsValue, llvm::AtomicOrdering AO,
214
                          bool IsVolatile);
215
216
    /// Emits atomic compare-and-exchange sequence.
217
    /// \param Expected Expected value.
218
    /// \param Desired Desired value.
219
    /// \param Success Atomic ordering for success operation.
220
    /// \param Failure Atomic ordering for failed operation.
221
    /// \param IsWeak true if atomic operation is weak, false otherwise.
222
    /// \returns Pair of values: previous value from storage (value type) and
223
    /// boolean flag (i1 type) with true if success and false otherwise.
224
    std::pair<RValue, llvm::Value *>
225
    EmitAtomicCompareExchange(RValue Expected, RValue Desired,
226
                              llvm::AtomicOrdering Success =
227
                                  llvm::AtomicOrdering::SequentiallyConsistent,
228
                              llvm::AtomicOrdering Failure =
229
                                  llvm::AtomicOrdering::SequentiallyConsistent,
230
                              bool IsWeak = false);
231
232
    /// Emits atomic update.
233
    /// \param AO Atomic ordering.
234
    /// \param UpdateOp Update operation for the current lvalue.
235
    void EmitAtomicUpdate(llvm::AtomicOrdering AO,
236
                          const llvm::function_ref<RValue(RValue)> &UpdateOp,
237
                          bool IsVolatile);
238
    /// Emits atomic update.
239
    /// \param AO Atomic ordering.
240
    void EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
241
                          bool IsVolatile);
242
243
    /// Materialize an atomic r-value in atomic-layout memory.
244
    Address materializeRValue(RValue rvalue) const;
245
246
    /// Creates temp alloca for intermediate operations on atomic value.
247
    Address CreateTempAlloca() const;
248
  private:
249
    bool requiresMemSetZero(llvm::Type *type) const;
250
251
252
    /// Emits atomic load as a libcall.
253
    void EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
254
                               llvm::AtomicOrdering AO, bool IsVolatile);
255
    /// Emits atomic load as LLVM instruction.
256
    llvm::Value *EmitAtomicLoadOp(llvm::AtomicOrdering AO, bool IsVolatile);
257
    /// Emits atomic compare-and-exchange op as a libcall.
258
    llvm::Value *EmitAtomicCompareExchangeLibcall(
259
        llvm::Value *ExpectedAddr, llvm::Value *DesiredAddr,
260
        llvm::AtomicOrdering Success =
261
            llvm::AtomicOrdering::SequentiallyConsistent,
262
        llvm::AtomicOrdering Failure =
263
            llvm::AtomicOrdering::SequentiallyConsistent);
264
    /// Emits atomic compare-and-exchange op as LLVM instruction.
265
    std::pair<llvm::Value *, llvm::Value *> EmitAtomicCompareExchangeOp(
266
        llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
267
        llvm::AtomicOrdering Success =
268
            llvm::AtomicOrdering::SequentiallyConsistent,
269
        llvm::AtomicOrdering Failure =
270
            llvm::AtomicOrdering::SequentiallyConsistent,
271
        bool IsWeak = false);
272
    /// Emit atomic update as libcalls.
273
    void
274
    EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
275
                            const llvm::function_ref<RValue(RValue)> &UpdateOp,
276
                            bool IsVolatile);
277
    /// Emit atomic update as LLVM instructions.
278
    void EmitAtomicUpdateOp(llvm::AtomicOrdering AO,
279
                            const llvm::function_ref<RValue(RValue)> &UpdateOp,
280
                            bool IsVolatile);
281
    /// Emit atomic update as libcalls.
282
    void EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, RValue UpdateRVal,
283
                                 bool IsVolatile);
284
    /// Emit atomic update as LLVM instructions.
285
    void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRal,
286
                            bool IsVolatile);
287
  };
288
}
289
290
1.36k
Address AtomicInfo::CreateTempAlloca() const {
291
1.36k
  Address TempAlloca = CGF.CreateMemTemp(
292
1.36k
      (LVal.isBitField() && 
ValueSizeInBits > AtomicSizeInBits176
) ?
ValueTy72
293
1.36k
                                                                : 
AtomicTy1.29k
,
294
1.36k
      getAtomicAlignment(),
295
1.36k
      "atomic-temp");
296
  // Cast to pointer to value type for bitfields.
297
1.36k
  if (LVal.isBitField())
298
176
    return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
299
176
        TempAlloca, getAtomicAddress().getType());
300
1.19k
  return TempAlloca;
301
1.36k
}
302
303
static RValue emitAtomicLibcall(CodeGenFunction &CGF,
304
                                StringRef fnName,
305
                                QualType resultType,
306
384
                                CallArgList &args) {
307
384
  const CGFunctionInfo &fnInfo =
308
384
    CGF.CGM.getTypes().arrangeBuiltinFunctionCall(resultType, args);
309
384
  llvm::FunctionType *fnTy = CGF.CGM.getTypes().GetFunctionType(fnInfo);
310
384
  llvm::AttrBuilder fnAttrB(CGF.getLLVMContext());
311
384
  fnAttrB.addAttribute(llvm::Attribute::NoUnwind);
312
384
  fnAttrB.addAttribute(llvm::Attribute::WillReturn);
313
384
  llvm::AttributeList fnAttrs = llvm::AttributeList::get(
314
384
      CGF.getLLVMContext(), llvm::AttributeList::FunctionIndex, fnAttrB);
315
316
384
  llvm::FunctionCallee fn =
317
384
      CGF.CGM.CreateRuntimeFunction(fnTy, fnName, fnAttrs);
318
384
  auto callee = CGCallee::forDirect(fn);
319
384
  return CGF.EmitCall(fnInfo, callee, ReturnValueSlot(), args);
320
384
}
321
322
/// Does a store of the given IR type modify the full expected width?
323
static bool isFullSizeType(CodeGenModule &CGM, llvm::Type *type,
324
427
                           uint64_t expectedSize) {
325
427
  return (CGM.getDataLayout().getTypeStoreSize(type) * 8 == expectedSize);
326
427
}
327
328
/// Does the atomic type require memsetting to zero before initialization?
329
///
330
/// The IR type is provided as a way of making certain queries faster.
331
454
bool AtomicInfo::requiresMemSetZero(llvm::Type *type) const {
332
  // If the atomic type has size padding, we definitely need a memset.
333
454
  if (hasPadding()) 
return true18
;
334
335
  // Otherwise, do some simple heuristics to try to avoid it:
336
436
  switch (getEvaluationKind()) {
337
  // For scalars and complexes, check whether the store size of the
338
  // type uses the full size.
339
377
  case TEK_Scalar:
340
377
    return !isFullSizeType(CGF.CGM, type, AtomicSizeInBits);
341
50
  case TEK_Complex:
342
50
    return !isFullSizeType(CGF.CGM, type->getStructElementType(0),
343
50
                           AtomicSizeInBits / 2);
344
345
  // Padding in structs has an undefined bit pattern.  User beware.
346
9
  case TEK_Aggregate:
347
9
    return false;
348
436
  }
349
0
  llvm_unreachable("bad evaluation kind");
350
0
}
351
352
190
bool AtomicInfo::emitMemSetZeroIfNecessary() const {
353
190
  assert(LVal.isSimple());
354
0
  llvm::Value *addr = LVal.getPointer(CGF);
355
190
  if (!requiresMemSetZero(addr->getType()->getPointerElementType()))
356
156
    return false;
357
358
34
  CGF.Builder.CreateMemSet(
359
34
      addr, llvm::ConstantInt::get(CGF.Int8Ty, 0),
360
34
      CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits).getQuantity(),
361
34
      LVal.getAlignment().getAsAlign());
362
34
  return true;
363
190
}
364
365
static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak,
366
                              Address Dest, Address Ptr,
367
                              Address Val1, Address Val2,
368
                              uint64_t Size,
369
                              llvm::AtomicOrdering SuccessOrder,
370
                              llvm::AtomicOrdering FailureOrder,
371
188
                              llvm::SyncScope::ID Scope) {
372
  // Note that cmpxchg doesn't support weak cmpxchg, at least at the moment.
373
188
  llvm::Value *Expected = CGF.Builder.CreateLoad(Val1);
374
188
  llvm::Value *Desired = CGF.Builder.CreateLoad(Val2);
375
376
188
  llvm::AtomicCmpXchgInst *Pair = CGF.Builder.CreateAtomicCmpXchg(
377
188
      Ptr.getPointer(), Expected, Desired, SuccessOrder, FailureOrder,
378
188
      Scope);
379
188
  Pair->setVolatile(E->isVolatile());
380
188
  Pair->setWeak(IsWeak);
381
382
  // Cmp holds the result of the compare-exchange operation: true on success,
383
  // false on failure.
384
188
  llvm::Value *Old = CGF.Builder.CreateExtractValue(Pair, 0);
385
188
  llvm::Value *Cmp = CGF.Builder.CreateExtractValue(Pair, 1);
386
387
  // This basic block is used to hold the store instruction if the operation
388
  // failed.
389
188
  llvm::BasicBlock *StoreExpectedBB =
390
188
      CGF.createBasicBlock("cmpxchg.store_expected", CGF.CurFn);
391
392
  // This basic block is the exit point of the operation, we should end up
393
  // here regardless of whether or not the operation succeeded.
394
188
  llvm::BasicBlock *ContinueBB =
395
188
      CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
396
397
  // Update Expected if Expected isn't equal to Old, otherwise branch to the
398
  // exit point.
399
188
  CGF.Builder.CreateCondBr(Cmp, ContinueBB, StoreExpectedBB);
400
401
188
  CGF.Builder.SetInsertPoint(StoreExpectedBB);
402
  // Update the memory at Expected with Old's value.
403
188
  CGF.Builder.CreateStore(Old, Val1);
404
  // Finally, branch to the exit point.
405
188
  CGF.Builder.CreateBr(ContinueBB);
406
407
188
  CGF.Builder.SetInsertPoint(ContinueBB);
408
  // Update the memory at Dest with Cmp's value.
409
188
  CGF.EmitStoreOfScalar(Cmp, CGF.MakeAddrLValue(Dest, E->getType()));
410
188
}
411
412
/// Given an ordering required on success, emit all possible cmpxchg
413
/// instructions to cope with the provided (but possibly only dynamically known)
414
/// FailureOrder.
415
static void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E,
416
                                        bool IsWeak, Address Dest, Address Ptr,
417
                                        Address Val1, Address Val2,
418
                                        llvm::Value *FailureOrderVal,
419
                                        uint64_t Size,
420
                                        llvm::AtomicOrdering SuccessOrder,
421
104
                                        llvm::SyncScope::ID Scope) {
422
104
  llvm::AtomicOrdering FailureOrder;
423
104
  if (llvm::ConstantInt *FO = dyn_cast<llvm::ConstantInt>(FailureOrderVal)) {
424
62
    auto FOS = FO->getSExtValue();
425
62
    if (!llvm::isValidAtomicOrderingCABI(FOS))
426
2
      FailureOrder = llvm::AtomicOrdering::Monotonic;
427
60
    else
428
60
      switch ((llvm::AtomicOrderingCABI)FOS) {
429
27
      case llvm::AtomicOrderingCABI::relaxed:
430
      // 31.7.2.18: "The failure argument shall not be memory_order_release
431
      // nor memory_order_acq_rel". Fallback to monotonic.
432
27
      case llvm::AtomicOrderingCABI::release:
433
27
      case llvm::AtomicOrderingCABI::acq_rel:
434
27
        FailureOrder = llvm::AtomicOrdering::Monotonic;
435
27
        break;
436
0
      case llvm::AtomicOrderingCABI::consume:
437
24
      case llvm::AtomicOrderingCABI::acquire:
438
24
        FailureOrder = llvm::AtomicOrdering::Acquire;
439
24
        break;
440
9
      case llvm::AtomicOrderingCABI::seq_cst:
441
9
        FailureOrder = llvm::AtomicOrdering::SequentiallyConsistent;
442
9
        break;
443
60
      }
444
    // Prior to c++17, "the failure argument shall be no stronger than the
445
    // success argument". This condition has been lifted and the only
446
    // precondition is 31.7.2.18. Effectively treat this as a DR and skip
447
    // language version checks.
448
62
    emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
449
62
                      FailureOrder, Scope);
450
62
    return;
451
62
  }
452
453
  // Create all the relevant BB's
454
42
  auto *MonotonicBB = CGF.createBasicBlock("monotonic_fail", CGF.CurFn);
455
42
  auto *AcquireBB = CGF.createBasicBlock("acquire_fail", CGF.CurFn);
456
42
  auto *SeqCstBB = CGF.createBasicBlock("seqcst_fail", CGF.CurFn);
457
42
  auto *ContBB = CGF.createBasicBlock("atomic.continue", CGF.CurFn);
458
459
  // MonotonicBB is arbitrarily chosen as the default case; in practice, this
460
  // doesn't matter unless someone is crazy enough to use something that
461
  // doesn't fold to a constant for the ordering.
462
42
  llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(FailureOrderVal, MonotonicBB);
463
  // Implemented as acquire, since it's the closest in LLVM.
464
42
  SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::consume),
465
42
              AcquireBB);
466
42
  SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire),
467
42
              AcquireBB);
468
42
  SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst),
469
42
              SeqCstBB);
470
471
  // Emit all the different atomics
472
42
  CGF.Builder.SetInsertPoint(MonotonicBB);
473
42
  emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
474
42
                    Size, SuccessOrder, llvm::AtomicOrdering::Monotonic, Scope);
475
42
  CGF.Builder.CreateBr(ContBB);
476
477
42
  CGF.Builder.SetInsertPoint(AcquireBB);
478
42
  emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
479
42
                    llvm::AtomicOrdering::Acquire, Scope);
480
42
  CGF.Builder.CreateBr(ContBB);
481
482
42
  CGF.Builder.SetInsertPoint(SeqCstBB);
483
42
  emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
484
42
                    llvm::AtomicOrdering::SequentiallyConsistent, Scope);
485
42
  CGF.Builder.CreateBr(ContBB);
486
487
42
  CGF.Builder.SetInsertPoint(ContBB);
488
42
}
489
490
/// Duplicate the atomic min/max operation in conventional IR for the builtin
491
/// variants that return the new rather than the original value.
492
static llvm::Value *EmitPostAtomicMinMax(CGBuilderTy &Builder,
493
                                         AtomicExpr::AtomicOp Op,
494
                                         bool IsSigned,
495
                                         llvm::Value *OldVal,
496
14
                                         llvm::Value *RHS) {
497
14
  llvm::CmpInst::Predicate Pred;
498
14
  switch (Op) {
499
0
  default:
500
0
    llvm_unreachable("Unexpected min/max operation");
501
4
  case AtomicExpr::AO__atomic_max_fetch:
502
4
    Pred = IsSigned ? 
llvm::CmpInst::ICMP_SGT2
:
llvm::CmpInst::ICMP_UGT2
;
503
4
    break;
504
10
  case AtomicExpr::AO__atomic_min_fetch:
505
10
    Pred = IsSigned ? 
llvm::CmpInst::ICMP_SLT4
:
llvm::CmpInst::ICMP_ULT6
;
506
10
    break;
507
14
  }
508
14
  llvm::Value *Cmp = Builder.CreateICmp(Pred, OldVal, RHS, "tst");
509
14
  return Builder.CreateSelect(Cmp, OldVal, RHS, "newval");
510
14
}
511
512
static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest,
513
                         Address Ptr, Address Val1, Address Val2,
514
                         llvm::Value *IsWeak, llvm::Value *FailureOrder,
515
                         uint64_t Size, llvm::AtomicOrdering Order,
516
1.95k
                         llvm::SyncScope::ID Scope) {
517
1.95k
  llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add;
518
1.95k
  bool PostOpMinMax = false;
519
1.95k
  unsigned PostOp = 0;
520
521
1.95k
  switch (E->getOp()) {
522
0
  case AtomicExpr::AO__c11_atomic_init:
523
0
  case AtomicExpr::AO__opencl_atomic_init:
524
0
    llvm_unreachable("Already handled!");
525
526
21
  case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
527
31
  case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
528
45
  case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
529
45
    emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
530
45
                                FailureOrder, Size, Order, Scope);
531
45
    return;
532
2
  case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
533
4
  case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
534
14
  case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
535
14
    emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
536
14
                                FailureOrder, Size, Order, Scope);
537
14
    return;
538
21
  case AtomicExpr::AO__atomic_compare_exchange:
539
31
  case AtomicExpr::AO__atomic_compare_exchange_n: {
540
31
    if (llvm::ConstantInt *IsWeakC = dyn_cast<llvm::ConstantInt>(IsWeak)) {
541
17
      emitAtomicCmpXchgFailureSet(CGF, E, IsWeakC->getZExtValue(), Dest, Ptr,
542
17
                                  Val1, Val2, FailureOrder, Size, Order, Scope);
543
17
    } else {
544
      // Create all the relevant BB's
545
14
      llvm::BasicBlock *StrongBB =
546
14
          CGF.createBasicBlock("cmpxchg.strong", CGF.CurFn);
547
14
      llvm::BasicBlock *WeakBB = CGF.createBasicBlock("cmxchg.weak", CGF.CurFn);
548
14
      llvm::BasicBlock *ContBB =
549
14
          CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
550
551
14
      llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(IsWeak, WeakBB);
552
14
      SI->addCase(CGF.Builder.getInt1(false), StrongBB);
553
554
14
      CGF.Builder.SetInsertPoint(StrongBB);
555
14
      emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
556
14
                                  FailureOrder, Size, Order, Scope);
557
14
      CGF.Builder.CreateBr(ContBB);
558
559
14
      CGF.Builder.SetInsertPoint(WeakBB);
560
14
      emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
561
14
                                  FailureOrder, Size, Order, Scope);
562
14
      CGF.Builder.CreateBr(ContBB);
563
564
14
      CGF.Builder.SetInsertPoint(ContBB);
565
14
    }
566
31
    return;
567
21
  }
568
271
  case AtomicExpr::AO__c11_atomic_load:
569
315
  case AtomicExpr::AO__opencl_atomic_load:
570
323
  case AtomicExpr::AO__hip_atomic_load:
571
342
  case AtomicExpr::AO__atomic_load_n:
572
393
  case AtomicExpr::AO__atomic_load: {
573
393
    llvm::LoadInst *Load = CGF.Builder.CreateLoad(Ptr);
574
393
    Load->setAtomic(Order, Scope);
575
393
    Load->setVolatile(E->isVolatile());
576
393
    CGF.Builder.CreateStore(Load, Dest);
577
393
    return;
578
342
  }
579
580
293
  case AtomicExpr::AO__c11_atomic_store:
581
303
  case AtomicExpr::AO__opencl_atomic_store:
582
318
  case AtomicExpr::AO__hip_atomic_store:
583
367
  case AtomicExpr::AO__atomic_store:
584
371
  case AtomicExpr::AO__atomic_store_n: {
585
371
    llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1);
586
371
    llvm::StoreInst *Store = CGF.Builder.CreateStore(LoadVal1, Ptr);
587
371
    Store->setAtomic(Order, Scope);
588
371
    Store->setVolatile(E->isVolatile());
589
371
    return;
590
367
  }
591
592
13
  case AtomicExpr::AO__c11_atomic_exchange:
593
23
  case AtomicExpr::AO__hip_atomic_exchange:
594
25
  case AtomicExpr::AO__opencl_atomic_exchange:
595
29
  case AtomicExpr::AO__atomic_exchange_n:
596
35
  case AtomicExpr::AO__atomic_exchange:
597
35
    Op = llvm::AtomicRMWInst::Xchg;
598
35
    break;
599
600
63
  case AtomicExpr::AO__atomic_add_fetch:
601
63
    PostOp = E->getValueType()->isFloatingType() ? 
llvm::Instruction::FAdd0
602
63
                                                 : llvm::Instruction::Add;
603
63
    LLVM_FALLTHROUGH;
604
477
  case AtomicExpr::AO__c11_atomic_fetch_add:
605
487
  case AtomicExpr::AO__hip_atomic_fetch_add:
606
491
  case AtomicExpr::AO__opencl_atomic_fetch_add:
607
515
  case AtomicExpr::AO__atomic_fetch_add:
608
515
    Op = E->getValueType()->isFloatingType() ? 
llvm::AtomicRMWInst::FAdd16
609
515
                                             : 
llvm::AtomicRMWInst::Add499
;
610
515
    break;
611
612
0
  case AtomicExpr::AO__atomic_sub_fetch:
613
0
    PostOp = E->getValueType()->isFloatingType() ? llvm::Instruction::FSub
614
0
                                                 : llvm::Instruction::Sub;
615
0
    LLVM_FALLTHROUGH;
616
391
  case AtomicExpr::AO__c11_atomic_fetch_sub:
617
391
  case AtomicExpr::AO__opencl_atomic_fetch_sub:
618
415
  case AtomicExpr::AO__atomic_fetch_sub:
619
415
    Op = E->getValueType()->isFloatingType() ? 
llvm::AtomicRMWInst::FSub12
620
415
                                             : 
llvm::AtomicRMWInst::Sub403
;
621
415
    break;
622
623
8
  case AtomicExpr::AO__atomic_min_fetch:
624
8
    PostOpMinMax = true;
625
8
    LLVM_FALLTHROUGH;
626
18
  case AtomicExpr::AO__c11_atomic_fetch_min:
627
38
  case AtomicExpr::AO__hip_atomic_fetch_min:
628
42
  case AtomicExpr::AO__opencl_atomic_fetch_min:
629
44
  case AtomicExpr::AO__atomic_fetch_min:
630
44
    Op = E->getValueType()->isSignedIntegerType() ? 
llvm::AtomicRMWInst::Min23
631
44
                                                  : 
llvm::AtomicRMWInst::UMin21
;
632
44
    break;
633
634
4
  case AtomicExpr::AO__atomic_max_fetch:
635
4
    PostOpMinMax = true;
636
4
    LLVM_FALLTHROUGH;
637
14
  case AtomicExpr::AO__c11_atomic_fetch_max:
638
34
  case AtomicExpr::AO__hip_atomic_fetch_max:
639
38
  case AtomicExpr::AO__opencl_atomic_fetch_max:
640
40
  case AtomicExpr::AO__atomic_fetch_max:
641
40
    Op = E->getValueType()->isSignedIntegerType() ? 
llvm::AtomicRMWInst::Max21
642
40
                                                  : 
llvm::AtomicRMWInst::UMax19
;
643
40
    break;
644
645
0
  case AtomicExpr::AO__atomic_and_fetch:
646
0
    PostOp = llvm::Instruction::And;
647
0
    LLVM_FALLTHROUGH;
648
4
  case AtomicExpr::AO__c11_atomic_fetch_and:
649
14
  case AtomicExpr::AO__hip_atomic_fetch_and:
650
16
  case AtomicExpr::AO__opencl_atomic_fetch_and:
651
16
  case AtomicExpr::AO__atomic_fetch_and:
652
16
    Op = llvm::AtomicRMWInst::And;
653
16
    break;
654
655
2
  case AtomicExpr::AO__atomic_or_fetch:
656
2
    PostOp = llvm::Instruction::Or;
657
2
    LLVM_FALLTHROUGH;
658
4
  case AtomicExpr::AO__c11_atomic_fetch_or:
659
14
  case AtomicExpr::AO__hip_atomic_fetch_or:
660
14
  case AtomicExpr::AO__opencl_atomic_fetch_or:
661
17
  case AtomicExpr::AO__atomic_fetch_or:
662
17
    Op = llvm::AtomicRMWInst::Or;
663
17
    break;
664
665
0
  case AtomicExpr::AO__atomic_xor_fetch:
666
0
    PostOp = llvm::Instruction::Xor;
667
0
    LLVM_FALLTHROUGH;
668
0
  case AtomicExpr::AO__c11_atomic_fetch_xor:
669
10
  case AtomicExpr::AO__hip_atomic_fetch_xor:
670
10
  case AtomicExpr::AO__opencl_atomic_fetch_xor:
671
14
  case AtomicExpr::AO__atomic_fetch_xor:
672
14
    Op = llvm::AtomicRMWInst::Xor;
673
14
    break;
674
675
4
  case AtomicExpr::AO__atomic_nand_fetch:
676
4
    PostOp = llvm::Instruction::And; // the NOT is special cased below
677
4
    LLVM_FALLTHROUGH;
678
4
  case AtomicExpr::AO__c11_atomic_fetch_nand:
679
8
  case AtomicExpr::AO__atomic_fetch_nand:
680
8
    Op = llvm::AtomicRMWInst::Nand;
681
8
    break;
682
1.95k
  }
683
684
1.10k
  llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1);
685
1.10k
  llvm::AtomicRMWInst *RMWI =
686
1.10k
      CGF.Builder.CreateAtomicRMW(Op, Ptr.getPointer(), LoadVal1, Order, Scope);
687
1.10k
  RMWI->setVolatile(E->isVolatile());
688
689
  // For __atomic_*_fetch operations, perform the operation again to
690
  // determine the value which was written.
691
1.10k
  llvm::Value *Result = RMWI;
692
1.10k
  if (PostOpMinMax)
693
12
    Result = EmitPostAtomicMinMax(CGF.Builder, E->getOp(),
694
12
                                  E->getValueType()->isSignedIntegerType(),
695
12
                                  RMWI, LoadVal1);
696
1.09k
  else if (PostOp)
697
69
    Result = CGF.Builder.CreateBinOp((llvm::Instruction::BinaryOps)PostOp, RMWI,
698
69
                                     LoadVal1);
699
1.10k
  if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch)
700
4
    Result = CGF.Builder.CreateNot(Result);
701
1.10k
  CGF.Builder.CreateStore(Result, Dest);
702
1.10k
}
703
704
// This function emits any expression (scalar, complex, or aggregate)
705
// into a temporary alloca.
706
static Address
707
778
EmitValToTemp(CodeGenFunction &CGF, Expr *E) {
708
778
  Address DeclPtr = CGF.CreateMemTemp(E->getType(), ".atomictmp");
709
778
  CGF.EmitAnyExprToMem(E, DeclPtr, E->getType().getQualifiers(),
710
778
                       /*Init*/ true);
711
778
  return DeclPtr;
712
778
}
713
714
static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *Expr, Address Dest,
715
                         Address Ptr, Address Val1, Address Val2,
716
                         llvm::Value *IsWeak, llvm::Value *FailureOrder,
717
                         uint64_t Size, llvm::AtomicOrdering Order,
718
1.93k
                         llvm::Value *Scope) {
719
1.93k
  auto ScopeModel = Expr->getScopeModel();
720
721
  // LLVM atomic instructions always have synch scope. If clang atomic
722
  // expression has no scope operand, use default LLVM synch scope.
723
1.93k
  if (!ScopeModel) {
724
1.73k
    EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
725
1.73k
                 Order, CGF.CGM.getLLVMContext().getOrInsertSyncScopeID(""));
726
1.73k
    return;
727
1.73k
  }
728
729
  // Handle constant scope.
730
195
  if (auto SC = dyn_cast<llvm::ConstantInt>(Scope)) {
731
187
    auto SCID = CGF.getTargetHooks().getLLVMSyncScopeID(
732
187
        CGF.CGM.getLangOpts(), ScopeModel->map(SC->getZExtValue()),
733
187
        Order, CGF.CGM.getLLVMContext());
734
187
    EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
735
187
                 Order, SCID);
736
187
    return;
737
187
  }
738
739
  // Handle non-constant scope.
740
8
  auto &Builder = CGF.Builder;
741
8
  auto Scopes = ScopeModel->getRuntimeValues();
742
8
  llvm::DenseMap<unsigned, llvm::BasicBlock *> BB;
743
8
  for (auto S : Scopes)
744
32
    BB[S] = CGF.createBasicBlock(getAsString(ScopeModel->map(S)), CGF.CurFn);
745
746
8
  llvm::BasicBlock *ContBB =
747
8
      CGF.createBasicBlock("atomic.scope.continue", CGF.CurFn);
748
749
8
  auto *SC = Builder.CreateIntCast(Scope, Builder.getInt32Ty(), false);
750
  // If unsupported synch scope is encountered at run time, assume a fallback
751
  // synch scope value.
752
8
  auto FallBack = ScopeModel->getFallBackValue();
753
8
  llvm::SwitchInst *SI = Builder.CreateSwitch(SC, BB[FallBack]);
754
32
  for (auto S : Scopes) {
755
32
    auto *B = BB[S];
756
32
    if (S != FallBack)
757
24
      SI->addCase(Builder.getInt32(S), B);
758
759
32
    Builder.SetInsertPoint(B);
760
32
    EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
761
32
                 Order,
762
32
                 CGF.getTargetHooks().getLLVMSyncScopeID(CGF.CGM.getLangOpts(),
763
32
                                                         ScopeModel->map(S),
764
32
                                                         Order,
765
32
                                                         CGF.getLLVMContext()));
766
32
    Builder.CreateBr(ContBB);
767
32
  }
768
769
8
  Builder.SetInsertPoint(ContBB);
770
8
}
771
772
static void
773
AddDirectArgument(CodeGenFunction &CGF, CallArgList &Args,
774
                  bool UseOptimizedLibcall, llvm::Value *Val, QualType ValTy,
775
133
                  SourceLocation Loc, CharUnits SizeInChars) {
776
133
  if (UseOptimizedLibcall) {
777
    // Load value and pass it to the function directly.
778
81
    CharUnits Align = CGF.getContext().getTypeAlignInChars(ValTy);
779
81
    int64_t SizeInBits = CGF.getContext().toBits(SizeInChars);
780
81
    ValTy =
781
81
        CGF.getContext().getIntTypeForBitwidth(SizeInBits, /*Signed=*/false);
782
81
    llvm::Type *IPtrTy = llvm::IntegerType::get(CGF.getLLVMContext(),
783
81
                                                SizeInBits)->getPointerTo();
784
81
    Address Ptr = Address(CGF.Builder.CreateBitCast(Val, IPtrTy), Align);
785
81
    Val = CGF.EmitLoadOfScalar(Ptr, false,
786
81
                               CGF.getContext().getPointerType(ValTy),
787
81
                               Loc);
788
    // Coerce the value into an appropriately sized integer type.
789
81
    Args.add(RValue::get(Val), ValTy);
790
81
  } else {
791
    // Non-optimized functions always take a reference.
792
52
    Args.add(RValue::get(CGF.EmitCastToVoidPtr(Val)),
793
52
                         CGF.getContext().VoidPtrTy);
794
52
  }
795
133
}
796
797
1.17k
RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) {
798
1.17k
  QualType AtomicTy = E->getPtr()->getType()->getPointeeType();
799
1.17k
  QualType MemTy = AtomicTy;
800
1.17k
  if (const AtomicType *AT = AtomicTy->getAs<AtomicType>())
801
628
    MemTy = AT->getValueType();
802
1.17k
  llvm::Value *IsWeak = nullptr, *OrderFail = nullptr;
803
804
1.17k
  Address Val1 = Address::invalid();
805
1.17k
  Address Val2 = Address::invalid();
806
1.17k
  Address Dest = Address::invalid();
807
1.17k
  Address Ptr = EmitPointerWithAlignment(E->getPtr());
808
809
1.17k
  if (E->getOp() == AtomicExpr::AO__c11_atomic_init ||
810
1.17k
      
E->getOp() == AtomicExpr::AO__opencl_atomic_init1.15k
) {
811
16
    LValue lvalue = MakeAddrLValue(Ptr, AtomicTy);
812
16
    EmitAtomicInit(E->getVal1(), lvalue);
813
16
    return RValue::get(nullptr);
814
16
  }
815
816
1.15k
  auto TInfo = getContext().getTypeInfoInChars(AtomicTy);
817
1.15k
  uint64_t Size = TInfo.Width.getQuantity();
818
1.15k
  unsigned MaxInlineWidthInBits = getTarget().getMaxAtomicInlineWidth();
819
820
1.15k
  bool Oversized = getContext().toBits(TInfo.Width) > MaxInlineWidthInBits;
821
1.15k
  bool Misaligned = (Ptr.getAlignment() % TInfo.Width) != 0;
822
1.15k
  bool UseLibcall = Misaligned | Oversized;
823
1.15k
  bool ShouldCastToIntPtrTy = true;
824
825
1.15k
  CharUnits MaxInlineWidth =
826
1.15k
      getContext().toCharUnitsFromBits(MaxInlineWidthInBits);
827
828
1.15k
  DiagnosticsEngine &Diags = CGM.getDiags();
829
830
1.15k
  if (Misaligned) {
831
80
    Diags.Report(E->getBeginLoc(), diag::warn_atomic_op_misaligned)
832
80
        << (int)TInfo.Width.getQuantity()
833
80
        << (int)Ptr.getAlignment().getQuantity();
834
80
  }
835
836
1.15k
  if (Oversized) {
837
148
    Diags.Report(E->getBeginLoc(), diag::warn_atomic_op_oversized)
838
148
        << (int)TInfo.Width.getQuantity() << (int)MaxInlineWidth.getQuantity();
839
148
  }
840
841
1.15k
  llvm::Value *Order = EmitScalarExpr(E->getOrder());
842
1.15k
  llvm::Value *Scope =
843
1.15k
      E->getScopeModel() ? 
EmitScalarExpr(E->getScope())210
:
nullptr947
;
844
845
1.15k
  switch (E->getOp()) {
846
0
  case AtomicExpr::AO__c11_atomic_init:
847
0
  case AtomicExpr::AO__opencl_atomic_init:
848
0
    llvm_unreachable("Already handled above with EmitAtomicInit!");
849
850
120
  case AtomicExpr::AO__c11_atomic_load:
851
138
  case AtomicExpr::AO__opencl_atomic_load:
852
146
  case AtomicExpr::AO__hip_atomic_load:
853
165
  case AtomicExpr::AO__atomic_load_n:
854
165
    break;
855
856
88
  case AtomicExpr::AO__atomic_load:
857
88
    Dest = EmitPointerWithAlignment(E->getVal1());
858
88
    break;
859
860
84
  case AtomicExpr::AO__atomic_store:
861
84
    Val1 = EmitPointerWithAlignment(E->getVal1());
862
84
    break;
863
864
14
  case AtomicExpr::AO__atomic_exchange:
865
14
    Val1 = EmitPointerWithAlignment(E->getVal1());
866
14
    Dest = EmitPointerWithAlignment(E->getVal2());
867
14
    break;
868
869
18
  case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
870
20
  case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
871
28
  case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
872
38
  case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
873
49
  case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
874
59
  case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
875
69
  case AtomicExpr::AO__atomic_compare_exchange_n:
876
91
  case AtomicExpr::AO__atomic_compare_exchange:
877
91
    Val1 = EmitPointerWithAlignment(E->getVal1());
878
91
    if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange)
879
22
      Val2 = EmitPointerWithAlignment(E->getVal2());
880
69
    else
881
69
      Val2 = EmitValToTemp(*this, E->getVal2());
882
91
    OrderFail = EmitScalarExpr(E->getOrderFail());
883
91
    if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange_n ||
884
91
        
E->getOp() == AtomicExpr::AO__atomic_compare_exchange81
)
885
32
      IsWeak = EmitScalarExpr(E->getWeak());
886
91
    break;
887
888
147
  case AtomicExpr::AO__c11_atomic_fetch_add:
889
228
  case AtomicExpr::AO__c11_atomic_fetch_sub:
890
238
  case AtomicExpr::AO__hip_atomic_fetch_add:
891
244
  case AtomicExpr::AO__opencl_atomic_fetch_add:
892
244
  case AtomicExpr::AO__opencl_atomic_fetch_sub:
893
244
    if (MemTy->isPointerType()) {
894
      // For pointer arithmetic, we're required to do a bit of math:
895
      // adding 1 to an int* is not the same as adding 1 to a uintptr_t.
896
      // ... but only for the C11 builtins. The GNU builtins expect the
897
      // user to multiply by sizeof(T).
898
6
      QualType Val1Ty = E->getVal1()->getType();
899
6
      llvm::Value *Val1Scalar = EmitScalarExpr(E->getVal1());
900
6
      CharUnits PointeeIncAmt =
901
6
          getContext().getTypeSizeInChars(MemTy->getPointeeType());
902
6
      Val1Scalar = Builder.CreateMul(Val1Scalar, CGM.getSize(PointeeIncAmt));
903
6
      auto Temp = CreateMemTemp(Val1Ty, ".atomictmp");
904
6
      Val1 = Temp;
905
6
      EmitStoreOfScalar(Val1Scalar, MakeAddrLValue(Temp, Val1Ty));
906
6
      break;
907
6
    }
908
244
    
LLVM_FALLTHROUGH238
;238
909
267
  case AtomicExpr::AO__atomic_fetch_add:
910
297
  case AtomicExpr::AO__atomic_fetch_sub:
911
361
  case AtomicExpr::AO__atomic_add_fetch:
912
362
  case AtomicExpr::AO__atomic_sub_fetch:
913
362
    ShouldCastToIntPtrTy = !MemTy->isFloatingType();
914
362
    LLVM_FALLTHROUGH;
915
916
485
  case AtomicExpr::AO__c11_atomic_store:
917
503
  case AtomicExpr::AO__c11_atomic_exchange:
918
521
  case AtomicExpr::AO__opencl_atomic_store:
919
536
  case AtomicExpr::AO__hip_atomic_store:
920
538
  case AtomicExpr::AO__opencl_atomic_exchange:
921
548
  case AtomicExpr::AO__hip_atomic_exchange:
922
552
  case AtomicExpr::AO__atomic_store_n:
923
556
  case AtomicExpr::AO__atomic_exchange_n:
924
560
  case AtomicExpr::AO__c11_atomic_fetch_and:
925
562
  case AtomicExpr::AO__c11_atomic_fetch_or:
926
562
  case AtomicExpr::AO__c11_atomic_fetch_xor:
927
562
  case AtomicExpr::AO__c11_atomic_fetch_nand:
928
572
  case AtomicExpr::AO__c11_atomic_fetch_max:
929
582
  case AtomicExpr::AO__c11_atomic_fetch_min:
930
584
  case AtomicExpr::AO__opencl_atomic_fetch_and:
931
584
  case AtomicExpr::AO__opencl_atomic_fetch_or:
932
584
  case AtomicExpr::AO__opencl_atomic_fetch_xor:
933
592
  case AtomicExpr::AO__opencl_atomic_fetch_min:
934
596
  case AtomicExpr::AO__opencl_atomic_fetch_max:
935
597
  case AtomicExpr::AO__atomic_fetch_and:
936
607
  case AtomicExpr::AO__hip_atomic_fetch_and:
937
611
  case AtomicExpr::AO__atomic_fetch_or:
938
621
  case AtomicExpr::AO__hip_atomic_fetch_or:
939
626
  case AtomicExpr::AO__atomic_fetch_xor:
940
636
  case AtomicExpr::AO__hip_atomic_fetch_xor:
941
641
  case AtomicExpr::AO__atomic_fetch_nand:
942
642
  case AtomicExpr::AO__atomic_and_fetch:
943
645
  case AtomicExpr::AO__atomic_or_fetch:
944
646
  case AtomicExpr::AO__atomic_xor_fetch:
945
651
  case AtomicExpr::AO__atomic_nand_fetch:
946
655
  case AtomicExpr::AO__atomic_max_fetch:
947
665
  case AtomicExpr::AO__atomic_min_fetch:
948
667
  case AtomicExpr::AO__atomic_fetch_max:
949
687
  case AtomicExpr::AO__hip_atomic_fetch_max:
950
689
  case AtomicExpr::AO__atomic_fetch_min:
951
709
  case AtomicExpr::AO__hip_atomic_fetch_min:
952
709
    Val1 = EmitValToTemp(*this, E->getVal1());
953
709
    break;
954
1.15k
  }
955
956
1.15k
  QualType RValTy = E->getType().getUnqualifiedType();
957
958
  // The inlined atomics only function on iN types, where N is a power of 2. We
959
  // need to make sure (via temporaries if necessary) that all incoming values
960
  // are compatible.
961
1.15k
  LValue AtomicVal = MakeAddrLValue(Ptr, AtomicTy);
962
1.15k
  AtomicInfo Atomics(*this, AtomicVal);
963
964
1.15k
  if (ShouldCastToIntPtrTy) {
965
1.12k
    Ptr = Atomics.emitCastToAtomicIntPointer(Ptr);
966
1.12k
    if (Val1.isValid())
967
876
      Val1 = Atomics.convertToAtomicIntPointer(Val1);
968
1.12k
    if (Val2.isValid())
969
91
      Val2 = Atomics.convertToAtomicIntPointer(Val2);
970
1.12k
  }
971
1.15k
  if (Dest.isValid()) {
972
102
    if (ShouldCastToIntPtrTy)
973
102
      Dest = Atomics.emitCastToAtomicIntPointer(Dest);
974
1.05k
  } else if (E->isCmpXChg())
975
91
    Dest = CreateMemTemp(RValTy, "cmpxchg.bool");
976
964
  else if (!RValTy->isVoidType()) {
977
720
    Dest = Atomics.CreateTempAlloca();
978
720
    if (ShouldCastToIntPtrTy)
979
692
      Dest = Atomics.emitCastToAtomicIntPointer(Dest);
980
720
  }
981
982
  // Use a library call.  See: http://gcc.gnu.org/wiki/Atomic/GCCMM/LIbrary .
983
1.15k
  if (UseLibcall) {
984
183
    bool UseOptimizedLibcall = false;
985
183
    switch (E->getOp()) {
986
0
    case AtomicExpr::AO__c11_atomic_init:
987
0
    case AtomicExpr::AO__opencl_atomic_init:
988
0
      llvm_unreachable("Already handled above with EmitAtomicInit!");
989
990
9
    case AtomicExpr::AO__c11_atomic_fetch_add:
991
11
    case AtomicExpr::AO__opencl_atomic_fetch_add:
992
16
    case AtomicExpr::AO__atomic_fetch_add:
993
16
    case AtomicExpr::AO__hip_atomic_fetch_add:
994
16
    case AtomicExpr::AO__c11_atomic_fetch_and:
995
16
    case AtomicExpr::AO__opencl_atomic_fetch_and:
996
16
    case AtomicExpr::AO__hip_atomic_fetch_and:
997
17
    case AtomicExpr::AO__atomic_fetch_and:
998
17
    case AtomicExpr::AO__c11_atomic_fetch_or:
999
17
    case AtomicExpr::AO__opencl_atomic_fetch_or:
1000
17
    case AtomicExpr::AO__hip_atomic_fetch_or:
1001
18
    case AtomicExpr::AO__atomic_fetch_or:
1002
18
    case AtomicExpr::AO__c11_atomic_fetch_nand:
1003
19
    case AtomicExpr::AO__atomic_fetch_nand:
1004
21
    case AtomicExpr::AO__c11_atomic_fetch_sub:
1005
21
    case AtomicExpr::AO__opencl_atomic_fetch_sub:
1006
27
    case AtomicExpr::AO__atomic_fetch_sub:
1007
27
    case AtomicExpr::AO__c11_atomic_fetch_xor:
1008
27
    case AtomicExpr::AO__opencl_atomic_fetch_xor:
1009
31
    case AtomicExpr::AO__opencl_atomic_fetch_min:
1010
31
    case AtomicExpr::AO__opencl_atomic_fetch_max:
1011
32
    case AtomicExpr::AO__atomic_fetch_xor:
1012
32
    case AtomicExpr::AO__hip_atomic_fetch_xor:
1013
32
    case AtomicExpr::AO__c11_atomic_fetch_max:
1014
32
    case AtomicExpr::AO__c11_atomic_fetch_min:
1015
33
    case AtomicExpr::AO__atomic_add_fetch:
1016
34
    case AtomicExpr::AO__atomic_and_fetch:
1017
35
    case AtomicExpr::AO__atomic_nand_fetch:
1018
36
    case AtomicExpr::AO__atomic_or_fetch:
1019
37
    case AtomicExpr::AO__atomic_sub_fetch:
1020
38
    case AtomicExpr::AO__atomic_xor_fetch:
1021
38
    case AtomicExpr::AO__atomic_fetch_max:
1022
38
    case AtomicExpr::AO__hip_atomic_fetch_max:
1023
38
    case AtomicExpr::AO__atomic_fetch_min:
1024
38
    case AtomicExpr::AO__hip_atomic_fetch_min:
1025
38
    case AtomicExpr::AO__atomic_max_fetch:
1026
40
    case AtomicExpr::AO__atomic_min_fetch:
1027
      // For these, only library calls for certain sizes exist.
1028
40
      UseOptimizedLibcall = true;
1029
40
      break;
1030
1031
37
    case AtomicExpr::AO__atomic_load:
1032
72
    case AtomicExpr::AO__atomic_store:
1033
80
    case AtomicExpr::AO__atomic_exchange:
1034
89
    case AtomicExpr::AO__atomic_compare_exchange:
1035
      // Use the generic version if we don't know that the operand will be
1036
      // suitably aligned for the optimized version.
1037
89
      if (Misaligned)
1038
63
        break;
1039
89
      
LLVM_FALLTHROUGH26
;26
1040
37
    case AtomicExpr::AO__c11_atomic_load:
1041
49
    case AtomicExpr::AO__c11_atomic_store:
1042
54
    case AtomicExpr::AO__c11_atomic_exchange:
1043
54
    case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
1044
59
    case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
1045
59
    case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
1046
61
    case AtomicExpr::AO__opencl_atomic_load:
1047
61
    case AtomicExpr::AO__hip_atomic_load:
1048
69
    case AtomicExpr::AO__opencl_atomic_store:
1049
69
    case AtomicExpr::AO__hip_atomic_store:
1050
69
    case AtomicExpr::AO__opencl_atomic_exchange:
1051
69
    case AtomicExpr::AO__hip_atomic_exchange:
1052
78
    case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
1053
78
    case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
1054
80
    case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
1055
80
    case AtomicExpr::AO__atomic_load_n:
1056
80
    case AtomicExpr::AO__atomic_store_n:
1057
80
    case AtomicExpr::AO__atomic_exchange_n:
1058
80
    case AtomicExpr::AO__atomic_compare_exchange_n:
1059
      // Only use optimized library calls for sizes for which they exist.
1060
      // FIXME: Size == 16 optimized library functions exist too.
1061
80
      if (Size == 1 || 
Size == 272
||
Size == 470
||
Size == 841
)
1062
63
        UseOptimizedLibcall = true;
1063
80
      break;
1064
183
    }
1065
1066
183
    CallArgList Args;
1067
183
    if (!UseOptimizedLibcall) {
1068
      // For non-optimized library calls, the size is the first parameter
1069
80
      Args.add(RValue::get(llvm::ConstantInt::get(SizeTy, Size)),
1070
80
               getContext().getSizeType());
1071
80
    }
1072
    // Atomic address is the first or second parameter
1073
    // The OpenCL atomic library functions only accept pointer arguments to
1074
    // generic address space.
1075
208
    auto CastToGenericAddrSpace = [&](llvm::Value *V, QualType PT) {
1076
208
      if (!E->isOpenCL())
1077
170
        return V;
1078
38
      auto AS = PT->castAs<PointerType>()->getPointeeType().getAddressSpace();
1079
38
      if (AS == LangAS::opencl_generic)
1080
21
        return V;
1081
17
      auto DestAS = getContext().getTargetAddressSpace(LangAS::opencl_generic);
1082
17
      auto T = llvm::cast<llvm::PointerType>(V->getType());
1083
17
      auto *DestType = llvm::PointerType::getWithSamePointeeType(T, DestAS);
1084
1085
17
      return getTargetHooks().performAddrSpaceCast(
1086
17
          *this, V, AS, LangAS::opencl_generic, DestType, false);
1087
38
    };
1088
1089
183
    Args.add(RValue::get(CastToGenericAddrSpace(
1090
183
                 EmitCastToVoidPtr(Ptr.getPointer()), E->getPtr()->getType())),
1091
183
             getContext().VoidPtrTy);
1092
1093
183
    std::string LibCallName;
1094
183
    QualType LoweredMemTy =
1095
183
      MemTy->isPointerType() ? 
getContext().getIntPtrType()3
:
MemTy180
;
1096
183
    QualType RetTy;
1097
183
    bool HaveRetTy = false;
1098
183
    llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0;
1099
183
    bool PostOpMinMax = false;
1100
183
    switch (E->getOp()) {
1101
0
    case AtomicExpr::AO__c11_atomic_init:
1102
0
    case AtomicExpr::AO__opencl_atomic_init:
1103
0
      llvm_unreachable("Already handled!");
1104
1105
    // There is only one libcall for compare an exchange, because there is no
1106
    // optimisation benefit possible from a libcall version of a weak compare
1107
    // and exchange.
1108
    // bool __atomic_compare_exchange(size_t size, void *mem, void *expected,
1109
    //                                void *desired, int success, int failure)
1110
    // bool __atomic_compare_exchange_N(T *mem, T *expected, T desired,
1111
    //                                  int success, int failure)
1112
0
    case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
1113
5
    case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
1114
14
    case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
1115
14
    case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
1116
16
    case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
1117
16
    case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
1118
25
    case AtomicExpr::AO__atomic_compare_exchange:
1119
25
    case AtomicExpr::AO__atomic_compare_exchange_n:
1120
25
      LibCallName = "__atomic_compare_exchange";
1121
25
      RetTy = getContext().BoolTy;
1122
25
      HaveRetTy = true;
1123
25
      Args.add(
1124
25
          RValue::get(CastToGenericAddrSpace(
1125
25
              EmitCastToVoidPtr(Val1.getPointer()), E->getVal1()->getType())),
1126
25
          getContext().VoidPtrTy);
1127
25
      AddDirectArgument(*this, Args, UseOptimizedLibcall, Val2.getPointer(),
1128
25
                        MemTy, E->getExprLoc(), TInfo.Width);
1129
25
      Args.add(RValue::get(Order), getContext().IntTy);
1130
25
      Order = OrderFail;
1131
25
      break;
1132
    // void __atomic_exchange(size_t size, void *mem, void *val, void *return,
1133
    //                        int order)
1134
    // T __atomic_exchange_N(T *mem, T val, int order)
1135
5
    case AtomicExpr::AO__c11_atomic_exchange:
1136
5
    case AtomicExpr::AO__opencl_atomic_exchange:
1137
5
    case AtomicExpr::AO__atomic_exchange_n:
1138
13
    case AtomicExpr::AO__atomic_exchange:
1139
13
    case AtomicExpr::AO__hip_atomic_exchange:
1140
13
      LibCallName = "__atomic_exchange";
1141
13
      AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1142
13
                        MemTy, E->getExprLoc(), TInfo.Width);
1143
13
      break;
1144
    // void __atomic_store(size_t size, void *mem, void *val, int order)
1145
    // void __atomic_store_N(T *mem, T val, int order)
1146
12
    case AtomicExpr::AO__c11_atomic_store:
1147
20
    case AtomicExpr::AO__opencl_atomic_store:
1148
20
    case AtomicExpr::AO__hip_atomic_store:
1149
55
    case AtomicExpr::AO__atomic_store:
1150
55
    case AtomicExpr::AO__atomic_store_n:
1151
55
      LibCallName = "__atomic_store";
1152
55
      RetTy = getContext().VoidTy;
1153
55
      HaveRetTy = true;
1154
55
      AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1155
55
                        MemTy, E->getExprLoc(), TInfo.Width);
1156
55
      break;
1157
    // void __atomic_load(size_t size, void *mem, void *return, int order)
1158
    // T __atomic_load_N(T *mem, int order)
1159
11
    case AtomicExpr::AO__c11_atomic_load:
1160
13
    case AtomicExpr::AO__opencl_atomic_load:
1161
13
    case AtomicExpr::AO__hip_atomic_load:
1162
50
    case AtomicExpr::AO__atomic_load:
1163
50
    case AtomicExpr::AO__atomic_load_n:
1164
50
      LibCallName = "__atomic_load";
1165
50
      break;
1166
    // T __atomic_add_fetch_N(T *mem, T val, int order)
1167
    // T __atomic_fetch_add_N(T *mem, T val, int order)
1168
1
    case AtomicExpr::AO__atomic_add_fetch:
1169
1
      PostOp = llvm::Instruction::Add;
1170
1
      LLVM_FALLTHROUGH;
1171
10
    case AtomicExpr::AO__c11_atomic_fetch_add:
1172
12
    case AtomicExpr::AO__opencl_atomic_fetch_add:
1173
17
    case AtomicExpr::AO__atomic_fetch_add:
1174
17
    case AtomicExpr::AO__hip_atomic_fetch_add:
1175
17
      LibCallName = "__atomic_fetch_add";
1176
17
      AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1177
17
                        LoweredMemTy, E->getExprLoc(), TInfo.Width);
1178
17
      break;
1179
    // T __atomic_and_fetch_N(T *mem, T val, int order)
1180
    // T __atomic_fetch_and_N(T *mem, T val, int order)
1181
1
    case AtomicExpr::AO__atomic_and_fetch:
1182
1
      PostOp = llvm::Instruction::And;
1183
1
      LLVM_FALLTHROUGH;
1184
1
    case AtomicExpr::AO__c11_atomic_fetch_and:
1185
1
    case AtomicExpr::AO__opencl_atomic_fetch_and:
1186
1
    case AtomicExpr::AO__hip_atomic_fetch_and:
1187
2
    case AtomicExpr::AO__atomic_fetch_and:
1188
2
      LibCallName = "__atomic_fetch_and";
1189
2
      AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1190
2
                        MemTy, E->getExprLoc(), TInfo.Width);
1191
2
      break;
1192
    // T __atomic_or_fetch_N(T *mem, T val, int order)
1193
    // T __atomic_fetch_or_N(T *mem, T val, int order)
1194
1
    case AtomicExpr::AO__atomic_or_fetch:
1195
1
      PostOp = llvm::Instruction::Or;
1196
1
      LLVM_FALLTHROUGH;
1197
1
    case AtomicExpr::AO__c11_atomic_fetch_or:
1198
1
    case AtomicExpr::AO__opencl_atomic_fetch_or:
1199
1
    case AtomicExpr::AO__hip_atomic_fetch_or:
1200
2
    case AtomicExpr::AO__atomic_fetch_or:
1201
2
      LibCallName = "__atomic_fetch_or";
1202
2
      AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1203
2
                        MemTy, E->getExprLoc(), TInfo.Width);
1204
2
      break;
1205
    // T __atomic_sub_fetch_N(T *mem, T val, int order)
1206
    // T __atomic_fetch_sub_N(T *mem, T val, int order)
1207
1
    case AtomicExpr::AO__atomic_sub_fetch:
1208
1
      PostOp = llvm::Instruction::Sub;
1209
1
      LLVM_FALLTHROUGH;
1210
3
    case AtomicExpr::AO__c11_atomic_fetch_sub:
1211
3
    case AtomicExpr::AO__opencl_atomic_fetch_sub:
1212
9
    case AtomicExpr::AO__atomic_fetch_sub:
1213
9
      LibCallName = "__atomic_fetch_sub";
1214
9
      AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1215
9
                        LoweredMemTy, E->getExprLoc(), TInfo.Width);
1216
9
      break;
1217
    // T __atomic_xor_fetch_N(T *mem, T val, int order)
1218
    // T __atomic_fetch_xor_N(T *mem, T val, int order)
1219
1
    case AtomicExpr::AO__atomic_xor_fetch:
1220
1
      PostOp = llvm::Instruction::Xor;
1221
1
      LLVM_FALLTHROUGH;
1222
1
    case AtomicExpr::AO__c11_atomic_fetch_xor:
1223
1
    case AtomicExpr::AO__opencl_atomic_fetch_xor:
1224
1
    case AtomicExpr::AO__hip_atomic_fetch_xor:
1225
2
    case AtomicExpr::AO__atomic_fetch_xor:
1226
2
      LibCallName = "__atomic_fetch_xor";
1227
2
      AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1228
2
                        MemTy, E->getExprLoc(), TInfo.Width);
1229
2
      break;
1230
2
    case AtomicExpr::AO__atomic_min_fetch:
1231
2
      PostOpMinMax = true;
1232
2
      LLVM_FALLTHROUGH;
1233
2
    case AtomicExpr::AO__c11_atomic_fetch_min:
1234
2
    case AtomicExpr::AO__atomic_fetch_min:
1235
2
    case AtomicExpr::AO__hip_atomic_fetch_min:
1236
6
    case AtomicExpr::AO__opencl_atomic_fetch_min:
1237
6
      LibCallName = E->getValueType()->isSignedIntegerType()
1238
6
                        ? 
"__atomic_fetch_min"2
1239
6
                        : 
"__atomic_fetch_umin"4
;
1240
6
      AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1241
6
                        LoweredMemTy, E->getExprLoc(), TInfo.Width);
1242
6
      break;
1243
0
    case AtomicExpr::AO__atomic_max_fetch:
1244
0
      PostOpMinMax = true;
1245
0
      LLVM_FALLTHROUGH;
1246
0
    case AtomicExpr::AO__c11_atomic_fetch_max:
1247
0
    case AtomicExpr::AO__atomic_fetch_max:
1248
0
    case AtomicExpr::AO__hip_atomic_fetch_max:
1249
0
    case AtomicExpr::AO__opencl_atomic_fetch_max:
1250
0
      LibCallName = E->getValueType()->isSignedIntegerType()
1251
0
                        ? "__atomic_fetch_max"
1252
0
                        : "__atomic_fetch_umax";
1253
0
      AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1254
0
                        LoweredMemTy, E->getExprLoc(), TInfo.Width);
1255
0
      break;
1256
    // T __atomic_nand_fetch_N(T *mem, T val, int order)
1257
    // T __atomic_fetch_nand_N(T *mem, T val, int order)
1258
1
    case AtomicExpr::AO__atomic_nand_fetch:
1259
1
      PostOp = llvm::Instruction::And; // the NOT is special cased below
1260
1
      LLVM_FALLTHROUGH;
1261
1
    case AtomicExpr::AO__c11_atomic_fetch_nand:
1262
2
    case AtomicExpr::AO__atomic_fetch_nand:
1263
2
      LibCallName = "__atomic_fetch_nand";
1264
2
      AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1265
2
                        MemTy, E->getExprLoc(), TInfo.Width);
1266
2
      break;
1267
183
    }
1268
1269
183
    if (E->isOpenCL()) {
1270
27
      LibCallName = std::string("__opencl") +
1271
27
          StringRef(LibCallName).drop_front(1).str();
1272
1273
27
    }
1274
    // Optimized functions have the size in their name.
1275
183
    if (UseOptimizedLibcall)
1276
103
      LibCallName += "_" + llvm::utostr(Size);
1277
    // By default, assume we return a value of the atomic type.
1278
183
    if (!HaveRetTy) {
1279
103
      if (UseOptimizedLibcall) {
1280
        // Value is returned directly.
1281
        // The function returns an appropriately sized integer type.
1282
63
        RetTy = getContext().getIntTypeForBitwidth(
1283
63
            getContext().toBits(TInfo.Width), /*Signed=*/false);
1284
63
      } else {
1285
        // Value is returned through parameter before the order.
1286
40
        RetTy = getContext().VoidTy;
1287
40
        Args.add(RValue::get(EmitCastToVoidPtr(Dest.getPointer())),
1288
40
                 getContext().VoidPtrTy);
1289
40
      }
1290
103
    }
1291
    // order is always the last parameter
1292
183
    Args.add(RValue::get(Order),
1293
183
             getContext().IntTy);
1294
183
    if (E->isOpenCL())
1295
27
      Args.add(RValue::get(Scope), getContext().IntTy);
1296
1297
    // PostOp is only needed for the atomic_*_fetch operations, and
1298
    // thus is only needed for and implemented in the
1299
    // UseOptimizedLibcall codepath.
1300
183
    assert(UseOptimizedLibcall || (!PostOp && !PostOpMinMax));
1301
1302
0
    RValue Res = emitAtomicLibcall(*this, LibCallName, RetTy, Args);
1303
    // The value is returned directly from the libcall.
1304
183
    if (E->isCmpXChg())
1305
25
      return Res;
1306
1307
    // The value is returned directly for optimized libcalls but the expr
1308
    // provided an out-param.
1309
158
    if (UseOptimizedLibcall && 
Res.getScalarVal()91
) {
1310
63
      llvm::Value *ResVal = Res.getScalarVal();
1311
63
      if (PostOpMinMax) {
1312
2
        llvm::Value *LoadVal1 = Args[1].getRValue(*this).getScalarVal();
1313
2
        ResVal = EmitPostAtomicMinMax(Builder, E->getOp(),
1314
2
                                      E->getValueType()->isSignedIntegerType(),
1315
2
                                      ResVal, LoadVal1);
1316
61
      } else if (PostOp) {
1317
6
        llvm::Value *LoadVal1 = Args[1].getRValue(*this).getScalarVal();
1318
6
        ResVal = Builder.CreateBinOp(PostOp, ResVal, LoadVal1);
1319
6
      }
1320
63
      if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch)
1321
1
        ResVal = Builder.CreateNot(ResVal);
1322
1323
63
      Builder.CreateStore(
1324
63
          ResVal, Builder.CreateElementBitCast(Dest, ResVal->getType()));
1325
63
    }
1326
1327
158
    if (RValTy->isVoidType())
1328
100
      return RValue::get(nullptr);
1329
1330
58
    return convertTempToRValue(
1331
58
        Builder.CreateElementBitCast(Dest, ConvertTypeForMem(RValTy)),
1332
58
        RValTy, E->getExprLoc());
1333
158
  }
1334
1335
974
  bool IsStore = E->getOp() == AtomicExpr::AO__c11_atomic_store ||
1336
974
                 
E->getOp() == AtomicExpr::AO__opencl_atomic_store863
||
1337
974
                 
E->getOp() == AtomicExpr::AO__hip_atomic_store853
||
1338
974
                 
E->getOp() == AtomicExpr::AO__atomic_store838
||
1339
974
                 
E->getOp() == AtomicExpr::AO__atomic_store_n789
;
1340
974
  bool IsLoad = E->getOp() == AtomicExpr::AO__c11_atomic_load ||
1341
974
                
E->getOp() == AtomicExpr::AO__opencl_atomic_load865
||
1342
974
                
E->getOp() == AtomicExpr::AO__hip_atomic_load849
||
1343
974
                
E->getOp() == AtomicExpr::AO__atomic_load841
||
1344
974
                
E->getOp() == AtomicExpr::AO__atomic_load_n790
;
1345
1346
974
  if (isa<llvm::ConstantInt>(Order)) {
1347
647
    auto ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
1348
    // We should not ever get to a case where the ordering isn't a valid C ABI
1349
    // value, but it's hard to enforce that in general.
1350
647
    if (llvm::isValidAtomicOrderingCABI(ord))
1351
647
      switch ((llvm::AtomicOrderingCABI)ord) {
1352
217
      case llvm::AtomicOrderingCABI::relaxed:
1353
217
        EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1354
217
                     llvm::AtomicOrdering::Monotonic, Scope);
1355
217
        break;
1356
0
      case llvm::AtomicOrderingCABI::consume:
1357
42
      case llvm::AtomicOrderingCABI::acquire:
1358
42
        if (IsStore)
1359
0
          break; // Avoid crashing on code with undefined behavior
1360
42
        EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1361
42
                     llvm::AtomicOrdering::Acquire, Scope);
1362
42
        break;
1363
22
      case llvm::AtomicOrderingCABI::release:
1364
22
        if (IsLoad)
1365
0
          break; // Avoid crashing on code with undefined behavior
1366
22
        EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1367
22
                     llvm::AtomicOrdering::Release, Scope);
1368
22
        break;
1369
54
      case llvm::AtomicOrderingCABI::acq_rel:
1370
54
        if (IsLoad || IsStore)
1371
0
          break; // Avoid crashing on code with undefined behavior
1372
54
        EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1373
54
                     llvm::AtomicOrdering::AcquireRelease, Scope);
1374
54
        break;
1375
312
      case llvm::AtomicOrderingCABI::seq_cst:
1376
312
        EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1377
312
                     llvm::AtomicOrdering::SequentiallyConsistent, Scope);
1378
312
        break;
1379
647
      }
1380
647
    if (RValTy->isVoidType())
1381
155
      return RValue::get(nullptr);
1382
1383
492
    return convertTempToRValue(
1384
492
        Builder.CreateElementBitCast(Dest, ConvertTypeForMem(RValTy)),
1385
492
        RValTy, E->getExprLoc());
1386
647
  }
1387
1388
  // Long case, when Order isn't obviously constant.
1389
1390
  // Create all the relevant BB's
1391
327
  llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr,
1392
327
                   *ReleaseBB = nullptr, *AcqRelBB = nullptr,
1393
327
                   *SeqCstBB = nullptr;
1394
327
  MonotonicBB = createBasicBlock("monotonic", CurFn);
1395
327
  if (!IsStore)
1396
236
    AcquireBB = createBasicBlock("acquire", CurFn);
1397
327
  if (!IsLoad)
1398
244
    ReleaseBB = createBasicBlock("release", CurFn);
1399
327
  if (!IsLoad && 
!IsStore244
)
1400
153
    AcqRelBB = createBasicBlock("acqrel", CurFn);
1401
327
  SeqCstBB = createBasicBlock("seqcst", CurFn);
1402
327
  llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn);
1403
1404
  // Create the switch for the split
1405
  // MonotonicBB is arbitrarily chosen as the default case; in practice, this
1406
  // doesn't matter unless someone is crazy enough to use something that
1407
  // doesn't fold to a constant for the ordering.
1408
327
  Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);
1409
327
  llvm::SwitchInst *SI = Builder.CreateSwitch(Order, MonotonicBB);
1410
1411
  // Emit all the different atomics
1412
327
  Builder.SetInsertPoint(MonotonicBB);
1413
327
  EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1414
327
               llvm::AtomicOrdering::Monotonic, Scope);
1415
327
  Builder.CreateBr(ContBB);
1416
327
  if (!IsStore) {
1417
236
    Builder.SetInsertPoint(AcquireBB);
1418
236
    EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1419
236
                 llvm::AtomicOrdering::Acquire, Scope);
1420
236
    Builder.CreateBr(ContBB);
1421
236
    SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::consume),
1422
236
                AcquireBB);
1423
236
    SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire),
1424
236
                AcquireBB);
1425
236
  }
1426
327
  if (!IsLoad) {
1427
244
    Builder.SetInsertPoint(ReleaseBB);
1428
244
    EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1429
244
                 llvm::AtomicOrdering::Release, Scope);
1430
244
    Builder.CreateBr(ContBB);
1431
244
    SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::release),
1432
244
                ReleaseBB);
1433
244
  }
1434
327
  if (!IsLoad && 
!IsStore244
) {
1435
153
    Builder.SetInsertPoint(AcqRelBB);
1436
153
    EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1437
153
                 llvm::AtomicOrdering::AcquireRelease, Scope);
1438
153
    Builder.CreateBr(ContBB);
1439
153
    SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acq_rel),
1440
153
                AcqRelBB);
1441
153
  }
1442
327
  Builder.SetInsertPoint(SeqCstBB);
1443
327
  EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1444
327
               llvm::AtomicOrdering::SequentiallyConsistent, Scope);
1445
327
  Builder.CreateBr(ContBB);
1446
327
  SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst),
1447
327
              SeqCstBB);
1448
1449
  // Cleanup and return
1450
327
  Builder.SetInsertPoint(ContBB);
1451
327
  if (RValTy->isVoidType())
1452
91
    return RValue::get(nullptr);
1453
1454
236
  assert(Atomics.getValueSizeInBits() <= Atomics.getAtomicSizeInBits());
1455
0
  return convertTempToRValue(
1456
236
      Builder.CreateElementBitCast(Dest, ConvertTypeForMem(RValTy)),
1457
236
      RValTy, E->getExprLoc());
1458
327
}
1459
1460
4.10k
Address AtomicInfo::emitCastToAtomicIntPointer(Address addr) const {
1461
4.10k
  llvm::IntegerType *ty =
1462
4.10k
    llvm::IntegerType::get(CGF.getLLVMContext(), AtomicSizeInBits);
1463
4.10k
  return CGF.Builder.CreateElementBitCast(addr, ty);
1464
4.10k
}
1465
1466
967
Address AtomicInfo::convertToAtomicIntPointer(Address Addr) const {
1467
967
  llvm::Type *Ty = Addr.getElementType();
1468
967
  uint64_t SourceSizeInBits = CGF.CGM.getDataLayout().getTypeSizeInBits(Ty);
1469
967
  if (SourceSizeInBits != AtomicSizeInBits) {
1470
9
    Address Tmp = CreateTempAlloca();
1471
9
    CGF.Builder.CreateMemCpy(Tmp, Addr,
1472
9
                             std::min(AtomicSizeInBits, SourceSizeInBits) / 8);
1473
9
    Addr = Tmp;
1474
9
  }
1475
1476
967
  return emitCastToAtomicIntPointer(Addr);
1477
967
}
1478
1479
RValue AtomicInfo::convertAtomicTempToRValue(Address addr,
1480
                                             AggValueSlot resultSlot,
1481
                                             SourceLocation loc,
1482
180
                                             bool asValue) const {
1483
180
  if (LVal.isSimple()) {
1484
144
    if (EvaluationKind == TEK_Aggregate)
1485
13
      return resultSlot.asRValue();
1486
1487
    // Drill into the padding structure if we have one.
1488
131
    if (hasPadding())
1489
0
      addr = CGF.Builder.CreateStructGEP(addr, 0);
1490
1491
    // Otherwise, just convert the temporary to an r-value using the
1492
    // normal conversion routine.
1493
131
    return CGF.convertTempToRValue(addr, getValueType(), loc);
1494
144
  }
1495
36
  if (!asValue)
1496
    // Get RValue from temp memory as atomic for non-simple lvalues
1497
12
    return RValue::get(CGF.Builder.CreateLoad(addr));
1498
24
  if (LVal.isBitField())
1499
20
    return CGF.EmitLoadOfBitfieldLValue(
1500
20
        LValue::MakeBitfield(addr, LVal.getBitFieldInfo(), LVal.getType(),
1501
20
                             LVal.getBaseInfo(), TBAAAccessInfo()), loc);
1502
4
  if (LVal.isVectorElt())
1503
2
    return CGF.EmitLoadOfLValue(
1504
2
        LValue::MakeVectorElt(addr, LVal.getVectorIdx(), LVal.getType(),
1505
2
                              LVal.getBaseInfo(), TBAAAccessInfo()), loc);
1506
2
  assert(LVal.isExtVectorElt());
1507
0
  return CGF.EmitLoadOfExtVectorElementLValue(LValue::MakeExtVectorElt(
1508
2
      addr, LVal.getExtVectorElts(), LVal.getType(),
1509
2
      LVal.getBaseInfo(), TBAAAccessInfo()));
1510
4
}
1511
1512
RValue AtomicInfo::ConvertIntToValueOrAtomic(llvm::Value *IntVal,
1513
                                             AggValueSlot ResultSlot,
1514
                                             SourceLocation Loc,
1515
414
                                             bool AsValue) const {
1516
  // Try not to in some easy cases.
1517
414
  assert(IntVal->getType()->isIntegerTy() && "Expected integer value");
1518
414
  if (getEvaluationKind() == TEK_Scalar &&
1519
414
      
(404
(404
(404
!LVal.isBitField()404
||
1520
404
         
LVal.getBitFieldInfo().Size == ValueSizeInBits64
) &&
1521
404
        
!hasPadding()340
) ||
1522
404
       
!AsValue80
)) {
1523
384
    auto *ValTy = AsValue
1524
384
                      ? 
CGF.ConvertTypeForMem(ValueTy)109
1525
384
                      : 
getAtomicAddress().getType()->getPointerElementType()275
;
1526
384
    if (ValTy->isIntegerTy()) {
1527
275
      assert(IntVal->getType() == ValTy && "Different integer types.");
1528
0
      return RValue::get(CGF.EmitFromMemory(IntVal, ValueTy));
1529
275
    } else 
if (109
ValTy->isPointerTy()109
)
1530
0
      return RValue::get(CGF.Builder.CreateIntToPtr(IntVal, ValTy));
1531
109
    else if (llvm::CastInst::isBitCastable(IntVal->getType(), ValTy))
1532
77
      return RValue::get(CGF.Builder.CreateBitCast(IntVal, ValTy));
1533
384
  }
1534
1535
  // Create a temporary.  This needs to be big enough to hold the
1536
  // atomic integer.
1537
62
  Address Temp = Address::invalid();
1538
62
  bool TempIsVolatile = false;
1539
62
  if (AsValue && 
getEvaluationKind() == TEK_Aggregate44
) {
1540
8
    assert(!ResultSlot.isIgnored());
1541
0
    Temp = ResultSlot.getAddress();
1542
8
    TempIsVolatile = ResultSlot.isVolatile();
1543
54
  } else {
1544
54
    Temp = CreateTempAlloca();
1545
54
  }
1546
1547
  // Slam the integer into the temporary.
1548
0
  Address CastTemp = emitCastToAtomicIntPointer(Temp);
1549
62
  CGF.Builder.CreateStore(IntVal, CastTemp)
1550
62
      ->setVolatile(TempIsVolatile);
1551
1552
62
  return convertAtomicTempToRValue(Temp, ResultSlot, Loc, AsValue);
1553
414
}
1554
1555
void AtomicInfo::EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
1556
105
                                       llvm::AtomicOrdering AO, bool) {
1557
  // void __atomic_load(size_t size, void *mem, void *return, int order);
1558
105
  CallArgList Args;
1559
105
  Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
1560
105
  Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())),
1561
105
           CGF.getContext().VoidPtrTy);
1562
105
  Args.add(RValue::get(CGF.EmitCastToVoidPtr(AddForLoaded)),
1563
105
           CGF.getContext().VoidPtrTy);
1564
105
  Args.add(
1565
105
      RValue::get(llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(AO))),
1566
105
      CGF.getContext().IntTy);
1567
105
  emitAtomicLibcall(CGF, "__atomic_load", CGF.getContext().VoidTy, Args);
1568
105
}
1569
1570
llvm::Value *AtomicInfo::EmitAtomicLoadOp(llvm::AtomicOrdering AO,
1571
422
                                          bool IsVolatile) {
1572
  // Okay, we're doing this natively.
1573
422
  Address Addr = getAtomicAddressAsAtomicIntPointer();
1574
422
  llvm::LoadInst *Load = CGF.Builder.CreateLoad(Addr, "atomic-load");
1575
422
  Load->setAtomic(AO);
1576
1577
  // Other decoration.
1578
422
  if (IsVolatile)
1579
12
    Load->setVolatile(true);
1580
422
  CGF.CGM.DecorateInstructionWithTBAA(Load, LVal.getTBAAInfo());
1581
422
  return Load;
1582
422
}
1583
1584
/// An LValue is a candidate for having its loads and stores be made atomic if
1585
/// we are operating under /volatile:ms *and* the LValue itself is volatile and
1586
/// performing such an operation can be performed without a libcall.
1587
1.41M
bool CodeGenFunction::LValueIsSuitableForInlineAtomic(LValue LV) {
1588
1.41M
  if (!CGM.getCodeGenOpts().MSVolatile) 
return false1.41M
;
1589
45
  AtomicInfo AI(*this, LV);
1590
45
  bool IsVolatile = LV.isVolatile() || 
hasVolatileMember(LV.getType())27
;
1591
  // An atomic is inline if we don't need to use a libcall.
1592
45
  bool AtomicIsInline = !AI.shouldUseLibcall();
1593
  // MSVC doesn't seem to do this for types wider than a pointer.
1594
45
  if (getContext().getTypeSize(LV.getType()) >
1595
45
      getContext().getTypeSize(getContext().getIntPtrType()))
1596
5
    return false;
1597
40
  return IsVolatile && 
AtomicIsInline15
;
1598
45
}
1599
1600
RValue CodeGenFunction::EmitAtomicLoad(LValue LV, SourceLocation SL,
1601
77
                                       AggValueSlot Slot) {
1602
77
  llvm::AtomicOrdering AO;
1603
77
  bool IsVolatile = LV.isVolatileQualified();
1604
77
  if (LV.getType()->isAtomicType()) {
1605
70
    AO = llvm::AtomicOrdering::SequentiallyConsistent;
1606
70
  } else {
1607
7
    AO = llvm::AtomicOrdering::Acquire;
1608
7
    IsVolatile = true;
1609
7
  }
1610
77
  return EmitAtomicLoad(LV, SL, AO, IsVolatile, Slot);
1611
77
}
1612
1613
RValue AtomicInfo::EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
1614
                                  bool AsValue, llvm::AtomicOrdering AO,
1615
183
                                  bool IsVolatile) {
1616
  // Check whether we should use a library call.
1617
183
  if (shouldUseLibcall()) {
1618
44
    Address TempAddr = Address::invalid();
1619
44
    if (LVal.isSimple() && 
!ResultSlot.isIgnored()40
) {
1620
5
      assert(getEvaluationKind() == TEK_Aggregate);
1621
0
      TempAddr = ResultSlot.getAddress();
1622
5
    } else
1623
39
      TempAddr = CreateTempAlloca();
1624
1625
0
    EmitAtomicLoadLibcall(TempAddr.getPointer(), AO, IsVolatile);
1626
1627
    // Okay, turn that back into the original value or whole atomic (for
1628
    // non-simple lvalues) type.
1629
44
    return convertAtomicTempToRValue(TempAddr, ResultSlot, Loc, AsValue);
1630
44
  }
1631
1632
  // Okay, we're doing this natively.
1633
139
  auto *Load = EmitAtomicLoadOp(AO, IsVolatile);
1634
1635
  // If we're ignoring an aggregate return, don't do anything.
1636
139
  if (getEvaluationKind() == TEK_Aggregate && 
ResultSlot.isIgnored()8
)
1637
0
    return RValue::getAggregate(Address::invalid(), false);
1638
1639
  // Okay, turn that back into the original value or atomic (for non-simple
1640
  // lvalues) type.
1641
139
  return ConvertIntToValueOrAtomic(Load, ResultSlot, Loc, AsValue);
1642
139
}
1643
1644
/// Emit a load from an l-value of atomic type.  Note that the r-value
1645
/// we produce is an r-value of the atomic *value* type.
1646
RValue CodeGenFunction::EmitAtomicLoad(LValue src, SourceLocation loc,
1647
                                       llvm::AtomicOrdering AO, bool IsVolatile,
1648
183
                                       AggValueSlot resultSlot) {
1649
183
  AtomicInfo Atomics(*this, src);
1650
183
  return Atomics.EmitAtomicLoad(resultSlot, loc, /*AsValue=*/true, AO,
1651
183
                                IsVolatile);
1652
183
}
1653
1654
/// Copy an r-value into memory as part of storing to an atomic type.
1655
/// This needs to create a bit-pattern suitable for atomic operations.
1656
185
void AtomicInfo::emitCopyIntoMemory(RValue rvalue) const {
1657
185
  assert(LVal.isSimple());
1658
  // If we have an r-value, the rvalue should be of the atomic type,
1659
  // which means that the caller is responsible for having zeroed
1660
  // any padding.  Just do an aggregate copy of that type.
1661
185
  if (rvalue.isAggregate()) {
1662
0
    LValue Dest = CGF.MakeAddrLValue(getAtomicAddress(), getAtomicType());
1663
0
    LValue Src = CGF.MakeAddrLValue(rvalue.getAggregateAddress(),
1664
0
                                    getAtomicType());
1665
0
    bool IsVolatile = rvalue.isVolatileQualified() ||
1666
0
                      LVal.isVolatileQualified();
1667
0
    CGF.EmitAggregateCopy(Dest, Src, getAtomicType(),
1668
0
                          AggValueSlot::DoesNotOverlap, IsVolatile);
1669
0
    return;
1670
0
  }
1671
1672
  // Okay, otherwise we're copying stuff.
1673
1674
  // Zero out the buffer if necessary.
1675
185
  emitMemSetZeroIfNecessary();
1676
1677
  // Drill past the padding if present.
1678
185
  LValue TempLVal = projectValue();
1679
1680
  // Okay, store the rvalue in.
1681
185
  if (rvalue.isScalar()) {
1682
167
    CGF.EmitStoreOfScalar(rvalue.getScalarVal(), TempLVal, /*init*/ true);
1683
167
  } else {
1684
18
    CGF.EmitStoreOfComplex(rvalue.getComplexVal(), TempLVal, /*init*/ true);
1685
18
  }
1686
185
}
1687
1688
1689
/// Materialize an r-value into memory for the purposes of storing it
1690
/// to an atomic type.
1691
153
Address AtomicInfo::materializeRValue(RValue rvalue) const {
1692
  // Aggregate r-values are already in memory, and EmitAtomicStore
1693
  // requires them to be values of the atomic type.
1694
153
  if (rvalue.isAggregate())
1695
12
    return rvalue.getAggregateAddress();
1696
1697
  // Otherwise, make a temporary and materialize into it.
1698
141
  LValue TempLV = CGF.MakeAddrLValue(CreateTempAlloca(), getAtomicType());
1699
141
  AtomicInfo Atomics(CGF, TempLV);
1700
141
  Atomics.emitCopyIntoMemory(rvalue);
1701
141
  return TempLV.getAddress(CGF);
1702
153
}
1703
1704
144
llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal) const {
1705
  // If we've got a scalar value of the right size, try to avoid going
1706
  // through memory.
1707
144
  if (RVal.isScalar() && 
(133
!hasPadding()133
||
!LVal.isSimple()0
)) {
1708
133
    llvm::Value *Value = RVal.getScalarVal();
1709
133
    if (isa<llvm::IntegerType>(Value->getType()))
1710
103
      return CGF.EmitToMemory(Value, ValueTy);
1711
30
    else {
1712
30
      llvm::IntegerType *InputIntTy = llvm::IntegerType::get(
1713
30
          CGF.getLLVMContext(),
1714
30
          LVal.isSimple() ? getValueSizeInBits() : 
getAtomicSizeInBits()0
);
1715
30
      if (isa<llvm::PointerType>(Value->getType()))
1716
0
        return CGF.Builder.CreatePtrToInt(Value, InputIntTy);
1717
30
      else if (llvm::BitCastInst::isBitCastable(Value->getType(), InputIntTy))
1718
12
        return CGF.Builder.CreateBitCast(Value, InputIntTy);
1719
30
    }
1720
133
  }
1721
  // Otherwise, we need to go through memory.
1722
  // Put the r-value in memory.
1723
29
  Address Addr = materializeRValue(RVal);
1724
1725
  // Cast the temporary to the atomic int type and pull a value out.
1726
29
  Addr = emitCastToAtomicIntPointer(Addr);
1727
29
  return CGF.Builder.CreateLoad(Addr);
1728
144
}
1729
1730
std::pair<llvm::Value *, llvm::Value *> AtomicInfo::EmitAtomicCompareExchangeOp(
1731
    llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
1732
295
    llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak) {
1733
  // Do the atomic store.
1734
295
  Address Addr = getAtomicAddressAsAtomicIntPointer();
1735
295
  auto *Inst = CGF.Builder.CreateAtomicCmpXchg(Addr.getPointer(),
1736
295
                                               ExpectedVal, DesiredVal,
1737
295
                                               Success, Failure);
1738
  // Other decoration.
1739
295
  Inst->setVolatile(LVal.isVolatileQualified());
1740
295
  Inst->setWeak(IsWeak);
1741
1742
  // Okay, turn that back into the original value type.
1743
295
  auto *PreviousVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/0);
1744
295
  auto *SuccessFailureVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/1);
1745
295
  return std::make_pair(PreviousVal, SuccessFailureVal);
1746
295
}
1747
1748
llvm::Value *
1749
AtomicInfo::EmitAtomicCompareExchangeLibcall(llvm::Value *ExpectedAddr,
1750
                                             llvm::Value *DesiredAddr,
1751
                                             llvm::AtomicOrdering Success,
1752
78
                                             llvm::AtomicOrdering Failure) {
1753
  // bool __atomic_compare_exchange(size_t size, void *obj, void *expected,
1754
  // void *desired, int success, int failure);
1755
78
  CallArgList Args;
1756
78
  Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
1757
78
  Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())),
1758
78
           CGF.getContext().VoidPtrTy);
1759
78
  Args.add(RValue::get(CGF.EmitCastToVoidPtr(ExpectedAddr)),
1760
78
           CGF.getContext().VoidPtrTy);
1761
78
  Args.add(RValue::get(CGF.EmitCastToVoidPtr(DesiredAddr)),
1762
78
           CGF.getContext().VoidPtrTy);
1763
78
  Args.add(RValue::get(
1764
78
               llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Success))),
1765
78
           CGF.getContext().IntTy);
1766
78
  Args.add(RValue::get(
1767
78
               llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Failure))),
1768
78
           CGF.getContext().IntTy);
1769
78
  auto SuccessFailureRVal = emitAtomicLibcall(CGF, "__atomic_compare_exchange",
1770
78
                                              CGF.getContext().BoolTy, Args);
1771
1772
78
  return SuccessFailureRVal.getScalarVal();
1773
78
}
1774
1775
std::pair<RValue, llvm::Value *> AtomicInfo::EmitAtomicCompareExchange(
1776
    RValue Expected, RValue Desired, llvm::AtomicOrdering Success,
1777
29
    llvm::AtomicOrdering Failure, bool IsWeak) {
1778
  // Check whether we should use a library call.
1779
29
  if (shouldUseLibcall()) {
1780
    // Produce a source address.
1781
17
    Address ExpectedAddr = materializeRValue(Expected);
1782
17
    Address DesiredAddr = materializeRValue(Desired);
1783
17
    auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1784
17
                                                 DesiredAddr.getPointer(),
1785
17
                                                 Success, Failure);
1786
17
    return std::make_pair(
1787
17
        convertAtomicTempToRValue(ExpectedAddr, AggValueSlot::ignored(),
1788
17
                                  SourceLocation(), /*AsValue=*/false),
1789
17
        Res);
1790
17
  }
1791
1792
  // If we've got a scalar value of the right size, try to avoid going
1793
  // through memory.
1794
12
  auto *ExpectedVal = convertRValueToInt(Expected);
1795
12
  auto *DesiredVal = convertRValueToInt(Desired);
1796
12
  auto Res = EmitAtomicCompareExchangeOp(ExpectedVal, DesiredVal, Success,
1797
12
                                         Failure, IsWeak);
1798
12
  return std::make_pair(
1799
12
      ConvertIntToValueOrAtomic(Res.first, AggValueSlot::ignored(),
1800
12
                                SourceLocation(), /*AsValue=*/false),
1801
12
      Res.second);
1802
29
}
1803
1804
static void
1805
EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, RValue OldRVal,
1806
                      const llvm::function_ref<RValue(RValue)> &UpdateOp,
1807
320
                      Address DesiredAddr) {
1808
320
  RValue UpRVal;
1809
320
  LValue AtomicLVal = Atomics.getAtomicLValue();
1810
320
  LValue DesiredLVal;
1811
320
  if (AtomicLVal.isSimple()) {
1812
248
    UpRVal = OldRVal;
1813
248
    DesiredLVal = CGF.MakeAddrLValue(DesiredAddr, AtomicLVal.getType());
1814
248
  } else {
1815
    // Build new lvalue for temp address.
1816
72
    Address Ptr = Atomics.materializeRValue(OldRVal);
1817
72
    LValue UpdateLVal;
1818
72
    if (AtomicLVal.isBitField()) {
1819
60
      UpdateLVal =
1820
60
          LValue::MakeBitfield(Ptr, AtomicLVal.getBitFieldInfo(),
1821
60
                               AtomicLVal.getType(),
1822
60
                               AtomicLVal.getBaseInfo(),
1823
60
                               AtomicLVal.getTBAAInfo());
1824
60
      DesiredLVal =
1825
60
          LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(),
1826
60
                               AtomicLVal.getType(), AtomicLVal.getBaseInfo(),
1827
60
                               AtomicLVal.getTBAAInfo());
1828
60
    } else 
if (12
AtomicLVal.isVectorElt()12
) {
1829
6
      UpdateLVal = LValue::MakeVectorElt(Ptr, AtomicLVal.getVectorIdx(),
1830
6
                                         AtomicLVal.getType(),
1831
6
                                         AtomicLVal.getBaseInfo(),
1832
6
                                         AtomicLVal.getTBAAInfo());
1833
6
      DesiredLVal = LValue::MakeVectorElt(
1834
6
          DesiredAddr, AtomicLVal.getVectorIdx(), AtomicLVal.getType(),
1835
6
          AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo());
1836
6
    } else {
1837
6
      assert(AtomicLVal.isExtVectorElt());
1838
0
      UpdateLVal = LValue::MakeExtVectorElt(Ptr, AtomicLVal.getExtVectorElts(),
1839
6
                                            AtomicLVal.getType(),
1840
6
                                            AtomicLVal.getBaseInfo(),
1841
6
                                            AtomicLVal.getTBAAInfo());
1842
6
      DesiredLVal = LValue::MakeExtVectorElt(
1843
6
          DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(),
1844
6
          AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo());
1845
6
    }
1846
0
    UpRVal = CGF.EmitLoadOfLValue(UpdateLVal, SourceLocation());
1847
72
  }
1848
  // Store new value in the corresponding memory area.
1849
0
  RValue NewRVal = UpdateOp(UpRVal);
1850
320
  if (NewRVal.isScalar()) {
1851
288
    CGF.EmitStoreThroughLValue(NewRVal, DesiredLVal);
1852
288
  } else {
1853
32
    assert(NewRVal.isComplex());
1854
0
    CGF.EmitStoreOfComplex(NewRVal.getComplexVal(), DesiredLVal,
1855
32
                           /*isInit=*/false);
1856
32
  }
1857
320
}
1858
1859
void AtomicInfo::EmitAtomicUpdateLibcall(
1860
    llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1861
57
    bool IsVolatile) {
1862
57
  auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1863
1864
57
  Address ExpectedAddr = CreateTempAlloca();
1865
1866
57
  EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile);
1867
57
  auto *ContBB = CGF.createBasicBlock("atomic_cont");
1868
57
  auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1869
57
  CGF.EmitBlock(ContBB);
1870
57
  Address DesiredAddr = CreateTempAlloca();
1871
57
  if ((LVal.isBitField() && 
BFI.Size != ValueSizeInBits12
) ||
1872
57
      
requiresMemSetZero(getAtomicAddress().getElementType())45
) {
1873
12
    auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr);
1874
12
    CGF.Builder.CreateStore(OldVal, DesiredAddr);
1875
12
  }
1876
57
  auto OldRVal = convertAtomicTempToRValue(ExpectedAddr,
1877
57
                                           AggValueSlot::ignored(),
1878
57
                                           SourceLocation(), /*AsValue=*/false);
1879
57
  EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, DesiredAddr);
1880
57
  auto *Res =
1881
57
      EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1882
57
                                       DesiredAddr.getPointer(),
1883
57
                                       AO, Failure);
1884
57
  CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
1885
57
  CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1886
57
}
1887
1888
void AtomicInfo::EmitAtomicUpdateOp(
1889
    llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1890
263
    bool IsVolatile) {
1891
263
  auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1892
1893
  // Do the atomic load.
1894
263
  auto *OldVal = EmitAtomicLoadOp(Failure, IsVolatile);
1895
  // For non-simple lvalues perform compare-and-swap procedure.
1896
263
  auto *ContBB = CGF.createBasicBlock("atomic_cont");
1897
263
  auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1898
263
  auto *CurBB = CGF.Builder.GetInsertBlock();
1899
263
  CGF.EmitBlock(ContBB);
1900
263
  llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(),
1901
263
                                             /*NumReservedValues=*/2);
1902
263
  PHI->addIncoming(OldVal, CurBB);
1903
263
  Address NewAtomicAddr = CreateTempAlloca();
1904
263
  Address NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr);
1905
263
  if ((LVal.isBitField() && 
BFI.Size != ValueSizeInBits48
) ||
1906
263
      
requiresMemSetZero(getAtomicAddress().getElementType())215
) {
1907
72
    CGF.Builder.CreateStore(PHI, NewAtomicIntAddr);
1908
72
  }
1909
263
  auto OldRVal = ConvertIntToValueOrAtomic(PHI, AggValueSlot::ignored(),
1910
263
                                           SourceLocation(), /*AsValue=*/false);
1911
263
  EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, NewAtomicAddr);
1912
263
  auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr);
1913
  // Try to write new value using cmpxchg operation.
1914
263
  auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure);
1915
263
  PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock());
1916
263
  CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB);
1917
263
  CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1918
263
}
1919
1920
static void EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics,
1921
24
                                  RValue UpdateRVal, Address DesiredAddr) {
1922
24
  LValue AtomicLVal = Atomics.getAtomicLValue();
1923
24
  LValue DesiredLVal;
1924
  // Build new lvalue for temp address.
1925
24
  if (AtomicLVal.isBitField()) {
1926
20
    DesiredLVal =
1927
20
        LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(),
1928
20
                             AtomicLVal.getType(), AtomicLVal.getBaseInfo(),
1929
20
                             AtomicLVal.getTBAAInfo());
1930
20
  } else 
if (4
AtomicLVal.isVectorElt()4
) {
1931
2
    DesiredLVal =
1932
2
        LValue::MakeVectorElt(DesiredAddr, AtomicLVal.getVectorIdx(),
1933
2
                              AtomicLVal.getType(), AtomicLVal.getBaseInfo(),
1934
2
                              AtomicLVal.getTBAAInfo());
1935
2
  } else {
1936
2
    assert(AtomicLVal.isExtVectorElt());
1937
0
    DesiredLVal = LValue::MakeExtVectorElt(
1938
2
        DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(),
1939
2
        AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo());
1940
2
  }
1941
  // Store new value in the corresponding memory area.
1942
0
  assert(UpdateRVal.isScalar());
1943
0
  CGF.EmitStoreThroughLValue(UpdateRVal, DesiredLVal);
1944
24
}
1945
1946
void AtomicInfo::EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
1947
4
                                         RValue UpdateRVal, bool IsVolatile) {
1948
4
  auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1949
1950
4
  Address ExpectedAddr = CreateTempAlloca();
1951
1952
4
  EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile);
1953
4
  auto *ContBB = CGF.createBasicBlock("atomic_cont");
1954
4
  auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1955
4
  CGF.EmitBlock(ContBB);
1956
4
  Address DesiredAddr = CreateTempAlloca();
1957
4
  if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1958
4
      
requiresMemSetZero(getAtomicAddress().getElementType())0
) {
1959
4
    auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr);
1960
4
    CGF.Builder.CreateStore(OldVal, DesiredAddr);
1961
4
  }
1962
4
  EmitAtomicUpdateValue(CGF, *this, UpdateRVal, DesiredAddr);
1963
4
  auto *Res =
1964
4
      EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1965
4
                                       DesiredAddr.getPointer(),
1966
4
                                       AO, Failure);
1967
4
  CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
1968
4
  CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1969
4
}
1970
1971
void AtomicInfo::EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRVal,
1972
20
                                    bool IsVolatile) {
1973
20
  auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1974
1975
  // Do the atomic load.
1976
20
  auto *OldVal = EmitAtomicLoadOp(Failure, IsVolatile);
1977
  // For non-simple lvalues perform compare-and-swap procedure.
1978
20
  auto *ContBB = CGF.createBasicBlock("atomic_cont");
1979
20
  auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1980
20
  auto *CurBB = CGF.Builder.GetInsertBlock();
1981
20
  CGF.EmitBlock(ContBB);
1982
20
  llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(),
1983
20
                                             /*NumReservedValues=*/2);
1984
20
  PHI->addIncoming(OldVal, CurBB);
1985
20
  Address NewAtomicAddr = CreateTempAlloca();
1986
20
  Address NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr);
1987
20
  if ((LVal.isBitField() && 
BFI.Size != ValueSizeInBits16
) ||
1988
20
      
requiresMemSetZero(getAtomicAddress().getElementType())4
) {
1989
20
    CGF.Builder.CreateStore(PHI, NewAtomicIntAddr);
1990
20
  }
1991
20
  EmitAtomicUpdateValue(CGF, *this, UpdateRVal, NewAtomicAddr);
1992
20
  auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr);
1993
  // Try to write new value using cmpxchg operation.
1994
20
  auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure);
1995
20
  PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock());
1996
20
  CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB);
1997
20
  CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1998
20
}
1999
2000
void AtomicInfo::EmitAtomicUpdate(
2001
    llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
2002
320
    bool IsVolatile) {
2003
320
  if (shouldUseLibcall()) {
2004
57
    EmitAtomicUpdateLibcall(AO, UpdateOp, IsVolatile);
2005
263
  } else {
2006
263
    EmitAtomicUpdateOp(AO, UpdateOp, IsVolatile);
2007
263
  }
2008
320
}
2009
2010
void AtomicInfo::EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
2011
24
                                  bool IsVolatile) {
2012
24
  if (shouldUseLibcall()) {
2013
4
    EmitAtomicUpdateLibcall(AO, UpdateRVal, IsVolatile);
2014
20
  } else {
2015
20
    EmitAtomicUpdateOp(AO, UpdateRVal, IsVolatile);
2016
20
  }
2017
24
}
2018
2019
void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue lvalue,
2020
83
                                      bool isInit) {
2021
83
  bool IsVolatile = lvalue.isVolatileQualified();
2022
83
  llvm::AtomicOrdering AO;
2023
83
  if (lvalue.getType()->isAtomicType()) {
2024
76
    AO = llvm::AtomicOrdering::SequentiallyConsistent;
2025
76
  } else {
2026
7
    AO = llvm::AtomicOrdering::Release;
2027
7
    IsVolatile = true;
2028
7
  }
2029
83
  return EmitAtomicStore(rvalue, lvalue, AO, IsVolatile, isInit);
2030
83
}
2031
2032
/// Emit a store to an l-value of atomic type.
2033
///
2034
/// Note that the r-value is expected to be an r-value *of the atomic
2035
/// type*; this means that for aggregate r-values, it should include
2036
/// storage for any padding that was necessary.
2037
void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest,
2038
                                      llvm::AtomicOrdering AO, bool IsVolatile,
2039
195
                                      bool isInit) {
2040
  // If this is an aggregate r-value, it should agree in type except
2041
  // maybe for address-space qualification.
2042
195
  assert(!rvalue.isAggregate() ||
2043
195
         rvalue.getAggregateAddress().getElementType() ==
2044
195
             dest.getAddress(*this).getElementType());
2045
2046
0
  AtomicInfo atomics(*this, dest);
2047
195
  LValue LVal = atomics.getAtomicLValue();
2048
2049
  // If this is an initialization, just put the value there normally.
2050
195
  if (LVal.isSimple()) {
2051
171
    if (isInit) {
2052
33
      atomics.emitCopyIntoMemory(rvalue);
2053
33
      return;
2054
33
    }
2055
2056
    // Check whether we should use a library call.
2057
138
    if (atomics.shouldUseLibcall()) {
2058
      // Produce a source address.
2059
18
      Address srcAddr = atomics.materializeRValue(rvalue);
2060
2061
      // void __atomic_store(size_t size, void *mem, void *val, int order)
2062
18
      CallArgList args;
2063
18
      args.add(RValue::get(atomics.getAtomicSizeValue()),
2064
18
               getContext().getSizeType());
2065
18
      args.add(RValue::get(EmitCastToVoidPtr(atomics.getAtomicPointer())),
2066
18
               getContext().VoidPtrTy);
2067
18
      args.add(RValue::get(EmitCastToVoidPtr(srcAddr.getPointer())),
2068
18
               getContext().VoidPtrTy);
2069
18
      args.add(
2070
18
          RValue::get(llvm::ConstantInt::get(IntTy, (int)llvm::toCABI(AO))),
2071
18
          getContext().IntTy);
2072
18
      emitAtomicLibcall(*this, "__atomic_store", getContext().VoidTy, args);
2073
18
      return;
2074
18
    }
2075
2076
    // Okay, we're doing this natively.
2077
120
    llvm::Value *intValue = atomics.convertRValueToInt(rvalue);
2078
2079
    // Do the atomic store.
2080
120
    Address addr =
2081
120
        atomics.emitCastToAtomicIntPointer(atomics.getAtomicAddress());
2082
120
    intValue = Builder.CreateIntCast(
2083
120
        intValue, addr.getElementType(), /*isSigned=*/false);
2084
120
    llvm::StoreInst *store = Builder.CreateStore(intValue, addr);
2085
2086
120
    if (AO == llvm::AtomicOrdering::Acquire)
2087
0
      AO = llvm::AtomicOrdering::Monotonic;
2088
120
    else if (AO == llvm::AtomicOrdering::AcquireRelease)
2089
0
      AO = llvm::AtomicOrdering::Release;
2090
    // Initializations don't need to be atomic.
2091
120
    if (!isInit)
2092
120
      store->setAtomic(AO);
2093
2094
    // Other decoration.
2095
120
    if (IsVolatile)
2096
10
      store->setVolatile(true);
2097
120
    CGM.DecorateInstructionWithTBAA(store, dest.getTBAAInfo());
2098
120
    return;
2099
138
  }
2100
2101
  // Emit simple atomic update operation.
2102
24
  atomics.EmitAtomicUpdate(AO, rvalue, IsVolatile);
2103
24
}
2104
2105
/// Emit a compare-and-exchange op for atomic type.
2106
///
2107
std::pair<RValue, llvm::Value *> CodeGenFunction::EmitAtomicCompareExchange(
2108
    LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc,
2109
    llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak,
2110
29
    AggValueSlot Slot) {
2111
  // If this is an aggregate r-value, it should agree in type except
2112
  // maybe for address-space qualification.
2113
29
  assert(!Expected.isAggregate() ||
2114
29
         Expected.getAggregateAddress().getElementType() ==
2115
29
             Obj.getAddress(*this).getElementType());
2116
0
  assert(!Desired.isAggregate() ||
2117
29
         Desired.getAggregateAddress().getElementType() ==
2118
29
             Obj.getAddress(*this).getElementType());
2119
0
  AtomicInfo Atomics(*this, Obj);
2120
2121
29
  return Atomics.EmitAtomicCompareExchange(Expected, Desired, Success, Failure,
2122
29
                                           IsWeak);
2123
29
}
2124
2125
void CodeGenFunction::EmitAtomicUpdate(
2126
    LValue LVal, llvm::AtomicOrdering AO,
2127
320
    const llvm::function_ref<RValue(RValue)> &UpdateOp, bool IsVolatile) {
2128
320
  AtomicInfo Atomics(*this, LVal);
2129
320
  Atomics.EmitAtomicUpdate(AO, UpdateOp, IsVolatile);
2130
320
}
2131
2132
20
void CodeGenFunction::EmitAtomicInit(Expr *init, LValue dest) {
2133
20
  AtomicInfo atomics(*this, dest);
2134
2135
20
  switch (atomics.getEvaluationKind()) {
2136
9
  case TEK_Scalar: {
2137
9
    llvm::Value *value = EmitScalarExpr(init);
2138
9
    atomics.emitCopyIntoMemory(RValue::get(value));
2139
9
    return;
2140
0
  }
2141
2142
2
  case TEK_Complex: {
2143
2
    ComplexPairTy value = EmitComplexExpr(init);
2144
2
    atomics.emitCopyIntoMemory(RValue::getComplex(value));
2145
2
    return;
2146
0
  }
2147
2148
9
  case TEK_Aggregate: {
2149
    // Fix up the destination if the initializer isn't an expression
2150
    // of atomic type.
2151
9
    bool Zeroed = false;
2152
9
    if (!init->getType()->isAtomicType()) {
2153
5
      Zeroed = atomics.emitMemSetZeroIfNecessary();
2154
5
      dest = atomics.projectValue();
2155
5
    }
2156
2157
    // Evaluate the expression directly into the destination.
2158
9
    AggValueSlot slot = AggValueSlot::forLValue(
2159
9
        dest, *this, AggValueSlot::IsNotDestructed,
2160
9
        AggValueSlot::DoesNotNeedGCBarriers, AggValueSlot::IsNotAliased,
2161
9
        AggValueSlot::DoesNotOverlap,
2162
9
        Zeroed ? 
AggValueSlot::IsZeroed2
:
AggValueSlot::IsNotZeroed7
);
2163
2164
9
    EmitAggExpr(init, slot);
2165
9
    return;
2166
0
  }
2167
20
  }
2168
0
  llvm_unreachable("bad evaluation kind");
2169
0
}