Coverage Report

Created: 2019-07-24 05:18

/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/tools/clang/lib/CodeGen/CGAtomic.cpp
Line
Count
Source (jump to first uncovered line)
1
//===--- CGAtomic.cpp - Emit LLVM IR for atomic operations ----------------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// This file contains the code for emitting atomic operations.
10
//
11
//===----------------------------------------------------------------------===//
12
13
#include "CGCall.h"
14
#include "CGRecordLayout.h"
15
#include "CodeGenFunction.h"
16
#include "CodeGenModule.h"
17
#include "TargetInfo.h"
18
#include "clang/AST/ASTContext.h"
19
#include "clang/CodeGen/CGFunctionInfo.h"
20
#include "clang/Frontend/FrontendDiagnostic.h"
21
#include "llvm/ADT/DenseMap.h"
22
#include "llvm/IR/DataLayout.h"
23
#include "llvm/IR/Intrinsics.h"
24
#include "llvm/IR/Operator.h"
25
26
using namespace clang;
27
using namespace CodeGen;
28
29
namespace {
30
  class AtomicInfo {
31
    CodeGenFunction &CGF;
32
    QualType AtomicTy;
33
    QualType ValueTy;
34
    uint64_t AtomicSizeInBits;
35
    uint64_t ValueSizeInBits;
36
    CharUnits AtomicAlign;
37
    CharUnits ValueAlign;
38
    TypeEvaluationKind EvaluationKind;
39
    bool UseLibcall;
40
    LValue LVal;
41
    CGBitFieldInfo BFI;
42
  public:
43
    AtomicInfo(CodeGenFunction &CGF, LValue &lvalue)
44
        : CGF(CGF), AtomicSizeInBits(0), ValueSizeInBits(0),
45
1.38k
          EvaluationKind(TEK_Scalar), UseLibcall(true) {
46
1.38k
      assert(!lvalue.isGlobalReg());
47
1.38k
      ASTContext &C = CGF.getContext();
48
1.38k
      if (lvalue.isSimple()) {
49
1.29k
        AtomicTy = lvalue.getType();
50
1.29k
        if (auto *ATy = AtomicTy->getAs<AtomicType>())
51
507
          ValueTy = ATy->getValueType();
52
785
        else
53
785
          ValueTy = AtomicTy;
54
1.29k
        EvaluationKind = CGF.getEvaluationKind(ValueTy);
55
1.29k
56
1.29k
        uint64_t ValueAlignInBits;
57
1.29k
        uint64_t AtomicAlignInBits;
58
1.29k
        TypeInfo ValueTI = C.getTypeInfo(ValueTy);
59
1.29k
        ValueSizeInBits = ValueTI.Width;
60
1.29k
        ValueAlignInBits = ValueTI.Align;
61
1.29k
62
1.29k
        TypeInfo AtomicTI = C.getTypeInfo(AtomicTy);
63
1.29k
        AtomicSizeInBits = AtomicTI.Width;
64
1.29k
        AtomicAlignInBits = AtomicTI.Align;
65
1.29k
66
1.29k
        assert(ValueSizeInBits <= AtomicSizeInBits);
67
1.29k
        assert(ValueAlignInBits <= AtomicAlignInBits);
68
1.29k
69
1.29k
        AtomicAlign = C.toCharUnitsFromBits(AtomicAlignInBits);
70
1.29k
        ValueAlign = C.toCharUnitsFromBits(ValueAlignInBits);
71
1.29k
        if (lvalue.getAlignment().isZero())
72
0
          lvalue.setAlignment(AtomicAlign);
73
1.29k
74
1.29k
        LVal = lvalue;
75
1.29k
      } else 
if (96
lvalue.isBitField()96
) {
76
80
        ValueTy = lvalue.getType();
77
80
        ValueSizeInBits = C.getTypeSize(ValueTy);
78
80
        auto &OrigBFI = lvalue.getBitFieldInfo();
79
80
        auto Offset = OrigBFI.Offset % C.toBits(lvalue.getAlignment());
80
80
        AtomicSizeInBits = C.toBits(
81
80
            C.toCharUnitsFromBits(Offset + OrigBFI.Size + C.getCharWidth() - 1)
82
80
                .alignTo(lvalue.getAlignment()));
83
80
        auto VoidPtrAddr = CGF.EmitCastToVoidPtr(lvalue.getBitFieldPointer());
84
80
        auto OffsetInChars =
85
80
            (C.toCharUnitsFromBits(OrigBFI.Offset) / lvalue.getAlignment()) *
86
80
            lvalue.getAlignment();
87
80
        VoidPtrAddr = CGF.Builder.CreateConstGEP1_64(
88
80
            VoidPtrAddr, OffsetInChars.getQuantity());
89
80
        auto Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
90
80
            VoidPtrAddr,
91
80
            CGF.Builder.getIntNTy(AtomicSizeInBits)->getPointerTo(),
92
80
            "atomic_bitfield_base");
93
80
        BFI = OrigBFI;
94
80
        BFI.Offset = Offset;
95
80
        BFI.StorageSize = AtomicSizeInBits;
96
80
        BFI.StorageOffset += OffsetInChars;
97
80
        LVal = LValue::MakeBitfield(Address(Addr, lvalue.getAlignment()),
98
80
                                    BFI, lvalue.getType(), lvalue.getBaseInfo(),
99
80
                                    lvalue.getTBAAInfo());
100
80
        AtomicTy = C.getIntTypeForBitwidth(AtomicSizeInBits, OrigBFI.IsSigned);
101
80
        if (AtomicTy.isNull()) {
102
8
          llvm::APInt Size(
103
8
              /*numBits=*/32,
104
8
              C.toCharUnitsFromBits(AtomicSizeInBits).getQuantity());
105
8
          AtomicTy = C.getConstantArrayType(C.CharTy, Size, ArrayType::Normal,
106
8
                                            /*IndexTypeQuals=*/0);
107
8
        }
108
80
        AtomicAlign = ValueAlign = lvalue.getAlignment();
109
80
      } else 
if (16
lvalue.isVectorElt()16
) {
110
8
        ValueTy = lvalue.getType()->getAs<VectorType>()->getElementType();
111
8
        ValueSizeInBits = C.getTypeSize(ValueTy);
112
8
        AtomicTy = lvalue.getType();
113
8
        AtomicSizeInBits = C.getTypeSize(AtomicTy);
114
8
        AtomicAlign = ValueAlign = lvalue.getAlignment();
115
8
        LVal = lvalue;
116
8
      } else {
117
8
        assert(lvalue.isExtVectorElt());
118
8
        ValueTy = lvalue.getType();
119
8
        ValueSizeInBits = C.getTypeSize(ValueTy);
120
8
        AtomicTy = ValueTy = CGF.getContext().getExtVectorType(
121
8
            lvalue.getType(), lvalue.getExtVectorAddress()
122
8
                                  .getElementType()->getVectorNumElements());
123
8
        AtomicSizeInBits = C.getTypeSize(AtomicTy);
124
8
        AtomicAlign = ValueAlign = lvalue.getAlignment();
125
8
        LVal = lvalue;
126
8
      }
127
1.38k
      UseLibcall = !C.getTargetInfo().hasBuiltinAtomic(
128
1.38k
          AtomicSizeInBits, C.toBits(lvalue.getAlignment()));
129
1.38k
    }
130
131
109
    QualType getAtomicType() const { return AtomicTy; }
132
240
    QualType getValueType() const { return ValueTy; }
133
1.99k
    CharUnits getAtomicAlignment() const { return AtomicAlign; }
134
0
    uint64_t getAtomicSizeInBits() const { return AtomicSizeInBits; }
135
30
    uint64_t getValueSizeInBits() const { return ValueSizeInBits; }
136
799
    TypeEvaluationKind getEvaluationKind() const { return EvaluationKind; }
137
569
    bool shouldUseLibcall() const { return UseLibcall; }
138
379
    const LValue &getAtomicLValue() const { return LVal; }
139
1.38k
    llvm::Value *getAtomicPointer() const {
140
1.38k
      if (LVal.isSimple())
141
1.03k
        return LVal.getPointer();
142
352
      else if (LVal.isBitField())
143
304
        return LVal.getBitFieldPointer();
144
48
      else if (LVal.isVectorElt())
145
24
        return LVal.getVectorPointer();
146
24
      assert(LVal.isExtVectorElt());
147
24
      return LVal.getExtVectorPointer();
148
24
    }
149
1.24k
    Address getAtomicAddress() const {
150
1.24k
      return Address(getAtomicPointer(), getAtomicAlignment());
151
1.24k
    }
152
153
525
    Address getAtomicAddressAsAtomicIntPointer() const {
154
525
      return emitCastToAtomicIntPointer(getAtomicAddress());
155
525
    }
156
157
    /// Is the atomic size larger than the underlying value type?
158
    ///
159
    /// Note that the absence of padding does not mean that atomic
160
    /// objects are completely interchangeable with non-atomic
161
    /// objects: we might have promoted the alignment of a type
162
    /// without making it bigger.
163
915
    bool hasPadding() const {
164
915
      return (ValueSizeInBits != AtomicSizeInBits);
165
915
    }
166
167
    bool emitMemSetZeroIfNecessary() const;
168
169
135
    llvm::Value *getAtomicSizeValue() const {
170
135
      CharUnits size = CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits);
171
135
      return CGF.CGM.getSize(size);
172
135
    }
173
174
    /// Cast the given pointer to an integer pointer suitable for atomic
175
    /// operations if the source.
176
    Address emitCastToAtomicIntPointer(Address Addr) const;
177
178
    /// If Addr is compatible with the iN that will be used for an atomic
179
    /// operation, bitcast it. Otherwise, create a temporary that is suitable
180
    /// and copy the value across.
181
    Address convertToAtomicIntPointer(Address Addr) const;
182
183
    /// Turn an atomic-layout object into an r-value.
184
    RValue convertAtomicTempToRValue(Address addr, AggValueSlot resultSlot,
185
                                     SourceLocation loc, bool AsValue) const;
186
187
    /// Converts a rvalue to integer value.
188
    llvm::Value *convertRValueToInt(RValue RVal) const;
189
190
    RValue ConvertIntToValueOrAtomic(llvm::Value *IntVal,
191
                                     AggValueSlot ResultSlot,
192
                                     SourceLocation Loc, bool AsValue) const;
193
194
    /// Copy an atomic r-value into atomic-layout memory.
195
    void emitCopyIntoMemory(RValue rvalue) const;
196
197
    /// Project an l-value down to the value field.
198
145
    LValue projectValue() const {
199
145
      assert(LVal.isSimple());
200
145
      Address addr = getAtomicAddress();
201
145
      if (hasPadding())
202
2
        addr = CGF.Builder.CreateStructGEP(addr, 0);
203
145
204
145
      return LValue::MakeAddr(addr, getValueType(), CGF.getContext(),
205
145
                              LVal.getBaseInfo(), LVal.getTBAAInfo());
206
145
    }
207
208
    /// Emits atomic load.
209
    /// \returns Loaded value.
210
    RValue EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
211
                          bool AsValue, llvm::AtomicOrdering AO,
212
                          bool IsVolatile);
213
214
    /// Emits atomic compare-and-exchange sequence.
215
    /// \param Expected Expected value.
216
    /// \param Desired Desired value.
217
    /// \param Success Atomic ordering for success operation.
218
    /// \param Failure Atomic ordering for failed operation.
219
    /// \param IsWeak true if atomic operation is weak, false otherwise.
220
    /// \returns Pair of values: previous value from storage (value type) and
221
    /// boolean flag (i1 type) with true if success and false otherwise.
222
    std::pair<RValue, llvm::Value *>
223
    EmitAtomicCompareExchange(RValue Expected, RValue Desired,
224
                              llvm::AtomicOrdering Success =
225
                                  llvm::AtomicOrdering::SequentiallyConsistent,
226
                              llvm::AtomicOrdering Failure =
227
                                  llvm::AtomicOrdering::SequentiallyConsistent,
228
                              bool IsWeak = false);
229
230
    /// Emits atomic update.
231
    /// \param AO Atomic ordering.
232
    /// \param UpdateOp Update operation for the current lvalue.
233
    void EmitAtomicUpdate(llvm::AtomicOrdering AO,
234
                          const llvm::function_ref<RValue(RValue)> &UpdateOp,
235
                          bool IsVolatile);
236
    /// Emits atomic update.
237
    /// \param AO Atomic ordering.
238
    void EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
239
                          bool IsVolatile);
240
241
    /// Materialize an atomic r-value in atomic-layout memory.
242
    Address materializeRValue(RValue rvalue) const;
243
244
    /// Creates temp alloca for intermediate operations on atomic value.
245
    Address CreateTempAlloca() const;
246
  private:
247
    bool requiresMemSetZero(llvm::Type *type) const;
248
249
250
    /// Emits atomic load as a libcall.
251
    void EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
252
                               llvm::AtomicOrdering AO, bool IsVolatile);
253
    /// Emits atomic load as LLVM instruction.
254
    llvm::Value *EmitAtomicLoadOp(llvm::AtomicOrdering AO, bool IsVolatile);
255
    /// Emits atomic compare-and-exchange op as a libcall.
256
    llvm::Value *EmitAtomicCompareExchangeLibcall(
257
        llvm::Value *ExpectedAddr, llvm::Value *DesiredAddr,
258
        llvm::AtomicOrdering Success =
259
            llvm::AtomicOrdering::SequentiallyConsistent,
260
        llvm::AtomicOrdering Failure =
261
            llvm::AtomicOrdering::SequentiallyConsistent);
262
    /// Emits atomic compare-and-exchange op as LLVM instruction.
263
    std::pair<llvm::Value *, llvm::Value *> EmitAtomicCompareExchangeOp(
264
        llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
265
        llvm::AtomicOrdering Success =
266
            llvm::AtomicOrdering::SequentiallyConsistent,
267
        llvm::AtomicOrdering Failure =
268
            llvm::AtomicOrdering::SequentiallyConsistent,
269
        bool IsWeak = false);
270
    /// Emit atomic update as libcalls.
271
    void
272
    EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
273
                            const llvm::function_ref<RValue(RValue)> &UpdateOp,
274
                            bool IsVolatile);
275
    /// Emit atomic update as LLVM instructions.
276
    void EmitAtomicUpdateOp(llvm::AtomicOrdering AO,
277
                            const llvm::function_ref<RValue(RValue)> &UpdateOp,
278
                            bool IsVolatile);
279
    /// Emit atomic update as libcalls.
280
    void EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, RValue UpdateRVal,
281
                                 bool IsVolatile);
282
    /// Emit atomic update as LLVM instructions.
283
    void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRal,
284
                            bool IsVolatile);
285
  };
286
}
287
288
752
Address AtomicInfo::CreateTempAlloca() const {
289
752
  Address TempAlloca = CGF.CreateMemTemp(
290
752
      (LVal.isBitField() && 
ValueSizeInBits > AtomicSizeInBits132
) ?
ValueTy54
291
752
                                                                : 
AtomicTy698
,
292
752
      getAtomicAlignment(),
293
752
      "atomic-temp");
294
752
  // Cast to pointer to value type for bitfields.
295
752
  if (LVal.isBitField())
296
132
    return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
297
132
        TempAlloca, getAtomicAddress().getType());
298
620
  return TempAlloca;
299
620
}
300
301
static RValue emitAtomicLibcall(CodeGenFunction &CGF,
302
                                StringRef fnName,
303
                                QualType resultType,
304
293
                                CallArgList &args) {
305
293
  const CGFunctionInfo &fnInfo =
306
293
    CGF.CGM.getTypes().arrangeBuiltinFunctionCall(resultType, args);
307
293
  llvm::FunctionType *fnTy = CGF.CGM.getTypes().GetFunctionType(fnInfo);
308
293
  llvm::FunctionCallee fn = CGF.CGM.CreateRuntimeFunction(fnTy, fnName);
309
293
  auto callee = CGCallee::forDirect(fn);
310
293
  return CGF.EmitCall(fnInfo, callee, ReturnValueSlot(), args);
311
293
}
312
313
/// Does a store of the given IR type modify the full expected width?
314
static bool isFullSizeType(CodeGenModule &CGM, llvm::Type *type,
315
291
                           uint64_t expectedSize) {
316
291
  return (CGM.getDataLayout().getTypeStoreSize(type) * 8 == expectedSize);
317
291
}
318
319
/// Does the atomic type require memsetting to zero before initialization?
320
///
321
/// The IR type is provided as a way of making certain queries faster.
322
312
bool AtomicInfo::requiresMemSetZero(llvm::Type *type) const {
323
312
  // If the atomic type has size padding, we definitely need a memset.
324
312
  if (hasPadding()) 
return true14
;
325
298
326
298
  // Otherwise, do some simple heuristics to try to avoid it:
327
298
  switch (getEvaluationKind()) {
328
298
  // For scalars and complexes, check whether the store size of the
329
298
  // type uses the full size.
330
298
  case TEK_Scalar:
331
251
    return !isFullSizeType(CGF.CGM, type, AtomicSizeInBits);
332
298
  case TEK_Complex:
333
40
    return !isFullSizeType(CGF.CGM, type->getStructElementType(0),
334
40
                           AtomicSizeInBits / 2);
335
298
336
298
  // Padding in structs has an undefined bit pattern.  User beware.
337
298
  case TEK_Aggregate:
338
7
    return false;
339
0
  }
340
0
  llvm_unreachable("bad evaluation kind");
341
0
}
342
343
145
bool AtomicInfo::emitMemSetZeroIfNecessary() const {
344
145
  assert(LVal.isSimple());
345
145
  llvm::Value *addr = LVal.getPointer();
346
145
  if (!requiresMemSetZero(addr->getType()->getPointerElementType()))
347
111
    return false;
348
34
349
34
  CGF.Builder.CreateMemSet(
350
34
      addr, llvm::ConstantInt::get(CGF.Int8Ty, 0),
351
34
      CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits).getQuantity(),
352
34
      LVal.getAlignment().getQuantity());
353
34
  return true;
354
34
}
355
356
static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak,
357
                              Address Dest, Address Ptr,
358
                              Address Val1, Address Val2,
359
                              uint64_t Size,
360
                              llvm::AtomicOrdering SuccessOrder,
361
                              llvm::AtomicOrdering FailureOrder,
362
114
                              llvm::SyncScope::ID Scope) {
363
114
  // Note that cmpxchg doesn't support weak cmpxchg, at least at the moment.
364
114
  llvm::Value *Expected = CGF.Builder.CreateLoad(Val1);
365
114
  llvm::Value *Desired = CGF.Builder.CreateLoad(Val2);
366
114
367
114
  llvm::AtomicCmpXchgInst *Pair = CGF.Builder.CreateAtomicCmpXchg(
368
114
      Ptr.getPointer(), Expected, Desired, SuccessOrder, FailureOrder,
369
114
      Scope);
370
114
  Pair->setVolatile(E->isVolatile());
371
114
  Pair->setWeak(IsWeak);
372
114
373
114
  // Cmp holds the result of the compare-exchange operation: true on success,
374
114
  // false on failure.
375
114
  llvm::Value *Old = CGF.Builder.CreateExtractValue(Pair, 0);
376
114
  llvm::Value *Cmp = CGF.Builder.CreateExtractValue(Pair, 1);
377
114
378
114
  // This basic block is used to hold the store instruction if the operation
379
114
  // failed.
380
114
  llvm::BasicBlock *StoreExpectedBB =
381
114
      CGF.createBasicBlock("cmpxchg.store_expected", CGF.CurFn);
382
114
383
114
  // This basic block is the exit point of the operation, we should end up
384
114
  // here regardless of whether or not the operation succeeded.
385
114
  llvm::BasicBlock *ContinueBB =
386
114
      CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
387
114
388
114
  // Update Expected if Expected isn't equal to Old, otherwise branch to the
389
114
  // exit point.
390
114
  CGF.Builder.CreateCondBr(Cmp, ContinueBB, StoreExpectedBB);
391
114
392
114
  CGF.Builder.SetInsertPoint(StoreExpectedBB);
393
114
  // Update the memory at Expected with Old's value.
394
114
  CGF.Builder.CreateStore(Old, Val1);
395
114
  // Finally, branch to the exit point.
396
114
  CGF.Builder.CreateBr(ContinueBB);
397
114
398
114
  CGF.Builder.SetInsertPoint(ContinueBB);
399
114
  // Update the memory at Dest with Cmp's value.
400
114
  CGF.EmitStoreOfScalar(Cmp, CGF.MakeAddrLValue(Dest, E->getType()));
401
114
}
402
403
/// Given an ordering required on success, emit all possible cmpxchg
404
/// instructions to cope with the provided (but possibly only dynamically known)
405
/// FailureOrder.
406
static void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E,
407
                                        bool IsWeak, Address Dest, Address Ptr,
408
                                        Address Val1, Address Val2,
409
                                        llvm::Value *FailureOrderVal,
410
                                        uint64_t Size,
411
                                        llvm::AtomicOrdering SuccessOrder,
412
80
                                        llvm::SyncScope::ID Scope) {
413
80
  llvm::AtomicOrdering FailureOrder;
414
80
  if (llvm::ConstantInt *FO = dyn_cast<llvm::ConstantInt>(FailureOrderVal)) {
415
38
    auto FOS = FO->getSExtValue();
416
38
    if (!llvm::isValidAtomicOrderingCABI(FOS))
417
2
      FailureOrder = llvm::AtomicOrdering::Monotonic;
418
36
    else
419
36
      switch ((llvm::AtomicOrderingCABI)FOS) {
420
36
      case llvm::AtomicOrderingCABI::relaxed:
421
7
      case llvm::AtomicOrderingCABI::release:
422
7
      case llvm::AtomicOrderingCABI::acq_rel:
423
7
        FailureOrder = llvm::AtomicOrdering::Monotonic;
424
7
        break;
425
20
      case llvm::AtomicOrderingCABI::consume:
426
20
      case llvm::AtomicOrderingCABI::acquire:
427
20
        FailureOrder = llvm::AtomicOrdering::Acquire;
428
20
        break;
429
20
      case llvm::AtomicOrderingCABI::seq_cst:
430
9
        FailureOrder = llvm::AtomicOrdering::SequentiallyConsistent;
431
9
        break;
432
38
      }
433
38
    if (isStrongerThan(FailureOrder, SuccessOrder)) {
434
0
      // Don't assert on undefined behavior "failure argument shall be no
435
0
      // stronger than the success argument".
436
0
      FailureOrder =
437
0
          llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrder);
438
0
    }
439
38
    emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
440
38
                      FailureOrder, Scope);
441
38
    return;
442
38
  }
443
42
444
42
  // Create all the relevant BB's
445
42
  llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr,
446
42
                   *SeqCstBB = nullptr;
447
42
  MonotonicBB = CGF.createBasicBlock("monotonic_fail", CGF.CurFn);
448
42
  if (SuccessOrder != llvm::AtomicOrdering::Monotonic &&
449
42
      
SuccessOrder != llvm::AtomicOrdering::Release34
)
450
26
    AcquireBB = CGF.createBasicBlock("acquire_fail", CGF.CurFn);
451
42
  if (SuccessOrder == llvm::AtomicOrdering::SequentiallyConsistent)
452
8
    SeqCstBB = CGF.createBasicBlock("seqcst_fail", CGF.CurFn);
453
42
454
42
  llvm::BasicBlock *ContBB = CGF.createBasicBlock("atomic.continue", CGF.CurFn);
455
42
456
42
  llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(FailureOrderVal, MonotonicBB);
457
42
458
42
  // Emit all the different atomics
459
42
460
42
  // MonotonicBB is arbitrarily chosen as the default case; in practice, this
461
42
  // doesn't matter unless someone is crazy enough to use something that
462
42
  // doesn't fold to a constant for the ordering.
463
42
  CGF.Builder.SetInsertPoint(MonotonicBB);
464
42
  emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
465
42
                    Size, SuccessOrder, llvm::AtomicOrdering::Monotonic, Scope);
466
42
  CGF.Builder.CreateBr(ContBB);
467
42
468
42
  if (AcquireBB) {
469
26
    CGF.Builder.SetInsertPoint(AcquireBB);
470
26
    emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
471
26
                      Size, SuccessOrder, llvm::AtomicOrdering::Acquire, Scope);
472
26
    CGF.Builder.CreateBr(ContBB);
473
26
    SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::consume),
474
26
                AcquireBB);
475
26
    SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire),
476
26
                AcquireBB);
477
26
  }
478
42
  if (SeqCstBB) {
479
8
    CGF.Builder.SetInsertPoint(SeqCstBB);
480
8
    emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
481
8
                      llvm::AtomicOrdering::SequentiallyConsistent, Scope);
482
8
    CGF.Builder.CreateBr(ContBB);
483
8
    SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst),
484
8
                SeqCstBB);
485
8
  }
486
42
487
42
  CGF.Builder.SetInsertPoint(ContBB);
488
42
}
489
490
static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest,
491
                         Address Ptr, Address Val1, Address Val2,
492
                         llvm::Value *IsWeak, llvm::Value *FailureOrder,
493
                         uint64_t Size, llvm::AtomicOrdering Order,
494
822
                         llvm::SyncScope::ID Scope) {
495
822
  llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add;
496
822
  llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0;
497
822
498
822
  switch (E->getOp()) {
499
822
  case AtomicExpr::AO__c11_atomic_init:
500
0
  case AtomicExpr::AO__opencl_atomic_init:
501
0
    llvm_unreachable("Already handled!");
502
0
503
35
  case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
504
35
  case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
505
35
    emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
506
35
                                FailureOrder, Size, Order, Scope);
507
35
    return;
508
35
  case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
509
4
  case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
510
4
    emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
511
4
                                FailureOrder, Size, Order, Scope);
512
4
    return;
513
29
  case AtomicExpr::AO__atomic_compare_exchange:
514
29
  case AtomicExpr::AO__atomic_compare_exchange_n: {
515
29
    if (llvm::ConstantInt *IsWeakC = dyn_cast<llvm::ConstantInt>(IsWeak)) {
516
17
      emitAtomicCmpXchgFailureSet(CGF, E, IsWeakC->getZExtValue(), Dest, Ptr,
517
17
                                  Val1, Val2, FailureOrder, Size, Order, Scope);
518
17
    } else {
519
12
      // Create all the relevant BB's
520
12
      llvm::BasicBlock *StrongBB =
521
12
          CGF.createBasicBlock("cmpxchg.strong", CGF.CurFn);
522
12
      llvm::BasicBlock *WeakBB = CGF.createBasicBlock("cmxchg.weak", CGF.CurFn);
523
12
      llvm::BasicBlock *ContBB =
524
12
          CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
525
12
526
12
      llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(IsWeak, WeakBB);
527
12
      SI->addCase(CGF.Builder.getInt1(false), StrongBB);
528
12
529
12
      CGF.Builder.SetInsertPoint(StrongBB);
530
12
      emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
531
12
                                  FailureOrder, Size, Order, Scope);
532
12
      CGF.Builder.CreateBr(ContBB);
533
12
534
12
      CGF.Builder.SetInsertPoint(WeakBB);
535
12
      emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
536
12
                                  FailureOrder, Size, Order, Scope);
537
12
      CGF.Builder.CreateBr(ContBB);
538
12
539
12
      CGF.Builder.SetInsertPoint(ContBB);
540
12
    }
541
29
    return;
542
29
  }
543
166
  case AtomicExpr::AO__c11_atomic_load:
544
166
  case AtomicExpr::AO__opencl_atomic_load:
545
166
  case AtomicExpr::AO__atomic_load_n:
546
166
  case AtomicExpr::AO__atomic_load: {
547
166
    llvm::LoadInst *Load = CGF.Builder.CreateLoad(Ptr);
548
166
    Load->setAtomic(Order, Scope);
549
166
    Load->setVolatile(E->isVolatile());
550
166
    CGF.Builder.CreateStore(Load, Dest);
551
166
    return;
552
166
  }
553
166
554
248
  case AtomicExpr::AO__c11_atomic_store:
555
248
  case AtomicExpr::AO__opencl_atomic_store:
556
248
  case AtomicExpr::AO__atomic_store:
557
248
  case AtomicExpr::AO__atomic_store_n: {
558
248
    llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1);
559
248
    llvm::StoreInst *Store = CGF.Builder.CreateStore(LoadVal1, Ptr);
560
248
    Store->setAtomic(Order, Scope);
561
248
    Store->setVolatile(E->isVolatile());
562
248
    return;
563
248
  }
564
248
565
248
  case AtomicExpr::AO__c11_atomic_exchange:
566
205
  case AtomicExpr::AO__opencl_atomic_exchange:
567
205
  case AtomicExpr::AO__atomic_exchange_n:
568
205
  case AtomicExpr::AO__atomic_exchange:
569
205
    Op = llvm::AtomicRMWInst::Xchg;
570
205
    break;
571
205
572
205
  case AtomicExpr::AO__atomic_add_fetch:
573
16
    PostOp = llvm::Instruction::Add;
574
16
    LLVM_FALLTHROUGH;
575
80
  case AtomicExpr::AO__c11_atomic_fetch_add:
576
80
  case AtomicExpr::AO__opencl_atomic_fetch_add:
577
80
  case AtomicExpr::AO__atomic_fetch_add:
578
80
    Op = llvm::AtomicRMWInst::Add;
579
80
    break;
580
80
581
80
  case AtomicExpr::AO__atomic_sub_fetch:
582
0
    PostOp = llvm::Instruction::Sub;
583
0
    LLVM_FALLTHROUGH;
584
18
  case AtomicExpr::AO__c11_atomic_fetch_sub:
585
18
  case AtomicExpr::AO__opencl_atomic_fetch_sub:
586
18
  case AtomicExpr::AO__atomic_fetch_sub:
587
18
    Op = llvm::AtomicRMWInst::Sub;
588
18
    break;
589
18
590
18
  case AtomicExpr::AO__opencl_atomic_fetch_min:
591
6
  case AtomicExpr::AO__atomic_fetch_min:
592
6
    Op = E->getValueType()->isSignedIntegerType() ? 
llvm::AtomicRMWInst::Min3
593
6
                                                  : 
llvm::AtomicRMWInst::UMin3
;
594
6
    break;
595
6
596
6
  case AtomicExpr::AO__opencl_atomic_fetch_max:
597
6
  case AtomicExpr::AO__atomic_fetch_max:
598
6
    Op = E->getValueType()->isSignedIntegerType() ? 
llvm::AtomicRMWInst::Max3
599
6
                                                  : 
llvm::AtomicRMWInst::UMax3
;
600
6
    break;
601
6
602
6
  case AtomicExpr::AO__atomic_and_fetch:
603
0
    PostOp = llvm::Instruction::And;
604
0
    LLVM_FALLTHROUGH;
605
6
  case AtomicExpr::AO__c11_atomic_fetch_and:
606
6
  case AtomicExpr::AO__opencl_atomic_fetch_and:
607
6
  case AtomicExpr::AO__atomic_fetch_and:
608
6
    Op = llvm::AtomicRMWInst::And;
609
6
    break;
610
6
611
6
  case AtomicExpr::AO__atomic_or_fetch:
612
2
    PostOp = llvm::Instruction::Or;
613
2
    LLVM_FALLTHROUGH;
614
7
  case AtomicExpr::AO__c11_atomic_fetch_or:
615
7
  case AtomicExpr::AO__opencl_atomic_fetch_or:
616
7
  case AtomicExpr::AO__atomic_fetch_or:
617
7
    Op = llvm::AtomicRMWInst::Or;
618
7
    break;
619
7
620
7
  case AtomicExpr::AO__atomic_xor_fetch:
621
0
    PostOp = llvm::Instruction::Xor;
622
0
    LLVM_FALLTHROUGH;
623
4
  case AtomicExpr::AO__c11_atomic_fetch_xor:
624
4
  case AtomicExpr::AO__opencl_atomic_fetch_xor:
625
4
  case AtomicExpr::AO__atomic_fetch_xor:
626
4
    Op = llvm::AtomicRMWInst::Xor;
627
4
    break;
628
4
629
4
  case AtomicExpr::AO__atomic_nand_fetch:
630
4
    PostOp = llvm::Instruction::And; // the NOT is special cased below
631
4
    LLVM_FALLTHROUGH;
632
8
  case AtomicExpr::AO__atomic_fetch_nand:
633
8
    Op = llvm::AtomicRMWInst::Nand;
634
8
    break;
635
340
  }
636
340
637
340
  llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1);
638
340
  llvm::AtomicRMWInst *RMWI =
639
340
      CGF.Builder.CreateAtomicRMW(Op, Ptr.getPointer(), LoadVal1, Order, Scope);
640
340
  RMWI->setVolatile(E->isVolatile());
641
340
642
340
  // For __atomic_*_fetch operations, perform the operation again to
643
340
  // determine the value which was written.
644
340
  llvm::Value *Result = RMWI;
645
340
  if (PostOp)
646
22
    Result = CGF.Builder.CreateBinOp(PostOp, RMWI, LoadVal1);
647
340
  if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch)
648
4
    Result = CGF.Builder.CreateNot(Result);
649
340
  CGF.Builder.CreateStore(Result, Dest);
650
340
}
651
652
// This function emits any expression (scalar, complex, or aggregate)
653
// into a temporary alloca.
654
static Address
655
383
EmitValToTemp(CodeGenFunction &CGF, Expr *E) {
656
383
  Address DeclPtr = CGF.CreateMemTemp(E->getType(), ".atomictmp");
657
383
  CGF.EmitAnyExprToMem(E, DeclPtr, E->getType().getQualifiers(),
658
383
                       /*Init*/ true);
659
383
  return DeclPtr;
660
383
}
661
662
static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *Expr, Address Dest,
663
                         Address Ptr, Address Val1, Address Val2,
664
                         llvm::Value *IsWeak, llvm::Value *FailureOrder,
665
                         uint64_t Size, llvm::AtomicOrdering Order,
666
798
                         llvm::Value *Scope) {
667
798
  auto ScopeModel = Expr->getScopeModel();
668
798
669
798
  // LLVM atomic instructions always have synch scope. If clang atomic
670
798
  // expression has no scope operand, use default LLVM synch scope.
671
798
  if (!ScopeModel) {
672
740
    EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
673
740
                 Order, CGF.CGM.getLLVMContext().getOrInsertSyncScopeID(""));
674
740
    return;
675
740
  }
676
58
677
58
  // Handle constant scope.
678
58
  if (auto SC = dyn_cast<llvm::ConstantInt>(Scope)) {
679
50
    auto SCID = CGF.getTargetHooks().getLLVMSyncScopeID(
680
50
        CGF.CGM.getLangOpts(), ScopeModel->map(SC->getZExtValue()),
681
50
        Order, CGF.CGM.getLLVMContext());
682
50
    EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
683
50
                 Order, SCID);
684
50
    return;
685
50
  }
686
8
687
8
  // Handle non-constant scope.
688
8
  auto &Builder = CGF.Builder;
689
8
  auto Scopes = ScopeModel->getRuntimeValues();
690
8
  llvm::DenseMap<unsigned, llvm::BasicBlock *> BB;
691
8
  for (auto S : Scopes)
692
32
    BB[S] = CGF.createBasicBlock(getAsString(ScopeModel->map(S)), CGF.CurFn);
693
8
694
8
  llvm::BasicBlock *ContBB =
695
8
      CGF.createBasicBlock("atomic.scope.continue", CGF.CurFn);
696
8
697
8
  auto *SC = Builder.CreateIntCast(Scope, Builder.getInt32Ty(), false);
698
8
  // If unsupported synch scope is encountered at run time, assume a fallback
699
8
  // synch scope value.
700
8
  auto FallBack = ScopeModel->getFallBackValue();
701
8
  llvm::SwitchInst *SI = Builder.CreateSwitch(SC, BB[FallBack]);
702
32
  for (auto S : Scopes) {
703
32
    auto *B = BB[S];
704
32
    if (S != FallBack)
705
24
      SI->addCase(Builder.getInt32(S), B);
706
32
707
32
    Builder.SetInsertPoint(B);
708
32
    EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
709
32
                 Order,
710
32
                 CGF.getTargetHooks().getLLVMSyncScopeID(CGF.CGM.getLangOpts(),
711
32
                                                         ScopeModel->map(S),
712
32
                                                         Order,
713
32
                                                         CGF.getLLVMContext()));
714
32
    Builder.CreateBr(ContBB);
715
32
  }
716
8
717
8
  Builder.SetInsertPoint(ContBB);
718
8
}
719
720
static void
721
AddDirectArgument(CodeGenFunction &CGF, CallArgList &Args,
722
                  bool UseOptimizedLibcall, llvm::Value *Val, QualType ValTy,
723
117
                  SourceLocation Loc, CharUnits SizeInChars) {
724
117
  if (UseOptimizedLibcall) {
725
65
    // Load value and pass it to the function directly.
726
65
    CharUnits Align = CGF.getContext().getTypeAlignInChars(ValTy);
727
65
    int64_t SizeInBits = CGF.getContext().toBits(SizeInChars);
728
65
    ValTy =
729
65
        CGF.getContext().getIntTypeForBitwidth(SizeInBits, /*Signed=*/false);
730
65
    llvm::Type *IPtrTy = llvm::IntegerType::get(CGF.getLLVMContext(),
731
65
                                                SizeInBits)->getPointerTo();
732
65
    Address Ptr = Address(CGF.Builder.CreateBitCast(Val, IPtrTy), Align);
733
65
    Val = CGF.EmitLoadOfScalar(Ptr, false,
734
65
                               CGF.getContext().getPointerType(ValTy),
735
65
                               Loc);
736
65
    // Coerce the value into an appropriately sized integer type.
737
65
    Args.add(RValue::get(Val), ValTy);
738
65
  } else {
739
52
    // Non-optimized functions always take a reference.
740
52
    Args.add(RValue::get(CGF.EmitCastToVoidPtr(Val)),
741
52
                         CGF.getContext().VoidPtrTy);
742
52
  }
743
117
}
744
745
686
RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) {
746
686
  QualType AtomicTy = E->getPtr()->getType()->getPointeeType();
747
686
  QualType MemTy = AtomicTy;
748
686
  if (const AtomicType *AT = AtomicTy->getAs<AtomicType>())
749
333
    MemTy = AT->getValueType();
750
686
  llvm::Value *IsWeak = nullptr, *OrderFail = nullptr;
751
686
752
686
  Address Val1 = Address::invalid();
753
686
  Address Val2 = Address::invalid();
754
686
  Address Dest = Address::invalid();
755
686
  Address Ptr = EmitPointerWithAlignment(E->getPtr());
756
686
757
686
  if (E->getOp() == AtomicExpr::AO__c11_atomic_init ||
758
686
      
E->getOp() == AtomicExpr::AO__opencl_atomic_init672
) {
759
16
    LValue lvalue = MakeAddrLValue(Ptr, AtomicTy);
760
16
    EmitAtomicInit(E->getVal1(), lvalue);
761
16
    return RValue::get(nullptr);
762
16
  }
763
670
764
670
  CharUnits sizeChars, alignChars;
765
670
  std::tie(sizeChars, alignChars) = getContext().getTypeInfoInChars(AtomicTy);
766
670
  uint64_t Size = sizeChars.getQuantity();
767
670
  unsigned MaxInlineWidthInBits = getTarget().getMaxAtomicInlineWidth();
768
670
769
670
  bool Oversized = getContext().toBits(sizeChars) > MaxInlineWidthInBits;
770
670
  bool Misaligned = (Ptr.getAlignment() % sizeChars) != 0;
771
670
  bool UseLibcall = Misaligned | Oversized;
772
670
773
670
  if (UseLibcall) {
774
158
    CGM.getDiags().Report(E->getBeginLoc(), diag::warn_atomic_op_misaligned)
775
158
        << !Oversized;
776
158
  }
777
670
778
670
  llvm::Value *Order = EmitScalarExpr(E->getOrder());
779
670
  llvm::Value *Scope =
780
670
      E->getScopeModel() ? 
EmitScalarExpr(E->getScope())73
:
nullptr597
;
781
670
782
670
  switch (E->getOp()) {
783
670
  case AtomicExpr::AO__c11_atomic_init:
784
0
  case AtomicExpr::AO__opencl_atomic_init:
785
0
    llvm_unreachable("Already handled above with EmitAtomicInit!");
786
0
787
75
  case AtomicExpr::AO__c11_atomic_load:
788
75
  case AtomicExpr::AO__opencl_atomic_load:
789
75
  case AtomicExpr::AO__atomic_load_n:
790
75
    break;
791
75
792
86
  case AtomicExpr::AO__atomic_load:
793
86
    Dest = EmitPointerWithAlignment(E->getVal1());
794
86
    break;
795
75
796
84
  case AtomicExpr::AO__atomic_store:
797
84
    Val1 = EmitPointerWithAlignment(E->getVal1());
798
84
    break;
799
75
800
75
  case AtomicExpr::AO__atomic_exchange:
801
14
    Val1 = EmitPointerWithAlignment(E->getVal1());
802
14
    Dest = EmitPointerWithAlignment(E->getVal2());
803
14
    break;
804
75
805
75
  case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
806
69
  case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
807
69
  case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
808
69
  case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
809
69
  case AtomicExpr::AO__atomic_compare_exchange_n:
810
69
  case AtomicExpr::AO__atomic_compare_exchange:
811
69
    Val1 = EmitPointerWithAlignment(E->getVal1());
812
69
    if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange)
813
22
      Val2 = EmitPointerWithAlignment(E->getVal2());
814
47
    else
815
47
      Val2 = EmitValToTemp(*this, E->getVal2());
816
69
    OrderFail = EmitScalarExpr(E->getOrderFail());
817
69
    if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange_n ||
818
69
        
E->getOp() == AtomicExpr::AO__atomic_compare_exchange61
)
819
30
      IsWeak = EmitScalarExpr(E->getWeak());
820
69
    break;
821
69
822
69
  case AtomicExpr::AO__c11_atomic_fetch_add:
823
20
  case AtomicExpr::AO__c11_atomic_fetch_sub:
824
20
  case AtomicExpr::AO__opencl_atomic_fetch_add:
825
20
  case AtomicExpr::AO__opencl_atomic_fetch_sub:
826
20
    if (MemTy->isPointerType()) {
827
6
      // For pointer arithmetic, we're required to do a bit of math:
828
6
      // adding 1 to an int* is not the same as adding 1 to a uintptr_t.
829
6
      // ... but only for the C11 builtins. The GNU builtins expect the
830
6
      // user to multiply by sizeof(T).
831
6
      QualType Val1Ty = E->getVal1()->getType();
832
6
      llvm::Value *Val1Scalar = EmitScalarExpr(E->getVal1());
833
6
      CharUnits PointeeIncAmt =
834
6
          getContext().getTypeSizeInChars(MemTy->getPointeeType());
835
6
      Val1Scalar = Builder.CreateMul(Val1Scalar, CGM.getSize(PointeeIncAmt));
836
6
      auto Temp = CreateMemTemp(Val1Ty, ".atomictmp");
837
6
      Val1 = Temp;
838
6
      EmitStoreOfScalar(Val1Scalar, MakeAddrLValue(Temp, Val1Ty));
839
6
      break;
840
6
    }
841
14
      LLVM_FALLTHROUGH;
842
336
  case AtomicExpr::AO__atomic_fetch_add:
843
336
  case AtomicExpr::AO__atomic_fetch_sub:
844
336
  case AtomicExpr::AO__atomic_add_fetch:
845
336
  case AtomicExpr::AO__atomic_sub_fetch:
846
336
  case AtomicExpr::AO__c11_atomic_store:
847
336
  case AtomicExpr::AO__c11_atomic_exchange:
848
336
  case AtomicExpr::AO__opencl_atomic_store:
849
336
  case AtomicExpr::AO__opencl_atomic_exchange:
850
336
  case AtomicExpr::AO__atomic_store_n:
851
336
  case AtomicExpr::AO__atomic_exchange_n:
852
336
  case AtomicExpr::AO__c11_atomic_fetch_and:
853
336
  case AtomicExpr::AO__c11_atomic_fetch_or:
854
336
  case AtomicExpr::AO__c11_atomic_fetch_xor:
855
336
  case AtomicExpr::AO__opencl_atomic_fetch_and:
856
336
  case AtomicExpr::AO__opencl_atomic_fetch_or:
857
336
  case AtomicExpr::AO__opencl_atomic_fetch_xor:
858
336
  case AtomicExpr::AO__opencl_atomic_fetch_min:
859
336
  case AtomicExpr::AO__opencl_atomic_fetch_max:
860
336
  case AtomicExpr::AO__atomic_fetch_and:
861
336
  case AtomicExpr::AO__atomic_fetch_or:
862
336
  case AtomicExpr::AO__atomic_fetch_xor:
863
336
  case AtomicExpr::AO__atomic_fetch_nand:
864
336
  case AtomicExpr::AO__atomic_and_fetch:
865
336
  case AtomicExpr::AO__atomic_or_fetch:
866
336
  case AtomicExpr::AO__atomic_xor_fetch:
867
336
  case AtomicExpr::AO__atomic_nand_fetch:
868
336
  case AtomicExpr::AO__atomic_fetch_min:
869
336
  case AtomicExpr::AO__atomic_fetch_max:
870
336
    Val1 = EmitValToTemp(*this, E->getVal1());
871
336
    break;
872
670
  }
873
670
874
670
  QualType RValTy = E->getType().getUnqualifiedType();
875
670
876
670
  // The inlined atomics only function on iN types, where N is a power of 2. We
877
670
  // need to make sure (via temporaries if necessary) that all incoming values
878
670
  // are compatible.
879
670
  LValue AtomicVal = MakeAddrLValue(Ptr, AtomicTy);
880
670
  AtomicInfo Atomics(*this, AtomicVal);
881
670
882
670
  Ptr = Atomics.emitCastToAtomicIntPointer(Ptr);
883
670
  if (Val1.isValid()) 
Val1 = Atomics.convertToAtomicIntPointer(Val1)509
;
884
670
  if (Val2.isValid()) 
Val2 = Atomics.convertToAtomicIntPointer(Val2)69
;
885
670
  if (Dest.isValid())
886
100
    Dest = Atomics.emitCastToAtomicIntPointer(Dest);
887
570
  else if (E->isCmpXChg())
888
69
    Dest = CreateMemTemp(RValTy, "cmpxchg.bool");
889
501
  else if (!RValTy->isVoidType())
890
289
    Dest = Atomics.emitCastToAtomicIntPointer(Atomics.CreateTempAlloca());
891
670
892
670
  // Use a library call.  See: http://gcc.gnu.org/wiki/Atomic/GCCMM/LIbrary .
893
670
  if (UseLibcall) {
894
158
    bool UseOptimizedLibcall = false;
895
158
    switch (E->getOp()) {
896
158
    case AtomicExpr::AO__c11_atomic_init:
897
0
    case AtomicExpr::AO__opencl_atomic_init:
898
0
      llvm_unreachable("Already handled above with EmitAtomicInit!");
899
0
900
31
    case AtomicExpr::AO__c11_atomic_fetch_add:
901
31
    case AtomicExpr::AO__opencl_atomic_fetch_add:
902
31
    case AtomicExpr::AO__atomic_fetch_add:
903
31
    case AtomicExpr::AO__c11_atomic_fetch_and:
904
31
    case AtomicExpr::AO__opencl_atomic_fetch_and:
905
31
    case AtomicExpr::AO__atomic_fetch_and:
906
31
    case AtomicExpr::AO__c11_atomic_fetch_or:
907
31
    case AtomicExpr::AO__opencl_atomic_fetch_or:
908
31
    case AtomicExpr::AO__atomic_fetch_or:
909
31
    case AtomicExpr::AO__atomic_fetch_nand:
910
31
    case AtomicExpr::AO__c11_atomic_fetch_sub:
911
31
    case AtomicExpr::AO__opencl_atomic_fetch_sub:
912
31
    case AtomicExpr::AO__atomic_fetch_sub:
913
31
    case AtomicExpr::AO__c11_atomic_fetch_xor:
914
31
    case AtomicExpr::AO__opencl_atomic_fetch_xor:
915
31
    case AtomicExpr::AO__opencl_atomic_fetch_min:
916
31
    case AtomicExpr::AO__opencl_atomic_fetch_max:
917
31
    case AtomicExpr::AO__atomic_fetch_xor:
918
31
    case AtomicExpr::AO__atomic_add_fetch:
919
31
    case AtomicExpr::AO__atomic_and_fetch:
920
31
    case AtomicExpr::AO__atomic_nand_fetch:
921
31
    case AtomicExpr::AO__atomic_or_fetch:
922
31
    case AtomicExpr::AO__atomic_sub_fetch:
923
31
    case AtomicExpr::AO__atomic_xor_fetch:
924
31
    case AtomicExpr::AO__atomic_fetch_min:
925
31
    case AtomicExpr::AO__atomic_fetch_max:
926
31
      // For these, only library calls for certain sizes exist.
927
31
      UseOptimizedLibcall = true;
928
31
      break;
929
31
930
87
    case AtomicExpr::AO__atomic_load:
931
87
    case AtomicExpr::AO__atomic_store:
932
87
    case AtomicExpr::AO__atomic_exchange:
933
87
    case AtomicExpr::AO__atomic_compare_exchange:
934
87
      // Use the generic version if we don't know that the operand will be
935
87
      // suitably aligned for the optimized version.
936
87
      if (Misaligned)
937
62
        break;
938
25
      LLVM_FALLTHROUGH;
939
65
    case AtomicExpr::AO__c11_atomic_load:
940
65
    case AtomicExpr::AO__c11_atomic_store:
941
65
    case AtomicExpr::AO__c11_atomic_exchange:
942
65
    case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
943
65
    case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
944
65
    case AtomicExpr::AO__opencl_atomic_load:
945
65
    case AtomicExpr::AO__opencl_atomic_store:
946
65
    case AtomicExpr::AO__opencl_atomic_exchange:
947
65
    case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
948
65
    case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
949
65
    case AtomicExpr::AO__atomic_load_n:
950
65
    case AtomicExpr::AO__atomic_store_n:
951
65
    case AtomicExpr::AO__atomic_exchange_n:
952
65
    case AtomicExpr::AO__atomic_compare_exchange_n:
953
65
      // Only use optimized library calls for sizes for which they exist.
954
65
      // FIXME: Size == 16 optimized library functions exist too.
955
65
      if (Size == 1 || 
Size == 261
||
Size == 459
||
Size == 834
)
956
49
        UseOptimizedLibcall = true;
957
65
      break;
958
158
    }
959
158
960
158
    CallArgList Args;
961
158
    if (!UseOptimizedLibcall) {
962
78
      // For non-optimized library calls, the size is the first parameter
963
78
      Args.add(RValue::get(llvm::ConstantInt::get(SizeTy, Size)),
964
78
               getContext().getSizeType());
965
78
    }
966
158
    // Atomic address is the first or second parameter
967
158
    // The OpenCL atomic library functions only accept pointer arguments to
968
158
    // generic address space.
969
183
    auto CastToGenericAddrSpace = [&](llvm::Value *V, QualType PT) {
970
183
      if (!E->isOpenCL())
971
145
        return V;
972
38
      auto AS = PT->getAs<PointerType>()->getPointeeType().getAddressSpace();
973
38
      if (AS == LangAS::opencl_generic)
974
21
        return V;
975
17
      auto DestAS = getContext().getTargetAddressSpace(LangAS::opencl_generic);
976
17
      auto T = V->getType();
977
17
      auto *DestType = T->getPointerElementType()->getPointerTo(DestAS);
978
17
979
17
      return getTargetHooks().performAddrSpaceCast(
980
17
          *this, V, AS, LangAS::opencl_generic, DestType, false);
981
17
    };
982
158
983
158
    Args.add(RValue::get(CastToGenericAddrSpace(
984
158
                 EmitCastToVoidPtr(Ptr.getPointer()), E->getPtr()->getType())),
985
158
             getContext().VoidPtrTy);
986
158
987
158
    std::string LibCallName;
988
158
    QualType LoweredMemTy =
989
158
      MemTy->isPointerType() ? 
getContext().getIntPtrType()3
:
MemTy155
;
990
158
    QualType RetTy;
991
158
    bool HaveRetTy = false;
992
158
    llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0;
993
158
    switch (E->getOp()) {
994
158
    case AtomicExpr::AO__c11_atomic_init:
995
0
    case AtomicExpr::AO__opencl_atomic_init:
996
0
      llvm_unreachable("Already handled!");
997
0
998
0
    // There is only one libcall for compare an exchange, because there is no
999
0
    // optimisation benefit possible from a libcall version of a weak compare
1000
0
    // and exchange.
1001
0
    // bool __atomic_compare_exchange(size_t size, void *mem, void *expected,
1002
0
    //                                void *desired, int success, int failure)
1003
0
    // bool __atomic_compare_exchange_N(T *mem, T *expected, T desired,
1004
0
    //                                  int success, int failure)
1005
25
    case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
1006
25
    case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
1007
25
    case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
1008
25
    case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
1009
25
    case AtomicExpr::AO__atomic_compare_exchange:
1010
25
    case AtomicExpr::AO__atomic_compare_exchange_n:
1011
25
      LibCallName = "__atomic_compare_exchange";
1012
25
      RetTy = getContext().BoolTy;
1013
25
      HaveRetTy = true;
1014
25
      Args.add(
1015
25
          RValue::get(CastToGenericAddrSpace(
1016
25
              EmitCastToVoidPtr(Val1.getPointer()), E->getVal1()->getType())),
1017
25
          getContext().VoidPtrTy);
1018
25
      AddDirectArgument(*this, Args, UseOptimizedLibcall, Val2.getPointer(),
1019
25
                        MemTy, E->getExprLoc(), sizeChars);
1020
25
      Args.add(RValue::get(Order), getContext().IntTy);
1021
25
      Order = OrderFail;
1022
25
      break;
1023
25
    // void __atomic_exchange(size_t size, void *mem, void *val, void *return,
1024
25
    //                        int order)
1025
25
    // T __atomic_exchange_N(T *mem, T val, int order)
1026
25
    case AtomicExpr::AO__c11_atomic_exchange:
1027
13
    case AtomicExpr::AO__opencl_atomic_exchange:
1028
13
    case AtomicExpr::AO__atomic_exchange_n:
1029
13
    case AtomicExpr::AO__atomic_exchange:
1030
13
      LibCallName = "__atomic_exchange";
1031
13
      AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1032
13
                        MemTy, E->getExprLoc(), sizeChars);
1033
13
      break;
1034
13
    // void __atomic_store(size_t size, void *mem, void *val, int order)
1035
13
    // void __atomic_store_N(T *mem, T val, int order)
1036
48
    case AtomicExpr::AO__c11_atomic_store:
1037
48
    case AtomicExpr::AO__opencl_atomic_store:
1038
48
    case AtomicExpr::AO__atomic_store:
1039
48
    case AtomicExpr::AO__atomic_store_n:
1040
48
      LibCallName = "__atomic_store";
1041
48
      RetTy = getContext().VoidTy;
1042
48
      HaveRetTy = true;
1043
48
      AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1044
48
                        MemTy, E->getExprLoc(), sizeChars);
1045
48
      break;
1046
48
    // void __atomic_load(size_t size, void *mem, void *return, int order)
1047
48
    // T __atomic_load_N(T *mem, int order)
1048
48
    case AtomicExpr::AO__c11_atomic_load:
1049
41
    case AtomicExpr::AO__opencl_atomic_load:
1050
41
    case AtomicExpr::AO__atomic_load:
1051
41
    case AtomicExpr::AO__atomic_load_n:
1052
41
      LibCallName = "__atomic_load";
1053
41
      break;
1054
41
    // T __atomic_add_fetch_N(T *mem, T val, int order)
1055
41
    // T __atomic_fetch_add_N(T *mem, T val, int order)
1056
41
    case AtomicExpr::AO__atomic_add_fetch:
1057
1
      PostOp = llvm::Instruction::Add;
1058
1
      LLVM_FALLTHROUGH;
1059
10
    case AtomicExpr::AO__c11_atomic_fetch_add:
1060
10
    case AtomicExpr::AO__opencl_atomic_fetch_add:
1061
10
    case AtomicExpr::AO__atomic_fetch_add:
1062
10
      LibCallName = "__atomic_fetch_add";
1063
10
      AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1064
10
                        LoweredMemTy, E->getExprLoc(), sizeChars);
1065
10
      break;
1066
10
    // T __atomic_and_fetch_N(T *mem, T val, int order)
1067
10
    // T __atomic_fetch_and_N(T *mem, T val, int order)
1068
10
    case AtomicExpr::AO__atomic_and_fetch:
1069
1
      PostOp = llvm::Instruction::And;
1070
1
      LLVM_FALLTHROUGH;
1071
2
    case AtomicExpr::AO__c11_atomic_fetch_and:
1072
2
    case AtomicExpr::AO__opencl_atomic_fetch_and:
1073
2
    case AtomicExpr::AO__atomic_fetch_and:
1074
2
      LibCallName = "__atomic_fetch_and";
1075
2
      AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1076
2
                        MemTy, E->getExprLoc(), sizeChars);
1077
2
      break;
1078
2
    // T __atomic_or_fetch_N(T *mem, T val, int order)
1079
2
    // T __atomic_fetch_or_N(T *mem, T val, int order)
1080
2
    case AtomicExpr::AO__atomic_or_fetch:
1081
1
      PostOp = llvm::Instruction::Or;
1082
1
      LLVM_FALLTHROUGH;
1083
2
    case AtomicExpr::AO__c11_atomic_fetch_or:
1084
2
    case AtomicExpr::AO__opencl_atomic_fetch_or:
1085
2
    case AtomicExpr::AO__atomic_fetch_or:
1086
2
      LibCallName = "__atomic_fetch_or";
1087
2
      AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1088
2
                        MemTy, E->getExprLoc(), sizeChars);
1089
2
      break;
1090
2
    // T __atomic_sub_fetch_N(T *mem, T val, int order)
1091
2
    // T __atomic_fetch_sub_N(T *mem, T val, int order)
1092
2
    case AtomicExpr::AO__atomic_sub_fetch:
1093
1
      PostOp = llvm::Instruction::Sub;
1094
1
      LLVM_FALLTHROUGH;
1095
9
    case AtomicExpr::AO__c11_atomic_fetch_sub:
1096
9
    case AtomicExpr::AO__opencl_atomic_fetch_sub:
1097
9
    case AtomicExpr::AO__atomic_fetch_sub:
1098
9
      LibCallName = "__atomic_fetch_sub";
1099
9
      AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1100
9
                        LoweredMemTy, E->getExprLoc(), sizeChars);
1101
9
      break;
1102
9
    // T __atomic_xor_fetch_N(T *mem, T val, int order)
1103
9
    // T __atomic_fetch_xor_N(T *mem, T val, int order)
1104
9
    case AtomicExpr::AO__atomic_xor_fetch:
1105
1
      PostOp = llvm::Instruction::Xor;
1106
1
      LLVM_FALLTHROUGH;
1107
2
    case AtomicExpr::AO__c11_atomic_fetch_xor:
1108
2
    case AtomicExpr::AO__opencl_atomic_fetch_xor:
1109
2
    case AtomicExpr::AO__atomic_fetch_xor:
1110
2
      LibCallName = "__atomic_fetch_xor";
1111
2
      AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1112
2
                        MemTy, E->getExprLoc(), sizeChars);
1113
2
      break;
1114
4
    case AtomicExpr::AO__atomic_fetch_min:
1115
4
    case AtomicExpr::AO__opencl_atomic_fetch_min:
1116
4
      LibCallName = E->getValueType()->isSignedIntegerType()
1117
4
                        ? 
"__atomic_fetch_min"2
1118
4
                        : 
"__atomic_fetch_umin"2
;
1119
4
      AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1120
4
                        LoweredMemTy, E->getExprLoc(), sizeChars);
1121
4
      break;
1122
4
    case AtomicExpr::AO__atomic_fetch_max:
1123
0
    case AtomicExpr::AO__opencl_atomic_fetch_max:
1124
0
      LibCallName = E->getValueType()->isSignedIntegerType()
1125
0
                        ? "__atomic_fetch_max"
1126
0
                        : "__atomic_fetch_umax";
1127
0
      AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1128
0
                        LoweredMemTy, E->getExprLoc(), sizeChars);
1129
0
      break;
1130
0
    // T __atomic_nand_fetch_N(T *mem, T val, int order)
1131
0
    // T __atomic_fetch_nand_N(T *mem, T val, int order)
1132
1
    case AtomicExpr::AO__atomic_nand_fetch:
1133
1
      PostOp = llvm::Instruction::And; // the NOT is special cased below
1134
1
      LLVM_FALLTHROUGH;
1135
2
    case AtomicExpr::AO__atomic_fetch_nand:
1136
2
      LibCallName = "__atomic_fetch_nand";
1137
2
      AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1138
2
                        MemTy, E->getExprLoc(), sizeChars);
1139
2
      break;
1140
158
    }
1141
158
1142
158
    if (E->isOpenCL()) {
1143
27
      LibCallName = std::string("__opencl") +
1144
27
          StringRef(LibCallName).drop_front(1).str();
1145
27
1146
27
    }
1147
158
    // Optimized functions have the size in their name.
1148
158
    if (UseOptimizedLibcall)
1149
80
      LibCallName += "_" + llvm::utostr(Size);
1150
158
    // By default, assume we return a value of the atomic type.
1151
158
    if (!HaveRetTy) {
1152
85
      if (UseOptimizedLibcall) {
1153
47
        // Value is returned directly.
1154
47
        // The function returns an appropriately sized integer type.
1155
47
        RetTy = getContext().getIntTypeForBitwidth(
1156
47
            getContext().toBits(sizeChars), /*Signed=*/false);
1157
47
      } else {
1158
38
        // Value is returned through parameter before the order.
1159
38
        RetTy = getContext().VoidTy;
1160
38
        Args.add(RValue::get(EmitCastToVoidPtr(Dest.getPointer())),
1161
38
                 getContext().VoidPtrTy);
1162
38
      }
1163
85
    }
1164
158
    // order is always the last parameter
1165
158
    Args.add(RValue::get(Order),
1166
158
             getContext().IntTy);
1167
158
    if (E->isOpenCL())
1168
27
      Args.add(RValue::get(Scope), getContext().IntTy);
1169
158
1170
158
    // PostOp is only needed for the atomic_*_fetch operations, and
1171
158
    // thus is only needed for and implemented in the
1172
158
    // UseOptimizedLibcall codepath.
1173
158
    assert(UseOptimizedLibcall || !PostOp);
1174
158
1175
158
    RValue Res = emitAtomicLibcall(*this, LibCallName, RetTy, Args);
1176
158
    // The value is returned directly from the libcall.
1177
158
    if (E->isCmpXChg())
1178
25
      return Res;
1179
133
1180
133
    // The value is returned directly for optimized libcalls but the expr
1181
133
    // provided an out-param.
1182
133
    if (UseOptimizedLibcall && 
Res.getScalarVal()68
) {
1183
47
      llvm::Value *ResVal = Res.getScalarVal();
1184
47
      if (PostOp) {
1185
6
        llvm::Value *LoadVal1 = Args[1].getRValue(*this).getScalarVal();
1186
6
        ResVal = Builder.CreateBinOp(PostOp, ResVal, LoadVal1);
1187
6
      }
1188
47
      if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch)
1189
1
        ResVal = Builder.CreateNot(ResVal);
1190
47
1191
47
      Builder.CreateStore(
1192
47
          ResVal,
1193
47
          Builder.CreateBitCast(Dest, ResVal->getType()->getPointerTo()));
1194
47
    }
1195
133
1196
133
    if (RValTy->isVoidType())
1197
91
      return RValue::get(nullptr);
1198
42
1199
42
    return convertTempToRValue(
1200
42
        Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo()),
1201
42
        RValTy, E->getExprLoc());
1202
42
  }
1203
512
1204
512
  bool IsStore = E->getOp() == AtomicExpr::AO__c11_atomic_store ||
1205
512
                 
E->getOp() == AtomicExpr::AO__opencl_atomic_store425
||
1206
512
                 
E->getOp() == AtomicExpr::AO__atomic_store415
||
1207
512
                 
E->getOp() == AtomicExpr::AO__atomic_store_n366
;
1208
512
  bool IsLoad = E->getOp() == AtomicExpr::AO__c11_atomic_load ||
1209
512
                
E->getOp() == AtomicExpr::AO__opencl_atomic_load486
||
1210
512
                
E->getOp() == AtomicExpr::AO__atomic_load470
||
1211
512
                
E->getOp() == AtomicExpr::AO__atomic_load_n419
;
1212
512
1213
512
  if (isa<llvm::ConstantInt>(Order)) {
1214
414
    auto ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
1215
414
    // We should not ever get to a case where the ordering isn't a valid C ABI
1216
414
    // value, but it's hard to enforce that in general.
1217
414
    if (llvm::isValidAtomicOrderingCABI(ord))
1218
414
      switch ((llvm::AtomicOrderingCABI)ord) {
1219
414
      case llvm::AtomicOrderingCABI::relaxed:
1220
51
        EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1221
51
                     llvm::AtomicOrdering::Monotonic, Scope);
1222
51
        break;
1223
414
      case llvm::AtomicOrderingCABI::consume:
1224
37
      case llvm::AtomicOrderingCABI::acquire:
1225
37
        if (IsStore)
1226
0
          break; // Avoid crashing on code with undefined behavior
1227
37
        EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1228
37
                     llvm::AtomicOrdering::Acquire, Scope);
1229
37
        break;
1230
37
      case llvm::AtomicOrderingCABI::release:
1231
22
        if (IsLoad)
1232
0
          break; // Avoid crashing on code with undefined behavior
1233
22
        EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1234
22
                     llvm::AtomicOrdering::Release, Scope);
1235
22
        break;
1236
22
      case llvm::AtomicOrderingCABI::acq_rel:
1237
13
        if (IsLoad || IsStore)
1238
0
          break; // Avoid crashing on code with undefined behavior
1239
13
        EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1240
13
                     llvm::AtomicOrdering::AcquireRelease, Scope);
1241
13
        break;
1242
291
      case llvm::AtomicOrderingCABI::seq_cst:
1243
291
        EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1244
291
                     llvm::AtomicOrdering::SequentiallyConsistent, Scope);
1245
291
        break;
1246
414
      }
1247
414
    if (RValTy->isVoidType())
1248
179
      return RValue::get(nullptr);
1249
235
1250
235
    return convertTempToRValue(
1251
235
        Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo(
1252
235
                                        Dest.getAddressSpace())),
1253
235
        RValTy, E->getExprLoc());
1254
235
  }
1255
98
1256
98
  // Long case, when Order isn't obviously constant.
1257
98
1258
98
  // Create all the relevant BB's
1259
98
  llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr,
1260
98
                   *ReleaseBB = nullptr, *AcqRelBB = nullptr,
1261
98
                   *SeqCstBB = nullptr;
1262
98
  MonotonicBB = createBasicBlock("monotonic", CurFn);
1263
98
  if (!IsStore)
1264
56
    AcquireBB = createBasicBlock("acquire", CurFn);
1265
98
  if (!IsLoad)
1266
87
    ReleaseBB = createBasicBlock("release", CurFn);
1267
98
  if (!IsLoad && 
!IsStore87
)
1268
45
    AcqRelBB = createBasicBlock("acqrel", CurFn);
1269
98
  SeqCstBB = createBasicBlock("seqcst", CurFn);
1270
98
  llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn);
1271
98
1272
98
  // Create the switch for the split
1273
98
  // MonotonicBB is arbitrarily chosen as the default case; in practice, this
1274
98
  // doesn't matter unless someone is crazy enough to use something that
1275
98
  // doesn't fold to a constant for the ordering.
1276
98
  Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);
1277
98
  llvm::SwitchInst *SI = Builder.CreateSwitch(Order, MonotonicBB);
1278
98
1279
98
  // Emit all the different atomics
1280
98
  Builder.SetInsertPoint(MonotonicBB);
1281
98
  EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1282
98
               llvm::AtomicOrdering::Monotonic, Scope);
1283
98
  Builder.CreateBr(ContBB);
1284
98
  if (!IsStore) {
1285
56
    Builder.SetInsertPoint(AcquireBB);
1286
56
    EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1287
56
                 llvm::AtomicOrdering::Acquire, Scope);
1288
56
    Builder.CreateBr(ContBB);
1289
56
    SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::consume),
1290
56
                AcquireBB);
1291
56
    SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire),
1292
56
                AcquireBB);
1293
56
  }
1294
98
  if (!IsLoad) {
1295
87
    Builder.SetInsertPoint(ReleaseBB);
1296
87
    EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1297
87
                 llvm::AtomicOrdering::Release, Scope);
1298
87
    Builder.CreateBr(ContBB);
1299
87
    SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::release),
1300
87
                ReleaseBB);
1301
87
  }
1302
98
  if (!IsLoad && 
!IsStore87
) {
1303
45
    Builder.SetInsertPoint(AcqRelBB);
1304
45
    EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1305
45
                 llvm::AtomicOrdering::AcquireRelease, Scope);
1306
45
    Builder.CreateBr(ContBB);
1307
45
    SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acq_rel),
1308
45
                AcqRelBB);
1309
45
  }
1310
98
  Builder.SetInsertPoint(SeqCstBB);
1311
98
  EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1312
98
               llvm::AtomicOrdering::SequentiallyConsistent, Scope);
1313
98
  Builder.CreateBr(ContBB);
1314
98
  SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst),
1315
98
              SeqCstBB);
1316
98
1317
98
  // Cleanup and return
1318
98
  Builder.SetInsertPoint(ContBB);
1319
98
  if (RValTy->isVoidType())
1320
42
    return RValue::get(nullptr);
1321
56
1322
56
  assert(Atomics.getValueSizeInBits() <= Atomics.getAtomicSizeInBits());
1323
56
  return convertTempToRValue(
1324
56
      Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo(
1325
56
                                      Dest.getAddressSpace())),
1326
56
      RValTy, E->getExprLoc());
1327
56
}
1328
1329
2.53k
Address AtomicInfo::emitCastToAtomicIntPointer(Address addr) const {
1330
2.53k
  unsigned addrspace =
1331
2.53k
    cast<llvm::PointerType>(addr.getPointer()->getType())->getAddressSpace();
1332
2.53k
  llvm::IntegerType *ty =
1333
2.53k
    llvm::IntegerType::get(CGF.getLLVMContext(), AtomicSizeInBits);
1334
2.53k
  return CGF.Builder.CreateBitCast(addr, ty->getPointerTo(addrspace));
1335
2.53k
}
1336
1337
578
Address AtomicInfo::convertToAtomicIntPointer(Address Addr) const {
1338
578
  llvm::Type *Ty = Addr.getElementType();
1339
578
  uint64_t SourceSizeInBits = CGF.CGM.getDataLayout().getTypeSizeInBits(Ty);
1340
578
  if (SourceSizeInBits != AtomicSizeInBits) {
1341
9
    Address Tmp = CreateTempAlloca();
1342
9
    CGF.Builder.CreateMemCpy(Tmp, Addr,
1343
9
                             std::min(AtomicSizeInBits, SourceSizeInBits) / 8);
1344
9
    Addr = Tmp;
1345
9
  }
1346
578
1347
578
  return emitCastToAtomicIntPointer(Addr);
1348
578
}
1349
1350
RValue AtomicInfo::convertAtomicTempToRValue(Address addr,
1351
                                             AggValueSlot resultSlot,
1352
                                             SourceLocation loc,
1353
138
                                             bool asValue) const {
1354
138
  if (LVal.isSimple()) {
1355
106
    if (EvaluationKind == TEK_Aggregate)
1356
11
      return resultSlot.asRValue();
1357
95
1358
95
    // Drill into the padding structure if we have one.
1359
95
    if (hasPadding())
1360
0
      addr = CGF.Builder.CreateStructGEP(addr, 0);
1361
95
1362
95
    // Otherwise, just convert the temporary to an r-value using the
1363
95
    // normal conversion routine.
1364
95
    return CGF.convertTempToRValue(addr, getValueType(), loc);
1365
95
  }
1366
32
  if (!asValue)
1367
8
    // Get RValue from temp memory as atomic for non-simple lvalues
1368
8
    return RValue::get(CGF.Builder.CreateLoad(addr));
1369
24
  if (LVal.isBitField())
1370
20
    return CGF.EmitLoadOfBitfieldLValue(
1371
20
        LValue::MakeBitfield(addr, LVal.getBitFieldInfo(), LVal.getType(),
1372
20
                             LVal.getBaseInfo(), TBAAAccessInfo()), loc);
1373
4
  if (LVal.isVectorElt())
1374
2
    return CGF.EmitLoadOfLValue(
1375
2
        LValue::MakeVectorElt(addr, LVal.getVectorIdx(), LVal.getType(),
1376
2
                              LVal.getBaseInfo(), TBAAAccessInfo()), loc);
1377
2
  assert(LVal.isExtVectorElt());
1378
2
  return CGF.EmitLoadOfExtVectorElementLValue(LValue::MakeExtVectorElt(
1379
2
      addr, LVal.getExtVectorElts(), LVal.getType(),
1380
2
      LVal.getBaseInfo(), TBAAAccessInfo()));
1381
2
}
1382
1383
RValue AtomicInfo::ConvertIntToValueOrAtomic(llvm::Value *IntVal,
1384
                                             AggValueSlot ResultSlot,
1385
                                             SourceLocation Loc,
1386
312
                                             bool AsValue) const {
1387
312
  // Try not to in some easy cases.
1388
312
  assert(IntVal->getType()->isIntegerTy() && "Expected integer value");
1389
312
  if (getEvaluationKind() == TEK_Scalar &&
1390
312
      
(302
(302
(302
!LVal.isBitField()302
||
1391
302
         
LVal.getBitFieldInfo().Size == ValueSizeInBits48
) &&
1392
302
        
!hasPadding()254
) ||
1393
302
       
!AsValue60
)) {
1394
282
    auto *ValTy = AsValue
1395
282
                      ? 
CGF.ConvertTypeForMem(ValueTy)95
1396
282
                      : 
getAtomicAddress().getType()->getPointerElementType()187
;
1397
282
    if (ValTy->isIntegerTy()) {
1398
191
      assert(IntVal->getType() == ValTy && "Different integer types.");
1399
191
      return RValue::get(CGF.EmitFromMemory(IntVal, ValueTy));
1400
191
    } else 
if (91
ValTy->isPointerTy()91
)
1401
0
      return RValue::get(CGF.Builder.CreateIntToPtr(IntVal, ValTy));
1402
91
    else if (llvm::CastInst::isBitCastable(IntVal->getType(), ValTy))
1403
63
      return RValue::get(CGF.Builder.CreateBitCast(IntVal, ValTy));
1404
58
  }
1405
58
1406
58
  // Create a temporary.  This needs to be big enough to hold the
1407
58
  // atomic integer.
1408
58
  Address Temp = Address::invalid();
1409
58
  bool TempIsVolatile = false;
1410
58
  if (AsValue && 
getEvaluationKind() == TEK_Aggregate44
) {
1411
8
    assert(!ResultSlot.isIgnored());
1412
8
    Temp = ResultSlot.getAddress();
1413
8
    TempIsVolatile = ResultSlot.isVolatile();
1414
50
  } else {
1415
50
    Temp = CreateTempAlloca();
1416
50
  }
1417
58
1418
58
  // Slam the integer into the temporary.
1419
58
  Address CastTemp = emitCastToAtomicIntPointer(Temp);
1420
58
  CGF.Builder.CreateStore(IntVal, CastTemp)
1421
58
      ->setVolatile(TempIsVolatile);
1422
58
1423
58
  return convertAtomicTempToRValue(Temp, ResultSlot, Loc, AsValue);
1424
58
}
1425
1426
void AtomicInfo::EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
1427
71
                                       llvm::AtomicOrdering AO, bool) {
1428
71
  // void __atomic_load(size_t size, void *mem, void *return, int order);
1429
71
  CallArgList Args;
1430
71
  Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
1431
71
  Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())),
1432
71
           CGF.getContext().VoidPtrTy);
1433
71
  Args.add(RValue::get(CGF.EmitCastToVoidPtr(AddForLoaded)),
1434
71
           CGF.getContext().VoidPtrTy);
1435
71
  Args.add(
1436
71
      RValue::get(llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(AO))),
1437
71
      CGF.getContext().IntTy);
1438
71
  emitAtomicLibcall(CGF, "__atomic_load", CGF.getContext().VoidTy, Args);
1439
71
}
1440
1441
llvm::Value *AtomicInfo::EmitAtomicLoadOp(llvm::AtomicOrdering AO,
1442
318
                                          bool IsVolatile) {
1443
318
  // Okay, we're doing this natively.
1444
318
  Address Addr = getAtomicAddressAsAtomicIntPointer();
1445
318
  llvm::LoadInst *Load = CGF.Builder.CreateLoad(Addr, "atomic-load");
1446
318
  Load->setAtomic(AO);
1447
318
1448
318
  // Other decoration.
1449
318
  if (IsVolatile)
1450
12
    Load->setVolatile(true);
1451
318
  CGF.CGM.DecorateInstructionWithTBAA(Load, LVal.getTBAAInfo());
1452
318
  return Load;
1453
318
}
1454
1455
/// An LValue is a candidate for having its loads and stores be made atomic if
1456
/// we are operating under /volatile:ms *and* the LValue itself is volatile and
1457
/// performing such an operation can be performed without a libcall.
1458
2.89M
bool CodeGenFunction::LValueIsSuitableForInlineAtomic(LValue LV) {
1459
2.89M
  if (!CGM.getCodeGenOpts().MSVolatile) 
return false2.89M
;
1460
45
  AtomicInfo AI(*this, LV);
1461
45
  bool IsVolatile = LV.isVolatile() || 
hasVolatileMember(LV.getType())27
;
1462
45
  // An atomic is inline if we don't need to use a libcall.
1463
45
  bool AtomicIsInline = !AI.shouldUseLibcall();
1464
45
  // MSVC doesn't seem to do this for types wider than a pointer.
1465
45
  if (getContext().getTypeSize(LV.getType()) >
1466
45
      getContext().getTypeSize(getContext().getIntPtrType()))
1467
5
    return false;
1468
40
  return IsVolatile && 
AtomicIsInline15
;
1469
40
}
1470
1471
RValue CodeGenFunction::EmitAtomicLoad(LValue LV, SourceLocation SL,
1472
62
                                       AggValueSlot Slot) {
1473
62
  llvm::AtomicOrdering AO;
1474
62
  bool IsVolatile = LV.isVolatileQualified();
1475
62
  if (LV.getType()->isAtomicType()) {
1476
55
    AO = llvm::AtomicOrdering::SequentiallyConsistent;
1477
55
  } else {
1478
7
    AO = llvm::AtomicOrdering::Acquire;
1479
7
    IsVolatile = true;
1480
7
  }
1481
62
  return EmitAtomicLoad(LV, SL, AO, IsVolatile, Slot);
1482
62
}
1483
1484
RValue AtomicInfo::EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
1485
                                  bool AsValue, llvm::AtomicOrdering AO,
1486
162
                                  bool IsVolatile) {
1487
162
  // Check whether we should use a library call.
1488
162
  if (shouldUseLibcall()) {
1489
37
    Address TempAddr = Address::invalid();
1490
37
    if (LVal.isSimple() && 
!ResultSlot.isIgnored()33
) {
1491
3
      assert(getEvaluationKind() == TEK_Aggregate);
1492
3
      TempAddr = ResultSlot.getAddress();
1493
3
    } else
1494
34
      TempAddr = CreateTempAlloca();
1495
37
1496
37
    EmitAtomicLoadLibcall(TempAddr.getPointer(), AO, IsVolatile);
1497
37
1498
37
    // Okay, turn that back into the original value or whole atomic (for
1499
37
    // non-simple lvalues) type.
1500
37
    return convertAtomicTempToRValue(TempAddr, ResultSlot, Loc, AsValue);
1501
37
  }
1502
125
1503
125
  // Okay, we're doing this natively.
1504
125
  auto *Load = EmitAtomicLoadOp(AO, IsVolatile);
1505
125
1506
125
  // If we're ignoring an aggregate return, don't do anything.
1507
125
  if (getEvaluationKind() == TEK_Aggregate && 
ResultSlot.isIgnored()8
)
1508
0
    return RValue::getAggregate(Address::invalid(), false);
1509
125
1510
125
  // Okay, turn that back into the original value or atomic (for non-simple
1511
125
  // lvalues) type.
1512
125
  return ConvertIntToValueOrAtomic(Load, ResultSlot, Loc, AsValue);
1513
125
}
1514
1515
/// Emit a load from an l-value of atomic type.  Note that the r-value
1516
/// we produce is an r-value of the atomic *value* type.
1517
RValue CodeGenFunction::EmitAtomicLoad(LValue src, SourceLocation loc,
1518
                                       llvm::AtomicOrdering AO, bool IsVolatile,
1519
162
                                       AggValueSlot resultSlot) {
1520
162
  AtomicInfo Atomics(*this, src);
1521
162
  return Atomics.EmitAtomicLoad(resultSlot, loc, /*AsValue=*/true, AO,
1522
162
                                IsVolatile);
1523
162
}
1524
1525
/// Copy an r-value into memory as part of storing to an atomic type.
1526
/// This needs to create a bit-pattern suitable for atomic operations.
1527
140
void AtomicInfo::emitCopyIntoMemory(RValue rvalue) const {
1528
140
  assert(LVal.isSimple());
1529
140
  // If we have an r-value, the rvalue should be of the atomic type,
1530
140
  // which means that the caller is responsible for having zeroed
1531
140
  // any padding.  Just do an aggregate copy of that type.
1532
140
  if (rvalue.isAggregate()) {
1533
0
    LValue Dest = CGF.MakeAddrLValue(getAtomicAddress(), getAtomicType());
1534
0
    LValue Src = CGF.MakeAddrLValue(rvalue.getAggregateAddress(),
1535
0
                                    getAtomicType());
1536
0
    bool IsVolatile = rvalue.isVolatileQualified() ||
1537
0
                      LVal.isVolatileQualified();
1538
0
    CGF.EmitAggregateCopy(Dest, Src, getAtomicType(),
1539
0
                          AggValueSlot::DoesNotOverlap, IsVolatile);
1540
0
    return;
1541
0
  }
1542
140
1543
140
  // Okay, otherwise we're copying stuff.
1544
140
1545
140
  // Zero out the buffer if necessary.
1546
140
  emitMemSetZeroIfNecessary();
1547
140
1548
140
  // Drill past the padding if present.
1549
140
  LValue TempLVal = projectValue();
1550
140
1551
140
  // Okay, store the rvalue in.
1552
140
  if (rvalue.isScalar()) {
1553
122
    CGF.EmitStoreOfScalar(rvalue.getScalarVal(), TempLVal, /*init*/ true);
1554
122
  } else {
1555
18
    CGF.EmitStoreOfComplex(rvalue.getComplexVal(), TempLVal, /*init*/ true);
1556
18
  }
1557
140
}
1558
1559
1560
/// Materialize an r-value into memory for the purposes of storing it
1561
/// to an atomic type.
1562
119
Address AtomicInfo::materializeRValue(RValue rvalue) const {
1563
119
  // Aggregate r-values are already in memory, and EmitAtomicStore
1564
119
  // requires them to be values of the atomic type.
1565
119
  if (rvalue.isAggregate())
1566
10
    return rvalue.getAggregateAddress();
1567
109
1568
109
  // Otherwise, make a temporary and materialize into it.
1569
109
  LValue TempLV = CGF.MakeAddrLValue(CreateTempAlloca(), getAtomicType());
1570
109
  AtomicInfo Atomics(CGF, TempLV);
1571
109
  Atomics.emitCopyIntoMemory(rvalue);
1572
109
  return TempLV.getAddress();
1573
109
}
1574
1575
119
llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal) const {
1576
119
  // If we've got a scalar value of the right size, try to avoid going
1577
119
  // through memory.
1578
119
  if (RVal.isScalar() && 
(109
!hasPadding()109
||
!LVal.isSimple()0
)) {
1579
109
    llvm::Value *Value = RVal.getScalarVal();
1580
109
    if (isa<llvm::IntegerType>(Value->getType()))
1581
79
      return CGF.EmitToMemory(Value, ValueTy);
1582
30
    else {
1583
30
      llvm::IntegerType *InputIntTy = llvm::IntegerType::get(
1584
30
          CGF.getLLVMContext(),
1585
30
          LVal.isSimple() ? getValueSizeInBits() : 
getAtomicSizeInBits()0
);
1586
30
      if (isa<llvm::PointerType>(Value->getType()))
1587
0
        return CGF.Builder.CreatePtrToInt(Value, InputIntTy);
1588
30
      else if (llvm::BitCastInst::isBitCastable(Value->getType(), InputIntTy))
1589
12
        return CGF.Builder.CreateBitCast(Value, InputIntTy);
1590
28
    }
1591
109
  }
1592
28
  // Otherwise, we need to go through memory.
1593
28
  // Put the r-value in memory.
1594
28
  Address Addr = materializeRValue(RVal);
1595
28
1596
28
  // Cast the temporary to the atomic int type and pull a value out.
1597
28
  Addr = emitCastToAtomicIntPointer(Addr);
1598
28
  return CGF.Builder.CreateLoad(Addr);
1599
28
}
1600
1601
std::pair<llvm::Value *, llvm::Value *> AtomicInfo::EmitAtomicCompareExchangeOp(
1602
    llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
1603
207
    llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak) {
1604
207
  // Do the atomic store.
1605
207
  Address Addr = getAtomicAddressAsAtomicIntPointer();
1606
207
  auto *Inst = CGF.Builder.CreateAtomicCmpXchg(Addr.getPointer(),
1607
207
                                               ExpectedVal, DesiredVal,
1608
207
                                               Success, Failure);
1609
207
  // Other decoration.
1610
207
  Inst->setVolatile(LVal.isVolatileQualified());
1611
207
  Inst->setWeak(IsWeak);
1612
207
1613
207
  // Okay, turn that back into the original value type.
1614
207
  auto *PreviousVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/0);
1615
207
  auto *SuccessFailureVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/1);
1616
207
  return std::make_pair(PreviousVal, SuccessFailureVal);
1617
207
}
1618
1619
llvm::Value *
1620
AtomicInfo::EmitAtomicCompareExchangeLibcall(llvm::Value *ExpectedAddr,
1621
                                             llvm::Value *DesiredAddr,
1622
                                             llvm::AtomicOrdering Success,
1623
47
                                             llvm::AtomicOrdering Failure) {
1624
47
  // bool __atomic_compare_exchange(size_t size, void *obj, void *expected,
1625
47
  // void *desired, int success, int failure);
1626
47
  CallArgList Args;
1627
47
  Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
1628
47
  Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())),
1629
47
           CGF.getContext().VoidPtrTy);
1630
47
  Args.add(RValue::get(CGF.EmitCastToVoidPtr(ExpectedAddr)),
1631
47
           CGF.getContext().VoidPtrTy);
1632
47
  Args.add(RValue::get(CGF.EmitCastToVoidPtr(DesiredAddr)),
1633
47
           CGF.getContext().VoidPtrTy);
1634
47
  Args.add(RValue::get(
1635
47
               llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Success))),
1636
47
           CGF.getContext().IntTy);
1637
47
  Args.add(RValue::get(
1638
47
               llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Failure))),
1639
47
           CGF.getContext().IntTy);
1640
47
  auto SuccessFailureRVal = emitAtomicLibcall(CGF, "__atomic_compare_exchange",
1641
47
                                              CGF.getContext().BoolTy, Args);
1642
47
1643
47
  return SuccessFailureRVal.getScalarVal();
1644
47
}
1645
1646
std::pair<RValue, llvm::Value *> AtomicInfo::EmitAtomicCompareExchange(
1647
    RValue Expected, RValue Desired, llvm::AtomicOrdering Success,
1648
27
    llvm::AtomicOrdering Failure, bool IsWeak) {
1649
27
  if (isStrongerThan(Failure, Success))
1650
0
    // Don't assert on undefined behavior "failure argument shall be no stronger
1651
0
    // than the success argument".
1652
0
    Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(Success);
1653
27
1654
27
  // Check whether we should use a library call.
1655
27
  if (shouldUseLibcall()) {
1656
13
    // Produce a source address.
1657
13
    Address ExpectedAddr = materializeRValue(Expected);
1658
13
    Address DesiredAddr = materializeRValue(Desired);
1659
13
    auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1660
13
                                                 DesiredAddr.getPointer(),
1661
13
                                                 Success, Failure);
1662
13
    return std::make_pair(
1663
13
        convertAtomicTempToRValue(ExpectedAddr, AggValueSlot::ignored(),
1664
13
                                  SourceLocation(), /*AsValue=*/false),
1665
13
        Res);
1666
13
  }
1667
14
1668
14
  // If we've got a scalar value of the right size, try to avoid going
1669
14
  // through memory.
1670
14
  auto *ExpectedVal = convertRValueToInt(Expected);
1671
14
  auto *DesiredVal = convertRValueToInt(Desired);
1672
14
  auto Res = EmitAtomicCompareExchangeOp(ExpectedVal, DesiredVal, Success,
1673
14
                                         Failure, IsWeak);
1674
14
  return std::make_pair(
1675
14
      ConvertIntToValueOrAtomic(Res.first, AggValueSlot::ignored(),
1676
14
                                SourceLocation(), /*AsValue=*/false),
1677
14
      Res.second);
1678
14
}
1679
1680
static void
1681
EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, RValue OldRVal,
1682
                      const llvm::function_ref<RValue(RValue)> &UpdateOp,
1683
203
                      Address DesiredAddr) {
1684
203
  RValue UpRVal;
1685
203
  LValue AtomicLVal = Atomics.getAtomicLValue();
1686
203
  LValue DesiredLVal;
1687
203
  if (AtomicLVal.isSimple()) {
1688
155
    UpRVal = OldRVal;
1689
155
    DesiredLVal = CGF.MakeAddrLValue(DesiredAddr, AtomicLVal.getType());
1690
155
  } else {
1691
48
    // Build new lvalue for temp address.
1692
48
    Address Ptr = Atomics.materializeRValue(OldRVal);
1693
48
    LValue UpdateLVal;
1694
48
    if (AtomicLVal.isBitField()) {
1695
40
      UpdateLVal =
1696
40
          LValue::MakeBitfield(Ptr, AtomicLVal.getBitFieldInfo(),
1697
40
                               AtomicLVal.getType(),
1698
40
                               AtomicLVal.getBaseInfo(),
1699
40
                               AtomicLVal.getTBAAInfo());
1700
40
      DesiredLVal =
1701
40
          LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(),
1702
40
                               AtomicLVal.getType(), AtomicLVal.getBaseInfo(),
1703
40
                               AtomicLVal.getTBAAInfo());
1704
40
    } else 
if (8
AtomicLVal.isVectorElt()8
) {
1705
4
      UpdateLVal = LValue::MakeVectorElt(Ptr, AtomicLVal.getVectorIdx(),
1706
4
                                         AtomicLVal.getType(),
1707
4
                                         AtomicLVal.getBaseInfo(),
1708
4
                                         AtomicLVal.getTBAAInfo());
1709
4
      DesiredLVal = LValue::MakeVectorElt(
1710
4
          DesiredAddr, AtomicLVal.getVectorIdx(), AtomicLVal.getType(),
1711
4
          AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo());
1712
4
    } else {
1713
4
      assert(AtomicLVal.isExtVectorElt());
1714
4
      UpdateLVal = LValue::MakeExtVectorElt(Ptr, AtomicLVal.getExtVectorElts(),
1715
4
                                            AtomicLVal.getType(),
1716
4
                                            AtomicLVal.getBaseInfo(),
1717
4
                                            AtomicLVal.getTBAAInfo());
1718
4
      DesiredLVal = LValue::MakeExtVectorElt(
1719
4
          DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(),
1720
4
          AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo());
1721
4
    }
1722
48
    UpRVal = CGF.EmitLoadOfLValue(UpdateLVal, SourceLocation());
1723
48
  }
1724
203
  // Store new value in the corresponding memory area.
1725
203
  RValue NewRVal = UpdateOp(UpRVal);
1726
203
  if (NewRVal.isScalar()) {
1727
181
    CGF.EmitStoreThroughLValue(NewRVal, DesiredLVal);
1728
181
  } else {
1729
22
    assert(NewRVal.isComplex());
1730
22
    CGF.EmitStoreOfComplex(NewRVal.getComplexVal(), DesiredLVal,
1731
22
                           /*isInit=*/false);
1732
22
  }
1733
203
}
1734
1735
void AtomicInfo::EmitAtomicUpdateLibcall(
1736
    llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1737
30
    bool IsVolatile) {
1738
30
  auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1739
30
1740
30
  Address ExpectedAddr = CreateTempAlloca();
1741
30
1742
30
  EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile);
1743
30
  auto *ContBB = CGF.createBasicBlock("atomic_cont");
1744
30
  auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1745
30
  CGF.EmitBlock(ContBB);
1746
30
  Address DesiredAddr = CreateTempAlloca();
1747
30
  if ((LVal.isBitField() && 
BFI.Size != ValueSizeInBits8
) ||
1748
30
      
requiresMemSetZero(getAtomicAddress().getElementType())22
) {
1749
8
    auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr);
1750
8
    CGF.Builder.CreateStore(OldVal, DesiredAddr);
1751
8
  }
1752
30
  auto OldRVal = convertAtomicTempToRValue(ExpectedAddr,
1753
30
                                           AggValueSlot::ignored(),
1754
30
                                           SourceLocation(), /*AsValue=*/false);
1755
30
  EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, DesiredAddr);
1756
30
  auto *Res =
1757
30
      EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1758
30
                                       DesiredAddr.getPointer(),
1759
30
                                       AO, Failure);
1760
30
  CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
1761
30
  CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1762
30
}
1763
1764
void AtomicInfo::EmitAtomicUpdateOp(
1765
    llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1766
173
    bool IsVolatile) {
1767
173
  auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1768
173
1769
173
  // Do the atomic load.
1770
173
  auto *OldVal = EmitAtomicLoadOp(AO, IsVolatile);
1771
173
  // For non-simple lvalues perform compare-and-swap procedure.
1772
173
  auto *ContBB = CGF.createBasicBlock("atomic_cont");
1773
173
  auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1774
173
  auto *CurBB = CGF.Builder.GetInsertBlock();
1775
173
  CGF.EmitBlock(ContBB);
1776
173
  llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(),
1777
173
                                             /*NumReservedValues=*/2);
1778
173
  PHI->addIncoming(OldVal, CurBB);
1779
173
  Address NewAtomicAddr = CreateTempAlloca();
1780
173
  Address NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr);
1781
173
  if ((LVal.isBitField() && 
BFI.Size != ValueSizeInBits32
) ||
1782
173
      
requiresMemSetZero(getAtomicAddress().getElementType())141
) {
1783
48
    CGF.Builder.CreateStore(PHI, NewAtomicIntAddr);
1784
48
  }
1785
173
  auto OldRVal = ConvertIntToValueOrAtomic(PHI, AggValueSlot::ignored(),
1786
173
                                           SourceLocation(), /*AsValue=*/false);
1787
173
  EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, NewAtomicAddr);
1788
173
  auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr);
1789
173
  // Try to write new value using cmpxchg operation.
1790
173
  auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure);
1791
173
  PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock());
1792
173
  CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB);
1793
173
  CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1794
173
}
1795
1796
static void EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics,
1797
24
                                  RValue UpdateRVal, Address DesiredAddr) {
1798
24
  LValue AtomicLVal = Atomics.getAtomicLValue();
1799
24
  LValue DesiredLVal;
1800
24
  // Build new lvalue for temp address.
1801
24
  if (AtomicLVal.isBitField()) {
1802
20
    DesiredLVal =
1803
20
        LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(),
1804
20
                             AtomicLVal.getType(), AtomicLVal.getBaseInfo(),
1805
20
                             AtomicLVal.getTBAAInfo());
1806
20
  } else 
if (4
AtomicLVal.isVectorElt()4
) {
1807
2
    DesiredLVal =
1808
2
        LValue::MakeVectorElt(DesiredAddr, AtomicLVal.getVectorIdx(),
1809
2
                              AtomicLVal.getType(), AtomicLVal.getBaseInfo(),
1810
2
                              AtomicLVal.getTBAAInfo());
1811
2
  } else {
1812
2
    assert(AtomicLVal.isExtVectorElt());
1813
2
    DesiredLVal = LValue::MakeExtVectorElt(
1814
2
        DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(),
1815
2
        AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo());
1816
2
  }
1817
24
  // Store new value in the corresponding memory area.
1818
24
  assert(UpdateRVal.isScalar());
1819
24
  CGF.EmitStoreThroughLValue(UpdateRVal, DesiredLVal);
1820
24
}
1821
1822
void AtomicInfo::EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
1823
4
                                         RValue UpdateRVal, bool IsVolatile) {
1824
4
  auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1825
4
1826
4
  Address ExpectedAddr = CreateTempAlloca();
1827
4
1828
4
  EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile);
1829
4
  auto *ContBB = CGF.createBasicBlock("atomic_cont");
1830
4
  auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1831
4
  CGF.EmitBlock(ContBB);
1832
4
  Address DesiredAddr = CreateTempAlloca();
1833
4
  if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1834
4
      
requiresMemSetZero(getAtomicAddress().getElementType())0
) {
1835
4
    auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr);
1836
4
    CGF.Builder.CreateStore(OldVal, DesiredAddr);
1837
4
  }
1838
4
  EmitAtomicUpdateValue(CGF, *this, UpdateRVal, DesiredAddr);
1839
4
  auto *Res =
1840
4
      EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1841
4
                                       DesiredAddr.getPointer(),
1842
4
                                       AO, Failure);
1843
4
  CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
1844
4
  CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1845
4
}
1846
1847
void AtomicInfo::EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRVal,
1848
20
                                    bool IsVolatile) {
1849
20
  auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1850
20
1851
20
  // Do the atomic load.
1852
20
  auto *OldVal = EmitAtomicLoadOp(AO, IsVolatile);
1853
20
  // For non-simple lvalues perform compare-and-swap procedure.
1854
20
  auto *ContBB = CGF.createBasicBlock("atomic_cont");
1855
20
  auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1856
20
  auto *CurBB = CGF.Builder.GetInsertBlock();
1857
20
  CGF.EmitBlock(ContBB);
1858
20
  llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(),
1859
20
                                             /*NumReservedValues=*/2);
1860
20
  PHI->addIncoming(OldVal, CurBB);
1861
20
  Address NewAtomicAddr = CreateTempAlloca();
1862
20
  Address NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr);
1863
20
  if ((LVal.isBitField() && 
BFI.Size != ValueSizeInBits16
) ||
1864
20
      
requiresMemSetZero(getAtomicAddress().getElementType())4
) {
1865
20
    CGF.Builder.CreateStore(PHI, NewAtomicIntAddr);
1866
20
  }
1867
20
  EmitAtomicUpdateValue(CGF, *this, UpdateRVal, NewAtomicAddr);
1868
20
  auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr);
1869
20
  // Try to write new value using cmpxchg operation.
1870
20
  auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure);
1871
20
  PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock());
1872
20
  CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB);
1873
20
  CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1874
20
}
1875
1876
void AtomicInfo::EmitAtomicUpdate(
1877
    llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1878
203
    bool IsVolatile) {
1879
203
  if (shouldUseLibcall()) {
1880
30
    EmitAtomicUpdateLibcall(AO, UpdateOp, IsVolatile);
1881
173
  } else {
1882
173
    EmitAtomicUpdateOp(AO, UpdateOp, IsVolatile);
1883
173
  }
1884
203
}
1885
1886
void AtomicInfo::EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
1887
24
                                  bool IsVolatile) {
1888
24
  if (shouldUseLibcall()) {
1889
4
    EmitAtomicUpdateLibcall(AO, UpdateRVal, IsVolatile);
1890
20
  } else {
1891
20
    EmitAtomicUpdateOp(AO, UpdateRVal, IsVolatile);
1892
20
  }
1893
24
}
1894
1895
void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue lvalue,
1896
48
                                      bool isInit) {
1897
48
  bool IsVolatile = lvalue.isVolatileQualified();
1898
48
  llvm::AtomicOrdering AO;
1899
48
  if (lvalue.getType()->isAtomicType()) {
1900
41
    AO = llvm::AtomicOrdering::SequentiallyConsistent;
1901
41
  } else {
1902
7
    AO = llvm::AtomicOrdering::Release;
1903
7
    IsVolatile = true;
1904
7
  }
1905
48
  return EmitAtomicStore(rvalue, lvalue, AO, IsVolatile, isInit);
1906
48
}
1907
1908
/// Emit a store to an l-value of atomic type.
1909
///
1910
/// Note that the r-value is expected to be an r-value *of the atomic
1911
/// type*; this means that for aggregate r-values, it should include
1912
/// storage for any padding that was necessary.
1913
void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest,
1914
                                      llvm::AtomicOrdering AO, bool IsVolatile,
1915
152
                                      bool isInit) {
1916
152
  // If this is an aggregate r-value, it should agree in type except
1917
152
  // maybe for address-space qualification.
1918
152
  assert(!rvalue.isAggregate() ||
1919
152
         rvalue.getAggregateAddress().getElementType()
1920
152
           == dest.getAddress().getElementType());
1921
152
1922
152
  AtomicInfo atomics(*this, dest);
1923
152
  LValue LVal = atomics.getAtomicLValue();
1924
152
1925
152
  // If this is an initialization, just put the value there normally.
1926
152
  if (LVal.isSimple()) {
1927
128
    if (isInit) {
1928
20
      atomics.emitCopyIntoMemory(rvalue);
1929
20
      return;
1930
20
    }
1931
108
1932
108
    // Check whether we should use a library call.
1933
108
    if (atomics.shouldUseLibcall()) {
1934
17
      // Produce a source address.
1935
17
      Address srcAddr = atomics.materializeRValue(rvalue);
1936
17
1937
17
      // void __atomic_store(size_t size, void *mem, void *val, int order)
1938
17
      CallArgList args;
1939
17
      args.add(RValue::get(atomics.getAtomicSizeValue()),
1940
17
               getContext().getSizeType());
1941
17
      args.add(RValue::get(EmitCastToVoidPtr(atomics.getAtomicPointer())),
1942
17
               getContext().VoidPtrTy);
1943
17
      args.add(RValue::get(EmitCastToVoidPtr(srcAddr.getPointer())),
1944
17
               getContext().VoidPtrTy);
1945
17
      args.add(
1946
17
          RValue::get(llvm::ConstantInt::get(IntTy, (int)llvm::toCABI(AO))),
1947
17
          getContext().IntTy);
1948
17
      emitAtomicLibcall(*this, "__atomic_store", getContext().VoidTy, args);
1949
17
      return;
1950
17
    }
1951
91
1952
91
    // Okay, we're doing this natively.
1953
91
    llvm::Value *intValue = atomics.convertRValueToInt(rvalue);
1954
91
1955
91
    // Do the atomic store.
1956
91
    Address addr =
1957
91
        atomics.emitCastToAtomicIntPointer(atomics.getAtomicAddress());
1958
91
    intValue = Builder.CreateIntCast(
1959
91
        intValue, addr.getElementType(), /*isSigned=*/false);
1960
91
    llvm::StoreInst *store = Builder.CreateStore(intValue, addr);
1961
91
1962
91
    // Initializations don't need to be atomic.
1963
91
    if (!isInit)
1964
91
      store->setAtomic(AO);
1965
91
1966
91
    // Other decoration.
1967
91
    if (IsVolatile)
1968
8
      store->setVolatile(true);
1969
91
    CGM.DecorateInstructionWithTBAA(store, dest.getTBAAInfo());
1970
91
    return;
1971
91
  }
1972
24
1973
24
  // Emit simple atomic update operation.
1974
24
  atomics.EmitAtomicUpdate(AO, rvalue, IsVolatile);
1975
24
}
1976
1977
/// Emit a compare-and-exchange op for atomic type.
1978
///
1979
std::pair<RValue, llvm::Value *> CodeGenFunction::EmitAtomicCompareExchange(
1980
    LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc,
1981
    llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak,
1982
27
    AggValueSlot Slot) {
1983
27
  // If this is an aggregate r-value, it should agree in type except
1984
27
  // maybe for address-space qualification.
1985
27
  assert(!Expected.isAggregate() ||
1986
27
         Expected.getAggregateAddress().getElementType() ==
1987
27
             Obj.getAddress().getElementType());
1988
27
  assert(!Desired.isAggregate() ||
1989
27
         Desired.getAggregateAddress().getElementType() ==
1990
27
             Obj.getAddress().getElementType());
1991
27
  AtomicInfo Atomics(*this, Obj);
1992
27
1993
27
  return Atomics.EmitAtomicCompareExchange(Expected, Desired, Success, Failure,
1994
27
                                           IsWeak);
1995
27
}
1996
1997
void CodeGenFunction::EmitAtomicUpdate(
1998
    LValue LVal, llvm::AtomicOrdering AO,
1999
203
    const llvm::function_ref<RValue(RValue)> &UpdateOp, bool IsVolatile) {
2000
203
  AtomicInfo Atomics(*this, LVal);
2001
203
  Atomics.EmitAtomicUpdate(AO, UpdateOp, IsVolatile);
2002
203
}
2003
2004
20
void CodeGenFunction::EmitAtomicInit(Expr *init, LValue dest) {
2005
20
  AtomicInfo atomics(*this, dest);
2006
20
2007
20
  switch (atomics.getEvaluationKind()) {
2008
20
  case TEK_Scalar: {
2009
9
    llvm::Value *value = EmitScalarExpr(init);
2010
9
    atomics.emitCopyIntoMemory(RValue::get(value));
2011
9
    return;
2012
20
  }
2013
20
2014
20
  case TEK_Complex: {
2015
2
    ComplexPairTy value = EmitComplexExpr(init);
2016
2
    atomics.emitCopyIntoMemory(RValue::getComplex(value));
2017
2
    return;
2018
20
  }
2019
20
2020
20
  case TEK_Aggregate: {
2021
9
    // Fix up the destination if the initializer isn't an expression
2022
9
    // of atomic type.
2023
9
    bool Zeroed = false;
2024
9
    if (!init->getType()->isAtomicType()) {
2025
5
      Zeroed = atomics.emitMemSetZeroIfNecessary();
2026
5
      dest = atomics.projectValue();
2027
5
    }
2028
9
2029
9
    // Evaluate the expression directly into the destination.
2030
9
    AggValueSlot slot = AggValueSlot::forLValue(dest,
2031
9
                                        AggValueSlot::IsNotDestructed,
2032
9
                                        AggValueSlot::DoesNotNeedGCBarriers,
2033
9
                                        AggValueSlot::IsNotAliased,
2034
9
                                        AggValueSlot::DoesNotOverlap,
2035
9
                                        Zeroed ? 
AggValueSlot::IsZeroed2
:
2036
9
                                                 
AggValueSlot::IsNotZeroed7
);
2037
9
2038
9
    EmitAggExpr(init, slot);
2039
9
    return;
2040
0
  }
2041
0
  }
2042
0
  llvm_unreachable("bad evaluation kind");
2043
0
}