Coverage Report

Created: 2022-07-16 07:03

/Users/buildslave/jenkins/workspace/coverage/llvm-project/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
Line
Count
Source (jump to first uncovered line)
1
//===- DependencyScanningFilesystem.cpp - clang-scan-deps fs --------------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
9
#include "clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h"
10
#include "llvm/Support/MemoryBuffer.h"
11
#include "llvm/Support/SmallVectorMemoryBuffer.h"
12
#include "llvm/Support/Threading.h"
13
14
using namespace clang;
15
using namespace tooling;
16
using namespace dependencies;
17
18
llvm::ErrorOr<DependencyScanningWorkerFilesystem::TentativeEntry>
19
3.85k
DependencyScanningWorkerFilesystem::readFile(StringRef Filename) {
20
  // Load the file and its content from the file system.
21
3.85k
  auto MaybeFile = getUnderlyingFS().openFileForRead(Filename);
22
3.85k
  if (!MaybeFile)
23
0
    return MaybeFile.getError();
24
3.85k
  auto File = std::move(*MaybeFile);
25
26
3.85k
  auto MaybeStat = File->status();
27
3.85k
  if (!MaybeStat)
28
0
    return MaybeStat.getError();
29
3.85k
  auto Stat = std::move(*MaybeStat);
30
31
3.85k
  auto MaybeBuffer = File->getBuffer(Stat.getName());
32
3.85k
  if (!MaybeBuffer)
33
0
    return MaybeBuffer.getError();
34
3.85k
  auto Buffer = std::move(*MaybeBuffer);
35
36
  // If the file size changed between read and stat, pretend it didn't.
37
3.85k
  if (Stat.getSize() != Buffer->getBufferSize())
38
0
    Stat = llvm::vfs::Status::copyWithNewSize(Stat, Buffer->getBufferSize());
39
40
3.85k
  return TentativeEntry(Stat, std::move(Buffer));
41
3.85k
}
42
43
EntryRef DependencyScanningWorkerFilesystem::scanForDirectivesIfNecessary(
44
6.46k
    const CachedFileSystemEntry &Entry, StringRef Filename, bool Disable) {
45
6.46k
  if (Entry.isError() || 
Entry.isDirectory()6.30k
||
Disable5.27k
||
46
6.46k
      
!shouldScanForDirectives(Filename)5.27k
)
47
1.73k
    return EntryRef(Filename, Entry);
48
49
4.73k
  CachedFileContents *Contents = Entry.getCachedContents();
50
4.73k
  assert(Contents && "contents not initialized");
51
52
  // Double-checked locking.
53
4.73k
  if (Contents->DepDirectives.load())
54
1.15k
    return EntryRef(Filename, Entry);
55
56
3.57k
  std::lock_guard<std::mutex> GuardLock(Contents->ValueLock);
57
58
  // Double-checked locking.
59
3.57k
  if (Contents->DepDirectives.load())
60
12
    return EntryRef(Filename, Entry);
61
62
3.56k
  SmallVector<dependency_directives_scan::Directive, 64> Directives;
63
  // Scan the file for preprocessor directives that might affect the
64
  // dependencies.
65
3.56k
  if (scanSourceForDependencyDirectives(Contents->Original->getBuffer(),
66
3.56k
                                        Contents->DepDirectiveTokens,
67
3.56k
                                        Directives)) {
68
0
    Contents->DepDirectiveTokens.clear();
69
    // FIXME: Propagate the diagnostic if desired by the client.
70
0
    Contents->DepDirectives.store(new Optional<DependencyDirectivesTy>());
71
0
    return EntryRef(Filename, Entry);
72
0
  }
73
74
  // This function performed double-checked locking using `DepDirectives`.
75
  // Assigning it must be the last thing this function does, otherwise other
76
  // threads may skip the
77
  // critical section (`DepDirectives != nullptr`), leading to a data race.
78
3.56k
  Contents->DepDirectives.store(
79
3.56k
      new Optional<DependencyDirectivesTy>(std::move(Directives)));
80
3.56k
  return EntryRef(Filename, Entry);
81
3.56k
}
82
83
DependencyScanningFilesystemSharedCache::
84
88
    DependencyScanningFilesystemSharedCache() {
85
  // This heuristic was chosen using a empirical testing on a
86
  // reasonably high core machine (iMacPro 18 cores / 36 threads). The cache
87
  // sharding gives a performance edge by reducing the lock contention.
88
  // FIXME: A better heuristic might also consider the OS to account for
89
  // the different cost of lock contention on different OSes.
90
88
  NumShards =
91
88
      std::max(2u, llvm::hardware_concurrency().compute_thread_count() / 4);
92
88
  CacheShards = std::make_unique<CacheShard[]>(NumShards);
93
88
}
94
95
DependencyScanningFilesystemSharedCache::CacheShard &
96
DependencyScanningFilesystemSharedCache::getShardForFilename(
97
11.5k
    StringRef Filename) const {
98
11.5k
  return CacheShards[llvm::hash_value(Filename) % NumShards];
99
11.5k
}
100
101
DependencyScanningFilesystemSharedCache::CacheShard &
102
DependencyScanningFilesystemSharedCache::getShardForUID(
103
9.52k
    llvm::sys::fs::UniqueID UID) const {
104
9.52k
  auto Hash = llvm::hash_combine(UID.getDevice(), UID.getFile());
105
9.52k
  return CacheShards[Hash % NumShards];
106
9.52k
}
107
108
const CachedFileSystemEntry *
109
DependencyScanningFilesystemSharedCache::CacheShard::findEntryByFilename(
110
6.70k
    StringRef Filename) const {
111
6.70k
  std::lock_guard<std::mutex> LockGuard(CacheLock);
112
6.70k
  auto It = EntriesByFilename.find(Filename);
113
6.70k
  return It == EntriesByFilename.end() ? 
nullptr6.61k
:
It->getValue()97
;
114
6.70k
}
115
116
const CachedFileSystemEntry *
117
DependencyScanningFilesystemSharedCache::CacheShard::findEntryByUID(
118
4.81k
    llvm::sys::fs::UniqueID UID) const {
119
4.81k
  std::lock_guard<std::mutex> LockGuard(CacheLock);
120
4.81k
  auto It = EntriesByUID.find(UID);
121
4.81k
  return It == EntriesByUID.end() ? 
nullptr4.70k
:
It->getSecond()109
;
122
4.81k
}
123
124
const CachedFileSystemEntry &
125
DependencyScanningFilesystemSharedCache::CacheShard::
126
    getOrEmplaceEntryForFilename(StringRef Filename,
127
137
                                 llvm::ErrorOr<llvm::vfs::Status> Stat) {
128
137
  std::lock_guard<std::mutex> LockGuard(CacheLock);
129
137
  auto Insertion = EntriesByFilename.insert({Filename, nullptr});
130
137
  if (Insertion.second)
131
120
    Insertion.first->second =
132
120
        new (EntryStorage.Allocate()) CachedFileSystemEntry(std::move(Stat));
133
137
  return *Insertion.first->second;
134
137
}
135
136
const CachedFileSystemEntry &
137
DependencyScanningFilesystemSharedCache::CacheShard::getOrEmplaceEntryForUID(
138
    llvm::sys::fs::UniqueID UID, llvm::vfs::Status Stat,
139
4.70k
    std::unique_ptr<llvm::MemoryBuffer> Contents) {
140
4.70k
  std::lock_guard<std::mutex> LockGuard(CacheLock);
141
4.70k
  auto Insertion = EntriesByUID.insert({UID, nullptr});
142
4.70k
  if (Insertion.second) {
143
4.65k
    CachedFileContents *StoredContents = nullptr;
144
4.65k
    if (Contents)
145
3.81k
      StoredContents = new (ContentsStorage.Allocate())
146
3.81k
          CachedFileContents(std::move(Contents));
147
4.65k
    Insertion.first->second = new (EntryStorage.Allocate())
148
4.65k
        CachedFileSystemEntry(std::move(Stat), StoredContents);
149
4.65k
  }
150
4.70k
  return *Insertion.first->second;
151
4.70k
}
152
153
const CachedFileSystemEntry &
154
DependencyScanningFilesystemSharedCache::CacheShard::
155
    getOrInsertEntryForFilename(StringRef Filename,
156
4.70k
                                const CachedFileSystemEntry &Entry) {
157
4.70k
  std::lock_guard<std::mutex> LockGuard(CacheLock);
158
4.70k
  return *EntriesByFilename.insert({Filename, &Entry}).first->getValue();
159
4.70k
}
160
161
/// Whitelist file extensions that should be minimized, treating no extension as
162
/// a source file that should be minimized.
163
///
164
/// This is kinda hacky, it would be better if we knew what kind of file Clang
165
/// was expecting instead.
166
6.57k
static bool shouldScanForDirectivesBasedOnExtension(StringRef Filename) {
167
6.57k
  StringRef Ext = llvm::sys::path::extension(Filename);
168
6.57k
  if (Ext.empty())
169
12
    return true; // C++ standard library
170
6.56k
  return llvm::StringSwitch<bool>(Ext)
171
6.56k
      .CasesLower(".c", ".cc", ".cpp", ".c++", ".cxx", true)
172
6.56k
      .CasesLower(".h", ".hh", ".hpp", ".h++", ".hxx", true)
173
6.56k
      .CasesLower(".m", ".mm", true)
174
6.56k
      .CasesLower(".i", ".ii", ".mi", ".mmi", true)
175
6.56k
      .CasesLower(".def", ".inc", true)
176
6.56k
      .Default(false);
177
6.57k
}
178
179
1.79k
static bool shouldCacheStatFailures(StringRef Filename) {
180
1.79k
  StringRef Ext = llvm::sys::path::extension(Filename);
181
1.79k
  if (Ext.empty())
182
490
    return false; // This may be the module cache directory.
183
  // Only cache stat failures on source files.
184
1.30k
  return shouldScanForDirectivesBasedOnExtension(Filename);
185
1.79k
}
186
187
bool DependencyScanningWorkerFilesystem::shouldScanForDirectives(
188
5.27k
    StringRef Filename) {
189
5.27k
  return shouldScanForDirectivesBasedOnExtension(Filename);
190
5.27k
}
191
192
const CachedFileSystemEntry &
193
DependencyScanningWorkerFilesystem::getOrEmplaceSharedEntryForUID(
194
4.70k
    TentativeEntry TEntry) {
195
4.70k
  auto &Shard = SharedCache.getShardForUID(TEntry.Status.getUniqueID());
196
4.70k
  return Shard.getOrEmplaceEntryForUID(TEntry.Status.getUniqueID(),
197
4.70k
                                       std::move(TEntry.Status),
198
4.70k
                                       std::move(TEntry.Contents));
199
4.70k
}
200
201
const CachedFileSystemEntry *
202
DependencyScanningWorkerFilesystem::findEntryByFilenameWithWriteThrough(
203
8.11k
    StringRef Filename) {
204
8.11k
  if (const auto *Entry = LocalCache.findEntryByFilename(Filename))
205
1.40k
    return Entry;
206
6.71k
  auto &Shard = SharedCache.getShardForFilename(Filename);
207
6.71k
  if (const auto *Entry = Shard.findEntryByFilename(Filename))
208
101
    return &LocalCache.insertEntryForFilename(Filename, *Entry);
209
6.61k
  return nullptr;
210
6.71k
}
211
212
llvm::ErrorOr<const CachedFileSystemEntry &>
213
6.61k
DependencyScanningWorkerFilesystem::computeAndStoreResult(StringRef Filename) {
214
6.61k
  llvm::ErrorOr<llvm::vfs::Status> Stat = getUnderlyingFS().status(Filename);
215
6.61k
  if (!Stat) {
216
1.79k
    if (!shouldCacheStatFailures(Filename))
217
1.65k
      return Stat.getError();
218
137
    const auto &Entry =
219
137
        getOrEmplaceSharedEntryForFilename(Filename, Stat.getError());
220
137
    return insertLocalEntryForFilename(Filename, Entry);
221
1.79k
  }
222
223
4.82k
  if (const auto *Entry = findSharedEntryByUID(*Stat))
224
112
    return insertLocalEntryForFilename(Filename, *Entry);
225
226
4.70k
  auto TEntry =
227
4.70k
      Stat->isDirectory() ? 
TentativeEntry(*Stat)848
:
readFile(Filename)3.86k
;
228
229
4.70k
  const CachedFileSystemEntry *SharedEntry = [&]() {
230
4.70k
    if (TEntry) {
231
4.70k
      const auto &UIDEntry = getOrEmplaceSharedEntryForUID(std::move(*TEntry));
232
4.70k
      return &getOrInsertSharedEntryForFilename(Filename, UIDEntry);
233
4.70k
    }
234
0
    return &getOrEmplaceSharedEntryForFilename(Filename, TEntry.getError());
235
4.70k
  }();
236
237
4.70k
  return insertLocalEntryForFilename(Filename, *SharedEntry);
238
4.82k
}
239
240
llvm::ErrorOr<EntryRef>
241
DependencyScanningWorkerFilesystem::getOrCreateFileSystemEntry(
242
8.11k
    StringRef Filename, bool DisableDirectivesScanning) {
243
8.11k
  if (const auto *Entry = findEntryByFilenameWithWriteThrough(Filename))
244
1.50k
    return scanForDirectivesIfNecessary(*Entry, Filename,
245
1.50k
                                        DisableDirectivesScanning)
246
1.50k
        .unwrapError();
247
6.60k
  auto MaybeEntry = computeAndStoreResult(Filename);
248
6.60k
  if (!MaybeEntry)
249
1.65k
    return MaybeEntry.getError();
250
4.95k
  return scanForDirectivesIfNecessary(*MaybeEntry, Filename,
251
4.95k
                                      DisableDirectivesScanning)
252
4.95k
      .unwrapError();
253
6.60k
}
254
255
llvm::ErrorOr<llvm::vfs::Status>
256
6.92k
DependencyScanningWorkerFilesystem::status(const Twine &Path) {
257
6.92k
  SmallString<256> OwnedFilename;
258
6.92k
  StringRef Filename = Path.toStringRef(OwnedFilename);
259
260
6.92k
  llvm::ErrorOr<EntryRef> Result = getOrCreateFileSystemEntry(Filename);
261
6.92k
  if (!Result)
262
1.67k
    return Result.getError();
263
5.24k
  return Result->getStatus();
264
6.92k
}
265
266
namespace {
267
268
/// The VFS that is used by clang consumes the \c CachedFileSystemEntry using
269
/// this subclass.
270
class DepScanFile final : public llvm::vfs::File {
271
public:
272
  DepScanFile(std::unique_ptr<llvm::MemoryBuffer> Buffer,
273
              llvm::vfs::Status Stat)
274
737
      : Buffer(std::move(Buffer)), Stat(std::move(Stat)) {}
275
276
  static llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>> create(EntryRef Entry);
277
278
784
  llvm::ErrorOr<llvm::vfs::Status> status() override { return Stat; }
279
280
  llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
281
  getBuffer(const Twine &Name, int64_t FileSize, bool RequiresNullTerminator,
282
626
            bool IsVolatile) override {
283
626
    return std::move(Buffer);
284
626
  }
285
286
0
  std::error_code close() override { return {}; }
287
288
private:
289
  std::unique_ptr<llvm::MemoryBuffer> Buffer;
290
  llvm::vfs::Status Stat;
291
};
292
293
} // end anonymous namespace
294
295
llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>
296
740
DepScanFile::create(EntryRef Entry) {
297
740
  assert(!Entry.isError() && "error");
298
299
740
  if (Entry.isDirectory())
300
2
    return std::make_error_code(std::errc::is_a_directory);
301
302
738
  auto Result = std::make_unique<DepScanFile>(
303
738
      llvm::MemoryBuffer::getMemBuffer(Entry.getContents(),
304
738
                                       Entry.getStatus().getName(),
305
738
                                       /*RequiresNullTerminator=*/false),
306
738
      Entry.getStatus());
307
308
738
  return llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>(
309
738
      std::unique_ptr<llvm::vfs::File>(std::move(Result)));
310
740
}
311
312
llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>
313
875
DependencyScanningWorkerFilesystem::openFileForRead(const Twine &Path) {
314
875
  SmallString<256> OwnedFilename;
315
875
  StringRef Filename = Path.toStringRef(OwnedFilename);
316
317
875
  llvm::ErrorOr<EntryRef> Result = getOrCreateFileSystemEntry(Filename);
318
875
  if (!Result)
319
136
    return Result.getError();
320
739
  return DepScanFile::create(Result.get());
321
875
}