Coverage Report

Created: 2022-05-17 06:19

/Users/buildslave/jenkins/workspace/coverage/llvm-project/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
Line
Count
Source (jump to first uncovered line)
1
//===- DependencyScanningFilesystem.cpp - clang-scan-deps fs --------------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
9
#include "clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h"
10
#include "clang/Lex/DependencyDirectivesSourceMinimizer.h"
11
#include "llvm/Support/MemoryBuffer.h"
12
#include "llvm/Support/SmallVectorMemoryBuffer.h"
13
#include "llvm/Support/Threading.h"
14
15
using namespace clang;
16
using namespace tooling;
17
using namespace dependencies;
18
19
llvm::ErrorOr<DependencyScanningWorkerFilesystem::TentativeEntry>
20
3.82k
DependencyScanningWorkerFilesystem::readFile(StringRef Filename) {
21
  // Load the file and its content from the file system.
22
3.82k
  auto MaybeFile = getUnderlyingFS().openFileForRead(Filename);
23
3.82k
  if (!MaybeFile)
24
0
    return MaybeFile.getError();
25
3.82k
  auto File = std::move(*MaybeFile);
26
27
3.82k
  auto MaybeStat = File->status();
28
3.82k
  if (!MaybeStat)
29
0
    return MaybeStat.getError();
30
3.82k
  auto Stat = std::move(*MaybeStat);
31
32
3.82k
  auto MaybeBuffer = File->getBuffer(Stat.getName());
33
3.82k
  if (!MaybeBuffer)
34
0
    return MaybeBuffer.getError();
35
3.82k
  auto Buffer = std::move(*MaybeBuffer);
36
37
  // If the file size changed between read and stat, pretend it didn't.
38
3.82k
  if (Stat.getSize() != Buffer->getBufferSize())
39
0
    Stat = llvm::vfs::Status::copyWithNewSize(Stat, Buffer->getBufferSize());
40
41
3.82k
  return TentativeEntry(Stat, std::move(Buffer));
42
3.82k
}
43
44
EntryRef DependencyScanningWorkerFilesystem::minimizeIfNecessary(
45
6.32k
    const CachedFileSystemEntry &Entry, StringRef Filename, bool Disable) {
46
6.32k
  if (Entry.isError() || 
Entry.isDirectory()6.16k
||
Disable4.94k
||
47
6.32k
      
!shouldMinimize(Filename, Entry.getUniqueID())4.90k
)
48
1.98k
    return EntryRef(/*Minimized=*/false, Filename, Entry);
49
50
4.33k
  CachedFileContents *Contents = Entry.getContents();
51
4.33k
  assert(Contents && "contents not initialized");
52
53
  // Double-checked locking.
54
4.33k
  if (Contents->MinimizedAccess.load())
55
836
    return EntryRef(/*Minimized=*/true, Filename, Entry);
56
57
3.50k
  std::lock_guard<std::mutex> GuardLock(Contents->ValueLock);
58
59
  // Double-checked locking.
60
3.50k
  if (Contents->MinimizedAccess.load())
61
11
    return EntryRef(/*Minimized=*/true, Filename, Entry);
62
63
3.49k
  llvm::SmallString<1024> MinimizedFileContents;
64
  // Minimize the file down to directives that might affect the dependencies.
65
3.49k
  SmallVector<minimize_source_to_dependency_directives::Token, 64> Tokens;
66
3.49k
  if (minimizeSourceToDependencyDirectives(Contents->Original->getBuffer(),
67
3.49k
                                           MinimizedFileContents, Tokens)) {
68
    // FIXME: Propagate the diagnostic if desired by the client.
69
    // Use the original file if the minimization failed.
70
0
    Contents->MinimizedStorage =
71
0
        llvm::MemoryBuffer::getMemBuffer(*Contents->Original);
72
0
    Contents->MinimizedAccess.store(Contents->MinimizedStorage.get());
73
0
    return EntryRef(/*Minimized=*/true, Filename, Entry);
74
0
  }
75
76
  // The contents produced by the minimizer must be null terminated.
77
3.49k
  assert(MinimizedFileContents.data()[MinimizedFileContents.size()] == '\0' &&
78
3.49k
         "not null terminated contents");
79
80
  // Compute the skipped PP ranges that speedup skipping over inactive
81
  // preprocessor blocks.
82
0
  llvm::SmallVector<minimize_source_to_dependency_directives::SkippedRange, 32>
83
3.49k
      SkippedRanges;
84
3.49k
  minimize_source_to_dependency_directives::computeSkippedRanges(Tokens,
85
3.49k
                                                                 SkippedRanges);
86
3.49k
  PreprocessorSkippedRangeMapping Mapping;
87
49.2k
  for (const auto &Range : SkippedRanges) {
88
49.2k
    if (Range.Length < 16) {
89
      // Ignore small ranges as non-profitable.
90
      // FIXME: This is a heuristic, its worth investigating the tradeoffs
91
      // when it should be applied.
92
146
      continue;
93
146
    }
94
49.0k
    Mapping[Range.Offset] = Range.Length;
95
49.0k
  }
96
3.49k
  Contents->PPSkippedRangeMapping = std::move(Mapping);
97
98
3.49k
  Contents->MinimizedStorage = std::make_unique<llvm::SmallVectorMemoryBuffer>(
99
3.49k
      std::move(MinimizedFileContents));
100
  // This function performed double-checked locking using `MinimizedAccess`.
101
  // Assigning it must be the last thing this function does. If we were to
102
  // assign it before `PPSkippedRangeMapping`, other threads may skip the
103
  // critical section (`MinimizedAccess != nullptr`) and access the mappings
104
  // that are about to be initialized, leading to a data race.
105
3.49k
  Contents->MinimizedAccess.store(Contents->MinimizedStorage.get());
106
3.49k
  return EntryRef(/*Minimized=*/true, Filename, Entry);
107
3.49k
}
108
109
DependencyScanningFilesystemSharedCache::
110
81
    DependencyScanningFilesystemSharedCache() {
111
  // This heuristic was chosen using a empirical testing on a
112
  // reasonably high core machine (iMacPro 18 cores / 36 threads). The cache
113
  // sharding gives a performance edge by reducing the lock contention.
114
  // FIXME: A better heuristic might also consider the OS to account for
115
  // the different cost of lock contention on different OSes.
116
81
  NumShards =
117
81
      std::max(2u, llvm::hardware_concurrency().compute_thread_count() / 4);
118
81
  CacheShards = std::make_unique<CacheShard[]>(NumShards);
119
81
}
120
121
DependencyScanningFilesystemSharedCache::CacheShard &
122
DependencyScanningFilesystemSharedCache::getShardForFilename(
123
11.6k
    StringRef Filename) const {
124
11.6k
  return CacheShards[llvm::hash_value(Filename) % NumShards];
125
11.6k
}
126
127
DependencyScanningFilesystemSharedCache::CacheShard &
128
DependencyScanningFilesystemSharedCache::getShardForUID(
129
9.45k
    llvm::sys::fs::UniqueID UID) const {
130
9.45k
  auto Hash = llvm::hash_combine(UID.getDevice(), UID.getFile());
131
9.45k
  return CacheShards[Hash % NumShards];
132
9.45k
}
133
134
const CachedFileSystemEntry *
135
DependencyScanningFilesystemSharedCache::CacheShard::findEntryByFilename(
136
6.87k
    StringRef Filename) const {
137
6.87k
  std::lock_guard<std::mutex> LockGuard(CacheLock);
138
6.87k
  auto It = EntriesByFilename.find(Filename);
139
6.87k
  return It == EntriesByFilename.end() ? 
nullptr6.78k
:
It->getValue()84
;
140
6.87k
}
141
142
const CachedFileSystemEntry *
143
DependencyScanningFilesystemSharedCache::CacheShard::findEntryByUID(
144
4.80k
    llvm::sys::fs::UniqueID UID) const {
145
4.80k
  std::lock_guard<std::mutex> LockGuard(CacheLock);
146
4.80k
  auto It = EntriesByUID.find(UID);
147
4.80k
  return It == EntriesByUID.end() ? 
nullptr4.64k
:
It->getSecond()155
;
148
4.80k
}
149
150
const CachedFileSystemEntry &
151
DependencyScanningFilesystemSharedCache::CacheShard::
152
    getOrEmplaceEntryForFilename(StringRef Filename,
153
137
                                 llvm::ErrorOr<llvm::vfs::Status> Stat) {
154
137
  std::lock_guard<std::mutex> LockGuard(CacheLock);
155
137
  auto Insertion = EntriesByFilename.insert({Filename, nullptr});
156
137
  if (Insertion.second)
157
120
    Insertion.first->second =
158
120
        new (EntryStorage.Allocate()) CachedFileSystemEntry(std::move(Stat));
159
137
  return *Insertion.first->second;
160
137
}
161
162
const CachedFileSystemEntry &
163
DependencyScanningFilesystemSharedCache::CacheShard::getOrEmplaceEntryForUID(
164
    llvm::sys::fs::UniqueID UID, llvm::vfs::Status Stat,
165
4.64k
    std::unique_ptr<llvm::MemoryBuffer> Contents) {
166
4.64k
  std::lock_guard<std::mutex> LockGuard(CacheLock);
167
4.64k
  auto Insertion = EntriesByUID.insert({UID, nullptr});
168
4.64k
  if (Insertion.second) {
169
4.58k
    CachedFileContents *StoredContents = nullptr;
170
4.58k
    if (Contents)
171
3.77k
      StoredContents = new (ContentsStorage.Allocate())
172
3.77k
          CachedFileContents(std::move(Contents));
173
4.58k
    Insertion.first->second = new (EntryStorage.Allocate())
174
4.58k
        CachedFileSystemEntry(std::move(Stat), StoredContents);
175
4.58k
  }
176
4.64k
  return *Insertion.first->second;
177
4.64k
}
178
179
const CachedFileSystemEntry &
180
DependencyScanningFilesystemSharedCache::CacheShard::
181
    getOrInsertEntryForFilename(StringRef Filename,
182
4.64k
                                const CachedFileSystemEntry &Entry) {
183
4.64k
  std::lock_guard<std::mutex> LockGuard(CacheLock);
184
4.64k
  return *EntriesByFilename.insert({Filename, &Entry}).first->getValue();
185
4.64k
}
186
187
/// Whitelist file extensions that should be minimized, treating no extension as
188
/// a source file that should be minimized.
189
///
190
/// This is kinda hacky, it would be better if we knew what kind of file Clang
191
/// was expecting instead.
192
6.17k
static bool shouldMinimizeBasedOnExtension(StringRef Filename) {
193
6.17k
  StringRef Ext = llvm::sys::path::extension(Filename);
194
6.17k
  if (Ext.empty())
195
8
    return true; // C++ standard library
196
6.16k
  return llvm::StringSwitch<bool>(Ext)
197
6.16k
      .CasesLower(".c", ".cc", ".cpp", ".c++", ".cxx", true)
198
6.16k
      .CasesLower(".h", ".hh", ".hpp", ".h++", ".hxx", true)
199
6.16k
      .CasesLower(".m", ".mm", true)
200
6.16k
      .CasesLower(".i", ".ii", ".mi", ".mmi", true)
201
6.16k
      .CasesLower(".def", ".inc", true)
202
6.16k
      .Default(false);
203
6.17k
}
204
205
1.97k
static bool shouldCacheStatFailures(StringRef Filename) {
206
1.97k
  StringRef Ext = llvm::sys::path::extension(Filename);
207
1.97k
  if (Ext.empty())
208
701
    return false; // This may be the module cache directory.
209
  // Only cache stat failures on source files.
210
1.27k
  return shouldMinimizeBasedOnExtension(Filename);
211
1.97k
}
212
213
void DependencyScanningWorkerFilesystem::disableMinimization(
214
532
    StringRef Filename) {
215
  // Since we're not done setting up `NotToBeMinimized` yet, we need to disable
216
  // minimization explicitly.
217
532
  if (llvm::ErrorOr<EntryRef> Result =
218
532
          getOrCreateFileSystemEntry(Filename, /*DisableMinimization=*/true))
219
273
    NotToBeMinimized.insert(Result->getStatus().getUniqueID());
220
532
}
221
222
bool DependencyScanningWorkerFilesystem::shouldMinimize(
223
4.90k
    StringRef Filename, llvm::sys::fs::UniqueID UID) {
224
4.90k
  return shouldMinimizeBasedOnExtension(Filename) &&
225
4.90k
         
!NotToBeMinimized.contains(UID)4.37k
;
226
4.90k
}
227
228
const CachedFileSystemEntry &
229
DependencyScanningWorkerFilesystem::getOrEmplaceSharedEntryForUID(
230
4.64k
    TentativeEntry TEntry) {
231
4.64k
  auto &Shard = SharedCache.getShardForUID(TEntry.Status.getUniqueID());
232
4.64k
  return Shard.getOrEmplaceEntryForUID(TEntry.Status.getUniqueID(),
233
4.64k
                                       std::move(TEntry.Status),
234
4.64k
                                       std::move(TEntry.Contents));
235
4.64k
}
236
237
const CachedFileSystemEntry *
238
DependencyScanningWorkerFilesystem::findEntryByFilenameWithWriteThrough(
239
8.16k
    StringRef Filename) {
240
8.16k
  if (const auto *Entry = LocalCache.findEntryByFilename(Filename))
241
1.27k
    return Entry;
242
6.88k
  auto &Shard = SharedCache.getShardForFilename(Filename);
243
6.88k
  if (const auto *Entry = Shard.findEntryByFilename(Filename))
244
96
    return &LocalCache.insertEntryForFilename(Filename, *Entry);
245
6.78k
  return nullptr;
246
6.88k
}
247
248
llvm::ErrorOr<const CachedFileSystemEntry &>
249
6.78k
DependencyScanningWorkerFilesystem::computeAndStoreResult(StringRef Filename) {
250
6.78k
  llvm::ErrorOr<llvm::vfs::Status> Stat = getUnderlyingFS().status(Filename);
251
6.78k
  if (!Stat) {
252
1.97k
    if (!shouldCacheStatFailures(Filename))
253
1.83k
      return Stat.getError();
254
139
    const auto &Entry =
255
139
        getOrEmplaceSharedEntryForFilename(Filename, Stat.getError());
256
139
    return insertLocalEntryForFilename(Filename, Entry);
257
1.97k
  }
258
259
4.81k
  if (const auto *Entry = findSharedEntryByUID(*Stat))
260
164
    return insertLocalEntryForFilename(Filename, *Entry);
261
262
4.64k
  auto TEntry =
263
4.64k
      Stat->isDirectory() ? 
TentativeEntry(*Stat)822
:
readFile(Filename)3.82k
;
264
265
4.64k
  const CachedFileSystemEntry *SharedEntry = [&]() {
266
4.64k
    if (
TEntry4.64k
) {
267
4.64k
      const auto &UIDEntry = getOrEmplaceSharedEntryForUID(std::move(*TEntry));
268
4.64k
      return &getOrInsertSharedEntryForFilename(Filename, UIDEntry);
269
4.64k
    }
270
18.4E
    return &getOrEmplaceSharedEntryForFilename(Filename, TEntry.getError());
271
4.64k
  }();
272
273
4.64k
  return insertLocalEntryForFilename(Filename, *SharedEntry);
274
4.81k
}
275
276
llvm::ErrorOr<EntryRef>
277
DependencyScanningWorkerFilesystem::getOrCreateFileSystemEntry(
278
8.15k
    StringRef Filename, bool DisableMinimization) {
279
8.15k
  if (const auto *Entry = findEntryByFilenameWithWriteThrough(Filename))
280
1.37k
    return minimizeIfNecessary(*Entry, Filename, DisableMinimization)
281
1.37k
        .unwrapError();
282
6.77k
  auto MaybeEntry = computeAndStoreResult(Filename);
283
6.77k
  if (!MaybeEntry)
284
1.83k
    return MaybeEntry.getError();
285
4.94k
  return minimizeIfNecessary(*MaybeEntry, Filename, DisableMinimization)
286
4.94k
      .unwrapError();
287
6.77k
}
288
289
llvm::ErrorOr<llvm::vfs::Status>
290
6.79k
DependencyScanningWorkerFilesystem::status(const Twine &Path) {
291
6.79k
  SmallString<256> OwnedFilename;
292
6.79k
  StringRef Filename = Path.toStringRef(OwnedFilename);
293
294
6.79k
  llvm::ErrorOr<EntryRef> Result = getOrCreateFileSystemEntry(Filename);
295
6.79k
  if (!Result)
296
1.60k
    return Result.getError();
297
5.19k
  return Result->getStatus();
298
6.79k
}
299
300
namespace {
301
302
/// The VFS that is used by clang consumes the \c CachedFileSystemEntry using
303
/// this subclass.
304
class MinimizedVFSFile final : public llvm::vfs::File {
305
public:
306
  MinimizedVFSFile(std::unique_ptr<llvm::MemoryBuffer> Buffer,
307
                   llvm::vfs::Status Stat)
308
695
      : Buffer(std::move(Buffer)), Stat(std::move(Stat)) {}
309
310
  static llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>
311
  create(EntryRef Entry,
312
         ExcludedPreprocessorDirectiveSkipMapping &PPSkipMappings);
313
314
704
  llvm::ErrorOr<llvm::vfs::Status> status() override { return Stat; }
315
316
  llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
317
  getBuffer(const Twine &Name, int64_t FileSize, bool RequiresNullTerminator,
318
596
            bool IsVolatile) override {
319
596
    return std::move(Buffer);
320
596
  }
321
322
0
  std::error_code close() override { return {}; }
323
324
private:
325
  std::unique_ptr<llvm::MemoryBuffer> Buffer;
326
  llvm::vfs::Status Stat;
327
};
328
329
} // end anonymous namespace
330
331
llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>> MinimizedVFSFile::create(
332
698
    EntryRef Entry, ExcludedPreprocessorDirectiveSkipMapping &PPSkipMappings) {
333
698
  assert(!Entry.isError() && "error");
334
335
698
  if (Entry.isDirectory())
336
2
    return std::make_error_code(std::errc::is_a_directory);
337
338
696
  auto Result = std::make_unique<MinimizedVFSFile>(
339
696
      llvm::MemoryBuffer::getMemBuffer(Entry.getContents(),
340
696
                                       Entry.getStatus().getName(),
341
696
                                       /*RequiresNullTerminator=*/false),
342
696
      Entry.getStatus());
343
344
696
  const auto *EntrySkipMappings = Entry.getPPSkippedRangeMapping();
345
696
  if (EntrySkipMappings && 
!EntrySkipMappings->empty()326
)
346
83
    PPSkipMappings[Result->Buffer->getBufferStart()] = EntrySkipMappings;
347
348
696
  return llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>(
349
696
      std::unique_ptr<llvm::vfs::File>(std::move(Result)));
350
698
}
351
352
llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>
353
833
DependencyScanningWorkerFilesystem::openFileForRead(const Twine &Path) {
354
833
  SmallString<256> OwnedFilename;
355
833
  StringRef Filename = Path.toStringRef(OwnedFilename);
356
357
833
  llvm::ErrorOr<EntryRef> Result = getOrCreateFileSystemEntry(Filename);
358
833
  if (!Result)
359
135
    return Result.getError();
360
698
  return MinimizedVFSFile::create(Result.get(), PPSkipMappings);
361
833
}