Coverage Report

Created: 2022-01-18 06:27

/Users/buildslave/jenkins/workspace/coverage/llvm-project/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
Line
Count
Source (jump to first uncovered line)
1
//===- DependencyScanningFilesystem.cpp - clang-scan-deps fs --------------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
9
#include "clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h"
10
#include "clang/Lex/DependencyDirectivesSourceMinimizer.h"
11
#include "llvm/Support/MemoryBuffer.h"
12
#include "llvm/Support/SmallVectorMemoryBuffer.h"
13
#include "llvm/Support/Threading.h"
14
15
using namespace clang;
16
using namespace tooling;
17
using namespace dependencies;
18
19
llvm::ErrorOr<llvm::vfs::Status>
20
2.58k
CachedFileSystemEntry::initFile(StringRef Filename, llvm::vfs::FileSystem &FS) {
21
  // Load the file and its content from the file system.
22
2.58k
  auto MaybeFile = FS.openFileForRead(Filename);
23
2.58k
  if (!MaybeFile)
24
0
    return MaybeFile.getError();
25
2.58k
  auto File = std::move(*MaybeFile);
26
27
2.58k
  auto MaybeStat = File->status();
28
2.58k
  if (!MaybeStat)
29
0
    return MaybeStat.getError();
30
2.58k
  auto Stat = std::move(*MaybeStat);
31
32
2.58k
  auto MaybeBuffer = File->getBuffer(Stat.getName());
33
2.58k
  if (!MaybeBuffer)
34
0
    return MaybeBuffer.getError();
35
2.58k
  auto Buffer = std::move(*MaybeBuffer);
36
37
2.58k
  OriginalContents = std::move(Buffer);
38
2.58k
  return Stat;
39
2.58k
}
40
41
2.39k
void CachedFileSystemEntry::minimizeFile() {
42
2.39k
  assert(OriginalContents && "minimizing missing contents");
43
44
0
  llvm::SmallString<1024> MinimizedFileContents;
45
  // Minimize the file down to directives that might affect the dependencies.
46
2.39k
  SmallVector<minimize_source_to_dependency_directives::Token, 64> Tokens;
47
2.39k
  if (minimizeSourceToDependencyDirectives(OriginalContents->getBuffer(),
48
2.39k
                                           MinimizedFileContents, Tokens)) {
49
    // FIXME: Propagate the diagnostic if desired by the client.
50
    // Use the original file if the minimization failed.
51
0
    MinimizedContentsStorage =
52
0
        llvm::MemoryBuffer::getMemBuffer(*OriginalContents);
53
0
    MinimizedContentsAccess.store(MinimizedContentsStorage.get());
54
0
    return;
55
0
  }
56
57
  // The contents produced by the minimizer must be null terminated.
58
2.39k
  assert(MinimizedFileContents.data()[MinimizedFileContents.size()] == '\0' &&
59
2.39k
         "not null terminated contents");
60
61
  // Compute the skipped PP ranges that speedup skipping over inactive
62
  // preprocessor blocks.
63
0
  llvm::SmallVector<minimize_source_to_dependency_directives::SkippedRange, 32>
64
2.39k
      SkippedRanges;
65
2.39k
  minimize_source_to_dependency_directives::computeSkippedRanges(Tokens,
66
2.39k
                                                                 SkippedRanges);
67
2.39k
  PreprocessorSkippedRangeMapping Mapping;
68
33.1k
  for (const auto &Range : SkippedRanges) {
69
33.1k
    if (Range.Length < 16) {
70
      // Ignore small ranges as non-profitable.
71
      // FIXME: This is a heuristic, its worth investigating the tradeoffs
72
      // when it should be applied.
73
112
      continue;
74
112
    }
75
32.9k
    Mapping[Range.Offset] = Range.Length;
76
32.9k
  }
77
2.39k
  PPSkippedRangeMapping = std::move(Mapping);
78
79
2.39k
  MinimizedContentsStorage = std::make_unique<llvm::SmallVectorMemoryBuffer>(
80
2.39k
      std::move(MinimizedFileContents));
81
  // The algorithm in `getOrCreateFileSystemEntry` uses the presence of
82
  // minimized contents to decide whether an entry is up-to-date or not.
83
  // If it is up-to-date, the skipped range mappings must be already computed.
84
  // This is why we need to store the minimized contents **after** storing the
85
  // skipped range mappings. Failing to do so would lead to a data race.
86
2.39k
  MinimizedContentsAccess.store(MinimizedContentsStorage.get());
87
2.39k
}
88
89
DependencyScanningFilesystemSharedCache::
90
74
    DependencyScanningFilesystemSharedCache() {
91
  // This heuristic was chosen using a empirical testing on a
92
  // reasonably high core machine (iMacPro 18 cores / 36 threads). The cache
93
  // sharding gives a performance edge by reducing the lock contention.
94
  // FIXME: A better heuristic might also consider the OS to account for
95
  // the different cost of lock contention on different OSes.
96
74
  NumShards =
97
74
      std::max(2u, llvm::hardware_concurrency().compute_thread_count() / 4);
98
74
  CacheShards = std::make_unique<CacheShard[]>(NumShards);
99
74
}
100
101
DependencyScanningFilesystemSharedCache::SharedFileSystemEntry &
102
5.54k
DependencyScanningFilesystemSharedCache::get(StringRef Key) {
103
5.54k
  CacheShard &Shard = CacheShards[llvm::hash_value(Key) % NumShards];
104
5.54k
  std::lock_guard<std::mutex> LockGuard(Shard.CacheLock);
105
5.54k
  auto It = Shard.Cache.try_emplace(Key);
106
5.54k
  return It.first->getValue();
107
5.54k
}
108
109
/// Whitelist file extensions that should be minimized, treating no extension as
110
/// a source file that should be minimized.
111
///
112
/// This is kinda hacky, it would be better if we knew what kind of file Clang
113
/// was expecting instead.
114
6.42k
static bool shouldMinimizeBasedOnExtension(StringRef Filename) {
115
6.42k
  StringRef Ext = llvm::sys::path::extension(Filename);
116
6.42k
  if (Ext.empty())
117
1.09k
    return true; // C++ standard library
118
5.32k
  return llvm::StringSwitch<bool>(Ext)
119
5.32k
      .CasesLower(".c", ".cc", ".cpp", ".c++", ".cxx", true)
120
5.32k
      .CasesLower(".h", ".hh", ".hpp", ".h++", ".hxx", true)
121
5.32k
      .CasesLower(".m", ".mm", true)
122
5.32k
      .CasesLower(".i", ".ii", ".mi", ".mmi", true)
123
5.32k
      .CasesLower(".def", ".inc", true)
124
5.32k
      .Default(false);
125
6.42k
}
126
127
1.26k
static bool shouldCacheStatFailures(StringRef Filename) {
128
1.26k
  StringRef Ext = llvm::sys::path::extension(Filename);
129
1.26k
  if (Ext.empty())
130
383
    return false; // This may be the module cache directory.
131
  // Only cache stat failures on source files.
132
881
  return shouldMinimizeBasedOnExtension(Filename);
133
1.26k
}
134
135
void DependencyScanningWorkerFilesystem::disableMinimization(
136
478
    StringRef RawFilename) {
137
478
  llvm::SmallString<256> Filename;
138
478
  llvm::sys::path::native(RawFilename, Filename);
139
478
  NotToBeMinimized.insert(Filename);
140
478
}
141
142
5.54k
bool DependencyScanningWorkerFilesystem::shouldMinimize(StringRef RawFilename) {
143
5.54k
  if (!shouldMinimizeBasedOnExtension(RawFilename))
144
1.32k
    return false;
145
146
4.22k
  llvm::SmallString<256> Filename;
147
4.22k
  llvm::sys::path::native(RawFilename, Filename);
148
4.22k
  return !NotToBeMinimized.contains(Filename);
149
5.54k
}
150
151
void CachedFileSystemEntry::init(llvm::ErrorOr<llvm::vfs::Status> &&MaybeStatus,
152
                                 StringRef Filename,
153
3.32k
                                 llvm::vfs::FileSystem &FS) {
154
3.32k
  if (!MaybeStatus || 
MaybeStatus->isDirectory()3.22k
)
155
741
    MaybeStat = std::move(MaybeStatus);
156
2.58k
  else
157
2.58k
    MaybeStat = initFile(Filename, FS);
158
3.32k
}
159
160
llvm::ErrorOr<EntryRef>
161
DependencyScanningWorkerFilesystem::getOrCreateFileSystemEntry(
162
5.54k
    StringRef Filename) {
163
5.54k
  bool ShouldBeMinimized = shouldMinimize(Filename);
164
165
5.54k
  const auto *Entry = LocalCache.getCachedEntry(Filename);
166
5.54k
  if (Entry && 
!Entry->needsUpdate(ShouldBeMinimized)0
)
167
0
    return EntryRef(ShouldBeMinimized, *Entry);
168
169
  // FIXME: Handle PCM/PCH files.
170
  // FIXME: Handle module map files.
171
172
5.54k
  auto &SharedCacheEntry = SharedCache.get(Filename);
173
5.54k
  {
174
5.54k
    std::lock_guard<std::mutex> LockGuard(SharedCacheEntry.ValueLock);
175
5.54k
    CachedFileSystemEntry &CacheEntry = SharedCacheEntry.Value;
176
177
5.54k
    if (!CacheEntry.isInitialized()) {
178
4.49k
      auto MaybeStatus = getUnderlyingFS().status(Filename);
179
4.49k
      if (!MaybeStatus && 
!shouldCacheStatFailures(Filename)1.26k
)
180
        // HACK: We need to always restat non source files if the stat fails.
181
        //   This is because Clang first looks up the module cache and module
182
        //   files before building them, and then looks for them again. If we
183
        //   cache the stat failure, it won't see them the second time.
184
1.17k
        return MaybeStatus.getError();
185
3.32k
      CacheEntry.init(std::move(MaybeStatus), Filename, getUnderlyingFS());
186
3.32k
    }
187
188
    // Checking `needsUpdate` verifies the entry represents an opened file.
189
    // Only checking `needsMinimization` could lead to minimization of files
190
    // that we failed to load (such files don't have `OriginalContents`).
191
4.37k
    if (CacheEntry.needsUpdate(ShouldBeMinimized))
192
2.39k
      CacheEntry.minimizeFile();
193
4.37k
  }
194
195
  // Store the result in the local cache.
196
0
  Entry = &SharedCacheEntry.Value;
197
4.37k
  return EntryRef(ShouldBeMinimized, *Entry);
198
5.54k
}
199
200
llvm::ErrorOr<llvm::vfs::Status>
201
4.83k
DependencyScanningWorkerFilesystem::status(const Twine &Path) {
202
4.83k
  SmallString<256> OwnedFilename;
203
4.83k
  StringRef Filename = Path.toStringRef(OwnedFilename);
204
205
4.83k
  llvm::ErrorOr<EntryRef> Result = getOrCreateFileSystemEntry(Filename);
206
4.83k
  if (!Result)
207
1.15k
    return Result.getError();
208
3.67k
  return Result->getStatus();
209
4.83k
}
210
211
namespace {
212
213
/// The VFS that is used by clang consumes the \c CachedFileSystemEntry using
214
/// this subclass.
215
class MinimizedVFSFile final : public llvm::vfs::File {
216
public:
217
  MinimizedVFSFile(std::unique_ptr<llvm::MemoryBuffer> Buffer,
218
                   llvm::vfs::Status Stat)
219
616
      : Buffer(std::move(Buffer)), Stat(std::move(Stat)) {}
220
221
  static llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>
222
  create(EntryRef Entry,
223
         ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings);
224
225
576
  llvm::ErrorOr<llvm::vfs::Status> status() override { return Stat; }
226
227
  llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
228
  getBuffer(const Twine &Name, int64_t FileSize, bool RequiresNullTerminator,
229
541
            bool IsVolatile) override {
230
541
    return std::move(Buffer);
231
541
  }
232
233
0
  std::error_code close() override { return {}; }
234
235
private:
236
  std::unique_ptr<llvm::MemoryBuffer> Buffer;
237
  llvm::vfs::Status Stat;
238
};
239
240
} // end anonymous namespace
241
242
llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>> MinimizedVFSFile::create(
243
700
    EntryRef Entry, ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings) {
244
700
  if (Entry.isDirectory())
245
2
    return std::make_error_code(std::errc::is_a_directory);
246
247
698
  llvm::ErrorOr<StringRef> Contents = Entry.getContents();
248
698
  if (!Contents)
249
82
    return Contents.getError();
250
616
  auto Result = std::make_unique<MinimizedVFSFile>(
251
616
      llvm::MemoryBuffer::getMemBuffer(*Contents, Entry.getName(),
252
616
                                       /*RequiresNullTerminator=*/false),
253
616
      *Entry.getStatus());
254
255
616
  const auto *EntrySkipMappings = Entry.getPPSkippedRangeMapping();
256
616
  if (EntrySkipMappings && 
!EntrySkipMappings->empty()296
&&
PPSkipMappings87
)
257
81
    (*PPSkipMappings)[Result->Buffer->getBufferStart()] = EntrySkipMappings;
258
259
616
  return llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>(
260
616
      std::unique_ptr<llvm::vfs::File>(std::move(Result)));
261
698
}
262
263
llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>
264
715
DependencyScanningWorkerFilesystem::openFileForRead(const Twine &Path) {
265
715
  SmallString<256> OwnedFilename;
266
715
  StringRef Filename = Path.toStringRef(OwnedFilename);
267
268
715
  llvm::ErrorOr<EntryRef> Result = getOrCreateFileSystemEntry(Filename);
269
715
  if (!Result)
270
15
    return Result.getError();
271
700
  return MinimizedVFSFile::create(Result.get(), PPSkipMappings);
272
715
}