Coverage Report

Created: 2020-09-19 12:23

/Users/buildslave/jenkins/workspace/coverage/llvm-project/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
Line
Count
Source (jump to first uncovered line)
1
//===- DependencyScanningFilesystem.cpp - clang-scan-deps fs --------------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
9
#include "clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h"
10
#include "clang/Lex/DependencyDirectivesSourceMinimizer.h"
11
#include "llvm/Support/MemoryBuffer.h"
12
#include "llvm/Support/Threading.h"
13
14
using namespace clang;
15
using namespace tooling;
16
using namespace dependencies;
17
18
CachedFileSystemEntry CachedFileSystemEntry::createFileEntry(
19
84
    StringRef Filename, llvm::vfs::FileSystem &FS, bool Minimize) {
20
  // Load the file and its content from the file system.
21
84
  llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>> MaybeFile =
22
84
      FS.openFileForRead(Filename);
23
84
  if (!MaybeFile)
24
0
    return MaybeFile.getError();
25
84
  llvm::ErrorOr<llvm::vfs::Status> Stat = (*MaybeFile)->status();
26
84
  if (!Stat)
27
0
    return Stat.getError();
28
84
29
84
  llvm::vfs::File &F = **MaybeFile;
30
84
  llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> MaybeBuffer =
31
84
      F.getBuffer(Stat->getName());
32
84
  if (!MaybeBuffer)
33
0
    return MaybeBuffer.getError();
34
84
35
84
  llvm::SmallString<1024> MinimizedFileContents;
36
  // Minimize the file down to directives that might affect the dependencies.
37
84
  const auto &Buffer = *MaybeBuffer;
38
84
  SmallVector<minimize_source_to_dependency_directives::Token, 64> Tokens;
39
84
  if (!Minimize || minimizeSourceToDependencyDirectives(
40
17
                       Buffer->getBuffer(), MinimizedFileContents, Tokens)) {
41
    // Use the original file unless requested otherwise, or
42
    // if the minimization failed.
43
    // FIXME: Propage the diagnostic if desired by the client.
44
17
    CachedFileSystemEntry Result;
45
17
    Result.MaybeStat = std::move(*Stat);
46
17
    Result.Contents.reserve(Buffer->getBufferSize() + 1);
47
17
    Result.Contents.append(Buffer->getBufferStart(), Buffer->getBufferEnd());
48
    // Implicitly null terminate the contents for Clang's lexer.
49
17
    Result.Contents.push_back('\0');
50
17
    Result.Contents.pop_back();
51
17
    return Result;
52
17
  }
53
67
54
67
  CachedFileSystemEntry Result;
55
67
  size_t Size = MinimizedFileContents.size();
56
67
  Result.MaybeStat = llvm::vfs::Status(Stat->getName(), Stat->getUniqueID(),
57
67
                                       Stat->getLastModificationTime(),
58
67
                                       Stat->getUser(), Stat->getGroup(), Size,
59
67
                                       Stat->getType(), Stat->getPermissions());
60
  // The contents produced by the minimizer must be null terminated.
61
67
  assert(MinimizedFileContents.data()[MinimizedFileContents.size()] == '\0' &&
62
67
         "not null terminated contents");
63
  // Even though there's an implicit null terminator in the minimized contents,
64
  // we want to temporarily make it explicit. This will ensure that the
65
  // std::move will preserve it even if it needs to do a copy if the
66
  // SmallString still has the small capacity.
67
67
  MinimizedFileContents.push_back('\0');
68
67
  Result.Contents = std::move(MinimizedFileContents);
69
  // Now make the null terminator implicit again, so that Clang's lexer can find
70
  // it right where the buffer ends.
71
67
  Result.Contents.pop_back();
72
67
73
  // Compute the skipped PP ranges that speedup skipping over inactive
74
  // preprocessor blocks.
75
67
  llvm::SmallVector<minimize_source_to_dependency_directives::SkippedRange, 32>
76
67
      SkippedRanges;
77
67
  minimize_source_to_dependency_directives::computeSkippedRanges(Tokens,
78
67
                                                                 SkippedRanges);
79
67
  PreprocessorSkippedRangeMapping Mapping;
80
30
  for (const auto &Range : SkippedRanges) {
81
30
    if (Range.Length < 16) {
82
      // Ignore small ranges as non-profitable.
83
      // FIXME: This is a heuristic, its worth investigating the tradeoffs
84
      // when it should be applied.
85
4
      continue;
86
4
    }
87
26
    Mapping[Range.Offset] = Range.Length;
88
26
  }
89
67
  Result.PPSkippedRangeMapping = std::move(Mapping);
90
67
91
67
  return Result;
92
67
}
93
94
CachedFileSystemEntry
95
129
CachedFileSystemEntry::createDirectoryEntry(llvm::vfs::Status &&Stat) {
96
129
  assert(Stat.isDirectory() && "not a directory!");
97
129
  auto Result = CachedFileSystemEntry();
98
129
  Result.MaybeStat = std::move(Stat);
99
129
  return Result;
100
129
}
101
102
DependencyScanningFilesystemSharedCache::
103
27
    DependencyScanningFilesystemSharedCache() {
104
  // This heuristic was chosen using a empirical testing on a
105
  // reasonably high core machine (iMacPro 18 cores / 36 threads). The cache
106
  // sharding gives a performance edge by reducing the lock contention.
107
  // FIXME: A better heuristic might also consider the OS to account for
108
  // the different cost of lock contention on different OSes.
109
27
  NumShards =
110
27
      std::max(2u, llvm::hardware_concurrency().compute_thread_count() / 4);
111
27
  CacheShards = std::make_unique<CacheShard[]>(NumShards);
112
27
}
113
114
/// Returns a cache entry for the corresponding key.
115
///
116
/// A new cache entry is created if the key is not in the cache. This is a
117
/// thread safe call.
118
DependencyScanningFilesystemSharedCache::SharedFileSystemEntry &
119
535
DependencyScanningFilesystemSharedCache::get(StringRef Key) {
120
535
  CacheShard &Shard = CacheShards[llvm::hash_value(Key) % NumShards];
121
535
  std::unique_lock<std::mutex> LockGuard(Shard.CacheLock);
122
535
  auto It = Shard.Cache.try_emplace(Key);
123
535
  return It.first->getValue();
124
535
}
125
126
/// Whitelist file extensions that should be minimized, treating no extension as
127
/// a source file that should be minimized.
128
///
129
/// This is kinda hacky, it would be better if we knew what kind of file Clang
130
/// was expecting instead.
131
484
static bool shouldMinimize(StringRef Filename) {
132
484
  StringRef Ext = llvm::sys::path::extension(Filename);
133
484
  if (Ext.empty())
134
228
    return true; // C++ standard library
135
256
  return llvm::StringSwitch<bool>(Ext)
136
256
    .CasesLower(".c", ".cc", ".cpp", ".c++", ".cxx", true)
137
256
    .CasesLower(".h", ".hh", ".hpp", ".h++", ".hxx", true)
138
256
    .CasesLower(".m", ".mm", true)
139
256
    .CasesLower(".i", ".ii", ".mi", ".mmi", true)
140
256
    .CasesLower(".def", ".inc", true)
141
256
    .Default(false);
142
256
}
143
144
145
260
static bool shouldCacheStatFailures(StringRef Filename) {
146
260
  StringRef Ext = llvm::sys::path::extension(Filename);
147
260
  if (Ext.empty())
148
202
    return false; // This may be the module cache directory.
149
58
  return shouldMinimize(Filename); // Only cache stat failures on source files.
150
58
}
151
152
llvm::ErrorOr<const CachedFileSystemEntry *>
153
DependencyScanningWorkerFilesystem::getOrCreateFileSystemEntry(
154
625
    const StringRef Filename) {
155
625
  if (const CachedFileSystemEntry *Entry = getCachedEntry(Filename)) {
156
87
    return Entry;
157
87
  }
158
538
159
  // FIXME: Handle PCM/PCH files.
160
  // FIXME: Handle module map files.
161
538
162
538
  bool KeepOriginalSource = IgnoredFiles.count(Filename) ||
163
426
                            !shouldMinimize(Filename);
164
538
  DependencyScanningFilesystemSharedCache::SharedFileSystemEntry
165
538
      &SharedCacheEntry = SharedCache.get(Filename);
166
538
  const CachedFileSystemEntry *Result;
167
538
  {
168
538
    std::unique_lock<std::mutex> LockGuard(SharedCacheEntry.ValueLock);
169
538
    CachedFileSystemEntry &CacheEntry = SharedCacheEntry.Value;
170
538
171
538
    if (!CacheEntry.isValid()) {
172
473
      llvm::vfs::FileSystem &FS = getUnderlyingFS();
173
473
      auto MaybeStatus = FS.status(Filename);
174
473
      if (!MaybeStatus) {
175
260
        if (!shouldCacheStatFailures(Filename))
176
          // HACK: We need to always restat non source files if the stat fails.
177
          //   This is because Clang first looks up the module cache and module
178
          //   files before building them, and then looks for them again. If we
179
          //   cache the stat failure, it won't see them the second time.
180
239
          return MaybeStatus.getError();
181
21
        else
182
21
          CacheEntry = CachedFileSystemEntry(MaybeStatus.getError());
183
213
      } else if (MaybeStatus->isDirectory())
184
129
        CacheEntry = CachedFileSystemEntry::createDirectoryEntry(
185
129
            std::move(*MaybeStatus));
186
84
      else
187
84
        CacheEntry = CachedFileSystemEntry::createFileEntry(
188
84
            Filename, FS, !KeepOriginalSource);
189
473
    }
190
538
191
299
    Result = &CacheEntry;
192
299
  }
193
299
194
  // Store the result in the local cache.
195
299
  setCachedEntry(Filename, Result);
196
299
  return Result;
197
538
}
198
199
llvm::ErrorOr<llvm::vfs::Status>
200
443
DependencyScanningWorkerFilesystem::status(const Twine &Path) {
201
443
  SmallString<256> OwnedFilename;
202
443
  StringRef Filename = Path.toStringRef(OwnedFilename);
203
443
  const llvm::ErrorOr<const CachedFileSystemEntry *> Result =
204
443
      getOrCreateFileSystemEntry(Filename);
205
443
  if (!Result)
206
236
    return Result.getError();
207
207
  return (*Result)->getStatus();
208
207
}
209
210
namespace {
211
212
/// The VFS that is used by clang consumes the \c CachedFileSystemEntry using
213
/// this subclass.
214
class MinimizedVFSFile final : public llvm::vfs::File {
215
public:
216
  MinimizedVFSFile(std::unique_ptr<llvm::MemoryBuffer> Buffer,
217
                   llvm::vfs::Status Stat)
218
149
      : Buffer(std::move(Buffer)), Stat(std::move(Stat)) {}
219
220
167
  llvm::ErrorOr<llvm::vfs::Status> status() override { return Stat; }
221
222
39
  const llvm::MemoryBuffer *getBufferPtr() const { return Buffer.get(); }
223
224
  llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
225
  getBuffer(const Twine &Name, int64_t FileSize, bool RequiresNullTerminator,
226
129
            bool IsVolatile) override {
227
129
    return std::move(Buffer);
228
129
  }
229
230
0
  std::error_code close() override { return {}; }
231
232
private:
233
  std::unique_ptr<llvm::MemoryBuffer> Buffer;
234
  llvm::vfs::Status Stat;
235
};
236
237
llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>
238
createFile(const CachedFileSystemEntry *Entry,
239
180
           ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings) {
240
180
  if (Entry->isDirectory())
241
2
    return llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>(
242
2
        std::make_error_code(std::errc::is_a_directory));
243
178
  llvm::ErrorOr<StringRef> Contents = Entry->getContents();
244
178
  if (!Contents)
245
29
    return Contents.getError();
246
149
  auto Result = std::make_unique<MinimizedVFSFile>(
247
149
      llvm::MemoryBuffer::getMemBuffer(*Contents, Entry->getName(),
248
149
                                       /*RequiresNullTerminator=*/false),
249
149
      *Entry->getStatus());
250
149
  if (!Entry->getPPSkippedRangeMapping().empty() && 
PPSkipMappings42
)
251
39
    (*PPSkipMappings)[Result->getBufferPtr()] =
252
39
        &Entry->getPPSkippedRangeMapping();
253
149
  return llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>(
254
149
      std::unique_ptr<llvm::vfs::File>(std::move(Result)));
255
149
}
256
257
} // end anonymous namespace
258
259
llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>
260
183
DependencyScanningWorkerFilesystem::openFileForRead(const Twine &Path) {
261
183
  SmallString<256> OwnedFilename;
262
183
  StringRef Filename = Path.toStringRef(OwnedFilename);
263
183
264
183
  const llvm::ErrorOr<const CachedFileSystemEntry *> Result =
265
183
      getOrCreateFileSystemEntry(Filename);
266
183
  if (!Result)
267
3
    return Result.getError();
268
180
  return createFile(Result.get(), PPSkipMappings);
269
180
}