Coverage Report

Created: 2022-01-25 06:29

/Users/buildslave/jenkins/workspace/coverage/llvm-project/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
Line
Count
Source (jump to first uncovered line)
1
//===- DependencyScanningWorker.cpp - clang-scan-deps worker --------------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
9
#include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h"
10
#include "clang/CodeGen/ObjectFilePCHContainerOperations.h"
11
#include "clang/Frontend/CompilerInstance.h"
12
#include "clang/Frontend/CompilerInvocation.h"
13
#include "clang/Frontend/FrontendActions.h"
14
#include "clang/Frontend/TextDiagnosticPrinter.h"
15
#include "clang/Frontend/Utils.h"
16
#include "clang/Lex/PreprocessorOptions.h"
17
#include "clang/Tooling/DependencyScanning/DependencyScanningService.h"
18
#include "clang/Tooling/DependencyScanning/ModuleDepCollector.h"
19
#include "clang/Tooling/Tooling.h"
20
21
using namespace clang;
22
using namespace tooling;
23
using namespace dependencies;
24
25
namespace {
26
27
/// Forwards the gatherered dependencies to the consumer.
28
class DependencyConsumerForwarder : public DependencyFileGenerator {
29
public:
30
  DependencyConsumerForwarder(std::unique_ptr<DependencyOutputOptions> Opts,
31
                              DependencyConsumer &C)
32
134
      : DependencyFileGenerator(*Opts), Opts(std::move(Opts)), C(C) {}
33
34
134
  void finishedMainFile(DiagnosticsEngine &Diags) override {
35
134
    C.handleDependencyOutputOpts(*Opts);
36
134
    llvm::SmallString<256> CanonPath;
37
383
    for (const auto &File : getDependencies()) {
38
383
      CanonPath = File;
39
383
      llvm::sys::path::remove_dots(CanonPath, /*remove_dot_dot=*/true);
40
383
      C.handleFileDependency(CanonPath);
41
383
    }
42
134
  }
43
44
private:
45
  std::unique_ptr<DependencyOutputOptions> Opts;
46
  DependencyConsumer &C;
47
};
48
49
/// A listener that collects the imported modules and optionally the input
50
/// files.
51
class PrebuiltModuleListener : public ASTReaderListener {
52
public:
53
  PrebuiltModuleListener(llvm::StringMap<std::string> &PrebuiltModuleFiles,
54
                         llvm::StringSet<> &InputFiles, bool VisitInputFiles)
55
      : PrebuiltModuleFiles(PrebuiltModuleFiles), InputFiles(InputFiles),
56
5
        VisitInputFiles(VisitInputFiles) {}
57
58
14
  bool needsImportVisitation() const override { return true; }
59
14
  bool needsInputFileVisitation() override { return VisitInputFiles; }
60
14
  bool needsSystemInputFileVisitation() override { return VisitInputFiles; }
61
62
11
  void visitImport(StringRef ModuleName, StringRef Filename) override {
63
11
    PrebuiltModuleFiles.insert({ModuleName, Filename.str()});
64
11
  }
65
66
  bool visitInputFile(StringRef Filename, bool isSystem, bool isOverridden,
67
24
                      bool isExplicitModule) override {
68
24
    InputFiles.insert(Filename);
69
24
    return true;
70
24
  }
71
72
private:
73
  llvm::StringMap<std::string> &PrebuiltModuleFiles;
74
  llvm::StringSet<> &InputFiles;
75
  bool VisitInputFiles;
76
};
77
78
using PrebuiltModuleFilesT = decltype(HeaderSearchOptions::PrebuiltModuleFiles);
79
80
/// Visit the given prebuilt module and collect all of the modules it
81
/// transitively imports and contributing input files.
82
static void visitPrebuiltModule(StringRef PrebuiltModuleFilename,
83
                                CompilerInstance &CI,
84
                                PrebuiltModuleFilesT &ModuleFiles,
85
                                llvm::StringSet<> &InputFiles,
86
5
                                bool VisitInputFiles) {
87
  // Maps the names of modules that weren't yet visited to their PCM path.
88
5
  llvm::StringMap<std::string> ModuleFilesWorklist;
89
  // Contains PCM paths of all visited modules.
90
5
  llvm::StringSet<> VisitedModuleFiles;
91
92
5
  PrebuiltModuleListener Listener(ModuleFilesWorklist, InputFiles,
93
5
                                  VisitInputFiles);
94
95
14
  auto GatherModuleFileInfo = [&](StringRef ASTFile) {
96
14
    ASTReader::readASTFileControlBlock(
97
14
        ASTFile, CI.getFileManager(), CI.getPCHContainerReader(),
98
14
        /*FindModuleFileExtensions=*/false, Listener,
99
14
        /*ValidateDiagnosticOptions=*/false);
100
14
  };
101
102
5
  GatherModuleFileInfo(PrebuiltModuleFilename);
103
16
  while (!ModuleFilesWorklist.empty()) {
104
11
    auto WorklistItemIt = ModuleFilesWorklist.begin();
105
106
11
    if (!VisitedModuleFiles.contains(WorklistItemIt->getValue())) {
107
9
      VisitedModuleFiles.insert(WorklistItemIt->getValue());
108
9
      GatherModuleFileInfo(WorklistItemIt->getValue());
109
9
      ModuleFiles[WorklistItemIt->getKey().str()] = WorklistItemIt->getValue();
110
9
    }
111
112
11
    ModuleFilesWorklist.erase(WorklistItemIt);
113
11
  }
114
5
}
115
116
/// Transform arbitrary file name into an object-like file name.
117
66
static std::string makeObjFileName(StringRef FileName) {
118
66
  SmallString<128> ObjFileName(FileName);
119
66
  llvm::sys::path::replace_extension(ObjFileName, "o");
120
66
  return std::string(ObjFileName.str());
121
66
}
122
123
/// Deduce the dependency target based on the output file and input files.
124
static std::string
125
deduceDepTarget(const std::string &OutputFile,
126
145
                const SmallVectorImpl<FrontendInputFile> &InputFiles) {
127
145
  if (OutputFile != "-")
128
76
    return OutputFile;
129
130
69
  if (InputFiles.empty() || 
!InputFiles.front().isFile()68
)
131
0
    return "clang-scan-deps\\ dependency";
132
133
69
  return makeObjFileName(InputFiles.front().getFile());
134
69
}
135
136
/// Sanitize diagnostic options for dependency scan.
137
356
static void sanitizeDiagOpts(DiagnosticOptions &DiagOpts) {
138
  // Don't print 'X warnings and Y errors generated'.
139
356
  DiagOpts.ShowCarets = false;
140
  // Don't write out diagnostic file.
141
356
  DiagOpts.DiagnosticSerializationFile.clear();
142
  // Don't treat warnings as errors.
143
356
  DiagOpts.Warnings.push_back("no-error");
144
356
}
145
146
/// A clang tool that runs the preprocessor in a mode that's optimized for
147
/// dependency scanning for the given compiler invocation.
148
class DependencyScanningAction : public tooling::ToolAction {
149
public:
150
  DependencyScanningAction(
151
      StringRef WorkingDirectory, DependencyConsumer &Consumer,
152
      llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS,
153
      ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings,
154
      ScanningOutputFormat Format, bool OptimizeArgs,
155
      llvm::Optional<StringRef> ModuleName = None)
156
      : WorkingDirectory(WorkingDirectory), Consumer(Consumer),
157
        DepFS(std::move(DepFS)), PPSkipMappings(PPSkipMappings), Format(Format),
158
179
        OptimizeArgs(OptimizeArgs), ModuleName(ModuleName) {}
159
160
  bool runInvocation(std::shared_ptr<CompilerInvocation> Invocation,
161
                     FileManager *FileMgr,
162
                     std::shared_ptr<PCHContainerOperations> PCHContainerOps,
163
177
                     DiagnosticConsumer *DiagConsumer) override {
164
    // Make a deep copy of the original Clang invocation.
165
177
    CompilerInvocation OriginalInvocation(*Invocation);
166
167
    // Create a compiler instance to handle the actual work.
168
177
    CompilerInstance ScanInstance(std::move(PCHContainerOps));
169
177
    ScanInstance.setInvocation(std::move(Invocation));
170
171
    // Create the compiler's actual diagnostics engine.
172
177
    sanitizeDiagOpts(ScanInstance.getDiagnosticOpts());
173
177
    ScanInstance.createDiagnostics(DiagConsumer, /*ShouldOwnClient=*/false);
174
177
    if (!ScanInstance.hasDiagnostics())
175
0
      return false;
176
177
177
    ScanInstance.getPreprocessorOpts().AllowPCHWithDifferentModulesCachePath =
178
177
        true;
179
180
177
    FileMgr->getFileSystemOpts().WorkingDir = std::string(WorkingDirectory);
181
177
    ScanInstance.setFileManager(FileMgr);
182
177
    ScanInstance.createSourceManager(*FileMgr);
183
184
177
    llvm::StringSet<> PrebuiltModulesInputFiles;
185
    // Store the list of prebuilt module files into header search options. This
186
    // will prevent the implicit build to create duplicate modules and will
187
    // force reuse of the existing prebuilt module files instead.
188
177
    if (!ScanInstance.getPreprocessorOpts().ImplicitPCHInclude.empty())
189
5
      visitPrebuiltModule(
190
5
          ScanInstance.getPreprocessorOpts().ImplicitPCHInclude, ScanInstance,
191
5
          ScanInstance.getHeaderSearchOpts().PrebuiltModuleFiles,
192
5
          PrebuiltModulesInputFiles, /*VisitInputFiles=*/DepFS != nullptr);
193
194
    // Use the dependency scanning optimized file system if requested to do so.
195
177
    if (DepFS) {
196
138
      DepFS->enableMinimizationOfAllFiles();
197
      // Don't minimize any files that contributed to prebuilt modules. The
198
      // implicit build validates the modules by comparing the reported sizes of
199
      // their inputs to the current state of the filesystem. Minimization would
200
      // throw this mechanism off.
201
138
      for (const auto &File : PrebuiltModulesInputFiles)
202
20
        DepFS->disableMinimization(File.getKey());
203
      // Don't minimize any files that were explicitly passed in the build
204
      // settings and that might be opened.
205
138
      for (const auto &E : ScanInstance.getHeaderSearchOpts().UserEntries)
206
463
        DepFS->disableMinimization(E.Path);
207
138
      for (const auto &F : ScanInstance.getHeaderSearchOpts().VFSOverlayFiles)
208
3
        DepFS->disableMinimization(F);
209
210
      // Support for virtual file system overlays on top of the caching
211
      // filesystem.
212
138
      FileMgr->setVirtualFileSystem(createVFSFromCompilerInvocation(
213
138
          ScanInstance.getInvocation(), ScanInstance.getDiagnostics(), DepFS));
214
215
      // Pass the skip mappings which should speed up excluded conditional block
216
      // skipping in the preprocessor.
217
138
      if (PPSkipMappings)
218
133
        ScanInstance.getPreprocessorOpts()
219
133
            .ExcludedConditionalDirectiveSkipMappings = PPSkipMappings;
220
138
    }
221
222
    // Create the dependency collector that will collect the produced
223
    // dependencies.
224
    //
225
    // This also moves the existing dependency output options from the
226
    // invocation to the collector. The options in the invocation are reset,
227
    // which ensures that the compiler won't create new dependency collectors,
228
    // and thus won't write out the extra '.d' files to disk.
229
177
    auto Opts = std::make_unique<DependencyOutputOptions>();
230
177
    std::swap(*Opts, ScanInstance.getInvocation().getDependencyOutputOpts());
231
    // We need at least one -MT equivalent for the generator of make dependency
232
    // files to work.
233
177
    if (Opts->Targets.empty())
234
144
      Opts->Targets = {
235
144
          deduceDepTarget(ScanInstance.getFrontendOpts().OutputFile,
236
144
                          ScanInstance.getFrontendOpts().Inputs)};
237
177
    Opts->IncludeSystemHeaders = true;
238
239
177
    switch (Format) {
240
134
    case ScanningOutputFormat::Make:
241
134
      ScanInstance.addDependencyCollector(
242
134
          std::make_shared<DependencyConsumerForwarder>(std::move(Opts),
243
134
                                                        Consumer));
244
134
      break;
245
43
    case ScanningOutputFormat::Full:
246
43
      ScanInstance.addDependencyCollector(std::make_shared<ModuleDepCollector>(
247
43
          std::move(Opts), ScanInstance, Consumer,
248
43
          std::move(OriginalInvocation), OptimizeArgs));
249
43
      break;
250
177
    }
251
252
    // Consider different header search and diagnostic options to create
253
    // different modules. This avoids the unsound aliasing of module PCMs.
254
    //
255
    // TODO: Implement diagnostic bucketing to reduce the impact of strict
256
    // context hashing.
257
173
    ScanInstance.getHeaderSearchOpts().ModulesStrictContextHash = true;
258
259
173
    std::unique_ptr<FrontendAction> Action;
260
261
173
    if (ModuleName.hasValue())
262
4
      Action = std::make_unique<GetDependenciesByModuleNameAction>(*ModuleName);
263
169
    else
264
169
      Action = std::make_unique<ReadPCHAndPreprocessAction>();
265
266
173
    const bool Result = ScanInstance.ExecuteAction(*Action);
267
173
    if (!DepFS)
268
38
      FileMgr->clearStatCache();
269
173
    return Result;
270
177
  }
271
272
private:
273
  StringRef WorkingDirectory;
274
  DependencyConsumer &Consumer;
275
  llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS;
276
  ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings;
277
  ScanningOutputFormat Format;
278
  bool OptimizeArgs;
279
  llvm::Optional<StringRef> ModuleName;
280
};
281
282
} // end anonymous namespace
283
284
DependencyScanningWorker::DependencyScanningWorker(
285
    DependencyScanningService &Service)
286
365
    : Format(Service.getFormat()), OptimizeArgs(Service.canOptimizeArgs()) {
287
365
  PCHContainerOps = std::make_shared<PCHContainerOperations>();
288
365
  PCHContainerOps->registerReader(
289
365
      std::make_unique<ObjectFilePCHContainerReader>());
290
  // We don't need to write object files, but the current PCH implementation
291
  // requires the writer to be registered as well.
292
365
  PCHContainerOps->registerWriter(
293
365
      std::make_unique<ObjectFilePCHContainerWriter>());
294
295
365
  auto OverlayFS = llvm::makeIntrusiveRefCnt<llvm::vfs::OverlayFileSystem>(
296
365
      llvm::vfs::createPhysicalFileSystem());
297
365
  InMemoryFS = llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>();
298
365
  OverlayFS->pushOverlay(InMemoryFS);
299
365
  RealFS = OverlayFS;
300
301
365
  if (Service.canSkipExcludedPPRanges())
302
363
    PPSkipMappings =
303
363
        std::make_unique<ExcludedPreprocessorDirectiveSkipMapping>();
304
365
  if (Service.getMode() == ScanningMode::MinimizedSourcePreprocessing)
305
345
    DepFS = new DependencyScanningWorkerFilesystem(
306
345
        Service.getSharedCache(), RealFS, PPSkipMappings.get());
307
365
  if (Service.canReuseFileManager())
308
363
    Files = new FileManager(FileSystemOptions(), RealFS);
309
365
}
310
311
static llvm::Error
312
runWithDiags(DiagnosticOptions *DiagOpts,
313
             llvm::function_ref<bool(DiagnosticConsumer &, DiagnosticOptions &)>
314
179
                 BodyShouldSucceed) {
315
179
  sanitizeDiagOpts(*DiagOpts);
316
317
  // Capture the emitted diagnostics and report them to the client
318
  // in the case of a failure.
319
179
  std::string DiagnosticOutput;
320
179
  llvm::raw_string_ostream DiagnosticsOS(DiagnosticOutput);
321
179
  TextDiagnosticPrinter DiagPrinter(DiagnosticsOS, DiagOpts);
322
323
179
  if (BodyShouldSucceed(DiagPrinter, *DiagOpts))
324
173
    return llvm::Error::success();
325
6
  return llvm::make_error<llvm::StringError>(DiagnosticsOS.str(),
326
6
                                             llvm::inconvertibleErrorCode());
327
179
}
328
329
llvm::Error DependencyScanningWorker::computeDependencies(
330
    StringRef WorkingDirectory, const std::vector<std::string> &CommandLine,
331
179
    DependencyConsumer &Consumer, llvm::Optional<StringRef> ModuleName) {
332
  // Reset what might have been modified in the previous worker invocation.
333
179
  RealFS->setCurrentWorkingDirectory(WorkingDirectory);
334
179
  if (Files)
335
175
    Files->setVirtualFileSystem(RealFS);
336
337
179
  llvm::IntrusiveRefCntPtr<FileManager> CurrentFiles =
338
179
      Files ? 
Files175
:
new FileManager(FileSystemOptions(), RealFS)4
;
339
340
179
  Optional<std::vector<std::string>> ModifiedCommandLine;
341
179
  if (ModuleName.hasValue()) {
342
4
    ModifiedCommandLine = CommandLine;
343
4
    InMemoryFS->addFile(*ModuleName, 0, llvm::MemoryBuffer::getMemBuffer(""));
344
4
    ModifiedCommandLine->emplace_back(*ModuleName);
345
4
  }
346
347
179
  const std::vector<std::string> &FinalCommandLine =
348
179
      ModifiedCommandLine ? 
*ModifiedCommandLine4
:
CommandLine175
;
349
350
179
  std::vector<const char *> FinalCCommandLine(CommandLine.size(), nullptr);
351
179
  llvm::transform(CommandLine, FinalCCommandLine.begin(),
352
1.62k
                  [](const std::string &Str) { return Str.c_str(); });
353
354
179
  return runWithDiags(CreateAndPopulateDiagOpts(FinalCCommandLine).release(),
355
179
                      [&](DiagnosticConsumer &DC, DiagnosticOptions &DiagOpts) {
356
179
                        DependencyScanningAction Action(
357
179
                            WorkingDirectory, Consumer, DepFS,
358
179
                            PPSkipMappings.get(), Format, OptimizeArgs,
359
179
                            ModuleName);
360
                        // Create an invocation that uses the underlying file
361
                        // system to ensure that any file system requests that
362
                        // are made by the driver do not go through the
363
                        // dependency scanning filesystem.
364
179
                        ToolInvocation Invocation(FinalCommandLine, &Action,
365
179
                                                  CurrentFiles.get(),
366
179
                                                  PCHContainerOps);
367
179
                        Invocation.setDiagnosticConsumer(&DC);
368
179
                        Invocation.setDiagnosticOptions(&DiagOpts);
369
179
                        return Invocation.run();
370
179
                      });
371
179
}