Coverage Report

Created: 2022-01-18 06:27

/Users/buildslave/jenkins/workspace/coverage/llvm-project/clang/lib/Serialization/GlobalModuleIndex.cpp
Line
Count
Source (jump to first uncovered line)
1
//===--- GlobalModuleIndex.cpp - Global Module Index ------------*- C++ -*-===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// This file implements the GlobalModuleIndex class.
10
//
11
//===----------------------------------------------------------------------===//
12
13
#include "clang/Serialization/GlobalModuleIndex.h"
14
#include "ASTReaderInternals.h"
15
#include "clang/Basic/FileManager.h"
16
#include "clang/Lex/HeaderSearch.h"
17
#include "clang/Serialization/ASTBitCodes.h"
18
#include "clang/Serialization/ModuleFile.h"
19
#include "clang/Serialization/PCHContainerOperations.h"
20
#include "llvm/ADT/DenseMap.h"
21
#include "llvm/ADT/MapVector.h"
22
#include "llvm/ADT/SmallString.h"
23
#include "llvm/ADT/StringRef.h"
24
#include "llvm/Bitstream/BitstreamReader.h"
25
#include "llvm/Bitstream/BitstreamWriter.h"
26
#include "llvm/Support/DJB.h"
27
#include "llvm/Support/FileSystem.h"
28
#include "llvm/Support/FileUtilities.h"
29
#include "llvm/Support/LockFileManager.h"
30
#include "llvm/Support/MemoryBuffer.h"
31
#include "llvm/Support/OnDiskHashTable.h"
32
#include "llvm/Support/Path.h"
33
#include "llvm/Support/TimeProfiler.h"
34
#include <cstdio>
35
using namespace clang;
36
using namespace serialization;
37
38
//----------------------------------------------------------------------------//
39
// Shared constants
40
//----------------------------------------------------------------------------//
41
namespace {
42
  enum {
43
    /// The block containing the index.
44
    GLOBAL_INDEX_BLOCK_ID = llvm::bitc::FIRST_APPLICATION_BLOCKID
45
  };
46
47
  /// Describes the record types in the index.
48
  enum IndexRecordTypes {
49
    /// Contains version information and potentially other metadata,
50
    /// used to determine if we can read this global index file.
51
    INDEX_METADATA,
52
    /// Describes a module, including its file name and dependencies.
53
    MODULE,
54
    /// The index for identifiers.
55
    IDENTIFIER_INDEX
56
  };
57
}
58
59
/// The name of the global index file.
60
static const char * const IndexFileName = "modules.idx";
61
62
/// The global index file version.
63
static const unsigned CurrentVersion = 1;
64
65
//----------------------------------------------------------------------------//
66
// Global module index reader.
67
//----------------------------------------------------------------------------//
68
69
namespace {
70
71
/// Trait used to read the identifier index from the on-disk hash
72
/// table.
73
class IdentifierIndexReaderTrait {
74
public:
75
  typedef StringRef external_key_type;
76
  typedef StringRef internal_key_type;
77
  typedef SmallVector<unsigned, 2> data_type;
78
  typedef unsigned hash_value_type;
79
  typedef unsigned offset_type;
80
81
93.6k
  static bool EqualKey(const internal_key_type& a, const internal_key_type& b) {
82
93.6k
    return a == b;
83
93.6k
  }
84
85
620k
  static hash_value_type ComputeHash(const internal_key_type& a) {
86
620k
    return llvm::djbHash(a);
87
620k
  }
88
89
  static std::pair<unsigned, unsigned>
90
389k
  ReadKeyDataLength(const unsigned char*& d) {
91
389k
    using namespace llvm::support;
92
389k
    unsigned KeyLen = endian::readNext<uint16_t, little, unaligned>(d);
93
389k
    unsigned DataLen = endian::readNext<uint16_t, little, unaligned>(d);
94
389k
    return std::make_pair(KeyLen, DataLen);
95
389k
  }
96
97
  static const internal_key_type&
98
620k
  GetInternalKey(const external_key_type& x) { return x; }
99
100
  static const external_key_type&
101
4.73k
  GetExternalKey(const internal_key_type& x) { return x; }
102
103
98.3k
  static internal_key_type ReadKey(const unsigned char* d, unsigned n) {
104
98.3k
    return StringRef((const char *)d, n);
105
98.3k
  }
106
107
  static data_type ReadData(const internal_key_type& k,
108
                            const unsigned char* d,
109
93.5k
                            unsigned DataLen) {
110
93.5k
    using namespace llvm::support;
111
112
93.5k
    data_type Result;
113
353k
    while (DataLen > 0) {
114
260k
      unsigned ID = endian::readNext<uint32_t, little, unaligned>(d);
115
260k
      Result.push_back(ID);
116
260k
      DataLen -= 4;
117
260k
    }
118
119
93.5k
    return Result;
120
93.5k
  }
121
};
122
123
typedef llvm::OnDiskIterableChainedHashTable<IdentifierIndexReaderTrait>
124
    IdentifierIndexTable;
125
126
}
127
128
GlobalModuleIndex::GlobalModuleIndex(
129
    std::unique_ptr<llvm::MemoryBuffer> IndexBuffer,
130
    llvm::BitstreamCursor Cursor)
131
    : Buffer(std::move(IndexBuffer)), IdentifierIndex(), NumIdentifierLookups(),
132
637
      NumIdentifierLookupHits() {
133
637
  auto Fail = [&](llvm::Error &&Err) {
134
0
    report_fatal_error("Module index '" + Buffer->getBufferIdentifier() +
135
0
                       "' failed: " + toString(std::move(Err)));
136
0
  };
137
138
637
  llvm::TimeTraceScope TimeScope("Module LoadIndex");
139
  // Read the global index.
140
637
  bool InGlobalIndexBlock = false;
141
637
  bool Done = false;
142
10.8k
  while (!Done) {
143
10.2k
    llvm::BitstreamEntry Entry;
144
10.2k
    if (Expected<llvm::BitstreamEntry> Res = Cursor.advance())
145
10.2k
      Entry = Res.get();
146
0
    else
147
0
      Fail(Res.takeError());
148
149
10.2k
    switch (Entry.Kind) {
150
0
    case llvm::BitstreamEntry::Error:
151
0
      return;
152
153
637
    case llvm::BitstreamEntry::EndBlock:
154
637
      if (InGlobalIndexBlock) {
155
637
        InGlobalIndexBlock = false;
156
637
        Done = true;
157
637
        continue;
158
637
      }
159
0
      return;
160
161
162
8.32k
    case llvm::BitstreamEntry::Record:
163
      // Entries in the global index block are handled below.
164
8.32k
      if (InGlobalIndexBlock)
165
8.32k
        break;
166
167
0
      return;
168
169
1.27k
    case llvm::BitstreamEntry::SubBlock:
170
1.27k
      if (!InGlobalIndexBlock && Entry.ID == GLOBAL_INDEX_BLOCK_ID) {
171
637
        if (llvm::Error Err = Cursor.EnterSubBlock(GLOBAL_INDEX_BLOCK_ID))
172
0
          Fail(std::move(Err));
173
637
        InGlobalIndexBlock = true;
174
637
      } else if (llvm::Error Err = Cursor.SkipBlock())
175
0
        Fail(std::move(Err));
176
1.27k
      continue;
177
10.2k
    }
178
179
8.32k
    SmallVector<uint64_t, 64> Record;
180
8.32k
    StringRef Blob;
181
8.32k
    Expected<unsigned> MaybeIndexRecord =
182
8.32k
        Cursor.readRecord(Entry.ID, Record, &Blob);
183
8.32k
    if (!MaybeIndexRecord)
184
0
      Fail(MaybeIndexRecord.takeError());
185
8.32k
    IndexRecordTypes IndexRecord =
186
8.32k
        static_cast<IndexRecordTypes>(MaybeIndexRecord.get());
187
8.32k
    switch (IndexRecord) {
188
637
    case INDEX_METADATA:
189
      // Make sure that the version matches.
190
637
      if (Record.size() < 1 || Record[0] != CurrentVersion)
191
0
        return;
192
637
      break;
193
194
7.05k
    case MODULE: {
195
7.05k
      unsigned Idx = 0;
196
7.05k
      unsigned ID = Record[Idx++];
197
198
      // Make room for this module's information.
199
7.05k
      if (ID == Modules.size())
200
7.05k
        Modules.push_back(ModuleInfo());
201
0
      else
202
0
        Modules.resize(ID + 1);
203
204
      // Size/modification time for this module file at the time the
205
      // global index was built.
206
7.05k
      Modules[ID].Size = Record[Idx++];
207
7.05k
      Modules[ID].ModTime = Record[Idx++];
208
209
      // File name.
210
7.05k
      unsigned NameLen = Record[Idx++];
211
7.05k
      Modules[ID].FileName.assign(Record.begin() + Idx,
212
7.05k
                                  Record.begin() + Idx + NameLen);
213
7.05k
      Idx += NameLen;
214
215
      // Dependencies
216
7.05k
      unsigned NumDeps = Record[Idx++];
217
7.05k
      Modules[ID].Dependencies.insert(Modules[ID].Dependencies.end(),
218
7.05k
                                      Record.begin() + Idx,
219
7.05k
                                      Record.begin() + Idx + NumDeps);
220
7.05k
      Idx += NumDeps;
221
222
      // Make sure we're at the end of the record.
223
7.05k
      assert(Idx == Record.size() && "More module info?");
224
225
      // Record this module as an unresolved module.
226
      // FIXME: this doesn't work correctly for module names containing path
227
      // separators.
228
0
      StringRef ModuleName = llvm::sys::path::stem(Modules[ID].FileName);
229
      // Remove the -<hash of ModuleMapPath>
230
7.05k
      ModuleName = ModuleName.rsplit('-').first;
231
7.05k
      UnresolvedModules[ModuleName] = ID;
232
7.05k
      break;
233
637
    }
234
235
637
    case IDENTIFIER_INDEX:
236
      // Wire up the identifier index.
237
637
      if (Record[0]) {
238
637
        IdentifierIndex = IdentifierIndexTable::Create(
239
637
            (const unsigned char *)Blob.data() + Record[0],
240
637
            (const unsigned char *)Blob.data() + sizeof(uint32_t),
241
637
            (const unsigned char *)Blob.data(), IdentifierIndexReaderTrait());
242
637
      }
243
637
      break;
244
8.32k
    }
245
8.32k
  }
246
637
}
247
248
273
GlobalModuleIndex::~GlobalModuleIndex() {
249
273
  delete static_cast<IdentifierIndexTable *>(IdentifierIndex);
250
273
}
251
252
std::pair<GlobalModuleIndex *, llvm::Error>
253
1.79k
GlobalModuleIndex::readIndex(StringRef Path) {
254
  // Load the index file, if it's there.
255
1.79k
  llvm::SmallString<128> IndexPath;
256
1.79k
  IndexPath += Path;
257
1.79k
  llvm::sys::path::append(IndexPath, IndexFileName);
258
259
1.79k
  llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> BufferOrErr =
260
1.79k
      llvm::MemoryBuffer::getFile(IndexPath.c_str());
261
1.79k
  if (!BufferOrErr)
262
1.15k
    return std::make_pair(nullptr,
263
1.15k
                          llvm::errorCodeToError(BufferOrErr.getError()));
264
637
  std::unique_ptr<llvm::MemoryBuffer> Buffer = std::move(BufferOrErr.get());
265
266
  /// The main bitstream cursor for the main block.
267
637
  llvm::BitstreamCursor Cursor(*Buffer);
268
269
  // Sniff for the signature.
270
2.54k
  for (unsigned char C : {'B', 'C', 'G', 'I'}) {
271
2.54k
    if (Expected<llvm::SimpleBitstreamCursor::word_t> Res = Cursor.Read(8)) {
272
2.54k
      if (Res.get() != C)
273
0
        return std::make_pair(
274
0
            nullptr, llvm::createStringError(std::errc::illegal_byte_sequence,
275
0
                                             "expected signature BCGI"));
276
2.54k
    } else
277
0
      return std::make_pair(nullptr, Res.takeError());
278
2.54k
  }
279
280
637
  return std::make_pair(new GlobalModuleIndex(std::move(Buffer), Cursor),
281
637
                        llvm::Error::success());
282
637
}
283
284
void
285
0
GlobalModuleIndex::getKnownModules(SmallVectorImpl<ModuleFile *> &ModuleFiles) {
286
0
  ModuleFiles.clear();
287
0
  for (unsigned I = 0, N = Modules.size(); I != N; ++I) {
288
0
    if (ModuleFile *MF = Modules[I].File)
289
0
      ModuleFiles.push_back(MF);
290
0
  }
291
0
}
292
293
void GlobalModuleIndex::getModuleDependencies(
294
       ModuleFile *File,
295
0
       SmallVectorImpl<ModuleFile *> &Dependencies) {
296
  // Look for information about this module file.
297
0
  llvm::DenseMap<ModuleFile *, unsigned>::iterator Known
298
0
    = ModulesByFile.find(File);
299
0
  if (Known == ModulesByFile.end())
300
0
    return;
301
302
  // Record dependencies.
303
0
  Dependencies.clear();
304
0
  ArrayRef<unsigned> StoredDependencies = Modules[Known->second].Dependencies;
305
0
  for (unsigned I = 0, N = StoredDependencies.size(); I != N; ++I) {
306
0
    if (ModuleFile *MF = Modules[I].File)
307
0
      Dependencies.push_back(MF);
308
0
  }
309
0
}
310
311
620k
bool GlobalModuleIndex::lookupIdentifier(StringRef Name, HitSet &Hits) {
312
620k
  Hits.clear();
313
314
  // If there's no identifier index, there is nothing we can do.
315
620k
  if (!IdentifierIndex)
316
0
    return false;
317
318
  // Look into the identifier index.
319
620k
  ++NumIdentifierLookups;
320
620k
  IdentifierIndexTable &Table
321
620k
    = *static_cast<IdentifierIndexTable *>(IdentifierIndex);
322
620k
  IdentifierIndexTable::iterator Known = Table.find(Name);
323
620k
  if (Known == Table.end()) {
324
527k
    return false;
325
527k
  }
326
327
93.5k
  SmallVector<unsigned, 2> ModuleIDs = *Known;
328
353k
  for (unsigned I = 0, N = ModuleIDs.size(); I != N; 
++I260k
) {
329
260k
    if (ModuleFile *MF = Modules[ModuleIDs[I]].File)
330
108k
      Hits.insert(MF);
331
260k
  }
332
333
93.5k
  ++NumIdentifierLookupHits;
334
93.5k
  return true;
335
620k
}
336
337
3.82k
bool GlobalModuleIndex::loadedModuleFile(ModuleFile *File) {
338
  // Look for the module in the global module index based on the module name.
339
3.82k
  StringRef Name = File->ModuleName;
340
3.82k
  llvm::StringMap<unsigned>::iterator Known = UnresolvedModules.find(Name);
341
3.82k
  if (Known == UnresolvedModules.end()) {
342
207
    return true;
343
207
  }
344
345
  // Rectify this module with the global module index.
346
3.61k
  ModuleInfo &Info = Modules[Known->second];
347
348
  //  If the size and modification time match what we expected, record this
349
  // module file.
350
3.61k
  bool Failed = true;
351
3.61k
  if (File->File->getSize() == Info.Size &&
352
3.61k
      
File->File->getModificationTime() == Info.ModTime3.56k
) {
353
3.54k
    Info.File = File;
354
3.54k
    ModulesByFile[File] = Known->second;
355
356
3.54k
    Failed = false;
357
3.54k
  }
358
359
  // One way or another, we have resolved this module file.
360
3.61k
  UnresolvedModules.erase(Known);
361
3.61k
  return Failed;
362
3.82k
}
363
364
1
void GlobalModuleIndex::printStats() {
365
1
  std::fprintf(stderr, "*** Global Module Index Statistics:\n");
366
1
  if (NumIdentifierLookups) {
367
1
    fprintf(stderr, "  %u / %u identifier lookups succeeded (%f%%)\n",
368
1
            NumIdentifierLookupHits, NumIdentifierLookups,
369
1
            (double)NumIdentifierLookupHits*100.0/NumIdentifierLookups);
370
1
  }
371
1
  std::fprintf(stderr, "\n");
372
1
}
373
374
0
LLVM_DUMP_METHOD void GlobalModuleIndex::dump() {
375
0
  llvm::errs() << "*** Global Module Index Dump:\n";
376
0
  llvm::errs() << "Module files:\n";
377
0
  for (auto &MI : Modules) {
378
0
    llvm::errs() << "** " << MI.FileName << "\n";
379
0
    if (MI.File)
380
0
      MI.File->dump();
381
0
    else
382
0
      llvm::errs() << "\n";
383
0
  }
384
0
  llvm::errs() << "\n";
385
0
}
386
387
//----------------------------------------------------------------------------//
388
// Global module index writer.
389
//----------------------------------------------------------------------------//
390
391
namespace {
392
  /// Provides information about a specific module file.
393
  struct ModuleFileInfo {
394
    /// The numberic ID for this module file.
395
    unsigned ID;
396
397
    /// The set of modules on which this module depends. Each entry is
398
    /// a module ID.
399
    SmallVector<unsigned, 4> Dependencies;
400
    ASTFileSignature Signature;
401
  };
402
403
  struct ImportedModuleFileInfo {
404
    off_t StoredSize;
405
    time_t StoredModTime;
406
    ASTFileSignature StoredSignature;
407
    ImportedModuleFileInfo(off_t Size, time_t ModTime, ASTFileSignature Sig)
408
1.77k
        : StoredSize(Size), StoredModTime(ModTime), StoredSignature(Sig) {}
409
  };
410
411
  /// Builder that generates the global module index file.
412
  class GlobalModuleIndexBuilder {
413
    FileManager &FileMgr;
414
    const PCHContainerReader &PCHContainerRdr;
415
416
    /// Mapping from files to module file information.
417
    typedef llvm::MapVector<const FileEntry *, ModuleFileInfo> ModuleFilesMap;
418
419
    /// Information about each of the known module files.
420
    ModuleFilesMap ModuleFiles;
421
422
    /// Mapping from the imported module file to the imported
423
    /// information.
424
    typedef std::multimap<const FileEntry *, ImportedModuleFileInfo>
425
        ImportedModuleFilesMap;
426
427
    /// Information about each importing of a module file.
428
    ImportedModuleFilesMap ImportedModuleFiles;
429
430
    /// Mapping from identifiers to the list of module file IDs that
431
    /// consider this identifier to be interesting.
432
    typedef llvm::StringMap<SmallVector<unsigned, 2> > InterestingIdentifierMap;
433
434
    /// A mapping from all interesting identifiers to the set of module
435
    /// files in which those identifiers are considered interesting.
436
    InterestingIdentifierMap InterestingIdentifiers;
437
438
    /// Write the block-info block for the global module index file.
439
    void emitBlockInfoBlock(llvm::BitstreamWriter &Stream);
440
441
    /// Retrieve the module file information for the given file.
442
11.0k
    ModuleFileInfo &getModuleFileInfo(const FileEntry *File) {
443
11.0k
      llvm::MapVector<const FileEntry *, ModuleFileInfo>::iterator Known
444
11.0k
        = ModuleFiles.find(File);
445
11.0k
      if (Known != ModuleFiles.end())
446
9.05k
        return Known->second;
447
448
2.00k
      unsigned NewID = ModuleFiles.size();
449
2.00k
      ModuleFileInfo &Info = ModuleFiles[File];
450
2.00k
      Info.ID = NewID;
451
2.00k
      return Info;
452
11.0k
    }
453
454
  public:
455
    explicit GlobalModuleIndexBuilder(
456
        FileManager &FileMgr, const PCHContainerReader &PCHContainerRdr)
457
789
        : FileMgr(FileMgr), PCHContainerRdr(PCHContainerRdr) {}
458
459
    /// Load the contents of the given module file into the builder.
460
    llvm::Error loadModuleFile(const FileEntry *File);
461
462
    /// Write the index to the given bitstream.
463
    /// \returns true if an error occurred, false otherwise.
464
    bool writeIndex(llvm::BitstreamWriter &Stream);
465
  };
466
}
467
468
static void emitBlockID(unsigned ID, const char *Name,
469
                        llvm::BitstreamWriter &Stream,
470
778
                        SmallVectorImpl<uint64_t> &Record) {
471
778
  Record.clear();
472
778
  Record.push_back(ID);
473
778
  Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_SETBID, Record);
474
475
  // Emit the block name if present.
476
778
  if (!Name || Name[0] == 0) 
return0
;
477
778
  Record.clear();
478
14.7k
  while (*Name)
479
14.0k
    Record.push_back(*Name++);
480
778
  Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_BLOCKNAME, Record);
481
778
}
482
483
static void emitRecordID(unsigned ID, const char *Name,
484
                         llvm::BitstreamWriter &Stream,
485
2.33k
                         SmallVectorImpl<uint64_t> &Record) {
486
2.33k
  Record.clear();
487
2.33k
  Record.push_back(ID);
488
30.3k
  while (*Name)
489
28.0k
    Record.push_back(*Name++);
490
2.33k
  Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_SETRECORDNAME, Record);
491
2.33k
}
492
493
void
494
778
GlobalModuleIndexBuilder::emitBlockInfoBlock(llvm::BitstreamWriter &Stream) {
495
778
  SmallVector<uint64_t, 64> Record;
496
778
  Stream.EnterBlockInfoBlock();
497
498
778
#define BLOCK(X) emitBlockID(X ## _ID, #X, Stream, Record)
499
2.33k
#define RECORD(X) emitRecordID(X, #X, Stream, Record)
500
778
  BLOCK(GLOBAL_INDEX_BLOCK);
501
778
  RECORD(INDEX_METADATA);
502
778
  RECORD(MODULE);
503
778
  RECORD(IDENTIFIER_INDEX);
504
778
#undef RECORD
505
778
#undef BLOCK
506
507
778
  Stream.ExitBlock();
508
778
}
509
510
namespace {
511
  class InterestingASTIdentifierLookupTrait
512
    : public serialization::reader::ASTIdentifierLookupTraitBase {
513
514
  public:
515
    /// The identifier and whether it is "interesting".
516
    typedef std::pair<StringRef, bool> data_type;
517
518
    data_type ReadData(const internal_key_type& k,
519
                       const unsigned char* d,
520
2.00M
                       unsigned DataLen) {
521
      // The first bit indicates whether this identifier is interesting.
522
      // That's all we care about.
523
2.00M
      using namespace llvm::support;
524
2.00M
      unsigned RawID = endian::readNext<uint32_t, little, unaligned>(d);
525
2.00M
      bool IsInteresting = RawID & 0x01;
526
2.00M
      return std::make_pair(k, IsInteresting);
527
2.00M
    }
528
  };
529
}
530
531
1.99k
llvm::Error GlobalModuleIndexBuilder::loadModuleFile(const FileEntry *File) {
532
  // Open the module file.
533
534
1.99k
  auto Buffer = FileMgr.getBufferForFile(File, /*isVolatile=*/true);
535
1.99k
  if (!Buffer)
536
0
    return llvm::createStringError(Buffer.getError(),
537
0
                                   "failed getting buffer for module file");
538
539
  // Initialize the input stream
540
1.99k
  llvm::BitstreamCursor InStream(PCHContainerRdr.ExtractPCH(**Buffer));
541
542
  // Sniff for the signature.
543
1.99k
  for (unsigned char C : {'C', 'P', 'C', 'H'})
544
7.98k
    if (Expected<llvm::SimpleBitstreamCursor::word_t> Res = InStream.Read(8)) {
545
7.98k
      if (Res.get() != C)
546
0
        return llvm::createStringError(std::errc::illegal_byte_sequence,
547
0
                                       "expected signature CPCH");
548
7.98k
    } else
549
0
      return Res.takeError();
550
551
  // Record this module file and assign it a unique ID (if it doesn't have
552
  // one already).
553
1.99k
  unsigned ID = getModuleFileInfo(File).ID;
554
555
  // Search for the blocks and records we care about.
556
1.99k
  enum { Other, ControlBlock, ASTBlock, DiagnosticOptionsBlock } State = Other;
557
1.99k
  bool Done = false;
558
96.6k
  while (!Done) {
559
94.6k
    Expected<llvm::BitstreamEntry> MaybeEntry = InStream.advance();
560
94.6k
    if (!MaybeEntry)
561
0
      return MaybeEntry.takeError();
562
94.6k
    llvm::BitstreamEntry Entry = MaybeEntry.get();
563
564
94.6k
    switch (Entry.Kind) {
565
1.98k
    case llvm::BitstreamEntry::Error:
566
1.98k
      Done = true;
567
1.98k
      continue;
568
569
64.7k
    case llvm::BitstreamEntry::Record:
570
      // In the 'other' state, just skip the record. We don't care.
571
64.7k
      if (State == Other) {
572
0
        if (llvm::Expected<unsigned> Skipped = InStream.skipRecord(Entry.ID))
573
0
          continue;
574
0
        else
575
0
          return Skipped.takeError();
576
0
      }
577
578
      // Handle potentially-interesting records below.
579
64.7k
      break;
580
581
64.7k
    case llvm::BitstreamEntry::SubBlock:
582
21.9k
      if (Entry.ID == CONTROL_BLOCK_ID) {
583
1.99k
        if (llvm::Error Err = InStream.EnterSubBlock(CONTROL_BLOCK_ID))
584
0
          return Err;
585
586
        // Found the control block.
587
1.99k
        State = ControlBlock;
588
1.99k
        continue;
589
1.99k
      }
590
591
19.9k
      if (Entry.ID == AST_BLOCK_ID) {
592
1.98k
        if (llvm::Error Err = InStream.EnterSubBlock(AST_BLOCK_ID))
593
0
          return Err;
594
595
        // Found the AST block.
596
1.98k
        State = ASTBlock;
597
1.98k
        continue;
598
1.98k
      }
599
600
17.9k
      if (Entry.ID == UNHASHED_CONTROL_BLOCK_ID) {
601
1.98k
        if (llvm::Error Err = InStream.EnterSubBlock(UNHASHED_CONTROL_BLOCK_ID))
602
0
          return Err;
603
604
        // Found the Diagnostic Options block.
605
1.98k
        State = DiagnosticOptionsBlock;
606
1.98k
        continue;
607
1.98k
      }
608
609
15.9k
      if (llvm::Error Err = InStream.SkipBlock())
610
0
        return Err;
611
612
15.9k
      continue;
613
614
15.9k
    case llvm::BitstreamEntry::EndBlock:
615
5.96k
      State = Other;
616
5.96k
      continue;
617
94.6k
    }
618
619
    // Read the given record.
620
64.7k
    SmallVector<uint64_t, 64> Record;
621
64.7k
    StringRef Blob;
622
64.7k
    Expected<unsigned> MaybeCode = InStream.readRecord(Entry.ID, Record, &Blob);
623
64.7k
    if (!MaybeCode)
624
0
      return MaybeCode.takeError();
625
64.7k
    unsigned Code = MaybeCode.get();
626
627
    // Handle module dependencies.
628
64.7k
    if (State == ControlBlock && 
Code == IMPORTS15.9k
) {
629
      // Load each of the imported PCH files.
630
1.99k
      unsigned Idx = 0, N = Record.size();
631
3.77k
      while (Idx < N) {
632
        // Read information about the AST file.
633
634
        // Skip the imported kind
635
1.78k
        ++Idx;
636
637
        // Skip the import location
638
1.78k
        ++Idx;
639
640
        // Load stored size/modification time.
641
1.78k
        off_t StoredSize = (off_t)Record[Idx++];
642
1.78k
        time_t StoredModTime = (time_t)Record[Idx++];
643
644
        // Skip the stored signature.
645
        // FIXME: we could read the signature out of the import and validate it.
646
1.78k
        auto FirstSignatureByte = Record.begin() + Idx;
647
1.78k
        ASTFileSignature StoredSignature = ASTFileSignature::create(
648
1.78k
            FirstSignatureByte, FirstSignatureByte + ASTFileSignature::size);
649
1.78k
        Idx += ASTFileSignature::size;
650
651
        // Skip the module name (currently this is only used for prebuilt
652
        // modules while here we are only dealing with cached).
653
1.78k
        Idx += Record[Idx] + 1;
654
655
        // Retrieve the imported file name.
656
1.78k
        unsigned Length = Record[Idx++];
657
1.78k
        SmallString<128> ImportedFile(Record.begin() + Idx,
658
1.78k
                                      Record.begin() + Idx + Length);
659
1.78k
        Idx += Length;
660
661
        // Find the imported module file.
662
1.78k
        auto DependsOnFile
663
1.78k
          = FileMgr.getFile(ImportedFile, /*OpenFile=*/false,
664
1.78k
                            /*CacheFailure=*/false);
665
666
1.78k
        if (!DependsOnFile)
667
8
          return llvm::createStringError(std::errc::bad_file_descriptor,
668
8
                                         "imported file \"%s\" not found",
669
8
                                         ImportedFile.c_str());
670
671
        // Save the information in ImportedModuleFileInfo so we can verify after
672
        // loading all pcms.
673
1.77k
        ImportedModuleFiles.insert(std::make_pair(
674
1.77k
            *DependsOnFile, ImportedModuleFileInfo(StoredSize, StoredModTime,
675
1.77k
                                                   StoredSignature)));
676
677
        // Record the dependency.
678
1.77k
        unsigned DependsOnID = getModuleFileInfo(*DependsOnFile).ID;
679
1.77k
        getModuleFileInfo(File).Dependencies.push_back(DependsOnID);
680
1.77k
      }
681
682
1.98k
      continue;
683
1.99k
    }
684
685
    // Handle the identifier table
686
62.7k
    if (State == ASTBlock && 
Code == IDENTIFIER_TABLE38.8k
&&
Record[0] > 01.98k
) {
687
1.98k
      typedef llvm::OnDiskIterableChainedHashTable<
688
1.98k
          InterestingASTIdentifierLookupTrait> InterestingIdentifierTable;
689
1.98k
      std::unique_ptr<InterestingIdentifierTable> Table(
690
1.98k
          InterestingIdentifierTable::Create(
691
1.98k
              (const unsigned char *)Blob.data() + Record[0],
692
1.98k
              (const unsigned char *)Blob.data() + sizeof(uint32_t),
693
1.98k
              (const unsigned char *)Blob.data()));
694
1.98k
      for (InterestingIdentifierTable::data_iterator D = Table->data_begin(),
695
1.98k
                                                     DEnd = Table->data_end();
696
2.00M
           D != DEnd; 
++D2.00M
) {
697
2.00M
        std::pair<StringRef, bool> Ident = *D;
698
2.00M
        if (Ident.second)
699
1.30M
          InterestingIdentifiers[Ident.first].push_back(ID);
700
699k
        else
701
699k
          (void)InterestingIdentifiers[Ident.first];
702
2.00M
      }
703
1.98k
    }
704
705
    // Get Signature.
706
62.7k
    if (State == DiagnosticOptionsBlock && 
Code == SIGNATURE9.90k
)
707
1.96k
      getModuleFileInfo(File).Signature = ASTFileSignature::create(
708
1.96k
          Record.begin(), Record.begin() + ASTFileSignature::size);
709
710
    // We don't care about this record.
711
62.7k
  }
712
713
1.98k
  return llvm::Error::success();
714
1.99k
}
715
716
namespace {
717
718
/// Trait used to generate the identifier index as an on-disk hash
719
/// table.
720
class IdentifierIndexWriterTrait {
721
public:
722
  typedef StringRef key_type;
723
  typedef StringRef key_type_ref;
724
  typedef SmallVector<unsigned, 2> data_type;
725
  typedef const SmallVector<unsigned, 2> &data_type_ref;
726
  typedef unsigned hash_value_type;
727
  typedef unsigned offset_type;
728
729
1.79M
  static hash_value_type ComputeHash(key_type_ref Key) {
730
1.79M
    return llvm::djbHash(Key);
731
1.79M
  }
732
733
  std::pair<unsigned,unsigned>
734
1.79M
  EmitKeyDataLength(raw_ostream& Out, key_type_ref Key, data_type_ref Data) {
735
1.79M
    using namespace llvm::support;
736
1.79M
    endian::Writer LE(Out, little);
737
1.79M
    unsigned KeyLen = Key.size();
738
1.79M
    unsigned DataLen = Data.size() * 4;
739
1.79M
    LE.write<uint16_t>(KeyLen);
740
1.79M
    LE.write<uint16_t>(DataLen);
741
1.79M
    return std::make_pair(KeyLen, DataLen);
742
1.79M
  }
743
744
1.79M
  void EmitKey(raw_ostream& Out, key_type_ref Key, unsigned KeyLen) {
745
1.79M
    Out.write(Key.data(), KeyLen);
746
1.79M
  }
747
748
  void EmitData(raw_ostream& Out, key_type_ref Key, data_type_ref Data,
749
1.79M
                unsigned DataLen) {
750
1.79M
    using namespace llvm::support;
751
3.09M
    for (unsigned I = 0, N = Data.size(); I != N; 
++I1.30M
)
752
1.30M
      endian::write<uint32_t>(Out, Data[I], little);
753
1.79M
  }
754
};
755
756
}
757
758
781
bool GlobalModuleIndexBuilder::writeIndex(llvm::BitstreamWriter &Stream) {
759
1.77k
  for (auto MapEntry : ImportedModuleFiles) {
760
1.77k
    auto *File = MapEntry.first;
761
1.77k
    ImportedModuleFileInfo &Info = MapEntry.second;
762
1.77k
    if (getModuleFileInfo(File).Signature) {
763
1.76k
      if (getModuleFileInfo(File).Signature != Info.StoredSignature)
764
        // Verify Signature.
765
3
        return true;
766
1.76k
    } else 
if (7
Info.StoredSize != File->getSize()7
||
767
7
               Info.StoredModTime != File->getModificationTime())
768
      // Verify Size and ModTime.
769
0
      return true;
770
1.77k
  }
771
772
778
  using namespace llvm;
773
778
  llvm::TimeTraceScope TimeScope("Module WriteIndex");
774
775
  // Emit the file header.
776
778
  Stream.Emit((unsigned)'B', 8);
777
778
  Stream.Emit((unsigned)'C', 8);
778
778
  Stream.Emit((unsigned)'G', 8);
779
778
  Stream.Emit((unsigned)'I', 8);
780
781
  // Write the block-info block, which describes the records in this bitcode
782
  // file.
783
778
  emitBlockInfoBlock(Stream);
784
785
778
  Stream.EnterSubblock(GLOBAL_INDEX_BLOCK_ID, 3);
786
787
  // Write the metadata.
788
778
  SmallVector<uint64_t, 2> Record;
789
778
  Record.push_back(CurrentVersion);
790
778
  Stream.EmitRecord(INDEX_METADATA, Record);
791
792
  // Write the set of known module files.
793
778
  for (ModuleFilesMap::iterator M = ModuleFiles.begin(),
794
778
                                MEnd = ModuleFiles.end();
795
2.75k
       M != MEnd; 
++M1.97k
) {
796
1.97k
    Record.clear();
797
1.97k
    Record.push_back(M->second.ID);
798
1.97k
    Record.push_back(M->first->getSize());
799
1.97k
    Record.push_back(M->first->getModificationTime());
800
801
    // File name
802
1.97k
    StringRef Name(M->first->getName());
803
1.97k
    Record.push_back(Name.size());
804
1.97k
    Record.append(Name.begin(), Name.end());
805
806
    // Dependencies
807
1.97k
    Record.push_back(M->second.Dependencies.size());
808
1.97k
    Record.append(M->second.Dependencies.begin(), M->second.Dependencies.end());
809
1.97k
    Stream.EmitRecord(MODULE, Record);
810
1.97k
  }
811
812
  // Write the identifier -> module file mapping.
813
778
  {
814
778
    llvm::OnDiskChainedHashTableGenerator<IdentifierIndexWriterTrait> Generator;
815
778
    IdentifierIndexWriterTrait Trait;
816
817
    // Populate the hash table.
818
778
    for (InterestingIdentifierMap::iterator I = InterestingIdentifiers.begin(),
819
778
                                            IEnd = InterestingIdentifiers.end();
820
1.79M
         I != IEnd; 
++I1.79M
) {
821
1.79M
      Generator.insert(I->first(), I->second, Trait);
822
1.79M
    }
823
824
    // Create the on-disk hash table in a buffer.
825
778
    SmallString<4096> IdentifierTable;
826
778
    uint32_t BucketOffset;
827
778
    {
828
778
      using namespace llvm::support;
829
778
      llvm::raw_svector_ostream Out(IdentifierTable);
830
      // Make sure that no bucket is at offset 0
831
778
      endian::write<uint32_t>(Out, 0, little);
832
778
      BucketOffset = Generator.Emit(Out, Trait);
833
778
    }
834
835
    // Create a blob abbreviation
836
778
    auto Abbrev = std::make_shared<BitCodeAbbrev>();
837
778
    Abbrev->Add(BitCodeAbbrevOp(IDENTIFIER_INDEX));
838
778
    Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
839
778
    Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
840
778
    unsigned IDTableAbbrev = Stream.EmitAbbrev(std::move(Abbrev));
841
842
    // Write the identifier table
843
778
    uint64_t Record[] = {IDENTIFIER_INDEX, BucketOffset};
844
778
    Stream.EmitRecordWithBlob(IDTableAbbrev, Record, IdentifierTable);
845
778
  }
846
847
778
  Stream.ExitBlock();
848
778
  return false;
849
781
}
850
851
llvm::Error
852
GlobalModuleIndex::writeIndex(FileManager &FileMgr,
853
                              const PCHContainerReader &PCHContainerRdr,
854
804
                              StringRef Path) {
855
804
  llvm::SmallString<128> IndexPath;
856
804
  IndexPath += Path;
857
804
  llvm::sys::path::append(IndexPath, IndexFileName);
858
859
  // Coordinate building the global index file with other processes that might
860
  // try to do the same.
861
804
  llvm::LockFileManager Locked(IndexPath);
862
804
  switch (Locked) {
863
15
  case llvm::LockFileManager::LFS_Error:
864
15
    return llvm::createStringError(std::errc::io_error, "LFS error");
865
866
789
  case llvm::LockFileManager::LFS_Owned:
867
    // We're responsible for building the index ourselves. Do so below.
868
789
    break;
869
870
0
  case llvm::LockFileManager::LFS_Shared:
871
    // Someone else is responsible for building the index. We don't care
872
    // when they finish, so we're done.
873
0
    return llvm::createStringError(std::errc::device_or_resource_busy,
874
0
                                   "someone else is building the index");
875
804
  }
876
877
  // The module index builder.
878
789
  GlobalModuleIndexBuilder Builder(FileMgr, PCHContainerRdr);
879
880
  // Load each of the module files.
881
789
  std::error_code EC;
882
789
  for (llvm::sys::fs::directory_iterator D(Path, EC), DEnd;
883
4.52k
       D != DEnd && 
!EC3.74k
;
884
3.74k
       
D.increment(EC)3.73k
) {
885
    // If this isn't a module file, we don't care.
886
3.74k
    if (llvm::sys::path::extension(D->path()) != ".pcm") {
887
      // ... unless it's a .pcm.lock file, which indicates that someone is
888
      // in the process of rebuilding a module. They'll rebuild the index
889
      // at the end of that translation unit, so we don't have to.
890
1.74k
      if (llvm::sys::path::extension(D->path()) == ".pcm.lock")
891
0
        return llvm::createStringError(std::errc::device_or_resource_busy,
892
0
                                       "someone else is building the index");
893
894
1.74k
      continue;
895
1.74k
    }
896
897
    // If we can't find the module file, skip it.
898
1.99k
    auto ModuleFile = FileMgr.getFile(D->path());
899
1.99k
    if (!ModuleFile)
900
0
      continue;
901
902
    // Load this module file.
903
1.99k
    if (llvm::Error Err = Builder.loadModuleFile(*ModuleFile))
904
8
      return Err;
905
1.99k
  }
906
907
  // The output buffer, into which the global index will be written.
908
781
  SmallString<16> OutputBuffer;
909
781
  {
910
781
    llvm::BitstreamWriter OutputStream(OutputBuffer);
911
781
    if (Builder.writeIndex(OutputStream))
912
3
      return llvm::createStringError(std::errc::io_error,
913
3
                                     "failed writing index");
914
781
  }
915
916
778
  return llvm::writeFileAtomically((IndexPath + "-%%%%%%%%").str(), IndexPath,
917
778
                                   OutputBuffer);
918
781
}
919
920
namespace {
921
  class GlobalIndexIdentifierIterator : public IdentifierIterator {
922
    /// The current position within the identifier lookup table.
923
    IdentifierIndexTable::key_iterator Current;
924
925
    /// The end position within the identifier lookup table.
926
    IdentifierIndexTable::key_iterator End;
927
928
  public:
929
16
    explicit GlobalIndexIdentifierIterator(IdentifierIndexTable &Idx) {
930
16
      Current = Idx.key_begin();
931
16
      End = Idx.key_end();
932
16
    }
933
934
4.74k
    StringRef Next() override {
935
4.74k
      if (Current == End)
936
16
        return StringRef();
937
938
4.73k
      StringRef Result = *Current;
939
4.73k
      ++Current;
940
4.73k
      return Result;
941
4.74k
    }
942
  };
943
}
944
945
16
IdentifierIterator *GlobalModuleIndex::createIdentifierIterator() const {
946
16
  IdentifierIndexTable &Table =
947
16
    *static_cast<IdentifierIndexTable *>(IdentifierIndex);
948
16
  return new GlobalIndexIdentifierIterator(Table);
949
16
}