Coverage Report

Created: 2019-07-24 05:18

/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/tools/clang/lib/Serialization/GlobalModuleIndex.cpp
Line
Count
Source (jump to first uncovered line)
1
//===--- GlobalModuleIndex.cpp - Global Module Index ------------*- C++ -*-===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// This file implements the GlobalModuleIndex class.
10
//
11
//===----------------------------------------------------------------------===//
12
13
14
#include "ASTReaderInternals.h"
15
#include "clang/Basic/FileManager.h"
16
#include "clang/Lex/HeaderSearch.h"
17
#include "clang/Serialization/ASTBitCodes.h"
18
#include "clang/Serialization/GlobalModuleIndex.h"
19
#include "clang/Serialization/Module.h"
20
#include "clang/Serialization/PCHContainerOperations.h"
21
#include "llvm/ADT/DenseMap.h"
22
#include "llvm/ADT/MapVector.h"
23
#include "llvm/ADT/SmallString.h"
24
#include "llvm/Bitstream/BitstreamReader.h"
25
#include "llvm/Bitstream/BitstreamWriter.h"
26
#include "llvm/Support/DJB.h"
27
#include "llvm/Support/FileSystem.h"
28
#include "llvm/Support/LockFileManager.h"
29
#include "llvm/Support/MemoryBuffer.h"
30
#include "llvm/Support/OnDiskHashTable.h"
31
#include "llvm/Support/Path.h"
32
#include "llvm/Support/TimeProfiler.h"
33
#include <cstdio>
34
using namespace clang;
35
using namespace serialization;
36
37
//----------------------------------------------------------------------------//
38
// Shared constants
39
//----------------------------------------------------------------------------//
40
namespace {
41
  enum {
42
    /// The block containing the index.
43
    GLOBAL_INDEX_BLOCK_ID = llvm::bitc::FIRST_APPLICATION_BLOCKID
44
  };
45
46
  /// Describes the record types in the index.
47
  enum IndexRecordTypes {
48
    /// Contains version information and potentially other metadata,
49
    /// used to determine if we can read this global index file.
50
    INDEX_METADATA,
51
    /// Describes a module, including its file name and dependencies.
52
    MODULE,
53
    /// The index for identifiers.
54
    IDENTIFIER_INDEX
55
  };
56
}
57
58
/// The name of the global index file.
59
static const char * const IndexFileName = "modules.idx";
60
61
/// The global index file version.
62
static const unsigned CurrentVersion = 1;
63
64
//----------------------------------------------------------------------------//
65
// Global module index reader.
66
//----------------------------------------------------------------------------//
67
68
namespace {
69
70
/// Trait used to read the identifier index from the on-disk hash
71
/// table.
72
class IdentifierIndexReaderTrait {
73
public:
74
  typedef StringRef external_key_type;
75
  typedef StringRef internal_key_type;
76
  typedef SmallVector<unsigned, 2> data_type;
77
  typedef unsigned hash_value_type;
78
  typedef unsigned offset_type;
79
80
7.09k
  static bool EqualKey(const internal_key_type& a, const internal_key_type& b) {
81
7.09k
    return a == b;
82
7.09k
  }
83
84
299k
  static hash_value_type ComputeHash(const internal_key_type& a) {
85
299k
    return llvm::djbHash(a);
86
299k
  }
87
88
  static std::pair<unsigned, unsigned>
89
191k
  ReadKeyDataLength(const unsigned char*& d) {
90
191k
    using namespace llvm::support;
91
191k
    unsigned KeyLen = endian::readNext<uint16_t, little, unaligned>(d);
92
191k
    unsigned DataLen = endian::readNext<uint16_t, little, unaligned>(d);
93
191k
    return std::make_pair(KeyLen, DataLen);
94
191k
  }
95
96
  static const internal_key_type&
97
299k
  GetInternalKey(const external_key_type& x) { return x; }
98
99
  static const external_key_type&
100
3.76k
  GetExternalKey(const internal_key_type& x) { return x; }
101
102
10.8k
  static internal_key_type ReadKey(const unsigned char* d, unsigned n) {
103
10.8k
    return StringRef((const char *)d, n);
104
10.8k
  }
105
106
  static data_type ReadData(const internal_key_type& k,
107
                            const unsigned char* d,
108
7.09k
                            unsigned DataLen) {
109
7.09k
    using namespace llvm::support;
110
7.09k
111
7.09k
    data_type Result;
112
10.2k
    while (DataLen > 0) {
113
3.12k
      unsigned ID = endian::readNext<uint32_t, little, unaligned>(d);
114
3.12k
      Result.push_back(ID);
115
3.12k
      DataLen -= 4;
116
3.12k
    }
117
7.09k
118
7.09k
    return Result;
119
7.09k
  }
120
};
121
122
typedef llvm::OnDiskIterableChainedHashTable<IdentifierIndexReaderTrait>
123
    IdentifierIndexTable;
124
125
}
126
127
GlobalModuleIndex::GlobalModuleIndex(std::unique_ptr<llvm::MemoryBuffer> Buffer,
128
                                     llvm::BitstreamCursor Cursor)
129
    : Buffer(std::move(Buffer)), IdentifierIndex(), NumIdentifierLookups(),
130
185
      NumIdentifierLookupHits() {
131
185
  auto Fail = [&Buffer](llvm::Error &&Err) {
132
0
    report_fatal_error("Module index '" + Buffer->getBufferIdentifier() +
133
0
                       "' failed: " + toString(std::move(Err)));
134
0
  };
135
185
136
185
  llvm::TimeTraceScope TimeScope("Module LoadIndex", StringRef(""));
137
185
  // Read the global index.
138
185
  bool InGlobalIndexBlock = false;
139
185
  bool Done = false;
140
1.45k
  while (!Done) {
141
1.26k
    llvm::BitstreamEntry Entry;
142
1.26k
    if (Expected<llvm::BitstreamEntry> Res = Cursor.advance())
143
1.26k
      Entry = Res.get();
144
0
    else
145
0
      Fail(Res.takeError());
146
1.26k
147
1.26k
    switch (Entry.Kind) {
148
1.26k
    case llvm::BitstreamEntry::Error:
149
0
      return;
150
1.26k
151
1.26k
    case llvm::BitstreamEntry::EndBlock:
152
185
      if (InGlobalIndexBlock) {
153
185
        InGlobalIndexBlock = false;
154
185
        Done = true;
155
185
        continue;
156
185
      }
157
0
      return;
158
0
159
0
160
712
    case llvm::BitstreamEntry::Record:
161
712
      // Entries in the global index block are handled below.
162
712
      if (InGlobalIndexBlock)
163
712
        break;
164
0
165
0
      return;
166
0
167
370
    case llvm::BitstreamEntry::SubBlock:
168
370
      if (!InGlobalIndexBlock && Entry.ID == GLOBAL_INDEX_BLOCK_ID) {
169
185
        if (llvm::Error Err = Cursor.EnterSubBlock(GLOBAL_INDEX_BLOCK_ID))
170
0
          Fail(std::move(Err));
171
185
        InGlobalIndexBlock = true;
172
185
      } else if (llvm::Error Err = Cursor.SkipBlock())
173
0
        Fail(std::move(Err));
174
370
      continue;
175
712
    }
176
712
177
712
    SmallVector<uint64_t, 64> Record;
178
712
    StringRef Blob;
179
712
    Expected<unsigned> MaybeIndexRecord =
180
712
        Cursor.readRecord(Entry.ID, Record, &Blob);
181
712
    if (!MaybeIndexRecord)
182
0
      Fail(MaybeIndexRecord.takeError());
183
712
    IndexRecordTypes IndexRecord =
184
712
        static_cast<IndexRecordTypes>(MaybeIndexRecord.get());
185
712
    switch (IndexRecord) {
186
712
    case INDEX_METADATA:
187
185
      // Make sure that the version matches.
188
185
      if (Record.size() < 1 || Record[0] != CurrentVersion)
189
0
        return;
190
185
      break;
191
185
192
342
    case MODULE: {
193
342
      unsigned Idx = 0;
194
342
      unsigned ID = Record[Idx++];
195
342
196
342
      // Make room for this module's information.
197
342
      if (ID == Modules.size())
198
342
        Modules.push_back(ModuleInfo());
199
0
      else
200
0
        Modules.resize(ID + 1);
201
342
202
342
      // Size/modification time for this module file at the time the
203
342
      // global index was built.
204
342
      Modules[ID].Size = Record[Idx++];
205
342
      Modules[ID].ModTime = Record[Idx++];
206
342
207
342
      // File name.
208
342
      unsigned NameLen = Record[Idx++];
209
342
      Modules[ID].FileName.assign(Record.begin() + Idx,
210
342
                                  Record.begin() + Idx + NameLen);
211
342
      Idx += NameLen;
212
342
213
342
      // Dependencies
214
342
      unsigned NumDeps = Record[Idx++];
215
342
      Modules[ID].Dependencies.insert(Modules[ID].Dependencies.end(),
216
342
                                      Record.begin() + Idx,
217
342
                                      Record.begin() + Idx + NumDeps);
218
342
      Idx += NumDeps;
219
342
220
342
      // Make sure we're at the end of the record.
221
342
      assert(Idx == Record.size() && "More module info?");
222
342
223
342
      // Record this module as an unresolved module.
224
342
      // FIXME: this doesn't work correctly for module names containing path
225
342
      // separators.
226
342
      StringRef ModuleName = llvm::sys::path::stem(Modules[ID].FileName);
227
342
      // Remove the -<hash of ModuleMapPath>
228
342
      ModuleName = ModuleName.rsplit('-').first;
229
342
      UnresolvedModules[ModuleName] = ID;
230
342
      break;
231
185
    }
232
185
233
185
    case IDENTIFIER_INDEX:
234
185
      // Wire up the identifier index.
235
185
      if (Record[0]) {
236
185
        IdentifierIndex = IdentifierIndexTable::Create(
237
185
            (const unsigned char *)Blob.data() + Record[0],
238
185
            (const unsigned char *)Blob.data() + sizeof(uint32_t),
239
185
            (const unsigned char *)Blob.data(), IdentifierIndexReaderTrait());
240
185
      }
241
185
      break;
242
712
    }
243
712
  }
244
185
}
245
246
181
GlobalModuleIndex::~GlobalModuleIndex() {
247
181
  delete static_cast<IdentifierIndexTable *>(IdentifierIndex);
248
181
}
249
250
std::pair<GlobalModuleIndex *, llvm::Error>
251
969
GlobalModuleIndex::readIndex(StringRef Path) {
252
969
  // Load the index file, if it's there.
253
969
  llvm::SmallString<128> IndexPath;
254
969
  IndexPath += Path;
255
969
  llvm::sys::path::append(IndexPath, IndexFileName);
256
969
257
969
  llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> BufferOrErr =
258
969
      llvm::MemoryBuffer::getFile(IndexPath.c_str());
259
969
  if (!BufferOrErr)
260
784
    return std::make_pair(nullptr,
261
784
                          llvm::errorCodeToError(BufferOrErr.getError()));
262
185
  std::unique_ptr<llvm::MemoryBuffer> Buffer = std::move(BufferOrErr.get());
263
185
264
185
  /// The main bitstream cursor for the main block.
265
185
  llvm::BitstreamCursor Cursor(*Buffer);
266
185
267
185
  // Sniff for the signature.
268
740
  for (unsigned char C : {'B', 'C', 'G', 'I'}) {
269
740
    if (Expected<llvm::SimpleBitstreamCursor::word_t> Res = Cursor.Read(8)) {
270
740
      if (Res.get() != C)
271
0
        return std::make_pair(
272
0
            nullptr, llvm::createStringError(std::errc::illegal_byte_sequence,
273
0
                                             "expected signature BCGI"));
274
0
    } else
275
0
      return std::make_pair(nullptr, Res.takeError());
276
740
  }
277
185
278
185
  return std::make_pair(new GlobalModuleIndex(std::move(Buffer), Cursor),
279
185
                        llvm::Error::success());
280
185
}
281
282
void
283
0
GlobalModuleIndex::getKnownModules(SmallVectorImpl<ModuleFile *> &ModuleFiles) {
284
0
  ModuleFiles.clear();
285
0
  for (unsigned I = 0, N = Modules.size(); I != N; ++I) {
286
0
    if (ModuleFile *MF = Modules[I].File)
287
0
      ModuleFiles.push_back(MF);
288
0
  }
289
0
}
290
291
void GlobalModuleIndex::getModuleDependencies(
292
       ModuleFile *File,
293
0
       SmallVectorImpl<ModuleFile *> &Dependencies) {
294
0
  // Look for information about this module file.
295
0
  llvm::DenseMap<ModuleFile *, unsigned>::iterator Known
296
0
    = ModulesByFile.find(File);
297
0
  if (Known == ModulesByFile.end())
298
0
    return;
299
0
300
0
  // Record dependencies.
301
0
  Dependencies.clear();
302
0
  ArrayRef<unsigned> StoredDependencies = Modules[Known->second].Dependencies;
303
0
  for (unsigned I = 0, N = StoredDependencies.size(); I != N; ++I) {
304
0
    if (ModuleFile *MF = Modules[I].File)
305
0
      Dependencies.push_back(MF);
306
0
  }
307
0
}
308
309
299k
bool GlobalModuleIndex::lookupIdentifier(StringRef Name, HitSet &Hits) {
310
299k
  Hits.clear();
311
299k
312
299k
  // If there's no identifier index, there is nothing we can do.
313
299k
  if (!IdentifierIndex)
314
0
    return false;
315
299k
316
299k
  // Look into the identifier index.
317
299k
  ++NumIdentifierLookups;
318
299k
  IdentifierIndexTable &Table
319
299k
    = *static_cast<IdentifierIndexTable *>(IdentifierIndex);
320
299k
  IdentifierIndexTable::iterator Known = Table.find(Name);
321
299k
  if (Known == Table.end()) {
322
292k
    return true;
323
292k
  }
324
7.09k
325
7.09k
  SmallVector<unsigned, 2> ModuleIDs = *Known;
326
10.2k
  for (unsigned I = 0, N = ModuleIDs.size(); I != N; 
++I3.12k
) {
327
3.12k
    if (ModuleFile *MF = Modules[ModuleIDs[I]].File)
328
2.11k
      Hits.insert(MF);
329
3.12k
  }
330
7.09k
331
7.09k
  ++NumIdentifierLookupHits;
332
7.09k
  return true;
333
7.09k
}
334
335
357
bool GlobalModuleIndex::loadedModuleFile(ModuleFile *File) {
336
357
  // Look for the module in the global module index based on the module name.
337
357
  StringRef Name = File->ModuleName;
338
357
  llvm::StringMap<unsigned>::iterator Known = UnresolvedModules.find(Name);
339
357
  if (Known == UnresolvedModules.end()) {
340
94
    return true;
341
94
  }
342
263
343
263
  // Rectify this module with the global module index.
344
263
  ModuleInfo &Info = Modules[Known->second];
345
263
346
263
  //  If the size and modification time match what we expected, record this
347
263
  // module file.
348
263
  bool Failed = true;
349
263
  if (File->File->getSize() == Info.Size &&
350
263
      
File->File->getModificationTime() == Info.ModTime239
) {
351
228
    Info.File = File;
352
228
    ModulesByFile[File] = Known->second;
353
228
354
228
    Failed = false;
355
228
  }
356
263
357
263
  // One way or another, we have resolved this module file.
358
263
  UnresolvedModules.erase(Known);
359
263
  return Failed;
360
263
}
361
362
1
void GlobalModuleIndex::printStats() {
363
1
  std::fprintf(stderr, "*** Global Module Index Statistics:\n");
364
1
  if (NumIdentifierLookups) {
365
1
    fprintf(stderr, "  %u / %u identifier lookups succeeded (%f%%)\n",
366
1
            NumIdentifierLookupHits, NumIdentifierLookups,
367
1
            (double)NumIdentifierLookupHits*100.0/NumIdentifierLookups);
368
1
  }
369
1
  std::fprintf(stderr, "\n");
370
1
}
371
372
0
LLVM_DUMP_METHOD void GlobalModuleIndex::dump() {
373
0
  llvm::errs() << "*** Global Module Index Dump:\n";
374
0
  llvm::errs() << "Module files:\n";
375
0
  for (auto &MI : Modules) {
376
0
    llvm::errs() << "** " << MI.FileName << "\n";
377
0
    if (MI.File)
378
0
      MI.File->dump();
379
0
    else
380
0
      llvm::errs() << "\n";
381
0
  }
382
0
  llvm::errs() << "\n";
383
0
}
384
385
//----------------------------------------------------------------------------//
386
// Global module index writer.
387
//----------------------------------------------------------------------------//
388
389
namespace {
390
  /// Provides information about a specific module file.
391
  struct ModuleFileInfo {
392
    /// The numberic ID for this module file.
393
    unsigned ID;
394
395
    /// The set of modules on which this module depends. Each entry is
396
    /// a module ID.
397
    SmallVector<unsigned, 4> Dependencies;
398
    ASTFileSignature Signature;
399
  };
400
401
  struct ImportedModuleFileInfo {
402
    off_t StoredSize;
403
    time_t StoredModTime;
404
    ASTFileSignature StoredSignature;
405
    ImportedModuleFileInfo(off_t Size, time_t ModTime, ASTFileSignature Sig)
406
503
        : StoredSize(Size), StoredModTime(ModTime), StoredSignature(Sig) {}
407
  };
408
409
  /// Builder that generates the global module index file.
410
  class GlobalModuleIndexBuilder {
411
    FileManager &FileMgr;
412
    const PCHContainerReader &PCHContainerRdr;
413
414
    /// Mapping from files to module file information.
415
    typedef llvm::MapVector<const FileEntry *, ModuleFileInfo> ModuleFilesMap;
416
417
    /// Information about each of the known module files.
418
    ModuleFilesMap ModuleFiles;
419
420
    /// Mapping from the imported module file to the imported
421
    /// information.
422
    typedef std::multimap<const FileEntry *, ImportedModuleFileInfo>
423
        ImportedModuleFilesMap;
424
425
    /// Information about each importing of a module file.
426
    ImportedModuleFilesMap ImportedModuleFiles;
427
428
    /// Mapping from identifiers to the list of module file IDs that
429
    /// consider this identifier to be interesting.
430
    typedef llvm::StringMap<SmallVector<unsigned, 2> > InterestingIdentifierMap;
431
432
    /// A mapping from all interesting identifiers to the set of module
433
    /// files in which those identifiers are considered interesting.
434
    InterestingIdentifierMap InterestingIdentifiers;
435
436
    /// Write the block-info block for the global module index file.
437
    void emitBlockInfoBlock(llvm::BitstreamWriter &Stream);
438
439
    /// Retrieve the module file information for the given file.
440
4.55k
    ModuleFileInfo &getModuleFileInfo(const FileEntry *File) {
441
4.55k
      llvm::MapVector<const FileEntry *, ModuleFileInfo>::iterator Known
442
4.55k
        = ModuleFiles.find(File);
443
4.55k
      if (Known != ModuleFiles.end())
444
3.25k
        return Known->second;
445
1.29k
446
1.29k
      unsigned NewID = ModuleFiles.size();
447
1.29k
      ModuleFileInfo &Info = ModuleFiles[File];
448
1.29k
      Info.ID = NewID;
449
1.29k
      return Info;
450
1.29k
    }
451
452
  public:
453
    explicit GlobalModuleIndexBuilder(
454
        FileManager &FileMgr, const PCHContainerReader &PCHContainerRdr)
455
631
        : FileMgr(FileMgr), PCHContainerRdr(PCHContainerRdr) {}
456
457
    /// Load the contents of the given module file into the builder.
458
    llvm::Error loadModuleFile(const FileEntry *File);
459
460
    /// Write the index to the given bitstream.
461
    /// \returns true if an error occurred, false otherwise.
462
    bool writeIndex(llvm::BitstreamWriter &Stream);
463
  };
464
}
465
466
static void emitBlockID(unsigned ID, const char *Name,
467
                        llvm::BitstreamWriter &Stream,
468
621
                        SmallVectorImpl<uint64_t> &Record) {
469
621
  Record.clear();
470
621
  Record.push_back(ID);
471
621
  Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_SETBID, Record);
472
621
473
621
  // Emit the block name if present.
474
621
  if (!Name || Name[0] == 0) 
return0
;
475
621
  Record.clear();
476
11.7k
  while (*Name)
477
11.1k
    Record.push_back(*Name++);
478
621
  Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_BLOCKNAME, Record);
479
621
}
480
481
static void emitRecordID(unsigned ID, const char *Name,
482
                         llvm::BitstreamWriter &Stream,
483
1.86k
                         SmallVectorImpl<uint64_t> &Record) {
484
1.86k
  Record.clear();
485
1.86k
  Record.push_back(ID);
486
24.2k
  while (*Name)
487
22.3k
    Record.push_back(*Name++);
488
1.86k
  Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_SETRECORDNAME, Record);
489
1.86k
}
490
491
void
492
621
GlobalModuleIndexBuilder::emitBlockInfoBlock(llvm::BitstreamWriter &Stream) {
493
621
  SmallVector<uint64_t, 64> Record;
494
621
  Stream.EnterBlockInfoBlock();
495
621
496
621
#define BLOCK(X) emitBlockID(X ## _ID, #X, Stream, Record)
497
1.86k
#define RECORD(X) emitRecordID(X, #X, Stream, Record)
498
621
  BLOCK(GLOBAL_INDEX_BLOCK);
499
621
  RECORD(INDEX_METADATA);
500
621
  RECORD(MODULE);
501
621
  RECORD(IDENTIFIER_INDEX);
502
621
#undef RECORD
503
621
#undef BLOCK
504
621
505
621
  Stream.ExitBlock();
506
621
}
507
508
namespace {
509
  class InterestingASTIdentifierLookupTrait
510
    : public serialization::reader::ASTIdentifierLookupTraitBase {
511
512
  public:
513
    /// The identifier and whether it is "interesting".
514
    typedef std::pair<StringRef, bool> data_type;
515
516
    data_type ReadData(const internal_key_type& k,
517
                       const unsigned char* d,
518
107k
                       unsigned DataLen) {
519
107k
      // The first bit indicates whether this identifier is interesting.
520
107k
      // That's all we care about.
521
107k
      using namespace llvm::support;
522
107k
      unsigned RawID = endian::readNext<uint32_t, little, unaligned>(d);
523
107k
      bool IsInteresting = RawID & 0x01;
524
107k
      return std::make_pair(k, IsInteresting);
525
107k
    }
526
  };
527
}
528
529
1.29k
llvm::Error GlobalModuleIndexBuilder::loadModuleFile(const FileEntry *File) {
530
1.29k
  // Open the module file.
531
1.29k
532
1.29k
  auto Buffer = FileMgr.getBufferForFile(File, /*isVolatile=*/true);
533
1.29k
  if (!Buffer)
534
0
    return llvm::createStringError(Buffer.getError(),
535
0
                                   "failed getting buffer for module file");
536
1.29k
537
1.29k
  // Initialize the input stream
538
1.29k
  llvm::BitstreamCursor InStream(PCHContainerRdr.ExtractPCH(**Buffer));
539
1.29k
540
1.29k
  // Sniff for the signature.
541
1.29k
  for (unsigned char C : {'C', 'P', 'C', 'H'})
542
5.16k
    if (Expected<llvm::SimpleBitstreamCursor::word_t> Res = InStream.Read(8)) {
543
5.16k
      if (Res.get() != C)
544
0
        return llvm::createStringError(std::errc::illegal_byte_sequence,
545
0
                                       "expected signature CPCH");
546
0
    } else
547
0
      return Res.takeError();
548
1.29k
549
1.29k
  // Record this module file and assign it a unique ID (if it doesn't have
550
1.29k
  // one already).
551
1.29k
  unsigned ID = getModuleFileInfo(File).ID;
552
1.29k
553
1.29k
  // Search for the blocks and records we care about.
554
1.29k
  enum { Other, ControlBlock, ASTBlock, DiagnosticOptionsBlock } State = Other;
555
1.29k
  bool Done = false;
556
59.6k
  while (!Done) {
557
58.3k
    Expected<llvm::BitstreamEntry> MaybeEntry = InStream.advance();
558
58.3k
    if (!MaybeEntry)
559
0
      return MaybeEntry.takeError();
560
58.3k
    llvm::BitstreamEntry Entry = MaybeEntry.get();
561
58.3k
562
58.3k
    switch (Entry.Kind) {
563
58.3k
    case llvm::BitstreamEntry::Error:
564
1.28k
      Done = true;
565
1.28k
      continue;
566
58.3k
567
58.3k
    case llvm::BitstreamEntry::Record:
568
39.0k
      // In the 'other' state, just skip the record. We don't care.
569
39.0k
      if (State == Other) {
570
0
        if (llvm::Expected<unsigned> Skipped = InStream.skipRecord(Entry.ID))
571
0
          continue;
572
0
        else
573
0
          return Skipped.takeError();
574
39.0k
      }
575
39.0k
576
39.0k
      // Handle potentially-interesting records below.
577
39.0k
      break;
578
39.0k
579
39.0k
    case llvm::BitstreamEntry::SubBlock:
580
14.1k
      if (Entry.ID == CONTROL_BLOCK_ID) {
581
1.29k
        if (llvm::Error Err = InStream.EnterSubBlock(CONTROL_BLOCK_ID))
582
0
          return Err;
583
1.29k
584
1.29k
        // Found the control block.
585
1.29k
        State = ControlBlock;
586
1.29k
        continue;
587
1.29k
      }
588
12.8k
589
12.8k
      if (Entry.ID == AST_BLOCK_ID) {
590
1.28k
        if (llvm::Error Err = InStream.EnterSubBlock(AST_BLOCK_ID))
591
0
          return Err;
592
1.28k
593
1.28k
        // Found the AST block.
594
1.28k
        State = ASTBlock;
595
1.28k
        continue;
596
1.28k
      }
597
11.6k
598
11.6k
      if (Entry.ID == UNHASHED_CONTROL_BLOCK_ID) {
599
1.28k
        if (llvm::Error Err = InStream.EnterSubBlock(UNHASHED_CONTROL_BLOCK_ID))
600
0
          return Err;
601
1.28k
602
1.28k
        // Found the Diagnostic Options block.
603
1.28k
        State = DiagnosticOptionsBlock;
604
1.28k
        continue;
605
1.28k
      }
606
10.3k
607
10.3k
      if (llvm::Error Err = InStream.SkipBlock())
608
0
        return Err;
609
10.3k
610
10.3k
      continue;
611
10.3k
612
10.3k
    case llvm::BitstreamEntry::EndBlock:
613
3.84k
      State = Other;
614
3.84k
      continue;
615
39.0k
    }
616
39.0k
617
39.0k
    // Read the given record.
618
39.0k
    SmallVector<uint64_t, 64> Record;
619
39.0k
    StringRef Blob;
620
39.0k
    Expected<unsigned> MaybeCode = InStream.readRecord(Entry.ID, Record, &Blob);
621
39.0k
    if (!MaybeCode)
622
0
      return MaybeCode.takeError();
623
39.0k
    unsigned Code = MaybeCode.get();
624
39.0k
625
39.0k
    // Handle module dependencies.
626
39.0k
    if (State == ControlBlock && 
Code == IMPORTS10.2k
) {
627
1.29k
      // Load each of the imported PCH files.
628
1.29k
      unsigned Idx = 0, N = Record.size();
629
1.79k
      while (Idx < N) {
630
511
        // Read information about the AST file.
631
511
632
511
        // Skip the imported kind
633
511
        ++Idx;
634
511
635
511
        // Skip the import location
636
511
        ++Idx;
637
511
638
511
        // Load stored size/modification time.
639
511
        off_t StoredSize = (off_t)Record[Idx++];
640
511
        time_t StoredModTime = (time_t)Record[Idx++];
641
511
642
511
        // Skip the stored signature.
643
511
        // FIXME: we could read the signature out of the import and validate it.
644
511
        ASTFileSignature StoredSignature = {
645
511
            {{(uint32_t)Record[Idx++], (uint32_t)Record[Idx++],
646
511
              (uint32_t)Record[Idx++], (uint32_t)Record[Idx++],
647
511
              (uint32_t)Record[Idx++]}}};
648
511
649
511
        // Skip the module name (currently this is only used for prebuilt
650
511
        // modules while here we are only dealing with cached).
651
511
        Idx += Record[Idx] + 1;
652
511
653
511
        // Retrieve the imported file name.
654
511
        unsigned Length = Record[Idx++];
655
511
        SmallString<128> ImportedFile(Record.begin() + Idx,
656
511
                                      Record.begin() + Idx + Length);
657
511
        Idx += Length;
658
511
659
511
        // Find the imported module file.
660
511
        const FileEntry *DependsOnFile
661
511
          = FileMgr.getFile(ImportedFile, /*OpenFile=*/false,
662
511
                            /*CacheFailure=*/false);
663
511
664
511
        if (!DependsOnFile)
665
8
          return llvm::createStringError(std::errc::bad_file_descriptor,
666
8
                                         "imported file \"%s\" not found",
667
8
                                         ImportedFile.c_str());
668
503
669
503
        // Save the information in ImportedModuleFileInfo so we can verify after
670
503
        // loading all pcms.
671
503
        ImportedModuleFiles.insert(std::make_pair(
672
503
            DependsOnFile, ImportedModuleFileInfo(StoredSize, StoredModTime,
673
503
                                                  StoredSignature)));
674
503
675
503
        // Record the dependency.
676
503
        unsigned DependsOnID = getModuleFileInfo(DependsOnFile).ID;
677
503
        getModuleFileInfo(File).Dependencies.push_back(DependsOnID);
678
503
      }
679
1.29k
680
1.29k
      
continue1.28k
;
681
37.7k
    }
682
37.7k
683
37.7k
    // Handle the identifier table
684
37.7k
    if (State == ASTBlock && 
Code == IDENTIFIER_TABLE24.9k
&&
Record[0] > 01.28k
) {
685
1.28k
      typedef llvm::OnDiskIterableChainedHashTable<
686
1.28k
          InterestingASTIdentifierLookupTrait> InterestingIdentifierTable;
687
1.28k
      std::unique_ptr<InterestingIdentifierTable> Table(
688
1.28k
          InterestingIdentifierTable::Create(
689
1.28k
              (const unsigned char *)Blob.data() + Record[0],
690
1.28k
              (const unsigned char *)Blob.data() + sizeof(uint32_t),
691
1.28k
              (const unsigned char *)Blob.data()));
692
1.28k
      for (InterestingIdentifierTable::data_iterator D = Table->data_begin(),
693
1.28k
                                                     DEnd = Table->data_end();
694
108k
           D != DEnd; 
++D107k
) {
695
107k
        std::pair<StringRef, bool> Ident = *D;
696
107k
        if (Ident.second)
697
78.6k
          InterestingIdentifiers[Ident.first].push_back(ID);
698
28.4k
        else
699
28.4k
          (void)InterestingIdentifiers[Ident.first];
700
107k
      }
701
1.28k
    }
702
37.7k
703
37.7k
    // Get Signature.
704
37.7k
    if (State == DiagnosticOptionsBlock && 
Code == SIGNATURE3.82k
)
705
1.26k
      getModuleFileInfo(File).Signature = {
706
1.26k
          {{(uint32_t)Record[0], (uint32_t)Record[1], (uint32_t)Record[2],
707
1.26k
            (uint32_t)Record[3], (uint32_t)Record[4]}}};
708
37.7k
709
37.7k
    // We don't care about this record.
710
37.7k
  }
711
1.29k
712
1.29k
  
return llvm::Error::success()1.28k
;
713
1.29k
}
714
715
namespace {
716
717
/// Trait used to generate the identifier index as an on-disk hash
718
/// table.
719
class IdentifierIndexWriterTrait {
720
public:
721
  typedef StringRef key_type;
722
  typedef StringRef key_type_ref;
723
  typedef SmallVector<unsigned, 2> data_type;
724
  typedef const SmallVector<unsigned, 2> &data_type_ref;
725
  typedef unsigned hash_value_type;
726
  typedef unsigned offset_type;
727
728
89.8k
  static hash_value_type ComputeHash(key_type_ref Key) {
729
89.8k
    return llvm::djbHash(Key);
730
89.8k
  }
731
732
  std::pair<unsigned,unsigned>
733
89.8k
  EmitKeyDataLength(raw_ostream& Out, key_type_ref Key, data_type_ref Data) {
734
89.8k
    using namespace llvm::support;
735
89.8k
    endian::Writer LE(Out, little);
736
89.8k
    unsigned KeyLen = Key.size();
737
89.8k
    unsigned DataLen = Data.size() * 4;
738
89.8k
    LE.write<uint16_t>(KeyLen);
739
89.8k
    LE.write<uint16_t>(DataLen);
740
89.8k
    return std::make_pair(KeyLen, DataLen);
741
89.8k
  }
742
743
89.8k
  void EmitKey(raw_ostream& Out, key_type_ref Key, unsigned KeyLen) {
744
89.8k
    Out.write(Key.data(), KeyLen);
745
89.8k
  }
746
747
  void EmitData(raw_ostream& Out, key_type_ref Key, data_type_ref Data,
748
89.8k
                unsigned DataLen) {
749
89.8k
    using namespace llvm::support;
750
168k
    for (unsigned I = 0, N = Data.size(); I != N; 
++I78.4k
)
751
78.4k
      endian::write<uint32_t>(Out, Data[I], little);
752
89.8k
  }
753
};
754
755
}
756
757
623
bool GlobalModuleIndexBuilder::writeIndex(llvm::BitstreamWriter &Stream) {
758
623
  for (auto MapEntry : ImportedModuleFiles) {
759
499
    auto *File = MapEntry.first;
760
499
    ImportedModuleFileInfo &Info = MapEntry.second;
761
499
    if (getModuleFileInfo(File).Signature) {
762
495
      if (getModuleFileInfo(File).Signature != Info.StoredSignature)
763
2
        // Verify Signature.
764
2
        return true;
765
4
    } else if (Info.StoredSize != File->getSize() ||
766
4
               Info.StoredModTime != File->getModificationTime())
767
0
      // Verify Size and ModTime.
768
0
      return true;
769
499
  }
770
623
771
623
  using namespace llvm;
772
621
  llvm::TimeTraceScope TimeScope("Module WriteIndex", StringRef(""));
773
621
774
621
  // Emit the file header.
775
621
  Stream.Emit((unsigned)'B', 8);
776
621
  Stream.Emit((unsigned)'C', 8);
777
621
  Stream.Emit((unsigned)'G', 8);
778
621
  Stream.Emit((unsigned)'I', 8);
779
621
780
621
  // Write the block-info block, which describes the records in this bitcode
781
621
  // file.
782
621
  emitBlockInfoBlock(Stream);
783
621
784
621
  Stream.EnterSubblock(GLOBAL_INDEX_BLOCK_ID, 3);
785
621
786
621
  // Write the metadata.
787
621
  SmallVector<uint64_t, 2> Record;
788
621
  Record.push_back(CurrentVersion);
789
621
  Stream.EmitRecord(INDEX_METADATA, Record);
790
621
791
621
  // Write the set of known module files.
792
621
  for (ModuleFilesMap::iterator M = ModuleFiles.begin(),
793
621
                                MEnd = ModuleFiles.end();
794
1.88k
       M != MEnd; 
++M1.26k
) {
795
1.26k
    Record.clear();
796
1.26k
    Record.push_back(M->second.ID);
797
1.26k
    Record.push_back(M->first->getSize());
798
1.26k
    Record.push_back(M->first->getModificationTime());
799
1.26k
800
1.26k
    // File name
801
1.26k
    StringRef Name(M->first->getName());
802
1.26k
    Record.push_back(Name.size());
803
1.26k
    Record.append(Name.begin(), Name.end());
804
1.26k
805
1.26k
    // Dependencies
806
1.26k
    Record.push_back(M->second.Dependencies.size());
807
1.26k
    Record.append(M->second.Dependencies.begin(), M->second.Dependencies.end());
808
1.26k
    Stream.EmitRecord(MODULE, Record);
809
1.26k
  }
810
621
811
621
  // Write the identifier -> module file mapping.
812
621
  {
813
621
    llvm::OnDiskChainedHashTableGenerator<IdentifierIndexWriterTrait> Generator;
814
621
    IdentifierIndexWriterTrait Trait;
815
621
816
621
    // Populate the hash table.
817
621
    for (InterestingIdentifierMap::iterator I = InterestingIdentifiers.begin(),
818
621
                                            IEnd = InterestingIdentifiers.end();
819
90.4k
         I != IEnd; 
++I89.8k
) {
820
89.8k
      Generator.insert(I->first(), I->second, Trait);
821
89.8k
    }
822
621
823
621
    // Create the on-disk hash table in a buffer.
824
621
    SmallString<4096> IdentifierTable;
825
621
    uint32_t BucketOffset;
826
621
    {
827
621
      using namespace llvm::support;
828
621
      llvm::raw_svector_ostream Out(IdentifierTable);
829
621
      // Make sure that no bucket is at offset 0
830
621
      endian::write<uint32_t>(Out, 0, little);
831
621
      BucketOffset = Generator.Emit(Out, Trait);
832
621
    }
833
621
834
621
    // Create a blob abbreviation
835
621
    auto Abbrev = std::make_shared<BitCodeAbbrev>();
836
621
    Abbrev->Add(BitCodeAbbrevOp(IDENTIFIER_INDEX));
837
621
    Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
838
621
    Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
839
621
    unsigned IDTableAbbrev = Stream.EmitAbbrev(std::move(Abbrev));
840
621
841
621
    // Write the identifier table
842
621
    uint64_t Record[] = {IDENTIFIER_INDEX, BucketOffset};
843
621
    Stream.EmitRecordWithBlob(IDTableAbbrev, Record, IdentifierTable);
844
621
  }
845
621
846
621
  Stream.ExitBlock();
847
621
  return false;
848
623
}
849
850
llvm::Error
851
GlobalModuleIndex::writeIndex(FileManager &FileMgr,
852
                              const PCHContainerReader &PCHContainerRdr,
853
641
                              StringRef Path) {
854
641
  llvm::SmallString<128> IndexPath;
855
641
  IndexPath += Path;
856
641
  llvm::sys::path::append(IndexPath, IndexFileName);
857
641
858
641
  // Coordinate building the global index file with other processes that might
859
641
  // try to do the same.
860
641
  llvm::LockFileManager Locked(IndexPath);
861
641
  switch (Locked) {
862
641
  case llvm::LockFileManager::LFS_Error:
863
10
    return llvm::createStringError(std::errc::io_error, "LFS error");
864
641
865
641
  case llvm::LockFileManager::LFS_Owned:
866
631
    // We're responsible for building the index ourselves. Do so below.
867
631
    break;
868
641
869
641
  case llvm::LockFileManager::LFS_Shared:
870
0
    // Someone else is responsible for building the index. We don't care
871
0
    // when they finish, so we're done.
872
0
    return llvm::createStringError(std::errc::device_or_resource_busy,
873
0
                                   "someone else is building the index");
874
631
  }
875
631
876
631
  // The module index builder.
877
631
  GlobalModuleIndexBuilder Builder(FileMgr, PCHContainerRdr);
878
631
879
631
  // Load each of the module files.
880
631
  std::error_code EC;
881
631
  for (llvm::sys::fs::directory_iterator D(Path, EC), DEnd;
882
3.28k
       D != DEnd && 
!EC2.66k
;
883
2.66k
       
D.increment(EC)2.65k
) {
884
2.66k
    // If this isn't a module file, we don't care.
885
2.66k
    if (llvm::sys::path::extension(D->path()) != ".pcm") {
886
1.37k
      // ... unless it's a .pcm.lock file, which indicates that someone is
887
1.37k
      // in the process of rebuilding a module. They'll rebuild the index
888
1.37k
      // at the end of that translation unit, so we don't have to.
889
1.37k
      if (llvm::sys::path::extension(D->path()) == ".pcm.lock")
890
0
        return llvm::createStringError(std::errc::device_or_resource_busy,
891
0
                                       "someone else is building the index");
892
1.37k
893
1.37k
      continue;
894
1.37k
    }
895
1.29k
896
1.29k
    // If we can't find the module file, skip it.
897
1.29k
    const FileEntry *ModuleFile = FileMgr.getFile(D->path());
898
1.29k
    if (!ModuleFile)
899
0
      continue;
900
1.29k
901
1.29k
    // Load this module file.
902
1.29k
    if (llvm::Error Err = Builder.loadModuleFile(ModuleFile))
903
8
      return Err;
904
1.29k
  }
905
631
906
631
  // The output buffer, into which the global index will be written.
907
631
  SmallVector<char, 16> OutputBuffer;
908
623
  {
909
623
    llvm::BitstreamWriter OutputStream(OutputBuffer);
910
623
    if (Builder.writeIndex(OutputStream))
911
2
      return llvm::createStringError(std::errc::io_error,
912
2
                                     "failed writing index");
913
621
  }
914
621
915
621
  // Write the global index file to a temporary file.
916
621
  llvm::SmallString<128> IndexTmpPath;
917
621
  int TmpFD;
918
621
  if (llvm::sys::fs::createUniqueFile(IndexPath + "-%%%%%%%%", TmpFD,
919
621
                                      IndexTmpPath))
920
0
    return llvm::createStringError(std::errc::io_error,
921
0
                                   "failed creating unique file");
922
621
923
621
  // Open the temporary global index file for output.
924
621
  llvm::raw_fd_ostream Out(TmpFD, true);
925
621
  if (Out.has_error())
926
0
    return llvm::createStringError(Out.error(), "failed outputting to stream");
927
621
928
621
  // Write the index.
929
621
  Out.write(OutputBuffer.data(), OutputBuffer.size());
930
621
  Out.close();
931
621
  if (Out.has_error())
932
0
    return llvm::createStringError(Out.error(), "failed writing to stream");
933
621
934
621
  // Remove the old index file. It isn't relevant any more.
935
621
  llvm::sys::fs::remove(IndexPath);
936
621
937
621
  // Rename the newly-written index file to the proper name.
938
621
  if (std::error_code Err = llvm::sys::fs::rename(IndexTmpPath, IndexPath)) {
939
0
    // Remove the file on failure, don't check whether removal succeeded.
940
0
    llvm::sys::fs::remove(IndexTmpPath);
941
0
    return llvm::createStringError(Err, "failed renaming file \"%s\" to \"%s\"",
942
0
                                   IndexTmpPath.c_str(), IndexPath.c_str());
943
0
  }
944
621
945
621
  return llvm::Error::success();
946
621
}
947
948
namespace {
949
  class GlobalIndexIdentifierIterator : public IdentifierIterator {
950
    /// The current position within the identifier lookup table.
951
    IdentifierIndexTable::key_iterator Current;
952
953
    /// The end position within the identifier lookup table.
954
    IdentifierIndexTable::key_iterator End;
955
956
  public:
957
14
    explicit GlobalIndexIdentifierIterator(IdentifierIndexTable &Idx) {
958
14
      Current = Idx.key_begin();
959
14
      End = Idx.key_end();
960
14
    }
961
962
3.78k
    StringRef Next() override {
963
3.78k
      if (Current == End)
964
14
        return StringRef();
965
3.76k
966
3.76k
      StringRef Result = *Current;
967
3.76k
      ++Current;
968
3.76k
      return Result;
969
3.76k
    }
970
  };
971
}
972
973
14
IdentifierIterator *GlobalModuleIndex::createIdentifierIterator() const {
974
14
  IdentifierIndexTable &Table =
975
14
    *static_cast<IdentifierIndexTable *>(IdentifierIndex);
976
14
  return new GlobalIndexIdentifierIterator(Table);
977
14
}