Coverage Report

Created: 2022-01-22 13:19

/Users/buildslave/jenkins/workspace/coverage/llvm-project/clang/lib/Lex/DependencyDirectivesSourceMinimizer.cpp
Line
Count
Source (jump to first uncovered line)
1
//===- DependencyDirectivesSourceMinimizer.cpp -  -------------------------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
///
9
/// \file
10
/// This is the implementation for minimizing header and source files to the
11
/// minimum necessary preprocessor directives for evaluating includes. It
12
/// reduces the source down to #define, #include, #import, @import, and any
13
/// conditional preprocessor logic that contains one of those.
14
///
15
//===----------------------------------------------------------------------===//
16
17
#include "clang/Lex/DependencyDirectivesSourceMinimizer.h"
18
#include "clang/Basic/CharInfo.h"
19
#include "clang/Basic/Diagnostic.h"
20
#include "clang/Lex/LexDiagnostic.h"
21
#include "llvm/ADT/StringMap.h"
22
#include "llvm/ADT/StringSwitch.h"
23
#include "llvm/Support/MemoryBuffer.h"
24
25
using namespace llvm;
26
using namespace clang;
27
using namespace clang::minimize_source_to_dependency_directives;
28
29
namespace {
30
31
struct Minimizer {
32
  /// Minimized output.
33
  SmallVectorImpl<char> &Out;
34
  /// The known tokens encountered during the minimization.
35
  SmallVectorImpl<Token> &Tokens;
36
37
  Minimizer(SmallVectorImpl<char> &Out, SmallVectorImpl<Token> &Tokens,
38
            StringRef Input, DiagnosticsEngine *Diags,
39
            SourceLocation InputSourceLoc)
40
      : Out(Out), Tokens(Tokens), Input(Input), Diags(Diags),
41
3.02k
        InputSourceLoc(InputSourceLoc) {}
42
43
  /// Lex the provided source and emit the minimized output.
44
  ///
45
  /// \returns True on error.
46
  bool minimize();
47
48
private:
49
  struct IdInfo {
50
    const char *Last;
51
    StringRef Name;
52
  };
53
54
  /// Lex an identifier.
55
  ///
56
  /// \pre First points at a valid identifier head.
57
  LLVM_NODISCARD IdInfo lexIdentifier(const char *First, const char *const End);
58
  LLVM_NODISCARD bool isNextIdentifier(StringRef Id, const char *&First,
59
                                       const char *const End);
60
  LLVM_NODISCARD bool minimizeImpl(const char *First, const char *const End);
61
  LLVM_NODISCARD bool lexPPLine(const char *&First, const char *const End);
62
  LLVM_NODISCARD bool lexAt(const char *&First, const char *const End);
63
  LLVM_NODISCARD bool lexModule(const char *&First, const char *const End);
64
  LLVM_NODISCARD bool lexDefine(const char *&First, const char *const End);
65
  LLVM_NODISCARD bool lexPragma(const char *&First, const char *const End);
66
  LLVM_NODISCARD bool lexEndif(const char *&First, const char *const End);
67
  LLVM_NODISCARD bool lexDefault(TokenKind Kind, StringRef Directive,
68
                                 const char *&First, const char *const End);
69
211k
  Token &makeToken(TokenKind K) {
70
211k
    Tokens.emplace_back(K, Out.size());
71
211k
    return Tokens.back();
72
211k
  }
73
11.3k
  void popToken() {
74
11.3k
    Out.resize(Tokens.back().Offset);
75
11.3k
    Tokens.pop_back();
76
11.3k
  }
77
129k
  TokenKind top() const { return Tokens.empty() ? 
pp_none0
: Tokens.back().K; }
78
79
567k
  Minimizer &put(char Byte) {
80
567k
    Out.push_back(Byte);
81
567k
    return *this;
82
567k
  }
83
320k
  Minimizer &append(StringRef S) { return append(S.begin(), S.end()); }
84
540k
  Minimizer &append(const char *First, const char *Last) {
85
540k
    Out.append(First, Last);
86
540k
    return *this;
87
540k
  }
88
89
  void printToNewline(const char *&First, const char *const End);
90
  void printAdjacentModuleNameParts(const char *&First, const char *const End);
91
  LLVM_NODISCARD bool printAtImportBody(const char *&First,
92
                                        const char *const End);
93
  void printDirectiveBody(const char *&First, const char *const End);
94
  void printAdjacentMacroArgs(const char *&First, const char *const End);
95
  LLVM_NODISCARD bool printMacroArgs(const char *&First, const char *const End);
96
97
  /// Reports a diagnostic if the diagnostic engine is provided. Always returns
98
  /// true at the end.
99
  bool reportError(const char *CurPtr, unsigned Err);
100
101
  StringMap<char> SplitIds;
102
  StringRef Input;
103
  DiagnosticsEngine *Diags;
104
  SourceLocation InputSourceLoc;
105
};
106
107
} // end anonymous namespace
108
109
8
bool Minimizer::reportError(const char *CurPtr, unsigned Err) {
110
8
  if (!Diags)
111
5
    return true;
112
3
  assert(CurPtr >= Input.data() && "invalid buffer ptr");
113
0
  Diags->Report(InputSourceLoc.getLocWithOffset(CurPtr - Input.data()), Err);
114
3
  return true;
115
8
}
116
117
1.45M
static void skipOverSpaces(const char *&First, const char *const End) {
118
3.12M
  while (First != End && 
isHorizontalWhitespace(*First)3.12M
)
119
1.67M
    ++First;
120
1.45M
}
121
122
LLVM_NODISCARD static bool isRawStringLiteral(const char *First,
123
25.0k
                                              const char *Current) {
124
25.0k
  assert(First <= Current);
125
126
  // Check if we can even back up.
127
25.0k
  if (*Current != '"' || 
First == Current13.4k
)
128
14.9k
    return false;
129
130
  // Check for an "R".
131
10.1k
  --Current;
132
10.1k
  if (*Current != 'R')
133
10.1k
    return false;
134
2
  if (First == Current || !isAsciiIdentifierContinue(*--Current))
135
3
    return true;
136
137
  // Check for a prefix of "u", "U", or "L".
138
18.4E
  if (*Current == 'u' || 
*Current == 'U'0
||
*Current == 'L'0
)
139
0
    return First == Current || !isAsciiIdentifierContinue(*--Current);
140
141
  // Check for a prefix of "u8".
142
18.4E
  if (*Current != '8' || 
First == Current0
||
*Current-- != 'u'0
)
143
0
    return false;
144
18.4E
  return First == Current || 
!isAsciiIdentifierContinue(*--Current)0
;
145
18.4E
}
146
147
3
static void skipRawString(const char *&First, const char *const End) {
148
3
  assert(First[0] == '"');
149
0
  assert(First[-1] == 'R');
150
151
0
  const char *Last = ++First;
152
6
  while (Last != End && *Last != '(')
153
3
    ++Last;
154
3
  if (Last == End) {
155
0
    First = Last; // Hit the end... just give up.
156
0
    return;
157
0
  }
158
159
3
  StringRef Terminator(First, Last - First);
160
3
  for (;;) {
161
    // Move First to just past the next ")".
162
3
    First = Last;
163
46
    while (First != End && *First != ')')
164
43
      ++First;
165
3
    if (First == End)
166
0
      return;
167
3
    ++First;
168
169
    // Look ahead for the terminator sequence.
170
3
    Last = First;
171
6
    while (Last != End && size_t(Last - First) < Terminator.size() &&
172
6
           
Terminator[Last - First] == *Last3
)
173
3
      ++Last;
174
175
    // Check if we hit it (or the end of the file).
176
3
    if (Last == End) {
177
0
      First = Last;
178
0
      return;
179
0
    }
180
3
    if (size_t(Last - First) < Terminator.size())
181
0
      continue;
182
3
    if (*Last != '"')
183
0
      continue;
184
3
    First = Last + 1;
185
3
    return;
186
3
  }
187
3
}
188
189
// Returns the length of EOL, either 0 (no end-of-line), 1 (\n) or 2 (\r\n)
190
973k
static unsigned isEOL(const char *First, const char *const End) {
191
973k
  if (First == End)
192
0
    return 0;
193
973k
  if (End - First > 1 && 
isVerticalWhitespace(First[0])970k
&&
194
973k
      
isVerticalWhitespace(First[1])613k
&&
First[0] != First[1]84.0k
)
195
20
    return 2;
196
973k
  return !!isVerticalWhitespace(First[0]);
197
973k
}
198
199
25.0k
static void skipString(const char *&First, const char *const End) {
200
25.0k
  assert(*First == '\'' || *First == '"' || *First == '<');
201
25.0k
  const char Terminator = *First == '<' ? 
'>'10.3k
:
*First14.7k
;
202
387k
  for (++First; First != End && 
*First != Terminator387k
;
++First362k
) {
203
    // String and character literals don't extend past the end of the line.
204
362k
    if (isVerticalWhitespace(*First))
205
3
      return;
206
362k
    if (*First != '\\')
207
362k
      continue;
208
    // Skip past backslash to the next character. This ensures that the
209
    // character right after it is skipped as well, which matters if it's
210
    // the terminator.
211
344
    if (++First == End)
212
0
      return;
213
344
    if (!isWhitespace(*First))
214
340
      continue;
215
    // Whitespace after the backslash might indicate a line continuation.
216
4
    const char *FirstAfterBackslashPastSpace = First;
217
4
    skipOverSpaces(FirstAfterBackslashPastSpace, End);
218
4
    if (unsigned NLSize = isEOL(FirstAfterBackslashPastSpace, End)) {
219
      // Advance the character pointer to the next line for the next
220
      // iteration.
221
3
      First = FirstAfterBackslashPastSpace + NLSize - 1;
222
3
    }
223
4
  }
224
25.0k
  if (First != End)
225
25.0k
    ++First; // Finish off the string.
226
25.0k
}
227
228
// Returns the length of the skipped newline
229
608k
static unsigned skipNewline(const char *&First, const char *End) {
230
608k
  if (First == End)
231
12
    return 0;
232
608k
  assert(isVerticalWhitespace(*First));
233
0
  unsigned Len = isEOL(First, End);
234
608k
  assert(Len && "expected newline");
235
0
  First += Len;
236
608k
  return Len;
237
608k
}
238
239
229k
static bool wasLineContinuation(const char *First, unsigned EOLLen) {
240
229k
  return *(First - (int)EOLLen - 1) == '\\';
241
229k
}
242
243
8.32k
static void skipToNewlineRaw(const char *&First, const char *const End) {
244
8.74k
  for (;;) {
245
8.74k
    if (First == End)
246
0
      return;
247
248
8.74k
    unsigned Len = isEOL(First, End);
249
8.74k
    if (Len)
250
489
      return;
251
252
358k
    
do 8.25k
{
253
358k
      if (++First == End)
254
0
        return;
255
358k
      Len = isEOL(First, End);
256
358k
    } while (!Len);
257
258
8.25k
    if (First[-1] != '\\')
259
7.81k
      return;
260
261
440
    First += Len;
262
    // Keep skipping lines...
263
440
  }
264
8.32k
}
265
266
207k
static const char *findLastNonSpace(const char *First, const char *Last) {
267
207k
  assert(First <= Last);
268
523k
  while (First != Last && 
isHorizontalWhitespace(Last[-1])494k
)
269
316k
    --Last;
270
207k
  return Last;
271
207k
}
272
273
static const char *findFirstTrailingSpace(const char *First,
274
40.6k
                                          const char *Last) {
275
40.6k
  const char *LastNonSpace = findLastNonSpace(First, Last);
276
40.6k
  if (Last == LastNonSpace)
277
1.78k
    return Last;
278
38.8k
  assert(isHorizontalWhitespace(LastNonSpace[0]));
279
0
  return LastNonSpace + 1;
280
40.6k
}
281
282
7.90k
static void skipLineComment(const char *&First, const char *const End) {
283
7.90k
  assert(First[0] == '/' && First[1] == '/');
284
0
  First += 2;
285
7.90k
  skipToNewlineRaw(First, End);
286
7.90k
}
287
288
122k
static void skipBlockComment(const char *&First, const char *const End) {
289
122k
  assert(First[0] == '/' && First[1] == '*');
290
122k
  if (End - First < 4) {
291
0
    First = End;
292
0
    return;
293
0
  }
294
14.6M
  
for (First += 3; 122k
First != End;
++First14.5M
)
295
14.6M
    if (First[-1] == '*' && 
First[0] == '/'501k
) {
296
122k
      ++First;
297
122k
      return;
298
122k
    }
299
122k
}
300
301
/// \returns True if the current single quotation mark character is a C++ 14
302
/// digit separator.
303
static bool isQuoteCppDigitSeparator(const char *const Start,
304
                                     const char *const Cur,
305
14
                                     const char *const End) {
306
14
  assert(*Cur == '\'' && "expected quotation character");
307
  // skipLine called in places where we don't expect a valid number
308
  // body before `start` on the same line, so always return false at the start.
309
14
  if (Start == Cur)
310
0
    return false;
311
  // The previous character must be a valid PP number character.
312
  // Make sure that the L, u, U, u8 prefixes don't get marked as a
313
  // separator though.
314
14
  char Prev = *(Cur - 1);
315
14
  if (Prev == 'L' || 
Prev == 'U'13
||
Prev == 'u'12
)
316
3
    return false;
317
11
  if (Prev == '8' && 
(Cur - 1 != Start)2
&&
*(Cur - 2) == 'u'2
)
318
1
    return false;
319
10
  if (!isPreprocessingNumberBody(Prev))
320
6
    return false;
321
  // The next character should be a valid identifier body character.
322
4
  return (Cur + 1) < End && isAsciiIdentifierContinue(*(Cur + 1));
323
10
}
324
325
433k
static void skipLine(const char *&First, const char *const End) {
326
433k
  for (;;) {
327
433k
    assert(First <= End);
328
433k
    if (First == End)
329
1
      return;
330
331
433k
    if (isVerticalWhitespace(*First)) {
332
204k
      skipNewline(First, End);
333
204k
      return;
334
204k
    }
335
229k
    const char *Start = First;
336
7.76M
    while (First != End && 
!isVerticalWhitespace(*First)7.76M
) {
337
      // Iterate over strings correctly to avoid comments and newlines.
338
7.53M
      if (*First == '"' ||
339
7.53M
          
(7.52M
*First == '\''7.52M
&&
!isQuoteCppDigitSeparator(Start, First, End)14
)) {
340
6.27k
        if (isRawStringLiteral(Start, First))
341
3
          skipRawString(First, End);
342
6.27k
        else
343
6.27k
          skipString(First, End);
344
6.27k
        continue;
345
6.27k
      }
346
347
      // Iterate over comments correctly.
348
7.52M
      if (*First != '/' || 
End - First < 238.0k
) {
349
7.48M
        ++First;
350
7.48M
        continue;
351
7.48M
      }
352
353
38.0k
      if (First[1] == '/') {
354
        // "//...".
355
1.24k
        skipLineComment(First, End);
356
1.24k
        continue;
357
1.24k
      }
358
359
36.8k
      if (First[1] != '*') {
360
230
        ++First;
361
230
        continue;
362
230
      }
363
364
      // "/*...*/".
365
36.5k
      skipBlockComment(First, End);
366
36.5k
    }
367
229k
    if (First == End)
368
12
      return;
369
370
    // Skip over the newline.
371
229k
    unsigned Len = skipNewline(First, End);
372
229k
    if (!wasLineContinuation(First, Len)) // Continue past line-continuations.
373
229k
      break;
374
229k
  }
375
433k
}
376
377
static void skipDirective(StringRef Name, const char *&First,
378
418
                          const char *const End) {
379
418
  if (llvm::StringSwitch<bool>(Name)
380
418
          .Case("warning", true)
381
418
          .Case("error", true)
382
418
          .Default(false))
383
    // Do not process quotes or comments.
384
418
    skipToNewlineRaw(First, End);
385
0
  else
386
0
    skipLine(First, End);
387
418
}
388
389
208k
void Minimizer::printToNewline(const char *&First, const char *const End) {
390
220k
  while (First != End && 
!isVerticalWhitespace(*First)220k
) {
391
166k
    const char *Last = First;
392
3.95M
    do {
393
      // Iterate over strings correctly to avoid comments and newlines.
394
3.95M
      if (*Last == '"' || 
*Last == '\''3.94M
||
395
3.95M
          
(3.94M
*Last == '<'3.94M
&&
top() == pp_include16.4k
)) {
396
18.8k
        if (LLVM_UNLIKELY(isRawStringLiteral(First, Last)))
397
0
          skipRawString(Last, End);
398
18.8k
        else
399
18.8k
          skipString(Last, End);
400
18.8k
        continue;
401
18.8k
      }
402
3.93M
      if (*Last != '/' || 
End - Last < 229.5k
) {
403
3.90M
        ++Last;
404
3.90M
        continue; // Gather the rest up to print verbatim.
405
3.90M
      }
406
407
29.5k
      if (Last[1] != '/' && 
Last[1] != '*'29.5k
) {
408
600
        ++Last;
409
600
        continue;
410
600
      }
411
412
      // Deal with "//..." and "/*...*/".
413
28.9k
      append(First, findFirstTrailingSpace(First, Last));
414
28.9k
      First = Last;
415
416
28.9k
      if (Last[1] == '/') {
417
25
        skipLineComment(First, End);
418
25
        return;
419
25
      }
420
421
28.9k
      put(' ');
422
28.9k
      skipBlockComment(First, End);
423
28.9k
      skipOverSpaces(First, End);
424
28.9k
      Last = First;
425
3.95M
    } while (Last != End && 
!isVerticalWhitespace(*Last)3.95M
);
426
427
    // Print out the string.
428
166k
    const char *LastBeforeTrailingSpace = findLastNonSpace(First, Last);
429
166k
    if (Last == End || 
LastBeforeTrailingSpace == First166k
||
430
166k
        
LastBeforeTrailingSpace[-1] != '\\'137k
) {
431
154k
      append(First, LastBeforeTrailingSpace);
432
154k
      First = Last;
433
154k
      skipNewline(First, End);
434
154k
      return;
435
154k
    }
436
437
    // Print up to the backslash, backing up over spaces. Preserve at least one
438
    // space, as the space matters when tokens are separated by a line
439
    // continuation.
440
11.6k
    append(First, findFirstTrailingSpace(
441
11.6k
                      First, LastBeforeTrailingSpace - 1));
442
443
11.6k
    First = Last;
444
11.6k
    skipNewline(First, End);
445
11.6k
    skipOverSpaces(First, End);
446
11.6k
  }
447
208k
}
448
449
1.34M
static void skipWhitespace(const char *&First, const char *const End) {
450
1.41M
  for (;;) {
451
1.41M
    assert(First <= End);
452
0
    skipOverSpaces(First, End);
453
454
1.41M
    if (End - First < 2)
455
5.00k
      return;
456
457
1.40M
    if (First[0] == '\\' && 
isVerticalWhitespace(First[1])8.82k
) {
458
8.82k
      skipNewline(++First, End);
459
8.82k
      continue;
460
8.82k
    }
461
462
    // Check for a non-comment character.
463
1.40M
    if (First[0] != '/')
464
1.33M
      return;
465
466
    // "// ...".
467
63.8k
    if (First[1] == '/') {
468
6.63k
      skipLineComment(First, End);
469
6.63k
      return;
470
6.63k
    }
471
472
    // Cannot be a comment.
473
57.1k
    if (First[1] != '*')
474
0
      return;
475
476
    // "/*...*/".
477
57.1k
    skipBlockComment(First, End);
478
57.1k
  }
479
1.34M
}
480
481
void Minimizer::printAdjacentModuleNameParts(const char *&First,
482
13
                                             const char *const End) {
483
  // Skip over parts of the body.
484
13
  const char *Last = First;
485
13
  do
486
19
    ++Last;
487
19
  while (Last != End && (isAsciiIdentifierContinue(*Last) || 
*Last == '.'14
));
488
13
  append(First, Last);
489
13
  First = Last;
490
13
}
491
492
12
bool Minimizer::printAtImportBody(const char *&First, const char *const End) {
493
29
  for (;;) {
494
29
    skipWhitespace(First, End);
495
29
    if (First == End)
496
2
      return true;
497
498
27
    if (isVerticalWhitespace(*First)) {
499
4
      skipNewline(First, End);
500
4
      continue;
501
4
    }
502
503
    // Found a semicolon.
504
23
    if (*First == ';') {
505
8
      put(*First++).put('\n');
506
8
      return false;
507
8
    }
508
509
    // Don't handle macro expansions inside @import for now.
510
15
    if (!isAsciiIdentifierContinue(*First) && 
*First != '.'3
)
511
2
      return true;
512
513
13
    printAdjacentModuleNameParts(First, End);
514
13
  }
515
12
}
516
517
208k
void Minimizer::printDirectiveBody(const char *&First, const char *const End) {
518
208k
  skipWhitespace(First, End); // Skip initial whitespace.
519
208k
  printToNewline(First, End);
520
300k
  while (Out.back() == ' ')
521
92.2k
    Out.pop_back();
522
208k
  put('\n');
523
208k
}
524
525
LLVM_NODISCARD static const char *lexRawIdentifier(const char *First,
526
384k
                                                   const char *const End) {
527
384k
  assert(isAsciiIdentifierContinue(*First) && "invalid identifer");
528
0
  const char *Last = First + 1;
529
4.12M
  while (Last != End && 
isAsciiIdentifierContinue(*Last)4.12M
)
530
3.74M
    ++Last;
531
384k
  return Last;
532
384k
}
533
534
LLVM_NODISCARD static const char *
535
384k
getIdentifierContinuation(const char *First, const char *const End) {
536
384k
  if (End - First < 3 || 
First[0] != '\\'384k
||
!isVerticalWhitespace(First[1])6
)
537
384k
    return nullptr;
538
539
5
  ++First;
540
5
  skipNewline(First, End);
541
5
  if (First == End)
542
0
    return nullptr;
543
5
  return isAsciiIdentifierContinue(First[0]) ? 
First4
:
nullptr1
;
544
5
}
545
546
Minimizer::IdInfo Minimizer::lexIdentifier(const char *First,
547
384k
                                           const char *const End) {
548
384k
  const char *Last = lexRawIdentifier(First, End);
549
384k
  const char *Next = getIdentifierContinuation(Last, End);
550
384k
  if (LLVM_LIKELY(!Next))
551
384k
    return IdInfo{Last, StringRef(First, Last - First)};
552
553
  // Slow path, where identifiers are split over lines.
554
9
  SmallVector<char, 64> Id(First, Last);
555
13
  while (Next) {
556
4
    Last = lexRawIdentifier(Next, End);
557
4
    Id.append(Next, Last);
558
4
    Next = getIdentifierContinuation(Last, End);
559
4
  }
560
9
  return IdInfo{
561
9
      Last,
562
9
      SplitIds.try_emplace(StringRef(Id.begin(), Id.size()), 0).first->first()};
563
384k
}
564
565
void Minimizer::printAdjacentMacroArgs(const char *&First,
566
24.0k
                                       const char *const End) {
567
  // Skip over parts of the body.
568
24.0k
  const char *Last = First;
569
24.0k
  do
570
103k
    ++Last;
571
103k
  while (Last != End &&
572
103k
         (isAsciiIdentifierContinue(*Last) || 
*Last == '.'35.9k
||
*Last == ','34.1k
));
573
24.0k
  append(First, Last);
574
24.0k
  First = Last;
575
24.0k
}
576
577
18.7k
bool Minimizer::printMacroArgs(const char *&First, const char *const End) {
578
18.7k
  assert(*First == '(');
579
0
  put(*First++);
580
42.7k
  for (;;) {
581
42.7k
    skipWhitespace(First, End);
582
42.7k
    if (First == End)
583
1
      return true;
584
585
42.7k
    if (*First == ')') {
586
18.7k
      put(*First++);
587
18.7k
      return false;
588
18.7k
    }
589
590
    // This is intentionally fairly liberal.
591
24.0k
    if (!(isAsciiIdentifierContinue(*First) || 
*First == '.'849
||
*First == ','2
))
592
2
      return true;
593
594
24.0k
    printAdjacentMacroArgs(First, End);
595
24.0k
  }
596
18.7k
}
597
598
/// Looks for an identifier starting from Last.
599
///
600
/// Updates "First" to just past the next identifier, if any.  Returns true iff
601
/// the identifier matches "Id".
602
bool Minimizer::isNextIdentifier(StringRef Id, const char *&First,
603
178
                                 const char *const End) {
604
178
  skipWhitespace(First, End);
605
178
  if (First == End || !isAsciiIdentifierStart(*First))
606
2
    return false;
607
608
176
  IdInfo FoundId = lexIdentifier(First, End);
609
176
  First = FoundId.Last;
610
176
  return FoundId.Name == Id;
611
178
}
612
613
137
bool Minimizer::lexAt(const char *&First, const char *const End) {
614
  // Handle "@import".
615
137
  const char *ImportLoc = First++;
616
137
  if (!isNextIdentifier("import", First, End)) {
617
125
    skipLine(First, End);
618
125
    return false;
619
125
  }
620
12
  makeToken(decl_at_import);
621
12
  append("@import ");
622
12
  if (printAtImportBody(First, End))
623
4
    return reportError(
624
4
        ImportLoc, diag::err_dep_source_minimizer_missing_sema_after_at_import);
625
8
  skipWhitespace(First, End);
626
8
  if (First == End)
627
2
    return false;
628
6
  if (!isVerticalWhitespace(*First))
629
1
    return reportError(
630
1
        ImportLoc, diag::err_dep_source_minimizer_unexpected_tokens_at_import);
631
5
  skipNewline(First, End);
632
5
  return false;
633
6
}
634
635
42.5k
bool Minimizer::lexModule(const char *&First, const char *const End) {
636
42.5k
  IdInfo Id = lexIdentifier(First, End);
637
42.5k
  First = Id.Last;
638
42.5k
  bool Export = false;
639
42.5k
  if (Id.Name == "export") {
640
4
    Export = true;
641
4
    skipWhitespace(First, End);
642
4
    if (!isAsciiIdentifierContinue(*First)) {
643
0
      skipLine(First, End);
644
0
      return false;
645
0
    }
646
4
    Id = lexIdentifier(First, End);
647
4
    First = Id.Last;
648
4
  }
649
650
42.5k
  if (Id.Name != "module" && 
Id.Name != "import"42.5k
) {
651
42.5k
    skipLine(First, End);
652
42.5k
    return false;
653
42.5k
  }
654
655
16
  skipWhitespace(First, End);
656
657
  // Ignore this as a module directive if the next character can't be part of
658
  // an import.
659
660
16
  switch (*First) {
661
1
  case ':':
662
2
  case '<':
663
2
  case '"':
664
2
    break;
665
14
  default:
666
14
    if (!isAsciiIdentifierContinue(*First)) {
667
6
      skipLine(First, End);
668
6
      return false;
669
6
    }
670
16
  }
671
672
10
  if (Export) {
673
3
    makeToken(cxx_export_decl);
674
3
    append("export ");
675
3
  }
676
677
10
  if (Id.Name == "module")
678
2
    makeToken(cxx_module_decl);
679
8
  else
680
8
    makeToken(cxx_import_decl);
681
10
  append(Id.Name);
682
10
  append(" ");
683
10
  printToNewline(First, End);
684
10
  append("\n");
685
10
  return false;
686
16
}
687
688
112k
bool Minimizer::lexDefine(const char *&First, const char *const End) {
689
112k
  makeToken(pp_define);
690
112k
  append("#define ");
691
112k
  skipWhitespace(First, End);
692
693
112k
  if (!isAsciiIdentifierStart(*First))
694
3
    return reportError(First, diag::err_pp_macro_not_identifier);
695
696
112k
  IdInfo Id = lexIdentifier(First, End);
697
112k
  const char *Last = Id.Last;
698
112k
  append(Id.Name);
699
112k
  if (Last == End)
700
1
    return false;
701
112k
  if (*Last == '(') {
702
18.7k
    size_t Size = Out.size();
703
18.7k
    if (printMacroArgs(Last, End)) {
704
      // Be robust to bad macro arguments, since they can show up in disabled
705
      // code.
706
3
      Out.resize(Size);
707
3
      append("(/* invalid */\n");
708
3
      skipLine(Last, End);
709
3
      return false;
710
3
    }
711
18.7k
  }
712
112k
  skipWhitespace(Last, End);
713
112k
  if (Last == End)
714
3
    return false;
715
112k
  if (!isVerticalWhitespace(*Last))
716
99.2k
    put(' ');
717
112k
  printDirectiveBody(Last, End);
718
112k
  First = Last;
719
112k
  return false;
720
112k
}
721
722
5.78k
bool Minimizer::lexPragma(const char *&First, const char *const End) {
723
  // #pragma.
724
5.78k
  skipWhitespace(First, End);
725
5.78k
  if (First == End || !isAsciiIdentifierStart(*First))
726
0
    return false;
727
728
5.78k
  IdInfo FoundId = lexIdentifier(First, End);
729
5.78k
  First = FoundId.Last;
730
5.78k
  if (FoundId.Name == "once") {
731
    // #pragma once
732
8
    skipLine(First, End);
733
8
    makeToken(pp_pragma_once);
734
8
    append("#pragma once\n");
735
8
    return false;
736
8
  }
737
5.78k
  if (FoundId.Name == "push_macro") {
738
    // #pragma push_macro
739
5
    makeToken(pp_pragma_push_macro);
740
5
    append("#pragma push_macro");
741
5
    printDirectiveBody(First, End);
742
5
    return false;
743
5
  }
744
5.77k
  if (FoundId.Name == "pop_macro") {
745
    // #pragma pop_macro
746
5
    makeToken(pp_pragma_pop_macro);
747
5
    append("#pragma pop_macro");
748
5
    printDirectiveBody(First, End);
749
5
    return false;
750
5
  }
751
5.77k
  if (FoundId.Name == "include_alias") {
752
    // #pragma include_alias
753
5
    makeToken(pp_pragma_include_alias);
754
5
    append("#pragma include_alias");
755
5
    printDirectiveBody(First, End);
756
5
    return false;
757
5
  }
758
759
5.76k
  if (FoundId.Name != "clang") {
760
5.72k
    skipLine(First, End);
761
5.72k
    return false;
762
5.72k
  }
763
764
  // #pragma clang.
765
36
  if (!isNextIdentifier("module", First, End)) {
766
31
    skipLine(First, End);
767
31
    return false;
768
31
  }
769
770
  // #pragma clang module.
771
5
  if (!isNextIdentifier("import", First, End)) {
772
2
    skipLine(First, End);
773
2
    return false;
774
2
  }
775
776
  // #pragma clang module import.
777
3
  makeToken(pp_pragma_import);
778
3
  append("#pragma clang module import ");
779
3
  printDirectiveBody(First, End);
780
3
  return false;
781
5
}
782
783
40.5k
bool Minimizer::lexEndif(const char *&First, const char *const End) {
784
  // Strip out "#else" if it's empty.
785
40.5k
  if (top() == pp_else)
786
2.25k
    popToken();
787
788
  // If "#ifdef" is empty, strip it and skip the "#endif".
789
  //
790
  // FIXME: Once/if Clang starts disallowing __has_include in macro expansions,
791
  // we can skip empty `#if` and `#elif` blocks as well after scanning for a
792
  // literal __has_include in the condition.  Even without that rule we could
793
  // drop the tokens if we scan for identifiers in the condition and find none.
794
40.5k
  if (top() == pp_ifdef || 
top() == pp_ifndef31.7k
) {
795
9.08k
    popToken();
796
9.08k
    skipLine(First, End);
797
9.08k
    return false;
798
9.08k
  }
799
800
31.4k
  return lexDefault(pp_endif, "endif", First, End);
801
40.5k
}
802
803
bool Minimizer::lexDefault(TokenKind Kind, StringRef Directive,
804
96.5k
                           const char *&First, const char *const End) {
805
96.5k
  makeToken(Kind);
806
96.5k
  put('#').append(Directive).put(' ');
807
96.5k
  printDirectiveBody(First, End);
808
96.5k
  return false;
809
96.5k
}
810
811
642k
static bool isStartOfRelevantLine(char First) {
812
642k
  switch (First) {
813
223k
  case '#':
814
224k
  case '@':
815
244k
  case 'i':
816
256k
  case 'e':
817
266k
  case 'm':
818
266k
    return true;
819
642k
  }
820
375k
  return false;
821
642k
}
822
823
642k
bool Minimizer::lexPPLine(const char *&First, const char *const End) {
824
642k
  assert(First != End);
825
826
0
  skipWhitespace(First, End);
827
642k
  assert(First <= End);
828
642k
  if (First == End)
829
3
    return false;
830
831
642k
  if (!isStartOfRelevantLine(*First)) {
832
375k
    skipLine(First, End);
833
375k
    assert(First <= End);
834
0
    return false;
835
375k
  }
836
837
  // Handle "@import".
838
266k
  if (*First == '@')
839
137
    return lexAt(First, End);
840
841
266k
  if (*First == 'i' || 
*First == 'e'246k
||
*First == 'm'233k
)
842
42.5k
    return lexModule(First, End);
843
844
  // Handle preprocessing directives.
845
223k
  ++First; // Skip over '#'.
846
223k
  skipWhitespace(First, End);
847
848
223k
  if (First == End)
849
0
    return reportError(First, diag::err_pp_expected_eol);
850
851
223k
  if (!isAsciiIdentifierStart(*First)) {
852
0
    skipLine(First, End);
853
0
    return false;
854
0
  }
855
856
  // Figure out the token.
857
223k
  IdInfo Id = lexIdentifier(First, End);
858
223k
  First = Id.Last;
859
860
223k
  if (Id.Name == "pragma")
861
5.78k
    return lexPragma(First, End);
862
863
218k
  auto Kind = llvm::StringSwitch<TokenKind>(Id.Name)
864
218k
                  .Case("include", pp_include)
865
218k
                  .Case("__include_macros", pp___include_macros)
866
218k
                  .Case("define", pp_define)
867
218k
                  .Case("undef", pp_undef)
868
218k
                  .Case("import", pp_import)
869
218k
                  .Case("include_next", pp_include_next)
870
218k
                  .Case("if", pp_if)
871
218k
                  .Case("ifdef", pp_ifdef)
872
218k
                  .Case("ifndef", pp_ifndef)
873
218k
                  .Case("elif", pp_elif)
874
218k
                  .Case("elifdef", pp_elifdef)
875
218k
                  .Case("elifndef", pp_elifndef)
876
218k
                  .Case("else", pp_else)
877
218k
                  .Case("endif", pp_endif)
878
218k
                  .Default(pp_none);
879
218k
  if (Kind == pp_none) {
880
418
    skipDirective(Id.Name, First, End);
881
418
    return false;
882
418
  }
883
884
217k
  if (Kind == pp_endif)
885
40.5k
    return lexEndif(First, End);
886
887
177k
  if (Kind == pp_define)
888
112k
    return lexDefine(First, End);
889
890
  // Everything else.
891
65.1k
  return lexDefault(Kind, Id.Name, First, End);
892
177k
}
893
894
3.02k
static void skipUTF8ByteOrderMark(const char *&First, const char *const End) {
895
3.02k
  if ((End - First) >= 3 && 
First[0] == '\xef'2.99k
&&
First[1] == '\xbb'1
&&
896
3.02k
      
First[2] == '\xbf'1
)
897
1
    First += 3;
898
3.02k
}
899
900
3.02k
bool Minimizer::minimizeImpl(const char *First, const char *const End) {
901
3.02k
  skipUTF8ByteOrderMark(First, End);
902
645k
  while (First != End)
903
642k
    if (lexPPLine(First, End))
904
8
      return true;
905
3.01k
  return false;
906
3.02k
}
907
908
3.02k
bool Minimizer::minimize() {
909
3.02k
  bool Error = minimizeImpl(Input.begin(), Input.end());
910
911
3.02k
  if (!Error) {
912
    // Add a trailing newline and an EOF on success.
913
3.01k
    if (!Out.empty() && 
Out.back() != '\n'2.93k
)
914
4
      Out.push_back('\n');
915
3.01k
    makeToken(pp_eof);
916
3.01k
  }
917
918
  // Null-terminate the output. This way the memory buffer that's passed to
919
  // Clang will not have to worry about the terminating '\0'.
920
3.02k
  Out.push_back(0);
921
3.02k
  Out.pop_back();
922
3.02k
  return Error;
923
3.02k
}
924
925
bool clang::minimize_source_to_dependency_directives::computeSkippedRanges(
926
2.91k
    ArrayRef<Token> Input, llvm::SmallVectorImpl<SkippedRange> &Range) {
927
2.91k
  struct Directive {
928
2.91k
    enum DirectiveKind {
929
2.91k
      If,  // if/ifdef/ifndef
930
2.91k
      Else // elif/elifdef/elifndef, else
931
2.91k
    };
932
2.91k
    int Offset;
933
2.91k
    DirectiveKind Kind;
934
2.91k
  };
935
2.91k
  llvm::SmallVector<Directive, 32> Offsets;
936
200k
  for (const Token &T : Input) {
937
200k
    switch (T.K) {
938
12.1k
    case pp_if:
939
14.0k
    case pp_ifdef:
940
31.4k
    case pp_ifndef:
941
31.4k
      Offsets.push_back({T.Offset, Directive::If});
942
31.4k
      break;
943
944
2.07k
    case pp_elif:
945
2.07k
    case pp_elifdef:
946
2.07k
    case pp_elifndef:
947
9.54k
    case pp_else: {
948
9.54k
      if (Offsets.empty())
949
0
        return true;
950
9.54k
      int PreviousOffset = Offsets.back().Offset;
951
9.54k
      Range.push_back({PreviousOffset, T.Offset - PreviousOffset});
952
9.54k
      Offsets.push_back({T.Offset, Directive::Else});
953
9.54k
      break;
954
9.54k
    }
955
956
31.4k
    case pp_endif: {
957
31.4k
      if (Offsets.empty())
958
0
        return true;
959
31.4k
      int PreviousOffset = Offsets.back().Offset;
960
31.4k
      Range.push_back({PreviousOffset, T.Offset - PreviousOffset});
961
41.0k
      do {
962
41.0k
        Directive::DirectiveKind Kind = Offsets.pop_back_val().Kind;
963
41.0k
        if (Kind == Directive::If)
964
31.4k
          break;
965
41.0k
      } while (
!Offsets.empty()9.54k
);
966
0
      break;
967
31.4k
    }
968
127k
    default:
969
127k
      break;
970
200k
    }
971
200k
  }
972
2.91k
  return false;
973
2.91k
}
974
975
bool clang::minimizeSourceToDependencyDirectives(
976
    StringRef Input, SmallVectorImpl<char> &Output,
977
    SmallVectorImpl<Token> &Tokens, DiagnosticsEngine *Diags,
978
3.02k
    SourceLocation InputSourceLoc) {
979
3.02k
  Output.clear();
980
3.02k
  Tokens.clear();
981
3.02k
  return Minimizer(Output, Tokens, Input, Diags, InputSourceLoc).minimize();
982
3.02k
}