xref: /freebsd/contrib/llvm-project/clang/lib/Frontend/PrintPreprocessedOutput.cpp (revision b1879975794772ee51f0b4865753364c7d7626c3)
1 //===--- PrintPreprocessedOutput.cpp - Implement the -E mode --------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This code simply runs the preprocessor on the input file and prints out the
10 // result.  This is the traditional behavior of the -E option.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "clang/Basic/CharInfo.h"
15 #include "clang/Basic/Diagnostic.h"
16 #include "clang/Basic/SourceManager.h"
17 #include "clang/Frontend/PreprocessorOutputOptions.h"
18 #include "clang/Frontend/Utils.h"
19 #include "clang/Lex/MacroInfo.h"
20 #include "clang/Lex/PPCallbacks.h"
21 #include "clang/Lex/Pragma.h"
22 #include "clang/Lex/Preprocessor.h"
23 #include "clang/Lex/TokenConcatenation.h"
24 #include "llvm/ADT/STLExtras.h"
25 #include "llvm/ADT/SmallString.h"
26 #include "llvm/ADT/StringRef.h"
27 #include "llvm/Support/ErrorHandling.h"
28 #include "llvm/Support/raw_ostream.h"
29 #include <cstdio>
30 using namespace clang;
31 
32 /// PrintMacroDefinition - Print a macro definition in a form that will be
33 /// properly accepted back as a definition.
34 static void PrintMacroDefinition(const IdentifierInfo &II, const MacroInfo &MI,
35                                  Preprocessor &PP, raw_ostream *OS) {
36   *OS << "#define " << II.getName();
37 
38   if (MI.isFunctionLike()) {
39     *OS << '(';
40     if (!MI.param_empty()) {
41       MacroInfo::param_iterator AI = MI.param_begin(), E = MI.param_end();
42       for (; AI+1 != E; ++AI) {
43         *OS << (*AI)->getName();
44         *OS << ',';
45       }
46 
47       // Last argument.
48       if ((*AI)->getName() == "__VA_ARGS__")
49         *OS << "...";
50       else
51         *OS << (*AI)->getName();
52     }
53 
54     if (MI.isGNUVarargs())
55       *OS << "...";  // #define foo(x...)
56 
57     *OS << ')';
58   }
59 
60   // GCC always emits a space, even if the macro body is empty.  However, do not
61   // want to emit two spaces if the first token has a leading space.
62   if (MI.tokens_empty() || !MI.tokens_begin()->hasLeadingSpace())
63     *OS << ' ';
64 
65   SmallString<128> SpellingBuffer;
66   for (const auto &T : MI.tokens()) {
67     if (T.hasLeadingSpace())
68       *OS << ' ';
69 
70     *OS << PP.getSpelling(T, SpellingBuffer);
71   }
72 }
73 
74 //===----------------------------------------------------------------------===//
75 // Preprocessed token printer
76 //===----------------------------------------------------------------------===//
77 
78 namespace {
79 class PrintPPOutputPPCallbacks : public PPCallbacks {
80   Preprocessor &PP;
81   SourceManager &SM;
82   TokenConcatenation ConcatInfo;
83 public:
84   raw_ostream *OS;
85 private:
86   unsigned CurLine;
87 
88   bool EmittedTokensOnThisLine;
89   bool EmittedDirectiveOnThisLine;
90   SrcMgr::CharacteristicKind FileType;
91   SmallString<512> CurFilename;
92   bool Initialized;
93   bool DisableLineMarkers;
94   bool DumpDefines;
95   bool DumpIncludeDirectives;
96   bool DumpEmbedDirectives;
97   bool UseLineDirectives;
98   bool IsFirstFileEntered;
99   bool MinimizeWhitespace;
100   bool DirectivesOnly;
101   bool KeepSystemIncludes;
102   raw_ostream *OrigOS;
103   std::unique_ptr<llvm::raw_null_ostream> NullOS;
104   unsigned NumToksToSkip;
105 
106   Token PrevTok;
107   Token PrevPrevTok;
108 
109 public:
110   PrintPPOutputPPCallbacks(Preprocessor &pp, raw_ostream *os, bool lineMarkers,
111                            bool defines, bool DumpIncludeDirectives,
112                            bool DumpEmbedDirectives, bool UseLineDirectives,
113                            bool MinimizeWhitespace, bool DirectivesOnly,
114                            bool KeepSystemIncludes)
115       : PP(pp), SM(PP.getSourceManager()), ConcatInfo(PP), OS(os),
116         DisableLineMarkers(lineMarkers), DumpDefines(defines),
117         DumpIncludeDirectives(DumpIncludeDirectives),
118         DumpEmbedDirectives(DumpEmbedDirectives),
119         UseLineDirectives(UseLineDirectives),
120         MinimizeWhitespace(MinimizeWhitespace), DirectivesOnly(DirectivesOnly),
121         KeepSystemIncludes(KeepSystemIncludes), OrigOS(os), NumToksToSkip(0) {
122     CurLine = 0;
123     CurFilename += "<uninit>";
124     EmittedTokensOnThisLine = false;
125     EmittedDirectiveOnThisLine = false;
126     FileType = SrcMgr::C_User;
127     Initialized = false;
128     IsFirstFileEntered = false;
129     if (KeepSystemIncludes)
130       NullOS = std::make_unique<llvm::raw_null_ostream>();
131 
132     PrevTok.startToken();
133     PrevPrevTok.startToken();
134   }
135 
136   /// Returns true if #embed directives should be expanded into a comma-
137   /// delimited list of integer constants or not.
138   bool expandEmbedContents() const { return !DumpEmbedDirectives; }
139 
140   bool isMinimizeWhitespace() const { return MinimizeWhitespace; }
141 
142   void setEmittedTokensOnThisLine() { EmittedTokensOnThisLine = true; }
143   bool hasEmittedTokensOnThisLine() const { return EmittedTokensOnThisLine; }
144 
145   void setEmittedDirectiveOnThisLine() { EmittedDirectiveOnThisLine = true; }
146   bool hasEmittedDirectiveOnThisLine() const {
147     return EmittedDirectiveOnThisLine;
148   }
149 
150   /// Ensure that the output stream position is at the beginning of a new line
151   /// and inserts one if it does not. It is intended to ensure that directives
152   /// inserted by the directives not from the input source (such as #line) are
153   /// in the first column. To insert newlines that represent the input, use
154   /// MoveToLine(/*...*/, /*RequireStartOfLine=*/true).
155   void startNewLineIfNeeded();
156 
157   void FileChanged(SourceLocation Loc, FileChangeReason Reason,
158                    SrcMgr::CharacteristicKind FileType,
159                    FileID PrevFID) override;
160   void EmbedDirective(SourceLocation HashLoc, StringRef FileName, bool IsAngled,
161                       OptionalFileEntryRef File,
162                       const LexEmbedParametersResult &Params) override;
163   void InclusionDirective(SourceLocation HashLoc, const Token &IncludeTok,
164                           StringRef FileName, bool IsAngled,
165                           CharSourceRange FilenameRange,
166                           OptionalFileEntryRef File, StringRef SearchPath,
167                           StringRef RelativePath, const Module *SuggestedModule,
168                           bool ModuleImported,
169                           SrcMgr::CharacteristicKind FileType) override;
170   void Ident(SourceLocation Loc, StringRef str) override;
171   void PragmaMessage(SourceLocation Loc, StringRef Namespace,
172                      PragmaMessageKind Kind, StringRef Str) override;
173   void PragmaDebug(SourceLocation Loc, StringRef DebugType) override;
174   void PragmaDiagnosticPush(SourceLocation Loc, StringRef Namespace) override;
175   void PragmaDiagnosticPop(SourceLocation Loc, StringRef Namespace) override;
176   void PragmaDiagnostic(SourceLocation Loc, StringRef Namespace,
177                         diag::Severity Map, StringRef Str) override;
178   void PragmaWarning(SourceLocation Loc, PragmaWarningSpecifier WarningSpec,
179                      ArrayRef<int> Ids) override;
180   void PragmaWarningPush(SourceLocation Loc, int Level) override;
181   void PragmaWarningPop(SourceLocation Loc) override;
182   void PragmaExecCharsetPush(SourceLocation Loc, StringRef Str) override;
183   void PragmaExecCharsetPop(SourceLocation Loc) override;
184   void PragmaAssumeNonNullBegin(SourceLocation Loc) override;
185   void PragmaAssumeNonNullEnd(SourceLocation Loc) override;
186 
187   /// Insert whitespace before emitting the next token.
188   ///
189   /// @param Tok             Next token to be emitted.
190   /// @param RequireSpace    Ensure at least one whitespace is emitted. Useful
191   ///                        if non-tokens have been emitted to the stream.
192   /// @param RequireSameLine Never emit newlines. Useful when semantics depend
193   ///                        on being on the same line, such as directives.
194   void HandleWhitespaceBeforeTok(const Token &Tok, bool RequireSpace,
195                                  bool RequireSameLine);
196 
197   /// Move to the line of the provided source location. This will
198   /// return true if a newline was inserted or if
199   /// the requested location is the first token on the first line.
200   /// In these cases the next output will be the first column on the line and
201   /// make it possible to insert indention. The newline was inserted
202   /// implicitly when at the beginning of the file.
203   ///
204   /// @param Tok                 Token where to move to.
205   /// @param RequireStartOfLine  Whether the next line depends on being in the
206   ///                            first column, such as a directive.
207   ///
208   /// @return Whether column adjustments are necessary.
209   bool MoveToLine(const Token &Tok, bool RequireStartOfLine) {
210     PresumedLoc PLoc = SM.getPresumedLoc(Tok.getLocation());
211     unsigned TargetLine = PLoc.isValid() ? PLoc.getLine() : CurLine;
212     bool IsFirstInFile =
213         Tok.isAtStartOfLine() && PLoc.isValid() && PLoc.getLine() == 1;
214     return MoveToLine(TargetLine, RequireStartOfLine) || IsFirstInFile;
215   }
216 
217   /// Move to the line of the provided source location. Returns true if a new
218   /// line was inserted.
219   bool MoveToLine(SourceLocation Loc, bool RequireStartOfLine) {
220     PresumedLoc PLoc = SM.getPresumedLoc(Loc);
221     unsigned TargetLine = PLoc.isValid() ? PLoc.getLine() : CurLine;
222     return MoveToLine(TargetLine, RequireStartOfLine);
223   }
224   bool MoveToLine(unsigned LineNo, bool RequireStartOfLine);
225 
226   bool AvoidConcat(const Token &PrevPrevTok, const Token &PrevTok,
227                    const Token &Tok) {
228     return ConcatInfo.AvoidConcat(PrevPrevTok, PrevTok, Tok);
229   }
230   void WriteLineInfo(unsigned LineNo, const char *Extra=nullptr,
231                      unsigned ExtraLen=0);
232   bool LineMarkersAreDisabled() const { return DisableLineMarkers; }
233   void HandleNewlinesInToken(const char *TokStr, unsigned Len);
234 
235   /// MacroDefined - This hook is called whenever a macro definition is seen.
236   void MacroDefined(const Token &MacroNameTok,
237                     const MacroDirective *MD) override;
238 
239   /// MacroUndefined - This hook is called whenever a macro #undef is seen.
240   void MacroUndefined(const Token &MacroNameTok,
241                       const MacroDefinition &MD,
242                       const MacroDirective *Undef) override;
243 
244   void BeginModule(const Module *M);
245   void EndModule(const Module *M);
246 
247   unsigned GetNumToksToSkip() const { return NumToksToSkip; }
248   void ResetSkipToks() { NumToksToSkip = 0; }
249 };
250 }  // end anonymous namespace
251 
252 void PrintPPOutputPPCallbacks::WriteLineInfo(unsigned LineNo,
253                                              const char *Extra,
254                                              unsigned ExtraLen) {
255   startNewLineIfNeeded();
256 
257   // Emit #line directives or GNU line markers depending on what mode we're in.
258   if (UseLineDirectives) {
259     *OS << "#line" << ' ' << LineNo << ' ' << '"';
260     OS->write_escaped(CurFilename);
261     *OS << '"';
262   } else {
263     *OS << '#' << ' ' << LineNo << ' ' << '"';
264     OS->write_escaped(CurFilename);
265     *OS << '"';
266 
267     if (ExtraLen)
268       OS->write(Extra, ExtraLen);
269 
270     if (FileType == SrcMgr::C_System)
271       OS->write(" 3", 2);
272     else if (FileType == SrcMgr::C_ExternCSystem)
273       OS->write(" 3 4", 4);
274   }
275   *OS << '\n';
276 }
277 
278 /// MoveToLine - Move the output to the source line specified by the location
279 /// object.  We can do this by emitting some number of \n's, or be emitting a
280 /// #line directive.  This returns false if already at the specified line, true
281 /// if some newlines were emitted.
282 bool PrintPPOutputPPCallbacks::MoveToLine(unsigned LineNo,
283                                           bool RequireStartOfLine) {
284   // If it is required to start a new line or finish the current, insert
285   // vertical whitespace now and take it into account when moving to the
286   // expected line.
287   bool StartedNewLine = false;
288   if ((RequireStartOfLine && EmittedTokensOnThisLine) ||
289       EmittedDirectiveOnThisLine) {
290     *OS << '\n';
291     StartedNewLine = true;
292     CurLine += 1;
293     EmittedTokensOnThisLine = false;
294     EmittedDirectiveOnThisLine = false;
295   }
296 
297   // If this line is "close enough" to the original line, just print newlines,
298   // otherwise print a #line directive.
299   if (CurLine == LineNo) {
300     // Nothing to do if we are already on the correct line.
301   } else if (MinimizeWhitespace && DisableLineMarkers) {
302     // With -E -P -fminimize-whitespace, don't emit anything if not necessary.
303   } else if (!StartedNewLine && LineNo - CurLine == 1) {
304     // Printing a single line has priority over printing a #line directive, even
305     // when minimizing whitespace which otherwise would print #line directives
306     // for every single line.
307     *OS << '\n';
308     StartedNewLine = true;
309   } else if (!DisableLineMarkers) {
310     if (LineNo - CurLine <= 8) {
311       const char *NewLines = "\n\n\n\n\n\n\n\n";
312       OS->write(NewLines, LineNo - CurLine);
313     } else {
314       // Emit a #line or line marker.
315       WriteLineInfo(LineNo, nullptr, 0);
316     }
317     StartedNewLine = true;
318   } else if (EmittedTokensOnThisLine) {
319     // If we are not on the correct line and don't need to be line-correct,
320     // at least ensure we start on a new line.
321     *OS << '\n';
322     StartedNewLine = true;
323   }
324 
325   if (StartedNewLine) {
326     EmittedTokensOnThisLine = false;
327     EmittedDirectiveOnThisLine = false;
328   }
329 
330   CurLine = LineNo;
331   return StartedNewLine;
332 }
333 
334 void PrintPPOutputPPCallbacks::startNewLineIfNeeded() {
335   if (EmittedTokensOnThisLine || EmittedDirectiveOnThisLine) {
336     *OS << '\n';
337     EmittedTokensOnThisLine = false;
338     EmittedDirectiveOnThisLine = false;
339   }
340 }
341 
342 /// FileChanged - Whenever the preprocessor enters or exits a #include file
343 /// it invokes this handler.  Update our conception of the current source
344 /// position.
345 void PrintPPOutputPPCallbacks::FileChanged(SourceLocation Loc,
346                                            FileChangeReason Reason,
347                                        SrcMgr::CharacteristicKind NewFileType,
348                                        FileID PrevFID) {
349   // Unless we are exiting a #include, make sure to skip ahead to the line the
350   // #include directive was at.
351   SourceManager &SourceMgr = SM;
352 
353   PresumedLoc UserLoc = SourceMgr.getPresumedLoc(Loc);
354   if (UserLoc.isInvalid())
355     return;
356 
357   unsigned NewLine = UserLoc.getLine();
358 
359   if (Reason == PPCallbacks::EnterFile) {
360     SourceLocation IncludeLoc = UserLoc.getIncludeLoc();
361     if (IncludeLoc.isValid())
362       MoveToLine(IncludeLoc, /*RequireStartOfLine=*/false);
363   } else if (Reason == PPCallbacks::SystemHeaderPragma) {
364     // GCC emits the # directive for this directive on the line AFTER the
365     // directive and emits a bunch of spaces that aren't needed. This is because
366     // otherwise we will emit a line marker for THIS line, which requires an
367     // extra blank line after the directive to avoid making all following lines
368     // off by one. We can do better by simply incrementing NewLine here.
369     NewLine += 1;
370   }
371 
372   CurLine = NewLine;
373 
374   // In KeepSystemIncludes mode, redirect OS as needed.
375   if (KeepSystemIncludes && (isSystem(FileType) != isSystem(NewFileType)))
376     OS = isSystem(FileType) ? OrigOS : NullOS.get();
377 
378   CurFilename.clear();
379   CurFilename += UserLoc.getFilename();
380   FileType = NewFileType;
381 
382   if (DisableLineMarkers) {
383     if (!MinimizeWhitespace)
384       startNewLineIfNeeded();
385     return;
386   }
387 
388   if (!Initialized) {
389     WriteLineInfo(CurLine);
390     Initialized = true;
391   }
392 
393   // Do not emit an enter marker for the main file (which we expect is the first
394   // entered file). This matches gcc, and improves compatibility with some tools
395   // which track the # line markers as a way to determine when the preprocessed
396   // output is in the context of the main file.
397   if (Reason == PPCallbacks::EnterFile && !IsFirstFileEntered) {
398     IsFirstFileEntered = true;
399     return;
400   }
401 
402   switch (Reason) {
403   case PPCallbacks::EnterFile:
404     WriteLineInfo(CurLine, " 1", 2);
405     break;
406   case PPCallbacks::ExitFile:
407     WriteLineInfo(CurLine, " 2", 2);
408     break;
409   case PPCallbacks::SystemHeaderPragma:
410   case PPCallbacks::RenameFile:
411     WriteLineInfo(CurLine);
412     break;
413   }
414 }
415 
416 void PrintPPOutputPPCallbacks::EmbedDirective(
417     SourceLocation HashLoc, StringRef FileName, bool IsAngled,
418     OptionalFileEntryRef File, const LexEmbedParametersResult &Params) {
419   if (!DumpEmbedDirectives)
420     return;
421 
422   // The EmbedDirective() callback is called before we produce the annotation
423   // token stream for the directive. We skip printing the annotation tokens
424   // within PrintPreprocessedTokens(), but we also need to skip the prefix,
425   // suffix, and if_empty tokens as those are inserted directly into the token
426   // stream and would otherwise be printed immediately after printing the
427   // #embed directive.
428   //
429   // FIXME: counting tokens to skip is a kludge but we have no way to know
430   // which tokens were inserted as part of the embed and which ones were
431   // explicitly written by the user.
432   MoveToLine(HashLoc, /*RequireStartOfLine=*/true);
433   *OS << "#embed " << (IsAngled ? '<' : '"') << FileName
434       << (IsAngled ? '>' : '"');
435 
436   auto PrintToks = [&](llvm::ArrayRef<Token> Toks) {
437     SmallString<128> SpellingBuffer;
438     for (const Token &T : Toks) {
439       if (T.hasLeadingSpace())
440         *OS << " ";
441       *OS << PP.getSpelling(T, SpellingBuffer);
442     }
443   };
444   bool SkipAnnotToks = true;
445   if (Params.MaybeIfEmptyParam) {
446     *OS << " if_empty(";
447     PrintToks(Params.MaybeIfEmptyParam->Tokens);
448     *OS << ")";
449     // If the file is empty, we can skip those tokens. If the file is not
450     // empty, we skip the annotation tokens.
451     if (File && !File->getSize()) {
452       NumToksToSkip += Params.MaybeIfEmptyParam->Tokens.size();
453       SkipAnnotToks = false;
454     }
455   }
456 
457   if (Params.MaybeLimitParam) {
458     *OS << " limit(" << Params.MaybeLimitParam->Limit << ")";
459   }
460   if (Params.MaybeOffsetParam) {
461     *OS << " clang::offset(" << Params.MaybeOffsetParam->Offset << ")";
462   }
463   if (Params.MaybePrefixParam) {
464     *OS << " prefix(";
465     PrintToks(Params.MaybePrefixParam->Tokens);
466     *OS << ")";
467     NumToksToSkip += Params.MaybePrefixParam->Tokens.size();
468   }
469   if (Params.MaybeSuffixParam) {
470     *OS << " suffix(";
471     PrintToks(Params.MaybeSuffixParam->Tokens);
472     *OS << ")";
473     NumToksToSkip += Params.MaybeSuffixParam->Tokens.size();
474   }
475 
476   // We may need to skip the annotation token.
477   if (SkipAnnotToks)
478     NumToksToSkip++;
479 
480   *OS << " /* clang -E -dE */";
481   setEmittedDirectiveOnThisLine();
482 }
483 
484 void PrintPPOutputPPCallbacks::InclusionDirective(
485     SourceLocation HashLoc, const Token &IncludeTok, StringRef FileName,
486     bool IsAngled, CharSourceRange FilenameRange, OptionalFileEntryRef File,
487     StringRef SearchPath, StringRef RelativePath, const Module *SuggestedModule,
488     bool ModuleImported, SrcMgr::CharacteristicKind FileType) {
489   // In -dI mode, dump #include directives prior to dumping their content or
490   // interpretation. Similar for -fkeep-system-includes.
491   if (DumpIncludeDirectives || (KeepSystemIncludes && isSystem(FileType))) {
492     MoveToLine(HashLoc, /*RequireStartOfLine=*/true);
493     const std::string TokenText = PP.getSpelling(IncludeTok);
494     assert(!TokenText.empty());
495     *OS << "#" << TokenText << " "
496         << (IsAngled ? '<' : '"') << FileName << (IsAngled ? '>' : '"')
497         << " /* clang -E "
498         << (DumpIncludeDirectives ? "-dI" : "-fkeep-system-includes")
499         << " */";
500     setEmittedDirectiveOnThisLine();
501   }
502 
503   // When preprocessing, turn implicit imports into module import pragmas.
504   if (ModuleImported) {
505     switch (IncludeTok.getIdentifierInfo()->getPPKeywordID()) {
506     case tok::pp_include:
507     case tok::pp_import:
508     case tok::pp_include_next:
509       MoveToLine(HashLoc, /*RequireStartOfLine=*/true);
510       *OS << "#pragma clang module import "
511           << SuggestedModule->getFullModuleName(true)
512           << " /* clang -E: implicit import for "
513           << "#" << PP.getSpelling(IncludeTok) << " "
514           << (IsAngled ? '<' : '"') << FileName << (IsAngled ? '>' : '"')
515           << " */";
516       setEmittedDirectiveOnThisLine();
517       break;
518 
519     case tok::pp___include_macros:
520       // #__include_macros has no effect on a user of a preprocessed source
521       // file; the only effect is on preprocessing.
522       //
523       // FIXME: That's not *quite* true: it causes the module in question to
524       // be loaded, which can affect downstream diagnostics.
525       break;
526 
527     default:
528       llvm_unreachable("unknown include directive kind");
529       break;
530     }
531   }
532 }
533 
534 /// Handle entering the scope of a module during a module compilation.
535 void PrintPPOutputPPCallbacks::BeginModule(const Module *M) {
536   startNewLineIfNeeded();
537   *OS << "#pragma clang module begin " << M->getFullModuleName(true);
538   setEmittedDirectiveOnThisLine();
539 }
540 
541 /// Handle leaving the scope of a module during a module compilation.
542 void PrintPPOutputPPCallbacks::EndModule(const Module *M) {
543   startNewLineIfNeeded();
544   *OS << "#pragma clang module end /*" << M->getFullModuleName(true) << "*/";
545   setEmittedDirectiveOnThisLine();
546 }
547 
548 /// Ident - Handle #ident directives when read by the preprocessor.
549 ///
550 void PrintPPOutputPPCallbacks::Ident(SourceLocation Loc, StringRef S) {
551   MoveToLine(Loc, /*RequireStartOfLine=*/true);
552 
553   OS->write("#ident ", strlen("#ident "));
554   OS->write(S.begin(), S.size());
555   setEmittedTokensOnThisLine();
556 }
557 
558 /// MacroDefined - This hook is called whenever a macro definition is seen.
559 void PrintPPOutputPPCallbacks::MacroDefined(const Token &MacroNameTok,
560                                             const MacroDirective *MD) {
561   const MacroInfo *MI = MD->getMacroInfo();
562   // Print out macro definitions in -dD mode and when we have -fdirectives-only
563   // for C++20 header units.
564   if ((!DumpDefines && !DirectivesOnly) ||
565       // Ignore __FILE__ etc.
566       MI->isBuiltinMacro())
567     return;
568 
569   SourceLocation DefLoc = MI->getDefinitionLoc();
570   if (DirectivesOnly && !MI->isUsed()) {
571     SourceManager &SM = PP.getSourceManager();
572     if (SM.isWrittenInBuiltinFile(DefLoc) ||
573         SM.isWrittenInCommandLineFile(DefLoc))
574       return;
575   }
576   MoveToLine(DefLoc, /*RequireStartOfLine=*/true);
577   PrintMacroDefinition(*MacroNameTok.getIdentifierInfo(), *MI, PP, OS);
578   setEmittedDirectiveOnThisLine();
579 }
580 
581 void PrintPPOutputPPCallbacks::MacroUndefined(const Token &MacroNameTok,
582                                               const MacroDefinition &MD,
583                                               const MacroDirective *Undef) {
584   // Print out macro definitions in -dD mode and when we have -fdirectives-only
585   // for C++20 header units.
586   if (!DumpDefines && !DirectivesOnly)
587     return;
588 
589   MoveToLine(MacroNameTok.getLocation(), /*RequireStartOfLine=*/true);
590   *OS << "#undef " << MacroNameTok.getIdentifierInfo()->getName();
591   setEmittedDirectiveOnThisLine();
592 }
593 
594 static void outputPrintable(raw_ostream *OS, StringRef Str) {
595   for (unsigned char Char : Str) {
596     if (isPrintable(Char) && Char != '\\' && Char != '"')
597       *OS << (char)Char;
598     else // Output anything hard as an octal escape.
599       *OS << '\\'
600           << (char)('0' + ((Char >> 6) & 7))
601           << (char)('0' + ((Char >> 3) & 7))
602           << (char)('0' + ((Char >> 0) & 7));
603   }
604 }
605 
606 void PrintPPOutputPPCallbacks::PragmaMessage(SourceLocation Loc,
607                                              StringRef Namespace,
608                                              PragmaMessageKind Kind,
609                                              StringRef Str) {
610   MoveToLine(Loc, /*RequireStartOfLine=*/true);
611   *OS << "#pragma ";
612   if (!Namespace.empty())
613     *OS << Namespace << ' ';
614   switch (Kind) {
615     case PMK_Message:
616       *OS << "message(\"";
617       break;
618     case PMK_Warning:
619       *OS << "warning \"";
620       break;
621     case PMK_Error:
622       *OS << "error \"";
623       break;
624   }
625 
626   outputPrintable(OS, Str);
627   *OS << '"';
628   if (Kind == PMK_Message)
629     *OS << ')';
630   setEmittedDirectiveOnThisLine();
631 }
632 
633 void PrintPPOutputPPCallbacks::PragmaDebug(SourceLocation Loc,
634                                            StringRef DebugType) {
635   MoveToLine(Loc, /*RequireStartOfLine=*/true);
636 
637   *OS << "#pragma clang __debug ";
638   *OS << DebugType;
639 
640   setEmittedDirectiveOnThisLine();
641 }
642 
643 void PrintPPOutputPPCallbacks::
644 PragmaDiagnosticPush(SourceLocation Loc, StringRef Namespace) {
645   MoveToLine(Loc, /*RequireStartOfLine=*/true);
646   *OS << "#pragma " << Namespace << " diagnostic push";
647   setEmittedDirectiveOnThisLine();
648 }
649 
650 void PrintPPOutputPPCallbacks::
651 PragmaDiagnosticPop(SourceLocation Loc, StringRef Namespace) {
652   MoveToLine(Loc, /*RequireStartOfLine=*/true);
653   *OS << "#pragma " << Namespace << " diagnostic pop";
654   setEmittedDirectiveOnThisLine();
655 }
656 
657 void PrintPPOutputPPCallbacks::PragmaDiagnostic(SourceLocation Loc,
658                                                 StringRef Namespace,
659                                                 diag::Severity Map,
660                                                 StringRef Str) {
661   MoveToLine(Loc, /*RequireStartOfLine=*/true);
662   *OS << "#pragma " << Namespace << " diagnostic ";
663   switch (Map) {
664   case diag::Severity::Remark:
665     *OS << "remark";
666     break;
667   case diag::Severity::Warning:
668     *OS << "warning";
669     break;
670   case diag::Severity::Error:
671     *OS << "error";
672     break;
673   case diag::Severity::Ignored:
674     *OS << "ignored";
675     break;
676   case diag::Severity::Fatal:
677     *OS << "fatal";
678     break;
679   }
680   *OS << " \"" << Str << '"';
681   setEmittedDirectiveOnThisLine();
682 }
683 
684 void PrintPPOutputPPCallbacks::PragmaWarning(SourceLocation Loc,
685                                              PragmaWarningSpecifier WarningSpec,
686                                              ArrayRef<int> Ids) {
687   MoveToLine(Loc, /*RequireStartOfLine=*/true);
688 
689   *OS << "#pragma warning(";
690   switch(WarningSpec) {
691     case PWS_Default:  *OS << "default"; break;
692     case PWS_Disable:  *OS << "disable"; break;
693     case PWS_Error:    *OS << "error"; break;
694     case PWS_Once:     *OS << "once"; break;
695     case PWS_Suppress: *OS << "suppress"; break;
696     case PWS_Level1:   *OS << '1'; break;
697     case PWS_Level2:   *OS << '2'; break;
698     case PWS_Level3:   *OS << '3'; break;
699     case PWS_Level4:   *OS << '4'; break;
700   }
701   *OS << ':';
702 
703   for (ArrayRef<int>::iterator I = Ids.begin(), E = Ids.end(); I != E; ++I)
704     *OS << ' ' << *I;
705   *OS << ')';
706   setEmittedDirectiveOnThisLine();
707 }
708 
709 void PrintPPOutputPPCallbacks::PragmaWarningPush(SourceLocation Loc,
710                                                  int Level) {
711   MoveToLine(Loc, /*RequireStartOfLine=*/true);
712   *OS << "#pragma warning(push";
713   if (Level >= 0)
714     *OS << ", " << Level;
715   *OS << ')';
716   setEmittedDirectiveOnThisLine();
717 }
718 
719 void PrintPPOutputPPCallbacks::PragmaWarningPop(SourceLocation Loc) {
720   MoveToLine(Loc, /*RequireStartOfLine=*/true);
721   *OS << "#pragma warning(pop)";
722   setEmittedDirectiveOnThisLine();
723 }
724 
725 void PrintPPOutputPPCallbacks::PragmaExecCharsetPush(SourceLocation Loc,
726                                                      StringRef Str) {
727   MoveToLine(Loc, /*RequireStartOfLine=*/true);
728   *OS << "#pragma character_execution_set(push";
729   if (!Str.empty())
730     *OS << ", " << Str;
731   *OS << ')';
732   setEmittedDirectiveOnThisLine();
733 }
734 
735 void PrintPPOutputPPCallbacks::PragmaExecCharsetPop(SourceLocation Loc) {
736   MoveToLine(Loc, /*RequireStartOfLine=*/true);
737   *OS << "#pragma character_execution_set(pop)";
738   setEmittedDirectiveOnThisLine();
739 }
740 
741 void PrintPPOutputPPCallbacks::
742 PragmaAssumeNonNullBegin(SourceLocation Loc) {
743   MoveToLine(Loc, /*RequireStartOfLine=*/true);
744   *OS << "#pragma clang assume_nonnull begin";
745   setEmittedDirectiveOnThisLine();
746 }
747 
748 void PrintPPOutputPPCallbacks::
749 PragmaAssumeNonNullEnd(SourceLocation Loc) {
750   MoveToLine(Loc, /*RequireStartOfLine=*/true);
751   *OS << "#pragma clang assume_nonnull end";
752   setEmittedDirectiveOnThisLine();
753 }
754 
755 void PrintPPOutputPPCallbacks::HandleWhitespaceBeforeTok(const Token &Tok,
756                                                          bool RequireSpace,
757                                                          bool RequireSameLine) {
758   // These tokens are not expanded to anything and don't need whitespace before
759   // them.
760   if (Tok.is(tok::eof) ||
761       (Tok.isAnnotation() && !Tok.is(tok::annot_header_unit) &&
762        !Tok.is(tok::annot_module_begin) && !Tok.is(tok::annot_module_end) &&
763        !Tok.is(tok::annot_repl_input_end) && !Tok.is(tok::annot_embed)))
764     return;
765 
766   // EmittedDirectiveOnThisLine takes priority over RequireSameLine.
767   if ((!RequireSameLine || EmittedDirectiveOnThisLine) &&
768       MoveToLine(Tok, /*RequireStartOfLine=*/EmittedDirectiveOnThisLine)) {
769     if (MinimizeWhitespace) {
770       // Avoid interpreting hash as a directive under -fpreprocessed.
771       if (Tok.is(tok::hash))
772         *OS << ' ';
773     } else {
774       // Print out space characters so that the first token on a line is
775       // indented for easy reading.
776       unsigned ColNo = SM.getExpansionColumnNumber(Tok.getLocation());
777 
778       // The first token on a line can have a column number of 1, yet still
779       // expect leading white space, if a macro expansion in column 1 starts
780       // with an empty macro argument, or an empty nested macro expansion. In
781       // this case, move the token to column 2.
782       if (ColNo == 1 && Tok.hasLeadingSpace())
783         ColNo = 2;
784 
785       // This hack prevents stuff like:
786       // #define HASH #
787       // HASH define foo bar
788       // From having the # character end up at column 1, which makes it so it
789       // is not handled as a #define next time through the preprocessor if in
790       // -fpreprocessed mode.
791       if (ColNo <= 1 && Tok.is(tok::hash))
792         *OS << ' ';
793 
794       // Otherwise, indent the appropriate number of spaces.
795       for (; ColNo > 1; --ColNo)
796         *OS << ' ';
797     }
798   } else {
799     // Insert whitespace between the previous and next token if either
800     // - The caller requires it
801     // - The input had whitespace between them and we are not in
802     //   whitespace-minimization mode
803     // - The whitespace is necessary to keep the tokens apart and there is not
804     //   already a newline between them
805     if (RequireSpace || (!MinimizeWhitespace && Tok.hasLeadingSpace()) ||
806         ((EmittedTokensOnThisLine || EmittedDirectiveOnThisLine) &&
807          AvoidConcat(PrevPrevTok, PrevTok, Tok)))
808       *OS << ' ';
809   }
810 
811   PrevPrevTok = PrevTok;
812   PrevTok = Tok;
813 }
814 
815 void PrintPPOutputPPCallbacks::HandleNewlinesInToken(const char *TokStr,
816                                                      unsigned Len) {
817   unsigned NumNewlines = 0;
818   for (; Len; --Len, ++TokStr) {
819     if (*TokStr != '\n' &&
820         *TokStr != '\r')
821       continue;
822 
823     ++NumNewlines;
824 
825     // If we have \n\r or \r\n, skip both and count as one line.
826     if (Len != 1 &&
827         (TokStr[1] == '\n' || TokStr[1] == '\r') &&
828         TokStr[0] != TokStr[1]) {
829       ++TokStr;
830       --Len;
831     }
832   }
833 
834   if (NumNewlines == 0) return;
835 
836   CurLine += NumNewlines;
837 }
838 
839 
840 namespace {
841 struct UnknownPragmaHandler : public PragmaHandler {
842   const char *Prefix;
843   PrintPPOutputPPCallbacks *Callbacks;
844 
845   // Set to true if tokens should be expanded
846   bool ShouldExpandTokens;
847 
848   UnknownPragmaHandler(const char *prefix, PrintPPOutputPPCallbacks *callbacks,
849                        bool RequireTokenExpansion)
850       : Prefix(prefix), Callbacks(callbacks),
851         ShouldExpandTokens(RequireTokenExpansion) {}
852   void HandlePragma(Preprocessor &PP, PragmaIntroducer Introducer,
853                     Token &PragmaTok) override {
854     // Figure out what line we went to and insert the appropriate number of
855     // newline characters.
856     Callbacks->MoveToLine(PragmaTok.getLocation(), /*RequireStartOfLine=*/true);
857     Callbacks->OS->write(Prefix, strlen(Prefix));
858     Callbacks->setEmittedTokensOnThisLine();
859 
860     if (ShouldExpandTokens) {
861       // The first token does not have expanded macros. Expand them, if
862       // required.
863       auto Toks = std::make_unique<Token[]>(1);
864       Toks[0] = PragmaTok;
865       PP.EnterTokenStream(std::move(Toks), /*NumToks=*/1,
866                           /*DisableMacroExpansion=*/false,
867                           /*IsReinject=*/false);
868       PP.Lex(PragmaTok);
869     }
870 
871     // Read and print all of the pragma tokens.
872     bool IsFirst = true;
873     while (PragmaTok.isNot(tok::eod)) {
874       Callbacks->HandleWhitespaceBeforeTok(PragmaTok, /*RequireSpace=*/IsFirst,
875                                            /*RequireSameLine=*/true);
876       IsFirst = false;
877       std::string TokSpell = PP.getSpelling(PragmaTok);
878       Callbacks->OS->write(&TokSpell[0], TokSpell.size());
879       Callbacks->setEmittedTokensOnThisLine();
880 
881       if (ShouldExpandTokens)
882         PP.Lex(PragmaTok);
883       else
884         PP.LexUnexpandedToken(PragmaTok);
885     }
886     Callbacks->setEmittedDirectiveOnThisLine();
887   }
888 };
889 } // end anonymous namespace
890 
891 
892 static void PrintPreprocessedTokens(Preprocessor &PP, Token &Tok,
893                                     PrintPPOutputPPCallbacks *Callbacks) {
894   bool DropComments = PP.getLangOpts().TraditionalCPP &&
895                       !PP.getCommentRetentionState();
896 
897   bool IsStartOfLine = false;
898   char Buffer[256];
899   while (true) {
900     // Two lines joined with line continuation ('\' as last character on the
901     // line) must be emitted as one line even though Tok.getLine() returns two
902     // different values. In this situation Tok.isAtStartOfLine() is false even
903     // though it may be the first token on the lexical line. When
904     // dropping/skipping a token that is at the start of a line, propagate the
905     // start-of-line-ness to the next token to not append it to the previous
906     // line.
907     IsStartOfLine = IsStartOfLine || Tok.isAtStartOfLine();
908 
909     Callbacks->HandleWhitespaceBeforeTok(Tok, /*RequireSpace=*/false,
910                                          /*RequireSameLine=*/!IsStartOfLine);
911 
912     if (DropComments && Tok.is(tok::comment)) {
913       // Skip comments. Normally the preprocessor does not generate
914       // tok::comment nodes at all when not keeping comments, but under
915       // -traditional-cpp the lexer keeps /all/ whitespace, including comments.
916       PP.Lex(Tok);
917       continue;
918     } else if (Tok.is(tok::annot_repl_input_end)) {
919       PP.Lex(Tok);
920       continue;
921     } else if (Tok.is(tok::eod)) {
922       // Don't print end of directive tokens, since they are typically newlines
923       // that mess up our line tracking. These come from unknown pre-processor
924       // directives or hash-prefixed comments in standalone assembly files.
925       PP.Lex(Tok);
926       // FIXME: The token on the next line after #include should have
927       // Tok.isAtStartOfLine() set.
928       IsStartOfLine = true;
929       continue;
930     } else if (Tok.is(tok::annot_module_include)) {
931       // PrintPPOutputPPCallbacks::InclusionDirective handles producing
932       // appropriate output here. Ignore this token entirely.
933       PP.Lex(Tok);
934       IsStartOfLine = true;
935       continue;
936     } else if (Tok.is(tok::annot_module_begin)) {
937       // FIXME: We retrieve this token after the FileChanged callback, and
938       // retrieve the module_end token before the FileChanged callback, so
939       // we render this within the file and render the module end outside the
940       // file, but this is backwards from the token locations: the module_begin
941       // token is at the include location (outside the file) and the module_end
942       // token is at the EOF location (within the file).
943       Callbacks->BeginModule(
944           reinterpret_cast<Module *>(Tok.getAnnotationValue()));
945       PP.Lex(Tok);
946       IsStartOfLine = true;
947       continue;
948     } else if (Tok.is(tok::annot_module_end)) {
949       Callbacks->EndModule(
950           reinterpret_cast<Module *>(Tok.getAnnotationValue()));
951       PP.Lex(Tok);
952       IsStartOfLine = true;
953       continue;
954     } else if (Tok.is(tok::annot_header_unit)) {
955       // This is a header-name that has been (effectively) converted into a
956       // module-name.
957       // FIXME: The module name could contain non-identifier module name
958       // components. We don't have a good way to round-trip those.
959       Module *M = reinterpret_cast<Module *>(Tok.getAnnotationValue());
960       std::string Name = M->getFullModuleName();
961       Callbacks->OS->write(Name.data(), Name.size());
962       Callbacks->HandleNewlinesInToken(Name.data(), Name.size());
963     } else if (Tok.is(tok::annot_embed)) {
964       // Manually explode the binary data out to a stream of comma-delimited
965       // integer values. If the user passed -dE, that is handled by the
966       // EmbedDirective() callback. We should only get here if the user did not
967       // pass -dE.
968       assert(Callbacks->expandEmbedContents() &&
969              "did not expect an embed annotation");
970       auto *Data =
971           reinterpret_cast<EmbedAnnotationData *>(Tok.getAnnotationValue());
972 
973       // Loop over the contents and print them as a comma-delimited list of
974       // values.
975       bool PrintComma = false;
976       for (auto Iter = Data->BinaryData.begin(), End = Data->BinaryData.end();
977            Iter != End; ++Iter) {
978         if (PrintComma)
979           *Callbacks->OS << ", ";
980         *Callbacks->OS << static_cast<unsigned>(*Iter);
981         PrintComma = true;
982       }
983       IsStartOfLine = true;
984     } else if (Tok.isAnnotation()) {
985       // Ignore annotation tokens created by pragmas - the pragmas themselves
986       // will be reproduced in the preprocessed output.
987       PP.Lex(Tok);
988       continue;
989     } else if (IdentifierInfo *II = Tok.getIdentifierInfo()) {
990       *Callbacks->OS << II->getName();
991     } else if (Tok.isLiteral() && !Tok.needsCleaning() &&
992                Tok.getLiteralData()) {
993       Callbacks->OS->write(Tok.getLiteralData(), Tok.getLength());
994     } else if (Tok.getLength() < std::size(Buffer)) {
995       const char *TokPtr = Buffer;
996       unsigned Len = PP.getSpelling(Tok, TokPtr);
997       Callbacks->OS->write(TokPtr, Len);
998 
999       // Tokens that can contain embedded newlines need to adjust our current
1000       // line number.
1001       // FIXME: The token may end with a newline in which case
1002       // setEmittedDirectiveOnThisLine/setEmittedTokensOnThisLine afterwards is
1003       // wrong.
1004       if (Tok.getKind() == tok::comment || Tok.getKind() == tok::unknown)
1005         Callbacks->HandleNewlinesInToken(TokPtr, Len);
1006       if (Tok.is(tok::comment) && Len >= 2 && TokPtr[0] == '/' &&
1007           TokPtr[1] == '/') {
1008         // It's a line comment;
1009         // Ensure that we don't concatenate anything behind it.
1010         Callbacks->setEmittedDirectiveOnThisLine();
1011       }
1012     } else {
1013       std::string S = PP.getSpelling(Tok);
1014       Callbacks->OS->write(S.data(), S.size());
1015 
1016       // Tokens that can contain embedded newlines need to adjust our current
1017       // line number.
1018       if (Tok.getKind() == tok::comment || Tok.getKind() == tok::unknown)
1019         Callbacks->HandleNewlinesInToken(S.data(), S.size());
1020       if (Tok.is(tok::comment) && S.size() >= 2 && S[0] == '/' && S[1] == '/') {
1021         // It's a line comment;
1022         // Ensure that we don't concatenate anything behind it.
1023         Callbacks->setEmittedDirectiveOnThisLine();
1024       }
1025     }
1026     Callbacks->setEmittedTokensOnThisLine();
1027     IsStartOfLine = false;
1028 
1029     if (Tok.is(tok::eof)) break;
1030 
1031     PP.Lex(Tok);
1032     // If lexing that token causes us to need to skip future tokens, do so now.
1033     for (unsigned I = 0, Skip = Callbacks->GetNumToksToSkip(); I < Skip; ++I)
1034       PP.Lex(Tok);
1035     Callbacks->ResetSkipToks();
1036   }
1037 }
1038 
1039 typedef std::pair<const IdentifierInfo *, MacroInfo *> id_macro_pair;
1040 static int MacroIDCompare(const id_macro_pair *LHS, const id_macro_pair *RHS) {
1041   return LHS->first->getName().compare(RHS->first->getName());
1042 }
1043 
1044 static void DoPrintMacros(Preprocessor &PP, raw_ostream *OS) {
1045   // Ignore unknown pragmas.
1046   PP.IgnorePragmas();
1047 
1048   // -dM mode just scans and ignores all tokens in the files, then dumps out
1049   // the macro table at the end.
1050   PP.EnterMainSourceFile();
1051 
1052   Token Tok;
1053   do PP.Lex(Tok);
1054   while (Tok.isNot(tok::eof));
1055 
1056   SmallVector<id_macro_pair, 128> MacrosByID;
1057   for (Preprocessor::macro_iterator I = PP.macro_begin(), E = PP.macro_end();
1058        I != E; ++I) {
1059     auto *MD = I->second.getLatest();
1060     if (MD && MD->isDefined())
1061       MacrosByID.push_back(id_macro_pair(I->first, MD->getMacroInfo()));
1062   }
1063   llvm::array_pod_sort(MacrosByID.begin(), MacrosByID.end(), MacroIDCompare);
1064 
1065   for (unsigned i = 0, e = MacrosByID.size(); i != e; ++i) {
1066     MacroInfo &MI = *MacrosByID[i].second;
1067     // Ignore computed macros like __LINE__ and friends.
1068     if (MI.isBuiltinMacro()) continue;
1069 
1070     PrintMacroDefinition(*MacrosByID[i].first, MI, PP, OS);
1071     *OS << '\n';
1072   }
1073 }
1074 
1075 /// DoPrintPreprocessedInput - This implements -E mode.
1076 ///
1077 void clang::DoPrintPreprocessedInput(Preprocessor &PP, raw_ostream *OS,
1078                                      const PreprocessorOutputOptions &Opts) {
1079   // Show macros with no output is handled specially.
1080   if (!Opts.ShowCPP) {
1081     assert(Opts.ShowMacros && "Not yet implemented!");
1082     DoPrintMacros(PP, OS);
1083     return;
1084   }
1085 
1086   // Inform the preprocessor whether we want it to retain comments or not, due
1087   // to -C or -CC.
1088   PP.SetCommentRetentionState(Opts.ShowComments, Opts.ShowMacroComments);
1089 
1090   PrintPPOutputPPCallbacks *Callbacks = new PrintPPOutputPPCallbacks(
1091       PP, OS, !Opts.ShowLineMarkers, Opts.ShowMacros,
1092       Opts.ShowIncludeDirectives, Opts.ShowEmbedDirectives,
1093       Opts.UseLineDirectives, Opts.MinimizeWhitespace, Opts.DirectivesOnly,
1094       Opts.KeepSystemIncludes);
1095 
1096   // Expand macros in pragmas with -fms-extensions.  The assumption is that
1097   // the majority of pragmas in such a file will be Microsoft pragmas.
1098   // Remember the handlers we will add so that we can remove them later.
1099   std::unique_ptr<UnknownPragmaHandler> MicrosoftExtHandler(
1100       new UnknownPragmaHandler(
1101           "#pragma", Callbacks,
1102           /*RequireTokenExpansion=*/PP.getLangOpts().MicrosoftExt));
1103 
1104   std::unique_ptr<UnknownPragmaHandler> GCCHandler(new UnknownPragmaHandler(
1105       "#pragma GCC", Callbacks,
1106       /*RequireTokenExpansion=*/PP.getLangOpts().MicrosoftExt));
1107 
1108   std::unique_ptr<UnknownPragmaHandler> ClangHandler(new UnknownPragmaHandler(
1109       "#pragma clang", Callbacks,
1110       /*RequireTokenExpansion=*/PP.getLangOpts().MicrosoftExt));
1111 
1112   PP.AddPragmaHandler(MicrosoftExtHandler.get());
1113   PP.AddPragmaHandler("GCC", GCCHandler.get());
1114   PP.AddPragmaHandler("clang", ClangHandler.get());
1115 
1116   // The tokens after pragma omp need to be expanded.
1117   //
1118   //  OpenMP [2.1, Directive format]
1119   //  Preprocessing tokens following the #pragma omp are subject to macro
1120   //  replacement.
1121   std::unique_ptr<UnknownPragmaHandler> OpenMPHandler(
1122       new UnknownPragmaHandler("#pragma omp", Callbacks,
1123                                /*RequireTokenExpansion=*/true));
1124   PP.AddPragmaHandler("omp", OpenMPHandler.get());
1125 
1126   PP.addPPCallbacks(std::unique_ptr<PPCallbacks>(Callbacks));
1127 
1128   // After we have configured the preprocessor, enter the main file.
1129   PP.EnterMainSourceFile();
1130   if (Opts.DirectivesOnly)
1131     PP.SetMacroExpansionOnlyInDirectives();
1132 
1133   // Consume all of the tokens that come from the predefines buffer.  Those
1134   // should not be emitted into the output and are guaranteed to be at the
1135   // start.
1136   const SourceManager &SourceMgr = PP.getSourceManager();
1137   Token Tok;
1138   do {
1139     PP.Lex(Tok);
1140     if (Tok.is(tok::eof) || !Tok.getLocation().isFileID())
1141       break;
1142 
1143     PresumedLoc PLoc = SourceMgr.getPresumedLoc(Tok.getLocation());
1144     if (PLoc.isInvalid())
1145       break;
1146 
1147     if (strcmp(PLoc.getFilename(), "<built-in>"))
1148       break;
1149   } while (true);
1150 
1151   // Read all the preprocessed tokens, printing them out to the stream.
1152   PrintPreprocessedTokens(PP, Tok, Callbacks);
1153   *OS << '\n';
1154 
1155   // Remove the handlers we just added to leave the preprocessor in a sane state
1156   // so that it can be reused (for example by a clang::Parser instance).
1157   PP.RemovePragmaHandler(MicrosoftExtHandler.get());
1158   PP.RemovePragmaHandler("GCC", GCCHandler.get());
1159   PP.RemovePragmaHandler("clang", ClangHandler.get());
1160   PP.RemovePragmaHandler("omp", OpenMPHandler.get());
1161 }
1162