xref: /freebsd/contrib/llvm-project/clang/lib/Frontend/Rewrite/InclusionRewriter.cpp (revision 2f9966ff63d65bd474478888c9088eeae3f9c669)
1 //===--- InclusionRewriter.cpp - Rewrite includes into their expansions ---===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This code rewrites include invocations into their expansions.  This gives you
10 // a file with all included files merged into it.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "clang/Rewrite/Frontend/Rewriters.h"
15 #include "clang/Basic/SourceManager.h"
16 #include "clang/Frontend/PreprocessorOutputOptions.h"
17 #include "clang/Lex/Pragma.h"
18 #include "clang/Lex/Preprocessor.h"
19 #include "llvm/ADT/SmallString.h"
20 #include "llvm/Support/raw_ostream.h"
21 #include <optional>
22 
23 using namespace clang;
24 using namespace llvm;
25 
26 namespace {
27 
28 class InclusionRewriter : public PPCallbacks {
29   /// Information about which #includes were actually performed,
30   /// created by preprocessor callbacks.
31   struct IncludedFile {
32     FileID Id;
33     SrcMgr::CharacteristicKind FileType;
34     IncludedFile(FileID Id, SrcMgr::CharacteristicKind FileType)
35         : Id(Id), FileType(FileType) {}
36   };
37   Preprocessor &PP; ///< Used to find inclusion directives.
38   SourceManager &SM; ///< Used to read and manage source files.
39   raw_ostream &OS; ///< The destination stream for rewritten contents.
40   StringRef MainEOL; ///< The line ending marker to use.
41   llvm::MemoryBufferRef PredefinesBuffer; ///< The preprocessor predefines.
42   bool ShowLineMarkers; ///< Show #line markers.
43   bool UseLineDirectives; ///< Use of line directives or line markers.
44   /// Tracks where inclusions that change the file are found.
45   std::map<SourceLocation, IncludedFile> FileIncludes;
46   /// Tracks where inclusions that import modules are found.
47   std::map<SourceLocation, const Module *> ModuleIncludes;
48   /// Tracks where inclusions that enter modules (in a module build) are found.
49   std::map<SourceLocation, const Module *> ModuleEntryIncludes;
50   /// Tracks where #if and #elif directives get evaluated and whether to true.
51   std::map<SourceLocation, bool> IfConditions;
52   /// Used transitively for building up the FileIncludes mapping over the
53   /// various \c PPCallbacks callbacks.
54   SourceLocation LastInclusionLocation;
55 public:
56   InclusionRewriter(Preprocessor &PP, raw_ostream &OS, bool ShowLineMarkers,
57                     bool UseLineDirectives);
58   void Process(FileID FileId, SrcMgr::CharacteristicKind FileType);
59   void setPredefinesBuffer(const llvm::MemoryBufferRef &Buf) {
60     PredefinesBuffer = Buf;
61   }
62   void detectMainFileEOL();
63   void handleModuleBegin(Token &Tok) {
64     assert(Tok.getKind() == tok::annot_module_begin);
65     ModuleEntryIncludes.insert(
66         {Tok.getLocation(), (Module *)Tok.getAnnotationValue()});
67   }
68 private:
69   void FileChanged(SourceLocation Loc, FileChangeReason Reason,
70                    SrcMgr::CharacteristicKind FileType,
71                    FileID PrevFID) override;
72   void FileSkipped(const FileEntryRef &SkippedFile, const Token &FilenameTok,
73                    SrcMgr::CharacteristicKind FileType) override;
74   void InclusionDirective(SourceLocation HashLoc, const Token &IncludeTok,
75                           StringRef FileName, bool IsAngled,
76                           CharSourceRange FilenameRange,
77                           OptionalFileEntryRef File, StringRef SearchPath,
78                           StringRef RelativePath, const Module *Imported,
79                           SrcMgr::CharacteristicKind FileType) override;
80   void If(SourceLocation Loc, SourceRange ConditionRange,
81           ConditionValueKind ConditionValue) override;
82   void Elif(SourceLocation Loc, SourceRange ConditionRange,
83             ConditionValueKind ConditionValue, SourceLocation IfLoc) override;
84   void WriteLineInfo(StringRef Filename, int Line,
85                      SrcMgr::CharacteristicKind FileType,
86                      StringRef Extra = StringRef());
87   void WriteImplicitModuleImport(const Module *Mod);
88   void OutputContentUpTo(const MemoryBufferRef &FromFile, unsigned &WriteFrom,
89                          unsigned WriteTo, StringRef EOL, int &lines,
90                          bool EnsureNewline);
91   void CommentOutDirective(Lexer &DirectivesLex, const Token &StartToken,
92                            const MemoryBufferRef &FromFile, StringRef EOL,
93                            unsigned &NextToWrite, int &Lines,
94                            const IncludedFile *Inc = nullptr);
95   const IncludedFile *FindIncludeAtLocation(SourceLocation Loc) const;
96   StringRef getIncludedFileName(const IncludedFile *Inc) const;
97   const Module *FindModuleAtLocation(SourceLocation Loc) const;
98   const Module *FindEnteredModule(SourceLocation Loc) const;
99   bool IsIfAtLocationTrue(SourceLocation Loc) const;
100   StringRef NextIdentifierName(Lexer &RawLex, Token &RawToken);
101 };
102 
103 }  // end anonymous namespace
104 
105 /// Initializes an InclusionRewriter with a \p PP source and \p OS destination.
106 InclusionRewriter::InclusionRewriter(Preprocessor &PP, raw_ostream &OS,
107                                      bool ShowLineMarkers,
108                                      bool UseLineDirectives)
109     : PP(PP), SM(PP.getSourceManager()), OS(OS), MainEOL("\n"),
110       ShowLineMarkers(ShowLineMarkers), UseLineDirectives(UseLineDirectives),
111       LastInclusionLocation(SourceLocation()) {}
112 
113 /// Write appropriate line information as either #line directives or GNU line
114 /// markers depending on what mode we're in, including the \p Filename and
115 /// \p Line we are located at, using the specified \p EOL line separator, and
116 /// any \p Extra context specifiers in GNU line directives.
117 void InclusionRewriter::WriteLineInfo(StringRef Filename, int Line,
118                                       SrcMgr::CharacteristicKind FileType,
119                                       StringRef Extra) {
120   if (!ShowLineMarkers)
121     return;
122   if (UseLineDirectives) {
123     OS << "#line" << ' ' << Line << ' ' << '"';
124     OS.write_escaped(Filename);
125     OS << '"';
126   } else {
127     // Use GNU linemarkers as described here:
128     // http://gcc.gnu.org/onlinedocs/cpp/Preprocessor-Output.html
129     OS << '#' << ' ' << Line << ' ' << '"';
130     OS.write_escaped(Filename);
131     OS << '"';
132     if (!Extra.empty())
133       OS << Extra;
134     if (FileType == SrcMgr::C_System)
135       // "`3' This indicates that the following text comes from a system header
136       // file, so certain warnings should be suppressed."
137       OS << " 3";
138     else if (FileType == SrcMgr::C_ExternCSystem)
139       // as above for `3', plus "`4' This indicates that the following text
140       // should be treated as being wrapped in an implicit extern "C" block."
141       OS << " 3 4";
142   }
143   OS << MainEOL;
144 }
145 
146 void InclusionRewriter::WriteImplicitModuleImport(const Module *Mod) {
147   OS << "#pragma clang module import " << Mod->getFullModuleName(true)
148      << " /* clang -frewrite-includes: implicit import */" << MainEOL;
149 }
150 
151 /// FileChanged - Whenever the preprocessor enters or exits a #include file
152 /// it invokes this handler.
153 void InclusionRewriter::FileChanged(SourceLocation Loc,
154                                     FileChangeReason Reason,
155                                     SrcMgr::CharacteristicKind NewFileType,
156                                     FileID) {
157   if (Reason != EnterFile)
158     return;
159   if (LastInclusionLocation.isInvalid())
160     // we didn't reach this file (eg: the main file) via an inclusion directive
161     return;
162   FileID Id = FullSourceLoc(Loc, SM).getFileID();
163   auto P = FileIncludes.insert(
164       std::make_pair(LastInclusionLocation, IncludedFile(Id, NewFileType)));
165   (void)P;
166   assert(P.second && "Unexpected revisitation of the same include directive");
167   LastInclusionLocation = SourceLocation();
168 }
169 
170 /// Called whenever an inclusion is skipped due to canonical header protection
171 /// macros.
172 void InclusionRewriter::FileSkipped(const FileEntryRef & /*SkippedFile*/,
173                                     const Token & /*FilenameTok*/,
174                                     SrcMgr::CharacteristicKind /*FileType*/) {
175   assert(LastInclusionLocation.isValid() &&
176          "A file, that wasn't found via an inclusion directive, was skipped");
177   LastInclusionLocation = SourceLocation();
178 }
179 
180 /// This should be called whenever the preprocessor encounters include
181 /// directives. It does not say whether the file has been included, but it
182 /// provides more information about the directive (hash location instead
183 /// of location inside the included file). It is assumed that the matching
184 /// FileChanged() or FileSkipped() is called after this (or neither is
185 /// called if this #include results in an error or does not textually include
186 /// anything).
187 void InclusionRewriter::InclusionDirective(
188     SourceLocation HashLoc, const Token & /*IncludeTok*/,
189     StringRef /*FileName*/, bool /*IsAngled*/,
190     CharSourceRange /*FilenameRange*/, OptionalFileEntryRef /*File*/,
191     StringRef /*SearchPath*/, StringRef /*RelativePath*/,
192     const Module *Imported, SrcMgr::CharacteristicKind FileType) {
193   if (Imported) {
194     auto P = ModuleIncludes.insert(std::make_pair(HashLoc, Imported));
195     (void)P;
196     assert(P.second && "Unexpected revisitation of the same include directive");
197   } else
198     LastInclusionLocation = HashLoc;
199 }
200 
201 void InclusionRewriter::If(SourceLocation Loc, SourceRange ConditionRange,
202                            ConditionValueKind ConditionValue) {
203   auto P = IfConditions.insert(std::make_pair(Loc, ConditionValue == CVK_True));
204   (void)P;
205   assert(P.second && "Unexpected revisitation of the same if directive");
206 }
207 
208 void InclusionRewriter::Elif(SourceLocation Loc, SourceRange ConditionRange,
209                              ConditionValueKind ConditionValue,
210                              SourceLocation IfLoc) {
211   auto P = IfConditions.insert(std::make_pair(Loc, ConditionValue == CVK_True));
212   (void)P;
213   assert(P.second && "Unexpected revisitation of the same elif directive");
214 }
215 
216 /// Simple lookup for a SourceLocation (specifically one denoting the hash in
217 /// an inclusion directive) in the map of inclusion information, FileChanges.
218 const InclusionRewriter::IncludedFile *
219 InclusionRewriter::FindIncludeAtLocation(SourceLocation Loc) const {
220   const auto I = FileIncludes.find(Loc);
221   if (I != FileIncludes.end())
222     return &I->second;
223   return nullptr;
224 }
225 
226 /// Simple lookup for a SourceLocation (specifically one denoting the hash in
227 /// an inclusion directive) in the map of module inclusion information.
228 const Module *
229 InclusionRewriter::FindModuleAtLocation(SourceLocation Loc) const {
230   const auto I = ModuleIncludes.find(Loc);
231   if (I != ModuleIncludes.end())
232     return I->second;
233   return nullptr;
234 }
235 
236 /// Simple lookup for a SourceLocation (specifically one denoting the hash in
237 /// an inclusion directive) in the map of module entry information.
238 const Module *
239 InclusionRewriter::FindEnteredModule(SourceLocation Loc) const {
240   const auto I = ModuleEntryIncludes.find(Loc);
241   if (I != ModuleEntryIncludes.end())
242     return I->second;
243   return nullptr;
244 }
245 
246 bool InclusionRewriter::IsIfAtLocationTrue(SourceLocation Loc) const {
247   const auto I = IfConditions.find(Loc);
248   if (I != IfConditions.end())
249     return I->second;
250   return false;
251 }
252 
253 void InclusionRewriter::detectMainFileEOL() {
254   std::optional<MemoryBufferRef> FromFile =
255       *SM.getBufferOrNone(SM.getMainFileID());
256   assert(FromFile);
257   if (!FromFile)
258     return; // Should never happen, but whatever.
259   MainEOL = FromFile->getBuffer().detectEOL();
260 }
261 
262 /// Writes out bytes from \p FromFile, starting at \p NextToWrite and ending at
263 /// \p WriteTo - 1.
264 void InclusionRewriter::OutputContentUpTo(const MemoryBufferRef &FromFile,
265                                           unsigned &WriteFrom, unsigned WriteTo,
266                                           StringRef LocalEOL, int &Line,
267                                           bool EnsureNewline) {
268   if (WriteTo <= WriteFrom)
269     return;
270   if (FromFile == PredefinesBuffer) {
271     // Ignore the #defines of the predefines buffer.
272     WriteFrom = WriteTo;
273     return;
274   }
275 
276   // If we would output half of a line ending, advance one character to output
277   // the whole line ending.  All buffers are null terminated, so looking ahead
278   // one byte is safe.
279   if (LocalEOL.size() == 2 &&
280       LocalEOL[0] == (FromFile.getBufferStart() + WriteTo)[-1] &&
281       LocalEOL[1] == (FromFile.getBufferStart() + WriteTo)[0])
282     WriteTo++;
283 
284   StringRef TextToWrite(FromFile.getBufferStart() + WriteFrom,
285                         WriteTo - WriteFrom);
286   // count lines manually, it's faster than getPresumedLoc()
287   Line += TextToWrite.count(LocalEOL);
288 
289   if (MainEOL == LocalEOL) {
290     OS << TextToWrite;
291   } else {
292     // Output the file one line at a time, rewriting the line endings as we go.
293     StringRef Rest = TextToWrite;
294     while (!Rest.empty()) {
295       // Identify and output the next line excluding an EOL sequence if present.
296       size_t Idx = Rest.find(LocalEOL);
297       StringRef LineText = Rest.substr(0, Idx);
298       OS << LineText;
299       if (Idx != StringRef::npos) {
300         // An EOL sequence was present, output the EOL sequence for the
301         // main source file and skip past the local EOL sequence.
302         OS << MainEOL;
303         Idx += LocalEOL.size();
304       }
305       // Strip the line just handled. If Idx is npos or matches the end of the
306       // text, Rest will be set to an empty string and the loop will terminate.
307       Rest = Rest.substr(Idx);
308     }
309   }
310   if (EnsureNewline && !TextToWrite.ends_with(LocalEOL))
311     OS << MainEOL;
312 
313   WriteFrom = WriteTo;
314 }
315 
316 StringRef
317 InclusionRewriter::getIncludedFileName(const IncludedFile *Inc) const {
318   if (Inc) {
319     auto B = SM.getBufferOrNone(Inc->Id);
320     assert(B && "Attempting to process invalid inclusion");
321     if (B)
322       return llvm::sys::path::filename(B->getBufferIdentifier());
323   }
324   return StringRef();
325 }
326 
327 /// Print characters from \p FromFile starting at \p NextToWrite up until the
328 /// inclusion directive at \p StartToken, then print out the inclusion
329 /// inclusion directive disabled by a #if directive, updating \p NextToWrite
330 /// and \p Line to track the number of source lines visited and the progress
331 /// through the \p FromFile buffer.
332 void InclusionRewriter::CommentOutDirective(Lexer &DirectiveLex,
333                                             const Token &StartToken,
334                                             const MemoryBufferRef &FromFile,
335                                             StringRef LocalEOL,
336                                             unsigned &NextToWrite, int &Line,
337                                             const IncludedFile *Inc) {
338   OutputContentUpTo(FromFile, NextToWrite,
339                     SM.getFileOffset(StartToken.getLocation()), LocalEOL, Line,
340                     false);
341   Token DirectiveToken;
342   do {
343     DirectiveLex.LexFromRawLexer(DirectiveToken);
344   } while (!DirectiveToken.is(tok::eod) && DirectiveToken.isNot(tok::eof));
345   if (FromFile == PredefinesBuffer) {
346     // OutputContentUpTo() would not output anything anyway.
347     return;
348   }
349   if (Inc) {
350     OS << "#if defined(__CLANG_REWRITTEN_INCLUDES) ";
351     if (isSystem(Inc->FileType))
352       OS << "|| defined(__CLANG_REWRITTEN_SYSTEM_INCLUDES) ";
353     OS << "/* " << getIncludedFileName(Inc);
354   } else {
355     OS << "#if 0 /*";
356   }
357   OS << " expanded by -frewrite-includes */" << MainEOL;
358   OutputContentUpTo(FromFile, NextToWrite,
359                     SM.getFileOffset(DirectiveToken.getLocation()) +
360                         DirectiveToken.getLength(),
361                     LocalEOL, Line, true);
362   OS << (Inc ? "#else /* " : "#endif /*") << getIncludedFileName(Inc)
363      << " expanded by -frewrite-includes */" << MainEOL;
364 }
365 
366 /// Find the next identifier in the pragma directive specified by \p RawToken.
367 StringRef InclusionRewriter::NextIdentifierName(Lexer &RawLex,
368                                                 Token &RawToken) {
369   RawLex.LexFromRawLexer(RawToken);
370   if (RawToken.is(tok::raw_identifier))
371     PP.LookUpIdentifierInfo(RawToken);
372   if (RawToken.is(tok::identifier))
373     return RawToken.getIdentifierInfo()->getName();
374   return StringRef();
375 }
376 
377 /// Use a raw lexer to analyze \p FileId, incrementally copying parts of it
378 /// and including content of included files recursively.
379 void InclusionRewriter::Process(FileID FileId,
380                                 SrcMgr::CharacteristicKind FileType) {
381   MemoryBufferRef FromFile;
382   {
383     auto B = SM.getBufferOrNone(FileId);
384     assert(B && "Attempting to process invalid inclusion");
385     if (B)
386       FromFile = *B;
387   }
388   StringRef FileName = FromFile.getBufferIdentifier();
389   Lexer RawLex(FileId, FromFile, PP.getSourceManager(), PP.getLangOpts());
390   RawLex.SetCommentRetentionState(false);
391 
392   StringRef LocalEOL = FromFile.getBuffer().detectEOL();
393 
394   // Per the GNU docs: "1" indicates entering a new file.
395   if (FileId == SM.getMainFileID() || FileId == PP.getPredefinesFileID())
396     WriteLineInfo(FileName, 1, FileType, "");
397   else
398     WriteLineInfo(FileName, 1, FileType, " 1");
399 
400   if (SM.getFileIDSize(FileId) == 0)
401     return;
402 
403   // The next byte to be copied from the source file, which may be non-zero if
404   // the lexer handled a BOM.
405   unsigned NextToWrite = SM.getFileOffset(RawLex.getSourceLocation());
406   assert(SM.getLineNumber(FileId, NextToWrite) == 1);
407   int Line = 1; // The current input file line number.
408 
409   Token RawToken;
410   RawLex.LexFromRawLexer(RawToken);
411 
412   // TODO: Consider adding a switch that strips possibly unimportant content,
413   // such as comments, to reduce the size of repro files.
414   while (RawToken.isNot(tok::eof)) {
415     if (RawToken.is(tok::hash) && RawToken.isAtStartOfLine()) {
416       RawLex.setParsingPreprocessorDirective(true);
417       Token HashToken = RawToken;
418       RawLex.LexFromRawLexer(RawToken);
419       if (RawToken.is(tok::raw_identifier))
420         PP.LookUpIdentifierInfo(RawToken);
421       if (RawToken.getIdentifierInfo() != nullptr) {
422         switch (RawToken.getIdentifierInfo()->getPPKeywordID()) {
423           case tok::pp_include:
424           case tok::pp_include_next:
425           case tok::pp_import: {
426             SourceLocation Loc = HashToken.getLocation();
427             const IncludedFile *Inc = FindIncludeAtLocation(Loc);
428             CommentOutDirective(RawLex, HashToken, FromFile, LocalEOL,
429                                 NextToWrite, Line, Inc);
430             if (FileId != PP.getPredefinesFileID())
431               WriteLineInfo(FileName, Line - 1, FileType, "");
432             StringRef LineInfoExtra;
433             if (const Module *Mod = FindModuleAtLocation(Loc))
434               WriteImplicitModuleImport(Mod);
435             else if (Inc) {
436               const Module *Mod = FindEnteredModule(Loc);
437               if (Mod)
438                 OS << "#pragma clang module begin "
439                    << Mod->getFullModuleName(true) << "\n";
440 
441               // Include and recursively process the file.
442               Process(Inc->Id, Inc->FileType);
443 
444               if (Mod)
445                 OS << "#pragma clang module end /*"
446                    << Mod->getFullModuleName(true) << "*/\n";
447               // There's no #include, therefore no #if, for -include files.
448               if (FromFile != PredefinesBuffer) {
449                 OS << "#endif /* " << getIncludedFileName(Inc)
450                    << " expanded by -frewrite-includes */" << LocalEOL;
451               }
452 
453               // Add line marker to indicate we're returning from an included
454               // file.
455               LineInfoExtra = " 2";
456             }
457             // fix up lineinfo (since commented out directive changed line
458             // numbers) for inclusions that were skipped due to header guards
459             WriteLineInfo(FileName, Line, FileType, LineInfoExtra);
460             break;
461           }
462           case tok::pp_pragma: {
463             StringRef Identifier = NextIdentifierName(RawLex, RawToken);
464             if (Identifier == "clang" || Identifier == "GCC") {
465               if (NextIdentifierName(RawLex, RawToken) == "system_header") {
466                 // keep the directive in, commented out
467                 CommentOutDirective(RawLex, HashToken, FromFile, LocalEOL,
468                   NextToWrite, Line);
469                 // update our own type
470                 FileType = SM.getFileCharacteristic(RawToken.getLocation());
471                 WriteLineInfo(FileName, Line, FileType);
472               }
473             } else if (Identifier == "once") {
474               // keep the directive in, commented out
475               CommentOutDirective(RawLex, HashToken, FromFile, LocalEOL,
476                 NextToWrite, Line);
477               WriteLineInfo(FileName, Line, FileType);
478             }
479             break;
480           }
481           case tok::pp_if:
482           case tok::pp_elif: {
483             bool elif = (RawToken.getIdentifierInfo()->getPPKeywordID() ==
484                          tok::pp_elif);
485             bool isTrue = IsIfAtLocationTrue(RawToken.getLocation());
486             OutputContentUpTo(FromFile, NextToWrite,
487                               SM.getFileOffset(HashToken.getLocation()),
488                               LocalEOL, Line, /*EnsureNewline=*/true);
489             do {
490               RawLex.LexFromRawLexer(RawToken);
491             } while (!RawToken.is(tok::eod) && RawToken.isNot(tok::eof));
492             // We need to disable the old condition, but that is tricky.
493             // Trying to comment it out can easily lead to comment nesting.
494             // So instead make the condition harmless by making it enclose
495             // and empty block. Moreover, put it itself inside an #if 0 block
496             // to disable it from getting evaluated (e.g. __has_include_next
497             // warns if used from the primary source file).
498             OS << "#if 0 /* disabled by -frewrite-includes */" << MainEOL;
499             if (elif) {
500               OS << "#if 0" << MainEOL;
501             }
502             OutputContentUpTo(FromFile, NextToWrite,
503                               SM.getFileOffset(RawToken.getLocation()) +
504                                   RawToken.getLength(),
505                               LocalEOL, Line, /*EnsureNewline=*/true);
506             // Close the empty block and the disabling block.
507             OS << "#endif" << MainEOL;
508             OS << "#endif /* disabled by -frewrite-includes */" << MainEOL;
509             OS << (elif ? "#elif " : "#if ") << (isTrue ? "1" : "0")
510                << " /* evaluated by -frewrite-includes */" << MainEOL;
511             WriteLineInfo(FileName, Line, FileType);
512             break;
513           }
514           case tok::pp_endif:
515           case tok::pp_else: {
516             // We surround every #include by #if 0 to comment it out, but that
517             // changes line numbers. These are fixed up right after that, but
518             // the whole #include could be inside a preprocessor conditional
519             // that is not processed. So it is necessary to fix the line
520             // numbers one the next line after each #else/#endif as well.
521             RawLex.SetKeepWhitespaceMode(true);
522             do {
523               RawLex.LexFromRawLexer(RawToken);
524             } while (RawToken.isNot(tok::eod) && RawToken.isNot(tok::eof));
525             OutputContentUpTo(FromFile, NextToWrite,
526                               SM.getFileOffset(RawToken.getLocation()) +
527                                   RawToken.getLength(),
528                               LocalEOL, Line, /*EnsureNewline=*/ true);
529             WriteLineInfo(FileName, Line, FileType);
530             RawLex.SetKeepWhitespaceMode(false);
531             break;
532           }
533           default:
534             break;
535         }
536       }
537       RawLex.setParsingPreprocessorDirective(false);
538     }
539     RawLex.LexFromRawLexer(RawToken);
540   }
541   OutputContentUpTo(FromFile, NextToWrite,
542                     SM.getFileOffset(SM.getLocForEndOfFile(FileId)), LocalEOL,
543                     Line, /*EnsureNewline=*/true);
544 }
545 
546 /// InclusionRewriterInInput - Implement -frewrite-includes mode.
547 void clang::RewriteIncludesInInput(Preprocessor &PP, raw_ostream *OS,
548                                    const PreprocessorOutputOptions &Opts) {
549   SourceManager &SM = PP.getSourceManager();
550   InclusionRewriter *Rewrite = new InclusionRewriter(
551       PP, *OS, Opts.ShowLineMarkers, Opts.UseLineDirectives);
552   Rewrite->detectMainFileEOL();
553 
554   PP.addPPCallbacks(std::unique_ptr<PPCallbacks>(Rewrite));
555   PP.IgnorePragmas();
556 
557   // First let the preprocessor process the entire file and call callbacks.
558   // Callbacks will record which #include's were actually performed.
559   PP.EnterMainSourceFile();
560   Token Tok;
561   // Only preprocessor directives matter here, so disable macro expansion
562   // everywhere else as an optimization.
563   // TODO: It would be even faster if the preprocessor could be switched
564   // to a mode where it would parse only preprocessor directives and comments,
565   // nothing else matters for parsing or processing.
566   PP.SetMacroExpansionOnlyInDirectives();
567   do {
568     PP.Lex(Tok);
569     if (Tok.is(tok::annot_module_begin))
570       Rewrite->handleModuleBegin(Tok);
571   } while (Tok.isNot(tok::eof));
572   Rewrite->setPredefinesBuffer(SM.getBufferOrFake(PP.getPredefinesFileID()));
573   Rewrite->Process(PP.getPredefinesFileID(), SrcMgr::C_User);
574   Rewrite->Process(SM.getMainFileID(), SrcMgr::C_User);
575   OS->flush();
576 }
577