xref: /freebsd/contrib/llvm-project/clang/lib/Frontend/Rewrite/InclusionRewriter.cpp (revision a2fda816eb054d5873be223ef2461741dfcc253c)
1  //===--- InclusionRewriter.cpp - Rewrite includes into their expansions ---===//
2  //
3  // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4  // See https://llvm.org/LICENSE.txt for license information.
5  // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6  //
7  //===----------------------------------------------------------------------===//
8  //
9  // This code rewrites include invocations into their expansions.  This gives you
10  // a file with all included files merged into it.
11  //
12  //===----------------------------------------------------------------------===//
13  
14  #include "clang/Rewrite/Frontend/Rewriters.h"
15  #include "clang/Basic/SourceManager.h"
16  #include "clang/Frontend/PreprocessorOutputOptions.h"
17  #include "clang/Lex/Pragma.h"
18  #include "clang/Lex/Preprocessor.h"
19  #include "llvm/ADT/SmallString.h"
20  #include "llvm/Support/raw_ostream.h"
21  #include <optional>
22  
23  using namespace clang;
24  using namespace llvm;
25  
26  namespace {
27  
28  class InclusionRewriter : public PPCallbacks {
29    /// Information about which #includes were actually performed,
30    /// created by preprocessor callbacks.
31    struct IncludedFile {
32      FileID Id;
33      SrcMgr::CharacteristicKind FileType;
34      IncludedFile(FileID Id, SrcMgr::CharacteristicKind FileType)
35          : Id(Id), FileType(FileType) {}
36    };
37    Preprocessor &PP; ///< Used to find inclusion directives.
38    SourceManager &SM; ///< Used to read and manage source files.
39    raw_ostream &OS; ///< The destination stream for rewritten contents.
40    StringRef MainEOL; ///< The line ending marker to use.
41    llvm::MemoryBufferRef PredefinesBuffer; ///< The preprocessor predefines.
42    bool ShowLineMarkers; ///< Show #line markers.
43    bool UseLineDirectives; ///< Use of line directives or line markers.
44    /// Tracks where inclusions that change the file are found.
45    std::map<SourceLocation, IncludedFile> FileIncludes;
46    /// Tracks where inclusions that import modules are found.
47    std::map<SourceLocation, const Module *> ModuleIncludes;
48    /// Tracks where inclusions that enter modules (in a module build) are found.
49    std::map<SourceLocation, const Module *> ModuleEntryIncludes;
50    /// Tracks where #if and #elif directives get evaluated and whether to true.
51    std::map<SourceLocation, bool> IfConditions;
52    /// Used transitively for building up the FileIncludes mapping over the
53    /// various \c PPCallbacks callbacks.
54    SourceLocation LastInclusionLocation;
55  public:
56    InclusionRewriter(Preprocessor &PP, raw_ostream &OS, bool ShowLineMarkers,
57                      bool UseLineDirectives);
58    void Process(FileID FileId, SrcMgr::CharacteristicKind FileType);
59    void setPredefinesBuffer(const llvm::MemoryBufferRef &Buf) {
60      PredefinesBuffer = Buf;
61    }
62    void detectMainFileEOL();
63    void handleModuleBegin(Token &Tok) {
64      assert(Tok.getKind() == tok::annot_module_begin);
65      ModuleEntryIncludes.insert(
66          {Tok.getLocation(), (Module *)Tok.getAnnotationValue()});
67    }
68  private:
69    void FileChanged(SourceLocation Loc, FileChangeReason Reason,
70                     SrcMgr::CharacteristicKind FileType,
71                     FileID PrevFID) override;
72    void FileSkipped(const FileEntryRef &SkippedFile, const Token &FilenameTok,
73                     SrcMgr::CharacteristicKind FileType) override;
74    void InclusionDirective(SourceLocation HashLoc, const Token &IncludeTok,
75                            StringRef FileName, bool IsAngled,
76                            CharSourceRange FilenameRange,
77                            OptionalFileEntryRef File, StringRef SearchPath,
78                            StringRef RelativePath, const Module *Imported,
79                            SrcMgr::CharacteristicKind FileType) override;
80    void If(SourceLocation Loc, SourceRange ConditionRange,
81            ConditionValueKind ConditionValue) override;
82    void Elif(SourceLocation Loc, SourceRange ConditionRange,
83              ConditionValueKind ConditionValue, SourceLocation IfLoc) override;
84    void WriteLineInfo(StringRef Filename, int Line,
85                       SrcMgr::CharacteristicKind FileType,
86                       StringRef Extra = StringRef());
87    void WriteImplicitModuleImport(const Module *Mod);
88    void OutputContentUpTo(const MemoryBufferRef &FromFile, unsigned &WriteFrom,
89                           unsigned WriteTo, StringRef EOL, int &lines,
90                           bool EnsureNewline);
91    void CommentOutDirective(Lexer &DirectivesLex, const Token &StartToken,
92                             const MemoryBufferRef &FromFile, StringRef EOL,
93                             unsigned &NextToWrite, int &Lines,
94                             const IncludedFile *Inc = nullptr);
95    const IncludedFile *FindIncludeAtLocation(SourceLocation Loc) const;
96    StringRef getIncludedFileName(const IncludedFile *Inc) const;
97    const Module *FindModuleAtLocation(SourceLocation Loc) const;
98    const Module *FindEnteredModule(SourceLocation Loc) const;
99    bool IsIfAtLocationTrue(SourceLocation Loc) const;
100    StringRef NextIdentifierName(Lexer &RawLex, Token &RawToken);
101  };
102  
103  }  // end anonymous namespace
104  
105  /// Initializes an InclusionRewriter with a \p PP source and \p OS destination.
106  InclusionRewriter::InclusionRewriter(Preprocessor &PP, raw_ostream &OS,
107                                       bool ShowLineMarkers,
108                                       bool UseLineDirectives)
109      : PP(PP), SM(PP.getSourceManager()), OS(OS), MainEOL("\n"),
110        ShowLineMarkers(ShowLineMarkers), UseLineDirectives(UseLineDirectives),
111        LastInclusionLocation(SourceLocation()) {}
112  
113  /// Write appropriate line information as either #line directives or GNU line
114  /// markers depending on what mode we're in, including the \p Filename and
115  /// \p Line we are located at, using the specified \p EOL line separator, and
116  /// any \p Extra context specifiers in GNU line directives.
117  void InclusionRewriter::WriteLineInfo(StringRef Filename, int Line,
118                                        SrcMgr::CharacteristicKind FileType,
119                                        StringRef Extra) {
120    if (!ShowLineMarkers)
121      return;
122    if (UseLineDirectives) {
123      OS << "#line" << ' ' << Line << ' ' << '"';
124      OS.write_escaped(Filename);
125      OS << '"';
126    } else {
127      // Use GNU linemarkers as described here:
128      // http://gcc.gnu.org/onlinedocs/cpp/Preprocessor-Output.html
129      OS << '#' << ' ' << Line << ' ' << '"';
130      OS.write_escaped(Filename);
131      OS << '"';
132      if (!Extra.empty())
133        OS << Extra;
134      if (FileType == SrcMgr::C_System)
135        // "`3' This indicates that the following text comes from a system header
136        // file, so certain warnings should be suppressed."
137        OS << " 3";
138      else if (FileType == SrcMgr::C_ExternCSystem)
139        // as above for `3', plus "`4' This indicates that the following text
140        // should be treated as being wrapped in an implicit extern "C" block."
141        OS << " 3 4";
142    }
143    OS << MainEOL;
144  }
145  
146  void InclusionRewriter::WriteImplicitModuleImport(const Module *Mod) {
147    OS << "#pragma clang module import " << Mod->getFullModuleName(true)
148       << " /* clang -frewrite-includes: implicit import */" << MainEOL;
149  }
150  
151  /// FileChanged - Whenever the preprocessor enters or exits a #include file
152  /// it invokes this handler.
153  void InclusionRewriter::FileChanged(SourceLocation Loc,
154                                      FileChangeReason Reason,
155                                      SrcMgr::CharacteristicKind NewFileType,
156                                      FileID) {
157    if (Reason != EnterFile)
158      return;
159    if (LastInclusionLocation.isInvalid())
160      // we didn't reach this file (eg: the main file) via an inclusion directive
161      return;
162    FileID Id = FullSourceLoc(Loc, SM).getFileID();
163    auto P = FileIncludes.insert(
164        std::make_pair(LastInclusionLocation, IncludedFile(Id, NewFileType)));
165    (void)P;
166    assert(P.second && "Unexpected revisitation of the same include directive");
167    LastInclusionLocation = SourceLocation();
168  }
169  
170  /// Called whenever an inclusion is skipped due to canonical header protection
171  /// macros.
172  void InclusionRewriter::FileSkipped(const FileEntryRef & /*SkippedFile*/,
173                                      const Token & /*FilenameTok*/,
174                                      SrcMgr::CharacteristicKind /*FileType*/) {
175    assert(LastInclusionLocation.isValid() &&
176           "A file, that wasn't found via an inclusion directive, was skipped");
177    LastInclusionLocation = SourceLocation();
178  }
179  
180  /// This should be called whenever the preprocessor encounters include
181  /// directives. It does not say whether the file has been included, but it
182  /// provides more information about the directive (hash location instead
183  /// of location inside the included file). It is assumed that the matching
184  /// FileChanged() or FileSkipped() is called after this (or neither is
185  /// called if this #include results in an error or does not textually include
186  /// anything).
187  void InclusionRewriter::InclusionDirective(
188      SourceLocation HashLoc, const Token & /*IncludeTok*/,
189      StringRef /*FileName*/, bool /*IsAngled*/,
190      CharSourceRange /*FilenameRange*/, OptionalFileEntryRef /*File*/,
191      StringRef /*SearchPath*/, StringRef /*RelativePath*/,
192      const Module *Imported, SrcMgr::CharacteristicKind FileType) {
193    if (Imported) {
194      auto P = ModuleIncludes.insert(std::make_pair(HashLoc, Imported));
195      (void)P;
196      assert(P.second && "Unexpected revisitation of the same include directive");
197    } else
198      LastInclusionLocation = HashLoc;
199  }
200  
201  void InclusionRewriter::If(SourceLocation Loc, SourceRange ConditionRange,
202                             ConditionValueKind ConditionValue) {
203    auto P = IfConditions.insert(std::make_pair(Loc, ConditionValue == CVK_True));
204    (void)P;
205    assert(P.second && "Unexpected revisitation of the same if directive");
206  }
207  
208  void InclusionRewriter::Elif(SourceLocation Loc, SourceRange ConditionRange,
209                               ConditionValueKind ConditionValue,
210                               SourceLocation IfLoc) {
211    auto P = IfConditions.insert(std::make_pair(Loc, ConditionValue == CVK_True));
212    (void)P;
213    assert(P.second && "Unexpected revisitation of the same elif directive");
214  }
215  
216  /// Simple lookup for a SourceLocation (specifically one denoting the hash in
217  /// an inclusion directive) in the map of inclusion information, FileChanges.
218  const InclusionRewriter::IncludedFile *
219  InclusionRewriter::FindIncludeAtLocation(SourceLocation Loc) const {
220    const auto I = FileIncludes.find(Loc);
221    if (I != FileIncludes.end())
222      return &I->second;
223    return nullptr;
224  }
225  
226  /// Simple lookup for a SourceLocation (specifically one denoting the hash in
227  /// an inclusion directive) in the map of module inclusion information.
228  const Module *
229  InclusionRewriter::FindModuleAtLocation(SourceLocation Loc) const {
230    const auto I = ModuleIncludes.find(Loc);
231    if (I != ModuleIncludes.end())
232      return I->second;
233    return nullptr;
234  }
235  
236  /// Simple lookup for a SourceLocation (specifically one denoting the hash in
237  /// an inclusion directive) in the map of module entry information.
238  const Module *
239  InclusionRewriter::FindEnteredModule(SourceLocation Loc) const {
240    const auto I = ModuleEntryIncludes.find(Loc);
241    if (I != ModuleEntryIncludes.end())
242      return I->second;
243    return nullptr;
244  }
245  
246  bool InclusionRewriter::IsIfAtLocationTrue(SourceLocation Loc) const {
247    const auto I = IfConditions.find(Loc);
248    if (I != IfConditions.end())
249      return I->second;
250    return false;
251  }
252  
253  void InclusionRewriter::detectMainFileEOL() {
254    std::optional<MemoryBufferRef> FromFile =
255        *SM.getBufferOrNone(SM.getMainFileID());
256    assert(FromFile);
257    if (!FromFile)
258      return; // Should never happen, but whatever.
259    MainEOL = FromFile->getBuffer().detectEOL();
260  }
261  
262  /// Writes out bytes from \p FromFile, starting at \p NextToWrite and ending at
263  /// \p WriteTo - 1.
264  void InclusionRewriter::OutputContentUpTo(const MemoryBufferRef &FromFile,
265                                            unsigned &WriteFrom, unsigned WriteTo,
266                                            StringRef LocalEOL, int &Line,
267                                            bool EnsureNewline) {
268    if (WriteTo <= WriteFrom)
269      return;
270    if (FromFile == PredefinesBuffer) {
271      // Ignore the #defines of the predefines buffer.
272      WriteFrom = WriteTo;
273      return;
274    }
275  
276    // If we would output half of a line ending, advance one character to output
277    // the whole line ending.  All buffers are null terminated, so looking ahead
278    // one byte is safe.
279    if (LocalEOL.size() == 2 &&
280        LocalEOL[0] == (FromFile.getBufferStart() + WriteTo)[-1] &&
281        LocalEOL[1] == (FromFile.getBufferStart() + WriteTo)[0])
282      WriteTo++;
283  
284    StringRef TextToWrite(FromFile.getBufferStart() + WriteFrom,
285                          WriteTo - WriteFrom);
286    // count lines manually, it's faster than getPresumedLoc()
287    Line += TextToWrite.count(LocalEOL);
288  
289    if (MainEOL == LocalEOL) {
290      OS << TextToWrite;
291    } else {
292      // Output the file one line at a time, rewriting the line endings as we go.
293      StringRef Rest = TextToWrite;
294      while (!Rest.empty()) {
295        // Identify and output the next line excluding an EOL sequence if present.
296        size_t Idx = Rest.find(LocalEOL);
297        StringRef LineText = Rest.substr(0, Idx);
298        OS << LineText;
299        if (Idx != StringRef::npos) {
300          // An EOL sequence was present, output the EOL sequence for the
301          // main source file and skip past the local EOL sequence.
302          OS << MainEOL;
303          Idx += LocalEOL.size();
304        }
305        // Strip the line just handled. If Idx is npos or matches the end of the
306        // text, Rest will be set to an empty string and the loop will terminate.
307        Rest = Rest.substr(Idx);
308      }
309    }
310    if (EnsureNewline && !TextToWrite.ends_with(LocalEOL))
311      OS << MainEOL;
312  
313    WriteFrom = WriteTo;
314  }
315  
316  StringRef
317  InclusionRewriter::getIncludedFileName(const IncludedFile *Inc) const {
318    if (Inc) {
319      auto B = SM.getBufferOrNone(Inc->Id);
320      assert(B && "Attempting to process invalid inclusion");
321      if (B)
322        return llvm::sys::path::filename(B->getBufferIdentifier());
323    }
324    return StringRef();
325  }
326  
327  /// Print characters from \p FromFile starting at \p NextToWrite up until the
328  /// inclusion directive at \p StartToken, then print out the inclusion
329  /// inclusion directive disabled by a #if directive, updating \p NextToWrite
330  /// and \p Line to track the number of source lines visited and the progress
331  /// through the \p FromFile buffer.
332  void InclusionRewriter::CommentOutDirective(Lexer &DirectiveLex,
333                                              const Token &StartToken,
334                                              const MemoryBufferRef &FromFile,
335                                              StringRef LocalEOL,
336                                              unsigned &NextToWrite, int &Line,
337                                              const IncludedFile *Inc) {
338    OutputContentUpTo(FromFile, NextToWrite,
339                      SM.getFileOffset(StartToken.getLocation()), LocalEOL, Line,
340                      false);
341    Token DirectiveToken;
342    do {
343      DirectiveLex.LexFromRawLexer(DirectiveToken);
344    } while (!DirectiveToken.is(tok::eod) && DirectiveToken.isNot(tok::eof));
345    if (FromFile == PredefinesBuffer) {
346      // OutputContentUpTo() would not output anything anyway.
347      return;
348    }
349    if (Inc) {
350      OS << "#if defined(__CLANG_REWRITTEN_INCLUDES) ";
351      if (isSystem(Inc->FileType))
352        OS << "|| defined(__CLANG_REWRITTEN_SYSTEM_INCLUDES) ";
353      OS << "/* " << getIncludedFileName(Inc);
354    } else {
355      OS << "#if 0 /*";
356    }
357    OS << " expanded by -frewrite-includes */" << MainEOL;
358    OutputContentUpTo(FromFile, NextToWrite,
359                      SM.getFileOffset(DirectiveToken.getLocation()) +
360                          DirectiveToken.getLength(),
361                      LocalEOL, Line, true);
362    OS << (Inc ? "#else /* " : "#endif /*") << getIncludedFileName(Inc)
363       << " expanded by -frewrite-includes */" << MainEOL;
364  }
365  
366  /// Find the next identifier in the pragma directive specified by \p RawToken.
367  StringRef InclusionRewriter::NextIdentifierName(Lexer &RawLex,
368                                                  Token &RawToken) {
369    RawLex.LexFromRawLexer(RawToken);
370    if (RawToken.is(tok::raw_identifier))
371      PP.LookUpIdentifierInfo(RawToken);
372    if (RawToken.is(tok::identifier))
373      return RawToken.getIdentifierInfo()->getName();
374    return StringRef();
375  }
376  
377  /// Use a raw lexer to analyze \p FileId, incrementally copying parts of it
378  /// and including content of included files recursively.
379  void InclusionRewriter::Process(FileID FileId,
380                                  SrcMgr::CharacteristicKind FileType) {
381    MemoryBufferRef FromFile;
382    {
383      auto B = SM.getBufferOrNone(FileId);
384      assert(B && "Attempting to process invalid inclusion");
385      if (B)
386        FromFile = *B;
387    }
388    StringRef FileName = FromFile.getBufferIdentifier();
389    Lexer RawLex(FileId, FromFile, PP.getSourceManager(), PP.getLangOpts());
390    RawLex.SetCommentRetentionState(false);
391  
392    StringRef LocalEOL = FromFile.getBuffer().detectEOL();
393  
394    // Per the GNU docs: "1" indicates entering a new file.
395    if (FileId == SM.getMainFileID() || FileId == PP.getPredefinesFileID())
396      WriteLineInfo(FileName, 1, FileType, "");
397    else
398      WriteLineInfo(FileName, 1, FileType, " 1");
399  
400    if (SM.getFileIDSize(FileId) == 0)
401      return;
402  
403    // The next byte to be copied from the source file, which may be non-zero if
404    // the lexer handled a BOM.
405    unsigned NextToWrite = SM.getFileOffset(RawLex.getSourceLocation());
406    assert(SM.getLineNumber(FileId, NextToWrite) == 1);
407    int Line = 1; // The current input file line number.
408  
409    Token RawToken;
410    RawLex.LexFromRawLexer(RawToken);
411  
412    // TODO: Consider adding a switch that strips possibly unimportant content,
413    // such as comments, to reduce the size of repro files.
414    while (RawToken.isNot(tok::eof)) {
415      if (RawToken.is(tok::hash) && RawToken.isAtStartOfLine()) {
416        RawLex.setParsingPreprocessorDirective(true);
417        Token HashToken = RawToken;
418        RawLex.LexFromRawLexer(RawToken);
419        if (RawToken.is(tok::raw_identifier))
420          PP.LookUpIdentifierInfo(RawToken);
421        if (RawToken.getIdentifierInfo() != nullptr) {
422          switch (RawToken.getIdentifierInfo()->getPPKeywordID()) {
423            case tok::pp_include:
424            case tok::pp_include_next:
425            case tok::pp_import: {
426              SourceLocation Loc = HashToken.getLocation();
427              const IncludedFile *Inc = FindIncludeAtLocation(Loc);
428              CommentOutDirective(RawLex, HashToken, FromFile, LocalEOL,
429                                  NextToWrite, Line, Inc);
430              if (FileId != PP.getPredefinesFileID())
431                WriteLineInfo(FileName, Line - 1, FileType, "");
432              StringRef LineInfoExtra;
433              if (const Module *Mod = FindModuleAtLocation(Loc))
434                WriteImplicitModuleImport(Mod);
435              else if (Inc) {
436                const Module *Mod = FindEnteredModule(Loc);
437                if (Mod)
438                  OS << "#pragma clang module begin "
439                     << Mod->getFullModuleName(true) << "\n";
440  
441                // Include and recursively process the file.
442                Process(Inc->Id, Inc->FileType);
443  
444                if (Mod)
445                  OS << "#pragma clang module end /*"
446                     << Mod->getFullModuleName(true) << "*/\n";
447                // There's no #include, therefore no #if, for -include files.
448                if (FromFile != PredefinesBuffer) {
449                  OS << "#endif /* " << getIncludedFileName(Inc)
450                     << " expanded by -frewrite-includes */" << LocalEOL;
451                }
452  
453                // Add line marker to indicate we're returning from an included
454                // file.
455                LineInfoExtra = " 2";
456              }
457              // fix up lineinfo (since commented out directive changed line
458              // numbers) for inclusions that were skipped due to header guards
459              WriteLineInfo(FileName, Line, FileType, LineInfoExtra);
460              break;
461            }
462            case tok::pp_pragma: {
463              StringRef Identifier = NextIdentifierName(RawLex, RawToken);
464              if (Identifier == "clang" || Identifier == "GCC") {
465                if (NextIdentifierName(RawLex, RawToken) == "system_header") {
466                  // keep the directive in, commented out
467                  CommentOutDirective(RawLex, HashToken, FromFile, LocalEOL,
468                    NextToWrite, Line);
469                  // update our own type
470                  FileType = SM.getFileCharacteristic(RawToken.getLocation());
471                  WriteLineInfo(FileName, Line, FileType);
472                }
473              } else if (Identifier == "once") {
474                // keep the directive in, commented out
475                CommentOutDirective(RawLex, HashToken, FromFile, LocalEOL,
476                  NextToWrite, Line);
477                WriteLineInfo(FileName, Line, FileType);
478              }
479              break;
480            }
481            case tok::pp_if:
482            case tok::pp_elif: {
483              bool elif = (RawToken.getIdentifierInfo()->getPPKeywordID() ==
484                           tok::pp_elif);
485              bool isTrue = IsIfAtLocationTrue(RawToken.getLocation());
486              OutputContentUpTo(FromFile, NextToWrite,
487                                SM.getFileOffset(HashToken.getLocation()),
488                                LocalEOL, Line, /*EnsureNewline=*/true);
489              do {
490                RawLex.LexFromRawLexer(RawToken);
491              } while (!RawToken.is(tok::eod) && RawToken.isNot(tok::eof));
492              // We need to disable the old condition, but that is tricky.
493              // Trying to comment it out can easily lead to comment nesting.
494              // So instead make the condition harmless by making it enclose
495              // and empty block. Moreover, put it itself inside an #if 0 block
496              // to disable it from getting evaluated (e.g. __has_include_next
497              // warns if used from the primary source file).
498              OS << "#if 0 /* disabled by -frewrite-includes */" << MainEOL;
499              if (elif) {
500                OS << "#if 0" << MainEOL;
501              }
502              OutputContentUpTo(FromFile, NextToWrite,
503                                SM.getFileOffset(RawToken.getLocation()) +
504                                    RawToken.getLength(),
505                                LocalEOL, Line, /*EnsureNewline=*/true);
506              // Close the empty block and the disabling block.
507              OS << "#endif" << MainEOL;
508              OS << "#endif /* disabled by -frewrite-includes */" << MainEOL;
509              OS << (elif ? "#elif " : "#if ") << (isTrue ? "1" : "0")
510                 << " /* evaluated by -frewrite-includes */" << MainEOL;
511              WriteLineInfo(FileName, Line, FileType);
512              break;
513            }
514            case tok::pp_endif:
515            case tok::pp_else: {
516              // We surround every #include by #if 0 to comment it out, but that
517              // changes line numbers. These are fixed up right after that, but
518              // the whole #include could be inside a preprocessor conditional
519              // that is not processed. So it is necessary to fix the line
520              // numbers one the next line after each #else/#endif as well.
521              RawLex.SetKeepWhitespaceMode(true);
522              do {
523                RawLex.LexFromRawLexer(RawToken);
524              } while (RawToken.isNot(tok::eod) && RawToken.isNot(tok::eof));
525              OutputContentUpTo(FromFile, NextToWrite,
526                                SM.getFileOffset(RawToken.getLocation()) +
527                                    RawToken.getLength(),
528                                LocalEOL, Line, /*EnsureNewline=*/ true);
529              WriteLineInfo(FileName, Line, FileType);
530              RawLex.SetKeepWhitespaceMode(false);
531              break;
532            }
533            default:
534              break;
535          }
536        }
537        RawLex.setParsingPreprocessorDirective(false);
538      }
539      RawLex.LexFromRawLexer(RawToken);
540    }
541    OutputContentUpTo(FromFile, NextToWrite,
542                      SM.getFileOffset(SM.getLocForEndOfFile(FileId)), LocalEOL,
543                      Line, /*EnsureNewline=*/true);
544  }
545  
546  /// InclusionRewriterInInput - Implement -frewrite-includes mode.
547  void clang::RewriteIncludesInInput(Preprocessor &PP, raw_ostream *OS,
548                                     const PreprocessorOutputOptions &Opts) {
549    SourceManager &SM = PP.getSourceManager();
550    InclusionRewriter *Rewrite = new InclusionRewriter(
551        PP, *OS, Opts.ShowLineMarkers, Opts.UseLineDirectives);
552    Rewrite->detectMainFileEOL();
553  
554    PP.addPPCallbacks(std::unique_ptr<PPCallbacks>(Rewrite));
555    PP.IgnorePragmas();
556  
557    // First let the preprocessor process the entire file and call callbacks.
558    // Callbacks will record which #include's were actually performed.
559    PP.EnterMainSourceFile();
560    Token Tok;
561    // Only preprocessor directives matter here, so disable macro expansion
562    // everywhere else as an optimization.
563    // TODO: It would be even faster if the preprocessor could be switched
564    // to a mode where it would parse only preprocessor directives and comments,
565    // nothing else matters for parsing or processing.
566    PP.SetMacroExpansionOnlyInDirectives();
567    do {
568      PP.Lex(Tok);
569      if (Tok.is(tok::annot_module_begin))
570        Rewrite->handleModuleBegin(Tok);
571    } while (Tok.isNot(tok::eof));
572    Rewrite->setPredefinesBuffer(SM.getBufferOrFake(PP.getPredefinesFileID()));
573    Rewrite->Process(PP.getPredefinesFileID(), SrcMgr::C_User);
574    Rewrite->Process(SM.getMainFileID(), SrcMgr::C_User);
575    OS->flush();
576  }
577