xref: /freebsd/contrib/llvm-project/clang/lib/Lex/PPDirectives.cpp (revision acb1f1269c6f4ff89a0d28ba742f6687e9ef779d)
1 //===--- PPDirectives.cpp - Directive Handling for Preprocessor -----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 ///
9 /// \file
10 /// Implements # directive processing for the Preprocessor.
11 ///
12 //===----------------------------------------------------------------------===//
13 
14 #include "clang/Basic/CharInfo.h"
15 #include "clang/Basic/FileManager.h"
16 #include "clang/Basic/IdentifierTable.h"
17 #include "clang/Basic/LangOptions.h"
18 #include "clang/Basic/Module.h"
19 #include "clang/Basic/SourceLocation.h"
20 #include "clang/Basic/SourceManager.h"
21 #include "clang/Basic/TokenKinds.h"
22 #include "clang/Lex/CodeCompletionHandler.h"
23 #include "clang/Lex/HeaderSearch.h"
24 #include "clang/Lex/LexDiagnostic.h"
25 #include "clang/Lex/LiteralSupport.h"
26 #include "clang/Lex/MacroInfo.h"
27 #include "clang/Lex/ModuleLoader.h"
28 #include "clang/Lex/ModuleMap.h"
29 #include "clang/Lex/PPCallbacks.h"
30 #include "clang/Lex/Pragma.h"
31 #include "clang/Lex/Preprocessor.h"
32 #include "clang/Lex/PreprocessorOptions.h"
33 #include "clang/Lex/Token.h"
34 #include "clang/Lex/VariadicMacroSupport.h"
35 #include "llvm/ADT/ArrayRef.h"
36 #include "llvm/ADT/ScopeExit.h"
37 #include "llvm/ADT/SmallString.h"
38 #include "llvm/ADT/SmallVector.h"
39 #include "llvm/ADT/STLExtras.h"
40 #include "llvm/ADT/StringSwitch.h"
41 #include "llvm/ADT/StringRef.h"
42 #include "llvm/Support/AlignOf.h"
43 #include "llvm/Support/ErrorHandling.h"
44 #include "llvm/Support/Path.h"
45 #include <algorithm>
46 #include <cassert>
47 #include <cstring>
48 #include <new>
49 #include <string>
50 #include <utility>
51 
52 using namespace clang;
53 
54 //===----------------------------------------------------------------------===//
55 // Utility Methods for Preprocessor Directive Handling.
56 //===----------------------------------------------------------------------===//
57 
58 MacroInfo *Preprocessor::AllocateMacroInfo(SourceLocation L) {
59   auto *MIChain = new (BP) MacroInfoChain{L, MIChainHead};
60   MIChainHead = MIChain;
61   return &MIChain->MI;
62 }
63 
64 DefMacroDirective *Preprocessor::AllocateDefMacroDirective(MacroInfo *MI,
65                                                            SourceLocation Loc) {
66   return new (BP) DefMacroDirective(MI, Loc);
67 }
68 
69 UndefMacroDirective *
70 Preprocessor::AllocateUndefMacroDirective(SourceLocation UndefLoc) {
71   return new (BP) UndefMacroDirective(UndefLoc);
72 }
73 
74 VisibilityMacroDirective *
75 Preprocessor::AllocateVisibilityMacroDirective(SourceLocation Loc,
76                                                bool isPublic) {
77   return new (BP) VisibilityMacroDirective(Loc, isPublic);
78 }
79 
80 /// Read and discard all tokens remaining on the current line until
81 /// the tok::eod token is found.
82 SourceRange Preprocessor::DiscardUntilEndOfDirective() {
83   Token Tmp;
84   SourceRange Res;
85 
86   LexUnexpandedToken(Tmp);
87   Res.setBegin(Tmp.getLocation());
88   while (Tmp.isNot(tok::eod)) {
89     assert(Tmp.isNot(tok::eof) && "EOF seen while discarding directive tokens");
90     LexUnexpandedToken(Tmp);
91   }
92   Res.setEnd(Tmp.getLocation());
93   return Res;
94 }
95 
96 /// Enumerates possible cases of #define/#undef a reserved identifier.
97 enum MacroDiag {
98   MD_NoWarn,        //> Not a reserved identifier
99   MD_KeywordDef,    //> Macro hides keyword, enabled by default
100   MD_ReservedMacro  //> #define of #undef reserved id, disabled by default
101 };
102 
103 /// Checks if the specified identifier is reserved in the specified
104 /// language.
105 /// This function does not check if the identifier is a keyword.
106 static bool isReservedId(StringRef Text, const LangOptions &Lang) {
107   // C++ [macro.names], C11 7.1.3:
108   // All identifiers that begin with an underscore and either an uppercase
109   // letter or another underscore are always reserved for any use.
110   if (Text.size() >= 2 && Text[0] == '_' &&
111       (isUppercase(Text[1]) || Text[1] == '_'))
112       return true;
113   // C++ [global.names]
114   // Each name that contains a double underscore ... is reserved to the
115   // implementation for any use.
116   if (Lang.CPlusPlus) {
117     if (Text.find("__") != StringRef::npos)
118       return true;
119   }
120   return false;
121 }
122 
123 // The -fmodule-name option tells the compiler to textually include headers in
124 // the specified module, meaning clang won't build the specified module. This is
125 // useful in a number of situations, for instance, when building a library that
126 // vends a module map, one might want to avoid hitting intermediate build
127 // products containimg the the module map or avoid finding the system installed
128 // modulemap for that library.
129 static bool isForModuleBuilding(Module *M, StringRef CurrentModule,
130                                 StringRef ModuleName) {
131   StringRef TopLevelName = M->getTopLevelModuleName();
132 
133   // When building framework Foo, we wanna make sure that Foo *and* Foo_Private
134   // are textually included and no modules are built for both.
135   if (M->getTopLevelModule()->IsFramework && CurrentModule == ModuleName &&
136       !CurrentModule.endswith("_Private") && TopLevelName.endswith("_Private"))
137     TopLevelName = TopLevelName.drop_back(8);
138 
139   return TopLevelName == CurrentModule;
140 }
141 
142 static MacroDiag shouldWarnOnMacroDef(Preprocessor &PP, IdentifierInfo *II) {
143   const LangOptions &Lang = PP.getLangOpts();
144   StringRef Text = II->getName();
145   if (isReservedId(Text, Lang))
146     return MD_ReservedMacro;
147   if (II->isKeyword(Lang))
148     return MD_KeywordDef;
149   if (Lang.CPlusPlus11 && (Text.equals("override") || Text.equals("final")))
150     return MD_KeywordDef;
151   return MD_NoWarn;
152 }
153 
154 static MacroDiag shouldWarnOnMacroUndef(Preprocessor &PP, IdentifierInfo *II) {
155   const LangOptions &Lang = PP.getLangOpts();
156   StringRef Text = II->getName();
157   // Do not warn on keyword undef.  It is generally harmless and widely used.
158   if (isReservedId(Text, Lang))
159     return MD_ReservedMacro;
160   return MD_NoWarn;
161 }
162 
163 // Return true if we want to issue a diagnostic by default if we
164 // encounter this name in a #include with the wrong case. For now,
165 // this includes the standard C and C++ headers, Posix headers,
166 // and Boost headers. Improper case for these #includes is a
167 // potential portability issue.
168 static bool warnByDefaultOnWrongCase(StringRef Include) {
169   // If the first component of the path is "boost", treat this like a standard header
170   // for the purposes of diagnostics.
171   if (::llvm::sys::path::begin(Include)->equals_lower("boost"))
172     return true;
173 
174   // "condition_variable" is the longest standard header name at 18 characters.
175   // If the include file name is longer than that, it can't be a standard header.
176   static const size_t MaxStdHeaderNameLen = 18u;
177   if (Include.size() > MaxStdHeaderNameLen)
178     return false;
179 
180   // Lowercase and normalize the search string.
181   SmallString<32> LowerInclude{Include};
182   for (char &Ch : LowerInclude) {
183     // In the ASCII range?
184     if (static_cast<unsigned char>(Ch) > 0x7f)
185       return false; // Can't be a standard header
186     // ASCII lowercase:
187     if (Ch >= 'A' && Ch <= 'Z')
188       Ch += 'a' - 'A';
189     // Normalize path separators for comparison purposes.
190     else if (::llvm::sys::path::is_separator(Ch))
191       Ch = '/';
192   }
193 
194   // The standard C/C++ and Posix headers
195   return llvm::StringSwitch<bool>(LowerInclude)
196     // C library headers
197     .Cases("assert.h", "complex.h", "ctype.h", "errno.h", "fenv.h", true)
198     .Cases("float.h", "inttypes.h", "iso646.h", "limits.h", "locale.h", true)
199     .Cases("math.h", "setjmp.h", "signal.h", "stdalign.h", "stdarg.h", true)
200     .Cases("stdatomic.h", "stdbool.h", "stddef.h", "stdint.h", "stdio.h", true)
201     .Cases("stdlib.h", "stdnoreturn.h", "string.h", "tgmath.h", "threads.h", true)
202     .Cases("time.h", "uchar.h", "wchar.h", "wctype.h", true)
203 
204     // C++ headers for C library facilities
205     .Cases("cassert", "ccomplex", "cctype", "cerrno", "cfenv", true)
206     .Cases("cfloat", "cinttypes", "ciso646", "climits", "clocale", true)
207     .Cases("cmath", "csetjmp", "csignal", "cstdalign", "cstdarg", true)
208     .Cases("cstdbool", "cstddef", "cstdint", "cstdio", "cstdlib", true)
209     .Cases("cstring", "ctgmath", "ctime", "cuchar", "cwchar", true)
210     .Case("cwctype", true)
211 
212     // C++ library headers
213     .Cases("algorithm", "fstream", "list", "regex", "thread", true)
214     .Cases("array", "functional", "locale", "scoped_allocator", "tuple", true)
215     .Cases("atomic", "future", "map", "set", "type_traits", true)
216     .Cases("bitset", "initializer_list", "memory", "shared_mutex", "typeindex", true)
217     .Cases("chrono", "iomanip", "mutex", "sstream", "typeinfo", true)
218     .Cases("codecvt", "ios", "new", "stack", "unordered_map", true)
219     .Cases("complex", "iosfwd", "numeric", "stdexcept", "unordered_set", true)
220     .Cases("condition_variable", "iostream", "ostream", "streambuf", "utility", true)
221     .Cases("deque", "istream", "queue", "string", "valarray", true)
222     .Cases("exception", "iterator", "random", "strstream", "vector", true)
223     .Cases("forward_list", "limits", "ratio", "system_error", true)
224 
225     // POSIX headers (which aren't also C headers)
226     .Cases("aio.h", "arpa/inet.h", "cpio.h", "dirent.h", "dlfcn.h", true)
227     .Cases("fcntl.h", "fmtmsg.h", "fnmatch.h", "ftw.h", "glob.h", true)
228     .Cases("grp.h", "iconv.h", "langinfo.h", "libgen.h", "monetary.h", true)
229     .Cases("mqueue.h", "ndbm.h", "net/if.h", "netdb.h", "netinet/in.h", true)
230     .Cases("netinet/tcp.h", "nl_types.h", "poll.h", "pthread.h", "pwd.h", true)
231     .Cases("regex.h", "sched.h", "search.h", "semaphore.h", "spawn.h", true)
232     .Cases("strings.h", "stropts.h", "sys/ipc.h", "sys/mman.h", "sys/msg.h", true)
233     .Cases("sys/resource.h", "sys/select.h",  "sys/sem.h", "sys/shm.h", "sys/socket.h", true)
234     .Cases("sys/stat.h", "sys/statvfs.h", "sys/time.h", "sys/times.h", "sys/types.h", true)
235     .Cases("sys/uio.h", "sys/un.h", "sys/utsname.h", "sys/wait.h", "syslog.h", true)
236     .Cases("tar.h", "termios.h", "trace.h", "ulimit.h", true)
237     .Cases("unistd.h", "utime.h", "utmpx.h", "wordexp.h", true)
238     .Default(false);
239 }
240 
241 bool Preprocessor::CheckMacroName(Token &MacroNameTok, MacroUse isDefineUndef,
242                                   bool *ShadowFlag) {
243   // Missing macro name?
244   if (MacroNameTok.is(tok::eod))
245     return Diag(MacroNameTok, diag::err_pp_missing_macro_name);
246 
247   IdentifierInfo *II = MacroNameTok.getIdentifierInfo();
248   if (!II)
249     return Diag(MacroNameTok, diag::err_pp_macro_not_identifier);
250 
251   if (II->isCPlusPlusOperatorKeyword()) {
252     // C++ 2.5p2: Alternative tokens behave the same as its primary token
253     // except for their spellings.
254     Diag(MacroNameTok, getLangOpts().MicrosoftExt
255                            ? diag::ext_pp_operator_used_as_macro_name
256                            : diag::err_pp_operator_used_as_macro_name)
257         << II << MacroNameTok.getKind();
258     // Allow #defining |and| and friends for Microsoft compatibility or
259     // recovery when legacy C headers are included in C++.
260   }
261 
262   if ((isDefineUndef != MU_Other) && II->getPPKeywordID() == tok::pp_defined) {
263     // Error if defining "defined": C99 6.10.8/4, C++ [cpp.predefined]p4.
264     return Diag(MacroNameTok, diag::err_defined_macro_name);
265   }
266 
267   if (isDefineUndef == MU_Undef) {
268     auto *MI = getMacroInfo(II);
269     if (MI && MI->isBuiltinMacro()) {
270       // Warn if undefining "__LINE__" and other builtins, per C99 6.10.8/4
271       // and C++ [cpp.predefined]p4], but allow it as an extension.
272       Diag(MacroNameTok, diag::ext_pp_undef_builtin_macro);
273     }
274   }
275 
276   // If defining/undefining reserved identifier or a keyword, we need to issue
277   // a warning.
278   SourceLocation MacroNameLoc = MacroNameTok.getLocation();
279   if (ShadowFlag)
280     *ShadowFlag = false;
281   if (!SourceMgr.isInSystemHeader(MacroNameLoc) &&
282       (SourceMgr.getBufferName(MacroNameLoc) != "<built-in>")) {
283     MacroDiag D = MD_NoWarn;
284     if (isDefineUndef == MU_Define) {
285       D = shouldWarnOnMacroDef(*this, II);
286     }
287     else if (isDefineUndef == MU_Undef)
288       D = shouldWarnOnMacroUndef(*this, II);
289     if (D == MD_KeywordDef) {
290       // We do not want to warn on some patterns widely used in configuration
291       // scripts.  This requires analyzing next tokens, so do not issue warnings
292       // now, only inform caller.
293       if (ShadowFlag)
294         *ShadowFlag = true;
295     }
296     if (D == MD_ReservedMacro)
297       Diag(MacroNameTok, diag::warn_pp_macro_is_reserved_id);
298   }
299 
300   // Okay, we got a good identifier.
301   return false;
302 }
303 
304 /// Lex and validate a macro name, which occurs after a
305 /// \#define or \#undef.
306 ///
307 /// This sets the token kind to eod and discards the rest of the macro line if
308 /// the macro name is invalid.
309 ///
310 /// \param MacroNameTok Token that is expected to be a macro name.
311 /// \param isDefineUndef Context in which macro is used.
312 /// \param ShadowFlag Points to a flag that is set if macro shadows a keyword.
313 void Preprocessor::ReadMacroName(Token &MacroNameTok, MacroUse isDefineUndef,
314                                  bool *ShadowFlag) {
315   // Read the token, don't allow macro expansion on it.
316   LexUnexpandedToken(MacroNameTok);
317 
318   if (MacroNameTok.is(tok::code_completion)) {
319     if (CodeComplete)
320       CodeComplete->CodeCompleteMacroName(isDefineUndef == MU_Define);
321     setCodeCompletionReached();
322     LexUnexpandedToken(MacroNameTok);
323   }
324 
325   if (!CheckMacroName(MacroNameTok, isDefineUndef, ShadowFlag))
326     return;
327 
328   // Invalid macro name, read and discard the rest of the line and set the
329   // token kind to tok::eod if necessary.
330   if (MacroNameTok.isNot(tok::eod)) {
331     MacroNameTok.setKind(tok::eod);
332     DiscardUntilEndOfDirective();
333   }
334 }
335 
336 /// Ensure that the next token is a tok::eod token.
337 ///
338 /// If not, emit a diagnostic and consume up until the eod.  If EnableMacros is
339 /// true, then we consider macros that expand to zero tokens as being ok.
340 ///
341 /// Returns the location of the end of the directive.
342 SourceLocation Preprocessor::CheckEndOfDirective(const char *DirType,
343                                                  bool EnableMacros) {
344   Token Tmp;
345   // Lex unexpanded tokens for most directives: macros might expand to zero
346   // tokens, causing us to miss diagnosing invalid lines.  Some directives (like
347   // #line) allow empty macros.
348   if (EnableMacros)
349     Lex(Tmp);
350   else
351     LexUnexpandedToken(Tmp);
352 
353   // There should be no tokens after the directive, but we allow them as an
354   // extension.
355   while (Tmp.is(tok::comment))  // Skip comments in -C mode.
356     LexUnexpandedToken(Tmp);
357 
358   if (Tmp.is(tok::eod))
359     return Tmp.getLocation();
360 
361   // Add a fixit in GNU/C99/C++ mode.  Don't offer a fixit for strict-C89,
362   // or if this is a macro-style preprocessing directive, because it is more
363   // trouble than it is worth to insert /**/ and check that there is no /**/
364   // in the range also.
365   FixItHint Hint;
366   if ((LangOpts.GNUMode || LangOpts.C99 || LangOpts.CPlusPlus) &&
367       !CurTokenLexer)
368     Hint = FixItHint::CreateInsertion(Tmp.getLocation(),"//");
369   Diag(Tmp, diag::ext_pp_extra_tokens_at_eol) << DirType << Hint;
370   return DiscardUntilEndOfDirective().getEnd();
371 }
372 
373 Optional<unsigned> Preprocessor::getSkippedRangeForExcludedConditionalBlock(
374     SourceLocation HashLoc) {
375   if (!ExcludedConditionalDirectiveSkipMappings)
376     return None;
377   if (!HashLoc.isFileID())
378     return None;
379 
380   std::pair<FileID, unsigned> HashFileOffset =
381       SourceMgr.getDecomposedLoc(HashLoc);
382   Optional<llvm::MemoryBufferRef> Buf =
383       SourceMgr.getBufferOrNone(HashFileOffset.first);
384   if (!Buf)
385     return None;
386   auto It =
387       ExcludedConditionalDirectiveSkipMappings->find(Buf->getBufferStart());
388   if (It == ExcludedConditionalDirectiveSkipMappings->end())
389     return None;
390 
391   const PreprocessorSkippedRangeMapping &SkippedRanges = *It->getSecond();
392   // Check if the offset of '#' is mapped in the skipped ranges.
393   auto MappingIt = SkippedRanges.find(HashFileOffset.second);
394   if (MappingIt == SkippedRanges.end())
395     return None;
396 
397   unsigned BytesToSkip = MappingIt->getSecond();
398   unsigned CurLexerBufferOffset = CurLexer->getCurrentBufferOffset();
399   assert(CurLexerBufferOffset >= HashFileOffset.second &&
400          "lexer is before the hash?");
401   // Take into account the fact that the lexer has already advanced, so the
402   // number of bytes to skip must be adjusted.
403   unsigned LengthDiff = CurLexerBufferOffset - HashFileOffset.second;
404   assert(BytesToSkip >= LengthDiff && "lexer is after the skipped range?");
405   return BytesToSkip - LengthDiff;
406 }
407 
408 /// SkipExcludedConditionalBlock - We just read a \#if or related directive and
409 /// decided that the subsequent tokens are in the \#if'd out portion of the
410 /// file.  Lex the rest of the file, until we see an \#endif.  If
411 /// FoundNonSkipPortion is true, then we have already emitted code for part of
412 /// this \#if directive, so \#else/\#elif blocks should never be entered.
413 /// If ElseOk is true, then \#else directives are ok, if not, then we have
414 /// already seen one so a \#else directive is a duplicate.  When this returns,
415 /// the caller can lex the first valid token.
416 void Preprocessor::SkipExcludedConditionalBlock(SourceLocation HashTokenLoc,
417                                                 SourceLocation IfTokenLoc,
418                                                 bool FoundNonSkipPortion,
419                                                 bool FoundElse,
420                                                 SourceLocation ElseLoc) {
421   ++NumSkipped;
422   assert(!CurTokenLexer && CurPPLexer && "Lexing a macro, not a file?");
423 
424   if (PreambleConditionalStack.reachedEOFWhileSkipping())
425     PreambleConditionalStack.clearSkipInfo();
426   else
427     CurPPLexer->pushConditionalLevel(IfTokenLoc, /*isSkipping*/ false,
428                                      FoundNonSkipPortion, FoundElse);
429 
430   // Enter raw mode to disable identifier lookup (and thus macro expansion),
431   // disabling warnings, etc.
432   CurPPLexer->LexingRawMode = true;
433   Token Tok;
434   if (auto SkipLength =
435           getSkippedRangeForExcludedConditionalBlock(HashTokenLoc)) {
436     // Skip to the next '#endif' / '#else' / '#elif'.
437     CurLexer->skipOver(*SkipLength);
438   }
439   SourceLocation endLoc;
440   while (true) {
441     CurLexer->Lex(Tok);
442 
443     if (Tok.is(tok::code_completion)) {
444       if (CodeComplete)
445         CodeComplete->CodeCompleteInConditionalExclusion();
446       setCodeCompletionReached();
447       continue;
448     }
449 
450     // If this is the end of the buffer, we have an error.
451     if (Tok.is(tok::eof)) {
452       // We don't emit errors for unterminated conditionals here,
453       // Lexer::LexEndOfFile can do that properly.
454       // Just return and let the caller lex after this #include.
455       if (PreambleConditionalStack.isRecording())
456         PreambleConditionalStack.SkipInfo.emplace(
457             HashTokenLoc, IfTokenLoc, FoundNonSkipPortion, FoundElse, ElseLoc);
458       break;
459     }
460 
461     // If this token is not a preprocessor directive, just skip it.
462     if (Tok.isNot(tok::hash) || !Tok.isAtStartOfLine())
463       continue;
464 
465     // We just parsed a # character at the start of a line, so we're in
466     // directive mode.  Tell the lexer this so any newlines we see will be
467     // converted into an EOD token (this terminates the macro).
468     CurPPLexer->ParsingPreprocessorDirective = true;
469     if (CurLexer) CurLexer->SetKeepWhitespaceMode(false);
470 
471 
472     // Read the next token, the directive flavor.
473     LexUnexpandedToken(Tok);
474 
475     // If this isn't an identifier directive (e.g. is "# 1\n" or "#\n", or
476     // something bogus), skip it.
477     if (Tok.isNot(tok::raw_identifier)) {
478       CurPPLexer->ParsingPreprocessorDirective = false;
479       // Restore comment saving mode.
480       if (CurLexer) CurLexer->resetExtendedTokenMode();
481       continue;
482     }
483 
484     // If the first letter isn't i or e, it isn't intesting to us.  We know that
485     // this is safe in the face of spelling differences, because there is no way
486     // to spell an i/e in a strange way that is another letter.  Skipping this
487     // allows us to avoid looking up the identifier info for #define/#undef and
488     // other common directives.
489     StringRef RI = Tok.getRawIdentifier();
490 
491     char FirstChar = RI[0];
492     if (FirstChar >= 'a' && FirstChar <= 'z' &&
493         FirstChar != 'i' && FirstChar != 'e') {
494       CurPPLexer->ParsingPreprocessorDirective = false;
495       // Restore comment saving mode.
496       if (CurLexer) CurLexer->resetExtendedTokenMode();
497       continue;
498     }
499 
500     // Get the identifier name without trigraphs or embedded newlines.  Note
501     // that we can't use Tok.getIdentifierInfo() because its lookup is disabled
502     // when skipping.
503     char DirectiveBuf[20];
504     StringRef Directive;
505     if (!Tok.needsCleaning() && RI.size() < 20) {
506       Directive = RI;
507     } else {
508       std::string DirectiveStr = getSpelling(Tok);
509       size_t IdLen = DirectiveStr.size();
510       if (IdLen >= 20) {
511         CurPPLexer->ParsingPreprocessorDirective = false;
512         // Restore comment saving mode.
513         if (CurLexer) CurLexer->resetExtendedTokenMode();
514         continue;
515       }
516       memcpy(DirectiveBuf, &DirectiveStr[0], IdLen);
517       Directive = StringRef(DirectiveBuf, IdLen);
518     }
519 
520     if (Directive.startswith("if")) {
521       StringRef Sub = Directive.substr(2);
522       if (Sub.empty() ||   // "if"
523           Sub == "def" ||   // "ifdef"
524           Sub == "ndef") {  // "ifndef"
525         // We know the entire #if/#ifdef/#ifndef block will be skipped, don't
526         // bother parsing the condition.
527         DiscardUntilEndOfDirective();
528         CurPPLexer->pushConditionalLevel(Tok.getLocation(), /*wasskipping*/true,
529                                        /*foundnonskip*/false,
530                                        /*foundelse*/false);
531       }
532     } else if (Directive[0] == 'e') {
533       StringRef Sub = Directive.substr(1);
534       if (Sub == "ndif") {  // "endif"
535         PPConditionalInfo CondInfo;
536         CondInfo.WasSkipping = true; // Silence bogus warning.
537         bool InCond = CurPPLexer->popConditionalLevel(CondInfo);
538         (void)InCond;  // Silence warning in no-asserts mode.
539         assert(!InCond && "Can't be skipping if not in a conditional!");
540 
541         // If we popped the outermost skipping block, we're done skipping!
542         if (!CondInfo.WasSkipping) {
543           // Restore the value of LexingRawMode so that trailing comments
544           // are handled correctly, if we've reached the outermost block.
545           CurPPLexer->LexingRawMode = false;
546           endLoc = CheckEndOfDirective("endif");
547           CurPPLexer->LexingRawMode = true;
548           if (Callbacks)
549             Callbacks->Endif(Tok.getLocation(), CondInfo.IfLoc);
550           break;
551         } else {
552           DiscardUntilEndOfDirective();
553         }
554       } else if (Sub == "lse") { // "else".
555         // #else directive in a skipping conditional.  If not in some other
556         // skipping conditional, and if #else hasn't already been seen, enter it
557         // as a non-skipping conditional.
558         PPConditionalInfo &CondInfo = CurPPLexer->peekConditionalLevel();
559 
560         // If this is a #else with a #else before it, report the error.
561         if (CondInfo.FoundElse) Diag(Tok, diag::pp_err_else_after_else);
562 
563         // Note that we've seen a #else in this conditional.
564         CondInfo.FoundElse = true;
565 
566         // If the conditional is at the top level, and the #if block wasn't
567         // entered, enter the #else block now.
568         if (!CondInfo.WasSkipping && !CondInfo.FoundNonSkip) {
569           CondInfo.FoundNonSkip = true;
570           // Restore the value of LexingRawMode so that trailing comments
571           // are handled correctly.
572           CurPPLexer->LexingRawMode = false;
573           endLoc = CheckEndOfDirective("else");
574           CurPPLexer->LexingRawMode = true;
575           if (Callbacks)
576             Callbacks->Else(Tok.getLocation(), CondInfo.IfLoc);
577           break;
578         } else {
579           DiscardUntilEndOfDirective();  // C99 6.10p4.
580         }
581       } else if (Sub == "lif") {  // "elif".
582         PPConditionalInfo &CondInfo = CurPPLexer->peekConditionalLevel();
583 
584         // If this is a #elif with a #else before it, report the error.
585         if (CondInfo.FoundElse) Diag(Tok, diag::pp_err_elif_after_else);
586 
587         // If this is in a skipping block or if we're already handled this #if
588         // block, don't bother parsing the condition.
589         if (CondInfo.WasSkipping || CondInfo.FoundNonSkip) {
590           DiscardUntilEndOfDirective();
591         } else {
592           // Restore the value of LexingRawMode so that identifiers are
593           // looked up, etc, inside the #elif expression.
594           assert(CurPPLexer->LexingRawMode && "We have to be skipping here!");
595           CurPPLexer->LexingRawMode = false;
596           IdentifierInfo *IfNDefMacro = nullptr;
597           DirectiveEvalResult DER = EvaluateDirectiveExpression(IfNDefMacro);
598           const bool CondValue = DER.Conditional;
599           CurPPLexer->LexingRawMode = true;
600           if (Callbacks) {
601             Callbacks->Elif(
602                 Tok.getLocation(), DER.ExprRange,
603                 (CondValue ? PPCallbacks::CVK_True : PPCallbacks::CVK_False),
604                 CondInfo.IfLoc);
605           }
606           // If this condition is true, enter it!
607           if (CondValue) {
608             CondInfo.FoundNonSkip = true;
609             break;
610           }
611         }
612       }
613     }
614 
615     CurPPLexer->ParsingPreprocessorDirective = false;
616     // Restore comment saving mode.
617     if (CurLexer) CurLexer->resetExtendedTokenMode();
618   }
619 
620   // Finally, if we are out of the conditional (saw an #endif or ran off the end
621   // of the file, just stop skipping and return to lexing whatever came after
622   // the #if block.
623   CurPPLexer->LexingRawMode = false;
624 
625   // The last skipped range isn't actually skipped yet if it's truncated
626   // by the end of the preamble; we'll resume parsing after the preamble.
627   if (Callbacks && (Tok.isNot(tok::eof) || !isRecordingPreamble()))
628     Callbacks->SourceRangeSkipped(
629         SourceRange(HashTokenLoc, endLoc.isValid()
630                                       ? endLoc
631                                       : CurPPLexer->getSourceLocation()),
632         Tok.getLocation());
633 }
634 
635 Module *Preprocessor::getModuleForLocation(SourceLocation Loc) {
636   if (!SourceMgr.isInMainFile(Loc)) {
637     // Try to determine the module of the include directive.
638     // FIXME: Look into directly passing the FileEntry from LookupFile instead.
639     FileID IDOfIncl = SourceMgr.getFileID(SourceMgr.getExpansionLoc(Loc));
640     if (const FileEntry *EntryOfIncl = SourceMgr.getFileEntryForID(IDOfIncl)) {
641       // The include comes from an included file.
642       return HeaderInfo.getModuleMap()
643           .findModuleForHeader(EntryOfIncl)
644           .getModule();
645     }
646   }
647 
648   // This is either in the main file or not in a file at all. It belongs
649   // to the current module, if there is one.
650   return getLangOpts().CurrentModule.empty()
651              ? nullptr
652              : HeaderInfo.lookupModule(getLangOpts().CurrentModule);
653 }
654 
655 const FileEntry *
656 Preprocessor::getHeaderToIncludeForDiagnostics(SourceLocation IncLoc,
657                                                SourceLocation Loc) {
658   Module *IncM = getModuleForLocation(IncLoc);
659 
660   // Walk up through the include stack, looking through textual headers of M
661   // until we hit a non-textual header that we can #include. (We assume textual
662   // headers of a module with non-textual headers aren't meant to be used to
663   // import entities from the module.)
664   auto &SM = getSourceManager();
665   while (!Loc.isInvalid() && !SM.isInMainFile(Loc)) {
666     auto ID = SM.getFileID(SM.getExpansionLoc(Loc));
667     auto *FE = SM.getFileEntryForID(ID);
668     if (!FE)
669       break;
670 
671     // We want to find all possible modules that might contain this header, so
672     // search all enclosing directories for module maps and load them.
673     HeaderInfo.hasModuleMap(FE->getName(), /*Root*/ nullptr,
674                             SourceMgr.isInSystemHeader(Loc));
675 
676     bool InPrivateHeader = false;
677     for (auto Header : HeaderInfo.findAllModulesForHeader(FE)) {
678       if (!Header.isAccessibleFrom(IncM)) {
679         // It's in a private header; we can't #include it.
680         // FIXME: If there's a public header in some module that re-exports it,
681         // then we could suggest including that, but it's not clear that's the
682         // expected way to make this entity visible.
683         InPrivateHeader = true;
684         continue;
685       }
686 
687       // We'll suggest including textual headers below if they're
688       // include-guarded.
689       if (Header.getRole() & ModuleMap::TextualHeader)
690         continue;
691 
692       // If we have a module import syntax, we shouldn't include a header to
693       // make a particular module visible. Let the caller know they should
694       // suggest an import instead.
695       if (getLangOpts().ObjC || getLangOpts().CPlusPlusModules ||
696           getLangOpts().ModulesTS)
697         return nullptr;
698 
699       // If this is an accessible, non-textual header of M's top-level module
700       // that transitively includes the given location and makes the
701       // corresponding module visible, this is the thing to #include.
702       return FE;
703     }
704 
705     // FIXME: If we're bailing out due to a private header, we shouldn't suggest
706     // an import either.
707     if (InPrivateHeader)
708       return nullptr;
709 
710     // If the header is includable and has an include guard, assume the
711     // intended way to expose its contents is by #include, not by importing a
712     // module that transitively includes it.
713     if (getHeaderSearchInfo().isFileMultipleIncludeGuarded(FE))
714       return FE;
715 
716     Loc = SM.getIncludeLoc(ID);
717   }
718 
719   return nullptr;
720 }
721 
722 Optional<FileEntryRef> Preprocessor::LookupFile(
723     SourceLocation FilenameLoc, StringRef Filename, bool isAngled,
724     const DirectoryLookup *FromDir, const FileEntry *FromFile,
725     const DirectoryLookup *&CurDir, SmallVectorImpl<char> *SearchPath,
726     SmallVectorImpl<char> *RelativePath,
727     ModuleMap::KnownHeader *SuggestedModule, bool *IsMapped,
728     bool *IsFrameworkFound, bool SkipCache) {
729   Module *RequestingModule = getModuleForLocation(FilenameLoc);
730   bool RequestingModuleIsModuleInterface = !SourceMgr.isInMainFile(FilenameLoc);
731 
732   // If the header lookup mechanism may be relative to the current inclusion
733   // stack, record the parent #includes.
734   SmallVector<std::pair<const FileEntry *, const DirectoryEntry *>, 16>
735       Includers;
736   bool BuildSystemModule = false;
737   if (!FromDir && !FromFile) {
738     FileID FID = getCurrentFileLexer()->getFileID();
739     const FileEntry *FileEnt = SourceMgr.getFileEntryForID(FID);
740 
741     // If there is no file entry associated with this file, it must be the
742     // predefines buffer or the module includes buffer. Any other file is not
743     // lexed with a normal lexer, so it won't be scanned for preprocessor
744     // directives.
745     //
746     // If we have the predefines buffer, resolve #include references (which come
747     // from the -include command line argument) from the current working
748     // directory instead of relative to the main file.
749     //
750     // If we have the module includes buffer, resolve #include references (which
751     // come from header declarations in the module map) relative to the module
752     // map file.
753     if (!FileEnt) {
754       if (FID == SourceMgr.getMainFileID() && MainFileDir) {
755         Includers.push_back(std::make_pair(nullptr, MainFileDir));
756         BuildSystemModule = getCurrentModule()->IsSystem;
757       } else if ((FileEnt =
758                     SourceMgr.getFileEntryForID(SourceMgr.getMainFileID())))
759         Includers.push_back(std::make_pair(FileEnt, *FileMgr.getDirectory(".")));
760     } else {
761       Includers.push_back(std::make_pair(FileEnt, FileEnt->getDir()));
762     }
763 
764     // MSVC searches the current include stack from top to bottom for
765     // headers included by quoted include directives.
766     // See: http://msdn.microsoft.com/en-us/library/36k2cdd4.aspx
767     if (LangOpts.MSVCCompat && !isAngled) {
768       for (IncludeStackInfo &ISEntry : llvm::reverse(IncludeMacroStack)) {
769         if (IsFileLexer(ISEntry))
770           if ((FileEnt = ISEntry.ThePPLexer->getFileEntry()))
771             Includers.push_back(std::make_pair(FileEnt, FileEnt->getDir()));
772       }
773     }
774   }
775 
776   CurDir = CurDirLookup;
777 
778   if (FromFile) {
779     // We're supposed to start looking from after a particular file. Search
780     // the include path until we find that file or run out of files.
781     const DirectoryLookup *TmpCurDir = CurDir;
782     const DirectoryLookup *TmpFromDir = nullptr;
783     while (Optional<FileEntryRef> FE = HeaderInfo.LookupFile(
784                Filename, FilenameLoc, isAngled, TmpFromDir, TmpCurDir,
785                Includers, SearchPath, RelativePath, RequestingModule,
786                SuggestedModule, /*IsMapped=*/nullptr,
787                /*IsFrameworkFound=*/nullptr, SkipCache)) {
788       // Keep looking as if this file did a #include_next.
789       TmpFromDir = TmpCurDir;
790       ++TmpFromDir;
791       if (&FE->getFileEntry() == FromFile) {
792         // Found it.
793         FromDir = TmpFromDir;
794         CurDir = TmpCurDir;
795         break;
796       }
797     }
798   }
799 
800   // Do a standard file entry lookup.
801   Optional<FileEntryRef> FE = HeaderInfo.LookupFile(
802       Filename, FilenameLoc, isAngled, FromDir, CurDir, Includers, SearchPath,
803       RelativePath, RequestingModule, SuggestedModule, IsMapped,
804       IsFrameworkFound, SkipCache, BuildSystemModule);
805   if (FE) {
806     if (SuggestedModule && !LangOpts.AsmPreprocessor)
807       HeaderInfo.getModuleMap().diagnoseHeaderInclusion(
808           RequestingModule, RequestingModuleIsModuleInterface, FilenameLoc,
809           Filename, &FE->getFileEntry());
810     return FE;
811   }
812 
813   const FileEntry *CurFileEnt;
814   // Otherwise, see if this is a subframework header.  If so, this is relative
815   // to one of the headers on the #include stack.  Walk the list of the current
816   // headers on the #include stack and pass them to HeaderInfo.
817   if (IsFileLexer()) {
818     if ((CurFileEnt = CurPPLexer->getFileEntry())) {
819       if (Optional<FileEntryRef> FE = HeaderInfo.LookupSubframeworkHeader(
820               Filename, CurFileEnt, SearchPath, RelativePath, RequestingModule,
821               SuggestedModule)) {
822         if (SuggestedModule && !LangOpts.AsmPreprocessor)
823           HeaderInfo.getModuleMap().diagnoseHeaderInclusion(
824               RequestingModule, RequestingModuleIsModuleInterface, FilenameLoc,
825               Filename, &FE->getFileEntry());
826         return FE;
827       }
828     }
829   }
830 
831   for (IncludeStackInfo &ISEntry : llvm::reverse(IncludeMacroStack)) {
832     if (IsFileLexer(ISEntry)) {
833       if ((CurFileEnt = ISEntry.ThePPLexer->getFileEntry())) {
834         if (Optional<FileEntryRef> FE = HeaderInfo.LookupSubframeworkHeader(
835                 Filename, CurFileEnt, SearchPath, RelativePath,
836                 RequestingModule, SuggestedModule)) {
837           if (SuggestedModule && !LangOpts.AsmPreprocessor)
838             HeaderInfo.getModuleMap().diagnoseHeaderInclusion(
839                 RequestingModule, RequestingModuleIsModuleInterface,
840                 FilenameLoc, Filename, &FE->getFileEntry());
841           return FE;
842         }
843       }
844     }
845   }
846 
847   // Otherwise, we really couldn't find the file.
848   return None;
849 }
850 
851 //===----------------------------------------------------------------------===//
852 // Preprocessor Directive Handling.
853 //===----------------------------------------------------------------------===//
854 
855 class Preprocessor::ResetMacroExpansionHelper {
856 public:
857   ResetMacroExpansionHelper(Preprocessor *pp)
858     : PP(pp), save(pp->DisableMacroExpansion) {
859     if (pp->MacroExpansionInDirectivesOverride)
860       pp->DisableMacroExpansion = false;
861   }
862 
863   ~ResetMacroExpansionHelper() {
864     PP->DisableMacroExpansion = save;
865   }
866 
867 private:
868   Preprocessor *PP;
869   bool save;
870 };
871 
872 /// Process a directive while looking for the through header or a #pragma
873 /// hdrstop. The following directives are handled:
874 /// #include (to check if it is the through header)
875 /// #define (to warn about macros that don't match the PCH)
876 /// #pragma (to check for pragma hdrstop).
877 /// All other directives are completely discarded.
878 void Preprocessor::HandleSkippedDirectiveWhileUsingPCH(Token &Result,
879                                                        SourceLocation HashLoc) {
880   if (const IdentifierInfo *II = Result.getIdentifierInfo()) {
881     if (II->getPPKeywordID() == tok::pp_define) {
882       return HandleDefineDirective(Result,
883                                    /*ImmediatelyAfterHeaderGuard=*/false);
884     }
885     if (SkippingUntilPCHThroughHeader &&
886         II->getPPKeywordID() == tok::pp_include) {
887       return HandleIncludeDirective(HashLoc, Result);
888     }
889     if (SkippingUntilPragmaHdrStop && II->getPPKeywordID() == tok::pp_pragma) {
890       Lex(Result);
891       auto *II = Result.getIdentifierInfo();
892       if (II && II->getName() == "hdrstop")
893         return HandlePragmaHdrstop(Result);
894     }
895   }
896   DiscardUntilEndOfDirective();
897 }
898 
899 /// HandleDirective - This callback is invoked when the lexer sees a # token
900 /// at the start of a line.  This consumes the directive, modifies the
901 /// lexer/preprocessor state, and advances the lexer(s) so that the next token
902 /// read is the correct one.
903 void Preprocessor::HandleDirective(Token &Result) {
904   // FIXME: Traditional: # with whitespace before it not recognized by K&R?
905 
906   // We just parsed a # character at the start of a line, so we're in directive
907   // mode.  Tell the lexer this so any newlines we see will be converted into an
908   // EOD token (which terminates the directive).
909   CurPPLexer->ParsingPreprocessorDirective = true;
910   if (CurLexer) CurLexer->SetKeepWhitespaceMode(false);
911 
912   bool ImmediatelyAfterTopLevelIfndef =
913       CurPPLexer->MIOpt.getImmediatelyAfterTopLevelIfndef();
914   CurPPLexer->MIOpt.resetImmediatelyAfterTopLevelIfndef();
915 
916   ++NumDirectives;
917 
918   // We are about to read a token.  For the multiple-include optimization FA to
919   // work, we have to remember if we had read any tokens *before* this
920   // pp-directive.
921   bool ReadAnyTokensBeforeDirective =CurPPLexer->MIOpt.getHasReadAnyTokensVal();
922 
923   // Save the '#' token in case we need to return it later.
924   Token SavedHash = Result;
925 
926   // Read the next token, the directive flavor.  This isn't expanded due to
927   // C99 6.10.3p8.
928   LexUnexpandedToken(Result);
929 
930   // C99 6.10.3p11: Is this preprocessor directive in macro invocation?  e.g.:
931   //   #define A(x) #x
932   //   A(abc
933   //     #warning blah
934   //   def)
935   // If so, the user is relying on undefined behavior, emit a diagnostic. Do
936   // not support this for #include-like directives, since that can result in
937   // terrible diagnostics, and does not work in GCC.
938   if (InMacroArgs) {
939     if (IdentifierInfo *II = Result.getIdentifierInfo()) {
940       switch (II->getPPKeywordID()) {
941       case tok::pp_include:
942       case tok::pp_import:
943       case tok::pp_include_next:
944       case tok::pp___include_macros:
945       case tok::pp_pragma:
946         Diag(Result, diag::err_embedded_directive) << II->getName();
947         Diag(*ArgMacro, diag::note_macro_expansion_here)
948             << ArgMacro->getIdentifierInfo();
949         DiscardUntilEndOfDirective();
950         return;
951       default:
952         break;
953       }
954     }
955     Diag(Result, diag::ext_embedded_directive);
956   }
957 
958   // Temporarily enable macro expansion if set so
959   // and reset to previous state when returning from this function.
960   ResetMacroExpansionHelper helper(this);
961 
962   if (SkippingUntilPCHThroughHeader || SkippingUntilPragmaHdrStop)
963     return HandleSkippedDirectiveWhileUsingPCH(Result, SavedHash.getLocation());
964 
965   switch (Result.getKind()) {
966   case tok::eod:
967     return;   // null directive.
968   case tok::code_completion:
969     if (CodeComplete)
970       CodeComplete->CodeCompleteDirective(
971                                     CurPPLexer->getConditionalStackDepth() > 0);
972     setCodeCompletionReached();
973     return;
974   case tok::numeric_constant:  // # 7  GNU line marker directive.
975     if (getLangOpts().AsmPreprocessor)
976       break;  // # 4 is not a preprocessor directive in .S files.
977     return HandleDigitDirective(Result);
978   default:
979     IdentifierInfo *II = Result.getIdentifierInfo();
980     if (!II) break; // Not an identifier.
981 
982     // Ask what the preprocessor keyword ID is.
983     switch (II->getPPKeywordID()) {
984     default: break;
985     // C99 6.10.1 - Conditional Inclusion.
986     case tok::pp_if:
987       return HandleIfDirective(Result, SavedHash, ReadAnyTokensBeforeDirective);
988     case tok::pp_ifdef:
989       return HandleIfdefDirective(Result, SavedHash, false,
990                                   true /*not valid for miopt*/);
991     case tok::pp_ifndef:
992       return HandleIfdefDirective(Result, SavedHash, true,
993                                   ReadAnyTokensBeforeDirective);
994     case tok::pp_elif:
995       return HandleElifDirective(Result, SavedHash);
996     case tok::pp_else:
997       return HandleElseDirective(Result, SavedHash);
998     case tok::pp_endif:
999       return HandleEndifDirective(Result);
1000 
1001     // C99 6.10.2 - Source File Inclusion.
1002     case tok::pp_include:
1003       // Handle #include.
1004       return HandleIncludeDirective(SavedHash.getLocation(), Result);
1005     case tok::pp___include_macros:
1006       // Handle -imacros.
1007       return HandleIncludeMacrosDirective(SavedHash.getLocation(), Result);
1008 
1009     // C99 6.10.3 - Macro Replacement.
1010     case tok::pp_define:
1011       return HandleDefineDirective(Result, ImmediatelyAfterTopLevelIfndef);
1012     case tok::pp_undef:
1013       return HandleUndefDirective();
1014 
1015     // C99 6.10.4 - Line Control.
1016     case tok::pp_line:
1017       return HandleLineDirective();
1018 
1019     // C99 6.10.5 - Error Directive.
1020     case tok::pp_error:
1021       return HandleUserDiagnosticDirective(Result, false);
1022 
1023     // C99 6.10.6 - Pragma Directive.
1024     case tok::pp_pragma:
1025       return HandlePragmaDirective({PIK_HashPragma, SavedHash.getLocation()});
1026 
1027     // GNU Extensions.
1028     case tok::pp_import:
1029       return HandleImportDirective(SavedHash.getLocation(), Result);
1030     case tok::pp_include_next:
1031       return HandleIncludeNextDirective(SavedHash.getLocation(), Result);
1032 
1033     case tok::pp_warning:
1034       Diag(Result, diag::ext_pp_warning_directive);
1035       return HandleUserDiagnosticDirective(Result, true);
1036     case tok::pp_ident:
1037       return HandleIdentSCCSDirective(Result);
1038     case tok::pp_sccs:
1039       return HandleIdentSCCSDirective(Result);
1040     case tok::pp_assert:
1041       //isExtension = true;  // FIXME: implement #assert
1042       break;
1043     case tok::pp_unassert:
1044       //isExtension = true;  // FIXME: implement #unassert
1045       break;
1046 
1047     case tok::pp___public_macro:
1048       if (getLangOpts().Modules)
1049         return HandleMacroPublicDirective(Result);
1050       break;
1051 
1052     case tok::pp___private_macro:
1053       if (getLangOpts().Modules)
1054         return HandleMacroPrivateDirective();
1055       break;
1056     }
1057     break;
1058   }
1059 
1060   // If this is a .S file, treat unknown # directives as non-preprocessor
1061   // directives.  This is important because # may be a comment or introduce
1062   // various pseudo-ops.  Just return the # token and push back the following
1063   // token to be lexed next time.
1064   if (getLangOpts().AsmPreprocessor) {
1065     auto Toks = std::make_unique<Token[]>(2);
1066     // Return the # and the token after it.
1067     Toks[0] = SavedHash;
1068     Toks[1] = Result;
1069 
1070     // If the second token is a hashhash token, then we need to translate it to
1071     // unknown so the token lexer doesn't try to perform token pasting.
1072     if (Result.is(tok::hashhash))
1073       Toks[1].setKind(tok::unknown);
1074 
1075     // Enter this token stream so that we re-lex the tokens.  Make sure to
1076     // enable macro expansion, in case the token after the # is an identifier
1077     // that is expanded.
1078     EnterTokenStream(std::move(Toks), 2, false, /*IsReinject*/false);
1079     return;
1080   }
1081 
1082   // If we reached here, the preprocessing token is not valid!
1083   Diag(Result, diag::err_pp_invalid_directive);
1084 
1085   // Read the rest of the PP line.
1086   DiscardUntilEndOfDirective();
1087 
1088   // Okay, we're done parsing the directive.
1089 }
1090 
1091 /// GetLineValue - Convert a numeric token into an unsigned value, emitting
1092 /// Diagnostic DiagID if it is invalid, and returning the value in Val.
1093 static bool GetLineValue(Token &DigitTok, unsigned &Val,
1094                          unsigned DiagID, Preprocessor &PP,
1095                          bool IsGNULineDirective=false) {
1096   if (DigitTok.isNot(tok::numeric_constant)) {
1097     PP.Diag(DigitTok, DiagID);
1098 
1099     if (DigitTok.isNot(tok::eod))
1100       PP.DiscardUntilEndOfDirective();
1101     return true;
1102   }
1103 
1104   SmallString<64> IntegerBuffer;
1105   IntegerBuffer.resize(DigitTok.getLength());
1106   const char *DigitTokBegin = &IntegerBuffer[0];
1107   bool Invalid = false;
1108   unsigned ActualLength = PP.getSpelling(DigitTok, DigitTokBegin, &Invalid);
1109   if (Invalid)
1110     return true;
1111 
1112   // Verify that we have a simple digit-sequence, and compute the value.  This
1113   // is always a simple digit string computed in decimal, so we do this manually
1114   // here.
1115   Val = 0;
1116   for (unsigned i = 0; i != ActualLength; ++i) {
1117     // C++1y [lex.fcon]p1:
1118     //   Optional separating single quotes in a digit-sequence are ignored
1119     if (DigitTokBegin[i] == '\'')
1120       continue;
1121 
1122     if (!isDigit(DigitTokBegin[i])) {
1123       PP.Diag(PP.AdvanceToTokenCharacter(DigitTok.getLocation(), i),
1124               diag::err_pp_line_digit_sequence) << IsGNULineDirective;
1125       PP.DiscardUntilEndOfDirective();
1126       return true;
1127     }
1128 
1129     unsigned NextVal = Val*10+(DigitTokBegin[i]-'0');
1130     if (NextVal < Val) { // overflow.
1131       PP.Diag(DigitTok, DiagID);
1132       PP.DiscardUntilEndOfDirective();
1133       return true;
1134     }
1135     Val = NextVal;
1136   }
1137 
1138   if (DigitTokBegin[0] == '0' && Val)
1139     PP.Diag(DigitTok.getLocation(), diag::warn_pp_line_decimal)
1140       << IsGNULineDirective;
1141 
1142   return false;
1143 }
1144 
1145 /// Handle a \#line directive: C99 6.10.4.
1146 ///
1147 /// The two acceptable forms are:
1148 /// \verbatim
1149 ///   # line digit-sequence
1150 ///   # line digit-sequence "s-char-sequence"
1151 /// \endverbatim
1152 void Preprocessor::HandleLineDirective() {
1153   // Read the line # and string argument.  Per C99 6.10.4p5, these tokens are
1154   // expanded.
1155   Token DigitTok;
1156   Lex(DigitTok);
1157 
1158   // Validate the number and convert it to an unsigned.
1159   unsigned LineNo;
1160   if (GetLineValue(DigitTok, LineNo, diag::err_pp_line_requires_integer,*this))
1161     return;
1162 
1163   if (LineNo == 0)
1164     Diag(DigitTok, diag::ext_pp_line_zero);
1165 
1166   // Enforce C99 6.10.4p3: "The digit sequence shall not specify ... a
1167   // number greater than 2147483647".  C90 requires that the line # be <= 32767.
1168   unsigned LineLimit = 32768U;
1169   if (LangOpts.C99 || LangOpts.CPlusPlus11)
1170     LineLimit = 2147483648U;
1171   if (LineNo >= LineLimit)
1172     Diag(DigitTok, diag::ext_pp_line_too_big) << LineLimit;
1173   else if (LangOpts.CPlusPlus11 && LineNo >= 32768U)
1174     Diag(DigitTok, diag::warn_cxx98_compat_pp_line_too_big);
1175 
1176   int FilenameID = -1;
1177   Token StrTok;
1178   Lex(StrTok);
1179 
1180   // If the StrTok is "eod", then it wasn't present.  Otherwise, it must be a
1181   // string followed by eod.
1182   if (StrTok.is(tok::eod))
1183     ; // ok
1184   else if (StrTok.isNot(tok::string_literal)) {
1185     Diag(StrTok, diag::err_pp_line_invalid_filename);
1186     DiscardUntilEndOfDirective();
1187     return;
1188   } else if (StrTok.hasUDSuffix()) {
1189     Diag(StrTok, diag::err_invalid_string_udl);
1190     DiscardUntilEndOfDirective();
1191     return;
1192   } else {
1193     // Parse and validate the string, converting it into a unique ID.
1194     StringLiteralParser Literal(StrTok, *this);
1195     assert(Literal.isAscii() && "Didn't allow wide strings in");
1196     if (Literal.hadError) {
1197       DiscardUntilEndOfDirective();
1198       return;
1199     }
1200     if (Literal.Pascal) {
1201       Diag(StrTok, diag::err_pp_linemarker_invalid_filename);
1202       DiscardUntilEndOfDirective();
1203       return;
1204     }
1205     FilenameID = SourceMgr.getLineTableFilenameID(Literal.GetString());
1206 
1207     // Verify that there is nothing after the string, other than EOD.  Because
1208     // of C99 6.10.4p5, macros that expand to empty tokens are ok.
1209     CheckEndOfDirective("line", true);
1210   }
1211 
1212   // Take the file kind of the file containing the #line directive. #line
1213   // directives are often used for generated sources from the same codebase, so
1214   // the new file should generally be classified the same way as the current
1215   // file. This is visible in GCC's pre-processed output, which rewrites #line
1216   // to GNU line markers.
1217   SrcMgr::CharacteristicKind FileKind =
1218       SourceMgr.getFileCharacteristic(DigitTok.getLocation());
1219 
1220   SourceMgr.AddLineNote(DigitTok.getLocation(), LineNo, FilenameID, false,
1221                         false, FileKind);
1222 
1223   if (Callbacks)
1224     Callbacks->FileChanged(CurPPLexer->getSourceLocation(),
1225                            PPCallbacks::RenameFile, FileKind);
1226 }
1227 
1228 /// ReadLineMarkerFlags - Parse and validate any flags at the end of a GNU line
1229 /// marker directive.
1230 static bool ReadLineMarkerFlags(bool &IsFileEntry, bool &IsFileExit,
1231                                 SrcMgr::CharacteristicKind &FileKind,
1232                                 Preprocessor &PP) {
1233   unsigned FlagVal;
1234   Token FlagTok;
1235   PP.Lex(FlagTok);
1236   if (FlagTok.is(tok::eod)) return false;
1237   if (GetLineValue(FlagTok, FlagVal, diag::err_pp_linemarker_invalid_flag, PP))
1238     return true;
1239 
1240   if (FlagVal == 1) {
1241     IsFileEntry = true;
1242 
1243     PP.Lex(FlagTok);
1244     if (FlagTok.is(tok::eod)) return false;
1245     if (GetLineValue(FlagTok, FlagVal, diag::err_pp_linemarker_invalid_flag,PP))
1246       return true;
1247   } else if (FlagVal == 2) {
1248     IsFileExit = true;
1249 
1250     SourceManager &SM = PP.getSourceManager();
1251     // If we are leaving the current presumed file, check to make sure the
1252     // presumed include stack isn't empty!
1253     FileID CurFileID =
1254       SM.getDecomposedExpansionLoc(FlagTok.getLocation()).first;
1255     PresumedLoc PLoc = SM.getPresumedLoc(FlagTok.getLocation());
1256     if (PLoc.isInvalid())
1257       return true;
1258 
1259     // If there is no include loc (main file) or if the include loc is in a
1260     // different physical file, then we aren't in a "1" line marker flag region.
1261     SourceLocation IncLoc = PLoc.getIncludeLoc();
1262     if (IncLoc.isInvalid() ||
1263         SM.getDecomposedExpansionLoc(IncLoc).first != CurFileID) {
1264       PP.Diag(FlagTok, diag::err_pp_linemarker_invalid_pop);
1265       PP.DiscardUntilEndOfDirective();
1266       return true;
1267     }
1268 
1269     PP.Lex(FlagTok);
1270     if (FlagTok.is(tok::eod)) return false;
1271     if (GetLineValue(FlagTok, FlagVal, diag::err_pp_linemarker_invalid_flag,PP))
1272       return true;
1273   }
1274 
1275   // We must have 3 if there are still flags.
1276   if (FlagVal != 3) {
1277     PP.Diag(FlagTok, diag::err_pp_linemarker_invalid_flag);
1278     PP.DiscardUntilEndOfDirective();
1279     return true;
1280   }
1281 
1282   FileKind = SrcMgr::C_System;
1283 
1284   PP.Lex(FlagTok);
1285   if (FlagTok.is(tok::eod)) return false;
1286   if (GetLineValue(FlagTok, FlagVal, diag::err_pp_linemarker_invalid_flag, PP))
1287     return true;
1288 
1289   // We must have 4 if there is yet another flag.
1290   if (FlagVal != 4) {
1291     PP.Diag(FlagTok, diag::err_pp_linemarker_invalid_flag);
1292     PP.DiscardUntilEndOfDirective();
1293     return true;
1294   }
1295 
1296   FileKind = SrcMgr::C_ExternCSystem;
1297 
1298   PP.Lex(FlagTok);
1299   if (FlagTok.is(tok::eod)) return false;
1300 
1301   // There are no more valid flags here.
1302   PP.Diag(FlagTok, diag::err_pp_linemarker_invalid_flag);
1303   PP.DiscardUntilEndOfDirective();
1304   return true;
1305 }
1306 
1307 /// HandleDigitDirective - Handle a GNU line marker directive, whose syntax is
1308 /// one of the following forms:
1309 ///
1310 ///     # 42
1311 ///     # 42 "file" ('1' | '2')?
1312 ///     # 42 "file" ('1' | '2')? '3' '4'?
1313 ///
1314 void Preprocessor::HandleDigitDirective(Token &DigitTok) {
1315   // Validate the number and convert it to an unsigned.  GNU does not have a
1316   // line # limit other than it fit in 32-bits.
1317   unsigned LineNo;
1318   if (GetLineValue(DigitTok, LineNo, diag::err_pp_linemarker_requires_integer,
1319                    *this, true))
1320     return;
1321 
1322   Token StrTok;
1323   Lex(StrTok);
1324 
1325   bool IsFileEntry = false, IsFileExit = false;
1326   int FilenameID = -1;
1327   SrcMgr::CharacteristicKind FileKind = SrcMgr::C_User;
1328 
1329   // If the StrTok is "eod", then it wasn't present.  Otherwise, it must be a
1330   // string followed by eod.
1331   if (StrTok.is(tok::eod)) {
1332     // Treat this like "#line NN", which doesn't change file characteristics.
1333     FileKind = SourceMgr.getFileCharacteristic(DigitTok.getLocation());
1334   } else if (StrTok.isNot(tok::string_literal)) {
1335     Diag(StrTok, diag::err_pp_linemarker_invalid_filename);
1336     DiscardUntilEndOfDirective();
1337     return;
1338   } else if (StrTok.hasUDSuffix()) {
1339     Diag(StrTok, diag::err_invalid_string_udl);
1340     DiscardUntilEndOfDirective();
1341     return;
1342   } else {
1343     // Parse and validate the string, converting it into a unique ID.
1344     StringLiteralParser Literal(StrTok, *this);
1345     assert(Literal.isAscii() && "Didn't allow wide strings in");
1346     if (Literal.hadError) {
1347       DiscardUntilEndOfDirective();
1348       return;
1349     }
1350     if (Literal.Pascal) {
1351       Diag(StrTok, diag::err_pp_linemarker_invalid_filename);
1352       DiscardUntilEndOfDirective();
1353       return;
1354     }
1355     FilenameID = SourceMgr.getLineTableFilenameID(Literal.GetString());
1356 
1357     // If a filename was present, read any flags that are present.
1358     if (ReadLineMarkerFlags(IsFileEntry, IsFileExit, FileKind, *this))
1359       return;
1360   }
1361 
1362   // Create a line note with this information.
1363   SourceMgr.AddLineNote(DigitTok.getLocation(), LineNo, FilenameID, IsFileEntry,
1364                         IsFileExit, FileKind);
1365 
1366   // If the preprocessor has callbacks installed, notify them of the #line
1367   // change.  This is used so that the line marker comes out in -E mode for
1368   // example.
1369   if (Callbacks) {
1370     PPCallbacks::FileChangeReason Reason = PPCallbacks::RenameFile;
1371     if (IsFileEntry)
1372       Reason = PPCallbacks::EnterFile;
1373     else if (IsFileExit)
1374       Reason = PPCallbacks::ExitFile;
1375 
1376     Callbacks->FileChanged(CurPPLexer->getSourceLocation(), Reason, FileKind);
1377   }
1378 }
1379 
1380 /// HandleUserDiagnosticDirective - Handle a #warning or #error directive.
1381 ///
1382 void Preprocessor::HandleUserDiagnosticDirective(Token &Tok,
1383                                                  bool isWarning) {
1384   // Read the rest of the line raw.  We do this because we don't want macros
1385   // to be expanded and we don't require that the tokens be valid preprocessing
1386   // tokens.  For example, this is allowed: "#warning `   'foo".  GCC does
1387   // collapse multiple consecutive white space between tokens, but this isn't
1388   // specified by the standard.
1389   SmallString<128> Message;
1390   CurLexer->ReadToEndOfLine(&Message);
1391 
1392   // Find the first non-whitespace character, so that we can make the
1393   // diagnostic more succinct.
1394   StringRef Msg = StringRef(Message).ltrim(' ');
1395 
1396   if (isWarning)
1397     Diag(Tok, diag::pp_hash_warning) << Msg;
1398   else
1399     Diag(Tok, diag::err_pp_hash_error) << Msg;
1400 }
1401 
1402 /// HandleIdentSCCSDirective - Handle a #ident/#sccs directive.
1403 ///
1404 void Preprocessor::HandleIdentSCCSDirective(Token &Tok) {
1405   // Yes, this directive is an extension.
1406   Diag(Tok, diag::ext_pp_ident_directive);
1407 
1408   // Read the string argument.
1409   Token StrTok;
1410   Lex(StrTok);
1411 
1412   // If the token kind isn't a string, it's a malformed directive.
1413   if (StrTok.isNot(tok::string_literal) &&
1414       StrTok.isNot(tok::wide_string_literal)) {
1415     Diag(StrTok, diag::err_pp_malformed_ident);
1416     if (StrTok.isNot(tok::eod))
1417       DiscardUntilEndOfDirective();
1418     return;
1419   }
1420 
1421   if (StrTok.hasUDSuffix()) {
1422     Diag(StrTok, diag::err_invalid_string_udl);
1423     DiscardUntilEndOfDirective();
1424     return;
1425   }
1426 
1427   // Verify that there is nothing after the string, other than EOD.
1428   CheckEndOfDirective("ident");
1429 
1430   if (Callbacks) {
1431     bool Invalid = false;
1432     std::string Str = getSpelling(StrTok, &Invalid);
1433     if (!Invalid)
1434       Callbacks->Ident(Tok.getLocation(), Str);
1435   }
1436 }
1437 
1438 /// Handle a #public directive.
1439 void Preprocessor::HandleMacroPublicDirective(Token &Tok) {
1440   Token MacroNameTok;
1441   ReadMacroName(MacroNameTok, MU_Undef);
1442 
1443   // Error reading macro name?  If so, diagnostic already issued.
1444   if (MacroNameTok.is(tok::eod))
1445     return;
1446 
1447   // Check to see if this is the last token on the #__public_macro line.
1448   CheckEndOfDirective("__public_macro");
1449 
1450   IdentifierInfo *II = MacroNameTok.getIdentifierInfo();
1451   // Okay, we finally have a valid identifier to undef.
1452   MacroDirective *MD = getLocalMacroDirective(II);
1453 
1454   // If the macro is not defined, this is an error.
1455   if (!MD) {
1456     Diag(MacroNameTok, diag::err_pp_visibility_non_macro) << II;
1457     return;
1458   }
1459 
1460   // Note that this macro has now been exported.
1461   appendMacroDirective(II, AllocateVisibilityMacroDirective(
1462                                 MacroNameTok.getLocation(), /*isPublic=*/true));
1463 }
1464 
1465 /// Handle a #private directive.
1466 void Preprocessor::HandleMacroPrivateDirective() {
1467   Token MacroNameTok;
1468   ReadMacroName(MacroNameTok, MU_Undef);
1469 
1470   // Error reading macro name?  If so, diagnostic already issued.
1471   if (MacroNameTok.is(tok::eod))
1472     return;
1473 
1474   // Check to see if this is the last token on the #__private_macro line.
1475   CheckEndOfDirective("__private_macro");
1476 
1477   IdentifierInfo *II = MacroNameTok.getIdentifierInfo();
1478   // Okay, we finally have a valid identifier to undef.
1479   MacroDirective *MD = getLocalMacroDirective(II);
1480 
1481   // If the macro is not defined, this is an error.
1482   if (!MD) {
1483     Diag(MacroNameTok, diag::err_pp_visibility_non_macro) << II;
1484     return;
1485   }
1486 
1487   // Note that this macro has now been marked private.
1488   appendMacroDirective(II, AllocateVisibilityMacroDirective(
1489                                MacroNameTok.getLocation(), /*isPublic=*/false));
1490 }
1491 
1492 //===----------------------------------------------------------------------===//
1493 // Preprocessor Include Directive Handling.
1494 //===----------------------------------------------------------------------===//
1495 
1496 /// GetIncludeFilenameSpelling - Turn the specified lexer token into a fully
1497 /// checked and spelled filename, e.g. as an operand of \#include. This returns
1498 /// true if the input filename was in <>'s or false if it were in ""'s.  The
1499 /// caller is expected to provide a buffer that is large enough to hold the
1500 /// spelling of the filename, but is also expected to handle the case when
1501 /// this method decides to use a different buffer.
1502 bool Preprocessor::GetIncludeFilenameSpelling(SourceLocation Loc,
1503                                               StringRef &Buffer) {
1504   // Get the text form of the filename.
1505   assert(!Buffer.empty() && "Can't have tokens with empty spellings!");
1506 
1507   // FIXME: Consider warning on some of the cases described in C11 6.4.7/3 and
1508   // C++20 [lex.header]/2:
1509   //
1510   // If `"`, `'`, `\`, `/*`, or `//` appears in a header-name, then
1511   //   in C: behavior is undefined
1512   //   in C++: program is conditionally-supported with implementation-defined
1513   //           semantics
1514 
1515   // Make sure the filename is <x> or "x".
1516   bool isAngled;
1517   if (Buffer[0] == '<') {
1518     if (Buffer.back() != '>') {
1519       Diag(Loc, diag::err_pp_expects_filename);
1520       Buffer = StringRef();
1521       return true;
1522     }
1523     isAngled = true;
1524   } else if (Buffer[0] == '"') {
1525     if (Buffer.back() != '"') {
1526       Diag(Loc, diag::err_pp_expects_filename);
1527       Buffer = StringRef();
1528       return true;
1529     }
1530     isAngled = false;
1531   } else {
1532     Diag(Loc, diag::err_pp_expects_filename);
1533     Buffer = StringRef();
1534     return true;
1535   }
1536 
1537   // Diagnose #include "" as invalid.
1538   if (Buffer.size() <= 2) {
1539     Diag(Loc, diag::err_pp_empty_filename);
1540     Buffer = StringRef();
1541     return true;
1542   }
1543 
1544   // Skip the brackets.
1545   Buffer = Buffer.substr(1, Buffer.size()-2);
1546   return isAngled;
1547 }
1548 
1549 /// Push a token onto the token stream containing an annotation.
1550 void Preprocessor::EnterAnnotationToken(SourceRange Range,
1551                                         tok::TokenKind Kind,
1552                                         void *AnnotationVal) {
1553   // FIXME: Produce this as the current token directly, rather than
1554   // allocating a new token for it.
1555   auto Tok = std::make_unique<Token[]>(1);
1556   Tok[0].startToken();
1557   Tok[0].setKind(Kind);
1558   Tok[0].setLocation(Range.getBegin());
1559   Tok[0].setAnnotationEndLoc(Range.getEnd());
1560   Tok[0].setAnnotationValue(AnnotationVal);
1561   EnterTokenStream(std::move(Tok), 1, true, /*IsReinject*/ false);
1562 }
1563 
1564 /// Produce a diagnostic informing the user that a #include or similar
1565 /// was implicitly treated as a module import.
1566 static void diagnoseAutoModuleImport(
1567     Preprocessor &PP, SourceLocation HashLoc, Token &IncludeTok,
1568     ArrayRef<std::pair<IdentifierInfo *, SourceLocation>> Path,
1569     SourceLocation PathEnd) {
1570   StringRef ImportKeyword;
1571   if (PP.getLangOpts().ObjC)
1572     ImportKeyword = "@import";
1573   else if (PP.getLangOpts().ModulesTS || PP.getLangOpts().CPlusPlusModules)
1574     ImportKeyword = "import";
1575   else
1576     return; // no import syntax available
1577 
1578   SmallString<128> PathString;
1579   for (size_t I = 0, N = Path.size(); I != N; ++I) {
1580     if (I)
1581       PathString += '.';
1582     PathString += Path[I].first->getName();
1583   }
1584   int IncludeKind = 0;
1585 
1586   switch (IncludeTok.getIdentifierInfo()->getPPKeywordID()) {
1587   case tok::pp_include:
1588     IncludeKind = 0;
1589     break;
1590 
1591   case tok::pp_import:
1592     IncludeKind = 1;
1593     break;
1594 
1595   case tok::pp_include_next:
1596     IncludeKind = 2;
1597     break;
1598 
1599   case tok::pp___include_macros:
1600     IncludeKind = 3;
1601     break;
1602 
1603   default:
1604     llvm_unreachable("unknown include directive kind");
1605   }
1606 
1607   CharSourceRange ReplaceRange(SourceRange(HashLoc, PathEnd),
1608                                /*IsTokenRange=*/false);
1609   PP.Diag(HashLoc, diag::warn_auto_module_import)
1610       << IncludeKind << PathString
1611       << FixItHint::CreateReplacement(
1612              ReplaceRange, (ImportKeyword + " " + PathString + ";").str());
1613 }
1614 
1615 // Given a vector of path components and a string containing the real
1616 // path to the file, build a properly-cased replacement in the vector,
1617 // and return true if the replacement should be suggested.
1618 static bool trySimplifyPath(SmallVectorImpl<StringRef> &Components,
1619                             StringRef RealPathName) {
1620   auto RealPathComponentIter = llvm::sys::path::rbegin(RealPathName);
1621   auto RealPathComponentEnd = llvm::sys::path::rend(RealPathName);
1622   int Cnt = 0;
1623   bool SuggestReplacement = false;
1624   // Below is a best-effort to handle ".." in paths. It is admittedly
1625   // not 100% correct in the presence of symlinks.
1626   for (auto &Component : llvm::reverse(Components)) {
1627     if ("." == Component) {
1628     } else if (".." == Component) {
1629       ++Cnt;
1630     } else if (Cnt) {
1631       --Cnt;
1632     } else if (RealPathComponentIter != RealPathComponentEnd) {
1633       if (Component != *RealPathComponentIter) {
1634         // If these path components differ by more than just case, then we
1635         // may be looking at symlinked paths. Bail on this diagnostic to avoid
1636         // noisy false positives.
1637         SuggestReplacement = RealPathComponentIter->equals_lower(Component);
1638         if (!SuggestReplacement)
1639           break;
1640         Component = *RealPathComponentIter;
1641       }
1642       ++RealPathComponentIter;
1643     }
1644   }
1645   return SuggestReplacement;
1646 }
1647 
1648 bool Preprocessor::checkModuleIsAvailable(const LangOptions &LangOpts,
1649                                           const TargetInfo &TargetInfo,
1650                                           DiagnosticsEngine &Diags, Module *M) {
1651   Module::Requirement Requirement;
1652   Module::UnresolvedHeaderDirective MissingHeader;
1653   Module *ShadowingModule = nullptr;
1654   if (M->isAvailable(LangOpts, TargetInfo, Requirement, MissingHeader,
1655                      ShadowingModule))
1656     return false;
1657 
1658   if (MissingHeader.FileNameLoc.isValid()) {
1659     Diags.Report(MissingHeader.FileNameLoc, diag::err_module_header_missing)
1660         << MissingHeader.IsUmbrella << MissingHeader.FileName;
1661   } else if (ShadowingModule) {
1662     Diags.Report(M->DefinitionLoc, diag::err_module_shadowed) << M->Name;
1663     Diags.Report(ShadowingModule->DefinitionLoc,
1664                  diag::note_previous_definition);
1665   } else {
1666     // FIXME: Track the location at which the requirement was specified, and
1667     // use it here.
1668     Diags.Report(M->DefinitionLoc, diag::err_module_unavailable)
1669         << M->getFullModuleName() << Requirement.second << Requirement.first;
1670   }
1671   return true;
1672 }
1673 
1674 /// HandleIncludeDirective - The "\#include" tokens have just been read, read
1675 /// the file to be included from the lexer, then include it!  This is a common
1676 /// routine with functionality shared between \#include, \#include_next and
1677 /// \#import.  LookupFrom is set when this is a \#include_next directive, it
1678 /// specifies the file to start searching from.
1679 void Preprocessor::HandleIncludeDirective(SourceLocation HashLoc,
1680                                           Token &IncludeTok,
1681                                           const DirectoryLookup *LookupFrom,
1682                                           const FileEntry *LookupFromFile) {
1683   Token FilenameTok;
1684   if (LexHeaderName(FilenameTok))
1685     return;
1686 
1687   if (FilenameTok.isNot(tok::header_name)) {
1688     Diag(FilenameTok.getLocation(), diag::err_pp_expects_filename);
1689     if (FilenameTok.isNot(tok::eod))
1690       DiscardUntilEndOfDirective();
1691     return;
1692   }
1693 
1694   // Verify that there is nothing after the filename, other than EOD.  Note
1695   // that we allow macros that expand to nothing after the filename, because
1696   // this falls into the category of "#include pp-tokens new-line" specified
1697   // in C99 6.10.2p4.
1698   SourceLocation EndLoc =
1699       CheckEndOfDirective(IncludeTok.getIdentifierInfo()->getNameStart(), true);
1700 
1701   auto Action = HandleHeaderIncludeOrImport(HashLoc, IncludeTok, FilenameTok,
1702                                             EndLoc, LookupFrom, LookupFromFile);
1703   switch (Action.Kind) {
1704   case ImportAction::None:
1705   case ImportAction::SkippedModuleImport:
1706     break;
1707   case ImportAction::ModuleBegin:
1708     EnterAnnotationToken(SourceRange(HashLoc, EndLoc),
1709                          tok::annot_module_begin, Action.ModuleForHeader);
1710     break;
1711   case ImportAction::ModuleImport:
1712     EnterAnnotationToken(SourceRange(HashLoc, EndLoc),
1713                          tok::annot_module_include, Action.ModuleForHeader);
1714     break;
1715   case ImportAction::Failure:
1716     assert(TheModuleLoader.HadFatalFailure &&
1717            "This should be an early exit only to a fatal error");
1718     TheModuleLoader.HadFatalFailure = true;
1719     IncludeTok.setKind(tok::eof);
1720     CurLexer->cutOffLexing();
1721     return;
1722   }
1723 }
1724 
1725 Optional<FileEntryRef> Preprocessor::LookupHeaderIncludeOrImport(
1726     const DirectoryLookup *&CurDir, StringRef& Filename,
1727     SourceLocation FilenameLoc, CharSourceRange FilenameRange,
1728     const Token &FilenameTok, bool &IsFrameworkFound, bool IsImportDecl,
1729     bool &IsMapped, const DirectoryLookup *LookupFrom,
1730     const FileEntry *LookupFromFile, StringRef& LookupFilename,
1731     SmallVectorImpl<char> &RelativePath, SmallVectorImpl<char> &SearchPath,
1732     ModuleMap::KnownHeader &SuggestedModule, bool isAngled) {
1733   Optional<FileEntryRef> File = LookupFile(
1734       FilenameLoc, LookupFilename,
1735       isAngled, LookupFrom, LookupFromFile, CurDir,
1736       Callbacks ? &SearchPath : nullptr, Callbacks ? &RelativePath : nullptr,
1737       &SuggestedModule, &IsMapped, &IsFrameworkFound);
1738   if (File)
1739     return File;
1740 
1741   if (Callbacks) {
1742     // Give the clients a chance to recover.
1743     SmallString<128> RecoveryPath;
1744     if (Callbacks->FileNotFound(Filename, RecoveryPath)) {
1745       if (auto DE = FileMgr.getOptionalDirectoryRef(RecoveryPath)) {
1746         // Add the recovery path to the list of search paths.
1747         DirectoryLookup DL(*DE, SrcMgr::C_User, false);
1748         HeaderInfo.AddSearchPath(DL, isAngled);
1749 
1750         // Try the lookup again, skipping the cache.
1751         Optional<FileEntryRef> File = LookupFile(
1752             FilenameLoc,
1753             LookupFilename, isAngled,
1754             LookupFrom, LookupFromFile, CurDir, nullptr, nullptr,
1755             &SuggestedModule, &IsMapped, /*IsFrameworkFound=*/nullptr,
1756             /*SkipCache*/ true);
1757         if (File)
1758           return File;
1759       }
1760     }
1761   }
1762 
1763   if (SuppressIncludeNotFoundError)
1764     return None;
1765 
1766   // If the file could not be located and it was included via angle
1767   // brackets, we can attempt a lookup as though it were a quoted path to
1768   // provide the user with a possible fixit.
1769   if (isAngled) {
1770     Optional<FileEntryRef> File = LookupFile(
1771         FilenameLoc, LookupFilename,
1772         false, LookupFrom, LookupFromFile, CurDir,
1773         Callbacks ? &SearchPath : nullptr, Callbacks ? &RelativePath : nullptr,
1774         &SuggestedModule, &IsMapped,
1775         /*IsFrameworkFound=*/nullptr);
1776     if (File) {
1777       Diag(FilenameTok, diag::err_pp_file_not_found_angled_include_not_fatal)
1778           << Filename << IsImportDecl
1779           << FixItHint::CreateReplacement(FilenameRange,
1780                                           "\"" + Filename.str() + "\"");
1781       return File;
1782     }
1783   }
1784 
1785   // Check for likely typos due to leading or trailing non-isAlphanumeric
1786   // characters
1787   StringRef OriginalFilename = Filename;
1788   if (LangOpts.SpellChecking) {
1789     // A heuristic to correct a typo file name by removing leading and
1790     // trailing non-isAlphanumeric characters.
1791     auto CorrectTypoFilename = [](llvm::StringRef Filename) {
1792       Filename = Filename.drop_until(isAlphanumeric);
1793       while (!Filename.empty() && !isAlphanumeric(Filename.back())) {
1794         Filename = Filename.drop_back();
1795       }
1796       return Filename;
1797     };
1798     StringRef TypoCorrectionName = CorrectTypoFilename(Filename);
1799     StringRef TypoCorrectionLookupName = CorrectTypoFilename(LookupFilename);
1800 
1801     Optional<FileEntryRef> File = LookupFile(
1802         FilenameLoc, TypoCorrectionLookupName, isAngled, LookupFrom, LookupFromFile,
1803         CurDir, Callbacks ? &SearchPath : nullptr,
1804         Callbacks ? &RelativePath : nullptr, &SuggestedModule, &IsMapped,
1805         /*IsFrameworkFound=*/nullptr);
1806     if (File) {
1807       auto Hint =
1808           isAngled ? FixItHint::CreateReplacement(
1809                          FilenameRange, "<" + TypoCorrectionName.str() + ">")
1810                    : FixItHint::CreateReplacement(
1811                          FilenameRange, "\"" + TypoCorrectionName.str() + "\"");
1812       Diag(FilenameTok, diag::err_pp_file_not_found_typo_not_fatal)
1813           << OriginalFilename << TypoCorrectionName << Hint;
1814       // We found the file, so set the Filename to the name after typo
1815       // correction.
1816       Filename = TypoCorrectionName;
1817       LookupFilename = TypoCorrectionLookupName;
1818       return File;
1819     }
1820   }
1821 
1822   // If the file is still not found, just go with the vanilla diagnostic
1823   assert(!File.hasValue() && "expected missing file");
1824   Diag(FilenameTok, diag::err_pp_file_not_found)
1825       << OriginalFilename << FilenameRange;
1826   if (IsFrameworkFound) {
1827     size_t SlashPos = OriginalFilename.find('/');
1828     assert(SlashPos != StringRef::npos &&
1829            "Include with framework name should have '/' in the filename");
1830     StringRef FrameworkName = OriginalFilename.substr(0, SlashPos);
1831     FrameworkCacheEntry &CacheEntry =
1832         HeaderInfo.LookupFrameworkCache(FrameworkName);
1833     assert(CacheEntry.Directory && "Found framework should be in cache");
1834     Diag(FilenameTok, diag::note_pp_framework_without_header)
1835         << OriginalFilename.substr(SlashPos + 1) << FrameworkName
1836         << CacheEntry.Directory->getName();
1837   }
1838 
1839   return None;
1840 }
1841 
1842 /// Handle either a #include-like directive or an import declaration that names
1843 /// a header file.
1844 ///
1845 /// \param HashLoc The location of the '#' token for an include, or
1846 ///        SourceLocation() for an import declaration.
1847 /// \param IncludeTok The include / include_next / import token.
1848 /// \param FilenameTok The header-name token.
1849 /// \param EndLoc The location at which any imported macros become visible.
1850 /// \param LookupFrom For #include_next, the starting directory for the
1851 ///        directory lookup.
1852 /// \param LookupFromFile For #include_next, the starting file for the directory
1853 ///        lookup.
1854 Preprocessor::ImportAction Preprocessor::HandleHeaderIncludeOrImport(
1855     SourceLocation HashLoc, Token &IncludeTok, Token &FilenameTok,
1856     SourceLocation EndLoc, const DirectoryLookup *LookupFrom,
1857     const FileEntry *LookupFromFile) {
1858   SmallString<128> FilenameBuffer;
1859   StringRef Filename = getSpelling(FilenameTok, FilenameBuffer);
1860   SourceLocation CharEnd = FilenameTok.getEndLoc();
1861 
1862   CharSourceRange FilenameRange
1863     = CharSourceRange::getCharRange(FilenameTok.getLocation(), CharEnd);
1864   StringRef OriginalFilename = Filename;
1865   bool isAngled =
1866     GetIncludeFilenameSpelling(FilenameTok.getLocation(), Filename);
1867 
1868   // If GetIncludeFilenameSpelling set the start ptr to null, there was an
1869   // error.
1870   if (Filename.empty())
1871     return {ImportAction::None};
1872 
1873   bool IsImportDecl = HashLoc.isInvalid();
1874   SourceLocation StartLoc = IsImportDecl ? IncludeTok.getLocation() : HashLoc;
1875 
1876   // Complain about attempts to #include files in an audit pragma.
1877   if (PragmaARCCFCodeAuditedInfo.second.isValid()) {
1878     Diag(StartLoc, diag::err_pp_include_in_arc_cf_code_audited) << IsImportDecl;
1879     Diag(PragmaARCCFCodeAuditedInfo.second, diag::note_pragma_entered_here);
1880 
1881     // Immediately leave the pragma.
1882     PragmaARCCFCodeAuditedInfo = {nullptr, SourceLocation()};
1883   }
1884 
1885   // Complain about attempts to #include files in an assume-nonnull pragma.
1886   if (PragmaAssumeNonNullLoc.isValid()) {
1887     Diag(StartLoc, diag::err_pp_include_in_assume_nonnull) << IsImportDecl;
1888     Diag(PragmaAssumeNonNullLoc, diag::note_pragma_entered_here);
1889 
1890     // Immediately leave the pragma.
1891     PragmaAssumeNonNullLoc = SourceLocation();
1892   }
1893 
1894   if (HeaderInfo.HasIncludeAliasMap()) {
1895     // Map the filename with the brackets still attached.  If the name doesn't
1896     // map to anything, fall back on the filename we've already gotten the
1897     // spelling for.
1898     StringRef NewName = HeaderInfo.MapHeaderToIncludeAlias(OriginalFilename);
1899     if (!NewName.empty())
1900       Filename = NewName;
1901   }
1902 
1903   // Search include directories.
1904   bool IsMapped = false;
1905   bool IsFrameworkFound = false;
1906   const DirectoryLookup *CurDir;
1907   SmallString<1024> SearchPath;
1908   SmallString<1024> RelativePath;
1909   // We get the raw path only if we have 'Callbacks' to which we later pass
1910   // the path.
1911   ModuleMap::KnownHeader SuggestedModule;
1912   SourceLocation FilenameLoc = FilenameTok.getLocation();
1913   StringRef LookupFilename = Filename;
1914 
1915 #ifdef _WIN32
1916   llvm::sys::path::Style BackslashStyle = llvm::sys::path::Style::windows;
1917 #else
1918   // Normalize slashes when compiling with -fms-extensions on non-Windows. This
1919   // is unnecessary on Windows since the filesystem there handles backslashes.
1920   SmallString<128> NormalizedPath;
1921   llvm::sys::path::Style BackslashStyle = llvm::sys::path::Style::posix;
1922   if (LangOpts.MicrosoftExt) {
1923     NormalizedPath = Filename.str();
1924     llvm::sys::path::native(NormalizedPath);
1925     LookupFilename = NormalizedPath;
1926     BackslashStyle = llvm::sys::path::Style::windows;
1927   }
1928 #endif
1929 
1930   Optional<FileEntryRef> File = LookupHeaderIncludeOrImport(
1931       CurDir, Filename, FilenameLoc, FilenameRange, FilenameTok,
1932       IsFrameworkFound, IsImportDecl, IsMapped, LookupFrom, LookupFromFile,
1933       LookupFilename, RelativePath, SearchPath, SuggestedModule, isAngled);
1934 
1935   if (usingPCHWithThroughHeader() && SkippingUntilPCHThroughHeader) {
1936     if (File && isPCHThroughHeader(&File->getFileEntry()))
1937       SkippingUntilPCHThroughHeader = false;
1938     return {ImportAction::None};
1939   }
1940 
1941   // Should we enter the source file? Set to Skip if either the source file is
1942   // known to have no effect beyond its effect on module visibility -- that is,
1943   // if it's got an include guard that is already defined, set to Import if it
1944   // is a modular header we've already built and should import.
1945   enum { Enter, Import, Skip, IncludeLimitReached } Action = Enter;
1946 
1947   if (PPOpts->SingleFileParseMode)
1948     Action = IncludeLimitReached;
1949 
1950   // If we've reached the max allowed include depth, it is usually due to an
1951   // include cycle. Don't enter already processed files again as it can lead to
1952   // reaching the max allowed include depth again.
1953   if (Action == Enter && HasReachedMaxIncludeDepth && File &&
1954       HeaderInfo.getFileInfo(&File->getFileEntry()).NumIncludes)
1955     Action = IncludeLimitReached;
1956 
1957   // Determine whether we should try to import the module for this #include, if
1958   // there is one. Don't do so if precompiled module support is disabled or we
1959   // are processing this module textually (because we're building the module).
1960   if (Action == Enter && File && SuggestedModule && getLangOpts().Modules &&
1961       !isForModuleBuilding(SuggestedModule.getModule(),
1962                            getLangOpts().CurrentModule,
1963                            getLangOpts().ModuleName)) {
1964     // If this include corresponds to a module but that module is
1965     // unavailable, diagnose the situation and bail out.
1966     // FIXME: Remove this; loadModule does the same check (but produces
1967     // slightly worse diagnostics).
1968     if (checkModuleIsAvailable(getLangOpts(), getTargetInfo(), getDiagnostics(),
1969                                SuggestedModule.getModule())) {
1970       Diag(FilenameTok.getLocation(),
1971            diag::note_implicit_top_level_module_import_here)
1972           << SuggestedModule.getModule()->getTopLevelModuleName();
1973       return {ImportAction::None};
1974     }
1975 
1976     // Compute the module access path corresponding to this module.
1977     // FIXME: Should we have a second loadModule() overload to avoid this
1978     // extra lookup step?
1979     SmallVector<std::pair<IdentifierInfo *, SourceLocation>, 2> Path;
1980     for (Module *Mod = SuggestedModule.getModule(); Mod; Mod = Mod->Parent)
1981       Path.push_back(std::make_pair(getIdentifierInfo(Mod->Name),
1982                                     FilenameTok.getLocation()));
1983     std::reverse(Path.begin(), Path.end());
1984 
1985     // Warn that we're replacing the include/import with a module import.
1986     if (!IsImportDecl)
1987       diagnoseAutoModuleImport(*this, StartLoc, IncludeTok, Path, CharEnd);
1988 
1989     // Load the module to import its macros. We'll make the declarations
1990     // visible when the parser gets here.
1991     // FIXME: Pass SuggestedModule in here rather than converting it to a path
1992     // and making the module loader convert it back again.
1993     ModuleLoadResult Imported = TheModuleLoader.loadModule(
1994         IncludeTok.getLocation(), Path, Module::Hidden,
1995         /*IsInclusionDirective=*/true);
1996     assert((Imported == nullptr || Imported == SuggestedModule.getModule()) &&
1997            "the imported module is different than the suggested one");
1998 
1999     if (Imported) {
2000       Action = Import;
2001     } else if (Imported.isMissingExpected()) {
2002       // We failed to find a submodule that we assumed would exist (because it
2003       // was in the directory of an umbrella header, for instance), but no
2004       // actual module containing it exists (because the umbrella header is
2005       // incomplete).  Treat this as a textual inclusion.
2006       SuggestedModule = ModuleMap::KnownHeader();
2007     } else if (Imported.isConfigMismatch()) {
2008       // On a configuration mismatch, enter the header textually. We still know
2009       // that it's part of the corresponding module.
2010     } else {
2011       // We hit an error processing the import. Bail out.
2012       if (hadModuleLoaderFatalFailure()) {
2013         // With a fatal failure in the module loader, we abort parsing.
2014         Token &Result = IncludeTok;
2015         assert(CurLexer && "#include but no current lexer set!");
2016         Result.startToken();
2017         CurLexer->FormTokenWithChars(Result, CurLexer->BufferEnd, tok::eof);
2018         CurLexer->cutOffLexing();
2019       }
2020       return {ImportAction::None};
2021     }
2022   }
2023 
2024   // The #included file will be considered to be a system header if either it is
2025   // in a system include directory, or if the #includer is a system include
2026   // header.
2027   SrcMgr::CharacteristicKind FileCharacter =
2028       SourceMgr.getFileCharacteristic(FilenameTok.getLocation());
2029   if (File)
2030     FileCharacter = std::max(HeaderInfo.getFileDirFlavor(&File->getFileEntry()),
2031                              FileCharacter);
2032 
2033   // If this is a '#import' or an import-declaration, don't re-enter the file.
2034   //
2035   // FIXME: If we have a suggested module for a '#include', and we've already
2036   // visited this file, don't bother entering it again. We know it has no
2037   // further effect.
2038   bool EnterOnce =
2039       IsImportDecl ||
2040       IncludeTok.getIdentifierInfo()->getPPKeywordID() == tok::pp_import;
2041 
2042   // Ask HeaderInfo if we should enter this #include file.  If not, #including
2043   // this file will have no effect.
2044   if (Action == Enter && File &&
2045       !HeaderInfo.ShouldEnterIncludeFile(*this, &File->getFileEntry(),
2046                                          EnterOnce, getLangOpts().Modules,
2047                                          SuggestedModule.getModule())) {
2048     // Even if we've already preprocessed this header once and know that we
2049     // don't need to see its contents again, we still need to import it if it's
2050     // modular because we might not have imported it from this submodule before.
2051     //
2052     // FIXME: We don't do this when compiling a PCH because the AST
2053     // serialization layer can't cope with it. This means we get local
2054     // submodule visibility semantics wrong in that case.
2055     Action = (SuggestedModule && !getLangOpts().CompilingPCH) ? Import : Skip;
2056   }
2057 
2058   // Check for circular inclusion of the main file.
2059   // We can't generate a consistent preamble with regard to the conditional
2060   // stack if the main file is included again as due to the preamble bounds
2061   // some directives (e.g. #endif of a header guard) will never be seen.
2062   // Since this will lead to confusing errors, avoid the inclusion.
2063   if (Action == Enter && File && PreambleConditionalStack.isRecording() &&
2064       SourceMgr.isMainFile(File->getFileEntry())) {
2065     Diag(FilenameTok.getLocation(),
2066          diag::err_pp_including_mainfile_in_preamble);
2067     return {ImportAction::None};
2068   }
2069 
2070   if (Callbacks && !IsImportDecl) {
2071     // Notify the callback object that we've seen an inclusion directive.
2072     // FIXME: Use a different callback for a pp-import?
2073     Callbacks->InclusionDirective(
2074         HashLoc, IncludeTok, LookupFilename, isAngled, FilenameRange,
2075         File ? &File->getFileEntry() : nullptr, SearchPath, RelativePath,
2076         Action == Import ? SuggestedModule.getModule() : nullptr,
2077         FileCharacter);
2078     if (Action == Skip && File)
2079       Callbacks->FileSkipped(*File, FilenameTok, FileCharacter);
2080   }
2081 
2082   if (!File)
2083     return {ImportAction::None};
2084 
2085   // If this is a C++20 pp-import declaration, diagnose if we didn't find any
2086   // module corresponding to the named header.
2087   if (IsImportDecl && !SuggestedModule) {
2088     Diag(FilenameTok, diag::err_header_import_not_header_unit)
2089       << OriginalFilename << File->getName();
2090     return {ImportAction::None};
2091   }
2092 
2093   // Issue a diagnostic if the name of the file on disk has a different case
2094   // than the one we're about to open.
2095   const bool CheckIncludePathPortability =
2096       !IsMapped && !File->getFileEntry().tryGetRealPathName().empty();
2097 
2098   if (CheckIncludePathPortability) {
2099     StringRef Name = LookupFilename;
2100     StringRef NameWithoriginalSlashes = Filename;
2101 #if defined(_WIN32)
2102     // Skip UNC prefix if present. (tryGetRealPathName() always
2103     // returns a path with the prefix skipped.)
2104     bool NameWasUNC = Name.consume_front("\\\\?\\");
2105     NameWithoriginalSlashes.consume_front("\\\\?\\");
2106 #endif
2107     StringRef RealPathName = File->getFileEntry().tryGetRealPathName();
2108     SmallVector<StringRef, 16> Components(llvm::sys::path::begin(Name),
2109                                           llvm::sys::path::end(Name));
2110 #if defined(_WIN32)
2111     // -Wnonportable-include-path is designed to diagnose includes using
2112     // case even on systems with a case-insensitive file system.
2113     // On Windows, RealPathName always starts with an upper-case drive
2114     // letter for absolute paths, but Name might start with either
2115     // case depending on if `cd c:\foo` or `cd C:\foo` was used in the shell.
2116     // ("foo" will always have on-disk case, no matter which case was
2117     // used in the cd command). To not emit this warning solely for
2118     // the drive letter, whose case is dependent on if `cd` is used
2119     // with upper- or lower-case drive letters, always consider the
2120     // given drive letter case as correct for the purpose of this warning.
2121     SmallString<128> FixedDriveRealPath;
2122     if (llvm::sys::path::is_absolute(Name) &&
2123         llvm::sys::path::is_absolute(RealPathName) &&
2124         toLowercase(Name[0]) == toLowercase(RealPathName[0]) &&
2125         isLowercase(Name[0]) != isLowercase(RealPathName[0])) {
2126       assert(Components.size() >= 3 && "should have drive, backslash, name");
2127       assert(Components[0].size() == 2 && "should start with drive");
2128       assert(Components[0][1] == ':' && "should have colon");
2129       FixedDriveRealPath = (Name.substr(0, 1) + RealPathName.substr(1)).str();
2130       RealPathName = FixedDriveRealPath;
2131     }
2132 #endif
2133 
2134     if (trySimplifyPath(Components, RealPathName)) {
2135       SmallString<128> Path;
2136       Path.reserve(Name.size()+2);
2137       Path.push_back(isAngled ? '<' : '"');
2138 
2139       const auto IsSep = [BackslashStyle](char c) {
2140         return llvm::sys::path::is_separator(c, BackslashStyle);
2141       };
2142 
2143       for (auto Component : Components) {
2144         // On POSIX, Components will contain a single '/' as first element
2145         // exactly if Name is an absolute path.
2146         // On Windows, it will contain "C:" followed by '\' for absolute paths.
2147         // The drive letter is optional for absolute paths on Windows, but
2148         // clang currently cannot process absolute paths in #include lines that
2149         // don't have a drive.
2150         // If the first entry in Components is a directory separator,
2151         // then the code at the bottom of this loop that keeps the original
2152         // directory separator style copies it. If the second entry is
2153         // a directory separator (the C:\ case), then that separator already
2154         // got copied when the C: was processed and we want to skip that entry.
2155         if (!(Component.size() == 1 && IsSep(Component[0])))
2156           Path.append(Component);
2157         else if (!Path.empty())
2158           continue;
2159 
2160         // Append the separator(s) the user used, or the close quote
2161         if (Path.size() > NameWithoriginalSlashes.size()) {
2162           Path.push_back(isAngled ? '>' : '"');
2163           continue;
2164         }
2165         assert(IsSep(NameWithoriginalSlashes[Path.size()-1]));
2166         do
2167           Path.push_back(NameWithoriginalSlashes[Path.size()-1]);
2168         while (Path.size() <= NameWithoriginalSlashes.size() &&
2169                IsSep(NameWithoriginalSlashes[Path.size()-1]));
2170       }
2171 
2172 #if defined(_WIN32)
2173       // Restore UNC prefix if it was there.
2174       if (NameWasUNC)
2175         Path = (Path.substr(0, 1) + "\\\\?\\" + Path.substr(1)).str();
2176 #endif
2177 
2178       // For user files and known standard headers, issue a diagnostic.
2179       // For other system headers, don't. They can be controlled separately.
2180       auto DiagId =
2181           (FileCharacter == SrcMgr::C_User || warnByDefaultOnWrongCase(Name))
2182               ? diag::pp_nonportable_path
2183               : diag::pp_nonportable_system_path;
2184       Diag(FilenameTok, DiagId) << Path <<
2185         FixItHint::CreateReplacement(FilenameRange, Path);
2186     }
2187   }
2188 
2189   switch (Action) {
2190   case Skip:
2191     // If we don't need to enter the file, stop now.
2192     if (Module *M = SuggestedModule.getModule())
2193       return {ImportAction::SkippedModuleImport, M};
2194     return {ImportAction::None};
2195 
2196   case IncludeLimitReached:
2197     // If we reached our include limit and don't want to enter any more files,
2198     // don't go any further.
2199     return {ImportAction::None};
2200 
2201   case Import: {
2202     // If this is a module import, make it visible if needed.
2203     Module *M = SuggestedModule.getModule();
2204     assert(M && "no module to import");
2205 
2206     makeModuleVisible(M, EndLoc);
2207 
2208     if (IncludeTok.getIdentifierInfo()->getPPKeywordID() ==
2209         tok::pp___include_macros)
2210       return {ImportAction::None};
2211 
2212     return {ImportAction::ModuleImport, M};
2213   }
2214 
2215   case Enter:
2216     break;
2217   }
2218 
2219   // Check that we don't have infinite #include recursion.
2220   if (IncludeMacroStack.size() == MaxAllowedIncludeStackDepth-1) {
2221     Diag(FilenameTok, diag::err_pp_include_too_deep);
2222     HasReachedMaxIncludeDepth = true;
2223     return {ImportAction::None};
2224   }
2225 
2226   // Look up the file, create a File ID for it.
2227   SourceLocation IncludePos = FilenameTok.getLocation();
2228   // If the filename string was the result of macro expansions, set the include
2229   // position on the file where it will be included and after the expansions.
2230   if (IncludePos.isMacroID())
2231     IncludePos = SourceMgr.getExpansionRange(IncludePos).getEnd();
2232   FileID FID = SourceMgr.createFileID(*File, IncludePos, FileCharacter);
2233   if (!FID.isValid()) {
2234     TheModuleLoader.HadFatalFailure = true;
2235     return ImportAction::Failure;
2236   }
2237 
2238   // If all is good, enter the new file!
2239   if (EnterSourceFile(FID, CurDir, FilenameTok.getLocation()))
2240     return {ImportAction::None};
2241 
2242   // Determine if we're switching to building a new submodule, and which one.
2243   if (auto *M = SuggestedModule.getModule()) {
2244     if (M->getTopLevelModule()->ShadowingModule) {
2245       // We are building a submodule that belongs to a shadowed module. This
2246       // means we find header files in the shadowed module.
2247       Diag(M->DefinitionLoc, diag::err_module_build_shadowed_submodule)
2248         << M->getFullModuleName();
2249       Diag(M->getTopLevelModule()->ShadowingModule->DefinitionLoc,
2250            diag::note_previous_definition);
2251       return {ImportAction::None};
2252     }
2253     // When building a pch, -fmodule-name tells the compiler to textually
2254     // include headers in the specified module. We are not building the
2255     // specified module.
2256     //
2257     // FIXME: This is the wrong way to handle this. We should produce a PCH
2258     // that behaves the same as the header would behave in a compilation using
2259     // that PCH, which means we should enter the submodule. We need to teach
2260     // the AST serialization layer to deal with the resulting AST.
2261     if (getLangOpts().CompilingPCH &&
2262         isForModuleBuilding(M, getLangOpts().CurrentModule,
2263                             getLangOpts().ModuleName))
2264       return {ImportAction::None};
2265 
2266     assert(!CurLexerSubmodule && "should not have marked this as a module yet");
2267     CurLexerSubmodule = M;
2268 
2269     // Let the macro handling code know that any future macros are within
2270     // the new submodule.
2271     EnterSubmodule(M, EndLoc, /*ForPragma*/false);
2272 
2273     // Let the parser know that any future declarations are within the new
2274     // submodule.
2275     // FIXME: There's no point doing this if we're handling a #__include_macros
2276     // directive.
2277     return {ImportAction::ModuleBegin, M};
2278   }
2279 
2280   assert(!IsImportDecl && "failed to diagnose missing module for import decl");
2281   return {ImportAction::None};
2282 }
2283 
2284 /// HandleIncludeNextDirective - Implements \#include_next.
2285 ///
2286 void Preprocessor::HandleIncludeNextDirective(SourceLocation HashLoc,
2287                                               Token &IncludeNextTok) {
2288   Diag(IncludeNextTok, diag::ext_pp_include_next_directive);
2289 
2290   // #include_next is like #include, except that we start searching after
2291   // the current found directory.  If we can't do this, issue a
2292   // diagnostic.
2293   const DirectoryLookup *Lookup = CurDirLookup;
2294   const FileEntry *LookupFromFile = nullptr;
2295   if (isInPrimaryFile() && LangOpts.IsHeaderFile) {
2296     // If the main file is a header, then it's either for PCH/AST generation,
2297     // or libclang opened it. Either way, handle it as a normal include below
2298     // and do not complain about include_next.
2299   } else if (isInPrimaryFile()) {
2300     Lookup = nullptr;
2301     Diag(IncludeNextTok, diag::pp_include_next_in_primary);
2302   } else if (CurLexerSubmodule) {
2303     // Start looking up in the directory *after* the one in which the current
2304     // file would be found, if any.
2305     assert(CurPPLexer && "#include_next directive in macro?");
2306     LookupFromFile = CurPPLexer->getFileEntry();
2307     Lookup = nullptr;
2308   } else if (!Lookup) {
2309     // The current file was not found by walking the include path. Either it
2310     // is the primary file (handled above), or it was found by absolute path,
2311     // or it was found relative to such a file.
2312     // FIXME: Track enough information so we know which case we're in.
2313     Diag(IncludeNextTok, diag::pp_include_next_absolute_path);
2314   } else {
2315     // Start looking up in the next directory.
2316     ++Lookup;
2317   }
2318 
2319   return HandleIncludeDirective(HashLoc, IncludeNextTok, Lookup,
2320                                 LookupFromFile);
2321 }
2322 
2323 /// HandleMicrosoftImportDirective - Implements \#import for Microsoft Mode
2324 void Preprocessor::HandleMicrosoftImportDirective(Token &Tok) {
2325   // The Microsoft #import directive takes a type library and generates header
2326   // files from it, and includes those.  This is beyond the scope of what clang
2327   // does, so we ignore it and error out.  However, #import can optionally have
2328   // trailing attributes that span multiple lines.  We're going to eat those
2329   // so we can continue processing from there.
2330   Diag(Tok, diag::err_pp_import_directive_ms );
2331 
2332   // Read tokens until we get to the end of the directive.  Note that the
2333   // directive can be split over multiple lines using the backslash character.
2334   DiscardUntilEndOfDirective();
2335 }
2336 
2337 /// HandleImportDirective - Implements \#import.
2338 ///
2339 void Preprocessor::HandleImportDirective(SourceLocation HashLoc,
2340                                          Token &ImportTok) {
2341   if (!LangOpts.ObjC) {  // #import is standard for ObjC.
2342     if (LangOpts.MSVCCompat)
2343       return HandleMicrosoftImportDirective(ImportTok);
2344     Diag(ImportTok, diag::ext_pp_import_directive);
2345   }
2346   return HandleIncludeDirective(HashLoc, ImportTok);
2347 }
2348 
2349 /// HandleIncludeMacrosDirective - The -imacros command line option turns into a
2350 /// pseudo directive in the predefines buffer.  This handles it by sucking all
2351 /// tokens through the preprocessor and discarding them (only keeping the side
2352 /// effects on the preprocessor).
2353 void Preprocessor::HandleIncludeMacrosDirective(SourceLocation HashLoc,
2354                                                 Token &IncludeMacrosTok) {
2355   // This directive should only occur in the predefines buffer.  If not, emit an
2356   // error and reject it.
2357   SourceLocation Loc = IncludeMacrosTok.getLocation();
2358   if (SourceMgr.getBufferName(Loc) != "<built-in>") {
2359     Diag(IncludeMacrosTok.getLocation(),
2360          diag::pp_include_macros_out_of_predefines);
2361     DiscardUntilEndOfDirective();
2362     return;
2363   }
2364 
2365   // Treat this as a normal #include for checking purposes.  If this is
2366   // successful, it will push a new lexer onto the include stack.
2367   HandleIncludeDirective(HashLoc, IncludeMacrosTok);
2368 
2369   Token TmpTok;
2370   do {
2371     Lex(TmpTok);
2372     assert(TmpTok.isNot(tok::eof) && "Didn't find end of -imacros!");
2373   } while (TmpTok.isNot(tok::hashhash));
2374 }
2375 
2376 //===----------------------------------------------------------------------===//
2377 // Preprocessor Macro Directive Handling.
2378 //===----------------------------------------------------------------------===//
2379 
2380 /// ReadMacroParameterList - The ( starting a parameter list of a macro
2381 /// definition has just been read.  Lex the rest of the parameters and the
2382 /// closing ), updating MI with what we learn.  Return true if an error occurs
2383 /// parsing the param list.
2384 bool Preprocessor::ReadMacroParameterList(MacroInfo *MI, Token &Tok) {
2385   SmallVector<IdentifierInfo*, 32> Parameters;
2386 
2387   while (true) {
2388     LexUnexpandedToken(Tok);
2389     switch (Tok.getKind()) {
2390     case tok::r_paren:
2391       // Found the end of the parameter list.
2392       if (Parameters.empty())  // #define FOO()
2393         return false;
2394       // Otherwise we have #define FOO(A,)
2395       Diag(Tok, diag::err_pp_expected_ident_in_arg_list);
2396       return true;
2397     case tok::ellipsis:  // #define X(... -> C99 varargs
2398       if (!LangOpts.C99)
2399         Diag(Tok, LangOpts.CPlusPlus11 ?
2400              diag::warn_cxx98_compat_variadic_macro :
2401              diag::ext_variadic_macro);
2402 
2403       // OpenCL v1.2 s6.9.e: variadic macros are not supported.
2404       if (LangOpts.OpenCL && !LangOpts.OpenCLCPlusPlus) {
2405         Diag(Tok, diag::ext_pp_opencl_variadic_macros);
2406       }
2407 
2408       // Lex the token after the identifier.
2409       LexUnexpandedToken(Tok);
2410       if (Tok.isNot(tok::r_paren)) {
2411         Diag(Tok, diag::err_pp_missing_rparen_in_macro_def);
2412         return true;
2413       }
2414       // Add the __VA_ARGS__ identifier as a parameter.
2415       Parameters.push_back(Ident__VA_ARGS__);
2416       MI->setIsC99Varargs();
2417       MI->setParameterList(Parameters, BP);
2418       return false;
2419     case tok::eod:  // #define X(
2420       Diag(Tok, diag::err_pp_missing_rparen_in_macro_def);
2421       return true;
2422     default:
2423       // Handle keywords and identifiers here to accept things like
2424       // #define Foo(for) for.
2425       IdentifierInfo *II = Tok.getIdentifierInfo();
2426       if (!II) {
2427         // #define X(1
2428         Diag(Tok, diag::err_pp_invalid_tok_in_arg_list);
2429         return true;
2430       }
2431 
2432       // If this is already used as a parameter, it is used multiple times (e.g.
2433       // #define X(A,A.
2434       if (llvm::find(Parameters, II) != Parameters.end()) { // C99 6.10.3p6
2435         Diag(Tok, diag::err_pp_duplicate_name_in_arg_list) << II;
2436         return true;
2437       }
2438 
2439       // Add the parameter to the macro info.
2440       Parameters.push_back(II);
2441 
2442       // Lex the token after the identifier.
2443       LexUnexpandedToken(Tok);
2444 
2445       switch (Tok.getKind()) {
2446       default:          // #define X(A B
2447         Diag(Tok, diag::err_pp_expected_comma_in_arg_list);
2448         return true;
2449       case tok::r_paren: // #define X(A)
2450         MI->setParameterList(Parameters, BP);
2451         return false;
2452       case tok::comma:  // #define X(A,
2453         break;
2454       case tok::ellipsis:  // #define X(A... -> GCC extension
2455         // Diagnose extension.
2456         Diag(Tok, diag::ext_named_variadic_macro);
2457 
2458         // Lex the token after the identifier.
2459         LexUnexpandedToken(Tok);
2460         if (Tok.isNot(tok::r_paren)) {
2461           Diag(Tok, diag::err_pp_missing_rparen_in_macro_def);
2462           return true;
2463         }
2464 
2465         MI->setIsGNUVarargs();
2466         MI->setParameterList(Parameters, BP);
2467         return false;
2468       }
2469     }
2470   }
2471 }
2472 
2473 static bool isConfigurationPattern(Token &MacroName, MacroInfo *MI,
2474                                    const LangOptions &LOptions) {
2475   if (MI->getNumTokens() == 1) {
2476     const Token &Value = MI->getReplacementToken(0);
2477 
2478     // Macro that is identity, like '#define inline inline' is a valid pattern.
2479     if (MacroName.getKind() == Value.getKind())
2480       return true;
2481 
2482     // Macro that maps a keyword to the same keyword decorated with leading/
2483     // trailing underscores is a valid pattern:
2484     //    #define inline __inline
2485     //    #define inline __inline__
2486     //    #define inline _inline (in MS compatibility mode)
2487     StringRef MacroText = MacroName.getIdentifierInfo()->getName();
2488     if (IdentifierInfo *II = Value.getIdentifierInfo()) {
2489       if (!II->isKeyword(LOptions))
2490         return false;
2491       StringRef ValueText = II->getName();
2492       StringRef TrimmedValue = ValueText;
2493       if (!ValueText.startswith("__")) {
2494         if (ValueText.startswith("_"))
2495           TrimmedValue = TrimmedValue.drop_front(1);
2496         else
2497           return false;
2498       } else {
2499         TrimmedValue = TrimmedValue.drop_front(2);
2500         if (TrimmedValue.endswith("__"))
2501           TrimmedValue = TrimmedValue.drop_back(2);
2502       }
2503       return TrimmedValue.equals(MacroText);
2504     } else {
2505       return false;
2506     }
2507   }
2508 
2509   // #define inline
2510   return MacroName.isOneOf(tok::kw_extern, tok::kw_inline, tok::kw_static,
2511                            tok::kw_const) &&
2512          MI->getNumTokens() == 0;
2513 }
2514 
2515 // ReadOptionalMacroParameterListAndBody - This consumes all (i.e. the
2516 // entire line) of the macro's tokens and adds them to MacroInfo, and while
2517 // doing so performs certain validity checks including (but not limited to):
2518 //   - # (stringization) is followed by a macro parameter
2519 //
2520 //  Returns a nullptr if an invalid sequence of tokens is encountered or returns
2521 //  a pointer to a MacroInfo object.
2522 
2523 MacroInfo *Preprocessor::ReadOptionalMacroParameterListAndBody(
2524     const Token &MacroNameTok, const bool ImmediatelyAfterHeaderGuard) {
2525 
2526   Token LastTok = MacroNameTok;
2527   // Create the new macro.
2528   MacroInfo *const MI = AllocateMacroInfo(MacroNameTok.getLocation());
2529 
2530   Token Tok;
2531   LexUnexpandedToken(Tok);
2532 
2533   // Ensure we consume the rest of the macro body if errors occur.
2534   auto _ = llvm::make_scope_exit([&]() {
2535     // The flag indicates if we are still waiting for 'eod'.
2536     if (CurLexer->ParsingPreprocessorDirective)
2537       DiscardUntilEndOfDirective();
2538   });
2539 
2540   // Used to un-poison and then re-poison identifiers of the __VA_ARGS__ ilk
2541   // within their appropriate context.
2542   VariadicMacroScopeGuard VariadicMacroScopeGuard(*this);
2543 
2544   // If this is a function-like macro definition, parse the argument list,
2545   // marking each of the identifiers as being used as macro arguments.  Also,
2546   // check other constraints on the first token of the macro body.
2547   if (Tok.is(tok::eod)) {
2548     if (ImmediatelyAfterHeaderGuard) {
2549       // Save this macro information since it may part of a header guard.
2550       CurPPLexer->MIOpt.SetDefinedMacro(MacroNameTok.getIdentifierInfo(),
2551                                         MacroNameTok.getLocation());
2552     }
2553     // If there is no body to this macro, we have no special handling here.
2554   } else if (Tok.hasLeadingSpace()) {
2555     // This is a normal token with leading space.  Clear the leading space
2556     // marker on the first token to get proper expansion.
2557     Tok.clearFlag(Token::LeadingSpace);
2558   } else if (Tok.is(tok::l_paren)) {
2559     // This is a function-like macro definition.  Read the argument list.
2560     MI->setIsFunctionLike();
2561     if (ReadMacroParameterList(MI, LastTok))
2562       return nullptr;
2563 
2564     // If this is a definition of an ISO C/C++ variadic function-like macro (not
2565     // using the GNU named varargs extension) inform our variadic scope guard
2566     // which un-poisons and re-poisons certain identifiers (e.g. __VA_ARGS__)
2567     // allowed only within the definition of a variadic macro.
2568 
2569     if (MI->isC99Varargs()) {
2570       VariadicMacroScopeGuard.enterScope();
2571     }
2572 
2573     // Read the first token after the arg list for down below.
2574     LexUnexpandedToken(Tok);
2575   } else if (LangOpts.C99 || LangOpts.CPlusPlus11) {
2576     // C99 requires whitespace between the macro definition and the body.  Emit
2577     // a diagnostic for something like "#define X+".
2578     Diag(Tok, diag::ext_c99_whitespace_required_after_macro_name);
2579   } else {
2580     // C90 6.8 TC1 says: "In the definition of an object-like macro, if the
2581     // first character of a replacement list is not a character required by
2582     // subclause 5.2.1, then there shall be white-space separation between the
2583     // identifier and the replacement list.".  5.2.1 lists this set:
2584     //   "A-Za-z0-9!"#%&'()*+,_./:;<=>?[\]^_{|}~" as well as whitespace, which
2585     // is irrelevant here.
2586     bool isInvalid = false;
2587     if (Tok.is(tok::at)) // @ is not in the list above.
2588       isInvalid = true;
2589     else if (Tok.is(tok::unknown)) {
2590       // If we have an unknown token, it is something strange like "`".  Since
2591       // all of valid characters would have lexed into a single character
2592       // token of some sort, we know this is not a valid case.
2593       isInvalid = true;
2594     }
2595     if (isInvalid)
2596       Diag(Tok, diag::ext_missing_whitespace_after_macro_name);
2597     else
2598       Diag(Tok, diag::warn_missing_whitespace_after_macro_name);
2599   }
2600 
2601   if (!Tok.is(tok::eod))
2602     LastTok = Tok;
2603 
2604   // Read the rest of the macro body.
2605   if (MI->isObjectLike()) {
2606     // Object-like macros are very simple, just read their body.
2607     while (Tok.isNot(tok::eod)) {
2608       LastTok = Tok;
2609       MI->AddTokenToBody(Tok);
2610       // Get the next token of the macro.
2611       LexUnexpandedToken(Tok);
2612     }
2613   } else {
2614     // Otherwise, read the body of a function-like macro.  While we are at it,
2615     // check C99 6.10.3.2p1: ensure that # operators are followed by macro
2616     // parameters in function-like macro expansions.
2617 
2618     VAOptDefinitionContext VAOCtx(*this);
2619 
2620     while (Tok.isNot(tok::eod)) {
2621       LastTok = Tok;
2622 
2623       if (!Tok.isOneOf(tok::hash, tok::hashat, tok::hashhash)) {
2624         MI->AddTokenToBody(Tok);
2625 
2626         if (VAOCtx.isVAOptToken(Tok)) {
2627           // If we're already within a VAOPT, emit an error.
2628           if (VAOCtx.isInVAOpt()) {
2629             Diag(Tok, diag::err_pp_vaopt_nested_use);
2630             return nullptr;
2631           }
2632           // Ensure VAOPT is followed by a '(' .
2633           LexUnexpandedToken(Tok);
2634           if (Tok.isNot(tok::l_paren)) {
2635             Diag(Tok, diag::err_pp_missing_lparen_in_vaopt_use);
2636             return nullptr;
2637           }
2638           MI->AddTokenToBody(Tok);
2639           VAOCtx.sawVAOptFollowedByOpeningParens(Tok.getLocation());
2640           LexUnexpandedToken(Tok);
2641           if (Tok.is(tok::hashhash)) {
2642             Diag(Tok, diag::err_vaopt_paste_at_start);
2643             return nullptr;
2644           }
2645           continue;
2646         } else if (VAOCtx.isInVAOpt()) {
2647           if (Tok.is(tok::r_paren)) {
2648             if (VAOCtx.sawClosingParen()) {
2649               const unsigned NumTokens = MI->getNumTokens();
2650               assert(NumTokens >= 3 && "Must have seen at least __VA_OPT__( "
2651                                        "and a subsequent tok::r_paren");
2652               if (MI->getReplacementToken(NumTokens - 2).is(tok::hashhash)) {
2653                 Diag(Tok, diag::err_vaopt_paste_at_end);
2654                 return nullptr;
2655               }
2656             }
2657           } else if (Tok.is(tok::l_paren)) {
2658             VAOCtx.sawOpeningParen(Tok.getLocation());
2659           }
2660         }
2661         // Get the next token of the macro.
2662         LexUnexpandedToken(Tok);
2663         continue;
2664       }
2665 
2666       // If we're in -traditional mode, then we should ignore stringification
2667       // and token pasting. Mark the tokens as unknown so as not to confuse
2668       // things.
2669       if (getLangOpts().TraditionalCPP) {
2670         Tok.setKind(tok::unknown);
2671         MI->AddTokenToBody(Tok);
2672 
2673         // Get the next token of the macro.
2674         LexUnexpandedToken(Tok);
2675         continue;
2676       }
2677 
2678       if (Tok.is(tok::hashhash)) {
2679         // If we see token pasting, check if it looks like the gcc comma
2680         // pasting extension.  We'll use this information to suppress
2681         // diagnostics later on.
2682 
2683         // Get the next token of the macro.
2684         LexUnexpandedToken(Tok);
2685 
2686         if (Tok.is(tok::eod)) {
2687           MI->AddTokenToBody(LastTok);
2688           break;
2689         }
2690 
2691         unsigned NumTokens = MI->getNumTokens();
2692         if (NumTokens && Tok.getIdentifierInfo() == Ident__VA_ARGS__ &&
2693             MI->getReplacementToken(NumTokens-1).is(tok::comma))
2694           MI->setHasCommaPasting();
2695 
2696         // Things look ok, add the '##' token to the macro.
2697         MI->AddTokenToBody(LastTok);
2698         continue;
2699       }
2700 
2701       // Our Token is a stringization operator.
2702       // Get the next token of the macro.
2703       LexUnexpandedToken(Tok);
2704 
2705       // Check for a valid macro arg identifier or __VA_OPT__.
2706       if (!VAOCtx.isVAOptToken(Tok) &&
2707           (Tok.getIdentifierInfo() == nullptr ||
2708            MI->getParameterNum(Tok.getIdentifierInfo()) == -1)) {
2709 
2710         // If this is assembler-with-cpp mode, we accept random gibberish after
2711         // the '#' because '#' is often a comment character.  However, change
2712         // the kind of the token to tok::unknown so that the preprocessor isn't
2713         // confused.
2714         if (getLangOpts().AsmPreprocessor && Tok.isNot(tok::eod)) {
2715           LastTok.setKind(tok::unknown);
2716           MI->AddTokenToBody(LastTok);
2717           continue;
2718         } else {
2719           Diag(Tok, diag::err_pp_stringize_not_parameter)
2720             << LastTok.is(tok::hashat);
2721           return nullptr;
2722         }
2723       }
2724 
2725       // Things look ok, add the '#' and param name tokens to the macro.
2726       MI->AddTokenToBody(LastTok);
2727 
2728       // If the token following '#' is VAOPT, let the next iteration handle it
2729       // and check it for correctness, otherwise add the token and prime the
2730       // loop with the next one.
2731       if (!VAOCtx.isVAOptToken(Tok)) {
2732         MI->AddTokenToBody(Tok);
2733         LastTok = Tok;
2734 
2735         // Get the next token of the macro.
2736         LexUnexpandedToken(Tok);
2737       }
2738     }
2739     if (VAOCtx.isInVAOpt()) {
2740       assert(Tok.is(tok::eod) && "Must be at End Of preprocessing Directive");
2741       Diag(Tok, diag::err_pp_expected_after)
2742         << LastTok.getKind() << tok::r_paren;
2743       Diag(VAOCtx.getUnmatchedOpeningParenLoc(), diag::note_matching) << tok::l_paren;
2744       return nullptr;
2745     }
2746   }
2747   MI->setDefinitionEndLoc(LastTok.getLocation());
2748   return MI;
2749 }
2750 /// HandleDefineDirective - Implements \#define.  This consumes the entire macro
2751 /// line then lets the caller lex the next real token.
2752 void Preprocessor::HandleDefineDirective(
2753     Token &DefineTok, const bool ImmediatelyAfterHeaderGuard) {
2754   ++NumDefined;
2755 
2756   Token MacroNameTok;
2757   bool MacroShadowsKeyword;
2758   ReadMacroName(MacroNameTok, MU_Define, &MacroShadowsKeyword);
2759 
2760   // Error reading macro name?  If so, diagnostic already issued.
2761   if (MacroNameTok.is(tok::eod))
2762     return;
2763 
2764   // If we are supposed to keep comments in #defines, reenable comment saving
2765   // mode.
2766   if (CurLexer) CurLexer->SetCommentRetentionState(KeepMacroComments);
2767 
2768   MacroInfo *const MI = ReadOptionalMacroParameterListAndBody(
2769       MacroNameTok, ImmediatelyAfterHeaderGuard);
2770 
2771   if (!MI) return;
2772 
2773   if (MacroShadowsKeyword &&
2774       !isConfigurationPattern(MacroNameTok, MI, getLangOpts())) {
2775     Diag(MacroNameTok, diag::warn_pp_macro_hides_keyword);
2776   }
2777   // Check that there is no paste (##) operator at the beginning or end of the
2778   // replacement list.
2779   unsigned NumTokens = MI->getNumTokens();
2780   if (NumTokens != 0) {
2781     if (MI->getReplacementToken(0).is(tok::hashhash)) {
2782       Diag(MI->getReplacementToken(0), diag::err_paste_at_start);
2783       return;
2784     }
2785     if (MI->getReplacementToken(NumTokens-1).is(tok::hashhash)) {
2786       Diag(MI->getReplacementToken(NumTokens-1), diag::err_paste_at_end);
2787       return;
2788     }
2789   }
2790 
2791   // When skipping just warn about macros that do not match.
2792   if (SkippingUntilPCHThroughHeader) {
2793     const MacroInfo *OtherMI = getMacroInfo(MacroNameTok.getIdentifierInfo());
2794     if (!OtherMI || !MI->isIdenticalTo(*OtherMI, *this,
2795                              /*Syntactic=*/LangOpts.MicrosoftExt))
2796       Diag(MI->getDefinitionLoc(), diag::warn_pp_macro_def_mismatch_with_pch)
2797           << MacroNameTok.getIdentifierInfo();
2798     // Issue the diagnostic but allow the change if msvc extensions are enabled
2799     if (!LangOpts.MicrosoftExt)
2800       return;
2801   }
2802 
2803   // Finally, if this identifier already had a macro defined for it, verify that
2804   // the macro bodies are identical, and issue diagnostics if they are not.
2805   if (const MacroInfo *OtherMI=getMacroInfo(MacroNameTok.getIdentifierInfo())) {
2806     // In Objective-C, ignore attempts to directly redefine the builtin
2807     // definitions of the ownership qualifiers.  It's still possible to
2808     // #undef them.
2809     auto isObjCProtectedMacro = [](const IdentifierInfo *II) -> bool {
2810       return II->isStr("__strong") ||
2811              II->isStr("__weak") ||
2812              II->isStr("__unsafe_unretained") ||
2813              II->isStr("__autoreleasing");
2814     };
2815    if (getLangOpts().ObjC &&
2816         SourceMgr.getFileID(OtherMI->getDefinitionLoc())
2817           == getPredefinesFileID() &&
2818         isObjCProtectedMacro(MacroNameTok.getIdentifierInfo())) {
2819       // Warn if it changes the tokens.
2820       if ((!getDiagnostics().getSuppressSystemWarnings() ||
2821            !SourceMgr.isInSystemHeader(DefineTok.getLocation())) &&
2822           !MI->isIdenticalTo(*OtherMI, *this,
2823                              /*Syntactic=*/LangOpts.MicrosoftExt)) {
2824         Diag(MI->getDefinitionLoc(), diag::warn_pp_objc_macro_redef_ignored);
2825       }
2826       assert(!OtherMI->isWarnIfUnused());
2827       return;
2828     }
2829 
2830     // It is very common for system headers to have tons of macro redefinitions
2831     // and for warnings to be disabled in system headers.  If this is the case,
2832     // then don't bother calling MacroInfo::isIdenticalTo.
2833     if (!getDiagnostics().getSuppressSystemWarnings() ||
2834         !SourceMgr.isInSystemHeader(DefineTok.getLocation())) {
2835       if (!OtherMI->isUsed() && OtherMI->isWarnIfUnused())
2836         Diag(OtherMI->getDefinitionLoc(), diag::pp_macro_not_used);
2837 
2838       // Warn if defining "__LINE__" and other builtins, per C99 6.10.8/4 and
2839       // C++ [cpp.predefined]p4, but allow it as an extension.
2840       if (OtherMI->isBuiltinMacro())
2841         Diag(MacroNameTok, diag::ext_pp_redef_builtin_macro);
2842       // Macros must be identical.  This means all tokens and whitespace
2843       // separation must be the same.  C99 6.10.3p2.
2844       else if (!OtherMI->isAllowRedefinitionsWithoutWarning() &&
2845                !MI->isIdenticalTo(*OtherMI, *this, /*Syntactic=*/LangOpts.MicrosoftExt)) {
2846         Diag(MI->getDefinitionLoc(), diag::ext_pp_macro_redef)
2847           << MacroNameTok.getIdentifierInfo();
2848         Diag(OtherMI->getDefinitionLoc(), diag::note_previous_definition);
2849       }
2850     }
2851     if (OtherMI->isWarnIfUnused())
2852       WarnUnusedMacroLocs.erase(OtherMI->getDefinitionLoc());
2853   }
2854 
2855   DefMacroDirective *MD =
2856       appendDefMacroDirective(MacroNameTok.getIdentifierInfo(), MI);
2857 
2858   assert(!MI->isUsed());
2859   // If we need warning for not using the macro, add its location in the
2860   // warn-because-unused-macro set. If it gets used it will be removed from set.
2861   if (getSourceManager().isInMainFile(MI->getDefinitionLoc()) &&
2862       !Diags->isIgnored(diag::pp_macro_not_used, MI->getDefinitionLoc()) &&
2863       !MacroExpansionInDirectivesOverride &&
2864       getSourceManager().getFileID(MI->getDefinitionLoc()) !=
2865           getPredefinesFileID()) {
2866     MI->setIsWarnIfUnused(true);
2867     WarnUnusedMacroLocs.insert(MI->getDefinitionLoc());
2868   }
2869 
2870   // If the callbacks want to know, tell them about the macro definition.
2871   if (Callbacks)
2872     Callbacks->MacroDefined(MacroNameTok, MD);
2873 }
2874 
2875 /// HandleUndefDirective - Implements \#undef.
2876 ///
2877 void Preprocessor::HandleUndefDirective() {
2878   ++NumUndefined;
2879 
2880   Token MacroNameTok;
2881   ReadMacroName(MacroNameTok, MU_Undef);
2882 
2883   // Error reading macro name?  If so, diagnostic already issued.
2884   if (MacroNameTok.is(tok::eod))
2885     return;
2886 
2887   // Check to see if this is the last token on the #undef line.
2888   CheckEndOfDirective("undef");
2889 
2890   // Okay, we have a valid identifier to undef.
2891   auto *II = MacroNameTok.getIdentifierInfo();
2892   auto MD = getMacroDefinition(II);
2893   UndefMacroDirective *Undef = nullptr;
2894 
2895   // If the macro is not defined, this is a noop undef.
2896   if (const MacroInfo *MI = MD.getMacroInfo()) {
2897     if (!MI->isUsed() && MI->isWarnIfUnused())
2898       Diag(MI->getDefinitionLoc(), diag::pp_macro_not_used);
2899 
2900     if (MI->isWarnIfUnused())
2901       WarnUnusedMacroLocs.erase(MI->getDefinitionLoc());
2902 
2903     Undef = AllocateUndefMacroDirective(MacroNameTok.getLocation());
2904   }
2905 
2906   // If the callbacks want to know, tell them about the macro #undef.
2907   // Note: no matter if the macro was defined or not.
2908   if (Callbacks)
2909     Callbacks->MacroUndefined(MacroNameTok, MD, Undef);
2910 
2911   if (Undef)
2912     appendMacroDirective(II, Undef);
2913 }
2914 
2915 //===----------------------------------------------------------------------===//
2916 // Preprocessor Conditional Directive Handling.
2917 //===----------------------------------------------------------------------===//
2918 
2919 /// HandleIfdefDirective - Implements the \#ifdef/\#ifndef directive.  isIfndef
2920 /// is true when this is a \#ifndef directive.  ReadAnyTokensBeforeDirective is
2921 /// true if any tokens have been returned or pp-directives activated before this
2922 /// \#ifndef has been lexed.
2923 ///
2924 void Preprocessor::HandleIfdefDirective(Token &Result,
2925                                         const Token &HashToken,
2926                                         bool isIfndef,
2927                                         bool ReadAnyTokensBeforeDirective) {
2928   ++NumIf;
2929   Token DirectiveTok = Result;
2930 
2931   Token MacroNameTok;
2932   ReadMacroName(MacroNameTok);
2933 
2934   // Error reading macro name?  If so, diagnostic already issued.
2935   if (MacroNameTok.is(tok::eod)) {
2936     // Skip code until we get to #endif.  This helps with recovery by not
2937     // emitting an error when the #endif is reached.
2938     SkipExcludedConditionalBlock(HashToken.getLocation(),
2939                                  DirectiveTok.getLocation(),
2940                                  /*Foundnonskip*/ false, /*FoundElse*/ false);
2941     return;
2942   }
2943 
2944   // Check to see if this is the last token on the #if[n]def line.
2945   CheckEndOfDirective(isIfndef ? "ifndef" : "ifdef");
2946 
2947   IdentifierInfo *MII = MacroNameTok.getIdentifierInfo();
2948   auto MD = getMacroDefinition(MII);
2949   MacroInfo *MI = MD.getMacroInfo();
2950 
2951   if (CurPPLexer->getConditionalStackDepth() == 0) {
2952     // If the start of a top-level #ifdef and if the macro is not defined,
2953     // inform MIOpt that this might be the start of a proper include guard.
2954     // Otherwise it is some other form of unknown conditional which we can't
2955     // handle.
2956     if (!ReadAnyTokensBeforeDirective && !MI) {
2957       assert(isIfndef && "#ifdef shouldn't reach here");
2958       CurPPLexer->MIOpt.EnterTopLevelIfndef(MII, MacroNameTok.getLocation());
2959     } else
2960       CurPPLexer->MIOpt.EnterTopLevelConditional();
2961   }
2962 
2963   // If there is a macro, process it.
2964   if (MI)  // Mark it used.
2965     markMacroAsUsed(MI);
2966 
2967   if (Callbacks) {
2968     if (isIfndef)
2969       Callbacks->Ifndef(DirectiveTok.getLocation(), MacroNameTok, MD);
2970     else
2971       Callbacks->Ifdef(DirectiveTok.getLocation(), MacroNameTok, MD);
2972   }
2973 
2974   bool RetainExcludedCB = PPOpts->RetainExcludedConditionalBlocks &&
2975     getSourceManager().isInMainFile(DirectiveTok.getLocation());
2976 
2977   // Should we include the stuff contained by this directive?
2978   if (PPOpts->SingleFileParseMode && !MI) {
2979     // In 'single-file-parse mode' undefined identifiers trigger parsing of all
2980     // the directive blocks.
2981     CurPPLexer->pushConditionalLevel(DirectiveTok.getLocation(),
2982                                      /*wasskip*/false, /*foundnonskip*/false,
2983                                      /*foundelse*/false);
2984   } else if (!MI == isIfndef || RetainExcludedCB) {
2985     // Yes, remember that we are inside a conditional, then lex the next token.
2986     CurPPLexer->pushConditionalLevel(DirectiveTok.getLocation(),
2987                                      /*wasskip*/false, /*foundnonskip*/true,
2988                                      /*foundelse*/false);
2989   } else {
2990     // No, skip the contents of this block.
2991     SkipExcludedConditionalBlock(HashToken.getLocation(),
2992                                  DirectiveTok.getLocation(),
2993                                  /*Foundnonskip*/ false,
2994                                  /*FoundElse*/ false);
2995   }
2996 }
2997 
2998 /// HandleIfDirective - Implements the \#if directive.
2999 ///
3000 void Preprocessor::HandleIfDirective(Token &IfToken,
3001                                      const Token &HashToken,
3002                                      bool ReadAnyTokensBeforeDirective) {
3003   ++NumIf;
3004 
3005   // Parse and evaluate the conditional expression.
3006   IdentifierInfo *IfNDefMacro = nullptr;
3007   const DirectiveEvalResult DER = EvaluateDirectiveExpression(IfNDefMacro);
3008   const bool ConditionalTrue = DER.Conditional;
3009 
3010   // If this condition is equivalent to #ifndef X, and if this is the first
3011   // directive seen, handle it for the multiple-include optimization.
3012   if (CurPPLexer->getConditionalStackDepth() == 0) {
3013     if (!ReadAnyTokensBeforeDirective && IfNDefMacro && ConditionalTrue)
3014       // FIXME: Pass in the location of the macro name, not the 'if' token.
3015       CurPPLexer->MIOpt.EnterTopLevelIfndef(IfNDefMacro, IfToken.getLocation());
3016     else
3017       CurPPLexer->MIOpt.EnterTopLevelConditional();
3018   }
3019 
3020   if (Callbacks)
3021     Callbacks->If(
3022         IfToken.getLocation(), DER.ExprRange,
3023         (ConditionalTrue ? PPCallbacks::CVK_True : PPCallbacks::CVK_False));
3024 
3025   bool RetainExcludedCB = PPOpts->RetainExcludedConditionalBlocks &&
3026     getSourceManager().isInMainFile(IfToken.getLocation());
3027 
3028   // Should we include the stuff contained by this directive?
3029   if (PPOpts->SingleFileParseMode && DER.IncludedUndefinedIds) {
3030     // In 'single-file-parse mode' undefined identifiers trigger parsing of all
3031     // the directive blocks.
3032     CurPPLexer->pushConditionalLevel(IfToken.getLocation(), /*wasskip*/false,
3033                                      /*foundnonskip*/false, /*foundelse*/false);
3034   } else if (ConditionalTrue || RetainExcludedCB) {
3035     // Yes, remember that we are inside a conditional, then lex the next token.
3036     CurPPLexer->pushConditionalLevel(IfToken.getLocation(), /*wasskip*/false,
3037                                    /*foundnonskip*/true, /*foundelse*/false);
3038   } else {
3039     // No, skip the contents of this block.
3040     SkipExcludedConditionalBlock(HashToken.getLocation(), IfToken.getLocation(),
3041                                  /*Foundnonskip*/ false,
3042                                  /*FoundElse*/ false);
3043   }
3044 }
3045 
3046 /// HandleEndifDirective - Implements the \#endif directive.
3047 ///
3048 void Preprocessor::HandleEndifDirective(Token &EndifToken) {
3049   ++NumEndif;
3050 
3051   // Check that this is the whole directive.
3052   CheckEndOfDirective("endif");
3053 
3054   PPConditionalInfo CondInfo;
3055   if (CurPPLexer->popConditionalLevel(CondInfo)) {
3056     // No conditionals on the stack: this is an #endif without an #if.
3057     Diag(EndifToken, diag::err_pp_endif_without_if);
3058     return;
3059   }
3060 
3061   // If this the end of a top-level #endif, inform MIOpt.
3062   if (CurPPLexer->getConditionalStackDepth() == 0)
3063     CurPPLexer->MIOpt.ExitTopLevelConditional();
3064 
3065   assert(!CondInfo.WasSkipping && !CurPPLexer->LexingRawMode &&
3066          "This code should only be reachable in the non-skipping case!");
3067 
3068   if (Callbacks)
3069     Callbacks->Endif(EndifToken.getLocation(), CondInfo.IfLoc);
3070 }
3071 
3072 /// HandleElseDirective - Implements the \#else directive.
3073 ///
3074 void Preprocessor::HandleElseDirective(Token &Result, const Token &HashToken) {
3075   ++NumElse;
3076 
3077   // #else directive in a non-skipping conditional... start skipping.
3078   CheckEndOfDirective("else");
3079 
3080   PPConditionalInfo CI;
3081   if (CurPPLexer->popConditionalLevel(CI)) {
3082     Diag(Result, diag::pp_err_else_without_if);
3083     return;
3084   }
3085 
3086   // If this is a top-level #else, inform the MIOpt.
3087   if (CurPPLexer->getConditionalStackDepth() == 0)
3088     CurPPLexer->MIOpt.EnterTopLevelConditional();
3089 
3090   // If this is a #else with a #else before it, report the error.
3091   if (CI.FoundElse) Diag(Result, diag::pp_err_else_after_else);
3092 
3093   if (Callbacks)
3094     Callbacks->Else(Result.getLocation(), CI.IfLoc);
3095 
3096   bool RetainExcludedCB = PPOpts->RetainExcludedConditionalBlocks &&
3097     getSourceManager().isInMainFile(Result.getLocation());
3098 
3099   if ((PPOpts->SingleFileParseMode && !CI.FoundNonSkip) || RetainExcludedCB) {
3100     // In 'single-file-parse mode' undefined identifiers trigger parsing of all
3101     // the directive blocks.
3102     CurPPLexer->pushConditionalLevel(CI.IfLoc, /*wasskip*/false,
3103                                      /*foundnonskip*/false, /*foundelse*/true);
3104     return;
3105   }
3106 
3107   // Finally, skip the rest of the contents of this block.
3108   SkipExcludedConditionalBlock(HashToken.getLocation(), CI.IfLoc,
3109                                /*Foundnonskip*/ true,
3110                                /*FoundElse*/ true, Result.getLocation());
3111 }
3112 
3113 /// HandleElifDirective - Implements the \#elif directive.
3114 ///
3115 void Preprocessor::HandleElifDirective(Token &ElifToken,
3116                                        const Token &HashToken) {
3117   ++NumElse;
3118 
3119   // #elif directive in a non-skipping conditional... start skipping.
3120   // We don't care what the condition is, because we will always skip it (since
3121   // the block immediately before it was included).
3122   SourceRange ConditionRange = DiscardUntilEndOfDirective();
3123 
3124   PPConditionalInfo CI;
3125   if (CurPPLexer->popConditionalLevel(CI)) {
3126     Diag(ElifToken, diag::pp_err_elif_without_if);
3127     return;
3128   }
3129 
3130   // If this is a top-level #elif, inform the MIOpt.
3131   if (CurPPLexer->getConditionalStackDepth() == 0)
3132     CurPPLexer->MIOpt.EnterTopLevelConditional();
3133 
3134   // If this is a #elif with a #else before it, report the error.
3135   if (CI.FoundElse) Diag(ElifToken, diag::pp_err_elif_after_else);
3136 
3137   if (Callbacks)
3138     Callbacks->Elif(ElifToken.getLocation(), ConditionRange,
3139                     PPCallbacks::CVK_NotEvaluated, CI.IfLoc);
3140 
3141   bool RetainExcludedCB = PPOpts->RetainExcludedConditionalBlocks &&
3142     getSourceManager().isInMainFile(ElifToken.getLocation());
3143 
3144   if ((PPOpts->SingleFileParseMode && !CI.FoundNonSkip) || RetainExcludedCB) {
3145     // In 'single-file-parse mode' undefined identifiers trigger parsing of all
3146     // the directive blocks.
3147     CurPPLexer->pushConditionalLevel(ElifToken.getLocation(), /*wasskip*/false,
3148                                      /*foundnonskip*/false, /*foundelse*/false);
3149     return;
3150   }
3151 
3152   // Finally, skip the rest of the contents of this block.
3153   SkipExcludedConditionalBlock(
3154       HashToken.getLocation(), CI.IfLoc, /*Foundnonskip*/ true,
3155       /*FoundElse*/ CI.FoundElse, ElifToken.getLocation());
3156 }
3157