Lines Matching full:token
1 //===- TokenLexer.cpp - Lex from a token stream ---------------------------===//
25 #include "clang/Lex/Token.h"
40 void TokenLexer::Init(Token &Tok, SourceLocation ELEnd, MacroInfo *MI, in Init()
74 // creating separate source location entries for each token. in Init()
94 /// Create a TokenLexer for the specified token stream. This does not
95 /// take ownership of the specified token vector.
96 void TokenLexer::Init(const Token *TokArray, unsigned NumToks, in Init()
118 // Set HasLeadingSpace/AtStartOfLine so that the first token will be in Init()
140 SmallVectorImpl<Token> &ResultToks, bool HasPasteOperator, MacroInfo *Macro, in MaybeRemoveCommaBeforeVaArgs()
174 // then removal of the comma should produce a placemarker token (in C99 in MaybeRemoveCommaBeforeVaArgs()
181 ResultToks.back().setFlag(Token::CommaAfterElided); in MaybeRemoveCommaBeforeVaArgs()
190 SmallVectorImpl<Token> &ResultToks, const VAOptExpansionContext &VCtx, in stringifyVAOPTContents()
194 Token *const VAOPTTokens = in stringifyVAOPTContents()
197 SmallVector<Token, 64> ConcatenatedVAOPTResultToks; in stringifyVAOPTContents()
201 // Perform token pasting (concatenation) prior to stringization. in stringifyVAOPTContents()
207 Token &LHS = VAOPTTokens[CurTokenIdx - 1]; in stringifyVAOPTContents()
210 // Replace the token prior to the first ## in this iteration. in stringifyVAOPTContents()
222 // macro definition, and use it to indicate that the stringified token in stringifyVAOPTContents()
229 Token StringifiedVAOPT = MacroArgs::StringifyArgument( in stringifyVAOPTContents()
234 StringifiedVAOPT.setFlag(Token::LeadingSpace); in stringifyVAOPTContents()
236 StringifiedVAOPT.setFlag(Token::StringifiedInMacro); in stringifyVAOPTContents()
237 // Resize (shrink) the token stream to just capture this stringified token. in stringifyVAOPTContents()
245 SmallVector<Token, 128> ResultToks; in ExpandFunctionArguments()
257 const Token &CurTok = Tokens[I]; in ExpandFunctionArguments()
258 // We don't want a space for the next token after a paste in ExpandFunctionArguments()
259 // operator. In valid code, the token will get smooshed onto the in ExpandFunctionArguments()
282 // If we are about to process a token that is either an argument to in ExpandFunctionArguments()
284 // 1) If the token is the closing rparen that exits us out of __VA_OPT__, in ExpandFunctionArguments()
286 // and/or skip to the next token. in ExpandFunctionArguments()
288 // token. in ExpandFunctionArguments()
289 // 3) else (macro was invoked with variadic arguments) process the token in ExpandFunctionArguments()
296 // this token. Note sawClosingParen() returns true only if the r_paren matches in ExpandFunctionArguments()
305 // Skip this token. in ExpandFunctionArguments()
309 // have a closing rparen - so process this token normally. in ExpandFunctionArguments()
311 // Current token is the closing r_paren which marks the end of the in ExpandFunctionArguments()
315 // but do so only after any token concatenation that needs to occur in ExpandFunctionArguments()
320 // stringified token. This requires token-pasting to eagerly occur in ExpandFunctionArguments()
323 // is a token that represents an empty string. in ExpandFunctionArguments()
329 // Treat VAOPT as a placemarker token. Eat either the '##' before the in ExpandFunctionArguments()
349 "no token paste before __VA_OPT__"); in ExpandFunctionArguments()
354 // following '##' token. in ExpandFunctionArguments()
362 // __VA_OPT__), so skip to the next token. in ExpandFunctionArguments()
368 // preprocessor already verified that the following token is a macro in ExpandFunctionArguments()
374 "Token following # is not an argument or __VA_OPT__!"); in ExpandFunctionArguments()
389 const Token *UnexpArg = ActualArgs->getUnexpArgument(ArgNo); in ExpandFunctionArguments()
390 Token Res = MacroArgs::StringifyArgument( in ExpandFunctionArguments()
392 Res.setFlag(Token::StringifiedInMacro); in ExpandFunctionArguments()
397 Res.setFlag(Token::LeadingSpace); in ExpandFunctionArguments()
406 // Find out if there is a paste (##) operator before or after the token. in ExpandFunctionArguments()
416 // Otherwise, if this is not an argument token, just add the token to the in ExpandFunctionArguments()
425 ResultToks.back().setFlag(Token::LeadingSpace); in ExpandFunctionArguments()
428 ResultToks.back().clearFlag(Token::LeadingSpace); in ExpandFunctionArguments()
451 const Token *ResultArgToks; in ExpandFunctionArguments()
455 const Token *ArgTok = ActualArgs->getUnexpArgument(ArgNo); in ExpandFunctionArguments()
461 // If the arg token expanded into anything, append it. in ExpandFunctionArguments()
469 // expansions as argument separators. Set a flag on the token so we can in ExpandFunctionArguments()
473 ResultToks.back().setFlag(Token::IgnoredComma); in ExpandFunctionArguments()
477 for (Token &Tok : llvm::drop_begin(ResultToks, FirstResult)) in ExpandFunctionArguments()
488 // before the first token should match the whitespace of the arg in ExpandFunctionArguments()
490 ResultToks[FirstResult].setFlagValue(Token::LeadingSpace, in ExpandFunctionArguments()
492 ResultToks[FirstResult].setFlagValue(Token::StartOfLine, false); in ExpandFunctionArguments()
495 // We're creating a placeholder token. Usually this doesn't matter, in ExpandFunctionArguments()
499 // We're imagining a placeholder token is inserted here. If this is in ExpandFunctionArguments()
500 // the first token in a __VA_OPT__ after a ##, delete the ##. in ExpandFunctionArguments()
509 // Okay, we have a token that is either the LHS or RHS of a paste (##) in ExpandFunctionArguments()
511 const Token *ArgToks = ActualArgs->getUnexpArgument(ArgNo); in ExpandFunctionArguments()
517 // the expander tries to paste ',' with the first token of the __VA_ARGS__ in ExpandFunctionArguments()
532 for (Token &Tok : llvm::make_range(ResultToks.end() - NumToks, in ExpandFunctionArguments()
543 // Transfer the leading whitespace information from the token in ExpandFunctionArguments()
544 // (the macro argument) onto the first token of the in ExpandFunctionArguments()
548 ResultToks[ResultToks.size() - NumToks].setFlagValue(Token::StartOfLine, in ExpandFunctionArguments()
551 Token::LeadingSpace, NextTokGetsSpace); in ExpandFunctionArguments()
563 // Discard the argument token and skip (don't copy to the expansion in ExpandFunctionArguments()
589 // If this is the __VA_ARGS__ token, and if the argument wasn't provided, in ExpandFunctionArguments()
590 // and if the macro had at least one real argument, and if the token before in ExpandFunctionArguments()
601 assert(!OwnsTokens && "This would leak if we already own the token list"); in ExpandFunctionArguments()
614 static bool isWideStringLiteralFromMacro(const Token &FirstTok, in isWideStringLiteralFromMacro()
615 const Token &SecondTok) { in isWideStringLiteralFromMacro()
621 /// Lex - Lex and return a token from this macro stream.
622 bool TokenLexer::Lex(Token &Tok) { in Lex()
625 // If this is a macro (not a token stream), mark the macro enabled now in Lex()
630 Tok.setFlagValue(Token::StartOfLine , AtStartOfLine); in Lex()
631 Tok.setFlagValue(Token::LeadingSpace, HasLeadingSpace || NextTokGetsSpace); in Lex()
633 Tok.setFlag(Token::LeadingEmptyMacro); in Lex()
639 // If this is the first token of the expanded result, we inherit spacing in Lex()
643 // Get the next token to return. in Lex()
646 Tok.setFlag(Token::IsReinjected); in Lex()
650 // If this token is followed by a token paste (##) operator, paste the tokens! in Lex()
651 // Note that ## is a normal token when not expanding a macro. in Lex()
659 // When handling the microsoft /##/ extension, the final token is in Lex()
660 // returned by pasteTokens, not the pasted token. in Lex()
667 // The token's current location indicate where the token was lexed from. We in Lex()
668 // need this information to compute the spelling of the token, but any in Lex()
669 // diagnostics for the expanded token should appear as if they came from in Lex()
672 if (ExpandLocStart.isValid() && // Don't do this for token streams. in Lex()
673 // Check that the token's location was not already set properly. in Lex()
688 // If this is the first token, set the lexical properties of the token to in Lex()
691 Tok.setFlagValue(Token::StartOfLine , AtStartOfLine); in Lex()
692 Tok.setFlagValue(Token::LeadingSpace, HasLeadingSpace); in Lex()
694 // If this is not the first token, we may still need to pass through in Lex()
696 if (AtStartOfLine) Tok.setFlag(Token::StartOfLine); in Lex()
697 if (HasLeadingSpace) Tok.setFlag(Token::LeadingSpace); in Lex()
704 // Change the kind of this identifier to the appropriate token kind, e.g. in Lex()
720 // Otherwise, return a normal token. in Lex()
724 bool TokenLexer::pasteTokens(Token &Tok) { in pasteTokens()
731 /// If this returns true, the caller should immediately return the token.
732 bool TokenLexer::pasteTokens(Token &LHSTok, ArrayRef<Token> TokenStream, in pasteTokens()
734 assert(CurIdx > 0 && "## can not be the first token within tokens"); in pasteTokens()
738 "Token at this Index must be ## or part of the MSVC 'L " in pasteTokens()
741 // MSVC: If previous token was pasted, this must be a recovery from an invalid in pasteTokens()
742 // paste operation. Ignore spaces before this token to mimic MSVC output. in pasteTokens()
746 LHSTok.clearFlag(Token::LeadingSpace); in pasteTokens()
762 assert(!IsAtEnd() && "No token on the RHS of a paste operator!"); in pasteTokens()
764 // Get the RHS token. in pasteTokens()
765 const Token &RHS = TokenStream[CurIdx]; in pasteTokens()
767 // Allocate space for the result token. This is guaranteed to be enough for in pasteTokens()
771 // Get the spelling of the LHS token in Buffer. in pasteTokens()
793 Token ResultTokTmp; in pasteTokens()
796 // Claim that the tmp token is a string_literal so that we can get the in pasteTokens()
803 // Lex the resultant pasted token into Result. in pasteTokens()
804 Token Result; in pasteTokens()
829 // Make a lexer to lex this string from. Lex just this one token. in pasteTokens()
835 // Lex a token in raw mode. This way it won't look up identifiers in pasteTokens()
836 // automatically, lexing off the end will return an eof token, and in pasteTokens()
837 // warnings are disabled. This returns true if the result token is the in pasteTokens()
841 // If we got an EOF token, we didn't form even ONE token. For example, we in pasteTokens()
845 // If pasting the two tokens didn't form a full new token, this is an in pasteTokens()
847 // unmodified and with RHS as the next token to lex. in pasteTokens()
849 // Explicitly convert the token location to have proper expansion in pasteTokens()
884 Result.setFlagValue(Token::StartOfLine , LHSTok.isAtStartOfLine()); in pasteTokens()
885 Result.setFlagValue(Token::LeadingSpace, LHSTok.hasLeadingSpace()); in pasteTokens()
894 // The token's current location indicate where the token was lexed from. We in pasteTokens()
895 // need this information to compute the spelling of the token, but any in pasteTokens()
896 // diagnostics for the expanded token should appear as if the token was in pasteTokens()
913 // Now that we got the result token, it will be subject to expansion. Since in pasteTokens()
914 // token pasting re-lexes the result token in raw mode, identifier information in pasteTokens()
917 // Look up the identifier info for the token. We disabled identifier lookup in pasteTokens()
924 /// isNextTokenLParen - If the next token lexed will pop this macro off the
925 /// expansion stack, return 2. If the next unexpanded token is a '(', return
944 /// first token on the next line.
945 void TokenLexer::HandleMicrosoftCommentPaste(Token &Tok, SourceLocation OpLoc) { in HandleMicrosoftCommentPaste()
953 assert(Macro && "Token streams can't paste comments"); in HandleMicrosoftCommentPaste()
966 "Not appropriate for token streams"); in getExpansionLocForMacroDefLoc()
979 /// creates a single SLocEntry, and assigns SourceLocations to each token that
989 Token *&begin_tokens, in updateConsecutiveMacroArgTokens()
990 Token * end_tokens) { in updateConsecutiveMacroArgTokens()
993 llvm::MutableArrayRef<Token> All(begin_tokens, end_tokens); in updateConsecutiveMacroArgTokens()
994 llvm::MutableArrayRef<Token> Partition; in updateConsecutiveMacroArgTokens()
1012 Partition = All.take_while([&](const Token &T) { in updateConsecutiveMacroArgTokens()
1021 Partition = All.take_while([&](const Token &T) { in updateConsecutiveMacroArgTokens()
1023 // single token past the end of the FileID, specifically the ) when a in updateConsecutiveMacroArgTokens()
1048 const Token &T) { in updateConsecutiveMacroArgTokens()
1055 for (Token& T : Partition) { in updateConsecutiveMacroArgTokens()
1069 Token *begin_tokens, in updateLocForMacroArgTokens()
1070 Token *end_tokens) { in updateLocForMacroArgTokens()
1077 // If there's only one token just create a SLocEntry for it. in updateLocForMacroArgTokens()
1079 Token &Tok = *begin_tokens; in updateLocForMacroArgTokens()
1090 void TokenLexer::PropagateLineStartLeadingSpaceInfo(Token &Result) { in PropagateLineStartLeadingSpaceInfo()