Lines Matching full:tokens
80 assert(Tokens.empty()); in lex()
83 Tokens.push_back(getNextToken()); in lex()
92 // This needs to come after tokens have been merged so that C# in lex()
100 if (Tokens.back()->NewlinesBefore > 0 || Tokens.back()->IsMultiline) in lex()
101 FirstInLineIndex = Tokens.size() - 1; in lex()
102 } while (Tokens.back()->isNot(tok::eof)); in lex()
104 auto &TokEOF = *Tokens.back(); in lex()
110 return Tokens; in lex()
138 Tokens.back()->Tok.setKind(tok::pipepipe); in tryMergePreviousTokens()
143 Tokens.back()->Tok.setKind(tok::period); in tryMergePreviousTokens()
163 Tokens.back()->Tok.setKind(tok::l_square); in tryMergePreviousTokens()
193 Tokens.back()->Tok.setKind(tok::starequal); in tryMergePreviousTokens()
199 Tokens.back()->Tok.setKind(tok::equal); in tryMergePreviousTokens()
215 if (Tokens.size() >= 3 && Tokens.end()[-3]->is(TT_VerilogNumberBase) && in tryMergePreviousTokens()
216 Tokens.end()[-2]->is(tok::numeric_constant) && in tryMergePreviousTokens()
217 Tokens.back()->isOneOf(tok::numeric_constant, tok::identifier, in tryMergePreviousTokens()
231 if (Tokens.back()->TokenText.size() == 1 && in tryMergePreviousTokens()
234 Tokens.back()->Tok.setKind(tok::caret); in tryMergePreviousTokens()
239 Tokens.back()->Tok.setKind(tok::lessless); in tryMergePreviousTokens()
243 Tokens.back()->Tok.setKind(tok::greatergreater); in tryMergePreviousTokens()
253 Tokens.back()->ForcedPrecedence = prec::Assignment; in tryMergePreviousTokens()
281 Tokens.back()->is(tok::arrow)) { in tryMergePreviousTokens()
282 Tokens.back()->ForcedPrecedence = prec::Comma; in tryMergePreviousTokens()
291 Tokens.back()->setFinalizedType(TT_TableGenMultiLineString); in tryMergePreviousTokens()
292 Tokens.back()->Tok.setKind(tok::string_literal); in tryMergePreviousTokens()
299 Tokens.back()->Tok.setKind(tok::identifier); in tryMergePreviousTokens()
300 Tokens.back()->Tok.setIdentifierInfo(nullptr); in tryMergePreviousTokens()
301 if (Tokens.back()->TokenText == "!cond") in tryMergePreviousTokens()
302 Tokens.back()->setFinalizedType(TT_TableGenCondOperator); in tryMergePreviousTokens()
304 Tokens.back()->setFinalizedType(TT_TableGenBangOperator); in tryMergePreviousTokens()
310 Tokens.back()->Tok.setKind(tok::identifier); in tryMergePreviousTokens()
311 Tokens.back()->Tok.setIdentifierInfo(nullptr); in tryMergePreviousTokens()
312 Tokens.back()->setFinalizedType(TT_TableGenBangOperator); in tryMergePreviousTokens()
317 Tokens.back()->Tok.setKind(tok::numeric_constant); in tryMergePreviousTokens()
321 Tokens.back()->Tok.setKind(tok::numeric_constant); in tryMergePreviousTokens()
328 if (Tokens.size() < 2) in tryMergeNSStringLiteral()
330 auto &At = *(Tokens.end() - 2); in tryMergeNSStringLiteral()
331 auto &String = *(Tokens.end() - 1); in tryMergeNSStringLiteral()
339 Tokens.erase(Tokens.end() - 1); in tryMergeNSStringLiteral()
346 if (Tokens.size() < 2) in tryMergeJSPrivateIdentifier()
348 auto &Hash = *(Tokens.end() - 2); in tryMergeJSPrivateIdentifier()
349 auto &Identifier = *(Tokens.end() - 1); in tryMergeJSPrivateIdentifier()
358 Tokens.erase(Tokens.end() - 1); in tryMergeJSPrivateIdentifier()
368 if (Tokens.size() < 2) in tryMergeCSharpStringLiteral()
372 const auto String = *(Tokens.end() - 1); in tryMergeCSharpStringLiteral()
376 auto Prefix = *(Tokens.end() - 2); in tryMergeCSharpStringLiteral()
380 if (Tokens.size() > 2) { in tryMergeCSharpStringLiteral()
381 const auto Tok = *(Tokens.end() - 3); in tryMergeCSharpStringLiteral()
384 // This looks like $@"aaa" or @$"aaa" so we need to combine all 3 tokens. in tryMergeCSharpStringLiteral()
386 Tokens.erase(Tokens.end() - 2); in tryMergeCSharpStringLiteral()
398 Tokens.erase(Tokens.end() - 1); in tryMergeCSharpStringLiteral()
410 if (Tokens.size() < 2) in tryMergeNullishCoalescingEqual()
412 auto &NullishCoalescing = *(Tokens.end() - 2); in tryMergeNullishCoalescingEqual()
413 auto &Equal = *(Tokens.end() - 1); in tryMergeNullishCoalescingEqual()
418 NullishCoalescing->Tok.setKind(tok::equal); // no '??=' in clang tokens. in tryMergeNullishCoalescingEqual()
424 Tokens.erase(Tokens.end() - 1); in tryMergeNullishCoalescingEqual()
429 if (Tokens.size() < 2) in tryMergeCSharpKeywordVariables()
431 const auto At = *(Tokens.end() - 2); in tryMergeCSharpKeywordVariables()
434 const auto Keyword = *(Tokens.end() - 1); in tryMergeCSharpKeywordVariables()
445 Tokens.erase(Tokens.end() - 1); in tryMergeCSharpKeywordVariables()
451 if (Tokens.size() < 1) in tryTransformCSharpForEach()
453 auto &Identifier = *(Tokens.end() - 1); in tryTransformCSharpForEach()
465 if (Tokens.size() < 2) in tryMergeForEach()
467 auto &For = *(Tokens.end() - 2); in tryMergeForEach()
468 auto &Each = *(Tokens.end() - 1); in tryMergeForEach()
482 Tokens.erase(Tokens.end() - 1); in tryMergeForEach()
487 if (Tokens.size() < 2) in tryTransformTryUsageForC()
489 auto &Try = *(Tokens.end() - 2); in tryTransformTryUsageForC()
492 auto &Next = *(Tokens.end() - 1); in tryTransformTryUsageForC()
496 if (Tokens.size() > 2) { in tryTransformTryUsageForC()
497 auto &At = *(Tokens.end() - 3); in tryTransformTryUsageForC()
508 if (Tokens.size() < 3) in tryMergeLessLess()
511 auto First = Tokens.end() - 3; in tryMergeLessLess()
519 auto X = Tokens.size() > 3 ? First[-1] : nullptr; in tryMergeLessLess()
530 Tokens.erase(Tokens.end() - 2); in tryMergeLessLess()
536 if (Tokens.size() < 2) in tryMergeGreaterGreater()
539 auto First = Tokens.end() - 2; in tryMergeGreaterGreater()
547 auto Tok = Tokens.size() > 2 ? First[-1] : nullptr; in tryMergeGreaterGreater()
554 Tokens.erase(Tokens.end() - 1); in tryMergeGreaterGreater()
560 if (Tokens.size() < Kinds.size()) in tryMergeTokens()
564 Tokens.end() - Kinds.size(); in tryMergeTokens()
573 if (Tokens.size() < Count) in tryMergeTokens()
576 SmallVectorImpl<FormatToken *>::const_iterator First = Tokens.end() - Count; in tryMergeTokens()
586 Tokens.resize(Tokens.size() - Count + 1); in tryMergeTokens()
626 return Tokens.size() < 3 || precedesOperand(Tokens[Tokens.size() - 3]); in canPrecedeRegexLiteral()
641 FormatToken *RegexToken = Tokens.back(); in tryParseJSRegexLiteral()
646 for (FormatToken *FT : llvm::drop_begin(llvm::reverse(Tokens))) { in tryParseJSRegexLiteral()
755 FormatToken *CSharpStringLiteral = Tokens.back(); in handleCSharpVerbatimAndInterpolatedStrings()
817 FormatToken *MultiLineString = Tokens.back(); in handleTableGenMultilineString()
848 FormatToken *Tok = Tokens.back(); in handleTableGenNumericLikeIdentifier()
849 // TableGen identifiers can begin with digits. Such tokens are lexed as in handleTableGenNumericLikeIdentifier()
886 FormatToken *BacktickToken = Tokens.back(); in handleTemplateStrings()
952 FormatToken *HashToken = Tokens.back(); in tryParsePythonComment()
973 if (Tokens.size() < 4) in tryMerge_TMacro()
975 FormatToken *Last = Tokens.back(); in tryMerge_TMacro()
979 FormatToken *String = Tokens[Tokens.size() - 2]; in tryMerge_TMacro()
983 if (Tokens[Tokens.size() - 3]->isNot(tok::l_paren)) in tryMerge_TMacro()
986 FormatToken *Macro = Tokens[Tokens.size() - 4]; in tryMerge_TMacro()
1002 Tokens.pop_back(); in tryMerge_TMacro()
1003 Tokens.pop_back(); in tryMerge_TMacro()
1004 Tokens.pop_back(); in tryMerge_TMacro()
1005 Tokens.back() = String; in tryMerge_TMacro()
1006 if (FirstInLineIndex >= Tokens.size()) in tryMerge_TMacro()
1007 FirstInLineIndex = Tokens.size() - 1; in tryMerge_TMacro()
1012 if (Tokens.back()->NewlinesBefore == 0 && Tokens.back()->isNot(tok::eof)) in tryMergeConflictMarkers()
1020 // We merge all tokens in a line that starts with a conflict marker in tryMergeConflictMarkers()
1028 Tokens[FirstInLineIndex]->getStartOfNonWhitespace()); in tryMergeConflictMarkers()
1055 FormatToken *Next = Tokens.back(); in tryMergeConflictMarkers()
1057 Tokens.resize(FirstInLineIndex + 1); in tryMergeConflictMarkers()
1061 Tokens.back()->setType(Type); in tryMergeConflictMarkers()
1062 Tokens.back()->Tok.setKind(tok::kw___unknown_anytype); in tryMergeConflictMarkers()
1064 Tokens.push_back(Next); in tryMergeConflictMarkers()
1156 // Some tok::unknown tokens are not just whitespace, e.g. whitespace in getNextToken()
1266 } else if (Tokens.size() > 0 && in getNextToken()
1267 Tokens.back()->is(Keywords.kw_apostrophe) && in getNextToken()
1271 // the rest of the number literal as two tokens. But if there is no in getNextToken()
1320 if (Style.isVerilog() && Tokens.size() > 0 && in getNextToken()
1321 Tokens.back()->is(TT_VerilogNumberBase) && in getNextToken()
1354 if (!(Tokens.size() > 0 && Tokens.back()->Tok.getIdentifierInfo() && in getNextToken()
1355 Tokens.back()->Tok.getIdentifierInfo()->getPPKeywordID() == in getNextToken()