1 //===- Tokens.cpp - collect tokens from preprocessing ---------------------===//
8 #include "clang/Tooling/Syntax/Tokens.h"
57 // Finds the range within FID corresponding to expanded tokens [First, Last].
71   //  - identifying which spelled range covers the expanded tokens  in spelledForExpandedSlow()
72   //  - validating that this range doesn't cover any extra tokens (First/Last)  in spelledForExpandedSlow()
174   assert(F.file() == L.file() && "tokens from different files");  in range()
176          "wrong order of tokens");  in range()
283   // Find the first mapping that produced tokens after \p Expanded.  in spelledForExpandedToken()
337     // This mapping applies to Spelled tokens.  in expandedForSpelled()
339       // Spelled tokens don't cover the entire mapping, returning empty result.  in expandedForSpelled()
342     // Spelled tokens start at the beginning of this mapping.  in expandedForSpelled()
345     // Spelled tokens start after the mapping ends (they start in the hole  in expandedForSpelled()
359     // This mapping applies to Spelled tokens.  in expandedForSpelled()
361       // Spelled tokens don't cover the entire mapping, returning empty result.  in expandedForSpelled()
366     // Spelled tokens end after the mapping ends.  in expandedForSpelled()
399       llvm::formatv("spelled tokens: [{0},{1}), expanded tokens: [{2},{3})",  in str()
481          "Tokens not in spelled range");  in fileForSpelled()
524                               llvm::ArrayRef<syntax::Token> Tokens) {  in spelledTokensTouching()  argument
528       Tokens, [&](const syntax::Token &Tok) { return Tok.location() < Loc; });  in spelledTokensTouching()
529   bool AcceptRight = Right != Tokens.end() && Right->location() <= Loc;  in spelledTokensTouching()
531       Right != Tokens.begin() && (Right - 1)->endLocation() >= Loc;  in spelledTokensTouching()
538                               const syntax::TokenBuffer &Tokens) {  in spelledTokensTouching()  argument
540       Loc, Tokens.spelledTokens(Tokens.sourceManager().getFileID(Loc)));  in spelledTokensTouching()
545                                   llvm::ArrayRef<syntax::Token> Tokens) {  in spelledIdentifierTouching()  argument
546   for (const syntax::Token &Tok : spelledTokensTouching(Loc, Tokens)) {  in spelledIdentifierTouching()
555                                   const syntax::TokenBuffer &Tokens) {  in spelledIdentifierTouching()  argument
557       Loc, Tokens.spelledTokens(Tokens.sourceManager().getFileID(Loc)));  in spelledIdentifierTouching()
578   std::vector<syntax::Token> Tokens;  in tokenize()  local
588     Tokens.push_back(syntax::Token(T));  in tokenize()
605   return Tokens;  in tokenize()
629     // Only record top-level expansions that directly produce expanded tokens.  in MacroExpands()
638     // Both A and B produce expanded tokens, though the macro name 'B' comes  in MacroExpands()
671 /// implementation tracks the tokens, macro expansions and directives coming
674 ///   spelled token stream or both. Stores the tokens appropriately.
688     DEBUG_WITH_TYPE("collect-tokens", llvm::dbgs()  in TokenCollector()
704 /// Builds mappings and spelled tokens in the TokenBuffer based on the expanded
719     // Tokenize every file that contributed tokens to the expanded stream.  in build()
722     // The expanded token stream consists of runs of tokens that came from  in build()
724     // Between these runs are the logical positions of spelled tokens that  in build()
727       // Create empty mappings for spelled tokens that expanded to nothing here.  in build()
730       // Create mapping for a contiguous run of expanded tokens.  in build()
737     // If any tokens remain in any of the files, they didn't expand to anything.  in build()
759   // Consume a sequence of spelled tokens that didn't expand to anything.
760   // In the simplest case, skips spelled tokens until finding one that produced
762   // If Drain is provided, skips remaining tokens from that file instead.
774     // When dropping trailing tokens from a file, the empty mapping should  in discard()
810   // (unless this is a run of file tokens, which we represent with no mapping).
819       // A run of file tokens continues while the expanded/spelled tokens match.  in advance()
827       // We need no mapping for file tokens copied to the expanded stream.  in advance()
837       // ... consumes spelled tokens within bounds we captured ...  in advance()
841       // ... consumes expanded tokens rooted at the same expansion ...  in advance()
864     llvm_unreachable("Couldn't map expanded token to spelled tokens!");  in diagnoseAdvanceFailure()
867   /// Initializes TokenBuffer::Files and fills spelled tokens and expanded
921                                         llvm::ArrayRef<syntax::Token> Tokens) {  in dumpForTests()  argument
922     if (Tokens.empty()) {  in dumpForTests()
926     OS << Tokens[0].text(*SourceMgr);  in dumpForTests()
927     for (unsigned I = 1; I < Tokens.size(); ++I) {  in dumpForTests()
928       if (Tokens[I].kind() == tok::eof)  in dumpForTests()
930       OS << " " << PrintToken(Tokens[I]);  in dumpForTests()
937   OS << "expanded tokens:\n"  in dumpForTests()
954     OS << llvm::formatv("file '{0}'\n", Path) << "  spelled tokens:\n"  in dumpForTests()