57 // Finds the range within FID corresponding to expanded tokens [First, Last].
71   //  - identifying which spelled range covers the expanded tokens  in spelledForExpandedSlow()
74   // We do these in order. However as we transform the expanded range into the  in spelledForExpandedSlow()
247     // AST. Ranges in AST are token ranges from expanded token stream.  in expandedTokens()
271 TokenBuffer::spelledForExpandedToken(const syntax::Token *Expanded) const {  in spelledForExpandedToken()
272   assert(Expanded);  in spelledForExpandedToken()
273   assert(ExpandedTokens.data() <= Expanded &&  in spelledForExpandedToken()
274          Expanded < ExpandedTokens.data() + ExpandedTokens.size());  in spelledForExpandedToken()
277       SourceMgr->getFileID(SourceMgr->getExpansionLoc(Expanded->location())));  in spelledForExpandedToken()
278   assert(FileIt != Files.end() && "no file for an expanded token");  in spelledForExpandedToken()
282   unsigned ExpandedIndex = Expanded - ExpandedTokens.data();  in spelledForExpandedToken()
283   // Find the first mapping that produced tokens after \p Expanded.  in spelledForExpandedToken()
399       llvm::formatv("spelled tokens: [{0},{1}), expanded tokens: [{2},{3})",  in str()
404 TokenBuffer::spelledForExpanded(llvm::ArrayRef<syntax::Token> Expanded) const {  in spelledForExpanded()
408   if (!Expanded.empty() && Expanded.back().kind() == tok::eof) {  in spelledForExpanded()
409     Expanded = Expanded.drop_back();  in spelledForExpanded()
413   if (Expanded.empty())  in spelledForExpanded()
415   const syntax::Token *First = &Expanded.front();  in spelledForExpanded()
416   const syntax::Token *Last = &Expanded.back();  in spelledForExpanded()
448   unsigned FirstExpanded = Expanded.begin() - ExpandedTokens.data();  in spelledForExpanded()
449   unsigned LastExpanded = Expanded.end() - ExpandedTokens.data();  in spelledForExpanded()
466   E.Expanded = llvm::ArrayRef(ExpandedTokens.data() + M.BeginExpanded,  in makeExpansion()
629     // Only record top-level expansions that directly produce expanded tokens.  in MacroExpands()
638     // Both A and B produce expanded tokens, though the macro name 'B' comes  in MacroExpands()
673 /// - for each token, figures out if it is a part of an expanded token stream,
675 /// - records mappings from the spelled to expanded token ranges, e.g. for macro
684   // Collect the expanded token stream during preprocessing.  in TokenCollector()
695     Expanded.push_back(syntax::Token(T));  in TokenCollector()
704 /// Builds mappings and spelled tokens in the TokenBuffer based on the expanded
708   Builder(std::vector<syntax::Token> Expanded, PPExpansions CollectedExpansions,  in Builder()  argument
712     Result.ExpandedTokens = std::move(Expanded);  in Builder()
719     // Tokenize every file that contributed tokens to the expanded stream.  in build()
722     // The expanded token stream consists of runs of tokens that came from  in build()
727       // Create empty mappings for spelled tokens that expanded to nothing here.  in build()
730       // Create mapping for a contiguous run of expanded tokens.  in build()
775     // be positioned within the file's expanded-token range (at the end).  in discard()
819       // A run of file tokens continues while the expanded/spelled tokens match.  in advance()
827       // We need no mapping for file tokens copied to the expanded stream.  in advance()
841       // ... consumes expanded tokens rooted at the same expansion ...  in advance()
864     llvm_unreachable("Couldn't map expanded token to spelled tokens!");  in diagnoseAdvanceFailure()
867   /// Initializes TokenBuffer::Files and fills spelled tokens and expanded
898   return Builder(std::move(Expanded), std::move(Expansions),  in consume()
937   OS << "expanded tokens:\n"  in dumpForTests()