1 //===- Tokens.cpp - collect tokens from preprocessing ---------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 #include "clang/Tooling/Syntax/Tokens.h" 9 10 #include "clang/Basic/Diagnostic.h" 11 #include "clang/Basic/IdentifierTable.h" 12 #include "clang/Basic/LLVM.h" 13 #include "clang/Basic/LangOptions.h" 14 #include "clang/Basic/SourceLocation.h" 15 #include "clang/Basic/SourceManager.h" 16 #include "clang/Basic/TokenKinds.h" 17 #include "clang/Lex/PPCallbacks.h" 18 #include "clang/Lex/Preprocessor.h" 19 #include "clang/Lex/Token.h" 20 #include "llvm/ADT/ArrayRef.h" 21 #include "llvm/ADT/None.h" 22 #include "llvm/ADT/Optional.h" 23 #include "llvm/ADT/STLExtras.h" 24 #include "llvm/Support/Debug.h" 25 #include "llvm/Support/ErrorHandling.h" 26 #include "llvm/Support/FormatVariadic.h" 27 #include "llvm/Support/raw_ostream.h" 28 #include <algorithm> 29 #include <cassert> 30 #include <iterator> 31 #include <string> 32 #include <utility> 33 #include <vector> 34 35 using namespace clang; 36 using namespace clang::syntax; 37 38 syntax::Token::Token(SourceLocation Location, unsigned Length, 39 tok::TokenKind Kind) 40 : Location(Location), Length(Length), Kind(Kind) { 41 assert(Location.isValid()); 42 } 43 44 syntax::Token::Token(const clang::Token &T) 45 : Token(T.getLocation(), T.getLength(), T.getKind()) { 46 assert(!T.isAnnotation()); 47 } 48 49 llvm::StringRef syntax::Token::text(const SourceManager &SM) const { 50 bool Invalid = false; 51 const char *Start = SM.getCharacterData(location(), &Invalid); 52 assert(!Invalid); 53 return llvm::StringRef(Start, length()); 54 } 55 56 FileRange syntax::Token::range(const SourceManager &SM) const { 57 assert(location().isFileID() && "must be a spelled token"); 58 FileID File; 59 unsigned StartOffset; 60 std::tie(File, StartOffset) = SM.getDecomposedLoc(location()); 61 return FileRange(File, StartOffset, StartOffset + length()); 62 } 63 64 FileRange syntax::Token::range(const SourceManager &SM, 65 const syntax::Token &First, 66 const syntax::Token &Last) { 67 auto F = First.range(SM); 68 auto L = Last.range(SM); 69 assert(F.file() == L.file() && "tokens from different files"); 70 assert(F.endOffset() <= L.beginOffset() && "wrong order of tokens"); 71 return FileRange(F.file(), F.beginOffset(), L.endOffset()); 72 } 73 74 llvm::raw_ostream &syntax::operator<<(llvm::raw_ostream &OS, const Token &T) { 75 return OS << T.str(); 76 } 77 78 FileRange::FileRange(FileID File, unsigned BeginOffset, unsigned EndOffset) 79 : File(File), Begin(BeginOffset), End(EndOffset) { 80 assert(File.isValid()); 81 assert(BeginOffset <= EndOffset); 82 } 83 84 FileRange::FileRange(const SourceManager &SM, SourceLocation BeginLoc, 85 unsigned Length) { 86 assert(BeginLoc.isValid()); 87 assert(BeginLoc.isFileID()); 88 89 std::tie(File, Begin) = SM.getDecomposedLoc(BeginLoc); 90 End = Begin + Length; 91 } 92 FileRange::FileRange(const SourceManager &SM, SourceLocation BeginLoc, 93 SourceLocation EndLoc) { 94 assert(BeginLoc.isValid()); 95 assert(BeginLoc.isFileID()); 96 assert(EndLoc.isValid()); 97 assert(EndLoc.isFileID()); 98 assert(SM.getFileID(BeginLoc) == SM.getFileID(EndLoc)); 99 assert(SM.getFileOffset(BeginLoc) <= SM.getFileOffset(EndLoc)); 100 101 std::tie(File, Begin) = SM.getDecomposedLoc(BeginLoc); 102 End = SM.getFileOffset(EndLoc); 103 } 104 105 llvm::raw_ostream &syntax::operator<<(llvm::raw_ostream &OS, 106 const FileRange &R) { 107 return OS << llvm::formatv("FileRange(file = {0}, offsets = {1}-{2})", 108 R.file().getHashValue(), R.beginOffset(), 109 R.endOffset()); 110 } 111 112 llvm::StringRef FileRange::text(const SourceManager &SM) const { 113 bool Invalid = false; 114 StringRef Text = SM.getBufferData(File, &Invalid); 115 if (Invalid) 116 return ""; 117 assert(Begin <= Text.size()); 118 assert(End <= Text.size()); 119 return Text.substr(Begin, length()); 120 } 121 122 std::pair<const syntax::Token *, const TokenBuffer::Mapping *> 123 TokenBuffer::spelledForExpandedToken(const syntax::Token *Expanded) const { 124 assert(Expanded); 125 assert(ExpandedTokens.data() <= Expanded && 126 Expanded < ExpandedTokens.data() + ExpandedTokens.size()); 127 128 auto FileIt = Files.find( 129 SourceMgr->getFileID(SourceMgr->getExpansionLoc(Expanded->location()))); 130 assert(FileIt != Files.end() && "no file for an expanded token"); 131 132 const MarkedFile &File = FileIt->second; 133 134 unsigned ExpandedIndex = Expanded - ExpandedTokens.data(); 135 // Find the first mapping that produced tokens after \p Expanded. 136 auto It = llvm::partition_point(File.Mappings, [&](const Mapping &M) { 137 return M.BeginExpanded <= ExpandedIndex; 138 }); 139 // Our token could only be produced by the previous mapping. 140 if (It == File.Mappings.begin()) { 141 // No previous mapping, no need to modify offsets. 142 return {&File.SpelledTokens[ExpandedIndex - File.BeginExpanded], nullptr}; 143 } 144 --It; // 'It' now points to last mapping that started before our token. 145 146 // Check if the token is part of the mapping. 147 if (ExpandedIndex < It->EndExpanded) 148 return {&File.SpelledTokens[It->BeginSpelled], /*Mapping*/ &*It}; 149 150 // Not part of the mapping, use the index from previous mapping to compute the 151 // corresponding spelled token. 152 return { 153 &File.SpelledTokens[It->EndSpelled + (ExpandedIndex - It->EndExpanded)], 154 /*Mapping*/ nullptr}; 155 } 156 157 llvm::ArrayRef<syntax::Token> TokenBuffer::spelledTokens(FileID FID) const { 158 auto It = Files.find(FID); 159 assert(It != Files.end()); 160 return It->second.SpelledTokens; 161 } 162 163 std::string TokenBuffer::Mapping::str() const { 164 return llvm::formatv("spelled tokens: [{0},{1}), expanded tokens: [{2},{3})", 165 BeginSpelled, EndSpelled, BeginExpanded, EndExpanded); 166 } 167 168 llvm::Optional<llvm::ArrayRef<syntax::Token>> 169 TokenBuffer::spelledForExpanded(llvm::ArrayRef<syntax::Token> Expanded) const { 170 // Mapping an empty range is ambiguous in case of empty mappings at either end 171 // of the range, bail out in that case. 172 if (Expanded.empty()) 173 return llvm::None; 174 175 // FIXME: also allow changes uniquely mapping to macro arguments. 176 177 const syntax::Token *BeginSpelled; 178 const Mapping *BeginMapping; 179 std::tie(BeginSpelled, BeginMapping) = 180 spelledForExpandedToken(&Expanded.front()); 181 182 const syntax::Token *LastSpelled; 183 const Mapping *LastMapping; 184 std::tie(LastSpelled, LastMapping) = 185 spelledForExpandedToken(&Expanded.back()); 186 187 FileID FID = SourceMgr->getFileID(BeginSpelled->location()); 188 // FIXME: Handle multi-file changes by trying to map onto a common root. 189 if (FID != SourceMgr->getFileID(LastSpelled->location())) 190 return llvm::None; 191 192 const MarkedFile &File = Files.find(FID)->second; 193 194 // Do not allow changes that cross macro expansion boundaries. 195 unsigned BeginExpanded = Expanded.begin() - ExpandedTokens.data(); 196 unsigned EndExpanded = Expanded.end() - ExpandedTokens.data(); 197 if (BeginMapping && BeginMapping->BeginExpanded < BeginExpanded) 198 return llvm::None; 199 if (LastMapping && EndExpanded < LastMapping->EndExpanded) 200 return llvm::None; 201 // All is good, return the result. 202 return llvm::makeArrayRef( 203 BeginMapping ? File.SpelledTokens.data() + BeginMapping->BeginSpelled 204 : BeginSpelled, 205 LastMapping ? File.SpelledTokens.data() + LastMapping->EndSpelled 206 : LastSpelled + 1); 207 } 208 209 llvm::Optional<TokenBuffer::Expansion> 210 TokenBuffer::expansionStartingAt(const syntax::Token *Spelled) const { 211 assert(Spelled); 212 assert(Spelled->location().isFileID() && "not a spelled token"); 213 auto FileIt = Files.find(SourceMgr->getFileID(Spelled->location())); 214 assert(FileIt != Files.end() && "file not tracked by token buffer"); 215 216 auto &File = FileIt->second; 217 assert(File.SpelledTokens.data() <= Spelled && 218 Spelled < (File.SpelledTokens.data() + File.SpelledTokens.size())); 219 220 unsigned SpelledIndex = Spelled - File.SpelledTokens.data(); 221 auto M = llvm::partition_point(File.Mappings, [&](const Mapping &M) { 222 return M.BeginSpelled < SpelledIndex; 223 }); 224 if (M == File.Mappings.end() || M->BeginSpelled != SpelledIndex) 225 return llvm::None; 226 227 Expansion E; 228 E.Spelled = llvm::makeArrayRef(File.SpelledTokens.data() + M->BeginSpelled, 229 File.SpelledTokens.data() + M->EndSpelled); 230 E.Expanded = llvm::makeArrayRef(ExpandedTokens.data() + M->BeginExpanded, 231 ExpandedTokens.data() + M->EndExpanded); 232 return E; 233 } 234 235 std::vector<const syntax::Token *> 236 TokenBuffer::macroExpansions(FileID FID) const { 237 auto FileIt = Files.find(FID); 238 assert(FileIt != Files.end() && "file not tracked by token buffer"); 239 auto &File = FileIt->second; 240 std::vector<const syntax::Token *> Expansions; 241 auto &Spelled = File.SpelledTokens; 242 for (auto Mapping : File.Mappings) { 243 const syntax::Token *Token = &Spelled[Mapping.BeginSpelled]; 244 if (Token->kind() == tok::TokenKind::identifier) 245 Expansions.push_back(Token); 246 } 247 return Expansions; 248 } 249 250 std::vector<syntax::Token> syntax::tokenize(FileID FID, const SourceManager &SM, 251 const LangOptions &LO) { 252 std::vector<syntax::Token> Tokens; 253 IdentifierTable Identifiers(LO); 254 auto AddToken = [&](clang::Token T) { 255 // Fill the proper token kind for keywords, etc. 256 if (T.getKind() == tok::raw_identifier && !T.needsCleaning() && 257 !T.hasUCN()) { // FIXME: support needsCleaning and hasUCN cases. 258 clang::IdentifierInfo &II = Identifiers.get(T.getRawIdentifier()); 259 T.setIdentifierInfo(&II); 260 T.setKind(II.getTokenID()); 261 } 262 Tokens.push_back(syntax::Token(T)); 263 }; 264 265 Lexer L(FID, SM.getBuffer(FID), SM, LO); 266 267 clang::Token T; 268 while (!L.LexFromRawLexer(T)) 269 AddToken(T); 270 // 'eof' is only the last token if the input is null-terminated. Never store 271 // it, for consistency. 272 if (T.getKind() != tok::eof) 273 AddToken(T); 274 return Tokens; 275 } 276 277 /// Records information reqired to construct mappings for the token buffer that 278 /// we are collecting. 279 class TokenCollector::CollectPPExpansions : public PPCallbacks { 280 public: 281 CollectPPExpansions(TokenCollector &C) : Collector(&C) {} 282 283 /// Disabled instance will stop reporting anything to TokenCollector. 284 /// This ensures that uses of the preprocessor after TokenCollector::consume() 285 /// is called do not access the (possibly invalid) collector instance. 286 void disable() { Collector = nullptr; } 287 288 void MacroExpands(const clang::Token &MacroNameTok, const MacroDefinition &MD, 289 SourceRange Range, const MacroArgs *Args) override { 290 if (!Collector) 291 return; 292 // Only record top-level expansions, not those where: 293 // - the macro use is inside a macro body, 294 // - the macro appears in an argument to another macro. 295 if (!MacroNameTok.getLocation().isFileID() || 296 (LastExpansionEnd.isValid() && 297 Collector->PP.getSourceManager().isBeforeInTranslationUnit( 298 Range.getBegin(), LastExpansionEnd))) 299 return; 300 Collector->Expansions[Range.getBegin().getRawEncoding()] = Range.getEnd(); 301 LastExpansionEnd = Range.getEnd(); 302 } 303 // FIXME: handle directives like #pragma, #include, etc. 304 private: 305 TokenCollector *Collector; 306 /// Used to detect recursive macro expansions. 307 SourceLocation LastExpansionEnd; 308 }; 309 310 /// Fills in the TokenBuffer by tracing the run of a preprocessor. The 311 /// implementation tracks the tokens, macro expansions and directives coming 312 /// from the preprocessor and: 313 /// - for each token, figures out if it is a part of an expanded token stream, 314 /// spelled token stream or both. Stores the tokens appropriately. 315 /// - records mappings from the spelled to expanded token ranges, e.g. for macro 316 /// expansions. 317 /// FIXME: also properly record: 318 /// - #include directives, 319 /// - #pragma, #line and other PP directives, 320 /// - skipped pp regions, 321 /// - ... 322 323 TokenCollector::TokenCollector(Preprocessor &PP) : PP(PP) { 324 // Collect the expanded token stream during preprocessing. 325 PP.setTokenWatcher([this](const clang::Token &T) { 326 if (T.isAnnotation()) 327 return; 328 DEBUG_WITH_TYPE("collect-tokens", llvm::dbgs() 329 << "Token: " 330 << syntax::Token(T).dumpForTests( 331 this->PP.getSourceManager()) 332 << "\n" 333 334 ); 335 Expanded.push_back(syntax::Token(T)); 336 }); 337 // And locations of macro calls, to properly recover boundaries of those in 338 // case of empty expansions. 339 auto CB = std::make_unique<CollectPPExpansions>(*this); 340 this->Collector = CB.get(); 341 PP.addPPCallbacks(std::move(CB)); 342 } 343 344 /// Builds mappings and spelled tokens in the TokenBuffer based on the expanded 345 /// token stream. 346 class TokenCollector::Builder { 347 public: 348 Builder(std::vector<syntax::Token> Expanded, PPExpansions CollectedExpansions, 349 const SourceManager &SM, const LangOptions &LangOpts) 350 : Result(SM), CollectedExpansions(std::move(CollectedExpansions)), SM(SM), 351 LangOpts(LangOpts) { 352 Result.ExpandedTokens = std::move(Expanded); 353 } 354 355 TokenBuffer build() && { 356 buildSpelledTokens(); 357 358 // Walk over expanded tokens and spelled tokens in parallel, building the 359 // mappings between those using source locations. 360 // To correctly recover empty macro expansions, we also take locations 361 // reported to PPCallbacks::MacroExpands into account as we do not have any 362 // expanded tokens with source locations to guide us. 363 364 // The 'eof' token is special, it is not part of spelled token stream. We 365 // handle it separately at the end. 366 assert(!Result.ExpandedTokens.empty()); 367 assert(Result.ExpandedTokens.back().kind() == tok::eof); 368 for (unsigned I = 0; I < Result.ExpandedTokens.size() - 1; ++I) { 369 // (!) I might be updated by the following call. 370 processExpandedToken(I); 371 } 372 373 // 'eof' not handled in the loop, do it here. 374 assert(SM.getMainFileID() == 375 SM.getFileID(Result.ExpandedTokens.back().location())); 376 fillGapUntil(Result.Files[SM.getMainFileID()], 377 Result.ExpandedTokens.back().location(), 378 Result.ExpandedTokens.size() - 1); 379 Result.Files[SM.getMainFileID()].EndExpanded = Result.ExpandedTokens.size(); 380 381 // Some files might have unaccounted spelled tokens at the end, add an empty 382 // mapping for those as they did not have expanded counterparts. 383 fillGapsAtEndOfFiles(); 384 385 return std::move(Result); 386 } 387 388 private: 389 /// Process the next token in an expanded stream and move corresponding 390 /// spelled tokens, record any mapping if needed. 391 /// (!) \p I will be updated if this had to skip tokens, e.g. for macros. 392 void processExpandedToken(unsigned &I) { 393 auto L = Result.ExpandedTokens[I].location(); 394 if (L.isMacroID()) { 395 processMacroExpansion(SM.getExpansionRange(L), I); 396 return; 397 } 398 if (L.isFileID()) { 399 auto FID = SM.getFileID(L); 400 TokenBuffer::MarkedFile &File = Result.Files[FID]; 401 402 fillGapUntil(File, L, I); 403 404 // Skip the token. 405 assert(File.SpelledTokens[NextSpelled[FID]].location() == L && 406 "no corresponding token in the spelled stream"); 407 ++NextSpelled[FID]; 408 return; 409 } 410 } 411 412 /// Skipped expanded and spelled tokens of a macro expansion that covers \p 413 /// SpelledRange. Add a corresponding mapping. 414 /// (!) \p I will be the index of the last token in an expansion after this 415 /// function returns. 416 void processMacroExpansion(CharSourceRange SpelledRange, unsigned &I) { 417 auto FID = SM.getFileID(SpelledRange.getBegin()); 418 assert(FID == SM.getFileID(SpelledRange.getEnd())); 419 TokenBuffer::MarkedFile &File = Result.Files[FID]; 420 421 fillGapUntil(File, SpelledRange.getBegin(), I); 422 423 // Skip all expanded tokens from the same macro expansion. 424 unsigned BeginExpanded = I; 425 for (; I + 1 < Result.ExpandedTokens.size(); ++I) { 426 auto NextL = Result.ExpandedTokens[I + 1].location(); 427 if (!NextL.isMacroID() || 428 SM.getExpansionLoc(NextL) != SpelledRange.getBegin()) 429 break; 430 } 431 unsigned EndExpanded = I + 1; 432 consumeMapping(File, SM.getFileOffset(SpelledRange.getEnd()), BeginExpanded, 433 EndExpanded, NextSpelled[FID]); 434 } 435 436 /// Initializes TokenBuffer::Files and fills spelled tokens and expanded 437 /// ranges for each of the files. 438 void buildSpelledTokens() { 439 for (unsigned I = 0; I < Result.ExpandedTokens.size(); ++I) { 440 auto FID = 441 SM.getFileID(SM.getExpansionLoc(Result.ExpandedTokens[I].location())); 442 auto It = Result.Files.try_emplace(FID); 443 TokenBuffer::MarkedFile &File = It.first->second; 444 445 File.EndExpanded = I + 1; 446 if (!It.second) 447 continue; // we have seen this file before. 448 449 // This is the first time we see this file. 450 File.BeginExpanded = I; 451 File.SpelledTokens = tokenize(FID, SM, LangOpts); 452 } 453 } 454 455 void consumeEmptyMapping(TokenBuffer::MarkedFile &File, unsigned EndOffset, 456 unsigned ExpandedIndex, unsigned &SpelledIndex) { 457 consumeMapping(File, EndOffset, ExpandedIndex, ExpandedIndex, SpelledIndex); 458 } 459 460 /// Consumes spelled tokens that form a macro expansion and adds a entry to 461 /// the resulting token buffer. 462 /// (!) SpelledIndex is updated in-place. 463 void consumeMapping(TokenBuffer::MarkedFile &File, unsigned EndOffset, 464 unsigned BeginExpanded, unsigned EndExpanded, 465 unsigned &SpelledIndex) { 466 // We need to record this mapping before continuing. 467 unsigned MappingBegin = SpelledIndex; 468 ++SpelledIndex; 469 470 bool HitMapping = 471 tryConsumeSpelledUntil(File, EndOffset + 1, SpelledIndex).hasValue(); 472 (void)HitMapping; 473 assert(!HitMapping && "recursive macro expansion?"); 474 475 TokenBuffer::Mapping M; 476 M.BeginExpanded = BeginExpanded; 477 M.EndExpanded = EndExpanded; 478 M.BeginSpelled = MappingBegin; 479 M.EndSpelled = SpelledIndex; 480 481 File.Mappings.push_back(M); 482 } 483 484 /// Consumes spelled tokens until location \p L is reached and adds a mapping 485 /// covering the consumed tokens. The mapping will point to an empty expanded 486 /// range at position \p ExpandedIndex. 487 void fillGapUntil(TokenBuffer::MarkedFile &File, SourceLocation L, 488 unsigned ExpandedIndex) { 489 assert(L.isFileID()); 490 FileID FID; 491 unsigned Offset; 492 std::tie(FID, Offset) = SM.getDecomposedLoc(L); 493 494 unsigned &SpelledIndex = NextSpelled[FID]; 495 unsigned MappingBegin = SpelledIndex; 496 while (true) { 497 auto EndLoc = tryConsumeSpelledUntil(File, Offset, SpelledIndex); 498 if (SpelledIndex != MappingBegin) { 499 TokenBuffer::Mapping M; 500 M.BeginSpelled = MappingBegin; 501 M.EndSpelled = SpelledIndex; 502 M.BeginExpanded = M.EndExpanded = ExpandedIndex; 503 File.Mappings.push_back(M); 504 } 505 if (!EndLoc) 506 break; 507 consumeEmptyMapping(File, SM.getFileOffset(*EndLoc), ExpandedIndex, 508 SpelledIndex); 509 510 MappingBegin = SpelledIndex; 511 } 512 }; 513 514 /// Consumes spelled tokens until it reaches Offset or a mapping boundary, 515 /// i.e. a name of a macro expansion or the start '#' token of a PP directive. 516 /// (!) NextSpelled is updated in place. 517 /// 518 /// returns None if \p Offset was reached, otherwise returns the end location 519 /// of a mapping that starts at \p NextSpelled. 520 llvm::Optional<SourceLocation> 521 tryConsumeSpelledUntil(TokenBuffer::MarkedFile &File, unsigned Offset, 522 unsigned &NextSpelled) { 523 for (; NextSpelled < File.SpelledTokens.size(); ++NextSpelled) { 524 auto L = File.SpelledTokens[NextSpelled].location(); 525 if (Offset <= SM.getFileOffset(L)) 526 return llvm::None; // reached the offset we are looking for. 527 auto Mapping = CollectedExpansions.find(L.getRawEncoding()); 528 if (Mapping != CollectedExpansions.end()) 529 return Mapping->second; // found a mapping before the offset. 530 } 531 return llvm::None; // no more tokens, we "reached" the offset. 532 } 533 534 /// Adds empty mappings for unconsumed spelled tokens at the end of each file. 535 void fillGapsAtEndOfFiles() { 536 for (auto &F : Result.Files) { 537 if (F.second.SpelledTokens.empty()) 538 continue; 539 fillGapUntil(F.second, F.second.SpelledTokens.back().endLocation(), 540 F.second.EndExpanded); 541 } 542 } 543 544 TokenBuffer Result; 545 /// For each file, a position of the next spelled token we will consume. 546 llvm::DenseMap<FileID, unsigned> NextSpelled; 547 PPExpansions CollectedExpansions; 548 const SourceManager &SM; 549 const LangOptions &LangOpts; 550 }; 551 552 TokenBuffer TokenCollector::consume() && { 553 PP.setTokenWatcher(nullptr); 554 Collector->disable(); 555 return Builder(std::move(Expanded), std::move(Expansions), 556 PP.getSourceManager(), PP.getLangOpts()) 557 .build(); 558 } 559 560 std::string syntax::Token::str() const { 561 return llvm::formatv("Token({0}, length = {1})", tok::getTokenName(kind()), 562 length()); 563 } 564 565 std::string syntax::Token::dumpForTests(const SourceManager &SM) const { 566 return llvm::formatv("{0} {1}", tok::getTokenName(kind()), text(SM)); 567 } 568 569 std::string TokenBuffer::dumpForTests() const { 570 auto PrintToken = [this](const syntax::Token &T) -> std::string { 571 if (T.kind() == tok::eof) 572 return "<eof>"; 573 return T.text(*SourceMgr); 574 }; 575 576 auto DumpTokens = [this, &PrintToken](llvm::raw_ostream &OS, 577 llvm::ArrayRef<syntax::Token> Tokens) { 578 if (Tokens.empty()) { 579 OS << "<empty>"; 580 return; 581 } 582 OS << Tokens[0].text(*SourceMgr); 583 for (unsigned I = 1; I < Tokens.size(); ++I) { 584 if (Tokens[I].kind() == tok::eof) 585 continue; 586 OS << " " << PrintToken(Tokens[I]); 587 } 588 }; 589 590 std::string Dump; 591 llvm::raw_string_ostream OS(Dump); 592 593 OS << "expanded tokens:\n" 594 << " "; 595 // (!) we do not show '<eof>'. 596 DumpTokens(OS, llvm::makeArrayRef(ExpandedTokens).drop_back()); 597 OS << "\n"; 598 599 std::vector<FileID> Keys; 600 for (auto F : Files) 601 Keys.push_back(F.first); 602 llvm::sort(Keys); 603 604 for (FileID ID : Keys) { 605 const MarkedFile &File = Files.find(ID)->second; 606 auto *Entry = SourceMgr->getFileEntryForID(ID); 607 if (!Entry) 608 continue; // Skip builtin files. 609 OS << llvm::formatv("file '{0}'\n", Entry->getName()) 610 << " spelled tokens:\n" 611 << " "; 612 DumpTokens(OS, File.SpelledTokens); 613 OS << "\n"; 614 615 if (File.Mappings.empty()) { 616 OS << " no mappings.\n"; 617 continue; 618 } 619 OS << " mappings:\n"; 620 for (auto &M : File.Mappings) { 621 OS << llvm::formatv( 622 " ['{0}'_{1}, '{2}'_{3}) => ['{4}'_{5}, '{6}'_{7})\n", 623 PrintToken(File.SpelledTokens[M.BeginSpelled]), M.BeginSpelled, 624 M.EndSpelled == File.SpelledTokens.size() 625 ? "<eof>" 626 : PrintToken(File.SpelledTokens[M.EndSpelled]), 627 M.EndSpelled, PrintToken(ExpandedTokens[M.BeginExpanded]), 628 M.BeginExpanded, PrintToken(ExpandedTokens[M.EndExpanded]), 629 M.EndExpanded); 630 } 631 } 632 return OS.str(); 633 } 634