Lines Matching +full:auto +full:- +full:range
1 //===- Tokens.cpp - collect tokens from preprocessing ---------------------===//
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
57 // Finds the range within FID corresponding to expanded tokens [First, Last].
59 // If no range satisfies the criteria, returns an invalid range.
63 // ~~ -> a1
64 // ~~ -> a2
65 // ~~~~~~~~~ -> a1 a2
71 // - identifying which spelled range covers the expanded tokens in spelledForExpandedSlow()
72 // - validating that this range doesn't cover any extra tokens (First/Last) in spelledForExpandedSlow()
74 // We do these in order. However as we transform the expanded range into the in spelledForExpandedSlow()
79 // range, i.e. the whole of any macros they are included in. in spelledForExpandedSlow()
82 // in TargetFile, we that slice of the arg, i.e. their spelling range. in spelledForExpandedSlow()
89 auto DecFirst = SM.getDecomposedLoc(First); in spelledForExpandedSlow()
90 auto DecLast = SM.getDecomposedLoc(Last); in spelledForExpandedSlow()
91 auto &ExpFirst = SM.getSLocEntry(DecFirst.first).getExpansion(); in spelledForExpandedSlow()
92 auto &ExpLast = SM.getSLocEntry(DecLast.first).getExpansion(); in spelledForExpandedSlow()
97 // (They may still have different FileIDs - an arg can have >1 chunks!) in spelledForExpandedSlow()
103 // The token `a` is wrapped in 4 arg-expansions, we only want to unwrap 2. in spelledForExpandedSlow()
106 auto ExpFileID = SM.getFileID(ExpFirst.getExpansionLocStart()); in spelledForExpandedSlow()
116 // If this overlaps Prev or Next, then no range is possible. in spelledForExpandedSlow()
119 auto DecFirst = SM.getDecomposedExpansionLoc(Candidate.getBegin()); in spelledForExpandedSlow()
120 auto DecLast = SM.getDecomposedExpansionLoc(Candidate.getEnd()); in spelledForExpandedSlow()
121 // Can end up in the wrong file due to bad input or token-pasting shenanigans. in spelledForExpandedSlow()
127 auto Dec = SM.getDecomposedLoc(SM.getExpansionRange(Prev).getBegin()); in spelledForExpandedSlow()
132 auto Dec = SM.getDecomposedLoc(SM.getExpansionRange(Next).getEnd()); in spelledForExpandedSlow()
136 // Now we know that Candidate is a file range that covers [First, Last] in spelledForExpandedSlow()
161 FileRange syntax::Token::range(const SourceManager &SM) const { in range() function in syntax::Token
169 FileRange syntax::Token::range(const SourceManager &SM, in range() function in syntax::Token
172 auto F = First.range(SM); in range()
173 auto L = Last.range(SM); in range()
213 return OS << llvm::formatv("FileRange(file = {0}, offsets = {1}-{2})", in operator <<()
229 // No-op if the index is already created. in indexExpandedTokens()
245 // Quick lookup if `R` is a token range. in expandedTokens()
248 const auto B = ExpandedTokIndex.find(R.getBegin()); in expandedTokens()
249 const auto E = ExpandedTokIndex.find(R.getEnd()); in expandedTokens()
251 const Token *L = ExpandedTokens.data() + B->getSecond(); in expandedTokens()
252 // Add 1 to End to make a half-open range. in expandedTokens()
253 const Token *R = ExpandedTokens.data() + E->getSecond() + 1; in expandedTokens()
260 // required range. in expandedTokens()
276 auto FileIt = Files.find( in spelledForExpandedToken()
277 SourceMgr->getFileID(SourceMgr->getExpansionLoc(Expanded->location()))); in spelledForExpandedToken()
280 const MarkedFile &File = FileIt->second; in spelledForExpandedToken()
282 unsigned ExpandedIndex = Expanded - ExpandedTokens.data(); in spelledForExpandedToken()
284 auto It = llvm::partition_point(File.Mappings, [&](const Mapping &M) { in spelledForExpandedToken()
290 return {&File.SpelledTokens[ExpandedIndex - File.BeginExpanded], in spelledForExpandedToken()
293 --It; // 'It' now points to last mapping that started before our token. in spelledForExpandedToken()
296 if (ExpandedIndex < It->EndExpanded) in spelledForExpandedToken()
297 return {&File.SpelledTokens[It->BeginSpelled], /*Mapping=*/&*It}; in spelledForExpandedToken()
302 &File.SpelledTokens[It->EndSpelled + (ExpandedIndex - It->EndExpanded)], in spelledForExpandedToken()
310 unsigned SpelledI = Spelled - F.SpelledTokens.data(); in mappingStartingBeforeSpelled()
313 auto It = llvm::partition_point(F.Mappings, [SpelledI](const Mapping &M) { in mappingStartingBeforeSpelled()
318 --It; in mappingStartingBeforeSpelled()
326 const auto &File = fileForSpelled(Spelled); in expandedForSpelled()
328 auto *FrontMapping = mappingStartingBeforeSpelled(File, &Spelled.front()); in expandedForSpelled()
329 unsigned SpelledFrontI = &Spelled.front() - File.SpelledTokens.data(); in expandedForSpelled()
336 } else if (SpelledFrontI < FrontMapping->EndSpelled) { in expandedForSpelled()
338 if (SpelledFrontI != FrontMapping->BeginSpelled) { in expandedForSpelled()
343 ExpandedBegin = FrontMapping->BeginExpanded; in expandedForSpelled()
348 FrontMapping->EndExpanded + (SpelledFrontI - FrontMapping->EndSpelled); in expandedForSpelled()
351 auto *BackMapping = mappingStartingBeforeSpelled(File, &Spelled.back()); in expandedForSpelled()
352 unsigned SpelledBackI = &Spelled.back() - File.SpelledTokens.data(); in expandedForSpelled()
358 } else if (SpelledBackI < BackMapping->EndSpelled) { in expandedForSpelled()
360 if (SpelledBackI + 1 != BackMapping->EndSpelled) { in expandedForSpelled()
364 ExpandedEnd = BackMapping->EndExpanded; in expandedForSpelled()
368 BackMapping->EndExpanded + (SpelledBackI - BackMapping->EndSpelled) + 1; in expandedForSpelled()
381 auto It = Files.find(FID); in spelledTokens()
383 return It->second.SpelledTokens; in spelledTokens()
389 const auto *Tok = llvm::partition_point( in spelledTokenContaining()
390 spelledTokens(SourceMgr->getFileID(Loc)), in spelledTokenContaining()
392 if (!Tok || Loc < Tok->location()) in spelledTokenContaining()
407 // the range. in spelledForExpanded()
411 // Mapping an empty range is ambiguous in case of empty mappings at either end in spelledForExpanded()
412 // of the range, bail out in that case. in spelledForExpanded()
417 auto [FirstSpelled, FirstMapping] = spelledForExpandedToken(First); in spelledForExpanded()
418 auto [LastSpelled, LastMapping] = spelledForExpandedToken(Last); in spelledForExpanded()
420 FileID FID = SourceMgr->getFileID(FirstSpelled->location()); in spelledForExpanded()
421 // FIXME: Handle multi-file changes by trying to map onto a common root. in spelledForExpanded()
422 if (FID != SourceMgr->getFileID(LastSpelled->location())) in spelledForExpanded()
425 const MarkedFile &File = Files.find(FID)->second; in spelledForExpanded()
427 // If the range is within one macro argument, the result may be only part of a in spelledForExpanded()
428 // Mapping. We must use the general (SourceManager-based) algorithm. in spelledForExpanded()
430 SourceMgr->isMacroArgExpansion(First->location()) && in spelledForExpanded()
431 SourceMgr->isMacroArgExpansion(Last->location())) { in spelledForExpanded()
435 : (First - 1)->location(); in spelledForExpanded()
438 : (Last + 1)->location(); in spelledForExpanded()
439 SourceRange Range = spelledForExpandedSlow( in spelledForExpanded() local
440 First->location(), Last->location(), Prev, Next, FID, *SourceMgr); in spelledForExpanded()
441 if (Range.isInvalid()) in spelledForExpanded()
443 return getTokensCovering(File.SpelledTokens, Range, *SourceMgr); in spelledForExpanded()
448 unsigned FirstExpanded = Expanded.begin() - ExpandedTokens.data(); in spelledForExpanded()
449 unsigned LastExpanded = Expanded.end() - ExpandedTokens.data(); in spelledForExpanded()
450 if (FirstMapping && FirstExpanded != FirstMapping->BeginExpanded) in spelledForExpanded()
452 if (LastMapping && LastMapping->EndExpanded != LastExpanded) in spelledForExpanded()
455 FirstMapping ? File.SpelledTokens.data() + FirstMapping->BeginSpelled in spelledForExpanded()
457 LastMapping ? File.SpelledTokens.data() + LastMapping->EndSpelled in spelledForExpanded()
475 auto FileIt = Files.find(SourceMgr->getFileID(Spelled.front().location())); in fileForSpelled()
477 const auto &File = FileIt->second; in fileForSpelled()
481 "Tokens not in spelled range"); in fileForSpelled()
483 auto T1 = Spelled.back().location(); in fileForSpelled()
484 auto T2 = File.SpelledTokens.back().location(); in fileForSpelled()
493 const auto &File = fileForSpelled(*Spelled); in expansionStartingAt()
495 unsigned SpelledIndex = Spelled - File.SpelledTokens.data(); in expansionStartingAt()
496 auto M = llvm::partition_point(File.Mappings, [&](const Mapping &M) { in expansionStartingAt()
499 if (M == File.Mappings.end() || M->BeginSpelled != SpelledIndex) in expansionStartingAt()
508 const auto &File = fileForSpelled(Spelled); in expansionsOverlapping()
510 // Find the first overlapping range, and then copy until we stop overlapping. in expansionsOverlapping()
511 unsigned SpelledBeginIndex = Spelled.begin() - File.SpelledTokens.data(); in expansionsOverlapping()
512 unsigned SpelledEndIndex = Spelled.end() - File.SpelledTokens.data(); in expansionsOverlapping()
513 auto M = llvm::partition_point(File.Mappings, [&](const Mapping &M) { in expansionsOverlapping()
517 for (; M != File.Mappings.end() && M->BeginSpelled < SpelledEndIndex; ++M) in expansionsOverlapping()
527 auto *Right = llvm::partition_point( in spelledTokensTouching()
529 bool AcceptRight = Right != Tokens.end() && Right->location() <= Loc; in spelledTokensTouching()
531 Right != Tokens.begin() && (Right - 1)->endLocation() >= Loc; in spelledTokensTouching()
532 return llvm::ArrayRef(Right - (AcceptLeft ? 1 : 0), in spelledTokensTouching()
562 auto FileIt = Files.find(FID); in macroExpansions()
564 auto &File = FileIt->second; in macroExpansions()
566 auto &Spelled = File.SpelledTokens; in macroExpansions()
567 for (auto Mapping : File.Mappings) { in macroExpansions()
569 if (Token->kind() == tok::TokenKind::identifier) in macroExpansions()
580 auto AddToken = [&](clang::Token T) { in tokenize()
591 auto SrcBuffer = SM.getBufferData(FR.file()); in tokenize()
602 // it iff it starts within the range we are interested in. in tokenize()
625 SourceRange Range, const MacroArgs *Args) override { in MacroExpands() argument
628 const auto &SM = Collector->PP.getSourceManager(); in MacroExpands()
629 // Only record top-level expansions that directly produce expanded tokens. in MacroExpands()
631 // - the macro use is inside a macro body, in MacroExpands()
632 // - the macro appears in an argument to another macro. in MacroExpands()
641 // The *last* token of any top-level macro expansion must be in a file. in MacroExpands()
643 if (!Range.getEnd().isFileID()) in MacroExpands()
646 // top-level. in MacroExpands()
648 !SM.isBeforeInTranslationUnit(LastExpansionEnd, Range.getEnd())) in MacroExpands()
654 if (!Range.getBegin().isFileID()) { in MacroExpands()
655 Range.setBegin(SM.getExpansionLoc(Range.getBegin())); in MacroExpands()
656 assert(Collector->Expansions.count(Range.getBegin()) && in MacroExpands()
660 Collector->Expansions[Range.getBegin()] = Range.getEnd(); in MacroExpands()
661 LastExpansionEnd = Range.getEnd(); in MacroExpands()
673 /// - for each token, figures out if it is a part of an expanded token stream,
675 /// - records mappings from the spelled to expanded token ranges, e.g. for macro
678 /// - #include directives,
679 /// - #pragma, #line and other PP directives,
680 /// - skipped pp regions,
681 /// - ...
688 DEBUG_WITH_TYPE("collect-tokens", llvm::dbgs() in TokenCollector()
691 this->PP.getSourceManager()) in TokenCollector()
699 auto CB = std::make_unique<CollectPPExpansions>(*this); in TokenCollector()
700 this->Collector = CB.get(); in TokenCollector()
726 while (NextExpanded < Result.ExpandedTokens.size() - 1 /* eof */) { in build()
739 for (const auto &File : Result.Files) in build()
743 for (auto &pair : Result.Files) { in build()
744 auto &mappings = pair.second.Mappings; in build()
769 const auto &SpelledTokens = Result.Files[File].SpelledTokens; in discard()
770 auto &NextSpelled = this->NextSpelled[File]; in discard()
775 // be positioned within the file's expanded-token range (at the end). in discard()
780 auto FlushMapping = [&, this] { in discard()
815 const auto &SpelledTokens = Result.Files[File].SpelledTokens; in advance()
816 auto &NextSpelled = this->NextSpelled[File]; in advance()
830 auto End = CollectedExpansions.lookup(Expansion); in advance()
853 // advance() is supposed to consume at least one token - if not, we crash.
856 // Show the failed-to-map token in context. in diagnoseAdvanceFailure()
857 for (unsigned I = (NextExpanded < 10) ? 0 : NextExpanded - 10; in diagnoseAdvanceFailure()
871 const auto &Tok = Result.ExpandedTokens[I]; in buildSpelledTokens()
872 auto FID = SM.getFileID(SM.getExpansionLoc(Tok.location())); in buildSpelledTokens()
873 auto It = Result.Files.try_emplace(FID); in buildSpelledTokens()
874 TokenBuffer::MarkedFile &File = It.first->second; in buildSpelledTokens()
876 // The eof token should not be considered part of the main-file's range. in buildSpelledTokens()
897 Collector->disable(); in consume()
914 auto PrintToken = [this](const syntax::Token &T) -> std::string { in dumpForTests()
920 auto DumpTokens = [this, &PrintToken](llvm::raw_ostream &OS, in dumpForTests()
944 for (const auto &F : Files) in dumpForTests()
949 const MarkedFile &File = Files.find(ID)->second; in dumpForTests()
950 auto Entry = SourceMgr->getFileEntryRefForID(ID); in dumpForTests()
953 std::string Path = llvm::sys::path::convert_to_slash(Entry->getName()); in dumpForTests()
964 for (auto &M : File.Mappings) { in dumpForTests()