Lines Matching +full:data +full:- +full:mapping
1 //===- Tokens.cpp - collect tokens from preprocessing ---------------------===//
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
63 // ~~ -> a1
64 // ~~ -> a2
65 // ~~~~~~~~~ -> a1 a2
71 // - identifying which spelled range covers the expanded tokens in spelledForExpandedSlow()
72 // - validating that this range doesn't cover any extra tokens (First/Last) in spelledForExpandedSlow()
97 // (They may still have different FileIDs - an arg can have >1 chunks!) in spelledForExpandedSlow()
103 // The token `a` is wrapped in 4 arg-expansions, we only want to unwrap 2. in spelledForExpandedSlow()
121 // Can end up in the wrong file due to bad input or token-pasting shenanigans. in spelledForExpandedSlow()
213 return OS << llvm::formatv("FileRange(file = {0}, offsets = {1}-{2})", in operator <<()
229 // No-op if the index is already created. in indexExpandedTokens()
251 const Token *L = ExpandedTokens.data() + B->getSecond(); in expandedTokens()
252 // Add 1 to End to make a half-open range. in expandedTokens()
253 const Token *R = ExpandedTokens.data() + E->getSecond() + 1; in expandedTokens()
270 std::pair<const syntax::Token *, const TokenBuffer::Mapping *>
273 assert(ExpandedTokens.data() <= Expanded && in spelledForExpandedToken()
274 Expanded < ExpandedTokens.data() + ExpandedTokens.size()); in spelledForExpandedToken()
277 SourceMgr->getFileID(SourceMgr->getExpansionLoc(Expanded->location()))); in spelledForExpandedToken()
280 const MarkedFile &File = FileIt->second; in spelledForExpandedToken()
282 unsigned ExpandedIndex = Expanded - ExpandedTokens.data(); in spelledForExpandedToken()
283 // Find the first mapping that produced tokens after \p Expanded. in spelledForExpandedToken()
284 auto It = llvm::partition_point(File.Mappings, [&](const Mapping &M) { in spelledForExpandedToken()
287 // Our token could only be produced by the previous mapping. in spelledForExpandedToken()
289 // No previous mapping, no need to modify offsets. in spelledForExpandedToken()
290 return {&File.SpelledTokens[ExpandedIndex - File.BeginExpanded], in spelledForExpandedToken()
291 /*Mapping=*/nullptr}; in spelledForExpandedToken()
293 --It; // 'It' now points to last mapping that started before our token. in spelledForExpandedToken()
295 // Check if the token is part of the mapping. in spelledForExpandedToken()
296 if (ExpandedIndex < It->EndExpanded) in spelledForExpandedToken()
297 return {&File.SpelledTokens[It->BeginSpelled], /*Mapping=*/&*It}; in spelledForExpandedToken()
299 // Not part of the mapping, use the index from previous mapping to compute the in spelledForExpandedToken()
302 &File.SpelledTokens[It->EndSpelled + (ExpandedIndex - It->EndExpanded)], in spelledForExpandedToken()
303 /*Mapping=*/nullptr}; in spelledForExpandedToken()
306 const TokenBuffer::Mapping *
309 assert(F.SpelledTokens.data() <= Spelled); in mappingStartingBeforeSpelled()
310 unsigned SpelledI = Spelled - F.SpelledTokens.data(); in mappingStartingBeforeSpelled()
313 auto It = llvm::partition_point(F.Mappings, [SpelledI](const Mapping &M) { in mappingStartingBeforeSpelled()
318 --It; in mappingStartingBeforeSpelled()
329 unsigned SpelledFrontI = &Spelled.front() - File.SpelledTokens.data(); in expandedForSpelled()
333 // No mapping that starts before the first token of Spelled, we don't have in expandedForSpelled()
336 } else if (SpelledFrontI < FrontMapping->EndSpelled) { in expandedForSpelled()
337 // This mapping applies to Spelled tokens. in expandedForSpelled()
338 if (SpelledFrontI != FrontMapping->BeginSpelled) { in expandedForSpelled()
339 // Spelled tokens don't cover the entire mapping, returning empty result. in expandedForSpelled()
342 // Spelled tokens start at the beginning of this mapping. in expandedForSpelled()
343 ExpandedBegin = FrontMapping->BeginExpanded; in expandedForSpelled()
345 // Spelled tokens start after the mapping ends (they start in the hole in expandedForSpelled()
346 // between 2 mappings, or between a mapping and end of the file). in expandedForSpelled()
348 FrontMapping->EndExpanded + (SpelledFrontI - FrontMapping->EndSpelled); in expandedForSpelled()
352 unsigned SpelledBackI = &Spelled.back() - File.SpelledTokens.data(); in expandedForSpelled()
355 // No mapping that starts before the last token of Spelled, we don't have to in expandedForSpelled()
358 } else if (SpelledBackI < BackMapping->EndSpelled) { in expandedForSpelled()
359 // This mapping applies to Spelled tokens. in expandedForSpelled()
360 if (SpelledBackI + 1 != BackMapping->EndSpelled) { in expandedForSpelled()
361 // Spelled tokens don't cover the entire mapping, returning empty result. in expandedForSpelled()
364 ExpandedEnd = BackMapping->EndExpanded; in expandedForSpelled()
366 // Spelled tokens end after the mapping ends. in expandedForSpelled()
368 BackMapping->EndExpanded + (SpelledBackI - BackMapping->EndSpelled) + 1; in expandedForSpelled()
376 return {llvm::ArrayRef(ExpandedTokens.data() + ExpandedBegin, in expandedForSpelled()
377 ExpandedTokens.data() + ExpandedEnd)}; in expandedForSpelled()
383 return It->second.SpelledTokens; in spelledTokens()
390 spelledTokens(SourceMgr->getFileID(Loc)), in spelledTokenContaining()
392 if (!Tok || Loc < Tok->location()) in spelledTokenContaining()
397 std::string TokenBuffer::Mapping::str() const { in str()
411 // Mapping an empty range is ambiguous in case of empty mappings at either end in spelledForExpanded()
420 FileID FID = SourceMgr->getFileID(FirstSpelled->location()); in spelledForExpanded()
421 // FIXME: Handle multi-file changes by trying to map onto a common root. in spelledForExpanded()
422 if (FID != SourceMgr->getFileID(LastSpelled->location())) in spelledForExpanded()
425 const MarkedFile &File = Files.find(FID)->second; in spelledForExpanded()
428 // Mapping. We must use the general (SourceManager-based) algorithm. in spelledForExpanded()
430 SourceMgr->isMacroArgExpansion(First->location()) && in spelledForExpanded()
431 SourceMgr->isMacroArgExpansion(Last->location())) { in spelledForExpanded()
435 : (First - 1)->location(); in spelledForExpanded()
438 : (Last + 1)->location(); in spelledForExpanded()
440 First->location(), Last->location(), Prev, Next, FID, *SourceMgr); in spelledForExpanded()
448 unsigned FirstExpanded = Expanded.begin() - ExpandedTokens.data(); in spelledForExpanded()
449 unsigned LastExpanded = Expanded.end() - ExpandedTokens.data(); in spelledForExpanded()
450 if (FirstMapping && FirstExpanded != FirstMapping->BeginExpanded) in spelledForExpanded()
452 if (LastMapping && LastMapping->EndExpanded != LastExpanded) in spelledForExpanded()
455 FirstMapping ? File.SpelledTokens.data() + FirstMapping->BeginSpelled in spelledForExpanded()
457 LastMapping ? File.SpelledTokens.data() + LastMapping->EndSpelled in spelledForExpanded()
462 const Mapping &M) const { in makeExpansion()
464 E.Spelled = llvm::ArrayRef(F.SpelledTokens.data() + M.BeginSpelled, in makeExpansion()
465 F.SpelledTokens.data() + M.EndSpelled); in makeExpansion()
466 E.Expanded = llvm::ArrayRef(ExpandedTokens.data() + M.BeginExpanded, in makeExpansion()
467 ExpandedTokens.data() + M.EndExpanded); in makeExpansion()
475 auto FileIt = Files.find(SourceMgr->getFileID(Spelled.front().location())); in fileForSpelled()
477 const auto &File = FileIt->second; in fileForSpelled()
478 assert(File.SpelledTokens.data() <= Spelled.data() && in fileForSpelled()
480 (File.SpelledTokens.data() + File.SpelledTokens.size()) && in fileForSpelled()
495 unsigned SpelledIndex = Spelled - File.SpelledTokens.data(); in expansionStartingAt()
496 auto M = llvm::partition_point(File.Mappings, [&](const Mapping &M) { in expansionStartingAt()
499 if (M == File.Mappings.end() || M->BeginSpelled != SpelledIndex) in expansionStartingAt()
511 unsigned SpelledBeginIndex = Spelled.begin() - File.SpelledTokens.data(); in expansionsOverlapping()
512 unsigned SpelledEndIndex = Spelled.end() - File.SpelledTokens.data(); in expansionsOverlapping()
513 auto M = llvm::partition_point(File.Mappings, [&](const Mapping &M) { in expansionsOverlapping()
517 for (; M != File.Mappings.end() && M->BeginSpelled < SpelledEndIndex; ++M) in expansionsOverlapping()
529 bool AcceptRight = Right != Tokens.end() && Right->location() <= Loc; in spelledTokensTouching()
531 Right != Tokens.begin() && (Right - 1)->endLocation() >= Loc; in spelledTokensTouching()
532 return llvm::ArrayRef(Right - (AcceptLeft ? 1 : 0), in spelledTokensTouching()
564 auto &File = FileIt->second; in macroExpansions()
567 for (auto Mapping : File.Mappings) { in macroExpansions() local
568 const syntax::Token *Token = &Spelled[Mapping.BeginSpelled]; in macroExpansions()
569 if (Token->kind() == tok::TokenKind::identifier) in macroExpansions()
592 Lexer L(SM.getLocForStartOfFile(FR.file()), LO, SrcBuffer.data(), in tokenize()
593 SrcBuffer.data() + FR.beginOffset(), in tokenize()
596 SrcBuffer.data() + SrcBuffer.size()); in tokenize()
628 const auto &SM = Collector->PP.getSourceManager(); in MacroExpands()
629 // Only record top-level expansions that directly produce expanded tokens. in MacroExpands()
631 // - the macro use is inside a macro body, in MacroExpands()
632 // - the macro appears in an argument to another macro. in MacroExpands()
641 // The *last* token of any top-level macro expansion must be in a file. in MacroExpands()
646 // top-level. in MacroExpands()
652 // we'll create a merged mapping for A + B by overwriting the endpoint for in MacroExpands()
656 assert(Collector->Expansions.count(Range.getBegin()) && in MacroExpands()
660 Collector->Expansions[Range.getBegin()] = Range.getEnd(); in MacroExpands()
673 /// - for each token, figures out if it is a part of an expanded token stream,
675 /// - records mappings from the spelled to expanded token ranges, e.g. for macro
678 /// - #include directives,
679 /// - #pragma, #line and other PP directives,
680 /// - skipped pp regions,
681 /// - ...
688 DEBUG_WITH_TYPE("collect-tokens", llvm::dbgs() in TokenCollector()
691 this->PP.getSourceManager()) in TokenCollector()
700 this->Collector = CB.get(); in TokenCollector()
726 while (NextExpanded < Result.ExpandedTokens.size() - 1 /* eof */) { in build()
730 // Create mapping for a contiguous run of expanded tokens. in build()
745 assert(llvm::is_sorted(mappings, [](const TokenBuffer::Mapping &M1, in build()
746 const TokenBuffer::Mapping &M2) { in build()
761 // the NextExpanded token, and creates an empty mapping for them.
770 auto &NextSpelled = this->NextSpelled[File]; in discard()
772 TokenBuffer::Mapping Mapping; in discard() local
773 Mapping.BeginSpelled = NextSpelled; in discard()
774 // When dropping trailing tokens from a file, the empty mapping should in discard()
775 // be positioned within the file's expanded-token range (at the end). in discard()
776 Mapping.BeginExpanded = Mapping.EndExpanded = in discard()
779 // FlushMapping() emits the current mapping and starts a new one. in discard()
781 Mapping.EndSpelled = NextSpelled; in discard()
782 if (Mapping.BeginSpelled != Mapping.EndSpelled) in discard()
783 Result.Files[File].Mappings.push_back(Mapping); in discard()
784 Mapping.BeginSpelled = NextSpelled; in discard()
789 // If we know mapping bounds at [NextSpelled, KnownEnd] (macro expansion) in discard()
790 // then we want to partition our (empty) mapping. in discard()
809 // Increases NextExpanded and NextSpelled by at least one, and adds a mapping
810 // (unless this is a run of file tokens, which we represent with no mapping).
816 auto &NextSpelled = this->NextSpelled[File]; in advance()
827 // We need no mapping for file tokens copied to the expanded stream. in advance()
833 // Mapping starts here... in advance()
834 TokenBuffer::Mapping Mapping; in advance() local
835 Mapping.BeginExpanded = NextExpanded; in advance()
836 Mapping.BeginSpelled = NextSpelled; in advance()
847 Mapping.EndExpanded = NextExpanded; in advance()
848 Mapping.EndSpelled = NextSpelled; in advance()
849 Result.Files[File].Mappings.push_back(Mapping); in advance()
853 // advance() is supposed to consume at least one token - if not, we crash.
856 // Show the failed-to-map token in context. in diagnoseAdvanceFailure()
857 for (unsigned I = (NextExpanded < 10) ? 0 : NextExpanded - 10; in diagnoseAdvanceFailure()
874 TokenBuffer::MarkedFile &File = It.first->second; in buildSpelledTokens()
876 // The eof token should not be considered part of the main-file's range. in buildSpelledTokens()
897 Collector->disable(); in consume()
914 auto PrintToken = [this](const syntax::Token &T) -> std::string { in dumpForTests()
949 const MarkedFile &File = Files.find(ID)->second; in dumpForTests()
950 auto Entry = SourceMgr->getFileEntryRefForID(ID); in dumpForTests()
953 std::string Path = llvm::sys::path::convert_to_slash(Entry->getName()); in dumpForTests()