1 //===--- BreakableToken.cpp - Format C++ code -----------------------------===//
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
13 //===----------------------------------------------------------------------===//
23 #define DEBUG_TYPE "format-token-breaker"
78   unsigned MaxSplit = ColumnLimit - ContentStartColumn + 1;  in getCommentSplit()
105   static const auto kNumberedListRegexp = llvm::Regex("^[1-9][0-9]?\\.");  in getCommentSplit()
108     // If a line-comment ends with `\`, the next line continues the comment,  in getCommentSplit()
110     // -Wcomment.  in getCommentSplit()
123     // as a numbered list, which would prevent re-flowing in subsequent passes.  in getCommentSplit()
153     // adaptStartOfLine will break after lines starting with /** if the comment  in getCommentSplit()
157     if (SpaceOffset == 1 && Text[SpaceOffset - 1] == '*')  in getCommentSplit()
165                                  AfterCut.begin() - BeforeCut.end());  in getCommentSplit()
178   unsigned MaxSplit = ColumnLimit - UsedColumns;  in getStringSplit()
223   return Content.starts_with("clang-format on") ||  in switchesFormatting()
224          Content.starts_with("clang-format off");  in switchesFormatting()
232   // - RemainingTokenColumns is the original number of columns, 10;  in getLengthAfterCompression()
233   // - Split is (4, 2), denoting the two spaces between the two words;  in getLengthAfterCompression()
241   return RemainingTokenColumns + 1 - Split.second;  in getLengthAfterCompression()
276       Prefix.size(), Tok.TokenText.size() - Prefix.size() - Postfix.size());  in BreakableStringLiteral()
283                         ColumnLimit - Postfix.size(), Style.TabWidth, Encoding);  in getSplit()
347   // Following lines are indented by the width of the brace and space if any.  in BreakableStringLiteralUsingOperators()
348   ContinuationIndent = BracesNeeded ? LeftBraceQuote.size() - 1 : 0;  in BreakableStringLiteralUsingOperators()
353     ContinuationIndent -= 2;  in BreakableStringLiteralUsingOperators()
371                  : (BracesNeeded ? static_cast<int>(LeftBraceQuote.size()) - 1  in getContentStartColumn()
404       Tok, /*Offset=*/Tok.TokenText.size() - 1, /*ReplaceChars=*/1,  in updateAfterBroken()
416 unsigned BreakableComment::getLineCount() const { return Lines.size(); }  in getLineCount()
422   // Don't break lines matching the comment pragmas regex.  in getSplit()
439       Text.data() - tokenAt(LineIndex).TokenText.data() + Split.first;  in compressWhitespace()
452   // Lines starting with '@' or '\' commonly have special meaning.  in mayReflowContent()
453   // Lines starting with '-', '-#', '+' or '*' are bulleted/numbered lists.  in mayReflowContent()
456        {"@", "\\", "TODO", "FIXME", "XXX", "-# ", "- ", "+ ", "* "}) {  in mayReflowContent()
466   static const auto kNumberedListRegexp = llvm::Regex("^[1-9][0-9]?\\. ");  in mayReflowContent()
472   // non-punctuation.  in mayReflowContent()
475          // Note that this is UTF-8 safe, since if isPunctuation(Content[0]) is  in mayReflowContent()
492   TokenText.substr(2, TokenText.size() - 4)  in BreakableBlockComment()
493       .split(Lines, UseCRLF ? "\r\n" : "\n");  in BreakableBlockComment()
495   int IndentDelta = StartColumn - OriginalStartColumn;  in BreakableBlockComment()
496   Content.resize(Lines.size());  in BreakableBlockComment()
497   Content[0] = Lines[0];  in BreakableBlockComment()
498   ContentColumn.resize(Lines.size());  in BreakableBlockComment()
501   Tokens.resize(Lines.size());  in BreakableBlockComment()
502   for (size_t i = 1; i < Lines.size(); ++i)  in BreakableBlockComment()
514   if (Lines.size() >= 2 && Content[1].starts_with("**") &&  in BreakableBlockComment()
520   if (Lines.size() == 1 && !FirstInLine) {  in BreakableBlockComment()
523     // lines with the first one.  in BreakableBlockComment()
543   for (size_t i = 1, e = Lines.size(); i < e; ++i) {  in BreakableBlockComment()
550         // Align the star in the last '*/' with the stars on the previous lines.  in BreakableBlockComment()
554         // For all other lines, set the start column to 0 if they're empty, so  in BreakableBlockComment()
563     // For all other lines, adjust the line to exclude the star and  in BreakableBlockComment()
580     if ((Lines[0] == "*" || Lines[0].starts_with("* ")) && Lines.size() > 1) {  in BreakableBlockComment()
583     } else if (Lines[0].starts_with("* ") && Lines.size() == 1) {  in BreakableBlockComment()
584       // Detect a long single-line comment, like:  in BreakableBlockComment()
589           encoding::columnWidthWithTabs(Lines[0], ContentColumn[0],  in BreakableBlockComment()
599     for (size_t i = 0; i < Lines.size(); ++i) {  in BreakableBlockComment()
602                    << "IN=" << (Content[i].data() - Lines[i].data()) << "\n";  in BreakableBlockComment()
610   // Don't break lines matching the comment pragmas regex.  in getSplit()
623   // trimming the trailing whitespace. The backslash will be re-added later when  in adjustWhitespace()
625   size_t EndOfPreviousLine = Lines[LineIndex - 1].size();  in adjustWhitespace()
626   if (InPPDirective && Lines[LineIndex - 1].ends_with("\\"))  in adjustWhitespace()
627     --EndOfPreviousLine;  in adjustWhitespace()
629   // Calculate the end of the non-whitespace text in the previous line.  in adjustWhitespace()
631       Lines[LineIndex - 1].find_last_not_of(Blanks, EndOfPreviousLine);  in adjustWhitespace()
636   // Calculate the start of the non-whitespace text in the current line.  in adjustWhitespace()
637   size_t StartOfLine = Lines[LineIndex].find_first_not_of(Blanks);  in adjustWhitespace()
639     StartOfLine = Lines[LineIndex].size();  in adjustWhitespace()
641   StringRef Whitespace = Lines[LineIndex].substr(0, StartOfLine);  in adjustWhitespace()
642   // Adjust Lines to only contain relevant text.  in adjustWhitespace()
644       Content[LineIndex - 1].data() - Lines[LineIndex - 1].data();  in adjustWhitespace()
645   Content[LineIndex - 1] = Lines[LineIndex - 1].substr(  in adjustWhitespace()
646       PreviousContentOffset, EndOfPreviousLine - PreviousContentOffset);  in adjustWhitespace()
647   Content[LineIndex] = Lines[LineIndex].substr(StartOfLine);  in adjustWhitespace()
649   // Adjust the start column uniformly across all lines.  in adjustWhitespace()
670   if (LineIndex + 1 == Lines.size()) {  in getRemainingLength()
675       bool HasDecoration = Lines[LineIndex].ltrim().starts_with(Decoration);  in getRemainingLength()
677         LineLength -= Decoration.size();  in getRemainingLength()
718   // the lines except for the last one, where the star in "*/" acts as a  in insertBreak()
721   if (LineIndex + 1 == Lines.size() &&  in insertBreak()
726       LocalIndentAtLineBreak -= 2;  in insertBreak()
731       Text.data() - tokenAt(LineIndex).TokenText.data() + Split.first;  in insertBreak()
739       /*Spaces=*/LocalIndentAtLineBreak + ContentIndent -  in insertBreak()
752     unsigned PreviousContentIndent = getContentIndent(LineIndex - 1);  in getReflowSplit()
765          Lines[0].substr(1).find_first_not_of(Blanks) != StringRef::npos;  in introducesBreakBeforeToken()
772   assert(Tokens[LineIndex - 1] == Tokens[LineIndex] &&  in reflow()
776   unsigned WhitespaceOffsetInToken = Content[LineIndex - 1].data() +  in reflow()
777                                      Content[LineIndex - 1].size() -  in reflow()
778                                      tokenAt(LineIndex).TokenText.data();  in reflow()
779   unsigned WhitespaceLength = TrimmedContent.data() -  in reflow()
780                               tokenAt(LineIndex).TokenText.data() -  in reflow()
797       size_t BreakLength = Lines[0].substr(1).find_first_not_of(Blanks);  in adaptStartOfLine()
809     if (LineIndex + 1 == Lines.size()) {  in adaptStartOfLine()
816       // For other empty lines, if we do have a decoration, adapt it to not  in adaptStartOfLine()
826   unsigned WhitespaceOffsetInToken = Content[LineIndex - 1].data() +  in adaptStartOfLine()
827                                      Content[LineIndex - 1].size() -  in adaptStartOfLine()
828                                      tokenAt(LineIndex).TokenText.data();  in adaptStartOfLine()
829   unsigned WhitespaceLength = Content[LineIndex].data() -  in adaptStartOfLine()
830                               tokenAt(LineIndex).TokenText.data() -  in adaptStartOfLine()
834       InPPDirective, /*Newlines=*/1, ContentColumn[LineIndex] - Prefix.size());  in adaptStartOfLine()
846       return Split(TrimmedLine.size(), Line.size() - TrimmedLine.size());  in getSplitAfterLastLine()
856   if (Lines[LineIndex].ltrim(Blanks).starts_with("*"))  in mayReflow()
857     IndentContent = Lines[LineIndex].ltrim(Blanks).substr(1);  in mayReflow()
872   // applied in all following lines  in BreakableLineCommentSection()
875        CurrentTok && CurrentTok->is(TT_LineComment);  in BreakableLineCommentSection()
876        CurrentTok = CurrentTok->Next) {  in BreakableLineCommentSection()
878     StringRef TokenText(CurrentTok->TokenText);  in BreakableLineCommentSection()
881     size_t FirstLineIndex = Lines.size();  in BreakableLineCommentSection()
882     TokenText.split(Lines, "\n");  in BreakableLineCommentSection()
883     Content.resize(Lines.size());  in BreakableLineCommentSection()
884     ContentColumn.resize(Lines.size());  in BreakableLineCommentSection()
885     PrefixSpaceChange.resize(Lines.size());  in BreakableLineCommentSection()
886     Tokens.resize(Lines.size());  in BreakableLineCommentSection()
887     Prefix.resize(Lines.size());  in BreakableLineCommentSection()
888     OriginalPrefix.resize(Lines.size());  in BreakableLineCommentSection()
889     for (size_t i = FirstLineIndex, e = Lines.size(); i < e; ++i) {  in BreakableLineCommentSection()
890       Lines[i] = Lines[i].ltrim(Blanks);  in BreakableLineCommentSection()
891       StringRef IndentPrefix = getLineCommentIndentPrefix(Lines[i], Style);  in BreakableLineCommentSection()
898         assert(Lines[i].size() > IndentPrefix.size());  in BreakableLineCommentSection()
899         const char FirstCommentChar = Lines[i][IndentPrefix.size()];  in BreakableLineCommentSection()
903                 Lines[i].substr(IndentPrefix.size(), FirstCharByteSize),  in BreakableLineCommentSection()
907         // In C-like comments, add a space before #. For example this is useful  in BreakableLineCommentSection()
923       // are to be added or removed, all lines after that just get only the  in BreakableLineCommentSection()
928                         OriginalPrefix[i - 1].rtrim(Blanks)) {  in BreakableLineCommentSection()
929         if (SpacesInPrefix < Minimum && Lines[i].size() > IndentPrefix.size() &&  in BreakableLineCommentSection()
931           FirstLineSpaceChange = Minimum - SpacesInPrefix;  in BreakableLineCommentSection()
935               Style.SpacesInLineCommentPrefix.Maximum - SpacesInPrefix;  in BreakableLineCommentSection()
941       if (Lines[i].size() != IndentPrefix.size()) {  in BreakableLineCommentSection()
946               Minimum - (SpacesInPrefix + PrefixSpaceChange[i]);  in BreakableLineCommentSection()
949         assert(Lines[i].size() > IndentPrefix.size());  in BreakableLineCommentSection()
950         const auto FirstNonSpace = Lines[i][IndentPrefix.size()];  in BreakableLineCommentSection()
965                               -PrefixSpaceChange[i], SpacesInPrefix))  in BreakableLineCommentSection()
977       Content[i] = Lines[i].substr(IndentPrefix.size());  in BreakableLineCommentSection()
982       // Calculate the end of the non-whitespace text in this line.  in BreakableLineCommentSection()
990     LineTok = CurrentTok->Next;  in BreakableLineCommentSection()
991     if (CurrentTok->Next && !CurrentTok->Next->ContinuesLineCommentSection) {  in BreakableLineCommentSection()
995       // would split the contents of the enum into two unwrapped lines in this  in BreakableLineCommentSection()
1033       Text.data() - tokenAt(LineIndex).TokenText.data() + Split.first;  in insertBreak()
1038       /*Spaces=*/ContentColumn[LineIndex] - Prefix[LineIndex].size());  in insertBreak()
1057   if (LineIndex > 0 && Tokens[LineIndex] != Tokens[LineIndex - 1]) {  in reflow()
1075     unsigned Offset = Lines[LineIndex - 1].data() +  in reflow()
1076                       Lines[LineIndex - 1].size() -  in reflow()
1077                       tokenAt(LineIndex - 1).TokenText.data();  in reflow()
1081         Lines[LineIndex].data() - tokenAt(LineIndex).TokenText.data() - Offset;  in reflow()
1092       Lines[LineIndex].data() - tokenAt(LineIndex).TokenText.data();  in reflow()
1094       Content[LineIndex].data() - Lines[LineIndex].data();  in reflow()
1112   if (LineIndex > 0 && Tokens[LineIndex] != Tokens[LineIndex - 1]) {  in adaptStartOfLine()
1119         ContentColumn[LineIndex] -  in adaptStartOfLine()
1120         (Content[LineIndex].data() - Lines[LineIndex].data()) +  in adaptStartOfLine()
1121         (OriginalPrefix[LineIndex].size() - Prefix[LineIndex].size());  in adaptStartOfLine()
1136     const auto SpacesToRemove = -std::min(PrefixSpaceChange[LineIndex], 0);  in adaptStartOfLine()
1139         tokenAt(LineIndex), OriginalPrefix[LineIndex].size() - SpacesToRemove,  in adaptStartOfLine()
1147     State.NextToken = LastLineTok->Next;  in updateNextToken()
1155   if (Lines[LineIndex].starts_with("//"))  in mayReflow()
1156     IndentContent = Lines[LineIndex].substr(2);  in mayReflow()
1157   // FIXME: Decide whether we want to reflow non-regular indents:  in mayReflow()
1159   // OriginalPrefix[LineIndex-1]. That means we don't reflow  in mayReflow()
1166          OriginalPrefix[LineIndex] == OriginalPrefix[LineIndex - 1];  in mayReflow()