1 //===--- BreakableToken.h - Format C++ code ---------------------*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 /// 9 /// \file 10 /// Declares BreakableToken, BreakableStringLiteral, BreakableComment, 11 /// BreakableBlockComment and BreakableLineCommentSection classes, that contain 12 /// token type-specific logic to break long lines in tokens and reflow content 13 /// between tokens. 14 /// 15 //===----------------------------------------------------------------------===// 16 17 #ifndef LLVM_CLANG_LIB_FORMAT_BREAKABLETOKEN_H 18 #define LLVM_CLANG_LIB_FORMAT_BREAKABLETOKEN_H 19 20 #include "Encoding.h" 21 #include "TokenAnnotator.h" 22 #include "WhitespaceManager.h" 23 #include "llvm/ADT/StringSet.h" 24 #include "llvm/Support/Regex.h" 25 #include <utility> 26 27 namespace clang { 28 namespace format { 29 30 /// Checks if \p Token switches formatting, like /* clang-format off */. 31 /// \p Token must be a comment. 32 bool switchesFormatting(const FormatToken &Token); 33 34 struct FormatStyle; 35 36 /// Base class for tokens / ranges of tokens that can allow breaking 37 /// within the tokens - for example, to avoid whitespace beyond the column 38 /// limit, or to reflow text. 39 /// 40 /// Generally, a breakable token consists of logical lines, addressed by a line 41 /// index. For example, in a sequence of line comments, each line comment is its 42 /// own logical line; similarly, for a block comment, each line in the block 43 /// comment is on its own logical line. 44 /// 45 /// There are two methods to compute the layout of the token: 46 /// - getRangeLength measures the number of columns needed for a range of text 47 /// within a logical line, and 48 /// - getContentStartColumn returns the start column at which we want the 49 /// content of a logical line to start (potentially after introducing a line 50 /// break). 51 /// 52 /// The mechanism to adapt the layout of the breakable token is organised 53 /// around the concept of a \c Split, which is a whitespace range that signifies 54 /// a position of the content of a token where a reformatting might be done. 55 /// 56 /// Operating with splits is divided into two operations: 57 /// - getSplit, for finding a split starting at a position, 58 /// - insertBreak, for executing the split using a whitespace manager. 59 /// 60 /// There is a pair of operations that are used to compress a long whitespace 61 /// range with a single space if that will bring the line length under the 62 /// column limit: 63 /// - getLineLengthAfterCompression, for calculating the size in columns of the 64 /// line after a whitespace range has been compressed, and 65 /// - compressWhitespace, for executing the whitespace compression using a 66 /// whitespace manager; note that the compressed whitespace may be in the 67 /// middle of the original line and of the reformatted line. 68 /// 69 /// For tokens where the whitespace before each line needs to be also 70 /// reformatted, for example for tokens supporting reflow, there are analogous 71 /// operations that might be executed before the main line breaking occurs: 72 /// - getReflowSplit, for finding a split such that the content preceding it 73 /// needs to be specially reflown, 74 /// - reflow, for executing the split using a whitespace manager, 75 /// - introducesBreakBefore, for checking if reformatting the beginning 76 /// of the content introduces a line break before it, 77 /// - adaptStartOfLine, for executing the reflow using a whitespace 78 /// manager. 79 /// 80 /// For tokens that require the whitespace after the last line to be 81 /// reformatted, for example in multiline jsdoc comments that require the 82 /// trailing '*/' to be on a line of itself, there are analogous operations 83 /// that might be executed after the last line has been reformatted: 84 /// - getSplitAfterLastLine, for finding a split after the last line that needs 85 /// to be reflown, 86 /// - replaceWhitespaceAfterLastLine, for executing the reflow using a 87 /// whitespace manager. 88 /// 89 class BreakableToken { 90 public: 91 /// Contains starting character index and length of split. 92 typedef std::pair<StringRef::size_type, unsigned> Split; 93 94 virtual ~BreakableToken() {} 95 96 /// Returns the number of lines in this token in the original code. 97 virtual unsigned getLineCount() const = 0; 98 99 /// Returns the number of columns required to format the text in the 100 /// byte range [\p Offset, \p Offset \c + \p Length). 101 /// 102 /// \p Offset is the byte offset from the start of the content of the line 103 /// at \p LineIndex. 104 /// 105 /// \p StartColumn is the column at which the text starts in the formatted 106 /// file, needed to compute tab stops correctly. 107 virtual unsigned getRangeLength(unsigned LineIndex, unsigned Offset, 108 StringRef::size_type Length, 109 unsigned StartColumn) const = 0; 110 111 /// Returns the number of columns required to format the text following 112 /// the byte \p Offset in the line \p LineIndex, including potentially 113 /// unbreakable sequences of tokens following after the end of the token. 114 /// 115 /// \p Offset is the byte offset from the start of the content of the line 116 /// at \p LineIndex. 117 /// 118 /// \p StartColumn is the column at which the text starts in the formatted 119 /// file, needed to compute tab stops correctly. 120 /// 121 /// For breakable tokens that never use extra space at the end of a line, this 122 /// is equivalent to getRangeLength with a Length of StringRef::npos. 123 virtual unsigned getRemainingLength(unsigned LineIndex, unsigned Offset, 124 unsigned StartColumn) const { 125 return getRangeLength(LineIndex, Offset, StringRef::npos, StartColumn); 126 } 127 128 /// Returns the column at which content in line \p LineIndex starts, 129 /// assuming no reflow. 130 /// 131 /// If \p Break is true, returns the column at which the line should start 132 /// after the line break. 133 /// If \p Break is false, returns the column at which the line itself will 134 /// start. 135 virtual unsigned getContentStartColumn(unsigned LineIndex, 136 bool Break) const = 0; 137 138 /// Returns additional content indent required for the second line after the 139 /// content at line \p LineIndex is broken. 140 /// 141 // (Next lines do not start with `///` since otherwise -Wdocumentation picks 142 // up the example annotations and generates warnings for them) 143 // For example, Javadoc @param annotations require and indent of 4 spaces and 144 // in this example getContentIndex(1) returns 4. 145 // /** 146 // * @param loooooooooooooong line 147 // * continuation 148 // */ 149 virtual unsigned getContentIndent(unsigned LineIndex) const { return 0; } 150 151 /// Returns a range (offset, length) at which to break the line at 152 /// \p LineIndex, if previously broken at \p TailOffset. If possible, do not 153 /// violate \p ColumnLimit, assuming the text starting at \p TailOffset in 154 /// the token is formatted starting at ContentStartColumn in the reformatted 155 /// file. 156 virtual Split getSplit(unsigned LineIndex, unsigned TailOffset, 157 unsigned ColumnLimit, unsigned ContentStartColumn, 158 const llvm::Regex &CommentPragmasRegex) const = 0; 159 160 /// Emits the previously retrieved \p Split via \p Whitespaces. 161 virtual void insertBreak(unsigned LineIndex, unsigned TailOffset, Split Split, 162 unsigned ContentIndent, 163 WhitespaceManager &Whitespaces) const = 0; 164 165 /// Returns the number of columns needed to format 166 /// \p RemainingTokenColumns, assuming that Split is within the range measured 167 /// by \p RemainingTokenColumns, and that the whitespace in Split is reduced 168 /// to a single space. 169 unsigned getLengthAfterCompression(unsigned RemainingTokenColumns, 170 Split Split) const; 171 172 /// Replaces the whitespace range described by \p Split with a single 173 /// space. 174 virtual void compressWhitespace(unsigned LineIndex, unsigned TailOffset, 175 Split Split, 176 WhitespaceManager &Whitespaces) const = 0; 177 178 /// Returns whether the token supports reflowing text. 179 virtual bool supportsReflow() const { return false; } 180 181 /// Returns a whitespace range (offset, length) of the content at \p 182 /// LineIndex such that the content of that line is reflown to the end of the 183 /// previous one. 184 /// 185 /// Returning (StringRef::npos, 0) indicates reflowing is not possible. 186 /// 187 /// The range will include any whitespace preceding the specified line's 188 /// content. 189 /// 190 /// If the split is not contained within one token, for example when reflowing 191 /// line comments, returns (0, <length>). 192 virtual Split getReflowSplit(unsigned LineIndex, 193 const llvm::Regex &CommentPragmasRegex) const { 194 return Split(StringRef::npos, 0); 195 } 196 197 /// Reflows the current line into the end of the previous one. 198 virtual void reflow(unsigned LineIndex, 199 WhitespaceManager &Whitespaces) const {} 200 201 /// Returns whether there will be a line break at the start of the 202 /// token. 203 virtual bool introducesBreakBeforeToken() const { return false; } 204 205 /// Replaces the whitespace between \p LineIndex-1 and \p LineIndex. 206 virtual void adaptStartOfLine(unsigned LineIndex, 207 WhitespaceManager &Whitespaces) const {} 208 209 /// Returns a whitespace range (offset, length) of the content at 210 /// the last line that needs to be reformatted after the last line has been 211 /// reformatted. 212 /// 213 /// A result having offset == StringRef::npos means that no reformat is 214 /// necessary. 215 virtual Split getSplitAfterLastLine(unsigned TailOffset) const { 216 return Split(StringRef::npos, 0); 217 } 218 219 /// Replaces the whitespace from \p SplitAfterLastLine on the last line 220 /// after the last line has been formatted by performing a reformatting. 221 void replaceWhitespaceAfterLastLine(unsigned TailOffset, 222 Split SplitAfterLastLine, 223 WhitespaceManager &Whitespaces) const { 224 insertBreak(getLineCount() - 1, TailOffset, SplitAfterLastLine, 225 /*ContentIndent=*/0, Whitespaces); 226 } 227 228 /// Updates the next token of \p State to the next token after this 229 /// one. This can be used when this token manages a set of underlying tokens 230 /// as a unit and is responsible for the formatting of the them. 231 virtual void updateNextToken(LineState &State) const {} 232 233 /// Adds replacements that are needed when the token is broken. Such as 234 /// wrapping a JavaScript string in parentheses after it gets broken with plus 235 /// signs. 236 virtual void updateAfterBroken(WhitespaceManager &Whitespaces) const {} 237 238 protected: 239 BreakableToken(const FormatToken &Tok, bool InPPDirective, 240 encoding::Encoding Encoding, const FormatStyle &Style) 241 : Tok(Tok), InPPDirective(InPPDirective), Encoding(Encoding), 242 Style(Style) {} 243 244 const FormatToken &Tok; 245 const bool InPPDirective; 246 const encoding::Encoding Encoding; 247 const FormatStyle &Style; 248 }; 249 250 class BreakableStringLiteral : public BreakableToken { 251 public: 252 /// Creates a breakable token for a single line string literal. 253 /// 254 /// \p StartColumn specifies the column in which the token will start 255 /// after formatting. 256 BreakableStringLiteral(const FormatToken &Tok, unsigned StartColumn, 257 StringRef Prefix, StringRef Postfix, 258 unsigned UnbreakableTailLength, bool InPPDirective, 259 encoding::Encoding Encoding, const FormatStyle &Style); 260 261 Split getSplit(unsigned LineIndex, unsigned TailOffset, unsigned ColumnLimit, 262 unsigned ContentStartColumn, 263 const llvm::Regex &CommentPragmasRegex) const override; 264 void insertBreak(unsigned LineIndex, unsigned TailOffset, Split Split, 265 unsigned ContentIndent, 266 WhitespaceManager &Whitespaces) const override; 267 void compressWhitespace(unsigned LineIndex, unsigned TailOffset, Split Split, 268 WhitespaceManager &Whitespaces) const override {} 269 unsigned getLineCount() const override; 270 unsigned getRangeLength(unsigned LineIndex, unsigned Offset, 271 StringRef::size_type Length, 272 unsigned StartColumn) const override; 273 unsigned getRemainingLength(unsigned LineIndex, unsigned Offset, 274 unsigned StartColumn) const override; 275 unsigned getContentStartColumn(unsigned LineIndex, bool Break) const override; 276 277 protected: 278 // The column in which the token starts. 279 unsigned StartColumn; 280 // The prefix a line needs after a break in the token. 281 StringRef Prefix; 282 // The postfix a line needs before introducing a break. 283 StringRef Postfix; 284 // The token text excluding the prefix and postfix. 285 StringRef Line; 286 // Length of the sequence of tokens after this string literal that cannot 287 // contain line breaks. 288 unsigned UnbreakableTailLength; 289 }; 290 291 class BreakableStringLiteralUsingOperators : public BreakableStringLiteral { 292 public: 293 enum QuoteStyleType { 294 DoubleQuotes, // The string is quoted with double quotes. 295 SingleQuotes, // The JavaScript string is quoted with single quotes. 296 AtDoubleQuotes, // The C# verbatim string is quoted with the at sign and 297 // double quotes. 298 }; 299 /// Creates a breakable token for a single line string literal for C#, Java, 300 /// JavaScript, or Verilog. 301 /// 302 /// \p StartColumn specifies the column in which the token will start 303 /// after formatting. 304 BreakableStringLiteralUsingOperators( 305 const FormatToken &Tok, QuoteStyleType QuoteStyle, bool UnindentPlus, 306 unsigned StartColumn, unsigned UnbreakableTailLength, bool InPPDirective, 307 encoding::Encoding Encoding, const FormatStyle &Style); 308 unsigned getRemainingLength(unsigned LineIndex, unsigned Offset, 309 unsigned StartColumn) const override; 310 unsigned getContentStartColumn(unsigned LineIndex, bool Break) const override; 311 void insertBreak(unsigned LineIndex, unsigned TailOffset, Split Split, 312 unsigned ContentIndent, 313 WhitespaceManager &Whitespaces) const override; 314 void updateAfterBroken(WhitespaceManager &Whitespaces) const override; 315 316 protected: 317 // Whether braces or parentheses should be inserted around the string to form 318 // a concatenation. 319 bool BracesNeeded; 320 QuoteStyleType QuoteStyle; 321 // The braces or parentheses along with the first character which they 322 // replace, either a quote or at sign. 323 StringRef LeftBraceQuote; 324 StringRef RightBraceQuote; 325 // Width added to the left due to the added brace or parenthesis. Does not 326 // apply to the first line. 327 int ContinuationIndent; 328 }; 329 330 class BreakableComment : public BreakableToken { 331 protected: 332 /// Creates a breakable token for a comment. 333 /// 334 /// \p StartColumn specifies the column in which the comment will start after 335 /// formatting. 336 BreakableComment(const FormatToken &Token, unsigned StartColumn, 337 bool InPPDirective, encoding::Encoding Encoding, 338 const FormatStyle &Style); 339 340 public: 341 bool supportsReflow() const override { return true; } 342 unsigned getLineCount() const override; 343 Split getSplit(unsigned LineIndex, unsigned TailOffset, unsigned ColumnLimit, 344 unsigned ContentStartColumn, 345 const llvm::Regex &CommentPragmasRegex) const override; 346 void compressWhitespace(unsigned LineIndex, unsigned TailOffset, Split Split, 347 WhitespaceManager &Whitespaces) const override; 348 349 protected: 350 // Returns the token containing the line at LineIndex. 351 const FormatToken &tokenAt(unsigned LineIndex) const; 352 353 // Checks if the content of line LineIndex may be reflown with the previous 354 // line. 355 virtual bool mayReflow(unsigned LineIndex, 356 const llvm::Regex &CommentPragmasRegex) const = 0; 357 358 // Contains the original text of the lines of the block comment. 359 // 360 // In case of a block comments, excludes the leading /* in the first line and 361 // trailing */ in the last line. In case of line comments, excludes the 362 // leading // and spaces. 363 SmallVector<StringRef, 16> Lines; 364 365 // Contains the text of the lines excluding all leading and trailing 366 // whitespace between the lines. Note that the decoration (if present) is also 367 // not considered part of the text. 368 SmallVector<StringRef, 16> Content; 369 370 // Tokens[i] contains a reference to the token containing Lines[i] if the 371 // whitespace range before that token is managed by this block. 372 // Otherwise, Tokens[i] is a null pointer. 373 SmallVector<FormatToken *, 16> Tokens; 374 375 // ContentColumn[i] is the target column at which Content[i] should be. 376 // Note that this excludes a leading "* " or "*" in case of block comments 377 // where all lines have a "*" prefix, or the leading "// " or "//" in case of 378 // line comments. 379 // 380 // In block comments, the first line's target column is always positive. The 381 // remaining lines' target columns are relative to the first line to allow 382 // correct indentation of comments in \c WhitespaceManager. Thus they can be 383 // negative as well (in case the first line needs to be unindented more than 384 // there's actual whitespace in another line). 385 SmallVector<int, 16> ContentColumn; 386 387 // The intended start column of the first line of text from this section. 388 unsigned StartColumn; 389 390 // The prefix to use in front a line that has been reflown up. 391 // For example, when reflowing the second line after the first here: 392 // // comment 1 393 // // comment 2 394 // we expect: 395 // // comment 1 comment 2 396 // and not: 397 // // comment 1comment 2 398 StringRef ReflowPrefix = " "; 399 }; 400 401 class BreakableBlockComment : public BreakableComment { 402 public: 403 BreakableBlockComment(const FormatToken &Token, unsigned StartColumn, 404 unsigned OriginalStartColumn, bool FirstInLine, 405 bool InPPDirective, encoding::Encoding Encoding, 406 const FormatStyle &Style, bool UseCRLF); 407 408 Split getSplit(unsigned LineIndex, unsigned TailOffset, unsigned ColumnLimit, 409 unsigned ContentStartColumn, 410 const llvm::Regex &CommentPragmasRegex) const override; 411 unsigned getRangeLength(unsigned LineIndex, unsigned Offset, 412 StringRef::size_type Length, 413 unsigned StartColumn) const override; 414 unsigned getRemainingLength(unsigned LineIndex, unsigned Offset, 415 unsigned StartColumn) const override; 416 unsigned getContentStartColumn(unsigned LineIndex, bool Break) const override; 417 unsigned getContentIndent(unsigned LineIndex) const override; 418 void insertBreak(unsigned LineIndex, unsigned TailOffset, Split Split, 419 unsigned ContentIndent, 420 WhitespaceManager &Whitespaces) const override; 421 Split getReflowSplit(unsigned LineIndex, 422 const llvm::Regex &CommentPragmasRegex) const override; 423 void reflow(unsigned LineIndex, 424 WhitespaceManager &Whitespaces) const override; 425 bool introducesBreakBeforeToken() const override; 426 void adaptStartOfLine(unsigned LineIndex, 427 WhitespaceManager &Whitespaces) const override; 428 Split getSplitAfterLastLine(unsigned TailOffset) const override; 429 430 bool mayReflow(unsigned LineIndex, 431 const llvm::Regex &CommentPragmasRegex) const override; 432 433 // Contains Javadoc annotations that require additional indent when continued 434 // on multiple lines. 435 static const llvm::StringSet<> ContentIndentingJavadocAnnotations; 436 437 private: 438 // Rearranges the whitespace between Lines[LineIndex-1] and Lines[LineIndex]. 439 // 440 // Updates Content[LineIndex-1] and Content[LineIndex] by stripping off 441 // leading and trailing whitespace. 442 // 443 // Sets ContentColumn to the intended column in which the text at 444 // Lines[LineIndex] starts (note that the decoration, if present, is not 445 // considered part of the text). 446 void adjustWhitespace(unsigned LineIndex, int IndentDelta); 447 448 // The column at which the text of a broken line should start. 449 // Note that an optional decoration would go before that column. 450 // IndentAtLineBreak is a uniform position for all lines in a block comment, 451 // regardless of their relative position. 452 // FIXME: Revisit the decision to do this; the main reason was to support 453 // patterns like 454 // /**************//** 455 // * Comment 456 // We could also support such patterns by special casing the first line 457 // instead. 458 unsigned IndentAtLineBreak; 459 460 // This is to distinguish between the case when the last line was empty and 461 // the case when it started with a decoration ("*" or "* "). 462 bool LastLineNeedsDecoration; 463 464 // Either "* " if all lines begin with a "*", or empty. 465 StringRef Decoration; 466 467 // If this block comment has decorations, this is the column of the start of 468 // the decorations. 469 unsigned DecorationColumn; 470 471 // If true, make sure that the opening '/**' and the closing '*/' ends on a 472 // line of itself. Styles like jsdoc require this for multiline comments. 473 bool DelimitersOnNewline; 474 475 // Length of the sequence of tokens after this string literal that cannot 476 // contain line breaks. 477 unsigned UnbreakableTailLength; 478 }; 479 480 class BreakableLineCommentSection : public BreakableComment { 481 public: 482 BreakableLineCommentSection(const FormatToken &Token, unsigned StartColumn, 483 bool InPPDirective, encoding::Encoding Encoding, 484 const FormatStyle &Style); 485 486 unsigned getRangeLength(unsigned LineIndex, unsigned Offset, 487 StringRef::size_type Length, 488 unsigned StartColumn) const override; 489 unsigned getContentStartColumn(unsigned LineIndex, bool Break) const override; 490 void insertBreak(unsigned LineIndex, unsigned TailOffset, Split Split, 491 unsigned ContentIndent, 492 WhitespaceManager &Whitespaces) const override; 493 Split getReflowSplit(unsigned LineIndex, 494 const llvm::Regex &CommentPragmasRegex) const override; 495 void reflow(unsigned LineIndex, 496 WhitespaceManager &Whitespaces) const override; 497 void adaptStartOfLine(unsigned LineIndex, 498 WhitespaceManager &Whitespaces) const override; 499 void updateNextToken(LineState &State) const override; 500 bool mayReflow(unsigned LineIndex, 501 const llvm::Regex &CommentPragmasRegex) const override; 502 503 private: 504 // OriginalPrefix[i] contains the original prefix of line i, including 505 // trailing whitespace before the start of the content. The indentation 506 // preceding the prefix is not included. 507 // For example, if the line is: 508 // // content 509 // then the original prefix is "// ". 510 SmallVector<StringRef, 16> OriginalPrefix; 511 512 /// Prefix[i] + SpacesToAdd[i] contains the intended leading "//" with 513 /// trailing spaces to account for the indentation of content within the 514 /// comment at line i after formatting. It can be different than the original 515 /// prefix. 516 /// When the original line starts like this: 517 /// //content 518 /// Then the OriginalPrefix[i] is "//", but the Prefix[i] is "// " in the LLVM 519 /// style. 520 /// When the line starts like: 521 /// // content 522 /// And we want to remove the spaces the OriginalPrefix[i] is "// " and 523 /// Prefix[i] is "//". 524 SmallVector<std::string, 16> Prefix; 525 526 /// How many spaces are added or removed from the OriginalPrefix to form 527 /// Prefix. 528 SmallVector<int, 16> PrefixSpaceChange; 529 530 /// The token to which the last line of this breakable token belongs 531 /// to; nullptr if that token is the initial token. 532 /// 533 /// The distinction is because if the token of the last line of this breakable 534 /// token is distinct from the initial token, this breakable token owns the 535 /// whitespace before the token of the last line, and the whitespace manager 536 /// must be able to modify it. 537 FormatToken *LastLineTok = nullptr; 538 }; 539 } // namespace format 540 } // namespace clang 541 542 #endif 543