1 //===--- RawCommentList.cpp - Processing raw comments -----------*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "clang/AST/RawCommentList.h" 10 #include "clang/AST/ASTContext.h" 11 #include "clang/AST/Comment.h" 12 #include "clang/AST/CommentBriefParser.h" 13 #include "clang/AST/CommentCommandTraits.h" 14 #include "clang/AST/CommentLexer.h" 15 #include "clang/AST/CommentParser.h" 16 #include "clang/AST/CommentSema.h" 17 #include "clang/Basic/CharInfo.h" 18 #include "llvm/ADT/STLExtras.h" 19 #include "llvm/Support/Allocator.h" 20 21 using namespace clang; 22 23 namespace { 24 /// Get comment kind and bool describing if it is a trailing comment. 25 std::pair<RawComment::CommentKind, bool> getCommentKind(StringRef Comment, 26 bool ParseAllComments) { 27 const size_t MinCommentLength = ParseAllComments ? 2 : 3; 28 if ((Comment.size() < MinCommentLength) || Comment[0] != '/') 29 return std::make_pair(RawComment::RCK_Invalid, false); 30 31 RawComment::CommentKind K; 32 if (Comment[1] == '/') { 33 if (Comment.size() < 3) 34 return std::make_pair(RawComment::RCK_OrdinaryBCPL, false); 35 36 if (Comment[2] == '/') 37 K = RawComment::RCK_BCPLSlash; 38 else if (Comment[2] == '!') 39 K = RawComment::RCK_BCPLExcl; 40 else 41 return std::make_pair(RawComment::RCK_OrdinaryBCPL, false); 42 } else { 43 assert(Comment.size() >= 4); 44 45 // Comment lexer does not understand escapes in comment markers, so pretend 46 // that this is not a comment. 47 if (Comment[1] != '*' || 48 Comment[Comment.size() - 2] != '*' || 49 Comment[Comment.size() - 1] != '/') 50 return std::make_pair(RawComment::RCK_Invalid, false); 51 52 if (Comment[2] == '*') 53 K = RawComment::RCK_JavaDoc; 54 else if (Comment[2] == '!') 55 K = RawComment::RCK_Qt; 56 else 57 return std::make_pair(RawComment::RCK_OrdinaryC, false); 58 } 59 const bool TrailingComment = (Comment.size() > 3) && (Comment[3] == '<'); 60 return std::make_pair(K, TrailingComment); 61 } 62 63 bool mergedCommentIsTrailingComment(StringRef Comment) { 64 return (Comment.size() > 3) && (Comment[3] == '<'); 65 } 66 67 /// Returns true if R1 and R2 both have valid locations that start on the same 68 /// column. 69 bool commentsStartOnSameColumn(const SourceManager &SM, const RawComment &R1, 70 const RawComment &R2) { 71 SourceLocation L1 = R1.getBeginLoc(); 72 SourceLocation L2 = R2.getBeginLoc(); 73 bool Invalid = false; 74 unsigned C1 = SM.getPresumedColumnNumber(L1, &Invalid); 75 if (!Invalid) { 76 unsigned C2 = SM.getPresumedColumnNumber(L2, &Invalid); 77 return !Invalid && (C1 == C2); 78 } 79 return false; 80 } 81 } // unnamed namespace 82 83 /// Determines whether there is only whitespace in `Buffer` between `P` 84 /// and the previous line. 85 /// \param Buffer The buffer to search in. 86 /// \param P The offset from the beginning of `Buffer` to start from. 87 /// \return true if all of the characters in `Buffer` ranging from the closest 88 /// line-ending character before `P` (or the beginning of `Buffer`) to `P - 1` 89 /// are whitespace. 90 static bool onlyWhitespaceOnLineBefore(const char *Buffer, unsigned P) { 91 // Search backwards until we see linefeed or carriage return. 92 for (unsigned I = P; I != 0; --I) { 93 char C = Buffer[I - 1]; 94 if (isVerticalWhitespace(C)) 95 return true; 96 if (!isHorizontalWhitespace(C)) 97 return false; 98 } 99 // We hit the beginning of the buffer. 100 return true; 101 } 102 103 /// Returns whether `K` is an ordinary comment kind. 104 static bool isOrdinaryKind(RawComment::CommentKind K) { 105 return (K == RawComment::RCK_OrdinaryBCPL) || 106 (K == RawComment::RCK_OrdinaryC); 107 } 108 109 RawComment::RawComment(const SourceManager &SourceMgr, SourceRange SR, 110 const CommentOptions &CommentOpts, bool Merged) : 111 Range(SR), RawTextValid(false), BriefTextValid(false), 112 IsAttached(false), IsTrailingComment(false), 113 IsAlmostTrailingComment(false) { 114 // Extract raw comment text, if possible. 115 if (SR.getBegin() == SR.getEnd() || getRawText(SourceMgr).empty()) { 116 Kind = RCK_Invalid; 117 return; 118 } 119 120 // Guess comment kind. 121 std::pair<CommentKind, bool> K = 122 getCommentKind(RawText, CommentOpts.ParseAllComments); 123 124 // Guess whether an ordinary comment is trailing. 125 if (CommentOpts.ParseAllComments && isOrdinaryKind(K.first)) { 126 FileID BeginFileID; 127 unsigned BeginOffset; 128 std::tie(BeginFileID, BeginOffset) = 129 SourceMgr.getDecomposedLoc(Range.getBegin()); 130 if (BeginOffset != 0) { 131 bool Invalid = false; 132 const char *Buffer = 133 SourceMgr.getBufferData(BeginFileID, &Invalid).data(); 134 IsTrailingComment |= 135 (!Invalid && !onlyWhitespaceOnLineBefore(Buffer, BeginOffset)); 136 } 137 } 138 139 if (!Merged) { 140 Kind = K.first; 141 IsTrailingComment |= K.second; 142 143 IsAlmostTrailingComment = RawText.startswith("//<") || 144 RawText.startswith("/*<"); 145 } else { 146 Kind = RCK_Merged; 147 IsTrailingComment = 148 IsTrailingComment || mergedCommentIsTrailingComment(RawText); 149 } 150 } 151 152 StringRef RawComment::getRawTextSlow(const SourceManager &SourceMgr) const { 153 FileID BeginFileID; 154 FileID EndFileID; 155 unsigned BeginOffset; 156 unsigned EndOffset; 157 158 std::tie(BeginFileID, BeginOffset) = 159 SourceMgr.getDecomposedLoc(Range.getBegin()); 160 std::tie(EndFileID, EndOffset) = SourceMgr.getDecomposedLoc(Range.getEnd()); 161 162 const unsigned Length = EndOffset - BeginOffset; 163 if (Length < 2) 164 return StringRef(); 165 166 // The comment can't begin in one file and end in another. 167 assert(BeginFileID == EndFileID); 168 169 bool Invalid = false; 170 const char *BufferStart = SourceMgr.getBufferData(BeginFileID, 171 &Invalid).data(); 172 if (Invalid) 173 return StringRef(); 174 175 return StringRef(BufferStart + BeginOffset, Length); 176 } 177 178 const char *RawComment::extractBriefText(const ASTContext &Context) const { 179 // Lazily initialize RawText using the accessor before using it. 180 (void)getRawText(Context.getSourceManager()); 181 182 // Since we will be copying the resulting text, all allocations made during 183 // parsing are garbage after resulting string is formed. Thus we can use 184 // a separate allocator for all temporary stuff. 185 llvm::BumpPtrAllocator Allocator; 186 187 comments::Lexer L(Allocator, Context.getDiagnostics(), 188 Context.getCommentCommandTraits(), 189 Range.getBegin(), 190 RawText.begin(), RawText.end()); 191 comments::BriefParser P(L, Context.getCommentCommandTraits()); 192 193 const std::string Result = P.Parse(); 194 const unsigned BriefTextLength = Result.size(); 195 char *BriefTextPtr = new (Context) char[BriefTextLength + 1]; 196 memcpy(BriefTextPtr, Result.c_str(), BriefTextLength + 1); 197 BriefText = BriefTextPtr; 198 BriefTextValid = true; 199 200 return BriefTextPtr; 201 } 202 203 comments::FullComment *RawComment::parse(const ASTContext &Context, 204 const Preprocessor *PP, 205 const Decl *D) const { 206 // Lazily initialize RawText using the accessor before using it. 207 (void)getRawText(Context.getSourceManager()); 208 209 comments::Lexer L(Context.getAllocator(), Context.getDiagnostics(), 210 Context.getCommentCommandTraits(), 211 getSourceRange().getBegin(), 212 RawText.begin(), RawText.end()); 213 comments::Sema S(Context.getAllocator(), Context.getSourceManager(), 214 Context.getDiagnostics(), 215 Context.getCommentCommandTraits(), 216 PP); 217 S.setDecl(D); 218 comments::Parser P(L, S, Context.getAllocator(), Context.getSourceManager(), 219 Context.getDiagnostics(), 220 Context.getCommentCommandTraits()); 221 222 return P.parseFullComment(); 223 } 224 225 static bool onlyWhitespaceBetween(SourceManager &SM, 226 SourceLocation Loc1, SourceLocation Loc2, 227 unsigned MaxNewlinesAllowed) { 228 std::pair<FileID, unsigned> Loc1Info = SM.getDecomposedLoc(Loc1); 229 std::pair<FileID, unsigned> Loc2Info = SM.getDecomposedLoc(Loc2); 230 231 // Question does not make sense if locations are in different files. 232 if (Loc1Info.first != Loc2Info.first) 233 return false; 234 235 bool Invalid = false; 236 const char *Buffer = SM.getBufferData(Loc1Info.first, &Invalid).data(); 237 if (Invalid) 238 return false; 239 240 unsigned NumNewlines = 0; 241 assert(Loc1Info.second <= Loc2Info.second && "Loc1 after Loc2!"); 242 // Look for non-whitespace characters and remember any newlines seen. 243 for (unsigned I = Loc1Info.second; I != Loc2Info.second; ++I) { 244 switch (Buffer[I]) { 245 default: 246 return false; 247 case ' ': 248 case '\t': 249 case '\f': 250 case '\v': 251 break; 252 case '\r': 253 case '\n': 254 ++NumNewlines; 255 256 // Check if we have found more than the maximum allowed number of 257 // newlines. 258 if (NumNewlines > MaxNewlinesAllowed) 259 return false; 260 261 // Collapse \r\n and \n\r into a single newline. 262 if (I + 1 != Loc2Info.second && 263 (Buffer[I + 1] == '\n' || Buffer[I + 1] == '\r') && 264 Buffer[I] != Buffer[I + 1]) 265 ++I; 266 break; 267 } 268 } 269 270 return true; 271 } 272 273 void RawCommentList::addComment(const RawComment &RC, 274 const CommentOptions &CommentOpts, 275 llvm::BumpPtrAllocator &Allocator) { 276 if (RC.isInvalid()) 277 return; 278 279 // Ordinary comments are not interesting for us. 280 if (RC.isOrdinary() && !CommentOpts.ParseAllComments) 281 return; 282 283 std::pair<FileID, unsigned> Loc = 284 SourceMgr.getDecomposedLoc(RC.getBeginLoc()); 285 286 const FileID CommentFile = Loc.first; 287 const unsigned CommentOffset = Loc.second; 288 289 // If this is the first Doxygen comment, save it (because there isn't 290 // anything to merge it with). 291 if (OrderedComments[CommentFile].empty()) { 292 OrderedComments[CommentFile][CommentOffset] = 293 new (Allocator) RawComment(RC); 294 return; 295 } 296 297 const RawComment &C1 = *OrderedComments[CommentFile].rbegin()->second; 298 const RawComment &C2 = RC; 299 300 // Merge comments only if there is only whitespace between them. 301 // Can't merge trailing and non-trailing comments unless the second is 302 // non-trailing ordinary in the same column, as in the case: 303 // int x; // documents x 304 // // more text 305 // versus: 306 // int x; // documents x 307 // int y; // documents y 308 // or: 309 // int x; // documents x 310 // // documents y 311 // int y; 312 // Merge comments if they are on same or consecutive lines. 313 if ((C1.isTrailingComment() == C2.isTrailingComment() || 314 (C1.isTrailingComment() && !C2.isTrailingComment() && 315 isOrdinaryKind(C2.getKind()) && 316 commentsStartOnSameColumn(SourceMgr, C1, C2))) && 317 onlyWhitespaceBetween(SourceMgr, C1.getEndLoc(), C2.getBeginLoc(), 318 /*MaxNewlinesAllowed=*/1)) { 319 SourceRange MergedRange(C1.getBeginLoc(), C2.getEndLoc()); 320 *OrderedComments[CommentFile].rbegin()->second = 321 RawComment(SourceMgr, MergedRange, CommentOpts, true); 322 } else { 323 OrderedComments[CommentFile][CommentOffset] = 324 new (Allocator) RawComment(RC); 325 } 326 } 327 328 const std::map<unsigned, RawComment *> * 329 RawCommentList::getCommentsInFile(FileID File) const { 330 auto CommentsInFile = OrderedComments.find(File); 331 if (CommentsInFile == OrderedComments.end()) 332 return nullptr; 333 334 return &CommentsInFile->second; 335 } 336 337 bool RawCommentList::empty() const { return OrderedComments.empty(); } 338 339 unsigned RawCommentList::getCommentBeginLine(RawComment *C, FileID File, 340 unsigned Offset) const { 341 auto Cached = CommentBeginLine.find(C); 342 if (Cached != CommentBeginLine.end()) 343 return Cached->second; 344 const unsigned Line = SourceMgr.getLineNumber(File, Offset); 345 CommentBeginLine[C] = Line; 346 return Line; 347 } 348 349 unsigned RawCommentList::getCommentEndOffset(RawComment *C) const { 350 auto Cached = CommentEndOffset.find(C); 351 if (Cached != CommentEndOffset.end()) 352 return Cached->second; 353 const unsigned Offset = 354 SourceMgr.getDecomposedLoc(C->getSourceRange().getEnd()).second; 355 CommentEndOffset[C] = Offset; 356 return Offset; 357 } 358 359 std::string RawComment::getFormattedText(const SourceManager &SourceMgr, 360 DiagnosticsEngine &Diags) const { 361 llvm::StringRef CommentText = getRawText(SourceMgr); 362 if (CommentText.empty()) 363 return ""; 364 365 llvm::BumpPtrAllocator Allocator; 366 // We do not parse any commands, so CommentOptions are ignored by 367 // comments::Lexer. Therefore, we just use default-constructed options. 368 CommentOptions DefOpts; 369 comments::CommandTraits EmptyTraits(Allocator, DefOpts); 370 comments::Lexer L(Allocator, Diags, EmptyTraits, getSourceRange().getBegin(), 371 CommentText.begin(), CommentText.end(), 372 /*ParseCommands=*/false); 373 374 std::string Result; 375 // A column number of the first non-whitespace token in the comment text. 376 // We skip whitespace up to this column, but keep the whitespace after this 377 // column. IndentColumn is calculated when lexing the first line and reused 378 // for the rest of lines. 379 unsigned IndentColumn = 0; 380 381 // Processes one line of the comment and adds it to the result. 382 // Handles skipping the indent at the start of the line. 383 // Returns false when eof is reached and true otherwise. 384 auto LexLine = [&](bool IsFirstLine) -> bool { 385 comments::Token Tok; 386 // Lex the first token on the line. We handle it separately, because we to 387 // fix up its indentation. 388 L.lex(Tok); 389 if (Tok.is(comments::tok::eof)) 390 return false; 391 if (Tok.is(comments::tok::newline)) { 392 Result += "\n"; 393 return true; 394 } 395 llvm::StringRef TokText = L.getSpelling(Tok, SourceMgr); 396 bool LocInvalid = false; 397 unsigned TokColumn = 398 SourceMgr.getSpellingColumnNumber(Tok.getLocation(), &LocInvalid); 399 assert(!LocInvalid && "getFormattedText for invalid location"); 400 401 // Amount of leading whitespace in TokText. 402 size_t WhitespaceLen = TokText.find_first_not_of(" \t"); 403 if (WhitespaceLen == StringRef::npos) 404 WhitespaceLen = TokText.size(); 405 // Remember the amount of whitespace we skipped in the first line to remove 406 // indent up to that column in the following lines. 407 if (IsFirstLine) 408 IndentColumn = TokColumn + WhitespaceLen; 409 410 // Amount of leading whitespace we actually want to skip. 411 // For the first line we skip all the whitespace. 412 // For the rest of the lines, we skip whitespace up to IndentColumn. 413 unsigned SkipLen = 414 IsFirstLine 415 ? WhitespaceLen 416 : std::min<size_t>( 417 WhitespaceLen, 418 std::max<int>(static_cast<int>(IndentColumn) - TokColumn, 0)); 419 llvm::StringRef Trimmed = TokText.drop_front(SkipLen); 420 Result += Trimmed; 421 // Lex all tokens in the rest of the line. 422 for (L.lex(Tok); Tok.isNot(comments::tok::eof); L.lex(Tok)) { 423 if (Tok.is(comments::tok::newline)) { 424 Result += "\n"; 425 return true; 426 } 427 Result += L.getSpelling(Tok, SourceMgr); 428 } 429 // We've reached the end of file token. 430 return false; 431 }; 432 433 auto DropTrailingNewLines = [](std::string &Str) { 434 while (!Str.empty() && Str.back() == '\n') 435 Str.pop_back(); 436 }; 437 438 // Process first line separately to remember indent for the following lines. 439 if (!LexLine(/*IsFirstLine=*/true)) { 440 DropTrailingNewLines(Result); 441 return Result; 442 } 443 // Process the rest of the lines. 444 while (LexLine(/*IsFirstLine=*/false)) 445 ; 446 DropTrailingNewLines(Result); 447 return Result; 448 } 449