1 //===--- CommentParser.cpp - Doxygen comment parser -----------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "clang/AST/CommentParser.h" 10 #include "clang/AST/CommentCommandTraits.h" 11 #include "clang/AST/CommentDiagnostic.h" 12 #include "clang/AST/CommentSema.h" 13 #include "clang/Basic/CharInfo.h" 14 #include "clang/Basic/SourceManager.h" 15 #include "llvm/Support/ErrorHandling.h" 16 17 namespace clang { 18 19 static inline bool isWhitespace(llvm::StringRef S) { 20 for (StringRef::const_iterator I = S.begin(), E = S.end(); I != E; ++I) { 21 if (!isWhitespace(*I)) 22 return false; 23 } 24 return true; 25 } 26 27 namespace comments { 28 29 /// Re-lexes a sequence of tok::text tokens. 30 class TextTokenRetokenizer { 31 llvm::BumpPtrAllocator &Allocator; 32 Parser &P; 33 34 /// This flag is set when there are no more tokens we can fetch from lexer. 35 bool NoMoreInterestingTokens; 36 37 /// Token buffer: tokens we have processed and lookahead. 38 SmallVector<Token, 16> Toks; 39 40 /// A position in \c Toks. 41 struct Position { 42 const char *BufferStart; 43 const char *BufferEnd; 44 const char *BufferPtr; 45 SourceLocation BufferStartLoc; 46 unsigned CurToken; 47 }; 48 49 /// Current position in Toks. 50 Position Pos; 51 52 bool isEnd() const { 53 return Pos.CurToken >= Toks.size(); 54 } 55 56 /// Sets up the buffer pointers to point to current token. 57 void setupBuffer() { 58 assert(!isEnd()); 59 const Token &Tok = Toks[Pos.CurToken]; 60 61 Pos.BufferStart = Tok.getText().begin(); 62 Pos.BufferEnd = Tok.getText().end(); 63 Pos.BufferPtr = Pos.BufferStart; 64 Pos.BufferStartLoc = Tok.getLocation(); 65 } 66 67 SourceLocation getSourceLocation() const { 68 const unsigned CharNo = Pos.BufferPtr - Pos.BufferStart; 69 return Pos.BufferStartLoc.getLocWithOffset(CharNo); 70 } 71 72 char peek() const { 73 assert(!isEnd()); 74 assert(Pos.BufferPtr != Pos.BufferEnd); 75 return *Pos.BufferPtr; 76 } 77 78 void consumeChar() { 79 assert(!isEnd()); 80 assert(Pos.BufferPtr != Pos.BufferEnd); 81 Pos.BufferPtr++; 82 if (Pos.BufferPtr == Pos.BufferEnd) { 83 Pos.CurToken++; 84 if (isEnd() && !addToken()) 85 return; 86 87 assert(!isEnd()); 88 setupBuffer(); 89 } 90 } 91 92 /// Add a token. 93 /// Returns true on success, false if there are no interesting tokens to 94 /// fetch from lexer. 95 bool addToken() { 96 if (NoMoreInterestingTokens) 97 return false; 98 99 if (P.Tok.is(tok::newline)) { 100 // If we see a single newline token between text tokens, skip it. 101 Token Newline = P.Tok; 102 P.consumeToken(); 103 if (P.Tok.isNot(tok::text)) { 104 P.putBack(Newline); 105 NoMoreInterestingTokens = true; 106 return false; 107 } 108 } 109 if (P.Tok.isNot(tok::text)) { 110 NoMoreInterestingTokens = true; 111 return false; 112 } 113 114 Toks.push_back(P.Tok); 115 P.consumeToken(); 116 if (Toks.size() == 1) 117 setupBuffer(); 118 return true; 119 } 120 121 void consumeWhitespace() { 122 while (!isEnd()) { 123 if (isWhitespace(peek())) 124 consumeChar(); 125 else 126 break; 127 } 128 } 129 130 void formTokenWithChars(Token &Result, 131 SourceLocation Loc, 132 const char *TokBegin, 133 unsigned TokLength, 134 StringRef Text) { 135 Result.setLocation(Loc); 136 Result.setKind(tok::text); 137 Result.setLength(TokLength); 138 #ifndef NDEBUG 139 Result.TextPtr = "<UNSET>"; 140 Result.IntVal = 7; 141 #endif 142 Result.setText(Text); 143 } 144 145 public: 146 TextTokenRetokenizer(llvm::BumpPtrAllocator &Allocator, Parser &P): 147 Allocator(Allocator), P(P), NoMoreInterestingTokens(false) { 148 Pos.CurToken = 0; 149 addToken(); 150 } 151 152 /// Extract a word -- sequence of non-whitespace characters. 153 bool lexWord(Token &Tok) { 154 if (isEnd()) 155 return false; 156 157 Position SavedPos = Pos; 158 159 consumeWhitespace(); 160 SmallString<32> WordText; 161 const char *WordBegin = Pos.BufferPtr; 162 SourceLocation Loc = getSourceLocation(); 163 while (!isEnd()) { 164 const char C = peek(); 165 if (!isWhitespace(C)) { 166 WordText.push_back(C); 167 consumeChar(); 168 } else 169 break; 170 } 171 const unsigned Length = WordText.size(); 172 if (Length == 0) { 173 Pos = SavedPos; 174 return false; 175 } 176 177 char *TextPtr = Allocator.Allocate<char>(Length + 1); 178 179 memcpy(TextPtr, WordText.c_str(), Length + 1); 180 StringRef Text = StringRef(TextPtr, Length); 181 182 formTokenWithChars(Tok, Loc, WordBegin, Length, Text); 183 return true; 184 } 185 186 bool lexDelimitedSeq(Token &Tok, char OpenDelim, char CloseDelim) { 187 if (isEnd()) 188 return false; 189 190 Position SavedPos = Pos; 191 192 consumeWhitespace(); 193 SmallString<32> WordText; 194 const char *WordBegin = Pos.BufferPtr; 195 SourceLocation Loc = getSourceLocation(); 196 bool Error = false; 197 if (!isEnd()) { 198 const char C = peek(); 199 if (C == OpenDelim) { 200 WordText.push_back(C); 201 consumeChar(); 202 } else 203 Error = true; 204 } 205 char C = '\0'; 206 while (!Error && !isEnd()) { 207 C = peek(); 208 WordText.push_back(C); 209 consumeChar(); 210 if (C == CloseDelim) 211 break; 212 } 213 if (!Error && C != CloseDelim) 214 Error = true; 215 216 if (Error) { 217 Pos = SavedPos; 218 return false; 219 } 220 221 const unsigned Length = WordText.size(); 222 char *TextPtr = Allocator.Allocate<char>(Length + 1); 223 224 memcpy(TextPtr, WordText.c_str(), Length + 1); 225 StringRef Text = StringRef(TextPtr, Length); 226 227 formTokenWithChars(Tok, Loc, WordBegin, 228 Pos.BufferPtr - WordBegin, Text); 229 return true; 230 } 231 232 /// Put back tokens that we didn't consume. 233 void putBackLeftoverTokens() { 234 if (isEnd()) 235 return; 236 237 bool HavePartialTok = false; 238 Token PartialTok; 239 if (Pos.BufferPtr != Pos.BufferStart) { 240 formTokenWithChars(PartialTok, getSourceLocation(), 241 Pos.BufferPtr, Pos.BufferEnd - Pos.BufferPtr, 242 StringRef(Pos.BufferPtr, 243 Pos.BufferEnd - Pos.BufferPtr)); 244 HavePartialTok = true; 245 Pos.CurToken++; 246 } 247 248 P.putBack(llvm::makeArrayRef(Toks.begin() + Pos.CurToken, Toks.end())); 249 Pos.CurToken = Toks.size(); 250 251 if (HavePartialTok) 252 P.putBack(PartialTok); 253 } 254 }; 255 256 Parser::Parser(Lexer &L, Sema &S, llvm::BumpPtrAllocator &Allocator, 257 const SourceManager &SourceMgr, DiagnosticsEngine &Diags, 258 const CommandTraits &Traits): 259 L(L), S(S), Allocator(Allocator), SourceMgr(SourceMgr), Diags(Diags), 260 Traits(Traits) { 261 consumeToken(); 262 } 263 264 void Parser::parseParamCommandArgs(ParamCommandComment *PC, 265 TextTokenRetokenizer &Retokenizer) { 266 Token Arg; 267 // Check if argument looks like direction specification: [dir] 268 // e.g., [in], [out], [in,out] 269 if (Retokenizer.lexDelimitedSeq(Arg, '[', ']')) 270 S.actOnParamCommandDirectionArg(PC, 271 Arg.getLocation(), 272 Arg.getEndLocation(), 273 Arg.getText()); 274 275 if (Retokenizer.lexWord(Arg)) 276 S.actOnParamCommandParamNameArg(PC, 277 Arg.getLocation(), 278 Arg.getEndLocation(), 279 Arg.getText()); 280 } 281 282 void Parser::parseTParamCommandArgs(TParamCommandComment *TPC, 283 TextTokenRetokenizer &Retokenizer) { 284 Token Arg; 285 if (Retokenizer.lexWord(Arg)) 286 S.actOnTParamCommandParamNameArg(TPC, 287 Arg.getLocation(), 288 Arg.getEndLocation(), 289 Arg.getText()); 290 } 291 292 void Parser::parseBlockCommandArgs(BlockCommandComment *BC, 293 TextTokenRetokenizer &Retokenizer, 294 unsigned NumArgs) { 295 typedef BlockCommandComment::Argument Argument; 296 Argument *Args = 297 new (Allocator.Allocate<Argument>(NumArgs)) Argument[NumArgs]; 298 unsigned ParsedArgs = 0; 299 Token Arg; 300 while (ParsedArgs < NumArgs && Retokenizer.lexWord(Arg)) { 301 Args[ParsedArgs] = Argument(SourceRange(Arg.getLocation(), 302 Arg.getEndLocation()), 303 Arg.getText()); 304 ParsedArgs++; 305 } 306 307 S.actOnBlockCommandArgs(BC, llvm::makeArrayRef(Args, ParsedArgs)); 308 } 309 310 BlockCommandComment *Parser::parseBlockCommand() { 311 assert(Tok.is(tok::backslash_command) || Tok.is(tok::at_command)); 312 313 ParamCommandComment *PC = nullptr; 314 TParamCommandComment *TPC = nullptr; 315 BlockCommandComment *BC = nullptr; 316 const CommandInfo *Info = Traits.getCommandInfo(Tok.getCommandID()); 317 CommandMarkerKind CommandMarker = 318 Tok.is(tok::backslash_command) ? CMK_Backslash : CMK_At; 319 if (Info->IsParamCommand) { 320 PC = S.actOnParamCommandStart(Tok.getLocation(), 321 Tok.getEndLocation(), 322 Tok.getCommandID(), 323 CommandMarker); 324 } else if (Info->IsTParamCommand) { 325 TPC = S.actOnTParamCommandStart(Tok.getLocation(), 326 Tok.getEndLocation(), 327 Tok.getCommandID(), 328 CommandMarker); 329 } else { 330 BC = S.actOnBlockCommandStart(Tok.getLocation(), 331 Tok.getEndLocation(), 332 Tok.getCommandID(), 333 CommandMarker); 334 } 335 consumeToken(); 336 337 if (isTokBlockCommand()) { 338 // Block command ahead. We can't nest block commands, so pretend that this 339 // command has an empty argument. 340 ParagraphComment *Paragraph = S.actOnParagraphComment(None); 341 if (PC) { 342 S.actOnParamCommandFinish(PC, Paragraph); 343 return PC; 344 } else if (TPC) { 345 S.actOnTParamCommandFinish(TPC, Paragraph); 346 return TPC; 347 } else { 348 S.actOnBlockCommandFinish(BC, Paragraph); 349 return BC; 350 } 351 } 352 353 if (PC || TPC || Info->NumArgs > 0) { 354 // In order to parse command arguments we need to retokenize a few 355 // following text tokens. 356 TextTokenRetokenizer Retokenizer(Allocator, *this); 357 358 if (PC) 359 parseParamCommandArgs(PC, Retokenizer); 360 else if (TPC) 361 parseTParamCommandArgs(TPC, Retokenizer); 362 else 363 parseBlockCommandArgs(BC, Retokenizer, Info->NumArgs); 364 365 Retokenizer.putBackLeftoverTokens(); 366 } 367 368 // If there's a block command ahead, we will attach an empty paragraph to 369 // this command. 370 bool EmptyParagraph = false; 371 if (isTokBlockCommand()) 372 EmptyParagraph = true; 373 else if (Tok.is(tok::newline)) { 374 Token PrevTok = Tok; 375 consumeToken(); 376 EmptyParagraph = isTokBlockCommand(); 377 putBack(PrevTok); 378 } 379 380 ParagraphComment *Paragraph; 381 if (EmptyParagraph) 382 Paragraph = S.actOnParagraphComment(None); 383 else { 384 BlockContentComment *Block = parseParagraphOrBlockCommand(); 385 // Since we have checked for a block command, we should have parsed a 386 // paragraph. 387 Paragraph = cast<ParagraphComment>(Block); 388 } 389 390 if (PC) { 391 S.actOnParamCommandFinish(PC, Paragraph); 392 return PC; 393 } else if (TPC) { 394 S.actOnTParamCommandFinish(TPC, Paragraph); 395 return TPC; 396 } else { 397 S.actOnBlockCommandFinish(BC, Paragraph); 398 return BC; 399 } 400 } 401 402 InlineCommandComment *Parser::parseInlineCommand() { 403 assert(Tok.is(tok::backslash_command) || Tok.is(tok::at_command)); 404 405 const Token CommandTok = Tok; 406 consumeToken(); 407 408 TextTokenRetokenizer Retokenizer(Allocator, *this); 409 410 Token ArgTok; 411 bool ArgTokValid = Retokenizer.lexWord(ArgTok); 412 413 InlineCommandComment *IC; 414 if (ArgTokValid) { 415 IC = S.actOnInlineCommand(CommandTok.getLocation(), 416 CommandTok.getEndLocation(), 417 CommandTok.getCommandID(), 418 ArgTok.getLocation(), 419 ArgTok.getEndLocation(), 420 ArgTok.getText()); 421 } else { 422 IC = S.actOnInlineCommand(CommandTok.getLocation(), 423 CommandTok.getEndLocation(), 424 CommandTok.getCommandID()); 425 426 Diag(CommandTok.getEndLocation().getLocWithOffset(1), 427 diag::warn_doc_inline_contents_no_argument) 428 << CommandTok.is(tok::at_command) 429 << Traits.getCommandInfo(CommandTok.getCommandID())->Name 430 << SourceRange(CommandTok.getLocation(), CommandTok.getEndLocation()); 431 } 432 433 Retokenizer.putBackLeftoverTokens(); 434 435 return IC; 436 } 437 438 HTMLStartTagComment *Parser::parseHTMLStartTag() { 439 assert(Tok.is(tok::html_start_tag)); 440 HTMLStartTagComment *HST = 441 S.actOnHTMLStartTagStart(Tok.getLocation(), 442 Tok.getHTMLTagStartName()); 443 consumeToken(); 444 445 SmallVector<HTMLStartTagComment::Attribute, 2> Attrs; 446 while (true) { 447 switch (Tok.getKind()) { 448 case tok::html_ident: { 449 Token Ident = Tok; 450 consumeToken(); 451 if (Tok.isNot(tok::html_equals)) { 452 Attrs.push_back(HTMLStartTagComment::Attribute(Ident.getLocation(), 453 Ident.getHTMLIdent())); 454 continue; 455 } 456 Token Equals = Tok; 457 consumeToken(); 458 if (Tok.isNot(tok::html_quoted_string)) { 459 Diag(Tok.getLocation(), 460 diag::warn_doc_html_start_tag_expected_quoted_string) 461 << SourceRange(Equals.getLocation()); 462 Attrs.push_back(HTMLStartTagComment::Attribute(Ident.getLocation(), 463 Ident.getHTMLIdent())); 464 while (Tok.is(tok::html_equals) || 465 Tok.is(tok::html_quoted_string)) 466 consumeToken(); 467 continue; 468 } 469 Attrs.push_back(HTMLStartTagComment::Attribute( 470 Ident.getLocation(), 471 Ident.getHTMLIdent(), 472 Equals.getLocation(), 473 SourceRange(Tok.getLocation(), 474 Tok.getEndLocation()), 475 Tok.getHTMLQuotedString())); 476 consumeToken(); 477 continue; 478 } 479 480 case tok::html_greater: 481 S.actOnHTMLStartTagFinish(HST, 482 S.copyArray(llvm::makeArrayRef(Attrs)), 483 Tok.getLocation(), 484 /* IsSelfClosing = */ false); 485 consumeToken(); 486 return HST; 487 488 case tok::html_slash_greater: 489 S.actOnHTMLStartTagFinish(HST, 490 S.copyArray(llvm::makeArrayRef(Attrs)), 491 Tok.getLocation(), 492 /* IsSelfClosing = */ true); 493 consumeToken(); 494 return HST; 495 496 case tok::html_equals: 497 case tok::html_quoted_string: 498 Diag(Tok.getLocation(), 499 diag::warn_doc_html_start_tag_expected_ident_or_greater); 500 while (Tok.is(tok::html_equals) || 501 Tok.is(tok::html_quoted_string)) 502 consumeToken(); 503 if (Tok.is(tok::html_ident) || 504 Tok.is(tok::html_greater) || 505 Tok.is(tok::html_slash_greater)) 506 continue; 507 508 S.actOnHTMLStartTagFinish(HST, 509 S.copyArray(llvm::makeArrayRef(Attrs)), 510 SourceLocation(), 511 /* IsSelfClosing = */ false); 512 return HST; 513 514 default: 515 // Not a token from an HTML start tag. Thus HTML tag prematurely ended. 516 S.actOnHTMLStartTagFinish(HST, 517 S.copyArray(llvm::makeArrayRef(Attrs)), 518 SourceLocation(), 519 /* IsSelfClosing = */ false); 520 bool StartLineInvalid; 521 const unsigned StartLine = SourceMgr.getPresumedLineNumber( 522 HST->getLocation(), 523 &StartLineInvalid); 524 bool EndLineInvalid; 525 const unsigned EndLine = SourceMgr.getPresumedLineNumber( 526 Tok.getLocation(), 527 &EndLineInvalid); 528 if (StartLineInvalid || EndLineInvalid || StartLine == EndLine) 529 Diag(Tok.getLocation(), 530 diag::warn_doc_html_start_tag_expected_ident_or_greater) 531 << HST->getSourceRange(); 532 else { 533 Diag(Tok.getLocation(), 534 diag::warn_doc_html_start_tag_expected_ident_or_greater); 535 Diag(HST->getLocation(), diag::note_doc_html_tag_started_here) 536 << HST->getSourceRange(); 537 } 538 return HST; 539 } 540 } 541 } 542 543 HTMLEndTagComment *Parser::parseHTMLEndTag() { 544 assert(Tok.is(tok::html_end_tag)); 545 Token TokEndTag = Tok; 546 consumeToken(); 547 SourceLocation Loc; 548 if (Tok.is(tok::html_greater)) { 549 Loc = Tok.getLocation(); 550 consumeToken(); 551 } 552 553 return S.actOnHTMLEndTag(TokEndTag.getLocation(), 554 Loc, 555 TokEndTag.getHTMLTagEndName()); 556 } 557 558 BlockContentComment *Parser::parseParagraphOrBlockCommand() { 559 SmallVector<InlineContentComment *, 8> Content; 560 561 while (true) { 562 switch (Tok.getKind()) { 563 case tok::verbatim_block_begin: 564 case tok::verbatim_line_name: 565 case tok::eof: 566 break; // Block content or EOF ahead, finish this parapgaph. 567 568 case tok::unknown_command: 569 Content.push_back(S.actOnUnknownCommand(Tok.getLocation(), 570 Tok.getEndLocation(), 571 Tok.getUnknownCommandName())); 572 consumeToken(); 573 continue; 574 575 case tok::backslash_command: 576 case tok::at_command: { 577 const CommandInfo *Info = Traits.getCommandInfo(Tok.getCommandID()); 578 if (Info->IsBlockCommand) { 579 if (Content.size() == 0) 580 return parseBlockCommand(); 581 break; // Block command ahead, finish this parapgaph. 582 } 583 if (Info->IsVerbatimBlockEndCommand) { 584 Diag(Tok.getLocation(), 585 diag::warn_verbatim_block_end_without_start) 586 << Tok.is(tok::at_command) 587 << Info->Name 588 << SourceRange(Tok.getLocation(), Tok.getEndLocation()); 589 consumeToken(); 590 continue; 591 } 592 if (Info->IsUnknownCommand) { 593 Content.push_back(S.actOnUnknownCommand(Tok.getLocation(), 594 Tok.getEndLocation(), 595 Info->getID())); 596 consumeToken(); 597 continue; 598 } 599 assert(Info->IsInlineCommand); 600 Content.push_back(parseInlineCommand()); 601 continue; 602 } 603 604 case tok::newline: { 605 consumeToken(); 606 if (Tok.is(tok::newline) || Tok.is(tok::eof)) { 607 consumeToken(); 608 break; // Two newlines -- end of paragraph. 609 } 610 // Also allow [tok::newline, tok::text, tok::newline] if the middle 611 // tok::text is just whitespace. 612 if (Tok.is(tok::text) && isWhitespace(Tok.getText())) { 613 Token WhitespaceTok = Tok; 614 consumeToken(); 615 if (Tok.is(tok::newline) || Tok.is(tok::eof)) { 616 consumeToken(); 617 break; 618 } 619 // We have [tok::newline, tok::text, non-newline]. Put back tok::text. 620 putBack(WhitespaceTok); 621 } 622 if (Content.size() > 0) 623 Content.back()->addTrailingNewline(); 624 continue; 625 } 626 627 // Don't deal with HTML tag soup now. 628 case tok::html_start_tag: 629 Content.push_back(parseHTMLStartTag()); 630 continue; 631 632 case tok::html_end_tag: 633 Content.push_back(parseHTMLEndTag()); 634 continue; 635 636 case tok::text: 637 Content.push_back(S.actOnText(Tok.getLocation(), 638 Tok.getEndLocation(), 639 Tok.getText())); 640 consumeToken(); 641 continue; 642 643 case tok::verbatim_block_line: 644 case tok::verbatim_block_end: 645 case tok::verbatim_line_text: 646 case tok::html_ident: 647 case tok::html_equals: 648 case tok::html_quoted_string: 649 case tok::html_greater: 650 case tok::html_slash_greater: 651 llvm_unreachable("should not see this token"); 652 } 653 break; 654 } 655 656 return S.actOnParagraphComment(S.copyArray(llvm::makeArrayRef(Content))); 657 } 658 659 VerbatimBlockComment *Parser::parseVerbatimBlock() { 660 assert(Tok.is(tok::verbatim_block_begin)); 661 662 VerbatimBlockComment *VB = 663 S.actOnVerbatimBlockStart(Tok.getLocation(), 664 Tok.getVerbatimBlockID()); 665 consumeToken(); 666 667 // Don't create an empty line if verbatim opening command is followed 668 // by a newline. 669 if (Tok.is(tok::newline)) 670 consumeToken(); 671 672 SmallVector<VerbatimBlockLineComment *, 8> Lines; 673 while (Tok.is(tok::verbatim_block_line) || 674 Tok.is(tok::newline)) { 675 VerbatimBlockLineComment *Line; 676 if (Tok.is(tok::verbatim_block_line)) { 677 Line = S.actOnVerbatimBlockLine(Tok.getLocation(), 678 Tok.getVerbatimBlockText()); 679 consumeToken(); 680 if (Tok.is(tok::newline)) { 681 consumeToken(); 682 } 683 } else { 684 // Empty line, just a tok::newline. 685 Line = S.actOnVerbatimBlockLine(Tok.getLocation(), ""); 686 consumeToken(); 687 } 688 Lines.push_back(Line); 689 } 690 691 if (Tok.is(tok::verbatim_block_end)) { 692 const CommandInfo *Info = Traits.getCommandInfo(Tok.getVerbatimBlockID()); 693 S.actOnVerbatimBlockFinish(VB, Tok.getLocation(), 694 Info->Name, 695 S.copyArray(llvm::makeArrayRef(Lines))); 696 consumeToken(); 697 } else { 698 // Unterminated \\verbatim block 699 S.actOnVerbatimBlockFinish(VB, SourceLocation(), "", 700 S.copyArray(llvm::makeArrayRef(Lines))); 701 } 702 703 return VB; 704 } 705 706 VerbatimLineComment *Parser::parseVerbatimLine() { 707 assert(Tok.is(tok::verbatim_line_name)); 708 709 Token NameTok = Tok; 710 consumeToken(); 711 712 SourceLocation TextBegin; 713 StringRef Text; 714 // Next token might not be a tok::verbatim_line_text if verbatim line 715 // starting command comes just before a newline or comment end. 716 if (Tok.is(tok::verbatim_line_text)) { 717 TextBegin = Tok.getLocation(); 718 Text = Tok.getVerbatimLineText(); 719 } else { 720 TextBegin = NameTok.getEndLocation(); 721 Text = ""; 722 } 723 724 VerbatimLineComment *VL = S.actOnVerbatimLine(NameTok.getLocation(), 725 NameTok.getVerbatimLineID(), 726 TextBegin, 727 Text); 728 consumeToken(); 729 return VL; 730 } 731 732 BlockContentComment *Parser::parseBlockContent() { 733 switch (Tok.getKind()) { 734 case tok::text: 735 case tok::unknown_command: 736 case tok::backslash_command: 737 case tok::at_command: 738 case tok::html_start_tag: 739 case tok::html_end_tag: 740 return parseParagraphOrBlockCommand(); 741 742 case tok::verbatim_block_begin: 743 return parseVerbatimBlock(); 744 745 case tok::verbatim_line_name: 746 return parseVerbatimLine(); 747 748 case tok::eof: 749 case tok::newline: 750 case tok::verbatim_block_line: 751 case tok::verbatim_block_end: 752 case tok::verbatim_line_text: 753 case tok::html_ident: 754 case tok::html_equals: 755 case tok::html_quoted_string: 756 case tok::html_greater: 757 case tok::html_slash_greater: 758 llvm_unreachable("should not see this token"); 759 } 760 llvm_unreachable("bogus token kind"); 761 } 762 763 FullComment *Parser::parseFullComment() { 764 // Skip newlines at the beginning of the comment. 765 while (Tok.is(tok::newline)) 766 consumeToken(); 767 768 SmallVector<BlockContentComment *, 8> Blocks; 769 while (Tok.isNot(tok::eof)) { 770 Blocks.push_back(parseBlockContent()); 771 772 // Skip extra newlines after paragraph end. 773 while (Tok.is(tok::newline)) 774 consumeToken(); 775 } 776 return S.actOnFullComment(S.copyArray(llvm::makeArrayRef(Blocks))); 777 } 778 779 } // end namespace comments 780 } // end namespace clang 781