lib/AST/CommentParser.cpp

//===--- CommentParser.cpp - Doxygen comment parser -----------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//

#include "clang/AST/CommentParser.h"
#include "clang/AST/CommentCommandTraits.h"
#include "clang/AST/CommentDiagnostic.h"
#include "clang/AST/CommentSema.h"
#include "clang/Basic/CharInfo.h"
#include "clang/Basic/SourceManager.h"
#include "llvm/Support/ErrorHandling.h"

namespace clang {

static inline bool isWhitespace(llvm::StringRef S) {
  for (StringRef::const_iterator I = S.begin(), E = S.end(); I != E; ++I) {
    if (!isWhitespace(*I))
      return false;
  }
  return true;
}

namespace comments {

/// Re-lexes a sequence of tok::text tokens.
class TextTokenRetokenizer {
  llvm::BumpPtrAllocator &Allocator;
  Parser &P;

  /// This flag is set when there are no more tokens we can fetch from lexer.
  bool NoMoreInterestingTokens;

  /// Token buffer: tokens we have processed and lookahead.
  SmallVector<Token, 16> Toks;

  /// A position in \c Toks.
  struct Position {
    const char *BufferStart;
    const char *BufferEnd;
    const char *BufferPtr;
    SourceLocation BufferStartLoc;
    unsigned CurToken;
  };

  /// Current position in Toks.
  Position Pos;

  bool isEnd() const {
    return Pos.CurToken >= Toks.size();
  }

  /// Sets up the buffer pointers to point to current token.
  void setupBuffer() {
    assert(!isEnd());
    const Token &Tok = Toks[Pos.CurToken];

    Pos.BufferStart = Tok.getText().begin();
    Pos.BufferEnd = Tok.getText().end();
    Pos.BufferPtr = Pos.BufferStart;
    Pos.BufferStartLoc = Tok.getLocation();
  }

  SourceLocation getSourceLocation() const {
    const unsigned CharNo = Pos.BufferPtr - Pos.BufferStart;
    return Pos.BufferStartLoc.getLocWithOffset(CharNo);
  }

  char peek() const {
    assert(!isEnd());
    assert(Pos.BufferPtr != Pos.BufferEnd);
    return *Pos.BufferPtr;
  }

  void consumeChar() {
    assert(!isEnd());
    assert(Pos.BufferPtr != Pos.BufferEnd);
    Pos.BufferPtr++;
    if (Pos.BufferPtr == Pos.BufferEnd) {
      Pos.CurToken++;
      if (isEnd() && !addToken())
        return;

      assert(!isEnd());
      setupBuffer();
    }
  }

  /// Extract a template type
  bool lexTemplate(SmallString<32> &WordText) {
    unsigned BracketCount = 0;
    while (!isEnd()) {
      const char C = peek();
      WordText.push_back(C);
      consumeChar();
      switch (C) {
      case '<': {
        BracketCount++;
        break;
      }
      case '>': {
        BracketCount--;
        if (!BracketCount)
          return true;
        break;
      }
      default:
        break;
      }
    }
    return false;
  }

  /// Add a token.
  /// Returns true on success, false if there are no interesting tokens to
  /// fetch from lexer.
  bool addToken() {
    if (NoMoreInterestingTokens)
      return false;

    if (P.Tok.is(tok::newline)) {
      // If we see a single newline token between text tokens, skip it.
      Token Newline = P.Tok;
      P.consumeToken();
      if (P.Tok.isNot(tok::text)) {
        P.putBack(Newline);
        NoMoreInterestingTokens = true;
        return false;
      }
    }
    if (P.Tok.isNot(tok::text)) {
      NoMoreInterestingTokens = true;
      return false;
    }

    Toks.push_back(P.Tok);
    P.consumeToken();
    if (Toks.size() == 1)
      setupBuffer();
    return true;
  }

  void consumeWhitespace() {
    while (!isEnd()) {
      if (isWhitespace(peek()))
        consumeChar();
      else
        break;
    }
  }

  void formTokenWithChars(Token &Result,
                          SourceLocation Loc,
                          const char *TokBegin,
                          unsigned TokLength,
                          StringRef Text) {
    Result.setLocation(Loc);
    Result.setKind(tok::text);
    Result.setLength(TokLength);
#ifndef NDEBUG
    Result.TextPtr = "<UNSET>";
    Result.IntVal = 7;
#endif
    Result.setText(Text);
  }

public:
  TextTokenRetokenizer(llvm::BumpPtrAllocator &Allocator, Parser &P):
      Allocator(Allocator), P(P), NoMoreInterestingTokens(false) {
    Pos.CurToken = 0;
    addToken();
  }

  /// Extract a type argument
  bool lexType(Token &Tok) {
    if (isEnd())
      return false;

    // Save current position in case we need to rollback because the type is
    // empty.
    Position SavedPos = Pos;

    // Consume any leading whitespace.
    consumeWhitespace();
    SmallString<32> WordText;
    const char *WordBegin = Pos.BufferPtr;
    SourceLocation Loc = getSourceLocation();

    while (!isEnd()) {
      const char C = peek();
      // For non-whitespace characters we check if it's a template or otherwise
      // continue reading the text into a word.
      if (!isWhitespace(C)) {
        if (C == '<') {
          if (!lexTemplate(WordText))
            return false;
        } else {
          WordText.push_back(C);
          consumeChar();
        }
      } else {
        consumeChar();
        break;
      }
    }

    const unsigned Length = WordText.size();
    if (Length == 0) {
      Pos = SavedPos;
      return false;
    }

    char *TextPtr = Allocator.Allocate<char>(Length + 1);

    memcpy(TextPtr, WordText.c_str(), Length + 1);
    StringRef Text = StringRef(TextPtr, Length);

    formTokenWithChars(Tok, Loc, WordBegin, Length, Text);
    return true;
  }

  // Check if this line starts with @par or \par
  bool startsWithParCommand() {
    unsigned Offset = 1;

    // Skip all whitespace characters at the beginning.
    // This needs to backtrack because Pos has already advanced past the
    // actual \par or @par command by the time this function is called.
    while (isWhitespace(*(Pos.BufferPtr - Offset)))
      Offset++;

    // Once we've reached the whitespace, backtrack and check if the previous
    // four characters are \par or @par.
    llvm::StringRef LineStart(Pos.BufferPtr - Offset - 3, 4);
    return LineStart.starts_with("\\par") || LineStart.starts_with("@par");
  }

  /// Extract a par command argument-header.
  bool lexParHeading(Token &Tok) {
    if (isEnd())
      return false;

    Position SavedPos = Pos;

    consumeWhitespace();
    SmallString<32> WordText;
    const char *WordBegin = Pos.BufferPtr;
    SourceLocation Loc = getSourceLocation();

    if (!startsWithParCommand())
      return false;

    // Read until the end of this token, which is effectively the end of the
    // line. This gets us the content of the par header, if there is one.
    while (!isEnd()) {
      WordText.push_back(peek());
      if (Pos.BufferPtr + 1 == Pos.BufferEnd) {
        consumeChar();
        break;
      }
      consumeChar();
    }

    unsigned Length = WordText.size();
    if (Length == 0) {
      Pos = SavedPos;
      return false;
    }

    char *TextPtr = Allocator.Allocate<char>(Length + 1);

    memcpy(TextPtr, WordText.c_str(), Length + 1);
    StringRef Text = StringRef(TextPtr, Length);

    formTokenWithChars(Tok, Loc, WordBegin, Length, Text);
    return true;
  }

  /// Extract a word -- sequence of non-whitespace characters.
  bool lexWord(Token &Tok) {
    if (isEnd())
      return false;

    Position SavedPos = Pos;

    consumeWhitespace();
    SmallString<32> WordText;
    const char *WordBegin = Pos.BufferPtr;
    SourceLocation Loc = getSourceLocation();
    while (!isEnd()) {
      const char C = peek();
      if (!isWhitespace(C)) {
        WordText.push_back(C);
        consumeChar();
      } else
        break;
    }
    const unsigned Length = WordText.size();
    if (Length == 0) {
      Pos = SavedPos;
      return false;
    }

    char *TextPtr = Allocator.Allocate<char>(Length + 1);

    memcpy(TextPtr, WordText.c_str(), Length + 1);
    StringRef Text = StringRef(TextPtr, Length);

    formTokenWithChars(Tok, Loc, WordBegin, Length, Text);
    return true;
  }

  bool lexDelimitedSeq(Token &Tok, char OpenDelim, char CloseDelim) {
    if (isEnd())
      return false;

    Position SavedPos = Pos;

    consumeWhitespace();
    SmallString<32> WordText;
    const char *WordBegin = Pos.BufferPtr;
    SourceLocation Loc = getSourceLocation();
    bool Error = false;
    if (!isEnd()) {
      const char C = peek();
      if (C == OpenDelim) {
        WordText.push_back(C);
        consumeChar();
      } else
        Error = true;
    }
    char C = '\0';
    while (!Error && !isEnd()) {
      C = peek();
      WordText.push_back(C);
      consumeChar();
      if (C == CloseDelim)
        break;
    }
    if (!Error && C != CloseDelim)
      Error = true;

    if (Error) {
      Pos = SavedPos;
      return false;
    }

    const unsigned Length = WordText.size();
    char *TextPtr = Allocator.Allocate<char>(Length + 1);

    memcpy(TextPtr, WordText.c_str(), Length + 1);
    StringRef Text = StringRef(TextPtr, Length);

    formTokenWithChars(Tok, Loc, WordBegin,
                       Pos.BufferPtr - WordBegin, Text);
    return true;
  }

  /// Put back tokens that we didn't consume.
  void putBackLeftoverTokens() {
    if (isEnd())
      return;

    bool HavePartialTok = false;
    Token PartialTok;
    if (Pos.BufferPtr != Pos.BufferStart) {
      formTokenWithChars(PartialTok, getSourceLocation(),
                         Pos.BufferPtr, Pos.BufferEnd - Pos.BufferPtr,
                         StringRef(Pos.BufferPtr,
                                   Pos.BufferEnd - Pos.BufferPtr));
      HavePartialTok = true;
      Pos.CurToken++;
    }

    P.putBack(llvm::ArrayRef(Toks.begin() + Pos.CurToken, Toks.end()));
    Pos.CurToken = Toks.size();

    if (HavePartialTok)
      P.putBack(PartialTok);
  }
};

Parser::Parser(Lexer &L, Sema &S, llvm::BumpPtrAllocator &Allocator,
               const SourceManager &SourceMgr, DiagnosticsEngine &Diags,
               const CommandTraits &Traits):
    L(L), S(S), Allocator(Allocator), SourceMgr(SourceMgr), Diags(Diags),
    Traits(Traits) {
  consumeToken();
}

void Parser::parseParamCommandArgs(ParamCommandComment *PC,
                                   TextTokenRetokenizer &Retokenizer) {
  Token Arg;
  // Check if argument looks like direction specification: [dir]
  // e.g., [in], [out], [in,out]
  if (Retokenizer.lexDelimitedSeq(Arg, '[', ']'))
    S.actOnParamCommandDirectionArg(PC,
                                    Arg.getLocation(),
                                    Arg.getEndLocation(),
                                    Arg.getText());

  if (Retokenizer.lexWord(Arg))
    S.actOnParamCommandParamNameArg(PC,
                                    Arg.getLocation(),
                                    Arg.getEndLocation(),
                                    Arg.getText());
}

void Parser::parseTParamCommandArgs(TParamCommandComment *TPC,
                                    TextTokenRetokenizer &Retokenizer) {
  Token Arg;
  if (Retokenizer.lexWord(Arg))
    S.actOnTParamCommandParamNameArg(TPC,
                                     Arg.getLocation(),
                                     Arg.getEndLocation(),
                                     Arg.getText());
}

ArrayRef<Comment::Argument>
Parser::parseCommandArgs(TextTokenRetokenizer &Retokenizer, unsigned NumArgs) {
  auto *Args = new (Allocator.Allocate<Comment::Argument>(NumArgs))
      Comment::Argument[NumArgs];
  unsigned ParsedArgs = 0;
  Token Arg;
  while (ParsedArgs < NumArgs && Retokenizer.lexWord(Arg)) {
    Args[ParsedArgs] = Comment::Argument{
        SourceRange(Arg.getLocation(), Arg.getEndLocation()), Arg.getText()};
    ParsedArgs++;
  }

  return llvm::ArrayRef(Args, ParsedArgs);
}

ArrayRef<Comment::Argument>
Parser::parseThrowCommandArgs(TextTokenRetokenizer &Retokenizer,
                              unsigned NumArgs) {
  auto *Args = new (Allocator.Allocate<Comment::Argument>(NumArgs))
      Comment::Argument[NumArgs];
  unsigned ParsedArgs = 0;
  Token Arg;

  while (ParsedArgs < NumArgs && Retokenizer.lexType(Arg)) {
    Args[ParsedArgs] = Comment::Argument{
        SourceRange(Arg.getLocation(), Arg.getEndLocation()), Arg.getText()};
    ParsedArgs++;
  }

  return llvm::ArrayRef(Args, ParsedArgs);
}

ArrayRef<Comment::Argument>
Parser::parseParCommandArgs(TextTokenRetokenizer &Retokenizer,
                            unsigned NumArgs) {
  assert(NumArgs > 0);
  auto *Args = new (Allocator.Allocate<Comment::Argument>(NumArgs))
      Comment::Argument[NumArgs];
  unsigned ParsedArgs = 0;
  Token Arg;

  while (ParsedArgs < NumArgs && Retokenizer.lexParHeading(Arg)) {
    Args[ParsedArgs] = Comment::Argument{
        SourceRange(Arg.getLocation(), Arg.getEndLocation()), Arg.getText()};
    ParsedArgs++;
  }

  return llvm::ArrayRef(Args, ParsedArgs);
}

BlockCommandComment *Parser::parseBlockCommand() {
  assert(Tok.is(tok::backslash_command) || Tok.is(tok::at_command));

  ParamCommandComment *PC = nullptr;
  TParamCommandComment *TPC = nullptr;
  BlockCommandComment *BC = nullptr;
  const CommandInfo *Info = Traits.getCommandInfo(Tok.getCommandID());
  CommandMarkerKind CommandMarker =
      Tok.is(tok::backslash_command) ? CMK_Backslash : CMK_At;
  if (Info->IsParamCommand) {
    PC = S.actOnParamCommandStart(Tok.getLocation(),
                                  Tok.getEndLocation(),
                                  Tok.getCommandID(),
                                  CommandMarker);
  } else if (Info->IsTParamCommand) {
    TPC = S.actOnTParamCommandStart(Tok.getLocation(),
                                    Tok.getEndLocation(),
                                    Tok.getCommandID(),
                                    CommandMarker);
  } else {
    BC = S.actOnBlockCommandStart(Tok.getLocation(),
                                  Tok.getEndLocation(),
                                  Tok.getCommandID(),
                                  CommandMarker);
  }
  consumeToken();

  if (isTokBlockCommand()) {
    // Block command ahead.  We can't nest block commands, so pretend that this
    // command has an empty argument.
    ParagraphComment *Paragraph = S.actOnParagraphComment(std::nullopt);
    if (PC) {
      S.actOnParamCommandFinish(PC, Paragraph);
      return PC;
    } else if (TPC) {
      S.actOnTParamCommandFinish(TPC, Paragraph);
      return TPC;
    } else {
      S.actOnBlockCommandFinish(BC, Paragraph);
      return BC;
    }
  }

  if (PC || TPC || Info->NumArgs > 0) {
    // In order to parse command arguments we need to retokenize a few
    // following text tokens.
    TextTokenRetokenizer Retokenizer(Allocator, *this);

    if (PC)
      parseParamCommandArgs(PC, Retokenizer);
    else if (TPC)
      parseTParamCommandArgs(TPC, Retokenizer);
    else if (Info->IsThrowsCommand)
      S.actOnBlockCommandArgs(
          BC, parseThrowCommandArgs(Retokenizer, Info->NumArgs));
    else if (Info->IsParCommand)
      S.actOnBlockCommandArgs(BC,
                              parseParCommandArgs(Retokenizer, Info->NumArgs));
    else
      S.actOnBlockCommandArgs(BC, parseCommandArgs(Retokenizer, Info->NumArgs));

    Retokenizer.putBackLeftoverTokens();
  }

  // If there's a block command ahead, we will attach an empty paragraph to
  // this command.
  bool EmptyParagraph = false;
  if (isTokBlockCommand())
    EmptyParagraph = true;
  else if (Tok.is(tok::newline)) {
    Token PrevTok = Tok;
    consumeToken();
    EmptyParagraph = isTokBlockCommand();
    putBack(PrevTok);
  }

  ParagraphComment *Paragraph;
  if (EmptyParagraph)
    Paragraph = S.actOnParagraphComment(std::nullopt);
  else {
    BlockContentComment *Block = parseParagraphOrBlockCommand();
    // Since we have checked for a block command, we should have parsed a
    // paragraph.
    Paragraph = cast<ParagraphComment>(Block);
  }

  if (PC) {
    S.actOnParamCommandFinish(PC, Paragraph);
    return PC;
  } else if (TPC) {
    S.actOnTParamCommandFinish(TPC, Paragraph);
    return TPC;
  } else {
    S.actOnBlockCommandFinish(BC, Paragraph);
    return BC;
  }
}

InlineCommandComment *Parser::parseInlineCommand() {
  assert(Tok.is(tok::backslash_command) || Tok.is(tok::at_command));
  const CommandInfo *Info = Traits.getCommandInfo(Tok.getCommandID());

  const Token CommandTok = Tok;
  consumeToken();

  TextTokenRetokenizer Retokenizer(Allocator, *this);
  ArrayRef<Comment::Argument> Args =
      parseCommandArgs(Retokenizer, Info->NumArgs);

  InlineCommandComment *IC = S.actOnInlineCommand(
      CommandTok.getLocation(), CommandTok.getEndLocation(),
      CommandTok.getCommandID(), Args);

  if (Args.size() < Info->NumArgs) {
    Diag(CommandTok.getEndLocation().getLocWithOffset(1),
         diag::warn_doc_inline_command_not_enough_arguments)
        << CommandTok.is(tok::at_command) << Info->Name << Args.size()
        << Info->NumArgs
        << SourceRange(CommandTok.getLocation(), CommandTok.getEndLocation());
  }

  Retokenizer.putBackLeftoverTokens();

  return IC;
}

HTMLStartTagComment *Parser::parseHTMLStartTag() {
  assert(Tok.is(tok::html_start_tag));
  HTMLStartTagComment *HST =
      S.actOnHTMLStartTagStart(Tok.getLocation(),
                               Tok.getHTMLTagStartName());
  consumeToken();

  SmallVector<HTMLStartTagComment::Attribute, 2> Attrs;
  while (true) {
    switch (Tok.getKind()) {
    case tok::html_ident: {
      Token Ident = Tok;
      consumeToken();
      if (Tok.isNot(tok::html_equals)) {
        Attrs.push_back(HTMLStartTagComment::Attribute(Ident.getLocation(),
                                                       Ident.getHTMLIdent()));
        continue;
      }
      Token Equals = Tok;
      consumeToken();
      if (Tok.isNot(tok::html_quoted_string)) {
        Diag(Tok.getLocation(),
             diag::warn_doc_html_start_tag_expected_quoted_string)
          << SourceRange(Equals.getLocation());
        Attrs.push_back(HTMLStartTagComment::Attribute(Ident.getLocation(),
                                                       Ident.getHTMLIdent()));
        while (Tok.is(tok::html_equals) ||
               Tok.is(tok::html_quoted_string))
          consumeToken();
        continue;
      }
      Attrs.push_back(HTMLStartTagComment::Attribute(
                              Ident.getLocation(),
                              Ident.getHTMLIdent(),
                              Equals.getLocation(),
                              SourceRange(Tok.getLocation(),
                                          Tok.getEndLocation()),
                              Tok.getHTMLQuotedString()));
      consumeToken();
      continue;
    }

    case tok::html_greater:
      S.actOnHTMLStartTagFinish(HST, S.copyArray(llvm::ArrayRef(Attrs)),
                                Tok.getLocation(),
                                /* IsSelfClosing = */ false);
      consumeToken();
      return HST;

    case tok::html_slash_greater:
      S.actOnHTMLStartTagFinish(HST, S.copyArray(llvm::ArrayRef(Attrs)),
                                Tok.getLocation(),
                                /* IsSelfClosing = */ true);
      consumeToken();
      return HST;

    case tok::html_equals:
    case tok::html_quoted_string:
      Diag(Tok.getLocation(),
           diag::warn_doc_html_start_tag_expected_ident_or_greater);
      while (Tok.is(tok::html_equals) ||
             Tok.is(tok::html_quoted_string))
        consumeToken();
      if (Tok.is(tok::html_ident) ||
          Tok.is(tok::html_greater) ||
          Tok.is(tok::html_slash_greater))
        continue;

      S.actOnHTMLStartTagFinish(HST, S.copyArray(llvm::ArrayRef(Attrs)),
                                SourceLocation(),
                                /* IsSelfClosing = */ false);
      return HST;

    default:
      // Not a token from an HTML start tag.  Thus HTML tag prematurely ended.
      S.actOnHTMLStartTagFinish(HST, S.copyArray(llvm::ArrayRef(Attrs)),
                                SourceLocation(),
                                /* IsSelfClosing = */ false);
      bool StartLineInvalid;
      const unsigned StartLine = SourceMgr.getPresumedLineNumber(
                                                  HST->getLocation(),
                                                  &StartLineInvalid);
      bool EndLineInvalid;
      const unsigned EndLine = SourceMgr.getPresumedLineNumber(
                                                  Tok.getLocation(),
                                                  &EndLineInvalid);
      if (StartLineInvalid || EndLineInvalid || StartLine == EndLine)
        Diag(Tok.getLocation(),
             diag::warn_doc_html_start_tag_expected_ident_or_greater)
          << HST->getSourceRange();
      else {
        Diag(Tok.getLocation(),
             diag::warn_doc_html_start_tag_expected_ident_or_greater);
        Diag(HST->getLocation(), diag::note_doc_html_tag_started_here)
          << HST->getSourceRange();
      }
      return HST;
    }
  }
}

HTMLEndTagComment *Parser::parseHTMLEndTag() {
  assert(Tok.is(tok::html_end_tag));
  Token TokEndTag = Tok;
  consumeToken();
  SourceLocation Loc;
  if (Tok.is(tok::html_greater)) {
    Loc = Tok.getLocation();
    consumeToken();
  }

  return S.actOnHTMLEndTag(TokEndTag.getLocation(),
                           Loc,
                           TokEndTag.getHTMLTagEndName());
}

BlockContentComment *Parser::parseParagraphOrBlockCommand() {
  SmallVector<InlineContentComment *, 8> Content;

  while (true) {
    switch (Tok.getKind()) {
    case tok::verbatim_block_begin:
    case tok::verbatim_line_name:
    case tok::eof:
      break; // Block content or EOF ahead, finish this parapgaph.

    case tok::unknown_command:
      Content.push_back(S.actOnUnknownCommand(Tok.getLocation(),
                                              Tok.getEndLocation(),
                                              Tok.getUnknownCommandName()));
      consumeToken();
      continue;

    case tok::backslash_command:
    case tok::at_command: {
      const CommandInfo *Info = Traits.getCommandInfo(Tok.getCommandID());
      if (Info->IsBlockCommand) {
        if (Content.size() == 0)
          return parseBlockCommand();
        break; // Block command ahead, finish this parapgaph.
      }
      if (Info->IsVerbatimBlockEndCommand) {
        Diag(Tok.getLocation(),
             diag::warn_verbatim_block_end_without_start)
          << Tok.is(tok::at_command)
          << Info->Name
          << SourceRange(Tok.getLocation(), Tok.getEndLocation());
        consumeToken();
        continue;
      }
      if (Info->IsUnknownCommand) {
        Content.push_back(S.actOnUnknownCommand(Tok.getLocation(),
                                                Tok.getEndLocation(),
                                                Info->getID()));
        consumeToken();
        continue;
      }
      assert(Info->IsInlineCommand);
      Content.push_back(parseInlineCommand());
      continue;
    }

    case tok::newline: {
      consumeToken();
      if (Tok.is(tok::newline) || Tok.is(tok::eof)) {
        consumeToken();
        break; // Two newlines -- end of paragraph.
      }
      // Also allow [tok::newline, tok::text, tok::newline] if the middle
      // tok::text is just whitespace.
      if (Tok.is(tok::text) && isWhitespace(Tok.getText())) {
        Token WhitespaceTok = Tok;
        consumeToken();
        if (Tok.is(tok::newline) || Tok.is(tok::eof)) {
          consumeToken();
          break;
        }
        // We have [tok::newline, tok::text, non-newline].  Put back tok::text.
        putBack(WhitespaceTok);
      }
      if (Content.size() > 0)
        Content.back()->addTrailingNewline();
      continue;
    }

    // Don't deal with HTML tag soup now.
    case tok::html_start_tag:
      Content.push_back(parseHTMLStartTag());
      continue;

    case tok::html_end_tag:
      Content.push_back(parseHTMLEndTag());
      continue;

    case tok::text:
      Content.push_back(S.actOnText(Tok.getLocation(),
                                    Tok.getEndLocation(),
                                    Tok.getText()));
      consumeToken();
      continue;

    case tok::verbatim_block_line:
    case tok::verbatim_block_end:
    case tok::verbatim_line_text:
    case tok::html_ident:
    case tok::html_equals:
    case tok::html_quoted_string:
    case tok::html_greater:
    case tok::html_slash_greater:
      llvm_unreachable("should not see this token");
    }
    break;
  }

  return S.actOnParagraphComment(S.copyArray(llvm::ArrayRef(Content)));
}

VerbatimBlockComment *Parser::parseVerbatimBlock() {
  assert(Tok.is(tok::verbatim_block_begin));

  VerbatimBlockComment *VB =
      S.actOnVerbatimBlockStart(Tok.getLocation(),
                                Tok.getVerbatimBlockID());
  consumeToken();

  // Don't create an empty line if verbatim opening command is followed
  // by a newline.
  if (Tok.is(tok::newline))
    consumeToken();

  SmallVector<VerbatimBlockLineComment *, 8> Lines;
  while (Tok.is(tok::verbatim_block_line) ||
         Tok.is(tok::newline)) {
    VerbatimBlockLineComment *Line;
    if (Tok.is(tok::verbatim_block_line)) {
      Line = S.actOnVerbatimBlockLine(Tok.getLocation(),
                                      Tok.getVerbatimBlockText());
      consumeToken();
      if (Tok.is(tok::newline)) {
        consumeToken();
      }
    } else {
      // Empty line, just a tok::newline.
      Line = S.actOnVerbatimBlockLine(Tok.getLocation(), "");
      consumeToken();
    }
    Lines.push_back(Line);
  }

  if (Tok.is(tok::verbatim_block_end)) {
    const CommandInfo *Info = Traits.getCommandInfo(Tok.getVerbatimBlockID());
    S.actOnVerbatimBlockFinish(VB, Tok.getLocation(), Info->Name,
                               S.copyArray(llvm::ArrayRef(Lines)));
    consumeToken();
  } else {
    // Unterminated \\verbatim block
    S.actOnVerbatimBlockFinish(VB, SourceLocation(), "",
                               S.copyArray(llvm::ArrayRef(Lines)));
  }

  return VB;
}

VerbatimLineComment *Parser::parseVerbatimLine() {
  assert(Tok.is(tok::verbatim_line_name));

  Token NameTok = Tok;
  consumeToken();

  SourceLocation TextBegin;
  StringRef Text;
  // Next token might not be a tok::verbatim_line_text if verbatim line
  // starting command comes just before a newline or comment end.
  if (Tok.is(tok::verbatim_line_text)) {
    TextBegin = Tok.getLocation();
    Text = Tok.getVerbatimLineText();
  } else {
    TextBegin = NameTok.getEndLocation();
    Text = "";
  }

  VerbatimLineComment *VL = S.actOnVerbatimLine(NameTok.getLocation(),
                                                NameTok.getVerbatimLineID(),
                                                TextBegin,
                                                Text);
  consumeToken();
  return VL;
}

BlockContentComment *Parser::parseBlockContent() {
  switch (Tok.getKind()) {
  case tok::text:
  case tok::unknown_command:
  case tok::backslash_command:
  case tok::at_command:
  case tok::html_start_tag:
  case tok::html_end_tag:
    return parseParagraphOrBlockCommand();

  case tok::verbatim_block_begin:
    return parseVerbatimBlock();

  case tok::verbatim_line_name:
    return parseVerbatimLine();

  case tok::eof:
  case tok::newline:
  case tok::verbatim_block_line:
  case tok::verbatim_block_end:
  case tok::verbatim_line_text:
  case tok::html_ident:
  case tok::html_equals:
  case tok::html_quoted_string:
  case tok::html_greater:
  case tok::html_slash_greater:
    llvm_unreachable("should not see this token");
  }
  llvm_unreachable("bogus token kind");
}

FullComment *Parser::parseFullComment() {
  // Skip newlines at the beginning of the comment.
  while (Tok.is(tok::newline))
    consumeToken();

  SmallVector<BlockContentComment *, 8> Blocks;
  while (Tok.isNot(tok::eof)) {
    Blocks.push_back(parseBlockContent());

    // Skip extra newlines after paragraph end.
    while (Tok.is(tok::newline))
      consumeToken();
  }
  return S.actOnFullComment(S.copyArray(llvm::ArrayRef(Blocks)));
}

} // end namespace comments
} // end namespace clang