Lines Matching refs:TokenPtr
274 const char *TokenPtr = BufferPtr; in skipTextToken() local
275 assert(TokenPtr < CommentEnd); in skipTextToken()
280 StringRef(TokenPtr, CommentEnd - TokenPtr).find_first_of(TokStartSymbols); in skipTextToken()
286 if (*(TokenPtr + End) == '\"') { in skipTextToken()
287 TokenPtr += End + 1; in skipTextToken()
288 End = StringRef(TokenPtr, CommentEnd - TokenPtr).find_first_of("\n\r\""); in skipTextToken()
289 if (End != StringRef::npos && *(TokenPtr + End) == '\"') in skipTextToken()
290 TokenPtr += End + 1; in skipTextToken()
293 return TokenPtr + End; in skipTextToken()
304 const char *TokenPtr = BufferPtr; in lexCommentText() local
305 assert(TokenPtr < CommentEnd); in lexCommentText()
306 switch (*TokenPtr) { in lexCommentText()
309 TokenPtr = skipNewline(TokenPtr, CommentEnd); in lexCommentText()
310 formTokenWithChars(T, TokenPtr, tok::newline); in lexCommentText()
345 const char *TokenPtr = BufferPtr; in lexCommentText() local
346 assert(TokenPtr < CommentEnd); in lexCommentText()
347 switch(*TokenPtr) { in lexCommentText()
354 (*TokenPtr == '@') ? tok::at_command : tok::backslash_command; in lexCommentText()
355 TokenPtr++; in lexCommentText()
356 if (TokenPtr == CommentEnd) { in lexCommentText()
357 formTextToken(T, TokenPtr); in lexCommentText()
360 char C = *TokenPtr; in lexCommentText()
369 TokenPtr++; in lexCommentText()
370 if (C == ':' && TokenPtr != CommentEnd && *TokenPtr == ':') { in lexCommentText()
372 TokenPtr++; in lexCommentText()
374 StringRef UnescapedText(BufferPtr + 1, TokenPtr - (BufferPtr + 1)); in lexCommentText()
375 formTokenWithChars(T, TokenPtr, tok::text); in lexCommentText()
381 if (!isCommandNameStartCharacter(*TokenPtr)) { in lexCommentText()
382 formTextToken(T, TokenPtr); in lexCommentText()
386 TokenPtr = skipCommandName(TokenPtr, CommentEnd); in lexCommentText()
387 unsigned Length = TokenPtr - (BufferPtr + 1); in lexCommentText()
391 if (Length == 1 && TokenPtr[-1] == 'f' && TokenPtr != CommentEnd) { in lexCommentText()
392 C = *TokenPtr; in lexCommentText()
395 TokenPtr++; in lexCommentText()
407 SourceLocation EndLoc = getSourceLocation(TokenPtr); in lexCommentText()
414 formTokenWithChars(T, TokenPtr, tok::unknown_command); in lexCommentText()
422 setupAndLexVerbatimBlock(T, TokenPtr, *BufferPtr, Info); in lexCommentText()
426 setupAndLexVerbatimLine(T, TokenPtr, Info); in lexCommentText()
429 formTokenWithChars(T, TokenPtr, CommandKind); in lexCommentText()
439 TokenPtr++; in lexCommentText()
440 if (TokenPtr == CommentEnd) { in lexCommentText()
441 formTextToken(T, TokenPtr); in lexCommentText()
444 const char C = *TokenPtr; in lexCommentText()
450 formTextToken(T, TokenPtr); in lexCommentText()
566 const char *TokenPtr = BufferPtr; in lexHTMLCharacterReference() local
567 assert(*TokenPtr == '&'); in lexHTMLCharacterReference()
568 TokenPtr++; in lexHTMLCharacterReference()
569 if (TokenPtr == CommentEnd) { in lexHTMLCharacterReference()
570 formTextToken(T, TokenPtr); in lexHTMLCharacterReference()
576 char C = *TokenPtr; in lexHTMLCharacterReference()
578 NamePtr = TokenPtr; in lexHTMLCharacterReference()
579 TokenPtr = skipNamedCharacterReference(TokenPtr, CommentEnd); in lexHTMLCharacterReference()
582 TokenPtr++; in lexHTMLCharacterReference()
583 if (TokenPtr == CommentEnd) { in lexHTMLCharacterReference()
584 formTextToken(T, TokenPtr); in lexHTMLCharacterReference()
587 C = *TokenPtr; in lexHTMLCharacterReference()
589 NamePtr = TokenPtr; in lexHTMLCharacterReference()
590 TokenPtr = skipDecimalCharacterReference(TokenPtr, CommentEnd); in lexHTMLCharacterReference()
593 TokenPtr++; in lexHTMLCharacterReference()
594 NamePtr = TokenPtr; in lexHTMLCharacterReference()
595 TokenPtr = skipHexCharacterReference(TokenPtr, CommentEnd); in lexHTMLCharacterReference()
597 formTextToken(T, TokenPtr); in lexHTMLCharacterReference()
601 formTextToken(T, TokenPtr); in lexHTMLCharacterReference()
604 if (NamePtr == TokenPtr || TokenPtr == CommentEnd || in lexHTMLCharacterReference()
605 *TokenPtr != ';') { in lexHTMLCharacterReference()
606 formTextToken(T, TokenPtr); in lexHTMLCharacterReference()
609 StringRef Name(NamePtr, TokenPtr - NamePtr); in lexHTMLCharacterReference()
610 TokenPtr++; // Skip semicolon. in lexHTMLCharacterReference()
620 formTextToken(T, TokenPtr); in lexHTMLCharacterReference()
623 formTokenWithChars(T, TokenPtr, tok::text); in lexHTMLCharacterReference()
651 const char *TokenPtr = BufferPtr; in lexHTMLStartTag() local
652 char C = *TokenPtr; in lexHTMLStartTag()
654 TokenPtr = skipHTMLIdentifier(TokenPtr, CommentEnd); in lexHTMLStartTag()
655 StringRef Ident(BufferPtr, TokenPtr - BufferPtr); in lexHTMLStartTag()
656 formTokenWithChars(T, TokenPtr, tok::html_ident); in lexHTMLStartTag()
661 TokenPtr++; in lexHTMLStartTag()
662 formTokenWithChars(T, TokenPtr, tok::html_equals); in lexHTMLStartTag()
666 const char *OpenQuote = TokenPtr; in lexHTMLStartTag()
667 TokenPtr = skipHTMLQuotedString(TokenPtr, CommentEnd); in lexHTMLStartTag()
668 const char *ClosingQuote = TokenPtr; in lexHTMLStartTag()
669 if (TokenPtr != CommentEnd) // Skip closing quote. in lexHTMLStartTag()
670 TokenPtr++; in lexHTMLStartTag()
671 formTokenWithChars(T, TokenPtr, tok::html_quoted_string); in lexHTMLStartTag()
677 TokenPtr++; in lexHTMLStartTag()
678 formTokenWithChars(T, TokenPtr, tok::html_greater); in lexHTMLStartTag()
682 TokenPtr++; in lexHTMLStartTag()
683 if (TokenPtr != CommentEnd && *TokenPtr == '>') { in lexHTMLStartTag()
684 TokenPtr++; in lexHTMLStartTag()
685 formTokenWithChars(T, TokenPtr, tok::html_slash_greater); in lexHTMLStartTag()
687 formTextToken(T, TokenPtr); in lexHTMLStartTag()