Lines Matching refs:tokenizer

508     def _search(self, tokenizer):  argument
535         for i, tok in enumerate(tokenizer.tokens):
571                 s = str(tokenizer)
574             yield start, len(tokenizer.tokens)
585             tokenizer = source
588             tokenizer = CTokenizer(source)
591         for start, end in self._search(tokenizer):
592             new_tokenizer = CTokenizer(tokenizer.tokens[start:end + 1])
618             tokenizer = source
621             tokenizer = CTokenizer(source)
637         for start, end in self._search(tokenizer):
638             new_tokenizer.tokens += tokenizer.tokens[pos:start]
640             new = CTokenizer(tokenizer.tokens[start:end + 1])
650         new_tokenizer.tokens += tokenizer.tokens[pos:]