Searched refs:CTokenizer (Results 1 – 3 of 3) sorted by relevance
169 class CTokenizer(): class342 self.sub_tokeninzer = CTokenizer(sub_str)455 new = CTokenizer()584 if isinstance(source, CTokenizer):588 tokenizer = CTokenizer(source)592 new_tokenizer = CTokenizer(tokenizer.tokens[start:end + 1])616 if isinstance(source, CTokenizer):621 tokenizer = CTokenizer(source)627 new_tokenizer = CTokenizer()640 new = CTokenizer(tokenizer.tokens[start:end + 1])
8 from kdoc.c_lex import CMatch, CTokenizer152 source = CTokenizer(source)
17 from kdoc.c_lex import CTokenizer, tokenizer_set_log90 tokens = CTokenizer(text)