Lines Matching refs:tokens
1499 tokens = collections.deque(_scan(string))
1501 def get_subscripted(tokens): argument
1507 if len(tokens) == 0 or tokens[0].ttype != 'word':
1509 if len(tokens) > 1 and tokens[1].ttype == 'type':
1510 word = tokens.popleft()
1511 return word, tokens.popleft()
1592 while tokens:
1593 token = tokens.popleft()
1601 if len(tokens) == 0:
1603 cond = tokens.popleft()
1616 if len(tokens) == 0 or tokens[0].ttype != '(':
1618 tokens.popleft()
1619 repeat = get_subscripted(tokens)
1622 if len(tokens) == 0 or tokens[0].ttype != ')':
1625 tokens.popleft()
1636 if len(tokens) == 0 or tokens[0].ttype != 'type':
1638 type_or_size = tokens.popleft()
1645 if len(tokens) > 0 and tokens[0].ttype == 'aux':
1646 aux = tokens.popleft()