1 /*********************************************************************** 2 * * 3 * This software is part of the ast package * 4 * Copyright (c) 1985-2007 AT&T Knowledge Ventures * 5 * and is licensed under the * 6 * Common Public License, Version 1.0 * 7 * by AT&T Knowledge Ventures * 8 * * 9 * A copy of the License is available at * 10 * http://www.opensource.org/licenses/cpl1.0.txt * 11 * (with md5 checksum 059e8cd6165cb4c31e351f2b69388fd9) * 12 * * 13 * Information and Software Systems Research * 14 * AT&T Research * 15 * Florham Park NJ * 16 * * 17 * Glenn Fowler <gsf@research.att.com> * 18 * David Korn <dgk@research.att.com> * 19 * Phong Vo <kpv@research.att.com> * 20 * * 21 ***********************************************************************/ 22 #pragma prototyped 23 /* 24 * Glenn Fowler 25 * AT&T Research 26 * 27 * token stream routines 28 */ 29 30 #include <ast.h> 31 #include <tok.h> 32 33 #define FLG_RESTORE 01 /* restore string on close */ 34 #define FLG_NEWLINE 02 /* return newline token next */ 35 36 typedef struct Tok_s /* token stream state */ 37 { 38 union 39 { 40 char* end; /* end ('\0') of last token */ 41 struct Tok_s* nxt; /* next in free list */ 42 } ptr; 43 char chr; /* replace *end with this */ 44 char flg; /* FLG_* */ 45 } Tok_t; 46 47 static Tok_t* freelist; 48 49 /* 50 * open a new token stream on s 51 * if f==0 then string is not restored 52 */ 53 54 char* 55 tokopen(register char* s, int f) 56 { 57 register Tok_t* p; 58 59 if (p = freelist) 60 freelist = freelist->ptr.nxt; 61 else if (!(p = newof(0, Tok_t, 1, 0))) 62 return 0; 63 p->chr = *(p->ptr.end = s); 64 p->flg = f ? FLG_RESTORE : 0; 65 return (char*)p; 66 } 67 68 /* 69 * close a token stream 70 * restore the string to its original state 71 */ 72 73 void 74 tokclose(char* u) 75 { 76 register Tok_t* p = (Tok_t*)u; 77 78 if (p->flg == FLG_RESTORE && *p->ptr.end != p->chr) 79 *p->ptr.end = p->chr; 80 p->ptr.nxt = freelist; 81 freelist = p; 82 } 83 84 /* 85 * return next space separated token 86 * "\n" is returned as a token 87 * 0 returned when no tokens remain 88 * "..." and '...' quotes are honored with \ escapes 89 */ 90 91 char* 92 tokread(char* u) 93 { 94 register Tok_t* p = (Tok_t*)u; 95 register char* s; 96 register char* r; 97 register int q; 98 register int c; 99 100 /* 101 * restore string on each call 102 */ 103 104 if (!p->chr) 105 return 0; 106 s = p->ptr.end; 107 switch (p->flg) 108 { 109 case FLG_NEWLINE: 110 p->flg = 0; 111 return "\n"; 112 case FLG_RESTORE: 113 if (*s != p->chr) 114 *s = p->chr; 115 break; 116 default: 117 if (!*s) 118 s++; 119 break; 120 } 121 122 /* 123 * skip leading space 124 */ 125 126 while (*s == ' ' || *s == '\t') 127 s++; 128 if (!*s) 129 { 130 p->ptr.end = s; 131 p->chr = 0; 132 return 0; 133 } 134 135 /* 136 * find the end of this token 137 */ 138 139 r = s; 140 q = 0; 141 for (;;) 142 switch (c = *r++) 143 { 144 case '\n': 145 if (!q) 146 { 147 if (s == (r - 1)) 148 { 149 if (!p->flg) 150 { 151 p->ptr.end = r; 152 return "\n"; 153 } 154 r++; 155 } 156 else if (!p->flg) 157 p->flg = FLG_NEWLINE; 158 } 159 /*FALLTHROUGH*/ 160 case ' ': 161 case '\t': 162 if (q) 163 break; 164 /*FALLTHROUGH*/ 165 case 0: 166 if (s == --r) 167 { 168 p->ptr.end = r; 169 p->chr = 0; 170 } 171 else 172 { 173 p->chr = *(p->ptr.end = r); 174 if (*r) 175 *r = 0; 176 } 177 return s; 178 case '\\': 179 if (*r) 180 r++; 181 break; 182 case '"': 183 case '\'': 184 if (c == q) 185 q = 0; 186 else if (!q) 187 q = c; 188 break; 189 } 190 } 191