xref: /illumos-gate/usr/src/cmd/fm/eversholt/common/esclex.c (revision bea83d026ee1bd1b2a2419e1d0232f107a5d7d9b)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  *
25  * esclex.c -- lexer for esc
26  *
27  * this module provides lexical analysis and error handling routine
28  * expected by the yacc-generated parser (i.e. yylex() and yyerror()).
29  * it also does lots of tracking of things like filenames, line numbers,
30  * and what tokens are seen on a line up to the point where a syntax error
31  * was found.  this module also arranges for the input source files to
32  * be run through cpp.
33  */
34 
35 #pragma ident	"%Z%%M%	%I%	%E% SMI"
36 
37 #include <stdio.h>
38 #include <ctype.h>
39 #include <string.h>
40 #include <stdlib.h>
41 #include <unistd.h>
42 #include <time.h>
43 #include <errno.h>
44 #include "out.h"
45 #include "alloc.h"
46 #include "stats.h"
47 #include "stable.h"
48 #include "lut.h"
49 #include "literals.h"
50 #include "tree.h"
51 #include "esclex.h"
52 #include "eftread.h"
53 #include "check.h"
54 #include "y.tab.h"
55 
56 /* ridiculously long token buffer -- disallow any token longer than this */
57 #define	MAXTOK	8192
58 static char Tok[MAXTOK];
59 
60 /* some misc stats we keep on the lexer & parser */
61 static struct stats *Tokcount;
62 static struct stats *Lexelapse;
63 struct stats *Filecount;
64 struct filestats {
65 	struct filestats *next;
66 	struct stats *stats;
67 	struct stats *idstats;
68 } *Fstats;
69 
70 static int Errcount;
71 
72 /* input file state */
73 static char **Files;
74 static const char *Fileopened;
75 static FILE *Fp;
76 static int Line;
77 static const char *File;
78 static const char *Cpp = "/usr/bin/cpp";
79 #ifdef	ESC
80 static const char *Cppargs;
81 static const char *Cppstdargs = "-undef -Y.";
82 #endif	/* ESC */
83 
84 /* for debugging */
85 static int Lexecho;	/* echo tokens as we read them */
86 
87 /* forward declarations of our internal routines */
88 static int record(int tok, const char *s);
89 static void dumpline(int flags);
90 static void doident();
91 static void dopragma(const char *tok);
92 
93 /*
94  * table of reserved words.  this table is only used by lex_init()
95  * to intialize the Rwords lookup table.
96  */
97 static const struct {
98 	const char *word;
99 	const int val;
100 } Rwords[] = {
101 	{ "asru", ASRU },
102 	{ "count", COUNT },
103 	{ "div", DIV },
104 	{ "engine", ENGINE },
105 	{ "event", EVENT },
106 	{ "fru", FRU },
107 	{ "if", IF },
108 	{ "mask", MASK },
109 	{ "prop", PROP },
110 	{ "config", CONFIG },
111 	/*
112 	 * PATHFUNC indicates functions that operate only on paths
113 	 * and quotes
114 	 */
115 	{ "is_connected", PATHFUNC },
116 	{ "is_under", PATHFUNC },
117 };
118 
119 /*
120  * Rwordslut is a lookup table of reserved words.  lhs is the word
121  * (in the string table) and the rhs is the token value returned
122  * by the yylex() for that word.
123  */
124 static struct lut *Rwordslut;
125 
126 static const struct {
127 	const char *suffix;
128 	const unsigned long long nsec;
129 } Timesuffix[] = {
130 	{ "nanosecond",		1ULL },
131 	{ "nanoseconds",	1ULL },
132 	{ "nsec",		1ULL },
133 	{ "nsecs",		1ULL },
134 	{ "ns",			1ULL },
135 	{ "microsecond",	1000ULL },
136 	{ "microseconds",	1000ULL },
137 	{ "usec",		1000ULL },
138 	{ "usecs",		1000ULL },
139 	{ "us",			1000ULL },
140 	{ "millisecond",	1000000ULL },
141 	{ "milliseconds",	1000000ULL },
142 	{ "msec",		1000000ULL },
143 	{ "msecs",		1000000ULL },
144 	{ "ms",			1000000ULL },
145 	{ "second",		1000000000ULL },
146 	{ "seconds",		1000000000ULL },
147 	{ "s",			1000000000ULL },
148 	{ "minute",		1000000000ULL * 60 },
149 	{ "minutes",		1000000000ULL * 60 },
150 	{ "min",		1000000000ULL * 60 },
151 	{ "mins",		1000000000ULL * 60 },
152 	{ "m",			1000000000ULL * 60 },
153 	{ "hour",		1000000000ULL * 60 * 60 },
154 	{ "hours",		1000000000ULL * 60 * 60 },
155 	{ "hr",			1000000000ULL * 60 * 60 },
156 	{ "hrs",		1000000000ULL * 60 * 60 },
157 	{ "h",			1000000000ULL * 60 * 60 },
158 	{ "day",		1000000000ULL * 60 * 60 * 24 },
159 	{ "days",		1000000000ULL * 60 * 60 * 24 },
160 	{ "d",			1000000000ULL * 60 * 60 * 24 },
161 	{ "week",		1000000000ULL * 60 * 60 * 24 * 7 },
162 	{ "weeks",		1000000000ULL * 60 * 60 * 24 * 7 },
163 	{ "wk",			1000000000ULL * 60 * 60 * 24 * 7 },
164 	{ "wks",		1000000000ULL * 60 * 60 * 24 * 7 },
165 	{ "month",		1000000000ULL * 60 * 60 * 24 * 30 },
166 	{ "months",		1000000000ULL * 60 * 60 * 24 * 30 },
167 	{ "year",		1000000000ULL * 60 * 60 * 24 * 365 },
168 	{ "years",		1000000000ULL * 60 * 60 * 24 * 365 },
169 	{ "yr",			1000000000ULL * 60 * 60 * 24 * 365 },
170 	{ "yrs",		1000000000ULL * 60 * 60 * 24 * 365 },
171 };
172 
173 /*
174  * some wrappers around the general lut functions to provide type checking...
175  */
176 
177 static struct lut *
178 lex_s2i_lut_add(struct lut *root, const char *s, int i)
179 {
180 	return (lut_add(root, (void *)s, (void *)i, NULL));
181 }
182 
183 static int
184 lex_s2i_lut_lookup(struct lut *root, const char *s)
185 {
186 	return ((int)lut_lookup(root, (void *)s, NULL));
187 }
188 
189 static struct lut *
190 lex_s2ullp_lut_add(struct lut *root, const char *s,
191     const unsigned long long *ullp)
192 {
193 	return (lut_add(root, (void *)s, (void *)ullp, NULL));
194 }
195 
196 const unsigned long long *
197 lex_s2ullp_lut_lookup(struct lut *root, const char *s)
198 {
199 	return ((unsigned long long *)lut_lookup(root, (void *)s, NULL));
200 }
201 
202 /*
203  * lex_init -- initialize the lexer with appropriate filenames & debug flags
204  */
205 
206 /*ARGSUSED*/
207 void
208 lex_init(char **av, const char *cppargs, int lexecho)
209 {
210 	int i;
211 #ifdef	ESC
212 	const char *ptr;
213 #endif	/* ESC */
214 
215 	Lexecho = lexecho;
216 	Tokcount = stats_new_counter("lex.tokens", "total tokens in", 1);
217 	Filecount = stats_new_counter("lex.files", "total files read", 0);
218 	Lexelapse = stats_new_elapse("lex.time", "elapsed lex/parse time", 1);
219 
220 #ifdef	ESC
221 	Cppargs = cppargs;
222 
223 	/* allow user to tell us where cpp is if it is some weird place */
224 	if (ptr = getenv("_ESC_CPP"))
225 		Cpp = ptr;
226 
227 	/* and in case it takes some special stdargs */
228 	if (ptr = getenv("_ESC_CPP_STDARGS"))
229 		Cppstdargs = ptr;
230 
231 	/* verify we can find cpp */
232 	if (access(Cpp, X_OK) < 0) {
233 		Cpp = "/usr/lib/cpp";
234 		if (access(Cpp, X_OK) < 0)
235 			out(O_DIE, "can't locate cpp");
236 	}
237 #endif	/* ESC */
238 
239 	Files = av;
240 
241 	/* verify we can find all the input files */
242 	while (*av) {
243 		if (strlen(*av) >= MAXTOK - strlen(Cpp) - 3)
244 			out(O_DIE, "filename too long: %.100s...", *av);
245 		if (access(*av, R_OK) < 0)
246 			out(O_DIE|O_SYS, "%s", *av);
247 		av++;
248 		stats_counter_bump(Filecount);
249 	}
250 
251 	/* put reserved words into the string table & a lookup table */
252 	for (i = 0; i < sizeof (Rwords) / sizeof (*Rwords); i++)
253 		Rwordslut = lex_s2i_lut_add(Rwordslut,
254 		    stable(Rwords[i].word), Rwords[i].val);
255 
256 	/* initialize table of timeval suffixes */
257 	for (i = 0; i < sizeof (Timesuffix) / sizeof (*Timesuffix); i++) {
258 		Timesuffixlut = lex_s2ullp_lut_add(Timesuffixlut,
259 		    stable(Timesuffix[i].suffix), &Timesuffix[i].nsec);
260 	}
261 
262 	/* record start time */
263 	stats_elapse_start(Lexelapse);
264 }
265 
266 void
267 closefile(void)
268 {
269 	if (Fp != NULL) {
270 #ifdef	ESC
271 		if (pclose(Fp) > 0)
272 			out(O_DIE, "cpp errors while reading \"%s\", "
273 			    "bailing out.", Fileopened);
274 #else
275 		(void) fclose(Fp);
276 #endif	/* ESC */
277 	}
278 	Fp = NULL;
279 }
280 
281 /*
282  * yylex -- the lexer, called yylex() because that's what yacc wants
283  */
284 
285 int
286 yylex()
287 {
288 	int c;
289 	int nextc;
290 	char *ptr = Tok;
291 	char *eptr = &Tok[MAXTOK];
292 	const char *cptr;
293 	int startline;
294 	int val;
295 	static int bol = 1;	/* true if we're at beginning of line */
296 
297 	for (;;) {
298 		while (Fp == NULL) {
299 			char ibuf[80];
300 
301 			if (*Files == NULL)
302 				return (record(EOF, NULL));
303 			Fileopened = stable(*Files++);
304 #ifdef	ESC
305 			sprintf(Tok, "%s %s %s %s",
306 			    Cpp, Cppstdargs, Cppargs, Fileopened);
307 			if ((Fp = popen(Tok, "r")) == NULL)
308 				out(O_DIE|O_SYS, "%s", Tok);
309 #else
310 			Fp = eftread_fopen(Fileopened, ibuf, sizeof (ibuf));
311 #endif	/* ESC */
312 			Line = 1;
313 			bol = 1;
314 
315 			/* add name to stats for visibility */
316 			if (Fp != NULL) {
317 				static int fnum;
318 				char nbuf[100];
319 				struct filestats *nfs = MALLOC(sizeof (*nfs));
320 
321 				(void) sprintf(nbuf, "lex.file%d", fnum);
322 				nfs->stats = stats_new_string(nbuf, "", 0);
323 				stats_string_set(nfs->stats, Fileopened);
324 
325 				if (ibuf[0] != '\0') {
326 					(void) sprintf(nbuf, "lex.file%d-ident",
327 					    fnum);
328 					nfs->idstats =
329 					    stats_new_string(nbuf, "", 0);
330 					stats_string_set(nfs->idstats, ibuf);
331 				} else {
332 					nfs->idstats = NULL;
333 				}
334 
335 				nfs->next = Fstats;
336 				Fstats = nfs;
337 				fnum++;
338 			}
339 		}
340 
341 		switch (c = getc(Fp)) {
342 		case '#':
343 			/* enforce that we're at beginning of line */
344 			if (!bol)
345 				return (record(c, NULL));
346 
347 			while ((c = getc(Fp)) != EOF &&
348 			    (c == ' ' || c == '\t'))
349 				;
350 			if (!isdigit(c)) {
351 				/*
352 				 * three cases here:
353 				 *	#pragma
354 				 *	#ident
355 				 *	#something-we-don't-understand
356 				 * anything we don't expect we just ignore.
357 				 */
358 				*ptr++ = c;
359 				while ((c = getc(Fp)) != EOF && isalnum(c))
360 					if (ptr < eptr - 1)
361 						*ptr++ = c;
362 				*ptr++ = '\0';
363 				if (strcmp(Tok, "pragma") == 0) {
364 					/* skip white space */
365 					while ((c = getc(Fp)) != EOF &&
366 					    (c == ' ' || c == '\t'))
367 						;
368 
369 					if (c == EOF || c == '\n')
370 						outfl(O_DIE, File, Line,
371 						    "bad #pragma");
372 
373 					/* pull in next token */
374 					ptr = Tok;
375 					*ptr++ = c;
376 					while ((c = getc(Fp)) != EOF &&
377 					    !isspace(c))
378 						if (ptr < eptr - 1)
379 							*ptr++ = c;
380 					*ptr++ = '\0';
381 					(void) ungetc(c, Fp);
382 
383 					dopragma(Tok);
384 				} else if (strcmp(Tok, "ident") == 0)
385 					doident();
386 			} else {
387 				/* handle file & line info from cpp */
388 				Line = 0;
389 				do {
390 					if (!isdigit(c))
391 						break;
392 					Line = Line * 10 + c - '0';
393 				} while ((c = getc(Fp)) != EOF);
394 				Line--;		/* newline will increment it */
395 				while (c != EOF && isspace(c))
396 					c = getc(Fp);
397 				if (c != '"')
398 					outfl(O_DIE, File, Line,
399 					    "bad # statement (file name)");
400 				while ((c = getc(Fp)) != EOF && c != '"')
401 					if (ptr < eptr - 1)
402 						*ptr++ = c;
403 				*ptr++ = '\0';
404 				if (c != '"')
405 					outfl(O_DIE, File, Line,
406 					    "bad # statement (quotes)");
407 				File = stable(Tok);
408 			}
409 			/* skip the rest of the cpp line */
410 			while ((c = getc(Fp)) != EOF && c != '\n' && c != '\r')
411 				;
412 			if (c == EOF)
413 				return (record(c, NULL));
414 			else
415 				(void) ungetc(c, Fp);
416 			ptr = Tok;
417 			break;
418 
419 		case EOF:
420 			closefile();
421 			continue;
422 
423 		case '\n':
424 			Line++;
425 			bol = 1;
426 			break;
427 
428 		case '\r':
429 		case ' ':
430 		case '\t':
431 			bol = 0;
432 			break;
433 
434 		case '/':
435 			bol = 0;
436 			/* comment handling */
437 			if ((nextc = getc(Fp)) == EOF)
438 				outfl(O_DIE, File, Line, "unexpected EOF");
439 			else if (nextc == '*') {
440 				startline = Line;
441 				while ((c = getc(Fp)) != EOF) {
442 					if (c == '\n')
443 						Line++;
444 					else if (c == '*' &&
445 						(((c = getc(Fp)) == EOF) ||
446 						(c == '/')))
447 						break;
448 				}
449 				if (c == EOF) {
450 					outfl(O_DIE, File, Line,
451 					    "end of comment not seen "
452 					    "(started on line %d)",
453 					    startline);
454 				}
455 			} else {
456 				/* wasn't a comment, return the '/' token */
457 				(void) ungetc(nextc, Fp);
458 				return (record(c, NULL));
459 			}
460 			break;
461 
462 		case '"': {
463 			int prevc;
464 
465 			bol = 0;
466 			prevc = '\0';
467 			/* quoted string handling */
468 			startline = Line;
469 			for (;;) {
470 				c = getc(Fp);
471 				if (c == EOF)
472 					outfl(O_DIE, File, Line,
473 					    "end of string not seen "
474 					    "(started on line %d)",
475 					    startline);
476 				else if (c == '\n')
477 					Line++;
478 				else if (c == '"' && prevc != '\\')
479 					break;
480 				else if (ptr < eptr)
481 					*ptr++ = c;
482 				prevc = c;
483 			}
484 			if (ptr >= eptr)
485 				out(O_DIE, File, Line, "string too long");
486 			*ptr++ = '\0';
487 			return (record(QUOTE, stable(Tok)));
488 		}
489 		case '&':
490 			bol = 0;
491 			/* && */
492 			if ((nextc = getc(Fp)) == '&')
493 				return (record(AND, NULL));
494 			else {
495 				(void) ungetc(nextc, Fp);
496 				return (record(c, NULL));
497 			}
498 			/*NOTREACHED*/
499 			break;
500 
501 		case '|':
502 			bol = 0;
503 			/* || */
504 			if ((nextc = getc(Fp)) == '|')
505 				return (record(OR, NULL));
506 			else {
507 				(void) ungetc(nextc, Fp);
508 				return (record(c, NULL));
509 			}
510 			/*NOTREACHED*/
511 			break;
512 
513 		case '!':
514 			bol = 0;
515 			/* ! or != */
516 			if ((nextc = getc(Fp)) == '=')
517 				return (record(NE, NULL));
518 			else {
519 				(void) ungetc(nextc, Fp);
520 				return (record(c, NULL));
521 			}
522 			/*NOTREACHED*/
523 			break;
524 
525 		case '=':
526 			bol = 0;
527 			/* == */
528 			if ((nextc = getc(Fp)) == '=')
529 				return (record(EQ, NULL));
530 			else {
531 				(void) ungetc(nextc, Fp);
532 				return (record(c, NULL));
533 			}
534 			/*NOTREACHED*/
535 			break;
536 
537 		case '-':
538 			bol = 0;
539 			/* -> */
540 			if ((nextc = getc(Fp)) == '>')
541 				return (record(ARROW, stable(Tok)));
542 			else {
543 				(void) ungetc(nextc, Fp);
544 				return (record(c, NULL));
545 			}
546 			/*NOTREACHED*/
547 			break;
548 
549 		case '<':
550 			bol = 0;
551 			if ((nextc = getc(Fp)) == '=')
552 				/* <= */
553 				return (record(LE, NULL));
554 			else if (nextc == '<')
555 				/* << */
556 				return (record(LSHIFT, NULL));
557 			else {
558 				(void) ungetc(nextc, Fp);
559 				return (record(c, NULL));
560 			}
561 			/*NOTREACHED*/
562 			break;
563 
564 		case '>':
565 			bol = 0;
566 			if ((nextc = getc(Fp)) == '=')
567 				/* >= */
568 				return (record(GE, NULL));
569 			else if (nextc == '>')
570 				/* >> */
571 				return (record(RSHIFT, NULL));
572 			else {
573 				(void) ungetc(nextc, Fp);
574 				return (record(c, NULL));
575 			}
576 			/*NOTREACHED*/
577 			break;
578 
579 		default:
580 			bol = 0;
581 			if (isdigit(c)) {
582 				int base;
583 
584 				/* collect rest of number */
585 				if (c == '0') {
586 					*ptr++ = c;
587 					if ((c = getc(Fp)) == EOF) {
588 						*ptr++ = '\0';
589 						return (record(NUMBER,
590 						    stable(Tok)));
591 					} else if (c == 'x' || c == 'X') {
592 						*ptr++ = c;
593 						base = 16;
594 					} else {
595 						(void) ungetc(c, Fp);
596 						base = 8;
597 					}
598 				} else {
599 					*ptr++ = c;
600 					base = 10;
601 				}
602 				while ((c = getc(Fp)) != EOF) {
603 					if (ptr >= eptr)
604 						out(O_DIE, File, Line,
605 						    "number too long");
606 
607 					switch (base) {
608 					case 16:
609 						if (c >= 'a' && c <= 'f' ||
610 						    c >= 'A' && c <= 'F') {
611 							*ptr++ = c;
612 							continue;
613 						}
614 						/*FALLTHRU*/
615 					case 10:
616 						if (c >= '8' && c <= '9') {
617 							*ptr++ = c;
618 							continue;
619 						}
620 						/*FALLTHRU*/
621 					case 8:
622 						if (c >= '0' && c <= '7') {
623 							*ptr++ = c;
624 							continue;
625 						}
626 						/* not valid for this base */
627 						*ptr++ = '\0';
628 						(void) ungetc(c, Fp);
629 						return (record(NUMBER,
630 						    stable(Tok)));
631 					}
632 				}
633 				*ptr++ = '\0';
634 				return (record(NUMBER, stable(Tok)));
635 			} else if (isalpha(c)) {
636 				/* collect identifier */
637 				*ptr++ = c;
638 				for (;;) {
639 					c = getc(Fp);
640 					if ((isalnum(c) || c == '_') &&
641 						ptr < eptr)
642 						*ptr++ = c;
643 					else {
644 						(void) ungetc(c, Fp);
645 						break;
646 					}
647 				}
648 				if (ptr >= eptr)
649 					out(O_DIE, File, Line,
650 					    "identifier too long");
651 				*ptr++ = '\0';
652 				cptr = stable(Tok);
653 				if (val = lex_s2i_lut_lookup(Rwordslut, cptr)) {
654 					return (record(val, cptr));
655 				}
656 				return (record(ID, cptr));
657 			} else
658 				return (record(c, NULL));
659 		}
660 		/*NOTREACHED*/
661 	}
662 }
663 
664 /*
665  * the record()/dumpline() routines are used to track & report
666  * the list of tokens seen on a given line.  this is used in two ways.
667  * first, syntax errors found by the parser are reported by us (via
668  * yyerror()) and we tack on the tokens processed so far on the current
669  * line to help indicate exactly where the error is.  second, if "lexecho"
670  * debugging is turned on, these routines provide it.
671  */
672 #define	MAXRECORD 1000
673 static int Recordedline;
674 static struct {
675 	int tok;
676 	const char *s;
677 } Recorded[MAXRECORD];
678 static int Recordnext;
679 
680 static int
681 record(int tok, const char *s)
682 {
683 	stats_counter_bump(Tokcount);
684 	if (Line != Recordedline) {
685 		/* starting new line, dump out the previous line */
686 		if (Lexecho && Recordedline) {
687 			outfl(O_NONL, File, Recordedline, "lex: ");
688 			dumpline(O_OK);
689 		}
690 		Recordedline = Line;
691 		Recordnext = 0;
692 	}
693 	if (Recordnext >= MAXRECORD)
694 		outfl(O_DIE, File, Line, "line too long, bailing out");
695 	Recorded[Recordnext].tok = tok;
696 	Recorded[Recordnext++].s = s;
697 
698 	yylval.tok.s = s;
699 	yylval.tok.file = File;
700 	yylval.tok.line = Line;
701 	return (tok);
702 }
703 
704 /*ARGSUSED*/
705 static void
706 dumpline(int flags)
707 {
708 	int i;
709 
710 	for (i = 0; i < Recordnext; i++)
711 		if (Recorded[i].s && Recorded[i].tok != ARROW)
712 			switch (Recorded[i].tok) {
713 			case T_QUOTE:
714 				out(flags|O_NONL, " \"%s\"",
715 					Recorded[i].s);
716 				break;
717 
718 			default:
719 				out(flags|O_NONL, " %s",
720 					Recorded[i].s);
721 				break;
722 			}
723 		else
724 			switch (Recorded[i].tok) {
725 			case EOF:
726 				out(flags|O_NONL, " EOF");
727 				break;
728 			case ARROW:
729 				out(flags|O_NONL, " ->%s",
730 					Recorded[i].s);
731 				break;
732 			case EQ:
733 				out(flags|O_NONL, " ==");
734 				break;
735 			case NE:
736 				out(flags|O_NONL, " !=");
737 				break;
738 			case OR:
739 				out(flags|O_NONL, " ||");
740 				break;
741 			case AND:
742 				out(flags|O_NONL, " &&");
743 				break;
744 			case LE:
745 				out(flags|O_NONL, " <=");
746 				break;
747 			case GE:
748 				out(flags|O_NONL, " >=");
749 				break;
750 			case LSHIFT:
751 				out(flags|O_NONL, " <<");
752 				break;
753 			case RSHIFT:
754 				out(flags|O_NONL, " >>");
755 				break;
756 			default:
757 				if (isprint(Recorded[i].tok))
758 					out(flags|O_NONL, " %c",
759 						Recorded[i].tok);
760 				else
761 					out(flags|O_NONL, " '\\%03o'",
762 						Recorded[i].tok);
763 				break;
764 			}
765 	out(flags, NULL);
766 }
767 
768 /*
769  * yyerror -- report a pareser error, called yyerror because yacc wants it
770  */
771 
772 void
773 yyerror(const char *s)
774 {
775 	Errcount++;
776 	outfl(O_ERR|O_NONL, File, Line, "%s, tokens: ", s);
777 	dumpline(O_ERR);
778 }
779 
780 /*
781  * doident -- handle "#pragma ident" directives
782  */
783 static void
784 doident()
785 {
786 	int c;
787 	char *ptr = Tok;
788 	char *eptr = &Tok[MAXTOK];
789 
790 	/* skip white space and quotes */
791 	while ((c = getc(Fp)) != EOF &&
792 	    (c == ' ' || c == '\t' || c == '"'))
793 		;
794 
795 	if (c == EOF || c == '\n')
796 		outfl(O_DIE, File, Line, "bad ident");
797 
798 	/* pull in next token */
799 	ptr = Tok;
800 	*ptr++ = c;
801 	while ((c = getc(Fp)) != EOF && c != '"' && c != '\n')
802 		if (ptr < eptr - 1)
803 			*ptr++ = c;
804 	*ptr++ = '\0';
805 	if (c != '\n') {
806 		/* skip to end of line (including close quote, if any) */
807 		while ((c = getc(Fp)) != EOF && c != '\n')
808 			;
809 	}
810 	(void) ungetc(c, Fp);
811 	Ident = lut_add(Ident, (void *)stable(Tok), (void *)0, NULL);
812 
813 	outfl(O_VERB, File, Line, "pragma set: ident \"%s\"", Tok);
814 }
815 
816 /*
817  * dodictionary -- handle "#pragma dictionary" directives
818  */
819 static void
820 dodictionary()
821 {
822 	int c;
823 	char *ptr = Tok;
824 	char *eptr = &Tok[MAXTOK];
825 
826 	/* skip white space and quotes */
827 	while ((c = getc(Fp)) != EOF &&
828 	    (c == ' ' || c == '\t' || c == '"'))
829 		;
830 
831 	if (c == EOF || c == '\n')
832 		outfl(O_DIE, File, Line, "bad dictionary");
833 
834 	/* pull in next token */
835 	ptr = Tok;
836 	*ptr++ = c;
837 	while ((c = getc(Fp)) != EOF && c != '"' && c != '\n')
838 		if (ptr < eptr - 1)
839 			*ptr++ = c;
840 	*ptr++ = '\0';
841 	if (c != '\n') {
842 		/* skip to end of line (including close quote, if any) */
843 		while ((c = getc(Fp)) != EOF && c != '\n')
844 			;
845 	}
846 	(void) ungetc(c, Fp);
847 	Dicts = lut_add(Dicts, (void *)stable(Tok), (void *)0, NULL);
848 
849 	outfl(O_VERB, File, Line, "pragma set: dictionary \"%s\"", Tok);
850 }
851 
852 /*
853  * doallow_cycles -- handle "#pragma allow_cycles" directives
854  */
855 static void
856 doallow_cycles()
857 {
858 	int c;
859 	char *ptr = Tok;
860 	char *eptr = &Tok[MAXTOK];
861 	unsigned long long newlevel;
862 
863 	/*
864 	 * by default the compiler does not allow cycles or loops
865 	 * in propagations.  when cycles are encountered, the
866 	 * compiler prints out an error message.
867 	 *
868 	 *   "#pragma allow_cycles" and
869 	 *   "#pragma allow_cycles 0"
870 	 * allow cycles, but any such cycle will produce a warning
871 	 * message.
872 	 *
873 	 *   "#pragma allow_cycles N"
874 	 * with N > 0 will allow cycles and not produce any
875 	 * warning messages.
876 	 */
877 
878 	/* skip white space and quotes */
879 	while ((c = getc(Fp)) != EOF &&
880 	    (c == ' ' || c == '\t' || c == '"'))
881 		;
882 
883 	if (c == EOF || c == '\n')
884 		newlevel = 0ULL;
885 	else {
886 
887 		/* pull in next token */
888 		ptr = Tok;
889 		*ptr++ = c;
890 		while ((c = getc(Fp)) != EOF && c != '"' && c != '\n')
891 			if (ptr < eptr - 1)
892 				*ptr++ = c;
893 		*ptr++ = '\0';
894 		if (c != '\n') {
895 			/* skip to end of line */
896 			while ((c = getc(Fp)) != EOF && c != '\n')
897 				;
898 		}
899 		newlevel = strtoll(Tok, NULL, 0);
900 	}
901 	(void) ungetc(c, Fp);
902 
903 	(void) check_cycle_level(newlevel);
904 	outfl(O_VERB, File, Line,
905 	    "pragma set: allow_cycles (%s)",
906 		    newlevel ? "no warnings" : "with warnings");
907 }
908 
909 /*
910  * dopragma -- handle #pragma directives
911  */
912 static void
913 dopragma(const char *tok)
914 {
915 	if (strcmp(tok, "ident") == 0)
916 		doident();
917 	else if (strcmp(tok, "dictionary") == 0)
918 		dodictionary();
919 	else if (strcmp(tok, "new_errors_only") == 0) {
920 		if (Pragma_new_errors_only++ == 0)
921 			outfl(O_VERB, File, Line,
922 			    "pragma set: new_errors_only");
923 	} else if (strcmp(tok, "trust_ereports") == 0) {
924 		if (Pragma_trust_ereports++ == 0)
925 			outfl(O_VERB, File, Line,
926 			    "pragma set: trust_ereports");
927 	} else if (strcmp(tok, "allow_cycles") == 0)
928 		doallow_cycles();
929 	else
930 		outfl(O_VERB, File, Line,
931 		    "unknown pragma ignored: \"%s\"", tok);
932 }
933 
934 /*
935  * lex_fini -- finalize the lexer
936  */
937 
938 int
939 lex_fini(void)
940 {
941 	stats_elapse_stop(Lexelapse);
942 	closefile();
943 	if (Lexecho) {
944 		outfl(O_OK, File, Line, "lex: ");
945 		dumpline(O_OK);
946 	}
947 	return (Errcount);
948 }
949 
950 void
951 lex_free(void)
952 {
953 	struct filestats *nfstats = Fstats;
954 
955 	/*
956 	 * Free up memory consumed by the lexer
957 	 */
958 	stats_delete(Tokcount);
959 	stats_delete(Filecount);
960 	stats_delete(Lexelapse);
961 	while (nfstats != NULL) {
962 		Fstats = nfstats->next;
963 		stats_delete(nfstats->stats);
964 		if (nfstats->idstats != NULL)
965 			stats_delete(nfstats->idstats);
966 		FREE(nfstats);
967 		nfstats = Fstats;
968 	}
969 	lut_free(Timesuffixlut, NULL, NULL);
970 	lut_free(Rwordslut, NULL, NULL);
971 	lut_free(Ident, NULL, NULL);
972 	lut_free(Dicts, NULL, NULL);
973 }
974