xref: /illumos-gate/usr/src/cmd/fm/eversholt/common/esclex.c (revision 9b664393d4fdda96221e6ea9ea95790d3c15be70)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  *
25  * esclex.c -- lexer for esc
26  *
27  * this module provides lexical analysis and error handling routine
28  * expected by the yacc-generated parser (i.e. yylex() and yyerror()).
29  * it also does lots of tracking of things like filenames, line numbers,
30  * and what tokens are seen on a line up to the point where a syntax error
31  * was found.  this module also arranges for the input source files to
32  * be run through cpp.
33  */
34 
35 #include <stdio.h>
36 #include <ctype.h>
37 #include <string.h>
38 #include <stdlib.h>
39 #include <unistd.h>
40 #include <time.h>
41 #include <errno.h>
42 #include "out.h"
43 #include "alloc.h"
44 #include "stats.h"
45 #include "stable.h"
46 #include "lut.h"
47 #include "literals.h"
48 #include "tree.h"
49 #include "esclex.h"
50 #include "eftread.h"
51 #include "check.h"
52 #include "y.tab.h"
53 
54 struct lut *Dicts;
55 struct lut *Ident;
56 struct lut *Timesuffixlut;
57 int Pragma_trust_ereports;
58 int Pragma_new_errors_only;
59 
60 /* ridiculously long token buffer -- disallow any token longer than this */
61 #define	MAXTOK	8192
62 static char Tok[MAXTOK];
63 
64 /* some misc stats we keep on the lexer & parser */
65 static struct stats *Tokcount;
66 static struct stats *Lexelapse;
67 struct stats *Filecount;
68 struct filestats {
69 	struct filestats *next;
70 	struct stats *stats;
71 	struct stats *idstats;
72 } *Fstats;
73 
74 static int Errcount;
75 
76 /* input file state */
77 static char **Files;
78 static const char *Fileopened;
79 static FILE *Fp;
80 static int Line;
81 static const char *File;
82 static const char *Cpp = "/usr/lib/cpp";
83 #ifdef	ESC
84 static const char *Cppargs;
85 static const char *Cppstdargs = "-undef -Y.";
86 #endif	/* ESC */
87 
88 /* for debugging */
89 static int Lexecho;	/* echo tokens as we read them */
90 
91 /* forward declarations of our internal routines */
92 static int record(int tok, const char *s);
93 static void dumpline(int flags);
94 static void doident();
95 static void dopragma(const char *tok);
96 
97 /*
98  * table of reserved words.  this table is only used by lex_init()
99  * to intialize the Rwords lookup table.
100  */
101 static const struct {
102 	const char *word;
103 	const int val;
104 } Rwords[] = {
105 	{ "asru", ASRU },
106 	{ "count", COUNT },
107 	{ "div", DIV },
108 	{ "engine", ENGINE },
109 	{ "event", EVENT },
110 	{ "fru", FRU },
111 	{ "if", IF },
112 	{ "mask", MASK },
113 	{ "prop", PROP },
114 	{ "config", CONFIG },
115 	/*
116 	 * PATHFUNC indicates functions that operate only on paths
117 	 * and quotes
118 	 */
119 	{ "is_connected", PATHFUNC },
120 	{ "is_under", PATHFUNC },
121 	{ "is_on", PATHFUNC },
122 	{ "is_present", PATHFUNC },
123 	{ "is_type", PATHFUNC },
124 	{ "has_fault", PATHFUNC },
125 	{ "confprop", PATHFUNC },
126 	{ "confprop_defined", PATHFUNC },
127 };
128 
129 /*
130  * Rwordslut is a lookup table of reserved words.  lhs is the word
131  * (in the string table) and the rhs is the token value returned
132  * by the yylex() for that word.
133  */
134 static struct lut *Rwordslut;
135 
136 static const struct {
137 	const char *suffix;
138 	const unsigned long long nsec;
139 } Timesuffix[] = {
140 	{ "nanosecond",		1ULL },
141 	{ "nanoseconds",	1ULL },
142 	{ "nsec",		1ULL },
143 	{ "nsecs",		1ULL },
144 	{ "ns",			1ULL },
145 	{ "microsecond",	1000ULL },
146 	{ "microseconds",	1000ULL },
147 	{ "usec",		1000ULL },
148 	{ "usecs",		1000ULL },
149 	{ "us",			1000ULL },
150 	{ "millisecond",	1000000ULL },
151 	{ "milliseconds",	1000000ULL },
152 	{ "msec",		1000000ULL },
153 	{ "msecs",		1000000ULL },
154 	{ "ms",			1000000ULL },
155 	{ "second",		1000000000ULL },
156 	{ "seconds",		1000000000ULL },
157 	{ "s",			1000000000ULL },
158 	{ "minute",		1000000000ULL * 60 },
159 	{ "minutes",		1000000000ULL * 60 },
160 	{ "min",		1000000000ULL * 60 },
161 	{ "mins",		1000000000ULL * 60 },
162 	{ "m",			1000000000ULL * 60 },
163 	{ "hour",		1000000000ULL * 60 * 60 },
164 	{ "hours",		1000000000ULL * 60 * 60 },
165 	{ "hr",			1000000000ULL * 60 * 60 },
166 	{ "hrs",		1000000000ULL * 60 * 60 },
167 	{ "h",			1000000000ULL * 60 * 60 },
168 	{ "day",		1000000000ULL * 60 * 60 * 24 },
169 	{ "days",		1000000000ULL * 60 * 60 * 24 },
170 	{ "d",			1000000000ULL * 60 * 60 * 24 },
171 	{ "week",		1000000000ULL * 60 * 60 * 24 * 7 },
172 	{ "weeks",		1000000000ULL * 60 * 60 * 24 * 7 },
173 	{ "wk",			1000000000ULL * 60 * 60 * 24 * 7 },
174 	{ "wks",		1000000000ULL * 60 * 60 * 24 * 7 },
175 	{ "month",		1000000000ULL * 60 * 60 * 24 * 30 },
176 	{ "months",		1000000000ULL * 60 * 60 * 24 * 30 },
177 	{ "year",		1000000000ULL * 60 * 60 * 24 * 365 },
178 	{ "years",		1000000000ULL * 60 * 60 * 24 * 365 },
179 	{ "yr",			1000000000ULL * 60 * 60 * 24 * 365 },
180 	{ "yrs",		1000000000ULL * 60 * 60 * 24 * 365 },
181 };
182 
183 /*
184  * some wrappers around the general lut functions to provide type checking...
185  */
186 
187 static struct lut *
188 lex_s2i_lut_add(struct lut *root, const char *s, intptr_t i)
189 {
190 	return (lut_add(root, (void *)s, (void *)i, NULL));
191 }
192 
193 static int
194 lex_s2i_lut_lookup(struct lut *root, const char *s)
195 {
196 	return ((intptr_t)lut_lookup(root, (void *)s, NULL));
197 }
198 
199 static struct lut *
200 lex_s2ullp_lut_add(struct lut *root, const char *s,
201     const unsigned long long *ullp)
202 {
203 	return (lut_add(root, (void *)s, (void *)ullp, NULL));
204 }
205 
206 const unsigned long long *
207 lex_s2ullp_lut_lookup(struct lut *root, const char *s)
208 {
209 	return ((unsigned long long *)lut_lookup(root, (void *)s, NULL));
210 }
211 
212 /*
213  * lex_init -- initialize the lexer with appropriate filenames & debug flags
214  */
215 
216 /*ARGSUSED*/
217 void
218 lex_init(char **av, const char *cppargs, int lexecho)
219 {
220 	int i;
221 #ifdef	ESC
222 	const char *ptr;
223 #endif	/* ESC */
224 
225 	Lexecho = lexecho;
226 	Tokcount = stats_new_counter("lex.tokens", "total tokens in", 1);
227 	Filecount = stats_new_counter("lex.files", "total files read", 0);
228 	Lexelapse = stats_new_elapse("lex.time", "elapsed lex/parse time", 1);
229 
230 #ifdef	ESC
231 	Cppargs = cppargs;
232 
233 	/* allow user to tell us where cpp is if it is some weird place */
234 	if (ptr = getenv("_ESC_CPP"))
235 		Cpp = ptr;
236 
237 	/* and in case it takes some special stdargs */
238 	if (ptr = getenv("_ESC_CPP_STDARGS"))
239 		Cppstdargs = ptr;
240 
241 	/* verify we can find cpp */
242 	if (access(Cpp, X_OK) < 0) {
243 		Cpp = "/usr/lib/cpp";
244 		if (access(Cpp, X_OK) < 0)
245 			out(O_DIE, "can't locate cpp");
246 	}
247 #endif	/* ESC */
248 
249 	Files = av;
250 
251 	/* verify we can find all the input files */
252 	while (*av) {
253 		if (strlen(*av) >= MAXTOK - strlen(Cpp) - 3)
254 			out(O_DIE, "filename too long: %.100s...", *av);
255 		if (access(*av, R_OK) < 0)
256 			out(O_DIE|O_SYS, "%s", *av);
257 		av++;
258 		stats_counter_bump(Filecount);
259 	}
260 
261 	/* put reserved words into the string table & a lookup table */
262 	for (i = 0; i < sizeof (Rwords) / sizeof (*Rwords); i++)
263 		Rwordslut = lex_s2i_lut_add(Rwordslut,
264 		    stable(Rwords[i].word), Rwords[i].val);
265 
266 	/* initialize table of timeval suffixes */
267 	for (i = 0; i < sizeof (Timesuffix) / sizeof (*Timesuffix); i++) {
268 		Timesuffixlut = lex_s2ullp_lut_add(Timesuffixlut,
269 		    stable(Timesuffix[i].suffix), &Timesuffix[i].nsec);
270 	}
271 
272 	/* record start time */
273 	stats_elapse_start(Lexelapse);
274 }
275 
276 void
277 closefile(void)
278 {
279 	if (Fp != NULL) {
280 #ifdef	ESC
281 		if (pclose(Fp) > 0)
282 			out(O_DIE, "cpp errors while reading \"%s\", "
283 			    "bailing out.", Fileopened);
284 #else
285 		(void) fclose(Fp);
286 #endif	/* ESC */
287 	}
288 	Fp = NULL;
289 }
290 
291 /*
292  * yylex -- the lexer, called yylex() because that's what yacc wants
293  */
294 
295 int
296 yylex()
297 {
298 	int c;
299 	int nextc;
300 	char *ptr = Tok;
301 	char *eptr = &Tok[MAXTOK];
302 	const char *cptr;
303 	int startline;
304 	int val;
305 	static int bol = 1;	/* true if we're at beginning of line */
306 
307 	for (;;) {
308 		while (Fp == NULL) {
309 			char ibuf[80];
310 
311 			if (*Files == NULL)
312 				return (record(EOF, NULL));
313 			Fileopened = stable(*Files++);
314 #ifdef	ESC
315 			sprintf(Tok, "%s %s %s %s",
316 			    Cpp, Cppstdargs, Cppargs, Fileopened);
317 			if ((Fp = popen(Tok, "r")) == NULL)
318 				out(O_DIE|O_SYS, "%s", Tok);
319 #else
320 			Fp = eftread_fopen(Fileopened, ibuf, sizeof (ibuf));
321 #endif	/* ESC */
322 			Line = 1;
323 			bol = 1;
324 
325 			/* add name to stats for visibility */
326 			if (Fp != NULL) {
327 				static int fnum;
328 				char nbuf[100];
329 				struct filestats *nfs = MALLOC(sizeof (*nfs));
330 
331 				(void) sprintf(nbuf, "lex.file%d", fnum);
332 				nfs->stats = stats_new_string(nbuf, "", 0);
333 				stats_string_set(nfs->stats, Fileopened);
334 
335 				if (ibuf[0] != '\0') {
336 					(void) sprintf(nbuf, "lex.file%d-ident",
337 					    fnum);
338 					nfs->idstats =
339 					    stats_new_string(nbuf, "", 0);
340 					stats_string_set(nfs->idstats, ibuf);
341 				} else {
342 					nfs->idstats = NULL;
343 				}
344 
345 				nfs->next = Fstats;
346 				Fstats = nfs;
347 				fnum++;
348 			}
349 		}
350 
351 		switch (c = getc(Fp)) {
352 		case '#':
353 			/* enforce that we're at beginning of line */
354 			if (!bol)
355 				return (record(c, NULL));
356 
357 			while ((c = getc(Fp)) != EOF &&
358 			    (c == ' ' || c == '\t'))
359 				;
360 			if (!isdigit(c)) {
361 				/*
362 				 * three cases here:
363 				 *	#pragma
364 				 *	#ident
365 				 *	#something-we-don't-understand
366 				 * anything we don't expect we just ignore.
367 				 */
368 				*ptr++ = c;
369 				while ((c = getc(Fp)) != EOF && isalnum(c))
370 					if (ptr < eptr - 1)
371 						*ptr++ = c;
372 				*ptr++ = '\0';
373 				if (strcmp(Tok, "pragma") == 0) {
374 					/* skip white space */
375 					while ((c = getc(Fp)) != EOF &&
376 					    (c == ' ' || c == '\t'))
377 						;
378 
379 					if (c == EOF || c == '\n')
380 						outfl(O_DIE, File, Line,
381 						    "bad #pragma");
382 
383 					/* pull in next token */
384 					ptr = Tok;
385 					*ptr++ = c;
386 					while ((c = getc(Fp)) != EOF &&
387 					    !isspace(c))
388 						if (ptr < eptr - 1)
389 							*ptr++ = c;
390 					*ptr++ = '\0';
391 					(void) ungetc(c, Fp);
392 
393 					dopragma(Tok);
394 				} else if (strcmp(Tok, "ident") == 0)
395 					doident();
396 			} else {
397 				/* handle file & line info from cpp */
398 				Line = 0;
399 				do {
400 					if (!isdigit(c))
401 						break;
402 					Line = Line * 10 + c - '0';
403 				} while ((c = getc(Fp)) != EOF);
404 				Line--;		/* newline will increment it */
405 				while (c != EOF && isspace(c))
406 					c = getc(Fp);
407 				if (c != '"')
408 					outfl(O_DIE, File, Line,
409 					    "bad # statement (file name)");
410 				while ((c = getc(Fp)) != EOF && c != '"')
411 					if (ptr < eptr - 1)
412 						*ptr++ = c;
413 				*ptr++ = '\0';
414 				if (c != '"')
415 					outfl(O_DIE, File, Line,
416 					    "bad # statement (quotes)");
417 				File = stable(Tok);
418 			}
419 			/* skip the rest of the cpp line */
420 			while ((c = getc(Fp)) != EOF && c != '\n' && c != '\r')
421 				;
422 			if (c == EOF)
423 				return (record(c, NULL));
424 			else
425 				(void) ungetc(c, Fp);
426 			ptr = Tok;
427 			break;
428 
429 		case EOF:
430 			closefile();
431 			continue;
432 
433 		case '\n':
434 			Line++;
435 			bol = 1;
436 			break;
437 
438 		case '\r':
439 		case ' ':
440 		case '\t':
441 			bol = 0;
442 			break;
443 
444 		case '/':
445 			bol = 0;
446 			/* comment handling */
447 			if ((nextc = getc(Fp)) == EOF)
448 				outfl(O_DIE, File, Line, "unexpected EOF");
449 			else if (nextc == '*') {
450 				startline = Line;
451 				while ((c = getc(Fp)) != EOF) {
452 					if (c == '\n')
453 						Line++;
454 					else if (c == '*' &&
455 					    (((c = getc(Fp)) == EOF) ||
456 					    (c == '/')))
457 						break;
458 				}
459 				if (c == EOF) {
460 					outfl(O_DIE, File, Line,
461 					    "end of comment not seen "
462 					    "(started on line %d)",
463 					    startline);
464 				}
465 			} else {
466 				/* wasn't a comment, return the '/' token */
467 				(void) ungetc(nextc, Fp);
468 				return (record(c, NULL));
469 			}
470 			break;
471 
472 		case '"': {
473 			int prevc;
474 
475 			bol = 0;
476 			prevc = '\0';
477 			/* quoted string handling */
478 			startline = Line;
479 			for (;;) {
480 				c = getc(Fp);
481 				if (c == EOF)
482 					outfl(O_DIE, File, Line,
483 					    "end of string not seen "
484 					    "(started on line %d)",
485 					    startline);
486 				else if (c == '\n')
487 					Line++;
488 				else if (c == '"' && prevc != '\\')
489 					break;
490 				else if (ptr < eptr)
491 					*ptr++ = c;
492 				prevc = c;
493 			}
494 			if (ptr >= eptr)
495 				out(O_DIE, File, Line, "string too long");
496 			*ptr++ = '\0';
497 			return (record(QUOTE, stable(Tok)));
498 		}
499 		case '&':
500 			bol = 0;
501 			/* && */
502 			if ((nextc = getc(Fp)) == '&')
503 				return (record(AND, NULL));
504 			else {
505 				(void) ungetc(nextc, Fp);
506 				return (record(c, NULL));
507 			}
508 			/*NOTREACHED*/
509 			break;
510 
511 		case '|':
512 			bol = 0;
513 			/* || */
514 			if ((nextc = getc(Fp)) == '|')
515 				return (record(OR, NULL));
516 			else {
517 				(void) ungetc(nextc, Fp);
518 				return (record(c, NULL));
519 			}
520 			/*NOTREACHED*/
521 			break;
522 
523 		case '!':
524 			bol = 0;
525 			/* ! or != */
526 			if ((nextc = getc(Fp)) == '=')
527 				return (record(NE, NULL));
528 			else {
529 				(void) ungetc(nextc, Fp);
530 				return (record(c, NULL));
531 			}
532 			/*NOTREACHED*/
533 			break;
534 
535 		case '=':
536 			bol = 0;
537 			/* == */
538 			if ((nextc = getc(Fp)) == '=')
539 				return (record(EQ, NULL));
540 			else {
541 				(void) ungetc(nextc, Fp);
542 				return (record(c, NULL));
543 			}
544 			/*NOTREACHED*/
545 			break;
546 
547 		case '-':
548 			bol = 0;
549 			/* -> */
550 			if ((nextc = getc(Fp)) == '>')
551 				return (record(ARROW, stable(Tok)));
552 			else {
553 				(void) ungetc(nextc, Fp);
554 				return (record(c, NULL));
555 			}
556 			/*NOTREACHED*/
557 			break;
558 
559 		case '<':
560 			bol = 0;
561 			if ((nextc = getc(Fp)) == '=')
562 				/* <= */
563 				return (record(LE, NULL));
564 			else if (nextc == '<')
565 				/* << */
566 				return (record(LSHIFT, NULL));
567 			else {
568 				(void) ungetc(nextc, Fp);
569 				return (record(c, NULL));
570 			}
571 			/*NOTREACHED*/
572 			break;
573 
574 		case '>':
575 			bol = 0;
576 			if ((nextc = getc(Fp)) == '=')
577 				/* >= */
578 				return (record(GE, NULL));
579 			else if (nextc == '>')
580 				/* >> */
581 				return (record(RSHIFT, NULL));
582 			else {
583 				(void) ungetc(nextc, Fp);
584 				return (record(c, NULL));
585 			}
586 			/*NOTREACHED*/
587 			break;
588 
589 		default:
590 			bol = 0;
591 			if (isdigit(c)) {
592 				int base;
593 
594 				/* collect rest of number */
595 				if (c == '0') {
596 					*ptr++ = c;
597 					if ((c = getc(Fp)) == EOF) {
598 						*ptr++ = '\0';
599 						return (record(NUMBER,
600 						    stable(Tok)));
601 					} else if (c == 'x' || c == 'X') {
602 						*ptr++ = c;
603 						base = 16;
604 					} else {
605 						(void) ungetc(c, Fp);
606 						base = 8;
607 					}
608 				} else {
609 					*ptr++ = c;
610 					base = 10;
611 				}
612 				while ((c = getc(Fp)) != EOF) {
613 					if (ptr >= eptr)
614 						out(O_DIE, File, Line,
615 						    "number too long");
616 
617 					switch (base) {
618 					case 16:
619 						if (c >= 'a' && c <= 'f' ||
620 						    c >= 'A' && c <= 'F') {
621 							*ptr++ = c;
622 							continue;
623 						}
624 						/*FALLTHRU*/
625 					case 10:
626 						if (c >= '8' && c <= '9') {
627 							*ptr++ = c;
628 							continue;
629 						}
630 						/*FALLTHRU*/
631 					case 8:
632 						if (c >= '0' && c <= '7') {
633 							*ptr++ = c;
634 							continue;
635 						}
636 						/* not valid for this base */
637 						*ptr++ = '\0';
638 						(void) ungetc(c, Fp);
639 						return (record(NUMBER,
640 						    stable(Tok)));
641 					}
642 				}
643 				*ptr++ = '\0';
644 				return (record(NUMBER, stable(Tok)));
645 			} else if (isalpha(c)) {
646 				/* collect identifier */
647 				*ptr++ = c;
648 				for (;;) {
649 					c = getc(Fp);
650 					if ((isalnum(c) || c == '_') &&
651 					    ptr < eptr)
652 						*ptr++ = c;
653 					else {
654 						(void) ungetc(c, Fp);
655 						break;
656 					}
657 				}
658 				if (ptr >= eptr)
659 					out(O_DIE, File, Line,
660 					    "identifier too long");
661 				*ptr++ = '\0';
662 				cptr = stable(Tok);
663 				if (val = lex_s2i_lut_lookup(Rwordslut, cptr)) {
664 					return (record(val, cptr));
665 				}
666 				return (record(ID, cptr));
667 			} else
668 				return (record(c, NULL));
669 		}
670 		/*NOTREACHED*/
671 	}
672 }
673 
674 /*
675  * the record()/dumpline() routines are used to track & report
676  * the list of tokens seen on a given line.  this is used in two ways.
677  * first, syntax errors found by the parser are reported by us (via
678  * yyerror()) and we tack on the tokens processed so far on the current
679  * line to help indicate exactly where the error is.  second, if "lexecho"
680  * debugging is turned on, these routines provide it.
681  */
682 #define	MAXRECORD 1000
683 static int Recordedline;
684 static struct {
685 	int tok;
686 	const char *s;
687 } Recorded[MAXRECORD];
688 static int Recordnext;
689 
690 static int
691 record(int tok, const char *s)
692 {
693 	stats_counter_bump(Tokcount);
694 	if (Line != Recordedline) {
695 		/* starting new line, dump out the previous line */
696 		if (Lexecho && Recordedline) {
697 			outfl(O_NONL, File, Recordedline, "lex: ");
698 			dumpline(O_OK);
699 		}
700 		Recordedline = Line;
701 		Recordnext = 0;
702 	}
703 	if (Recordnext >= MAXRECORD)
704 		outfl(O_DIE, File, Line, "line too long, bailing out");
705 	Recorded[Recordnext].tok = tok;
706 	Recorded[Recordnext++].s = s;
707 
708 	yylval.tok.s = s;
709 	yylval.tok.file = File;
710 	yylval.tok.line = Line;
711 	return (tok);
712 }
713 
714 /*ARGSUSED*/
715 static void
716 dumpline(int flags)
717 {
718 	int i;
719 
720 	for (i = 0; i < Recordnext; i++)
721 		if (Recorded[i].s && Recorded[i].tok != ARROW)
722 			switch (Recorded[i].tok) {
723 			case T_QUOTE:
724 				out(flags|O_NONL, " \"%s\"",
725 				    Recorded[i].s);
726 				break;
727 
728 			default:
729 				out(flags|O_NONL, " %s",
730 				    Recorded[i].s);
731 				break;
732 			}
733 		else
734 			switch (Recorded[i].tok) {
735 			case EOF:
736 				out(flags|O_NONL, " EOF");
737 				break;
738 			case ARROW:
739 				out(flags|O_NONL, " ->%s",
740 				    Recorded[i].s);
741 				break;
742 			case EQ:
743 				out(flags|O_NONL, " ==");
744 				break;
745 			case NE:
746 				out(flags|O_NONL, " !=");
747 				break;
748 			case OR:
749 				out(flags|O_NONL, " ||");
750 				break;
751 			case AND:
752 				out(flags|O_NONL, " &&");
753 				break;
754 			case LE:
755 				out(flags|O_NONL, " <=");
756 				break;
757 			case GE:
758 				out(flags|O_NONL, " >=");
759 				break;
760 			case LSHIFT:
761 				out(flags|O_NONL, " <<");
762 				break;
763 			case RSHIFT:
764 				out(flags|O_NONL, " >>");
765 				break;
766 			default:
767 				if (isprint(Recorded[i].tok))
768 					out(flags|O_NONL, " %c",
769 					    Recorded[i].tok);
770 				else
771 					out(flags|O_NONL, " '\\%03o'",
772 					    Recorded[i].tok);
773 				break;
774 			}
775 	out(flags, NULL);
776 }
777 
778 /*
779  * yyerror -- report a pareser error, called yyerror because yacc wants it
780  */
781 
782 void
783 yyerror(const char *s)
784 {
785 	Errcount++;
786 	outfl(O_ERR|O_NONL, File, Line, "%s, tokens: ", s);
787 	dumpline(O_ERR);
788 }
789 
790 /*
791  * doident -- handle "#pragma ident" directives
792  */
793 static void
794 doident()
795 {
796 	int c;
797 	char *ptr = Tok;
798 	char *eptr = &Tok[MAXTOK];
799 
800 	/* skip white space and quotes */
801 	while ((c = getc(Fp)) != EOF &&
802 	    (c == ' ' || c == '\t' || c == '"'))
803 		;
804 
805 	if (c == EOF || c == '\n')
806 		outfl(O_DIE, File, Line, "bad ident");
807 
808 	/* pull in next token */
809 	ptr = Tok;
810 	*ptr++ = c;
811 	while ((c = getc(Fp)) != EOF && c != '"' && c != '\n')
812 		if (ptr < eptr - 1)
813 			*ptr++ = c;
814 	*ptr++ = '\0';
815 	if (c != '\n') {
816 		/* skip to end of line (including close quote, if any) */
817 		while ((c = getc(Fp)) != EOF && c != '\n')
818 			;
819 	}
820 	(void) ungetc(c, Fp);
821 	Ident = lut_add(Ident, (void *)stable(Tok), (void *)0, NULL);
822 
823 	outfl(O_VERB, File, Line, "pragma set: ident \"%s\"", Tok);
824 }
825 
826 /*
827  * dodictionary -- handle "#pragma dictionary" directives
828  */
829 static void
830 dodictionary()
831 {
832 	int c;
833 	char *ptr = Tok;
834 	char *eptr = &Tok[MAXTOK];
835 
836 	/* skip white space and quotes */
837 	while ((c = getc(Fp)) != EOF &&
838 	    (c == ' ' || c == '\t' || c == '"'))
839 		;
840 
841 	if (c == EOF || c == '\n')
842 		outfl(O_DIE, File, Line, "bad dictionary");
843 
844 	/* pull in next token */
845 	ptr = Tok;
846 	*ptr++ = c;
847 	while ((c = getc(Fp)) != EOF && c != '"' && c != '\n')
848 		if (ptr < eptr - 1)
849 			*ptr++ = c;
850 	*ptr++ = '\0';
851 	if (c != '\n') {
852 		/* skip to end of line (including close quote, if any) */
853 		while ((c = getc(Fp)) != EOF && c != '\n')
854 			;
855 	}
856 	(void) ungetc(c, Fp);
857 	Dicts = lut_add(Dicts, (void *)stable(Tok), (void *)0, NULL);
858 
859 	outfl(O_VERB, File, Line, "pragma set: dictionary \"%s\"", Tok);
860 }
861 
862 /*
863  * doallow_cycles -- handle "#pragma allow_cycles" directives
864  */
865 static void
866 doallow_cycles()
867 {
868 	int c;
869 	char *ptr = Tok;
870 	char *eptr = &Tok[MAXTOK];
871 	unsigned long long newlevel;
872 
873 	/*
874 	 * by default the compiler does not allow cycles or loops
875 	 * in propagations.  when cycles are encountered, the
876 	 * compiler prints out an error message.
877 	 *
878 	 *   "#pragma allow_cycles" and
879 	 *   "#pragma allow_cycles 0"
880 	 * allow cycles, but any such cycle will produce a warning
881 	 * message.
882 	 *
883 	 *   "#pragma allow_cycles N"
884 	 * with N > 0 will allow cycles and not produce any
885 	 * warning messages.
886 	 */
887 
888 	/* skip white space and quotes */
889 	while ((c = getc(Fp)) != EOF &&
890 	    (c == ' ' || c == '\t' || c == '"'))
891 		;
892 
893 	if (c == EOF || c == '\n')
894 		newlevel = 0ULL;
895 	else {
896 
897 		/* pull in next token */
898 		ptr = Tok;
899 		*ptr++ = c;
900 		while ((c = getc(Fp)) != EOF && c != '"' && c != '\n')
901 			if (ptr < eptr - 1)
902 				*ptr++ = c;
903 		*ptr++ = '\0';
904 		if (c != '\n') {
905 			/* skip to end of line */
906 			while ((c = getc(Fp)) != EOF && c != '\n')
907 				;
908 		}
909 		newlevel = strtoll(Tok, NULL, 0);
910 	}
911 	(void) ungetc(c, Fp);
912 
913 	(void) check_cycle_level(newlevel);
914 	outfl(O_VERB, File, Line,
915 	    "pragma set: allow_cycles (%s)",
916 	    newlevel ? "no warnings" : "with warnings");
917 }
918 
919 /*
920  * dopragma -- handle #pragma directives
921  */
922 static void
923 dopragma(const char *tok)
924 {
925 	if (strcmp(tok, "ident") == 0)
926 		doident();
927 	else if (strcmp(tok, "dictionary") == 0)
928 		dodictionary();
929 	else if (strcmp(tok, "new_errors_only") == 0) {
930 		if (Pragma_new_errors_only++ == 0)
931 			outfl(O_VERB, File, Line,
932 			    "pragma set: new_errors_only");
933 	} else if (strcmp(tok, "trust_ereports") == 0) {
934 		if (Pragma_trust_ereports++ == 0)
935 			outfl(O_VERB, File, Line,
936 			    "pragma set: trust_ereports");
937 	} else if (strcmp(tok, "allow_cycles") == 0)
938 		doallow_cycles();
939 	else
940 		outfl(O_VERB, File, Line,
941 		    "unknown pragma ignored: \"%s\"", tok);
942 }
943 
944 /*
945  * lex_fini -- finalize the lexer
946  */
947 
948 int
949 lex_fini(void)
950 {
951 	stats_elapse_stop(Lexelapse);
952 	closefile();
953 	if (Lexecho) {
954 		outfl(O_OK, File, Line, "lex: ");
955 		dumpline(O_OK);
956 	}
957 	return (Errcount);
958 }
959 
960 void
961 lex_free(void)
962 {
963 	struct filestats *nfstats = Fstats;
964 
965 	/*
966 	 * Free up memory consumed by the lexer
967 	 */
968 	stats_delete(Tokcount);
969 	stats_delete(Filecount);
970 	stats_delete(Lexelapse);
971 	while (nfstats != NULL) {
972 		Fstats = nfstats->next;
973 		stats_delete(nfstats->stats);
974 		if (nfstats->idstats != NULL)
975 			stats_delete(nfstats->idstats);
976 		FREE(nfstats);
977 		nfstats = Fstats;
978 	}
979 	lut_free(Timesuffixlut, NULL, NULL);
980 	lut_free(Rwordslut, NULL, NULL);
981 	lut_free(Ident, NULL, NULL);
982 	lut_free(Dicts, NULL, NULL);
983 }
984