xref: /illumos-gate/usr/src/cmd/fm/eversholt/common/esclex.c (revision 2bbdd445a21f9d61f4a0ca0faf05d5ceb2bd91f3)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  *
25  * esclex.c -- lexer for esc
26  *
27  * this module provides lexical analysis and error handling routine
28  * expected by the yacc-generated parser (i.e. yylex() and yyerror()).
29  * it also does lots of tracking of things like filenames, line numbers,
30  * and what tokens are seen on a line up to the point where a syntax error
31  * was found.  this module also arranges for the input source files to
32  * be run through cpp.
33  */
34 
35 #include <stdio.h>
36 #include <ctype.h>
37 #include <string.h>
38 #include <stdlib.h>
39 #include <unistd.h>
40 #include <time.h>
41 #include <errno.h>
42 #include "out.h"
43 #include "alloc.h"
44 #include "stats.h"
45 #include "stable.h"
46 #include "lut.h"
47 #include "literals.h"
48 #include "tree.h"
49 #include "esclex.h"
50 #include "eftread.h"
51 #include "check.h"
52 #include "y.tab.h"
53 
54 /* ridiculously long token buffer -- disallow any token longer than this */
55 #define	MAXTOK	8192
56 static char Tok[MAXTOK];
57 
58 /* some misc stats we keep on the lexer & parser */
59 static struct stats *Tokcount;
60 static struct stats *Lexelapse;
61 struct stats *Filecount;
62 struct filestats {
63 	struct filestats *next;
64 	struct stats *stats;
65 	struct stats *idstats;
66 } *Fstats;
67 
68 static int Errcount;
69 
70 /* input file state */
71 static char **Files;
72 static const char *Fileopened;
73 static FILE *Fp;
74 static int Line;
75 static const char *File;
76 static const char *Cpp = "/usr/bin/cpp";
77 #ifdef	ESC
78 static const char *Cppargs;
79 static const char *Cppstdargs = "-undef -Y.";
80 #endif	/* ESC */
81 
82 /* for debugging */
83 static int Lexecho;	/* echo tokens as we read them */
84 
85 /* forward declarations of our internal routines */
86 static int record(int tok, const char *s);
87 static void dumpline(int flags);
88 static void doident();
89 static void dopragma(const char *tok);
90 
91 /*
92  * table of reserved words.  this table is only used by lex_init()
93  * to intialize the Rwords lookup table.
94  */
95 static const struct {
96 	const char *word;
97 	const int val;
98 } Rwords[] = {
99 	{ "asru", ASRU },
100 	{ "count", COUNT },
101 	{ "div", DIV },
102 	{ "engine", ENGINE },
103 	{ "event", EVENT },
104 	{ "fru", FRU },
105 	{ "if", IF },
106 	{ "mask", MASK },
107 	{ "prop", PROP },
108 	{ "config", CONFIG },
109 	/*
110 	 * PATHFUNC indicates functions that operate only on paths
111 	 * and quotes
112 	 */
113 	{ "is_connected", PATHFUNC },
114 	{ "is_under", PATHFUNC },
115 	{ "is_on", PATHFUNC },
116 	{ "is_present", PATHFUNC },
117 	{ "is_type", PATHFUNC },
118 	{ "has_fault", PATHFUNC },
119 	{ "confprop", PATHFUNC },
120 	{ "confprop_defined", PATHFUNC },
121 };
122 
123 /*
124  * Rwordslut is a lookup table of reserved words.  lhs is the word
125  * (in the string table) and the rhs is the token value returned
126  * by the yylex() for that word.
127  */
128 static struct lut *Rwordslut;
129 
130 static const struct {
131 	const char *suffix;
132 	const unsigned long long nsec;
133 } Timesuffix[] = {
134 	{ "nanosecond",		1ULL },
135 	{ "nanoseconds",	1ULL },
136 	{ "nsec",		1ULL },
137 	{ "nsecs",		1ULL },
138 	{ "ns",			1ULL },
139 	{ "microsecond",	1000ULL },
140 	{ "microseconds",	1000ULL },
141 	{ "usec",		1000ULL },
142 	{ "usecs",		1000ULL },
143 	{ "us",			1000ULL },
144 	{ "millisecond",	1000000ULL },
145 	{ "milliseconds",	1000000ULL },
146 	{ "msec",		1000000ULL },
147 	{ "msecs",		1000000ULL },
148 	{ "ms",			1000000ULL },
149 	{ "second",		1000000000ULL },
150 	{ "seconds",		1000000000ULL },
151 	{ "s",			1000000000ULL },
152 	{ "minute",		1000000000ULL * 60 },
153 	{ "minutes",		1000000000ULL * 60 },
154 	{ "min",		1000000000ULL * 60 },
155 	{ "mins",		1000000000ULL * 60 },
156 	{ "m",			1000000000ULL * 60 },
157 	{ "hour",		1000000000ULL * 60 * 60 },
158 	{ "hours",		1000000000ULL * 60 * 60 },
159 	{ "hr",			1000000000ULL * 60 * 60 },
160 	{ "hrs",		1000000000ULL * 60 * 60 },
161 	{ "h",			1000000000ULL * 60 * 60 },
162 	{ "day",		1000000000ULL * 60 * 60 * 24 },
163 	{ "days",		1000000000ULL * 60 * 60 * 24 },
164 	{ "d",			1000000000ULL * 60 * 60 * 24 },
165 	{ "week",		1000000000ULL * 60 * 60 * 24 * 7 },
166 	{ "weeks",		1000000000ULL * 60 * 60 * 24 * 7 },
167 	{ "wk",			1000000000ULL * 60 * 60 * 24 * 7 },
168 	{ "wks",		1000000000ULL * 60 * 60 * 24 * 7 },
169 	{ "month",		1000000000ULL * 60 * 60 * 24 * 30 },
170 	{ "months",		1000000000ULL * 60 * 60 * 24 * 30 },
171 	{ "year",		1000000000ULL * 60 * 60 * 24 * 365 },
172 	{ "years",		1000000000ULL * 60 * 60 * 24 * 365 },
173 	{ "yr",			1000000000ULL * 60 * 60 * 24 * 365 },
174 	{ "yrs",		1000000000ULL * 60 * 60 * 24 * 365 },
175 };
176 
177 /*
178  * some wrappers around the general lut functions to provide type checking...
179  */
180 
181 static struct lut *
182 lex_s2i_lut_add(struct lut *root, const char *s, intptr_t i)
183 {
184 	return (lut_add(root, (void *)s, (void *)i, NULL));
185 }
186 
187 static int
188 lex_s2i_lut_lookup(struct lut *root, const char *s)
189 {
190 	return ((intptr_t)lut_lookup(root, (void *)s, NULL));
191 }
192 
193 static struct lut *
194 lex_s2ullp_lut_add(struct lut *root, const char *s,
195     const unsigned long long *ullp)
196 {
197 	return (lut_add(root, (void *)s, (void *)ullp, NULL));
198 }
199 
200 const unsigned long long *
201 lex_s2ullp_lut_lookup(struct lut *root, const char *s)
202 {
203 	return ((unsigned long long *)lut_lookup(root, (void *)s, NULL));
204 }
205 
206 /*
207  * lex_init -- initialize the lexer with appropriate filenames & debug flags
208  */
209 
210 /*ARGSUSED*/
211 void
212 lex_init(char **av, const char *cppargs, int lexecho)
213 {
214 	int i;
215 #ifdef	ESC
216 	const char *ptr;
217 #endif	/* ESC */
218 
219 	Lexecho = lexecho;
220 	Tokcount = stats_new_counter("lex.tokens", "total tokens in", 1);
221 	Filecount = stats_new_counter("lex.files", "total files read", 0);
222 	Lexelapse = stats_new_elapse("lex.time", "elapsed lex/parse time", 1);
223 
224 #ifdef	ESC
225 	Cppargs = cppargs;
226 
227 	/* allow user to tell us where cpp is if it is some weird place */
228 	if (ptr = getenv("_ESC_CPP"))
229 		Cpp = ptr;
230 
231 	/* and in case it takes some special stdargs */
232 	if (ptr = getenv("_ESC_CPP_STDARGS"))
233 		Cppstdargs = ptr;
234 
235 	/* verify we can find cpp */
236 	if (access(Cpp, X_OK) < 0) {
237 		Cpp = "/usr/lib/cpp";
238 		if (access(Cpp, X_OK) < 0)
239 			out(O_DIE, "can't locate cpp");
240 	}
241 #endif	/* ESC */
242 
243 	Files = av;
244 
245 	/* verify we can find all the input files */
246 	while (*av) {
247 		if (strlen(*av) >= MAXTOK - strlen(Cpp) - 3)
248 			out(O_DIE, "filename too long: %.100s...", *av);
249 		if (access(*av, R_OK) < 0)
250 			out(O_DIE|O_SYS, "%s", *av);
251 		av++;
252 		stats_counter_bump(Filecount);
253 	}
254 
255 	/* put reserved words into the string table & a lookup table */
256 	for (i = 0; i < sizeof (Rwords) / sizeof (*Rwords); i++)
257 		Rwordslut = lex_s2i_lut_add(Rwordslut,
258 		    stable(Rwords[i].word), Rwords[i].val);
259 
260 	/* initialize table of timeval suffixes */
261 	for (i = 0; i < sizeof (Timesuffix) / sizeof (*Timesuffix); i++) {
262 		Timesuffixlut = lex_s2ullp_lut_add(Timesuffixlut,
263 		    stable(Timesuffix[i].suffix), &Timesuffix[i].nsec);
264 	}
265 
266 	/* record start time */
267 	stats_elapse_start(Lexelapse);
268 }
269 
270 void
271 closefile(void)
272 {
273 	if (Fp != NULL) {
274 #ifdef	ESC
275 		if (pclose(Fp) > 0)
276 			out(O_DIE, "cpp errors while reading \"%s\", "
277 			    "bailing out.", Fileopened);
278 #else
279 		(void) fclose(Fp);
280 #endif	/* ESC */
281 	}
282 	Fp = NULL;
283 }
284 
285 /*
286  * yylex -- the lexer, called yylex() because that's what yacc wants
287  */
288 
289 int
290 yylex()
291 {
292 	int c;
293 	int nextc;
294 	char *ptr = Tok;
295 	char *eptr = &Tok[MAXTOK];
296 	const char *cptr;
297 	int startline;
298 	int val;
299 	static int bol = 1;	/* true if we're at beginning of line */
300 
301 	for (;;) {
302 		while (Fp == NULL) {
303 			char ibuf[80];
304 
305 			if (*Files == NULL)
306 				return (record(EOF, NULL));
307 			Fileopened = stable(*Files++);
308 #ifdef	ESC
309 			sprintf(Tok, "%s %s %s %s",
310 			    Cpp, Cppstdargs, Cppargs, Fileopened);
311 			if ((Fp = popen(Tok, "r")) == NULL)
312 				out(O_DIE|O_SYS, "%s", Tok);
313 #else
314 			Fp = eftread_fopen(Fileopened, ibuf, sizeof (ibuf));
315 #endif	/* ESC */
316 			Line = 1;
317 			bol = 1;
318 
319 			/* add name to stats for visibility */
320 			if (Fp != NULL) {
321 				static int fnum;
322 				char nbuf[100];
323 				struct filestats *nfs = MALLOC(sizeof (*nfs));
324 
325 				(void) sprintf(nbuf, "lex.file%d", fnum);
326 				nfs->stats = stats_new_string(nbuf, "", 0);
327 				stats_string_set(nfs->stats, Fileopened);
328 
329 				if (ibuf[0] != '\0') {
330 					(void) sprintf(nbuf, "lex.file%d-ident",
331 					    fnum);
332 					nfs->idstats =
333 					    stats_new_string(nbuf, "", 0);
334 					stats_string_set(nfs->idstats, ibuf);
335 				} else {
336 					nfs->idstats = NULL;
337 				}
338 
339 				nfs->next = Fstats;
340 				Fstats = nfs;
341 				fnum++;
342 			}
343 		}
344 
345 		switch (c = getc(Fp)) {
346 		case '#':
347 			/* enforce that we're at beginning of line */
348 			if (!bol)
349 				return (record(c, NULL));
350 
351 			while ((c = getc(Fp)) != EOF &&
352 			    (c == ' ' || c == '\t'))
353 				;
354 			if (!isdigit(c)) {
355 				/*
356 				 * three cases here:
357 				 *	#pragma
358 				 *	#ident
359 				 *	#something-we-don't-understand
360 				 * anything we don't expect we just ignore.
361 				 */
362 				*ptr++ = c;
363 				while ((c = getc(Fp)) != EOF && isalnum(c))
364 					if (ptr < eptr - 1)
365 						*ptr++ = c;
366 				*ptr++ = '\0';
367 				if (strcmp(Tok, "pragma") == 0) {
368 					/* skip white space */
369 					while ((c = getc(Fp)) != EOF &&
370 					    (c == ' ' || c == '\t'))
371 						;
372 
373 					if (c == EOF || c == '\n')
374 						outfl(O_DIE, File, Line,
375 						    "bad #pragma");
376 
377 					/* pull in next token */
378 					ptr = Tok;
379 					*ptr++ = c;
380 					while ((c = getc(Fp)) != EOF &&
381 					    !isspace(c))
382 						if (ptr < eptr - 1)
383 							*ptr++ = c;
384 					*ptr++ = '\0';
385 					(void) ungetc(c, Fp);
386 
387 					dopragma(Tok);
388 				} else if (strcmp(Tok, "ident") == 0)
389 					doident();
390 			} else {
391 				/* handle file & line info from cpp */
392 				Line = 0;
393 				do {
394 					if (!isdigit(c))
395 						break;
396 					Line = Line * 10 + c - '0';
397 				} while ((c = getc(Fp)) != EOF);
398 				Line--;		/* newline will increment it */
399 				while (c != EOF && isspace(c))
400 					c = getc(Fp);
401 				if (c != '"')
402 					outfl(O_DIE, File, Line,
403 					    "bad # statement (file name)");
404 				while ((c = getc(Fp)) != EOF && c != '"')
405 					if (ptr < eptr - 1)
406 						*ptr++ = c;
407 				*ptr++ = '\0';
408 				if (c != '"')
409 					outfl(O_DIE, File, Line,
410 					    "bad # statement (quotes)");
411 				File = stable(Tok);
412 			}
413 			/* skip the rest of the cpp line */
414 			while ((c = getc(Fp)) != EOF && c != '\n' && c != '\r')
415 				;
416 			if (c == EOF)
417 				return (record(c, NULL));
418 			else
419 				(void) ungetc(c, Fp);
420 			ptr = Tok;
421 			break;
422 
423 		case EOF:
424 			closefile();
425 			continue;
426 
427 		case '\n':
428 			Line++;
429 			bol = 1;
430 			break;
431 
432 		case '\r':
433 		case ' ':
434 		case '\t':
435 			bol = 0;
436 			break;
437 
438 		case '/':
439 			bol = 0;
440 			/* comment handling */
441 			if ((nextc = getc(Fp)) == EOF)
442 				outfl(O_DIE, File, Line, "unexpected EOF");
443 			else if (nextc == '*') {
444 				startline = Line;
445 				while ((c = getc(Fp)) != EOF) {
446 					if (c == '\n')
447 						Line++;
448 					else if (c == '*' &&
449 					    (((c = getc(Fp)) == EOF) ||
450 					    (c == '/')))
451 						break;
452 				}
453 				if (c == EOF) {
454 					outfl(O_DIE, File, Line,
455 					    "end of comment not seen "
456 					    "(started on line %d)",
457 					    startline);
458 				}
459 			} else {
460 				/* wasn't a comment, return the '/' token */
461 				(void) ungetc(nextc, Fp);
462 				return (record(c, NULL));
463 			}
464 			break;
465 
466 		case '"': {
467 			int prevc;
468 
469 			bol = 0;
470 			prevc = '\0';
471 			/* quoted string handling */
472 			startline = Line;
473 			for (;;) {
474 				c = getc(Fp);
475 				if (c == EOF)
476 					outfl(O_DIE, File, Line,
477 					    "end of string not seen "
478 					    "(started on line %d)",
479 					    startline);
480 				else if (c == '\n')
481 					Line++;
482 				else if (c == '"' && prevc != '\\')
483 					break;
484 				else if (ptr < eptr)
485 					*ptr++ = c;
486 				prevc = c;
487 			}
488 			if (ptr >= eptr)
489 				out(O_DIE, File, Line, "string too long");
490 			*ptr++ = '\0';
491 			return (record(QUOTE, stable(Tok)));
492 		}
493 		case '&':
494 			bol = 0;
495 			/* && */
496 			if ((nextc = getc(Fp)) == '&')
497 				return (record(AND, NULL));
498 			else {
499 				(void) ungetc(nextc, Fp);
500 				return (record(c, NULL));
501 			}
502 			/*NOTREACHED*/
503 			break;
504 
505 		case '|':
506 			bol = 0;
507 			/* || */
508 			if ((nextc = getc(Fp)) == '|')
509 				return (record(OR, NULL));
510 			else {
511 				(void) ungetc(nextc, Fp);
512 				return (record(c, NULL));
513 			}
514 			/*NOTREACHED*/
515 			break;
516 
517 		case '!':
518 			bol = 0;
519 			/* ! or != */
520 			if ((nextc = getc(Fp)) == '=')
521 				return (record(NE, NULL));
522 			else {
523 				(void) ungetc(nextc, Fp);
524 				return (record(c, NULL));
525 			}
526 			/*NOTREACHED*/
527 			break;
528 
529 		case '=':
530 			bol = 0;
531 			/* == */
532 			if ((nextc = getc(Fp)) == '=')
533 				return (record(EQ, NULL));
534 			else {
535 				(void) ungetc(nextc, Fp);
536 				return (record(c, NULL));
537 			}
538 			/*NOTREACHED*/
539 			break;
540 
541 		case '-':
542 			bol = 0;
543 			/* -> */
544 			if ((nextc = getc(Fp)) == '>')
545 				return (record(ARROW, stable(Tok)));
546 			else {
547 				(void) ungetc(nextc, Fp);
548 				return (record(c, NULL));
549 			}
550 			/*NOTREACHED*/
551 			break;
552 
553 		case '<':
554 			bol = 0;
555 			if ((nextc = getc(Fp)) == '=')
556 				/* <= */
557 				return (record(LE, NULL));
558 			else if (nextc == '<')
559 				/* << */
560 				return (record(LSHIFT, NULL));
561 			else {
562 				(void) ungetc(nextc, Fp);
563 				return (record(c, NULL));
564 			}
565 			/*NOTREACHED*/
566 			break;
567 
568 		case '>':
569 			bol = 0;
570 			if ((nextc = getc(Fp)) == '=')
571 				/* >= */
572 				return (record(GE, NULL));
573 			else if (nextc == '>')
574 				/* >> */
575 				return (record(RSHIFT, NULL));
576 			else {
577 				(void) ungetc(nextc, Fp);
578 				return (record(c, NULL));
579 			}
580 			/*NOTREACHED*/
581 			break;
582 
583 		default:
584 			bol = 0;
585 			if (isdigit(c)) {
586 				int base;
587 
588 				/* collect rest of number */
589 				if (c == '0') {
590 					*ptr++ = c;
591 					if ((c = getc(Fp)) == EOF) {
592 						*ptr++ = '\0';
593 						return (record(NUMBER,
594 						    stable(Tok)));
595 					} else if (c == 'x' || c == 'X') {
596 						*ptr++ = c;
597 						base = 16;
598 					} else {
599 						(void) ungetc(c, Fp);
600 						base = 8;
601 					}
602 				} else {
603 					*ptr++ = c;
604 					base = 10;
605 				}
606 				while ((c = getc(Fp)) != EOF) {
607 					if (ptr >= eptr)
608 						out(O_DIE, File, Line,
609 						    "number too long");
610 
611 					switch (base) {
612 					case 16:
613 						if (c >= 'a' && c <= 'f' ||
614 						    c >= 'A' && c <= 'F') {
615 							*ptr++ = c;
616 							continue;
617 						}
618 						/*FALLTHRU*/
619 					case 10:
620 						if (c >= '8' && c <= '9') {
621 							*ptr++ = c;
622 							continue;
623 						}
624 						/*FALLTHRU*/
625 					case 8:
626 						if (c >= '0' && c <= '7') {
627 							*ptr++ = c;
628 							continue;
629 						}
630 						/* not valid for this base */
631 						*ptr++ = '\0';
632 						(void) ungetc(c, Fp);
633 						return (record(NUMBER,
634 						    stable(Tok)));
635 					}
636 				}
637 				*ptr++ = '\0';
638 				return (record(NUMBER, stable(Tok)));
639 			} else if (isalpha(c)) {
640 				/* collect identifier */
641 				*ptr++ = c;
642 				for (;;) {
643 					c = getc(Fp);
644 					if ((isalnum(c) || c == '_') &&
645 					    ptr < eptr)
646 						*ptr++ = c;
647 					else {
648 						(void) ungetc(c, Fp);
649 						break;
650 					}
651 				}
652 				if (ptr >= eptr)
653 					out(O_DIE, File, Line,
654 					    "identifier too long");
655 				*ptr++ = '\0';
656 				cptr = stable(Tok);
657 				if (val = lex_s2i_lut_lookup(Rwordslut, cptr)) {
658 					return (record(val, cptr));
659 				}
660 				return (record(ID, cptr));
661 			} else
662 				return (record(c, NULL));
663 		}
664 		/*NOTREACHED*/
665 	}
666 }
667 
668 /*
669  * the record()/dumpline() routines are used to track & report
670  * the list of tokens seen on a given line.  this is used in two ways.
671  * first, syntax errors found by the parser are reported by us (via
672  * yyerror()) and we tack on the tokens processed so far on the current
673  * line to help indicate exactly where the error is.  second, if "lexecho"
674  * debugging is turned on, these routines provide it.
675  */
676 #define	MAXRECORD 1000
677 static int Recordedline;
678 static struct {
679 	int tok;
680 	const char *s;
681 } Recorded[MAXRECORD];
682 static int Recordnext;
683 
684 static int
685 record(int tok, const char *s)
686 {
687 	stats_counter_bump(Tokcount);
688 	if (Line != Recordedline) {
689 		/* starting new line, dump out the previous line */
690 		if (Lexecho && Recordedline) {
691 			outfl(O_NONL, File, Recordedline, "lex: ");
692 			dumpline(O_OK);
693 		}
694 		Recordedline = Line;
695 		Recordnext = 0;
696 	}
697 	if (Recordnext >= MAXRECORD)
698 		outfl(O_DIE, File, Line, "line too long, bailing out");
699 	Recorded[Recordnext].tok = tok;
700 	Recorded[Recordnext++].s = s;
701 
702 	yylval.tok.s = s;
703 	yylval.tok.file = File;
704 	yylval.tok.line = Line;
705 	return (tok);
706 }
707 
708 /*ARGSUSED*/
709 static void
710 dumpline(int flags)
711 {
712 	int i;
713 
714 	for (i = 0; i < Recordnext; i++)
715 		if (Recorded[i].s && Recorded[i].tok != ARROW)
716 			switch (Recorded[i].tok) {
717 			case T_QUOTE:
718 				out(flags|O_NONL, " \"%s\"",
719 				    Recorded[i].s);
720 				break;
721 
722 			default:
723 				out(flags|O_NONL, " %s",
724 				    Recorded[i].s);
725 				break;
726 			}
727 		else
728 			switch (Recorded[i].tok) {
729 			case EOF:
730 				out(flags|O_NONL, " EOF");
731 				break;
732 			case ARROW:
733 				out(flags|O_NONL, " ->%s",
734 				    Recorded[i].s);
735 				break;
736 			case EQ:
737 				out(flags|O_NONL, " ==");
738 				break;
739 			case NE:
740 				out(flags|O_NONL, " !=");
741 				break;
742 			case OR:
743 				out(flags|O_NONL, " ||");
744 				break;
745 			case AND:
746 				out(flags|O_NONL, " &&");
747 				break;
748 			case LE:
749 				out(flags|O_NONL, " <=");
750 				break;
751 			case GE:
752 				out(flags|O_NONL, " >=");
753 				break;
754 			case LSHIFT:
755 				out(flags|O_NONL, " <<");
756 				break;
757 			case RSHIFT:
758 				out(flags|O_NONL, " >>");
759 				break;
760 			default:
761 				if (isprint(Recorded[i].tok))
762 					out(flags|O_NONL, " %c",
763 					    Recorded[i].tok);
764 				else
765 					out(flags|O_NONL, " '\\%03o'",
766 					    Recorded[i].tok);
767 				break;
768 			}
769 	out(flags, NULL);
770 }
771 
772 /*
773  * yyerror -- report a pareser error, called yyerror because yacc wants it
774  */
775 
776 void
777 yyerror(const char *s)
778 {
779 	Errcount++;
780 	outfl(O_ERR|O_NONL, File, Line, "%s, tokens: ", s);
781 	dumpline(O_ERR);
782 }
783 
784 /*
785  * doident -- handle "#pragma ident" directives
786  */
787 static void
788 doident()
789 {
790 	int c;
791 	char *ptr = Tok;
792 	char *eptr = &Tok[MAXTOK];
793 
794 	/* skip white space and quotes */
795 	while ((c = getc(Fp)) != EOF &&
796 	    (c == ' ' || c == '\t' || c == '"'))
797 		;
798 
799 	if (c == EOF || c == '\n')
800 		outfl(O_DIE, File, Line, "bad ident");
801 
802 	/* pull in next token */
803 	ptr = Tok;
804 	*ptr++ = c;
805 	while ((c = getc(Fp)) != EOF && c != '"' && c != '\n')
806 		if (ptr < eptr - 1)
807 			*ptr++ = c;
808 	*ptr++ = '\0';
809 	if (c != '\n') {
810 		/* skip to end of line (including close quote, if any) */
811 		while ((c = getc(Fp)) != EOF && c != '\n')
812 			;
813 	}
814 	(void) ungetc(c, Fp);
815 	Ident = lut_add(Ident, (void *)stable(Tok), (void *)0, NULL);
816 
817 	outfl(O_VERB, File, Line, "pragma set: ident \"%s\"", Tok);
818 }
819 
820 /*
821  * dodictionary -- handle "#pragma dictionary" directives
822  */
823 static void
824 dodictionary()
825 {
826 	int c;
827 	char *ptr = Tok;
828 	char *eptr = &Tok[MAXTOK];
829 
830 	/* skip white space and quotes */
831 	while ((c = getc(Fp)) != EOF &&
832 	    (c == ' ' || c == '\t' || c == '"'))
833 		;
834 
835 	if (c == EOF || c == '\n')
836 		outfl(O_DIE, File, Line, "bad dictionary");
837 
838 	/* pull in next token */
839 	ptr = Tok;
840 	*ptr++ = c;
841 	while ((c = getc(Fp)) != EOF && c != '"' && c != '\n')
842 		if (ptr < eptr - 1)
843 			*ptr++ = c;
844 	*ptr++ = '\0';
845 	if (c != '\n') {
846 		/* skip to end of line (including close quote, if any) */
847 		while ((c = getc(Fp)) != EOF && c != '\n')
848 			;
849 	}
850 	(void) ungetc(c, Fp);
851 	Dicts = lut_add(Dicts, (void *)stable(Tok), (void *)0, NULL);
852 
853 	outfl(O_VERB, File, Line, "pragma set: dictionary \"%s\"", Tok);
854 }
855 
856 /*
857  * doallow_cycles -- handle "#pragma allow_cycles" directives
858  */
859 static void
860 doallow_cycles()
861 {
862 	int c;
863 	char *ptr = Tok;
864 	char *eptr = &Tok[MAXTOK];
865 	unsigned long long newlevel;
866 
867 	/*
868 	 * by default the compiler does not allow cycles or loops
869 	 * in propagations.  when cycles are encountered, the
870 	 * compiler prints out an error message.
871 	 *
872 	 *   "#pragma allow_cycles" and
873 	 *   "#pragma allow_cycles 0"
874 	 * allow cycles, but any such cycle will produce a warning
875 	 * message.
876 	 *
877 	 *   "#pragma allow_cycles N"
878 	 * with N > 0 will allow cycles and not produce any
879 	 * warning messages.
880 	 */
881 
882 	/* skip white space and quotes */
883 	while ((c = getc(Fp)) != EOF &&
884 	    (c == ' ' || c == '\t' || c == '"'))
885 		;
886 
887 	if (c == EOF || c == '\n')
888 		newlevel = 0ULL;
889 	else {
890 
891 		/* pull in next token */
892 		ptr = Tok;
893 		*ptr++ = c;
894 		while ((c = getc(Fp)) != EOF && c != '"' && c != '\n')
895 			if (ptr < eptr - 1)
896 				*ptr++ = c;
897 		*ptr++ = '\0';
898 		if (c != '\n') {
899 			/* skip to end of line */
900 			while ((c = getc(Fp)) != EOF && c != '\n')
901 				;
902 		}
903 		newlevel = strtoll(Tok, NULL, 0);
904 	}
905 	(void) ungetc(c, Fp);
906 
907 	(void) check_cycle_level(newlevel);
908 	outfl(O_VERB, File, Line,
909 	    "pragma set: allow_cycles (%s)",
910 	    newlevel ? "no warnings" : "with warnings");
911 }
912 
913 /*
914  * dopragma -- handle #pragma directives
915  */
916 static void
917 dopragma(const char *tok)
918 {
919 	if (strcmp(tok, "ident") == 0)
920 		doident();
921 	else if (strcmp(tok, "dictionary") == 0)
922 		dodictionary();
923 	else if (strcmp(tok, "new_errors_only") == 0) {
924 		if (Pragma_new_errors_only++ == 0)
925 			outfl(O_VERB, File, Line,
926 			    "pragma set: new_errors_only");
927 	} else if (strcmp(tok, "trust_ereports") == 0) {
928 		if (Pragma_trust_ereports++ == 0)
929 			outfl(O_VERB, File, Line,
930 			    "pragma set: trust_ereports");
931 	} else if (strcmp(tok, "allow_cycles") == 0)
932 		doallow_cycles();
933 	else
934 		outfl(O_VERB, File, Line,
935 		    "unknown pragma ignored: \"%s\"", tok);
936 }
937 
938 /*
939  * lex_fini -- finalize the lexer
940  */
941 
942 int
943 lex_fini(void)
944 {
945 	stats_elapse_stop(Lexelapse);
946 	closefile();
947 	if (Lexecho) {
948 		outfl(O_OK, File, Line, "lex: ");
949 		dumpline(O_OK);
950 	}
951 	return (Errcount);
952 }
953 
954 void
955 lex_free(void)
956 {
957 	struct filestats *nfstats = Fstats;
958 
959 	/*
960 	 * Free up memory consumed by the lexer
961 	 */
962 	stats_delete(Tokcount);
963 	stats_delete(Filecount);
964 	stats_delete(Lexelapse);
965 	while (nfstats != NULL) {
966 		Fstats = nfstats->next;
967 		stats_delete(nfstats->stats);
968 		if (nfstats->idstats != NULL)
969 			stats_delete(nfstats->idstats);
970 		FREE(nfstats);
971 		nfstats = Fstats;
972 	}
973 	lut_free(Timesuffixlut, NULL, NULL);
974 	lut_free(Rwordslut, NULL, NULL);
975 	lut_free(Ident, NULL, NULL);
976 	lut_free(Dicts, NULL, NULL);
977 }
978