1 /* BEGIN CSTYLED */ 2 /* 3 ** $Id: llex.c,v 2.63.1.3 2015/02/09 17:56:34 roberto Exp $ 4 ** Lexical Analyzer 5 ** See Copyright Notice in lua.h 6 */ 7 8 #define llex_c 9 #define LUA_CORE 10 11 #include <sys/lua/lua.h> 12 13 #include "lctype.h" 14 #include "ldo.h" 15 #include "llex.h" 16 #include "lobject.h" 17 #include "lparser.h" 18 #include "lstate.h" 19 #include "lstring.h" 20 #include "ltable.h" 21 #include "lzio.h" 22 23 24 25 #define next(ls) (ls->current = zgetc(ls->z)) 26 27 28 29 #define currIsNewline(ls) (ls->current == '\n' || ls->current == '\r') 30 31 32 /* ORDER RESERVED */ 33 static const char *const luaX_tokens [] = { 34 "and", "break", "do", "else", "elseif", 35 "end", "false", "for", "function", "goto", "if", 36 "in", "local", "nil", "not", "or", "repeat", 37 "return", "then", "true", "until", "while", 38 "..", "...", "==", ">=", "<=", "~=", "::", "<eof>", 39 "<number>", "<name>", "<string>" 40 }; 41 42 43 #define save_and_next(ls) (save(ls, ls->current), next(ls)) 44 45 46 static l_noret lexerror (LexState *ls, const char *msg, int token); 47 48 49 static void save (LexState *ls, int c) { 50 Mbuffer *b = ls->buff; 51 if (luaZ_bufflen(b) + 1 > luaZ_sizebuffer(b)) { 52 size_t newsize; 53 if (luaZ_sizebuffer(b) >= MAX_SIZET/2) 54 lexerror(ls, "lexical element too long", 0); 55 newsize = luaZ_sizebuffer(b) * 2; 56 luaZ_resizebuffer(ls->L, b, newsize); 57 } 58 b->buffer[luaZ_bufflen(b)++] = cast(char, c); 59 } 60 61 62 void luaX_init (lua_State *L) { 63 int i; 64 for (i=0; i<NUM_RESERVED; i++) { 65 TString *ts = luaS_new(L, luaX_tokens[i]); 66 luaS_fix(ts); /* reserved words are never collected */ 67 ts->tsv.extra = cast_byte(i+1); /* reserved word */ 68 } 69 } 70 71 72 const char *luaX_token2str (LexState *ls, int token) { 73 if (token < FIRST_RESERVED) { /* single-byte symbols? */ 74 lua_assert(token == cast(unsigned char, token)); 75 return (lisprint(token)) ? luaO_pushfstring(ls->L, LUA_QL("%c"), token) : 76 luaO_pushfstring(ls->L, "char(%d)", token); 77 } 78 else { 79 const char *s = luaX_tokens[token - FIRST_RESERVED]; 80 if (token < TK_EOS) /* fixed format (symbols and reserved words)? */ 81 return luaO_pushfstring(ls->L, LUA_QS, s); 82 else /* names, strings, and numerals */ 83 return s; 84 } 85 } 86 87 88 static const char *txtToken (LexState *ls, int token) { 89 switch (token) { 90 case TK_NAME: 91 case TK_STRING: 92 case TK_NUMBER: 93 save(ls, '\0'); 94 return luaO_pushfstring(ls->L, LUA_QS, luaZ_buffer(ls->buff)); 95 default: 96 return luaX_token2str(ls, token); 97 } 98 } 99 100 101 static l_noret lexerror (LexState *ls, const char *msg, int token) { 102 char buff[LUA_IDSIZE]; 103 luaO_chunkid(buff, getstr(ls->source), LUA_IDSIZE); 104 msg = luaO_pushfstring(ls->L, "%s:%d: %s", buff, ls->linenumber, msg); 105 if (token) 106 luaO_pushfstring(ls->L, "%s near %s", msg, txtToken(ls, token)); 107 luaD_throw(ls->L, LUA_ERRSYNTAX); 108 } 109 110 111 l_noret luaX_syntaxerror (LexState *ls, const char *msg) { 112 lexerror(ls, msg, ls->t.token); 113 } 114 115 116 /* 117 ** creates a new string and anchors it in function's table so that 118 ** it will not be collected until the end of the function's compilation 119 ** (by that time it should be anchored in function's prototype) 120 */ 121 TString *luaX_newstring (LexState *ls, const char *str, size_t l) { 122 lua_State *L = ls->L; 123 TValue *o; /* entry for `str' */ 124 TString *ts = luaS_newlstr(L, str, l); /* create new string */ 125 setsvalue2s(L, L->top++, ts); /* temporarily anchor it in stack */ 126 o = luaH_set(L, ls->fs->h, L->top - 1); 127 if (ttisnil(o)) { /* not in use yet? (see 'addK') */ 128 /* boolean value does not need GC barrier; 129 table has no metatable, so it does not need to invalidate cache */ 130 setbvalue(o, 1); /* t[string] = true */ 131 luaC_checkGC(L); 132 } 133 else { /* string already present */ 134 ts = rawtsvalue(keyfromval(o)); /* re-use value previously stored */ 135 } 136 L->top--; /* remove string from stack */ 137 return ts; 138 } 139 140 141 /* 142 ** increment line number and skips newline sequence (any of 143 ** \n, \r, \n\r, or \r\n) 144 */ 145 static void inclinenumber (LexState *ls) { 146 int old = ls->current; 147 lua_assert(currIsNewline(ls)); 148 next(ls); /* skip `\n' or `\r' */ 149 if (currIsNewline(ls) && ls->current != old) 150 next(ls); /* skip `\n\r' or `\r\n' */ 151 if (++ls->linenumber >= MAX_INT) 152 lexerror(ls, "chunk has too many lines", 0); 153 } 154 155 156 void luaX_setinput (lua_State *L, LexState *ls, ZIO *z, TString *source, 157 int firstchar) { 158 ls->decpoint = '.'; 159 ls->L = L; 160 ls->current = firstchar; 161 ls->lookahead.token = TK_EOS; /* no look-ahead token */ 162 ls->z = z; 163 ls->fs = NULL; 164 ls->linenumber = 1; 165 ls->lastline = 1; 166 ls->source = source; 167 ls->envn = luaS_new(L, LUA_ENV); /* create env name */ 168 luaS_fix(ls->envn); /* never collect this name */ 169 luaZ_resizebuffer(ls->L, ls->buff, LUA_MINBUFFER); /* initialize buffer */ 170 } 171 172 173 174 /* 175 ** ======================================================= 176 ** LEXICAL ANALYZER 177 ** ======================================================= 178 */ 179 180 181 182 static int check_next (LexState *ls, const char *set) { 183 if (ls->current == '\0' || !strchr(set, ls->current)) 184 return 0; 185 save_and_next(ls); 186 return 1; 187 } 188 189 190 /* 191 ** change all characters 'from' in buffer to 'to' 192 */ 193 static void buffreplace (LexState *ls, char from, char to) { 194 size_t n = luaZ_bufflen(ls->buff); 195 char *p = luaZ_buffer(ls->buff); 196 while (n--) 197 if (p[n] == from) p[n] = to; 198 } 199 200 201 #if !defined(getlocaledecpoint) 202 #define getlocaledecpoint() (localeconv()->decimal_point[0]) 203 #endif 204 205 206 #define buff2d(b,e) luaO_str2d(luaZ_buffer(b), luaZ_bufflen(b) - 1, e) 207 208 /* 209 ** in case of format error, try to change decimal point separator to 210 ** the one defined in the current locale and check again 211 */ 212 static void trydecpoint (LexState *ls, SemInfo *seminfo) { 213 char old = ls->decpoint; 214 ls->decpoint = getlocaledecpoint(); 215 buffreplace(ls, old, ls->decpoint); /* try new decimal separator */ 216 if (!buff2d(ls->buff, &seminfo->r)) { 217 /* format error with correct decimal point: no more options */ 218 buffreplace(ls, ls->decpoint, '.'); /* undo change (for error message) */ 219 lexerror(ls, "malformed number", TK_NUMBER); 220 } 221 } 222 223 224 /* LUA_NUMBER */ 225 /* 226 ** this function is quite liberal in what it accepts, as 'luaO_str2d' 227 ** will reject ill-formed numerals. 228 */ 229 static void read_numeral (LexState *ls, SemInfo *seminfo) { 230 const char *expo = "Ee"; 231 int first = ls->current; 232 lua_assert(lisdigit(ls->current)); 233 save_and_next(ls); 234 if (first == '0' && check_next(ls, "Xx")) /* hexadecimal? */ 235 expo = "Pp"; 236 for (;;) { 237 if (check_next(ls, expo)) /* exponent part? */ 238 (void) check_next(ls, "+-"); /* optional exponent sign */ 239 if (lisxdigit(ls->current) || ls->current == '.') 240 save_and_next(ls); 241 else break; 242 } 243 save(ls, '\0'); 244 buffreplace(ls, '.', ls->decpoint); /* follow locale for decimal point */ 245 if (!buff2d(ls->buff, &seminfo->r)) /* format error? */ 246 trydecpoint(ls, seminfo); /* try to update decimal point separator */ 247 } 248 249 250 /* 251 ** skip a sequence '[=*[' or ']=*]' and return its number of '='s or 252 ** -1 if sequence is malformed 253 */ 254 static int skip_sep (LexState *ls) { 255 int count = 0; 256 int s = ls->current; 257 lua_assert(s == '[' || s == ']'); 258 save_and_next(ls); 259 while (ls->current == '=') { 260 save_and_next(ls); 261 count++; 262 } 263 return (ls->current == s) ? count : (-count) - 1; 264 } 265 266 267 static void read_long_string (LexState *ls, SemInfo *seminfo, int sep) { 268 save_and_next(ls); /* skip 2nd `[' */ 269 if (currIsNewline(ls)) /* string starts with a newline? */ 270 inclinenumber(ls); /* skip it */ 271 for (;;) { 272 switch (ls->current) { 273 case EOZ: 274 lexerror(ls, (seminfo) ? "unfinished long string" : 275 "unfinished long comment", TK_EOS); 276 break; /* to avoid warnings */ 277 case ']': { 278 if (skip_sep(ls) == sep) { 279 save_and_next(ls); /* skip 2nd `]' */ 280 goto endloop; 281 } 282 break; 283 } 284 case '\n': case '\r': { 285 save(ls, '\n'); 286 inclinenumber(ls); 287 if (!seminfo) luaZ_resetbuffer(ls->buff); /* avoid wasting space */ 288 break; 289 } 290 default: { 291 if (seminfo) save_and_next(ls); 292 else next(ls); 293 } 294 } 295 } endloop: 296 if (seminfo) 297 seminfo->ts = luaX_newstring(ls, luaZ_buffer(ls->buff) + (2 + sep), 298 luaZ_bufflen(ls->buff) - 2*(2 + sep)); 299 } 300 301 302 static void escerror (LexState *ls, int *c, int n, const char *msg) { 303 int i; 304 luaZ_resetbuffer(ls->buff); /* prepare error message */ 305 save(ls, '\\'); 306 for (i = 0; i < n && c[i] != EOZ; i++) 307 save(ls, c[i]); 308 lexerror(ls, msg, TK_STRING); 309 } 310 311 312 static int readhexaesc (LexState *ls) { 313 int c[3], i; /* keep input for error message */ 314 int r = 0; /* result accumulator */ 315 c[0] = 'x'; /* for error message */ 316 for (i = 1; i < 3; i++) { /* read two hexadecimal digits */ 317 c[i] = next(ls); 318 if (!lisxdigit(c[i])) 319 escerror(ls, c, i + 1, "hexadecimal digit expected"); 320 r = (r << 4) + luaO_hexavalue(c[i]); 321 } 322 return r; 323 } 324 325 326 static int readdecesc (LexState *ls) { 327 int c[3], i; 328 int r = 0; /* result accumulator */ 329 for (i = 0; i < 3 && lisdigit(ls->current); i++) { /* read up to 3 digits */ 330 c[i] = ls->current; 331 r = 10*r + c[i] - '0'; 332 next(ls); 333 } 334 if (r > UCHAR_MAX) 335 escerror(ls, c, i, "decimal escape too large"); 336 return r; 337 } 338 339 340 static void read_string (LexState *ls, int del, SemInfo *seminfo) { 341 save_and_next(ls); /* keep delimiter (for error messages) */ 342 while (ls->current != del) { 343 switch (ls->current) { 344 case EOZ: 345 lexerror(ls, "unfinished string", TK_EOS); 346 break; /* to avoid warnings */ 347 case '\n': 348 case '\r': 349 lexerror(ls, "unfinished string", TK_STRING); 350 break; /* to avoid warnings */ 351 case '\\': { /* escape sequences */ 352 int c; /* final character to be saved */ 353 next(ls); /* do not save the `\' */ 354 switch (ls->current) { 355 case 'a': c = '\a'; goto read_save; 356 case 'b': c = '\b'; goto read_save; 357 case 'f': c = '\f'; goto read_save; 358 case 'n': c = '\n'; goto read_save; 359 case 'r': c = '\r'; goto read_save; 360 case 't': c = '\t'; goto read_save; 361 case 'v': c = '\v'; goto read_save; 362 case 'x': c = readhexaesc(ls); goto read_save; 363 case '\n': case '\r': 364 inclinenumber(ls); c = '\n'; goto only_save; 365 case '\\': case '\"': case '\'': 366 c = ls->current; goto read_save; 367 case EOZ: goto no_save; /* will raise an error next loop */ 368 case 'z': { /* zap following span of spaces */ 369 next(ls); /* skip the 'z' */ 370 while (lisspace(ls->current)) { 371 if (currIsNewline(ls)) inclinenumber(ls); 372 else next(ls); 373 } 374 goto no_save; 375 } 376 default: { 377 if (!lisdigit(ls->current)) 378 escerror(ls, &ls->current, 1, "invalid escape sequence"); 379 /* digital escape \ddd */ 380 c = readdecesc(ls); 381 goto only_save; 382 } 383 } 384 read_save: next(ls); /* read next character */ 385 only_save: save(ls, c); /* save 'c' */ 386 no_save: break; 387 } 388 default: 389 save_and_next(ls); 390 } 391 } 392 save_and_next(ls); /* skip delimiter */ 393 seminfo->ts = luaX_newstring(ls, luaZ_buffer(ls->buff) + 1, 394 luaZ_bufflen(ls->buff) - 2); 395 } 396 397 398 static int llex (LexState *ls, SemInfo *seminfo) { 399 luaZ_resetbuffer(ls->buff); 400 for (;;) { 401 switch (ls->current) { 402 case '\n': case '\r': { /* line breaks */ 403 inclinenumber(ls); 404 break; 405 } 406 case ' ': case '\f': case '\t': case '\v': { /* spaces */ 407 next(ls); 408 break; 409 } 410 case '-': { /* '-' or '--' (comment) */ 411 next(ls); 412 if (ls->current != '-') return '-'; 413 /* else is a comment */ 414 next(ls); 415 if (ls->current == '[') { /* long comment? */ 416 int sep = skip_sep(ls); 417 luaZ_resetbuffer(ls->buff); /* `skip_sep' may dirty the buffer */ 418 if (sep >= 0) { 419 read_long_string(ls, NULL, sep); /* skip long comment */ 420 luaZ_resetbuffer(ls->buff); /* previous call may dirty the buff. */ 421 break; 422 } 423 } 424 /* else short comment */ 425 while (!currIsNewline(ls) && ls->current != EOZ) 426 next(ls); /* skip until end of line (or end of file) */ 427 break; 428 } 429 case '[': { /* long string or simply '[' */ 430 int sep = skip_sep(ls); 431 if (sep >= 0) { 432 read_long_string(ls, seminfo, sep); 433 return TK_STRING; 434 } else if (sep == -1) { 435 return '['; 436 } else { 437 lexerror(ls, "invalid long string delimiter", TK_STRING); 438 break; 439 } 440 } 441 case '=': { 442 next(ls); 443 if (ls->current != '=') return '='; 444 else { next(ls); return TK_EQ; } 445 } 446 case '<': { 447 next(ls); 448 if (ls->current != '=') return '<'; 449 else { next(ls); return TK_LE; } 450 } 451 case '>': { 452 next(ls); 453 if (ls->current != '=') return '>'; 454 else { next(ls); return TK_GE; } 455 } 456 case '~': { 457 next(ls); 458 if (ls->current != '=') return '~'; 459 else { next(ls); return TK_NE; } 460 } 461 case ':': { 462 next(ls); 463 if (ls->current != ':') return ':'; 464 else { next(ls); return TK_DBCOLON; } 465 } 466 case '"': case '\'': { /* short literal strings */ 467 read_string(ls, ls->current, seminfo); 468 return TK_STRING; 469 } 470 case '.': { /* '.', '..', '...', or number */ 471 save_and_next(ls); 472 if (check_next(ls, ".")) { 473 if (check_next(ls, ".")) 474 return TK_DOTS; /* '...' */ 475 else return TK_CONCAT; /* '..' */ 476 } 477 else if (!lisdigit(ls->current)) return '.'; 478 /* else go through */ 479 } 480 /* FALLTHROUGH */ 481 case '0': case '1': case '2': case '3': case '4': 482 case '5': case '6': case '7': case '8': case '9': { 483 read_numeral(ls, seminfo); 484 return TK_NUMBER; 485 } 486 case EOZ: { 487 return TK_EOS; 488 } 489 default: { 490 if (lislalpha(ls->current)) { /* identifier or reserved word? */ 491 TString *ts; 492 do { 493 save_and_next(ls); 494 } while (lislalnum(ls->current)); 495 ts = luaX_newstring(ls, luaZ_buffer(ls->buff), 496 luaZ_bufflen(ls->buff)); 497 seminfo->ts = ts; 498 if (isreserved(ts)) /* reserved word? */ 499 return ts->tsv.extra - 1 + FIRST_RESERVED; 500 else { 501 return TK_NAME; 502 } 503 } 504 else { /* single-char tokens (+ - / ...) */ 505 int c = ls->current; 506 next(ls); 507 return c; 508 } 509 } 510 } 511 } 512 } 513 514 515 void luaX_next (LexState *ls) { 516 ls->lastline = ls->linenumber; 517 if (ls->lookahead.token != TK_EOS) { /* is there a look-ahead token? */ 518 ls->t = ls->lookahead; /* use this one */ 519 ls->lookahead.token = TK_EOS; /* and discharge it */ 520 } 521 else 522 ls->t.token = llex(ls, &ls->t.seminfo); /* read next token */ 523 } 524 525 526 int luaX_lookahead (LexState *ls) { 527 lua_assert(ls->lookahead.token == TK_EOS); 528 ls->lookahead.token = llex(ls, &ls->lookahead.seminfo); 529 return ls->lookahead.token; 530 } 531 /* END CSTYLED */ 532