1 /*
2 ** $Id: lutf8lib.c $
3 ** Standard library for UTF-8 manipulation
4 ** See Copyright Notice in lua.h
5 */
6
7 #define lutf8lib_c
8 #define LUA_LIB
9
10 #include "lprefix.h"
11
12
13 #include <assert.h>
14 #include <limits.h>
15 #include <stdlib.h>
16 #include <string.h>
17
18 #include "lua.h"
19
20 #include "lauxlib.h"
21 #include "lualib.h"
22
23
24 #define MAXUNICODE 0x10FFFFu
25
26 #define MAXUTF 0x7FFFFFFFu
27
28
29 #define MSGInvalid "invalid UTF-8 code"
30
31 /*
32 ** Integer type for decoded UTF-8 values; MAXUTF needs 31 bits.
33 */
34 #if (UINT_MAX >> 30) >= 1
35 typedef unsigned int utfint;
36 #else
37 typedef unsigned long utfint;
38 #endif
39
40
41 #define iscont(c) (((c) & 0xC0) == 0x80)
42 #define iscontp(p) iscont(*(p))
43
44
45 /* from strlib */
46 /* translate a relative string position: negative means back from end */
u_posrelat(lua_Integer pos,size_t len)47 static lua_Integer u_posrelat (lua_Integer pos, size_t len) {
48 if (pos >= 0) return pos;
49 else if (0u - (size_t)pos > len) return 0;
50 else return (lua_Integer)len + pos + 1;
51 }
52
53
54 /*
55 ** Decode one UTF-8 sequence, returning NULL if byte sequence is
56 ** invalid. The array 'limits' stores the minimum value for each
57 ** sequence length, to check for overlong representations. Its first
58 ** entry forces an error for non-ascii bytes with no continuation
59 ** bytes (count == 0).
60 */
utf8_decode(const char * s,utfint * val,int strict)61 static const char *utf8_decode (const char *s, utfint *val, int strict) {
62 static const utfint limits[] =
63 {~(utfint)0, 0x80, 0x800, 0x10000u, 0x200000u, 0x4000000u};
64 unsigned int c = (unsigned char)s[0];
65 utfint res = 0; /* final result */
66 if (c < 0x80) /* ascii? */
67 res = c;
68 else {
69 int count = 0; /* to count number of continuation bytes */
70 for (; c & 0x40; c <<= 1) { /* while it needs continuation bytes... */
71 unsigned int cc = (unsigned char)s[++count]; /* read next byte */
72 if (!iscont(cc)) /* not a continuation byte? */
73 return NULL; /* invalid byte sequence */
74 res = (res << 6) | (cc & 0x3F); /* add lower 6 bits from cont. byte */
75 }
76 res |= ((utfint)(c & 0x7F) << (count * 5)); /* add first byte */
77 if (count > 5 || res > MAXUTF || res < limits[count])
78 return NULL; /* invalid byte sequence */
79 s += count; /* skip continuation bytes read */
80 }
81 if (strict) {
82 /* check for invalid code points; too large or surrogates */
83 if (res > MAXUNICODE || (0xD800u <= res && res <= 0xDFFFu))
84 return NULL;
85 }
86 if (val) *val = res;
87 return s + 1; /* +1 to include first byte */
88 }
89
90
91 /*
92 ** utf8len(s [, i [, j [, lax]]]) --> number of characters that
93 ** start in the range [i,j], or nil + current position if 's' is not
94 ** well formed in that interval
95 */
utflen(lua_State * L)96 static int utflen (lua_State *L) {
97 lua_Integer n = 0; /* counter for the number of characters */
98 size_t len; /* string length in bytes */
99 const char *s = luaL_checklstring(L, 1, &len);
100 lua_Integer posi = u_posrelat(luaL_optinteger(L, 2, 1), len);
101 lua_Integer posj = u_posrelat(luaL_optinteger(L, 3, -1), len);
102 int lax = lua_toboolean(L, 4);
103 luaL_argcheck(L, 1 <= posi && --posi <= (lua_Integer)len, 2,
104 "initial position out of bounds");
105 luaL_argcheck(L, --posj < (lua_Integer)len, 3,
106 "final position out of bounds");
107 while (posi <= posj) {
108 const char *s1 = utf8_decode(s + posi, NULL, !lax);
109 if (s1 == NULL) { /* conversion error? */
110 luaL_pushfail(L); /* return fail ... */
111 lua_pushinteger(L, posi + 1); /* ... and current position */
112 return 2;
113 }
114 posi = s1 - s;
115 n++;
116 }
117 lua_pushinteger(L, n);
118 return 1;
119 }
120
121
122 /*
123 ** codepoint(s, [i, [j [, lax]]]) -> returns codepoints for all
124 ** characters that start in the range [i,j]
125 */
codepoint(lua_State * L)126 static int codepoint (lua_State *L) {
127 size_t len;
128 const char *s = luaL_checklstring(L, 1, &len);
129 lua_Integer posi = u_posrelat(luaL_optinteger(L, 2, 1), len);
130 lua_Integer pose = u_posrelat(luaL_optinteger(L, 3, posi), len);
131 int lax = lua_toboolean(L, 4);
132 int n;
133 const char *se;
134 luaL_argcheck(L, posi >= 1, 2, "out of bounds");
135 luaL_argcheck(L, pose <= (lua_Integer)len, 3, "out of bounds");
136 if (posi > pose) return 0; /* empty interval; return no values */
137 if (pose - posi >= INT_MAX) /* (lua_Integer -> int) overflow? */
138 return luaL_error(L, "string slice too long");
139 n = (int)(pose - posi) + 1; /* upper bound for number of returns */
140 luaL_checkstack(L, n, "string slice too long");
141 n = 0; /* count the number of returns */
142 se = s + pose; /* string end */
143 for (s += posi - 1; s < se;) {
144 utfint code;
145 s = utf8_decode(s, &code, !lax);
146 if (s == NULL)
147 return luaL_error(L, MSGInvalid);
148 lua_pushinteger(L, code);
149 n++;
150 }
151 return n;
152 }
153
154
pushutfchar(lua_State * L,int arg)155 static void pushutfchar (lua_State *L, int arg) {
156 lua_Unsigned code = (lua_Unsigned)luaL_checkinteger(L, arg);
157 luaL_argcheck(L, code <= MAXUTF, arg, "value out of range");
158 lua_pushfstring(L, "%U", (long)code);
159 }
160
161
162 /*
163 ** utfchar(n1, n2, ...) -> char(n1)..char(n2)...
164 */
utfchar(lua_State * L)165 static int utfchar (lua_State *L) {
166 int n = lua_gettop(L); /* number of arguments */
167 if (n == 1) /* optimize common case of single char */
168 pushutfchar(L, 1);
169 else {
170 int i;
171 luaL_Buffer b;
172 luaL_buffinit(L, &b);
173 for (i = 1; i <= n; i++) {
174 pushutfchar(L, i);
175 luaL_addvalue(&b);
176 }
177 luaL_pushresult(&b);
178 }
179 return 1;
180 }
181
182
183 /*
184 ** offset(s, n, [i]) -> index where n-th character counting from
185 ** position 'i' starts; 0 means character at 'i'.
186 */
byteoffset(lua_State * L)187 static int byteoffset (lua_State *L) {
188 size_t len;
189 const char *s = luaL_checklstring(L, 1, &len);
190 lua_Integer n = luaL_checkinteger(L, 2);
191 lua_Integer posi = (n >= 0) ? 1 : len + 1;
192 posi = u_posrelat(luaL_optinteger(L, 3, posi), len);
193 luaL_argcheck(L, 1 <= posi && --posi <= (lua_Integer)len, 3,
194 "position out of bounds");
195 if (n == 0) {
196 /* find beginning of current byte sequence */
197 while (posi > 0 && iscontp(s + posi)) posi--;
198 }
199 else {
200 if (iscontp(s + posi))
201 return luaL_error(L, "initial position is a continuation byte");
202 if (n < 0) {
203 while (n < 0 && posi > 0) { /* move back */
204 do { /* find beginning of previous character */
205 posi--;
206 } while (posi > 0 && iscontp(s + posi));
207 n++;
208 }
209 }
210 else {
211 n--; /* do not move for 1st character */
212 while (n > 0 && posi < (lua_Integer)len) {
213 do { /* find beginning of next character */
214 posi++;
215 } while (iscontp(s + posi)); /* (cannot pass final '\0') */
216 n--;
217 }
218 }
219 }
220 if (n == 0) /* did it find given character? */
221 lua_pushinteger(L, posi + 1);
222 else /* no such character */
223 luaL_pushfail(L);
224 return 1;
225 }
226
227
iter_aux(lua_State * L,int strict)228 static int iter_aux (lua_State *L, int strict) {
229 size_t len;
230 const char *s = luaL_checklstring(L, 1, &len);
231 lua_Unsigned n = (lua_Unsigned)lua_tointeger(L, 2);
232 if (n < len) {
233 while (iscontp(s + n)) n++; /* go to next character */
234 }
235 if (n >= len) /* (also handles original 'n' being negative) */
236 return 0; /* no more codepoints */
237 else {
238 utfint code;
239 const char *next = utf8_decode(s + n, &code, strict);
240 if (next == NULL || iscontp(next))
241 return luaL_error(L, MSGInvalid);
242 lua_pushinteger(L, n + 1);
243 lua_pushinteger(L, code);
244 return 2;
245 }
246 }
247
248
iter_auxstrict(lua_State * L)249 static int iter_auxstrict (lua_State *L) {
250 return iter_aux(L, 1);
251 }
252
iter_auxlax(lua_State * L)253 static int iter_auxlax (lua_State *L) {
254 return iter_aux(L, 0);
255 }
256
257
iter_codes(lua_State * L)258 static int iter_codes (lua_State *L) {
259 int lax = lua_toboolean(L, 2);
260 const char *s = luaL_checkstring(L, 1);
261 luaL_argcheck(L, !iscontp(s), 1, MSGInvalid);
262 lua_pushcfunction(L, lax ? iter_auxlax : iter_auxstrict);
263 lua_pushvalue(L, 1);
264 lua_pushinteger(L, 0);
265 return 3;
266 }
267
268
269 /* pattern to match a single UTF-8 character */
270 #define UTF8PATT "[\0-\x7F\xC2-\xFD][\x80-\xBF]*"
271
272
273 static const luaL_Reg funcs[] = {
274 {"offset", byteoffset},
275 {"codepoint", codepoint},
276 {"char", utfchar},
277 {"len", utflen},
278 {"codes", iter_codes},
279 /* placeholders */
280 {"charpattern", NULL},
281 {NULL, NULL}
282 };
283
284
luaopen_utf8(lua_State * L)285 LUAMOD_API int luaopen_utf8 (lua_State *L) {
286 luaL_newlib(L, funcs);
287 lua_pushlstring(L, UTF8PATT, sizeof(UTF8PATT)/sizeof(char) - 1);
288 lua_setfield(L, -2, "charpattern");
289 return 1;
290 }
291
292