3 Copyright 2006-2012 Taco Hoekwater <taco@luatex.org>
5 This file is part of LuaTeX.
7 LuaTeX is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 2 of the License, or (at your
10 option) any later version.
12 LuaTeX is distributed in the hope that it will be useful, but WITHOUT
13 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
15 License for more details.
17 You should have received a copy of the GNU General Public License along
18 with LuaTeX; if not, see <http://www.gnu.org/licenses/>. */
22 This module is unfinished and an intermediate step to removal of the old
23 token lib. Between version 0.80 and 0.85 the transition will be complete
24 and at the same time the input (buffer) handling will be cleaned up and
25 simplified. At that moment we can feed back tokens into the input.
27 The code can be optimized a bit by faster key checking. The scan functions
28 implemented here will stay functionally the same but can be improved if
29 needed. The old triplet model will disappear.
34 #include "lua/luatex-api.h"
36 typedef struct lua_token
{
41 typedef struct saved_tex_scanner
{
44 int save_cmd
, save_chr
, save_cs
, save_tok
;
47 #define save_tex_scanner(a) do { \
48 a.save_cmd = cur_cmd; \
49 a.save_chr = cur_chr; \
51 a.save_tok = cur_tok; \
54 #define unsave_tex_scanner(a) do { \
55 cur_cmd = a.save_cmd; \
56 cur_chr = a.save_chr; \
58 cur_tok = a.save_tok; \
61 #define TEX_ORIGIN 0 /* not used yet */
64 static lua_token
*check_istoken(lua_State
* L
, int ud
);
66 #define TOKEN_METATABLE "luatex_token"
69 #define DEBUG_OUT stdout
71 #define DEFAULT_SCAN_CODE_SET 2048 + 4096 /* default: letter and other */
73 /* two core helpers */
75 #define is_active_string(s) (strlen((char *)s)>3 && *s==0xEF && *(s+1)==0xBF && *(s+2)==0xBF)
77 void make_token_table(lua_State
* L
, int cmd
, int chr
, int cs
)
79 lua_createtable(L
, 3, 0);
80 lua_pushinteger(L
, cmd
);
81 lua_rawseti(L
, -2, 1);
82 lua_pushinteger(L
, chr
);
83 lua_rawseti(L
, -2, 2);
84 lua_pushinteger(L
, cs
);
85 lua_rawseti(L
, -2, 3);
88 static unsigned char *get_cs_text(int cs
)
91 return (unsigned char *) xstrdup("\\csname\\endcsname");
92 else if ((cs_text(cs
) < 0) || (cs_text(cs
) >= str_ptr
))
93 return (unsigned char *) xstrdup("");
95 return (unsigned char *) makecstring(cs_text(cs
));
98 /* maybe this qualify as a macro, not function */
100 static lua_token
*maybe_istoken(lua_State
* L
, int ud
)
102 lua_token
*p
= lua_touserdata(L
, ud
);
104 if (lua_getmetatable(L
, ud
)) {
105 lua_rawgeti(L
, LUA_REGISTRYINDEX
, luaS_index(luatex_token
));
106 lua_gettable(L
, LUA_REGISTRYINDEX
);
107 if (!lua_rawequal(L
, -1, -2))
115 /* we could make the message a function and just inline the rest (via a macro) */
117 lua_token
*check_istoken(lua_State
* L
, int ud
)
119 lua_token
*p
= maybe_istoken(L
, ud
);
122 formatted_error("token lib","lua <token> expected, not an object with type %s", luaL_typename(L
, ud
));
126 /* token library functions */
128 static void make_new_token(lua_State
* L
, int cmd
, int chr
, int cs
)
131 lua_token
*thetok
= lua_newuserdata(L
, sizeof(lua_token
));
132 thetok
->origin
= LUA_ORIGIN
;
133 fast_get_avail(thetok
->token
);
134 tok
= (cs
? cs_token_flag
+ cs
: token_val(cmd
, chr
));
135 set_token_info(thetok
->token
, tok
);
136 lua_rawgeti(L
, LUA_REGISTRYINDEX
, luaS_index(luatex_token
));
137 lua_gettable(L
, LUA_REGISTRYINDEX
);
138 lua_setmetatable(L
, -2);
141 static void push_token(lua_State
* L
, int tok
)
143 lua_token
*thetok
= lua_newuserdata(L
, sizeof(lua_token
));
144 thetok
->origin
= LUA_ORIGIN
;
146 lua_rawgeti(L
, LUA_REGISTRYINDEX
, luaS_index(luatex_token
));
147 lua_gettable(L
, LUA_REGISTRYINDEX
);
148 lua_setmetatable(L
, -2);
151 /* static int run_get_cs_offset(lua_State * L) */
153 /* lua_pushinteger(L, cs_token_flag); */
157 /* static int run_get_command_id(lua_State * L) */
160 /* if (lua_type(L, -1) == LUA_TSTRING) { */
161 /* cs = get_command_id(lua_tostring(L, -1)); */
163 /* lua_pushinteger(L, cs); */
167 /* static int run_get_csname_id(lua_State * L) */
170 /* size_t k, cs = 0; */
171 /* if (lua_type(L, -1) == LUA_TSTRING) { */
172 /* s = lua_tolstring(L, -1, &k); */
173 /* cs = (size_t) string_lookup(s, k); */
175 /* lua_pushinteger(L, (lua_Number) cs); */
179 static int run_get_next(lua_State
* L
)
181 saved_tex_scanner texstate
;
182 save_tex_scanner(texstate
);
184 make_new_token(L
, cur_cmd
, cur_chr
, cur_cs
);
185 unsave_tex_scanner(texstate
);
190 This is experimental code:
192 local t1 = token.get_next()
193 local t2 = token.get_next()
194 local t3 = token.get_next()
195 local t4 = token.get_next()
196 -- watch out, we flush in sequence
197 token.put_next { t1, t2 }
198 -- but this one gets pushed in front
199 token.put_next ( t3, t4 )
200 -- so when we get wxyz we put yzwx!
202 At some point we can consider a token.print that delays and goes via
203 the same rope mechanism as texio.prints and friends but then one can
204 as well serialize the tokens and do a normal print so there is no real
205 gain in it. After all, the tokenlib operates at the input level so we
206 might as well keep it there.
210 inline static int run_put_next(lua_State
* L
)
212 int n
= lua_gettop(L
);
220 /* we accept a single nil argument */
223 lua_rawgeti(L
, LUA_REGISTRYINDEX
, luaS_index(luatex_token
)); /* n+1 */
224 lua_gettable(L
, LUA_REGISTRYINDEX
); /* n+1 */
226 if (lua_type(L
,1) == LUA_TTABLE
) {
228 normal_error("token lib","only one table permitted in put_next");
231 lua_rawgeti(L
, 1, i
); /* table mt token */
232 if (lua_type(L
,-1) == LUA_TNIL
) {
235 p
= lua_touserdata(L
, -1);
237 normal_error("token lib","lua <token> expected in put_next (1)");
238 } else if (!lua_getmetatable(L
, -1)) { /* table mt token mt */
239 normal_error("token lib","lua <token> expected in put_next (2)");
240 } else if (!lua_rawequal(L
, m
, -1)) {
241 normal_error("token lib","lua <token> expected in put_next (3)");
244 token_info(x
) = token_info(p
->token
);
257 for (i
= 1; i
<= n
; i
++) {
258 p
= lua_touserdata(L
,i
);
260 normal_error("token lib","lua <token> expected in put_next (4)");
261 } else if (!lua_getmetatable(L
, i
)) { /* table mt token mt */
262 normal_error("token lib","lua <token> expected in put_next (5)");
263 } else if (!lua_rawequal(L
, m
, -1)) {
264 normal_error("token lib","lua <token> expected in put_next (6)");
267 token_info(x
) = token_info(p
->token
);
281 begin_token_list(h
,0);
287 static int run_scan_keyword(lua_State
* L
)
289 saved_tex_scanner texstate
;
290 const char *s
= luaL_checkstring(L
, -1);
293 save_tex_scanner(texstate
);
294 if (scan_keyword(s
)) {
297 unsave_tex_scanner(texstate
);
299 lua_pushboolean(L
,v
);
303 static int run_scan_csname(lua_State
* L
)
307 saved_tex_scanner texstate
;
308 save_tex_scanner(texstate
);
310 t
= (cur_cs
? cs_token_flag
+ cur_cs
: token_val(cur_cmd
, cur_chr
));
311 if (t
>= cs_token_flag
&& ((s
= get_cs_text(t
- cs_token_flag
)) != (unsigned char *) NULL
)) {
312 if (is_active_string(s
))
313 lua_pushstring(L
, (char *) (s
+ 3));
315 lua_pushstring(L
, (char *) s
);
319 unsave_tex_scanner(texstate
);
323 static int run_scan_int(lua_State
* L
)
325 saved_tex_scanner texstate
;
327 save_tex_scanner(texstate
);
330 unsave_tex_scanner(texstate
);
331 lua_pushinteger(L
,(int)v
);
335 static int run_scan_dimen(lua_State
* L
)
337 saved_tex_scanner texstate
;
339 int inf
= false, mu
= false;
340 int t
= lua_gettop(L
);
342 inf
= lua_toboolean(L
,1); /* inf values allowed ?*/
344 mu
= lua_toboolean(L
,2); /* mu units required ?*/
345 save_tex_scanner(texstate
);
346 scan_dimen( mu
,inf
, false); /* arg3 = shortcut */
349 unsave_tex_scanner(texstate
);
350 lua_pushinteger(L
,v
);
352 lua_pushinteger(L
,(lua_Number
)o
);
359 static int run_scan_glue(lua_State
* L
)
361 saved_tex_scanner texstate
;
364 int t
= lua_gettop(L
);
366 mu
= lua_toboolean(L
,1); /* mu units required ?*/
367 save_tex_scanner(texstate
);
368 scan_glue((mu
? mu_val_level
: glue_val_level
));
369 v
= cur_val
; /* which is a glue_spec node */
370 unsave_tex_scanner(texstate
);
371 lua_nodelib_push_fast(L
,(halfword
)v
);
375 static int run_scan_toks(lua_State
* L
)
377 saved_tex_scanner texstate
;
378 int macro_def
= false, xpand
= false;
379 halfword t
, saved_defref
;
381 int top
= lua_gettop(L
);
383 macro_def
= lua_toboolean(L
,1); /* \\def ? */
385 xpand
= lua_toboolean(L
,2); /* expand ? */
386 save_tex_scanner(texstate
);
387 saved_defref
= def_ref
;
388 (void) scan_toks(macro_def
, xpand
);
390 unsave_tex_scanner(texstate
);
391 def_ref
= saved_defref
;
392 /* This function returns a pointer to the tail of a new token
393 list, and it also makes |def_ref| point to the reference count at the
394 head of that list. */
396 while (token_link(t
)) {
399 lua_rawseti(L
,-2,i
++);
404 static int run_scan_string(lua_State
* L
) /* HH */
405 { /* can be simplified, no need for intermediate list */
406 saved_tex_scanner texstate
;
407 halfword t
, saved_defref
;
408 save_tex_scanner(texstate
);
411 } while ((cur_cmd
== spacer_cmd
) || (cur_cmd
== relax_cmd
));
412 if (cur_cmd
== left_brace_cmd
) {
414 saved_defref
= def_ref
;
415 (void) scan_toks(false, true);
417 def_ref
= saved_defref
;
418 tokenlist_to_luastring(L
,t
);
419 } else if (cur_cmd
== call_cmd
) {
420 t
= token_link(cur_chr
);
421 tokenlist_to_luastring(L
,t
);
423 if (cur_cmd
== 11 || cur_cmd
== 12 ) {
426 luaL_buffinit(L
,&b
) ;
428 str
= (char *) uni2str(cur_chr
);
429 luaL_addstring(&b
,(char *) str
);
431 if (cur_cmd
!= 11 && cur_cmd
!= 12 ) {
442 unsave_tex_scanner(texstate
);
446 static int run_scan_word(lua_State
* L
) /* HH */
448 saved_tex_scanner texstate
;
449 save_tex_scanner(texstate
);
452 } while ((cur_cmd
== spacer_cmd
) || (cur_cmd
== relax_cmd
));
453 if (cur_cmd
== 11 || cur_cmd
== 12 ) {
456 luaL_buffinit(L
,&b
) ;
458 str
= (char *) uni2str(cur_chr
);
459 luaL_addstring(&b
,str
);
462 if (cur_cmd
!= 11 && cur_cmd
!= 12 ) {
472 unsave_tex_scanner(texstate
);
476 static int run_scan_code(lua_State
* L
) /* HH */
478 saved_tex_scanner texstate
;
479 int cc
= DEFAULT_SCAN_CODE_SET
;
480 save_tex_scanner(texstate
);
483 if (lua_gettop(L
)>0) {
484 cc
= (int) lua_tointeger(L
,-1);
486 /* todo: message that we choose a default */
487 cc
= DEFAULT_SCAN_CODE_SET
;
490 if (cc
& (1<<(cur_cmd
))) {
491 lua_pushinteger(L
,(int)cur_chr
);
500 unsave_tex_scanner(texstate
);
504 static int lua_tokenlib_is_token(lua_State
* L
) /* HH */
506 lua_pushboolean(L
,maybe_istoken(L
,1)==NULL
? 0 : 1);
510 /* static int run_expand(lua_State * L) */
517 static int run_lookup(lua_State
* L
)
522 if (lua_type(L
, -1) == LUA_TSTRING
) {
523 s
= lua_tolstring(L
, -1, &l
);
525 cs
= string_lookup(s
, l
);
528 make_new_token(L
, cmd
, chr
, cs
);
536 static int run_build(lua_State
* L
)
538 if (lua_type(L
, 1) == LUA_TNUMBER
) {
540 int chr
= (int) lua_tointeger(L
, 1);
541 int cmd
= (int) luaL_optinteger(L
, 2, get_cat_code(int_par(cat_code_table_code
),chr
));
542 if (cmd
== 0 || cmd
== 9 || cmd
== 14 || cmd
== 15) {
544 "\n\nluatex error: not a good token.\nCatcode %i can not be returned, so I replaced it by 12 (other)",
548 } else if (cmd
== 13) {
549 cs
= active_to_cs(chr
, false);
553 make_new_token(L
, cmd
, chr
, cs
);
556 return run_lookup(L
);
560 /* token instance functions */
562 static int lua_tokenlib_free(lua_State
* L
)
565 n
= check_istoken(L
, 1);
566 if (n
->origin
== LUA_ORIGIN
) {
567 free_avail(n
->token
);
574 inline static int lua_tokenlib_get_command(lua_State
* L
)
576 lua_token
*n
= check_istoken(L
, 1);
577 halfword t
= token_info(n
->token
);
578 if (t
>= cs_token_flag
) {
579 lua_pushinteger(L
,(int) eq_type((t
- cs_token_flag
)));
581 lua_pushinteger(L
, token_cmd(t
));
586 inline static int lua_tokenlib_get_index(lua_State
* L
)
588 lua_token
*n
= check_istoken(L
, 1);
589 halfword t
= token_info(n
->token
);
590 int cmd
= (t
>= cs_token_flag
? eq_type(t
- cs_token_flag
) : token_cmd(t
));
591 halfword e
= equiv(t
- cs_token_flag
);
596 case assign_attr_cmd
:
599 case assign_dimen_cmd
:
602 case assign_glue_cmd
:
605 case assign_mu_glue_cmd
:
608 case assign_toks_cmd
:
615 if ((e
>= 0) && (e
<= 65535)) {
616 lua_pushinteger(L
, e
);
623 inline static int lua_tokenlib_get_mode(lua_State
* L
)
625 lua_token
*n
= check_istoken(L
, 1);
626 halfword t
= token_info(n
->token
);
627 if (t
>= cs_token_flag
) {
628 lua_pushinteger(L
, equiv(t
- cs_token_flag
));
630 lua_pushinteger(L
, token_chr(t
));
635 inline static int lua_tokenlib_get_cmdname(lua_State
* L
)
637 lua_token
*n
= check_istoken(L
, 1);
638 halfword t
= token_info(n
->token
);
639 int cmd
= (t
>= cs_token_flag
? eq_type(t
- cs_token_flag
) : token_cmd(t
));
640 lua_pushstring(L
, command_names
[cmd
].cmd_name
); /* can be sped up */
644 inline static int lua_tokenlib_get_csname(lua_State
* L
)
646 lua_token
*n
= check_istoken(L
, 1);
647 halfword t
= token_info(n
->token
);
649 if (t
>= cs_token_flag
&& ((s
= get_cs_text(t
- cs_token_flag
)) != (unsigned char *) NULL
)) {
650 if (is_active_string(s
))
651 lua_pushstring(L
, (char *) (s
+ 3));
653 lua_pushstring(L
, (char *) s
);
660 inline static int lua_tokenlib_get_id(lua_State
* L
)
662 lua_token
*n
= check_istoken(L
, 1);
663 lua_pushinteger(L
, n
->token
);
667 inline static int lua_tokenlib_get_tok(lua_State
* L
)
669 lua_token
*n
= check_istoken(L
, 1);
670 halfword t
= token_info(n
->token
);
671 lua_pushinteger(L
, t
);
675 inline static int lua_tokenlib_get_active(lua_State
* L
)
677 lua_token
*n
= check_istoken(L
, 1);
678 halfword t
= token_info(n
->token
);
680 if (t
>= cs_token_flag
&& ((s
= get_cs_text(t
- cs_token_flag
)) != (unsigned char *) NULL
)) {
681 if (is_active_string(s
))
682 lua_pushboolean(L
,1);
684 lua_pushboolean(L
,0);
687 lua_pushboolean(L
,0);
692 inline static int lua_tokenlib_get_expandable(lua_State
* L
)
694 lua_token
*n
= check_istoken(L
, 1);
695 halfword t
= token_info(n
->token
);
696 int cmd
= (t
>= cs_token_flag
? eq_type(t
- cs_token_flag
) : token_cmd(t
));
697 if (cmd
> max_command_cmd
) {
698 lua_pushboolean(L
, 1);
700 lua_pushboolean(L
, 0);
705 inline static int lua_tokenlib_get_protected(lua_State
* L
)
707 lua_token
*n
= check_istoken(L
, 1);
708 halfword t
= token_info(n
->token
);
709 int cmd
= (t
>= cs_token_flag
? eq_type(t
- cs_token_flag
) : token_cmd(t
));
710 if (cmd
> max_command_cmd
&& (cmd
>= call_cmd
) && (cmd
< end_template_cmd
)) {
711 int chr
= (t
>= cs_token_flag
? equiv(t
- cs_token_flag
) : token_chr(t
));
712 if (token_info(token_link(chr
)) == protected_token
) {
713 lua_pushboolean(L
, 1);
715 lua_pushboolean(L
, 0);
718 lua_pushboolean(L
, 0);
723 static int lua_tokenlib_getfield(lua_State
* L
)
725 const char *s
= lua_tostring(L
, 2);
726 if (lua_key_eq(s
, command
)) {
727 return lua_tokenlib_get_command(L
);
728 } else if (lua_key_eq(s
, index
)) {
729 return lua_tokenlib_get_index(L
);
730 } else if (lua_key_eq(s
, mode
)) {
731 return lua_tokenlib_get_mode(L
);
732 } else if (lua_key_eq(s
, cmdname
)) {
733 return lua_tokenlib_get_cmdname(L
);
734 } else if (lua_key_eq(s
, csname
)) {
735 return lua_tokenlib_get_csname(L
);
736 } else if (lua_key_eq(s
, id
)) {
737 return lua_tokenlib_get_id(L
);
738 } else if (lua_key_eq(s
, tok
)) {
739 return lua_tokenlib_get_tok(L
);
740 } else if (lua_key_eq(s
, active
)) {
741 return lua_tokenlib_get_active(L
);
742 } else if (lua_key_eq(s
, expandable
)) {
743 return lua_tokenlib_get_expandable(L
);
744 } else if (lua_key_eq(s
, protected)) {
745 return lua_tokenlib_get_protected(L
);
754 static int lua_tokenlib_equal(lua_State
* L
)
757 n
= check_istoken(L
, 1);
758 m
= check_istoken(L
, 2);
759 if (token_info(n
->token
) == token_info(m
->token
)) {
760 lua_pushboolean(L
,1);
763 lua_pushboolean(L
,0);
767 static int lua_tokenlib_tostring(lua_State
* L
)
771 n
= check_istoken(L
, 1);
773 snprintf(msg
, 255, "<%s token %d: %d>", (n
->origin
==LUA_ORIGIN
?"lua":"tex"), n
->token
, token_info(n
->token
));
774 lua_pushstring(L
, msg
);
779 static int lua_tokenlib_type(lua_State
* L
)
781 if (maybe_istoken(L
,1)!=NULL
) {
782 lua_pushstring(L
,"token");
789 static int run_scan_token(lua_State
* L
)
791 saved_tex_scanner texstate
;
792 save_tex_scanner(texstate
);
794 make_new_token(L
, cur_cmd
, cur_chr
, cur_cs
);
795 unsave_tex_scanner(texstate
);
801 /* [catcodetable] csname content : \def\csname{content} */
802 /* [catcodetable] csname content global : \gdef\csname{content} */
803 /* [catcodetable] csname : \def\csname{} */
805 /* TODO: check for a quick way to set a macro to empty (HH) */
807 static int set_macro(lua_State
* L
)
809 const char *name
= null
;
810 const char *str
= null
;
811 const char *s
= null
;
815 int n
= lua_gettop(L
);
816 int a
= 0 ; /* global state */
817 int nncs
= no_new_control_sequence
;
821 if (lua_type(L
, 1) == LUA_TNUMBER
) {
824 ct
= (int) lua_tointeger(L
, 1);
825 name
= lua_tolstring(L
, 2, &lname
);
827 str
= lua_tolstring(L
, 3, &lstr
);
829 s
= lua_tostring(L
, 4);
831 ct
= int_par(cat_code_table_code
) ;
832 name
= lua_tolstring(L
, 1, &lname
);
834 str
= lua_tolstring(L
, 2, &lstr
);
836 s
= lua_tostring(L
, 3);
841 if (s
&& (lua_key_eq(s
, global
))) {
844 no_new_control_sequence
= false ;
845 cs
= string_lookup(name
, lname
);
846 no_new_control_sequence
= nncs
;
848 halfword p
; /* tail of the token list */
849 halfword q
; /* new node being added to the token list via |store_new_token| */
850 halfword t
; /* token being appended */
851 const char *se
= str
+ lstr
;
853 set_token_link(p
, null
);
854 /* this left brace is used to store the number of arguments */
855 fast_store_new_token(left_brace_token
);
856 /* and this ends the not present arguments, and no: we will not support arguments here*/
857 fast_store_new_token(end_match_token
);
859 /* hh: str2uni could return len too (also elsewhere) */
860 t
= (halfword
) str2uni((const unsigned char *) str
);
862 cc
= get_cat_code(ct
,t
);
863 /* this is a relating simple converter; if more is needed one can just use */
864 /* tex.print with a regular \def or \gdef and feed the string into the regular */
867 /* we have a potential control sequence so we check for it */
871 halfword _cs
= null
;
872 const char *_name
= str
;
874 t
= (halfword
) str2uni((const unsigned char *) str
);
876 _c
= get_cat_code(ct
,t
);
879 _lname
= _lname
+ _s
;
880 } else if (_c
== 10) {
881 /* we ignore a trailing space like normal scanning does */
889 /* we have a potential \cs */
890 _cs
= string_lookup(_name
, _lname
);
891 if (_cs
== undefined_control_sequence
) {
892 /* let's play safe and backtrack */
893 t
= cc
* (1<<21) + t
;
896 t
= cs_token_flag
+ _cs
;
899 /* just a character with some meaning, so \unknown becomes effectively */
900 /* \\unknown assuming that \\ has some useful meaning of course */
901 t
= cc
* (1<<21) + t
;
906 /* whatever token, so for instance $x^2$ just works given a tex */
908 t
= cc
* (1<<21) + t
;
910 fast_store_new_token(t
);
912 /* there is no fast_store_new_token(right_brace_token) needed */
913 define(cs
, call_cmd
+ (a
% 4), token_link(temp_token_head
));
916 halfword q
; /* new node being added to the token list via |store_new_token| */
918 set_token_info(p
,null
);
919 fast_store_new_token(left_brace_token
);
920 fast_store_new_token(end_match_token
);
921 define(cs
, call_cmd
+ (a
% 4), token_link(temp_token_head
));
926 static const struct luaL_Reg tokenlib
[] = {
927 { "type", lua_tokenlib_type
},
928 { "create", run_build
},
929 { "is_token", lua_tokenlib_is_token
},
931 { "get_next", run_get_next
},
932 { "put_next", run_put_next
},
933 { "scan_keyword", run_scan_keyword
},
934 { "scan_int", run_scan_int
},
935 { "scan_dimen", run_scan_dimen
},
936 { "scan_glue", run_scan_glue
},
937 { "scan_toks", run_scan_toks
},
938 { "scan_code", run_scan_code
},
939 { "scan_string", run_scan_string
},
940 { "scan_word", run_scan_word
},
941 { "scan_csname", run_scan_csname
},
942 { "scan_token", run_scan_token
}, /* expands next token if needed */
943 /* push into input stream */
945 { "write",luatwrite },
948 { "get_command", lua_tokenlib_get_command
},
949 { "get_index", lua_tokenlib_get_index
},
950 { "get_mode", lua_tokenlib_get_mode
},
951 { "get_cmdname", lua_tokenlib_get_cmdname
},
952 { "get_csname", lua_tokenlib_get_csname
},
953 { "get_id", lua_tokenlib_get_id
},
954 { "get_tok", lua_tokenlib_get_tok
},
955 { "get_active", lua_tokenlib_get_active
},
956 { "get_expandable", lua_tokenlib_get_expandable
},
957 { "get_protected", lua_tokenlib_get_protected
},
958 /* maybe more setters */
959 { "set_macro", set_macro
},
961 /* {"expand", run_expand}, */ /* does not work yet! */
962 /* {"csname_id", run_get_csname_id}, */ /* yes or no */
963 /* {"command_id", run_get_command_id}, */ /* yes or no */
964 /* {"cs_offset", run_get_cs_offset}, */ /* not that useful */
968 static const struct luaL_Reg tokenlib_m
[] = {
969 {"__index", lua_tokenlib_getfield
},
970 {"__tostring", lua_tokenlib_tostring
},
971 {"__eq", lua_tokenlib_equal
},
972 {"__gc", lua_tokenlib_free
},
973 {NULL
, NULL
} /* sentinel */
976 int luaopen_token(lua_State
* L
)
978 /* the main metatable of token userdata */
979 luaL_newmetatable(L
, TOKEN_METATABLE
);
980 luaL_register(L
, NULL
, tokenlib_m
);
981 luaL_register(L
, "token", tokenlib
);