| blob | f8fb8b467a0a3444cddebe2856cc4d712eb3deb3 |
1 % textoken.w
2 %
4 %
5 % This file is part of LuaTeX.
6 %
8 % the terms of the GNU General Public License as published by the Free
11 %
15 % License for more details.
16 %
17 % You should have received a copy of the GNU General Public License along
20 @ @c
22 #include "ptexlib.h"
24 @ @c
41 else \
46 @ The \TeX\ system does nearly all of its own memory allocation, so that it can
47 readily be transported into environments that do not have automatic facilities
49 error messages the user receives. The dynamic storage requirements of \TeX\ are
50 handled by providing two large arrays called |fixmem| and |varmem| in which
51 consecutive blocks of words are used as nodes by the \TeX\ routines.
53 Pointer variables are indices into this array, or into another array called
54 |eqtb| that will be explained later. A pointer variable might also be a special
55 flag that lies outside the bounds of |mem|, so we allow pointers to assume any
56 |halfword| value. The minimum halfword value represents a null pointer. \TeX\
59 @ Locations in |fixmem| are used for storing one-word records; a conventional
62 @c
67 @ In order to study the memory requirements of particular applications, it is
68 possible to prepare a version of \TeX\ that keeps track of current and maximum
69 memory usage. When code between the delimiters |@!stat| $\ldots$ |tats| is not
70 commented out, \TeX\ will run a bit slower but it will report these statistics
71 when |tracing_stats| is sufficiently large.
73 @c
86 @ @c
88 {
89 halfword p;
111 }
113 @ The function |get_avail| returns a pointer to a new one-word node whose |link|
114 field is null. However, \TeX\ will halt if there is no more room left.
115 @^inner loop@>
120 work, we have to quit.
122 @c
126 unsigned t;
136 new_fixmem =
137 fixmemcast(realloc
144 }
148 }
152 }
155 that starts at position |p|.
156 @^inner loop@>
158 @c
164 do {
171 }
172 }
176 number is |c| and whose command code is |m| is represented as the number
178 whose |eqtb| address is |p| is represented as the number |cs_token_flag+p|. Here
181 halfword.
183 A token |t| represents a |left_brace| command if and only if
187 following definitions take care of these token-oriented constants and a few
188 others.
190 @ A token list is a singly linked list of one-word nodes in |mem|, where each
193 form of token lists, usually preceded by a node with a reference count in its
196 Three special commands appear in the token lists of macro definitions. When
200 insert parameter number |c| into the text at this point.
203 are omitted, but the final right brace of an output routine is included at the
204 end of its token list.
206 Here is an example macro definition that illustrates these conventions. After
207 \TeX\ processes the text
221 The procedure |scan_toks| builds such token lists, and |macro_call| does the
222 parameter matching. @^reference counts@>
227 entry before the token list has been fully consumed, so we dare not simply
228 destroy a token list when its control sequence is being redefined.
231 corresponding token list will have `\.\{' just before the `|end_match|' and also
233 one keeps the first from disappearing.
235 The |print_meaning| subroutine displays |cur_cmd| and |cur_chr| in symbolic form,
236 including the expansion of a macro or mark.
238 @c
240 {
247 }
248 } */
252 print_ln();
258 print_ln();
260 case first_mark_code:
262 break;
263 case bot_mark_code:
265 break;
266 case split_first_mark_code:
268 break;
269 case split_bot_mark_code:
271 break;
272 default:
274 break;
275 }
276 }
277 }
278 }
280 @ The procedure |show_token_list|, which prints a symbolic form of the token list
281 that starts at a given node |p|, illustrates these conventions. The token list
282 being displayed should not begin with a reference count. However, the procedure
283 is intended to be robust, so that if the memory links are awry or if |p| is not
284 really a pointer to a token list, nothing catastrophic will happen.
286 An additional parameter |q| is also given; this parameter is either null or it
287 points to a node in the token list where a certain magic computation takes place
289 the two-line context information at the time of an error message; |q| marks the
290 place corresponding to where the second line should begin.)
292 For example, if |p| points to the node containing the first \.a in the token list
295 second \.a, the magic computation will be performed just before the second \.a is
296 printed.
299 printing exceeds a given limit~|l|. Anomalous entries are printed in the form of
301 this cannot be confused with actual control sequences because a real control
304 @c
307 case assign_int_cmd: \
310 break; \
311 case assign_dimen_cmd: \
314 break; \
315 case assign_toks_cmd: \
318 break; \
319 default: \
321 break; \
322 }
325 {
335 set_trick_count();
336 }
340 return;
341 }
351 /*
354 The procedure usually ``learns'' the character code used for macro
355 parameters by seeing one in a |match| command before it runs into any
356 |out_param| commands.
357 */
359 case left_brace_cmd:
360 case right_brace_cmd:
361 case math_shift_cmd:
362 case tab_mark_cmd:
363 case sup_mark_cmd:
364 case sub_mark_cmd:
365 case spacer_cmd:
366 case letter_cmd:
367 case other_char_cmd:
369 break;
370 case mac_param_cmd:
374 break;
375 case out_param_cmd:
381 return;
382 }
383 break;
384 case match_cmd:
390 return;
391 break;
392 case end_match_cmd:
395 break;
396 default:
398 break;
399 }
400 }
401 }
403 }
406 }
408 @ @c
414 @ Here's the way we sometimes want to display a token list, given a pointer to
415 its reference count; the pointer may be null.
417 @c
419 {
422 }
424 @ |delete_token_ref|, is called when a pointer to a token list's reference count
425 is being removed. This means that the token list should disappear if the
426 reference count was |null|, otherwise the count should be decreased by one.
427 @^reference counts@>
429 @ |p| points to the reference count of a token list that is losing one
430 reference.
432 @c
434 {
437 else
439 }
441 @ @c
443 {
444 int a;
446 return a;
447 }
449 @ @c
451 {
455 NULL
456 };
460 }
462 @ @c
467 typedef enum {
468 next_line_ok,
469 next_line_return,
470 next_line_restart
477 |scan_keyword| routine checks to see whether the next tokens of input match this
478 string. The match must be exact, except that uppercase letters will match their
479 lowercase counterparts; uppercase equivalents are determined by subtracting
481 only for its own limited set of keywords.
483 If a match is found, the characters are effectively removed from the input and
484 |true| is returned. Otherwise |false| is returned, and the input is left
485 essentially unchanged (except for the fact that some macros may have been
488 @c
506 k++;
517 back_input();
518 }
520 return false;
521 }
522 }
528 return true;
529 }
531 @ We can not return |undefined_control_sequence| under some conditions
534 @c
536 /*
538 {
539 halfword curcs;
555 }
559 return curcs;
560 }
561 */
566 {
567 halfword curcs;
571 }
582 }
584 return curcs;
585 }
587 /*
590 {
616 }
618 return buf;
619 }
622 {
623 halfword curcs;
627 }
634 }
636 return curcs;
637 }
639 */
643 @ prints a control sequence
645 @c
647 {
657 }
662 }
673 }
679 }
681 }
683 }
685 }
687 @ TODO this is a quick hack, will be solved differently soon
689 @c
691 {
693 str_number str;
701 return s;
702 }
704 @ The heart of \TeX's input mechanism is the |get_next| procedure, which we shall
705 develop in the next few sections of the program. Perhaps we shouldn't actually
707 reading the source files and gobbling them up. And it also helps \TeX\ to
708 regurgitate stored token lists that are to be processed again. @^eyes and mouth@>
710 The main duty of |get_next| is to input one token and to set |cur_cmd| and
711 |cur_chr| to that token's command code and modifier. Furthermore, if the input
712 token is a control sequence, the |eqtb| location of that control sequence is
713 stored in |cur_cs|; otherwise |cur_cs| is set to zero.
715 Underlying this simple description is a certain amount of complexity because of
716 all the cases that need to be handled. However, the inner loop of |get_next| is
717 reasonably short and fast.
722 |end_line_char| has |ignore| as its catcode.)
725 needed because a blank line of input is supposed to be exactly equivalent to the
727 line.
729 @c
733 @ Parts |get_next| are executed more often than any other instructions of \TeX.
736 The global variable |force_eof| is normally |false|; it is set |true| by an
738 waiting to be input, it is changed by |luatokencall|.
740 @c
744 @ If the user has set the |pausing| parameter to some positive value, and if
745 nonstop mode has not been selected, each line of input is displayed on the
747 response. If the response is simply |carriage_return|, the line is accepted as it
748 stands, otherwise the line typed is used instead of the line in the file.
750 @c
752 {
757 wake_up_terminal();
758 print_ln();
762 }
769 }
770 }
771 }
772 }
774 @ Before getting into |get_next|, let's consider the subroutine that is called
776 has been reached. These two cases are distinguished by |cur_cs|, which is zero at
777 the end of a file.
779 @c
781 {
785 return;
796 }
799 }
806 NULL
807 };
818 }
820 and insert tokens that should lead to recovery; */
821 /* The recovery procedure can't be fully understood without knowing more
822 about the \TeX\ routines that should be aborted, but we can sketch the
829 case defining:
832 break;
833 case matching:
837 break;
838 case aligning:
846 break;
847 case absorbing:
850 break;
853 break;
865 NULL
866 };
871 NULL
872 };
878 }
886 {
888 back_input();
892 }
893 }
895 }
896 }
898 @ @c
900 #if 0
902 /*
903 The other variant gives less clutter in tracing cache usage when profiling and for
905 */
908 {
909 SWITCH:
914 RESWITCH:
919 }
920 /*
921 Change state if necessary, and |goto switch| if the current
922 character should be ignored, or |goto reswitch| if the current
923 character changes to another;
926 that a decent C compiler has translated the code. Note that the numeric
929 command code to the state to get a single number that characterizes both.
932 cause of many branch prediction errors but changing it to:
936 return true;
939 }
942 }
945 }
948 return true;
949 }
951 gives as many prediction errors. So, we can indeed assume that the compiler
952 does the right job, or that there is simply no other way.
953 */
962 goto SWITCH;
963 break;
970 check_outer_validity();
971 break;
981 check_outer_validity();
982 break;
988 goto RESWITCH;
989 else
991 break;
996 invalid_character_error();
998 break;
1003 break;
1005 /*
1009 treated alike when macro parameters are being matched. We do this since such
1010 characters are indistinguishable on most computer terminal displays.
1011 */
1015 break;
1022 goto SWITCH;
1023 break;
1031 check_outer_validity();
1032 break;
1038 align_state++;
1039 break;
1045 align_state--;
1046 break;
1053 break;
1054 /*
1067 */
1068 default:
1070 break;
1071 }
1075 /*
1076 Move to next line of file,
1077 or |goto restart| if there is no next line,
1079 */
1080 do {
1083 return true;
1085 return false;
1086 }
1088 check_interrupt();
1089 goto SWITCH;
1090 }
1091 return true;
1092 }
1094 #else
1098 /*
1099 escape_cmd left_brace_cmd right_brace_cmd math_shift_cmd
1100 tab_mark_cmd car_ret_cmd mac_param_cmd sup_mark_cmd
1101 sub_mark_cmd ignore_cmd spacer_cmd letter_cmd
1102 other_char_cmd active_char_cmd comment_cmd invalid_char_cmd
1103 */
1106 {
1108 SWITCH:
1112 RESWITCH:
1117 }
1118 /*
1119 Change state if necessary, and |goto switch| if the current
1120 character should be ignored, or |goto reswitch| if the current
1121 character changes to another;
1122 */
1125 return true;
1128 case escape_cmd:
1131 check_outer_validity();
1132 return true;
1133 case left_brace_cmd:
1135 align_state++;
1136 return true;
1137 case right_brace_cmd:
1139 align_state--;
1140 return true;
1141 case math_shift_cmd:
1143 return true;
1144 case tab_mark_cmd:
1146 return true;
1147 case car_ret_cmd:
1154 check_outer_validity();
1155 return true;
1156 case mac_param_cmd:
1158 return true;
1159 case sup_mark_cmd:
1161 goto RESWITCH;
1162 else
1164 return true;
1165 case sub_mark_cmd:
1167 return true;
1168 case ignore_cmd:
1169 goto SWITCH;
1170 return true;
1171 case spacer_cmd:
1173 goto SWITCH;
1174 case letter_cmd:
1176 return true;
1177 case other_char_cmd:
1179 return true;
1180 case active_char_cmd:
1186 check_outer_validity();
1187 return true;
1188 case comment_cmd:
1190 goto SWITCH;
1191 case invalid_char_cmd:
1192 invalid_character_error();
1194 default:
1196 return true;
1197 }
1200 case escape_cmd:
1204 check_outer_validity();
1205 return true;
1206 case left_brace_cmd:
1208 align_state++;
1209 return true;
1210 case right_brace_cmd:
1212 align_state--;
1213 return true;
1214 case math_shift_cmd:
1216 return true;
1217 case tab_mark_cmd:
1219 return true;
1220 case car_ret_cmd:
1222 goto SWITCH;
1223 case mac_param_cmd:
1225 return true;
1226 case sup_mark_cmd:
1229 goto RESWITCH;
1230 else
1232 return true;
1233 case sub_mark_cmd:
1235 return true;
1236 case ignore_cmd:
1237 goto SWITCH;
1238 case spacer_cmd:
1239 goto SWITCH;
1240 case letter_cmd:
1242 return true;
1243 case other_char_cmd:
1245 return true;
1246 case active_char_cmd:
1252 check_outer_validity();
1253 return true;
1254 case comment_cmd:
1257 goto SWITCH;
1258 case invalid_char_cmd:
1260 invalid_character_error();
1262 default:
1264 return true;
1265 }
1268 case escape_cmd:
1271 check_outer_validity();
1272 return true;
1273 case left_brace_cmd:
1274 align_state++;
1275 return true;
1276 case right_brace_cmd:
1277 align_state--;
1278 return true;
1279 case math_shift_cmd:
1280 return true;
1281 case tab_mark_cmd:
1282 return true;
1283 case car_ret_cmd:
1284 /*
1288 treated alike when macro parameters are being matched. We do this since such
1289 characters are indistinguishable on most computer terminal displays.
1290 */
1294 return true;
1295 case mac_param_cmd:
1296 return true;
1297 case sup_mark_cmd:
1299 goto RESWITCH;
1300 else
1302 return true;
1303 case sub_mark_cmd:
1304 return true;
1305 case ignore_cmd:
1306 goto SWITCH;
1307 case spacer_cmd:
1311 return true;
1312 case letter_cmd:
1314 return true;
1315 case other_char_cmd:
1317 return true;
1318 case active_char_cmd:
1324 check_outer_validity();
1325 return true;
1326 case comment_cmd:
1328 goto SWITCH;
1329 case invalid_char_cmd:
1330 invalid_character_error();
1332 default:
1334 return true;
1335 }
1338 return true;
1339 }
1343 }
1344 /*
1347 */
1348 do {
1351 return true;
1353 return false;
1354 }
1356 check_interrupt();
1357 goto SWITCH;
1358 }
1359 return true;
1360 }
1362 #endif
1365 We only support a limited set:
1367 ^^^^^^XXXXXX
1368 ^^^^XXXXXX
1371 @c
1380 }
1387 }
1407 {
1424 return true;
1427 }
1430 }
1441 return true;
1444 }
1447 }
1448 }
1457 return true;
1458 }
1459 }
1461 }
1462 }
1464 {
1473 return true;
1474 }
1475 }
1477 return true;
1478 }
1479 }
1480 }
1481 return false;
1482 }
1484 @ Control sequence names are scanned only when they appear in some line of a
1486 a unique identification, so \TeX\ doesn't need to refer to the original name any
1487 more except when it prints the equivalent in symbolic form.
1489 The program that scans a control sequence has been written carefully in order to
1490 avoid the blowups that might otherwise occur if a malicious user tried something
1495 following a control sequence name, they are converted to single characters in the
1496 buffer and the process is repeated, slowly but surely.
1498 @c
1502 {
1515 continue;
1518 do {
1524 continue;
1527 /*
1535 */
1544 }
1546 }
1547 }
1550 break;
1551 }
1552 }
1555 return retval;
1556 }
1558 @ Whenever we reach the following piece of code, we will have
1563 left two or three places.
1565 @c
1567 {
1568 int l;
1586 }
1589 }
1601 }
1604 }
1605 }
1617 }
1620 }
1621 }
1622 }
1625 else
1626 d++;
1631 k++;
1632 d--;
1636 k++;
1637 d--;
1639 k++;
1640 d--;
1644 k++;
1645 d--;
1647 k++;
1648 d--;
1650 k++;
1651 d--;
1653 }
1658 l++;
1659 }
1661 return true;
1662 }
1663 return false;
1664 }
1666 @ All of the easy branches of |get_next| have now been taken care of. There is
1667 one more branch.
1670 {
1688 return next_line_restart;
1691 }
1695 firm_up_the_line();
1705 }
1714 return next_line_restart;
1717 }
1718 }
1719 }
1720 }
1726 file_warning();
1727 }
1731 }
1735 end_file_reading();
1737 end_file_reading();
1739 check_outer_validity();
1740 }
1741 return next_line_restart;
1742 }
1744 ilimit--;
1745 else
1755 }
1758 end_file_reading();
1760 }
1762 open_log_file();
1765 ilimit++;
1769 }
1770 print_ln();
1775 ilimit--;
1776 else
1781 /*
1783 never waits for on-line input.
1784 */
1786 }
1787 }
1788 return next_line_ok;
1789 }
1791 @ Let's consider now what happens when |get_next| is looking at a token list.
1793 @c
1795 {
1804 /*
1805 Get the next token, suppressing expansion. The present point in the program
1806 is reached only when the |expand| routine has inserted a special marker into
1809 */
1816 return true;
1817 }
1819 check_outer_validity();
1820 }
1821 }
1827 case left_brace_cmd:
1828 align_state++;
1829 break;
1830 case right_brace_cmd:
1831 align_state--;
1832 break;
1833 case out_param_cmd:
1836 return false;
1837 break;
1838 }
1839 }
1840 return true;
1841 }
1843 @ Now we're ready to take the plunge into |get_next| itself. Parts of this
1844 routine are executed more often than any other instructions of \TeX.
1849 @c
1851 {
1852 RESTART:
1857 goto RESTART;
1860 end_token_list();
1864 }
1865 }
1868 insert_vj_template();
1869 goto RESTART;
1870 }
1871 }
1873 @ Since |get_next| is used so frequently in \TeX, it is convenient to define
1874 three related procedures that do a little more:
1876 \yskip\hang|get_token| not only sets |cur_cmd| and |cur_chr|, it also sets
1877 |cur_tok|, a packed halfword version of the current token.
1883 the expansion of the macro or the evaluation of the conditional.
1885 \yskip\hang|x_token| is like |get_x_token| except that it assumes that |get_next|
1886 has already been called.
1888 \yskip\noindent In fact, these three procedures account for almost every use of
1889 |get_next|.
1891 No new control sequences will be defined except during a call of |get_token|, or
1893 always |true| at other times.
1897 @c
1899 {
1901 get_next();
1905 else
1907 }
1909 @ changes the string |s| to a token list
1911 @c
1913 {
1926 else
1929 }
1931 }
1936 Before we get into the details of |scan_toks|, let's consider a much simpler
1937 task, that of converting the current string into a token list. The |str_toks|
1938 function does this; it classifies spaces as type |spacer| and everything else as
1939 type |other_char|.
1944 |lua_str_toks| is almost identical, but it also escapes the three symbols that
1945 |lua| considers special while scanning a literal string
1949 @c
1951 {
1966 fast_store_new_token(other_token + '\\');
1967 if (t == 10)
1968 t = 'n';
1969 if (t == 13)
1970 t = 'r';
1971 t = other_token + t;
1972 }
1973 fast_store_new_token(t);
1974 }
1975 return p;
1976 }
1978 @ Incidentally, the main reason for wanting |str_toks| is the function
1979 |the_toks|, which has similar input/output characteristics.
1981 @ changes the string |str_pool[b..pool_ptr]| to a token list
1983 @c
1984 halfword str_toks(lstring s)
1985 {
1986 halfword p; /* tail of the token list */
1987 halfword q; /* new node being added to the token list via |store_new_token| */
1988 halfword t; /* token being appended */
1989 unsigned char *k, *l; /* index into string */
1990 p = temp_token_head;
1991 set_token_link(p, null);
1992 k = s.s;
1993 l = k + s.l;
1994 while (k < l) {
1995 t = pool_to_unichar(k);
1996 k += utf8_size(t);
1997 if (t == ' ')
1998 t = space_token;
1999 else
2000 t = other_token + t;
2001 fast_store_new_token(t);
2002 }
2003 return p;
2004 }
2006 /*
2007 hh: most of the converter is similar to the one i made for macro so at some point i
2008 can make a helper; also todo: there is no need to go through the pool
2010 */
2012 halfword str_scan_toks(int ct, lstring s)
2013 { /* changes the string |str_pool[b..pool_ptr]| to a token list */
2014 halfword p; /* tail of the token list */
2015 halfword q; /* new node being added to the token list via |store_new_token| */
2016 halfword t; /* token being appended */
2017 unsigned char *k, *l; /* index into string */
2018 int cc;
2019 p = temp_token_head;
2020 set_token_link(p, null);
2021 k = s.s;
2022 l = k + s.l;
2023 while (k < l) {
2024 t = pool_to_unichar(k);
2025 k += utf8_size(t);
2026 cc = get_cat_code(ct,t);
2027 if (cc == 0) {
2028 /* we have a potential control sequence so we check for it */
2029 int _lname = 0 ;
2030 int _s = 0 ;
2031 int _c = 0 ;
2032 halfword _cs = null ;
2033 unsigned char *_name = k ;
2034 while (k < l) {
2035 t = (halfword) str2uni((const unsigned char *) k);
2036 _s = utf8_size(t);
2037 _c = get_cat_code(ct,t);
2038 if (_c == 11) {
2039 k += _s ;
2040 _lname = _lname + _s ;
2041 } else if (_c == 10) {
2042 /* we ignore a trailing space like normal scanning does */
2043 k += _s ;
2044 break ;
2045 } else {
2046 break ;
2047 }
2048 }
2049 if (_s > 0) {
2050 /* we have a potential \cs */
2051 _cs = string_lookup((const char *) _name, _lname);
2052 if (_cs == undefined_control_sequence) {
2053 /* let's play safe and backtrack */
2054 t = cc * (1<<21) + t ;
2055 k = _name ;
2056 } else {
2057 t = cs_token_flag + _cs;
2058 }
2059 } else {
2060 /* just a character with some meaning, so \unknown becomes effectively */
2061 /* \\unknown assuming that \\ has some useful meaning of course */
2062 t = cc * (1<<21) + t ;
2063 k = _name ;
2064 }
2066 } else {
2067 /* whatever token, so for instance $x^2$ just works given a tex */
2068 /* catcode regime */
2069 t = cc * (1<<21) + t ;
2070 }
2071 fast_store_new_token(t);
2073 }
2074 return p;
2075 }
2077 @ Here's part of the |expand| subroutine that we are now ready to complete:
2079 @c
2080 void ins_the_toks(void)
2081 {
2082 (void) the_toks();
2083 ins_list(token_link(temp_token_head));
2084 }
2086 #define set_toks_register(n,t,g) { \
2087 int a = (g>0) ? 4 : 0; \
2088 halfword ref = get_avail(); \
2089 set_token_ref_count(ref, 0); \
2090 set_token_link(ref, token_link(t)); \
2091 define(n + toks_base, call_cmd, ref); \
2092 }
2094 void combine_the_toks(int how)
2095 {
2096 halfword nt;
2097 get_x_token();
2098 /* target */
2099 if (cur_cmd == assign_toks_cmd) {
2100 nt = equiv(cur_cs) - toks_base;
2101 /* check range */
2102 } else {
2103 back_input();
2104 scan_int();
2105 nt = cur_val;
2106 }
2107 /* source */
2108 do {
2109 get_x_token();
2110 } while (cur_cmd == spacer_cmd);
2111 if (cur_cmd == left_brace_cmd) {
2112 halfword x, source;
2113 back_input();
2114 x = scan_toks(false,how > 1); /* expanded or not */
2115 source = def_ref;
2116 /* action */
2117 if (source != null) {
2118 halfword target = toks(nt);
2119 if (target == null) {
2120 set_toks_register(nt,source,0);
2121 } else {
2122 halfword s = token_link(source);
2123 if (s != null) {
2124 halfword t = token_link(target);
2125 if (t == null) {
2126 /* can this happen ? */
2127 set_token_link(target, s);
2128 } else if (odd(how)) {
2129 /* prepend */
2130 if (cur_level != eq_level_field(eqtb[toks_base+nt])) {
2131 halfword p = temp_token_head;
2132 halfword q;
2133 set_token_link(p, s); /* s = head, x = tail */
2134 p = x;
2135 while (t != null) {
2136 fast_store_new_token(token_info(t));
2137 t = token_link(t);
2138 }
2139 set_toks_register(nt,temp_token_head,0);
2140 } else {
2141 set_token_link(x,t);
2142 set_token_link(target,s);
2143 }
2144 } else {
2145 /* append */
2146 if (cur_level != eq_level_field(eqtb[toks_base+nt])) {
2147 halfword p = temp_token_head;
2148 halfword q;
2149 set_token_link(p, null);
2150 while (t != null) {
2151 fast_store_new_token(token_info(t));
2152 t = token_link(t);
2153 }
2154 set_token_link(p,s);
2155 set_toks_register(nt,temp_token_head,0);
2156 } else {
2157 while (token_link(t) != null) {
2158 t = token_link(t);
2159 }
2160 set_token_link(t,s);
2161 }
2162 }
2163 }
2164 }
2165 }
2166 } else {
2167 halfword source, ns;
2168 if (cur_cmd == assign_toks_cmd) {
2169 ns = equiv(cur_cs) - toks_base;
2170 /* check range */
2171 } else {
2172 back_input();
2173 scan_int();
2174 ns = cur_val;
2175 }
2176 /* action */
2177 source = toks(ns);
2178 if (source != null) {
2179 halfword target = toks(nt);
2180 if (target == null) {
2181 equiv(toks_base+nt) = source;
2182 equiv(toks_base+ns) = null;
2183 } else {
2184 halfword s = token_link(source);
2185 if (s != null) {
2186 halfword t = token_link(target);
2187 if (t == null) {
2188 set_token_link(target, s);
2189 } else if (odd(how)) {
2190 /* prepend */
2191 halfword x = s;
2192 while (token_link(x) != null) {
2193 x = token_link(x);
2194 }
2195 set_token_link(x,t);
2196 set_token_link(target,s);
2197 } else {
2198 /* append */
2199 while (token_link(t) != null) {
2200 t = token_link(t);
2201 }
2202 set_token_link(t,s);
2203 }
2204 equiv(toks_base+ns) = null;
2205 }
2206 }
2207 }
2208 }
2209 }
2211 @ This routine, used in the next one, prints the job name, possibly modified by
2212 the |process_jobname| callback.
2214 @c
2215 static void print_job_name(void)
2216 {
2217 if (job_name) {
2218 char *s, *ss; /* C strings for jobname before and after processing */
2219 int callback_id, lua_retval;
2220 s = (char*)str_string(job_name);
2221 callback_id = callback_defined(process_jobname_callback);
2222 if (callback_id > 0) {
2224 if ((lua_retval == true) && (ss != NULL))
2225 s = ss;
2226 }
2227 tprint(s);
2228 } else {
2229 print(job_name);
2230 }
2231 }
2233 @ Here is a routine that print the result of a convert command, using the
2234 argument |i|. It returns |false | if it does not know to print the code |c|. The
2235 function exists because lua code and tex code can both call it to convert
2236 something.
2238 @ Parse optional lua state integer, or an instance name to be stored in |sn| and
2239 get the next non-blank non-relax non-call token.
2241 @c
2243 int scan_lua_state(void)
2244 {
2245 int sn = 0;
2246 do {
2247 get_x_token();
2248 } while ((cur_cmd == spacer_cmd) || (cur_cmd == relax_cmd));
2249 back_input();
2250 if (cur_cmd != left_brace_cmd) {
2252 (void) scan_toks(false, true);
2253 sn = def_ref;
2254 } else {
2255 scan_register_num();
2256 if (get_lua_name(cur_val))
2257 sn = (cur_val - 65536);
2258 }
2259 }
2260 return sn;
2261 }
2263 @ The procedure |conv_toks| uses |str_toks| to insert the token list for
2264 |convert| functions into the scanner; `\.{\\outer}' control sequences are allowed
2265 to follow `\.{\\string}' and `\.{\\meaning}'.
2267 The extra temp string |u| is needed because |pdf_scan_ext_toks| incorporates any
2268 pending string in its output. In order to save such a pending string, we have to
2269 create a temporary string that is destroyed immediately after.
2271 @c
2272 #define push_selector { \
2273 old_setting = selector; \
2274 selector = new_string; \
2275 }
2277 #define pop_selector { \
2278 selector = old_setting; \
2279 }
2281 static int do_variable_dvi(halfword c)
2282 {
2283 return 0;
2284 }
2286 #define do_variable_backend_int(i) \
2287 cur_cmd = assign_int_cmd; \
2288 cur_val = backend_int_base + i; \
2289 cur_tok = token_val(cur_cmd, cur_val); \
2290 back_input();
2292 #define do_variable_backend_dimen(i) \
2293 cur_cmd = assign_dimen_cmd; \
2294 cur_val = backend_dimen_base + i; \
2295 cur_tok = token_val(cur_cmd, cur_val); \
2296 back_input();
2298 #define do_variable_backend_toks(i) \
2299 cur_cmd = assign_toks_cmd; \
2300 cur_val = backend_toks_base + i ; \
2301 cur_tok = token_val(cur_cmd, cur_val); \
2302 back_input();
2304 static int do_variable_pdf(halfword c)
2305 {
2313 else if (scan_keyword("inclusionerrorlevel")) { do_variable_backend_int(c_pdf_inclusion_errorlevel); }
2314 else if (scan_keyword("ignoreunknownimages")) { do_variable_backend_int(c_pdf_ignore_unknown_images); }
2316 else if (scan_keyword("imageapplygamma")) { do_variable_backend_int(c_pdf_image_apply_gamma); }
2319 else if (scan_keyword("imageaddfilename")) { do_variable_backend_int(c_pdf_image_addfilename); }
2320 else if (scan_keyword("objcompresslevel")) { do_variable_backend_int(c_pdf_objcompresslevel); }
2321 else if (scan_keyword("inclusioncopyfonts")) { do_variable_backend_int(c_pdf_inclusion_copy_font); }
2339 else
2340 return 0;
2341 return 1;
2342 }
2344 static int do_feedback_dvi(halfword c)
2345 {
2346 return 0;
2347 }
2349 /* codes not really needed but cleaner when testing */
2351 #define pdftex_version 40 /* these values will not change any more */
2354 static int do_feedback_pdf(halfword c)
2355 {
2356 int old_setting; /* holds |selector| setting */
2357 int save_scanner_status; /* |scanner_status| upon entry */
2358 halfword save_def_ref; /* |def_ref| upon entry, important if inside `\.{\\message}' */
2359 halfword save_warning_index;
2360 boolean bool; /* temp boolean */
2361 str_number s; /* first temp string */
2362 int ff; /* for use with |set_ff| */
2363 str_number u = 0; /* third temp string, will become non-nil if a string is already being built */
2364 char *str; /* color stack init str */
2367 push_selector;
2368 print_int(pdf_last_link);
2369 pop_selector;
2371 push_selector;
2372 print_int(pdf_retval);
2373 pop_selector;
2375 push_selector;
2376 print_int(pdf_last_obj);
2377 pop_selector;
2379 push_selector;
2380 print_int(pdf_last_annot);
2381 pop_selector;
2383 scan_int();
2384 check_obj_type(static_pdf, obj_type_xform, cur_val);
2385 push_selector;
2386 print_int(obj_info(static_pdf, cur_val));
2387 pop_selector;
2389 ins_list(string_to_toks(getcreationdate(static_pdf)));
2390 /* no further action */
2391 return 2;
2393 scan_font_ident();
2394 if (cur_val == null_font)
2396 pdf_check_vf(cur_val);
2397 if (!font_used(cur_val))
2398 pdf_init_font(static_pdf, cur_val);
2399 push_selector;
2400 set_ff(cur_val);
2401 print_int(obj_info(static_pdf, pdf_font_num(ff)));
2402 pop_selector;
2404 scan_font_ident();
2405 if (cur_val == null_font)
2407 pdf_check_vf(cur_val);
2408 if (!font_used(cur_val))
2409 pdf_init_font(static_pdf, cur_val);
2410 push_selector;
2411 set_ff(cur_val);
2412 print_int(pdf_font_num(ff));
2413 pop_selector;
2415 scan_font_ident();
2416 if (cur_val == null_font)
2418 push_selector;
2419 print_scaled(font_size(cur_val));
2421 pop_selector;
2423 scan_int();
2424 if (cur_val <= 0)
2426 push_selector;
2427 print_int(pdf_get_obj(static_pdf, obj_type_page, cur_val, false));
2428 pop_selector;
2432 cur_val = direct_always;
2434 cur_val = direct_page;
2435 else
2436 cur_val = set_origin;
2437 save_scanner_status = scanner_status;
2438 save_warning_index = warning_index;
2439 save_def_ref = def_ref;
2440 u = save_cur_string();
2441 scan_toks(false, true);
2442 s = tokens_to_string(def_ref);
2443 delete_token_ref(def_ref);
2444 def_ref = save_def_ref;
2445 warning_index = save_warning_index;
2446 scanner_status = save_scanner_status;
2447 str = makecstring(s);
2448 cur_val = newcolorstack(str, cur_val, bool);
2449 free(str);
2450 flush_str(s);
2451 cur_val_level = int_val_level;
2452 if (cur_val < 0) {
2456 error();
2457 cur_val = 0;
2458 restore_cur_string(u);
2459 }
2460 push_selector;
2461 print_int(cur_val);
2462 pop_selector;
2464 push_selector;
2465 print_int(pdftex_version);
2466 pop_selector;
2468 ins_list(string_to_toks(pdftex_revision));
2469 return 2;
2470 } else {
2471 return 0;
2472 }
2473 return 1;
2474 }
2476 void conv_toks(void)
2477 {
2478 int old_setting; /* holds |selector| setting */
2479 halfword p, q;
2480 int save_scanner_status; /* |scanner_status| upon entry */
2481 halfword save_def_ref; /* |def_ref| upon entry, important if inside `\.{\\message}' */
2482 halfword save_warning_index;
2483 boolean bool; /* temp boolean */
2484 str_number s; /* first temp string */
2485 int sn; /* lua chunk name */
2486 str_number u = 0; /* third temp string, will become non-nil if a string is already being built */
2487 int c = cur_chr; /* desired type of conversion */
2488 str_number str;
2489 int i = 0;
2490 /* Scan the argument for command |c| */
2491 switch (c) {
2492 case number_code:
2493 scan_int();
2494 push_selector;
2495 print_int(cur_val);
2496 pop_selector;
2497 break;
2498 case lua_function_code:
2499 scan_int();
2500 if (cur_val <= 0) {
2502 } else {
2503 u = save_cur_string();
2504 luacstrings = 0;
2505 luafunctioncall(cur_val);
2506 restore_cur_string(u);
2507 if (luacstrings > 0)
2508 lua_string_start();
2509 }
2510 /* no further action */
2511 return;
2512 break;
2513 case lua_code:
2514 u = save_cur_string();
2515 save_scanner_status = scanner_status;
2516 save_def_ref = def_ref;
2517 save_warning_index = warning_index;
2518 sn = scan_lua_state();
2519 scan_toks(false, true);
2520 s = def_ref;
2521 warning_index = save_warning_index;
2522 def_ref = save_def_ref;
2523 scanner_status = save_scanner_status;
2524 luacstrings = 0;
2525 luatokencall(s, sn);
2526 delete_token_ref(s);
2527 restore_cur_string(u); /* TODO: check this, was different */
2528 if (luacstrings > 0)
2529 lua_string_start();
2530 /* no further action */
2531 return;
2532 break;
2533 case expanded_code:
2534 save_scanner_status = scanner_status;
2535 save_warning_index = warning_index;
2536 save_def_ref = def_ref;
2537 u = save_cur_string();
2538 scan_toks(false, true);
2539 warning_index = save_warning_index;
2540 scanner_status = save_scanner_status;
2541 ins_list(token_link(def_ref));
2542 def_ref = save_def_ref;
2543 restore_cur_string(u);
2544 /* no further action */
2545 return;
2546 break;
2547 case math_style_code:
2548 push_selector;
2549 print_math_style();
2550 pop_selector;
2551 break;
2552 case string_code:
2553 save_scanner_status = scanner_status;
2554 scanner_status = normal;
2555 get_token();
2556 scanner_status = save_scanner_status;
2557 push_selector;
2558 if (cur_cs != 0)
2559 sprint_cs(cur_cs);
2560 else
2561 print(cur_chr);
2562 pop_selector;
2563 break;
2564 case cs_string_code:
2565 save_scanner_status = scanner_status;
2566 scanner_status = normal;
2567 get_token();
2568 scanner_status = save_scanner_status;
2569 push_selector;
2570 if (cur_cs != 0)
2571 sprint_cs_name(cur_cs);
2572 else
2573 print(cur_chr);
2574 pop_selector;
2575 break;
2576 case roman_numeral_code:
2577 scan_int();
2578 push_selector;
2579 print_roman_int(cur_val);
2580 pop_selector;
2581 break;
2582 case meaning_code:
2583 save_scanner_status = scanner_status;
2584 scanner_status = normal;
2585 get_token();
2586 scanner_status = save_scanner_status;
2587 push_selector;
2588 print_meaning();
2589 pop_selector;
2590 break;
2591 case uchar_code:
2592 scan_char_num();
2593 push_selector;
2594 print(cur_val);
2595 pop_selector;
2596 break;
2597 case lua_escape_string_code:
2598 {
2599 lstring escstr;
2600 int l = 0;
2601 save_scanner_status = scanner_status;
2602 save_def_ref = def_ref;
2603 save_warning_index = warning_index;
2604 scan_toks(false, true);
2605 bool = in_lua_escape;
2606 in_lua_escape = true;
2607 escstr.s = (unsigned char *) tokenlist_to_cstring(def_ref, false, &l);
2608 escstr.l = (unsigned) l;
2609 in_lua_escape = bool;
2610 delete_token_ref(def_ref);
2611 def_ref = save_def_ref;
2612 warning_index = save_warning_index;
2613 scanner_status = save_scanner_status;
2614 (void) lua_str_toks(escstr);
2615 ins_list(token_link(temp_token_head));
2616 free(escstr.s);
2617 return;
2618 }
2619 /* no further action */
2620 break;
2621 case font_id_code:
2622 scan_font_ident();
2623 push_selector;
2624 print_int(cur_val);
2625 pop_selector;
2626 break;
2627 case font_name_code:
2628 scan_font_ident();
2629 push_selector;
2630 append_string((unsigned char *) font_name(cur_val),(unsigned) strlen(font_name(cur_val)));
2631 if (font_size(cur_val) != font_dsize(cur_val)) {
2633 print_scaled(font_size(cur_val));
2635 }
2636 pop_selector;
2637 break;
2638 case left_margin_kern_code:
2639 scan_int();
2640 if ((box(cur_val) == null) || (type(box(cur_val)) != hlist_node))
2642 push_selector;
2643 p = list_ptr(box(cur_val));
2644 while ((p != null) && (type(p) == glue_node)) {
2645 p = vlink(p);
2646 }
2647 if ((p != null) && (type(p) == margin_kern_node) && (subtype(p) == left_side))
2648 print_scaled(width(p));
2649 else
2650 print_char('0');
2652 pop_selector;
2653 break;
2654 case right_margin_kern_code:
2655 scan_int();
2656 if ((box(cur_val) == null) || (type(box(cur_val)) != hlist_node))
2658 push_selector;
2659 p = list_ptr(box(cur_val));
2660 if (p != null) {
2661 p = tail_of_list(p);
2662 /*
2663 there can be a leftskip, rightskip, penalty and yes, also a disc node with a nesting
2664 node that points to glue spec ... and we don't want to analyze that messy lot
2665 */
2666 while ((p != null) && (type(p) == glue_node)) {
2667 p = alink(p);
2668 }
2669 if ((p != null) && ! ((type(p) == margin_kern_node) && (subtype(p) == right_side))) {
2670 if (type(p) == disc_node) {
2671 q = alink(p);
2672 if ((q != null) && ((type(q) == margin_kern_node) && (subtype(q) == right_side))) {
2673 p = q;
2674 } else {
2675 /*
2676 officially we should look in the replace but currently protrusion doesn't
2678 don't need it now
2679 */
2680 }
2681 }
2682 }
2683 }
2684 if ((p != null) && (type(p) == margin_kern_node) && (subtype(p) == right_side))
2685 print_scaled(width(p));
2686 else
2687 print_char('0');
2689 pop_selector;
2690 break;
2691 case uniform_deviate_code:
2692 scan_int();
2693 push_selector;
2694 print_int(unif_rand(cur_val));
2695 pop_selector;
2696 break;
2697 case normal_deviate_code:
2698 scan_int();
2699 push_selector;
2700 print_int(norm_rand());
2701 pop_selector;
2702 break;
2703 case math_char_class_code:
2704 {
2705 mathcodeval mval;
2706 scan_int();
2707 mval = get_math_code(cur_val);
2708 push_selector;
2709 print_int(mval.class_value);
2710 pop_selector;
2711 }
2712 break;
2713 case math_char_fam_code:
2714 {
2715 mathcodeval mval;
2716 scan_int();
2717 mval = get_math_code(cur_val);
2718 push_selector;
2719 print_int(mval.family_value);
2720 pop_selector;
2721 }
2722 break;
2723 case math_char_slot_code:
2724 {
2725 mathcodeval mval;
2726 scan_int();
2727 mval = get_math_code(cur_val);
2728 push_selector;
2729 print_int(mval.character_value);
2730 pop_selector;
2731 }
2732 break;
2733 case insert_ht_code:
2734 scan_register_num();
2735 push_selector;
2736 i = cur_val;
2737 p = page_ins_head;
2738 while (i >= subtype(vlink(p)))
2739 p = vlink(p);
2740 if (subtype(p) == i)
2741 print_scaled(height(p));
2742 else
2743 print_char('0');
2745 pop_selector;
2746 break;
2747 case job_name_code:
2748 if (job_name == 0)
2749 open_log_file();
2750 push_selector;
2751 print_job_name();
2752 pop_selector;
2753 break;
2754 case format_name_code:
2755 if (job_name == 0)
2756 open_log_file();
2757 push_selector;
2758 print(format_name);
2759 pop_selector;
2760 break;
2761 case luatex_banner_code:
2762 push_selector;
2763 tprint(luatex_banner);
2764 pop_selector;
2765 break;
2766 case luatex_revision_code:
2767 push_selector;
2768 print(get_luatexrevision());
2769 pop_selector;
2770 break;
2771 case luatex_date_code:
2772 push_selector;
2773 print_int(get_luatex_date_info());
2774 pop_selector;
2775 break;
2776 case etex_code:
2777 push_selector;
2778 tprint(eTeX_version_string);
2779 pop_selector;
2780 break;
2781 case eTeX_revision_code:
2782 push_selector;
2783 tprint(eTeX_revision);
2784 pop_selector;
2785 break;
2786 case font_identifier_code:
2788 break;
2789 default:
2791 break;
2792 }
2793 str = make_string();
2794 (void) str_toks(str_lstring(str));
2795 flush_str(str);
2796 ins_list(token_link(temp_token_head));
2797 }
2799 void do_feedback(void)
2800 {
2801 int c = cur_chr;
2802 str_number str;
2803 int done = 1;
2804 switch (c) {
2805 case dvi_feedback_code:
2806 if (get_o_mode() == OMODE_DVI) {
2807 done = do_feedback_dvi(c);
2808 } else {
2810 return ;
2811 }
2812 if (done==0) {
2813 /* we recover */
2815 return;
2816 } else if (done==2) {
2817 return;
2818 }
2819 break;
2820 case pdf_feedback_code:
2821 if (get_o_mode() == OMODE_PDF) {
2822 done = do_feedback_pdf(c);
2823 } else {
2825 return ;
2826 }
2827 if (done==0) {
2828 /* we recover */
2830 return;
2831 } else if (done==2) {
2832 return;
2833 }
2834 break;
2835 default:
2837 break;
2838 }
2839 str = make_string();
2840 (void) str_toks(str_lstring(str));
2841 flush_str(str);
2842 ins_list(token_link(temp_token_head));
2843 }
2845 void do_variable(void)
2846 {
2847 int c = cur_chr;
2848 int done = 1;
2849 switch (c) {
2850 case dvi_variable_code:
2851 done = do_variable_dvi(c);
2852 if (done==0) {
2853 /* we recover */
2855 }
2856 return;
2857 break;
2858 case pdf_variable_code:
2859 done = do_variable_pdf(c);
2860 if (done==0) {
2861 /* we recover */
2863 }
2864 return;
2865 break;
2866 default:
2868 break;
2869 }
2870 }
2872 @ This boolean is keeping track of the lua string escape state
2873 @c
2874 boolean in_lua_escape;
2876 static int the_convert_string_dvi(halfword c, int i)
2877 {
2878 return 0 ;
2879 }
2881 static int the_convert_string_pdf(halfword c, int i)
2882 {
2883 int ff;
2884 if (get_o_mode() != OMODE_PDF) {
2885 return 0;
2887 print_int(pdf_last_link);
2889 print_int(pdf_retval);
2891 print_int(pdf_last_obj);
2893 print_int(pdf_last_annot);
2895 print_int(obj_info(static_pdf, i));
2897 return 0;
2899 set_ff(i);
2900 print_int(obj_info(static_pdf, pdf_font_num(ff)));
2902 set_ff(i);
2903 print_int(pdf_font_num(ff));
2905 print_scaled(font_size(i));
2908 print_int(pdf_get_obj(static_pdf, obj_type_page, i, false));
2910 return 0;
2911 } else {
2912 return 0;
2913 }
2914 return 1;
2915 }
2917 str_number the_convert_string(halfword c, int i)
2918 {
2919 int old_setting; /* saved |selector| setting */
2920 str_number ret = 0;
2921 boolean done = true ;
2922 old_setting = selector;
2923 selector = new_string;
2924 switch (c) {
2925 case number_code:
2926 print_int(i);
2927 break;
2928 /* case lua_function_code: */
2929 /* case lua_code: */
2930 /* case expanded_code: */
2931 case math_style_code:
2932 print_math_style();
2933 break;
2934 /* case string_code: */
2935 /* case cs_string_code: */
2936 case roman_numeral_code:
2937 print_roman_int(i);
2938 break;
2939 /* case meaning_code: */
2940 case uchar_code:
2941 print(i);
2942 break;
2943 /* lua_escape_string_code: */
2944 case font_id_code:
2945 print_int(i);
2946 break;
2947 case font_name_code:
2948 append_string((unsigned char *) font_name(i),(unsigned) strlen(font_name(i)));
2949 if (font_size(i) != font_dsize(i)) {
2951 print_scaled(font_size(i));
2953 }
2954 break;
2955 /* left_margin_kern_code: */
2956 /* right_margin_kern_code: */
2957 case uniform_deviate_code:
2958 print_int(unif_rand(i));
2959 break;
2960 case normal_deviate_code:
2961 print_int(norm_rand());
2962 break;
2963 /* math_char_class_code: */
2964 /* math_char_fam_code: */
2965 /* math_char_slot_code: */
2966 /* insert_ht_code: */
2967 case job_name_code:
2968 print_job_name();
2969 break;
2970 case format_name_code:
2971 print(format_name);
2972 break;
2973 case luatex_banner_code:
2974 tprint(luatex_banner);
2975 break;
2976 case luatex_revision_code:
2977 print(get_luatexrevision());
2978 break;
2979 case luatex_date_code:
2980 print_int(get_luatex_date_info());
2981 break;
2982 case etex_code:
2983 tprint(eTeX_version_string);
2984 break;
2985 case eTeX_revision_code:
2986 tprint(eTeX_revision);
2987 break;
2988 case font_identifier_code:
2989 print_font_identifier(i);
2990 break;
2991 /* backend: this might become obsolete */
2992 case dvi_feedback_code:
2993 done = the_convert_string_dvi(c,i);
2994 break;
2995 case pdf_feedback_code:
2996 done = the_convert_string_pdf(c,i);
2997 break;
2998 /* done */
2999 default:
3000 done = false;
3001 break;
3002 }
3003 if (done) {
3004 ret = make_string();
3005 }
3006 selector = old_setting;
3007 return ret;
3008 }
3010 @ Another way to create a token list is via the \.{\\read} command. The sixteen
3011 files potentially usable for reading appear in the following global variables.
3012 The value of |read_open[n]| will be |closed| if stream number |n| has not been
3013 opened or if it has been fully read; |just_open| if an \.{\\openin} but not a
3014 \.{\\read} has been done; and |normal| if it is open and ready to read the next
3015 line.
3017 @c
3018 FILE *read_file[16]; /* used for \.{\\read} */
3019 int read_open[17]; /* state of |read_file[n]| */
3021 void initialize_read(void)
3022 {
3023 int k;
3024 for (k = 0; k <= 16; k++)
3025 read_open[k] = closed;
3026 }
3028 @ The |read_toks| procedure constructs a token list like that for any macro
3029 definition, and makes |cur_val| point to it. Parameter |r| points to the control
3030 sequence that will receive this token list.
3032 @c
3033 void read_toks(int n, halfword r, halfword j)
3034 {
3035 halfword p; /* tail of the token list */
3036 halfword q; /* new node being added to the token list via |store_new_token| */
3037 int s; /* saved value of |align_state| */
3038 int m; /* stream number */
3039 scanner_status = defining;
3040 warning_index = r;
3041 p = get_avail();
3042 def_ref = p;
3043 set_token_ref_count(def_ref, 0);
3044 p = def_ref; /* the reference count */
3045 store_new_token(end_match_token);
3046 if ((n < 0) || (n > 15))
3047 m = 16;
3048 else
3049 m = n;
3050 s = align_state;
3051 align_state = 1000000; /* disable tab marks, etc. */
3052 do {
3053 /* Input and store tokens from the next line of the file */
3054 begin_file_reading();
3055 iname = m + 1;
3056 if (read_open[m] == closed) {
3057 /*
3058 Input for \.{\\read} from the terminal
3060 Here we input on-line into the |buffer| array, prompting the user explicitly
3061 if |n>=0|. The value of |n| is set negative so that additional prompts
3062 will not be given in the case of multi-line input.
3063 */
3064 if (interaction > nonstop_mode) {
3065 if (n < 0) {
3067 } else {
3068 wake_up_terminal();
3069 print_ln();
3070 sprint_cs(r);
3072 n = -1;
3073 }
3074 } else {
3075 fatal_error
3077 }
3079 } else if (read_open[m] == just_open) {
3080 /*
3081 Input the first line of |read_file[m]|
3083 The first line of a file must be treated specially, since |lua_input_ln|
3084 must be told not to start with |get|.
3085 */
3086 if (lua_input_ln(read_file[m], (m + 1), false)) {
3087 read_open[m] = normal;
3088 } else {
3089 lua_a_close_in(read_file[m], (m + 1));
3090 read_open[m] = closed;
3091 }
3093 } else {
3094 /*
3095 Input the next line of |read_file[m]|
3097 An empty line is appended at the end of a |read_file|.
3098 */
3099 if (!lua_input_ln(read_file[m], (m + 1), true)) {
3100 lua_a_close_in(read_file[m], (m + 1));
3101 read_open[m] = closed;
3102 if (align_state != 1000000) {
3103 runaway();
3106 align_state = 1000000;
3107 error();
3108 }
3109 }
3111 }
3112 ilimit = last;
3113 if (end_line_char_inactive)
3114 decr(ilimit);
3115 else
3116 buffer[ilimit] = (packed_ASCII_code) int_par(end_line_char_code);
3117 first = ilimit + 1;
3118 iloc = istart;
3119 istate = new_line;
3120 /* Handle \.{\\readline} and |goto done|; */
3121 if (j == 1) {
3122 while (iloc <= ilimit) {
3123 /* current line not yet finished */
3124 do_buffer_to_unichar(cur_chr, iloc);
3125 if (cur_chr == ' ')
3126 cur_tok = space_token;
3127 else
3128 cur_tok = cur_chr + other_token;
3129 store_new_token(cur_tok);
3130 }
3131 } else {
3132 while (1) {
3133 get_token();
3134 if (cur_tok == 0) {
3135 /* |cur_cmd=cur_chr=0| will occur at the end of the line */
3136 break;
3137 }
3138 if (align_state < 1000000) {
3139 /* unmatched `\.\}' aborts the line */
3140 do {
3141 get_token();
3142 } while (cur_tok != 0);
3143 align_state = 1000000;
3144 break;
3145 }
3146 store_new_token(cur_tok);
3147 }
3148 }
3149 end_file_reading();
3151 } while (align_state != 1000000);
3152 cur_val = def_ref;
3153 scanner_status = normal;
3154 align_state = s;
3155 }
3157 @ return a string from tokens list
3159 @c
3160 str_number tokens_to_string(halfword p)
3161 {
3162 int old_setting;
3163 if (selector == new_string)
3165 old_setting = selector;
3166 selector = new_string;
3167 show_token_list(token_link(p), null, -1);
3168 selector = old_setting;
3169 return make_string();
3170 }
3172 @ @c
3173 #define make_room(a) \
3174 if ((unsigned)i+a+1>alloci) { \
3175 ret = xrealloc(ret,(alloci+64)); \
3176 alloci = alloci + 64; \
3177 }
3179 #define append_i_byte(a) ret[i++] = (char)(a)
3181 #define Print_char(a) make_room(1); append_i_byte(a)
3183 #define Print_uchar(s) { \
3184 make_room(4); \
3185 if (s<=0x7F) { \
3186 append_i_byte(s); \
3187 } else if (s<=0x7FF) { \
3188 append_i_byte(0xC0 + (s / 0x40)); \
3189 append_i_byte(0x80 + (s % 0x40)); \
3190 } else if (s<=0xFFFF) { \
3191 append_i_byte(0xE0 + (s / 0x1000)); \
3192 append_i_byte(0x80 + ((s % 0x1000) / 0x40)); \
3193 append_i_byte(0x80 + ((s % 0x1000) % 0x40)); \
3194 } else if (s>=0x110000) { \
3195 append_i_byte(s-0x11000); \
3196 } else { \
3197 append_i_byte(0xF0 + (s / 0x40000)); \
3198 append_i_byte(0x80 + ((s % 0x40000) / 0x1000)); \
3199 append_i_byte(0x80 + (((s % 0x40000) % 0x1000) / 0x40)); \
3200 append_i_byte(0x80 + (((s % 0x40000) % 0x1000) % 0x40)); \
3201 } }
3203 #define Print_esc(b) { \
3204 const char *v = b; \
3205 if (e>0 && e<STRING_OFFSET) { \
3206 Print_uchar (e); \
3207 } \
3208 make_room(strlen(v)); \
3209 while (*v) { append_i_byte(*v); v++; } \
3210 }
3212 #define Print_str(b) { \
3213 const char *v = b; \
3214 make_room(strlen(v)); \
3215 while (*v) { append_i_byte(*v); v++; } \
3216 }
3218 #define is_cat_letter(a) \
3219 (get_char_cat_code(pool_to_unichar(str_string((a)))) == 11)
3221 @ the actual token conversion in this function is now functionally equivalent to
3222 |show_token_list|, except that it always prints the whole token list. TODO: check
3223 whether this causes problems in the lua library.
3225 @c
3226 char *tokenlist_to_cstring(int pp, int inhibit_par, int *siz)
3227 {
3228 register int p, c, m;
3229 int q;
3230 int infop;
3231 char *s, *sh;
3232 int e = 0;
3233 char *ret;
3234 int match_chr = '#';
3235 int n = '0';
3236 unsigned alloci = 1024;
3237 int i = 0;
3238 p = pp;
3239 if (p == null) {
3240 if (siz != NULL)
3241 *siz = 0;
3242 return NULL;
3243 }
3244 ret = xmalloc(alloci);
3245 p = token_link(p); /* skip refcount */
3246 if (p != null) {
3247 e = int_par(escape_char_code);
3248 }
3249 while (p != null) {
3250 if (p < (int) fix_mem_min || p > (int) fix_mem_end) {
3252 break;
3253 }
3254 infop = token_info(p);
3255 if (infop >= cs_token_flag) {
3256 if (!(inhibit_par && infop == par_token)) {
3257 q = infop - cs_token_flag;
3258 if (q < hash_base) {
3259 if (q == null_cs) {
3262 } else {
3264 }
3265 } else if ((q >= undefined_control_sequence) && ((q <= eqtb_size) || (q > eqtb_size + hash_extra))) {
3267 } else if ((cs_text(q) < 0) || (cs_text(q) >= str_ptr)) {
3269 } else {
3270 str_number txt = cs_text(q);
3271 sh = makecstring(txt);
3272 s = sh;
3273 if (is_active_cs(txt)) {
3274 s = s + 3;
3275 while (*s) {
3276 Print_char(*s);
3277 s++;
3278 }
3279 } else {
3280 if (e>=0 && e<0x110000) Print_uchar(e);
3281 while (*s) {
3282 Print_char(*s);
3283 s++;
3284 }
3285 if ((!single_letter(txt)) || is_cat_letter(txt)) {
3286 Print_char(' ');
3287 }
3288 }
3289 free(sh);
3290 }
3291 }
3292 } else {
3293 if (infop < 0) {
3295 } else {
3296 m = token_cmd(infop);
3297 c = token_chr(infop);
3298 switch (m) {
3299 case left_brace_cmd:
3300 case right_brace_cmd:
3301 case math_shift_cmd:
3302 case tab_mark_cmd:
3303 case sup_mark_cmd:
3304 case sub_mark_cmd:
3305 case spacer_cmd:
3306 case letter_cmd:
3307 case other_char_cmd:
3308 Print_uchar(c);
3309 break;
3310 case mac_param_cmd:
3311 if (!in_lua_escape && (is_in_csname==0))
3312 Print_uchar(c);
3313 Print_uchar(c);
3314 break;
3315 case out_param_cmd:
3316 Print_uchar(match_chr);
3317 if (c <= 9) {
3318 Print_char(c + '0');
3319 } else {
3320 Print_char('!');
3321 goto EXIT;
3322 }
3323 break;
3324 case match_cmd:
3325 match_chr = c;
3326 Print_uchar(c);
3327 n++;
3328 Print_char(n);
3329 if (n > '9')
3330 goto EXIT;
3331 break;
3332 case end_match_cmd:
3333 if (c == 0) {
3334 Print_char('-');
3335 Print_char('>');
3336 }
3337 break;
3338 default:
3339 not_so_bad(Print_esc);
3340 break;
3341 }
3342 }
3343 }
3344 p = token_link(p);
3345 }
3346 EXIT:
3347 ret[i] = '\0';
3348 if (siz != NULL)
3349 *siz = i;
3350 return ret;
3351 }
3353 @ @c
3354 lstring *tokenlist_to_lstring(int pp, int inhibit_par)
3355 {
3356 int siz;
3357 lstring *ret = xmalloc(sizeof(lstring));
3358 ret->s = (unsigned char *) tokenlist_to_cstring(pp, inhibit_par, &siz);
3359 ret->l = (size_t) siz;
3360 return ret;
3361 }
3363 @ @c
3364 void free_lstring(lstring * ls)
3365 {
3366 if (ls == NULL)
3367 return;
3368 if (ls->s != NULL)
3369 free(ls->s);
3370 free(ls);
3371 }