| blob | 196ecf8c39924ec429b7506bcf3bef8cf30f2f34 |
1 % textoken.w
2 %
4 %
5 % This file is part of LuaTeX.
6 %
8 % the terms of the GNU General Public License as published by the Free
11 %
15 % License for more details.
16 %
17 % You should have received a copy of the GNU General Public License along
20 @ @c
22 #include "ptexlib.h"
24 @ @c
27 /*
33 else \
36 */
43 else \
48 @ The \TeX\ system does nearly all of its own memory allocation, so that it can
49 readily be transported into environments that do not have automatic facilities
51 error messages the user receives. The dynamic storage requirements of \TeX\ are
52 handled by providing two large arrays called |fixmem| and |varmem| in which
53 consecutive blocks of words are used as nodes by the \TeX\ routines.
55 Pointer variables are indices into this array, or into another array called
56 |eqtb| that will be explained later. A pointer variable might also be a special
57 flag that lies outside the bounds of |mem|, so we allow pointers to assume any
58 |halfword| value. The minimum halfword value represents a null pointer. \TeX\
61 @ Locations in |fixmem| are used for storing one-word records; a conventional
64 @c
69 @ In order to study the memory requirements of particular applications, it is
70 possible to prepare a version of \TeX\ that keeps track of current and maximum
71 memory usage. When code between the delimiters |@!stat| $\ldots$ |tats| is not
72 commented out, \TeX\ will run a bit slower but it will report these statistics
73 when |tracing_stats| is sufficiently large.
75 @c
88 @ @c
90 {
91 halfword p;
113 }
115 @ The function |get_avail| returns a pointer to a new one-word node whose |link|
116 field is null. However, \TeX\ will halt if there is no more room left.
117 @^inner loop@>
122 work, we have to quit.
124 @c
128 unsigned t;
138 new_fixmem =
139 fixmemcast(realloc
146 }
150 }
154 }
157 that starts at position |p|.
158 @^inner loop@>
160 @c
166 do {
173 }
174 }
178 number is |c| and whose command code is |m| is represented as the number
180 whose |eqtb| address is |p| is represented as the number |cs_token_flag+p|. Here
183 halfword.
185 A token |t| represents a |left_brace| command if and only if
189 following definitions take care of these token-oriented constants and a few
190 others.
192 @ A token list is a singly linked list of one-word nodes in |mem|, where each
195 form of token lists, usually preceded by a node with a reference count in its
198 Three special commands appear in the token lists of macro definitions. When
202 insert parameter number |c| into the text at this point.
205 are omitted, but the final right brace of an output routine is included at the
206 end of its token list.
208 Here is an example macro definition that illustrates these conventions. After
209 \TeX\ processes the text
223 The procedure |scan_toks| builds such token lists, and |macro_call| does the
224 parameter matching. @^reference counts@>
229 entry before the token list has been fully consumed, so we dare not simply
230 destroy a token list when its control sequence is being redefined.
233 corresponding token list will have `\.\{' just before the `|end_match|' and also
235 one keeps the first from disappearing.
237 The |print_meaning| subroutine displays |cur_cmd| and |cur_chr| in symbolic form,
238 including the expansion of a macro or mark.
240 @c
242 {
244 /*
247 }
248 */
252 print_ln();
258 print_ln();
260 case first_mark_code:
262 break;
263 case bot_mark_code:
265 break;
266 case split_first_mark_code:
268 break;
269 case split_bot_mark_code:
271 break;
272 default:
274 break;
275 }
276 }
277 }
278 }
280 @ The procedure |show_token_list|, which prints a symbolic form of the token list
281 that starts at a given node |p|, illustrates these conventions. The token list
282 being displayed should not begin with a reference count. However, the procedure
283 is intended to be robust, so that if the memory links are awry or if |p| is not
284 really a pointer to a token list, nothing catastrophic will happen.
286 An additional parameter |q| is also given; this parameter is either null or it
287 points to a node in the token list where a certain magic computation takes place
289 the two-line context information at the time of an error message; |q| marks the
290 place corresponding to where the second line should begin.)
292 For example, if |p| points to the node containing the first \.a in the token list
295 second \.a, the magic computation will be performed just before the second \.a is
296 printed.
299 printing exceeds a given limit~|l|. Anomalous entries are printed in the form of
301 this cannot be confused with actual control sequences because a real control
304 @c
307 case assign_int_cmd: \
310 break; \
311 case assign_dimen_cmd: \
314 break; \
315 case assign_toks_cmd: \
318 break; \
319 default: \
321 break; \
322 }
325 {
335 set_trick_count();
336 }
340 return;
341 }
351 /*
354 The procedure usually ``learns'' the character code used for macro
355 parameters by seeing one in a |match| command before it runs into any
356 |out_param| commands.
357 */
359 case left_brace_cmd:
360 case right_brace_cmd:
361 case math_shift_cmd:
362 case tab_mark_cmd:
363 case sup_mark_cmd:
364 case sub_mark_cmd:
365 case spacer_cmd:
366 case letter_cmd:
367 case other_char_cmd:
369 break;
370 case mac_param_cmd:
374 break;
375 case out_param_cmd:
381 return;
382 }
383 break;
384 case match_cmd:
390 return;
391 break;
392 case end_match_cmd:
395 break;
396 default:
398 break;
399 }
400 }
401 }
403 }
406 }
408 @ @c
414 @ Here's the way we sometimes want to display a token list, given a pointer to
415 its reference count; the pointer may be null.
417 @c
419 {
422 }
424 @ |delete_token_ref|, is called when a pointer to a token list's reference count
425 is being removed. This means that the token list should disappear if the
426 reference count was |null|, otherwise the count should be decreased by one.
427 @^reference counts@>
429 @ |p| points to the reference count of a token list that is losing one
430 reference.
432 @c
434 {
437 else
439 }
441 @ @c
443 {
444 int a;
446 return a;
447 }
449 @ @c
451 {
455 NULL
456 };
460 }
462 @ @c
467 typedef enum {
468 next_line_ok,
469 next_line_return,
470 next_line_restart
477 |scan_keyword| routine checks to see whether the next tokens of input match this
478 string. The match must be exact, except that uppercase letters will match their
479 lowercase counterparts; uppercase equivalents are determined by subtracting
481 only for its own limited set of keywords.
483 If a match is found, the characters are effectively removed from the input and
484 |true| is returned. Otherwise |false| is returned, and the input is left
485 essentially unchanged (except for the fact that some macros may have been
488 @c
504 k++;
506 /*
507 crashes on some alignments:
516 back_input();
517 }
518 */
519 back_input();
522 }
523 /* */
525 return false;
526 }
527 }
531 return true;
532 }
534 @ We can not return |undefined_control_sequence| under some conditions
537 @c
539 /*
541 {
542 halfword curcs;
558 }
562 return curcs;
563 }
564 */
569 {
570 halfword curcs;
574 }
585 }
587 return curcs;
588 }
590 /*
593 {
619 }
621 return buf;
622 }
625 {
626 halfword curcs;
630 }
637 }
639 return curcs;
640 }
642 */
646 @ prints a control sequence
648 @c
650 {
660 }
665 }
676 }
682 }
684 }
686 }
688 }
690 @ TODO this is a quick hack, will be solved differently soon
692 @c
694 {
696 str_number str;
704 return s;
705 }
707 @ The heart of \TeX's input mechanism is the |get_next| procedure, which we shall
708 develop in the next few sections of the program. Perhaps we shouldn't actually
710 reading the source files and gobbling them up. And it also helps \TeX\ to
711 regurgitate stored token lists that are to be processed again. @^eyes and mouth@>
713 The main duty of |get_next| is to input one token and to set |cur_cmd| and
714 |cur_chr| to that token's command code and modifier. Furthermore, if the input
715 token is a control sequence, the |eqtb| location of that control sequence is
716 stored in |cur_cs|; otherwise |cur_cs| is set to zero.
718 Underlying this simple description is a certain amount of complexity because of
719 all the cases that need to be handled. However, the inner loop of |get_next| is
720 reasonably short and fast.
725 |end_line_char| has |ignore| as its catcode.)
728 needed because a blank line of input is supposed to be exactly equivalent to the
730 line.
732 @c
736 @ Parts |get_next| are executed more often than any other instructions of \TeX.
739 The global variable |force_eof| is normally |false|; it is set |true| by an
741 waiting to be input, it is changed by |luatokencall|.
743 @c
747 @ If the user has set the |pausing| parameter to some positive value, and if
748 nonstop mode has not been selected, each line of input is displayed on the
750 response. If the response is simply |carriage_return|, the line is accepted as it
751 stands, otherwise the line typed is used instead of the line in the file.
753 @c
755 {
760 wake_up_terminal();
761 print_ln();
765 }
772 }
773 }
774 }
775 }
777 @ Before getting into |get_next|, let's consider the subroutine that is called
779 has been reached. These two cases are distinguished by |cur_cs|, which is zero at
780 the end of a file.
782 @c
784 {
788 return;
799 }
802 }
809 NULL
810 };
821 }
823 and insert tokens that should lead to recovery; */
824 /* The recovery procedure can't be fully understood without knowing more
825 about the \TeX\ routines that should be aborted, but we can sketch the
832 case defining:
835 break;
836 case matching:
840 break;
841 case aligning:
849 break;
850 case absorbing:
853 break;
856 break;
868 NULL
869 };
874 NULL
875 };
881 }
889 {
891 back_input();
895 }
896 }
898 }
899 }
901 @ @c
903 #if 0
905 /*
906 The other variant gives less clutter in tracing cache usage when profiling and for
908 */
911 {
912 SWITCH:
917 RESWITCH:
922 }
923 /*
924 Change state if necessary, and |goto switch| if the current
925 character should be ignored, or |goto reswitch| if the current
926 character changes to another;
929 that a decent C compiler has translated the code. Note that the numeric
932 command code to the state to get a single number that characterizes both.
935 cause of many branch prediction errors but changing it to:
939 return true;
942 }
945 }
948 }
951 return true;
952 }
954 gives as many prediction errors. So, we can indeed assume that the compiler
955 does the right job, or that there is simply no other way.
956 */
965 goto SWITCH;
966 break;
973 check_outer_validity();
974 break;
984 check_outer_validity();
985 break;
991 goto RESWITCH;
992 else
994 break;
999 invalid_character_error();
1001 break;
1006 break;
1008 /*
1012 treated alike when macro parameters are being matched. We do this since such
1013 characters are indistinguishable on most computer terminal displays.
1014 */
1018 break;
1025 goto SWITCH;
1026 break;
1034 check_outer_validity();
1035 break;
1041 align_state++;
1042 break;
1048 align_state--;
1049 break;
1056 break;
1057 /*
1070 */
1071 default:
1073 break;
1074 }
1078 /*
1079 Move to next line of file,
1080 or |goto restart| if there is no next line,
1082 */
1083 do {
1086 return true;
1088 return false;
1089 }
1091 check_interrupt();
1092 goto SWITCH;
1093 }
1094 return true;
1095 }
1097 #else
1101 /*
1102 escape_cmd left_brace_cmd right_brace_cmd math_shift_cmd
1103 tab_mark_cmd car_ret_cmd mac_param_cmd sup_mark_cmd
1104 sub_mark_cmd ignore_cmd spacer_cmd letter_cmd
1105 other_char_cmd active_char_cmd comment_cmd invalid_char_cmd
1106 */
1109 {
1111 SWITCH:
1115 RESWITCH:
1120 }
1121 /*
1122 Change state if necessary, and |goto switch| if the current
1123 character should be ignored, or |goto reswitch| if the current
1124 character changes to another;
1125 */
1128 return true;
1131 case escape_cmd:
1134 check_outer_validity();
1135 return true;
1136 case left_brace_cmd:
1138 align_state++;
1139 return true;
1140 case right_brace_cmd:
1142 align_state--;
1143 return true;
1144 case math_shift_cmd:
1146 return true;
1147 case tab_mark_cmd:
1149 return true;
1150 case car_ret_cmd:
1157 check_outer_validity();
1158 return true;
1159 case mac_param_cmd:
1161 return true;
1162 case sup_mark_cmd:
1164 goto RESWITCH;
1165 else
1167 return true;
1168 case sub_mark_cmd:
1170 return true;
1171 case ignore_cmd:
1172 goto SWITCH;
1173 return true;
1174 case spacer_cmd:
1176 goto SWITCH;
1177 case letter_cmd:
1179 return true;
1180 case other_char_cmd:
1182 return true;
1183 case active_char_cmd:
1189 check_outer_validity();
1190 return true;
1191 case comment_cmd:
1193 goto SWITCH;
1194 case invalid_char_cmd:
1195 invalid_character_error();
1197 default:
1199 return true;
1200 }
1203 case escape_cmd:
1207 check_outer_validity();
1208 return true;
1209 case left_brace_cmd:
1211 align_state++;
1212 return true;
1213 case right_brace_cmd:
1215 align_state--;
1216 return true;
1217 case math_shift_cmd:
1219 return true;
1220 case tab_mark_cmd:
1222 return true;
1223 case car_ret_cmd:
1225 goto SWITCH;
1226 case mac_param_cmd:
1228 return true;
1229 case sup_mark_cmd:
1232 goto RESWITCH;
1233 else
1235 return true;
1236 case sub_mark_cmd:
1238 return true;
1239 case ignore_cmd:
1240 goto SWITCH;
1241 case spacer_cmd:
1242 goto SWITCH;
1243 case letter_cmd:
1245 return true;
1246 case other_char_cmd:
1248 return true;
1249 case active_char_cmd:
1255 check_outer_validity();
1256 return true;
1257 case comment_cmd:
1260 goto SWITCH;
1261 case invalid_char_cmd:
1263 invalid_character_error();
1265 default:
1267 return true;
1268 }
1271 case escape_cmd:
1274 check_outer_validity();
1275 return true;
1276 case left_brace_cmd:
1277 align_state++;
1278 return true;
1279 case right_brace_cmd:
1280 align_state--;
1281 return true;
1282 case math_shift_cmd:
1283 return true;
1284 case tab_mark_cmd:
1285 return true;
1286 case car_ret_cmd:
1287 /*
1291 treated alike when macro parameters are being matched. We do this since such
1292 characters are indistinguishable on most computer terminal displays.
1293 */
1297 return true;
1298 case mac_param_cmd:
1299 return true;
1300 case sup_mark_cmd:
1302 goto RESWITCH;
1303 else
1305 return true;
1306 case sub_mark_cmd:
1307 return true;
1308 case ignore_cmd:
1309 goto SWITCH;
1310 case spacer_cmd:
1314 return true;
1315 case letter_cmd:
1317 return true;
1318 case other_char_cmd:
1320 return true;
1321 case active_char_cmd:
1327 check_outer_validity();
1328 return true;
1329 case comment_cmd:
1331 goto SWITCH;
1332 case invalid_char_cmd:
1333 invalid_character_error();
1335 default:
1337 return true;
1338 }
1341 return true;
1342 }
1346 }
1347 /*
1350 */
1351 do {
1354 return true;
1356 return false;
1357 }
1359 check_interrupt();
1360 goto SWITCH;
1361 }
1362 return true;
1363 }
1365 #endif
1368 We only support a limited set:
1370 ^^^^^^XXXXXX
1371 ^^^^XXXXXX
1374 @c
1383 }
1390 }
1410 {
1427 return true;
1430 }
1433 }
1444 return true;
1447 }
1450 }
1451 }
1460 return true;
1461 }
1462 }
1464 }
1465 }
1467 {
1476 return true;
1477 }
1478 }
1480 return true;
1481 }
1482 }
1483 }
1484 return false;
1485 }
1487 @ Control sequence names are scanned only when they appear in some line of a
1489 a unique identification, so \TeX\ doesn't need to refer to the original name any
1490 more except when it prints the equivalent in symbolic form.
1492 The program that scans a control sequence has been written carefully in order to
1493 avoid the blowups that might otherwise occur if a malicious user tried something
1498 following a control sequence name, they are converted to single characters in the
1499 buffer and the process is repeated, slowly but surely.
1501 @c
1505 {
1518 continue;
1521 do {
1527 continue;
1530 /*
1538 */
1547 }
1549 }
1550 }
1553 break;
1554 }
1555 }
1558 return retval;
1559 }
1561 @ Whenever we reach the following piece of code, we will have
1566 left two or three places.
1568 @c
1570 {
1571 int l;
1589 }
1592 }
1604 }
1607 }
1608 }
1620 }
1623 }
1624 }
1625 }
1628 else
1629 d++;
1634 k++;
1635 d--;
1639 k++;
1640 d--;
1642 k++;
1643 d--;
1647 k++;
1648 d--;
1650 k++;
1651 d--;
1653 k++;
1654 d--;
1656 }
1661 l++;
1662 }
1664 return true;
1665 }
1666 return false;
1667 }
1669 @ All of the easy branches of |get_next| have now been taken care of. There is
1670 one more branch.
1673 {
1691 return next_line_restart;
1694 }
1698 firm_up_the_line();
1708 }
1717 return next_line_restart;
1720 }
1721 }
1722 }
1723 }
1729 file_warning();
1730 }
1734 }
1738 end_file_reading();
1740 end_file_reading();
1742 check_outer_validity();
1743 }
1744 return next_line_restart;
1745 }
1747 ilimit--;
1748 else
1758 }
1761 end_file_reading();
1763 }
1765 open_log_file();
1768 ilimit++;
1772 }
1773 print_ln();
1778 ilimit--;
1779 else
1784 /*
1786 never waits for on-line input.
1787 */
1789 }
1790 }
1791 return next_line_ok;
1792 }
1794 @ Let's consider now what happens when |get_next| is looking at a token list.
1796 @c
1798 {
1807 /*
1808 Get the next token, suppressing expansion. The present point in the program
1809 is reached only when the |expand| routine has inserted a special marker into
1812 */
1819 return true;
1820 }
1822 check_outer_validity();
1823 }
1824 }
1830 case left_brace_cmd:
1831 align_state++;
1832 break;
1833 case right_brace_cmd:
1834 align_state--;
1835 break;
1836 case out_param_cmd:
1839 return false;
1840 break;
1841 }
1842 }
1843 return true;
1844 }
1846 @ Now we're ready to take the plunge into |get_next| itself. Parts of this
1847 routine are executed more often than any other instructions of \TeX.
1852 @c
1854 {
1855 RESTART:
1860 goto RESTART;
1863 end_token_list();
1867 }
1868 }
1871 insert_vj_template();
1872 goto RESTART;
1873 }
1874 }
1876 @ Since |get_next| is used so frequently in \TeX, it is convenient to define
1877 three related procedures that do a little more:
1879 \yskip\hang|get_token| not only sets |cur_cmd| and |cur_chr|, it also sets
1880 |cur_tok|, a packed halfword version of the current token.
1886 the expansion of the macro or the evaluation of the conditional.
1888 \yskip\hang|x_token| is like |get_x_token| except that it assumes that |get_next|
1889 has already been called.
1891 \yskip\noindent In fact, these three procedures account for almost every use of
1892 |get_next|.
1894 No new control sequences will be defined except during a call of |get_token|, or
1896 always |true| at other times.
1900 @c
1902 {
1904 get_next();
1908 else
1910 }
1912 @ changes the string |s| to a token list
1914 @c
1916 {
1929 else
1932 }
1934 }
1939 Before we get into the details of |scan_toks|, let's consider a much simpler
1940 task, that of converting the current string into a token list. The |str_toks|
1941 function does this; it classifies spaces as type |spacer| and everything else as
1942 type |other_char|.
1947 |lua_str_toks| is almost identical, but it also escapes the three symbols that
1948 |lua| considers special while scanning a literal string
1952 @c
1954 {
1969 fast_store_new_token(other_token + '\\');
1970 if (t == 10)
1971 t = 'n';
1972 if (t == 13)
1973 t = 'r';
1974 t = other_token + t;
1975 }
1976 fast_store_new_token(t);
1977 }
1978 return p;
1979 }
1981 @ Incidentally, the main reason for wanting |str_toks| is the function
1982 |the_toks|, which has similar input/output characteristics.
1984 @ changes the string |str_pool[b..pool_ptr]| to a token list
1986 @c
1987 halfword str_toks(lstring s)
1988 {
1989 halfword p; /* tail of the token list */
1990 halfword q; /* new node being added to the token list via |store_new_token| */
1991 halfword t; /* token being appended */
1992 unsigned char *k, *l; /* index into string */
1993 p = temp_token_head;
1994 set_token_link(p, null);
1995 k = s.s;
1996 l = k + s.l;
1997 while (k < l) {
1998 t = pool_to_unichar(k);
1999 k += utf8_size(t);
2000 if (t == ' ')
2001 t = space_token;
2002 else
2003 t = other_token + t;
2004 fast_store_new_token(t);
2005 }
2006 return p;
2007 }
2009 /*
2010 hh: most of the converter is similar to the one i made for macro so at some point i
2011 can make a helper; also todo: there is no need to go through the pool
2013 */
2015 halfword str_scan_toks(int ct, lstring s)
2016 { /* changes the string |str_pool[b..pool_ptr]| to a token list */
2017 halfword p; /* tail of the token list */
2018 halfword q; /* new node being added to the token list via |store_new_token| */
2019 halfword t; /* token being appended */
2020 unsigned char *k, *l; /* index into string */
2021 int cc;
2022 p = temp_token_head;
2023 set_token_link(p, null);
2024 k = s.s;
2025 l = k + s.l;
2026 while (k < l) {
2027 t = pool_to_unichar(k);
2028 k += utf8_size(t);
2029 cc = get_cat_code(ct,t);
2030 if (cc == 0) {
2031 /* we have a potential control sequence so we check for it */
2032 int _lname = 0 ;
2033 int _s = 0 ;
2034 int _c = 0 ;
2035 halfword _cs = null ;
2036 unsigned char *_name = k ;
2037 while (k < l) {
2038 t = (halfword) str2uni((const unsigned char *) k);
2039 _s = utf8_size(t);
2040 _c = get_cat_code(ct,t);
2041 if (_c == 11) {
2042 k += _s ;
2043 _lname = _lname + _s ;
2044 } else if (_c == 10) {
2045 /* we ignore a trailing space like normal scanning does */
2046 k += _s ;
2047 break ;
2048 } else {
2049 break ;
2050 }
2051 }
2052 if (_s > 0) {
2053 /* we have a potential \cs */
2054 _cs = string_lookup((const char *) _name, _lname);
2055 if (_cs == undefined_control_sequence) {
2056 /* let's play safe and backtrack */
2057 t = cc * (1<<21) + t ;
2058 k = _name ;
2059 } else {
2060 t = cs_token_flag + _cs;
2061 }
2062 } else {
2063 /* just a character with some meaning, so \unknown becomes effectively */
2064 /* \\unknown assuming that \\ has some useful meaning of course */
2065 t = cc * (1<<21) + t ;
2066 k = _name ;
2067 }
2069 } else {
2070 /* whatever token, so for instance $x^2$ just works given a tex */
2071 /* catcode regime */
2072 t = cc * (1<<21) + t ;
2073 }
2074 fast_store_new_token(t);
2076 }
2077 return p;
2078 }
2080 @ Here's part of the |expand| subroutine that we are now ready to complete:
2082 @c
2083 void ins_the_toks(void)
2084 {
2085 (void) the_toks();
2086 ins_list(token_link(temp_token_head));
2087 }
2089 #define set_toks_register(n,t,g) { \
2090 int a = (g>0) ? 4 : 0; \
2091 halfword ref = get_avail(); \
2092 set_token_ref_count(ref, 0); \
2093 set_token_link(ref, token_link(t)); \
2094 define(n + toks_base, call_cmd, ref); \
2095 }
2097 void combine_the_toks(int how)
2098 {
2099 halfword nt;
2100 get_x_token();
2101 /* target */
2102 if (cur_cmd == assign_toks_cmd) {
2103 nt = equiv(cur_cs) - toks_base;
2104 /* check range */
2105 } else {
2106 back_input();
2107 scan_int();
2108 nt = cur_val;
2109 }
2110 /* source */
2111 do {
2112 get_x_token();
2113 } while (cur_cmd == spacer_cmd);
2114 if (cur_cmd == left_brace_cmd) {
2115 halfword x, source;
2116 back_input();
2117 x = scan_toks(false,how > 1); /* expanded or not */
2118 source = def_ref;
2119 /* action */
2120 if (source != null) {
2121 halfword target = toks(nt);
2122 if (target == null) {
2123 set_toks_register(nt,source,0);
2124 } else {
2125 halfword s = token_link(source);
2126 if (s != null) {
2127 halfword t = token_link(target);
2128 if (t == null) {
2129 /* can this happen ? */
2130 set_token_link(target, s);
2131 } else if (odd(how)) {
2132 /* prepend */
2133 if (cur_level != eq_level_field(eqtb[toks_base+nt])) {
2134 halfword p = temp_token_head;
2135 halfword q;
2136 set_token_link(p, s); /* s = head, x = tail */
2137 p = x;
2138 while (t != null) {
2139 fast_store_new_token(token_info(t));
2140 t = token_link(t);
2141 }
2142 set_toks_register(nt,temp_token_head,0);
2143 } else {
2144 set_token_link(x,t);
2145 set_token_link(target,s);
2146 }
2147 } else {
2148 /* append */
2149 if (cur_level != eq_level_field(eqtb[toks_base+nt])) {
2150 halfword p = temp_token_head;
2151 halfword q;
2152 set_token_link(p, null);
2153 while (t != null) {
2154 fast_store_new_token(token_info(t));
2155 t = token_link(t);
2156 }
2157 set_token_link(p,s);
2158 set_toks_register(nt,temp_token_head,0);
2159 } else {
2160 while (token_link(t) != null) {
2161 t = token_link(t);
2162 }
2163 set_token_link(t,s);
2164 }
2165 }
2166 }
2167 }
2168 }
2169 } else {
2170 halfword source, ns;
2171 if (cur_cmd == assign_toks_cmd) {
2172 ns = equiv(cur_cs) - toks_base;
2173 /* check range */
2174 } else {
2175 back_input();
2176 scan_int();
2177 ns = cur_val;
2178 }
2179 /* action */
2180 source = toks(ns);
2181 if (source != null) {
2182 halfword target = toks(nt);
2183 if (target == null) {
2184 equiv(toks_base+nt) = source;
2185 equiv(toks_base+ns) = null;
2186 } else {
2187 halfword s = token_link(source);
2188 if (s != null) {
2189 halfword t = token_link(target);
2190 if (t == null) {
2191 set_token_link(target, s);
2192 } else if (odd(how)) {
2193 /* prepend */
2194 halfword x = s;
2195 while (token_link(x) != null) {
2196 x = token_link(x);
2197 }
2198 set_token_link(x,t);
2199 set_token_link(target,s);
2200 } else {
2201 /* append */
2202 while (token_link(t) != null) {
2203 t = token_link(t);
2204 }
2205 set_token_link(t,s);
2206 }
2207 equiv(toks_base+ns) = null;
2208 }
2209 }
2210 }
2211 }
2212 }
2214 @ This routine, used in the next one, prints the job name, possibly modified by
2215 the |process_jobname| callback.
2217 @c
2218 static void print_job_name(void)
2219 {
2220 if (job_name) {
2221 char *s, *ss; /* C strings for jobname before and after processing */
2222 int callback_id, lua_retval;
2223 s = (char*)str_string(job_name);
2224 callback_id = callback_defined(process_jobname_callback);
2225 if (callback_id > 0) {
2227 if ((lua_retval == true) && (ss != NULL))
2228 s = ss;
2229 }
2230 tprint(s);
2231 } else {
2232 print(job_name);
2233 }
2234 }
2236 @ Here is a routine that print the result of a convert command, using the
2237 argument |i|. It returns |false | if it does not know to print the code |c|. The
2238 function exists because lua code and tex code can both call it to convert
2239 something.
2241 @ Parse optional lua state integer, or an instance name to be stored in |sn| and
2242 get the next non-blank non-relax non-call token.
2244 @c
2246 int scan_lua_state(void)
2247 {
2248 int sn = 0;
2249 do {
2250 get_x_token();
2251 } while ((cur_cmd == spacer_cmd) || (cur_cmd == relax_cmd));
2252 back_input();
2253 if (cur_cmd != left_brace_cmd) {
2255 (void) scan_toks(false, true);
2256 sn = def_ref;
2257 } else {
2258 scan_register_num();
2259 if (get_lua_name(cur_val))
2260 sn = (cur_val - 65536);
2261 }
2262 }
2263 return sn;
2264 }
2266 @ The procedure |conv_toks| uses |str_toks| to insert the token list for
2267 |convert| functions into the scanner; `\.{\\outer}' control sequences are allowed
2268 to follow `\.{\\string}' and `\.{\\meaning}'.
2270 The extra temp string |u| is needed because |pdf_scan_ext_toks| incorporates any
2271 pending string in its output. In order to save such a pending string, we have to
2272 create a temporary string that is destroyed immediately after.
2274 @c
2275 #define push_selector { \
2276 old_setting = selector; \
2277 selector = new_string; \
2278 }
2280 #define pop_selector { \
2281 selector = old_setting; \
2282 }
2284 static int do_variable_dvi(halfword c)
2285 {
2286 return 0;
2287 }
2289 #define do_variable_backend_int(i) \
2290 cur_cmd = assign_int_cmd; \
2291 cur_val = backend_int_base + i; \
2292 cur_tok = token_val(cur_cmd, cur_val); \
2293 back_input();
2295 #define do_variable_backend_dimen(i) \
2296 cur_cmd = assign_dimen_cmd; \
2297 cur_val = backend_dimen_base + i; \
2298 cur_tok = token_val(cur_cmd, cur_val); \
2299 back_input();
2301 #define do_variable_backend_toks(i) \
2302 cur_cmd = assign_toks_cmd; \
2303 cur_val = backend_toks_base + i ; \
2304 cur_tok = token_val(cur_cmd, cur_val); \
2305 back_input();
2307 static int do_variable_pdf(halfword c)
2308 {
2316 else if (scan_keyword("inclusionerrorlevel")) { do_variable_backend_int(c_pdf_inclusion_errorlevel); }
2317 else if (scan_keyword("ignoreunknownimages")) { do_variable_backend_int(c_pdf_ignore_unknown_images); }
2319 else if (scan_keyword("imageapplygamma")) { do_variable_backend_int(c_pdf_image_apply_gamma); }
2322 else if (scan_keyword("imageaddfilename")) { do_variable_backend_int(c_pdf_image_addfilename); }
2323 else if (scan_keyword("objcompresslevel")) { do_variable_backend_int(c_pdf_obj_compress_level); }
2324 else if (scan_keyword("inclusioncopyfonts")) { do_variable_backend_int(c_pdf_inclusion_copy_font); }
2327 else if (scan_keyword("suppressoptionalinfo")) { do_variable_backend_int(c_pdf_suppress_optional_info); }
2345 else
2346 return 0;
2347 return 1;
2348 }
2350 static int do_feedback_dvi(halfword c)
2351 {
2352 return 0;
2353 }
2355 /* codes not really needed but cleaner when testing */
2357 #define pdftex_version 140 /* these values will not change any more */
2360 static int do_feedback_pdf(halfword c)
2361 {
2362 int old_setting; /* holds |selector| setting */
2363 int save_scanner_status; /* |scanner_status| upon entry */
2364 halfword save_def_ref; /* |def_ref| upon entry, important if inside `\.{\\message}' */
2365 halfword save_warning_index;
2366 boolean bool; /* temp boolean */
2367 str_number s; /* first temp string */
2368 int ff; /* for use with |set_ff| */
2369 str_number u = 0; /* third temp string, will become non-nil if a string is already being built */
2370 char *str; /* color stack init str */
2373 push_selector;
2374 print_int(pdf_last_link);
2375 pop_selector;
2377 push_selector;
2378 print_int(pdf_retval);
2379 pop_selector;
2381 push_selector;
2382 print_int(pdf_last_obj);
2383 pop_selector;
2385 push_selector;
2386 print_int(pdf_last_annot);
2387 pop_selector;
2389 scan_int();
2390 check_obj_type(static_pdf, obj_type_xform, cur_val);
2391 push_selector;
2392 print_int(obj_info(static_pdf, cur_val));
2393 pop_selector;
2395 ins_list(string_to_toks(getcreationdate(static_pdf)));
2396 /* no further action */
2397 return 2;
2399 scan_font_ident();
2400 if (cur_val == null_font)
2402 pdf_check_vf(cur_val);
2403 if (!font_used(cur_val))
2404 pdf_init_font(static_pdf, cur_val);
2405 push_selector;
2406 set_ff(cur_val);
2407 print_int(obj_info(static_pdf, pdf_font_num(ff)));
2408 pop_selector;
2410 scan_font_ident();
2411 if (cur_val == null_font)
2413 pdf_check_vf(cur_val);
2414 if (!font_used(cur_val))
2415 pdf_init_font(static_pdf, cur_val);
2416 push_selector;
2417 set_ff(cur_val);
2418 print_int(pdf_font_num(ff));
2419 pop_selector;
2421 scan_font_ident();
2422 if (cur_val == null_font)
2424 push_selector;
2425 print_scaled(font_size(cur_val));
2427 pop_selector;
2429 scan_int();
2430 if (cur_val <= 0)
2432 push_selector;
2433 print_int(pdf_get_obj(static_pdf, obj_type_page, cur_val, false));
2434 pop_selector;
2438 cur_val = direct_always;
2440 cur_val = direct_page;
2442 cur_val = direct_raw;
2443 else
2444 cur_val = set_origin;
2445 save_scanner_status = scanner_status;
2446 save_warning_index = warning_index;
2447 save_def_ref = def_ref;
2448 u = save_cur_string();
2449 scan_toks(false, true);
2450 s = tokens_to_string(def_ref);
2451 delete_token_ref(def_ref);
2452 def_ref = save_def_ref;
2453 warning_index = save_warning_index;
2454 scanner_status = save_scanner_status;
2455 str = makecstring(s);
2456 cur_val = newcolorstack(str, cur_val, bool);
2457 free(str);
2458 flush_str(s);
2459 cur_val_level = int_val_level;
2460 if (cur_val < 0) {
2464 error();
2465 cur_val = 0;
2466 restore_cur_string(u);
2467 }
2468 push_selector;
2469 print_int(cur_val);
2470 pop_selector;
2472 push_selector;
2473 print_int(pdftex_version);
2474 pop_selector;
2476 ins_list(string_to_toks(pdftex_revision));
2477 return 2;
2478 } else {
2479 return 0;
2480 }
2481 return 1;
2482 }
2484 void conv_toks(void)
2485 {
2486 int old_setting; /* holds |selector| setting */
2487 halfword p, q;
2488 int save_scanner_status; /* |scanner_status| upon entry */
2489 halfword save_def_ref; /* |def_ref| upon entry, important if inside `\.{\\message}' */
2490 halfword save_warning_index;
2491 boolean bool; /* temp boolean */
2492 str_number s; /* first temp string */
2493 int sn; /* lua chunk name */
2494 str_number u = 0; /* third temp string, will become non-nil if a string is already being built */
2495 int c = cur_chr; /* desired type of conversion */
2496 str_number str;
2497 int i = 0;
2498 /* Scan the argument for command |c| */
2499 switch (c) {
2500 case number_code:
2501 scan_int();
2502 push_selector;
2503 print_int(cur_val);
2504 pop_selector;
2505 break;
2506 case lua_function_code:
2507 scan_int();
2508 if (cur_val <= 0) {
2510 } else {
2511 u = save_cur_string();
2512 luacstrings = 0;
2513 luafunctioncall(cur_val);
2514 restore_cur_string(u);
2515 if (luacstrings > 0)
2516 lua_string_start();
2517 }
2518 /* no further action */
2519 return;
2520 break;
2521 case lua_code:
2522 u = save_cur_string();
2523 save_scanner_status = scanner_status;
2524 save_def_ref = def_ref;
2525 save_warning_index = warning_index;
2526 sn = scan_lua_state();
2527 scan_toks(false, true);
2528 s = def_ref;
2529 warning_index = save_warning_index;
2530 def_ref = save_def_ref;
2531 scanner_status = save_scanner_status;
2532 luacstrings = 0;
2533 luatokencall(s, sn);
2534 delete_token_ref(s);
2535 restore_cur_string(u); /* TODO: check this, was different */
2536 if (luacstrings > 0)
2537 lua_string_start();
2538 /* no further action */
2539 return;
2540 break;
2541 case expanded_code:
2542 save_scanner_status = scanner_status;
2543 save_warning_index = warning_index;
2544 save_def_ref = def_ref;
2545 u = save_cur_string();
2546 scan_toks(false, true);
2547 warning_index = save_warning_index;
2548 scanner_status = save_scanner_status;
2549 ins_list(token_link(def_ref));
2550 def_ref = save_def_ref;
2551 restore_cur_string(u);
2552 /* no further action */
2553 return;
2554 break;
2555 case math_style_code:
2556 push_selector;
2557 print_math_style();
2558 pop_selector;
2559 break;
2560 case string_code:
2561 save_scanner_status = scanner_status;
2562 scanner_status = normal;
2563 get_token();
2564 scanner_status = save_scanner_status;
2565 push_selector;
2566 if (cur_cs != 0)
2567 sprint_cs(cur_cs);
2568 else
2569 print(cur_chr);
2570 pop_selector;
2571 break;
2572 case cs_string_code:
2573 save_scanner_status = scanner_status;
2574 scanner_status = normal;
2575 get_token();
2576 scanner_status = save_scanner_status;
2577 push_selector;
2578 if (cur_cs != 0)
2579 sprint_cs_name(cur_cs);
2580 else
2581 print(cur_chr);
2582 pop_selector;
2583 break;
2584 case roman_numeral_code:
2585 scan_int();
2586 push_selector;
2587 print_roman_int(cur_val);
2588 pop_selector;
2589 break;
2590 case meaning_code:
2591 save_scanner_status = scanner_status;
2592 scanner_status = normal;
2593 get_token();
2594 scanner_status = save_scanner_status;
2595 push_selector;
2596 print_meaning();
2597 pop_selector;
2598 break;
2599 case uchar_code:
2600 scan_char_num();
2601 push_selector;
2602 print(cur_val);
2603 pop_selector;
2604 break;
2605 case lua_escape_string_code:
2606 {
2607 lstring escstr;
2608 int l = 0;
2609 save_scanner_status = scanner_status;
2610 save_def_ref = def_ref;
2611 save_warning_index = warning_index;
2612 scan_toks(false, true);
2613 bool = in_lua_escape;
2614 in_lua_escape = true;
2615 escstr.s = (unsigned char *) tokenlist_to_cstring(def_ref, false, &l);
2616 escstr.l = (unsigned) l;
2617 in_lua_escape = bool;
2618 delete_token_ref(def_ref);
2619 def_ref = save_def_ref;
2620 warning_index = save_warning_index;
2621 scanner_status = save_scanner_status;
2622 (void) lua_str_toks(escstr);
2623 ins_list(token_link(temp_token_head));
2624 free(escstr.s);
2625 return;
2626 }
2627 /* no further action */
2628 break;
2629 case font_id_code:
2630 scan_font_ident();
2631 push_selector;
2632 print_int(cur_val);
2633 pop_selector;
2634 break;
2635 case font_name_code:
2636 scan_font_ident();
2637 push_selector;
2638 append_string((unsigned char *) font_name(cur_val),(unsigned) strlen(font_name(cur_val)));
2639 if (font_size(cur_val) != font_dsize(cur_val)) {
2641 print_scaled(font_size(cur_val));
2643 }
2644 pop_selector;
2645 break;
2646 case left_margin_kern_code:
2647 scan_int();
2648 if ((box(cur_val) == null) || (type(box(cur_val)) != hlist_node))
2650 push_selector;
2651 p = list_ptr(box(cur_val));
2652 while ((p != null) && (type(p) == glue_node)) {
2653 p = vlink(p);
2654 }
2655 if ((p != null) && (type(p) == margin_kern_node) && (subtype(p) == left_side))
2656 print_scaled(width(p));
2657 else
2658 print_char('0');
2660 pop_selector;
2661 break;
2662 case right_margin_kern_code:
2663 scan_int();
2664 if ((box(cur_val) == null) || (type(box(cur_val)) != hlist_node))
2666 push_selector;
2667 p = list_ptr(box(cur_val));
2668 if (p != null) {
2669 p = tail_of_list(p);
2670 /*
2671 there can be a leftskip, rightskip, penalty and yes, also a disc node with a nesting
2672 node that points to glue spec ... and we don't want to analyze that messy lot
2673 */
2674 while ((p != null) && (type(p) == glue_node)) {
2675 p = alink(p);
2676 }
2677 if ((p != null) && ! ((type(p) == margin_kern_node) && (subtype(p) == right_side))) {
2678 if (type(p) == disc_node) {
2679 q = alink(p);
2680 if ((q != null) && ((type(q) == margin_kern_node) && (subtype(q) == right_side))) {
2681 p = q;
2682 } else {
2683 /*
2684 officially we should look in the replace but currently protrusion doesn't
2686 don't need it now
2687 */
2688 }
2689 }
2690 }
2691 }
2692 if ((p != null) && (type(p) == margin_kern_node) && (subtype(p) == right_side))
2693 print_scaled(width(p));
2694 else
2695 print_char('0');
2697 pop_selector;
2698 break;
2699 case uniform_deviate_code:
2700 scan_int();
2701 push_selector;
2702 print_int(unif_rand(cur_val));
2703 pop_selector;
2704 break;
2705 case normal_deviate_code:
2706 push_selector;
2707 print_int(norm_rand());
2708 pop_selector;
2709 break;
2710 case math_char_class_code:
2711 {
2712 mathcodeval mval;
2713 scan_int();
2714 mval = get_math_code(cur_val);
2715 push_selector;
2716 print_int(mval.class_value);
2717 pop_selector;
2718 }
2719 break;
2720 case math_char_fam_code:
2721 {
2722 mathcodeval mval;
2723 scan_int();
2724 mval = get_math_code(cur_val);
2725 push_selector;
2726 print_int(mval.family_value);
2727 pop_selector;
2728 }
2729 break;
2730 case math_char_slot_code:
2731 {
2732 mathcodeval mval;
2733 scan_int();
2734 mval = get_math_code(cur_val);
2735 push_selector;
2736 print_int(mval.character_value);
2737 pop_selector;
2738 }
2739 break;
2740 case insert_ht_code:
2741 scan_register_num();
2742 push_selector;
2743 i = cur_val;
2744 p = page_ins_head;
2745 while (i >= subtype(vlink(p)))
2746 p = vlink(p);
2747 if (subtype(p) == i)
2748 print_scaled(height(p));
2749 else
2750 print_char('0');
2752 pop_selector;
2753 break;
2754 case job_name_code:
2755 if (job_name == 0)
2756 open_log_file();
2757 push_selector;
2758 print_job_name();
2759 pop_selector;
2760 break;
2761 case format_name_code:
2762 if (job_name == 0)
2763 open_log_file();
2764 push_selector;
2765 print(format_name);
2766 pop_selector;
2767 break;
2768 case luatex_banner_code:
2769 push_selector;
2770 tprint(luatex_banner);
2771 pop_selector;
2772 break;
2773 case luatex_revision_code:
2774 push_selector;
2775 print(get_luatexrevision());
2776 pop_selector;
2777 break;
2778 case luatex_date_code:
2779 push_selector;
2780 print_int(get_luatex_date_info());
2781 pop_selector;
2782 break;
2783 case etex_code:
2784 push_selector;
2785 tprint(eTeX_version_string);
2786 pop_selector;
2787 break;
2788 case eTeX_revision_code:
2789 push_selector;
2790 tprint(eTeX_revision);
2791 pop_selector;
2792 break;
2793 case font_identifier_code:
2795 break;
2796 default:
2798 break;
2799 }
2800 str = make_string();
2801 (void) str_toks(str_lstring(str));
2802 flush_str(str);
2803 ins_list(token_link(temp_token_head));
2804 }
2806 void do_feedback(void)
2807 {
2808 int c = cur_chr;
2809 str_number str;
2810 int done = 1;
2811 switch (c) {
2812 case dvi_feedback_code:
2813 if (get_o_mode() == OMODE_DVI) {
2814 done = do_feedback_dvi(c);
2815 } else {
2817 return ;
2818 }
2819 if (done==0) {
2820 /* we recover */
2822 return;
2823 } else if (done==2) {
2824 return;
2825 }
2826 break;
2827 case pdf_feedback_code:
2828 if (get_o_mode() == OMODE_PDF) {
2829 done = do_feedback_pdf(c);
2830 } else {
2832 return ;
2833 }
2834 if (done==0) {
2835 /* we recover */
2837 return;
2838 } else if (done==2) {
2839 return;
2840 }
2841 break;
2842 default:
2844 break;
2845 }
2846 str = make_string();
2847 (void) str_toks(str_lstring(str));
2848 flush_str(str);
2849 ins_list(token_link(temp_token_head));
2850 }
2852 void do_variable(void)
2853 {
2854 int c = cur_chr;
2855 int done = 1;
2856 switch (c) {
2857 case dvi_variable_code:
2858 done = do_variable_dvi(c);
2859 if (done==0) {
2860 /* we recover */
2862 }
2863 return;
2864 break;
2865 case pdf_variable_code:
2866 done = do_variable_pdf(c);
2867 if (done==0) {
2868 /* we recover */
2870 }
2871 return;
2872 break;
2873 default:
2875 break;
2876 }
2877 }
2879 /*
2880 The following code is not used as we can only set math options and not query them. If
2881 an option is really important we will provide a proper variable. Most options are not
2882 meant for users anyway but for development.
2883 */
2885 /*
2887 #define do_mathoption_int(i) \
2888 cur_cmd = assign_int_cmd; \
2889 cur_val = mathoption_int_base + i; \
2890 cur_tok = token_val(cur_cmd, cur_val); \
2891 back_input();
2893 void do_mathoption(void)
2894 {
2896 if (scan_keyword("noitaliccompensation")) { do_mathoption_int(c_mathoption_no_char_italic_code); }
2897 else if (scan_keyword("nocharitalic")) { do_mathoption_int(c_mathoption_use_old_fraction_scaling_code); }
2899 else if (scan_keyword("umathcodemeaning")) { do_mathoption_int(c_mathoption_umathcode_meaning_code); }
2900 }
2902 */
2904 @ This boolean is keeping track of the lua string escape state
2905 @c
2906 boolean in_lua_escape;
2908 static int the_convert_string_dvi(halfword c, int i)
2909 {
2910 return 0 ;
2911 }
2913 static int the_convert_string_pdf(halfword c, int i)
2914 {
2915 int ff;
2916 if (get_o_mode() != OMODE_PDF) {
2917 return 0;
2919 print_int(pdf_last_link);
2921 print_int(pdf_retval);
2923 print_int(pdf_last_obj);
2925 print_int(pdf_last_annot);
2927 print_int(obj_info(static_pdf, i));
2929 return 0;
2931 set_ff(i);
2932 print_int(obj_info(static_pdf, pdf_font_num(ff)));
2934 set_ff(i);
2935 print_int(pdf_font_num(ff));
2937 print_scaled(font_size(i));
2940 print_int(pdf_get_obj(static_pdf, obj_type_page, i, false));
2942 return 0;
2943 } else {
2944 return 0;
2945 }
2946 return 1;
2947 }
2949 str_number the_convert_string(halfword c, int i)
2950 {
2951 int old_setting; /* saved |selector| setting */
2952 str_number ret = 0;
2953 boolean done = true ;
2954 old_setting = selector;
2955 selector = new_string;
2956 switch (c) {
2957 case number_code:
2958 print_int(i);
2959 break;
2960 /* case lua_function_code: */
2961 /* case lua_code: */
2962 /* case expanded_code: */
2963 case math_style_code:
2964 print_math_style();
2965 break;
2966 /* case string_code: */
2967 /* case cs_string_code: */
2968 case roman_numeral_code:
2969 print_roman_int(i);
2970 break;
2971 /* case meaning_code: */
2972 case uchar_code:
2973 print(i);
2974 break;
2975 /* lua_escape_string_code: */
2976 case font_id_code:
2977 print_int(i);
2978 break;
2979 case font_name_code:
2980 append_string((unsigned char *) font_name(i),(unsigned) strlen(font_name(i)));
2981 if (font_size(i) != font_dsize(i)) {
2983 print_scaled(font_size(i));
2985 }
2986 break;
2987 /* left_margin_kern_code: */
2988 /* right_margin_kern_code: */
2989 case uniform_deviate_code:
2990 print_int(unif_rand(i));
2991 break;
2992 case normal_deviate_code:
2993 print_int(norm_rand());
2994 break;
2995 /* math_char_class_code: */
2996 /* math_char_fam_code: */
2997 /* math_char_slot_code: */
2998 /* insert_ht_code: */
2999 case job_name_code:
3000 print_job_name();
3001 break;
3002 case format_name_code:
3003 print(format_name);
3004 break;
3005 case luatex_banner_code:
3006 tprint(luatex_banner);
3007 break;
3008 case luatex_revision_code:
3009 print(get_luatexrevision());
3010 break;
3011 case luatex_date_code:
3012 print_int(get_luatex_date_info());
3013 break;
3014 case etex_code:
3015 tprint(eTeX_version_string);
3016 break;
3017 case eTeX_revision_code:
3018 tprint(eTeX_revision);
3019 break;
3020 case font_identifier_code:
3021 print_font_identifier(i);
3022 break;
3023 /* backend: this might become obsolete */
3024 case dvi_feedback_code:
3025 done = the_convert_string_dvi(c,i);
3026 break;
3027 case pdf_feedback_code:
3028 done = the_convert_string_pdf(c,i);
3029 break;
3030 /* done */
3031 default:
3032 done = false;
3033 break;
3034 }
3035 if (done) {
3036 ret = make_string();
3037 }
3038 selector = old_setting;
3039 return ret;
3040 }
3042 @ Another way to create a token list is via the \.{\\read} command. The sixteen
3043 files potentially usable for reading appear in the following global variables.
3044 The value of |read_open[n]| will be |closed| if stream number |n| has not been
3045 opened or if it has been fully read; |just_open| if an \.{\\openin} but not a
3046 \.{\\read} has been done; and |normal| if it is open and ready to read the next
3047 line.
3049 @c
3050 FILE *read_file[16]; /* used for \.{\\read} */
3051 int read_open[17]; /* state of |read_file[n]| */
3053 void initialize_read(void)
3054 {
3055 int k;
3056 for (k = 0; k <= 16; k++)
3057 read_open[k] = closed;
3058 }
3060 @ The |read_toks| procedure constructs a token list like that for any macro
3061 definition, and makes |cur_val| point to it. Parameter |r| points to the control
3062 sequence that will receive this token list.
3064 @c
3065 void read_toks(int n, halfword r, halfword j)
3066 {
3067 halfword p; /* tail of the token list */
3068 halfword q; /* new node being added to the token list via |store_new_token| */
3069 int s; /* saved value of |align_state| */
3070 int m; /* stream number */
3071 scanner_status = defining;
3072 warning_index = r;
3073 p = get_avail();
3074 def_ref = p;
3075 set_token_ref_count(def_ref, 0);
3076 p = def_ref; /* the reference count */
3077 store_new_token(end_match_token);
3078 if ((n < 0) || (n > 15))
3079 m = 16;
3080 else
3081 m = n;
3082 s = align_state;
3083 align_state = 1000000; /* disable tab marks, etc. */
3084 do {
3085 /* Input and store tokens from the next line of the file */
3086 begin_file_reading();
3087 iname = m + 1;
3088 if (read_open[m] == closed) {
3089 /*
3090 Input for \.{\\read} from the terminal
3092 Here we input on-line into the |buffer| array, prompting the user explicitly
3093 if |n>=0|. The value of |n| is set negative so that additional prompts
3094 will not be given in the case of multi-line input.
3095 */
3096 if (interaction > nonstop_mode) {
3097 if (n < 0) {
3099 } else {
3100 wake_up_terminal();
3101 print_ln();
3102 sprint_cs(r);
3104 n = -1;
3105 }
3106 } else {
3107 fatal_error
3109 }
3111 } else if (read_open[m] == just_open) {
3112 /*
3113 Input the first line of |read_file[m]|
3115 The first line of a file must be treated specially, since |lua_input_ln|
3116 must be told not to start with |get|.
3117 */
3118 if (lua_input_ln(read_file[m], (m + 1), false)) {
3119 read_open[m] = normal;
3120 } else {
3121 lua_a_close_in(read_file[m], (m + 1));
3122 read_open[m] = closed;
3123 }
3125 } else {
3126 /*
3127 Input the next line of |read_file[m]|
3129 An empty line is appended at the end of a |read_file|.
3130 */
3131 if (!lua_input_ln(read_file[m], (m + 1), true)) {
3132 lua_a_close_in(read_file[m], (m + 1));
3133 read_open[m] = closed;
3134 if (align_state != 1000000) {
3135 runaway();
3138 align_state = 1000000;
3139 error();
3140 }
3141 }
3143 }
3144 ilimit = last;
3145 if (end_line_char_inactive)
3146 decr(ilimit);
3147 else
3148 buffer[ilimit] = (packed_ASCII_code) end_line_char_par;
3149 first = ilimit + 1;
3150 iloc = istart;
3151 istate = new_line;
3152 /* Handle \.{\\readline} and |goto done|; */
3153 if (j == 1) {
3154 while (iloc <= ilimit) {
3155 /* current line not yet finished */
3156 do_buffer_to_unichar(cur_chr, iloc);
3157 if (cur_chr == ' ')
3158 cur_tok = space_token;
3159 else
3160 cur_tok = cur_chr + other_token;
3161 store_new_token(cur_tok);
3162 }
3163 } else {
3164 while (1) {
3165 get_token();
3166 if (cur_tok == 0) {
3167 /* |cur_cmd=cur_chr=0| will occur at the end of the line */
3168 break;
3169 }
3170 if (align_state < 1000000) {
3171 /* unmatched `\.\}' aborts the line */
3172 do {
3173 get_token();
3174 } while (cur_tok != 0);
3175 align_state = 1000000;
3176 break;
3177 }
3178 store_new_token(cur_tok);
3179 }
3180 }
3181 end_file_reading();
3183 } while (align_state != 1000000);
3184 cur_val = def_ref;
3185 scanner_status = normal;
3186 align_state = s;
3187 }
3189 @ return a string from tokens list
3191 @c
3192 str_number tokens_to_string(halfword p)
3193 {
3194 int old_setting;
3195 if (selector == new_string)
3197 old_setting = selector;
3198 selector = new_string;
3199 show_token_list(token_link(p), null, -1);
3200 selector = old_setting;
3201 return make_string();
3202 }
3204 @ @c
3205 #define make_room(a) \
3206 if ((unsigned)i+a+1>alloci) { \
3207 ret = xrealloc(ret,(alloci+64)); \
3208 alloci = alloci + 64; \
3209 }
3211 #define append_i_byte(a) ret[i++] = (char)(a)
3213 #define Print_char(a) make_room(1); append_i_byte(a)
3215 #define Print_uchar(s) { \
3216 make_room(4); \
3217 if (s<=0x7F) { \
3218 append_i_byte(s); \
3219 } else if (s<=0x7FF) { \
3220 append_i_byte(0xC0 + (s / 0x40)); \
3221 append_i_byte(0x80 + (s % 0x40)); \
3222 } else if (s<=0xFFFF) { \
3223 append_i_byte(0xE0 + (s / 0x1000)); \
3224 append_i_byte(0x80 + ((s % 0x1000) / 0x40)); \
3225 append_i_byte(0x80 + ((s % 0x1000) % 0x40)); \
3226 } else if (s>=0x110000) { \
3227 append_i_byte(s-0x11000); \
3228 } else { \
3229 append_i_byte(0xF0 + (s / 0x40000)); \
3230 append_i_byte(0x80 + ((s % 0x40000) / 0x1000)); \
3231 append_i_byte(0x80 + (((s % 0x40000) % 0x1000) / 0x40)); \
3232 append_i_byte(0x80 + (((s % 0x40000) % 0x1000) % 0x40)); \
3233 } }
3235 #define Print_esc(b) { \
3236 const char *v = b; \
3237 if (e>0 && e<STRING_OFFSET) { \
3238 Print_uchar (e); \
3239 } \
3240 make_room(strlen(v)); \
3241 while (*v) { append_i_byte(*v); v++; } \
3242 }
3244 #define Print_str(b) { \
3245 const char *v = b; \
3246 make_room(strlen(v)); \
3247 while (*v) { append_i_byte(*v); v++; } \
3248 }
3250 #define is_cat_letter(a) \
3251 (get_char_cat_code(pool_to_unichar(str_string((a)))) == 11)
3253 @ the actual token conversion in this function is now functionally equivalent to
3254 |show_token_list|, except that it always prints the whole token list. TODO: check
3255 whether this causes problems in the lua library.
3257 @c
3258 char *tokenlist_to_cstring(int pp, int inhibit_par, int *siz)
3259 {
3260 register int p, c, m;
3261 int q;
3262 int infop;
3263 char *s, *sh;
3264 int e = 0;
3265 char *ret;
3266 int match_chr = '#';
3267 int n = '0';
3268 unsigned alloci = 1024;
3269 int i = 0;
3270 p = pp;
3271 if (p == null) {
3272 if (siz != NULL)
3273 *siz = 0;
3274 return NULL;
3275 }
3276 ret = xmalloc(alloci);
3277 p = token_link(p); /* skip refcount */
3278 if (p != null) {
3279 e = escape_char_par;
3280 }
3281 while (p != null) {
3282 if (p < (int) fix_mem_min || p > (int) fix_mem_end) {
3284 break;
3285 }
3286 infop = token_info(p);
3287 if (infop >= cs_token_flag) {
3288 if (!(inhibit_par && infop == par_token)) {
3289 q = infop - cs_token_flag;
3290 if (q < hash_base) {
3291 if (q == null_cs) {
3294 } else {
3296 }
3297 } else if ((q >= undefined_control_sequence) && ((q <= eqtb_size) || (q > eqtb_size + hash_extra))) {
3299 } else if ((cs_text(q) < 0) || (cs_text(q) >= str_ptr)) {
3301 } else {
3302 str_number txt = cs_text(q);
3303 sh = makecstring(txt);
3304 s = sh;
3305 if (is_active_cs(txt)) {
3306 s = s + 3;
3307 while (*s) {
3308 Print_char(*s);
3309 s++;
3310 }
3311 } else {
3312 if (e>=0 && e<0x110000) Print_uchar(e);
3313 while (*s) {
3314 Print_char(*s);
3315 s++;
3316 }
3317 if ((!single_letter(txt)) || is_cat_letter(txt)) {
3318 Print_char(' ');
3319 }
3320 }
3321 free(sh);
3322 }
3323 }
3324 } else {
3325 if (infop < 0) {
3327 } else {
3328 m = token_cmd(infop);
3329 c = token_chr(infop);
3330 switch (m) {
3331 case left_brace_cmd:
3332 case right_brace_cmd:
3333 case math_shift_cmd:
3334 case tab_mark_cmd:
3335 case sup_mark_cmd:
3336 case sub_mark_cmd:
3337 case spacer_cmd:
3338 case letter_cmd:
3339 case other_char_cmd:
3340 Print_uchar(c);
3341 break;
3342 case mac_param_cmd:
3343 if (!in_lua_escape && (is_in_csname==0))
3344 Print_uchar(c);
3345 Print_uchar(c);
3346 break;
3347 case out_param_cmd:
3348 Print_uchar(match_chr);
3349 if (c <= 9) {
3350 Print_char(c + '0');
3351 } else {
3352 Print_char('!');
3353 goto EXIT;
3354 }
3355 break;
3356 case match_cmd:
3357 match_chr = c;
3358 Print_uchar(c);
3359 n++;
3360 Print_char(n);
3361 if (n > '9')
3362 goto EXIT;
3363 break;
3364 case end_match_cmd:
3365 if (c == 0) {
3366 Print_char('-');
3367 Print_char('>');
3368 }
3369 break;
3370 default:
3371 not_so_bad(Print_esc);
3372 break;
3373 }
3374 }
3375 }
3376 p = token_link(p);
3377 }
3378 EXIT:
3379 ret[i] = '\0';
3380 if (siz != NULL)
3381 *siz = i;
3382 return ret;
3383 }
3385 @ @c
3386 lstring *tokenlist_to_lstring(int pp, int inhibit_par)
3387 {
3388 int siz;
3389 lstring *ret = xmalloc(sizeof(lstring));
3390 ret->s = (unsigned char *) tokenlist_to_cstring(pp, inhibit_par, &siz);
3391 ret->l = (size_t) siz;
3392 return ret;
3393 }
3395 @ @c
3396 void free_lstring(lstring * ls)
3397 {
3398 if (ls == NULL)
3399 return;
3400 if (ls->s != NULL)
3401 free(ls->s);
3402 free(ls);
3403 }