| blob | 8d147f4e2a8da4a91ba90ee5bdff7c51b679e70e |
1 % textoken.w
2 %
4 %
5 % This file is part of LuaTeX.
6 %
8 % the terms of the GNU General Public License as published by the Free
11 %
15 % License for more details.
16 %
17 % You should have received a copy of the GNU General Public License along
20 @ @c
22 #include "ptexlib.h"
24 @ @c
38 /*
44 else \
47 */
54 else \
59 @ The \TeX\ system does nearly all of its own memory allocation, so that it can
60 readily be transported into environments that do not have automatic facilities
62 error messages the user receives. The dynamic storage requirements of \TeX\ are
63 handled by providing two large arrays called |fixmem| and |varmem| in which
64 consecutive blocks of words are used as nodes by the \TeX\ routines.
66 Pointer variables are indices into this array, or into another array called
67 |eqtb| that will be explained later. A pointer variable might also be a special
68 flag that lies outside the bounds of |mem|, so we allow pointers to assume any
69 |halfword| value. The minimum halfword value represents a null pointer. \TeX\
72 @ Locations in |fixmem| are used for storing one-word records; a conventional
75 @c
80 @ In order to study the memory requirements of particular applications, it is
81 possible to prepare a version of \TeX\ that keeps track of current and maximum
82 memory usage. When code between the delimiters |@!stat| $\ldots$ |tats| is not
83 commented out, \TeX\ will run a bit slower but it will report these statistics
84 when |tracing_stats| is sufficiently large.
86 @c
99 @ @c
101 {
102 halfword p;
124 }
126 @ The function |get_avail| returns a pointer to a new one-word node whose |link|
127 field is null. However, \TeX\ will halt if there is no more room left.
128 @^inner loop@>
133 work, we have to quit.
135 @c
139 unsigned t;
149 new_fixmem =
150 fixmemcast(realloc
157 }
161 }
165 }
168 that starts at position |p|.
169 @^inner loop@>
171 @c
177 do {
184 }
185 }
189 number is |c| and whose command code is |m| is represented as the number
191 whose |eqtb| address is |p| is represented as the number |cs_token_flag+p|. Here
194 halfword.
196 A token |t| represents a |left_brace| command if and only if
200 following definitions take care of these token-oriented constants and a few
201 others.
203 @ A token list is a singly linked list of one-word nodes in |mem|, where each
206 form of token lists, usually preceded by a node with a reference count in its
209 Three special commands appear in the token lists of macro definitions. When
213 insert parameter number |c| into the text at this point.
216 are omitted, but the final right brace of an output routine is included at the
217 end of its token list.
219 Here is an example macro definition that illustrates these conventions. After
220 \TeX\ processes the text
234 The procedure |scan_toks| builds such token lists, and |macro_call| does the
235 parameter matching. @^reference counts@>
240 entry before the token list has been fully consumed, so we dare not simply
241 destroy a token list when its control sequence is being redefined.
244 corresponding token list will have `\.\{' just before the `|end_match|' and also
246 one keeps the first from disappearing.
248 The |print_meaning| subroutine displays |cur_cmd| and |cur_chr| in symbolic form,
249 including the expansion of a macro or mark.
251 @c
253 {
255 /*
258 }
259 */
263 print_ln();
269 print_ln();
271 case first_mark_code:
273 break;
274 case bot_mark_code:
276 break;
277 case split_first_mark_code:
279 break;
280 case split_bot_mark_code:
282 break;
283 default:
285 break;
286 }
287 }
288 }
289 }
291 @ The procedure |show_token_list|, which prints a symbolic form of the token list
292 that starts at a given node |p|, illustrates these conventions. The token list
293 being displayed should not begin with a reference count. However, the procedure
294 is intended to be robust, so that if the memory links are awry or if |p| is not
295 really a pointer to a token list, nothing catastrophic will happen.
297 An additional parameter |q| is also given; this parameter is either null or it
298 points to a node in the token list where a certain magic computation takes place
300 the two-line context information at the time of an error message; |q| marks the
301 place corresponding to where the second line should begin.)
303 For example, if |p| points to the node containing the first \.a in the token list
306 second \.a, the magic computation will be performed just before the second \.a is
307 printed.
310 printing exceeds a given limit~|l|. Anomalous entries are printed in the form of
312 this cannot be confused with actual control sequences because a real control
315 @c
318 case assign_int_cmd: \
321 break; \
322 case assign_dimen_cmd: \
325 break; \
326 case assign_toks_cmd: \
329 break; \
330 default: \
332 break; \
333 }
336 {
346 set_trick_count();
347 }
351 return;
352 }
362 /*
365 The procedure usually ``learns'' the character code used for macro
366 parameters by seeing one in a |match| command before it runs into any
367 |out_param| commands.
368 */
370 case left_brace_cmd:
371 case right_brace_cmd:
372 case math_shift_cmd:
373 case tab_mark_cmd:
374 case sup_mark_cmd:
375 case sub_mark_cmd:
376 case spacer_cmd:
377 case letter_cmd:
378 case other_char_cmd:
380 break;
381 case mac_param_cmd:
385 break;
386 case out_param_cmd:
392 return;
393 }
394 break;
395 case match_cmd:
401 return;
402 break;
403 case end_match_cmd:
406 break;
407 default:
409 break;
410 }
411 }
412 }
414 }
417 }
419 @ @c
425 @ Here's the way we sometimes want to display a token list, given a pointer to
426 its reference count; the pointer may be null.
428 @c
430 {
433 }
435 @ |delete_token_ref|, is called when a pointer to a token list's reference count
436 is being removed. This means that the token list should disappear if the
437 reference count was |null|, otherwise the count should be decreased by one.
438 @^reference counts@>
440 @ |p| points to the reference count of a token list that is losing one
441 reference.
443 @c
445 {
448 else
450 }
452 @ @c
454 {
455 int a;
457 return a;
458 }
460 @ @c
462 {
466 NULL
467 };
471 }
473 @ @c
478 typedef enum {
479 next_line_ok,
480 next_line_return,
481 next_line_restart
488 |scan_keyword| routine checks to see whether the next tokens of input match this
489 string. The match must be exact, except that uppercase letters will match their
490 lowercase counterparts; uppercase equivalents are determined by subtracting
492 only for its own limited set of keywords.
494 If a match is found, the characters are effectively removed from the input and
495 |true| is returned. Otherwise |false| is returned, and the input is left
496 essentially unchanged (except for the fact that some macros may have been
499 @c
515 k++;
517 /*
518 crashes on some alignments:
527 back_input();
528 }
529 */
530 back_input();
533 }
534 /* */
536 return false;
537 }
538 }
542 return true;
543 }
545 @ We can not return |undefined_control_sequence| under some conditions
548 @c
550 /*
552 {
553 halfword curcs;
569 }
573 return curcs;
574 }
575 */
580 {
581 halfword curcs;
585 }
596 }
598 return curcs;
599 }
601 /*
604 {
630 }
632 return buf;
633 }
636 {
637 halfword curcs;
641 }
648 }
650 return curcs;
651 }
653 */
657 @ prints a control sequence
659 @c
661 {
671 }
676 }
687 }
693 }
695 }
697 }
699 }
701 @ TODO this is a quick hack, will be solved differently soon
703 @c
705 {
707 str_number str;
715 return s;
716 }
718 @ The heart of \TeX's input mechanism is the |get_next| procedure, which we shall
719 develop in the next few sections of the program. Perhaps we shouldn't actually
721 reading the source files and gobbling them up. And it also helps \TeX\ to
722 regurgitate stored token lists that are to be processed again. @^eyes and mouth@>
724 The main duty of |get_next| is to input one token and to set |cur_cmd| and
725 |cur_chr| to that token's command code and modifier. Furthermore, if the input
726 token is a control sequence, the |eqtb| location of that control sequence is
727 stored in |cur_cs|; otherwise |cur_cs| is set to zero.
729 Underlying this simple description is a certain amount of complexity because of
730 all the cases that need to be handled. However, the inner loop of |get_next| is
731 reasonably short and fast.
736 |end_line_char| has |ignore| as its catcode.)
739 needed because a blank line of input is supposed to be exactly equivalent to the
741 line.
743 @c
747 @ Parts |get_next| are executed more often than any other instructions of \TeX.
750 The global variable |force_eof| is normally |false|; it is set |true| by an
752 waiting to be input, it is changed by |luatokencall|.
754 @c
758 @ If the user has set the |pausing| parameter to some positive value, and if
759 nonstop mode has not been selected, each line of input is displayed on the
761 response. If the response is simply |carriage_return|, the line is accepted as it
762 stands, otherwise the line typed is used instead of the line in the file.
764 @c
766 {
771 wake_up_terminal();
772 print_ln();
776 }
783 }
784 }
785 }
786 }
788 @ Before getting into |get_next|, let's consider the subroutine that is called
790 has been reached. These two cases are distinguished by |cur_cs|, which is zero at
791 the end of a file.
793 @c
795 {
799 return;
810 }
813 }
820 NULL
821 };
832 }
834 and insert tokens that should lead to recovery; */
835 /* The recovery procedure can't be fully understood without knowing more
836 about the \TeX\ routines that should be aborted, but we can sketch the
843 case defining:
846 break;
847 case matching:
851 break;
852 case aligning:
860 break;
861 case absorbing:
864 break;
867 break;
879 NULL
880 };
885 NULL
886 };
892 }
900 {
902 back_input();
906 }
907 }
909 }
910 }
912 @ @c
914 #if 0
916 /*
917 The other variant gives less clutter in tracing cache usage when profiling and for
919 */
922 {
923 SWITCH:
928 RESWITCH:
933 }
934 /*
935 Change state if necessary, and |goto switch| if the current
936 character should be ignored, or |goto reswitch| if the current
937 character changes to another;
940 that a decent C compiler has translated the code. Note that the numeric
943 command code to the state to get a single number that characterizes both.
946 cause of many branch prediction errors but changing it to:
950 return true;
953 }
956 }
959 }
962 return true;
963 }
965 gives as many prediction errors. So, we can indeed assume that the compiler
966 does the right job, or that there is simply no other way.
967 */
976 goto SWITCH;
977 break;
984 check_outer_validity();
985 break;
995 check_outer_validity();
996 break;
1002 goto RESWITCH;
1003 else
1005 break;
1010 invalid_character_error();
1012 break;
1017 break;
1019 /*
1023 treated alike when macro parameters are being matched. We do this since such
1024 characters are indistinguishable on most computer terminal displays.
1025 */
1029 break;
1036 goto SWITCH;
1037 break;
1045 check_outer_validity();
1046 break;
1052 align_state++;
1053 break;
1059 align_state--;
1060 break;
1067 break;
1068 /*
1081 */
1082 default:
1084 break;
1085 }
1089 /*
1090 Move to next line of file,
1091 or |goto restart| if there is no next line,
1093 */
1094 do {
1097 return true;
1099 return false;
1100 }
1102 check_interrupt();
1103 goto SWITCH;
1104 }
1105 return true;
1106 }
1108 #else
1112 /*
1113 escape_cmd left_brace_cmd right_brace_cmd math_shift_cmd
1114 tab_mark_cmd car_ret_cmd mac_param_cmd sup_mark_cmd
1115 sub_mark_cmd ignore_cmd spacer_cmd letter_cmd
1116 other_char_cmd active_char_cmd comment_cmd invalid_char_cmd
1117 */
1120 {
1122 SWITCH:
1126 RESWITCH:
1131 }
1132 /*
1133 Change state if necessary, and |goto switch| if the current
1134 character should be ignored, or |goto reswitch| if the current
1135 character changes to another;
1136 */
1139 return true;
1142 case escape_cmd:
1145 check_outer_validity();
1146 return true;
1147 case left_brace_cmd:
1149 align_state++;
1150 return true;
1151 case right_brace_cmd:
1153 align_state--;
1154 return true;
1155 case math_shift_cmd:
1157 return true;
1158 case tab_mark_cmd:
1160 return true;
1161 case car_ret_cmd:
1168 check_outer_validity();
1169 return true;
1170 case mac_param_cmd:
1172 return true;
1173 case sup_mark_cmd:
1175 goto RESWITCH;
1176 else
1178 return true;
1179 case sub_mark_cmd:
1181 return true;
1182 case ignore_cmd:
1183 goto SWITCH;
1184 return true;
1185 case spacer_cmd:
1187 goto SWITCH;
1188 case letter_cmd:
1190 return true;
1191 case other_char_cmd:
1193 return true;
1194 case active_char_cmd:
1200 check_outer_validity();
1201 return true;
1202 case comment_cmd:
1204 goto SWITCH;
1205 case invalid_char_cmd:
1206 invalid_character_error();
1208 default:
1210 return true;
1211 }
1214 case escape_cmd:
1218 check_outer_validity();
1219 return true;
1220 case left_brace_cmd:
1222 align_state++;
1223 return true;
1224 case right_brace_cmd:
1226 align_state--;
1227 return true;
1228 case math_shift_cmd:
1230 return true;
1231 case tab_mark_cmd:
1233 return true;
1234 case car_ret_cmd:
1236 goto SWITCH;
1237 case mac_param_cmd:
1239 return true;
1240 case sup_mark_cmd:
1243 goto RESWITCH;
1244 else
1246 return true;
1247 case sub_mark_cmd:
1249 return true;
1250 case ignore_cmd:
1251 goto SWITCH;
1252 case spacer_cmd:
1253 goto SWITCH;
1254 case letter_cmd:
1256 return true;
1257 case other_char_cmd:
1259 return true;
1260 case active_char_cmd:
1266 check_outer_validity();
1267 return true;
1268 case comment_cmd:
1271 goto SWITCH;
1272 case invalid_char_cmd:
1274 invalid_character_error();
1276 default:
1278 return true;
1279 }
1282 case escape_cmd:
1285 check_outer_validity();
1286 return true;
1287 case left_brace_cmd:
1288 align_state++;
1289 return true;
1290 case right_brace_cmd:
1291 align_state--;
1292 return true;
1293 case math_shift_cmd:
1294 return true;
1295 case tab_mark_cmd:
1296 return true;
1297 case car_ret_cmd:
1298 /*
1302 treated alike when macro parameters are being matched. We do this since such
1303 characters are indistinguishable on most computer terminal displays.
1304 */
1308 return true;
1309 case mac_param_cmd:
1310 return true;
1311 case sup_mark_cmd:
1313 goto RESWITCH;
1314 else
1316 return true;
1317 case sub_mark_cmd:
1318 return true;
1319 case ignore_cmd:
1320 goto SWITCH;
1321 case spacer_cmd:
1325 return true;
1326 case letter_cmd:
1328 return true;
1329 case other_char_cmd:
1331 return true;
1332 case active_char_cmd:
1338 check_outer_validity();
1339 return true;
1340 case comment_cmd:
1342 goto SWITCH;
1343 case invalid_char_cmd:
1344 invalid_character_error();
1346 default:
1348 return true;
1349 }
1352 return true;
1353 }
1357 }
1358 /*
1361 */
1362 do {
1365 return true;
1367 return false;
1368 }
1370 check_interrupt();
1371 goto SWITCH;
1372 }
1373 return true;
1374 }
1376 #endif
1379 We only support a limited set:
1381 ^^^^^^XXXXXX
1382 ^^^^XXXXXX
1385 @c
1394 }
1401 }
1421 {
1438 return true;
1441 }
1444 }
1455 return true;
1458 }
1461 }
1462 }
1471 return true;
1472 }
1473 }
1475 }
1476 }
1478 {
1487 return true;
1488 }
1489 }
1491 return true;
1492 }
1493 }
1494 }
1495 return false;
1496 }
1498 @ Control sequence names are scanned only when they appear in some line of a
1500 a unique identification, so \TeX\ doesn't need to refer to the original name any
1501 more except when it prints the equivalent in symbolic form.
1503 The program that scans a control sequence has been written carefully in order to
1504 avoid the blowups that might otherwise occur if a malicious user tried something
1509 following a control sequence name, they are converted to single characters in the
1510 buffer and the process is repeated, slowly but surely.
1512 @c
1516 {
1529 continue;
1532 do {
1538 continue;
1541 /*
1549 */
1558 }
1560 }
1561 }
1564 break;
1565 }
1566 }
1569 return retval;
1570 }
1572 @ Whenever we reach the following piece of code, we will have
1577 left two or three places.
1579 @c
1581 {
1582 int l;
1600 }
1603 }
1615 }
1618 }
1619 }
1631 }
1634 }
1635 }
1636 }
1639 else
1640 d++;
1645 k++;
1646 d--;
1650 k++;
1651 d--;
1653 k++;
1654 d--;
1658 k++;
1659 d--;
1661 k++;
1662 d--;
1664 k++;
1665 d--;
1667 }
1672 l++;
1673 }
1675 return true;
1676 }
1677 return false;
1678 }
1680 @ All of the easy branches of |get_next| have now been taken care of. There is
1681 one more branch.
1684 {
1702 return next_line_restart;
1705 }
1709 firm_up_the_line();
1719 }
1728 return next_line_restart;
1731 }
1732 }
1733 }
1734 }
1740 file_warning();
1741 }
1745 }
1749 end_file_reading();
1751 end_file_reading();
1753 check_outer_validity();
1754 }
1755 return next_line_restart;
1756 }
1758 ilimit--;
1759 else
1769 }
1772 end_file_reading();
1774 }
1776 open_log_file();
1779 ilimit++;
1783 }
1784 print_ln();
1789 ilimit--;
1790 else
1795 /*
1797 never waits for on-line input.
1798 */
1800 }
1801 }
1802 return next_line_ok;
1803 }
1805 @ Let's consider now what happens when |get_next| is looking at a token list.
1807 @c
1809 {
1818 /*
1819 Get the next token, suppressing expansion. The present point in the program
1820 is reached only when the |expand| routine has inserted a special marker into
1823 */
1830 return true;
1831 }
1833 check_outer_validity();
1834 }
1835 }
1841 case left_brace_cmd:
1842 align_state++;
1843 break;
1844 case right_brace_cmd:
1845 align_state--;
1846 break;
1847 case out_param_cmd:
1850 return false;
1851 break;
1852 }
1853 }
1854 return true;
1855 }
1857 @ Now we're ready to take the plunge into |get_next| itself. Parts of this
1858 routine are executed more often than any other instructions of \TeX.
1863 @c
1865 {
1866 RESTART:
1871 goto RESTART;
1874 end_token_list();
1878 }
1879 }
1882 insert_vj_template();
1883 goto RESTART;
1884 }
1885 }
1887 @ Since |get_next| is used so frequently in \TeX, it is convenient to define
1888 three related procedures that do a little more:
1890 \yskip\hang|get_token| not only sets |cur_cmd| and |cur_chr|, it also sets
1891 |cur_tok|, a packed halfword version of the current token.
1897 the expansion of the macro or the evaluation of the conditional.
1899 \yskip\hang|x_token| is like |get_x_token| except that it assumes that |get_next|
1900 has already been called.
1902 \yskip\noindent In fact, these three procedures account for almost every use of
1903 |get_next|.
1905 No new control sequences will be defined except during a call of |get_token|, or
1907 always |true| at other times.
1911 @c
1913 {
1915 get_next();
1919 else
1921 }
1923 @ changes the string |s| to a token list
1925 @c
1927 {
1940 else
1943 }
1945 }
1950 Before we get into the details of |scan_toks|, let's consider a much simpler
1951 task, that of converting the current string into a token list. The |str_toks|
1952 function does this; it classifies spaces as type |spacer| and everything else as
1953 type |other_char|.
1958 |lua_str_toks| is almost identical, but it also escapes the three symbols that
1959 |lua| considers special while scanning a literal string
1963 @c
1965 {
1980 fast_store_new_token(other_token + '\\');
1981 if (t == 10)
1982 t = 'n';
1983 if (t == 13)
1984 t = 'r';
1985 t = other_token + t;
1986 }
1987 fast_store_new_token(t);
1988 }
1989 return p;
1990 }
1992 @ Incidentally, the main reason for wanting |str_toks| is the function
1993 |the_toks|, which has similar input/output characteristics.
1995 @ changes the string |str_pool[b..pool_ptr]| to a token list
1997 @c
1998 halfword str_toks(lstring s)
1999 {
2000 halfword p; /* tail of the token list */
2001 halfword q; /* new node being added to the token list via |store_new_token| */
2002 halfword t; /* token being appended */
2003 unsigned char *k, *l; /* index into string */
2004 p = temp_token_head;
2005 set_token_link(p, null);
2006 k = s.s;
2007 l = k + s.l;
2008 while (k < l) {
2009 t = pool_to_unichar(k);
2010 k += utf8_size(t);
2011 if (t == ' ')
2012 t = space_token;
2013 else
2014 t = other_token + t;
2015 fast_store_new_token(t);
2016 }
2017 return p;
2018 }
2020 /*
2021 hh: most of the converter is similar to the one i made for macro so at some point i
2022 can make a helper; also todo: there is no need to go through the pool
2024 */
2026 halfword str_scan_toks(int ct, lstring s)
2027 { /* changes the string |str_pool[b..pool_ptr]| to a token list */
2028 halfword p; /* tail of the token list */
2029 halfword q; /* new node being added to the token list via |store_new_token| */
2030 halfword t; /* token being appended */
2031 unsigned char *k, *l; /* index into string */
2032 int cc;
2033 p = temp_token_head;
2034 set_token_link(p, null);
2035 k = s.s;
2036 l = k + s.l;
2037 while (k < l) {
2038 t = pool_to_unichar(k);
2039 k += utf8_size(t);
2040 cc = get_cat_code(ct,t);
2041 if (cc == 0) {
2042 /* we have a potential control sequence so we check for it */
2043 int _lname = 0 ;
2044 int _s = 0 ;
2045 int _c = 0 ;
2046 halfword _cs = null ;
2047 unsigned char *_name = k ;
2048 while (k < l) {
2049 t = (halfword) str2uni((const unsigned char *) k);
2050 _s = utf8_size(t);
2051 _c = get_cat_code(ct,t);
2052 if (_c == 11) {
2053 k += _s ;
2054 _lname = _lname + _s ;
2055 } else if (_c == 10) {
2056 /* we ignore a trailing space like normal scanning does */
2057 k += _s ;
2058 break ;
2059 } else {
2060 break ;
2061 }
2062 }
2063 if (_s > 0) {
2064 /* we have a potential \cs */
2065 _cs = string_lookup((const char *) _name, _lname);
2066 if (_cs == undefined_control_sequence) {
2067 /* let's play safe and backtrack */
2068 t = cc * (1<<21) + t ;
2069 k = _name ;
2070 } else {
2071 t = cs_token_flag + _cs;
2072 }
2073 } else {
2074 /* just a character with some meaning, so \unknown becomes effectively */
2075 /* \\unknown assuming that \\ has some useful meaning of course */
2076 t = cc * (1<<21) + t ;
2077 k = _name ;
2078 }
2080 } else {
2081 /* whatever token, so for instance $x^2$ just works given a tex */
2082 /* catcode regime */
2083 t = cc * (1<<21) + t ;
2084 }
2085 fast_store_new_token(t);
2087 }
2088 return p;
2089 }
2091 @ Here's part of the |expand| subroutine that we are now ready to complete:
2093 @c
2094 void ins_the_toks(void)
2095 {
2096 (void) the_toks();
2097 ins_list(token_link(temp_token_head));
2098 }
2100 #define set_toks_register(n,t,g) { \
2101 int a = (g>0) ? 4 : 0; \
2102 halfword ref = get_avail(); \
2103 set_token_ref_count(ref, 0); \
2104 set_token_link(ref, token_link(t)); \
2105 define(n + toks_base, call_cmd, ref); \
2106 }
2108 void combine_the_toks(int how)
2109 {
2110 halfword nt;
2111 get_x_token();
2112 /* target */
2113 if (cur_cmd == assign_toks_cmd) {
2114 nt = equiv(cur_cs) - toks_base;
2115 /* check range */
2116 } else {
2117 back_input();
2118 scan_int();
2119 nt = cur_val;
2120 }
2121 /* source */
2122 do {
2123 get_x_token();
2124 } while (cur_cmd == spacer_cmd);
2125 if (cur_cmd == left_brace_cmd) {
2126 halfword x, source;
2127 back_input();
2128 x = scan_toks(false,how > 1); /* expanded or not */
2129 source = def_ref;
2130 /* action */
2131 if (source != null) {
2132 halfword target = toks(nt);
2133 if (target == null) {
2134 set_toks_register(nt,source,0);
2135 } else {
2136 halfword s = token_link(source);
2137 if (s != null) {
2138 halfword t = token_link(target);
2139 if (t == null) {
2140 /* can this happen ? */
2141 set_token_link(target, s);
2142 } else if (odd(how)) {
2143 /* prepend */
2144 if (cur_level != eq_level_field(eqtb[toks_base+nt])) {
2145 halfword p = temp_token_head;
2146 halfword q;
2147 set_token_link(p, s); /* s = head, x = tail */
2148 p = x;
2149 while (t != null) {
2150 fast_store_new_token(token_info(t));
2151 t = token_link(t);
2152 }
2153 set_toks_register(nt,temp_token_head,0);
2154 } else {
2155 set_token_link(x,t);
2156 set_token_link(target,s);
2157 }
2158 } else {
2159 /* append */
2160 if (cur_level != eq_level_field(eqtb[toks_base+nt])) {
2161 halfword p = temp_token_head;
2162 halfword q;
2163 set_token_link(p, null);
2164 while (t != null) {
2165 fast_store_new_token(token_info(t));
2166 t = token_link(t);
2167 }
2168 set_token_link(p,s);
2169 set_toks_register(nt,temp_token_head,0);
2170 } else {
2171 while (token_link(t) != null) {
2172 t = token_link(t);
2173 }
2174 set_token_link(t,s);
2175 }
2176 }
2177 }
2178 }
2179 }
2180 } else {
2181 halfword source, ns;
2182 if (cur_cmd == assign_toks_cmd) {
2183 ns = equiv(cur_cs) - toks_base;
2184 /* check range */
2185 } else {
2186 back_input();
2187 scan_int();
2188 ns = cur_val;
2189 }
2190 /* action */
2191 source = toks(ns);
2192 if (source != null) {
2193 halfword target = toks(nt);
2194 if (target == null) {
2195 equiv(toks_base+nt) = source;
2196 equiv(toks_base+ns) = null;
2197 } else {
2198 halfword s = token_link(source);
2199 if (s != null) {
2200 halfword t = token_link(target);
2201 if (t == null) {
2202 set_token_link(target, s);
2203 } else if (odd(how)) {
2204 /* prepend */
2205 halfword x = s;
2206 while (token_link(x) != null) {
2207 x = token_link(x);
2208 }
2209 set_token_link(x,t);
2210 set_token_link(target,s);
2211 } else {
2212 /* append */
2213 while (token_link(t) != null) {
2214 t = token_link(t);
2215 }
2216 set_token_link(t,s);
2217 }
2218 equiv(toks_base+ns) = null;
2219 }
2220 }
2221 }
2222 }
2223 }
2225 @ This routine, used in the next one, prints the job name, possibly modified by
2226 the |process_jobname| callback.
2228 @c
2229 static void print_job_name(void)
2230 {
2231 if (job_name) {
2232 char *s, *ss; /* C strings for jobname before and after processing */
2233 int callback_id, lua_retval;
2234 s = (char*)str_string(job_name);
2235 callback_id = callback_defined(process_jobname_callback);
2236 if (callback_id > 0) {
2238 if ((lua_retval == true) && (ss != NULL))
2239 s = ss;
2240 }
2241 tprint(s);
2242 } else {
2243 print(job_name);
2244 }
2245 }
2247 @ Here is a routine that print the result of a convert command, using the
2248 argument |i|. It returns |false | if it does not know to print the code |c|. The
2249 function exists because lua code and tex code can both call it to convert
2250 something.
2252 @ Parse optional lua state integer, or an instance name to be stored in |sn| and
2253 get the next non-blank non-relax non-call token.
2255 @c
2257 int scan_lua_state(void)
2258 {
2259 int sn = 0;
2260 do {
2261 get_x_token();
2262 } while ((cur_cmd == spacer_cmd) || (cur_cmd == relax_cmd));
2263 back_input();
2264 if (cur_cmd != left_brace_cmd) {
2266 (void) scan_toks(false, true);
2267 sn = def_ref;
2268 } else {
2269 scan_register_num();
2270 if (get_lua_name(cur_val))
2271 sn = (cur_val - 65536);
2272 }
2273 }
2274 return sn;
2275 }
2277 @ The procedure |conv_toks| uses |str_toks| to insert the token list for
2278 |convert| functions into the scanner; `\.{\\outer}' control sequences are allowed
2279 to follow `\.{\\string}' and `\.{\\meaning}'.
2281 The extra temp string |u| is needed because |pdf_scan_ext_toks| incorporates any
2282 pending string in its output. In order to save such a pending string, we have to
2283 create a temporary string that is destroyed immediately after.
2285 @c
2286 #define push_selector { \
2287 old_setting = selector; \
2288 selector = new_string; \
2289 }
2291 #define pop_selector { \
2292 selector = old_setting; \
2293 }
2295 static int do_variable_dvi(halfword c)
2296 {
2297 return 0;
2298 }
2300 #define do_variable_backend_int(i) \
2301 cur_cmd = assign_int_cmd; \
2302 cur_val = backend_int_base + i; \
2303 cur_tok = token_val(cur_cmd, cur_val); \
2304 back_input();
2306 #define do_variable_backend_dimen(i) \
2307 cur_cmd = assign_dimen_cmd; \
2308 cur_val = backend_dimen_base + i; \
2309 cur_tok = token_val(cur_cmd, cur_val); \
2310 back_input();
2312 #define do_variable_backend_toks(i) \
2313 cur_cmd = assign_toks_cmd; \
2314 cur_val = backend_toks_base + i ; \
2315 cur_tok = token_val(cur_cmd, cur_val); \
2316 back_input();
2318 static int do_variable_pdf(halfword c)
2319 {
2327 else if (scan_keyword("inclusionerrorlevel")) { do_variable_backend_int(c_pdf_inclusion_errorlevel); }
2328 else if (scan_keyword("ignoreunknownimages")) { do_variable_backend_int(c_pdf_ignore_unknown_images); }
2330 else if (scan_keyword("imageapplygamma")) { do_variable_backend_int(c_pdf_image_apply_gamma); }
2333 else if (scan_keyword("imageaddfilename")) { do_variable_backend_int(c_pdf_image_addfilename); }
2334 else if (scan_keyword("objcompresslevel")) { do_variable_backend_int(c_pdf_obj_compress_level); }
2335 else if (scan_keyword("inclusioncopyfonts")) { do_variable_backend_int(c_pdf_inclusion_copy_font); }
2338 else if (scan_keyword("suppressoptionalinfo")) { do_variable_backend_int(c_pdf_suppress_optional_info); }
2355 else
2356 return 0;
2357 return 1;
2358 }
2360 static int do_feedback_dvi(halfword c)
2361 {
2362 return 0;
2363 }
2365 /* codes not really needed but cleaner when testing */
2367 #define pdftex_version 40 /* these values will not change any more */
2370 static int do_feedback_pdf(halfword c)
2371 {
2372 int old_setting; /* holds |selector| setting */
2373 int save_scanner_status; /* |scanner_status| upon entry */
2374 halfword save_def_ref; /* |def_ref| upon entry, important if inside `\.{\\message}' */
2375 halfword save_warning_index;
2376 boolean bool; /* temp boolean */
2377 str_number s; /* first temp string */
2378 int ff; /* for use with |set_ff| */
2379 str_number u = 0; /* third temp string, will become non-nil if a string is already being built */
2380 char *str; /* color stack init str */
2383 push_selector;
2384 print_int(pdf_last_link);
2385 pop_selector;
2387 push_selector;
2388 print_int(pdf_retval);
2389 pop_selector;
2391 push_selector;
2392 print_int(pdf_last_obj);
2393 pop_selector;
2395 push_selector;
2396 print_int(pdf_last_annot);
2397 pop_selector;
2399 scan_int();
2400 check_obj_type(static_pdf, obj_type_xform, cur_val);
2401 push_selector;
2402 print_int(obj_info(static_pdf, cur_val));
2403 pop_selector;
2405 ins_list(string_to_toks(getcreationdate(static_pdf)));
2406 /* no further action */
2407 return 2;
2409 scan_font_ident();
2410 if (cur_val == null_font)
2412 pdf_check_vf(cur_val);
2413 if (!font_used(cur_val))
2414 pdf_init_font(static_pdf, cur_val);
2415 push_selector;
2416 set_ff(cur_val);
2417 print_int(obj_info(static_pdf, pdf_font_num(ff)));
2418 pop_selector;
2420 scan_font_ident();
2421 if (cur_val == null_font)
2423 pdf_check_vf(cur_val);
2424 if (!font_used(cur_val))
2425 pdf_init_font(static_pdf, cur_val);
2426 push_selector;
2427 set_ff(cur_val);
2428 print_int(pdf_font_num(ff));
2429 pop_selector;
2431 scan_font_ident();
2432 if (cur_val == null_font)
2434 push_selector;
2435 print_scaled(font_size(cur_val));
2437 pop_selector;
2439 scan_int();
2440 if (cur_val <= 0)
2442 push_selector;
2443 print_int(pdf_get_obj(static_pdf, obj_type_page, cur_val, false));
2444 pop_selector;
2448 cur_val = direct_always;
2450 cur_val = direct_page;
2451 else
2452 cur_val = set_origin;
2453 save_scanner_status = scanner_status;
2454 save_warning_index = warning_index;
2455 save_def_ref = def_ref;
2456 u = save_cur_string();
2457 scan_toks(false, true);
2458 s = tokens_to_string(def_ref);
2459 delete_token_ref(def_ref);
2460 def_ref = save_def_ref;
2461 warning_index = save_warning_index;
2462 scanner_status = save_scanner_status;
2463 str = makecstring(s);
2464 cur_val = newcolorstack(str, cur_val, bool);
2465 free(str);
2466 flush_str(s);
2467 cur_val_level = int_val_level;
2468 if (cur_val < 0) {
2472 error();
2473 cur_val = 0;
2474 restore_cur_string(u);
2475 }
2476 push_selector;
2477 print_int(cur_val);
2478 pop_selector;
2480 push_selector;
2481 print_int(pdftex_version);
2482 pop_selector;
2484 ins_list(string_to_toks(pdftex_revision));
2485 return 2;
2486 } else {
2487 return 0;
2488 }
2489 return 1;
2490 }
2492 void conv_toks(void)
2493 {
2494 int old_setting; /* holds |selector| setting */
2495 halfword p, q;
2496 int save_scanner_status; /* |scanner_status| upon entry */
2497 halfword save_def_ref; /* |def_ref| upon entry, important if inside `\.{\\message}' */
2498 halfword save_warning_index;
2499 boolean bool; /* temp boolean */
2500 str_number s; /* first temp string */
2501 int sn; /* lua chunk name */
2502 str_number u = 0; /* third temp string, will become non-nil if a string is already being built */
2503 int c = cur_chr; /* desired type of conversion */
2504 str_number str;
2505 int i = 0;
2506 /* Scan the argument for command |c| */
2507 switch (c) {
2508 case number_code:
2509 scan_int();
2510 push_selector;
2511 print_int(cur_val);
2512 pop_selector;
2513 break;
2514 case lua_function_code:
2515 scan_int();
2516 if (cur_val <= 0) {
2518 } else {
2519 u = save_cur_string();
2520 luacstrings = 0;
2521 luafunctioncall(cur_val);
2522 restore_cur_string(u);
2523 if (luacstrings > 0)
2524 lua_string_start();
2525 }
2526 /* no further action */
2527 return;
2528 break;
2529 case lua_code:
2530 u = save_cur_string();
2531 save_scanner_status = scanner_status;
2532 save_def_ref = def_ref;
2533 save_warning_index = warning_index;
2534 sn = scan_lua_state();
2535 scan_toks(false, true);
2536 s = def_ref;
2537 warning_index = save_warning_index;
2538 def_ref = save_def_ref;
2539 scanner_status = save_scanner_status;
2540 luacstrings = 0;
2541 luatokencall(s, sn);
2542 delete_token_ref(s);
2543 restore_cur_string(u); /* TODO: check this, was different */
2544 if (luacstrings > 0)
2545 lua_string_start();
2546 /* no further action */
2547 return;
2548 break;
2549 case expanded_code:
2550 save_scanner_status = scanner_status;
2551 save_warning_index = warning_index;
2552 save_def_ref = def_ref;
2553 u = save_cur_string();
2554 scan_toks(false, true);
2555 warning_index = save_warning_index;
2556 scanner_status = save_scanner_status;
2557 ins_list(token_link(def_ref));
2558 def_ref = save_def_ref;
2559 restore_cur_string(u);
2560 /* no further action */
2561 return;
2562 break;
2563 case math_style_code:
2564 push_selector;
2565 print_math_style();
2566 pop_selector;
2567 break;
2568 case string_code:
2569 save_scanner_status = scanner_status;
2570 scanner_status = normal;
2571 get_token();
2572 scanner_status = save_scanner_status;
2573 push_selector;
2574 if (cur_cs != 0)
2575 sprint_cs(cur_cs);
2576 else
2577 print(cur_chr);
2578 pop_selector;
2579 break;
2580 case cs_string_code:
2581 save_scanner_status = scanner_status;
2582 scanner_status = normal;
2583 get_token();
2584 scanner_status = save_scanner_status;
2585 push_selector;
2586 if (cur_cs != 0)
2587 sprint_cs_name(cur_cs);
2588 else
2589 print(cur_chr);
2590 pop_selector;
2591 break;
2592 case roman_numeral_code:
2593 scan_int();
2594 push_selector;
2595 print_roman_int(cur_val);
2596 pop_selector;
2597 break;
2598 case meaning_code:
2599 save_scanner_status = scanner_status;
2600 scanner_status = normal;
2601 get_token();
2602 scanner_status = save_scanner_status;
2603 push_selector;
2604 print_meaning();
2605 pop_selector;
2606 break;
2607 case uchar_code:
2608 scan_char_num();
2609 push_selector;
2610 print(cur_val);
2611 pop_selector;
2612 break;
2613 case lua_escape_string_code:
2614 {
2615 lstring escstr;
2616 int l = 0;
2617 save_scanner_status = scanner_status;
2618 save_def_ref = def_ref;
2619 save_warning_index = warning_index;
2620 scan_toks(false, true);
2621 bool = in_lua_escape;
2622 in_lua_escape = true;
2623 escstr.s = (unsigned char *) tokenlist_to_cstring(def_ref, false, &l);
2624 escstr.l = (unsigned) l;
2625 in_lua_escape = bool;
2626 delete_token_ref(def_ref);
2627 def_ref = save_def_ref;
2628 warning_index = save_warning_index;
2629 scanner_status = save_scanner_status;
2630 (void) lua_str_toks(escstr);
2631 ins_list(token_link(temp_token_head));
2632 free(escstr.s);
2633 return;
2634 }
2635 /* no further action */
2636 break;
2637 case font_id_code:
2638 scan_font_ident();
2639 push_selector;
2640 print_int(cur_val);
2641 pop_selector;
2642 break;
2643 case font_name_code:
2644 scan_font_ident();
2645 push_selector;
2646 append_string((unsigned char *) font_name(cur_val),(unsigned) strlen(font_name(cur_val)));
2647 if (font_size(cur_val) != font_dsize(cur_val)) {
2649 print_scaled(font_size(cur_val));
2651 }
2652 pop_selector;
2653 break;
2654 case left_margin_kern_code:
2655 scan_int();
2656 if ((box(cur_val) == null) || (type(box(cur_val)) != hlist_node))
2658 push_selector;
2659 p = list_ptr(box(cur_val));
2660 while ((p != null) && (type(p) == glue_node)) {
2661 p = vlink(p);
2662 }
2663 if ((p != null) && (type(p) == margin_kern_node) && (subtype(p) == left_side))
2664 print_scaled(width(p));
2665 else
2666 print_char('0');
2668 pop_selector;
2669 break;
2670 case right_margin_kern_code:
2671 scan_int();
2672 if ((box(cur_val) == null) || (type(box(cur_val)) != hlist_node))
2674 push_selector;
2675 p = list_ptr(box(cur_val));
2676 if (p != null) {
2677 p = tail_of_list(p);
2678 /*
2679 there can be a leftskip, rightskip, penalty and yes, also a disc node with a nesting
2680 node that points to glue spec ... and we don't want to analyze that messy lot
2681 */
2682 while ((p != null) && (type(p) == glue_node)) {
2683 p = alink(p);
2684 }
2685 if ((p != null) && ! ((type(p) == margin_kern_node) && (subtype(p) == right_side))) {
2686 if (type(p) == disc_node) {
2687 q = alink(p);
2688 if ((q != null) && ((type(q) == margin_kern_node) && (subtype(q) == right_side))) {
2689 p = q;
2690 } else {
2691 /*
2692 officially we should look in the replace but currently protrusion doesn't
2694 don't need it now
2695 */
2696 }
2697 }
2698 }
2699 }
2700 if ((p != null) && (type(p) == margin_kern_node) && (subtype(p) == right_side))
2701 print_scaled(width(p));
2702 else
2703 print_char('0');
2705 pop_selector;
2706 break;
2707 case uniform_deviate_code:
2708 scan_int();
2709 push_selector;
2710 print_int(unif_rand(cur_val));
2711 pop_selector;
2712 break;
2713 case normal_deviate_code:
2714 scan_int();
2715 push_selector;
2716 print_int(norm_rand());
2717 pop_selector;
2718 break;
2719 case math_char_class_code:
2720 {
2721 mathcodeval mval;
2722 scan_int();
2723 mval = get_math_code(cur_val);
2724 push_selector;
2725 print_int(mval.class_value);
2726 pop_selector;
2727 }
2728 break;
2729 case math_char_fam_code:
2730 {
2731 mathcodeval mval;
2732 scan_int();
2733 mval = get_math_code(cur_val);
2734 push_selector;
2735 print_int(mval.family_value);
2736 pop_selector;
2737 }
2738 break;
2739 case math_char_slot_code:
2740 {
2741 mathcodeval mval;
2742 scan_int();
2743 mval = get_math_code(cur_val);
2744 push_selector;
2745 print_int(mval.character_value);
2746 pop_selector;
2747 }
2748 break;
2749 case insert_ht_code:
2750 scan_register_num();
2751 push_selector;
2752 i = cur_val;
2753 p = page_ins_head;
2754 while (i >= subtype(vlink(p)))
2755 p = vlink(p);
2756 if (subtype(p) == i)
2757 print_scaled(height(p));
2758 else
2759 print_char('0');
2761 pop_selector;
2762 break;
2763 case job_name_code:
2764 if (job_name == 0)
2765 open_log_file();
2766 push_selector;
2767 print_job_name();
2768 pop_selector;
2769 break;
2770 case format_name_code:
2771 if (job_name == 0)
2772 open_log_file();
2773 push_selector;
2774 print(format_name);
2775 pop_selector;
2776 break;
2777 case luatex_banner_code:
2778 push_selector;
2779 tprint(luatex_banner);
2780 pop_selector;
2781 break;
2782 case luatex_revision_code:
2783 push_selector;
2784 print(get_luatexrevision());
2785 pop_selector;
2786 break;
2787 case luatex_date_code:
2788 push_selector;
2789 print_int(get_luatex_date_info());
2790 pop_selector;
2791 break;
2792 case etex_code:
2793 push_selector;
2794 tprint(eTeX_version_string);
2795 pop_selector;
2796 break;
2797 case eTeX_revision_code:
2798 push_selector;
2799 tprint(eTeX_revision);
2800 pop_selector;
2801 break;
2802 case font_identifier_code:
2804 break;
2805 default:
2807 break;
2808 }
2809 str = make_string();
2810 (void) str_toks(str_lstring(str));
2811 flush_str(str);
2812 ins_list(token_link(temp_token_head));
2813 }
2815 void do_feedback(void)
2816 {
2817 int c = cur_chr;
2818 str_number str;
2819 int done = 1;
2820 switch (c) {
2821 case dvi_feedback_code:
2822 if (get_o_mode() == OMODE_DVI) {
2823 done = do_feedback_dvi(c);
2824 } else {
2826 return ;
2827 }
2828 if (done==0) {
2829 /* we recover */
2831 return;
2832 } else if (done==2) {
2833 return;
2834 }
2835 break;
2836 case pdf_feedback_code:
2837 if (get_o_mode() == OMODE_PDF) {
2838 done = do_feedback_pdf(c);
2839 } else {
2841 return ;
2842 }
2843 if (done==0) {
2844 /* we recover */
2846 return;
2847 } else if (done==2) {
2848 return;
2849 }
2850 break;
2851 default:
2853 break;
2854 }
2855 str = make_string();
2856 (void) str_toks(str_lstring(str));
2857 flush_str(str);
2858 ins_list(token_link(temp_token_head));
2859 }
2861 void do_variable(void)
2862 {
2863 int c = cur_chr;
2864 int done = 1;
2865 switch (c) {
2866 case dvi_variable_code:
2867 done = do_variable_dvi(c);
2868 if (done==0) {
2869 /* we recover */
2871 }
2872 return;
2873 break;
2874 case pdf_variable_code:
2875 done = do_variable_pdf(c);
2876 if (done==0) {
2877 /* we recover */
2879 }
2880 return;
2881 break;
2882 default:
2884 break;
2885 }
2886 }
2888 /*
2889 The following code is not used as we can only set math options and not query them. If
2890 an option is really important we will provide a proper variable. Most options are not
2891 meant for users anyway but for development.
2892 */
2894 /*
2896 #define do_mathoption_int(i) \
2897 cur_cmd = assign_int_cmd; \
2898 cur_val = mathoption_int_base + i; \
2899 cur_tok = token_val(cur_cmd, cur_val); \
2900 back_input();
2902 void do_mathoption(void)
2903 {
2905 if (scan_keyword("noitaliccompensation")) { do_mathoption_int(c_mathoption_no_char_italic_code); }
2906 else if (scan_keyword("nocharitalic")) { do_mathoption_int(c_mathoption_use_old_fraction_scaling_code); }
2908 else if (scan_keyword("umathcodemeaning")) { do_mathoption_int(c_mathoption_umathcode_meaning_code); }
2909 }
2911 */
2913 @ This boolean is keeping track of the lua string escape state
2914 @c
2915 boolean in_lua_escape;
2917 static int the_convert_string_dvi(halfword c, int i)
2918 {
2919 return 0 ;
2920 }
2922 static int the_convert_string_pdf(halfword c, int i)
2923 {
2924 int ff;
2925 if (get_o_mode() != OMODE_PDF) {
2926 return 0;
2928 print_int(pdf_last_link);
2930 print_int(pdf_retval);
2932 print_int(pdf_last_obj);
2934 print_int(pdf_last_annot);
2936 print_int(obj_info(static_pdf, i));
2938 return 0;
2940 set_ff(i);
2941 print_int(obj_info(static_pdf, pdf_font_num(ff)));
2943 set_ff(i);
2944 print_int(pdf_font_num(ff));
2946 print_scaled(font_size(i));
2949 print_int(pdf_get_obj(static_pdf, obj_type_page, i, false));
2951 return 0;
2952 } else {
2953 return 0;
2954 }
2955 return 1;
2956 }
2958 str_number the_convert_string(halfword c, int i)
2959 {
2960 int old_setting; /* saved |selector| setting */
2961 str_number ret = 0;
2962 boolean done = true ;
2963 old_setting = selector;
2964 selector = new_string;
2965 switch (c) {
2966 case number_code:
2967 print_int(i);
2968 break;
2969 /* case lua_function_code: */
2970 /* case lua_code: */
2971 /* case expanded_code: */
2972 case math_style_code:
2973 print_math_style();
2974 break;
2975 /* case string_code: */
2976 /* case cs_string_code: */
2977 case roman_numeral_code:
2978 print_roman_int(i);
2979 break;
2980 /* case meaning_code: */
2981 case uchar_code:
2982 print(i);
2983 break;
2984 /* lua_escape_string_code: */
2985 case font_id_code:
2986 print_int(i);
2987 break;
2988 case font_name_code:
2989 append_string((unsigned char *) font_name(i),(unsigned) strlen(font_name(i)));
2990 if (font_size(i) != font_dsize(i)) {
2992 print_scaled(font_size(i));
2994 }
2995 break;
2996 /* left_margin_kern_code: */
2997 /* right_margin_kern_code: */
2998 case uniform_deviate_code:
2999 print_int(unif_rand(i));
3000 break;
3001 case normal_deviate_code:
3002 print_int(norm_rand());
3003 break;
3004 /* math_char_class_code: */
3005 /* math_char_fam_code: */
3006 /* math_char_slot_code: */
3007 /* insert_ht_code: */
3008 case job_name_code:
3009 print_job_name();
3010 break;
3011 case format_name_code:
3012 print(format_name);
3013 break;
3014 case luatex_banner_code:
3015 tprint(luatex_banner);
3016 break;
3017 case luatex_revision_code:
3018 print(get_luatexrevision());
3019 break;
3020 case luatex_date_code:
3021 print_int(get_luatex_date_info());
3022 break;
3023 case etex_code:
3024 tprint(eTeX_version_string);
3025 break;
3026 case eTeX_revision_code:
3027 tprint(eTeX_revision);
3028 break;
3029 case font_identifier_code:
3030 print_font_identifier(i);
3031 break;
3032 /* backend: this might become obsolete */
3033 case dvi_feedback_code:
3034 done = the_convert_string_dvi(c,i);
3035 break;
3036 case pdf_feedback_code:
3037 done = the_convert_string_pdf(c,i);
3038 break;
3039 /* done */
3040 default:
3041 done = false;
3042 break;
3043 }
3044 if (done) {
3045 ret = make_string();
3046 }
3047 selector = old_setting;
3048 return ret;
3049 }
3051 @ Another way to create a token list is via the \.{\\read} command. The sixteen
3052 files potentially usable for reading appear in the following global variables.
3053 The value of |read_open[n]| will be |closed| if stream number |n| has not been
3054 opened or if it has been fully read; |just_open| if an \.{\\openin} but not a
3055 \.{\\read} has been done; and |normal| if it is open and ready to read the next
3056 line.
3058 @c
3059 FILE *read_file[16]; /* used for \.{\\read} */
3060 int read_open[17]; /* state of |read_file[n]| */
3062 void initialize_read(void)
3063 {
3064 int k;
3065 for (k = 0; k <= 16; k++)
3066 read_open[k] = closed;
3067 }
3069 @ The |read_toks| procedure constructs a token list like that for any macro
3070 definition, and makes |cur_val| point to it. Parameter |r| points to the control
3071 sequence that will receive this token list.
3073 @c
3074 void read_toks(int n, halfword r, halfword j)
3075 {
3076 halfword p; /* tail of the token list */
3077 halfword q; /* new node being added to the token list via |store_new_token| */
3078 int s; /* saved value of |align_state| */
3079 int m; /* stream number */
3080 scanner_status = defining;
3081 warning_index = r;
3082 p = get_avail();
3083 def_ref = p;
3084 set_token_ref_count(def_ref, 0);
3085 p = def_ref; /* the reference count */
3086 store_new_token(end_match_token);
3087 if ((n < 0) || (n > 15))
3088 m = 16;
3089 else
3090 m = n;
3091 s = align_state;
3092 align_state = 1000000; /* disable tab marks, etc. */
3093 do {
3094 /* Input and store tokens from the next line of the file */
3095 begin_file_reading();
3096 iname = m + 1;
3097 if (read_open[m] == closed) {
3098 /*
3099 Input for \.{\\read} from the terminal
3101 Here we input on-line into the |buffer| array, prompting the user explicitly
3102 if |n>=0|. The value of |n| is set negative so that additional prompts
3103 will not be given in the case of multi-line input.
3104 */
3105 if (interaction > nonstop_mode) {
3106 if (n < 0) {
3108 } else {
3109 wake_up_terminal();
3110 print_ln();
3111 sprint_cs(r);
3113 n = -1;
3114 }
3115 } else {
3116 fatal_error
3118 }
3120 } else if (read_open[m] == just_open) {
3121 /*
3122 Input the first line of |read_file[m]|
3124 The first line of a file must be treated specially, since |lua_input_ln|
3125 must be told not to start with |get|.
3126 */
3127 if (lua_input_ln(read_file[m], (m + 1), false)) {
3128 read_open[m] = normal;
3129 } else {
3130 lua_a_close_in(read_file[m], (m + 1));
3131 read_open[m] = closed;
3132 }
3134 } else {
3135 /*
3136 Input the next line of |read_file[m]|
3138 An empty line is appended at the end of a |read_file|.
3139 */
3140 if (!lua_input_ln(read_file[m], (m + 1), true)) {
3141 lua_a_close_in(read_file[m], (m + 1));
3142 read_open[m] = closed;
3143 if (align_state != 1000000) {
3144 runaway();
3147 align_state = 1000000;
3148 error();
3149 }
3150 }
3152 }
3153 ilimit = last;
3154 if (end_line_char_inactive)
3155 decr(ilimit);
3156 else
3157 buffer[ilimit] = (packed_ASCII_code) int_par(end_line_char_code);
3158 first = ilimit + 1;
3159 iloc = istart;
3160 istate = new_line;
3161 /* Handle \.{\\readline} and |goto done|; */
3162 if (j == 1) {
3163 while (iloc <= ilimit) {
3164 /* current line not yet finished */
3165 do_buffer_to_unichar(cur_chr, iloc);
3166 if (cur_chr == ' ')
3167 cur_tok = space_token;
3168 else
3169 cur_tok = cur_chr + other_token;
3170 store_new_token(cur_tok);
3171 }
3172 } else {
3173 while (1) {
3174 get_token();
3175 if (cur_tok == 0) {
3176 /* |cur_cmd=cur_chr=0| will occur at the end of the line */
3177 break;
3178 }
3179 if (align_state < 1000000) {
3180 /* unmatched `\.\}' aborts the line */
3181 do {
3182 get_token();
3183 } while (cur_tok != 0);
3184 align_state = 1000000;
3185 break;
3186 }
3187 store_new_token(cur_tok);
3188 }
3189 }
3190 end_file_reading();
3192 } while (align_state != 1000000);
3193 cur_val = def_ref;
3194 scanner_status = normal;
3195 align_state = s;
3196 }
3198 @ return a string from tokens list
3200 @c
3201 str_number tokens_to_string(halfword p)
3202 {
3203 int old_setting;
3204 if (selector == new_string)
3206 old_setting = selector;
3207 selector = new_string;
3208 show_token_list(token_link(p), null, -1);
3209 selector = old_setting;
3210 return make_string();
3211 }
3213 @ @c
3214 #define make_room(a) \
3215 if ((unsigned)i+a+1>alloci) { \
3216 ret = xrealloc(ret,(alloci+64)); \
3217 alloci = alloci + 64; \
3218 }
3220 #define append_i_byte(a) ret[i++] = (char)(a)
3222 #define Print_char(a) make_room(1); append_i_byte(a)
3224 #define Print_uchar(s) { \
3225 make_room(4); \
3226 if (s<=0x7F) { \
3227 append_i_byte(s); \
3228 } else if (s<=0x7FF) { \
3229 append_i_byte(0xC0 + (s / 0x40)); \
3230 append_i_byte(0x80 + (s % 0x40)); \
3231 } else if (s<=0xFFFF) { \
3232 append_i_byte(0xE0 + (s / 0x1000)); \
3233 append_i_byte(0x80 + ((s % 0x1000) / 0x40)); \
3234 append_i_byte(0x80 + ((s % 0x1000) % 0x40)); \
3235 } else if (s>=0x110000) { \
3236 append_i_byte(s-0x11000); \
3237 } else { \
3238 append_i_byte(0xF0 + (s / 0x40000)); \
3239 append_i_byte(0x80 + ((s % 0x40000) / 0x1000)); \
3240 append_i_byte(0x80 + (((s % 0x40000) % 0x1000) / 0x40)); \
3241 append_i_byte(0x80 + (((s % 0x40000) % 0x1000) % 0x40)); \
3242 } }
3244 #define Print_esc(b) { \
3245 const char *v = b; \
3246 if (e>0 && e<STRING_OFFSET) { \
3247 Print_uchar (e); \
3248 } \
3249 make_room(strlen(v)); \
3250 while (*v) { append_i_byte(*v); v++; } \
3251 }
3253 #define Print_str(b) { \
3254 const char *v = b; \
3255 make_room(strlen(v)); \
3256 while (*v) { append_i_byte(*v); v++; } \
3257 }
3259 #define is_cat_letter(a) \
3260 (get_char_cat_code(pool_to_unichar(str_string((a)))) == 11)
3262 @ the actual token conversion in this function is now functionally equivalent to
3263 |show_token_list|, except that it always prints the whole token list. TODO: check
3264 whether this causes problems in the lua library.
3266 @c
3267 char *tokenlist_to_cstring(int pp, int inhibit_par, int *siz)
3268 {
3269 register int p, c, m;
3270 int q;
3271 int infop;
3272 char *s, *sh;
3273 int e = 0;
3274 char *ret;
3275 int match_chr = '#';
3276 int n = '0';
3277 unsigned alloci = 1024;
3278 int i = 0;
3279 p = pp;
3280 if (p == null) {
3281 if (siz != NULL)
3282 *siz = 0;
3283 return NULL;
3284 }
3285 ret = xmalloc(alloci);
3286 p = token_link(p); /* skip refcount */
3287 if (p != null) {
3288 e = int_par(escape_char_code);
3289 }
3290 while (p != null) {
3291 if (p < (int) fix_mem_min || p > (int) fix_mem_end) {
3293 break;
3294 }
3295 infop = token_info(p);
3296 if (infop >= cs_token_flag) {
3297 if (!(inhibit_par && infop == par_token)) {
3298 q = infop - cs_token_flag;
3299 if (q < hash_base) {
3300 if (q == null_cs) {
3303 } else {
3305 }
3306 } else if ((q >= undefined_control_sequence) && ((q <= eqtb_size) || (q > eqtb_size + hash_extra))) {
3308 } else if ((cs_text(q) < 0) || (cs_text(q) >= str_ptr)) {
3310 } else {
3311 str_number txt = cs_text(q);
3312 sh = makecstring(txt);
3313 s = sh;
3314 if (is_active_cs(txt)) {
3315 s = s + 3;
3316 while (*s) {
3317 Print_char(*s);
3318 s++;
3319 }
3320 } else {
3321 if (e>=0 && e<0x110000) Print_uchar(e);
3322 while (*s) {
3323 Print_char(*s);
3324 s++;
3325 }
3326 if ((!single_letter(txt)) || is_cat_letter(txt)) {
3327 Print_char(' ');
3328 }
3329 }
3330 free(sh);
3331 }
3332 }
3333 } else {
3334 if (infop < 0) {
3336 } else {
3337 m = token_cmd(infop);
3338 c = token_chr(infop);
3339 switch (m) {
3340 case left_brace_cmd:
3341 case right_brace_cmd:
3342 case math_shift_cmd:
3343 case tab_mark_cmd:
3344 case sup_mark_cmd:
3345 case sub_mark_cmd:
3346 case spacer_cmd:
3347 case letter_cmd:
3348 case other_char_cmd:
3349 Print_uchar(c);
3350 break;
3351 case mac_param_cmd:
3352 if (!in_lua_escape && (is_in_csname==0))
3353 Print_uchar(c);
3354 Print_uchar(c);
3355 break;
3356 case out_param_cmd:
3357 Print_uchar(match_chr);
3358 if (c <= 9) {
3359 Print_char(c + '0');
3360 } else {
3361 Print_char('!');
3362 goto EXIT;
3363 }
3364 break;
3365 case match_cmd:
3366 match_chr = c;
3367 Print_uchar(c);
3368 n++;
3369 Print_char(n);
3370 if (n > '9')
3371 goto EXIT;
3372 break;
3373 case end_match_cmd:
3374 if (c == 0) {
3375 Print_char('-');
3376 Print_char('>');
3377 }
3378 break;
3379 default:
3380 not_so_bad(Print_esc);
3381 break;
3382 }
3383 }
3384 }
3385 p = token_link(p);
3386 }
3387 EXIT:
3388 ret[i] = '\0';
3389 if (siz != NULL)
3390 *siz = i;
3391 return ret;
3392 }
3394 @ @c
3395 lstring *tokenlist_to_lstring(int pp, int inhibit_par)
3396 {
3397 int siz;
3398 lstring *ret = xmalloc(sizeof(lstring));
3399 ret->s = (unsigned char *) tokenlist_to_cstring(pp, inhibit_par, &siz);
3400 ret->l = (size_t) siz;
3401 return ret;
3402 }
3404 @ @c
3405 void free_lstring(lstring * ls)
3406 {
3407 if (ls == NULL)
3408 return;
3409 if (ls->s != NULL)
3410 free(ls->s);
3411 free(ls);
3412 }