1 /* $Id: read.c,v 1.196 2018/07/28 18:34:15 schwarze Exp $ */
3 * Copyright (c) 2008, 2009, 2010, 2011 Kristaps Dzonsons <kristaps@bsd.lv>
4 * Copyright (c) 2010-2018 Ingo Schwarze <schwarze@openbsd.org>
5 * Copyright (c) 2010, 2012 Joerg Sonnenberger <joerg@netbsd.org>
7 * Permission to use, copy, modify, and distribute this software for any
8 * purpose with or without fee is hereby granted, provided that the above
9 * copyright notice and this permission notice appear in all copies.
11 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHORS DISCLAIM ALL WARRANTIES
12 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
13 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR
14 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
15 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
16 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
17 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
21 #include <sys/types.h>
36 #include "mandoc_aux.h"
41 #include "libmandoc.h"
43 #define REPARSE_LIMIT 1000
46 struct roff
*roff
; /* roff parser (!NULL) */
47 struct roff_man
*man
; /* man parser */
48 char *sodest
; /* filename pointed to by .so */
49 const char *file
; /* filename of current input file */
50 struct buf
*primary
; /* buffer currently being parsed */
51 struct buf
*secondary
; /* preprocessed copy of input */
52 const char *os_s
; /* default operating system */
53 mandocmsg mmsg
; /* warning/error message handler */
54 enum mandoclevel file_status
; /* status of current parse */
55 enum mandocerr mmin
; /* ignore messages below this */
56 int options
; /* parser options */
57 int gzip
; /* current input file is gzipped */
58 int filenc
; /* encoding of the current file */
59 int reparse_count
; /* finite interp. stack */
60 int line
; /* line number in the file */
63 static void choose_parser(struct mparse
*);
64 static void resize_buf(struct buf
*, size_t);
65 static int mparse_buf_r(struct mparse
*, struct buf
, size_t, int);
66 static int read_whole_file(struct mparse
*, const char *, int,
68 static void mparse_end(struct mparse
*);
69 static void mparse_parse_buffer(struct mparse
*, struct buf
,
72 static const enum mandocerr mandoclimits
[MANDOCLEVEL_MAX
] = {
82 static const char * const mandocerrs
[MANDOCERR_MAX
] = {
85 "base system convention",
89 "unknown architecture",
90 "operating system explicitly specified",
92 "referenced manual not found",
94 "generic style suggestion",
96 "legacy man(7) date format",
97 "normalizing date format to",
98 "lower case character in document title",
100 "possible typo in section name",
101 "unterminated quoted argument",
103 "consider using OS macro",
104 "errnos out of order",
106 "trailing delimiter",
107 "no blank before trailing delimiter",
108 "fill mode already enabled, skipping",
109 "fill mode already disabled, skipping",
110 "verbatim \"--\", maybe consider using \\(em",
111 "function name without markup",
112 "whitespace at end of input line",
117 /* related to the prologue */
118 "missing manual title, using UNTITLED",
119 "missing manual title, using \"\"",
120 "missing manual section, using \"\"",
121 "unknown manual section",
122 "missing date, using today's date",
123 "cannot parse date, using it verbatim",
124 "date in the future, using it anyway",
125 "missing Os macro, using \"\"",
126 "late prologue macro",
127 "prologue macros out of order",
129 /* related to document structure */
130 ".so is fragile, better use ln(1)",
132 "content before first section header",
133 "first section is not \"NAME\"",
134 "NAME section without Nm before Nd",
135 "NAME section without description",
136 "description not at the end of NAME",
137 "bad NAME section content",
138 "missing comma before name",
139 "missing description line, using \"\"",
140 "description line outside NAME section",
141 "sections out of conventional order",
142 "duplicate section title",
143 "unexpected section",
144 "cross reference to self",
146 "unusual Xr punctuation",
147 "AUTHORS section without An macro",
149 /* related to macros and nesting */
151 "macro neither callable nor escaped",
152 "skipping paragraph macro",
153 "moving paragraph macro out of list",
154 "skipping no-space macro",
155 "blocks badly nested",
156 "nested displays are not portable",
157 "moving content out of list",
158 "first macro on line",
160 "skipping blank line in line scope",
162 /* related to missing macro arguments */
163 "skipping empty request",
164 "conditional request controls empty scope",
165 "skipping empty macro",
167 "empty argument, using 0n",
168 "missing display type, using -ragged",
169 "list type is not the first argument",
170 "missing -width in -tag list, using 6n",
171 "missing utility name, using \"\"",
172 "missing function name, using \"\"",
173 "empty head in list item",
175 "missing argument, using next line",
176 "missing font type, using \\fR",
177 "unknown font type, using \\fR",
178 "nothing follows prefix",
179 "empty reference block",
180 "missing section argument",
181 "missing -std argument, adding it",
182 "missing option string, using \"\"",
183 "missing resource identifier, using \"\"",
184 "missing eqn box, using \"\"",
186 /* related to bad macro arguments */
187 "duplicate argument",
188 "skipping duplicate argument",
189 "skipping duplicate display type",
190 "skipping duplicate list type",
191 "skipping -width argument",
192 "wrong number of cells",
193 "unknown AT&T UNIX version",
194 "comma in function argument",
195 "parenthesis in function name",
196 "unknown library name",
197 "invalid content in Rs block",
198 "invalid Boolean argument",
199 "unknown font, skipping request",
200 "odd number of characters in request",
202 /* related to plain text */
203 "blank line in fill mode, using .sp",
204 "tab in filled text",
205 "new sentence, new line",
206 "invalid escape sequence",
207 "undefined string, using \"\"",
209 /* related to tables */
210 "tbl line starts with span",
211 "tbl column starts with span",
212 "skipping vertical bar in tbl layout",
216 /* related to tables */
217 "non-alphabetic character in tbl options",
218 "skipping unknown tbl option",
219 "missing tbl option argument",
220 "wrong tbl option argument size",
222 "invalid character in tbl layout",
223 "unmatched parenthesis in tbl layout",
224 "tbl without any data cells",
225 "ignoring data in spanned tbl cell",
226 "ignoring extra tbl data cells",
227 "data block open at end of tbl",
229 /* related to document structure and macros */
231 "duplicate prologue macro",
232 "skipping late title macro",
233 "input stack limit exceeded, infinite loop?",
234 "skipping bad character",
235 "skipping unknown macro",
236 "skipping insecure request",
237 "skipping item outside list",
238 "skipping column outside column list",
239 "skipping end of block that is not open",
240 "fewer RS blocks open, skipping",
241 "inserting missing end of block",
242 "appending missing end of block",
244 /* related to request and macro arguments */
245 "escaped character not allowed in a name",
246 "NOT IMPLEMENTED: Bd -file",
247 "skipping display without arguments",
248 "missing list type, using -item",
249 "argument is not numeric, using 1",
250 "missing manual name, using \"\"",
251 "uname(3) system call failed, using UNKNOWN",
252 "unknown standard specifier",
253 "skipping request without numeric argument",
254 "NOT IMPLEMENTED: .so with absolute path or \"..\"",
255 ".so request failed",
256 "skipping all arguments",
257 "skipping excess arguments",
260 "unsupported feature",
262 "unsupported control character",
263 "unsupported roff request",
264 "eqn delim option in tbl",
265 "unsupported tbl layout modifier",
266 "ignoring macro in table",
269 static const char * const mandoclevels
[MANDOCLEVEL_MAX
] = {
281 resize_buf(struct buf
*buf
, size_t initial
)
284 buf
->sz
= buf
->sz
> initial
/2 ? 2 * buf
->sz
: initial
;
285 buf
->buf
= mandoc_realloc(buf
->buf
, buf
->sz
);
289 choose_parser(struct mparse
*curp
)
295 * If neither command line arguments -mdoc or -man select
296 * a parser nor the roff parser found a .Dd or .TH macro
297 * yet, look ahead in the main input buffer.
300 if ((format
= roff_getformat(curp
->roff
)) == 0) {
301 cp
= curp
->primary
->buf
;
302 ep
= cp
+ curp
->primary
->sz
;
304 if (*cp
== '.' || *cp
== '\'') {
306 if (cp
[0] == 'D' && cp
[1] == 'd') {
307 format
= MPARSE_MDOC
;
310 if (cp
[0] == 'T' && cp
[1] == 'H') {
315 cp
= memchr(cp
, '\n', ep
- cp
);
322 if (format
== MPARSE_MDOC
) {
323 curp
->man
->macroset
= MACROSET_MDOC
;
324 if (curp
->man
->mdocmac
== NULL
)
325 curp
->man
->mdocmac
= roffhash_alloc(MDOC_Dd
, MDOC_MAX
);
327 curp
->man
->macroset
= MACROSET_MAN
;
328 if (curp
->man
->manmac
== NULL
)
329 curp
->man
->manmac
= roffhash_alloc(MAN_TH
, MAN_MAX
);
331 curp
->man
->first
->tok
= TOKEN_NONE
;
335 * Main parse routine for a buffer.
336 * It assumes encoding and line numbering are already set up.
337 * It can recurse directly (for invocations of user-defined
338 * macros, inline equations, and input line traps)
339 * and indirectly (for .so file inclusion).
342 mparse_buf_r(struct mparse
*curp
, struct buf blk
, size_t i
, int start
)
345 const char *save_file
;
347 size_t pos
; /* byte number in the ln buffer */
350 int lnn
; /* line number in the real file */
354 memset(&ln
, 0, sizeof(ln
));
360 if (0 == pos
&& '\0' == blk
.buf
[i
])
365 curp
->reparse_count
= 0;
368 curp
->filenc
& MPARSE_UTF8
&&
369 curp
->filenc
& MPARSE_LATIN1
)
370 curp
->filenc
= preconv_cue(&blk
, i
);
373 while (i
< blk
.sz
&& (start
|| blk
.buf
[i
] != '\0')) {
376 * When finding an unescaped newline character,
377 * leave the character loop to process the line.
378 * Skip a preceding carriage return, if any.
381 if ('\r' == blk
.buf
[i
] && i
+ 1 < blk
.sz
&&
382 '\n' == blk
.buf
[i
+ 1])
384 if ('\n' == blk
.buf
[i
]) {
391 * Make sure we have space for the worst
392 * case of 11 bytes: "\\[u10ffff]\0"
395 if (pos
+ 11 > ln
.sz
)
396 resize_buf(&ln
, 256);
399 * Encode 8-bit input.
404 if ( ! (curp
->filenc
&& preconv_encode(
405 &blk
, &i
, &ln
, &pos
, &curp
->filenc
))) {
406 mandoc_vmsg(MANDOCERR_CHAR_BAD
, curp
,
407 curp
->line
, pos
, "0x%x", c
);
415 * Exclude control characters.
418 if (c
== 0x7f || (c
< 0x20 && c
!= 0x09)) {
419 mandoc_vmsg(c
== 0x00 || c
== 0x04 ||
420 c
> 0x0a ? MANDOCERR_CHAR_BAD
:
421 MANDOCERR_CHAR_UNSUPP
,
422 curp
, curp
->line
, pos
, "0x%x", c
);
429 ln
.buf
[pos
++] = blk
.buf
[i
++];
432 if (pos
+ 1 >= ln
.sz
)
433 resize_buf(&ln
, 256);
435 if (i
== blk
.sz
|| blk
.buf
[i
] == '\0')
436 ln
.buf
[pos
++] = '\n';
440 * A significant amount of complexity is contained by
441 * the roff preprocessor. It's line-oriented but can be
442 * expressed on one line, so we need at times to
443 * readjust our starting point and re-run it. The roff
444 * preprocessor can also readjust the buffers with new
445 * data, so we pass them in wholesale.
451 * Maintain a lookaside buffer of all parsed lines. We
452 * only do this if mparse_keep() has been invoked (the
453 * buffer may be accessed with mparse_getkeep()).
456 if (curp
->secondary
) {
457 curp
->secondary
->buf
= mandoc_realloc(
458 curp
->secondary
->buf
,
459 curp
->secondary
->sz
+ pos
+ 2);
460 memcpy(curp
->secondary
->buf
+
463 curp
->secondary
->sz
+= pos
;
465 [curp
->secondary
->sz
] = '\n';
466 curp
->secondary
->sz
++;
468 [curp
->secondary
->sz
] = '\0';
471 rr
= roff_parseln(curp
->roff
, curp
->line
, &ln
, &of
);
475 if (++curp
->reparse_count
> REPARSE_LIMIT
)
476 mandoc_msg(MANDOCERR_ROFFLOOP
, curp
,
477 curp
->line
, pos
, NULL
);
478 else if (mparse_buf_r(curp
, ln
, of
, 0) == 1 ||
486 pos
= strlen(ln
.buf
);
494 if ( ! (curp
->options
& MPARSE_SO
) &&
495 (i
>= blk
.sz
|| blk
.buf
[i
] == '\0')) {
496 curp
->sodest
= mandoc_strdup(ln
.buf
+ of
);
501 * We remove `so' clauses from our lookaside
502 * buffer because we're going to descend into
503 * the file recursively.
506 curp
->secondary
->sz
-= pos
+ 1;
507 save_file
= curp
->file
;
508 if ((fd
= mparse_open(curp
, ln
.buf
+ of
)) != -1) {
509 mparse_readfd(curp
, fd
, ln
.buf
+ of
);
511 curp
->file
= save_file
;
513 curp
->file
= save_file
;
514 mandoc_vmsg(MANDOCERR_SO_FAIL
,
515 curp
, curp
->line
, pos
,
516 ".so %s", ln
.buf
+ of
);
517 ln
.sz
= mandoc_asprintf(&cp
,
518 ".sp\nSee the file %s.\n.sp",
523 mparse_buf_r(curp
, ln
, of
, 0);
531 if (curp
->man
->macroset
== MACROSET_NONE
)
534 if ((curp
->man
->macroset
== MACROSET_MDOC
?
535 mdoc_parseln(curp
->man
, curp
->line
, ln
.buf
, of
) :
536 man_parseln(curp
->man
, curp
->line
, ln
.buf
, of
)) == 2)
539 /* Temporary buffers typically are not full. */
541 if (0 == start
&& '\0' == blk
.buf
[i
])
544 /* Start the next input line. */
554 read_whole_file(struct mparse
*curp
, const char *file
, int fd
,
555 struct buf
*fb
, int *with_mmap
)
561 int gzerrnum
, retval
;
563 if (fstat(fd
, &st
) == -1) {
564 mandoc_vmsg(MANDOCERR_FILE
, curp
, 0, 0,
565 "fstat: %s", strerror(errno
));
570 * If we're a regular file, try just reading in the whole entry
571 * via mmap(). This is faster than reading it into blocks, and
572 * since each file is only a few bytes to begin with, I'm not
573 * concerned that this is going to tank any machines.
576 if (curp
->gzip
== 0 && S_ISREG(st
.st_mode
)) {
577 if (st
.st_size
> 0x7fffffff) {
578 mandoc_msg(MANDOCERR_TOOLARGE
, curp
, 0, 0, NULL
);
582 fb
->sz
= (size_t)st
.st_size
;
583 fb
->buf
= mmap(NULL
, fb
->sz
, PROT_READ
, MAP_SHARED
, fd
, 0);
584 if (fb
->buf
!= MAP_FAILED
)
590 * Duplicating the file descriptor is required
591 * because we will have to call gzclose(3)
592 * to free memory used internally by zlib,
593 * but that will also close the file descriptor,
594 * which this function must not do.
596 if ((fd
= dup(fd
)) == -1) {
597 mandoc_vmsg(MANDOCERR_FILE
, curp
, 0, 0,
598 "dup: %s", strerror(errno
));
601 if ((gz
= gzdopen(fd
, "rb")) == NULL
) {
602 mandoc_vmsg(MANDOCERR_FILE
, curp
, 0, 0,
603 "gzdopen: %s", strerror(errno
));
611 * If this isn't a regular file (like, say, stdin), then we must
612 * go the old way and just read things in bit by bit.
622 if (fb
->sz
== (1U << 31)) {
623 mandoc_msg(MANDOCERR_TOOLARGE
, curp
,
627 resize_buf(fb
, 65536);
630 gzread(gz
, fb
->buf
+ (int)off
, fb
->sz
- off
) :
631 read(fd
, fb
->buf
+ (int)off
, fb
->sz
- off
);
639 (void)gzerror(gz
, &gzerrnum
);
640 mandoc_vmsg(MANDOCERR_FILE
, curp
, 0, 0, "read: %s",
641 curp
->gzip
&& gzerrnum
!= Z_ERRNO
?
642 zError(gzerrnum
) : strerror(errno
));
648 if (curp
->gzip
&& (gzerrnum
= gzclose(gz
)) != Z_OK
)
649 mandoc_vmsg(MANDOCERR_FILE
, curp
, 0, 0, "gzclose: %s",
650 gzerrnum
== Z_ERRNO
? strerror(errno
) :
660 mparse_end(struct mparse
*curp
)
662 if (curp
->man
->macroset
== MACROSET_NONE
)
663 curp
->man
->macroset
= MACROSET_MAN
;
664 if (curp
->man
->macroset
== MACROSET_MDOC
)
665 mdoc_endparse(curp
->man
);
667 man_endparse(curp
->man
);
668 roff_endparse(curp
->roff
);
672 mparse_parse_buffer(struct mparse
*curp
, struct buf blk
, const char *file
)
674 struct buf
*svprimary
;
677 static int recursion_depth
;
679 if (64 < recursion_depth
) {
680 mandoc_msg(MANDOCERR_ROFFLOOP
, curp
, curp
->line
, 0, NULL
);
684 /* Line number is per-file. */
687 svprimary
= curp
->primary
;
688 curp
->primary
= &blk
;
692 /* Skip an UTF-8 byte order mark. */
693 if (curp
->filenc
& MPARSE_UTF8
&& blk
.sz
> 2 &&
694 (unsigned char)blk
.buf
[0] == 0xef &&
695 (unsigned char)blk
.buf
[1] == 0xbb &&
696 (unsigned char)blk
.buf
[2] == 0xbf) {
698 curp
->filenc
&= ~MPARSE_LATIN1
;
702 mparse_buf_r(curp
, blk
, offset
, 1);
704 if (--recursion_depth
== 0)
707 curp
->primary
= svprimary
;
712 mparse_readmem(struct mparse
*curp
, void *buf
, size_t len
,
720 mparse_parse_buffer(curp
, blk
, file
);
721 return curp
->file_status
;
725 * Read the whole file into memory and call the parsers.
726 * Called recursively when an .so request is encountered.
729 mparse_readfd(struct mparse
*curp
, int fd
, const char *file
)
735 if (read_whole_file(curp
, file
, fd
, &blk
, &with_mmap
)) {
736 save_filenc
= curp
->filenc
;
737 curp
->filenc
= curp
->options
&
738 (MPARSE_UTF8
| MPARSE_LATIN1
);
739 mparse_parse_buffer(curp
, blk
, file
);
740 curp
->filenc
= save_filenc
;
742 munmap(blk
.buf
, blk
.sz
);
746 return curp
->file_status
;
750 mparse_open(struct mparse
*curp
, const char *file
)
756 cp
= strrchr(file
, '.');
757 curp
->gzip
= (cp
!= NULL
&& ! strcmp(cp
+ 1, "gz"));
759 /* First try to use the filename as it is. */
761 if ((fd
= open(file
, O_RDONLY
)) != -1)
765 * If that doesn't work and the filename doesn't
766 * already end in .gz, try appending .gz.
770 mandoc_asprintf(&cp
, "%s.gz", file
);
771 fd
= open(cp
, O_RDONLY
);
779 /* Neither worked, give up. */
781 mandoc_msg(MANDOCERR_FILE
, curp
, 0, 0, strerror(errno
));
786 mparse_alloc(int options
, enum mandocerr mmin
, mandocmsg mmsg
,
787 enum mandoc_os os_e
, const char *os_s
)
791 curp
= mandoc_calloc(1, sizeof(struct mparse
));
793 curp
->options
= options
;
798 curp
->roff
= roff_alloc(curp
, options
);
799 curp
->man
= roff_man_alloc(curp
->roff
, curp
, curp
->os_s
,
800 curp
->options
& MPARSE_QUICK
? 1 : 0);
801 if (curp
->options
& MPARSE_MDOC
) {
802 curp
->man
->macroset
= MACROSET_MDOC
;
803 if (curp
->man
->mdocmac
== NULL
)
804 curp
->man
->mdocmac
= roffhash_alloc(MDOC_Dd
, MDOC_MAX
);
805 } else if (curp
->options
& MPARSE_MAN
) {
806 curp
->man
->macroset
= MACROSET_MAN
;
807 if (curp
->man
->manmac
== NULL
)
808 curp
->man
->manmac
= roffhash_alloc(MAN_TH
, MAN_MAX
);
810 curp
->man
->first
->tok
= TOKEN_NONE
;
811 curp
->man
->meta
.os_e
= os_e
;
816 mparse_reset(struct mparse
*curp
)
818 roff_reset(curp
->roff
);
819 roff_man_reset(curp
->man
);
825 curp
->secondary
->sz
= 0;
827 curp
->file_status
= MANDOCLEVEL_OK
;
832 mparse_free(struct mparse
*curp
)
835 roffhash_free(curp
->man
->mdocmac
);
836 roffhash_free(curp
->man
->manmac
);
837 roff_man_free(curp
->man
);
838 roff_free(curp
->roff
);
840 free(curp
->secondary
->buf
);
842 free(curp
->secondary
);
848 mparse_result(struct mparse
*curp
, struct roff_man
**man
,
852 if (sodest
&& NULL
!= (*sodest
= curp
->sodest
)) {
861 mparse_updaterc(struct mparse
*curp
, enum mandoclevel
*rc
)
863 if (curp
->file_status
> *rc
)
864 *rc
= curp
->file_status
;
868 mandoc_vmsg(enum mandocerr t
, struct mparse
*m
,
869 int ln
, int pos
, const char *fmt
, ...)
875 (void)vsnprintf(buf
, sizeof(buf
), fmt
, ap
);
878 mandoc_msg(t
, m
, ln
, pos
, buf
);
882 mandoc_msg(enum mandocerr er
, struct mparse
*m
,
883 int ln
, int col
, const char *msg
)
885 enum mandoclevel level
;
887 if (er
< m
->mmin
&& er
!= MANDOCERR_FILE
)
890 level
= MANDOCLEVEL_UNSUPP
;
891 while (er
< mandoclimits
[level
])
895 (*m
->mmsg
)(er
, level
, m
->file
, ln
, col
, msg
);
897 if (m
->file_status
< level
)
898 m
->file_status
= level
;
902 mparse_strerror(enum mandocerr er
)
905 return mandocerrs
[er
];
909 mparse_strlevel(enum mandoclevel lvl
)
911 return mandoclevels
[lvl
];
915 mparse_keep(struct mparse
*p
)
918 assert(NULL
== p
->secondary
);
919 p
->secondary
= mandoc_calloc(1, sizeof(struct buf
));
923 mparse_getkeep(const struct mparse
*p
)
926 assert(p
->secondary
);
927 return p
->secondary
->sz
? p
->secondary
->buf
: NULL
;