1 /* Stream reading and decoding (mostly decompression) */
10 #include <sys/stat.h> /* OS/2 needs this after sys/types.h */
11 #include <sys/types.h>
13 #include <fcntl.h> /* OS/2 needs this after sys/types.h */
21 #include "config/options.h"
22 #include "encoding/encoding.h"
23 #include "network/state.h"
24 #include "osdep/osdep.h"
25 #include "util/memory.h"
26 #include "util/string.h"
29 /*************************************************************************
30 Dummy encoding (ENCODING_NONE)
31 *************************************************************************/
33 struct dummy_enc_data
{
38 dummy_open(struct stream_encoded
*stream
, int fd
)
40 stream
->data
= mem_alloc(sizeof(struct dummy_enc_data
));
41 if (!stream
->data
) return -1;
43 ((struct dummy_enc_data
*) stream
->data
)->fd
= fd
;
49 dummy_read(struct stream_encoded
*stream
, unsigned char *data
, int len
)
51 return safe_read(((struct dummy_enc_data
*) stream
->data
)->fd
, data
, len
);
54 static unsigned char *
55 dummy_decode_buffer(unsigned char *data
, int len
, int *new_len
)
57 unsigned char *buffer
= memacpy(data
, len
);
59 if (!buffer
) return NULL
;
66 dummy_close(struct stream_encoded
*stream
)
68 close(((struct dummy_enc_data
*) stream
->data
)->fd
);
69 mem_free(stream
->data
);
72 static const unsigned char *const dummy_extensions
[] = { NULL
};
74 static const struct decoding_backend dummy_decoding_backend
= {
84 /* Dynamic backend area */
86 #include "encoding/bzip2.h"
87 #include "encoding/deflate.h"
88 #include "encoding/lzma.h"
90 static const struct decoding_backend
*const decoding_backends
[] = {
91 &dummy_decoding_backend
,
92 &gzip_decoding_backend
,
93 &bzip2_decoding_backend
,
94 &lzma_decoding_backend
,
95 &deflate_decoding_backend
,
99 /*************************************************************************
101 *************************************************************************/
104 /* Associates encoded stream with a fd. */
105 struct stream_encoded
*
106 open_encoded(int fd
, enum stream_encoding encoding
)
108 struct stream_encoded
*stream
;
110 stream
= mem_alloc(sizeof(*stream
));
111 if (!stream
) return NULL
;
113 stream
->encoding
= encoding
;
114 if (decoding_backends
[stream
->encoding
]->open(stream
, fd
) >= 0)
121 /* Read available data from stream and decode them. Note that when data change
122 * their size during decoding, 'len' indicates desired size of _returned_ data,
123 * not desired size of data read from stream. */
125 read_encoded(struct stream_encoded
*stream
, unsigned char *data
, int len
)
127 return decoding_backends
[stream
->encoding
]->read(stream
, data
, len
);
130 /* Decode an entire file from a buffer. This function is not suitable
131 * for parts of files. @data contains the original data, @len bytes
132 * long. The resulting decoded data chunk is *@new_len bytes long. */
134 decode_encoded_buffer(enum stream_encoding encoding
, unsigned char *data
, int len
,
137 return decoding_backends
[encoding
]->decode_buffer(data
, len
, new_len
);
140 /* Closes encoded stream. Note that fd associated with the stream will be
143 close_encoded(struct stream_encoded
*stream
)
145 decoding_backends
[stream
->encoding
]->close(stream
);
150 /* Return a list of extensions associated with that encoding. */
151 const unsigned char *const *listext_encoded(enum stream_encoding encoding
)
153 return decoding_backends
[encoding
]->extensions
;
157 guess_encoding(unsigned char *filename
)
159 int fname_len
= strlen(filename
);
160 unsigned char *fname_end
= filename
+ fname_len
;
163 for (enc
= 1; enc
< ENCODINGS_KNOWN
; enc
++) {
164 const unsigned char *const *ext
= decoding_backends
[enc
]->extensions
;
166 while (ext
&& *ext
) {
167 int len
= strlen(*ext
);
169 if (fname_len
>= len
&& !strcmp(fname_end
- len
, *ext
))
176 return ENCODING_NONE
;
179 const unsigned char *
180 get_encoding_name(enum stream_encoding encoding
)
182 return decoding_backends
[encoding
]->name
;
188 /* Tries to open @prefixname with each of the supported encoding extensions
190 static inline enum stream_encoding
191 try_encoding_extensions(struct string
*filename
, int *fd
)
193 int length
= filename
->length
;
196 /* No file of that name was found, try some others names. */
197 for (encoding
= 1; encoding
< ENCODINGS_KNOWN
; encoding
++) {
198 const unsigned char *const *ext
= listext_encoded(encoding
);
200 for (; ext
&& *ext
; ext
++) {
201 add_to_string(filename
, *ext
);
203 /* We try with some extensions. */
204 *fd
= open(filename
->source
, O_RDONLY
| O_NOCTTY
);
207 /* Ok, found one, use it. */
210 filename
->source
[length
] = 0;
211 filename
->length
= length
;
215 return ENCODING_NONE
;
218 /* Reads the file from @stream in chunks of size @readsize. */
219 /* Returns a connection state. S_OK if all is well. */
220 enum connection_state
221 read_file(struct stream_encoded
*stream
, int readsize
, struct string
*page
)
223 if (!init_string(page
)) return S_OUT_OF_MEM
;
225 /* We read with granularity of stt.st_size (given as @readsize) - this
226 * does best job for uncompressed files, and doesn't hurt for
227 * compressed ones anyway - very large files usually tend to inflate
228 * fast anyway. At least I hope ;). --pasky */
229 /* Also there because of bug in Linux. Read returns -EACCES when
230 * reading 0 bytes to invalid address so ensure never to try and
231 * allocate zero number of bytes. */
232 if (!readsize
) readsize
= 4096;
234 while (realloc_string(page
, page
->length
+ readsize
)) {
235 unsigned char *string_pos
= page
->source
+ page
->length
;
236 int readlen
= read_encoded(stream
, string_pos
, readsize
);
241 /* If it is some I/O error (and errno is set) that will
242 * do. Since errno == 0 == S_WAIT and we cannot have
245 return (enum connection_state
) -errno
;
247 /* FIXME: This is indeed an internal error. If readed from a
248 * corrupted encoded file nothing or only some of the
249 * data will be read. */
250 return S_ENCODE_ERROR
;
252 } else if (readlen
== 0) {
253 /* NUL-terminate just in case */
254 page
->source
[page
->length
] = '\0';
258 page
->length
+= readlen
;
260 /* This didn't work so well as it should (I had to implement
261 * end of stream handling to bzip2 anyway), so I rather
263 if (readlen
< readsize
) {
264 /* This is much safer. It should always mean that we
265 * already read everything possible, and it permits us
266 * more elegant of handling end of file with bzip2. */
277 is_stdin_pipe(struct stat
*stt
, struct string
*filename
)
279 /* On Mac OS X, /dev/stdin has type S_IFSOCK. (bug 616) */
280 return !strlcmp(filename
->source
, filename
->length
, "/dev/stdin", 10)
283 S_ISSOCK(stt
->st_mode
) ||
285 S_ISFIFO(stt
->st_mode
));
288 enum connection_state
289 read_encoded_file(struct string
*filename
, struct string
*page
)
291 struct stream_encoded
*stream
;
293 enum stream_encoding encoding
= ENCODING_NONE
;
294 int fd
= open(filename
->source
, O_RDONLY
| O_NOCTTY
);
295 enum connection_state state
= -errno
;
297 if (fd
== -1 && get_opt_bool("protocol.file.try_encoding_extensions", NULL
)) {
298 encoding
= try_encoding_extensions(filename
, &fd
);
300 } else if (fd
!= -1) {
301 encoding
= guess_encoding(filename
->source
);
307 /* Some file was opened so let's get down to bi'ness */
310 /* Do all the necessary checks before trying to read the file.
311 * @state code is used to block further progress. */
312 if (fstat(fd
, &stt
)) {
315 } else if (!S_ISREG(stt
.st_mode
) && encoding
!= ENCODING_NONE
) {
316 /* We only want to open regular encoded files. */
317 /* Leave @state being the saved errno */
319 } else if (!S_ISREG(stt
.st_mode
) && !is_stdin_pipe(&stt
, filename
)
320 && !get_opt_bool("protocol.file.allow_special_files", NULL
)) {
323 } else if (!(stream
= open_encoded(fd
, encoding
))) {
324 state
= S_OUT_OF_MEM
;
327 int readsize
= (int) stt
.st_size
;
329 /* Check if st_size will cause overflow. */
330 /* FIXME: See bug 497 for info about support for big files. */
331 if (readsize
!= stt
.st_size
|| readsize
< 0) {
333 state
= (enum connection_state
) -(EFBIG
);
335 state
= S_FILE_ERROR
;
339 state
= read_file(stream
, stt
.st_size
, page
);
341 close_encoded(stream
);