4 * the network chunk-API
13 #include <sys/types.h>
25 /* default 1MB, upper limit 128MB */
26 #define DEFAULT_TEMPFILE_SIZE (1 * 1024 * 1024)
27 #define MAX_TEMPFILE_SIZE (128 * 1024 * 1024)
29 static array
*chunkqueue_default_tempdirs
= NULL
;
30 static unsigned int chunkqueue_default_tempfile_size
= DEFAULT_TEMPFILE_SIZE
;
32 chunkqueue
*chunkqueue_init(void) {
35 cq
= calloc(1, sizeof(*cq
));
36 force_assert(NULL
!= cq
);
43 cq
->tempdirs
= chunkqueue_default_tempdirs
;
44 cq
->upload_temp_file_size
= chunkqueue_default_tempfile_size
;
49 static chunk
*chunk_init(void) {
52 c
= calloc(1, sizeof(*c
));
53 force_assert(NULL
!= c
);
56 c
->mem
= buffer_init();
57 c
->file
.name
= buffer_init();
58 c
->file
.start
= c
->file
.length
= c
->file
.mmap
.offset
= 0;
60 c
->file
.mmap
.start
= MAP_FAILED
;
61 c
->file
.mmap
.length
= 0;
69 static void chunk_reset(chunk
*c
) {
70 if (NULL
== c
) return;
76 if (c
->file
.is_temp
&& !buffer_string_is_empty(c
->file
.name
)) {
77 unlink(c
->file
.name
->ptr
);
80 buffer_reset(c
->file
.name
);
82 if (c
->file
.fd
!= -1) {
86 if (MAP_FAILED
!= c
->file
.mmap
.start
) {
87 munmap(c
->file
.mmap
.start
, c
->file
.mmap
.length
);
88 c
->file
.mmap
.start
= MAP_FAILED
;
90 c
->file
.start
= c
->file
.length
= c
->file
.mmap
.offset
= 0;
91 c
->file
.mmap
.length
= 0;
97 static void chunk_free(chunk
*c
) {
98 if (NULL
== c
) return;
103 buffer_free(c
->file
.name
);
108 static off_t
chunk_remaining_length(const chunk
*c
) {
112 len
= buffer_string_length(c
->mem
);
115 len
= c
->file
.length
;
118 force_assert(c
->type
== MEM_CHUNK
|| c
->type
== FILE_CHUNK
);
121 force_assert(c
->offset
<= len
);
122 return len
- c
->offset
;
125 void chunkqueue_free(chunkqueue
*cq
) {
128 if (NULL
== cq
) return;
130 for (c
= cq
->first
; c
; ) {
136 for (c
= cq
->unused
; c
; ) {
145 static void chunkqueue_push_unused_chunk(chunkqueue
*cq
, chunk
*c
) {
146 force_assert(NULL
!= cq
&& NULL
!= c
);
148 /* keep at max 4 chunks in the 'unused'-cache */
149 if (cq
->unused_chunks
> 4) {
153 c
->next
= cq
->unused
;
159 static chunk
*chunkqueue_get_unused_chunk(chunkqueue
*cq
) {
162 force_assert(NULL
!= cq
);
164 /* check if we have a unused chunk */
165 if (0 == cq
->unused
) {
168 /* take the first element from the list (a stack) */
170 cq
->unused
= c
->next
;
178 static void chunkqueue_prepend_chunk(chunkqueue
*cq
, chunk
*c
) {
182 if (NULL
== cq
->last
) {
185 cq
->bytes_in
+= chunk_remaining_length(c
);
188 static void chunkqueue_append_chunk(chunkqueue
*cq
, chunk
*c
) {
195 if (NULL
== cq
->first
) {
198 cq
->bytes_in
+= chunk_remaining_length(c
);
201 void chunkqueue_reset(chunkqueue
*cq
) {
202 chunk
*cur
= cq
->first
;
204 cq
->first
= cq
->last
= NULL
;
206 while (NULL
!= cur
) {
207 chunk
*next
= cur
->next
;
208 chunkqueue_push_unused_chunk(cq
, cur
);
217 void chunkqueue_append_file_fd(chunkqueue
*cq
, buffer
*fn
, int fd
, off_t offset
, off_t len
) {
225 c
= chunkqueue_get_unused_chunk(cq
);
227 c
->type
= FILE_CHUNK
;
229 buffer_copy_buffer(c
->file
.name
, fn
);
230 c
->file
.start
= offset
;
231 c
->file
.length
= len
;
235 chunkqueue_append_chunk(cq
, c
);
238 void chunkqueue_append_file(chunkqueue
*cq
, buffer
*fn
, off_t offset
, off_t len
) {
241 if (0 == len
) return;
243 c
= chunkqueue_get_unused_chunk(cq
);
245 c
->type
= FILE_CHUNK
;
247 buffer_copy_buffer(c
->file
.name
, fn
);
248 c
->file
.start
= offset
;
249 c
->file
.length
= len
;
252 chunkqueue_append_chunk(cq
, c
);
255 void chunkqueue_append_buffer(chunkqueue
*cq
, buffer
*mem
) {
258 if (buffer_string_is_empty(mem
)) return;
260 c
= chunkqueue_get_unused_chunk(cq
);
262 force_assert(NULL
!= c
->mem
);
263 buffer_move(c
->mem
, mem
);
265 chunkqueue_append_chunk(cq
, c
);
268 void chunkqueue_prepend_buffer(chunkqueue
*cq
, buffer
*mem
) {
271 if (buffer_string_is_empty(mem
)) return;
273 c
= chunkqueue_get_unused_chunk(cq
);
275 force_assert(NULL
!= c
->mem
);
276 buffer_move(c
->mem
, mem
);
278 chunkqueue_prepend_chunk(cq
, c
);
282 void chunkqueue_append_mem(chunkqueue
*cq
, const char * mem
, size_t len
) {
285 if (0 == len
) return;
287 c
= chunkqueue_get_unused_chunk(cq
);
289 buffer_copy_string_len(c
->mem
, mem
, len
);
291 chunkqueue_append_chunk(cq
, c
);
295 void chunkqueue_append_chunkqueue(chunkqueue
*cq
, chunkqueue
*src
) {
296 if (src
== NULL
|| NULL
== src
->first
) return;
298 if (NULL
== cq
->first
) {
299 cq
->first
= src
->first
;
301 cq
->last
->next
= src
->first
;
303 cq
->last
= src
->last
;
304 cq
->bytes_in
+= (src
->bytes_in
- src
->bytes_out
);
308 src
->bytes_out
= src
->bytes_in
;
312 void chunkqueue_get_memory(chunkqueue
*cq
, char **mem
, size_t *len
, size_t min_size
, size_t alloc_size
) {
313 static const size_t REALLOC_MAX_SIZE
= 256;
319 force_assert(NULL
!= cq
);
320 if (NULL
== mem
) mem
= &dummy_mem
;
321 if (NULL
== len
) len
= &dummy_len
;
323 /* default values: */
324 if (0 == min_size
) min_size
= 1024;
325 if (0 == alloc_size
) alloc_size
= 4096;
326 if (alloc_size
< min_size
) alloc_size
= min_size
;
328 if (NULL
!= cq
->last
&& MEM_CHUNK
== cq
->last
->type
) {
332 have
= buffer_string_space(b
);
334 /* unused buffer: allocate space */
335 if (buffer_string_is_empty(b
)) {
336 buffer_string_prepare_copy(b
, alloc_size
);
337 have
= buffer_string_space(b
);
339 /* if buffer is really small just make it bigger */
340 else if (have
< min_size
&& b
->size
<= REALLOC_MAX_SIZE
) {
341 size_t cur_len
= buffer_string_length(b
);
342 size_t new_size
= cur_len
+ min_size
, append
;
343 if (new_size
< alloc_size
) new_size
= alloc_size
;
345 append
= new_size
- cur_len
;
346 if (append
>= min_size
) {
347 buffer_string_prepare_append(b
, append
);
348 have
= buffer_string_space(b
);
352 /* return pointer into existing buffer if large enough */
353 if (have
>= min_size
) {
354 *mem
= b
->ptr
+ buffer_string_length(b
);
360 /* allocate new chunk */
361 c
= chunkqueue_get_unused_chunk(cq
);
363 chunkqueue_append_chunk(cq
, c
);
366 buffer_string_prepare_append(b
, alloc_size
);
368 *mem
= b
->ptr
+ buffer_string_length(b
);
369 *len
= buffer_string_space(b
);
372 void chunkqueue_use_memory(chunkqueue
*cq
, size_t len
) {
375 force_assert(NULL
!= cq
);
376 force_assert(NULL
!= cq
->last
&& MEM_CHUNK
== cq
->last
->type
);
380 buffer_commit(b
, len
);
382 } else if (buffer_string_is_empty(b
)) {
383 /* unused buffer: can't remove chunk easily from
384 * end of list, so just reset the buffer
390 void chunkqueue_set_tempdirs_default (array
*tempdirs
, unsigned int upload_temp_file_size
) {
391 chunkqueue_default_tempdirs
= tempdirs
;
392 chunkqueue_default_tempfile_size
393 = (0 == upload_temp_file_size
) ? DEFAULT_TEMPFILE_SIZE
394 : (upload_temp_file_size
> MAX_TEMPFILE_SIZE
) ? MAX_TEMPFILE_SIZE
395 : upload_temp_file_size
;
399 void chunkqueue_set_tempdirs(chunkqueue
*cq
, array
*tempdirs
, unsigned int upload_temp_file_size
) {
400 force_assert(NULL
!= cq
);
401 cq
->tempdirs
= tempdirs
;
402 cq
->upload_temp_file_size
403 = (0 == upload_temp_file_size
) ? DEFAULT_TEMPFILE_SIZE
404 : (upload_temp_file_size
> MAX_TEMPFILE_SIZE
) ? MAX_TEMPFILE_SIZE
405 : upload_temp_file_size
;
410 void chunkqueue_steal(chunkqueue
*dest
, chunkqueue
*src
, off_t len
) {
412 chunk
*c
= src
->first
;
415 if (NULL
== c
) break;
417 clen
= chunk_remaining_length(c
);
419 /* drop empty chunk */
420 src
->first
= c
->next
;
421 if (c
== src
->last
) src
->last
= NULL
;
422 chunkqueue_push_unused_chunk(src
, c
);
426 use
= len
>= clen
? clen
: len
;
430 /* move complete chunk */
431 src
->first
= c
->next
;
432 if (c
== src
->last
) src
->last
= NULL
;
434 chunkqueue_append_chunk(dest
, c
);
436 /* partial chunk with length "use" */
440 chunkqueue_append_mem(dest
, c
->mem
->ptr
+ c
->offset
, use
);
443 /* tempfile flag is in "last" chunk after the split */
444 chunkqueue_append_file(dest
, c
->file
.name
, c
->file
.start
+ c
->offset
, use
);
449 force_assert(0 == len
);
452 src
->bytes_out
+= use
;
456 static chunk
*chunkqueue_get_append_tempfile(chunkqueue
*cq
) {
458 buffer
*template = buffer_init_string("/var/tmp/lighttpd-upload-XXXXXX");
461 if (cq
->tempdirs
&& cq
->tempdirs
->used
) {
462 /* we have several tempdirs, only if all of them fail we jump out */
464 for (errno
= EIO
; cq
->tempdir_idx
< cq
->tempdirs
->used
; ++cq
->tempdir_idx
) {
465 data_string
*ds
= (data_string
*)cq
->tempdirs
->data
[cq
->tempdir_idx
];
467 buffer_copy_buffer(template, ds
->value
);
468 buffer_append_slash(template);
469 buffer_append_string_len(template, CONST_STR_LEN("lighttpd-upload-XXXXXX"));
471 if (-1 != (fd
= mkstemp(template->ptr
))) break;
474 /* coverity[secure_temp : FALSE] */
475 fd
= mkstemp(template->ptr
);
479 buffer_free(template);
483 fd_close_on_exec(fd
);
484 (void)fcntl(fd
, F_SETFL
, fcntl(fd
, F_GETFL
, 0) | O_APPEND
);
486 c
= chunkqueue_get_unused_chunk(cq
);
487 c
->type
= FILE_CHUNK
;
490 buffer_copy_buffer(c
->file
.name
, template);
493 chunkqueue_append_chunk(cq
, c
);
495 buffer_free(template);
500 static void chunkqueue_remove_empty_chunks(chunkqueue
*cq
);
502 int chunkqueue_append_mem_to_tempfile(server
*srv
, chunkqueue
*dest
, const char *mem
, size_t len
) {
508 * if the last chunk is
509 * - smaller than dest->upload_temp_file_size
510 * - not read yet (offset == 0)
511 * -> append to it (so it might actually become larger than dest->upload_temp_file_size)
513 * -> create a new chunk
519 && FILE_CHUNK
== dst_c
->type
520 && dst_c
->file
.is_temp
521 && dst_c
->file
.fd
>= 0
522 && 0 == dst_c
->offset
) {
523 /* ok, take the last chunk for our job */
525 if (dst_c
->file
.length
>= (off_t
)dest
->upload_temp_file_size
) {
526 /* the chunk is too large now, close it */
527 int rc
= close(dst_c
->file
.fd
);
530 log_error_write(srv
, __FILE__
, __LINE__
, "sbss",
531 "close() temp-file", dst_c
->file
.name
, "failed:",
541 if (NULL
== dst_c
&& NULL
== (dst_c
= chunkqueue_get_append_tempfile(dest
))) {
542 /* we don't have file to write to,
543 * EACCES might be one reason.
545 * Instead of sending 500 we send 413 and say the request is too large
548 log_error_write(srv
, __FILE__
, __LINE__
, "ss",
549 "opening temp-file failed:", strerror(errno
));
554 /* (dst_c->file.fd >= 0) */
555 /* coverity[negative_returns : FALSE] */
556 written
= write(dst_c
->file
.fd
, mem
, len
);
558 if ((size_t) written
== len
) {
559 dst_c
->file
.length
+= len
;
560 dest
->bytes_in
+= len
;
563 } else if (written
>= 0) {
564 /*(assume EINTR if partial write and retry write();
565 * retry write() might fail with ENOSPC if no more space on volume)*/
566 dest
->bytes_in
+= written
;
568 len
-= (size_t)written
;
569 dst_c
->file
.length
+= (size_t)written
;
570 /* continue; retry */
571 } else if (errno
== EINTR
) {
572 /* continue; retry */
574 int retry
= (errno
== ENOSPC
&& dest
->tempdirs
&& ++dest
->tempdir_idx
< dest
->tempdirs
->used
);
576 log_error_write(srv
, __FILE__
, __LINE__
, "sbs",
577 "write() temp-file", dst_c
->file
.name
, "failed:",
581 if (0 == chunk_remaining_length(dst_c
)) {
582 /*(remove empty chunk and unlink tempfile)*/
583 chunkqueue_remove_empty_chunks(dest
);
584 } else {/*(close tempfile; avoid later attempts to append)*/
585 int rc
= close(dst_c
->file
.fd
);
588 log_error_write(srv
, __FILE__
, __LINE__
, "sbss",
589 "close() temp-file", dst_c
->file
.name
, "failed:",
594 if (!retry
) break; /* return -1; */
596 /* continue; retry */
604 int chunkqueue_steal_with_tempfiles(server
*srv
, chunkqueue
*dest
, chunkqueue
*src
, off_t len
) {
606 chunk
*c
= src
->first
;
609 if (NULL
== c
) break;
611 clen
= chunk_remaining_length(c
);
613 /* drop empty chunk */
614 src
->first
= c
->next
;
615 if (c
== src
->last
) src
->last
= NULL
;
616 chunkqueue_push_unused_chunk(src
, c
);
620 use
= (len
>= clen
) ? clen
: len
;
626 /* move complete chunk */
627 src
->first
= c
->next
;
628 if (c
== src
->last
) src
->last
= NULL
;
629 chunkqueue_append_chunk(dest
, c
);
631 /* partial chunk with length "use" */
632 /* tempfile flag is in "last" chunk after the split */
633 chunkqueue_append_file(dest
, c
->file
.name
, c
->file
.start
+ c
->offset
, use
);
636 force_assert(0 == len
);
641 /* store "use" bytes from memory chunk in tempfile */
642 if (0 != chunkqueue_append_mem_to_tempfile(srv
, dest
, c
->mem
->ptr
+ c
->offset
, use
)) {
648 src
->first
= c
->next
;
649 if (c
== src
->last
) src
->last
= NULL
;
650 chunkqueue_push_unused_chunk(src
, c
);
654 force_assert(0 == len
);
659 src
->bytes_out
+= use
;
665 off_t
chunkqueue_length(chunkqueue
*cq
) {
669 for (c
= cq
->first
; c
; c
= c
->next
) {
670 len
+= chunk_remaining_length(c
);
676 int chunkqueue_is_empty(chunkqueue
*cq
) {
677 return NULL
== cq
->first
;
680 void chunkqueue_mark_written(chunkqueue
*cq
, off_t len
) {
683 force_assert(len
>= 0);
685 for (c
= cq
->first
; NULL
!= c
; c
= cq
->first
) {
686 off_t c_len
= chunk_remaining_length(c
);
688 if (0 == written
&& 0 != c_len
) break; /* no more finished chunks */
690 if (written
>= c_len
) { /* chunk got finished */
695 if (c
== cq
->last
) cq
->last
= NULL
;
697 chunkqueue_push_unused_chunk(cq
, c
);
698 } else { /* partial chunk */
699 c
->offset
+= written
;
701 break; /* chunk not finished */
705 force_assert(0 == written
);
706 cq
->bytes_out
+= len
;
709 void chunkqueue_remove_finished_chunks(chunkqueue
*cq
) {
712 for (c
= cq
->first
; c
; c
= cq
->first
) {
713 if (0 != chunk_remaining_length(c
)) break; /* not finished yet */
716 if (c
== cq
->last
) cq
->last
= NULL
;
718 chunkqueue_push_unused_chunk(cq
, c
);
722 static void chunkqueue_remove_empty_chunks(chunkqueue
*cq
) {
724 chunkqueue_remove_finished_chunks(cq
);
725 if (chunkqueue_is_empty(cq
)) return;
727 for (c
= cq
->first
; c
->next
; c
= c
->next
) {
728 if (0 == chunk_remaining_length(c
->next
)) {
729 chunk
*empty
= c
->next
;
730 c
->next
= empty
->next
;
731 if (empty
== cq
->last
) cq
->last
= c
;
733 chunkqueue_push_unused_chunk(cq
, empty
);