[core] set chunkqueue tempdirs at startup
[lighttpd.git] / src / chunk.c
bloba6b1f1f05f9d246506e7529d9a274e59de205a23
1 #include "first.h"
3 /**
4 * the network chunk-API
7 */
9 #include "chunk.h"
10 #include "base.h"
11 #include "log.h"
13 #include <sys/types.h>
14 #include <sys/stat.h>
15 #include "sys-mmap.h"
17 #include <stdlib.h>
18 #include <fcntl.h>
19 #include <unistd.h>
21 #include <stdio.h>
22 #include <errno.h>
23 #include <string.h>
25 /* default 1MB, upper limit 128MB */
26 #define DEFAULT_TEMPFILE_SIZE (1 * 1024 * 1024)
27 #define MAX_TEMPFILE_SIZE (128 * 1024 * 1024)
29 static array *chunkqueue_default_tempdirs = NULL;
30 static unsigned int chunkqueue_default_tempfile_size = DEFAULT_TEMPFILE_SIZE;
32 chunkqueue *chunkqueue_init(void) {
33 chunkqueue *cq;
35 cq = calloc(1, sizeof(*cq));
36 force_assert(NULL != cq);
38 cq->first = NULL;
39 cq->last = NULL;
41 cq->unused = NULL;
43 cq->tempdirs = chunkqueue_default_tempdirs;
44 cq->upload_temp_file_size = chunkqueue_default_tempfile_size;
46 return cq;
49 static chunk *chunk_init(void) {
50 chunk *c;
52 c = calloc(1, sizeof(*c));
53 force_assert(NULL != c);
55 c->type = MEM_CHUNK;
56 c->mem = buffer_init();
57 c->file.name = buffer_init();
58 c->file.start = c->file.length = c->file.mmap.offset = 0;
59 c->file.fd = -1;
60 c->file.mmap.start = MAP_FAILED;
61 c->file.mmap.length = 0;
62 c->file.is_temp = 0;
63 c->offset = 0;
64 c->next = NULL;
66 return c;
69 static void chunk_reset(chunk *c) {
70 if (NULL == c) return;
72 c->type = MEM_CHUNK;
74 buffer_reset(c->mem);
76 if (c->file.is_temp && !buffer_string_is_empty(c->file.name)) {
77 unlink(c->file.name->ptr);
80 buffer_reset(c->file.name);
82 if (c->file.fd != -1) {
83 close(c->file.fd);
84 c->file.fd = -1;
86 if (MAP_FAILED != c->file.mmap.start) {
87 munmap(c->file.mmap.start, c->file.mmap.length);
88 c->file.mmap.start = MAP_FAILED;
90 c->file.start = c->file.length = c->file.mmap.offset = 0;
91 c->file.mmap.length = 0;
92 c->file.is_temp = 0;
93 c->offset = 0;
94 c->next = NULL;
97 static void chunk_free(chunk *c) {
98 if (NULL == c) return;
100 chunk_reset(c);
102 buffer_free(c->mem);
103 buffer_free(c->file.name);
105 free(c);
108 static off_t chunk_remaining_length(const chunk *c) {
109 off_t len = 0;
110 switch (c->type) {
111 case MEM_CHUNK:
112 len = buffer_string_length(c->mem);
113 break;
114 case FILE_CHUNK:
115 len = c->file.length;
116 break;
117 default:
118 force_assert(c->type == MEM_CHUNK || c->type == FILE_CHUNK);
119 break;
121 force_assert(c->offset <= len);
122 return len - c->offset;
125 void chunkqueue_free(chunkqueue *cq) {
126 chunk *c, *pc;
128 if (NULL == cq) return;
130 for (c = cq->first; c; ) {
131 pc = c;
132 c = c->next;
133 chunk_free(pc);
136 for (c = cq->unused; c; ) {
137 pc = c;
138 c = c->next;
139 chunk_free(pc);
142 free(cq);
145 static void chunkqueue_push_unused_chunk(chunkqueue *cq, chunk *c) {
146 force_assert(NULL != cq && NULL != c);
148 /* keep at max 4 chunks in the 'unused'-cache */
149 if (cq->unused_chunks > 4) {
150 chunk_free(c);
151 } else {
152 chunk_reset(c);
153 c->next = cq->unused;
154 cq->unused = c;
155 cq->unused_chunks++;
159 static chunk *chunkqueue_get_unused_chunk(chunkqueue *cq) {
160 chunk *c;
162 force_assert(NULL != cq);
164 /* check if we have a unused chunk */
165 if (0 == cq->unused) {
166 c = chunk_init();
167 } else {
168 /* take the first element from the list (a stack) */
169 c = cq->unused;
170 cq->unused = c->next;
171 c->next = NULL;
172 cq->unused_chunks--;
175 return c;
178 static void chunkqueue_prepend_chunk(chunkqueue *cq, chunk *c) {
179 c->next = cq->first;
180 cq->first = c;
182 if (NULL == cq->last) {
183 cq->last = c;
185 cq->bytes_in += chunk_remaining_length(c);
188 static void chunkqueue_append_chunk(chunkqueue *cq, chunk *c) {
189 c->next = NULL;
190 if (cq->last) {
191 cq->last->next = c;
193 cq->last = c;
195 if (NULL == cq->first) {
196 cq->first = c;
198 cq->bytes_in += chunk_remaining_length(c);
201 void chunkqueue_reset(chunkqueue *cq) {
202 chunk *cur = cq->first;
204 cq->first = cq->last = NULL;
206 while (NULL != cur) {
207 chunk *next = cur->next;
208 chunkqueue_push_unused_chunk(cq, cur);
209 cur = next;
212 cq->bytes_in = 0;
213 cq->bytes_out = 0;
214 cq->tempdir_idx = 0;
217 void chunkqueue_append_file_fd(chunkqueue *cq, buffer *fn, int fd, off_t offset, off_t len) {
218 chunk *c;
220 if (0 == len) {
221 close(fd);
222 return;
225 c = chunkqueue_get_unused_chunk(cq);
227 c->type = FILE_CHUNK;
229 buffer_copy_buffer(c->file.name, fn);
230 c->file.start = offset;
231 c->file.length = len;
232 c->file.fd = fd;
233 c->offset = 0;
235 chunkqueue_append_chunk(cq, c);
238 void chunkqueue_append_file(chunkqueue *cq, buffer *fn, off_t offset, off_t len) {
239 chunk *c;
241 if (0 == len) return;
243 c = chunkqueue_get_unused_chunk(cq);
245 c->type = FILE_CHUNK;
247 buffer_copy_buffer(c->file.name, fn);
248 c->file.start = offset;
249 c->file.length = len;
250 c->offset = 0;
252 chunkqueue_append_chunk(cq, c);
255 void chunkqueue_append_buffer(chunkqueue *cq, buffer *mem) {
256 chunk *c;
258 if (buffer_string_is_empty(mem)) return;
260 c = chunkqueue_get_unused_chunk(cq);
261 c->type = MEM_CHUNK;
262 force_assert(NULL != c->mem);
263 buffer_move(c->mem, mem);
265 chunkqueue_append_chunk(cq, c);
268 void chunkqueue_prepend_buffer(chunkqueue *cq, buffer *mem) {
269 chunk *c;
271 if (buffer_string_is_empty(mem)) return;
273 c = chunkqueue_get_unused_chunk(cq);
274 c->type = MEM_CHUNK;
275 force_assert(NULL != c->mem);
276 buffer_move(c->mem, mem);
278 chunkqueue_prepend_chunk(cq, c);
282 void chunkqueue_append_mem(chunkqueue *cq, const char * mem, size_t len) {
283 chunk *c;
285 if (0 == len) return;
287 c = chunkqueue_get_unused_chunk(cq);
288 c->type = MEM_CHUNK;
289 buffer_copy_string_len(c->mem, mem, len);
291 chunkqueue_append_chunk(cq, c);
295 void chunkqueue_append_chunkqueue(chunkqueue *cq, chunkqueue *src) {
296 if (src == NULL || NULL == src->first) return;
298 if (NULL == cq->first) {
299 cq->first = src->first;
300 } else {
301 cq->last->next = src->first;
303 cq->last = src->last;
304 cq->bytes_in += (src->bytes_in - src->bytes_out);
306 src->first = NULL;
307 src->last = NULL;
308 src->bytes_out = src->bytes_in;
312 void chunkqueue_get_memory(chunkqueue *cq, char **mem, size_t *len, size_t min_size, size_t alloc_size) {
313 static const size_t REALLOC_MAX_SIZE = 256;
314 chunk *c;
315 buffer *b;
316 char *dummy_mem;
317 size_t dummy_len;
319 force_assert(NULL != cq);
320 if (NULL == mem) mem = &dummy_mem;
321 if (NULL == len) len = &dummy_len;
323 /* default values: */
324 if (0 == min_size) min_size = 1024;
325 if (0 == alloc_size) alloc_size = 4096;
326 if (alloc_size < min_size) alloc_size = min_size;
328 if (NULL != cq->last && MEM_CHUNK == cq->last->type) {
329 size_t have;
331 b = cq->last->mem;
332 have = buffer_string_space(b);
334 /* unused buffer: allocate space */
335 if (buffer_string_is_empty(b)) {
336 buffer_string_prepare_copy(b, alloc_size);
337 have = buffer_string_space(b);
339 /* if buffer is really small just make it bigger */
340 else if (have < min_size && b->size <= REALLOC_MAX_SIZE) {
341 size_t cur_len = buffer_string_length(b);
342 size_t new_size = cur_len + min_size, append;
343 if (new_size < alloc_size) new_size = alloc_size;
345 append = new_size - cur_len;
346 if (append >= min_size) {
347 buffer_string_prepare_append(b, append);
348 have = buffer_string_space(b);
352 /* return pointer into existing buffer if large enough */
353 if (have >= min_size) {
354 *mem = b->ptr + buffer_string_length(b);
355 *len = have;
356 return;
360 /* allocate new chunk */
361 c = chunkqueue_get_unused_chunk(cq);
362 c->type = MEM_CHUNK;
363 chunkqueue_append_chunk(cq, c);
365 b = c->mem;
366 buffer_string_prepare_append(b, alloc_size);
368 *mem = b->ptr + buffer_string_length(b);
369 *len = buffer_string_space(b);
372 void chunkqueue_use_memory(chunkqueue *cq, size_t len) {
373 buffer *b;
375 force_assert(NULL != cq);
376 force_assert(NULL != cq->last && MEM_CHUNK == cq->last->type);
377 b = cq->last->mem;
379 if (len > 0) {
380 buffer_commit(b, len);
381 cq->bytes_in += len;
382 } else if (buffer_string_is_empty(b)) {
383 /* unused buffer: can't remove chunk easily from
384 * end of list, so just reset the buffer
386 buffer_reset(b);
390 void chunkqueue_set_tempdirs_default (array *tempdirs, unsigned int upload_temp_file_size) {
391 chunkqueue_default_tempdirs = tempdirs;
392 chunkqueue_default_tempfile_size
393 = (0 == upload_temp_file_size) ? DEFAULT_TEMPFILE_SIZE
394 : (upload_temp_file_size > MAX_TEMPFILE_SIZE) ? MAX_TEMPFILE_SIZE
395 : upload_temp_file_size;
398 #if 0
399 void chunkqueue_set_tempdirs(chunkqueue *cq, array *tempdirs, unsigned int upload_temp_file_size) {
400 force_assert(NULL != cq);
401 cq->tempdirs = tempdirs;
402 cq->upload_temp_file_size
403 = (0 == upload_temp_file_size) ? DEFAULT_TEMPFILE_SIZE
404 : (upload_temp_file_size > MAX_TEMPFILE_SIZE) ? MAX_TEMPFILE_SIZE
405 : upload_temp_file_size;
406 cq->tempdir_idx = 0;
408 #endif
410 void chunkqueue_steal(chunkqueue *dest, chunkqueue *src, off_t len) {
411 while (len > 0) {
412 chunk *c = src->first;
413 off_t clen = 0, use;
415 if (NULL == c) break;
417 clen = chunk_remaining_length(c);
418 if (0 == clen) {
419 /* drop empty chunk */
420 src->first = c->next;
421 if (c == src->last) src->last = NULL;
422 chunkqueue_push_unused_chunk(src, c);
423 continue;
426 use = len >= clen ? clen : len;
427 len -= use;
429 if (use == clen) {
430 /* move complete chunk */
431 src->first = c->next;
432 if (c == src->last) src->last = NULL;
434 chunkqueue_append_chunk(dest, c);
435 } else {
436 /* partial chunk with length "use" */
438 switch (c->type) {
439 case MEM_CHUNK:
440 chunkqueue_append_mem(dest, c->mem->ptr + c->offset, use);
441 break;
442 case FILE_CHUNK:
443 /* tempfile flag is in "last" chunk after the split */
444 chunkqueue_append_file(dest, c->file.name, c->file.start + c->offset, use);
445 break;
448 c->offset += use;
449 force_assert(0 == len);
452 src->bytes_out += use;
456 static chunk *chunkqueue_get_append_tempfile(chunkqueue *cq) {
457 chunk *c;
458 buffer *template = buffer_init_string("/var/tmp/lighttpd-upload-XXXXXX");
459 int fd = -1;
461 if (cq->tempdirs && cq->tempdirs->used) {
462 /* we have several tempdirs, only if all of them fail we jump out */
464 for (errno = EIO; cq->tempdir_idx < cq->tempdirs->used; ++cq->tempdir_idx) {
465 data_string *ds = (data_string *)cq->tempdirs->data[cq->tempdir_idx];
467 buffer_copy_buffer(template, ds->value);
468 buffer_append_slash(template);
469 buffer_append_string_len(template, CONST_STR_LEN("lighttpd-upload-XXXXXX"));
471 if (-1 != (fd = mkstemp(template->ptr))) break;
473 } else {
474 fd = mkstemp(template->ptr);
477 if (fd < 0) {
478 buffer_free(template);
479 return NULL;
482 c = chunkqueue_get_unused_chunk(cq);
483 c->type = FILE_CHUNK;
484 c->file.fd = fd;
485 c->file.is_temp = 1;
486 buffer_copy_buffer(c->file.name, template);
487 c->file.length = 0;
489 chunkqueue_append_chunk(cq, c);
491 buffer_free(template);
493 return c;
496 static void chunkqueue_remove_empty_chunks(chunkqueue *cq);
498 int chunkqueue_append_mem_to_tempfile(server *srv, chunkqueue *dest, const char *mem, size_t len) {
499 chunk *dst_c;
500 ssize_t written;
502 do {
504 * if the last chunk is
505 * - smaller than dest->upload_temp_file_size
506 * - not read yet (offset == 0)
507 * -> append to it (so it might actually become larger than dest->upload_temp_file_size)
508 * otherwise
509 * -> create a new chunk
511 * */
513 dst_c = dest->last;
514 if (NULL != dst_c
515 && FILE_CHUNK == dst_c->type
516 && dst_c->file.is_temp
517 && dst_c->file.fd >= 0
518 && 0 == dst_c->offset) {
519 /* ok, take the last chunk for our job */
521 if (dst_c->file.length >= (off_t)dest->upload_temp_file_size) {
522 /* the chunk is too large now, close it */
523 int rc = close(dst_c->file.fd);
524 dst_c->file.fd = -1;
525 if (0 != rc) {
526 log_error_write(srv, __FILE__, __LINE__, "sbss",
527 "close() temp-file", dst_c->file.name, "failed:",
528 strerror(errno));
529 return -1;
531 dst_c = NULL;
533 } else {
534 dst_c = NULL;
537 if (NULL == dst_c && NULL == (dst_c = chunkqueue_get_append_tempfile(dest))) {
538 /* we don't have file to write to,
539 * EACCES might be one reason.
541 * Instead of sending 500 we send 413 and say the request is too large
544 log_error_write(srv, __FILE__, __LINE__, "ss",
545 "opening temp-file failed:", strerror(errno));
547 return -1;
550 written = write(dst_c->file.fd, mem, len);
552 if ((size_t) written == len) {
553 dst_c->file.length += len;
554 dest->bytes_in += len;
556 return 0;
557 } else if (written >= 0) {
558 /*(assume EINTR if partial write and retry write();
559 * retry write() might fail with ENOSPC if no more space on volume)*/
560 dest->bytes_in += written;
561 mem += written;
562 len -= (size_t)written;
563 dst_c->file.length += (size_t)written;
564 /* continue; retry */
565 } else if (errno == EINTR) {
566 /* continue; retry */
567 } else {
568 int retry = (errno == ENOSPC && dest->tempdirs && ++dest->tempdir_idx < dest->tempdirs->used);
569 if (!retry) {
570 log_error_write(srv, __FILE__, __LINE__, "sbs",
571 "write() temp-file", dst_c->file.name, "failed:",
572 strerror(errno));
575 if (0 == chunk_remaining_length(dst_c)) {
576 /*(remove empty chunk and unlink tempfile)*/
577 chunkqueue_remove_empty_chunks(dest);
578 } else {/*(close tempfile; avoid later attempts to append)*/
579 int rc = close(dst_c->file.fd);
580 dst_c->file.fd = -1;
581 if (0 != rc) {
582 log_error_write(srv, __FILE__, __LINE__, "sbss",
583 "close() temp-file", dst_c->file.name, "failed:",
584 strerror(errno));
585 return -1;
588 if (!retry) return -1;
590 /* continue; retry */
593 } while (dst_c);
595 return -1; /*(not reached)*/
598 int chunkqueue_steal_with_tempfiles(server *srv, chunkqueue *dest, chunkqueue *src, off_t len) {
599 while (len > 0) {
600 chunk *c = src->first;
601 off_t clen = 0, use;
603 if (NULL == c) break;
605 clen = chunk_remaining_length(c);
606 if (0 == clen) {
607 /* drop empty chunk */
608 src->first = c->next;
609 if (c == src->last) src->last = NULL;
610 chunkqueue_push_unused_chunk(src, c);
611 continue;
614 use = (len >= clen) ? clen : len;
615 len -= use;
617 switch (c->type) {
618 case FILE_CHUNK:
619 if (use == clen) {
620 /* move complete chunk */
621 src->first = c->next;
622 if (c == src->last) src->last = NULL;
623 chunkqueue_append_chunk(dest, c);
624 } else {
625 /* partial chunk with length "use" */
626 /* tempfile flag is in "last" chunk after the split */
627 chunkqueue_append_file(dest, c->file.name, c->file.start + c->offset, use);
629 c->offset += use;
630 force_assert(0 == len);
632 break;
634 case MEM_CHUNK:
635 /* store "use" bytes from memory chunk in tempfile */
636 if (0 != chunkqueue_append_mem_to_tempfile(srv, dest, c->mem->ptr + c->offset, use)) {
637 return -1;
640 if (use == clen) {
641 /* finished chunk */
642 src->first = c->next;
643 if (c == src->last) src->last = NULL;
644 chunkqueue_push_unused_chunk(src, c);
645 } else {
646 /* partial chunk */
647 c->offset += use;
648 force_assert(0 == len);
650 break;
653 src->bytes_out += use;
656 return 0;
659 off_t chunkqueue_length(chunkqueue *cq) {
660 off_t len = 0;
661 chunk *c;
663 for (c = cq->first; c; c = c->next) {
664 len += chunk_remaining_length(c);
667 return len;
670 int chunkqueue_is_empty(chunkqueue *cq) {
671 return NULL == cq->first;
674 void chunkqueue_mark_written(chunkqueue *cq, off_t len) {
675 off_t written = len;
676 chunk *c;
677 force_assert(len >= 0);
679 for (c = cq->first; NULL != c; c = cq->first) {
680 off_t c_len = chunk_remaining_length(c);
682 if (0 == written && 0 != c_len) break; /* no more finished chunks */
684 if (written >= c_len) { /* chunk got finished */
685 c->offset += c_len;
686 written -= c_len;
688 cq->first = c->next;
689 if (c == cq->last) cq->last = NULL;
691 chunkqueue_push_unused_chunk(cq, c);
692 } else { /* partial chunk */
693 c->offset += written;
694 written = 0;
695 break; /* chunk not finished */
699 force_assert(0 == written);
700 cq->bytes_out += len;
703 void chunkqueue_remove_finished_chunks(chunkqueue *cq) {
704 chunk *c;
706 for (c = cq->first; c; c = cq->first) {
707 if (0 != chunk_remaining_length(c)) break; /* not finished yet */
709 cq->first = c->next;
710 if (c == cq->last) cq->last = NULL;
712 chunkqueue_push_unused_chunk(cq, c);
716 static void chunkqueue_remove_empty_chunks(chunkqueue *cq) {
717 chunk *c;
718 chunkqueue_remove_finished_chunks(cq);
719 if (chunkqueue_is_empty(cq)) return;
721 for (c = cq->first; c->next; c = c->next) {
722 if (0 == chunk_remaining_length(c->next)) {
723 chunk *empty = c->next;
724 c->next = empty->next;
725 if (empty == cq->last) cq->last = c;
727 chunkqueue_push_unused_chunk(cq, empty);