src/chunk.c

   1 #include "first.h"
   2
   3 /**
   4  * the network chunk-API
   5  *
   6  *
   7  */
   8
   9 #include "chunk.h"
  10 #include "fdevent.h"
  11 #include "log.h"
  12
  13 #include <sys/types.h>
  14 #include <sys/stat.h>
  15 #include "sys-mmap.h"
  16
  17 #include <stdlib.h>
  18 #include <fcntl.h>
  19 #include <unistd.h>
  20
  21 #include <errno.h>
  22 #include <string.h>
  23
  24 /* default 1MB, upper limit 128MB */
  25 #define DEFAULT_TEMPFILE_SIZE (1 * 1024 * 1024)
  26 #define MAX_TEMPFILE_SIZE (128 * 1024 * 1024)
  27
  28 static size_t chunk_buf_sz = 4096;
  29 static chunk *chunks;
  30 static chunk *chunk_buffers;
  31 static array *chunkqueue_default_tempdirs = NULL;
  32 static unsigned int chunkqueue_default_tempfile_size = DEFAULT_TEMPFILE_SIZE;
  33
  34 void chunkqueue_set_chunk_size (size_t sz)
  35 {
  36     chunk_buf_sz = sz > 0 ? ((sz + 1023) & ~1023uL) : 4096;
  37 }
  38
  39 void chunkqueue_set_tempdirs_default_reset (void)
  40 {
  41     chunkqueue_default_tempdirs = NULL;
  42     chunkqueue_default_tempfile_size = DEFAULT_TEMPFILE_SIZE;
  43 }
  44
  45 chunkqueue *chunkqueue_init(void) {
  46         chunkqueue *cq;
  47
  48         cq = calloc(1, sizeof(*cq));
  49         force_assert(NULL != cq);
  50
  51         cq->first = NULL;
  52         cq->last = NULL;
  53
  54         cq->tempdirs              = chunkqueue_default_tempdirs;
  55         cq->upload_temp_file_size = chunkqueue_default_tempfile_size;
  56
  57         return cq;
  58 }
  59
  60 static chunk *chunk_init(size_t sz) {
  61         chunk *c;
  62
  63         c = calloc(1, sizeof(*c));
  64         force_assert(NULL != c);
  65
  66         c->type = MEM_CHUNK;
  67         c->mem = buffer_init();
  68         c->file.start = c->file.length = c->file.mmap.offset = 0;
  69         c->file.fd = -1;
  70         c->file.mmap.start = MAP_FAILED;
  71         c->file.mmap.length = 0;
  72         c->file.is_temp = 0;
  73         c->offset = 0;
  74         c->next = NULL;
  75
  76         buffer_string_prepare_copy(c->mem, sz-1);
  77
  78         return c;
  79 }
  80
  81 static void chunk_reset_file_chunk(chunk *c) {
  82         if (c->file.is_temp && !buffer_string_is_empty(c->mem)) {
  83                 unlink(c->mem->ptr);
  84         }
  85         if (c->file.fd != -1) {
  86                 close(c->file.fd);
  87                 c->file.fd = -1;
  88         }
  89         if (MAP_FAILED != c->file.mmap.start) {
  90                 munmap(c->file.mmap.start, c->file.mmap.length);
  91                 c->file.mmap.start = MAP_FAILED;
  92         }
  93         c->file.start = c->file.length = c->file.mmap.offset = 0;
  94         c->file.mmap.length = 0;
  95         c->file.is_temp = 0;
  96         c->type = MEM_CHUNK;
  97 }
  98
  99 static void chunk_reset(chunk *c) {
 100         if (c->type == FILE_CHUNK) chunk_reset_file_chunk(c);
 101
 102         buffer_clear(c->mem);
 103         c->offset = 0;
 104 }
 105
 106 static void chunk_free(chunk *c) {
 107         if (c->type == FILE_CHUNK) chunk_reset_file_chunk(c);
 108         buffer_free(c->mem);
 109         free(c);
 110 }
 111
 112 buffer * chunk_buffer_acquire(void) {
 113     chunk *c;
 114     buffer *b;
 115     if (chunks) {
 116         c = chunks;
 117         chunks = c->next;
 118     }
 119     else {
 120         c = chunk_init(chunk_buf_sz);
 121     }
 122     c->next = chunk_buffers;
 123     chunk_buffers = c;
 124     b = c->mem;
 125     c->mem = NULL;
 126     return b;
 127 }
 128
 129 void chunk_buffer_release(buffer *b) {
 130     if (NULL == b) return;
 131     if (b->size >= chunk_buf_sz && chunk_buffers) {
 132         chunk *c = chunk_buffers;
 133         chunk_buffers = c->next;
 134         c->mem = b;
 135         c->next = chunks;
 136         chunks = c;
 137         buffer_clear(b);
 138     }
 139     else {
 140         buffer_free(b);
 141     }
 142 }
 143
 144 static chunk * chunk_acquire(void) {
 145     if (chunks) {
 146         chunk *c = chunks;
 147         chunks = c->next;
 148         return c;
 149     }
 150     else {
 151         return chunk_init(chunk_buf_sz);
 152     }
 153 }
 154
 155 static void chunk_release(chunk *c) {
 156     if (c->mem->size >= chunk_buf_sz) {
 157         chunk_reset(c);
 158         c->next = chunks;
 159         chunks = c;
 160     }
 161     else {
 162         chunk_free(c);
 163     }
 164 }
 165
 166 void chunkqueue_chunk_pool_clear(void)
 167 {
 168     for (chunk *next, *c = chunks; c; c = next) {
 169         next = c->next;
 170         chunk_free(c);
 171     }
 172     chunks = NULL;
 173 }
 174
 175 void chunkqueue_chunk_pool_free(void)
 176 {
 177     chunkqueue_chunk_pool_clear();
 178     for (chunk *next, *c = chunk_buffers; c; c = next) {
 179         next = c->next;
 180         c->mem = buffer_init(); /*(chunk_reset() expects c->mem != NULL)*/
 181         chunk_free(c);
 182     }
 183     chunk_buffers = NULL;
 184 }
 185
 186 static off_t chunk_remaining_length(const chunk *c) {
 187         off_t len = 0;
 188         switch (c->type) {
 189         case MEM_CHUNK:
 190                 len = buffer_string_length(c->mem);
 191                 break;
 192         case FILE_CHUNK:
 193                 len = c->file.length;
 194                 break;
 195         default:
 196                 force_assert(c->type == MEM_CHUNK || c->type == FILE_CHUNK);
 197                 break;
 198         }
 199         force_assert(c->offset <= len);
 200         return len - c->offset;
 201 }
 202
 203 void chunkqueue_free(chunkqueue *cq) {
 204         chunk *c, *pc;
 205
 206         if (NULL == cq) return;
 207
 208         for (c = cq->first; c; ) {
 209                 pc = c;
 210                 c = c->next;
 211                 chunk_release(pc);
 212         }
 213
 214         free(cq);
 215 }
 216
 217 static void chunkqueue_prepend_chunk(chunkqueue *cq, chunk *c) {
 218         c->next = cq->first;
 219         cq->first = c;
 220
 221         if (NULL == cq->last) {
 222                 cq->last = c;
 223         }
 224 }
 225
 226 static void chunkqueue_append_chunk(chunkqueue *cq, chunk *c) {
 227         c->next = NULL;
 228         if (cq->last) {
 229                 cq->last->next = c;
 230         }
 231         cq->last = c;
 232
 233         if (NULL == cq->first) {
 234                 cq->first = c;
 235         }
 236 }
 237
 238 static chunk * chunkqueue_prepend_mem_chunk(chunkqueue *cq) {
 239     chunk *c = chunk_acquire();
 240     chunkqueue_prepend_chunk(cq, c);
 241     return c;
 242 }
 243
 244 static chunk * chunkqueue_append_mem_chunk(chunkqueue *cq) {
 245     chunk *c = chunk_acquire();
 246     chunkqueue_append_chunk(cq, c);
 247     return c;
 248 }
 249
 250 static chunk * chunkqueue_append_file_chunk(chunkqueue *cq, buffer *fn, off_t offset, off_t len) {
 251     chunk *c = chunk_acquire();
 252     chunkqueue_append_chunk(cq, c);
 253     c->type = FILE_CHUNK;
 254     c->file.start = offset;
 255     c->file.length = len;
 256     cq->bytes_in += len;
 257     buffer_copy_buffer(c->mem, fn);
 258     return c;
 259 }
 260
 261 void chunkqueue_reset(chunkqueue *cq) {
 262         chunk *cur = cq->first;
 263
 264         cq->first = cq->last = NULL;
 265
 266         while (NULL != cur) {
 267                 chunk *next = cur->next;
 268                 chunk_release(cur);
 269                 cur = next;
 270         }
 271
 272         cq->bytes_in = 0;
 273         cq->bytes_out = 0;
 274         cq->tempdir_idx = 0;
 275 }
 276
 277 void chunkqueue_append_file_fd(chunkqueue *cq, buffer *fn, int fd, off_t offset, off_t len) {
 278     if (len > 0) {
 279         (chunkqueue_append_file_chunk(cq, fn, offset, len))->file.fd = fd;
 280     }
 281     else {
 282         close(fd);
 283     }
 284 }
 285
 286 void chunkqueue_append_file(chunkqueue *cq, buffer *fn, off_t offset, off_t len) {
 287     if (len > 0) {
 288         chunkqueue_append_file_chunk(cq, fn, offset, len);
 289     }
 290 }
 291
 292
 293 static int chunkqueue_append_mem_extend_chunk(chunkqueue *cq, const char *mem, size_t len) {
 294         chunk *c = cq->last;
 295         if (0 == len) return 1;
 296         if (c != NULL && c->type == MEM_CHUNK
 297             && buffer_string_space(c->mem) >= len) {
 298                 buffer_append_string_len(c->mem, mem, len);
 299                 cq->bytes_in += len;
 300                 return 1;
 301         }
 302         return 0;
 303 }
 304
 305
 306 void chunkqueue_append_buffer(chunkqueue *cq, buffer *mem) {
 307         chunk *c;
 308         size_t len = buffer_string_length(mem);
 309         if (len < 256 && chunkqueue_append_mem_extend_chunk(cq, mem->ptr, len)) return;
 310
 311         c = chunkqueue_append_mem_chunk(cq);
 312         cq->bytes_in += len;
 313         buffer_move(c->mem, mem);
 314 }
 315
 316
 317 void chunkqueue_append_mem(chunkqueue *cq, const char * mem, size_t len) {
 318         chunk *c;
 319         if (len < chunk_buf_sz && chunkqueue_append_mem_extend_chunk(cq, mem, len))
 320                 return;
 321
 322         c = chunkqueue_append_mem_chunk(cq);
 323         cq->bytes_in += len;
 324         buffer_copy_string_len(c->mem, mem, len);
 325 }
 326
 327
 328 void chunkqueue_append_mem_min(chunkqueue *cq, const char * mem, size_t len) {
 329         chunk *c;
 330         if (len < chunk_buf_sz && chunkqueue_append_mem_extend_chunk(cq, mem, len))
 331                 return;
 332
 333         c = chunk_init(len+1);
 334         chunkqueue_append_chunk(cq, c);
 335         cq->bytes_in += len;
 336         buffer_copy_string_len(c->mem, mem, len);
 337 }
 338
 339
 340 void chunkqueue_append_chunkqueue(chunkqueue *cq, chunkqueue *src) {
 341         if (src == NULL || NULL == src->first) return;
 342
 343         if (NULL == cq->first) {
 344                 cq->first = src->first;
 345         } else {
 346                 cq->last->next = src->first;
 347         }
 348         cq->last = src->last;
 349         cq->bytes_in += (src->bytes_in - src->bytes_out);
 350
 351         src->first = NULL;
 352         src->last = NULL;
 353         src->bytes_out = src->bytes_in;
 354 }
 355
 356
 357 __attribute_cold__
 358 static void chunkqueue_buffer_open_resize(chunk *c, size_t sz) {
 359         chunk * const n = chunk_init((sz + 4095) & ~4095uL);
 360         buffer * const b = c->mem;
 361         c->mem = n->mem;
 362         n->mem = b;
 363         chunk_release(n);
 364 }
 365
 366
 367 buffer * chunkqueue_prepend_buffer_open_sz(chunkqueue *cq, size_t sz) {
 368         chunk * const c = chunkqueue_prepend_mem_chunk(cq);
 369         if (buffer_string_space(c->mem) < sz) {
 370                 chunkqueue_buffer_open_resize(c, sz);
 371         }
 372         return c->mem;
 373 }
 374
 375
 376 buffer * chunkqueue_prepend_buffer_open(chunkqueue *cq) {
 377         chunk *c = chunkqueue_prepend_mem_chunk(cq);
 378         return c->mem;
 379 }
 380
 381
 382 void chunkqueue_prepend_buffer_commit(chunkqueue *cq) {
 383         cq->bytes_in += buffer_string_length(cq->first->mem);
 384 }
 385
 386
 387 buffer * chunkqueue_append_buffer_open_sz(chunkqueue *cq, size_t sz) {
 388         chunk * const c = chunkqueue_append_mem_chunk(cq);
 389         if (buffer_string_space(c->mem) < sz) {
 390                 chunkqueue_buffer_open_resize(c, sz);
 391         }
 392         return c->mem;
 393 }
 394
 395
 396 buffer * chunkqueue_append_buffer_open(chunkqueue *cq) {
 397         chunk *c = chunkqueue_append_mem_chunk(cq);
 398         return c->mem;
 399 }
 400
 401
 402 void chunkqueue_append_buffer_commit(chunkqueue *cq) {
 403         cq->bytes_in += buffer_string_length(cq->last->mem);
 404 }
 405
 406
 407 static void chunkqueue_remove_empty_chunks(chunkqueue *cq);
 408
 409
 410 char * chunkqueue_get_memory(chunkqueue *cq, size_t *len) {
 411         size_t sz = *len ? *len : (chunk_buf_sz >> 1);
 412         buffer *b;
 413         chunk *c = cq->last;
 414         if (NULL != c && MEM_CHUNK == c->type) {
 415                 /* return pointer into existing buffer if large enough */
 416                 size_t avail = buffer_string_space(c->mem);
 417                 if (avail >= sz) {
 418                         *len = avail;
 419                         b = c->mem;
 420                         return b->ptr + buffer_string_length(b);
 421                 }
 422         }
 423
 424         /* allocate new chunk */
 425         b = chunkqueue_append_buffer_open_sz(cq, sz);
 426         *len = buffer_string_space(b);
 427         return b->ptr;
 428 }
 429
 430 void chunkqueue_use_memory(chunkqueue *cq, size_t len) {
 431         buffer *b;
 432
 433         force_assert(NULL != cq);
 434         force_assert(NULL != cq->last && MEM_CHUNK == cq->last->type);
 435         b = cq->last->mem;
 436
 437         if (len > 0) {
 438                 buffer_commit(b, len);
 439                 cq->bytes_in += len;
 440         } else if (buffer_string_is_empty(b)) {
 441                 /* scan chunkqueue to remove empty last chunk
 442                  * (generally not expecting a deep queue) */
 443                 chunkqueue_remove_empty_chunks(cq);
 444         }
 445 }
 446
 447 void chunkqueue_set_tempdirs_default (array *tempdirs, unsigned int upload_temp_file_size) {
 448         chunkqueue_default_tempdirs = tempdirs;
 449         chunkqueue_default_tempfile_size
 450                 = (0 == upload_temp_file_size)                ? DEFAULT_TEMPFILE_SIZE
 451                 : (upload_temp_file_size > MAX_TEMPFILE_SIZE) ? MAX_TEMPFILE_SIZE
 452                                                               : upload_temp_file_size;
 453 }
 454
 455 #if 0
 456 void chunkqueue_set_tempdirs(chunkqueue *cq, array *tempdirs, unsigned int upload_temp_file_size) {
 457         force_assert(NULL != cq);
 458         cq->tempdirs = tempdirs;
 459         cq->upload_temp_file_size
 460                 = (0 == upload_temp_file_size)                ? DEFAULT_TEMPFILE_SIZE
 461                 : (upload_temp_file_size > MAX_TEMPFILE_SIZE) ? MAX_TEMPFILE_SIZE
 462                                                               : upload_temp_file_size;
 463         cq->tempdir_idx = 0;
 464 }
 465 #endif
 466
 467 void chunkqueue_steal(chunkqueue *dest, chunkqueue *src, off_t len) {
 468         while (len > 0) {
 469                 chunk *c = src->first;
 470                 off_t clen = 0, use;
 471
 472                 if (NULL == c) break;
 473
 474                 clen = chunk_remaining_length(c);
 475                 if (0 == clen) {
 476                         /* drop empty chunk */
 477                         src->first = c->next;
 478                         if (c == src->last) src->last = NULL;
 479                         chunk_release(c);
 480                         continue;
 481                 }
 482
 483                 use = len >= clen ? clen : len;
 484                 len -= use;
 485
 486                 if (use == clen) {
 487                         /* move complete chunk */
 488                         src->first = c->next;
 489                         if (c == src->last) src->last = NULL;
 490
 491                         chunkqueue_append_chunk(dest, c);
 492                         dest->bytes_in += use;
 493                 } else {
 494                         /* partial chunk with length "use" */
 495
 496                         switch (c->type) {
 497                         case MEM_CHUNK:
 498                                 chunkqueue_append_mem(dest, c->mem->ptr + c->offset, use);
 499                                 break;
 500                         case FILE_CHUNK:
 501                                 /* tempfile flag is in "last" chunk after the split */
 502                                 chunkqueue_append_file(dest, c->mem, c->file.start + c->offset, use);
 503                                 break;
 504                         }
 505
 506                         c->offset += use;
 507                         force_assert(0 == len);
 508                 }
 509
 510                 src->bytes_out += use;
 511         }
 512 }
 513
 514 static chunk *chunkqueue_get_append_tempfile(server *srv, chunkqueue *cq) {
 515         chunk *c;
 516         buffer *template = buffer_init_string("/var/tmp/lighttpd-upload-XXXXXX");
 517         int fd = -1;
 518
 519         if (cq->tempdirs && cq->tempdirs->used) {
 520                 /* we have several tempdirs, only if all of them fail we jump out */
 521
 522                 for (errno = EIO; cq->tempdir_idx < cq->tempdirs->used; ++cq->tempdir_idx) {
 523                         data_string *ds = (data_string *)cq->tempdirs->data[cq->tempdir_idx];
 524
 525                         buffer_copy_buffer(template, ds->value);
 526                         buffer_append_path_len(template, CONST_STR_LEN("lighttpd-upload-XXXXXX"));
 527
 528                       #ifdef __COVERITY__
 529                         /* POSIX-2008 requires mkstemp create file with 0600 perms */
 530                         umask(0600);
 531                       #endif
 532                         /* coverity[secure_temp : FALSE] */
 533                         if (-1 != (fd = mkstemp(template->ptr))) break;
 534                 }
 535         } else {
 536               #ifdef __COVERITY__
 537                 /* POSIX-2008 requires mkstemp create file with 0600 perms */
 538                 umask(0600);
 539               #endif
 540                 /* coverity[secure_temp : FALSE] */
 541                 fd = mkstemp(template->ptr);
 542         }
 543
 544         if (fd < 0) {
 545                 /* (report only the last error to mkstemp()
 546                  *  if multiple temp dirs attempted) */
 547                 log_error_write(srv, __FILE__, __LINE__, "sbs",
 548                                 "opening temp-file failed:",
 549                                 template, strerror(errno));
 550                 buffer_free(template);
 551                 return NULL;
 552         }
 553
 554         if (0 != fcntl(fd, F_SETFL, fcntl(fd, F_GETFL, 0) | O_APPEND)) {
 555                 /* (should not happen; fd is regular file) */
 556                 log_error_write(srv, __FILE__, __LINE__, "sbs",
 557                                 "fcntl():", template, strerror(errno));
 558                 close(fd);
 559                 buffer_free(template);
 560                 return NULL;
 561         }
 562         fdevent_setfd_cloexec(fd);
 563
 564         c = chunkqueue_append_file_chunk(cq, template, 0, 0);
 565         c->file.fd = fd;
 566         c->file.is_temp = 1;
 567
 568         buffer_free(template);
 569
 570         return c;
 571 }
 572
 573 int chunkqueue_append_mem_to_tempfile(server *srv, chunkqueue *dest, const char *mem, size_t len) {
 574         chunk *dst_c;
 575         ssize_t written;
 576
 577         do {
 578                 /*
 579                  * if the last chunk is
 580                  * - smaller than dest->upload_temp_file_size
 581                  * - not read yet (offset == 0)
 582                  * -> append to it (so it might actually become larger than dest->upload_temp_file_size)
 583                  * otherwise
 584                  * -> create a new chunk
 585                  *
 586                  * */
 587
 588                 dst_c = dest->last;
 589                 if (NULL != dst_c
 590                         && FILE_CHUNK == dst_c->type
 591                         && dst_c->file.is_temp
 592                         && dst_c->file.fd >= 0
 593                         && 0 == dst_c->offset) {
 594                         /* ok, take the last chunk for our job */
 595
 596                         if (dst_c->file.length >= (off_t)dest->upload_temp_file_size) {
 597                                 /* the chunk is too large now, close it */
 598                                 int rc = close(dst_c->file.fd);
 599                                 dst_c->file.fd = -1;
 600                                 if (0 != rc) {
 601                                         log_error_write(srv, __FILE__, __LINE__, "sbss",
 602                                                 "close() temp-file", dst_c->mem, "failed:",
 603                                                 strerror(errno));
 604                                         return -1;
 605                                 }
 606                                 dst_c = NULL;
 607                         }
 608                 } else {
 609                         dst_c = NULL;
 610                 }
 611
 612                 if (NULL == dst_c && NULL == (dst_c = chunkqueue_get_append_tempfile(srv, dest))) {
 613                         return -1;
 614                 }
 615               #ifdef __COVERITY__
 616                 if (dst_c->file.fd < 0) return -1;
 617               #endif
 618
 619                 /* (dst_c->file.fd >= 0) */
 620                 /* coverity[negative_returns : FALSE] */
 621                 written = write(dst_c->file.fd, mem, len);
 622
 623                 if ((size_t) written == len) {
 624                         dst_c->file.length += len;
 625                         dest->bytes_in += len;
 626
 627                         return 0;
 628                 } else if (written >= 0) {
 629                         /*(assume EINTR if partial write and retry write();
 630                          * retry write() might fail with ENOSPC if no more space on volume)*/
 631                         dest->bytes_in += written;
 632                         mem += written;
 633                         len -= (size_t)written;
 634                         dst_c->file.length += (size_t)written;
 635                         /* continue; retry */
 636                 } else if (errno == EINTR) {
 637                         /* continue; retry */
 638                 } else {
 639                         int retry = (errno == ENOSPC && dest->tempdirs && ++dest->tempdir_idx < dest->tempdirs->used);
 640                         if (!retry) {
 641                                 log_error_write(srv, __FILE__, __LINE__, "sbs",
 642                                                 "write() temp-file", dst_c->mem, "failed:",
 643                                                 strerror(errno));
 644                         }
 645
 646                         if (0 == chunk_remaining_length(dst_c)) {
 647                                 /*(remove empty chunk and unlink tempfile)*/
 648                                 chunkqueue_remove_empty_chunks(dest);
 649                         } else {/*(close tempfile; avoid later attempts to append)*/
 650                                 int rc = close(dst_c->file.fd);
 651                                 dst_c->file.fd = -1;
 652                                 if (0 != rc) {
 653                                         log_error_write(srv, __FILE__, __LINE__, "sbss",
 654                                                 "close() temp-file", dst_c->mem, "failed:",
 655                                                 strerror(errno));
 656                                         return -1;
 657                                 }
 658                         }
 659                         if (!retry) break; /* return -1; */
 660
 661                         /* continue; retry */
 662                 }
 663
 664         } while (dst_c);
 665
 666         return -1;
 667 }
 668
 669 int chunkqueue_steal_with_tempfiles(server *srv, chunkqueue *dest, chunkqueue *src, off_t len) {
 670         while (len > 0) {
 671                 chunk *c = src->first;
 672                 off_t clen = 0, use;
 673
 674                 if (NULL == c) break;
 675
 676                 clen = chunk_remaining_length(c);
 677                 if (0 == clen) {
 678                         /* drop empty chunk */
 679                         src->first = c->next;
 680                         if (c == src->last) src->last = NULL;
 681                         chunk_release(c);
 682                         continue;
 683                 }
 684
 685                 use = (len >= clen) ? clen : len;
 686                 len -= use;
 687
 688                 switch (c->type) {
 689                 case FILE_CHUNK:
 690                         if (use == clen) {
 691                                 /* move complete chunk */
 692                                 src->first = c->next;
 693                                 if (c == src->last) src->last = NULL;
 694                                 chunkqueue_append_chunk(dest, c);
 695                                 dest->bytes_in += use;
 696                         } else {
 697                                 /* partial chunk with length "use" */
 698                                 /* tempfile flag is in "last" chunk after the split */
 699                                 chunkqueue_append_file(dest, c->mem, c->file.start + c->offset, use);
 700
 701                                 c->offset += use;
 702                                 force_assert(0 == len);
 703                         }
 704                         break;
 705
 706                 case MEM_CHUNK:
 707                         /* store "use" bytes from memory chunk in tempfile */
 708                         if (0 != chunkqueue_append_mem_to_tempfile(srv, dest, c->mem->ptr + c->offset, use)) {
 709                                 return -1;
 710                         }
 711
 712                         if (use == clen) {
 713                                 /* finished chunk */
 714                                 src->first = c->next;
 715                                 if (c == src->last) src->last = NULL;
 716                                 chunk_release(c);
 717                         } else {
 718                                 /* partial chunk */
 719                                 c->offset += use;
 720                                 force_assert(0 == len);
 721                         }
 722                         break;
 723                 }
 724
 725                 src->bytes_out += use;
 726         }
 727
 728         return 0;
 729 }
 730
 731 off_t chunkqueue_length(chunkqueue *cq) {
 732         off_t len = 0;
 733         chunk *c;
 734
 735         for (c = cq->first; c; c = c->next) {
 736                 len += chunk_remaining_length(c);
 737         }
 738
 739         return len;
 740 }
 741
 742 void chunkqueue_mark_written(chunkqueue *cq, off_t len) {
 743         off_t written = len;
 744         chunk *c;
 745         force_assert(len >= 0);
 746
 747         for (c = cq->first; NULL != c; c = cq->first) {
 748                 off_t c_len = chunk_remaining_length(c);
 749
 750                 if (0 == written && 0 != c_len) break; /* no more finished chunks */
 751
 752                 if (written >= c_len) { /* chunk got finished */
 753                         c->offset += c_len;
 754                         written -= c_len;
 755
 756                         cq->first = c->next;
 757                         if (c == cq->last) cq->last = NULL;
 758                         chunk_release(c);
 759                 } else { /* partial chunk */
 760                         c->offset += written;
 761                         written = 0;
 762                         break; /* chunk not finished */
 763                 }
 764         }
 765
 766         force_assert(0 == written);
 767         cq->bytes_out += len;
 768 }
 769
 770 void chunkqueue_remove_finished_chunks(chunkqueue *cq) {
 771         chunk *c;
 772
 773         for (c = cq->first; c; c = cq->first) {
 774                 if (0 != chunk_remaining_length(c)) break; /* not finished yet */
 775
 776                 cq->first = c->next;
 777                 if (c == cq->last) cq->last = NULL;
 778                 chunk_release(c);
 779         }
 780 }
 781
 782 static void chunkqueue_remove_empty_chunks(chunkqueue *cq) {
 783         chunk *c;
 784         chunkqueue_remove_finished_chunks(cq);
 785         if (chunkqueue_is_empty(cq)) return;
 786
 787         for (c = cq->first; c && c->next; c = c->next) {
 788                 if (0 == chunk_remaining_length(c->next)) {
 789                         chunk *empty = c->next;
 790                         c->next = empty->next;
 791                         if (empty == cq->last) cq->last = c;
 792                         chunk_release(empty);
 793                 }
 794         }
 795 }
 796
 797 int chunkqueue_open_file_chunk(server *srv, chunkqueue *cq) {
 798         chunk* const c = cq->first;
 799         off_t offset, toSend;
 800         struct stat st;
 801
 802         force_assert(NULL != c);
 803         force_assert(FILE_CHUNK == c->type);
 804         force_assert(c->offset >= 0 && c->offset <= c->file.length);
 805
 806         offset = c->file.start + c->offset;
 807         toSend = c->file.length - c->offset;
 808
 809         if (-1 == c->file.fd) {
 810                 if (-1 == (c->file.fd = fdevent_open_cloexec(c->mem->ptr, O_RDONLY, 0))) {
 811                         log_error_write(srv, __FILE__, __LINE__, "ssb", "open failed:", strerror(errno), c->mem);
 812                         return -1;
 813                 }
 814         }
 815
 816         /*(skip file size checks if file is temp file created by lighttpd)*/
 817         if (c->file.is_temp) return 0;
 818
 819         if (-1 == fstat(c->file.fd, &st)) {
 820                 log_error_write(srv, __FILE__, __LINE__, "ss", "fstat failed:", strerror(errno));
 821                 return -1;
 822         }
 823
 824         if (offset > st.st_size || toSend > st.st_size || offset > st.st_size - toSend) {
 825                 log_error_write(srv, __FILE__, __LINE__, "sb", "file shrunk:", c->mem);
 826                 return -1;
 827         }
 828
 829         return 0;
 830 }