src/chunk.c

   1 #include "first.h"
   2
   3 /**
   4  * the network chunk-API
   5  *
   6  *
   7  */
   8
   9 #include "chunk.h"
  10 #include "fdevent.h"
  11 #include "log.h"
  12
  13 #include <sys/types.h>
  14 #include <sys/stat.h>
  15 #include "sys-mmap.h"
  16
  17 #include <stdlib.h>
  18 #include <fcntl.h>
  19 #include <unistd.h>
  20
  21 #include <errno.h>
  22 #include <string.h>
  23
  24 /* default 1MB, upper limit 128MB */
  25 #define DEFAULT_TEMPFILE_SIZE (1 * 1024 * 1024)
  26 #define MAX_TEMPFILE_SIZE (128 * 1024 * 1024)
  27
  28 static size_t chunk_buf_sz = 4096;
  29 static chunk *chunks;
  30 static chunk *chunk_buffers;
  31 static array *chunkqueue_default_tempdirs = NULL;
  32 static off_t chunkqueue_default_tempfile_size = DEFAULT_TEMPFILE_SIZE;
  33
  34 void chunkqueue_set_chunk_size (size_t sz)
  35 {
  36     chunk_buf_sz = sz > 0 ? ((sz + 1023) & ~1023uL) : 4096;
  37 }
  38
  39 void chunkqueue_set_tempdirs_default_reset (void)
  40 {
  41     chunkqueue_default_tempdirs = NULL;
  42     chunkqueue_default_tempfile_size = DEFAULT_TEMPFILE_SIZE;
  43 }
  44
  45 chunkqueue *chunkqueue_init(void) {
  46         chunkqueue *cq;
  47
  48         cq = calloc(1, sizeof(*cq));
  49         force_assert(NULL != cq);
  50
  51         cq->first = NULL;
  52         cq->last = NULL;
  53
  54         cq->tempdirs              = chunkqueue_default_tempdirs;
  55         cq->upload_temp_file_size = chunkqueue_default_tempfile_size;
  56
  57         return cq;
  58 }
  59
  60 static chunk *chunk_init(size_t sz) {
  61         chunk *c;
  62
  63         c = calloc(1, sizeof(*c));
  64         force_assert(NULL != c);
  65
  66         c->type = MEM_CHUNK;
  67         c->mem = buffer_init();
  68         c->file.start = c->file.length = c->file.mmap.offset = 0;
  69         c->file.fd = -1;
  70         c->file.mmap.start = MAP_FAILED;
  71         c->file.mmap.length = 0;
  72         c->file.is_temp = 0;
  73         c->offset = 0;
  74         c->next = NULL;
  75
  76         buffer_string_prepare_copy(c->mem, sz-1);
  77
  78         return c;
  79 }
  80
  81 static void chunk_reset_file_chunk(chunk *c) {
  82         if (c->file.is_temp && !buffer_string_is_empty(c->mem)) {
  83                 unlink(c->mem->ptr);
  84         }
  85         if (c->file.fd != -1) {
  86                 close(c->file.fd);
  87                 c->file.fd = -1;
  88         }
  89         if (MAP_FAILED != c->file.mmap.start) {
  90                 munmap(c->file.mmap.start, c->file.mmap.length);
  91                 c->file.mmap.start = MAP_FAILED;
  92         }
  93         c->file.start = c->file.length = c->file.mmap.offset = 0;
  94         c->file.mmap.length = 0;
  95         c->file.is_temp = 0;
  96         c->type = MEM_CHUNK;
  97 }
  98
  99 static void chunk_reset(chunk *c) {
 100         if (c->type == FILE_CHUNK) chunk_reset_file_chunk(c);
 101
 102         buffer_clear(c->mem);
 103         c->offset = 0;
 104 }
 105
 106 static void chunk_free(chunk *c) {
 107         if (c->type == FILE_CHUNK) chunk_reset_file_chunk(c);
 108         buffer_free(c->mem);
 109         free(c);
 110 }
 111
 112 buffer * chunk_buffer_acquire(void) {
 113     chunk *c;
 114     buffer *b;
 115     if (chunks) {
 116         c = chunks;
 117         chunks = c->next;
 118     }
 119     else {
 120         c = chunk_init(chunk_buf_sz);
 121     }
 122     c->next = chunk_buffers;
 123     chunk_buffers = c;
 124     b = c->mem;
 125     c->mem = NULL;
 126     return b;
 127 }
 128
 129 void chunk_buffer_release(buffer *b) {
 130     if (NULL == b) return;
 131     if (b->size >= chunk_buf_sz && chunk_buffers) {
 132         chunk *c = chunk_buffers;
 133         chunk_buffers = c->next;
 134         c->mem = b;
 135         c->next = chunks;
 136         chunks = c;
 137         buffer_clear(b);
 138     }
 139     else {
 140         buffer_free(b);
 141     }
 142 }
 143
 144 static chunk * chunk_acquire(void) {
 145     if (chunks) {
 146         chunk *c = chunks;
 147         chunks = c->next;
 148         return c;
 149     }
 150     else {
 151         return chunk_init(chunk_buf_sz);
 152     }
 153 }
 154
 155 static void chunk_release(chunk *c) {
 156     if (c->mem->size >= chunk_buf_sz) {
 157         chunk_reset(c);
 158         c->next = chunks;
 159         chunks = c;
 160     }
 161     else {
 162         chunk_free(c);
 163     }
 164 }
 165
 166 void chunkqueue_chunk_pool_clear(void)
 167 {
 168     for (chunk *next, *c = chunks; c; c = next) {
 169         next = c->next;
 170         chunk_free(c);
 171     }
 172     chunks = NULL;
 173 }
 174
 175 void chunkqueue_chunk_pool_free(void)
 176 {
 177     chunkqueue_chunk_pool_clear();
 178     for (chunk *next, *c = chunk_buffers; c; c = next) {
 179         next = c->next;
 180         c->mem = buffer_init(); /*(chunk_reset() expects c->mem != NULL)*/
 181         chunk_free(c);
 182     }
 183     chunk_buffers = NULL;
 184 }
 185
 186 static off_t chunk_remaining_length(const chunk *c) {
 187         off_t len = 0;
 188         switch (c->type) {
 189         case MEM_CHUNK:
 190                 len = buffer_string_length(c->mem);
 191                 break;
 192         case FILE_CHUNK:
 193                 len = c->file.length;
 194                 break;
 195         default:
 196                 force_assert(c->type == MEM_CHUNK || c->type == FILE_CHUNK);
 197                 break;
 198         }
 199         force_assert(c->offset <= len);
 200         return len - c->offset;
 201 }
 202
 203 void chunkqueue_free(chunkqueue *cq) {
 204         chunk *c, *pc;
 205
 206         if (NULL == cq) return;
 207
 208         for (c = cq->first; c; ) {
 209                 pc = c;
 210                 c = c->next;
 211                 chunk_release(pc);
 212         }
 213
 214         free(cq);
 215 }
 216
 217 static void chunkqueue_prepend_chunk(chunkqueue *cq, chunk *c) {
 218         c->next = cq->first;
 219         cq->first = c;
 220
 221         if (NULL == cq->last) {
 222                 cq->last = c;
 223         }
 224 }
 225
 226 static void chunkqueue_append_chunk(chunkqueue *cq, chunk *c) {
 227         c->next = NULL;
 228         if (cq->last) {
 229                 cq->last->next = c;
 230         }
 231         cq->last = c;
 232
 233         if (NULL == cq->first) {
 234                 cq->first = c;
 235         }
 236 }
 237
 238 static chunk * chunkqueue_prepend_mem_chunk(chunkqueue *cq) {
 239     chunk *c = chunk_acquire();
 240     chunkqueue_prepend_chunk(cq, c);
 241     return c;
 242 }
 243
 244 static chunk * chunkqueue_append_mem_chunk(chunkqueue *cq) {
 245     chunk *c = chunk_acquire();
 246     chunkqueue_append_chunk(cq, c);
 247     return c;
 248 }
 249
 250 static chunk * chunkqueue_append_file_chunk(chunkqueue *cq, buffer *fn, off_t offset, off_t len) {
 251     chunk *c = chunk_acquire();
 252     chunkqueue_append_chunk(cq, c);
 253     c->type = FILE_CHUNK;
 254     c->file.start = offset;
 255     c->file.length = len;
 256     cq->bytes_in += len;
 257     buffer_copy_buffer(c->mem, fn);
 258     return c;
 259 }
 260
 261 void chunkqueue_reset(chunkqueue *cq) {
 262         chunk *cur = cq->first;
 263
 264         cq->first = cq->last = NULL;
 265
 266         while (NULL != cur) {
 267                 chunk *next = cur->next;
 268                 chunk_release(cur);
 269                 cur = next;
 270         }
 271
 272         cq->bytes_in = 0;
 273         cq->bytes_out = 0;
 274         cq->tempdir_idx = 0;
 275 }
 276
 277 void chunkqueue_append_file_fd(chunkqueue *cq, buffer *fn, int fd, off_t offset, off_t len) {
 278     if (len > 0) {
 279         (chunkqueue_append_file_chunk(cq, fn, offset, len))->file.fd = fd;
 280     }
 281     else {
 282         close(fd);
 283     }
 284 }
 285
 286 void chunkqueue_append_file(chunkqueue *cq, buffer *fn, off_t offset, off_t len) {
 287     if (len > 0) {
 288         chunkqueue_append_file_chunk(cq, fn, offset, len);
 289     }
 290 }
 291
 292
 293 static int chunkqueue_append_mem_extend_chunk(chunkqueue *cq, const char *mem, size_t len) {
 294         chunk *c = cq->last;
 295         if (0 == len) return 1;
 296         if (c != NULL && c->type == MEM_CHUNK
 297             && buffer_string_space(c->mem) >= len) {
 298                 buffer_append_string_len(c->mem, mem, len);
 299                 cq->bytes_in += len;
 300                 return 1;
 301         }
 302         return 0;
 303 }
 304
 305
 306 void chunkqueue_append_buffer(chunkqueue *cq, buffer *mem) {
 307         chunk *c;
 308         size_t len = buffer_string_length(mem);
 309         if (len < 256 && chunkqueue_append_mem_extend_chunk(cq, mem->ptr, len)) return;
 310
 311         c = chunkqueue_append_mem_chunk(cq);
 312         cq->bytes_in += len;
 313         buffer_move(c->mem, mem);
 314 }
 315
 316
 317 void chunkqueue_append_mem(chunkqueue *cq, const char * mem, size_t len) {
 318         chunk *c;
 319         if (len < chunk_buf_sz && chunkqueue_append_mem_extend_chunk(cq, mem, len))
 320                 return;
 321
 322         c = chunkqueue_append_mem_chunk(cq);
 323         cq->bytes_in += len;
 324         buffer_copy_string_len(c->mem, mem, len);
 325 }
 326
 327
 328 void chunkqueue_append_mem_min(chunkqueue *cq, const char * mem, size_t len) {
 329         chunk *c;
 330         if (len < chunk_buf_sz && chunkqueue_append_mem_extend_chunk(cq, mem, len))
 331                 return;
 332
 333         c = chunk_init(len+1);
 334         chunkqueue_append_chunk(cq, c);
 335         cq->bytes_in += len;
 336         buffer_copy_string_len(c->mem, mem, len);
 337 }
 338
 339
 340 void chunkqueue_append_chunkqueue(chunkqueue *cq, chunkqueue *src) {
 341         if (src == NULL || NULL == src->first) return;
 342
 343         if (NULL == cq->first) {
 344                 cq->first = src->first;
 345         } else {
 346                 cq->last->next = src->first;
 347         }
 348         cq->last = src->last;
 349         cq->bytes_in += (src->bytes_in - src->bytes_out);
 350
 351         src->first = NULL;
 352         src->last = NULL;
 353         src->bytes_out = src->bytes_in;
 354 }
 355
 356
 357 __attribute_cold__
 358 static void chunkqueue_buffer_open_resize(chunk *c, size_t sz) {
 359         chunk * const n = chunk_init((sz + 4095) & ~4095uL);
 360         buffer * const b = c->mem;
 361         c->mem = n->mem;
 362         n->mem = b;
 363         chunk_release(n);
 364 }
 365
 366
 367 buffer * chunkqueue_prepend_buffer_open_sz(chunkqueue *cq, size_t sz) {
 368         chunk * const c = chunkqueue_prepend_mem_chunk(cq);
 369         if (buffer_string_space(c->mem) < sz) {
 370                 chunkqueue_buffer_open_resize(c, sz);
 371         }
 372         return c->mem;
 373 }
 374
 375
 376 buffer * chunkqueue_prepend_buffer_open(chunkqueue *cq) {
 377         chunk *c = chunkqueue_prepend_mem_chunk(cq);
 378         return c->mem;
 379 }
 380
 381
 382 void chunkqueue_prepend_buffer_commit(chunkqueue *cq) {
 383         cq->bytes_in += buffer_string_length(cq->first->mem);
 384 }
 385
 386
 387 buffer * chunkqueue_append_buffer_open_sz(chunkqueue *cq, size_t sz) {
 388         chunk * const c = chunkqueue_append_mem_chunk(cq);
 389         if (buffer_string_space(c->mem) < sz) {
 390                 chunkqueue_buffer_open_resize(c, sz);
 391         }
 392         return c->mem;
 393 }
 394
 395
 396 buffer * chunkqueue_append_buffer_open(chunkqueue *cq) {
 397         chunk *c = chunkqueue_append_mem_chunk(cq);
 398         return c->mem;
 399 }
 400
 401
 402 void chunkqueue_append_buffer_commit(chunkqueue *cq) {
 403         cq->bytes_in += buffer_string_length(cq->last->mem);
 404 }
 405
 406
 407 static void chunkqueue_remove_empty_chunks(chunkqueue *cq);
 408
 409
 410 char * chunkqueue_get_memory(chunkqueue *cq, size_t *len) {
 411         size_t sz = *len ? *len : (chunk_buf_sz >> 1);
 412         buffer *b;
 413         chunk *c = cq->last;
 414         if (NULL != c && MEM_CHUNK == c->type) {
 415                 /* return pointer into existing buffer if large enough */
 416                 size_t avail = buffer_string_space(c->mem);
 417                 if (avail >= sz) {
 418                         *len = avail;
 419                         b = c->mem;
 420                         return b->ptr + buffer_string_length(b);
 421                 }
 422         }
 423
 424         /* allocate new chunk */
 425         b = chunkqueue_append_buffer_open_sz(cq, sz);
 426         *len = buffer_string_space(b);
 427         return b->ptr;
 428 }
 429
 430 void chunkqueue_use_memory(chunkqueue *cq, size_t len) {
 431         buffer *b;
 432
 433         force_assert(NULL != cq);
 434         force_assert(NULL != cq->last && MEM_CHUNK == cq->last->type);
 435         b = cq->last->mem;
 436
 437         if (len > 0) {
 438                 buffer_commit(b, len);
 439                 cq->bytes_in += len;
 440         } else if (buffer_string_is_empty(b)) {
 441                 /* scan chunkqueue to remove empty last chunk
 442                  * (generally not expecting a deep queue) */
 443                 chunkqueue_remove_empty_chunks(cq);
 444         }
 445 }
 446
 447 void chunkqueue_set_tempdirs_default (array *tempdirs, off_t upload_temp_file_size) {
 448         chunkqueue_default_tempdirs = tempdirs;
 449         chunkqueue_default_tempfile_size
 450                 = (0 == upload_temp_file_size)                ? DEFAULT_TEMPFILE_SIZE
 451                 : (upload_temp_file_size > MAX_TEMPFILE_SIZE) ? MAX_TEMPFILE_SIZE
 452                                                               : upload_temp_file_size;
 453 }
 454
 455 void chunkqueue_set_tempdirs(chunkqueue *cq, array *tempdirs, off_t upload_temp_file_size) {
 456         force_assert(NULL != cq);
 457         cq->tempdirs = tempdirs;
 458         cq->upload_temp_file_size
 459                 = (0 == upload_temp_file_size)                ? DEFAULT_TEMPFILE_SIZE
 460                 : (upload_temp_file_size > MAX_TEMPFILE_SIZE) ? MAX_TEMPFILE_SIZE
 461                                                               : upload_temp_file_size;
 462         cq->tempdir_idx = 0;
 463 }
 464
 465 void chunkqueue_steal(chunkqueue *dest, chunkqueue *src, off_t len) {
 466         while (len > 0) {
 467                 chunk *c = src->first;
 468                 off_t clen = 0, use;
 469
 470                 if (NULL == c) break;
 471
 472                 clen = chunk_remaining_length(c);
 473                 if (0 == clen) {
 474                         /* drop empty chunk */
 475                         src->first = c->next;
 476                         if (c == src->last) src->last = NULL;
 477                         chunk_release(c);
 478                         continue;
 479                 }
 480
 481                 use = len >= clen ? clen : len;
 482                 len -= use;
 483
 484                 if (use == clen) {
 485                         /* move complete chunk */
 486                         src->first = c->next;
 487                         if (c == src->last) src->last = NULL;
 488
 489                         chunkqueue_append_chunk(dest, c);
 490                         dest->bytes_in += use;
 491                 } else {
 492                         /* partial chunk with length "use" */
 493
 494                         switch (c->type) {
 495                         case MEM_CHUNK:
 496                                 chunkqueue_append_mem(dest, c->mem->ptr + c->offset, use);
 497                                 break;
 498                         case FILE_CHUNK:
 499                                 /* tempfile flag is in "last" chunk after the split */
 500                                 chunkqueue_append_file(dest, c->mem, c->file.start + c->offset, use);
 501                                 break;
 502                         }
 503
 504                         c->offset += use;
 505                         force_assert(0 == len);
 506                 }
 507
 508                 src->bytes_out += use;
 509         }
 510 }
 511
 512 static chunk *chunkqueue_get_append_tempfile(server *srv, chunkqueue *cq) {
 513         chunk *c;
 514         buffer *template = buffer_init_string("/var/tmp/lighttpd-upload-XXXXXX");
 515         int fd = -1;
 516
 517         if (cq->tempdirs && cq->tempdirs->used) {
 518                 /* we have several tempdirs, only if all of them fail we jump out */
 519
 520                 for (errno = EIO; cq->tempdir_idx < cq->tempdirs->used; ++cq->tempdir_idx) {
 521                         data_string *ds = (data_string *)cq->tempdirs->data[cq->tempdir_idx];
 522
 523                         buffer_copy_buffer(template, ds->value);
 524                         buffer_append_path_len(template, CONST_STR_LEN("lighttpd-upload-XXXXXX"));
 525                         if (-1 != (fd = fdevent_mkstemp_append(template->ptr))) break;
 526                 }
 527         } else {
 528                 fd = fdevent_mkstemp_append(template->ptr);
 529         }
 530
 531         if (fd < 0) {
 532                 /* (report only the last error to mkstemp()
 533                  *  if multiple temp dirs attempted) */
 534                 log_error_write(srv, __FILE__, __LINE__, "sbs",
 535                                 "opening temp-file failed:",
 536                                 template, strerror(errno));
 537                 buffer_free(template);
 538                 return NULL;
 539         }
 540
 541         c = chunkqueue_append_file_chunk(cq, template, 0, 0);
 542         c->file.fd = fd;
 543         c->file.is_temp = 1;
 544
 545         buffer_free(template);
 546
 547         return c;
 548 }
 549
 550 int chunkqueue_append_mem_to_tempfile(server *srv, chunkqueue *dest, const char *mem, size_t len) {
 551         chunk *dst_c;
 552         ssize_t written;
 553
 554         do {
 555                 /*
 556                  * if the last chunk is
 557                  * - smaller than dest->upload_temp_file_size
 558                  * - not read yet (offset == 0)
 559                  * -> append to it (so it might actually become larger than dest->upload_temp_file_size)
 560                  * otherwise
 561                  * -> create a new chunk
 562                  *
 563                  * */
 564
 565                 dst_c = dest->last;
 566                 if (NULL != dst_c
 567                         && FILE_CHUNK == dst_c->type
 568                         && dst_c->file.is_temp
 569                         && dst_c->file.fd >= 0
 570                         && 0 == dst_c->offset) {
 571                         /* ok, take the last chunk for our job */
 572
 573                         if (dst_c->file.length >= (off_t)dest->upload_temp_file_size) {
 574                                 /* the chunk is too large now, close it */
 575                                 int rc = close(dst_c->file.fd);
 576                                 dst_c->file.fd = -1;
 577                                 if (0 != rc) {
 578                                         log_error_write(srv, __FILE__, __LINE__, "sbss",
 579                                                 "close() temp-file", dst_c->mem, "failed:",
 580                                                 strerror(errno));
 581                                         return -1;
 582                                 }
 583                                 dst_c = NULL;
 584                         }
 585                 } else {
 586                         dst_c = NULL;
 587                 }
 588
 589                 if (NULL == dst_c && NULL == (dst_c = chunkqueue_get_append_tempfile(srv, dest))) {
 590                         return -1;
 591                 }
 592               #ifdef __COVERITY__
 593                 if (dst_c->file.fd < 0) return -1;
 594               #endif
 595
 596                 /* (dst_c->file.fd >= 0) */
 597                 /* coverity[negative_returns : FALSE] */
 598                 written = write(dst_c->file.fd, mem, len);
 599
 600                 if ((size_t) written == len) {
 601                         dst_c->file.length += len;
 602                         dest->bytes_in += len;
 603
 604                         return 0;
 605                 } else if (written >= 0) {
 606                         /*(assume EINTR if partial write and retry write();
 607                          * retry write() might fail with ENOSPC if no more space on volume)*/
 608                         dest->bytes_in += written;
 609                         mem += written;
 610                         len -= (size_t)written;
 611                         dst_c->file.length += (size_t)written;
 612                         /* continue; retry */
 613                 } else if (errno == EINTR) {
 614                         /* continue; retry */
 615                 } else {
 616                         int retry = (errno == ENOSPC && dest->tempdirs && ++dest->tempdir_idx < dest->tempdirs->used);
 617                         if (!retry) {
 618                                 log_error_write(srv, __FILE__, __LINE__, "sbs",
 619                                                 "write() temp-file", dst_c->mem, "failed:",
 620                                                 strerror(errno));
 621                         }
 622
 623                         if (0 == chunk_remaining_length(dst_c)) {
 624                                 /*(remove empty chunk and unlink tempfile)*/
 625                                 chunkqueue_remove_empty_chunks(dest);
 626                         } else {/*(close tempfile; avoid later attempts to append)*/
 627                                 int rc = close(dst_c->file.fd);
 628                                 dst_c->file.fd = -1;
 629                                 if (0 != rc) {
 630                                         log_error_write(srv, __FILE__, __LINE__, "sbss",
 631                                                 "close() temp-file", dst_c->mem, "failed:",
 632                                                 strerror(errno));
 633                                         return -1;
 634                                 }
 635                         }
 636                         if (!retry) break; /* return -1; */
 637
 638                         /* continue; retry */
 639                 }
 640
 641         } while (dst_c);
 642
 643         return -1;
 644 }
 645
 646 int chunkqueue_steal_with_tempfiles(server *srv, chunkqueue *dest, chunkqueue *src, off_t len) {
 647         while (len > 0) {
 648                 chunk *c = src->first;
 649                 off_t clen = 0, use;
 650
 651                 if (NULL == c) break;
 652
 653                 clen = chunk_remaining_length(c);
 654                 if (0 == clen) {
 655                         /* drop empty chunk */
 656                         src->first = c->next;
 657                         if (c == src->last) src->last = NULL;
 658                         chunk_release(c);
 659                         continue;
 660                 }
 661
 662                 use = (len >= clen) ? clen : len;
 663                 len -= use;
 664
 665                 switch (c->type) {
 666                 case FILE_CHUNK:
 667                         if (use == clen) {
 668                                 /* move complete chunk */
 669                                 src->first = c->next;
 670                                 if (c == src->last) src->last = NULL;
 671                                 chunkqueue_append_chunk(dest, c);
 672                                 dest->bytes_in += use;
 673                         } else {
 674                                 /* partial chunk with length "use" */
 675                                 /* tempfile flag is in "last" chunk after the split */
 676                                 chunkqueue_append_file(dest, c->mem, c->file.start + c->offset, use);
 677
 678                                 c->offset += use;
 679                                 force_assert(0 == len);
 680                         }
 681                         break;
 682
 683                 case MEM_CHUNK:
 684                         /* store "use" bytes from memory chunk in tempfile */
 685                         if (0 != chunkqueue_append_mem_to_tempfile(srv, dest, c->mem->ptr + c->offset, use)) {
 686                                 return -1;
 687                         }
 688
 689                         if (use == clen) {
 690                                 /* finished chunk */
 691                                 src->first = c->next;
 692                                 if (c == src->last) src->last = NULL;
 693                                 chunk_release(c);
 694                         } else {
 695                                 /* partial chunk */
 696                                 c->offset += use;
 697                                 force_assert(0 == len);
 698                         }
 699                         break;
 700                 }
 701
 702                 src->bytes_out += use;
 703         }
 704
 705         return 0;
 706 }
 707
 708 off_t chunkqueue_length(chunkqueue *cq) {
 709         off_t len = 0;
 710         chunk *c;
 711
 712         for (c = cq->first; c; c = c->next) {
 713                 len += chunk_remaining_length(c);
 714         }
 715
 716         return len;
 717 }
 718
 719 void chunkqueue_mark_written(chunkqueue *cq, off_t len) {
 720         off_t written = len;
 721         chunk *c;
 722         force_assert(len >= 0);
 723
 724         for (c = cq->first; NULL != c; c = cq->first) {
 725                 off_t c_len = chunk_remaining_length(c);
 726
 727                 if (0 == written && 0 != c_len) break; /* no more finished chunks */
 728
 729                 if (written >= c_len) { /* chunk got finished */
 730                         c->offset += c_len;
 731                         written -= c_len;
 732
 733                         cq->first = c->next;
 734                         if (c == cq->last) cq->last = NULL;
 735                         chunk_release(c);
 736                 } else { /* partial chunk */
 737                         c->offset += written;
 738                         written = 0;
 739                         break; /* chunk not finished */
 740                 }
 741         }
 742
 743         force_assert(0 == written);
 744         cq->bytes_out += len;
 745 }
 746
 747 void chunkqueue_remove_finished_chunks(chunkqueue *cq) {
 748         chunk *c;
 749
 750         for (c = cq->first; c; c = cq->first) {
 751                 if (0 != chunk_remaining_length(c)) break; /* not finished yet */
 752
 753                 cq->first = c->next;
 754                 if (c == cq->last) cq->last = NULL;
 755                 chunk_release(c);
 756         }
 757 }
 758
 759 static void chunkqueue_remove_empty_chunks(chunkqueue *cq) {
 760         chunk *c;
 761         chunkqueue_remove_finished_chunks(cq);
 762         if (chunkqueue_is_empty(cq)) return;
 763
 764         for (c = cq->first; c && c->next; c = c->next) {
 765                 if (0 == chunk_remaining_length(c->next)) {
 766                         chunk *empty = c->next;
 767                         c->next = empty->next;
 768                         if (empty == cq->last) cq->last = c;
 769                         chunk_release(empty);
 770                 }
 771         }
 772 }
 773
 774 int chunkqueue_open_file_chunk(server *srv, chunkqueue *cq) {
 775         chunk* const c = cq->first;
 776         off_t offset, toSend;
 777         struct stat st;
 778
 779         force_assert(NULL != c);
 780         force_assert(FILE_CHUNK == c->type);
 781         force_assert(c->offset >= 0 && c->offset <= c->file.length);
 782
 783         offset = c->file.start + c->offset;
 784         toSend = c->file.length - c->offset;
 785
 786         if (-1 == c->file.fd) {
 787                 /* (permit symlinks; should already have been checked.  However, TOC-TOU remains) */
 788                 if (-1 == (c->file.fd = fdevent_open_cloexec(c->mem->ptr, 1, O_RDONLY, 0))) {
 789                         log_error_write(srv, __FILE__, __LINE__, "ssb", "open failed:", strerror(errno), c->mem);
 790                         return -1;
 791                 }
 792         }
 793
 794         /*(skip file size checks if file is temp file created by lighttpd)*/
 795         if (c->file.is_temp) return 0;
 796
 797         if (-1 == fstat(c->file.fd, &st)) {
 798                 log_error_write(srv, __FILE__, __LINE__, "ss", "fstat failed:", strerror(errno));
 799                 return -1;
 800         }
 801
 802         if (offset > st.st_size || toSend > st.st_size || offset > st.st_size - toSend) {
 803                 log_error_write(srv, __FILE__, __LINE__, "sb", "file shrunk:", c->mem);
 804                 return -1;
 805         }
 806
 807         return 0;
 808 }