src/chunk.c

   1 #include "first.h"
   2
   3 /**
   4  * the network chunk-API
   5  *
   6  *
   7  */
   8
   9 #include "chunk.h"
  10 #include "base.h"
  11 #include "fdevent.h"
  12 #include "log.h"
  13
  14 #include <sys/types.h>
  15 #include <sys/stat.h>
  16 #include "sys-mmap.h"
  17
  18 #include <stdlib.h>
  19 #include <fcntl.h>
  20 #include <unistd.h>
  21
  22 #include <errno.h>
  23 #include <string.h>
  24
  25 /* default 1MB, upper limit 128MB */
  26 #define DEFAULT_TEMPFILE_SIZE (1 * 1024 * 1024)
  27 #define MAX_TEMPFILE_SIZE (128 * 1024 * 1024)
  28
  29 static array *chunkqueue_default_tempdirs = NULL;
  30 static unsigned int chunkqueue_default_tempfile_size = DEFAULT_TEMPFILE_SIZE;
  31
  32 void chunkqueue_set_tempdirs_default_reset (void)
  33 {
  34     chunkqueue_default_tempdirs = NULL;
  35     chunkqueue_default_tempfile_size = DEFAULT_TEMPFILE_SIZE;
  36 }
  37
  38 chunkqueue *chunkqueue_init(void) {
  39         chunkqueue *cq;
  40
  41         cq = calloc(1, sizeof(*cq));
  42         force_assert(NULL != cq);
  43
  44         cq->first = NULL;
  45         cq->last = NULL;
  46
  47         cq->unused = NULL;
  48
  49         cq->tempdirs              = chunkqueue_default_tempdirs;
  50         cq->upload_temp_file_size = chunkqueue_default_tempfile_size;
  51
  52         return cq;
  53 }
  54
  55 static chunk *chunk_init(void) {
  56         chunk *c;
  57
  58         c = calloc(1, sizeof(*c));
  59         force_assert(NULL != c);
  60
  61         c->type = MEM_CHUNK;
  62         c->mem = buffer_init();
  63         c->file.name = buffer_init();
  64         c->file.start = c->file.length = c->file.mmap.offset = 0;
  65         c->file.fd = -1;
  66         c->file.mmap.start = MAP_FAILED;
  67         c->file.mmap.length = 0;
  68         c->file.is_temp = 0;
  69         c->offset = 0;
  70         c->next = NULL;
  71
  72         return c;
  73 }
  74
  75 static void chunk_reset(chunk *c) {
  76         if (NULL == c) return;
  77
  78         c->type = MEM_CHUNK;
  79
  80         buffer_reset(c->mem);
  81
  82         if (c->file.is_temp && !buffer_string_is_empty(c->file.name)) {
  83                 unlink(c->file.name->ptr);
  84         }
  85
  86         buffer_reset(c->file.name);
  87
  88         if (c->file.fd != -1) {
  89                 close(c->file.fd);
  90                 c->file.fd = -1;
  91         }
  92         if (MAP_FAILED != c->file.mmap.start) {
  93                 munmap(c->file.mmap.start, c->file.mmap.length);
  94                 c->file.mmap.start = MAP_FAILED;
  95         }
  96         c->file.start = c->file.length = c->file.mmap.offset = 0;
  97         c->file.mmap.length = 0;
  98         c->file.is_temp = 0;
  99         c->offset = 0;
 100         c->next = NULL;
 101 }
 102
 103 static void chunk_free(chunk *c) {
 104         if (NULL == c) return;
 105
 106         chunk_reset(c);
 107
 108         buffer_free(c->mem);
 109         buffer_free(c->file.name);
 110
 111         free(c);
 112 }
 113
 114 static off_t chunk_remaining_length(const chunk *c) {
 115         off_t len = 0;
 116         switch (c->type) {
 117         case MEM_CHUNK:
 118                 len = buffer_string_length(c->mem);
 119                 break;
 120         case FILE_CHUNK:
 121                 len = c->file.length;
 122                 break;
 123         default:
 124                 force_assert(c->type == MEM_CHUNK || c->type == FILE_CHUNK);
 125                 break;
 126         }
 127         force_assert(c->offset <= len);
 128         return len - c->offset;
 129 }
 130
 131 void chunkqueue_free(chunkqueue *cq) {
 132         chunk *c, *pc;
 133
 134         if (NULL == cq) return;
 135
 136         for (c = cq->first; c; ) {
 137                 pc = c;
 138                 c = c->next;
 139                 chunk_free(pc);
 140         }
 141
 142         for (c = cq->unused; c; ) {
 143                 pc = c;
 144                 c = c->next;
 145                 chunk_free(pc);
 146         }
 147
 148         free(cq);
 149 }
 150
 151 static void chunkqueue_push_unused_chunk(chunkqueue *cq, chunk *c) {
 152         force_assert(NULL != cq && NULL != c);
 153
 154         /* keep at max 4 chunks in the 'unused'-cache */
 155         if (cq->unused_chunks > 4) {
 156                 chunk_free(c);
 157         } else {
 158                 chunk_reset(c);
 159                 c->next = cq->unused;
 160                 cq->unused = c;
 161                 cq->unused_chunks++;
 162         }
 163 }
 164
 165 static chunk *chunkqueue_get_unused_chunk(chunkqueue *cq) {
 166         chunk *c;
 167
 168         force_assert(NULL != cq);
 169
 170         /* check if we have a unused chunk */
 171         if (0 == cq->unused) {
 172                 c = chunk_init();
 173         } else {
 174                 /* take the first element from the list (a stack) */
 175                 c = cq->unused;
 176                 cq->unused = c->next;
 177                 c->next = NULL;
 178                 cq->unused_chunks--;
 179         }
 180
 181         return c;
 182 }
 183
 184 static void chunkqueue_prepend_chunk(chunkqueue *cq, chunk *c) {
 185         c->next = cq->first;
 186         cq->first = c;
 187
 188         if (NULL == cq->last) {
 189                 cq->last = c;
 190         }
 191         cq->bytes_in += chunk_remaining_length(c);
 192 }
 193
 194 static void chunkqueue_append_chunk(chunkqueue *cq, chunk *c) {
 195         c->next = NULL;
 196         if (cq->last) {
 197                 cq->last->next = c;
 198         }
 199         cq->last = c;
 200
 201         if (NULL == cq->first) {
 202                 cq->first = c;
 203         }
 204         cq->bytes_in += chunk_remaining_length(c);
 205 }
 206
 207 void chunkqueue_reset(chunkqueue *cq) {
 208         chunk *cur = cq->first;
 209
 210         cq->first = cq->last = NULL;
 211
 212         while (NULL != cur) {
 213                 chunk *next = cur->next;
 214                 chunkqueue_push_unused_chunk(cq, cur);
 215                 cur = next;
 216         }
 217
 218         cq->bytes_in = 0;
 219         cq->bytes_out = 0;
 220         cq->tempdir_idx = 0;
 221 }
 222
 223 void chunkqueue_append_file_fd(chunkqueue *cq, buffer *fn, int fd, off_t offset, off_t len) {
 224         chunk *c;
 225
 226         if (0 == len) {
 227                 close(fd);
 228                 return;
 229         }
 230
 231         c = chunkqueue_get_unused_chunk(cq);
 232
 233         c->type = FILE_CHUNK;
 234
 235         buffer_copy_buffer(c->file.name, fn);
 236         c->file.start = offset;
 237         c->file.length = len;
 238         c->file.fd = fd;
 239         c->offset = 0;
 240
 241         chunkqueue_append_chunk(cq, c);
 242 }
 243
 244 void chunkqueue_append_file(chunkqueue *cq, buffer *fn, off_t offset, off_t len) {
 245         chunk *c;
 246
 247         if (0 == len) return;
 248
 249         c = chunkqueue_get_unused_chunk(cq);
 250
 251         c->type = FILE_CHUNK;
 252
 253         buffer_copy_buffer(c->file.name, fn);
 254         c->file.start = offset;
 255         c->file.length = len;
 256         c->offset = 0;
 257
 258         chunkqueue_append_chunk(cq, c);
 259 }
 260
 261 void chunkqueue_append_buffer(chunkqueue *cq, buffer *mem) {
 262         chunk *c;
 263
 264         if (buffer_string_is_empty(mem)) return;
 265
 266         c = chunkqueue_get_unused_chunk(cq);
 267         c->type = MEM_CHUNK;
 268         force_assert(NULL != c->mem);
 269         buffer_move(c->mem, mem);
 270
 271         chunkqueue_append_chunk(cq, c);
 272 }
 273
 274 void chunkqueue_prepend_buffer(chunkqueue *cq, buffer *mem) {
 275         chunk *c;
 276
 277         if (buffer_string_is_empty(mem)) return;
 278
 279         c = chunkqueue_get_unused_chunk(cq);
 280         c->type = MEM_CHUNK;
 281         force_assert(NULL != c->mem);
 282         buffer_move(c->mem, mem);
 283
 284         chunkqueue_prepend_chunk(cq, c);
 285 }
 286
 287
 288 void chunkqueue_append_mem(chunkqueue *cq, const char * mem, size_t len) {
 289         chunk *c;
 290
 291         if (0 == len) return;
 292
 293         c = chunkqueue_get_unused_chunk(cq);
 294         c->type = MEM_CHUNK;
 295         buffer_copy_string_len(c->mem, mem, len);
 296
 297         chunkqueue_append_chunk(cq, c);
 298 }
 299
 300
 301 void chunkqueue_append_chunkqueue(chunkqueue *cq, chunkqueue *src) {
 302         if (src == NULL || NULL == src->first) return;
 303
 304         if (NULL == cq->first) {
 305                 cq->first = src->first;
 306         } else {
 307                 cq->last->next = src->first;
 308         }
 309         cq->last = src->last;
 310         cq->bytes_in += (src->bytes_in - src->bytes_out);
 311
 312         src->first = NULL;
 313         src->last = NULL;
 314         src->bytes_out = src->bytes_in;
 315 }
 316
 317
 318 void chunkqueue_get_memory(chunkqueue *cq, char **mem, size_t *len, size_t min_size, size_t alloc_size) {
 319         static const size_t REALLOC_MAX_SIZE = 256;
 320         chunk *c;
 321         buffer *b;
 322         char *dummy_mem;
 323         size_t dummy_len;
 324
 325         force_assert(NULL != cq);
 326         if (NULL == mem) mem = &dummy_mem;
 327         if (NULL == len) len = &dummy_len;
 328
 329         /* default values: */
 330         if (0 == min_size) min_size = 1024;
 331         if (0 == alloc_size) alloc_size = 4096;
 332         if (alloc_size < min_size) alloc_size = min_size;
 333
 334         if (NULL != cq->last && MEM_CHUNK == cq->last->type) {
 335                 size_t have;
 336
 337                 b = cq->last->mem;
 338                 have = buffer_string_space(b);
 339
 340                 /* unused buffer: allocate space */
 341                 if (buffer_string_is_empty(b)) {
 342                         buffer_string_prepare_copy(b, alloc_size);
 343                         have = buffer_string_space(b);
 344                 }
 345                 /* if buffer is really small just make it bigger */
 346                 else if (have < min_size && b->size <= REALLOC_MAX_SIZE) {
 347                         size_t cur_len = buffer_string_length(b);
 348                         size_t new_size = cur_len + min_size, append;
 349                         if (new_size < alloc_size) new_size = alloc_size;
 350
 351                         append = new_size - cur_len;
 352                         if (append >= min_size) {
 353                                 buffer_string_prepare_append(b, append);
 354                                 have = buffer_string_space(b);
 355                         }
 356                 }
 357
 358                 /* return pointer into existing buffer if large enough */
 359                 if (have >= min_size) {
 360                         *mem = b->ptr + buffer_string_length(b);
 361                         *len = have;
 362                         return;
 363                 }
 364         }
 365
 366         /* allocate new chunk */
 367         c = chunkqueue_get_unused_chunk(cq);
 368         c->type = MEM_CHUNK;
 369         chunkqueue_append_chunk(cq, c);
 370
 371         b = c->mem;
 372         buffer_string_prepare_append(b, alloc_size);
 373
 374         *mem = b->ptr + buffer_string_length(b);
 375         *len = buffer_string_space(b);
 376 }
 377
 378 void chunkqueue_use_memory(chunkqueue *cq, size_t len) {
 379         buffer *b;
 380
 381         force_assert(NULL != cq);
 382         force_assert(NULL != cq->last && MEM_CHUNK == cq->last->type);
 383         b = cq->last->mem;
 384
 385         if (len > 0) {
 386                 buffer_commit(b, len);
 387                 cq->bytes_in += len;
 388         } else if (buffer_string_is_empty(b)) {
 389                 /* unused buffer: can't remove chunk easily from
 390                  * end of list, so just reset the buffer
 391                  */
 392                 buffer_reset(b);
 393         }
 394 }
 395
 396 void chunkqueue_set_tempdirs_default (array *tempdirs, unsigned int upload_temp_file_size) {
 397         chunkqueue_default_tempdirs = tempdirs;
 398         chunkqueue_default_tempfile_size
 399                 = (0 == upload_temp_file_size)                ? DEFAULT_TEMPFILE_SIZE
 400                 : (upload_temp_file_size > MAX_TEMPFILE_SIZE) ? MAX_TEMPFILE_SIZE
 401                                                               : upload_temp_file_size;
 402 }
 403
 404 #if 0
 405 void chunkqueue_set_tempdirs(chunkqueue *cq, array *tempdirs, unsigned int upload_temp_file_size) {
 406         force_assert(NULL != cq);
 407         cq->tempdirs = tempdirs;
 408         cq->upload_temp_file_size
 409                 = (0 == upload_temp_file_size)                ? DEFAULT_TEMPFILE_SIZE
 410                 : (upload_temp_file_size > MAX_TEMPFILE_SIZE) ? MAX_TEMPFILE_SIZE
 411                                                               : upload_temp_file_size;
 412         cq->tempdir_idx = 0;
 413 }
 414 #endif
 415
 416 void chunkqueue_steal(chunkqueue *dest, chunkqueue *src, off_t len) {
 417         while (len > 0) {
 418                 chunk *c = src->first;
 419                 off_t clen = 0, use;
 420
 421                 if (NULL == c) break;
 422
 423                 clen = chunk_remaining_length(c);
 424                 if (0 == clen) {
 425                         /* drop empty chunk */
 426                         src->first = c->next;
 427                         if (c == src->last) src->last = NULL;
 428                         chunkqueue_push_unused_chunk(src, c);
 429                         continue;
 430                 }
 431
 432                 use = len >= clen ? clen : len;
 433                 len -= use;
 434
 435                 if (use == clen) {
 436                         /* move complete chunk */
 437                         src->first = c->next;
 438                         if (c == src->last) src->last = NULL;
 439
 440                         chunkqueue_append_chunk(dest, c);
 441                 } else {
 442                         /* partial chunk with length "use" */
 443
 444                         switch (c->type) {
 445                         case MEM_CHUNK:
 446                                 chunkqueue_append_mem(dest, c->mem->ptr + c->offset, use);
 447                                 break;
 448                         case FILE_CHUNK:
 449                                 /* tempfile flag is in "last" chunk after the split */
 450                                 chunkqueue_append_file(dest, c->file.name, c->file.start + c->offset, use);
 451                                 break;
 452                         }
 453
 454                         c->offset += use;
 455                         force_assert(0 == len);
 456                 }
 457
 458                 src->bytes_out += use;
 459         }
 460 }
 461
 462 static chunk *chunkqueue_get_append_tempfile(server *srv, chunkqueue *cq) {
 463         chunk *c;
 464         buffer *template = buffer_init_string("/var/tmp/lighttpd-upload-XXXXXX");
 465         int fd = -1;
 466
 467         if (cq->tempdirs && cq->tempdirs->used) {
 468                 /* we have several tempdirs, only if all of them fail we jump out */
 469
 470                 for (errno = EIO; cq->tempdir_idx < cq->tempdirs->used; ++cq->tempdir_idx) {
 471                         data_string *ds = (data_string *)cq->tempdirs->data[cq->tempdir_idx];
 472
 473                         buffer_copy_buffer(template, ds->value);
 474                         buffer_append_slash(template);
 475                         buffer_append_string_len(template, CONST_STR_LEN("lighttpd-upload-XXXXXX"));
 476
 477                       #ifdef __COVERITY__
 478                         /* POSIX-2008 requires mkstemp create file with 0600 perms */
 479                         umask(0600);
 480                       #endif
 481                         /* coverity[secure_temp : FALSE] */
 482                         if (-1 != (fd = mkstemp(template->ptr))) break;
 483                 }
 484         } else {
 485               #ifdef __COVERITY__
 486                 /* POSIX-2008 requires mkstemp create file with 0600 perms */
 487                 umask(0600);
 488               #endif
 489                 /* coverity[secure_temp : FALSE] */
 490                 fd = mkstemp(template->ptr);
 491         }
 492
 493         if (fd < 0) {
 494                 /* (report only the last error to mkstemp()
 495                  *  if multiple temp dirs attempted) */
 496                 log_error_write(srv, __FILE__, __LINE__, "sbs",
 497                                 "opening temp-file failed:",
 498                                 template, strerror(errno));
 499                 buffer_free(template);
 500                 return NULL;
 501         }
 502
 503         if (0 != fcntl(fd, F_SETFL, fcntl(fd, F_GETFL, 0) | O_APPEND)) {
 504                 /* (should not happen; fd is regular file) */
 505                 log_error_write(srv, __FILE__, __LINE__, "sbs",
 506                                 "fcntl():", template, strerror(errno));
 507                 close(fd);
 508                 buffer_free(template);
 509                 return NULL;
 510         }
 511         fdevent_setfd_cloexec(fd);
 512
 513         c = chunkqueue_get_unused_chunk(cq);
 514         c->type = FILE_CHUNK;
 515         c->file.fd = fd;
 516         c->file.is_temp = 1;
 517         buffer_copy_buffer(c->file.name, template);
 518         c->file.length = 0;
 519
 520         chunkqueue_append_chunk(cq, c);
 521
 522         buffer_free(template);
 523
 524         return c;
 525 }
 526
 527 static void chunkqueue_remove_empty_chunks(chunkqueue *cq);
 528
 529 int chunkqueue_append_mem_to_tempfile(server *srv, chunkqueue *dest, const char *mem, size_t len) {
 530         chunk *dst_c;
 531         ssize_t written;
 532
 533         do {
 534                 /*
 535                  * if the last chunk is
 536                  * - smaller than dest->upload_temp_file_size
 537                  * - not read yet (offset == 0)
 538                  * -> append to it (so it might actually become larger than dest->upload_temp_file_size)
 539                  * otherwise
 540                  * -> create a new chunk
 541                  *
 542                  * */
 543
 544                 dst_c = dest->last;
 545                 if (NULL != dst_c
 546                         && FILE_CHUNK == dst_c->type
 547                         && dst_c->file.is_temp
 548                         && dst_c->file.fd >= 0
 549                         && 0 == dst_c->offset) {
 550                         /* ok, take the last chunk for our job */
 551
 552                         if (dst_c->file.length >= (off_t)dest->upload_temp_file_size) {
 553                                 /* the chunk is too large now, close it */
 554                                 int rc = close(dst_c->file.fd);
 555                                 dst_c->file.fd = -1;
 556                                 if (0 != rc) {
 557                                         log_error_write(srv, __FILE__, __LINE__, "sbss",
 558                                                 "close() temp-file", dst_c->file.name, "failed:",
 559                                                 strerror(errno));
 560                                         return -1;
 561                                 }
 562                                 dst_c = NULL;
 563                         }
 564                 } else {
 565                         dst_c = NULL;
 566                 }
 567
 568                 if (NULL == dst_c && NULL == (dst_c = chunkqueue_get_append_tempfile(srv, dest))) {
 569                         return -1;
 570                 }
 571               #ifdef __COVERITY__
 572                 if (dst_c->file.fd < 0) return -1;
 573               #endif
 574
 575                 /* (dst_c->file.fd >= 0) */
 576                 /* coverity[negative_returns : FALSE] */
 577                 written = write(dst_c->file.fd, mem, len);
 578
 579                 if ((size_t) written == len) {
 580                         dst_c->file.length += len;
 581                         dest->bytes_in += len;
 582
 583                         return 0;
 584                 } else if (written >= 0) {
 585                         /*(assume EINTR if partial write and retry write();
 586                          * retry write() might fail with ENOSPC if no more space on volume)*/
 587                         dest->bytes_in += written;
 588                         mem += written;
 589                         len -= (size_t)written;
 590                         dst_c->file.length += (size_t)written;
 591                         /* continue; retry */
 592                 } else if (errno == EINTR) {
 593                         /* continue; retry */
 594                 } else {
 595                         int retry = (errno == ENOSPC && dest->tempdirs && ++dest->tempdir_idx < dest->tempdirs->used);
 596                         if (!retry) {
 597                                 log_error_write(srv, __FILE__, __LINE__, "sbs",
 598                                                 "write() temp-file", dst_c->file.name, "failed:",
 599                                                 strerror(errno));
 600                         }
 601
 602                         if (0 == chunk_remaining_length(dst_c)) {
 603                                 /*(remove empty chunk and unlink tempfile)*/
 604                                 chunkqueue_remove_empty_chunks(dest);
 605                         } else {/*(close tempfile; avoid later attempts to append)*/
 606                                 int rc = close(dst_c->file.fd);
 607                                 dst_c->file.fd = -1;
 608                                 if (0 != rc) {
 609                                         log_error_write(srv, __FILE__, __LINE__, "sbss",
 610                                                 "close() temp-file", dst_c->file.name, "failed:",
 611                                                 strerror(errno));
 612                                         return -1;
 613                                 }
 614                         }
 615                         if (!retry) break; /* return -1; */
 616
 617                         /* continue; retry */
 618                 }
 619
 620         } while (dst_c);
 621
 622         return -1;
 623 }
 624
 625 int chunkqueue_steal_with_tempfiles(server *srv, chunkqueue *dest, chunkqueue *src, off_t len) {
 626         while (len > 0) {
 627                 chunk *c = src->first;
 628                 off_t clen = 0, use;
 629
 630                 if (NULL == c) break;
 631
 632                 clen = chunk_remaining_length(c);
 633                 if (0 == clen) {
 634                         /* drop empty chunk */
 635                         src->first = c->next;
 636                         if (c == src->last) src->last = NULL;
 637                         chunkqueue_push_unused_chunk(src, c);
 638                         continue;
 639                 }
 640
 641                 use = (len >= clen) ? clen : len;
 642                 len -= use;
 643
 644                 switch (c->type) {
 645                 case FILE_CHUNK:
 646                         if (use == clen) {
 647                                 /* move complete chunk */
 648                                 src->first = c->next;
 649                                 if (c == src->last) src->last = NULL;
 650                                 chunkqueue_append_chunk(dest, c);
 651                         } else {
 652                                 /* partial chunk with length "use" */
 653                                 /* tempfile flag is in "last" chunk after the split */
 654                                 chunkqueue_append_file(dest, c->file.name, c->file.start + c->offset, use);
 655
 656                                 c->offset += use;
 657                                 force_assert(0 == len);
 658                         }
 659                         break;
 660
 661                 case MEM_CHUNK:
 662                         /* store "use" bytes from memory chunk in tempfile */
 663                         if (0 != chunkqueue_append_mem_to_tempfile(srv, dest, c->mem->ptr + c->offset, use)) {
 664                                 return -1;
 665                         }
 666
 667                         if (use == clen) {
 668                                 /* finished chunk */
 669                                 src->first = c->next;
 670                                 if (c == src->last) src->last = NULL;
 671                                 chunkqueue_push_unused_chunk(src, c);
 672                         } else {
 673                                 /* partial chunk */
 674                                 c->offset += use;
 675                                 force_assert(0 == len);
 676                         }
 677                         break;
 678                 }
 679
 680                 src->bytes_out += use;
 681         }
 682
 683         return 0;
 684 }
 685
 686 off_t chunkqueue_length(chunkqueue *cq) {
 687         off_t len = 0;
 688         chunk *c;
 689
 690         for (c = cq->first; c; c = c->next) {
 691                 len += chunk_remaining_length(c);
 692         }
 693
 694         return len;
 695 }
 696
 697 void chunkqueue_mark_written(chunkqueue *cq, off_t len) {
 698         off_t written = len;
 699         chunk *c;
 700         force_assert(len >= 0);
 701
 702         for (c = cq->first; NULL != c; c = cq->first) {
 703                 off_t c_len = chunk_remaining_length(c);
 704
 705                 if (0 == written && 0 != c_len) break; /* no more finished chunks */
 706
 707                 if (written >= c_len) { /* chunk got finished */
 708                         c->offset += c_len;
 709                         written -= c_len;
 710
 711                         cq->first = c->next;
 712                         if (c == cq->last) cq->last = NULL;
 713
 714                         chunkqueue_push_unused_chunk(cq, c);
 715                 } else { /* partial chunk */
 716                         c->offset += written;
 717                         written = 0;
 718                         break; /* chunk not finished */
 719                 }
 720         }
 721
 722         force_assert(0 == written);
 723         cq->bytes_out += len;
 724 }
 725
 726 void chunkqueue_remove_finished_chunks(chunkqueue *cq) {
 727         chunk *c;
 728
 729         for (c = cq->first; c; c = cq->first) {
 730                 if (0 != chunk_remaining_length(c)) break; /* not finished yet */
 731
 732                 cq->first = c->next;
 733                 if (c == cq->last) cq->last = NULL;
 734
 735                 chunkqueue_push_unused_chunk(cq, c);
 736         }
 737 }
 738
 739 static void chunkqueue_remove_empty_chunks(chunkqueue *cq) {
 740         chunk *c;
 741         chunkqueue_remove_finished_chunks(cq);
 742         if (chunkqueue_is_empty(cq)) return;
 743
 744         for (c = cq->first; c->next; c = c->next) {
 745                 if (0 == chunk_remaining_length(c->next)) {
 746                         chunk *empty = c->next;
 747                         c->next = empty->next;
 748                         if (empty == cq->last) cq->last = c;
 749
 750                         chunkqueue_push_unused_chunk(cq, empty);
 751                 }
 752         }
 753 }
 754
 755 int chunkqueue_open_file_chunk(server *srv, chunkqueue *cq) {
 756         chunk* const c = cq->first;
 757         off_t offset, toSend;
 758         struct stat st;
 759
 760         force_assert(NULL != c);
 761         force_assert(FILE_CHUNK == c->type);
 762         force_assert(c->offset >= 0 && c->offset <= c->file.length);
 763
 764         offset = c->file.start + c->offset;
 765         toSend = c->file.length - c->offset;
 766
 767         if (-1 == c->file.fd) {
 768                 if (-1 == (c->file.fd = fdevent_open_cloexec(c->file.name->ptr, O_RDONLY, 0))) {
 769                         log_error_write(srv, __FILE__, __LINE__, "ssb", "open failed:", strerror(errno), c->file.name);
 770                         return -1;
 771                 }
 772         }
 773
 774         /*(skip file size checks if file is temp file created by lighttpd)*/
 775         if (c->file.is_temp) return 0;
 776
 777         if (-1 == fstat(c->file.fd, &st)) {
 778                 log_error_write(srv, __FILE__, __LINE__, "ss", "fstat failed:", strerror(errno));
 779                 return -1;
 780         }
 781
 782         if (offset > st.st_size || toSend > st.st_size || offset > st.st_size - toSend) {
 783                 log_error_write(srv, __FILE__, __LINE__, "sb", "file shrunk:", c->file.name);
 784                 return -1;
 785         }
 786
 787         return 0;
 788 }