release/src/router/nginx/src/core/ngx_output_chain.c

   1
   2 /*
   3  * Copyright (C) Igor Sysoev
   4  * Copyright (C) Nginx, Inc.
   5  */
   6
   7
   8 #include <ngx_config.h>
   9 #include <ngx_core.h>
  10 #include <ngx_event.h>
  11
  12
  13 #if 0
  14 #define NGX_SENDFILE_LIMIT  4096
  15 #endif
  16
  17 /*
  18  * When DIRECTIO is enabled FreeBSD, Solaris, and MacOSX read directly
  19  * to an application memory from a device if parameters are aligned
  20  * to device sector boundary (512 bytes).  They fallback to usual read
  21  * operation if the parameters are not aligned.
  22  * Linux allows DIRECTIO only if the parameters are aligned to a filesystem
  23  * sector boundary, otherwise it returns EINVAL.  The sector size is
  24  * usually 512 bytes, however, on XFS it may be 4096 bytes.
  25  */
  26
  27 #define NGX_NONE            1
  28
  29
  30 static ngx_inline ngx_int_t
  31     ngx_output_chain_as_is(ngx_output_chain_ctx_t *ctx, ngx_buf_t *buf);
  32 static ngx_int_t ngx_output_chain_add_copy(ngx_pool_t *pool,
  33     ngx_chain_t **chain, ngx_chain_t *in);
  34 static ngx_int_t ngx_output_chain_align_file_buf(ngx_output_chain_ctx_t *ctx,
  35     off_t bsize);
  36 static ngx_int_t ngx_output_chain_get_buf(ngx_output_chain_ctx_t *ctx,
  37     off_t bsize);
  38 static ngx_int_t ngx_output_chain_copy_buf(ngx_output_chain_ctx_t *ctx);
  39
  40
  41 ngx_int_t
  42 ngx_output_chain(ngx_output_chain_ctx_t *ctx, ngx_chain_t *in)
  43 {
  44     off_t         bsize;
  45     ngx_int_t     rc, last;
  46     ngx_chain_t  *cl, *out, **last_out;
  47
  48     if (ctx->in == NULL && ctx->busy == NULL) {
  49
  50         /*
  51          * the short path for the case when the ctx->in and ctx->busy chains
  52          * are empty, the incoming chain is empty too or has the single buf
  53          * that does not require the copy
  54          */
  55
  56         if (in == NULL) {
  57             return ctx->output_filter(ctx->filter_ctx, in);
  58         }
  59
  60         if (in->next == NULL
  61 #if (NGX_SENDFILE_LIMIT)
  62             && !(in->buf->in_file && in->buf->file_last > NGX_SENDFILE_LIMIT)
  63 #endif
  64             && ngx_output_chain_as_is(ctx, in->buf))
  65         {
  66             return ctx->output_filter(ctx->filter_ctx, in);
  67         }
  68     }
  69
  70     /* add the incoming buf to the chain ctx->in */
  71
  72     if (in) {
  73         if (ngx_output_chain_add_copy(ctx->pool, &ctx->in, in) == NGX_ERROR) {
  74             return NGX_ERROR;
  75         }
  76     }
  77
  78     out = NULL;
  79     last_out = &out;
  80     last = NGX_NONE;
  81
  82     for ( ;; ) {
  83
  84 #if (NGX_HAVE_FILE_AIO)
  85         if (ctx->aio) {
  86             return NGX_AGAIN;
  87         }
  88 #endif
  89
  90         while (ctx->in) {
  91
  92             /*
  93              * cycle while there are the ctx->in bufs
  94              * and there are the free output bufs to copy in
  95              */
  96
  97             bsize = ngx_buf_size(ctx->in->buf);
  98
  99             if (bsize == 0 && !ngx_buf_special(ctx->in->buf)) {
 100
 101                 ngx_log_error(NGX_LOG_ALERT, ctx->pool->log, 0,
 102                               "zero size buf in output "
 103                               "t:%d r:%d f:%d %p %p-%p %p %O-%O",
 104                               ctx->in->buf->temporary,
 105                               ctx->in->buf->recycled,
 106                               ctx->in->buf->in_file,
 107                               ctx->in->buf->start,
 108                               ctx->in->buf->pos,
 109                               ctx->in->buf->last,
 110                               ctx->in->buf->file,
 111                               ctx->in->buf->file_pos,
 112                               ctx->in->buf->file_last);
 113
 114                 ngx_debug_point();
 115
 116                 ctx->in = ctx->in->next;
 117
 118                 continue;
 119             }
 120
 121             if (ngx_output_chain_as_is(ctx, ctx->in->buf)) {
 122
 123                 /* move the chain link to the output chain */
 124
 125                 cl = ctx->in;
 126                 ctx->in = cl->next;
 127
 128                 *last_out = cl;
 129                 last_out = &cl->next;
 130                 cl->next = NULL;
 131
 132                 continue;
 133             }
 134
 135             if (ctx->buf == NULL) {
 136
 137                 rc = ngx_output_chain_align_file_buf(ctx, bsize);
 138
 139                 if (rc == NGX_ERROR) {
 140                     return NGX_ERROR;
 141                 }
 142
 143                 if (rc != NGX_OK) {
 144
 145                     if (ctx->free) {
 146
 147                         /* get the free buf */
 148
 149                         cl = ctx->free;
 150                         ctx->buf = cl->buf;
 151                         ctx->free = cl->next;
 152
 153                         ngx_free_chain(ctx->pool, cl);
 154
 155                     } else if (out || ctx->allocated == ctx->bufs.num) {
 156
 157                         break;
 158
 159                     } else if (ngx_output_chain_get_buf(ctx, bsize) != NGX_OK) {
 160                         return NGX_ERROR;
 161                     }
 162                 }
 163             }
 164
 165             rc = ngx_output_chain_copy_buf(ctx);
 166
 167             if (rc == NGX_ERROR) {
 168                 return rc;
 169             }
 170
 171             if (rc == NGX_AGAIN) {
 172                 if (out) {
 173                     break;
 174                 }
 175
 176                 return rc;
 177             }
 178
 179             /* delete the completed buf from the ctx->in chain */
 180
 181             if (ngx_buf_size(ctx->in->buf) == 0) {
 182                 ctx->in = ctx->in->next;
 183             }
 184
 185             cl = ngx_alloc_chain_link(ctx->pool);
 186             if (cl == NULL) {
 187                 return NGX_ERROR;
 188             }
 189
 190             cl->buf = ctx->buf;
 191             cl->next = NULL;
 192             *last_out = cl;
 193             last_out = &cl->next;
 194             ctx->buf = NULL;
 195         }
 196
 197         if (out == NULL && last != NGX_NONE) {
 198
 199             if (ctx->in) {
 200                 return NGX_AGAIN;
 201             }
 202
 203             return last;
 204         }
 205
 206         last = ctx->output_filter(ctx->filter_ctx, out);
 207
 208         if (last == NGX_ERROR || last == NGX_DONE) {
 209             return last;
 210         }
 211
 212         ngx_chain_update_chains(ctx->pool, &ctx->free, &ctx->busy, &out,
 213                                 ctx->tag);
 214         last_out = &out;
 215     }
 216 }
 217
 218
 219 static ngx_inline ngx_int_t
 220 ngx_output_chain_as_is(ngx_output_chain_ctx_t *ctx, ngx_buf_t *buf)
 221 {
 222     ngx_uint_t  sendfile;
 223
 224     if (ngx_buf_special(buf)) {
 225         return 1;
 226     }
 227
 228     if (buf->in_file && buf->file->directio) {
 229         return 0;
 230     }
 231
 232     sendfile = ctx->sendfile;
 233
 234 #if (NGX_SENDFILE_LIMIT)
 235
 236     if (buf->in_file && buf->file_pos >= NGX_SENDFILE_LIMIT) {
 237         sendfile = 0;
 238     }
 239
 240 #endif
 241
 242     if (!sendfile) {
 243
 244         if (!ngx_buf_in_memory(buf)) {
 245             return 0;
 246         }
 247
 248         buf->in_file = 0;
 249     }
 250
 251     if (ctx->need_in_memory && !ngx_buf_in_memory(buf)) {
 252         return 0;
 253     }
 254
 255     if (ctx->need_in_temp && (buf->memory || buf->mmap)) {
 256         return 0;
 257     }
 258
 259     return 1;
 260 }
 261
 262
 263 static ngx_int_t
 264 ngx_output_chain_add_copy(ngx_pool_t *pool, ngx_chain_t **chain,
 265     ngx_chain_t *in)
 266 {
 267     ngx_chain_t  *cl, **ll;
 268 #if (NGX_SENDFILE_LIMIT)
 269     ngx_buf_t    *b, *buf;
 270 #endif
 271
 272     ll = chain;
 273
 274     for (cl = *chain; cl; cl = cl->next) {
 275         ll = &cl->next;
 276     }
 277
 278     while (in) {
 279
 280         cl = ngx_alloc_chain_link(pool);
 281         if (cl == NULL) {
 282             return NGX_ERROR;
 283         }
 284
 285 #if (NGX_SENDFILE_LIMIT)
 286
 287         buf = in->buf;
 288
 289         if (buf->in_file
 290             && buf->file_pos < NGX_SENDFILE_LIMIT
 291             && buf->file_last > NGX_SENDFILE_LIMIT)
 292         {
 293             /* split a file buf on two bufs by the sendfile limit */
 294
 295             b = ngx_calloc_buf(pool);
 296             if (b == NULL) {
 297                 return NGX_ERROR;
 298             }
 299
 300             ngx_memcpy(b, buf, sizeof(ngx_buf_t));
 301
 302             if (ngx_buf_in_memory(buf)) {
 303                 buf->pos += (ssize_t) (NGX_SENDFILE_LIMIT - buf->file_pos);
 304                 b->last = buf->pos;
 305             }
 306
 307             buf->file_pos = NGX_SENDFILE_LIMIT;
 308             b->file_last = NGX_SENDFILE_LIMIT;
 309
 310             cl->buf = b;
 311
 312         } else {
 313             cl->buf = buf;
 314             in = in->next;
 315         }
 316
 317 #else
 318         cl->buf = in->buf;
 319         in = in->next;
 320
 321 #endif
 322
 323         cl->next = NULL;
 324         *ll = cl;
 325         ll = &cl->next;
 326     }
 327
 328     return NGX_OK;
 329 }
 330
 331
 332 static ngx_int_t
 333 ngx_output_chain_align_file_buf(ngx_output_chain_ctx_t *ctx, off_t bsize)
 334 {
 335     size_t      size;
 336     ngx_buf_t  *in;
 337
 338     in = ctx->in->buf;
 339
 340     if (in->file == NULL || !in->file->directio) {
 341         return NGX_DECLINED;
 342     }
 343
 344     ctx->directio = 1;
 345
 346     size = (size_t) (in->file_pos - (in->file_pos & ~(ctx->alignment - 1)));
 347
 348     if (size == 0) {
 349
 350         if (bsize >= (off_t) ctx->bufs.size) {
 351             return NGX_DECLINED;
 352         }
 353
 354         size = (size_t) bsize;
 355
 356     } else {
 357         size = (size_t) ctx->alignment - size;
 358
 359         if ((off_t) size > bsize) {
 360             size = (size_t) bsize;
 361         }
 362     }
 363
 364     ctx->buf = ngx_create_temp_buf(ctx->pool, size);
 365     if (ctx->buf == NULL) {
 366         return NGX_ERROR;
 367     }
 368
 369     /*
 370      * we do not set ctx->buf->tag, because we do not want
 371      * to reuse the buf via ctx->free list
 372      */
 373
 374 #if (NGX_HAVE_ALIGNED_DIRECTIO)
 375     ctx->unaligned = 1;
 376 #endif
 377
 378     return NGX_OK;
 379 }
 380
 381
 382 static ngx_int_t
 383 ngx_output_chain_get_buf(ngx_output_chain_ctx_t *ctx, off_t bsize)
 384 {
 385     size_t       size;
 386     ngx_buf_t   *b, *in;
 387     ngx_uint_t   recycled;
 388
 389     in = ctx->in->buf;
 390     size = ctx->bufs.size;
 391     recycled = 1;
 392
 393     if (in->last_in_chain) {
 394
 395         if (bsize < (off_t) size) {
 396
 397             /*
 398              * allocate a small temp buf for a small last buf
 399              * or its small last part
 400              */
 401
 402             size = (size_t) bsize;
 403             recycled = 0;
 404
 405         } else if (!ctx->directio
 406                    && ctx->bufs.num == 1
 407                    && (bsize < (off_t) (size + size / 4)))
 408         {
 409             /*
 410              * allocate a temp buf that equals to a last buf,
 411              * if there is no directio, the last buf size is lesser
 412              * than 1.25 of bufs.size and the temp buf is single
 413              */
 414
 415             size = (size_t) bsize;
 416             recycled = 0;
 417         }
 418     }
 419
 420     b = ngx_calloc_buf(ctx->pool);
 421     if (b == NULL) {
 422         return NGX_ERROR;
 423     }
 424
 425     if (ctx->directio) {
 426
 427         /*
 428          * allocate block aligned to a disk sector size to enable
 429          * userland buffer direct usage conjunctly with directio
 430          */
 431
 432         b->start = ngx_pmemalign(ctx->pool, size, (size_t) ctx->alignment);
 433         if (b->start == NULL) {
 434             return NGX_ERROR;
 435         }
 436
 437     } else {
 438         b->start = ngx_palloc(ctx->pool, size);
 439         if (b->start == NULL) {
 440             return NGX_ERROR;
 441         }
 442     }
 443
 444     b->pos = b->start;
 445     b->last = b->start;
 446     b->end = b->last + size;
 447     b->temporary = 1;
 448     b->tag = ctx->tag;
 449     b->recycled = recycled;
 450
 451     ctx->buf = b;
 452     ctx->allocated++;
 453
 454     return NGX_OK;
 455 }
 456
 457
 458 static ngx_int_t
 459 ngx_output_chain_copy_buf(ngx_output_chain_ctx_t *ctx)
 460 {
 461     off_t        size;
 462     ssize_t      n;
 463     ngx_buf_t   *src, *dst;
 464     ngx_uint_t   sendfile;
 465
 466     src = ctx->in->buf;
 467     dst = ctx->buf;
 468
 469     size = ngx_buf_size(src);
 470     size = ngx_min(size, dst->end - dst->pos);
 471
 472     sendfile = ctx->sendfile & !ctx->directio;
 473
 474 #if (NGX_SENDFILE_LIMIT)
 475
 476     if (src->in_file && src->file_pos >= NGX_SENDFILE_LIMIT) {
 477         sendfile = 0;
 478     }
 479
 480 #endif
 481
 482     if (ngx_buf_in_memory(src)) {
 483         ngx_memcpy(dst->pos, src->pos, (size_t) size);
 484         src->pos += (size_t) size;
 485         dst->last += (size_t) size;
 486
 487         if (src->in_file) {
 488
 489             if (sendfile) {
 490                 dst->in_file = 1;
 491                 dst->file = src->file;
 492                 dst->file_pos = src->file_pos;
 493                 dst->file_last = src->file_pos + size;
 494
 495             } else {
 496                 dst->in_file = 0;
 497             }
 498
 499             src->file_pos += size;
 500
 501         } else {
 502             dst->in_file = 0;
 503         }
 504
 505         if (src->pos == src->last) {
 506             dst->flush = src->flush;
 507             dst->last_buf = src->last_buf;
 508             dst->last_in_chain = src->last_in_chain;
 509         }
 510
 511     } else {
 512
 513 #if (NGX_HAVE_ALIGNED_DIRECTIO)
 514
 515         if (ctx->unaligned) {
 516             if (ngx_directio_off(src->file->fd) == NGX_FILE_ERROR) {
 517                 ngx_log_error(NGX_LOG_ALERT, ctx->pool->log, ngx_errno,
 518                               ngx_directio_off_n " \"%s\" failed",
 519                               src->file->name.data);
 520             }
 521         }
 522
 523 #endif
 524
 525 #if (NGX_HAVE_FILE_AIO)
 526
 527         if (ctx->aio_handler) {
 528             n = ngx_file_aio_read(src->file, dst->pos, (size_t) size,
 529                                   src->file_pos, ctx->pool);
 530             if (n == NGX_AGAIN) {
 531                 ctx->aio_handler(ctx, src->file);
 532                 return NGX_AGAIN;
 533             }
 534
 535         } else {
 536             n = ngx_read_file(src->file, dst->pos, (size_t) size,
 537                               src->file_pos);
 538         }
 539 #else
 540
 541         n = ngx_read_file(src->file, dst->pos, (size_t) size, src->file_pos);
 542
 543 #endif
 544
 545 #if (NGX_HAVE_ALIGNED_DIRECTIO)
 546
 547         if (ctx->unaligned) {
 548             ngx_err_t  err;
 549
 550             err = ngx_errno;
 551
 552             if (ngx_directio_on(src->file->fd) == NGX_FILE_ERROR) {
 553                 ngx_log_error(NGX_LOG_ALERT, ctx->pool->log, ngx_errno,
 554                               ngx_directio_on_n " \"%s\" failed",
 555                               src->file->name.data);
 556             }
 557
 558             ngx_set_errno(err);
 559
 560             ctx->unaligned = 0;
 561         }
 562
 563 #endif
 564
 565         if (n == NGX_ERROR) {
 566             return (ngx_int_t) n;
 567         }
 568
 569         if (n != size) {
 570             ngx_log_error(NGX_LOG_ALERT, ctx->pool->log, 0,
 571                           ngx_read_file_n " read only %z of %O from \"%s\"",
 572                           n, size, src->file->name.data);
 573             return NGX_ERROR;
 574         }
 575
 576         dst->last += n;
 577
 578         if (sendfile) {
 579             dst->in_file = 1;
 580             dst->file = src->file;
 581             dst->file_pos = src->file_pos;
 582             dst->file_last = src->file_pos + n;
 583
 584         } else {
 585             dst->in_file = 0;
 586         }
 587
 588         src->file_pos += n;
 589
 590         if (src->file_pos == src->file_last) {
 591             dst->flush = src->flush;
 592             dst->last_buf = src->last_buf;
 593             dst->last_in_chain = src->last_in_chain;
 594         }
 595     }
 596
 597     return NGX_OK;
 598 }
 599
 600
 601 ngx_int_t
 602 ngx_chain_writer(void *data, ngx_chain_t *in)
 603 {
 604     ngx_chain_writer_ctx_t *ctx = data;
 605
 606     off_t              size;
 607     ngx_chain_t       *cl;
 608     ngx_connection_t  *c;
 609
 610     c = ctx->connection;
 611
 612     for (size = 0; in; in = in->next) {
 613
 614 #if 1
 615         if (ngx_buf_size(in->buf) == 0 && !ngx_buf_special(in->buf)) {
 616             ngx_debug_point();
 617         }
 618 #endif
 619
 620         size += ngx_buf_size(in->buf);
 621
 622         ngx_log_debug2(NGX_LOG_DEBUG_CORE, c->log, 0,
 623                        "chain writer buf fl:%d s:%uO",
 624                        in->buf->flush, ngx_buf_size(in->buf));
 625
 626         cl = ngx_alloc_chain_link(ctx->pool);
 627         if (cl == NULL) {
 628             return NGX_ERROR;
 629         }
 630
 631         cl->buf = in->buf;
 632         cl->next = NULL;
 633         *ctx->last = cl;
 634         ctx->last = &cl->next;
 635     }
 636
 637     ngx_log_debug1(NGX_LOG_DEBUG_CORE, c->log, 0,
 638                    "chain writer in: %p", ctx->out);
 639
 640     for (cl = ctx->out; cl; cl = cl->next) {
 641
 642 #if 1
 643         if (ngx_buf_size(cl->buf) == 0 && !ngx_buf_special(cl->buf)) {
 644             ngx_debug_point();
 645         }
 646
 647 #endif
 648
 649         size += ngx_buf_size(cl->buf);
 650     }
 651
 652     if (size == 0 && !c->buffered) {
 653         return NGX_OK;
 654     }
 655
 656     ctx->out = c->send_chain(c, ctx->out, ctx->limit);
 657
 658     ngx_log_debug1(NGX_LOG_DEBUG_CORE, c->log, 0,
 659                    "chain writer out: %p", ctx->out);
 660
 661     if (ctx->out == NGX_CHAIN_ERROR) {
 662         return NGX_ERROR;
 663     }
 664
 665     if (ctx->out == NULL) {
 666         ctx->last = &ctx->out;
 667
 668         if (!c->buffered) {
 669             return NGX_OK;
 670         }
 671     }
 672
 673     return NGX_AGAIN;
 674 }