src/core/ngx_output_chain.c

   1
   2 /*
   3  * Copyright (C) Igor Sysoev
   4  */
   5
   6
   7 #include <ngx_config.h>
   8 #include <ngx_core.h>
   9 #include <ngx_event.h>
  10
  11
  12 #if 0
  13 #define NGX_SENDFILE_LIMIT  4096
  14 #endif
  15
  16
  17 #define NGX_NONE            1
  18
  19
  20 static ngx_inline ngx_int_t
  21     ngx_output_chain_need_to_copy(ngx_output_chain_ctx_t *ctx, ngx_buf_t *buf);
  22 static ngx_int_t ngx_output_chain_add_copy(ngx_pool_t *pool,
  23     ngx_chain_t **chain, ngx_chain_t *in);
  24 static ngx_int_t ngx_output_chain_copy_buf(ngx_buf_t *dst, ngx_buf_t *src,
  25     ngx_uint_t sendfile);
  26
  27
  28 ngx_int_t
  29 ngx_output_chain(ngx_output_chain_ctx_t *ctx, ngx_chain_t *in)
  30 {
  31     off_t         bsize;
  32     size_t        size;
  33     ngx_int_t     rc, last;
  34     ngx_uint_t    recycled;
  35     ngx_chain_t  *cl, *out, **last_out;
  36
  37     if (ctx->in == NULL && ctx->busy == NULL) {
  38
  39         /*
  40          * the short path for the case when the ctx->in and ctx->busy chains
  41          * are empty, the incoming chain is empty too or has the single buf
  42          * that does not require the copy
  43          */
  44
  45         if (in == NULL) {
  46             return ctx->output_filter(ctx->filter_ctx, in);
  47         }
  48
  49         if (in->next == NULL
  50 #if (NGX_SENDFILE_LIMIT)
  51             && !(in->buf->in_file && in->buf->file_last > NGX_SENDFILE_LIMIT)
  52 #endif
  53             && !ngx_output_chain_need_to_copy(ctx, in->buf))
  54         {
  55             return ctx->output_filter(ctx->filter_ctx, in);
  56         }
  57     }
  58
  59     /* add the incoming buf to the chain ctx->in */
  60
  61     if (in) {
  62         if (ngx_output_chain_add_copy(ctx->pool, &ctx->in, in) == NGX_ERROR) {
  63             return NGX_ERROR;
  64         }
  65     }
  66
  67     out = NULL;
  68     last_out = &out;
  69     last = NGX_NONE;
  70
  71     for ( ;; ) {
  72
  73         while (ctx->in) {
  74
  75             /*
  76              * cycle while there are the ctx->in bufs
  77              * or there are the free output bufs to copy in
  78              */
  79
  80             bsize = ngx_buf_size(ctx->in->buf);
  81
  82             if (bsize == 0 && !ngx_buf_special(ctx->in->buf)) {
  83
  84                 ngx_log_error(NGX_LOG_ALERT, ctx->pool->log, 0,
  85                               "zero size buf in output "
  86                               "t:%d r:%d f:%d %p %p-%p %p %O-%O",
  87                               ctx->in->buf->temporary,
  88                               ctx->in->buf->recycled,
  89                               ctx->in->buf->in_file,
  90                               ctx->in->buf->start,
  91                               ctx->in->buf->pos,
  92                               ctx->in->buf->last,
  93                               ctx->in->buf->file,
  94                               ctx->in->buf->file_pos,
  95                               ctx->in->buf->file_last);
  96
  97                 ngx_debug_point();
  98
  99                 ctx->in = ctx->in->next;
 100
 101                 continue;
 102             }
 103
 104             if (!ngx_output_chain_need_to_copy(ctx, ctx->in->buf)) {
 105
 106                 /* move the chain link to the output chain */
 107
 108                 cl = ctx->in;
 109                 ctx->in = cl->next;
 110
 111                 *last_out = cl;
 112                 last_out = &cl->next;
 113                 cl->next = NULL;
 114
 115                 continue;
 116             }
 117
 118             if (ctx->buf == NULL) {
 119
 120                 /* get the free buf */
 121
 122                 if (ctx->free) {
 123                     cl = ctx->free;
 124                     ctx->buf = cl->buf;
 125                     ctx->free = cl->next;
 126                     ngx_free_chain(ctx->pool, cl);
 127
 128                 } else if (out || ctx->allocated == ctx->bufs.num) {
 129
 130                     break;
 131
 132                 } else {
 133
 134                     size = ctx->bufs.size;
 135                     recycled = 1;
 136
 137                     if (ctx->in->buf->last_in_chain) {
 138
 139                         if (bsize < (off_t) ctx->bufs.size) {
 140
 141                            /*
 142                             * allocate small temp buf for the small last buf
 143                             * or its small last part
 144                             */
 145
 146                             size = (size_t) bsize;
 147                             recycled = 0;
 148
 149                         } else if (ctx->bufs.num == 1
 150                                    && (bsize < (off_t) (ctx->bufs.size
 151                                                      + (ctx->bufs.size >> 2))))
 152                         {
 153                             /*
 154                              * allocate a temp buf that equals
 155                              * to the last buf if the last buf size is lesser
 156                              * than 1.25 of bufs.size and a temp buf is single
 157                              */
 158
 159                             size = (size_t) bsize;
 160                             recycled = 0;
 161                         }
 162                     }
 163
 164                     ctx->buf = ngx_create_temp_buf(ctx->pool, size);
 165                     if (ctx->buf == NULL) {
 166                         return NGX_ERROR;
 167                     }
 168
 169                     ctx->buf->tag = ctx->tag;
 170                     ctx->buf->recycled = recycled;
 171                     ctx->allocated++;
 172                 }
 173             }
 174
 175             rc = ngx_output_chain_copy_buf(ctx->buf, ctx->in->buf,
 176                                            ctx->sendfile);
 177
 178             if (rc == NGX_ERROR) {
 179                 return rc;
 180             }
 181
 182             if (rc == NGX_AGAIN) {
 183                 if (out) {
 184                     break;
 185                 }
 186
 187                 return rc;
 188             }
 189
 190             /* delete the completed buf from the ctx->in chain */
 191
 192             if (ngx_buf_size(ctx->in->buf) == 0) {
 193                 ctx->in = ctx->in->next;
 194             }
 195
 196             cl = ngx_alloc_chain_link(ctx->pool);
 197             if (cl == NULL) {
 198                 return NGX_ERROR;
 199             }
 200
 201             cl->buf = ctx->buf;
 202             cl->next = NULL;
 203             *last_out = cl;
 204             last_out = &cl->next;
 205             ctx->buf = NULL;
 206         }
 207
 208         if (out == NULL && last != NGX_NONE) {
 209
 210             if (ctx->in) {
 211                 return NGX_AGAIN;
 212             }
 213
 214             return last;
 215         }
 216
 217         last = ctx->output_filter(ctx->filter_ctx, out);
 218
 219         if (last == NGX_ERROR || last == NGX_DONE) {
 220             return last;
 221         }
 222
 223         ngx_chain_update_chains(&ctx->free, &ctx->busy, &out, ctx->tag);
 224         last_out = &out;
 225     }
 226 }
 227
 228
 229 static ngx_inline ngx_int_t
 230 ngx_output_chain_need_to_copy(ngx_output_chain_ctx_t *ctx, ngx_buf_t *buf)
 231 {
 232     ngx_uint_t  sendfile;
 233
 234     if (ngx_buf_special(buf)) {
 235         return 0;
 236     }
 237
 238     sendfile = ctx->sendfile;
 239
 240 #if (NGX_SENDFILE_LIMIT)
 241
 242     if (buf->in_file && buf->file_pos >= NGX_SENDFILE_LIMIT) {
 243         sendfile = 0;
 244     }
 245
 246 #endif
 247
 248     if (!sendfile) {
 249
 250         if (!ngx_buf_in_memory(buf)) {
 251             return 1;
 252         }
 253
 254         buf->in_file = 0;
 255     }
 256
 257     if (ctx->need_in_memory && !ngx_buf_in_memory(buf)) {
 258         return 1;
 259     }
 260
 261     if (ctx->need_in_temp && (buf->memory || buf->mmap)) {
 262         return 1;
 263     }
 264
 265     return 0;
 266 }
 267
 268
 269 static ngx_int_t
 270 ngx_output_chain_add_copy(ngx_pool_t *pool, ngx_chain_t **chain,
 271     ngx_chain_t *in)
 272 {
 273     ngx_chain_t  *cl, **ll;
 274 #if (NGX_SENDFILE_LIMIT)
 275     ngx_buf_t    *b, *buf;
 276 #endif
 277
 278     ll = chain;
 279
 280     for (cl = *chain; cl; cl = cl->next) {
 281         ll = &cl->next;
 282     }
 283
 284     while (in) {
 285
 286         cl = ngx_alloc_chain_link(pool);
 287         if (cl == NULL) {
 288             return NGX_ERROR;
 289         }
 290
 291 #if (NGX_SENDFILE_LIMIT)
 292
 293         buf = in->buf;
 294
 295         if (buf->in_file
 296             && buf->file_pos < NGX_SENDFILE_LIMIT
 297             && buf->file_last > NGX_SENDFILE_LIMIT)
 298         {
 299             b = ngx_calloc_buf(pool);
 300             if (b == NULL) {
 301                 return NGX_ERROR;
 302             }
 303
 304             ngx_memcpy(b, buf, sizeof(ngx_buf_t));
 305
 306             if (ngx_buf_in_memory(buf)) {
 307                 buf->pos += (ssize_t) (NGX_SENDFILE_LIMIT - buf->file_pos);
 308                 b->last = buf->pos;
 309             }
 310
 311             buf->file_pos = NGX_SENDFILE_LIMIT;
 312             b->file_last = NGX_SENDFILE_LIMIT;
 313
 314             cl->buf = b;
 315
 316         } else {
 317             cl->buf = buf;
 318             in = in->next;
 319         }
 320
 321 #else
 322         cl->buf = in->buf;
 323         in = in->next;
 324
 325 #endif
 326
 327         *ll = cl;
 328         ll = &cl->next;
 329     }
 330
 331     *ll = NULL;
 332
 333     return NGX_OK;
 334 }
 335
 336
 337 static ngx_int_t
 338 ngx_output_chain_copy_buf(ngx_buf_t *dst, ngx_buf_t *src, ngx_uint_t sendfile)
 339 {
 340     off_t    size;
 341     ssize_t  n;
 342
 343     size = ngx_buf_size(src);
 344
 345     if (size > dst->end - dst->pos) {
 346         size = dst->end - dst->pos;
 347     }
 348
 349 #if (NGX_SENDFILE_LIMIT)
 350
 351     if (src->in_file && src->file_pos >= NGX_SENDFILE_LIMIT) {
 352         sendfile = 0;
 353     }
 354
 355 #endif
 356
 357     if (ngx_buf_in_memory(src)) {
 358         ngx_memcpy(dst->pos, src->pos, (size_t) size);
 359         src->pos += (size_t) size;
 360         dst->last += (size_t) size;
 361
 362         if (src->in_file) {
 363
 364             if (sendfile) {
 365                 dst->in_file = 1;
 366                 dst->file = src->file;
 367                 dst->file_pos = src->file_pos;
 368                 dst->file_last = src->file_pos + size;
 369
 370             } else {
 371                 dst->in_file = 0;
 372             }
 373
 374             src->file_pos += size;
 375
 376         } else {
 377             dst->in_file = 0;
 378         }
 379
 380         if (src->last_buf && src->pos == src->last) {
 381             dst->last_buf = 1;
 382         }
 383
 384     } else {
 385         n = ngx_read_file(src->file, dst->pos, (size_t) size, src->file_pos);
 386
 387         if (n == NGX_ERROR) {
 388             return (ngx_int_t) n;
 389         }
 390
 391 #if (NGX_FILE_AIO_READ)
 392         if (n == NGX_AGAIN) {
 393             return (ngx_int_t) n;
 394         }
 395 #endif
 396
 397         if (n != size) {
 398             ngx_log_error(NGX_LOG_ALERT, src->file->log, 0,
 399                           ngx_read_file_n " reads only %z of %O from file",
 400                           n, size);
 401             if (n == 0) {
 402                 return NGX_ERROR;
 403             }
 404         }
 405
 406         dst->last += n;
 407
 408         if (sendfile) {
 409             dst->in_file = 1;
 410             dst->file = src->file;
 411             dst->file_pos = src->file_pos;
 412             dst->file_last = src->file_pos + n;
 413
 414         } else {
 415             dst->in_file = 0;
 416         }
 417
 418         src->file_pos += n;
 419
 420         if (src->last_buf && src->file_pos == src->file_last) {
 421             dst->last_buf = 1;
 422         }
 423     }
 424
 425     return NGX_OK;
 426 }
 427
 428
 429 ngx_int_t
 430 ngx_chain_writer(void *data, ngx_chain_t *in)
 431 {
 432     ngx_chain_writer_ctx_t *ctx = data;
 433
 434     off_t         size;
 435     ngx_chain_t  *cl;
 436
 437     for (size = 0; in; in = in->next) {
 438
 439 #if 1
 440         if (ngx_buf_size(in->buf) == 0 && !ngx_buf_special(in->buf)) {
 441             ngx_debug_point();
 442         }
 443 #endif
 444
 445         size += ngx_buf_size(in->buf);
 446
 447         ngx_log_debug1(NGX_LOG_DEBUG_CORE, ctx->connection->log, 0,
 448                        "chain writer buf size: %uO", ngx_buf_size(in->buf));
 449
 450         cl = ngx_alloc_chain_link(ctx->pool);
 451         if (cl == NULL) {
 452             return NGX_ERROR;
 453         }
 454
 455         cl->buf = in->buf;
 456         cl->next = NULL;
 457         *ctx->last = cl;
 458         ctx->last = &cl->next;
 459     }
 460
 461     ngx_log_debug1(NGX_LOG_DEBUG_CORE, ctx->connection->log, 0,
 462                    "chain writer in: %p", ctx->out);
 463
 464     for (cl = ctx->out; cl; cl = cl->next) {
 465
 466 #if 1
 467         if (ngx_buf_size(cl->buf) == 0 && !ngx_buf_special(cl->buf)) {
 468             ngx_debug_point();
 469         }
 470
 471 #endif
 472
 473         size += ngx_buf_size(cl->buf);
 474     }
 475
 476     if (size == 0 && !ctx->connection->buffered) {
 477         return NGX_OK;
 478     }
 479
 480     ctx->out = ctx->connection->send_chain(ctx->connection, ctx->out,
 481                                            ctx->limit);
 482
 483     ngx_log_debug1(NGX_LOG_DEBUG_CORE, ctx->connection->log, 0,
 484                    "chain writer out: %p", ctx->out);
 485
 486     if (ctx->out == NGX_CHAIN_ERROR) {
 487         return NGX_ERROR;
 488     }
 489
 490     if (ctx->out == NULL) {
 491         ctx->last = &ctx->out;
 492     }
 493
 494     if (!ctx->connection->buffered) {
 495         return NGX_OK;
 496     }
 497
 498     return NGX_AGAIN;
 499 }