release/src/router/nginx/src/os/unix/ngx_linux_sendfile_chain.c

   1
   2 /*
   3  * Copyright (C) Igor Sysoev
   4  * Copyright (C) Nginx, Inc.
   5  */
   6
   7
   8 #include <ngx_config.h>
   9 #include <ngx_core.h>
  10 #include <ngx_event.h>
  11
  12
  13 /*
  14  * On Linux up to 2.4.21 sendfile() (syscall #187) works with 32-bit
  15  * offsets only, and the including <sys/sendfile.h> breaks the compiling,
  16  * if off_t is 64 bit wide.  So we use own sendfile() definition, where offset
  17  * parameter is int32_t, and use sendfile() for the file parts below 2G only,
  18  * see src/os/unix/ngx_linux_config.h
  19  *
  20  * Linux 2.4.21 has the new sendfile64() syscall #239.
  21  *
  22  * On Linux up to 2.6.16 sendfile() does not allow to pass the count parameter
  23  * more than 2G-1 bytes even on 64-bit platforms: it returns EINVAL,
  24  * so we limit it to 2G-1 bytes.
  25  */
  26
  27 #define NGX_SENDFILE_LIMIT  2147483647L
  28
  29
  30 #if (IOV_MAX > 64)
  31 #define NGX_HEADERS  64
  32 #else
  33 #define NGX_HEADERS  IOV_MAX
  34 #endif
  35
  36
  37 ngx_chain_t *
  38 ngx_linux_sendfile_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit)
  39 {
  40     int            rc, tcp_nodelay;
  41     off_t          size, send, prev_send, aligned, sent, fprev;
  42     u_char        *prev;
  43     size_t         file_size;
  44     ngx_err_t      err;
  45     ngx_buf_t     *file;
  46     ngx_uint_t     eintr, complete;
  47     ngx_array_t    header;
  48     ngx_event_t   *wev;
  49     ngx_chain_t   *cl;
  50     struct iovec  *iov, headers[NGX_HEADERS];
  51 #if (NGX_HAVE_SENDFILE64)
  52     off_t          offset;
  53 #else
  54     int32_t        offset;
  55 #endif
  56
  57     wev = c->write;
  58
  59     if (!wev->ready) {
  60         return in;
  61     }
  62
  63
  64     /* the maximum limit size is 2G-1 - the page size */
  65
  66     if (limit == 0 || limit > (off_t) (NGX_SENDFILE_LIMIT - ngx_pagesize)) {
  67         limit = NGX_SENDFILE_LIMIT - ngx_pagesize;
  68     }
  69
  70
  71     send = 0;
  72
  73     header.elts = headers;
  74     header.size = sizeof(struct iovec);
  75     header.nalloc = NGX_HEADERS;
  76     header.pool = c->pool;
  77
  78     for ( ;; ) {
  79         file = NULL;
  80         file_size = 0;
  81         eintr = 0;
  82         complete = 0;
  83         prev_send = send;
  84
  85         header.nelts = 0;
  86
  87         prev = NULL;
  88         iov = NULL;
  89
  90         /* create the iovec and coalesce the neighbouring bufs */
  91
  92         for (cl = in; cl && send < limit; cl = cl->next) {
  93
  94             if (ngx_buf_special(cl->buf)) {
  95                 continue;
  96             }
  97
  98 #if 1
  99             if (!ngx_buf_in_memory(cl->buf) && !cl->buf->in_file) {
 100                 ngx_log_error(NGX_LOG_ALERT, c->log, 0,
 101                               "zero size buf in sendfile "
 102                               "t:%d r:%d f:%d %p %p-%p %p %O-%O",
 103                               cl->buf->temporary,
 104                               cl->buf->recycled,
 105                               cl->buf->in_file,
 106                               cl->buf->start,
 107                               cl->buf->pos,
 108                               cl->buf->last,
 109                               cl->buf->file,
 110                               cl->buf->file_pos,
 111                               cl->buf->file_last);
 112
 113                 ngx_debug_point();
 114
 115                 return NGX_CHAIN_ERROR;
 116             }
 117 #endif
 118
 119             if (!ngx_buf_in_memory_only(cl->buf)) {
 120                 break;
 121             }
 122
 123             size = cl->buf->last - cl->buf->pos;
 124
 125             if (send + size > limit) {
 126                 size = limit - send;
 127             }
 128
 129             if (prev == cl->buf->pos) {
 130                 iov->iov_len += (size_t) size;
 131
 132             } else {
 133                 if (header.nelts >= IOV_MAX) {
 134                     break;
 135                 }
 136
 137                 iov = ngx_array_push(&header);
 138                 if (iov == NULL) {
 139                     return NGX_CHAIN_ERROR;
 140                 }
 141
 142                 iov->iov_base = (void *) cl->buf->pos;
 143                 iov->iov_len = (size_t) size;
 144             }
 145
 146             prev = cl->buf->pos + (size_t) size;
 147             send += size;
 148         }
 149
 150         /* set TCP_CORK if there is a header before a file */
 151
 152         if (c->tcp_nopush == NGX_TCP_NOPUSH_UNSET
 153             && header.nelts != 0
 154             && cl
 155             && cl->buf->in_file)
 156         {
 157             /* the TCP_CORK and TCP_NODELAY are mutually exclusive */
 158
 159             if (c->tcp_nodelay == NGX_TCP_NODELAY_SET) {
 160
 161                 tcp_nodelay = 0;
 162
 163                 if (setsockopt(c->fd, IPPROTO_TCP, TCP_NODELAY,
 164                                (const void *) &tcp_nodelay, sizeof(int)) == -1)
 165                 {
 166                     err = ngx_errno;
 167
 168                     /*
 169                      * there is a tiny chance to be interrupted, however,
 170                      * we continue a processing with the TCP_NODELAY
 171                      * and without the TCP_CORK
 172                      */
 173
 174                     if (err != NGX_EINTR) {
 175                         wev->error = 1;
 176                         ngx_connection_error(c, err,
 177                                              "setsockopt(TCP_NODELAY) failed");
 178                         return NGX_CHAIN_ERROR;
 179                     }
 180
 181                 } else {
 182                     c->tcp_nodelay = NGX_TCP_NODELAY_UNSET;
 183
 184                     ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0,
 185                                    "no tcp_nodelay");
 186                 }
 187             }
 188
 189             if (c->tcp_nodelay == NGX_TCP_NODELAY_UNSET) {
 190
 191                 if (ngx_tcp_nopush(c->fd) == NGX_ERROR) {
 192                     err = ngx_errno;
 193
 194                     /*
 195                      * there is a tiny chance to be interrupted, however,
 196                      * we continue a processing without the TCP_CORK
 197                      */
 198
 199                     if (err != NGX_EINTR) {
 200                         wev->error = 1;
 201                         ngx_connection_error(c, err,
 202                                              ngx_tcp_nopush_n " failed");
 203                         return NGX_CHAIN_ERROR;
 204                     }
 205
 206                 } else {
 207                     c->tcp_nopush = NGX_TCP_NOPUSH_SET;
 208
 209                     ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0,
 210                                    "tcp_nopush");
 211                 }
 212             }
 213         }
 214
 215         /* get the file buf */
 216
 217         if (header.nelts == 0 && cl && cl->buf->in_file && send < limit) {
 218             file = cl->buf;
 219
 220             /* coalesce the neighbouring file bufs */
 221
 222             do {
 223                 size = cl->buf->file_last - cl->buf->file_pos;
 224
 225                 if (send + size > limit) {
 226                     size = limit - send;
 227
 228                     aligned = (cl->buf->file_pos + size + ngx_pagesize - 1)
 229                                & ~((off_t) ngx_pagesize - 1);
 230
 231                     if (aligned <= cl->buf->file_last) {
 232                         size = aligned - cl->buf->file_pos;
 233                     }
 234                 }
 235
 236                 file_size += (size_t) size;
 237                 send += size;
 238                 fprev = cl->buf->file_pos + size;
 239                 cl = cl->next;
 240
 241             } while (cl
 242                      && cl->buf->in_file
 243                      && send < limit
 244                      && file->file->fd == cl->buf->file->fd
 245                      && fprev == cl->buf->file_pos);
 246         }
 247
 248         if (file) {
 249 #if 1
 250             if (file_size == 0) {
 251                 ngx_debug_point();
 252                 return NGX_CHAIN_ERROR;
 253             }
 254 #endif
 255 #if (NGX_HAVE_SENDFILE64)
 256             offset = file->file_pos;
 257 #else
 258             offset = (int32_t) file->file_pos;
 259 #endif
 260
 261             ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0,
 262                            "sendfile: @%O %uz", file->file_pos, file_size);
 263
 264             rc = sendfile(c->fd, file->file->fd, &offset, file_size);
 265
 266             if (rc == -1) {
 267                 err = ngx_errno;
 268
 269                 switch (err) {
 270                 case NGX_EAGAIN:
 271                     break;
 272
 273                 case NGX_EINTR:
 274                     eintr = 1;
 275                     break;
 276
 277                 default:
 278                     wev->error = 1;
 279                     ngx_connection_error(c, err, "sendfile() failed");
 280                     return NGX_CHAIN_ERROR;
 281                 }
 282
 283                 ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, err,
 284                                "sendfile() is not ready");
 285             }
 286
 287             sent = rc > 0 ? rc : 0;
 288
 289             ngx_log_debug4(NGX_LOG_DEBUG_EVENT, c->log, 0,
 290                            "sendfile: %d, @%O %O:%uz",
 291                            rc, file->file_pos, sent, file_size);
 292
 293         } else {
 294             rc = writev(c->fd, header.elts, header.nelts);
 295
 296             if (rc == -1) {
 297                 err = ngx_errno;
 298
 299                 switch (err) {
 300                 case NGX_EAGAIN:
 301                     break;
 302
 303                 case NGX_EINTR:
 304                     eintr = 1;
 305                     break;
 306
 307                 default:
 308                     wev->error = 1;
 309                     ngx_connection_error(c, err, "writev() failed");
 310                     return NGX_CHAIN_ERROR;
 311                 }
 312
 313                 ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, err,
 314                                "writev() not ready");
 315             }
 316
 317             sent = rc > 0 ? rc : 0;
 318
 319             ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, "writev: %O", sent);
 320         }
 321
 322         if (send - prev_send == sent) {
 323             complete = 1;
 324         }
 325
 326         c->sent += sent;
 327
 328         for (cl = in; cl; cl = cl->next) {
 329
 330             if (ngx_buf_special(cl->buf)) {
 331                 continue;
 332             }
 333
 334             if (sent == 0) {
 335                 break;
 336             }
 337
 338             size = ngx_buf_size(cl->buf);
 339
 340             if (sent >= size) {
 341                 sent -= size;
 342
 343                 if (ngx_buf_in_memory(cl->buf)) {
 344                     cl->buf->pos = cl->buf->last;
 345                 }
 346
 347                 if (cl->buf->in_file) {
 348                     cl->buf->file_pos = cl->buf->file_last;
 349                 }
 350
 351                 continue;
 352             }
 353
 354             if (ngx_buf_in_memory(cl->buf)) {
 355                 cl->buf->pos += (size_t) sent;
 356             }
 357
 358             if (cl->buf->in_file) {
 359                 cl->buf->file_pos += sent;
 360             }
 361
 362             break;
 363         }
 364
 365         if (eintr) {
 366             continue;
 367         }
 368
 369         if (!complete) {
 370             wev->ready = 0;
 371             return cl;
 372         }
 373
 374         if (send >= limit || cl == NULL) {
 375             return cl;
 376         }
 377
 378         in = cl;
 379     }
 380 }