src/os/unix/ngx_linux_sendfile_chain.c

   1
   2 /*
   3  * Copyright (C) Igor Sysoev
   4  */
   5
   6
   7 #include <ngx_config.h>
   8 #include <ngx_core.h>
   9 #include <ngx_event.h>
  10
  11
  12 /*
  13  * On Linux up to 2.4.21 sendfile() (syscall #187) works with 32-bit
  14  * offsets only, and the including <sys/sendfile.h> breaks the compiling,
  15  * if off_t is 64 bit wide.  So we use own sendfile() definition, where offset
  16  * parameter is int32_t, and use sendfile() for the file parts below 2G only,
  17  * see src/os/unix/ngx_linux_config.h
  18  *
  19  * Linux 2.4.21 has a new sendfile64() syscall #239.
  20  */
  21
  22
  23 #define NGX_HEADERS   8
  24
  25
  26 ngx_chain_t *ngx_linux_sendfile_chain(ngx_connection_t *c, ngx_chain_t *in,
  27                                       off_t limit)
  28 {
  29     int              rc, tcp_nodelay;
  30     u_char          *prev;
  31     off_t            size, send, prev_send, aligned, sent, fprev;
  32     size_t           file_size;
  33     ngx_uint_t       eintr, complete;
  34     ngx_err_t        err;
  35     ngx_buf_t       *file;
  36     ngx_array_t      header;
  37     ngx_event_t     *wev;
  38     ngx_chain_t     *cl;
  39     struct iovec    *iov, headers[NGX_HEADERS];
  40 #if (NGX_HAVE_SENDFILE64)
  41     off_t            offset;
  42 #else
  43     int32_t          offset;
  44 #endif
  45
  46     wev = c->write;
  47
  48     if (!wev->ready) {
  49         return in;
  50     }
  51
  52
  53     /* the maximum limit size is the maximum size_t value - the page size */
  54
  55     if (limit == 0 || limit > NGX_MAX_SIZE_T_VALUE - ngx_pagesize) {
  56         limit = NGX_MAX_SIZE_T_VALUE - ngx_pagesize;
  57     }
  58
  59
  60     send = 0;
  61
  62     header.elts = headers;
  63     header.size = sizeof(struct iovec);
  64     header.nalloc = NGX_HEADERS;
  65     header.pool = c->pool;
  66
  67     for ( ;; ) {
  68         file = NULL;
  69         file_size = 0;
  70         eintr = 0;
  71         complete = 0;
  72         prev_send = send;
  73
  74         header.nelts = 0;
  75
  76         prev = NULL;
  77         iov = NULL;
  78
  79         /* create the iovec and coalesce the neighbouring bufs */
  80
  81         for (cl = in;
  82              cl && header.nelts < IOV_MAX && send < limit;
  83              cl = cl->next)
  84         {
  85             if (ngx_buf_special(cl->buf)) {
  86                 continue;
  87             }
  88
  89 #if 1
  90             if (!ngx_buf_in_memory(cl->buf) && !cl->buf->in_file) {
  91                 ngx_log_error(NGX_LOG_ALERT, c->log, 0,
  92                               "zero size buf in sendfile "
  93                               "t:%d r:%d f:%d %p %p-%p %p %O-%O",
  94                               cl->buf->temporary,
  95                               cl->buf->recycled,
  96                               cl->buf->in_file,
  97                               cl->buf->start,
  98                               cl->buf->pos,
  99                               cl->buf->last,
 100                               cl->buf->file,
 101                               cl->buf->file_pos,
 102                               cl->buf->file_last);
 103
 104                 ngx_debug_point();
 105
 106                 return NGX_CHAIN_ERROR;
 107             }
 108 #endif
 109
 110             if (!ngx_buf_in_memory_only(cl->buf)) {
 111                 break;
 112             }
 113
 114             size = cl->buf->last - cl->buf->pos;
 115
 116             if (send + size > limit) {
 117                 size = limit - send;
 118             }
 119
 120             if (prev == cl->buf->pos) {
 121                 iov->iov_len += (size_t) size;
 122
 123             } else {
 124                 if (!(iov = ngx_array_push(&header))) {
 125                     return NGX_CHAIN_ERROR;
 126                 }
 127
 128                 iov->iov_base = (void *) cl->buf->pos;
 129                 iov->iov_len = (size_t) size;
 130             }
 131
 132             prev = cl->buf->pos + (size_t) size;
 133             send += size;
 134         }
 135
 136         /* set TCP_CORK if there is a header before a file */
 137
 138         if (c->tcp_nopush == NGX_TCP_NOPUSH_UNSET
 139             && header.nelts != 0
 140             && cl
 141             && cl->buf->in_file)
 142         {
 143             /* the TCP_CORK and TCP_NODELAY are mutually exclusive */
 144
 145             if (c->tcp_nodelay == NGX_TCP_NODELAY_SET) {
 146
 147                 tcp_nodelay = 0;
 148
 149                 if (setsockopt(c->fd, IPPROTO_TCP, TCP_NODELAY,
 150                                (const void *) &tcp_nodelay, sizeof(int)) == -1)
 151                 {
 152                     err = ngx_errno;
 153
 154                     /*
 155                      * there is a tiny chance to be interrupted, however,
 156                      * we continue a processing with the TCP_NODELAY
 157                      * and without the TCP_CORK
 158                      */
 159
 160                     if (err != NGX_EINTR) {
 161                         wev->error = 1;
 162                         ngx_connection_error(c, err,
 163                                              "setsockopt(TCP_NODELAY) failed");
 164                         return NGX_CHAIN_ERROR;
 165                     }
 166
 167                 } else {
 168                     c->tcp_nodelay = NGX_TCP_NODELAY_UNSET;
 169
 170                     ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0,
 171                                    "no tcp_nodelay");
 172                 }
 173             }
 174
 175             if (c->tcp_nodelay == NGX_TCP_NODELAY_UNSET) {
 176
 177                 if (ngx_tcp_nopush(c->fd) == NGX_ERROR) {
 178                     err = ngx_errno;
 179
 180                     /*
 181                      * there is a tiny chance to be interrupted, however,
 182                      * we continue a processing without the TCP_CORK
 183                      */
 184
 185                     if (err != NGX_EINTR) {
 186                         wev->error = 1;
 187                         ngx_connection_error(c, err,
 188                                              ngx_tcp_nopush_n " failed");
 189                         return NGX_CHAIN_ERROR;
 190                     }
 191
 192                 } else {
 193                     c->tcp_nopush = NGX_TCP_NOPUSH_SET;
 194
 195                     ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0,
 196                                    "tcp_nopush");
 197                 }
 198             }
 199         }
 200
 201         /* get the file buf */
 202
 203         if (header.nelts == 0 && cl && cl->buf->in_file && send < limit) {
 204             file = cl->buf;
 205
 206             /* coalesce the neighbouring file bufs */
 207
 208             do {
 209                 size = cl->buf->file_last - cl->buf->file_pos;
 210
 211                 if (send + size > limit) {
 212                     size = limit - send;
 213
 214                     aligned = (cl->buf->file_pos + size + ngx_pagesize - 1)
 215                                                       & ~(ngx_pagesize - 1);
 216
 217                     if (aligned <= cl->buf->file_last) {
 218                         size = aligned - cl->buf->file_pos;
 219                     }
 220                 }
 221
 222                 file_size += (size_t) size;
 223                 send += size;
 224                 fprev = cl->buf->file_pos + size;
 225                 cl = cl->next;
 226
 227             } while (cl
 228                      && cl->buf->in_file
 229                      && send < limit
 230                      && file->file->fd == cl->buf->file->fd
 231                      && fprev == cl->buf->file_pos);
 232         }
 233
 234         if (file) {
 235 #if (NGX_HAVE_SENDFILE64)
 236             offset = file->file_pos;
 237 #else
 238             offset = (int32_t) file->file_pos;
 239 #endif
 240             rc = sendfile(c->fd, file->file->fd, &offset, file_size);
 241
 242             if (rc == -1) {
 243                 err = ngx_errno;
 244
 245                 if (err == NGX_EAGAIN || err == NGX_EINTR) {
 246                     if (err == NGX_EINTR) {
 247                         eintr = 1;
 248                     }
 249
 250                     ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, err,
 251                                    "sendfile() is not ready");
 252
 253                 } else {
 254                     wev->error = 1;
 255                     ngx_connection_error(c, err, "sendfile() failed");
 256                     return NGX_CHAIN_ERROR;
 257                 }
 258             }
 259
 260             sent = rc > 0 ? rc : 0;
 261
 262             ngx_log_debug4(NGX_LOG_DEBUG_EVENT, c->log, 0,
 263                            "sendfile: %d, @%O %O:%uz",
 264                            rc, file->file_pos, sent, file_size);
 265
 266         } else {
 267             rc = writev(c->fd, header.elts, header.nelts);
 268
 269             if (rc == -1) {
 270                 err = ngx_errno;
 271
 272                 if (err == NGX_EAGAIN || err == NGX_EINTR) {
 273                     if (err == NGX_EINTR) {
 274                         eintr = 1;
 275                     }
 276
 277                     ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, err,
 278                                    "writev() not ready");
 279
 280                 } else {
 281                     wev->error = 1;
 282                     ngx_connection_error(c, err, "writev() failed");
 283                     return NGX_CHAIN_ERROR;
 284                 }
 285             }
 286
 287             sent = rc > 0 ? rc : 0;
 288
 289             ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, "writev: %O", sent);
 290         }
 291
 292         if (send - prev_send == sent) {
 293             complete = 1;
 294         }
 295
 296         c->sent += sent;
 297
 298         for (cl = in; cl; cl = cl->next) {
 299
 300             if (ngx_buf_special(cl->buf)) {
 301                 continue;
 302             }
 303
 304             if (sent == 0) {
 305                 break;
 306             }
 307
 308             size = ngx_buf_size(cl->buf);
 309
 310             if (sent >= size) {
 311                 sent -= size;
 312
 313                 if (ngx_buf_in_memory(cl->buf)) {
 314                     cl->buf->pos = cl->buf->last;
 315                 }
 316
 317                 if (cl->buf->in_file) {
 318                     cl->buf->file_pos = cl->buf->file_last;
 319                 }
 320
 321                 continue;
 322             }
 323
 324             if (ngx_buf_in_memory(cl->buf)) {
 325                 cl->buf->pos += (size_t) sent;
 326             }
 327
 328             if (cl->buf->in_file) {
 329                 cl->buf->file_pos += sent;
 330             }
 331
 332             break;
 333         }
 334
 335         if (eintr) {
 336             continue;
 337         }
 338
 339         if (!complete) {
 340             wev->ready = 0;
 341             return cl;
 342         }
 343
 344         if (send >= limit || cl == NULL) {
 345             return cl;
 346         }
 347
 348         in = cl;
 349     }
 350 }