release/src/router/nginx/src/os/unix/ngx_freebsd_sendfile_chain.c

   1
   2 /*
   3  * Copyright (C) Igor Sysoev
   4  * Copyright (C) Nginx, Inc.
   5  */
   6
   7
   8 #include <ngx_config.h>
   9 #include <ngx_core.h>
  10 #include <ngx_event.h>
  11
  12
  13 /*
  14  * Although FreeBSD sendfile() allows to pass a header and a trailer,
  15  * it cannot send a header with a part of the file in one packet until
  16  * FreeBSD 5.3.  Besides, over the fast ethernet connection sendfile()
  17  * may send the partially filled packets, i.e. the 8 file pages may be sent
  18  * as the 11 full 1460-bytes packets, then one incomplete 324-bytes packet,
  19  * and then again the 11 full 1460-bytes packets.
  20  *
  21  * Therefore we use the TCP_NOPUSH option (similar to Linux's TCP_CORK)
  22  * to postpone the sending - it not only sends a header and the first part of
  23  * the file in one packet, but also sends the file pages in the full packets.
  24  *
  25  * But until FreeBSD 4.5 turning TCP_NOPUSH off does not flush a pending
  26  * data that less than MSS, so that data may be sent with 5 second delay.
  27  * So we do not use TCP_NOPUSH on FreeBSD prior to 4.5, although it can be used
  28  * for non-keepalive HTTP connections.
  29  */
  30
  31
  32 #if (IOV_MAX > 64)
  33 #define NGX_HEADERS   64
  34 #define NGX_TRAILERS  64
  35 #else
  36 #define NGX_HEADERS   IOV_MAX
  37 #define NGX_TRAILERS  IOV_MAX
  38 #endif
  39
  40
  41 ngx_chain_t *
  42 ngx_freebsd_sendfile_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit)
  43 {
  44     int              rc, flags;
  45     u_char          *prev;
  46     off_t            size, send, prev_send, aligned, sent, fprev;
  47     size_t           header_size, file_size;
  48     ngx_uint_t       eintr, eagain, complete;
  49     ngx_err_t        err;
  50     ngx_buf_t       *file;
  51     ngx_array_t      header, trailer;
  52     ngx_event_t     *wev;
  53     ngx_chain_t     *cl;
  54     struct sf_hdtr   hdtr;
  55     struct iovec    *iov, headers[NGX_HEADERS], trailers[NGX_TRAILERS];
  56
  57     wev = c->write;
  58
  59     if (!wev->ready) {
  60         return in;
  61     }
  62
  63 #if (NGX_HAVE_KQUEUE)
  64
  65     if ((ngx_event_flags & NGX_USE_KQUEUE_EVENT) && wev->pending_eof) {
  66         (void) ngx_connection_error(c, wev->kq_errno,
  67                                "kevent() reported about an closed connection");
  68         wev->error = 1;
  69         return NGX_CHAIN_ERROR;
  70     }
  71
  72 #endif
  73
  74     /* the maximum limit size is the maximum size_t value - the page size */
  75
  76     if (limit == 0 || limit > (off_t) (NGX_MAX_SIZE_T_VALUE - ngx_pagesize)) {
  77         limit = NGX_MAX_SIZE_T_VALUE - ngx_pagesize;
  78     }
  79
  80     send = 0;
  81     eagain = 0;
  82     flags = 0;
  83
  84     header.elts = headers;
  85     header.size = sizeof(struct iovec);
  86     header.nalloc = NGX_HEADERS;
  87     header.pool = c->pool;
  88
  89     trailer.elts = trailers;
  90     trailer.size = sizeof(struct iovec);
  91     trailer.nalloc = NGX_TRAILERS;
  92     trailer.pool = c->pool;
  93
  94     for ( ;; ) {
  95         file = NULL;
  96         file_size = 0;
  97         header_size = 0;
  98         eintr = 0;
  99         complete = 0;
 100         prev_send = send;
 101
 102         header.nelts = 0;
 103         trailer.nelts = 0;
 104
 105         /* create the header iovec and coalesce the neighbouring bufs */
 106
 107         prev = NULL;
 108         iov = NULL;
 109
 110         for (cl = in; cl && send < limit; cl = cl->next) {
 111
 112             if (ngx_buf_special(cl->buf)) {
 113                 continue;
 114             }
 115
 116             if (!ngx_buf_in_memory_only(cl->buf)) {
 117                 break;
 118             }
 119
 120             size = cl->buf->last - cl->buf->pos;
 121
 122             if (send + size > limit) {
 123                 size = limit - send;
 124             }
 125
 126             if (prev == cl->buf->pos) {
 127                 iov->iov_len += (size_t) size;
 128
 129             } else {
 130                 if (header.nelts >= IOV_MAX){
 131                     break;
 132                 }
 133
 134                 iov = ngx_array_push(&header);
 135                 if (iov == NULL) {
 136                     return NGX_CHAIN_ERROR;
 137                 }
 138
 139                 iov->iov_base = (void *) cl->buf->pos;
 140                 iov->iov_len = (size_t) size;
 141             }
 142
 143             prev = cl->buf->pos + (size_t) size;
 144             header_size += (size_t) size;
 145             send += size;
 146         }
 147
 148
 149         if (cl && cl->buf->in_file && send < limit) {
 150             file = cl->buf;
 151
 152             /* coalesce the neighbouring file bufs */
 153
 154             do {
 155                 size = cl->buf->file_last - cl->buf->file_pos;
 156
 157                 if (send + size > limit) {
 158                     size = limit - send;
 159
 160                     aligned = (cl->buf->file_pos + size + ngx_pagesize - 1)
 161                                & ~((off_t) ngx_pagesize - 1);
 162
 163                     if (aligned <= cl->buf->file_last) {
 164                         size = aligned - cl->buf->file_pos;
 165                     }
 166                 }
 167
 168                 file_size += (size_t) size;
 169                 send += size;
 170                 fprev = cl->buf->file_pos + size;
 171                 cl = cl->next;
 172
 173             } while (cl
 174                      && cl->buf->in_file
 175                      && send < limit
 176                      && file->file->fd == cl->buf->file->fd
 177                      && fprev == cl->buf->file_pos);
 178         }
 179
 180
 181         if (file) {
 182
 183             /* create the trailer iovec and coalesce the neighbouring bufs */
 184
 185             prev = NULL;
 186             iov = NULL;
 187
 188             while (cl && send < limit) {
 189
 190                 if (ngx_buf_special(cl->buf)) {
 191                     cl = cl->next;
 192                     continue;
 193                 }
 194
 195                 if (!ngx_buf_in_memory_only(cl->buf)) {
 196                     break;
 197                 }
 198
 199                 size = cl->buf->last - cl->buf->pos;
 200
 201                 if (send + size > limit) {
 202                     size = limit - send;
 203                 }
 204
 205                 if (prev == cl->buf->pos) {
 206                     iov->iov_len += (size_t) size;
 207
 208                 } else {
 209                     if (trailer.nelts >= IOV_MAX){
 210                         break;
 211                     }
 212
 213                     iov = ngx_array_push(&trailer);
 214                     if (iov == NULL) {
 215                         return NGX_CHAIN_ERROR;
 216                     }
 217
 218                     iov->iov_base = (void *) cl->buf->pos;
 219                     iov->iov_len = (size_t) size;
 220                 }
 221
 222                 prev = cl->buf->pos + (size_t) size;
 223                 send += size;
 224                 cl = cl->next;
 225             }
 226         }
 227
 228         if (file) {
 229
 230             if (ngx_freebsd_use_tcp_nopush
 231                 && c->tcp_nopush == NGX_TCP_NOPUSH_UNSET)
 232             {
 233                 if (ngx_tcp_nopush(c->fd) == NGX_ERROR) {
 234                     err = ngx_errno;
 235
 236                     /*
 237                      * there is a tiny chance to be interrupted, however,
 238                      * we continue a processing without the TCP_NOPUSH
 239                      */
 240
 241                     if (err != NGX_EINTR) {
 242                         wev->error = 1;
 243                         (void) ngx_connection_error(c, err,
 244                                                     ngx_tcp_nopush_n " failed");
 245                         return NGX_CHAIN_ERROR;
 246                     }
 247
 248                 } else {
 249                     c->tcp_nopush = NGX_TCP_NOPUSH_SET;
 250
 251                     ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0,
 252                                    "tcp_nopush");
 253                 }
 254             }
 255
 256             /*
 257              * sendfile() does unneeded work if sf_hdtr's count is 0,
 258              * but corresponding pointer is not NULL
 259              */
 260
 261             hdtr.headers = header.nelts ? (struct iovec *) header.elts: NULL;
 262             hdtr.hdr_cnt = header.nelts;
 263             hdtr.trailers = trailer.nelts ? (struct iovec *) trailer.elts: NULL;
 264             hdtr.trl_cnt = trailer.nelts;
 265
 266             /*
 267              * the "nbytes bug" of the old sendfile() syscall:
 268              * http://www.freebsd.org/cgi/query-pr.cgi?pr=33771
 269              */
 270
 271             if (!ngx_freebsd_sendfile_nbytes_bug) {
 272                 header_size = 0;
 273             }
 274
 275             sent = 0;
 276
 277 #if (NGX_HAVE_AIO_SENDFILE)
 278             flags = c->aio_sendfile ? SF_NODISKIO : 0;
 279 #endif
 280
 281             rc = sendfile(file->file->fd, c->fd, file->file_pos,
 282                           file_size + header_size, &hdtr, &sent, flags);
 283
 284             if (rc == -1) {
 285                 err = ngx_errno;
 286
 287                 switch (err) {
 288                 case NGX_EAGAIN:
 289                     eagain = 1;
 290                     break;
 291
 292                 case NGX_EINTR:
 293                     eintr = 1;
 294                     break;
 295
 296 #if (NGX_HAVE_AIO_SENDFILE)
 297                 case NGX_EBUSY:
 298                     c->busy_sendfile = file;
 299                     break;
 300 #endif
 301
 302                 default:
 303                     wev->error = 1;
 304                     (void) ngx_connection_error(c, err, "sendfile() failed");
 305                     return NGX_CHAIN_ERROR;
 306                 }
 307
 308                 ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, err,
 309                                "sendfile() sent only %O bytes", sent);
 310
 311             /*
 312              * sendfile() in FreeBSD 3.x-4.x may return value >= 0
 313              * on success, although only 0 is documented
 314              */
 315
 316             } else if (rc >= 0 && sent == 0) {
 317
 318                 /*
 319                  * if rc is OK and sent equal to zero, then someone
 320                  * has truncated the file, so the offset became beyond
 321                  * the end of the file
 322                  */
 323
 324                 ngx_log_error(NGX_LOG_ALERT, c->log, 0,
 325                          "sendfile() reported that \"%s\" was truncated at %O",
 326                          file->file->name.data, file->file_pos);
 327
 328                 return NGX_CHAIN_ERROR;
 329             }
 330
 331             ngx_log_debug4(NGX_LOG_DEBUG_EVENT, c->log, 0,
 332                            "sendfile: %d, @%O %O:%uz",
 333                            rc, file->file_pos, sent, file_size + header_size);
 334
 335         } else {
 336             rc = writev(c->fd, header.elts, header.nelts);
 337
 338             ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0,
 339                            "writev: %d of %uz", rc, header_size);
 340
 341             if (rc == -1) {
 342                 err = ngx_errno;
 343
 344                 switch (err) {
 345                 case NGX_EAGAIN:
 346                     break;
 347
 348                 case NGX_EINTR:
 349                     eintr = 1;
 350                     break;
 351
 352                 default:
 353                     wev->error = 1;
 354                     ngx_connection_error(c, err, "writev() failed");
 355                     return NGX_CHAIN_ERROR;
 356                 }
 357
 358                 ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, err,
 359                                "writev() not ready");
 360             }
 361
 362             sent = rc > 0 ? rc : 0;
 363         }
 364
 365         if (send - prev_send == sent) {
 366             complete = 1;
 367         }
 368
 369         c->sent += sent;
 370
 371         for (cl = in; cl; cl = cl->next) {
 372
 373             if (ngx_buf_special(cl->buf)) {
 374                 continue;
 375             }
 376
 377             if (sent == 0) {
 378                 break;
 379             }
 380
 381             size = ngx_buf_size(cl->buf);
 382
 383             if (sent >= size) {
 384                 sent -= size;
 385
 386                 if (ngx_buf_in_memory(cl->buf)) {
 387                     cl->buf->pos = cl->buf->last;
 388                 }
 389
 390                 if (cl->buf->in_file) {
 391                     cl->buf->file_pos = cl->buf->file_last;
 392                 }
 393
 394                 continue;
 395             }
 396
 397             if (ngx_buf_in_memory(cl->buf)) {
 398                 cl->buf->pos += (size_t) sent;
 399             }
 400
 401             if (cl->buf->in_file) {
 402                 cl->buf->file_pos += sent;
 403             }
 404
 405             break;
 406         }
 407
 408 #if (NGX_HAVE_AIO_SENDFILE)
 409         if (c->busy_sendfile) {
 410             return cl;
 411         }
 412 #endif
 413
 414         if (eagain) {
 415
 416             /*
 417              * sendfile() may return EAGAIN, even if it has sent a whole file
 418              * part, it indicates that the successive sendfile() call would
 419              * return EAGAIN right away and would not send anything.
 420              * We use it as a hint.
 421              */
 422
 423             wev->ready = 0;
 424             return cl;
 425         }
 426
 427         if (eintr) {
 428             continue;
 429         }
 430
 431         if (!complete) {
 432             wev->ready = 0;
 433             return cl;
 434         }
 435
 436         if (send >= limit || cl == NULL) {
 437             return cl;
 438         }
 439
 440         in = cl;
 441     }
 442 }