2 * Copyright (c) 2002-2007 Niels Provos <provos@citi.umich.edu>
3 * Copyright (c) 2007-2012 Niels Provos and Nick Mathewson
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 * 3. The name of the author may not be used to endorse or promote products
14 * derived from this software without specific prior written permission.
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 #include "event2/event-config.h"
36 #ifdef _EVENT_HAVE_VASPRINTF
37 /* If we have vasprintf, we need to define this before we include stdio.h. */
43 #include <sys/types.h>
45 #ifdef _EVENT_HAVE_SYS_TIME_H
49 #ifdef _EVENT_HAVE_SYS_SOCKET_H
50 #include <sys/socket.h>
53 #ifdef _EVENT_HAVE_SYS_UIO_H
57 #ifdef _EVENT_HAVE_SYS_IOCTL_H
58 #include <sys/ioctl.h>
61 #ifdef _EVENT_HAVE_SYS_MMAN_H
65 #ifdef _EVENT_HAVE_SYS_SENDFILE_H
66 #include <sys/sendfile.h>
73 #ifdef _EVENT_HAVE_STDARG_H
76 #ifdef _EVENT_HAVE_UNISTD_H
81 #include "event2/event.h"
82 #include "event2/buffer.h"
83 #include "event2/buffer_compat.h"
84 #include "event2/bufferevent.h"
85 #include "event2/bufferevent_compat.h"
86 #include "event2/bufferevent_struct.h"
87 #include "event2/thread.h"
88 #include "event2/event-config.h"
89 #include "log-internal.h"
90 #include "mm-internal.h"
91 #include "util-internal.h"
92 #include "evthread-internal.h"
93 #include "evbuffer-internal.h"
94 #include "bufferevent-internal.h"
96 /* some systems do not have MAP_FAILED */
98 #define MAP_FAILED ((void *)-1)
101 /* send file support */
102 #if defined(_EVENT_HAVE_SYS_SENDFILE_H) && defined(_EVENT_HAVE_SENDFILE) && defined(__linux__)
103 #define USE_SENDFILE 1
104 #define SENDFILE_IS_LINUX 1
105 #elif defined(_EVENT_HAVE_SENDFILE) && defined(__FreeBSD__)
106 #define USE_SENDFILE 1
107 #define SENDFILE_IS_FREEBSD 1
108 #elif defined(_EVENT_HAVE_SENDFILE) && defined(__APPLE__)
109 #define USE_SENDFILE 1
110 #define SENDFILE_IS_MACOSX 1
111 #elif defined(_EVENT_HAVE_SENDFILE) && defined(__sun__) && defined(__svr4__)
112 #define USE_SENDFILE 1
113 #define SENDFILE_IS_SOLARIS 1
117 static int use_sendfile
= 1;
119 #ifdef _EVENT_HAVE_MMAP
120 static int use_mmap
= 1;
124 /* Mask of user-selectable callback flags. */
125 #define EVBUFFER_CB_USER_FLAGS 0xffff
126 /* Mask of all internal-use-only flags. */
127 #define EVBUFFER_CB_INTERNAL_FLAGS 0xffff0000
129 /* Flag set if the callback is using the cb_obsolete function pointer */
130 #define EVBUFFER_CB_OBSOLETE 0x00040000
132 /* evbuffer_chain support */
133 #define CHAIN_SPACE_PTR(ch) ((ch)->buffer + (ch)->misalign + (ch)->off)
134 #define CHAIN_SPACE_LEN(ch) ((ch)->flags & EVBUFFER_IMMUTABLE ? \
135 0 : (ch)->buffer_len - ((ch)->misalign + (ch)->off))
137 #define CHAIN_PINNED(ch) (((ch)->flags & EVBUFFER_MEM_PINNED_ANY) != 0)
138 #define CHAIN_PINNED_R(ch) (((ch)->flags & EVBUFFER_MEM_PINNED_R) != 0)
140 static void evbuffer_chain_align(struct evbuffer_chain
*chain
);
141 static int evbuffer_chain_should_realign(struct evbuffer_chain
*chain
,
143 static void evbuffer_deferred_callback(struct deferred_cb
*cb
, void *arg
);
144 static int evbuffer_ptr_memcmp(const struct evbuffer
*buf
,
145 const struct evbuffer_ptr
*pos
, const char *mem
, size_t len
);
146 static struct evbuffer_chain
*evbuffer_expand_singlechain(struct evbuffer
*buf
,
150 static int evbuffer_readfile(struct evbuffer
*buf
, evutil_socket_t fd
,
153 #define evbuffer_readfile evbuffer_read
156 static struct evbuffer_chain
*
157 evbuffer_chain_new(size_t size
)
159 struct evbuffer_chain
*chain
;
162 size
+= EVBUFFER_CHAIN_SIZE
;
164 /* get the next largest memory that can hold the buffer */
165 to_alloc
= MIN_BUFFER_SIZE
;
166 while (to_alloc
< size
)
169 /* we get everything in one chunk */
170 if ((chain
= mm_malloc(to_alloc
)) == NULL
)
173 memset(chain
, 0, EVBUFFER_CHAIN_SIZE
);
175 chain
->buffer_len
= to_alloc
- EVBUFFER_CHAIN_SIZE
;
177 /* this way we can manipulate the buffer to different addresses,
178 * which is required for mmap for example.
180 chain
->buffer
= EVBUFFER_CHAIN_EXTRA(u_char
, chain
);
186 evbuffer_chain_free(struct evbuffer_chain
*chain
)
188 if (CHAIN_PINNED(chain
)) {
189 chain
->flags
|= EVBUFFER_DANGLING
;
192 if (chain
->flags
& (EVBUFFER_MMAP
|EVBUFFER_SENDFILE
|
193 EVBUFFER_REFERENCE
)) {
194 if (chain
->flags
& EVBUFFER_REFERENCE
) {
195 struct evbuffer_chain_reference
*info
=
196 EVBUFFER_CHAIN_EXTRA(
197 struct evbuffer_chain_reference
,
200 (*info
->cleanupfn
)(chain
->buffer
,
204 #ifdef _EVENT_HAVE_MMAP
205 if (chain
->flags
& EVBUFFER_MMAP
) {
206 struct evbuffer_chain_fd
*info
=
207 EVBUFFER_CHAIN_EXTRA(struct evbuffer_chain_fd
,
209 if (munmap(chain
->buffer
, chain
->buffer_len
) == -1)
210 event_warn("%s: munmap failed", __func__
);
211 if (close(info
->fd
) == -1)
212 event_warn("%s: close(%d) failed",
217 if (chain
->flags
& EVBUFFER_SENDFILE
) {
218 struct evbuffer_chain_fd
*info
=
219 EVBUFFER_CHAIN_EXTRA(struct evbuffer_chain_fd
,
221 if (close(info
->fd
) == -1)
222 event_warn("%s: close(%d) failed",
232 evbuffer_free_all_chains(struct evbuffer_chain
*chain
)
234 struct evbuffer_chain
*next
;
235 for (; chain
; chain
= next
) {
237 evbuffer_chain_free(chain
);
243 evbuffer_chains_all_empty(struct evbuffer_chain
*chain
)
245 for (; chain
; chain
= chain
->next
) {
252 /* The definition is needed for EVUTIL_ASSERT, which uses sizeof to avoid
253 "unused variable" warnings. */
254 static inline int evbuffer_chains_all_empty(struct evbuffer_chain
*chain
) {
259 /* Free all trailing chains in 'buf' that are neither pinned nor empty, prior
260 * to replacing them all with a new chain. Return a pointer to the place
261 * where the new chain will go.
263 * Internal; requires lock. The caller must fix up buf->last and buf->first
264 * as needed; they might have been freed.
266 static struct evbuffer_chain
**
267 evbuffer_free_trailing_empty_chains(struct evbuffer
*buf
)
269 struct evbuffer_chain
**ch
= buf
->last_with_datap
;
270 /* Find the first victim chain. It might be *last_with_datap */
271 while ((*ch
) && ((*ch
)->off
!= 0 || CHAIN_PINNED(*ch
)))
274 EVUTIL_ASSERT(evbuffer_chains_all_empty(*ch
));
275 evbuffer_free_all_chains(*ch
);
281 /* Add a single chain 'chain' to the end of 'buf', freeing trailing empty
282 * chains as necessary. Requires lock. Does not schedule callbacks.
285 evbuffer_chain_insert(struct evbuffer
*buf
,
286 struct evbuffer_chain
*chain
)
288 ASSERT_EVBUFFER_LOCKED(buf
);
289 if (*buf
->last_with_datap
== NULL
) {
290 /* There are no chains data on the buffer at all. */
291 EVUTIL_ASSERT(buf
->last_with_datap
== &buf
->first
);
292 EVUTIL_ASSERT(buf
->first
== NULL
);
293 buf
->first
= buf
->last
= chain
;
295 struct evbuffer_chain
**ch
= buf
->last_with_datap
;
296 /* Find the first victim chain. It might be *last_with_datap */
297 while ((*ch
) && ((*ch
)->off
!= 0 || CHAIN_PINNED(*ch
)))
300 /* There is no victim; just append this new chain. */
301 buf
->last
->next
= chain
;
303 buf
->last_with_datap
= &buf
->last
->next
;
305 /* Replace all victim chains with this chain. */
306 EVUTIL_ASSERT(evbuffer_chains_all_empty(*ch
));
307 evbuffer_free_all_chains(*ch
);
312 buf
->total_len
+= chain
->off
;
315 static inline struct evbuffer_chain
*
316 evbuffer_chain_insert_new(struct evbuffer
*buf
, size_t datlen
)
318 struct evbuffer_chain
*chain
;
319 if ((chain
= evbuffer_chain_new(datlen
)) == NULL
)
321 evbuffer_chain_insert(buf
, chain
);
326 _evbuffer_chain_pin(struct evbuffer_chain
*chain
, unsigned flag
)
328 EVUTIL_ASSERT((chain
->flags
& flag
) == 0);
329 chain
->flags
|= flag
;
333 _evbuffer_chain_unpin(struct evbuffer_chain
*chain
, unsigned flag
)
335 EVUTIL_ASSERT((chain
->flags
& flag
) != 0);
336 chain
->flags
&= ~flag
;
337 if (chain
->flags
& EVBUFFER_DANGLING
)
338 evbuffer_chain_free(chain
);
344 struct evbuffer
*buffer
;
346 buffer
= mm_calloc(1, sizeof(struct evbuffer
));
350 TAILQ_INIT(&buffer
->callbacks
);
352 buffer
->last_with_datap
= &buffer
->first
;
358 evbuffer_set_flags(struct evbuffer
*buf
, ev_uint64_t flags
)
361 buf
->flags
|= (ev_uint32_t
)flags
;
362 EVBUFFER_UNLOCK(buf
);
367 evbuffer_clear_flags(struct evbuffer
*buf
, ev_uint64_t flags
)
370 buf
->flags
&= ~(ev_uint32_t
)flags
;
371 EVBUFFER_UNLOCK(buf
);
376 _evbuffer_incref(struct evbuffer
*buf
)
380 EVBUFFER_UNLOCK(buf
);
384 _evbuffer_incref_and_lock(struct evbuffer
*buf
)
391 evbuffer_defer_callbacks(struct evbuffer
*buffer
, struct event_base
*base
)
393 EVBUFFER_LOCK(buffer
);
394 buffer
->cb_queue
= event_base_get_deferred_cb_queue(base
);
395 buffer
->deferred_cbs
= 1;
396 event_deferred_cb_init(&buffer
->deferred
,
397 evbuffer_deferred_callback
, buffer
);
398 EVBUFFER_UNLOCK(buffer
);
403 evbuffer_enable_locking(struct evbuffer
*buf
, void *lock
)
405 #ifdef _EVENT_DISABLE_THREAD_SUPPORT
412 EVTHREAD_ALLOC_LOCK(lock
, EVTHREAD_LOCKTYPE_RECURSIVE
);
427 evbuffer_set_parent(struct evbuffer
*buf
, struct bufferevent
*bev
)
431 EVBUFFER_UNLOCK(buf
);
435 evbuffer_run_callbacks(struct evbuffer
*buffer
, int running_deferred
)
437 struct evbuffer_cb_entry
*cbent
, *next
;
438 struct evbuffer_cb_info info
;
440 ev_uint32_t mask
, masked_val
;
443 if (running_deferred
) {
444 mask
= EVBUFFER_CB_NODEFER
|EVBUFFER_CB_ENABLED
;
445 masked_val
= EVBUFFER_CB_ENABLED
;
446 } else if (buffer
->deferred_cbs
) {
447 mask
= EVBUFFER_CB_NODEFER
|EVBUFFER_CB_ENABLED
;
448 masked_val
= EVBUFFER_CB_NODEFER
|EVBUFFER_CB_ENABLED
;
449 /* Don't zero-out n_add/n_del, since the deferred callbacks
450 will want to see them. */
453 mask
= EVBUFFER_CB_ENABLED
;
454 masked_val
= EVBUFFER_CB_ENABLED
;
457 ASSERT_EVBUFFER_LOCKED(buffer
);
459 if (TAILQ_EMPTY(&buffer
->callbacks
)) {
460 buffer
->n_add_for_cb
= buffer
->n_del_for_cb
= 0;
463 if (buffer
->n_add_for_cb
== 0 && buffer
->n_del_for_cb
== 0)
466 new_size
= buffer
->total_len
;
467 info
.orig_size
= new_size
+ buffer
->n_del_for_cb
- buffer
->n_add_for_cb
;
468 info
.n_added
= buffer
->n_add_for_cb
;
469 info
.n_deleted
= buffer
->n_del_for_cb
;
471 buffer
->n_add_for_cb
= 0;
472 buffer
->n_del_for_cb
= 0;
474 for (cbent
= TAILQ_FIRST(&buffer
->callbacks
);
475 cbent
!= TAILQ_END(&buffer
->callbacks
);
477 /* Get the 'next' pointer now in case this callback decides
478 * to remove itself or something. */
479 next
= TAILQ_NEXT(cbent
, next
);
481 if ((cbent
->flags
& mask
) != masked_val
)
484 if ((cbent
->flags
& EVBUFFER_CB_OBSOLETE
))
485 cbent
->cb
.cb_obsolete(buffer
,
486 info
.orig_size
, new_size
, cbent
->cbarg
);
488 cbent
->cb
.cb_func(buffer
, &info
, cbent
->cbarg
);
493 evbuffer_invoke_callbacks(struct evbuffer
*buffer
)
495 if (TAILQ_EMPTY(&buffer
->callbacks
)) {
496 buffer
->n_add_for_cb
= buffer
->n_del_for_cb
= 0;
500 if (buffer
->deferred_cbs
) {
501 if (buffer
->deferred
.queued
)
503 _evbuffer_incref_and_lock(buffer
);
505 bufferevent_incref(buffer
->parent
);
506 EVBUFFER_UNLOCK(buffer
);
507 event_deferred_cb_schedule(buffer
->cb_queue
, &buffer
->deferred
);
510 evbuffer_run_callbacks(buffer
, 0);
514 evbuffer_deferred_callback(struct deferred_cb
*cb
, void *arg
)
516 struct bufferevent
*parent
= NULL
;
517 struct evbuffer
*buffer
= arg
;
519 /* XXXX It would be better to run these callbacks without holding the
521 EVBUFFER_LOCK(buffer
);
522 parent
= buffer
->parent
;
523 evbuffer_run_callbacks(buffer
, 1);
524 _evbuffer_decref_and_unlock(buffer
);
526 bufferevent_decref(parent
);
530 evbuffer_remove_all_callbacks(struct evbuffer
*buffer
)
532 struct evbuffer_cb_entry
*cbent
;
534 while ((cbent
= TAILQ_FIRST(&buffer
->callbacks
))) {
535 TAILQ_REMOVE(&buffer
->callbacks
, cbent
, next
);
541 _evbuffer_decref_and_unlock(struct evbuffer
*buffer
)
543 struct evbuffer_chain
*chain
, *next
;
544 ASSERT_EVBUFFER_LOCKED(buffer
);
546 EVUTIL_ASSERT(buffer
->refcnt
> 0);
548 if (--buffer
->refcnt
> 0) {
549 EVBUFFER_UNLOCK(buffer
);
553 for (chain
= buffer
->first
; chain
!= NULL
; chain
= next
) {
555 evbuffer_chain_free(chain
);
557 evbuffer_remove_all_callbacks(buffer
);
558 if (buffer
->deferred_cbs
)
559 event_deferred_cb_cancel(buffer
->cb_queue
, &buffer
->deferred
);
561 EVBUFFER_UNLOCK(buffer
);
562 if (buffer
->own_lock
)
563 EVTHREAD_FREE_LOCK(buffer
->lock
, EVTHREAD_LOCKTYPE_RECURSIVE
);
568 evbuffer_free(struct evbuffer
*buffer
)
570 EVBUFFER_LOCK(buffer
);
571 _evbuffer_decref_and_unlock(buffer
);
575 evbuffer_lock(struct evbuffer
*buf
)
581 evbuffer_unlock(struct evbuffer
*buf
)
583 EVBUFFER_UNLOCK(buf
);
587 evbuffer_get_length(const struct evbuffer
*buffer
)
591 EVBUFFER_LOCK(buffer
);
593 result
= (buffer
->total_len
);
595 EVBUFFER_UNLOCK(buffer
);
601 evbuffer_get_contiguous_space(const struct evbuffer
*buf
)
603 struct evbuffer_chain
*chain
;
608 result
= (chain
!= NULL
? chain
->off
: 0);
609 EVBUFFER_UNLOCK(buf
);
615 evbuffer_reserve_space(struct evbuffer
*buf
, ev_ssize_t size
,
616 struct evbuffer_iovec
*vec
, int n_vecs
)
618 struct evbuffer_chain
*chain
, **chainp
;
627 if ((chain
= evbuffer_expand_singlechain(buf
, size
)) == NULL
)
630 vec
[0].iov_base
= CHAIN_SPACE_PTR(chain
);
631 vec
[0].iov_len
= (size_t) CHAIN_SPACE_LEN(chain
);
632 EVUTIL_ASSERT(size
<0 || (size_t)vec
[0].iov_len
>= (size_t)size
);
635 if (_evbuffer_expand_fast(buf
, size
, n_vecs
)<0)
637 n
= _evbuffer_read_setup_vecs(buf
, size
, vec
, n_vecs
,
642 EVBUFFER_UNLOCK(buf
);
648 advance_last_with_data(struct evbuffer
*buf
)
651 ASSERT_EVBUFFER_LOCKED(buf
);
653 if (!*buf
->last_with_datap
)
656 while ((*buf
->last_with_datap
)->next
&& (*buf
->last_with_datap
)->next
->off
) {
657 buf
->last_with_datap
= &(*buf
->last_with_datap
)->next
;
664 evbuffer_commit_space(struct evbuffer
*buf
,
665 struct evbuffer_iovec
*vec
, int n_vecs
)
667 struct evbuffer_chain
*chain
, **firstchainp
, **chainp
;
679 } else if (n_vecs
== 1 &&
680 (buf
->last
&& vec
[0].iov_base
== (void*)CHAIN_SPACE_PTR(buf
->last
))) {
681 /* The user only got or used one chain; it might not
682 * be the first one with space in it. */
683 if ((size_t)vec
[0].iov_len
> (size_t)CHAIN_SPACE_LEN(buf
->last
))
685 buf
->last
->off
+= vec
[0].iov_len
;
686 added
= vec
[0].iov_len
;
688 advance_last_with_data(buf
);
692 /* Advance 'firstchain' to the first chain with space in it. */
693 firstchainp
= buf
->last_with_datap
;
696 if (CHAIN_SPACE_LEN(*firstchainp
) == 0) {
697 firstchainp
= &(*firstchainp
)->next
;
700 chain
= *firstchainp
;
701 /* pass 1: make sure that the pointers and lengths of vecs[] are in
702 * bounds before we try to commit anything. */
703 for (i
=0; i
<n_vecs
; ++i
) {
706 if (vec
[i
].iov_base
!= (void*)CHAIN_SPACE_PTR(chain
) ||
707 (size_t)vec
[i
].iov_len
> CHAIN_SPACE_LEN(chain
))
711 /* pass 2: actually adjust all the chains. */
712 chainp
= firstchainp
;
713 for (i
=0; i
<n_vecs
; ++i
) {
714 (*chainp
)->off
+= vec
[i
].iov_len
;
715 added
+= vec
[i
].iov_len
;
716 if (vec
[i
].iov_len
) {
717 buf
->last_with_datap
= chainp
;
719 chainp
= &(*chainp
)->next
;
723 buf
->total_len
+= added
;
724 buf
->n_add_for_cb
+= added
;
726 evbuffer_invoke_callbacks(buf
);
729 EVBUFFER_UNLOCK(buf
);
734 HAS_PINNED_R(struct evbuffer
*buf
)
736 return (buf
->last
&& CHAIN_PINNED_R(buf
->last
));
740 ZERO_CHAIN(struct evbuffer
*dst
)
742 ASSERT_EVBUFFER_LOCKED(dst
);
745 dst
->last_with_datap
= &(dst
)->first
;
749 /* Prepares the contents of src to be moved to another buffer by removing
750 * read-pinned chains. The first pinned chain is saved in first, and the
751 * last in last. If src has no read-pinned chains, first and last are set
754 PRESERVE_PINNED(struct evbuffer
*src
, struct evbuffer_chain
**first
,
755 struct evbuffer_chain
**last
)
757 struct evbuffer_chain
*chain
, **pinned
;
759 ASSERT_EVBUFFER_LOCKED(src
);
761 if (!HAS_PINNED_R(src
)) {
762 *first
= *last
= NULL
;
766 pinned
= src
->last_with_datap
;
767 if (!CHAIN_PINNED_R(*pinned
))
768 pinned
= &(*pinned
)->next
;
769 EVUTIL_ASSERT(CHAIN_PINNED_R(*pinned
));
770 chain
= *first
= *pinned
;
773 /* If there's data in the first pinned chain, we need to allocate
774 * a new chain and copy the data over. */
776 struct evbuffer_chain
*tmp
;
778 EVUTIL_ASSERT(pinned
== src
->last_with_datap
);
779 tmp
= evbuffer_chain_new(chain
->off
);
782 memcpy(tmp
->buffer
, chain
->buffer
+ chain
->misalign
,
784 tmp
->off
= chain
->off
;
785 *src
->last_with_datap
= tmp
;
787 chain
->misalign
+= chain
->off
;
790 src
->last
= *src
->last_with_datap
;
798 RESTORE_PINNED(struct evbuffer
*src
, struct evbuffer_chain
*pinned
,
799 struct evbuffer_chain
*last
)
801 ASSERT_EVBUFFER_LOCKED(src
);
810 src
->last_with_datap
= &src
->first
;
815 COPY_CHAIN(struct evbuffer
*dst
, struct evbuffer
*src
)
817 ASSERT_EVBUFFER_LOCKED(dst
);
818 ASSERT_EVBUFFER_LOCKED(src
);
819 dst
->first
= src
->first
;
820 if (src
->last_with_datap
== &src
->first
)
821 dst
->last_with_datap
= &dst
->first
;
823 dst
->last_with_datap
= src
->last_with_datap
;
824 dst
->last
= src
->last
;
825 dst
->total_len
= src
->total_len
;
829 APPEND_CHAIN(struct evbuffer
*dst
, struct evbuffer
*src
)
831 ASSERT_EVBUFFER_LOCKED(dst
);
832 ASSERT_EVBUFFER_LOCKED(src
);
833 dst
->last
->next
= src
->first
;
834 if (src
->last_with_datap
== &src
->first
)
835 dst
->last_with_datap
= &dst
->last
->next
;
837 dst
->last_with_datap
= src
->last_with_datap
;
838 dst
->last
= src
->last
;
839 dst
->total_len
+= src
->total_len
;
843 PREPEND_CHAIN(struct evbuffer
*dst
, struct evbuffer
*src
)
845 ASSERT_EVBUFFER_LOCKED(dst
);
846 ASSERT_EVBUFFER_LOCKED(src
);
847 src
->last
->next
= dst
->first
;
848 dst
->first
= src
->first
;
849 dst
->total_len
+= src
->total_len
;
850 if (*dst
->last_with_datap
== NULL
) {
851 if (src
->last_with_datap
== &(src
)->first
)
852 dst
->last_with_datap
= &dst
->first
;
854 dst
->last_with_datap
= src
->last_with_datap
;
855 } else if (dst
->last_with_datap
== &dst
->first
) {
856 dst
->last_with_datap
= &src
->last
->next
;
861 evbuffer_add_buffer(struct evbuffer
*outbuf
, struct evbuffer
*inbuf
)
863 struct evbuffer_chain
*pinned
, *last
;
864 size_t in_total_len
, out_total_len
;
867 EVBUFFER_LOCK2(inbuf
, outbuf
);
868 in_total_len
= inbuf
->total_len
;
869 out_total_len
= outbuf
->total_len
;
871 if (in_total_len
== 0 || outbuf
== inbuf
)
874 if (outbuf
->freeze_end
|| inbuf
->freeze_start
) {
879 if (PRESERVE_PINNED(inbuf
, &pinned
, &last
) < 0) {
884 if (out_total_len
== 0) {
885 /* There might be an empty chain at the start of outbuf; free
887 evbuffer_free_all_chains(outbuf
->first
);
888 COPY_CHAIN(outbuf
, inbuf
);
890 APPEND_CHAIN(outbuf
, inbuf
);
893 RESTORE_PINNED(inbuf
, pinned
, last
);
895 inbuf
->n_del_for_cb
+= in_total_len
;
896 outbuf
->n_add_for_cb
+= in_total_len
;
898 evbuffer_invoke_callbacks(inbuf
);
899 evbuffer_invoke_callbacks(outbuf
);
902 EVBUFFER_UNLOCK2(inbuf
, outbuf
);
907 evbuffer_prepend_buffer(struct evbuffer
*outbuf
, struct evbuffer
*inbuf
)
909 struct evbuffer_chain
*pinned
, *last
;
910 size_t in_total_len
, out_total_len
;
913 EVBUFFER_LOCK2(inbuf
, outbuf
);
915 in_total_len
= inbuf
->total_len
;
916 out_total_len
= outbuf
->total_len
;
918 if (!in_total_len
|| inbuf
== outbuf
)
921 if (outbuf
->freeze_start
|| inbuf
->freeze_start
) {
926 if (PRESERVE_PINNED(inbuf
, &pinned
, &last
) < 0) {
931 if (out_total_len
== 0) {
932 /* There might be an empty chain at the start of outbuf; free
934 evbuffer_free_all_chains(outbuf
->first
);
935 COPY_CHAIN(outbuf
, inbuf
);
937 PREPEND_CHAIN(outbuf
, inbuf
);
940 RESTORE_PINNED(inbuf
, pinned
, last
);
942 inbuf
->n_del_for_cb
+= in_total_len
;
943 outbuf
->n_add_for_cb
+= in_total_len
;
945 evbuffer_invoke_callbacks(inbuf
);
946 evbuffer_invoke_callbacks(outbuf
);
948 EVBUFFER_UNLOCK2(inbuf
, outbuf
);
953 evbuffer_drain(struct evbuffer
*buf
, size_t len
)
955 struct evbuffer_chain
*chain
, *next
;
956 size_t remaining
, old_len
;
960 old_len
= buf
->total_len
;
965 if (buf
->freeze_start
) {
970 if (len
>= old_len
&& !HAS_PINNED_R(buf
)) {
972 for (chain
= buf
->first
; chain
!= NULL
; chain
= next
) {
974 evbuffer_chain_free(chain
);
982 buf
->total_len
-= len
;
984 for (chain
= buf
->first
;
985 remaining
>= chain
->off
;
988 remaining
-= chain
->off
;
990 if (chain
== *buf
->last_with_datap
) {
991 buf
->last_with_datap
= &buf
->first
;
993 if (&chain
->next
== buf
->last_with_datap
)
994 buf
->last_with_datap
= &buf
->first
;
996 if (CHAIN_PINNED_R(chain
)) {
997 EVUTIL_ASSERT(remaining
== 0);
998 chain
->misalign
+= chain
->off
;
1002 evbuffer_chain_free(chain
);
1007 chain
->misalign
+= remaining
;
1008 chain
->off
-= remaining
;
1012 buf
->n_del_for_cb
+= len
;
1013 /* Tell someone about changes in this buffer */
1014 evbuffer_invoke_callbacks(buf
);
1017 EVBUFFER_UNLOCK(buf
);
1021 /* Reads data from an event buffer and drains the bytes read */
1023 evbuffer_remove(struct evbuffer
*buf
, void *data_out
, size_t datlen
)
1027 n
= evbuffer_copyout(buf
, data_out
, datlen
);
1029 if (evbuffer_drain(buf
, n
)<0)
1032 EVBUFFER_UNLOCK(buf
);
1037 evbuffer_copyout(struct evbuffer
*buf
, void *data_out
, size_t datlen
)
1039 /*XXX fails badly on sendfile case. */
1040 struct evbuffer_chain
*chain
;
1041 char *data
= data_out
;
1043 ev_ssize_t result
= 0;
1049 if (datlen
>= buf
->total_len
)
1050 datlen
= buf
->total_len
;
1055 if (buf
->freeze_start
) {
1062 while (datlen
&& datlen
>= chain
->off
) {
1063 memcpy(data
, chain
->buffer
+ chain
->misalign
, chain
->off
);
1065 datlen
-= chain
->off
;
1067 chain
= chain
->next
;
1068 EVUTIL_ASSERT(chain
|| datlen
==0);
1072 EVUTIL_ASSERT(chain
);
1073 memcpy(data
, chain
->buffer
+ chain
->misalign
, datlen
);
1078 EVBUFFER_UNLOCK(buf
);
1082 /* reads data from the src buffer to the dst buffer, avoids memcpy as
1084 /* XXXX should return ev_ssize_t */
1086 evbuffer_remove_buffer(struct evbuffer
*src
, struct evbuffer
*dst
,
1089 /*XXX We should have an option to force this to be zero-copy.*/
1091 /*XXX can fail badly on sendfile case. */
1092 struct evbuffer_chain
*chain
, *previous
;
1096 EVBUFFER_LOCK2(src
, dst
);
1098 chain
= previous
= src
->first
;
1100 if (datlen
== 0 || dst
== src
) {
1105 if (dst
->freeze_end
|| src
->freeze_start
) {
1110 /* short-cut if there is no more data buffered */
1111 if (datlen
>= src
->total_len
) {
1112 datlen
= src
->total_len
;
1113 evbuffer_add_buffer(dst
, src
);
1114 result
= (int)datlen
; /*XXXX should return ev_ssize_t*/
1118 /* removes chains if possible */
1119 while (chain
->off
<= datlen
) {
1120 /* We can't remove the last with data from src unless we
1121 * remove all chains, in which case we would have done the if
1123 EVUTIL_ASSERT(chain
!= *src
->last_with_datap
);
1124 nread
+= chain
->off
;
1125 datlen
-= chain
->off
;
1127 if (src
->last_with_datap
== &chain
->next
)
1128 src
->last_with_datap
= &src
->first
;
1129 chain
= chain
->next
;
1133 /* we can remove the chain */
1134 struct evbuffer_chain
**chp
;
1135 chp
= evbuffer_free_trailing_empty_chains(dst
);
1137 if (dst
->first
== NULL
) {
1138 dst
->first
= src
->first
;
1142 dst
->last
= previous
;
1143 previous
->next
= NULL
;
1145 advance_last_with_data(dst
);
1147 dst
->total_len
+= nread
;
1148 dst
->n_add_for_cb
+= nread
;
1151 /* we know that there is more data in the src buffer than
1152 * we want to read, so we manually drain the chain */
1153 evbuffer_add(dst
, chain
->buffer
+ chain
->misalign
, datlen
);
1154 chain
->misalign
+= datlen
;
1155 chain
->off
-= datlen
;
1158 /* You might think we would want to increment dst->n_add_for_cb
1159 * here too. But evbuffer_add above already took care of that.
1161 src
->total_len
-= nread
;
1162 src
->n_del_for_cb
+= nread
;
1165 evbuffer_invoke_callbacks(dst
);
1166 evbuffer_invoke_callbacks(src
);
1168 result
= (int)nread
;/*XXXX should change return type */
1171 EVBUFFER_UNLOCK2(src
, dst
);
1176 evbuffer_pullup(struct evbuffer
*buf
, ev_ssize_t size
)
1178 struct evbuffer_chain
*chain
, *next
, *tmp
, *last_with_data
;
1179 unsigned char *buffer
, *result
= NULL
;
1180 ev_ssize_t remaining
;
1181 int removed_last_with_data
= 0;
1182 int removed_last_with_datap
= 0;
1189 size
= buf
->total_len
;
1190 /* if size > buf->total_len, we cannot guarantee to the user that she
1191 * is going to have a long enough buffer afterwards; so we return
1193 if (size
== 0 || (size_t)size
> buf
->total_len
)
1196 /* No need to pull up anything; the first size bytes are
1198 if (chain
->off
>= (size_t)size
) {
1199 result
= chain
->buffer
+ chain
->misalign
;
1203 /* Make sure that none of the chains we need to copy from is pinned. */
1204 remaining
= size
- chain
->off
;
1205 EVUTIL_ASSERT(remaining
>= 0);
1206 for (tmp
=chain
->next
; tmp
; tmp
=tmp
->next
) {
1207 if (CHAIN_PINNED(tmp
))
1209 if (tmp
->off
>= (size_t)remaining
)
1211 remaining
-= tmp
->off
;
1214 if (CHAIN_PINNED(chain
)) {
1215 size_t old_off
= chain
->off
;
1216 if (CHAIN_SPACE_LEN(chain
) < size
- chain
->off
) {
1217 /* not enough room at end of chunk. */
1220 buffer
= CHAIN_SPACE_PTR(chain
);
1224 chain
= chain
->next
;
1225 } else if (chain
->buffer_len
- chain
->misalign
>= (size_t)size
) {
1226 /* already have enough space in the first chain */
1227 size_t old_off
= chain
->off
;
1228 buffer
= chain
->buffer
+ chain
->misalign
+ chain
->off
;
1232 chain
= chain
->next
;
1234 if ((tmp
= evbuffer_chain_new(size
)) == NULL
) {
1235 event_warn("%s: out of memory", __func__
);
1238 buffer
= tmp
->buffer
;
1243 /* TODO(niels): deal with buffers that point to NULL like sendfile */
1245 /* Copy and free every chunk that will be entirely pulled into tmp */
1246 last_with_data
= *buf
->last_with_datap
;
1247 for (; chain
!= NULL
&& (size_t)size
>= chain
->off
; chain
= next
) {
1250 memcpy(buffer
, chain
->buffer
+ chain
->misalign
, chain
->off
);
1252 buffer
+= chain
->off
;
1253 if (chain
== last_with_data
)
1254 removed_last_with_data
= 1;
1255 if (&chain
->next
== buf
->last_with_datap
)
1256 removed_last_with_datap
= 1;
1258 evbuffer_chain_free(chain
);
1261 if (chain
!= NULL
) {
1262 memcpy(buffer
, chain
->buffer
+ chain
->misalign
, size
);
1263 chain
->misalign
+= size
;
1271 if (removed_last_with_data
) {
1272 buf
->last_with_datap
= &buf
->first
;
1273 } else if (removed_last_with_datap
) {
1274 if (buf
->first
->next
&& buf
->first
->next
->off
)
1275 buf
->last_with_datap
= &buf
->first
->next
;
1277 buf
->last_with_datap
= &buf
->first
;
1280 result
= (tmp
->buffer
+ tmp
->misalign
);
1283 EVBUFFER_UNLOCK(buf
);
1288 * Reads a line terminated by either '\r\n', '\n\r' or '\r' or '\n'.
1289 * The returned buffer needs to be freed by the called.
1292 evbuffer_readline(struct evbuffer
*buffer
)
1294 return evbuffer_readln(buffer
, NULL
, EVBUFFER_EOL_ANY
);
1297 static inline ev_ssize_t
1298 evbuffer_strchr(struct evbuffer_ptr
*it
, const char chr
)
1300 struct evbuffer_chain
*chain
= it
->_internal
.chain
;
1301 size_t i
= it
->_internal
.pos_in_chain
;
1302 while (chain
!= NULL
) {
1303 char *buffer
= (char *)chain
->buffer
+ chain
->misalign
;
1304 char *cp
= memchr(buffer
+i
, chr
, chain
->off
-i
);
1306 it
->_internal
.chain
= chain
;
1307 it
->_internal
.pos_in_chain
= cp
- buffer
;
1308 it
->pos
+= (cp
- buffer
- i
);
1311 it
->pos
+= chain
->off
- i
;
1313 chain
= chain
->next
;
1319 static inline char *
1320 find_eol_char(char *s
, size_t len
)
1322 #define CHUNK_SZ 128
1323 /* Lots of benchmarking found this approach to be faster in practice
1324 * than doing two memchrs over the whole buffer, doin a memchr on each
1325 * char of the buffer, or trying to emulate memchr by hand. */
1326 char *s_end
, *cr
, *lf
;
1329 size_t chunk
= (s
+ CHUNK_SZ
< s_end
) ? CHUNK_SZ
: (s_end
- s
);
1330 cr
= memchr(s
, '\r', chunk
);
1331 lf
= memchr(s
, '\n', chunk
);
1347 evbuffer_find_eol_char(struct evbuffer_ptr
*it
)
1349 struct evbuffer_chain
*chain
= it
->_internal
.chain
;
1350 size_t i
= it
->_internal
.pos_in_chain
;
1351 while (chain
!= NULL
) {
1352 char *buffer
= (char *)chain
->buffer
+ chain
->misalign
;
1353 char *cp
= find_eol_char(buffer
+i
, chain
->off
-i
);
1355 it
->_internal
.chain
= chain
;
1356 it
->_internal
.pos_in_chain
= cp
- buffer
;
1357 it
->pos
+= (cp
- buffer
) - i
;
1360 it
->pos
+= chain
->off
- i
;
1362 chain
= chain
->next
;
1370 struct evbuffer_ptr
*ptr
, const char *chrset
)
1373 struct evbuffer_chain
*chain
= ptr
->_internal
.chain
;
1374 size_t i
= ptr
->_internal
.pos_in_chain
;
1380 char *buffer
= (char *)chain
->buffer
+ chain
->misalign
;
1381 for (; i
< chain
->off
; ++i
) {
1382 const char *p
= chrset
;
1384 if (buffer
[i
] == *p
++)
1387 ptr
->_internal
.chain
= chain
;
1388 ptr
->_internal
.pos_in_chain
= i
;
1396 if (! chain
->next
) {
1397 ptr
->_internal
.chain
= chain
;
1398 ptr
->_internal
.pos_in_chain
= i
;
1403 chain
= chain
->next
;
1409 evbuffer_getchr(struct evbuffer_ptr
*it
)
1411 struct evbuffer_chain
*chain
= it
->_internal
.chain
;
1412 size_t off
= it
->_internal
.pos_in_chain
;
1414 return chain
->buffer
[chain
->misalign
+ off
];
1418 evbuffer_search_eol(struct evbuffer
*buffer
,
1419 struct evbuffer_ptr
*start
, size_t *eol_len_out
,
1420 enum evbuffer_eol_style eol_style
)
1422 struct evbuffer_ptr it
, it2
;
1423 size_t extra_drain
= 0;
1426 EVBUFFER_LOCK(buffer
);
1429 memcpy(&it
, start
, sizeof(it
));
1432 it
._internal
.chain
= buffer
->first
;
1433 it
._internal
.pos_in_chain
= 0;
1436 /* the eol_style determines our first stop character and how many
1437 * characters we are going to drain afterwards. */
1438 switch (eol_style
) {
1439 case EVBUFFER_EOL_ANY
:
1440 if (evbuffer_find_eol_char(&it
) < 0)
1442 memcpy(&it2
, &it
, sizeof(it
));
1443 extra_drain
= evbuffer_strspn(&it2
, "\r\n");
1445 case EVBUFFER_EOL_CRLF_STRICT
: {
1446 it
= evbuffer_search(buffer
, "\r\n", 2, &it
);
1452 case EVBUFFER_EOL_CRLF
:
1454 if (evbuffer_find_eol_char(&it
) < 0)
1456 if (evbuffer_getchr(&it
) == '\n') {
1459 } else if (!evbuffer_ptr_memcmp(
1460 buffer
, &it
, "\r\n", 2)) {
1464 if (evbuffer_ptr_set(buffer
, &it
, 1,
1465 EVBUFFER_PTR_ADD
)<0)
1470 case EVBUFFER_EOL_LF
:
1471 if (evbuffer_strchr(&it
, '\n') < 0)
1481 EVBUFFER_UNLOCK(buffer
);
1487 *eol_len_out
= extra_drain
;
1493 evbuffer_readln(struct evbuffer
*buffer
, size_t *n_read_out
,
1494 enum evbuffer_eol_style eol_style
)
1496 struct evbuffer_ptr it
;
1498 size_t n_to_copy
=0, extra_drain
=0;
1499 char *result
= NULL
;
1501 EVBUFFER_LOCK(buffer
);
1503 if (buffer
->freeze_start
) {
1507 it
= evbuffer_search_eol(buffer
, NULL
, &extra_drain
, eol_style
);
1512 if ((line
= mm_malloc(n_to_copy
+1)) == NULL
) {
1513 event_warn("%s: out of memory", __func__
);
1517 evbuffer_remove(buffer
, line
, n_to_copy
);
1518 line
[n_to_copy
] = '\0';
1520 evbuffer_drain(buffer
, extra_drain
);
1523 EVBUFFER_UNLOCK(buffer
);
1526 *n_read_out
= result
? n_to_copy
: 0;
1531 #define EVBUFFER_CHAIN_MAX_AUTO_SIZE 4096
1533 /* Adds data to an event buffer */
1536 evbuffer_add(struct evbuffer
*buf
, const void *data_in
, size_t datlen
)
1538 struct evbuffer_chain
*chain
, *tmp
;
1539 const unsigned char *data
= data_in
;
1540 size_t remain
, to_alloc
;
1545 if (buf
->freeze_end
) {
1551 /* If there are no chains allocated for this buffer, allocate one
1552 * big enough to hold all the data. */
1553 if (chain
== NULL
) {
1554 chain
= evbuffer_chain_new(datlen
);
1557 evbuffer_chain_insert(buf
, chain
);
1560 if ((chain
->flags
& EVBUFFER_IMMUTABLE
) == 0) {
1561 remain
= (size_t)(chain
->buffer_len
- chain
->misalign
- chain
->off
);
1562 if (remain
>= datlen
) {
1563 /* there's enough space to hold all the data in the
1564 * current last chain */
1565 memcpy(chain
->buffer
+ chain
->misalign
+ chain
->off
,
1567 chain
->off
+= datlen
;
1568 buf
->total_len
+= datlen
;
1569 buf
->n_add_for_cb
+= datlen
;
1571 } else if (!CHAIN_PINNED(chain
) &&
1572 evbuffer_chain_should_realign(chain
, datlen
)) {
1573 /* we can fit the data into the misalignment */
1574 evbuffer_chain_align(chain
);
1576 memcpy(chain
->buffer
+ chain
->off
, data
, datlen
);
1577 chain
->off
+= datlen
;
1578 buf
->total_len
+= datlen
;
1579 buf
->n_add_for_cb
+= datlen
;
1583 /* we cannot write any data to the last chain */
1587 /* we need to add another chain */
1588 to_alloc
= chain
->buffer_len
;
1589 if (to_alloc
<= EVBUFFER_CHAIN_MAX_AUTO_SIZE
/2)
1591 if (datlen
> to_alloc
)
1593 tmp
= evbuffer_chain_new(to_alloc
);
1598 memcpy(chain
->buffer
+ chain
->misalign
+ chain
->off
,
1600 chain
->off
+= remain
;
1601 buf
->total_len
+= remain
;
1602 buf
->n_add_for_cb
+= remain
;
1608 memcpy(tmp
->buffer
, data
, datlen
);
1610 evbuffer_chain_insert(buf
, tmp
);
1611 buf
->n_add_for_cb
+= datlen
;
1614 evbuffer_invoke_callbacks(buf
);
1617 EVBUFFER_UNLOCK(buf
);
1622 evbuffer_prepend(struct evbuffer
*buf
, const void *data
, size_t datlen
)
1624 struct evbuffer_chain
*chain
, *tmp
;
1629 if (buf
->freeze_start
) {
1635 if (chain
== NULL
) {
1636 chain
= evbuffer_chain_new(datlen
);
1639 evbuffer_chain_insert(buf
, chain
);
1642 /* we cannot touch immutable buffers */
1643 if ((chain
->flags
& EVBUFFER_IMMUTABLE
) == 0) {
1644 /* If this chain is empty, we can treat it as
1645 * 'empty at the beginning' rather than 'empty at the end' */
1646 if (chain
->off
== 0)
1647 chain
->misalign
= chain
->buffer_len
;
1649 if ((size_t)chain
->misalign
>= datlen
) {
1650 /* we have enough space to fit everything */
1651 memcpy(chain
->buffer
+ chain
->misalign
- datlen
,
1653 chain
->off
+= datlen
;
1654 chain
->misalign
-= datlen
;
1655 buf
->total_len
+= datlen
;
1656 buf
->n_add_for_cb
+= datlen
;
1658 } else if (chain
->misalign
) {
1659 /* we can only fit some of the data. */
1660 memcpy(chain
->buffer
,
1661 (char*)data
+ datlen
- chain
->misalign
,
1662 (size_t)chain
->misalign
);
1663 chain
->off
+= (size_t)chain
->misalign
;
1664 buf
->total_len
+= (size_t)chain
->misalign
;
1665 buf
->n_add_for_cb
+= (size_t)chain
->misalign
;
1666 datlen
-= (size_t)chain
->misalign
;
1667 chain
->misalign
= 0;
1671 /* we need to add another chain */
1672 if ((tmp
= evbuffer_chain_new(datlen
)) == NULL
)
1675 if (buf
->last_with_datap
== &buf
->first
)
1676 buf
->last_with_datap
= &tmp
->next
;
1681 tmp
->misalign
= tmp
->buffer_len
- datlen
;
1683 memcpy(tmp
->buffer
+ tmp
->misalign
, data
, datlen
);
1684 buf
->total_len
+= datlen
;
1685 buf
->n_add_for_cb
+= (size_t)chain
->misalign
;
1688 evbuffer_invoke_callbacks(buf
);
1691 EVBUFFER_UNLOCK(buf
);
1695 /** Helper: realigns the memory in chain->buffer so that misalign is 0. */
1697 evbuffer_chain_align(struct evbuffer_chain
*chain
)
1699 EVUTIL_ASSERT(!(chain
->flags
& EVBUFFER_IMMUTABLE
));
1700 EVUTIL_ASSERT(!(chain
->flags
& EVBUFFER_MEM_PINNED_ANY
));
1701 memmove(chain
->buffer
, chain
->buffer
+ chain
->misalign
, chain
->off
);
1702 chain
->misalign
= 0;
1705 #define MAX_TO_COPY_IN_EXPAND 4096
1706 #define MAX_TO_REALIGN_IN_EXPAND 2048
1708 /** Helper: return true iff we should realign chain to fit datalen bytes of
1711 evbuffer_chain_should_realign(struct evbuffer_chain
*chain
,
1714 return chain
->buffer_len
- chain
->off
>= datlen
&&
1715 (chain
->off
< chain
->buffer_len
/ 2) &&
1716 (chain
->off
<= MAX_TO_REALIGN_IN_EXPAND
);
1719 /* Expands the available space in the event buffer to at least datlen, all in
1720 * a single chunk. Return that chunk. */
1721 static struct evbuffer_chain
*
1722 evbuffer_expand_singlechain(struct evbuffer
*buf
, size_t datlen
)
1724 struct evbuffer_chain
*chain
, **chainp
;
1725 struct evbuffer_chain
*result
= NULL
;
1726 ASSERT_EVBUFFER_LOCKED(buf
);
1728 chainp
= buf
->last_with_datap
;
1730 /* XXX If *chainp is no longer writeable, but has enough space in its
1731 * misalign, this might be a bad idea: we could still use *chainp, not
1732 * (*chainp)->next. */
1733 if (*chainp
&& CHAIN_SPACE_LEN(*chainp
) == 0)
1734 chainp
= &(*chainp
)->next
;
1736 /* 'chain' now points to the first chain with writable space (if any)
1737 * We will either use it, realign it, replace it, or resize it. */
1740 if (chain
== NULL
||
1741 (chain
->flags
& (EVBUFFER_IMMUTABLE
|EVBUFFER_MEM_PINNED_ANY
))) {
1742 /* We can't use the last_with_data chain at all. Just add a
1743 * new one that's big enough. */
1747 /* If we can fit all the data, then we don't have to do anything */
1748 if (CHAIN_SPACE_LEN(chain
) >= datlen
) {
1753 /* If the chain is completely empty, just replace it by adding a new
1755 if (chain
->off
== 0) {
1759 /* If the misalignment plus the remaining space fulfills our data
1760 * needs, we could just force an alignment to happen. Afterwards, we
1761 * have enough space. But only do this if we're saving a lot of space
1762 * and not moving too much data. Otherwise the space savings are
1763 * probably offset by the time lost in copying.
1765 if (evbuffer_chain_should_realign(chain
, datlen
)) {
1766 evbuffer_chain_align(chain
);
1771 /* At this point, we can either resize the last chunk with space in
1772 * it, use the next chunk after it, or If we add a new chunk, we waste
1773 * CHAIN_SPACE_LEN(chain) bytes in the former last chunk. If we
1774 * resize, we have to copy chain->off bytes.
1777 /* Would expanding this chunk be affordable and worthwhile? */
1778 if (CHAIN_SPACE_LEN(chain
) < chain
->buffer_len
/ 8 ||
1779 chain
->off
> MAX_TO_COPY_IN_EXPAND
) {
1780 /* It's not worth resizing this chain. Can the next one be
1782 if (chain
->next
&& CHAIN_SPACE_LEN(chain
->next
) >= datlen
) {
1783 /* Yes, we can just use the next chain (which should
1785 result
= chain
->next
;
1788 /* No; append a new chain (which will free all
1789 * terminal empty chains.) */
1793 /* Okay, we're going to try to resize this chain: Not doing so
1794 * would waste at least 1/8 of its current allocation, and we
1795 * can do so without having to copy more than
1796 * MAX_TO_COPY_IN_EXPAND bytes. */
1797 /* figure out how much space we need */
1798 size_t length
= chain
->off
+ datlen
;
1799 struct evbuffer_chain
*tmp
= evbuffer_chain_new(length
);
1803 /* copy the data over that we had so far */
1804 tmp
->off
= chain
->off
;
1805 memcpy(tmp
->buffer
, chain
->buffer
+ chain
->misalign
,
1807 /* fix up the list */
1808 EVUTIL_ASSERT(*chainp
== chain
);
1809 result
= *chainp
= tmp
;
1811 if (buf
->last
== chain
)
1814 tmp
->next
= chain
->next
;
1815 evbuffer_chain_free(chain
);
1820 result
= evbuffer_chain_insert_new(buf
, datlen
);
1824 EVUTIL_ASSERT(result
);
1825 EVUTIL_ASSERT(CHAIN_SPACE_LEN(result
) >= datlen
);
1830 /* Make sure that datlen bytes are available for writing in the last n
1831 * chains. Never copies or moves data. */
1833 _evbuffer_expand_fast(struct evbuffer
*buf
, size_t datlen
, int n
)
1835 struct evbuffer_chain
*chain
= buf
->last
, *tmp
, *next
;
1839 ASSERT_EVBUFFER_LOCKED(buf
);
1840 EVUTIL_ASSERT(n
>= 2);
1842 if (chain
== NULL
|| (chain
->flags
& EVBUFFER_IMMUTABLE
)) {
1843 /* There is no last chunk, or we can't touch the last chunk.
1844 * Just add a new chunk. */
1845 chain
= evbuffer_chain_new(datlen
);
1849 evbuffer_chain_insert(buf
, chain
);
1853 used
= 0; /* number of chains we're using space in. */
1854 avail
= 0; /* how much space they have. */
1855 /* How many bytes can we stick at the end of buffer as it is? Iterate
1856 * over the chains at the end of the buffer, tring to see how much
1857 * space we have in the first n. */
1858 for (chain
= *buf
->last_with_datap
; chain
; chain
= chain
->next
) {
1860 size_t space
= (size_t) CHAIN_SPACE_LEN(chain
);
1861 EVUTIL_ASSERT(chain
== *buf
->last_with_datap
);
1867 /* No data in chain; realign it. */
1868 chain
->misalign
= 0;
1869 avail
+= chain
->buffer_len
;
1872 if (avail
>= datlen
) {
1873 /* There is already enough space. Just return */
1880 /* There wasn't enough space in the first n chains with space in
1881 * them. Either add a new chain with enough space, or replace all
1882 * empty chains with one that has enough space, depending on n. */
1884 /* The loop ran off the end of the chains before it hit n
1885 * chains; we can add another. */
1886 EVUTIL_ASSERT(chain
== NULL
);
1888 tmp
= evbuffer_chain_new(datlen
- avail
);
1892 buf
->last
->next
= tmp
;
1894 /* (we would only set last_with_data if we added the first
1895 * chain. But if the buffer had no chains, we would have
1896 * just allocated a new chain earlier) */
1899 /* Nuke _all_ the empty chains. */
1900 int rmv_all
= 0; /* True iff we removed last_with_data. */
1901 chain
= *buf
->last_with_datap
;
1903 EVUTIL_ASSERT(chain
== buf
->first
);
1907 avail
= (size_t) CHAIN_SPACE_LEN(chain
);
1908 chain
= chain
->next
;
1912 for (; chain
; chain
= next
) {
1914 EVUTIL_ASSERT(chain
->off
== 0);
1915 evbuffer_chain_free(chain
);
1917 tmp
= evbuffer_chain_new(datlen
- avail
);
1922 buf
->last
= *buf
->last_with_datap
;
1923 (*buf
->last_with_datap
)->next
= NULL
;
1929 buf
->first
= buf
->last
= tmp
;
1930 buf
->last_with_datap
= &buf
->first
;
1932 (*buf
->last_with_datap
)->next
= tmp
;
1940 evbuffer_expand(struct evbuffer
*buf
, size_t datlen
)
1942 struct evbuffer_chain
*chain
;
1945 chain
= evbuffer_expand_singlechain(buf
, datlen
);
1946 EVBUFFER_UNLOCK(buf
);
1947 return chain
? 0 : -1;
1951 * Reads data from a file descriptor into a buffer.
1954 #if defined(_EVENT_HAVE_SYS_UIO_H) || defined(WIN32)
1955 #define USE_IOVEC_IMPL
1958 #ifdef USE_IOVEC_IMPL
1960 #ifdef _EVENT_HAVE_SYS_UIO_H
1961 /* number of iovec we use for writev, fragmentation is going to determine
1962 * how much we end up writing */
1964 #define DEFAULT_WRITE_IOVEC 128
1966 #if defined(UIO_MAXIOV) && UIO_MAXIOV < DEFAULT_WRITE_IOVEC
1967 #define NUM_WRITE_IOVEC UIO_MAXIOV
1968 #elif defined(IOV_MAX) && IOV_MAX < DEFAULT_WRITE_IOVEC
1969 #define NUM_WRITE_IOVEC IOV_MAX
1971 #define NUM_WRITE_IOVEC DEFAULT_WRITE_IOVEC
1974 #define IOV_TYPE struct iovec
1975 #define IOV_PTR_FIELD iov_base
1976 #define IOV_LEN_FIELD iov_len
1977 #define IOV_LEN_TYPE size_t
1979 #define NUM_WRITE_IOVEC 16
1980 #define IOV_TYPE WSABUF
1981 #define IOV_PTR_FIELD buf
1982 #define IOV_LEN_FIELD len
1983 #define IOV_LEN_TYPE unsigned long
1986 #define NUM_READ_IOVEC 4
1988 #define EVBUFFER_MAX_READ 4096
1990 /** Helper function to figure out which space to use for reading data into
1991 an evbuffer. Internal use only.
1993 @param buf The buffer to read into
1994 @param howmuch How much we want to read.
1995 @param vecs An array of two or more iovecs or WSABUFs.
1996 @param n_vecs_avail The length of vecs
1997 @param chainp A pointer to a variable to hold the first chain we're
1999 @param exact Boolean: if true, we do not provide more than 'howmuch'
2000 space in the vectors, even if more space is available.
2001 @return The number of buffers we're using.
2004 _evbuffer_read_setup_vecs(struct evbuffer
*buf
, ev_ssize_t howmuch
,
2005 struct evbuffer_iovec
*vecs
, int n_vecs_avail
,
2006 struct evbuffer_chain
***chainp
, int exact
)
2008 struct evbuffer_chain
*chain
;
2009 struct evbuffer_chain
**firstchainp
;
2012 ASSERT_EVBUFFER_LOCKED(buf
);
2018 /* Let firstchain be the first chain with any space on it */
2019 firstchainp
= buf
->last_with_datap
;
2020 if (CHAIN_SPACE_LEN(*firstchainp
) == 0) {
2021 firstchainp
= &(*firstchainp
)->next
;
2024 chain
= *firstchainp
;
2025 for (i
= 0; i
< n_vecs_avail
&& so_far
< (size_t)howmuch
; ++i
) {
2026 size_t avail
= (size_t) CHAIN_SPACE_LEN(chain
);
2027 if (avail
> (howmuch
- so_far
) && exact
)
2028 avail
= howmuch
- so_far
;
2029 vecs
[i
].iov_base
= CHAIN_SPACE_PTR(chain
);
2030 vecs
[i
].iov_len
= avail
;
2032 chain
= chain
->next
;
2035 *chainp
= firstchainp
;
2040 get_n_bytes_readable_on_socket(evutil_socket_t fd
)
2042 #if defined(FIONREAD) && defined(WIN32)
2043 unsigned long lng
= EVBUFFER_MAX_READ
;
2044 if (ioctlsocket(fd
, FIONREAD
, &lng
) < 0)
2047 #elif defined(FIONREAD)
2048 int n
= EVBUFFER_MAX_READ
;
2049 if (ioctl(fd
, FIONREAD
, &n
) < 0)
2053 return EVBUFFER_MAX_READ
;
2057 /* TODO(niels): should this function return ev_ssize_t and take ev_ssize_t
2060 evbuffer_read(struct evbuffer
*buf
, evutil_socket_t fd
, int howmuch
)
2062 struct evbuffer_chain
**chainp
;
2066 #ifdef USE_IOVEC_IMPL
2067 int nvecs
, i
, remaining
;
2069 struct evbuffer_chain
*chain
;
2075 if (buf
->freeze_end
) {
2080 n
= get_n_bytes_readable_on_socket(fd
);
2081 if (n
<= 0 || n
> EVBUFFER_MAX_READ
)
2082 n
= EVBUFFER_MAX_READ
;
2083 if (howmuch
< 0 || howmuch
> n
)
2086 #ifdef USE_IOVEC_IMPL
2087 /* Since we can use iovecs, we're willing to use the last
2088 * NUM_READ_IOVEC chains. */
2089 if (_evbuffer_expand_fast(buf
, howmuch
, NUM_READ_IOVEC
) == -1) {
2093 IOV_TYPE vecs
[NUM_READ_IOVEC
];
2094 #ifdef _EVBUFFER_IOVEC_IS_NATIVE
2095 nvecs
= _evbuffer_read_setup_vecs(buf
, howmuch
, vecs
,
2096 NUM_READ_IOVEC
, &chainp
, 1);
2098 /* We aren't using the native struct iovec. Therefore,
2100 struct evbuffer_iovec ev_vecs
[NUM_READ_IOVEC
];
2101 nvecs
= _evbuffer_read_setup_vecs(buf
, howmuch
, ev_vecs
, 2,
2104 for (i
=0; i
< nvecs
; ++i
)
2105 WSABUF_FROM_EVBUFFER_IOV(&vecs
[i
], &ev_vecs
[i
]);
2112 if (WSARecv(fd
, vecs
, nvecs
, &bytesRead
, &flags
, NULL
, NULL
)) {
2113 /* The read failed. It might be a close,
2114 * or it might be an error. */
2115 if (WSAGetLastError() == WSAECONNABORTED
)
2123 n
= readv(fd
, vecs
, nvecs
);
2127 #else /*!USE_IOVEC_IMPL*/
2128 /* If we don't have FIONREAD, we might waste some space here */
2129 /* XXX we _will_ waste some space here if there is any space left
2130 * over on buf->last. */
2131 if ((chain
= evbuffer_expand_singlechain(buf
, howmuch
)) == NULL
) {
2136 /* We can append new data at this point */
2137 p
= chain
->buffer
+ chain
->misalign
+ chain
->off
;
2140 n
= read(fd
, p
, howmuch
);
2142 n
= recv(fd
, p
, howmuch
, 0);
2144 #endif /* USE_IOVEC_IMPL */
2155 #ifdef USE_IOVEC_IMPL
2157 for (i
=0; i
< nvecs
; ++i
) {
2158 ev_ssize_t space
= (ev_ssize_t
) CHAIN_SPACE_LEN(*chainp
);
2159 if (space
< remaining
) {
2160 (*chainp
)->off
+= space
;
2161 remaining
-= (int)space
;
2163 (*chainp
)->off
+= remaining
;
2164 buf
->last_with_datap
= chainp
;
2167 chainp
= &(*chainp
)->next
;
2171 advance_last_with_data(buf
);
2173 buf
->total_len
+= n
;
2174 buf
->n_add_for_cb
+= n
;
2176 /* Tell someone about changes in this buffer */
2177 evbuffer_invoke_callbacks(buf
);
2180 EVBUFFER_UNLOCK(buf
);
2186 evbuffer_readfile(struct evbuffer
*buf
, evutil_socket_t fd
, ev_ssize_t howmuch
)
2190 struct evbuffer_iovec v
[2];
2194 if (buf
->freeze_end
) {
2203 /* XXX we _will_ waste some space here if there is any space left
2204 * over on buf->last. */
2205 nchains
= evbuffer_reserve_space(buf
, howmuch
, v
, 2);
2206 if (nchains
< 1 || nchains
> 2) {
2210 n
= read((int)fd
, v
[0].iov_base
, (unsigned int)v
[0].iov_len
);
2215 v
[0].iov_len
= (IOV_LEN_TYPE
) n
; /* XXXX another problem with big n.*/
2217 n
= read((int)fd
, v
[1].iov_base
, (unsigned int)v
[1].iov_len
);
2219 result
= (unsigned long) v
[0].iov_len
;
2220 evbuffer_commit_space(buf
, v
, 1);
2225 evbuffer_commit_space(buf
, v
, nchains
);
2229 EVBUFFER_UNLOCK(buf
);
2234 #ifdef USE_IOVEC_IMPL
2236 evbuffer_write_iovec(struct evbuffer
*buffer
, evutil_socket_t fd
,
2239 IOV_TYPE iov
[NUM_WRITE_IOVEC
];
2240 struct evbuffer_chain
*chain
= buffer
->first
;
2246 ASSERT_EVBUFFER_LOCKED(buffer
);
2247 /* XXX make this top out at some maximal data length? if the
2248 * buffer has (say) 1MB in it, split over 128 chains, there's
2249 * no way it all gets written in one go. */
2250 while (chain
!= NULL
&& i
< NUM_WRITE_IOVEC
&& howmuch
) {
2252 /* we cannot write the file info via writev */
2253 if (chain
->flags
& EVBUFFER_SENDFILE
)
2256 iov
[i
].IOV_PTR_FIELD
= (void *) (chain
->buffer
+ chain
->misalign
);
2257 if ((size_t)howmuch
>= chain
->off
) {
2258 /* XXXcould be problematic when windows supports mmap*/
2259 iov
[i
++].IOV_LEN_FIELD
= (IOV_LEN_TYPE
)chain
->off
;
2260 howmuch
-= chain
->off
;
2262 /* XXXcould be problematic when windows supports mmap*/
2263 iov
[i
++].IOV_LEN_FIELD
= (IOV_LEN_TYPE
)howmuch
;
2266 chain
= chain
->next
;
2273 if (WSASend(fd
, iov
, i
, &bytesSent
, 0, NULL
, NULL
))
2279 n
= writev(fd
, iov
, i
);
2287 evbuffer_write_sendfile(struct evbuffer
*buffer
, evutil_socket_t fd
,
2290 struct evbuffer_chain
*chain
= buffer
->first
;
2291 struct evbuffer_chain_fd
*info
=
2292 EVBUFFER_CHAIN_EXTRA(struct evbuffer_chain_fd
, chain
);
2293 #if defined(SENDFILE_IS_MACOSX) || defined(SENDFILE_IS_FREEBSD)
2295 off_t len
= chain
->off
;
2296 #elif defined(SENDFILE_IS_LINUX) || defined(SENDFILE_IS_SOLARIS)
2298 off_t offset
= chain
->misalign
;
2301 ASSERT_EVBUFFER_LOCKED(buffer
);
2303 #if defined(SENDFILE_IS_MACOSX)
2304 res
= sendfile(info
->fd
, fd
, chain
->misalign
, &len
, NULL
, 0);
2305 if (res
== -1 && !EVUTIL_ERR_RW_RETRIABLE(errno
))
2309 #elif defined(SENDFILE_IS_FREEBSD)
2310 res
= sendfile(info
->fd
, fd
, chain
->misalign
, chain
->off
, NULL
, &len
, 0);
2311 if (res
== -1 && !EVUTIL_ERR_RW_RETRIABLE(errno
))
2315 #elif defined(SENDFILE_IS_LINUX)
2316 /* TODO(niels): implement splice */
2317 res
= sendfile(fd
, info
->fd
, &offset
, chain
->off
);
2318 if (res
== -1 && EVUTIL_ERR_RW_RETRIABLE(errno
)) {
2319 /* if this is EAGAIN or EINTR return 0; otherwise, -1 */
2323 #elif defined(SENDFILE_IS_SOLARIS)
2325 const off_t offset_orig
= offset
;
2326 res
= sendfile(fd
, info
->fd
, &offset
, chain
->off
);
2327 if (res
== -1 && EVUTIL_ERR_RW_RETRIABLE(errno
)) {
2328 if (offset
- offset_orig
)
2329 return offset
- offset_orig
;
2330 /* if this is EAGAIN or EINTR and no bytes were
2331 * written, return 0 */
2341 evbuffer_write_atmost(struct evbuffer
*buffer
, evutil_socket_t fd
,
2346 EVBUFFER_LOCK(buffer
);
2348 if (buffer
->freeze_start
) {
2352 if (howmuch
< 0 || (size_t)howmuch
> buffer
->total_len
)
2353 howmuch
= buffer
->total_len
;
2357 struct evbuffer_chain
*chain
= buffer
->first
;
2358 if (chain
!= NULL
&& (chain
->flags
& EVBUFFER_SENDFILE
))
2359 n
= evbuffer_write_sendfile(buffer
, fd
, howmuch
);
2362 #ifdef USE_IOVEC_IMPL
2363 n
= evbuffer_write_iovec(buffer
, fd
, howmuch
);
2364 #elif defined(WIN32)
2365 /* XXX(nickm) Don't disable this code until we know if
2366 * the WSARecv code above works. */
2367 void *p
= evbuffer_pullup(buffer
, howmuch
);
2368 n
= send(fd
, p
, howmuch
, 0);
2370 void *p
= evbuffer_pullup(buffer
, howmuch
);
2371 n
= write(fd
, p
, howmuch
);
2379 evbuffer_drain(buffer
, n
);
2382 EVBUFFER_UNLOCK(buffer
);
2387 evbuffer_write(struct evbuffer
*buffer
, evutil_socket_t fd
)
2389 return evbuffer_write_atmost(buffer
, fd
, -1);
2393 evbuffer_find(struct evbuffer
*buffer
, const unsigned char *what
, size_t len
)
2395 unsigned char *search
;
2396 struct evbuffer_ptr ptr
;
2398 EVBUFFER_LOCK(buffer
);
2400 ptr
= evbuffer_search(buffer
, (const char *)what
, len
, NULL
);
2404 search
= evbuffer_pullup(buffer
, ptr
.pos
+ len
);
2408 EVBUFFER_UNLOCK(buffer
);
2413 evbuffer_ptr_set(struct evbuffer
*buf
, struct evbuffer_ptr
*pos
,
2414 size_t position
, enum evbuffer_ptr_how how
)
2416 size_t left
= position
;
2417 struct evbuffer_chain
*chain
= NULL
;
2422 case EVBUFFER_PTR_SET
:
2424 pos
->pos
= position
;
2427 case EVBUFFER_PTR_ADD
:
2428 /* this avoids iterating over all previous chains if
2429 we just want to advance the position */
2430 chain
= pos
->_internal
.chain
;
2431 pos
->pos
+= position
;
2432 position
= pos
->_internal
.pos_in_chain
;
2436 while (chain
&& position
+ left
>= chain
->off
) {
2437 left
-= chain
->off
- position
;
2438 chain
= chain
->next
;
2442 pos
->_internal
.chain
= chain
;
2443 pos
->_internal
.pos_in_chain
= position
+ left
;
2445 pos
->_internal
.chain
= NULL
;
2449 EVBUFFER_UNLOCK(buf
);
2451 return chain
!= NULL
? 0 : -1;
2455 Compare the bytes in buf at position pos to the len bytes in mem. Return
2456 less than 0, 0, or greater than 0 as memcmp.
2459 evbuffer_ptr_memcmp(const struct evbuffer
*buf
, const struct evbuffer_ptr
*pos
,
2460 const char *mem
, size_t len
)
2462 struct evbuffer_chain
*chain
;
2466 ASSERT_EVBUFFER_LOCKED(buf
);
2468 if (pos
->pos
+ len
> buf
->total_len
)
2471 chain
= pos
->_internal
.chain
;
2472 position
= pos
->_internal
.pos_in_chain
;
2473 while (len
&& chain
) {
2474 size_t n_comparable
;
2475 if (len
+ position
> chain
->off
)
2476 n_comparable
= chain
->off
- position
;
2479 r
= memcmp(chain
->buffer
+ chain
->misalign
+ position
, mem
,
2483 mem
+= n_comparable
;
2484 len
-= n_comparable
;
2486 chain
= chain
->next
;
2493 evbuffer_search(struct evbuffer
*buffer
, const char *what
, size_t len
, const struct evbuffer_ptr
*start
)
2495 return evbuffer_search_range(buffer
, what
, len
, start
, NULL
);
2499 evbuffer_search_range(struct evbuffer
*buffer
, const char *what
, size_t len
, const struct evbuffer_ptr
*start
, const struct evbuffer_ptr
*end
)
2501 struct evbuffer_ptr pos
;
2502 struct evbuffer_chain
*chain
, *last_chain
= NULL
;
2503 const unsigned char *p
;
2506 EVBUFFER_LOCK(buffer
);
2509 memcpy(&pos
, start
, sizeof(pos
));
2510 chain
= pos
._internal
.chain
;
2513 chain
= pos
._internal
.chain
= buffer
->first
;
2514 pos
._internal
.pos_in_chain
= 0;
2518 last_chain
= end
->_internal
.chain
;
2520 if (!len
|| len
> EV_SSIZE_MAX
)
2526 const unsigned char *start_at
=
2527 chain
->buffer
+ chain
->misalign
+
2528 pos
._internal
.pos_in_chain
;
2529 p
= memchr(start_at
, first
,
2530 chain
->off
- pos
._internal
.pos_in_chain
);
2532 pos
.pos
+= p
- start_at
;
2533 pos
._internal
.pos_in_chain
+= p
- start_at
;
2534 if (!evbuffer_ptr_memcmp(buffer
, &pos
, what
, len
)) {
2535 if (end
&& pos
.pos
+ (ev_ssize_t
)len
> end
->pos
)
2541 ++pos
._internal
.pos_in_chain
;
2542 if (pos
._internal
.pos_in_chain
== chain
->off
) {
2543 chain
= pos
._internal
.chain
= chain
->next
;
2544 pos
._internal
.pos_in_chain
= 0;
2547 if (chain
== last_chain
)
2549 pos
.pos
+= chain
->off
- pos
._internal
.pos_in_chain
;
2550 chain
= pos
._internal
.chain
= chain
->next
;
2551 pos
._internal
.pos_in_chain
= 0;
2557 pos
._internal
.chain
= NULL
;
2559 EVBUFFER_UNLOCK(buffer
);
2564 evbuffer_peek(struct evbuffer
*buffer
, ev_ssize_t len
,
2565 struct evbuffer_ptr
*start_at
,
2566 struct evbuffer_iovec
*vec
, int n_vec
)
2568 struct evbuffer_chain
*chain
;
2570 ev_ssize_t len_so_far
= 0;
2572 EVBUFFER_LOCK(buffer
);
2575 chain
= start_at
->_internal
.chain
;
2576 len_so_far
= chain
->off
2577 - start_at
->_internal
.pos_in_chain
;
2580 vec
[0].iov_base
= chain
->buffer
+ chain
->misalign
2581 + start_at
->_internal
.pos_in_chain
;
2582 vec
[0].iov_len
= len_so_far
;
2584 chain
= chain
->next
;
2586 chain
= buffer
->first
;
2589 if (n_vec
== 0 && len
< 0) {
2590 /* If no vectors are provided and they asked for "everything",
2591 * pretend they asked for the actual available amount. */
2592 len
= buffer
->total_len
- len_so_far
;
2596 if (len
>= 0 && len_so_far
>= len
)
2599 vec
[idx
].iov_base
= chain
->buffer
+ chain
->misalign
;
2600 vec
[idx
].iov_len
= chain
->off
;
2605 len_so_far
+= chain
->off
;
2606 chain
= chain
->next
;
2609 EVBUFFER_UNLOCK(buffer
);
2616 evbuffer_add_vprintf(struct evbuffer
*buf
, const char *fmt
, va_list ap
)
2620 int sz
, result
= -1;
2622 struct evbuffer_chain
*chain
;
2627 if (buf
->freeze_end
) {
2631 /* make sure that at least some space is available */
2632 if ((chain
= evbuffer_expand_singlechain(buf
, 64)) == NULL
)
2637 size_t used
= chain
->misalign
+ chain
->off
;
2638 buffer
= (char *)chain
->buffer
+ chain
->misalign
+ chain
->off
;
2639 EVUTIL_ASSERT(chain
->buffer_len
>= used
);
2640 space
= chain
->buffer_len
- used
;
2642 buffer
= (char*) CHAIN_SPACE_PTR(chain
);
2643 space
= (size_t) CHAIN_SPACE_LEN(chain
);
2646 #define va_copy(dst, src) memcpy(&(dst), &(src), sizeof(va_list))
2650 sz
= evutil_vsnprintf(buffer
, space
, fmt
, aq
);
2656 if ((size_t)sz
< space
) {
2658 buf
->total_len
+= sz
;
2659 buf
->n_add_for_cb
+= sz
;
2661 advance_last_with_data(buf
);
2662 evbuffer_invoke_callbacks(buf
);
2666 if ((chain
= evbuffer_expand_singlechain(buf
, sz
+ 1)) == NULL
)
2672 EVBUFFER_UNLOCK(buf
);
2677 evbuffer_add_printf(struct evbuffer
*buf
, const char *fmt
, ...)
2683 res
= evbuffer_add_vprintf(buf
, fmt
, ap
);
2690 evbuffer_add_reference(struct evbuffer
*outbuf
,
2691 const void *data
, size_t datlen
,
2692 evbuffer_ref_cleanup_cb cleanupfn
, void *extra
)
2694 struct evbuffer_chain
*chain
;
2695 struct evbuffer_chain_reference
*info
;
2698 chain
= evbuffer_chain_new(sizeof(struct evbuffer_chain_reference
));
2701 chain
->flags
|= EVBUFFER_REFERENCE
| EVBUFFER_IMMUTABLE
;
2702 chain
->buffer
= (u_char
*)data
;
2703 chain
->buffer_len
= datlen
;
2704 chain
->off
= datlen
;
2706 info
= EVBUFFER_CHAIN_EXTRA(struct evbuffer_chain_reference
, chain
);
2707 info
->cleanupfn
= cleanupfn
;
2708 info
->extra
= extra
;
2710 EVBUFFER_LOCK(outbuf
);
2711 if (outbuf
->freeze_end
) {
2712 /* don't call chain_free; we do not want to actually invoke
2713 * the cleanup function */
2717 evbuffer_chain_insert(outbuf
, chain
);
2718 outbuf
->n_add_for_cb
+= datlen
;
2720 evbuffer_invoke_callbacks(outbuf
);
2724 EVBUFFER_UNLOCK(outbuf
);
2729 /* TODO(niels): maybe we don't want to own the fd, however, in that
2730 * case, we should dup it - dup is cheap. Perhaps, we should use a
2733 /* TODO(niels): we may want to add to automagically convert to mmap, in
2734 * case evbuffer_remove() or evbuffer_pullup() are being used.
2737 evbuffer_add_file(struct evbuffer
*outbuf
, int fd
,
2738 ev_off_t offset
, ev_off_t length
)
2740 #if defined(USE_SENDFILE) || defined(_EVENT_HAVE_MMAP)
2741 struct evbuffer_chain
*chain
;
2742 struct evbuffer_chain_fd
*info
;
2744 #if defined(USE_SENDFILE)
2745 int sendfile_okay
= 1;
2749 #if defined(USE_SENDFILE)
2751 EVBUFFER_LOCK(outbuf
);
2752 sendfile_okay
= outbuf
->flags
& EVBUFFER_FLAG_DRAINS_TO_FD
;
2753 EVBUFFER_UNLOCK(outbuf
);
2756 if (use_sendfile
&& sendfile_okay
) {
2757 chain
= evbuffer_chain_new(sizeof(struct evbuffer_chain_fd
));
2758 if (chain
== NULL
) {
2759 event_warn("%s: out of memory", __func__
);
2763 chain
->flags
|= EVBUFFER_SENDFILE
| EVBUFFER_IMMUTABLE
;
2764 chain
->buffer
= NULL
; /* no reading possible */
2765 chain
->buffer_len
= length
+ offset
;
2766 chain
->off
= length
;
2767 chain
->misalign
= offset
;
2769 info
= EVBUFFER_CHAIN_EXTRA(struct evbuffer_chain_fd
, chain
);
2772 EVBUFFER_LOCK(outbuf
);
2773 if (outbuf
->freeze_end
) {
2777 outbuf
->n_add_for_cb
+= length
;
2778 evbuffer_chain_insert(outbuf
, chain
);
2782 #if defined(_EVENT_HAVE_MMAP)
2784 void *mapped
= mmap(NULL
, length
+ offset
, PROT_READ
,
2793 /* some mmap implementations require offset to be a multiple of
2794 * the page size. most users of this api, are likely to use 0
2795 * so mapping everything is not likely to be a problem.
2796 * TODO(niels): determine page size and round offset to that
2797 * page size to avoid mapping too much memory.
2799 if (mapped
== MAP_FAILED
) {
2800 event_warn("%s: mmap(%d, %d, %zu) failed",
2801 __func__
, fd
, 0, (size_t)(offset
+ length
));
2804 chain
= evbuffer_chain_new(sizeof(struct evbuffer_chain_fd
));
2805 if (chain
== NULL
) {
2806 event_warn("%s: out of memory", __func__
);
2807 munmap(mapped
, length
);
2811 chain
->flags
|= EVBUFFER_MMAP
| EVBUFFER_IMMUTABLE
;
2812 chain
->buffer
= mapped
;
2813 chain
->buffer_len
= length
+ offset
;
2814 chain
->off
= length
+ offset
;
2816 info
= EVBUFFER_CHAIN_EXTRA(struct evbuffer_chain_fd
, chain
);
2819 EVBUFFER_LOCK(outbuf
);
2820 if (outbuf
->freeze_end
) {
2822 evbuffer_chain_free(chain
);
2825 outbuf
->n_add_for_cb
+= length
;
2827 evbuffer_chain_insert(outbuf
, chain
);
2829 /* we need to subtract whatever we don't need */
2830 evbuffer_drain(outbuf
, offset
);
2835 /* the default implementation */
2836 struct evbuffer
*tmp
= evbuffer_new();
2843 #define lseek _lseeki64
2845 if (lseek(fd
, offset
, SEEK_SET
) == -1) {
2850 /* we add everything to a temporary buffer, so that we
2851 * can abort without side effects if the read fails.
2854 read
= evbuffer_readfile(tmp
, fd
, (ev_ssize_t
)length
);
2863 EVBUFFER_LOCK(outbuf
);
2864 if (outbuf
->freeze_end
) {
2868 evbuffer_add_buffer(outbuf
, tmp
);
2872 #define close _close
2879 evbuffer_invoke_callbacks(outbuf
);
2880 EVBUFFER_UNLOCK(outbuf
);
2887 evbuffer_setcb(struct evbuffer
*buffer
, evbuffer_cb cb
, void *cbarg
)
2889 EVBUFFER_LOCK(buffer
);
2891 if (!TAILQ_EMPTY(&buffer
->callbacks
))
2892 evbuffer_remove_all_callbacks(buffer
);
2895 struct evbuffer_cb_entry
*ent
=
2896 evbuffer_add_cb(buffer
, NULL
, cbarg
);
2897 ent
->cb
.cb_obsolete
= cb
;
2898 ent
->flags
|= EVBUFFER_CB_OBSOLETE
;
2900 EVBUFFER_UNLOCK(buffer
);
2903 struct evbuffer_cb_entry
*
2904 evbuffer_add_cb(struct evbuffer
*buffer
, evbuffer_cb_func cb
, void *cbarg
)
2906 struct evbuffer_cb_entry
*e
;
2907 if (! (e
= mm_calloc(1, sizeof(struct evbuffer_cb_entry
))))
2909 EVBUFFER_LOCK(buffer
);
2912 e
->flags
= EVBUFFER_CB_ENABLED
;
2913 TAILQ_INSERT_HEAD(&buffer
->callbacks
, e
, next
);
2914 EVBUFFER_UNLOCK(buffer
);
2919 evbuffer_remove_cb_entry(struct evbuffer
*buffer
,
2920 struct evbuffer_cb_entry
*ent
)
2922 EVBUFFER_LOCK(buffer
);
2923 TAILQ_REMOVE(&buffer
->callbacks
, ent
, next
);
2924 EVBUFFER_UNLOCK(buffer
);
2930 evbuffer_remove_cb(struct evbuffer
*buffer
, evbuffer_cb_func cb
, void *cbarg
)
2932 struct evbuffer_cb_entry
*cbent
;
2934 EVBUFFER_LOCK(buffer
);
2935 TAILQ_FOREACH(cbent
, &buffer
->callbacks
, next
) {
2936 if (cb
== cbent
->cb
.cb_func
&& cbarg
== cbent
->cbarg
) {
2937 result
= evbuffer_remove_cb_entry(buffer
, cbent
);
2942 EVBUFFER_UNLOCK(buffer
);
2947 evbuffer_cb_set_flags(struct evbuffer
*buffer
,
2948 struct evbuffer_cb_entry
*cb
, ev_uint32_t flags
)
2950 /* the user isn't allowed to mess with these. */
2951 flags
&= ~EVBUFFER_CB_INTERNAL_FLAGS
;
2952 EVBUFFER_LOCK(buffer
);
2954 EVBUFFER_UNLOCK(buffer
);
2959 evbuffer_cb_clear_flags(struct evbuffer
*buffer
,
2960 struct evbuffer_cb_entry
*cb
, ev_uint32_t flags
)
2962 /* the user isn't allowed to mess with these. */
2963 flags
&= ~EVBUFFER_CB_INTERNAL_FLAGS
;
2964 EVBUFFER_LOCK(buffer
);
2965 cb
->flags
&= ~flags
;
2966 EVBUFFER_UNLOCK(buffer
);
2971 evbuffer_freeze(struct evbuffer
*buffer
, int start
)
2973 EVBUFFER_LOCK(buffer
);
2975 buffer
->freeze_start
= 1;
2977 buffer
->freeze_end
= 1;
2978 EVBUFFER_UNLOCK(buffer
);
2983 evbuffer_unfreeze(struct evbuffer
*buffer
, int start
)
2985 EVBUFFER_LOCK(buffer
);
2987 buffer
->freeze_start
= 0;
2989 buffer
->freeze_end
= 0;
2990 EVBUFFER_UNLOCK(buffer
);
2996 evbuffer_cb_suspend(struct evbuffer
*buffer
, struct evbuffer_cb_entry
*cb
)
2998 if (!(cb
->flags
& EVBUFFER_CB_SUSPENDED
)) {
2999 cb
->size_before_suspend
= evbuffer_get_length(buffer
);
3000 cb
->flags
|= EVBUFFER_CB_SUSPENDED
;
3005 evbuffer_cb_unsuspend(struct evbuffer
*buffer
, struct evbuffer_cb_entry
*cb
)
3007 if ((cb
->flags
& EVBUFFER_CB_SUSPENDED
)) {
3008 unsigned call
= (cb
->flags
& EVBUFFER_CB_CALL_ON_UNSUSPEND
);
3009 size_t sz
= cb
->size_before_suspend
;
3010 cb
->flags
&= ~(EVBUFFER_CB_SUSPENDED
|
3011 EVBUFFER_CB_CALL_ON_UNSUSPEND
);
3012 cb
->size_before_suspend
= 0;
3013 if (call
&& (cb
->flags
& EVBUFFER_CB_ENABLED
)) {
3014 cb
->cb(buffer
, sz
, evbuffer_get_length(buffer
), cb
->cbarg
);
3020 /* These hooks are exposed so that the unit tests can temporarily disable
3021 * sendfile support in order to test mmap, or both to test linear
3022 * access. Don't use it; if we need to add a way to disable sendfile support
3023 * in the future, it will probably be via an alternate version of
3024 * evbuffer_add_file() with a 'flags' argument.
3026 int _evbuffer_testing_use_sendfile(void);
3027 int _evbuffer_testing_use_mmap(void);
3028 int _evbuffer_testing_use_linear_file_access(void);
3031 _evbuffer_testing_use_sendfile(void)
3038 #ifdef _EVENT_HAVE_MMAP
3044 _evbuffer_testing_use_mmap(void)
3050 #ifdef _EVENT_HAVE_MMAP
3057 _evbuffer_testing_use_linear_file_access(void)
3062 #ifdef _EVENT_HAVE_MMAP