2 * Copyright (c) 2002-2007 Niels Provos <provos@citi.umich.edu>
3 * Copyright (c) 2007-2012 Niels Provos and Nick Mathewson
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 * 3. The name of the author may not be used to endorse or promote products
14 * derived from this software without specific prior written permission.
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 #include "event2/event-config.h"
36 #ifdef _EVENT_HAVE_VASPRINTF
37 /* If we have vasprintf, we need to define this before we include stdio.h. */
43 #include <sys/types.h>
45 #ifdef _EVENT_HAVE_SYS_TIME_H
49 #ifdef _EVENT_HAVE_SYS_SOCKET_H
50 #include <sys/socket.h>
53 #ifdef _EVENT_HAVE_SYS_UIO_H
57 #ifdef _EVENT_HAVE_SYS_IOCTL_H
58 #include <sys/ioctl.h>
61 #ifdef _EVENT_HAVE_SYS_MMAN_H
65 #ifdef _EVENT_HAVE_SYS_SENDFILE_H
66 #include <sys/sendfile.h>
73 #ifdef _EVENT_HAVE_STDARG_H
76 #ifdef _EVENT_HAVE_UNISTD_H
81 #include "event2/event.h"
82 #include "event2/buffer.h"
83 #include "event2/buffer_compat.h"
84 #include "event2/bufferevent.h"
85 #include "event2/bufferevent_compat.h"
86 #include "event2/bufferevent_struct.h"
87 #include "event2/thread.h"
88 #include "event2/event-config.h"
89 #include "log-internal.h"
90 #include "mm-internal.h"
91 #include "util-internal.h"
92 #include "evthread-internal.h"
93 #include "evbuffer-internal.h"
94 #include "bufferevent-internal.h"
96 /* some systems do not have MAP_FAILED */
98 #define MAP_FAILED ((void *)-1)
101 /* send file support */
102 #if defined(_EVENT_HAVE_SYS_SENDFILE_H) && defined(_EVENT_HAVE_SENDFILE) && defined(__linux__)
103 #define USE_SENDFILE 1
104 #define SENDFILE_IS_LINUX 1
105 #elif defined(_EVENT_HAVE_SENDFILE) && defined(__FreeBSD__)
106 #define USE_SENDFILE 1
107 #define SENDFILE_IS_FREEBSD 1
108 #elif defined(_EVENT_HAVE_SENDFILE) && defined(__APPLE__)
109 #define USE_SENDFILE 1
110 #define SENDFILE_IS_MACOSX 1
111 #elif defined(_EVENT_HAVE_SENDFILE) && defined(__sun__) && defined(__svr4__)
112 #define USE_SENDFILE 1
113 #define SENDFILE_IS_SOLARIS 1
117 static int use_sendfile
= 1;
119 #ifdef _EVENT_HAVE_MMAP
120 static int use_mmap
= 1;
124 /* Mask of user-selectable callback flags. */
125 #define EVBUFFER_CB_USER_FLAGS 0xffff
126 /* Mask of all internal-use-only flags. */
127 #define EVBUFFER_CB_INTERNAL_FLAGS 0xffff0000
129 /* Flag set if the callback is using the cb_obsolete function pointer */
130 #define EVBUFFER_CB_OBSOLETE 0x00040000
132 /* evbuffer_chain support */
133 #define CHAIN_SPACE_PTR(ch) ((ch)->buffer + (ch)->misalign + (ch)->off)
134 #define CHAIN_SPACE_LEN(ch) ((ch)->flags & EVBUFFER_IMMUTABLE ? \
135 0 : (ch)->buffer_len - ((ch)->misalign + (ch)->off))
137 #define CHAIN_PINNED(ch) (((ch)->flags & EVBUFFER_MEM_PINNED_ANY) != 0)
138 #define CHAIN_PINNED_R(ch) (((ch)->flags & EVBUFFER_MEM_PINNED_R) != 0)
140 static void evbuffer_chain_align(struct evbuffer_chain
*chain
);
141 static int evbuffer_chain_should_realign(struct evbuffer_chain
*chain
,
143 static void evbuffer_deferred_callback(struct deferred_cb
*cb
, void *arg
);
144 static int evbuffer_ptr_memcmp(const struct evbuffer
*buf
,
145 const struct evbuffer_ptr
*pos
, const char *mem
, size_t len
);
146 static struct evbuffer_chain
*evbuffer_expand_singlechain(struct evbuffer
*buf
,
150 static int evbuffer_readfile(struct evbuffer
*buf
, evutil_socket_t fd
,
153 #define evbuffer_readfile evbuffer_read
156 static struct evbuffer_chain
*
157 evbuffer_chain_new(size_t size
)
159 struct evbuffer_chain
*chain
;
162 if (size
> EVBUFFER_CHAIN_MAX
- EVBUFFER_CHAIN_SIZE
)
165 size
+= EVBUFFER_CHAIN_SIZE
;
167 /* get the next largest memory that can hold the buffer */
168 if (size
< EVBUFFER_CHAIN_MAX
/ 2) {
169 to_alloc
= MIN_BUFFER_SIZE
;
170 while (to_alloc
< size
) {
177 /* we get everything in one chunk */
178 if ((chain
= mm_malloc(to_alloc
)) == NULL
)
181 memset(chain
, 0, EVBUFFER_CHAIN_SIZE
);
183 chain
->buffer_len
= to_alloc
- EVBUFFER_CHAIN_SIZE
;
185 /* this way we can manipulate the buffer to different addresses,
186 * which is required for mmap for example.
188 chain
->buffer
= EVBUFFER_CHAIN_EXTRA(u_char
, chain
);
194 evbuffer_chain_free(struct evbuffer_chain
*chain
)
196 if (CHAIN_PINNED(chain
)) {
197 chain
->flags
|= EVBUFFER_DANGLING
;
200 if (chain
->flags
& (EVBUFFER_MMAP
|EVBUFFER_SENDFILE
|
201 EVBUFFER_REFERENCE
)) {
202 if (chain
->flags
& EVBUFFER_REFERENCE
) {
203 struct evbuffer_chain_reference
*info
=
204 EVBUFFER_CHAIN_EXTRA(
205 struct evbuffer_chain_reference
,
208 (*info
->cleanupfn
)(chain
->buffer
,
212 #ifdef _EVENT_HAVE_MMAP
213 if (chain
->flags
& EVBUFFER_MMAP
) {
214 struct evbuffer_chain_fd
*info
=
215 EVBUFFER_CHAIN_EXTRA(struct evbuffer_chain_fd
,
217 if (munmap(chain
->buffer
, chain
->buffer_len
) == -1)
218 event_warn("%s: munmap failed", __func__
);
219 if (close(info
->fd
) == -1)
220 event_warn("%s: close(%d) failed",
225 if (chain
->flags
& EVBUFFER_SENDFILE
) {
226 struct evbuffer_chain_fd
*info
=
227 EVBUFFER_CHAIN_EXTRA(struct evbuffer_chain_fd
,
229 if (close(info
->fd
) == -1)
230 event_warn("%s: close(%d) failed",
240 evbuffer_free_all_chains(struct evbuffer_chain
*chain
)
242 struct evbuffer_chain
*next
;
243 for (; chain
; chain
= next
) {
245 evbuffer_chain_free(chain
);
251 evbuffer_chains_all_empty(struct evbuffer_chain
*chain
)
253 for (; chain
; chain
= chain
->next
) {
260 /* The definition is needed for EVUTIL_ASSERT, which uses sizeof to avoid
261 "unused variable" warnings. */
262 static inline int evbuffer_chains_all_empty(struct evbuffer_chain
*chain
) {
267 /* Free all trailing chains in 'buf' that are neither pinned nor empty, prior
268 * to replacing them all with a new chain. Return a pointer to the place
269 * where the new chain will go.
271 * Internal; requires lock. The caller must fix up buf->last and buf->first
272 * as needed; they might have been freed.
274 static struct evbuffer_chain
**
275 evbuffer_free_trailing_empty_chains(struct evbuffer
*buf
)
277 struct evbuffer_chain
**ch
= buf
->last_with_datap
;
278 /* Find the first victim chain. It might be *last_with_datap */
279 while ((*ch
) && ((*ch
)->off
!= 0 || CHAIN_PINNED(*ch
)))
282 EVUTIL_ASSERT(evbuffer_chains_all_empty(*ch
));
283 evbuffer_free_all_chains(*ch
);
289 /* Add a single chain 'chain' to the end of 'buf', freeing trailing empty
290 * chains as necessary. Requires lock. Does not schedule callbacks.
293 evbuffer_chain_insert(struct evbuffer
*buf
,
294 struct evbuffer_chain
*chain
)
296 ASSERT_EVBUFFER_LOCKED(buf
);
297 if (*buf
->last_with_datap
== NULL
) {
298 /* There are no chains data on the buffer at all. */
299 EVUTIL_ASSERT(buf
->last_with_datap
== &buf
->first
);
300 EVUTIL_ASSERT(buf
->first
== NULL
);
301 buf
->first
= buf
->last
= chain
;
303 struct evbuffer_chain
**ch
= buf
->last_with_datap
;
304 /* Find the first victim chain. It might be *last_with_datap */
305 while ((*ch
) && ((*ch
)->off
!= 0 || CHAIN_PINNED(*ch
)))
308 /* There is no victim; just append this new chain. */
309 buf
->last
->next
= chain
;
311 buf
->last_with_datap
= &buf
->last
->next
;
313 /* Replace all victim chains with this chain. */
314 EVUTIL_ASSERT(evbuffer_chains_all_empty(*ch
));
315 evbuffer_free_all_chains(*ch
);
320 buf
->total_len
+= chain
->off
;
323 static inline struct evbuffer_chain
*
324 evbuffer_chain_insert_new(struct evbuffer
*buf
, size_t datlen
)
326 struct evbuffer_chain
*chain
;
327 if ((chain
= evbuffer_chain_new(datlen
)) == NULL
)
329 evbuffer_chain_insert(buf
, chain
);
334 _evbuffer_chain_pin(struct evbuffer_chain
*chain
, unsigned flag
)
336 EVUTIL_ASSERT((chain
->flags
& flag
) == 0);
337 chain
->flags
|= flag
;
341 _evbuffer_chain_unpin(struct evbuffer_chain
*chain
, unsigned flag
)
343 EVUTIL_ASSERT((chain
->flags
& flag
) != 0);
344 chain
->flags
&= ~flag
;
345 if (chain
->flags
& EVBUFFER_DANGLING
)
346 evbuffer_chain_free(chain
);
352 struct evbuffer
*buffer
;
354 buffer
= mm_calloc(1, sizeof(struct evbuffer
));
358 TAILQ_INIT(&buffer
->callbacks
);
360 buffer
->last_with_datap
= &buffer
->first
;
366 evbuffer_set_flags(struct evbuffer
*buf
, ev_uint64_t flags
)
369 buf
->flags
|= (ev_uint32_t
)flags
;
370 EVBUFFER_UNLOCK(buf
);
375 evbuffer_clear_flags(struct evbuffer
*buf
, ev_uint64_t flags
)
378 buf
->flags
&= ~(ev_uint32_t
)flags
;
379 EVBUFFER_UNLOCK(buf
);
384 _evbuffer_incref(struct evbuffer
*buf
)
388 EVBUFFER_UNLOCK(buf
);
392 _evbuffer_incref_and_lock(struct evbuffer
*buf
)
399 evbuffer_defer_callbacks(struct evbuffer
*buffer
, struct event_base
*base
)
401 EVBUFFER_LOCK(buffer
);
402 buffer
->cb_queue
= event_base_get_deferred_cb_queue(base
);
403 buffer
->deferred_cbs
= 1;
404 event_deferred_cb_init(&buffer
->deferred
,
405 evbuffer_deferred_callback
, buffer
);
406 EVBUFFER_UNLOCK(buffer
);
411 evbuffer_enable_locking(struct evbuffer
*buf
, void *lock
)
413 #ifdef _EVENT_DISABLE_THREAD_SUPPORT
420 EVTHREAD_ALLOC_LOCK(lock
, EVTHREAD_LOCKTYPE_RECURSIVE
);
435 evbuffer_set_parent(struct evbuffer
*buf
, struct bufferevent
*bev
)
439 EVBUFFER_UNLOCK(buf
);
443 evbuffer_run_callbacks(struct evbuffer
*buffer
, int running_deferred
)
445 struct evbuffer_cb_entry
*cbent
, *next
;
446 struct evbuffer_cb_info info
;
448 ev_uint32_t mask
, masked_val
;
451 if (running_deferred
) {
452 mask
= EVBUFFER_CB_NODEFER
|EVBUFFER_CB_ENABLED
;
453 masked_val
= EVBUFFER_CB_ENABLED
;
454 } else if (buffer
->deferred_cbs
) {
455 mask
= EVBUFFER_CB_NODEFER
|EVBUFFER_CB_ENABLED
;
456 masked_val
= EVBUFFER_CB_NODEFER
|EVBUFFER_CB_ENABLED
;
457 /* Don't zero-out n_add/n_del, since the deferred callbacks
458 will want to see them. */
461 mask
= EVBUFFER_CB_ENABLED
;
462 masked_val
= EVBUFFER_CB_ENABLED
;
465 ASSERT_EVBUFFER_LOCKED(buffer
);
467 if (TAILQ_EMPTY(&buffer
->callbacks
)) {
468 buffer
->n_add_for_cb
= buffer
->n_del_for_cb
= 0;
471 if (buffer
->n_add_for_cb
== 0 && buffer
->n_del_for_cb
== 0)
474 new_size
= buffer
->total_len
;
475 info
.orig_size
= new_size
+ buffer
->n_del_for_cb
- buffer
->n_add_for_cb
;
476 info
.n_added
= buffer
->n_add_for_cb
;
477 info
.n_deleted
= buffer
->n_del_for_cb
;
479 buffer
->n_add_for_cb
= 0;
480 buffer
->n_del_for_cb
= 0;
482 for (cbent
= TAILQ_FIRST(&buffer
->callbacks
);
483 cbent
!= TAILQ_END(&buffer
->callbacks
);
485 /* Get the 'next' pointer now in case this callback decides
486 * to remove itself or something. */
487 next
= TAILQ_NEXT(cbent
, next
);
489 if ((cbent
->flags
& mask
) != masked_val
)
492 if ((cbent
->flags
& EVBUFFER_CB_OBSOLETE
))
493 cbent
->cb
.cb_obsolete(buffer
,
494 info
.orig_size
, new_size
, cbent
->cbarg
);
496 cbent
->cb
.cb_func(buffer
, &info
, cbent
->cbarg
);
501 evbuffer_invoke_callbacks(struct evbuffer
*buffer
)
503 if (TAILQ_EMPTY(&buffer
->callbacks
)) {
504 buffer
->n_add_for_cb
= buffer
->n_del_for_cb
= 0;
508 if (buffer
->deferred_cbs
) {
509 if (buffer
->deferred
.queued
)
511 _evbuffer_incref_and_lock(buffer
);
513 bufferevent_incref(buffer
->parent
);
514 EVBUFFER_UNLOCK(buffer
);
515 event_deferred_cb_schedule(buffer
->cb_queue
, &buffer
->deferred
);
518 evbuffer_run_callbacks(buffer
, 0);
522 evbuffer_deferred_callback(struct deferred_cb
*cb
, void *arg
)
524 struct bufferevent
*parent
= NULL
;
525 struct evbuffer
*buffer
= arg
;
527 /* XXXX It would be better to run these callbacks without holding the
529 EVBUFFER_LOCK(buffer
);
530 parent
= buffer
->parent
;
531 evbuffer_run_callbacks(buffer
, 1);
532 _evbuffer_decref_and_unlock(buffer
);
534 bufferevent_decref(parent
);
538 evbuffer_remove_all_callbacks(struct evbuffer
*buffer
)
540 struct evbuffer_cb_entry
*cbent
;
542 while ((cbent
= TAILQ_FIRST(&buffer
->callbacks
))) {
543 TAILQ_REMOVE(&buffer
->callbacks
, cbent
, next
);
549 _evbuffer_decref_and_unlock(struct evbuffer
*buffer
)
551 struct evbuffer_chain
*chain
, *next
;
552 ASSERT_EVBUFFER_LOCKED(buffer
);
554 EVUTIL_ASSERT(buffer
->refcnt
> 0);
556 if (--buffer
->refcnt
> 0) {
557 EVBUFFER_UNLOCK(buffer
);
561 for (chain
= buffer
->first
; chain
!= NULL
; chain
= next
) {
563 evbuffer_chain_free(chain
);
565 evbuffer_remove_all_callbacks(buffer
);
566 if (buffer
->deferred_cbs
)
567 event_deferred_cb_cancel(buffer
->cb_queue
, &buffer
->deferred
);
569 EVBUFFER_UNLOCK(buffer
);
570 if (buffer
->own_lock
)
571 EVTHREAD_FREE_LOCK(buffer
->lock
, EVTHREAD_LOCKTYPE_RECURSIVE
);
576 evbuffer_free(struct evbuffer
*buffer
)
578 EVBUFFER_LOCK(buffer
);
579 _evbuffer_decref_and_unlock(buffer
);
583 evbuffer_lock(struct evbuffer
*buf
)
589 evbuffer_unlock(struct evbuffer
*buf
)
591 EVBUFFER_UNLOCK(buf
);
595 evbuffer_get_length(const struct evbuffer
*buffer
)
599 EVBUFFER_LOCK(buffer
);
601 result
= (buffer
->total_len
);
603 EVBUFFER_UNLOCK(buffer
);
609 evbuffer_get_contiguous_space(const struct evbuffer
*buf
)
611 struct evbuffer_chain
*chain
;
616 result
= (chain
!= NULL
? chain
->off
: 0);
617 EVBUFFER_UNLOCK(buf
);
623 evbuffer_reserve_space(struct evbuffer
*buf
, ev_ssize_t size
,
624 struct evbuffer_iovec
*vec
, int n_vecs
)
626 struct evbuffer_chain
*chain
, **chainp
;
635 if ((chain
= evbuffer_expand_singlechain(buf
, size
)) == NULL
)
638 vec
[0].iov_base
= CHAIN_SPACE_PTR(chain
);
639 vec
[0].iov_len
= (size_t) CHAIN_SPACE_LEN(chain
);
640 EVUTIL_ASSERT(size
<0 || (size_t)vec
[0].iov_len
>= (size_t)size
);
643 if (_evbuffer_expand_fast(buf
, size
, n_vecs
)<0)
645 n
= _evbuffer_read_setup_vecs(buf
, size
, vec
, n_vecs
,
650 EVBUFFER_UNLOCK(buf
);
656 advance_last_with_data(struct evbuffer
*buf
)
659 ASSERT_EVBUFFER_LOCKED(buf
);
661 if (!*buf
->last_with_datap
)
664 while ((*buf
->last_with_datap
)->next
&& (*buf
->last_with_datap
)->next
->off
) {
665 buf
->last_with_datap
= &(*buf
->last_with_datap
)->next
;
672 evbuffer_commit_space(struct evbuffer
*buf
,
673 struct evbuffer_iovec
*vec
, int n_vecs
)
675 struct evbuffer_chain
*chain
, **firstchainp
, **chainp
;
687 } else if (n_vecs
== 1 &&
688 (buf
->last
&& vec
[0].iov_base
== (void*)CHAIN_SPACE_PTR(buf
->last
))) {
689 /* The user only got or used one chain; it might not
690 * be the first one with space in it. */
691 if ((size_t)vec
[0].iov_len
> (size_t)CHAIN_SPACE_LEN(buf
->last
))
693 buf
->last
->off
+= vec
[0].iov_len
;
694 added
= vec
[0].iov_len
;
696 advance_last_with_data(buf
);
700 /* Advance 'firstchain' to the first chain with space in it. */
701 firstchainp
= buf
->last_with_datap
;
704 if (CHAIN_SPACE_LEN(*firstchainp
) == 0) {
705 firstchainp
= &(*firstchainp
)->next
;
708 chain
= *firstchainp
;
709 /* pass 1: make sure that the pointers and lengths of vecs[] are in
710 * bounds before we try to commit anything. */
711 for (i
=0; i
<n_vecs
; ++i
) {
714 if (vec
[i
].iov_base
!= (void*)CHAIN_SPACE_PTR(chain
) ||
715 (size_t)vec
[i
].iov_len
> CHAIN_SPACE_LEN(chain
))
719 /* pass 2: actually adjust all the chains. */
720 chainp
= firstchainp
;
721 for (i
=0; i
<n_vecs
; ++i
) {
722 (*chainp
)->off
+= vec
[i
].iov_len
;
723 added
+= vec
[i
].iov_len
;
724 if (vec
[i
].iov_len
) {
725 buf
->last_with_datap
= chainp
;
727 chainp
= &(*chainp
)->next
;
731 buf
->total_len
+= added
;
732 buf
->n_add_for_cb
+= added
;
734 evbuffer_invoke_callbacks(buf
);
737 EVBUFFER_UNLOCK(buf
);
742 HAS_PINNED_R(struct evbuffer
*buf
)
744 return (buf
->last
&& CHAIN_PINNED_R(buf
->last
));
748 ZERO_CHAIN(struct evbuffer
*dst
)
750 ASSERT_EVBUFFER_LOCKED(dst
);
753 dst
->last_with_datap
= &(dst
)->first
;
757 /* Prepares the contents of src to be moved to another buffer by removing
758 * read-pinned chains. The first pinned chain is saved in first, and the
759 * last in last. If src has no read-pinned chains, first and last are set
762 PRESERVE_PINNED(struct evbuffer
*src
, struct evbuffer_chain
**first
,
763 struct evbuffer_chain
**last
)
765 struct evbuffer_chain
*chain
, **pinned
;
767 ASSERT_EVBUFFER_LOCKED(src
);
769 if (!HAS_PINNED_R(src
)) {
770 *first
= *last
= NULL
;
774 pinned
= src
->last_with_datap
;
775 if (!CHAIN_PINNED_R(*pinned
))
776 pinned
= &(*pinned
)->next
;
777 EVUTIL_ASSERT(CHAIN_PINNED_R(*pinned
));
778 chain
= *first
= *pinned
;
781 /* If there's data in the first pinned chain, we need to allocate
782 * a new chain and copy the data over. */
784 struct evbuffer_chain
*tmp
;
786 EVUTIL_ASSERT(pinned
== src
->last_with_datap
);
787 tmp
= evbuffer_chain_new(chain
->off
);
790 memcpy(tmp
->buffer
, chain
->buffer
+ chain
->misalign
,
792 tmp
->off
= chain
->off
;
793 *src
->last_with_datap
= tmp
;
795 chain
->misalign
+= chain
->off
;
798 src
->last
= *src
->last_with_datap
;
806 RESTORE_PINNED(struct evbuffer
*src
, struct evbuffer_chain
*pinned
,
807 struct evbuffer_chain
*last
)
809 ASSERT_EVBUFFER_LOCKED(src
);
818 src
->last_with_datap
= &src
->first
;
823 COPY_CHAIN(struct evbuffer
*dst
, struct evbuffer
*src
)
825 ASSERT_EVBUFFER_LOCKED(dst
);
826 ASSERT_EVBUFFER_LOCKED(src
);
827 dst
->first
= src
->first
;
828 if (src
->last_with_datap
== &src
->first
)
829 dst
->last_with_datap
= &dst
->first
;
831 dst
->last_with_datap
= src
->last_with_datap
;
832 dst
->last
= src
->last
;
833 dst
->total_len
= src
->total_len
;
837 APPEND_CHAIN(struct evbuffer
*dst
, struct evbuffer
*src
)
839 ASSERT_EVBUFFER_LOCKED(dst
);
840 ASSERT_EVBUFFER_LOCKED(src
);
841 dst
->last
->next
= src
->first
;
842 if (src
->last_with_datap
== &src
->first
)
843 dst
->last_with_datap
= &dst
->last
->next
;
845 dst
->last_with_datap
= src
->last_with_datap
;
846 dst
->last
= src
->last
;
847 dst
->total_len
+= src
->total_len
;
851 PREPEND_CHAIN(struct evbuffer
*dst
, struct evbuffer
*src
)
853 ASSERT_EVBUFFER_LOCKED(dst
);
854 ASSERT_EVBUFFER_LOCKED(src
);
855 src
->last
->next
= dst
->first
;
856 dst
->first
= src
->first
;
857 dst
->total_len
+= src
->total_len
;
858 if (*dst
->last_with_datap
== NULL
) {
859 if (src
->last_with_datap
== &(src
)->first
)
860 dst
->last_with_datap
= &dst
->first
;
862 dst
->last_with_datap
= src
->last_with_datap
;
863 } else if (dst
->last_with_datap
== &dst
->first
) {
864 dst
->last_with_datap
= &src
->last
->next
;
869 evbuffer_add_buffer(struct evbuffer
*outbuf
, struct evbuffer
*inbuf
)
871 struct evbuffer_chain
*pinned
, *last
;
872 size_t in_total_len
, out_total_len
;
875 EVBUFFER_LOCK2(inbuf
, outbuf
);
876 in_total_len
= inbuf
->total_len
;
877 out_total_len
= outbuf
->total_len
;
879 if (in_total_len
== 0 || outbuf
== inbuf
)
882 if (outbuf
->freeze_end
|| inbuf
->freeze_start
) {
887 if (PRESERVE_PINNED(inbuf
, &pinned
, &last
) < 0) {
892 if (out_total_len
== 0) {
893 /* There might be an empty chain at the start of outbuf; free
895 evbuffer_free_all_chains(outbuf
->first
);
896 COPY_CHAIN(outbuf
, inbuf
);
898 APPEND_CHAIN(outbuf
, inbuf
);
901 RESTORE_PINNED(inbuf
, pinned
, last
);
903 inbuf
->n_del_for_cb
+= in_total_len
;
904 outbuf
->n_add_for_cb
+= in_total_len
;
906 evbuffer_invoke_callbacks(inbuf
);
907 evbuffer_invoke_callbacks(outbuf
);
910 EVBUFFER_UNLOCK2(inbuf
, outbuf
);
915 evbuffer_prepend_buffer(struct evbuffer
*outbuf
, struct evbuffer
*inbuf
)
917 struct evbuffer_chain
*pinned
, *last
;
918 size_t in_total_len
, out_total_len
;
921 EVBUFFER_LOCK2(inbuf
, outbuf
);
923 in_total_len
= inbuf
->total_len
;
924 out_total_len
= outbuf
->total_len
;
926 if (!in_total_len
|| inbuf
== outbuf
)
929 if (outbuf
->freeze_start
|| inbuf
->freeze_start
) {
934 if (PRESERVE_PINNED(inbuf
, &pinned
, &last
) < 0) {
939 if (out_total_len
== 0) {
940 /* There might be an empty chain at the start of outbuf; free
942 evbuffer_free_all_chains(outbuf
->first
);
943 COPY_CHAIN(outbuf
, inbuf
);
945 PREPEND_CHAIN(outbuf
, inbuf
);
948 RESTORE_PINNED(inbuf
, pinned
, last
);
950 inbuf
->n_del_for_cb
+= in_total_len
;
951 outbuf
->n_add_for_cb
+= in_total_len
;
953 evbuffer_invoke_callbacks(inbuf
);
954 evbuffer_invoke_callbacks(outbuf
);
956 EVBUFFER_UNLOCK2(inbuf
, outbuf
);
961 evbuffer_drain(struct evbuffer
*buf
, size_t len
)
963 struct evbuffer_chain
*chain
, *next
;
964 size_t remaining
, old_len
;
968 old_len
= buf
->total_len
;
973 if (buf
->freeze_start
) {
978 if (len
>= old_len
&& !HAS_PINNED_R(buf
)) {
980 for (chain
= buf
->first
; chain
!= NULL
; chain
= next
) {
982 evbuffer_chain_free(chain
);
990 buf
->total_len
-= len
;
992 for (chain
= buf
->first
;
993 remaining
>= chain
->off
;
996 remaining
-= chain
->off
;
998 if (chain
== *buf
->last_with_datap
) {
999 buf
->last_with_datap
= &buf
->first
;
1001 if (&chain
->next
== buf
->last_with_datap
)
1002 buf
->last_with_datap
= &buf
->first
;
1004 if (CHAIN_PINNED_R(chain
)) {
1005 EVUTIL_ASSERT(remaining
== 0);
1006 chain
->misalign
+= chain
->off
;
1010 evbuffer_chain_free(chain
);
1015 EVUTIL_ASSERT(remaining
<= chain
->off
);
1016 chain
->misalign
+= remaining
;
1017 chain
->off
-= remaining
;
1021 buf
->n_del_for_cb
+= len
;
1022 /* Tell someone about changes in this buffer */
1023 evbuffer_invoke_callbacks(buf
);
1026 EVBUFFER_UNLOCK(buf
);
1030 /* Reads data from an event buffer and drains the bytes read */
1032 evbuffer_remove(struct evbuffer
*buf
, void *data_out
, size_t datlen
)
1036 n
= evbuffer_copyout(buf
, data_out
, datlen
);
1038 if (evbuffer_drain(buf
, n
)<0)
1041 EVBUFFER_UNLOCK(buf
);
1046 evbuffer_copyout(struct evbuffer
*buf
, void *data_out
, size_t datlen
)
1048 /*XXX fails badly on sendfile case. */
1049 struct evbuffer_chain
*chain
;
1050 char *data
= data_out
;
1052 ev_ssize_t result
= 0;
1058 if (datlen
>= buf
->total_len
)
1059 datlen
= buf
->total_len
;
1064 if (buf
->freeze_start
) {
1071 while (datlen
&& datlen
>= chain
->off
) {
1072 memcpy(data
, chain
->buffer
+ chain
->misalign
, chain
->off
);
1074 datlen
-= chain
->off
;
1076 chain
= chain
->next
;
1077 EVUTIL_ASSERT(chain
|| datlen
==0);
1081 EVUTIL_ASSERT(chain
);
1082 EVUTIL_ASSERT(datlen
<= chain
->off
);
1083 memcpy(data
, chain
->buffer
+ chain
->misalign
, datlen
);
1088 EVBUFFER_UNLOCK(buf
);
1092 /* reads data from the src buffer to the dst buffer, avoids memcpy as
1094 /* XXXX should return ev_ssize_t */
1096 evbuffer_remove_buffer(struct evbuffer
*src
, struct evbuffer
*dst
,
1099 /*XXX We should have an option to force this to be zero-copy.*/
1101 /*XXX can fail badly on sendfile case. */
1102 struct evbuffer_chain
*chain
, *previous
;
1106 EVBUFFER_LOCK2(src
, dst
);
1108 chain
= previous
= src
->first
;
1110 if (datlen
== 0 || dst
== src
) {
1115 if (dst
->freeze_end
|| src
->freeze_start
) {
1120 /* short-cut if there is no more data buffered */
1121 if (datlen
>= src
->total_len
) {
1122 datlen
= src
->total_len
;
1123 evbuffer_add_buffer(dst
, src
);
1124 result
= (int)datlen
; /*XXXX should return ev_ssize_t*/
1128 /* removes chains if possible */
1129 while (chain
->off
<= datlen
) {
1130 /* We can't remove the last with data from src unless we
1131 * remove all chains, in which case we would have done the if
1133 EVUTIL_ASSERT(chain
!= *src
->last_with_datap
);
1134 nread
+= chain
->off
;
1135 datlen
-= chain
->off
;
1137 if (src
->last_with_datap
== &chain
->next
)
1138 src
->last_with_datap
= &src
->first
;
1139 chain
= chain
->next
;
1143 /* we can remove the chain */
1144 struct evbuffer_chain
**chp
;
1145 chp
= evbuffer_free_trailing_empty_chains(dst
);
1147 if (dst
->first
== NULL
) {
1148 dst
->first
= src
->first
;
1152 dst
->last
= previous
;
1153 previous
->next
= NULL
;
1155 advance_last_with_data(dst
);
1157 dst
->total_len
+= nread
;
1158 dst
->n_add_for_cb
+= nread
;
1161 /* we know that there is more data in the src buffer than
1162 * we want to read, so we manually drain the chain */
1163 evbuffer_add(dst
, chain
->buffer
+ chain
->misalign
, datlen
);
1164 chain
->misalign
+= datlen
;
1165 chain
->off
-= datlen
;
1168 /* You might think we would want to increment dst->n_add_for_cb
1169 * here too. But evbuffer_add above already took care of that.
1171 src
->total_len
-= nread
;
1172 src
->n_del_for_cb
+= nread
;
1175 evbuffer_invoke_callbacks(dst
);
1176 evbuffer_invoke_callbacks(src
);
1178 result
= (int)nread
;/*XXXX should change return type */
1181 EVBUFFER_UNLOCK2(src
, dst
);
1186 evbuffer_pullup(struct evbuffer
*buf
, ev_ssize_t size
)
1188 struct evbuffer_chain
*chain
, *next
, *tmp
, *last_with_data
;
1189 unsigned char *buffer
, *result
= NULL
;
1190 ev_ssize_t remaining
;
1191 int removed_last_with_data
= 0;
1192 int removed_last_with_datap
= 0;
1199 size
= buf
->total_len
;
1200 /* if size > buf->total_len, we cannot guarantee to the user that she
1201 * is going to have a long enough buffer afterwards; so we return
1203 if (size
== 0 || (size_t)size
> buf
->total_len
)
1206 /* No need to pull up anything; the first size bytes are
1208 if (chain
->off
>= (size_t)size
) {
1209 result
= chain
->buffer
+ chain
->misalign
;
1213 /* Make sure that none of the chains we need to copy from is pinned. */
1214 remaining
= size
- chain
->off
;
1215 EVUTIL_ASSERT(remaining
>= 0);
1216 for (tmp
=chain
->next
; tmp
; tmp
=tmp
->next
) {
1217 if (CHAIN_PINNED(tmp
))
1219 if (tmp
->off
>= (size_t)remaining
)
1221 remaining
-= tmp
->off
;
1224 if (CHAIN_PINNED(chain
)) {
1225 size_t old_off
= chain
->off
;
1226 if (CHAIN_SPACE_LEN(chain
) < size
- chain
->off
) {
1227 /* not enough room at end of chunk. */
1230 buffer
= CHAIN_SPACE_PTR(chain
);
1234 chain
= chain
->next
;
1235 } else if (chain
->buffer_len
- chain
->misalign
>= (size_t)size
) {
1236 /* already have enough space in the first chain */
1237 size_t old_off
= chain
->off
;
1238 buffer
= chain
->buffer
+ chain
->misalign
+ chain
->off
;
1242 chain
= chain
->next
;
1244 if ((tmp
= evbuffer_chain_new(size
)) == NULL
) {
1245 event_warn("%s: out of memory", __func__
);
1248 buffer
= tmp
->buffer
;
1253 /* TODO(niels): deal with buffers that point to NULL like sendfile */
1255 /* Copy and free every chunk that will be entirely pulled into tmp */
1256 last_with_data
= *buf
->last_with_datap
;
1257 for (; chain
!= NULL
&& (size_t)size
>= chain
->off
; chain
= next
) {
1260 memcpy(buffer
, chain
->buffer
+ chain
->misalign
, chain
->off
);
1262 buffer
+= chain
->off
;
1263 if (chain
== last_with_data
)
1264 removed_last_with_data
= 1;
1265 if (&chain
->next
== buf
->last_with_datap
)
1266 removed_last_with_datap
= 1;
1268 evbuffer_chain_free(chain
);
1271 if (chain
!= NULL
) {
1272 memcpy(buffer
, chain
->buffer
+ chain
->misalign
, size
);
1273 chain
->misalign
+= size
;
1281 if (removed_last_with_data
) {
1282 buf
->last_with_datap
= &buf
->first
;
1283 } else if (removed_last_with_datap
) {
1284 if (buf
->first
->next
&& buf
->first
->next
->off
)
1285 buf
->last_with_datap
= &buf
->first
->next
;
1287 buf
->last_with_datap
= &buf
->first
;
1290 result
= (tmp
->buffer
+ tmp
->misalign
);
1293 EVBUFFER_UNLOCK(buf
);
1298 * Reads a line terminated by either '\r\n', '\n\r' or '\r' or '\n'.
1299 * The returned buffer needs to be freed by the called.
1302 evbuffer_readline(struct evbuffer
*buffer
)
1304 return evbuffer_readln(buffer
, NULL
, EVBUFFER_EOL_ANY
);
1307 static inline ev_ssize_t
1308 evbuffer_strchr(struct evbuffer_ptr
*it
, const char chr
)
1310 struct evbuffer_chain
*chain
= it
->_internal
.chain
;
1311 size_t i
= it
->_internal
.pos_in_chain
;
1312 while (chain
!= NULL
) {
1313 char *buffer
= (char *)chain
->buffer
+ chain
->misalign
;
1314 char *cp
= memchr(buffer
+i
, chr
, chain
->off
-i
);
1316 it
->_internal
.chain
= chain
;
1317 it
->_internal
.pos_in_chain
= cp
- buffer
;
1318 it
->pos
+= (cp
- buffer
- i
);
1321 it
->pos
+= chain
->off
- i
;
1323 chain
= chain
->next
;
1329 static inline char *
1330 find_eol_char(char *s
, size_t len
)
1332 #define CHUNK_SZ 128
1333 /* Lots of benchmarking found this approach to be faster in practice
1334 * than doing two memchrs over the whole buffer, doin a memchr on each
1335 * char of the buffer, or trying to emulate memchr by hand. */
1336 char *s_end
, *cr
, *lf
;
1339 size_t chunk
= (s
+ CHUNK_SZ
< s_end
) ? CHUNK_SZ
: (s_end
- s
);
1340 cr
= memchr(s
, '\r', chunk
);
1341 lf
= memchr(s
, '\n', chunk
);
1357 evbuffer_find_eol_char(struct evbuffer_ptr
*it
)
1359 struct evbuffer_chain
*chain
= it
->_internal
.chain
;
1360 size_t i
= it
->_internal
.pos_in_chain
;
1361 while (chain
!= NULL
) {
1362 char *buffer
= (char *)chain
->buffer
+ chain
->misalign
;
1363 char *cp
= find_eol_char(buffer
+i
, chain
->off
-i
);
1365 it
->_internal
.chain
= chain
;
1366 it
->_internal
.pos_in_chain
= cp
- buffer
;
1367 it
->pos
+= (cp
- buffer
) - i
;
1370 it
->pos
+= chain
->off
- i
;
1372 chain
= chain
->next
;
1380 struct evbuffer_ptr
*ptr
, const char *chrset
)
1383 struct evbuffer_chain
*chain
= ptr
->_internal
.chain
;
1384 size_t i
= ptr
->_internal
.pos_in_chain
;
1390 char *buffer
= (char *)chain
->buffer
+ chain
->misalign
;
1391 for (; i
< chain
->off
; ++i
) {
1392 const char *p
= chrset
;
1394 if (buffer
[i
] == *p
++)
1397 ptr
->_internal
.chain
= chain
;
1398 ptr
->_internal
.pos_in_chain
= i
;
1406 if (! chain
->next
) {
1407 ptr
->_internal
.chain
= chain
;
1408 ptr
->_internal
.pos_in_chain
= i
;
1413 chain
= chain
->next
;
1419 evbuffer_getchr(struct evbuffer_ptr
*it
)
1421 struct evbuffer_chain
*chain
= it
->_internal
.chain
;
1422 size_t off
= it
->_internal
.pos_in_chain
;
1424 return chain
->buffer
[chain
->misalign
+ off
];
1428 evbuffer_search_eol(struct evbuffer
*buffer
,
1429 struct evbuffer_ptr
*start
, size_t *eol_len_out
,
1430 enum evbuffer_eol_style eol_style
)
1432 struct evbuffer_ptr it
, it2
;
1433 size_t extra_drain
= 0;
1436 EVBUFFER_LOCK(buffer
);
1439 memcpy(&it
, start
, sizeof(it
));
1442 it
._internal
.chain
= buffer
->first
;
1443 it
._internal
.pos_in_chain
= 0;
1446 /* the eol_style determines our first stop character and how many
1447 * characters we are going to drain afterwards. */
1448 switch (eol_style
) {
1449 case EVBUFFER_EOL_ANY
:
1450 if (evbuffer_find_eol_char(&it
) < 0)
1452 memcpy(&it2
, &it
, sizeof(it
));
1453 extra_drain
= evbuffer_strspn(&it2
, "\r\n");
1455 case EVBUFFER_EOL_CRLF_STRICT
: {
1456 it
= evbuffer_search(buffer
, "\r\n", 2, &it
);
1462 case EVBUFFER_EOL_CRLF
:
1464 if (evbuffer_find_eol_char(&it
) < 0)
1466 if (evbuffer_getchr(&it
) == '\n') {
1469 } else if (!evbuffer_ptr_memcmp(
1470 buffer
, &it
, "\r\n", 2)) {
1474 if (evbuffer_ptr_set(buffer
, &it
, 1,
1475 EVBUFFER_PTR_ADD
)<0)
1480 case EVBUFFER_EOL_LF
:
1481 if (evbuffer_strchr(&it
, '\n') < 0)
1491 EVBUFFER_UNLOCK(buffer
);
1497 *eol_len_out
= extra_drain
;
1503 evbuffer_readln(struct evbuffer
*buffer
, size_t *n_read_out
,
1504 enum evbuffer_eol_style eol_style
)
1506 struct evbuffer_ptr it
;
1508 size_t n_to_copy
=0, extra_drain
=0;
1509 char *result
= NULL
;
1511 EVBUFFER_LOCK(buffer
);
1513 if (buffer
->freeze_start
) {
1517 it
= evbuffer_search_eol(buffer
, NULL
, &extra_drain
, eol_style
);
1522 if ((line
= mm_malloc(n_to_copy
+1)) == NULL
) {
1523 event_warn("%s: out of memory", __func__
);
1527 evbuffer_remove(buffer
, line
, n_to_copy
);
1528 line
[n_to_copy
] = '\0';
1530 evbuffer_drain(buffer
, extra_drain
);
1533 EVBUFFER_UNLOCK(buffer
);
1536 *n_read_out
= result
? n_to_copy
: 0;
1541 #define EVBUFFER_CHAIN_MAX_AUTO_SIZE 4096
1543 /* Adds data to an event buffer */
1546 evbuffer_add(struct evbuffer
*buf
, const void *data_in
, size_t datlen
)
1548 struct evbuffer_chain
*chain
, *tmp
;
1549 const unsigned char *data
= data_in
;
1550 size_t remain
, to_alloc
;
1555 if (buf
->freeze_end
) {
1558 /* Prevent buf->total_len overflow */
1559 if (datlen
> EV_SIZE_MAX
- buf
->total_len
) {
1565 /* If there are no chains allocated for this buffer, allocate one
1566 * big enough to hold all the data. */
1567 if (chain
== NULL
) {
1568 chain
= evbuffer_chain_new(datlen
);
1571 evbuffer_chain_insert(buf
, chain
);
1574 if ((chain
->flags
& EVBUFFER_IMMUTABLE
) == 0) {
1575 /* Always true for mutable buffers */
1576 EVUTIL_ASSERT(chain
->misalign
>= 0 &&
1577 (ev_uint64_t
)chain
->misalign
<= EVBUFFER_CHAIN_MAX
);
1578 remain
= chain
->buffer_len
- (size_t)chain
->misalign
- chain
->off
;
1579 if (remain
>= datlen
) {
1580 /* there's enough space to hold all the data in the
1581 * current last chain */
1582 memcpy(chain
->buffer
+ chain
->misalign
+ chain
->off
,
1584 chain
->off
+= datlen
;
1585 buf
->total_len
+= datlen
;
1586 buf
->n_add_for_cb
+= datlen
;
1588 } else if (!CHAIN_PINNED(chain
) &&
1589 evbuffer_chain_should_realign(chain
, datlen
)) {
1590 /* we can fit the data into the misalignment */
1591 evbuffer_chain_align(chain
);
1593 memcpy(chain
->buffer
+ chain
->off
, data
, datlen
);
1594 chain
->off
+= datlen
;
1595 buf
->total_len
+= datlen
;
1596 buf
->n_add_for_cb
+= datlen
;
1600 /* we cannot write any data to the last chain */
1604 /* we need to add another chain */
1605 to_alloc
= chain
->buffer_len
;
1606 if (to_alloc
<= EVBUFFER_CHAIN_MAX_AUTO_SIZE
/2)
1608 if (datlen
> to_alloc
)
1610 tmp
= evbuffer_chain_new(to_alloc
);
1615 memcpy(chain
->buffer
+ chain
->misalign
+ chain
->off
,
1617 chain
->off
+= remain
;
1618 buf
->total_len
+= remain
;
1619 buf
->n_add_for_cb
+= remain
;
1625 memcpy(tmp
->buffer
, data
, datlen
);
1627 evbuffer_chain_insert(buf
, tmp
);
1628 buf
->n_add_for_cb
+= datlen
;
1631 evbuffer_invoke_callbacks(buf
);
1634 EVBUFFER_UNLOCK(buf
);
1639 evbuffer_prepend(struct evbuffer
*buf
, const void *data
, size_t datlen
)
1641 struct evbuffer_chain
*chain
, *tmp
;
1646 if (buf
->freeze_start
) {
1649 if (datlen
> EV_SIZE_MAX
- buf
->total_len
) {
1655 if (chain
== NULL
) {
1656 chain
= evbuffer_chain_new(datlen
);
1659 evbuffer_chain_insert(buf
, chain
);
1662 /* we cannot touch immutable buffers */
1663 if ((chain
->flags
& EVBUFFER_IMMUTABLE
) == 0) {
1664 /* Always true for mutable buffers */
1665 EVUTIL_ASSERT(chain
->misalign
>= 0 &&
1666 (ev_uint64_t
)chain
->misalign
<= EVBUFFER_CHAIN_MAX
);
1668 /* If this chain is empty, we can treat it as
1669 * 'empty at the beginning' rather than 'empty at the end' */
1670 if (chain
->off
== 0)
1671 chain
->misalign
= chain
->buffer_len
;
1673 if ((size_t)chain
->misalign
>= datlen
) {
1674 /* we have enough space to fit everything */
1675 memcpy(chain
->buffer
+ chain
->misalign
- datlen
,
1677 chain
->off
+= datlen
;
1678 chain
->misalign
-= datlen
;
1679 buf
->total_len
+= datlen
;
1680 buf
->n_add_for_cb
+= datlen
;
1682 } else if (chain
->misalign
) {
1683 /* we can only fit some of the data. */
1684 memcpy(chain
->buffer
,
1685 (char*)data
+ datlen
- chain
->misalign
,
1686 (size_t)chain
->misalign
);
1687 chain
->off
+= (size_t)chain
->misalign
;
1688 buf
->total_len
+= (size_t)chain
->misalign
;
1689 buf
->n_add_for_cb
+= (size_t)chain
->misalign
;
1690 datlen
-= (size_t)chain
->misalign
;
1691 chain
->misalign
= 0;
1695 /* we need to add another chain */
1696 if ((tmp
= evbuffer_chain_new(datlen
)) == NULL
)
1699 if (buf
->last_with_datap
== &buf
->first
)
1700 buf
->last_with_datap
= &tmp
->next
;
1705 EVUTIL_ASSERT(datlen
<= tmp
->buffer_len
);
1706 tmp
->misalign
= tmp
->buffer_len
- datlen
;
1708 memcpy(tmp
->buffer
+ tmp
->misalign
, data
, datlen
);
1709 buf
->total_len
+= datlen
;
1710 buf
->n_add_for_cb
+= (size_t)chain
->misalign
;
1713 evbuffer_invoke_callbacks(buf
);
1716 EVBUFFER_UNLOCK(buf
);
1720 /** Helper: realigns the memory in chain->buffer so that misalign is 0. */
1722 evbuffer_chain_align(struct evbuffer_chain
*chain
)
1724 EVUTIL_ASSERT(!(chain
->flags
& EVBUFFER_IMMUTABLE
));
1725 EVUTIL_ASSERT(!(chain
->flags
& EVBUFFER_MEM_PINNED_ANY
));
1726 memmove(chain
->buffer
, chain
->buffer
+ chain
->misalign
, chain
->off
);
1727 chain
->misalign
= 0;
1730 #define MAX_TO_COPY_IN_EXPAND 4096
1731 #define MAX_TO_REALIGN_IN_EXPAND 2048
1733 /** Helper: return true iff we should realign chain to fit datalen bytes of
1736 evbuffer_chain_should_realign(struct evbuffer_chain
*chain
,
1739 return chain
->buffer_len
- chain
->off
>= datlen
&&
1740 (chain
->off
< chain
->buffer_len
/ 2) &&
1741 (chain
->off
<= MAX_TO_REALIGN_IN_EXPAND
);
1744 /* Expands the available space in the event buffer to at least datlen, all in
1745 * a single chunk. Return that chunk. */
1746 static struct evbuffer_chain
*
1747 evbuffer_expand_singlechain(struct evbuffer
*buf
, size_t datlen
)
1749 struct evbuffer_chain
*chain
, **chainp
;
1750 struct evbuffer_chain
*result
= NULL
;
1751 ASSERT_EVBUFFER_LOCKED(buf
);
1753 chainp
= buf
->last_with_datap
;
1755 /* XXX If *chainp is no longer writeable, but has enough space in its
1756 * misalign, this might be a bad idea: we could still use *chainp, not
1757 * (*chainp)->next. */
1758 if (*chainp
&& CHAIN_SPACE_LEN(*chainp
) == 0)
1759 chainp
= &(*chainp
)->next
;
1761 /* 'chain' now points to the first chain with writable space (if any)
1762 * We will either use it, realign it, replace it, or resize it. */
1765 if (chain
== NULL
||
1766 (chain
->flags
& (EVBUFFER_IMMUTABLE
|EVBUFFER_MEM_PINNED_ANY
))) {
1767 /* We can't use the last_with_data chain at all. Just add a
1768 * new one that's big enough. */
1772 /* If we can fit all the data, then we don't have to do anything */
1773 if (CHAIN_SPACE_LEN(chain
) >= datlen
) {
1778 /* If the chain is completely empty, just replace it by adding a new
1780 if (chain
->off
== 0) {
1784 /* If the misalignment plus the remaining space fulfills our data
1785 * needs, we could just force an alignment to happen. Afterwards, we
1786 * have enough space. But only do this if we're saving a lot of space
1787 * and not moving too much data. Otherwise the space savings are
1788 * probably offset by the time lost in copying.
1790 if (evbuffer_chain_should_realign(chain
, datlen
)) {
1791 evbuffer_chain_align(chain
);
1796 /* At this point, we can either resize the last chunk with space in
1797 * it, use the next chunk after it, or If we add a new chunk, we waste
1798 * CHAIN_SPACE_LEN(chain) bytes in the former last chunk. If we
1799 * resize, we have to copy chain->off bytes.
1802 /* Would expanding this chunk be affordable and worthwhile? */
1803 if (CHAIN_SPACE_LEN(chain
) < chain
->buffer_len
/ 8 ||
1804 chain
->off
> MAX_TO_COPY_IN_EXPAND
||
1805 (datlen
< EVBUFFER_CHAIN_MAX
&&
1806 EVBUFFER_CHAIN_MAX
- datlen
>= chain
->off
)) {
1807 /* It's not worth resizing this chain. Can the next one be
1809 if (chain
->next
&& CHAIN_SPACE_LEN(chain
->next
) >= datlen
) {
1810 /* Yes, we can just use the next chain (which should
1812 result
= chain
->next
;
1815 /* No; append a new chain (which will free all
1816 * terminal empty chains.) */
1820 /* Okay, we're going to try to resize this chain: Not doing so
1821 * would waste at least 1/8 of its current allocation, and we
1822 * can do so without having to copy more than
1823 * MAX_TO_COPY_IN_EXPAND bytes. */
1824 /* figure out how much space we need */
1825 size_t length
= chain
->off
+ datlen
;
1826 struct evbuffer_chain
*tmp
= evbuffer_chain_new(length
);
1830 /* copy the data over that we had so far */
1831 tmp
->off
= chain
->off
;
1832 memcpy(tmp
->buffer
, chain
->buffer
+ chain
->misalign
,
1834 /* fix up the list */
1835 EVUTIL_ASSERT(*chainp
== chain
);
1836 result
= *chainp
= tmp
;
1838 if (buf
->last
== chain
)
1841 tmp
->next
= chain
->next
;
1842 evbuffer_chain_free(chain
);
1847 result
= evbuffer_chain_insert_new(buf
, datlen
);
1851 EVUTIL_ASSERT(result
);
1852 EVUTIL_ASSERT(CHAIN_SPACE_LEN(result
) >= datlen
);
1857 /* Make sure that datlen bytes are available for writing in the last n
1858 * chains. Never copies or moves data. */
1860 _evbuffer_expand_fast(struct evbuffer
*buf
, size_t datlen
, int n
)
1862 struct evbuffer_chain
*chain
= buf
->last
, *tmp
, *next
;
1866 ASSERT_EVBUFFER_LOCKED(buf
);
1867 EVUTIL_ASSERT(n
>= 2);
1869 if (chain
== NULL
|| (chain
->flags
& EVBUFFER_IMMUTABLE
)) {
1870 /* There is no last chunk, or we can't touch the last chunk.
1871 * Just add a new chunk. */
1872 chain
= evbuffer_chain_new(datlen
);
1876 evbuffer_chain_insert(buf
, chain
);
1880 used
= 0; /* number of chains we're using space in. */
1881 avail
= 0; /* how much space they have. */
1882 /* How many bytes can we stick at the end of buffer as it is? Iterate
1883 * over the chains at the end of the buffer, tring to see how much
1884 * space we have in the first n. */
1885 for (chain
= *buf
->last_with_datap
; chain
; chain
= chain
->next
) {
1887 size_t space
= (size_t) CHAIN_SPACE_LEN(chain
);
1888 EVUTIL_ASSERT(chain
== *buf
->last_with_datap
);
1894 /* No data in chain; realign it. */
1895 chain
->misalign
= 0;
1896 avail
+= chain
->buffer_len
;
1899 if (avail
>= datlen
) {
1900 /* There is already enough space. Just return */
1907 /* There wasn't enough space in the first n chains with space in
1908 * them. Either add a new chain with enough space, or replace all
1909 * empty chains with one that has enough space, depending on n. */
1911 /* The loop ran off the end of the chains before it hit n
1912 * chains; we can add another. */
1913 EVUTIL_ASSERT(chain
== NULL
);
1915 tmp
= evbuffer_chain_new(datlen
- avail
);
1919 buf
->last
->next
= tmp
;
1921 /* (we would only set last_with_data if we added the first
1922 * chain. But if the buffer had no chains, we would have
1923 * just allocated a new chain earlier) */
1926 /* Nuke _all_ the empty chains. */
1927 int rmv_all
= 0; /* True iff we removed last_with_data. */
1928 chain
= *buf
->last_with_datap
;
1930 EVUTIL_ASSERT(chain
== buf
->first
);
1934 /* can't overflow, since only mutable chains have
1935 * huge misaligns. */
1936 avail
= (size_t) CHAIN_SPACE_LEN(chain
);
1937 chain
= chain
->next
;
1941 for (; chain
; chain
= next
) {
1943 EVUTIL_ASSERT(chain
->off
== 0);
1944 evbuffer_chain_free(chain
);
1946 EVUTIL_ASSERT(datlen
>= avail
);
1947 tmp
= evbuffer_chain_new(datlen
- avail
);
1952 buf
->last
= *buf
->last_with_datap
;
1953 (*buf
->last_with_datap
)->next
= NULL
;
1959 buf
->first
= buf
->last
= tmp
;
1960 buf
->last_with_datap
= &buf
->first
;
1962 (*buf
->last_with_datap
)->next
= tmp
;
1970 evbuffer_expand(struct evbuffer
*buf
, size_t datlen
)
1972 struct evbuffer_chain
*chain
;
1975 chain
= evbuffer_expand_singlechain(buf
, datlen
);
1976 EVBUFFER_UNLOCK(buf
);
1977 return chain
? 0 : -1;
1981 * Reads data from a file descriptor into a buffer.
1984 #if defined(_EVENT_HAVE_SYS_UIO_H) || defined(WIN32)
1985 #define USE_IOVEC_IMPL
1988 #ifdef USE_IOVEC_IMPL
1990 #ifdef _EVENT_HAVE_SYS_UIO_H
1991 /* number of iovec we use for writev, fragmentation is going to determine
1992 * how much we end up writing */
1994 #define DEFAULT_WRITE_IOVEC 128
1996 #if defined(UIO_MAXIOV) && UIO_MAXIOV < DEFAULT_WRITE_IOVEC
1997 #define NUM_WRITE_IOVEC UIO_MAXIOV
1998 #elif defined(IOV_MAX) && IOV_MAX < DEFAULT_WRITE_IOVEC
1999 #define NUM_WRITE_IOVEC IOV_MAX
2001 #define NUM_WRITE_IOVEC DEFAULT_WRITE_IOVEC
2004 #define IOV_TYPE struct iovec
2005 #define IOV_PTR_FIELD iov_base
2006 #define IOV_LEN_FIELD iov_len
2007 #define IOV_LEN_TYPE size_t
2009 #define NUM_WRITE_IOVEC 16
2010 #define IOV_TYPE WSABUF
2011 #define IOV_PTR_FIELD buf
2012 #define IOV_LEN_FIELD len
2013 #define IOV_LEN_TYPE unsigned long
2016 #define NUM_READ_IOVEC 4
2018 #define EVBUFFER_MAX_READ 4096
2020 /** Helper function to figure out which space to use for reading data into
2021 an evbuffer. Internal use only.
2023 @param buf The buffer to read into
2024 @param howmuch How much we want to read.
2025 @param vecs An array of two or more iovecs or WSABUFs.
2026 @param n_vecs_avail The length of vecs
2027 @param chainp A pointer to a variable to hold the first chain we're
2029 @param exact Boolean: if true, we do not provide more than 'howmuch'
2030 space in the vectors, even if more space is available.
2031 @return The number of buffers we're using.
2034 _evbuffer_read_setup_vecs(struct evbuffer
*buf
, ev_ssize_t howmuch
,
2035 struct evbuffer_iovec
*vecs
, int n_vecs_avail
,
2036 struct evbuffer_chain
***chainp
, int exact
)
2038 struct evbuffer_chain
*chain
;
2039 struct evbuffer_chain
**firstchainp
;
2042 ASSERT_EVBUFFER_LOCKED(buf
);
2048 /* Let firstchain be the first chain with any space on it */
2049 firstchainp
= buf
->last_with_datap
;
2050 if (CHAIN_SPACE_LEN(*firstchainp
) == 0) {
2051 firstchainp
= &(*firstchainp
)->next
;
2054 chain
= *firstchainp
;
2055 for (i
= 0; i
< n_vecs_avail
&& so_far
< (size_t)howmuch
; ++i
) {
2056 size_t avail
= (size_t) CHAIN_SPACE_LEN(chain
);
2057 if (avail
> (howmuch
- so_far
) && exact
)
2058 avail
= howmuch
- so_far
;
2059 vecs
[i
].iov_base
= CHAIN_SPACE_PTR(chain
);
2060 vecs
[i
].iov_len
= avail
;
2062 chain
= chain
->next
;
2065 *chainp
= firstchainp
;
2070 get_n_bytes_readable_on_socket(evutil_socket_t fd
)
2072 #if defined(FIONREAD) && defined(WIN32)
2073 unsigned long lng
= EVBUFFER_MAX_READ
;
2074 if (ioctlsocket(fd
, FIONREAD
, &lng
) < 0)
2076 /* Can overflow, but mostly harmlessly. XXXX */
2078 #elif defined(FIONREAD)
2079 int n
= EVBUFFER_MAX_READ
;
2080 if (ioctl(fd
, FIONREAD
, &n
) < 0)
2084 return EVBUFFER_MAX_READ
;
2088 /* TODO(niels): should this function return ev_ssize_t and take ev_ssize_t
2091 evbuffer_read(struct evbuffer
*buf
, evutil_socket_t fd
, int howmuch
)
2093 struct evbuffer_chain
**chainp
;
2097 #ifdef USE_IOVEC_IMPL
2098 int nvecs
, i
, remaining
;
2100 struct evbuffer_chain
*chain
;
2106 if (buf
->freeze_end
) {
2111 n
= get_n_bytes_readable_on_socket(fd
);
2112 if (n
<= 0 || n
> EVBUFFER_MAX_READ
)
2113 n
= EVBUFFER_MAX_READ
;
2114 if (howmuch
< 0 || howmuch
> n
)
2117 #ifdef USE_IOVEC_IMPL
2118 /* Since we can use iovecs, we're willing to use the last
2119 * NUM_READ_IOVEC chains. */
2120 if (_evbuffer_expand_fast(buf
, howmuch
, NUM_READ_IOVEC
) == -1) {
2124 IOV_TYPE vecs
[NUM_READ_IOVEC
];
2125 #ifdef _EVBUFFER_IOVEC_IS_NATIVE
2126 nvecs
= _evbuffer_read_setup_vecs(buf
, howmuch
, vecs
,
2127 NUM_READ_IOVEC
, &chainp
, 1);
2129 /* We aren't using the native struct iovec. Therefore,
2131 struct evbuffer_iovec ev_vecs
[NUM_READ_IOVEC
];
2132 nvecs
= _evbuffer_read_setup_vecs(buf
, howmuch
, ev_vecs
, 2,
2135 for (i
=0; i
< nvecs
; ++i
)
2136 WSABUF_FROM_EVBUFFER_IOV(&vecs
[i
], &ev_vecs
[i
]);
2143 if (WSARecv(fd
, vecs
, nvecs
, &bytesRead
, &flags
, NULL
, NULL
)) {
2144 /* The read failed. It might be a close,
2145 * or it might be an error. */
2146 if (WSAGetLastError() == WSAECONNABORTED
)
2154 n
= readv(fd
, vecs
, nvecs
);
2158 #else /*!USE_IOVEC_IMPL*/
2159 /* If we don't have FIONREAD, we might waste some space here */
2160 /* XXX we _will_ waste some space here if there is any space left
2161 * over on buf->last. */
2162 if ((chain
= evbuffer_expand_singlechain(buf
, howmuch
)) == NULL
) {
2167 /* We can append new data at this point */
2168 p
= chain
->buffer
+ chain
->misalign
+ chain
->off
;
2171 n
= read(fd
, p
, howmuch
);
2173 n
= recv(fd
, p
, howmuch
, 0);
2175 #endif /* USE_IOVEC_IMPL */
2186 #ifdef USE_IOVEC_IMPL
2188 for (i
=0; i
< nvecs
; ++i
) {
2189 /* can't overflow, since only mutable chains have
2190 * huge misaligns. */
2191 size_t space
= (size_t) CHAIN_SPACE_LEN(*chainp
);
2192 /* XXXX This is a kludge that can waste space in perverse
2194 if (space
> EVBUFFER_CHAIN_MAX
)
2195 space
= EVBUFFER_CHAIN_MAX
;
2196 if ((ev_ssize_t
)space
< remaining
) {
2197 (*chainp
)->off
+= space
;
2198 remaining
-= (int)space
;
2200 (*chainp
)->off
+= remaining
;
2201 buf
->last_with_datap
= chainp
;
2204 chainp
= &(*chainp
)->next
;
2208 advance_last_with_data(buf
);
2210 buf
->total_len
+= n
;
2211 buf
->n_add_for_cb
+= n
;
2213 /* Tell someone about changes in this buffer */
2214 evbuffer_invoke_callbacks(buf
);
2217 EVBUFFER_UNLOCK(buf
);
2223 evbuffer_readfile(struct evbuffer
*buf
, evutil_socket_t fd
, ev_ssize_t howmuch
)
2227 struct evbuffer_iovec v
[2];
2231 if (buf
->freeze_end
) {
2240 /* XXX we _will_ waste some space here if there is any space left
2241 * over on buf->last. */
2242 nchains
= evbuffer_reserve_space(buf
, howmuch
, v
, 2);
2243 if (nchains
< 1 || nchains
> 2) {
2247 n
= read((int)fd
, v
[0].iov_base
, (unsigned int)v
[0].iov_len
);
2252 v
[0].iov_len
= (IOV_LEN_TYPE
) n
; /* XXXX another problem with big n.*/
2254 n
= read((int)fd
, v
[1].iov_base
, (unsigned int)v
[1].iov_len
);
2256 result
= (unsigned long) v
[0].iov_len
;
2257 evbuffer_commit_space(buf
, v
, 1);
2262 evbuffer_commit_space(buf
, v
, nchains
);
2266 EVBUFFER_UNLOCK(buf
);
2271 #ifdef USE_IOVEC_IMPL
2273 evbuffer_write_iovec(struct evbuffer
*buffer
, evutil_socket_t fd
,
2276 IOV_TYPE iov
[NUM_WRITE_IOVEC
];
2277 struct evbuffer_chain
*chain
= buffer
->first
;
2283 ASSERT_EVBUFFER_LOCKED(buffer
);
2284 /* XXX make this top out at some maximal data length? if the
2285 * buffer has (say) 1MB in it, split over 128 chains, there's
2286 * no way it all gets written in one go. */
2287 while (chain
!= NULL
&& i
< NUM_WRITE_IOVEC
&& howmuch
) {
2289 /* we cannot write the file info via writev */
2290 if (chain
->flags
& EVBUFFER_SENDFILE
)
2293 iov
[i
].IOV_PTR_FIELD
= (void *) (chain
->buffer
+ chain
->misalign
);
2294 if ((size_t)howmuch
>= chain
->off
) {
2295 /* XXXcould be problematic when windows supports mmap*/
2296 iov
[i
++].IOV_LEN_FIELD
= (IOV_LEN_TYPE
)chain
->off
;
2297 howmuch
-= chain
->off
;
2299 /* XXXcould be problematic when windows supports mmap*/
2300 iov
[i
++].IOV_LEN_FIELD
= (IOV_LEN_TYPE
)howmuch
;
2303 chain
= chain
->next
;
2310 if (WSASend(fd
, iov
, i
, &bytesSent
, 0, NULL
, NULL
))
2316 n
= writev(fd
, iov
, i
);
2324 evbuffer_write_sendfile(struct evbuffer
*buffer
, evutil_socket_t fd
,
2327 struct evbuffer_chain
*chain
= buffer
->first
;
2328 struct evbuffer_chain_fd
*info
=
2329 EVBUFFER_CHAIN_EXTRA(struct evbuffer_chain_fd
, chain
);
2330 #if defined(SENDFILE_IS_MACOSX) || defined(SENDFILE_IS_FREEBSD)
2332 off_t len
= chain
->off
;
2333 #elif defined(SENDFILE_IS_LINUX) || defined(SENDFILE_IS_SOLARIS)
2335 off_t offset
= chain
->misalign
;
2338 ASSERT_EVBUFFER_LOCKED(buffer
);
2340 #if defined(SENDFILE_IS_MACOSX)
2341 res
= sendfile(info
->fd
, fd
, chain
->misalign
, &len
, NULL
, 0);
2342 if (res
== -1 && !EVUTIL_ERR_RW_RETRIABLE(errno
))
2346 #elif defined(SENDFILE_IS_FREEBSD)
2347 res
= sendfile(info
->fd
, fd
, chain
->misalign
, chain
->off
, NULL
, &len
, 0);
2348 if (res
== -1 && !EVUTIL_ERR_RW_RETRIABLE(errno
))
2352 #elif defined(SENDFILE_IS_LINUX)
2353 /* TODO(niels): implement splice */
2354 res
= sendfile(fd
, info
->fd
, &offset
, chain
->off
);
2355 if (res
== -1 && EVUTIL_ERR_RW_RETRIABLE(errno
)) {
2356 /* if this is EAGAIN or EINTR return 0; otherwise, -1 */
2360 #elif defined(SENDFILE_IS_SOLARIS)
2362 const off_t offset_orig
= offset
;
2363 res
= sendfile(fd
, info
->fd
, &offset
, chain
->off
);
2364 if (res
== -1 && EVUTIL_ERR_RW_RETRIABLE(errno
)) {
2365 if (offset
- offset_orig
)
2366 return offset
- offset_orig
;
2367 /* if this is EAGAIN or EINTR and no bytes were
2368 * written, return 0 */
2378 evbuffer_write_atmost(struct evbuffer
*buffer
, evutil_socket_t fd
,
2383 EVBUFFER_LOCK(buffer
);
2385 if (buffer
->freeze_start
) {
2389 if (howmuch
< 0 || (size_t)howmuch
> buffer
->total_len
)
2390 howmuch
= buffer
->total_len
;
2394 struct evbuffer_chain
*chain
= buffer
->first
;
2395 if (chain
!= NULL
&& (chain
->flags
& EVBUFFER_SENDFILE
))
2396 n
= evbuffer_write_sendfile(buffer
, fd
, howmuch
);
2399 #ifdef USE_IOVEC_IMPL
2400 n
= evbuffer_write_iovec(buffer
, fd
, howmuch
);
2401 #elif defined(WIN32)
2402 /* XXX(nickm) Don't disable this code until we know if
2403 * the WSARecv code above works. */
2404 void *p
= evbuffer_pullup(buffer
, howmuch
);
2405 EVUTIL_ASSERT(p
|| !howmuch
);
2406 n
= send(fd
, p
, howmuch
, 0);
2408 void *p
= evbuffer_pullup(buffer
, howmuch
);
2409 EVUTIL_ASSERT(p
|| !howmuch
);
2410 n
= write(fd
, p
, howmuch
);
2418 evbuffer_drain(buffer
, n
);
2421 EVBUFFER_UNLOCK(buffer
);
2426 evbuffer_write(struct evbuffer
*buffer
, evutil_socket_t fd
)
2428 return evbuffer_write_atmost(buffer
, fd
, -1);
2432 evbuffer_find(struct evbuffer
*buffer
, const unsigned char *what
, size_t len
)
2434 unsigned char *search
;
2435 struct evbuffer_ptr ptr
;
2437 EVBUFFER_LOCK(buffer
);
2439 ptr
= evbuffer_search(buffer
, (const char *)what
, len
, NULL
);
2443 search
= evbuffer_pullup(buffer
, ptr
.pos
+ len
);
2447 EVBUFFER_UNLOCK(buffer
);
2452 evbuffer_ptr_set(struct evbuffer
*buf
, struct evbuffer_ptr
*pos
,
2453 size_t position
, enum evbuffer_ptr_how how
)
2455 size_t left
= position
;
2456 struct evbuffer_chain
*chain
= NULL
;
2461 case EVBUFFER_PTR_SET
:
2463 pos
->pos
= position
;
2466 case EVBUFFER_PTR_ADD
:
2467 /* this avoids iterating over all previous chains if
2468 we just want to advance the position */
2469 if (pos
->pos
< 0 || EV_SIZE_MAX
- position
< (size_t)pos
->pos
) {
2470 EVBUFFER_UNLOCK(buf
);
2473 chain
= pos
->_internal
.chain
;
2474 pos
->pos
+= position
;
2475 position
= pos
->_internal
.pos_in_chain
;
2479 EVUTIL_ASSERT(EV_SIZE_MAX
- left
>= position
);
2480 while (chain
&& position
+ left
>= chain
->off
) {
2481 left
-= chain
->off
- position
;
2482 chain
= chain
->next
;
2486 pos
->_internal
.chain
= chain
;
2487 pos
->_internal
.pos_in_chain
= position
+ left
;
2489 pos
->_internal
.chain
= NULL
;
2493 EVBUFFER_UNLOCK(buf
);
2495 return chain
!= NULL
? 0 : -1;
2499 Compare the bytes in buf at position pos to the len bytes in mem. Return
2500 less than 0, 0, or greater than 0 as memcmp.
2503 evbuffer_ptr_memcmp(const struct evbuffer
*buf
, const struct evbuffer_ptr
*pos
,
2504 const char *mem
, size_t len
)
2506 struct evbuffer_chain
*chain
;
2510 ASSERT_EVBUFFER_LOCKED(buf
);
2513 EV_SIZE_MAX
- len
< (size_t)pos
->pos
||
2514 pos
->pos
+ len
> buf
->total_len
)
2517 chain
= pos
->_internal
.chain
;
2518 position
= pos
->_internal
.pos_in_chain
;
2519 while (len
&& chain
) {
2520 size_t n_comparable
;
2521 if (len
+ position
> chain
->off
)
2522 n_comparable
= chain
->off
- position
;
2525 r
= memcmp(chain
->buffer
+ chain
->misalign
+ position
, mem
,
2529 mem
+= n_comparable
;
2530 len
-= n_comparable
;
2532 chain
= chain
->next
;
2539 evbuffer_search(struct evbuffer
*buffer
, const char *what
, size_t len
, const struct evbuffer_ptr
*start
)
2541 return evbuffer_search_range(buffer
, what
, len
, start
, NULL
);
2545 evbuffer_search_range(struct evbuffer
*buffer
, const char *what
, size_t len
, const struct evbuffer_ptr
*start
, const struct evbuffer_ptr
*end
)
2547 struct evbuffer_ptr pos
;
2548 struct evbuffer_chain
*chain
, *last_chain
= NULL
;
2549 const unsigned char *p
;
2552 EVBUFFER_LOCK(buffer
);
2555 memcpy(&pos
, start
, sizeof(pos
));
2556 chain
= pos
._internal
.chain
;
2559 chain
= pos
._internal
.chain
= buffer
->first
;
2560 pos
._internal
.pos_in_chain
= 0;
2564 last_chain
= end
->_internal
.chain
;
2566 if (!len
|| len
> EV_SSIZE_MAX
)
2572 const unsigned char *start_at
=
2573 chain
->buffer
+ chain
->misalign
+
2574 pos
._internal
.pos_in_chain
;
2575 p
= memchr(start_at
, first
,
2576 chain
->off
- pos
._internal
.pos_in_chain
);
2578 pos
.pos
+= p
- start_at
;
2579 pos
._internal
.pos_in_chain
+= p
- start_at
;
2580 if (!evbuffer_ptr_memcmp(buffer
, &pos
, what
, len
)) {
2581 if (end
&& pos
.pos
+ (ev_ssize_t
)len
> end
->pos
)
2587 ++pos
._internal
.pos_in_chain
;
2588 if (pos
._internal
.pos_in_chain
== chain
->off
) {
2589 chain
= pos
._internal
.chain
= chain
->next
;
2590 pos
._internal
.pos_in_chain
= 0;
2593 if (chain
== last_chain
)
2595 pos
.pos
+= chain
->off
- pos
._internal
.pos_in_chain
;
2596 chain
= pos
._internal
.chain
= chain
->next
;
2597 pos
._internal
.pos_in_chain
= 0;
2603 pos
._internal
.chain
= NULL
;
2605 EVBUFFER_UNLOCK(buffer
);
2610 evbuffer_peek(struct evbuffer
*buffer
, ev_ssize_t len
,
2611 struct evbuffer_ptr
*start_at
,
2612 struct evbuffer_iovec
*vec
, int n_vec
)
2614 struct evbuffer_chain
*chain
;
2616 ev_ssize_t len_so_far
= 0;
2618 EVBUFFER_LOCK(buffer
);
2621 chain
= start_at
->_internal
.chain
;
2622 len_so_far
= chain
->off
2623 - start_at
->_internal
.pos_in_chain
;
2626 vec
[0].iov_base
= chain
->buffer
+ chain
->misalign
2627 + start_at
->_internal
.pos_in_chain
;
2628 vec
[0].iov_len
= len_so_far
;
2630 chain
= chain
->next
;
2632 chain
= buffer
->first
;
2635 if (n_vec
== 0 && len
< 0) {
2636 /* If no vectors are provided and they asked for "everything",
2637 * pretend they asked for the actual available amount. */
2638 len
= buffer
->total_len
;
2640 len
-= start_at
->pos
;
2645 if (len
>= 0 && len_so_far
>= len
)
2648 vec
[idx
].iov_base
= chain
->buffer
+ chain
->misalign
;
2649 vec
[idx
].iov_len
= chain
->off
;
2654 len_so_far
+= chain
->off
;
2655 chain
= chain
->next
;
2658 EVBUFFER_UNLOCK(buffer
);
2665 evbuffer_add_vprintf(struct evbuffer
*buf
, const char *fmt
, va_list ap
)
2669 int sz
, result
= -1;
2671 struct evbuffer_chain
*chain
;
2676 if (buf
->freeze_end
) {
2680 /* make sure that at least some space is available */
2681 if ((chain
= evbuffer_expand_singlechain(buf
, 64)) == NULL
)
2686 size_t used
= chain
->misalign
+ chain
->off
;
2687 buffer
= (char *)chain
->buffer
+ chain
->misalign
+ chain
->off
;
2688 EVUTIL_ASSERT(chain
->buffer_len
>= used
);
2689 space
= chain
->buffer_len
- used
;
2691 buffer
= (char*) CHAIN_SPACE_PTR(chain
);
2692 space
= (size_t) CHAIN_SPACE_LEN(chain
);
2695 #define va_copy(dst, src) memcpy(&(dst), &(src), sizeof(va_list))
2699 sz
= evutil_vsnprintf(buffer
, space
, fmt
, aq
);
2705 if (INT_MAX
>= EVBUFFER_CHAIN_MAX
&&
2706 (size_t)sz
>= EVBUFFER_CHAIN_MAX
)
2708 if ((size_t)sz
< space
) {
2710 buf
->total_len
+= sz
;
2711 buf
->n_add_for_cb
+= sz
;
2713 advance_last_with_data(buf
);
2714 evbuffer_invoke_callbacks(buf
);
2718 if ((chain
= evbuffer_expand_singlechain(buf
, sz
+ 1)) == NULL
)
2724 EVBUFFER_UNLOCK(buf
);
2729 evbuffer_add_printf(struct evbuffer
*buf
, const char *fmt
, ...)
2735 res
= evbuffer_add_vprintf(buf
, fmt
, ap
);
2742 evbuffer_add_reference(struct evbuffer
*outbuf
,
2743 const void *data
, size_t datlen
,
2744 evbuffer_ref_cleanup_cb cleanupfn
, void *extra
)
2746 struct evbuffer_chain
*chain
;
2747 struct evbuffer_chain_reference
*info
;
2750 chain
= evbuffer_chain_new(sizeof(struct evbuffer_chain_reference
));
2753 chain
->flags
|= EVBUFFER_REFERENCE
| EVBUFFER_IMMUTABLE
;
2754 chain
->buffer
= (u_char
*)data
;
2755 chain
->buffer_len
= datlen
;
2756 chain
->off
= datlen
;
2758 info
= EVBUFFER_CHAIN_EXTRA(struct evbuffer_chain_reference
, chain
);
2759 info
->cleanupfn
= cleanupfn
;
2760 info
->extra
= extra
;
2762 EVBUFFER_LOCK(outbuf
);
2763 if (outbuf
->freeze_end
) {
2764 /* don't call chain_free; we do not want to actually invoke
2765 * the cleanup function */
2769 evbuffer_chain_insert(outbuf
, chain
);
2770 outbuf
->n_add_for_cb
+= datlen
;
2772 evbuffer_invoke_callbacks(outbuf
);
2776 EVBUFFER_UNLOCK(outbuf
);
2781 /* TODO(niels): maybe we don't want to own the fd, however, in that
2782 * case, we should dup it - dup is cheap. Perhaps, we should use a
2785 /* TODO(niels): we may want to add to automagically convert to mmap, in
2786 * case evbuffer_remove() or evbuffer_pullup() are being used.
2789 evbuffer_add_file(struct evbuffer
*outbuf
, int fd
,
2790 ev_off_t offset
, ev_off_t length
)
2792 #if defined(USE_SENDFILE) || defined(_EVENT_HAVE_MMAP)
2793 struct evbuffer_chain
*chain
;
2794 struct evbuffer_chain_fd
*info
;
2796 #if defined(USE_SENDFILE)
2797 int sendfile_okay
= 1;
2801 if (offset
< 0 || length
< 0 ||
2802 ((ev_uint64_t
)length
> EVBUFFER_CHAIN_MAX
) ||
2803 (ev_uint64_t
)offset
> (ev_uint64_t
)(EVBUFFER_CHAIN_MAX
- length
))
2806 #if defined(USE_SENDFILE)
2808 EVBUFFER_LOCK(outbuf
);
2809 sendfile_okay
= outbuf
->flags
& EVBUFFER_FLAG_DRAINS_TO_FD
;
2810 EVBUFFER_UNLOCK(outbuf
);
2813 if (use_sendfile
&& sendfile_okay
) {
2814 chain
= evbuffer_chain_new(sizeof(struct evbuffer_chain_fd
));
2815 if (chain
== NULL
) {
2816 event_warn("%s: out of memory", __func__
);
2820 chain
->flags
|= EVBUFFER_SENDFILE
| EVBUFFER_IMMUTABLE
;
2821 chain
->buffer
= NULL
; /* no reading possible */
2822 chain
->buffer_len
= length
+ offset
;
2823 chain
->off
= length
;
2824 chain
->misalign
= offset
;
2826 info
= EVBUFFER_CHAIN_EXTRA(struct evbuffer_chain_fd
, chain
);
2829 EVBUFFER_LOCK(outbuf
);
2830 if (outbuf
->freeze_end
) {
2834 outbuf
->n_add_for_cb
+= length
;
2835 evbuffer_chain_insert(outbuf
, chain
);
2839 #if defined(_EVENT_HAVE_MMAP)
2841 void *mapped
= mmap(NULL
, length
+ offset
, PROT_READ
,
2850 /* some mmap implementations require offset to be a multiple of
2851 * the page size. most users of this api, are likely to use 0
2852 * so mapping everything is not likely to be a problem.
2853 * TODO(niels): determine page size and round offset to that
2854 * page size to avoid mapping too much memory.
2856 if (mapped
== MAP_FAILED
) {
2857 event_warn("%s: mmap(%d, %d, %zu) failed",
2858 __func__
, fd
, 0, (size_t)(offset
+ length
));
2861 chain
= evbuffer_chain_new(sizeof(struct evbuffer_chain_fd
));
2862 if (chain
== NULL
) {
2863 event_warn("%s: out of memory", __func__
);
2864 munmap(mapped
, length
);
2868 chain
->flags
|= EVBUFFER_MMAP
| EVBUFFER_IMMUTABLE
;
2869 chain
->buffer
= mapped
;
2870 chain
->buffer_len
= length
+ offset
;
2871 chain
->off
= length
+ offset
;
2873 info
= EVBUFFER_CHAIN_EXTRA(struct evbuffer_chain_fd
, chain
);
2876 EVBUFFER_LOCK(outbuf
);
2877 if (outbuf
->freeze_end
) {
2879 evbuffer_chain_free(chain
);
2882 outbuf
->n_add_for_cb
+= length
;
2884 evbuffer_chain_insert(outbuf
, chain
);
2886 /* we need to subtract whatever we don't need */
2887 evbuffer_drain(outbuf
, offset
);
2892 /* the default implementation */
2893 struct evbuffer
*tmp
= evbuffer_new();
2900 #define lseek _lseeki64
2902 if (lseek(fd
, offset
, SEEK_SET
) == -1) {
2907 /* we add everything to a temporary buffer, so that we
2908 * can abort without side effects if the read fails.
2911 ev_ssize_t to_read
= length
> EV_SSIZE_MAX
? EV_SSIZE_MAX
: (ev_ssize_t
)length
;
2912 read
= evbuffer_readfile(tmp
, fd
, to_read
);
2921 EVBUFFER_LOCK(outbuf
);
2922 if (outbuf
->freeze_end
) {
2926 evbuffer_add_buffer(outbuf
, tmp
);
2930 #define close _close
2937 evbuffer_invoke_callbacks(outbuf
);
2938 EVBUFFER_UNLOCK(outbuf
);
2945 evbuffer_setcb(struct evbuffer
*buffer
, evbuffer_cb cb
, void *cbarg
)
2947 EVBUFFER_LOCK(buffer
);
2949 if (!TAILQ_EMPTY(&buffer
->callbacks
))
2950 evbuffer_remove_all_callbacks(buffer
);
2953 struct evbuffer_cb_entry
*ent
=
2954 evbuffer_add_cb(buffer
, NULL
, cbarg
);
2955 ent
->cb
.cb_obsolete
= cb
;
2956 ent
->flags
|= EVBUFFER_CB_OBSOLETE
;
2958 EVBUFFER_UNLOCK(buffer
);
2961 struct evbuffer_cb_entry
*
2962 evbuffer_add_cb(struct evbuffer
*buffer
, evbuffer_cb_func cb
, void *cbarg
)
2964 struct evbuffer_cb_entry
*e
;
2965 if (! (e
= mm_calloc(1, sizeof(struct evbuffer_cb_entry
))))
2967 EVBUFFER_LOCK(buffer
);
2970 e
->flags
= EVBUFFER_CB_ENABLED
;
2971 TAILQ_INSERT_HEAD(&buffer
->callbacks
, e
, next
);
2972 EVBUFFER_UNLOCK(buffer
);
2977 evbuffer_remove_cb_entry(struct evbuffer
*buffer
,
2978 struct evbuffer_cb_entry
*ent
)
2980 EVBUFFER_LOCK(buffer
);
2981 TAILQ_REMOVE(&buffer
->callbacks
, ent
, next
);
2982 EVBUFFER_UNLOCK(buffer
);
2988 evbuffer_remove_cb(struct evbuffer
*buffer
, evbuffer_cb_func cb
, void *cbarg
)
2990 struct evbuffer_cb_entry
*cbent
;
2992 EVBUFFER_LOCK(buffer
);
2993 TAILQ_FOREACH(cbent
, &buffer
->callbacks
, next
) {
2994 if (cb
== cbent
->cb
.cb_func
&& cbarg
== cbent
->cbarg
) {
2995 result
= evbuffer_remove_cb_entry(buffer
, cbent
);
3000 EVBUFFER_UNLOCK(buffer
);
3005 evbuffer_cb_set_flags(struct evbuffer
*buffer
,
3006 struct evbuffer_cb_entry
*cb
, ev_uint32_t flags
)
3008 /* the user isn't allowed to mess with these. */
3009 flags
&= ~EVBUFFER_CB_INTERNAL_FLAGS
;
3010 EVBUFFER_LOCK(buffer
);
3012 EVBUFFER_UNLOCK(buffer
);
3017 evbuffer_cb_clear_flags(struct evbuffer
*buffer
,
3018 struct evbuffer_cb_entry
*cb
, ev_uint32_t flags
)
3020 /* the user isn't allowed to mess with these. */
3021 flags
&= ~EVBUFFER_CB_INTERNAL_FLAGS
;
3022 EVBUFFER_LOCK(buffer
);
3023 cb
->flags
&= ~flags
;
3024 EVBUFFER_UNLOCK(buffer
);
3029 evbuffer_freeze(struct evbuffer
*buffer
, int start
)
3031 EVBUFFER_LOCK(buffer
);
3033 buffer
->freeze_start
= 1;
3035 buffer
->freeze_end
= 1;
3036 EVBUFFER_UNLOCK(buffer
);
3041 evbuffer_unfreeze(struct evbuffer
*buffer
, int start
)
3043 EVBUFFER_LOCK(buffer
);
3045 buffer
->freeze_start
= 0;
3047 buffer
->freeze_end
= 0;
3048 EVBUFFER_UNLOCK(buffer
);
3054 evbuffer_cb_suspend(struct evbuffer
*buffer
, struct evbuffer_cb_entry
*cb
)
3056 if (!(cb
->flags
& EVBUFFER_CB_SUSPENDED
)) {
3057 cb
->size_before_suspend
= evbuffer_get_length(buffer
);
3058 cb
->flags
|= EVBUFFER_CB_SUSPENDED
;
3063 evbuffer_cb_unsuspend(struct evbuffer
*buffer
, struct evbuffer_cb_entry
*cb
)
3065 if ((cb
->flags
& EVBUFFER_CB_SUSPENDED
)) {
3066 unsigned call
= (cb
->flags
& EVBUFFER_CB_CALL_ON_UNSUSPEND
);
3067 size_t sz
= cb
->size_before_suspend
;
3068 cb
->flags
&= ~(EVBUFFER_CB_SUSPENDED
|
3069 EVBUFFER_CB_CALL_ON_UNSUSPEND
);
3070 cb
->size_before_suspend
= 0;
3071 if (call
&& (cb
->flags
& EVBUFFER_CB_ENABLED
)) {
3072 cb
->cb(buffer
, sz
, evbuffer_get_length(buffer
), cb
->cbarg
);
3078 /* These hooks are exposed so that the unit tests can temporarily disable
3079 * sendfile support in order to test mmap, or both to test linear
3080 * access. Don't use it; if we need to add a way to disable sendfile support
3081 * in the future, it will probably be via an alternate version of
3082 * evbuffer_add_file() with a 'flags' argument.
3084 int _evbuffer_testing_use_sendfile(void);
3085 int _evbuffer_testing_use_mmap(void);
3086 int _evbuffer_testing_use_linear_file_access(void);
3089 _evbuffer_testing_use_sendfile(void)
3096 #ifdef _EVENT_HAVE_MMAP
3102 _evbuffer_testing_use_mmap(void)
3108 #ifdef _EVENT_HAVE_MMAP
3115 _evbuffer_testing_use_linear_file_access(void)
3120 #ifdef _EVENT_HAVE_MMAP