2 * spillbuf.c : an in-memory buffer that can spill to disk
4 * ====================================================================
5 * Licensed to the Apache Software Foundation (ASF) under one
6 * or more contributor license agreements. See the NOTICE file
7 * distributed with this work for additional information
8 * regarding copyright ownership. The ASF licenses this file
9 * to you under the Apache License, Version 2.0 (the
10 * "License"); you may not use this file except in compliance
11 * with the License. You may obtain a copy of the License at
13 * http://www.apache.org/licenses/LICENSE-2.0
15 * Unless required by applicable law or agreed to in writing,
16 * software distributed under the License is distributed on an
17 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
18 * KIND, either express or implied. See the License for the
19 * specific language governing permissions and limitations
21 * ====================================================================
24 #include <apr_file_io.h>
27 #include "svn_pools.h"
29 #include "private/svn_subr_private.h"
36 struct memblock_t
*next
;
40 struct svn_spillbuf_t
{
41 /* Pool for allocating blocks and the spill file. */
44 /* Size of in-memory blocks. */
47 /* Maximum in-memory size; start spilling when we reach this size. */
50 /* The amount of content in memory. */
51 apr_size_t memory_size
;
53 /* HEAD points to the first block of the linked list of buffers.
54 TAIL points to the last block, for quickly appending more blocks
55 to the overall list. */
56 struct memblock_t
*head
;
57 struct memblock_t
*tail
;
59 /* Available blocks for storing pending data. These were allocated
60 previously, then the data consumed and returned to this list. */
61 struct memblock_t
*avail
;
63 /* When a block is borrowed for reading, it is listed here. */
64 struct memblock_t
*out_for_reading
;
66 /* Once MEMORY_SIZE exceeds SPILL_SIZE, then arriving content will be
67 appended to the (temporary) file indicated by SPILL. */
70 /* As we consume content from SPILL, this value indicates where we
71 will begin reading. */
72 apr_off_t spill_start
;
74 /* How much content remains in SPILL. */
75 svn_filesize_t spill_size
;
79 struct svn_spillbuf_reader_t
{
80 /* Embed the spill-buffer within the reader. */
81 struct svn_spillbuf_t buf
;
83 /* When we read content from the underlying spillbuf, these fields store
84 the ptr/len pair. The ptr will be incremented as we "read" out of this
85 buffer since we don't have to retain the original pointer (it is
86 managed inside of the spillbuf). */
90 /* If a write comes in, then we may need to save content from our
91 borrowed buffer (since that buffer may be destroyed by our call into
92 the spillbuf code). Note that we retain the original pointer since
93 this buffer is allocated by the reader code and re-used. The SAVE_POS
94 field indicates the current position within this save buffer. The
95 SAVE_LEN field describes how much content is present. */
103 svn_spillbuf__create(apr_size_t blocksize
,
105 apr_pool_t
*result_pool
)
107 svn_spillbuf_t
*buf
= apr_pcalloc(result_pool
, sizeof(*buf
));
109 buf
->pool
= result_pool
;
110 buf
->blocksize
= blocksize
;
111 buf
->maxsize
= maxsize
;
112 /* Note: changes here should also go into svn_spillbuf__reader_create() */
119 svn_spillbuf__get_size(const svn_spillbuf_t
*buf
)
121 return buf
->memory_size
+ buf
->spill_size
;
125 /* Get a memblock from the spill-buffer. It will be the block that we
126 passed out for reading, come from the free list, or allocated. */
127 static struct memblock_t
*
128 get_buffer(svn_spillbuf_t
*buf
)
130 struct memblock_t
*mem
= buf
->out_for_reading
;
134 buf
->out_for_reading
= NULL
;
138 if (buf
->avail
== NULL
)
140 mem
= apr_palloc(buf
->pool
, sizeof(*mem
));
141 mem
->data
= apr_palloc(buf
->pool
, buf
->blocksize
);
146 buf
->avail
= mem
->next
;
151 /* Return MEM to the list of available buffers in BUF. */
153 return_buffer(svn_spillbuf_t
*buf
,
154 struct memblock_t
*mem
)
156 mem
->next
= buf
->avail
;
162 svn_spillbuf__write(svn_spillbuf_t
*buf
,
165 apr_pool_t
*scratch_pool
)
167 struct memblock_t
*mem
;
169 /* We do not (yet) have a spill file, but the amount stored in memory
170 will grow too large. Create the file and place the pending data into
171 the temporary file. */
172 if (buf
->spill
== NULL
173 && (buf
->memory_size
+ len
) > buf
->maxsize
)
175 SVN_ERR(svn_io_open_unique_file3(&buf
->spill
,
176 NULL
/* temp_path */,
178 svn_io_file_del_on_close
,
179 buf
->pool
, scratch_pool
));
182 /* Once a spill file has been constructed, then we need to put all
183 arriving data into the file. We will no longer attempt to hold it
185 if (buf
->spill
!= NULL
)
187 apr_off_t output_unused
= 0; /* ### stupid API */
189 /* Seek to the end of the spill file. We don't know if a read has
190 occurred since our last write, and moved the file position. */
191 SVN_ERR(svn_io_file_seek(buf
->spill
,
192 APR_END
, &output_unused
,
195 SVN_ERR(svn_io_file_write_full(buf
->spill
, data
, len
,
196 NULL
, scratch_pool
));
197 buf
->spill_size
+= len
;
206 if (buf
->tail
== NULL
|| buf
->tail
->size
== buf
->blocksize
)
208 /* There is no existing memblock (that may have space), or the
209 tail memblock has no space, so we need a new memblock. */
210 mem
= get_buffer(buf
);
219 /* Compute how much to write into the memblock. */
220 amt
= buf
->blocksize
- mem
->size
;
224 /* Copy some data into this memblock. */
225 memcpy(&mem
->data
[mem
->size
], data
, amt
);
230 /* We need to record how much is buffered in memory. Once we reach
231 buf->maxsize (or thereabouts, it doesn't have to be precise), then
232 we'll switch to putting the content into a file. */
233 buf
->memory_size
+= amt
;
235 /* Start a list of buffers, or (if we're not writing into the tail)
236 append to the end of the linked list of buffers. */
237 if (buf
->tail
== NULL
)
242 else if (mem
!= buf
->tail
)
244 buf
->tail
->next
= mem
;
253 /* Return a memblock of content, if any is available. *mem will be NULL if
254 no further content is available. The memblock should eventually be
255 passed to return_buffer() (or stored into buf->out_for_reading which
256 will grab that block at the next get_buffer() call). */
258 read_data(struct memblock_t
**mem
,
260 apr_pool_t
*scratch_pool
)
264 /* If we have some in-memory blocks, then return one. */
265 if (buf
->head
!= NULL
)
268 if (buf
->tail
== *mem
)
269 buf
->head
= buf
->tail
= NULL
;
271 buf
->head
= (*mem
)->next
;
273 /* We're using less memory now. If we haven't hit the spill file,
274 then we may be able to keep using memory. */
275 buf
->memory_size
-= (*mem
)->size
;
281 if (buf
->spill
== NULL
)
287 /* Assume that the caller has seeked the spill file to the correct pos. */
289 /* Get a buffer that we can read content into. */
290 *mem
= get_buffer(buf
);
291 /* NOTE: mem's size/next are uninitialized. */
293 if ((apr_uint64_t
)buf
->spill_size
< (apr_uint64_t
)buf
->blocksize
)
294 (*mem
)->size
= (apr_size_t
)buf
->spill_size
;
296 (*mem
)->size
= buf
->blocksize
; /* The size of (*mem)->data */
299 /* Read some data from the spill file into the memblock. */
300 err
= svn_io_file_read(buf
->spill
, (*mem
)->data
, &(*mem
)->size
,
304 return_buffer(buf
, *mem
);
305 return svn_error_trace(err
);
308 /* Mark the data that we consumed from the spill file. */
309 buf
->spill_start
+= (*mem
)->size
;
311 /* Did we consume all the data from the spill file? */
312 if ((buf
->spill_size
-= (*mem
)->size
) == 0)
314 /* Close and reset our spill file information. */
315 SVN_ERR(svn_io_file_close(buf
->spill
, scratch_pool
));
317 buf
->spill_start
= 0;
320 /* *mem has been initialized. Done. */
325 /* If the next read would consume data from the file, then seek to the
328 maybe_seek(svn_boolean_t
*seeked
,
329 const svn_spillbuf_t
*buf
,
330 apr_pool_t
*scratch_pool
)
332 if (buf
->head
== NULL
&& buf
->spill
!= NULL
)
334 apr_off_t output_unused
;
336 /* Seek to where we left off reading. */
337 output_unused
= buf
->spill_start
; /* ### stupid API */
338 SVN_ERR(svn_io_file_seek(buf
->spill
,
339 APR_SET
, &output_unused
,
344 else if (seeked
!= NULL
)
354 svn_spillbuf__read(const char **data
,
357 apr_pool_t
*scratch_pool
)
359 struct memblock_t
*mem
;
361 /* Possibly seek... */
362 SVN_ERR(maybe_seek(NULL
, buf
, scratch_pool
));
364 SVN_ERR(read_data(&mem
, buf
, scratch_pool
));
375 /* If a block was out for reading, then return it. */
376 if (buf
->out_for_reading
!= NULL
)
377 return_buffer(buf
, buf
->out_for_reading
);
379 /* Remember that we've passed this block out for reading. */
380 buf
->out_for_reading
= mem
;
388 svn_spillbuf__process(svn_boolean_t
*exhausted
,
390 svn_spillbuf_read_t read_func
,
392 apr_pool_t
*scratch_pool
)
394 svn_boolean_t has_seeked
= FALSE
;
395 apr_pool_t
*iterpool
= svn_pool_create(scratch_pool
);
401 struct memblock_t
*mem
;
405 svn_pool_clear(iterpool
);
407 /* If this call to read_data() will read from the spill file, and we
408 have not seek'd the file... then do it now. */
410 SVN_ERR(maybe_seek(&has_seeked
, buf
, iterpool
));
412 /* Get some content to pass to the read callback. */
413 SVN_ERR(read_data(&mem
, buf
, iterpool
));
420 err
= read_func(&stop
, read_baton
, mem
->data
, mem
->size
, iterpool
);
422 return_buffer(buf
, mem
);
425 return svn_error_trace(err
);
427 /* If the callbacks told us to stop, then we're done for now. */
432 svn_pool_destroy(iterpool
);
437 svn_spillbuf_reader_t
*
438 svn_spillbuf__reader_create(apr_size_t blocksize
,
440 apr_pool_t
*result_pool
)
442 svn_spillbuf_reader_t
*sbr
= apr_pcalloc(result_pool
, sizeof(*sbr
));
444 /* See svn_spillbuf__create() */
445 sbr
->buf
.pool
= result_pool
;
446 sbr
->buf
.blocksize
= blocksize
;
447 sbr
->buf
.maxsize
= maxsize
;
454 svn_spillbuf__reader_read(apr_size_t
*amt
,
455 svn_spillbuf_reader_t
*reader
,
458 apr_pool_t
*scratch_pool
)
461 return svn_error_create(SVN_ERR_INCORRECT_PARAMS
, NULL
, NULL
);
469 if (reader
->save_len
> 0)
471 /* We have some saved content, so use this first. */
473 if (len
< reader
->save_len
)
476 copy_amt
= reader
->save_len
;
478 memcpy(data
, reader
->save_ptr
+ reader
->save_pos
, copy_amt
);
479 reader
->save_pos
+= copy_amt
;
480 reader
->save_len
-= copy_amt
;
484 /* No saved content. We should now copy from spillbuf-provided
485 buffers of content. */
487 /* We may need more content from the spillbuf. */
488 if (reader
->sb_len
== 0)
490 SVN_ERR(svn_spillbuf__read(&reader
->sb_ptr
, &reader
->sb_len
,
494 /* We've run out of content, so return with whatever has
495 been copied into DATA and stored into AMT. */
496 if (reader
->sb_ptr
== NULL
)
498 /* For safety, read() may not have set SB_LEN. We use it
499 as an indicator, so it needs to be cleared. */
505 if (len
< reader
->sb_len
)
508 copy_amt
= reader
->sb_len
;
510 memcpy(data
, reader
->sb_ptr
, copy_amt
);
511 reader
->sb_ptr
+= copy_amt
;
512 reader
->sb_len
-= copy_amt
;
525 svn_spillbuf__reader_getc(char *c
,
526 svn_spillbuf_reader_t
*reader
,
527 apr_pool_t
*scratch_pool
)
531 SVN_ERR(svn_spillbuf__reader_read(&amt
, reader
, c
, 1, scratch_pool
));
533 return svn_error_create(SVN_ERR_STREAM_UNEXPECTED_EOF
, NULL
, NULL
);
540 svn_spillbuf__reader_write(svn_spillbuf_reader_t
*reader
,
543 apr_pool_t
*scratch_pool
)
545 /* If we have a buffer of content from the spillbuf, then we need to
546 move that content to a safe place. */
547 if (reader
->sb_len
> 0)
549 if (reader
->save_ptr
== NULL
)
550 reader
->save_ptr
= apr_palloc(reader
->buf
.pool
, reader
->buf
.blocksize
);
552 memcpy(reader
->save_ptr
, reader
->sb_ptr
, reader
->sb_len
);
553 reader
->save_len
= reader
->sb_len
;
554 reader
->save_pos
= 0;
556 /* No more content in the spillbuf-borrowed buffer. */
560 return svn_error_trace(svn_spillbuf__write(&reader
->buf
, data
, len
,
565 struct spillbuf_baton
567 svn_spillbuf_reader_t
*reader
;
568 apr_pool_t
*scratch_pool
;
573 read_handler_spillbuf(void *baton
, char *buffer
, apr_size_t
*len
)
575 struct spillbuf_baton
*sb
= baton
;
577 SVN_ERR(svn_spillbuf__reader_read(len
, sb
->reader
, buffer
, *len
,
580 svn_pool_clear(sb
->scratch_pool
);
586 write_handler_spillbuf(void *baton
, const char *data
, apr_size_t
*len
)
588 struct spillbuf_baton
*sb
= baton
;
590 SVN_ERR(svn_spillbuf__reader_write(sb
->reader
, data
, *len
,
593 svn_pool_clear(sb
->scratch_pool
);
599 svn_stream__from_spillbuf(apr_size_t blocksize
,
601 apr_pool_t
*result_pool
)
603 svn_stream_t
*stream
;
604 struct spillbuf_baton
*sb
= apr_palloc(result_pool
, sizeof(*sb
));
606 sb
->reader
= svn_spillbuf__reader_create(blocksize
, maxsize
, result_pool
);
607 sb
->scratch_pool
= svn_pool_create(result_pool
);
609 stream
= svn_stream_create(sb
, result_pool
);
611 svn_stream_set_read(stream
, read_handler_spillbuf
);
612 svn_stream_set_write(stream
, write_handler_spillbuf
);