8 #define GUARD_SIZE (32*1024) /* 32K */
9 #define BUFFER_SIZE (32*1024*1024) /* 32M */
10 #define MIN_BUFFER_SIZE (1024) /* smallest amount of data to try buffering */
12 #define BUFFER_REBUFFER_WATERMARK (BUFFER_SIZE/4) /* rebuffer is usage < this */
13 #define MAX_WASTED_SPACE (BUFFER_SIZE/2)
15 #define MAX_HANDLES 128
17 typedef unsigned char BYTE
; /* incase its not defined yet */
20 bool inuse
; /* is handle being used currently */
22 int fd
; /* file fd, <0 means not open */
23 bool finished
; /* true if finished reading the file */
24 size_t file_offset
; /* offset of the start of the data buffered */
25 /* memory vairables */
26 BYTE
*data
; /* start of the buffered data */
27 size_t data_len
; /* length of the buffered data */
28 BYTE
*last_read
; /* end of the last read block from this file */
32 /* Global vairables */
34 BYTE
*BUFFER_END
; /* saves recalculating this every read */
36 BYTE
*write_ptr
; /* write data here */
37 BYTE
*last_read
; /* pointer to the end of the last block of data "unbuffered" */
38 BYTE
*valid_data
;/* only data "between" valid_data and write_ptr is still valid */
40 struct handle handles
[MAX_HANDLES
];
41 int handles_used
; /* number of handles actually in use */
42 char filename
[MAX_PATH
]; /* filename of the next file to open. */
43 /* internal functions */
44 /* returns the number of bytes currently used.
45 *wasted_space is set to the number of bytes
46 between last_read and valid_data */
47 static size_t buffer_usage(size_t *wasted_space
)
49 size_t usage
= 0, temp
;
50 if (handles_used
== 0)
52 //printf("buffer: %p, write_ptr: %p end:%p\n",buffer, write_ptr, BUFFER_END);
55 if (wasted_space
== NULL
)
57 /* Case 1: |----V**L***W---| */
58 if (valid_data
<= write_ptr
)
60 *wasted_space
= last_read
- valid_data
;
61 usage
= (write_ptr
- valid_data
);
63 /* Case 2: |***W----V**L***| */
64 else if (write_ptr
< valid_data
)
66 if (last_read
< valid_data
) /* |*L**W----V*****| */
67 *wasted_space
= (BUFFER_END
- valid_data
) + (last_read
- buffer
);
68 else /* |***W----V**L***| */
69 *wasted_space
= last_read
- valid_data
;
70 usage
= BUFFER_SIZE
- (valid_data
- write_ptr
);
72 //printf("used: %d, wasted: %d, free %d\n", usage, *wasted_space, BUFFER_SIZE - usage);
76 bool need_rebuffer(void)
78 size_t usage
, wasted
= 0, free
;
79 usage
= buffer_usage(&wasted
);
80 free
= BUFFER_SIZE
- usage
;
81 return ((free
>= BUFFER_REBUFFER_WATERMARK
) ||
82 wasted
> MAX_WASTED_SPACE
);
85 static void limit_wasted_space(void)
87 size_t usage
, wasted
= 0;
88 usage
= buffer_usage(&wasted
);
89 /* this should be done more smartly so some rewind buffer still exists */
90 if (wasted
> MAX_WASTED_SPACE
)
91 valid_data
= last_read
;
94 static void move_last_read(BYTE
* new_pos
)
96 if (new_pos
> BUFFER_END
)
97 new_pos
-= BUFFER_SIZE
;
102 fd - file to read from
103 buffer - buffer to read into
104 *length - maximum amount of data to read.
105 On return *length becomes the amount of data actually read.
106 Returns false if the file is finished, or error.
107 true if there is still data to rbe read.
109 static bool read_from_file(int fd
, char *buffer
, size_t *length
)
112 size_t ret
= 0, data_read
= 0;
115 ret
= read(fd
, &buffer
[data_read
], *length
);
119 return false; /* file finished */
126 /* possibly yield() here */
132 /* buffer as much of handle_id as possible.
133 returns true if there is more data to buffer,
134 false if handle isnt being used or file is finished */
135 static bool buffer_handle(int handle_id
)
137 struct handle
*h
= &handles
[handle_id
];
138 size_t available_space
;
139 bool adding_to_existing
= false;
140 if (h
->inuse
== false ||
143 if (h
->fd
< 0 && filename
[0]) /* file closed, reopen */
145 h
->fd
= open(filename
, O_RDONLY
);
148 if (h
->file_offset
> 0)
149 lseek(h
->fd
, h
->file_offset
, SEEK_SET
);
151 limit_wasted_space(); /* free up some more space possibly */
152 available_space
= BUFFER_SIZE
- buffer_usage(NULL
);
153 //printf("$$ %d\n", available_space);
154 /* check if we are adding more data to a handle */
157 BYTE
* data_end
= h
->data
+ h
->data_len
;
158 if (data_end
> BUFFER_END
)
159 data_end
-= BUFFER_SIZE
;
160 if (data_end
== write_ptr
)
161 adding_to_existing
= true;
162 printf("appending %s... handle %d\n", adding_to_existing
?"yes":"no",handle_id
);
164 if (BUFFER_END
- write_ptr
< available_space
)
166 /* for the moment dont buffer around the wrap,
167 it will be called again automatically to rebuffer from the start */
168 available_space
= BUFFER_END
- write_ptr
;
169 printf("wrapping... handle %d\n", handle_id
);
171 h
->finished
= !read_from_file(h
->fd
, write_ptr
, &available_space
);
172 if (adding_to_existing
)
173 h
->data_len
+= available_space
;
177 h
->data_len
= available_space
;
181 write_ptr
+= available_space
;
183 if (BUFFER_END
<= write_ptr
)
187 printf("%d fininshed buffering\n", handle_id
);
188 return h
->finished
== false;
190 /* exported functions */
191 void buffer_init(void);
192 int bufopen(char *filename
, size_t offset
);
193 int bufseek(int handle_id
, size_t offset
);
194 int bufclose(int handle_id
);
195 long bufgetdata(int handle_id
, size_t size
, char **destptr
);
197 void buffer_init(void)
200 buffer
= (BYTE
*)malloc(sizeof(BYTE
)*(BUFFER_SIZE
+GUARD_SIZE
));
203 printf("couldnt allocated buffer\n");
206 BUFFER_END
= buffer
+ BUFFER_SIZE
;
211 for (i
=0; i
<MAX_HANDLES
; i
++)
212 handles
[i
].inuse
= false;
215 /* Request a file be buffered
216 filename: name of the file t open
217 offset: starting offset to buffer from the file
218 RETURNS: <0 if the file cannot be opened, or one file already
219 queued to be opened, otherwise the handle for the file in the buffer
221 int bufopen(char *file
, size_t offset
)
224 if (handles_used
>= MAX_HANDLES
)
226 for (i
=0; i
<MAX_HANDLES
; i
++)
228 if (handles
[i
].inuse
== false)
230 if (/*ata_is_active()*/1) /* open now seen as the disk is being used */
232 handles
[i
].fd
= open(file
, O_RDONLY
);
233 if (handles
[i
].fd
< 0)
234 return handles
[i
].fd
;
236 lseek(handles
[i
].fd
, offset
, SEEK_SET
);
238 else if (filename
[0]) /* already one file queued to open */
243 strcpy(filename
, file
);
245 handles
[i
].file_offset
= offset
;
246 handles
[i
].data
= NULL
;
247 handles
[i
].inuse
= true;
248 handles
[i
].finished
= false;
249 handles
[i
].last_read
= NULL
;
256 // rb->queue_post(&buffer_event_queue, B_BUFFER_FILE, i);
262 /* set *destptr to the start of the buffer for the handle.
263 size is the min it wants to read, data may be copied into the guard buffer
264 so the data can be one continuous block.
265 returns number of bytes available.
266 No guarentees are made about how long the buffer will be valid for.
268 long bufgetdata(int handle_id
, size_t size
, char **destptr
)
272 struct handle
*h
= &handles
[handle_id
];
273 if (h
->inuse
== false)
276 if (h
->last_read
== NULL
)
277 h
->last_read
= h
->data
;
278 if ((h
->last_read
+ size
< BUFFER_END
) &&
279 (h
->last_read
+ size
< h
->data
+ h
->data_len
))
281 *destptr
= h
->last_read
;
282 /* only give as much as requested for now */
284 h
->last_read
+= size
;
285 last_read
== h
->last_read
;
288 move_last_read(h
->data
+ h
->data_len
);
292 /* close the handle */
293 int bufclose(int handle_id
)
295 struct handle
*h
= &handles
[handle_id
];
296 if (h
->data
)// -- not sure about these...
297 move_last_read(h
->data
+ h
->data_len
);
306 long bufread(int handle_id
, size_t size
, BYTE
*dest
)
308 struct handle
*h
= &handles
[handle_id
];
309 size_t buffered_data
;
310 if (h
->inuse
== false ||
313 if (h
->last_read
== NULL
)
314 h
->last_read
= h
->data
;
315 buffered_data
= h
->data_len
- (h
->last_read
- h
->data
);
316 if (buffered_data
== 0)
318 if (buffered_data
< size
)
320 size
= buffered_data
;
322 if (h
->last_read
+ size
> BUFFER_END
) /* copy in 2 bits */
324 size_t read
= BUFFER_END
- h
->last_read
;
325 memcpy(dest
, h
->last_read
, read
);
326 memcpy(dest
+read
, buffer
, size
- read
);
328 else memcpy(dest
, h
->last_read
, size
);
329 h
->last_read
+= size
;
330 move_last_read(h
->last_read
);
347 int main(int argc
, char *argv
[])
350 int last_handle
= -1;
351 int handle_order
[MAX_HANDLES
];
352 int reading_handle
= 0;
354 BYTE read_buffer
[GUARD_SIZE
];
356 while (done
== false)
358 if (next_file
<= argc
&& need_rebuffer())
360 printf("buffer usage: %d handles_used: %d\n", buffer_usage(NULL
),handles_used
);
361 if ( (!handles_used
||
362 handles
[handle_order
[last_handle
]].finished
== true))
364 int h
= bufopen(argv
[next_file
++], 0);
367 printf("new handle %d\n",h
);
369 handle_order
[last_handle
] = h
;
370 buffer_handle(handle_order
[last_handle
]);
375 if (handles
[handle_order
[last_handle
]].finished
== false)
377 printf("buffering handle %d\n",handle_order
[last_handle
]);
378 buffer_handle(handle_order
[last_handle
]);
384 long total
= 0, read
;
387 if (next_file
> argc
&& reading_handle
>= last_handle
)
389 snprintf(file
, MAX_PATH
, "/home/jonno/buffering_test/file%d.mp3", reading_handle
);
390 fd
= open(file
, O_CREAT
|O_TRUNC
|O_RDWR
);
393 printf("ERROROROROR\n");
398 read
= bufread(handle_order
[reading_handle
], GUARD_SIZE
,read_buffer
);
400 write(fd
, read_buffer
, read
);
403 printf("finished reading %d, %d\n",handle_order
[reading_handle
], total
);
404 bufclose(handle_order
[reading_handle
]);