jdgordons_attempt/buffering.c

   1 #include <stdlib.h>
   2 #include <stdio.h>
   3 #include <string.h>
   4 #include <stdbool.h>
   5 #include <fcntl.h>
   6
   7 /* definitions */
   8 #define GUARD_SIZE      (32*1024)       /* 32K */
   9 #define BUFFER_SIZE     (32*1024*1024)  /* 32M */
  10 #define MIN_BUFFER_SIZE (1024) /* smallest amount of data to try buffering */
  11
  12 #define BUFFER_REBUFFER_WATERMARK (BUFFER_SIZE/4) /* rebuffer is usage < this */
  13 #define MAX_WASTED_SPACE (BUFFER_SIZE/2)
  14
  15 #define MAX_HANDLES 128
  16
  17 typedef unsigned char BYTE; /* incase its not defined yet */
  18 #define MAX_PATH 290
  19 struct handle {
  20     bool inuse;         /* is handle being used currently */
  21     /* file variables */
  22     int    fd;          /* file fd, <0 means not open */
  23     bool   finished;    /* true if finished reading the file */
  24     size_t file_offset; /* offset of the start of the data buffered */
  25     /* memory vairables */
  26     BYTE  *data;        /* start of the buffered data */
  27     size_t data_len;    /* length of the buffered data */
  28     BYTE  *last_read;   /* end of the last read block from this file */
  29 };
  30
  31
  32 /* Global vairables */
  33 BYTE *buffer;
  34 BYTE *BUFFER_END; /* saves recalculating this every read */
  35
  36 BYTE *write_ptr; /* write data here */
  37 BYTE *last_read; /* pointer to the end of the last block of data "unbuffered" */
  38 BYTE *valid_data;/* only data "between" valid_data and write_ptr is still valid */
  39
  40 struct handle handles[MAX_HANDLES];
  41 int handles_used; /* number of handles actually in use */
  42 char filename[MAX_PATH]; /* filename of the next file to open. */
  43 /* internal functions */
  44 /* returns the number of bytes currently used.
  45    *wasted_space is set to the number of bytes
  46     between last_read and valid_data */
  47 static size_t buffer_usage(size_t *wasted_space)
  48 {
  49     size_t usage = 0, temp;
  50     if (handles_used == 0)
  51     {
  52         //printf("buffer: %p,  write_ptr: %p end:%p\n",buffer, write_ptr, BUFFER_END);
  53         return 0;
  54     }
  55     if (wasted_space == NULL)
  56         wasted_space = &temp;
  57     /* Case 1: |----V**L***W---| */
  58     if (valid_data <= write_ptr)
  59     {
  60         *wasted_space = last_read - valid_data;
  61         usage = (write_ptr - valid_data);
  62     }
  63     /* Case 2: |***W----V**L***| */
  64     else if (write_ptr < valid_data)
  65     {
  66         if (last_read < valid_data) /* |*L**W----V*****| */
  67             *wasted_space = (BUFFER_END - valid_data) + (last_read - buffer);
  68         else                        /* |***W----V**L***| */
  69             *wasted_space = last_read - valid_data;
  70         usage = BUFFER_SIZE - (valid_data - write_ptr);
  71     }
  72     //printf("used: %d, wasted: %d, free %d\n", usage, *wasted_space, BUFFER_SIZE - usage);
  73     return usage;
  74 }
  75
  76 bool need_rebuffer(void)
  77 {
  78     size_t usage, wasted = 0, free;
  79     usage = buffer_usage(&wasted);
  80     free = BUFFER_SIZE - usage;
  81     return ((free >= BUFFER_REBUFFER_WATERMARK) ||
  82             wasted > MAX_WASTED_SPACE);
  83 }
  84
  85 static void limit_wasted_space(void)
  86 {
  87     size_t usage, wasted = 0;
  88     usage = buffer_usage(&wasted);
  89     /* this should be done more smartly so some rewind buffer still exists */
  90     if (wasted > MAX_WASTED_SPACE)
  91         valid_data = last_read;
  92 }
  93
  94 static void move_last_read(BYTE* new_pos)
  95 {
  96     if (new_pos > BUFFER_END)
  97         new_pos -= BUFFER_SIZE;
  98     last_read = new_pos;
  99 }
 100
 101 /* read_from_from:
 102     fd - file to read from
 103     buffer - buffer to read into
 104     *length - maximum amount of data to read.
 105         On return *length becomes the amount of data actually read.
 106     Returns false if the file is finished, or error.
 107             true if there is still data to rbe read.
 108 */
 109 static bool read_from_file(int fd, char *buffer, size_t *length)
 110 {
 111     char buf[32];
 112     size_t ret = 0, data_read = 0;
 113     while (*length > 0)
 114     {
 115         ret = read(fd, &buffer[data_read], *length);
 116         if (ret <= 0)
 117         {
 118             *length = data_read;
 119             return false; /* file finished */
 120         }
 121         else
 122         {
 123             data_read += ret;
 124             *length -= ret;
 125         }
 126         /* possibly yield() here */
 127     }
 128     *length = data_read;
 129     return true;
 130 }
 131
 132 /* buffer as much of handle_id as possible.
 133    returns true if there is more data to buffer,
 134    false if handle isnt being used or file is finished */
 135 static bool buffer_handle(int handle_id)
 136 {
 137     struct handle *h = &handles[handle_id];
 138     size_t available_space;
 139     bool adding_to_existing = false;
 140     if (h->inuse == false ||
 141         h->finished == true)
 142         return false;
 143     if (h->fd < 0 && filename[0]) /* file closed, reopen */
 144     {
 145         h->fd = open(filename, O_RDONLY);
 146         if (h->fd < 0)
 147             return false;
 148         if (h->file_offset > 0)
 149             lseek(h->fd, h->file_offset, SEEK_SET);
 150     }
 151     limit_wasted_space(); /* free up some more space possibly */
 152     available_space = BUFFER_SIZE - buffer_usage(NULL);
 153     //printf("$$ %d\n", available_space);
 154     /* check if we are adding more data to a handle */
 155     if (h->data)
 156     {
 157         BYTE* data_end = h->data + h->data_len;
 158         if (data_end > BUFFER_END)
 159             data_end -= BUFFER_SIZE;
 160         if (data_end == write_ptr)
 161             adding_to_existing = true;
 162         printf("appending %s... handle %d\n", adding_to_existing?"yes":"no",handle_id);
 163     }
 164     if (BUFFER_END - write_ptr < available_space)
 165     {
 166         /* for the moment dont buffer around the wrap,
 167            it will be called again automatically to rebuffer from the start */
 168         available_space = BUFFER_END - write_ptr;
 169         printf("wrapping... handle %d\n", handle_id);
 170     }
 171     h->finished = !read_from_file(h->fd, write_ptr, &available_space);
 172     if (adding_to_existing)
 173         h->data_len += available_space;
 174     else
 175     {
 176         h->data = write_ptr;
 177         h->data_len = available_space;
 178     }
 179     if (h->finished)
 180         close(h->fd);
 181     write_ptr += available_space;
 182
 183     if (BUFFER_END <= write_ptr)
 184     {
 185         write_ptr = buffer;
 186     }
 187     printf("%d fininshed buffering\n", handle_id);
 188     return h->finished == false;
 189 }
 190 /* exported functions */
 191 void buffer_init(void);
 192 int bufopen(char *filename, size_t offset);
 193 int bufseek(int handle_id, size_t offset);
 194 int bufclose(int handle_id);
 195 long bufgetdata(int handle_id, size_t size, char **destptr);
 196
 197 void buffer_init(void)
 198 {
 199     int i;
 200     buffer = (BYTE*)malloc(sizeof(BYTE)*(BUFFER_SIZE+GUARD_SIZE));
 201     if (!buffer)
 202     {
 203         printf("couldnt allocated buffer\n");
 204         exit(1);
 205     }
 206     BUFFER_END = buffer + BUFFER_SIZE;
 207     write_ptr = buffer;
 208     last_read = buffer;
 209     valid_data = buffer;
 210     handles_used = 0;
 211     for (i=0; i<MAX_HANDLES; i++)
 212         handles[i].inuse = false;
 213 }
 214
 215 /* Request a file be buffered
 216     filename: name of the file t open
 217     offset:   starting offset to buffer from the file
 218     RETURNS:  <0 if the file cannot be opened, or one file already
 219                 queued to be opened, otherwise the handle for the file in the buffer
 220 */
 221 int bufopen(char *file, size_t offset)
 222 {
 223     int i;
 224     if (handles_used >= MAX_HANDLES)
 225         return -1;
 226     for (i=0; i<MAX_HANDLES; i++)
 227     {
 228         if (handles[i].inuse == false)
 229         {
 230             if (/*ata_is_active()*/1) /* open now seen as the disk is being used */
 231             {
 232                 handles[i].fd = open(file, O_RDONLY);
 233                 if (handles[i].fd < 0)
 234                     return handles[i].fd;
 235                 if (offset)
 236                     lseek(handles[i].fd, offset, SEEK_SET);
 237             }
 238             else if (filename[0]) /* already one file queued to open */
 239                 return -1;
 240             else
 241             {
 242                 handles[i].fd = -1;
 243                 strcpy(filename, file);
 244             }
 245             handles[i].file_offset = offset;
 246             handles[i].data = NULL;
 247             handles[i].inuse = true;
 248             handles[i].finished = false;
 249             handles[i].last_read = NULL;
 250             break;
 251         }
 252     }
 253     if (i<MAX_HANDLES)
 254     {
 255         handles_used++;
 256     //    rb->queue_post(&buffer_event_queue, B_BUFFER_FILE, i);
 257         return i;
 258     }
 259     return -1;
 260 }
 261
 262 /* set *destptr to the start of the buffer for the handle.
 263    size is the min it wants to read, data may be copied into the guard buffer
 264    so the data can be one continuous block.
 265    returns number of bytes available.
 266    No guarentees are made about how long the buffer will be valid for.
 267 */
 268 long bufgetdata(int handle_id, size_t size, char **destptr)
 269 {
 270     long ret = 0;
 271     size_t data_len;
 272     struct handle *h = &handles[handle_id];
 273     if (h->inuse == false)
 274         return 0;
 275 #if 0
 276     if (h->last_read == NULL)
 277         h->last_read = h->data;
 278     if ((h->last_read + size < BUFFER_END) &&
 279          (h->last_read + size < h->data + h->data_len))
 280     {
 281         *destptr = h->last_read;
 282         /* only give as much as requested for now */
 283         ret = size;
 284         h->last_read += size;
 285         last_read == h->last_read;
 286     }
 287 #endif
 288     move_last_read(h->data + h->data_len);
 289     return ret;
 290 }
 291
 292 /* close the handle */
 293 int bufclose(int handle_id)
 294 {
 295     struct handle *h = &handles[handle_id];
 296     if (h->data)// -- not sure about these...
 297         move_last_read(h->data + h->data_len);
 298     if (h->fd >= 0)
 299         close(h->fd);
 300     h->data = NULL;
 301     h->inuse = false;
 302     handles_used--;
 303     return 0;
 304 }
 305
 306 long bufread(int handle_id, size_t size, BYTE *dest)
 307 {
 308     struct handle *h = &handles[handle_id];
 309     size_t buffered_data;
 310     if (h->inuse == false ||
 311         h->data == NULL)
 312         return -1;
 313     if (h->last_read == NULL)
 314         h->last_read = h->data;
 315     buffered_data = h->data_len - (h->last_read - h->data);
 316     if (buffered_data == 0)
 317         return 0;
 318     if (buffered_data < size)
 319     {
 320         size = buffered_data;
 321     }
 322     if (h->last_read + size > BUFFER_END) /* copy in 2 bits */
 323     {
 324         size_t read = BUFFER_END - h->last_read;
 325         memcpy(dest, h->last_read, read);
 326         memcpy(dest+read, buffer, size - read);
 327     }
 328     else memcpy(dest, h->last_read, size);
 329     h->last_read += size;
 330     move_last_read(h->last_read);
 331     return size;
 332 }
 333
 334
 335
 336
 337
 338
 339
 340
 341
 342
 343
 344
 345
 346
 347 int main(int argc, char *argv[])
 348 {
 349     int next_file = 1;
 350     int last_handle = -1;
 351     int handle_order[MAX_HANDLES];
 352     int reading_handle = 0;
 353     bool done = false;
 354     BYTE read_buffer[GUARD_SIZE];
 355     buffer_init();
 356     while (done == false)
 357     {
 358         if (next_file <= argc && need_rebuffer())
 359         {
 360             printf("buffer usage: %d handles_used: %d\n", buffer_usage(NULL),handles_used);
 361             if ( (!handles_used ||
 362                   handles[handle_order[last_handle]].finished == true))
 363             {
 364                 int h = bufopen(argv[next_file++], 0);
 365                 if (h >= 0)
 366                 {
 367                     printf("new handle %d\n",h);
 368                     last_handle++;
 369                     handle_order[last_handle] = h;
 370                     buffer_handle(handle_order[last_handle]);
 371                 }
 372             }
 373             else
 374             {
 375                 if (handles[handle_order[last_handle]].finished == false)
 376                 {
 377                     printf("buffering handle %d\n",handle_order[last_handle]);
 378                     buffer_handle(handle_order[last_handle]);
 379                 }
 380             }
 381         }
 382         else
 383         {
 384             long total = 0, read;
 385             char file[MAX_PATH];
 386             int fd;
 387             if (next_file > argc && reading_handle >= last_handle)
 388                     done = true;
 389             snprintf(file, MAX_PATH, "/home/jonno/buffering_test/file%d.mp3", reading_handle);
 390             fd = open(file, O_CREAT|O_TRUNC|O_RDWR);
 391             if (fd < 0)
 392             {
 393                 printf("ERROROROROR\n");
 394                 exit(1);
 395             }
 396             while (1)
 397             {
 398                 read = bufread(handle_order[reading_handle], GUARD_SIZE,read_buffer);
 399                 total += read;
 400                 write(fd, read_buffer, read);
 401                 if (read <= 0)
 402                 {
 403                     printf("finished reading %d, %d\n",handle_order[reading_handle], total);
 404                     bufclose(handle_order[reading_handle]);
 405                     close(fd);
 406                     reading_handle++;
 407                     break;
 408                 }
 409             }
 410         }
 411     }
 412
 413     free(buffer);
 414     return 0;
 415 }