Build script now builds an oggenc binary with flac support.
[vorbis-lancer-gcc.git] / aotuv-b5_20061024 / vq / bookutil.c
blobcdef31bc89be45dedc5e7222b82ca9a1563060f2
1 /********************************************************************
2 * *
3 * THIS FILE IS PART OF THE OggVorbis SOFTWARE CODEC SOURCE CODE. *
4 * USE, DISTRIBUTION AND REPRODUCTION OF THIS LIBRARY SOURCE IS *
5 * GOVERNED BY A BSD-STYLE SOURCE LICENSE INCLUDED WITH THIS SOURCE *
6 * IN 'COPYING'. PLEASE READ THESE TERMS BEFORE DISTRIBUTING. *
7 * *
8 * THE OggVorbis SOURCE CODE IS (C) COPYRIGHT 1994-2001 *
9 * by the XIPHOPHORUS Company http://www.xiph.org/ *
10 * *
11 ********************************************************************
13 function: utility functions for loading .vqh and .vqd files
14 last mod: $Id: bookutil.c 7187 2004-07-20 07:24:27Z xiphmont $
16 ********************************************************************/
18 #include <stdlib.h>
19 #include <stdio.h>
20 #include <math.h>
21 #include <string.h>
22 #include <errno.h>
23 #include "bookutil.h"
25 /* A few little utils for reading files */
26 /* read a line. Use global, persistent buffering */
27 static char *linebuffer=NULL;
28 static int lbufsize=0;
29 char *get_line(FILE *in){
30 long sofar=0;
31 if(feof(in))return NULL;
33 while(1){
34 int gotline=0;
36 while(!gotline){
37 if(sofar+1>=lbufsize){
38 if(!lbufsize){
39 lbufsize=1024;
40 linebuffer=_ogg_malloc(lbufsize);
41 }else{
42 lbufsize*=2;
43 linebuffer=_ogg_realloc(linebuffer,lbufsize);
47 long c=fgetc(in);
48 switch(c){
49 case EOF:
50 if(sofar==0)return(NULL);
51 /* fallthrough correct */
52 case '\n':
53 linebuffer[sofar]='\0';
54 gotline=1;
55 break;
56 default:
57 linebuffer[sofar++]=c;
58 linebuffer[sofar]='\0';
59 break;
64 if(linebuffer[0]=='#'){
65 sofar=0;
66 }else{
67 return(linebuffer);
72 /* read the next numerical value from the given file */
73 static char *value_line_buff=NULL;
75 int get_line_value(FILE *in,float *value){
76 char *next;
78 if(!value_line_buff)return(-1);
80 *value=strtod(value_line_buff, &next);
81 if(next==value_line_buff){
82 value_line_buff=NULL;
83 return(-1);
84 }else{
85 value_line_buff=next;
86 while(*value_line_buff>44)value_line_buff++;
87 if(*value_line_buff==44)value_line_buff++;
88 return(0);
92 int get_next_value(FILE *in,float *value){
93 while(1){
94 if(get_line_value(in,value)){
95 value_line_buff=get_line(in);
96 if(!value_line_buff)return(-1);
97 }else{
98 return(0);
103 int get_next_ivalue(FILE *in,long *ivalue){
104 float value;
105 int ret=get_next_value(in,&value);
106 *ivalue=value;
107 return(ret);
110 static float sequence_base=0.f;
111 static int v_sofar=0;
112 void reset_next_value(void){
113 value_line_buff=NULL;
114 sequence_base=0.f;
115 v_sofar=0;
118 char *setup_line(FILE *in){
119 reset_next_value();
120 value_line_buff=get_line(in);
121 return(value_line_buff);
125 int get_vector(codebook *b,FILE *in,int start, int n,float *a){
126 int i;
127 const static_codebook *c=b->c;
129 while(1){
131 if(v_sofar==n || get_line_value(in,a)){
132 reset_next_value();
133 if(get_next_value(in,a))
134 break;
135 for(i=0;i<start;i++){
136 sequence_base=*a;
137 get_line_value(in,a);
141 for(i=1;i<c->dim;i++)
142 if(get_line_value(in,a+i))
143 break;
145 if(i==c->dim){
146 float temp=a[c->dim-1];
147 for(i=0;i<c->dim;i++)a[i]-=sequence_base;
148 if(c->q_sequencep)sequence_base=temp;
149 v_sofar++;
150 return(0);
152 sequence_base=0.f;
155 return(-1);
158 /* read lines fromt he beginning until we find one containing the
159 specified string */
160 char *find_seek_to(FILE *in,char *s){
161 rewind(in);
162 while(1){
163 char *line=get_line(in);
164 if(line){
165 if(strstr(line,s))
166 return(line);
167 }else
168 return(NULL);
173 /* this reads the format as written by vqbuild/latticebuild; innocent
174 (legal) tweaking of the file that would not affect its valid
175 header-ness will break this routine */
177 codebook *codebook_load(char *filename){
178 codebook *b=_ogg_calloc(1,sizeof(codebook));
179 static_codebook *c=(static_codebook *)(b->c=_ogg_calloc(1,sizeof(static_codebook)));
180 encode_aux_nearestmatch *a=NULL;
181 encode_aux_threshmatch *t=NULL;
182 encode_aux_pigeonhole *p=NULL;
183 int quant_to_read=0;
184 FILE *in=fopen(filename,"r");
185 char *line;
186 long i;
188 if(in==NULL){
189 fprintf(stderr,"Couldn't open codebook %s\n",filename);
190 exit(1);
193 /* find the codebook struct */
194 find_seek_to(in,"static static_codebook ");
196 /* get the major important values */
197 line=get_line(in);
198 if(sscanf(line,"%ld, %ld,",
199 &(c->dim),&(c->entries))!=2){
200 fprintf(stderr,"1: syntax in %s in line:\t %s",filename,line);
201 exit(1);
203 line=get_line(in);
204 line=get_line(in);
205 if(sscanf(line,"%d, %ld, %ld, %d, %d,",
206 &(c->maptype),&(c->q_min),&(c->q_delta),&(c->q_quant),
207 &(c->q_sequencep))!=5){
208 fprintf(stderr,"1: syntax in %s in line:\t %s",filename,line);
209 exit(1);
212 /* find the auxiliary encode struct[s] (if any) */
213 if(find_seek_to(in,"static encode_aux_nearestmatch _vq_aux")){
214 /* how big? */
215 c->nearest_tree=a=_ogg_calloc(1,sizeof(encode_aux_nearestmatch));
216 line=get_line(in);
217 line=get_line(in);
218 line=get_line(in);
219 line=get_line(in);
220 line=get_line(in);
221 if(sscanf(line,"%ld, %ld",&(a->aux),&(a->alloc))!=2){
222 fprintf(stderr,"2: syntax in %s in line:\t %s",filename,line);
223 exit(1);
226 /* load ptr0 */
227 find_seek_to(in,"static long _vq_ptr0");
228 reset_next_value();
229 a->ptr0=_ogg_malloc(sizeof(long)*a->aux);
230 for(i=0;i<a->aux;i++)
231 if(get_next_ivalue(in,a->ptr0+i)){
232 fprintf(stderr,"out of data while reading codebook %s\n",filename);
233 exit(1);
236 /* load ptr1 */
237 find_seek_to(in,"static long _vq_ptr1");
238 reset_next_value();
239 a->ptr1=_ogg_malloc(sizeof(long)*a->aux);
240 for(i=0;i<a->aux;i++)
241 if(get_next_ivalue(in,a->ptr1+i)){
242 fprintf(stderr,"out of data while reading codebook %s\n",filename);
243 exit(1);
247 /* load p */
248 find_seek_to(in,"static long _vq_p_");
249 reset_next_value();
250 a->p=_ogg_malloc(sizeof(long)*a->aux);
251 for(i=0;i<a->aux;i++)
252 if(get_next_ivalue(in,a->p+i)){
253 fprintf(stderr,"out of data while reading codebook %s\n",filename);
254 exit(1);
257 /* load q */
258 find_seek_to(in,"static long _vq_q_");
259 reset_next_value();
260 a->q=_ogg_malloc(sizeof(long)*a->aux);
261 for(i=0;i<a->aux;i++)
262 if(get_next_ivalue(in,a->q+i)){
263 fprintf(stderr,"out of data while reading codebook %s\n",filename);
264 exit(1);
268 if(find_seek_to(in,"static encode_aux_threshmatch _vq_aux")){
269 /* how big? */
270 c->thresh_tree=t=_ogg_calloc(1,sizeof(encode_aux_threshmatch));
271 line=get_line(in);
272 line=get_line(in);
273 line=get_line(in);
274 if(sscanf(line,"%d",&(t->quantvals))!=1){
275 fprintf(stderr,"3: syntax in %s in line:\t %s",filename,line);
276 exit(1);
278 line=get_line(in);
279 if(sscanf(line,"%d",&(t->threshvals))!=1){
280 fprintf(stderr,"4: syntax in %s in line:\t %s",filename,line);
281 exit(1);
283 /* load quantthresh */
284 find_seek_to(in,"static float _vq_quantthresh_");
285 reset_next_value();
286 t->quantthresh=_ogg_malloc(sizeof(float)*t->threshvals);
287 for(i=0;i<t->threshvals-1;i++)
288 if(get_next_value(in,t->quantthresh+i)){
289 fprintf(stderr,"out of data 1 while reading codebook %s\n",filename);
290 exit(1);
292 /* load quantmap */
293 find_seek_to(in,"static long _vq_quantmap_");
294 reset_next_value();
295 t->quantmap=_ogg_malloc(sizeof(long)*t->threshvals);
296 for(i=0;i<t->threshvals;i++)
297 if(get_next_ivalue(in,t->quantmap+i)){
298 fprintf(stderr,"out of data 2 while reading codebook %s\n",filename);
299 exit(1);
303 if(find_seek_to(in,"static encode_aux_pigeonhole _vq_aux")){
304 int pigeons=1,i;
305 /* how big? */
306 c->pigeon_tree=p=_ogg_calloc(1,sizeof(encode_aux_pigeonhole));
307 line=get_line(in);
308 if(sscanf(line,"%f, %f, %d, %d",&(p->min),&(p->del),
309 &(p->mapentries),&(p->quantvals))!=4){
310 fprintf(stderr,"5: syntax in %s in line:\t %s",filename,line);
311 exit(1);
313 line=get_line(in);
314 line=get_line(in);
315 if(sscanf(line,"%ld",&(p->fittotal))!=1){
316 fprintf(stderr,"6: syntax in %s in line:\t %s",filename,line);
317 exit(1);
319 /* load pigeonmap */
320 find_seek_to(in,"static long _vq_pigeonmap_");
321 reset_next_value();
322 p->pigeonmap=_ogg_malloc(sizeof(long)*p->mapentries);
323 for(i=0;i<p->mapentries;i++)
324 if(get_next_ivalue(in,p->pigeonmap+i)){
325 fprintf(stderr,"out of data (pigeonmap) while reading codebook %s\n",filename);
326 exit(1);
328 /* load fitlist */
329 find_seek_to(in,"static long _vq_fitlist_");
330 reset_next_value();
331 p->fitlist=_ogg_malloc(sizeof(long)*p->fittotal);
332 for(i=0;i<p->fittotal;i++)
333 if(get_next_ivalue(in,p->fitlist+i)){
334 fprintf(stderr,"out of data (fitlist) while reading codebook %s\n",filename);
335 exit(1);
337 /* load fitmap */
338 find_seek_to(in,"static long _vq_fitmap_");
339 reset_next_value();
340 for(i=0;i<c->dim;i++)pigeons*=p->quantvals;
341 p->fitmap=_ogg_malloc(sizeof(long)*pigeons);
342 for(i=0;i<pigeons;i++)
343 if(get_next_ivalue(in,p->fitmap+i)){
344 fprintf(stderr,"out of data (fitmap) while reading codebook %s\n",filename);
345 exit(1);
348 /* load fitlength */
349 find_seek_to(in,"static long _vq_fitlength_");
350 reset_next_value();
351 p->fitlength=_ogg_malloc(sizeof(long)*pigeons);
352 for(i=0;i<pigeons;i++)
353 if(get_next_ivalue(in,p->fitlength+i)){
354 fprintf(stderr,"out of data (fitlength) while reading codebook %s\n",filename);
355 exit(1);
359 switch(c->maptype){
360 case 0:
361 quant_to_read=0;
362 break;
363 case 1:
364 quant_to_read=_book_maptype1_quantvals(c);
365 break;
366 case 2:
367 quant_to_read=c->entries*c->dim;
368 break;
371 /* load the quantized entries */
372 find_seek_to(in,"static long _vq_quantlist_");
373 reset_next_value();
374 c->quantlist=_ogg_malloc(sizeof(long)*quant_to_read);
375 for(i=0;i<quant_to_read;i++)
376 if(get_next_ivalue(in,c->quantlist+i)){
377 fprintf(stderr,"out of data while reading codebook %s\n",filename);
378 exit(1);
381 /* load the lengthlist */
382 find_seek_to(in,"_lengthlist");
383 reset_next_value();
384 c->lengthlist=_ogg_malloc(sizeof(long)*c->entries);
385 for(i=0;i<c->entries;i++)
386 if(get_next_ivalue(in,c->lengthlist+i)){
387 fprintf(stderr,"out of data while reading codebook %s\n",filename);
388 exit(1);
391 /* got it all */
392 fclose(in);
394 vorbis_book_init_encode(b,c);
396 return(b);
399 void spinnit(char *s,int n){
400 static int p=0;
401 static long lasttime=0;
402 long test;
403 struct timeval thistime;
405 gettimeofday(&thistime,NULL);
406 test=thistime.tv_sec*10+thistime.tv_usec/100000;
407 if(lasttime!=test){
408 lasttime=test;
410 fprintf(stderr,"%s%d ",s,n);
412 p++;if(p>3)p=0;
413 switch(p){
414 case 0:
415 fprintf(stderr,"| \r");
416 break;
417 case 1:
418 fprintf(stderr,"/ \r");
419 break;
420 case 2:
421 fprintf(stderr,"- \r");
422 break;
423 case 3:
424 fprintf(stderr,"\\ \r");
425 break;
427 fflush(stderr);
431 void build_tree_from_lengths(int vals, long *hist, long *lengths){
432 int i,j;
433 long *membership=_ogg_malloc(vals*sizeof(long));
434 long *histsave=alloca(vals*sizeof(long));
435 memcpy(histsave,hist,vals*sizeof(long));
437 for(i=0;i<vals;i++)membership[i]=i;
439 /* find codeword lengths */
440 /* much more elegant means exist. Brute force n^2, minimum thought */
441 for(i=vals;i>1;i--){
442 int first=-1,second=-1;
443 long least=-1;
445 spinnit("building... ",i);
447 /* find the two nodes to join */
448 for(j=0;j<vals;j++)
449 if(least==-1 || hist[j]<=least){
450 least=hist[j];
451 first=membership[j];
453 least=-1;
454 for(j=0;j<vals;j++)
455 if((least==-1 || hist[j]<=least) && membership[j]!=first){
456 least=hist[j];
457 second=membership[j];
459 if(first==-1 || second==-1){
460 fprintf(stderr,"huffman fault; no free branch\n");
461 exit(1);
464 /* join them */
465 least=hist[first]+hist[second];
466 for(j=0;j<vals;j++)
467 if(membership[j]==first || membership[j]==second){
468 membership[j]=first;
469 hist[j]=least;
470 lengths[j]++;
473 for(i=0;i<vals-1;i++)
474 if(membership[i]!=membership[i+1]){
475 fprintf(stderr,"huffman fault; failed to build single tree\n");
476 exit(1);
479 /* for sanity check purposes: how many bits would it have taken to
480 encode the training set? */
482 long bitsum=0;
483 long samples=0;
484 for(i=0;i<vals;i++){
485 bitsum+=(histsave[i]-1)*lengths[i];
486 samples+=histsave[i]-1;
489 if(samples){
490 fprintf(stderr,"\rTotal samples in training set: %ld \n",samples);
491 fprintf(stderr,"\rTotal bits used to represent training set: %ld\n",
492 bitsum);
496 free(membership);
499 /* wrap build_tree_from_lengths to allow zero entries in the histogram */
500 void build_tree_from_lengths0(int vals, long *hist, long *lengths){
502 /* pack the 'sparse' hit list into a dense list, then unpack
503 the lengths after the build */
505 int upper=0,i;
506 long *lengthlist=_ogg_calloc(vals,sizeof(long));
507 long *newhist=alloca(vals*sizeof(long));
509 for(i=0;i<vals;i++)
510 if(hist[i]>0)
511 newhist[upper++]=hist[i];
513 if(upper != vals){
514 fprintf(stderr,"\rEliminating %d unused entries; %d entries remain\n",
515 vals-upper,upper);
518 build_tree_from_lengths(upper,newhist,lengthlist);
520 upper=0;
521 for(i=0;i<vals;i++)
522 if(hist[i]>0)
523 lengths[i]=lengthlist[upper++];
524 else
525 lengths[i]=0;
527 free(lengthlist);
530 void write_codebook(FILE *out,char *name,const static_codebook *c){
531 encode_aux_pigeonhole *p=c->pigeon_tree;
532 encode_aux_threshmatch *t=c->thresh_tree;
533 encode_aux_nearestmatch *n=c->nearest_tree;
534 int i,j,k;
536 /* save the book in C header form */
538 /* first, the static vectors, then the book structure to tie it together. */
539 /* quantlist */
540 if(c->quantlist){
541 long vals=(c->maptype==1?_book_maptype1_quantvals(c):c->entries*c->dim);
542 fprintf(out,"static long _vq_quantlist_%s[] = {\n",name);
543 for(j=0;j<vals;j++){
544 fprintf(out,"\t%ld,\n",c->quantlist[j]);
546 fprintf(out,"};\n\n");
549 /* lengthlist */
550 fprintf(out,"static long _vq_lengthlist_%s[] = {\n",name);
551 for(j=0;j<c->entries;){
552 fprintf(out,"\t");
553 for(k=0;k<16 && j<c->entries;k++,j++)
554 fprintf(out,"%2ld,",c->lengthlist[j]);
555 fprintf(out,"\n");
557 fprintf(out,"};\n\n");
559 if(t){
560 /* quantthresh */
561 fprintf(out,"static float _vq_quantthresh_%s[] = {\n",name);
562 for(j=0;j<t->threshvals-1;){
563 fprintf(out,"\t");
564 for(k=0;k<8 && j<t->threshvals-1;k++,j++)
565 fprintf(out,"%.5g, ",t->quantthresh[j]);
566 fprintf(out,"\n");
568 fprintf(out,"};\n\n");
570 /* quantmap */
571 fprintf(out,"static long _vq_quantmap_%s[] = {\n",name);
572 for(j=0;j<t->threshvals;){
573 fprintf(out,"\t");
574 for(k=0;k<8 && j<t->threshvals;k++,j++)
575 fprintf(out,"%5ld,",t->quantmap[j]);
576 fprintf(out,"\n");
578 fprintf(out,"};\n\n");
580 fprintf(out,"static encode_aux_threshmatch _vq_auxt_%s = {\n",name);
581 fprintf(out,"\t_vq_quantthresh_%s,\n",name);
582 fprintf(out,"\t_vq_quantmap_%s,\n",name);
583 fprintf(out,"\t%d,\n",t->quantvals);
584 fprintf(out,"\t%d\n};\n\n",t->threshvals);
587 if(p){
588 int pigeons=1;
589 for(i=0;i<c->dim;i++)pigeons*=p->quantvals;
591 /* pigeonmap */
592 fprintf(out,"static long _vq_pigeonmap_%s[] = {\n",name);
593 for(j=0;j<p->mapentries;){
594 fprintf(out,"\t");
595 for(k=0;k<8 && j<p->mapentries;k++,j++)
596 fprintf(out,"%5ld, ",p->pigeonmap[j]);
597 fprintf(out,"\n");
599 fprintf(out,"};\n\n");
600 /* fitlist */
601 fprintf(out,"static long _vq_fitlist_%s[] = {\n",name);
602 for(j=0;j<p->fittotal;){
603 fprintf(out,"\t");
604 for(k=0;k<8 && j<p->fittotal;k++,j++)
605 fprintf(out,"%5ld, ",p->fitlist[j]);
606 fprintf(out,"\n");
608 fprintf(out,"};\n\n");
609 /* fitmap */
610 fprintf(out,"static long _vq_fitmap_%s[] = {\n",name);
611 for(j=0;j<pigeons;){
612 fprintf(out,"\t");
613 for(k=0;k<8 && j<pigeons;k++,j++)
614 fprintf(out,"%5ld, ",p->fitmap[j]);
615 fprintf(out,"\n");
617 fprintf(out,"};\n\n");
618 /* fitlength */
619 fprintf(out,"static long _vq_fitlength_%s[] = {\n",name);
620 for(j=0;j<pigeons;){
621 fprintf(out,"\t");
622 for(k=0;k<8 && j<pigeons;k++,j++)
623 fprintf(out,"%5ld, ",p->fitlength[j]);
624 fprintf(out,"\n");
626 fprintf(out,"};\n\n");
628 fprintf(out,"static encode_aux_pigeonhole _vq_auxp_%s = {\n",name);
629 fprintf(out,"\t%g, %g, %d, %d,\n",
630 p->min,p->del,p->mapentries,p->quantvals);
632 fprintf(out,"\t_vq_pigeonmap_%s,\n",name);
634 fprintf(out,"\t%ld,\n",p->fittotal);
635 fprintf(out,"\t_vq_fitlist_%s,\n",name);
636 fprintf(out,"\t_vq_fitmap_%s,\n",name);
637 fprintf(out,"\t_vq_fitlength_%s\n};\n\n",name);
640 if(n){
642 /* ptr0 */
643 fprintf(out,"static long _vq_ptr0_%s[] = {\n",name);
644 for(j=0;j<n->aux;){
645 fprintf(out,"\t");
646 for(k=0;k<8 && j<n->aux;k++,j++)
647 fprintf(out,"%6ld,",n->ptr0[j]);
648 fprintf(out,"\n");
650 fprintf(out,"};\n\n");
652 /* ptr1 */
653 fprintf(out,"static long _vq_ptr1_%s[] = {\n",name);
654 for(j=0;j<n->aux;){
655 fprintf(out,"\t");
656 for(k=0;k<8 && j<n->aux;k++,j++)
657 fprintf(out,"%6ld,",n->ptr1[j]);
658 fprintf(out,"\n");
660 fprintf(out,"};\n\n");
662 /* p */
663 fprintf(out,"static long _vq_p_%s[] = {\n",name);
664 for(j=0;j<n->aux;){
665 fprintf(out,"\t");
666 for(k=0;k<8 && j<n->aux;k++,j++)
667 fprintf(out,"%6ld,",n->p[j]*c->dim);
668 fprintf(out,"\n");
670 fprintf(out,"};\n\n");
672 /* q */
673 fprintf(out,"static long _vq_q_%s[] = {\n",name);
674 for(j=0;j<n->aux;){
675 fprintf(out,"\t");
676 for(k=0;k<8 && j<n->aux;k++,j++)
677 fprintf(out,"%6ld,",n->q[j]*c->dim);
678 fprintf(out,"\n");
680 fprintf(out,"};\n\n");
682 fprintf(out,"static encode_aux_nearestmatch _vq_auxn_%s = {\n",name);
683 fprintf(out,"\t_vq_ptr0_%s,\n",name);
684 fprintf(out,"\t_vq_ptr1_%s,\n",name);
685 fprintf(out,"\t_vq_p_%s,\n",name);
686 fprintf(out,"\t_vq_q_%s,\n",name);
687 fprintf(out,"\t%ld, %ld\n};\n\n",n->aux,n->aux);
690 /* tie it all together */
692 fprintf(out,"static static_codebook %s = {\n",name);
694 fprintf(out,"\t%ld, %ld,\n",c->dim,c->entries);
695 fprintf(out,"\t_vq_lengthlist_%s,\n",name);
696 fprintf(out,"\t%d, %ld, %ld, %d, %d,\n",
697 c->maptype,c->q_min,c->q_delta,c->q_quant,c->q_sequencep);
698 if(c->quantlist)
699 fprintf(out,"\t_vq_quantlist_%s,\n",name);
700 else
701 fprintf(out,"\tNULL,\n");
703 if(n)
704 fprintf(out,"\t&_vq_auxn_%s,\n",name);
705 else
706 fprintf(out,"\tNULL,\n");
707 if(t)
708 fprintf(out,"\t&_vq_auxt_%s,\n",name);
709 else
710 fprintf(out,"\tNULL,\n");
711 if(p)
712 fprintf(out,"\t&_vq_auxp_%s,\n",name);
713 else
714 fprintf(out,"\tNULL,\n");
716 fprintf(out,"\t0\n};\n\n");