tools/makefsdata

   1 #!/usr/bin/perl
   2 #Generate a .c source that preinitializes a file system
   3 #David Kopf <dak664@embarqmail.com> July 2009
   4 #Extended from the existing non-coffee tool
   5
   6 #The simplest format is used with httpd-fs.c. It contains a linked
   7 #list pointing to file names and file contents.
   8
   9 #An extension uses the same linked list that points to a the file names
  10 #and contents within a coffee file system having fixed file sizes.
  11 #This allows the content to be rewritten by coffee while still being
  12 #readable by httpd-fs.c New files or sector extension of existing files
  13 #is not possible, but a larger initial sector allocation could be given
  14 #to some files to allow a limited size increase. Since this would leave the
  15 #wrong file size in the linked list, httpd-fs would have to terminate on
  16 #a reverse zero scan like coffee does. Rewriting the linked list would
  17 #probably be OK if in eeprom, but not if in program flash. Suggestions
  18 #for a workaround would be welcome!
  19
  20 #Lastly a full coffee file system can be preinitialized. File reads must
  21 #then be done using coffee.
  22
  23 #Assumes the coffee file_header structure is
  24 #struct file_header {
  25 # coffee_page_t log_page;
  26 # uint16_t log_records;
  27 # uint16_t log_record_size;
  28 # coffee_page_t max_pages;
  29 # uint8_t deprecated_eof_hint;
  30 # uint8_t flags=3 for the initial value;
  31 # char name[COFFEE_NAME_LENGTH];
  32 # } __attribute__((packed));
  33
  34 goto DEFAULTS;
  35 START:$version="1.1";
  36
  37 #Process options
  38 for($n=0;$n<=$#ARGV;$n++) {
  39   $arg=$ARGV[$n];
  40   if      ($arg eq "-v") {
  41     print "makefsdata Version $version\n";
  42   } elsif ($arg eq "-A") {
  43     $n++;$attribute=$ARGV[$n];
  44   } elsif ($arg eq "-C") {
  45     $coffee=1;
  46   } elsif ($arg eq "-c") {
  47     $complement=1;
  48   } elsif ($arg eq "-i") {
  49     $n++;$includefile=$ARGV[$n];
  50 # } elsif ($arg eq "-p") {
  51 #   $n++;$coffee_page_length=$ARGV[$n];
  52   } elsif ($arg eq "-s") {
  53     $n++;$coffee_sector_size=$ARGV[$n];
  54   } elsif ($arg eq "-t") {
  55     $n++;$coffee_page_t     =$ARGV[$n];
  56   } elsif ($arg eq "-f") {
  57     $n++;$coffee_name_length=$ARGV[$n];
  58    } elsif ($arg eq "-S") {
  59     $n++;$sectionname=$ARGV[$n];
  60   } elsif ($arg eq "-l") {
  61     $linkedlist=1;
  62   } elsif ($arg eq "-d") {
  63     $n++;$directory=$ARGV[$n];
  64   } elsif ($arg eq "-o") {
  65     $n++;$outputfile=$ARGV[$n];
  66     $coffeefile=$outputfile;
  67   } else {
  68 DEFAULTS:
  69 #Set up defaults
  70 $coffee=0;
  71 #$coffee_page_length=256;
  72 $coffee_sector_size=256;
  73 $coffee_page_t=1;
  74 $coffee_name_length=16;
  75 $complement=0;
  76 $directory="";
  77 $outputfile="httpd-fsdata.c";
  78 $coffeefile="httpd-coffeedata.c";
  79 $includefile="makefsdata.h";
  80 $linkedlist=0;
  81 $attribute="";
  82 $sectionname=".coffeefiles";
  83 if (!$version) {goto START;}
  84     print "\n";
  85     print "Usage: makefsdata <option(s)> <-d input_directory> <-o output_file>\n\n";
  86     print " Generates c source file to pre-initialize a contiki file system.\n";
  87     print " The default output is a simple linked list readable by httpd-fs.c\n";
  88     print " and written to $outputfile.\n\n";
  89     print " The -C option makes a coffee file system to default output $coffeefile.\n";
  90     print " A linked list can be still be generated for use with httpd-fs.c so long\n";
  91     print " as coffee does not extend, delete, or add any files.\n\n";
  92     print " The input directory structure is copied. If input_directory is specified\n";
  93     print " it becomes the root \"/\". If no input_directory is specified the first\n";
  94     print " subdirectory found in the current directory is used as the root.\n\n";
  95     print " WARNING : If the output file exists it will be overwritten without confirmation!\n\n";
  96     print " Options are:\n";
  97     print " -v               Display the version number\n";
  98     print " -A attribute     Append \"attribute\" to the declaration, e.g. PROGMEM to put data in AVR program flash memory\n";
  99     print " -C               Use coffee file system format\n";
 100     print " -c               Complement the data, useful for obscurity or fast page erases for coffee\n";
 101     print " -i filename      Treat any input files with name \"filename\" as include files.\n";
 102     print "                  Useful for giving a server a name and ip address associated with the web content.\n";
 103     print "                  The default is $includefile.\n\n";
 104     print "   The following apply only to coffee file system\n";
 105 #   print " -p pagesize      Page size in bytes (default $coffee_page_length)\n";
 106     print " -s sectorsize    Sector size in bytes (default $coffee_sector_size)\n";
 107     print " -t page_t        Number of bytes in coffee_page_t (1,2,or 4, default $coffee_page_t)\n";
 108     print " -f namesize      File name field size in bytes (default $coffee_name_length)\n";
 109     print " -S section       Section name for data (default $sectionname)\n";
 110     print " -l               Append a linked list for use with httpd-fs\n";
 111     exit;
 112   }
 113 }
 114
 115 #--------------------Configure parameters-----------------------
 116 if ($coffee) {
 117   $outputfile=$coffeefile;
 118   $coffee_header_length=2*$coffee_page_t+$coffee_name_length+6;
 119   if ($coffee_page_t==1) {
 120     $coffeemax=0xff;
 121   } elsif ($coffee_page_t==2) {
 122     $coffeemax=0xffff;
 123   } elsif ($coffee_page_t==4) {
 124     $coffeemax=0xffffffff;
 125   } else {
 126    die "Unsupported coffee_page_t $coffee_page_t\n";
 127   }
 128 } else {
 129 # $coffee_page_length=1;
 130   $coffee_sector_size=1;
 131   $linkedlist=1;
 132   $coffee_name_length=256;
 133   $coffee_max=0xffffffff;
 134   $coffee_header_length=0;
 135 }
 136 $null="0x00";if ($complement) {$null="0xff";}
 137 $tab="  ";  #optional tabs or spaces at beginning of line, e.g. "\t\t"
 138
 139 #--------------------Create output file-------------------------
 140 #awkward but could not figure out how to compare paths later unless the file exists -- dak
 141 if (!open(OUTPUT, "> $outputfile")) {die "Aborted: Could not create output file $outputfile";}
 142 print(OUTPUT "\n");
 143 close($outputfile);
 144 use Cwd qw(abs_path);
 145 if (!open(OUTPUT, "> $outputfile")) {die "Aborted: Could not create output file $outputfile";}
 146 $outputfile=abs_path($outputfile);
 147
 148 #--------------------Get a list of input files------------------
 149 if ($directory eq "") {
 150   opendir(DIR, ".");
 151   @files =  grep { !/^\./ && !/(CVS|~)/ } readdir(DIR);
 152   closedir(DIR);
 153   foreach $file (@files) {
 154     if(-d $file && $file !~ /^\./) {
 155       $directory=$file;
 156       break;
 157     }
 158   }
 159 }
 160 if ($directory eq "") {die "Aborted: No subdirectory in current directory";}
 161 if (!chdir("$directory")) {die "Aborted: Directory \"$directory\" does not exist!";}
 162
 163 if ($coffee) {
 164   print "Processing directory $directory as root of coffee file system\n";
 165 } else {
 166   print "Processing directory $directory as root of packed httpd-fs file system\n";
 167 }
 168 opendir(DIR, ".");
 169 @files =  grep { !/^\./ && !/(CVS|~)/ } readdir(DIR);
 170 closedir(DIR);
 171
 172 foreach $file (@files) {
 173   if(-d $file && $file !~ /^\./) {
 174     print "Adding subdirectory $file\n";
 175     opendir(DIR, $file);
 176     @newfiles =  grep { !/^\./ && !/(CVS|~)/ } readdir(DIR);
 177     closedir(DIR);
 178 #   printf "Adding files @newfiles\n";
 179     @files = (@files, map { $_ = "$file/$_" } @newfiles);
 180     next;
 181   }
 182 }
 183 #--------------------Write the output file-------------------
 184 print "Writing to $outputfile\n";
 185 ($DAY, $MONTH, $YEAR) = (localtime)[3,4,5];
 186 printf(OUTPUT "/*********Generated by contiki/tools/makefsdata on %04d-%02d-%02d*********/\n", $YEAR+1900, $MONTH+1, $DAY);
 187 if ($coffee) {
 188   print(OUTPUT "/*For coffee filesystem of sector size $coffee_sector_size and header length $coffee_header_length bytes*/\n");
 189 }
 190 print(OUTPUT "\n");
 191 #--------------------Process include file-------------------
 192 foreach $file (@files) {if ($file eq $includefile) {
 193   open(FILE, $file) || die "Aborted: Could not open include file $file\n";
 194   print "Including text from $file\n";
 195   $file_length= -s FILE;
 196   read(FILE, $data, $file_length);
 197   print OUTPUT ($data);
 198   close(FILE);
 199   next;        #include all include file matches
 200 # break;       #include only first include file match
 201 }}
 202
 203 #--------------------Process data files-------------------
 204 $n=0;$coffeesize=0;$coffeesectors=0;
 205 foreach $file (@files) {if(-f $file) {
 206   if (length($file)>($coffee_name_length-1)) {die "Aborted: File name $file is too long";}
 207   if (abs_path("$file") eq abs_path("$outputfile")) {
 208     print "Skipping output file $outputfile - recursive input NOT allowed\n";
 209     next;
 210   }
 211   if ($file eq $includefile) {next;}
 212   open(FILE, $file) || die "Aborted: Could not open file $file\n";
 213   print "Adding /$file\n";
 214   if (grep /.png/||/.jpg/||/jpeg/||/.pdf/||/.gif/||/.bin/||/.zip/,$file) {binmode FILE;}
 215
 216   $file_length= -s FILE;
 217   $file =~ s-^-/-;
 218   $fvar = $file;
 219   $fvar =~ s-/-_-g;
 220   $fvar =~ s-\.-_-g;
 221
 222   if ($coffee) {
 223     $coffee_sectors=int(($coffee_header_length+$file_length+$coffee_sector_size-1)/$coffee_sector_size);
 224 #   $coffee_sectors=sprintf("%.0f",($coffee_header_length+$file_length+$coffee_sector_size-1)/$coffee_sector_size)-1;
 225     $coffee_length=$coffee_sectors*$coffee_sector_size;
 226   } else {
 227     $coffee_length=$file_length+length($file)+2;
 228   }
 229   $flen[$n]=$file_length;
 230   $clen[$n]=$coffee_length;
 231   $n++;$coffeesectors+=$coffee_sectors;$coffeesize+=$coffee_length;
 232   if ($coffee) {
 233     if ($coffeesectors>$coffeemax) {
 234       print "Warning: sector number $coffeesectors overflows allocated sector size in coffee header\n";
 235     }
 236     print(OUTPUT "\n__attribute__ ((section (\"$sectionname\")))\n");
 237     print(OUTPUT "volatile const char data".$fvar."[$coffee_length] = {\n");
 238   } else {
 239     print(OUTPUT "\nconst char data".$fvar."[$coffee_length] $attribute = {\n");
 240   }
 241   print(OUTPUT "$tab/* $file */\n$tab");
 242 #--------------------Header-----------------------------
 243 #log_page
 244   if ($coffee) {
 245     print (OUTPUT " ");
 246     for($j=0;$j<$coffee_page_t;$j++) {print (OUTPUT "$ null ,");}
 247 #log_records, log_record_size
 248     for($j=0;$j<4;$j++) {print (OUTPUT "$null, ");}
 249 #max_pages
 250     if ($complement) {$coffee_sectors=$coffee_sectors^0xffffffff;}
 251     if ($coffee_page_t==1) {
 252       printf(OUTPUT "0x%2.2x, ",($coffee_sectors    )&0xff);
 253     } elsif ($coffee_page_t==2) {
 254       printf(OUTPUT "0x%2.2x, ",($coffee_sectors>> 8)&0xff);
 255       printf(OUTPUT "0x%2.2x, ",($coffee_sectors    )&0xff);
 256     } elsif ($coffee_page_t==4) {
 257       printf(OUTPUT "0x%2.2x, ",($coffee_sectors>>24)&0xff);
 258       printf(OUTPUT "0x%2.2x, ",($coffee_sectors>>16)&0xff);
 259       printf(OUTPUT "0x%2.2x, ",($coffee_sectors>> 8)&0xff);
 260       printf(OUTPUT "0x%2.2x, ",($coffee_sectors    )&0xff);
 261     }
 262     if ($complement) {$coffee_sectors=$coffee_sectors^0xffffffff;}
 263 #eof hint and flags
 264     if ($complement) {
 265       print(OUTPUT "0xff, 0xfc,\n$tab");
 266     } else {
 267       print(OUTPUT "0x00, 0x03,\n$tab");
 268     }
 269   }
 270
 271 #-------------------File name--------------------------
 272   for($j = 0; $j < length($file); $j++) {
 273     $temp=unpack("C", substr($file, $j, 1));
 274     if ($complement) {$temp=$temp^0xff;}
 275     printf(OUTPUT " %#02.2x,", $temp);
 276   }
 277   if ($coffee) {
 278     for(; $j < $coffee_name_length; $j++) {printf(OUTPUT " $null,");}
 279   } else {
 280    {printf(OUTPUT " $null,");}
 281   }
 282   print(OUTPUT "\n$tab");
 283 #------------------File Data---------------------------
 284   $coffee_length-=$coffee_header_length;
 285   $i = 0;
 286   while(read(FILE, $data, 1)) {
 287     $temp=unpack("C", $data);
 288     if ($complement) {$temp=$temp^0xff;}
 289     printf(OUTPUT " 0x%2.2x,", $temp);
 290     $i++;$coffee_length--;
 291     if($i == 10) {
 292       print(OUTPUT "\n$tab");
 293       $i = 0;
 294     }
 295   }
 296   if ($coffee) {
 297     while (--$coffee_length>1) {
 298       print (OUTPUT " $null,");
 299       if($i++ == 9) {
 300         print(OUTPUT "\n$tab");
 301         $i = 0;
 302       }
 303     }
 304   }
 305   print (OUTPUT " $null};\n");
 306   close(FILE);
 307   push(@fvars, $fvar);
 308   push(@pfiles, $file);
 309 }}
 310
 311 if ($linkedlist) {
 312 #-------------------httpd_fsdata_file links-------------------
 313 #The non-coffee PROGMEM flash file system for the Raven webserver uses a linked flash list as follows:
 314 print(OUTPUT "\n\n/* Structure of linked list (all offsets relative to start of section):\n");
 315 print(OUTPUT "struct httpd_fsdata_file {\n");
 316 print(OUTPUT "$tab const struct httpd_fsdata_file *next; //actual flash address of next link\n");
 317 print(OUTPUT "$tab const char *name;                     //offset to coffee file name\n");
 318 print(OUTPUT "$tab const char *data;                     //offset to coffee file data\n");
 319 print(OUTPUT "$tab const int len;                        //length of file data\n");
 320 print(OUTPUT "#if HTTPD_FS_STATISTICS == 1             //not enabled since list is in PROGMEM\n");
 321 print(OUTPUT "$tab u16_t count;                          //storage for file statistics\n");
 322 print(OUTPUT "#endif\n");
 323 print(OUTPUT "}\n*/\n");
 324
 325 # For the static httpd-fs.c file system the file name and data addresses in the linked list
 326 # point to the actual memory locations.
 327 # For the coffee file system the addresses start from zero. The starting address must be added
 328 # in the coffee read routine.
 329
 330 for($i = 0; $i < @fvars; $i++) {
 331   $file = $pfiles[$i];
 332   $fvar = $fvars[$i];
 333   if($i == 0) {
 334       $prevfile = "NULL";
 335       $data_offset=0;
 336   } else {
 337       $data_offset=$data_offset+$clen[$i-1];
 338       $prevfile = "file" . $fvars[$i - 1];
 339   }
 340   $filename_offset=$data_offset+6+2*$coffee_page_t;
 341   $coffee_offset=$data_offset+$coffee_header_length;
 342   if ($coffee_offset>0xffff) {print "Warning : Linked list offset field overflow\n";}
 343   print(OUTPUT "const struct httpd_fsdata_file");
 344   for ($t=length($file);$t<16;$t++) {print(OUTPUT " ")};
 345   print(OUTPUT " file".$fvar."[] ");
 346   if ($attribute) {print(OUTPUT "$attribute ");}
 347   print(OUTPUT "={{");
 348   for ($t=length($prevfile);$t<20;$t++) {print(OUTPUT " ")};
 349   print(OUTPUT "$prevfile, ");
 350   if ($coffee) {
 351     printf(OUTPUT "(const char *)0x%4.4x, ",$filename_offset);
 352     printf(OUTPUT "(const char *)0x%4.4x, ",$coffee_offset);
 353     printf(OUTPUT "%5u}};\n",$flen[$i]);
 354   } else {
 355     print(OUTPUT "data$fvar");
 356     for ($t=length($file);$t<15;$t++) {print(OUTPUT " ")};
 357     print(OUTPUT ", data$fvar");
 358     for ($t=length($file);$t<15;$t++) {print(OUTPUT " ")};
 359     print(OUTPUT " +".(length($file)+1).", sizeof(data$fvar)");
 360     for ($t=length($file);$t<16;$t++) {print(OUTPUT " ")};
 361     print(OUTPUT " -".(length($file)+1)."}};\n");
 362   }
 363 }
 364 print(OUTPUT "\n#define HTTPD_FS_ROOT  file$fvars[$n-1]\n");
 365 print(OUTPUT "#define HTTPD_FS_NUMFILES  $n\n");
 366 print(OUTPUT "#define HTTPD_FS_SIZE $coffeesize\n");
 367 }
 368 print "All done, files occupy $coffeesize bytes\n";
 369