tools/build_db.sh

   1 #!/bin/sh
   2
   3 DB_VERSION="0"
   4
   5 DIR_WRK="./wrk"
   6
   7 FILE_AOI="../aoi"
   8 FILE_SENTENCES="../data/sentences.csv.xz"
   9 FILE_INDICES="../data/jpn_indices.csv.xz"
  10 FILE_JMDICT="../data/JMdict_e.gz"
  11 FILE_KANJIDIC="../data/kanjidic2.xml.gz"
  12 FILE_KRAD="../data/kradfile-u.gz"
  13
  14 mkdir "$DIR_WRK"
  15
  16 echo "Copying files"
  17 cp "$FILE_SENTENCES" "$DIR_WRK/"
  18 cp "$FILE_INDICES" "$DIR_WRK/"
  19 cp "$FILE_JMDICT" "$DIR_WRK/"
  20 cp "$FILE_KRAD" "$DIR_WRK/"
  21 cp "$FILE_KANJIDIC" "$DIR_WRK/"
  22 cp "$FILE_AOI" "$DIR_WRK/"
  23
  24 pushd "$DIR_WRK"
  25
  26 echo "Extracting files"
  27 unxz $(basename "$FILE_SENTENCES")
  28 unxz $(basename "$FILE_INDICES")
  29 gunzip $(basename "$FILE_JMDICT")
  30 gunzip $(basename "$FILE_KANJIDIC")
  31 gunzip $(basename "$FILE_KRAD")
  32
  33 echo "Parsing files"
  34 ./aoi -parse jmdict
  35 ./aoi -parse kanjidic
  36 ../parse_tatoeba.py
  37 ../parse_kradfile.py
  38
  39
  40 echo "Building the database"
  41 rm -f new.db
  42 echo "  jmdict"
  43 sqlite3 new.db < "script.jmdict.sql"      # must be first - creates table aoi
  44 echo "  kanjidic"
  45 sqlite3 new.db < "script.kanjidic.sql"
  46 echo "  components"
  47 sqlite3 new.db < "script.components.sql"  # must be after kanjidic
  48 echo "  tatoeba"
  49 sqlite3 new.db < "script.tatoeba.sql"
  50 sqlite3 new.db "replace into aoi(key,val) values('db_version','$DB_VERSION');"
  51 sqlite3 new.db "replace into aoi(key,val) values('db_created','$(date +%Y-%m-%d)');"
  52
  53 popd