admin/charsets/mapconv

   1 #!/bin/sh
   2
   3 # Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011
   4 #   National Institute of Advanced Industrial Science and Technology (AIST)
   5 #   Registration Number H13PRO009
   6
   7 # This file is part of GNU Emacs.
   8
   9 # GNU Emacs is free software: you can redistribute it and/or modify
  10 # it under the terms of the GNU General Public License as published by
  11 # the Free Software Foundation, either version 3 of the License, or
  12 # (at your option) any later version.
  13
  14 # GNU Emacs is distributed in the hope that it will be useful,
  15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17 # GNU General Public License for more details.
  18
  19 # You should have received a copy of the GNU General Public License
  20 # along with GNU Emacs.  If not, see <http://www.gnu.org/licenses/>.
  21
  22 # Commentary:
  23
  24 # Convert charset map of various format into this:
  25 #       0xXX 0xYYYY
  26 # where,
  27 #   XX is a code point of the charset in hexa-decimal,
  28 #   YYYY is the corresponding Unicode character code in hexa-decimal.
  29 # Arguments are:
  30 #   $1: source map file
  31 #   $2: address pattern for sed (optionally with substitution command)
  32 #   $3: format of source map file
  33 #       GLIBC-1 GLIBC-2 GLIBC-2-7 CZYBORRA IANA UNICODE UNICODE2 YASUOKA
  34 #   $4: awk script
  35
  36 FILE="admin/charsets/$1"
  37 BASE=`basename $1 .gz`
  38
  39 case "$3" in
  40     GLIBC*)
  41         FILE="$BASE in localedate/charmaps of glibc";
  42         SOURCE="";;
  43     CZYBORRA)
  44         BASE="$BASE.gz";
  45         SOURCE="http://czyborra.com/charsets/${BASE}";;
  46     IANA)
  47         SOURCE="http://www.iana.org/assignments/charset-reg/${BASE}";;
  48     UNICODE)
  49         SOURCE="http://www.unicode.org/Public/MAPPINGS/VENDORS/ADOBE/${BASE}";;
  50     UNICODE2)
  51         SOURCE="http://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/WINDOWS/${BASE}";;
  52     YASUOKA)
  53         BASE="$BASE.Z";
  54         SOURCE="http://kanji.zinbun.kyoto-u.ac.jp/~yasuoka/ftp/CJKtable/${BASE}";;
  55     KANJI-DATABASE)
  56         SOURCE="http://kanji-database.cvs.sourceforge.net/viewvc/*checkout*/kanji-database/kanji-database/data/cns2ucsdkw.txt?revision=1.4";;
  57     *)
  58         echo "Unknown file type: $3";
  59         exit 1;;
  60 esac
  61
  62 if [ -n "$SOURCE" ] ; then
  63     echo "# Generated from $FILE which is a copy of";
  64     echo "# $SOURCE"
  65 else
  66     echo "# Generated from $FILE"
  67 fi
  68
  69
  70 if [ -n "$4" ] ; then
  71     if [ -f "$4" ] ; then
  72         AWKPROG="gawk -f $4"
  73     else
  74         echo "Awk program does not exist: $4"
  75         exit 1
  76     fi
  77 else
  78     AWKPROG=cat
  79 fi
  80
  81 if [ "$3" = "GLIBC-1" ] ; then
  82     # Source format is:
  83     #   <UYYYY> /xXX
  84     zcat $1 | sed -n -e "$2 p" \
  85         | sed -e 's,<U\([^>]*\)>[       ]*/x\(..\).*,0x\2 0x\1,' \
  86         | sort | ${AWKPROG}
  87 elif [ "$3" = "GLIBC-2" ] ; then
  88     # Source format is:
  89     #   <UYYYY> /xXX/xZZ
  90     zcat $1 | sed -n -e "$2 p" \
  91         | sed -e 's,<U\([^>]*\)>[       ]*/x\(..\)/x\(..\).*,0x\2\3 0x\1,' \
  92         | sort | ${AWKPROG}
  93 elif [ "$3" = "GLIBC-2-7" ] ; then
  94     # Source format is:
  95     #   <UYYYY> /xXX/xZZ
  96     # We must drop MSBs of XX and ZZ
  97     zcat $1 | sed -n -e "$2 p" \
  98         | sed -e 's/xa/x2/g' -e 's/xb/x3/g' -e 's/xc/x4/g' \
  99               -e 's/xd/x5/g' -e 's/xe/x6/g' -e 's/xf/x7/g' \
 100               -e 's,<U\([^>]*\)>[       ]*/x\(..\)/x\(..\).*,0x\2\3 0x\1,' \
 101         | sort | ${AWKPROG}
 102 elif [ "$3" = "CZYBORRA" ] ; then
 103     # Source format is:
 104     #   =XX     U+YYYY
 105     sed -n -e "$2 p" < $1 \
 106         | sed -e 's/=\(..\)[^U]*U+\([0-9A-F]*\).*/0x\1 0x\2/' \
 107         | sort | ${AWKPROG}
 108 elif [ "$3" = "IANA" ] ; then
 109     # Source format is:
 110     #   0xXX    0xYYYY
 111     sed -n -e "$2 p" < $1 \
 112         | sed -e 's/\(0x[0-9A-Fa-f]*\)[^0]*\(0x[0-9A-Fa-f]*\).*/\1 \2/' \
 113         | sort | ${AWKPROG}
 114 elif [ "$3" = "UNICODE" ] ; then
 115     # Source format is:
 116     #   YYYY    XX
 117     # We perform reverse sort to prefer the first one in the
 118     # duplicated mappings (e.g. 0x20->U+0020, 0x20->U+00A0).
 119     sed -n -e "$2 p" < $1 \
 120         | sed -e 's/\([0-9A-F]*\)[^0-9A-F]*\([0-9A-F]*\).*/0x\2 0x\1/' \
 121         | sort -r
 122 elif [ "$3" = "UNICODE2" ] ; then
 123     # Source format is:
 124     #   0xXXXX  0xYYYY  # ...
 125     sed -n -e "$2 p" < $1 \
 126         | sed -e 's/\([0-9A-Fx]*\)[^0]*\([0-9A-Fx]*\).*/\1 \2/' \
 127         | ${AWKPROG} | sort -n -k 4,4
 128 elif [ "$3" = "YASUOKA" ] ; then
 129     # Source format is:
 130     # YYYY      0-XXXX (XXXX is a Kuten code)
 131     sed -n -e "$2 p" < $1 \
 132         | sed -e 's/\([0-9A-F]*\)[^0]*0-\([0-9]*\).*/0x\2 0x\1/' \
 133         | sort | ${AWKPROG}
 134 elif [ "$3" = "KANJI-DATABASE" ] ; then
 135     # Source format is:
 136     # C?-XXXX U+YYYYY .....
 137     sed -n -e "$2 p" < $1 \
 138         | sed -e 's/...\(....\) U+\([0-9A-F]*\).*/0x\1 0x\2/' \
 139         | sort | ${AWKPROG}
 140 else
 141     echo "Invalid arguments: $3"
 142     exit 1
 143 fi
 144