admin/charsets/mapconv

   1 #!/bin/sh
   2
   3 # Copyright (C) 2015-2024 Free Software Foundation, Inc.
   4
   5 # Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011
   6 #   National Institute of Advanced Industrial Science and Technology (AIST)
   7 #   Registration Number H13PRO009
   8
   9 # This file is part of GNU Emacs.
  10
  11 # GNU Emacs is free software: you can redistribute it and/or modify
  12 # it under the terms of the GNU General Public License as published by
  13 # the Free Software Foundation, either version 3 of the License, or
  14 # (at your option) any later version.
  15
  16 # GNU Emacs is distributed in the hope that it will be useful,
  17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  19 # GNU General Public License for more details.
  20
  21 # You should have received a copy of the GNU General Public License
  22 # along with GNU Emacs.  If not, see <https://www.gnu.org/licenses/>.
  23
  24 # Commentary:
  25
  26 # Convert charset map of various format into this:
  27 #       0xXX 0xYYYY
  28 # where,
  29 #   XX is a code point of the charset in hexa-decimal,
  30 #   YYYY is the corresponding Unicode character code in hexa-decimal.
  31 # Arguments are:
  32 #   $1: source map file
  33 #   $2: address pattern for sed (optionally with substitution command)
  34 #   $3: format of source map file
  35 #       GLIBC-1 GLIBC-2 GLIBC-2-7 CZYBORRA IANA UNICODE UNICODE2 YASUOKA
  36 #   $4: awk script
  37
  38 ## So that eg [A-F] as used by KANJI-DATABASE branch below works as expected.
  39 ## Otherwise with LANG=en_US.utf8, CNS-6.map was generated with a
  40 ## bogus entry.  By experiment, LC_COLLATE=C was not enough.
  41 LC_ALL=C
  42 export LC_ALL
  43
  44 BASE=`expr "$1" : '.*/\(.*\)' '|' "$1"` # basename
  45 FILE="admin/charsets/mapfiles/$BASE"
  46 BASE=`expr "$BASE" : '\(.*\)\.gz$' '|' "$BASE"` # remove any .gz suffix
  47 AWK=${AWK:-awk}
  48
  49 case "$3" in
  50     GLIBC*)
  51         FILE="$BASE in localedata/charmaps of glibc";
  52         SOURCE="";;
  53     CZYBORRA)
  54         BASE="$BASE.gz";
  55         SOURCE="https://czyborra.com/charsets/${BASE}";;
  56     IANA)
  57         SOURCE="https://www.iana.org/assignments/charset-reg/${BASE}";;
  58     UNICODE)
  59         SOURCE="https://www.unicode.org/Public/MAPPINGS/VENDORS/ADOBE/${BASE}";;
  60     UNICODE2)
  61         SOURCE="https://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/WINDOWS/${BASE}";;
  62     YASUOKA)
  63         BASE="$BASE.Z";
  64         SOURCE="http://kanji.zinbun.kyoto-u.ac.jp/~yasuoka/ftp/CJKtable/${BASE}";;
  65     KANJI-DATABASE)
  66         # FIXME: This URL no longer works.
  67         SOURCE="http://kanji-database.cvs.sourceforge.net/viewvc/*checkout*/kanji-database/kanji-database/data/cns2ucsdkw.txt?revision=1.4";;
  68     *)
  69         printf 'Unknown file type: %s\n' "$3"
  70         exit 1;;
  71 esac
  72
  73 if [ -n "$SOURCE" ] ; then
  74     echo "# Generated from $FILE which is a copy of";
  75     echo "# $SOURCE"
  76 else
  77     echo "# Generated from $FILE"
  78 fi
  79
  80
  81 if [ -n "$4" ] ; then
  82     if [ -f "$4" ] ; then
  83         AWKPROG="$AWK -f $4"
  84     else
  85         echo "Awk program does not exist: $4"
  86         exit 1
  87     fi
  88 else
  89     AWKPROG=cat
  90 fi
  91
  92 if [ "$3" = "GLIBC-1" ] ; then
  93     # Source format is:
  94     #   <UYYYY> /xXX
  95     gunzip -c $1 | sed -n -e "${2}p" \
  96         | sed -e 's,<U\([^>]*\)>[       ]*/x\(..\).*,0x\2 0x\1,' \
  97         | sort | ${AWKPROG}
  98 elif [ "$3" = "GLIBC-2" ] ; then
  99     # Source format is:
 100     #   <UYYYY> /xXX/xZZ
 101     gunzip -c $1 | sed -n -e "${2}p" \
 102         | sed -e 's,<U\([^>]*\)>[       ]*/x\(..\)/x\(..\).*,0x\2\3 0x\1,' \
 103         | sort | ${AWKPROG}
 104 elif [ "$3" = "GLIBC-2-7" ] ; then
 105     # Source format is:
 106     #   <UYYYY> /xXX/xZZ
 107     # We must drop MSBs of XX and ZZ
 108     gunzip -c $1 | sed -n -e "${2}p" \
 109         | sed -e 's/xa/x2/g' -e 's/xb/x3/g' -e 's/xc/x4/g' \
 110               -e 's/xd/x5/g' -e 's/xe/x6/g' -e 's/xf/x7/g' \
 111               -e 's,<U\([^>]*\)>[       ]*/x\(..\)/x\(..\).*,0x\2\3 0x\1,' \
 112         | sort | ${AWKPROG}
 113 elif [ "$3" = "CZYBORRA" ] ; then
 114     # Source format is:
 115     #   =XX     U+YYYY
 116     sed -n -e "${2}p" < $1 \
 117         | sed -e 's/=\(..\)[^U]*U+\([0-9A-F]*\).*/0x\1 0x\2/' \
 118         | sort | ${AWKPROG}
 119 elif [ "$3" = "IANA" ] ; then
 120     # Source format is:
 121     #   0xXX    0xYYYY
 122     sed -n -e "${2}p" < $1 \
 123         | sed -e 's/\(0x[0-9A-Fa-f]*\)[^0]*\(0x[0-9A-Fa-f]*\).*/\1 \2/' \
 124         | sort | ${AWKPROG}
 125 elif [ "$3" = "UNICODE" ] ; then
 126     # Source format is:
 127     #   YYYY    XX
 128     # We perform reverse sort to prefer the first one in the
 129     # duplicated mappings (e.g. 0x20->U+0020, 0x20->U+00A0).
 130     sed -n -e "${2}p" < $1 \
 131         | sed -e 's/\([0-9A-F]*\)[^0-9A-F]*\([0-9A-F]*\).*/0x\2 0x\1/' \
 132         | sort -r
 133 elif [ "$3" = "UNICODE2" ] ; then
 134     # Source format is:
 135     #   0xXXXX  0xYYYY  # ...
 136     sed -n -e "${2}p" < $1 \
 137         | sed -e 's/\([0-9A-Fx]*\)[^0]*\([0-9A-Fx]*\).*/\1 \2/' \
 138         | ${AWKPROG} | sort -n -k 4,4
 139 elif [ "$3" = "YASUOKA" ] ; then
 140     # Source format is:
 141     # YYYY      0-XXXX (XXXX is a Kuten code)
 142     sed -n -e "${2}p" < $1 \
 143         | sed -e 's/\([0-9A-F]*\)[^0]*0-\([0-9]*\).*/0x\2 0x\1/' \
 144         | sort | ${AWKPROG}
 145 elif [ "$3" = "KANJI-DATABASE" ] ; then
 146     # Source format is:
 147     # C?-XXXX U+YYYYY .....
 148     sed -n -e "${2}p" < $1 \
 149         | sed -e 's/...\(....\) U+\([0-9A-F]*\).*/0x\1 0x\2/' \
 150         | sort | ${AWKPROG}
 151 else
 152     printf 'Invalid arguments: %s\n' "$3"
 153     exit 1
 154 fi