apertium-transfer-tools/src/preprocess-corpus.awk

   1
   2 # This script reads a corpus that has been analysed. It preprocess the corpus
   3 # before training the word alignments using GIZA++.
   4 #
   5 # It remove all character not belonging to apertium words and replace each
   6 # space within apertium words by the character '_'.
   7 #
   8 # Example of input:
   9 #
  10 # Example of output:
  11 #
  12 # author: Felipe Sánchez-Martínez
  13
  14 function trim(palabra) {
  15    for(i=1;i<=length(palabra);i++){
  16      if(substr(palabra,i,1)~/[ \t\r\n]/);
  17      else break;
  18    }
  19    liminf=i;
  20
  21    for(i=length(palabra);i>=1;i--){
  22      if(substr(palabra,i,1)~/[ \t\r\n]/);
  23      else break;
  24    }
  25
  26    limsup=i;
  27
  28    return substr(palabra,liminf,limsup-liminf+1);
  29 }
  30
  31 BEGIN{FS="\\$"}
  32 {
  33   c="";
  34   for (j=1; j<=NF; j++) {
  35     w=trim($j);
  36     w=substr(w,index(w,"^"));
  37
  38     if ((length(w)>0) && (index(w,"^")>0)) {
  39       gsub(" ", "_", w);
  40
  41       if (length(c)>0)
  42         c = c " ";
  43
  44       c = c w "$";
  45     }
  46   }
  47   #if (length(c)>0)
  48   print c;
  49 }