Lib/test/test_normalization.py

   1 from test.test_support import (verbose, TestFailed, TestSkipped, verify,
   2                                open_urlresource)
   3 import sys
   4 import os
   5 from unicodedata import normalize
   6
   7 TESTDATAFILE = "NormalizationTest-3.2.0" + os.extsep + "txt"
   8 TESTDATAURL = "http://www.unicode.org/Public/3.2-Update/" + TESTDATAFILE
   9
  10 class RangeError:
  11     pass
  12
  13 def NFC(str):
  14     return normalize("NFC", str)
  15
  16 def NFKC(str):
  17     return normalize("NFKC", str)
  18
  19 def NFD(str):
  20     return normalize("NFD", str)
  21
  22 def NFKD(str):
  23     return normalize("NFKD", str)
  24
  25 def unistr(data):
  26     data = [int(x, 16) for x in data.split(" ")]
  27     for x in data:
  28         if x > sys.maxunicode:
  29             raise RangeError
  30     return u"".join([unichr(x) for x in data])
  31
  32 def test_main():
  33     part1_data = {}
  34     for line in open_urlresource(TESTDATAURL):
  35         if '#' in line:
  36             line = line.split('#')[0]
  37         line = line.strip()
  38         if not line:
  39             continue
  40         if line.startswith("@Part"):
  41             part = line
  42             continue
  43         try:
  44             c1,c2,c3,c4,c5 = [unistr(x) for x in line.split(';')[:-1]]
  45         except RangeError:
  46             # Skip unsupported characters
  47             continue
  48
  49         if verbose:
  50             print line
  51
  52         # Perform tests
  53         verify(c2 ==  NFC(c1) ==  NFC(c2) ==  NFC(c3), line)
  54         verify(c4 ==  NFC(c4) ==  NFC(c5), line)
  55         verify(c3 ==  NFD(c1) ==  NFD(c2) ==  NFD(c3), line)
  56         verify(c5 ==  NFD(c4) ==  NFD(c5), line)
  57         verify(c4 == NFKC(c1) == NFKC(c2) == NFKC(c3) == NFKC(c4) == NFKC(c5),
  58                line)
  59         verify(c5 == NFKD(c1) == NFKD(c2) == NFKD(c3) == NFKD(c4) == NFKD(c5),
  60                line)
  61
  62         # Record part 1 data
  63         if part == "@Part1":
  64             part1_data[c1] = 1
  65
  66     # Perform tests for all other data
  67     for c in range(sys.maxunicode+1):
  68         X = unichr(c)
  69         if X in part1_data:
  70             continue
  71         assert X == NFC(X) == NFD(X) == NFKC(X) == NFKD(X), c
  72
  73     # Check for bug 834676
  74     normalize('NFC',u'\ud55c\uae00')
  75
  76 if __name__ == "__main__":
  77     test_main()