libgo/go/exp/norm/iter_test.go

   1 // Copyright 2011 The Go Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style
   3 // license that can be found in the LICENSE file.
   4
   5 package norm
   6
   7 import (
   8         "strings"
   9         "testing"
  10 )
  11
  12 var iterBufSizes = []int{
  13         MaxSegmentSize,
  14         1.5 * MaxSegmentSize,
  15         2 * MaxSegmentSize,
  16         3 * MaxSegmentSize,
  17         100 * MaxSegmentSize,
  18 }
  19
  20 func doIterNorm(f Form, buf []byte, s string) []byte {
  21         acc := []byte{}
  22         i := Iter{}
  23         i.SetInputString(f, s)
  24         for !i.Done() {
  25                 n := i.Next(buf)
  26                 acc = append(acc, buf[:n]...)
  27         }
  28         return acc
  29 }
  30
  31 func runIterTests(t *testing.T, name string, f Form, tests []AppendTest, norm bool) {
  32         for i, test := range tests {
  33                 in := test.left + test.right
  34                 gold := test.out
  35                 if norm {
  36                         gold = string(f.AppendString(nil, test.out))
  37                 }
  38                 for _, sz := range iterBufSizes {
  39                         buf := make([]byte, sz)
  40                         out := string(doIterNorm(f, buf, in))
  41                         if len(out) != len(gold) {
  42                                 const msg = "%s:%d:%d: length is %d; want %d"
  43                                 t.Errorf(msg, name, i, sz, len(out), len(gold))
  44                         }
  45                         if out != gold {
  46                                 // Find first rune that differs and show context.
  47                                 ir := []rune(out)
  48                                 ig := []rune(gold)
  49                                 for j := 0; j < len(ir) && j < len(ig); j++ {
  50                                         if ir[j] == ig[j] {
  51                                                 continue
  52                                         }
  53                                         if j -= 3; j < 0 {
  54                                                 j = 0
  55                                         }
  56                                         for e := j + 7; j < e && j < len(ir) && j < len(ig); j++ {
  57                                                 const msg = "%s:%d:%d: runeAt(%d) = %U; want %U"
  58                                                 t.Errorf(msg, name, i, sz, j, ir[j], ig[j])
  59                                         }
  60                                         break
  61                                 }
  62                         }
  63                 }
  64         }
  65 }
  66
  67 func rep(r rune, n int) string {
  68         return strings.Repeat(string(r), n)
  69 }
  70
  71 var iterTests = []AppendTest{
  72         {"", ascii, ascii},
  73         {"", txt_all, txt_all},
  74         {"", "a" + rep(0x0300, MaxSegmentSize/2), "a" + rep(0x0300, MaxSegmentSize/2)},
  75 }
  76
  77 var iterTestsD = []AppendTest{
  78         { // segment overflow on unchanged character
  79                 "",
  80                 "a" + rep(0x0300, MaxSegmentSize/2) + "\u0316",
  81                 "a" + rep(0x0300, MaxSegmentSize/2-1) + "\u0316\u0300",
  82         },
  83         { // segment overflow on unchanged character + start value
  84                 "",
  85                 "a" + rep(0x0300, MaxSegmentSize/2+maxCombiningChars+4) + "\u0316",
  86                 "a" + rep(0x0300, MaxSegmentSize/2+maxCombiningChars) + "\u0316" + rep(0x300, 4),
  87         },
  88         { // segment overflow on decomposition
  89                 "",
  90                 "a" + rep(0x0300, MaxSegmentSize/2-1) + "\u0340",
  91                 "a" + rep(0x0300, MaxSegmentSize/2),
  92         },
  93         { // segment overflow on decomposition + start value
  94                 "",
  95                 "a" + rep(0x0300, MaxSegmentSize/2-1) + "\u0340" + rep(0x300, maxCombiningChars+4) + "\u0320",
  96                 "a" + rep(0x0300, MaxSegmentSize/2-1) + rep(0x300, maxCombiningChars+1) + "\u0320" + rep(0x300, 4),
  97         },
  98         { // start value after ASCII overflow
  99                 "",
 100                 rep('a', MaxSegmentSize) + rep(0x300, maxCombiningChars+2) + "\u0320",
 101                 rep('a', MaxSegmentSize) + rep(0x300, maxCombiningChars) + "\u0320\u0300\u0300",
 102         },
 103         { // start value after Hangul overflow
 104                 "",
 105                 rep(0xAC00, MaxSegmentSize/6) + rep(0x300, maxCombiningChars+2) + "\u0320",
 106                 strings.Repeat("\u1100\u1161", MaxSegmentSize/6) + rep(0x300, maxCombiningChars-1) + "\u0320" + rep(0x300, 3),
 107         },
 108         { // start value after cc=0
 109                 "",
 110                 "您您" + rep(0x300, maxCombiningChars+4) + "\u0320",
 111                 "您您" + rep(0x300, maxCombiningChars) + "\u0320" + rep(0x300, 4),
 112         },
 113         { // start value after normalization
 114                 "",
 115                 "\u0300\u0320a" + rep(0x300, maxCombiningChars+4) + "\u0320",
 116                 "\u0320\u0300a" + rep(0x300, maxCombiningChars) + "\u0320" + rep(0x300, 4),
 117         },
 118 }
 119
 120 var iterTestsC = []AppendTest{
 121         { // ordering of non-composing combining characters
 122                 "",
 123                 "\u0305\u0316",
 124                 "\u0316\u0305",
 125         },
 126         { // segment overflow
 127                 "",
 128                 "a" + rep(0x0305, MaxSegmentSize/2+4) + "\u0316",
 129                 "a" + rep(0x0305, MaxSegmentSize/2-1) + "\u0316" + rep(0x305, 5),
 130         },
 131 }
 132
 133 func TestIterNextD(t *testing.T) {
 134         runIterTests(t, "IterNextD1", NFKD, appendTests, true)
 135         runIterTests(t, "IterNextD2", NFKD, iterTests, true)
 136         runIterTests(t, "IterNextD3", NFKD, iterTestsD, false)
 137 }
 138
 139 func TestIterNextC(t *testing.T) {
 140         runIterTests(t, "IterNextC1", NFKC, appendTests, true)
 141         runIterTests(t, "IterNextC2", NFKC, iterTests, true)
 142         runIterTests(t, "IterNextC3", NFKC, iterTestsC, false)
 143 }
 144
 145 type SegmentTest struct {
 146         in  string
 147         out []string
 148 }
 149
 150 var segmentTests = []SegmentTest{
 151         {rep('a', MaxSegmentSize), []string{rep('a', MaxSegmentSize), ""}},
 152         {rep('a', MaxSegmentSize+2), []string{rep('a', MaxSegmentSize-1), "aaa", ""}},
 153         {rep('a', MaxSegmentSize) + "\u0300aa", []string{rep('a', MaxSegmentSize-1), "a\u0300", "aa", ""}},
 154 }
 155
 156 // Note that, by design, segmentation is equal for composing and decomposing forms.
 157 func TestIterSegmentation(t *testing.T) {
 158         segmentTest(t, "SegmentTestD", NFD, segmentTests)
 159         segmentTest(t, "SegmentTestC", NFC, segmentTests)
 160 }
 161
 162 func segmentTest(t *testing.T, name string, f Form, tests []SegmentTest) {
 163         iter := Iter{}
 164         for i, tt := range segmentTests {
 165                 buf := make([]byte, MaxSegmentSize)
 166                 iter.SetInputString(f, tt.in)
 167                 for j, seg := range tt.out {
 168                         if seg == "" {
 169                                 if !iter.Done() {
 170                                         n := iter.Next(buf)
 171                                         res := string(buf[:n])
 172                                         t.Errorf(`%s:%d:%d: expected Done()==true, found segment "%s"`, name, i, j, res)
 173                                 }
 174                                 continue
 175                         }
 176                         if iter.Done() {
 177                                 t.Errorf("%s:%d:%d: Done()==true, want false", name, i, j)
 178                         }
 179                         n := iter.Next(buf)
 180                         seg = f.String(seg)
 181                         if res := string(buf[:n]); res != seg {
 182                                 t.Errorf(`%s:%d:%d" segment was "%s" (%d); want "%s" (%d)`, name, i, j, res, len(res), seg, len(seg))
 183                         }
 184                 }
 185         }
 186 }