1 // Copyright 2013 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
18 // Test white space table matches the Unicode definition.
19 func TestSpace(t
*testing
.T
) {
20 for r
:= rune(0); r
<= utf8
.MaxRune
; r
++ {
21 if IsSpace(r
) != unicode
.IsSpace(r
) {
22 t
.Fatalf("white space property disagrees: %#U should be %t", r
, unicode
.IsSpace(r
))
27 var scanTests
= []string{
32 "\x81", // UTF-8 error
33 "\uFFFD", // correctly encoded RuneError
36 "abc¼☹\x81\uFFFD日本語\x82abc",
39 func TestScanByte(t
*testing
.T
) {
40 for n
, test
:= range scanTests
{
41 buf
:= bytes
.NewBufferString(test
)
45 for i
= 0; s
.Scan(); i
++ {
46 if b
:= s
.Bytes(); len(b
) != 1 || b
[0] != test
[i
] {
47 t
.Errorf("#%d: %d: expected %q got %q", n
, i
, test
, b
)
51 t
.Errorf("#%d: termination expected at %d; got %d", n
, len(test
), i
)
55 t
.Errorf("#%d: %v", n
, err
)
60 // Test that the rune splitter returns same sequence of runes (not bytes) as for range string.
61 func TestScanRune(t
*testing
.T
) {
62 for n
, test
:= range scanTests
{
63 buf
:= bytes
.NewBufferString(test
)
68 // Use a string range loop to validate the sequence of runes.
69 for i
, expect
= range string(test
) {
74 got
, _
:= utf8
.DecodeRune(s
.Bytes())
76 t
.Errorf("#%d: %d: expected %q got %q", n
, i
, expect
, got
)
80 t
.Errorf("#%d: scan ran too long, got %q", n
, s
.Text())
82 testRuneCount
:= utf8
.RuneCountInString(test
)
83 if runeCount
!= testRuneCount
{
84 t
.Errorf("#%d: termination expected at %d; got %d", n
, testRuneCount
, runeCount
)
88 t
.Errorf("#%d: %v", n
, err
)
93 var wordScanTests
= []string{
101 " abc\tdef\nghi\rjkl\fmno\vpqr\u0085stu\u00a0\n",
104 // Test that the word splitter returns the same data as strings.Fields.
105 func TestScanWords(t
*testing
.T
) {
106 for n
, test
:= range wordScanTests
{
107 buf
:= bytes
.NewBufferString(test
)
110 words
:= strings
.Fields(test
)
112 for wordCount
= 0; wordCount
< len(words
); wordCount
++ {
117 if got
!= words
[wordCount
] {
118 t
.Errorf("#%d: %d: expected %q got %q", n
, wordCount
, words
[wordCount
], got
)
122 t
.Errorf("#%d: scan ran too long, got %q", n
, s
.Text())
124 if wordCount
!= len(words
) {
125 t
.Errorf("#%d: termination expected at %d; got %d", n
, len(words
), wordCount
)
129 t
.Errorf("#%d: %v", n
, err
)
134 // slowReader is a reader that returns only a few bytes at a time, to test the incremental
135 // reads in Scanner.Scan.
136 type slowReader
struct {
141 func (sr
*slowReader
) Read(p
[]byte) (n
int, err error
) {
145 return sr
.buf
.Read(p
)
148 // genLine writes to buf a predictable but non-trivial line of text of length
149 // n, including the terminal newline and an occasional carriage return.
150 // If addNewline is false, the \r and \n are not emitted.
151 func genLine(buf
*bytes
.Buffer
, lineNum
, n
int, addNewline
bool) {
153 doCR
:= lineNum%5
== 0
157 for i
:= 0; i
< n
-1; i
++ { // Stop early for \n.
158 c
:= 'a' + byte(lineNum
+i
)
159 if c
== '\n' || c
== '\r' { // Don't confuse us.
173 // Test the line splitter, including some carriage returns but no long lines.
174 func TestScanLongLines(t
*testing
.T
) {
175 const smallMaxTokenSize
= 256 // Much smaller for more efficient testing.
176 // Build a buffer of lots of line lengths up to but not exceeding smallMaxTokenSize.
177 tmp
:= new(bytes
.Buffer
)
178 buf
:= new(bytes
.Buffer
)
181 for i
:= 0; i
< 2*smallMaxTokenSize
; i
++ {
182 genLine(tmp
, lineNum
, j
, true)
183 if j
< smallMaxTokenSize
{
188 buf
.Write(tmp
.Bytes())
191 s
:= NewScanner(&slowReader
{1, buf
})
193 s
.MaxTokenSize(smallMaxTokenSize
)
195 for lineNum
:= 0; s
.Scan(); lineNum
++ {
196 genLine(tmp
, lineNum
, j
, false)
197 if j
< smallMaxTokenSize
{
202 line
:= tmp
.String() // We use the string-valued token here, for variety.
203 if s
.Text() != line
{
204 t
.Errorf("%d: bad line: %d %d\n%.100q\n%.100q\n", lineNum
, len(s
.Bytes()), len(line
), s
.Text(), line
)
213 // Test that the line splitter errors out on a long line.
214 func TestScanLineTooLong(t
*testing
.T
) {
215 const smallMaxTokenSize
= 256 // Much smaller for more efficient testing.
216 // Build a buffer of lots of line lengths up to but not exceeding smallMaxTokenSize.
217 tmp
:= new(bytes
.Buffer
)
218 buf
:= new(bytes
.Buffer
)
221 for i
:= 0; i
< 2*smallMaxTokenSize
; i
++ {
222 genLine(tmp
, lineNum
, j
, true)
224 buf
.Write(tmp
.Bytes())
227 s
:= NewScanner(&slowReader
{3, buf
})
229 s
.MaxTokenSize(smallMaxTokenSize
)
231 for lineNum
:= 0; s
.Scan(); lineNum
++ {
232 genLine(tmp
, lineNum
, j
, false)
233 if j
< smallMaxTokenSize
{
239 if !bytes
.Equal(s
.Bytes(), line
) {
240 t
.Errorf("%d: bad line: %d %d\n%.100q\n%.100q\n", lineNum
, len(s
.Bytes()), len(line
), s
.Bytes(), line
)
244 if err
!= ErrTooLong
{
245 t
.Fatalf("expected ErrTooLong; got %s", err
)
249 // Test that the line splitter handles a final line without a newline.
250 func testNoNewline(text
string, lines
[]string, t
*testing
.T
) {
251 buf
:= bytes
.NewBufferString(text
)
252 s
:= NewScanner(&slowReader
{7, buf
})
254 for lineNum
:= 0; s
.Scan(); lineNum
++ {
255 line
:= lines
[lineNum
]
256 if s
.Text() != line
{
257 t
.Errorf("%d: bad line: %d %d\n%.100q\n%.100q\n", lineNum
, len(s
.Bytes()), len(line
), s
.Bytes(), line
)
266 var noNewlineLines
= []string{
267 "abcdefghijklmn\nopqrstuvwxyz",
270 // Test that the line splitter handles a final line without a newline.
271 func TestScanLineNoNewline(t
*testing
.T
) {
272 const text
= "abcdefghijklmn\nopqrstuvwxyz"
277 testNoNewline(text
, lines
, t
)
280 // Test that the line splitter handles a final line with a carriage return but nonewline.
281 func TestScanLineReturnButNoNewline(t
*testing
.T
) {
282 const text
= "abcdefghijklmn\nopqrstuvwxyz\r"
287 testNoNewline(text
, lines
, t
)
290 // Test that the line splitter handles a final empty line.
291 func TestScanLineEmptyFinalLine(t
*testing
.T
) {
292 const text
= "abcdefghijklmn\nopqrstuvwxyz\n\n"
298 testNoNewline(text
, lines
, t
)
301 // Test that the line splitter handles a final empty line with a carriage return but no newline.
302 func TestScanLineEmptyFinalLineWithCR(t
*testing
.T
) {
303 const text
= "abcdefghijklmn\nopqrstuvwxyz\n\r"
309 testNoNewline(text
, lines
, t
)
312 var testError
= errors
.New("testError")
314 // Test the correct error is returned when the split function errors out.
315 func TestSplitError(t
*testing
.T
) {
316 // Create a split function that delivers a little data, then a predictable error.
319 errorSplit
:= func(data
[]byte, atEOF
bool) (advance
int, token
[]byte, err error
) {
321 panic("didn't get enough data")
323 if numSplits
>= okCount
{
324 return 0, nil, testError
327 return 1, data
[0:1], nil
330 const text
= "abcdefghijklmnopqrstuvwxyz"
331 buf
:= bytes
.NewBufferString(text
)
332 s
:= NewScanner(&slowReader
{1, buf
})
335 for i
= 0; s
.Scan(); i
++ {
336 if len(s
.Bytes()) != 1 || text
[i
] != s
.Bytes()[0] {
337 t
.Errorf("#%d: expected %q got %q", i
, text
[i
], s
.Bytes()[0])
340 // Check correct termination location and error.
342 t
.Errorf("unexpected termination; expected %d tokens got %d", okCount
, i
)
345 if err
!= testError
{
346 t
.Fatalf("expected %q got %v", testError
, err
)
350 // Test that an EOF is overridden by a user-generated scan error.
351 func TestErrAtEOF(t
*testing
.T
) {
352 s
:= NewScanner(strings
.NewReader("1 2 33"))
353 // This spitter will fail on last entry, after s.err==EOF.
354 split
:= func(data
[]byte, atEOF
bool) (advance
int, token
[]byte, err error
) {
355 advance
, token
, err
= ScanWords(data
, atEOF
)
357 if s
.ErrOrEOF() != io
.EOF
{
358 t
.Fatal("not testing EOF")
367 if s
.Err() != testError
{
368 t
.Fatal("wrong error:", s
.Err())
372 // Test for issue 5268.
373 type alwaysError
struct{}
375 func (alwaysError
) Read(p
[]byte) (int, error
) {
376 return 0, io
.ErrUnexpectedEOF
379 func TestNonEOFWithEmptyRead(t
*testing
.T
) {
380 scanner
:= NewScanner(alwaysError
{})
382 t
.Fatal("read should fail")
385 if err
!= io
.ErrUnexpectedEOF
{
386 t
.Errorf("unexpected error: %v", err
)
390 // Test that Scan finishes if we have endless empty reads.
391 type endlessZeros
struct{}
393 func (endlessZeros
) Read(p
[]byte) (int, error
) {
397 func TestBadReader(t
*testing
.T
) {
398 scanner
:= NewScanner(endlessZeros
{})
400 t
.Fatal("read should fail")
403 if err
!= io
.ErrNoProgress
{
404 t
.Errorf("unexpected error: %v", err
)