gcc/ChangeLog:

[official-gcc.git] / libgo / go / bufio / scan_test.go

// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package bufio_test

import (
        . "bufio"
        "bytes"
        "errors"
        "io"
        "strings"
        "testing"
        "unicode"
        "unicode/utf8"
)

const smallMaxTokenSize = 256 // Much smaller for more efficient testing.

// Test white space table matches the Unicode definition.
func TestSpace(t *testing.T) {
        for r := rune(0); r <= utf8.MaxRune; r++ {
                if IsSpace(r) != unicode.IsSpace(r) {
                        t.Fatalf("white space property disagrees: %#U should be %t", r, unicode.IsSpace(r))
                }
        }
}

var scanTests = []string{
        "",
        "a",
        "¼",
        "☹",
        "\x81",   // UTF-8 error
        "\uFFFD", // correctly encoded RuneError
        "abcdefgh",
        "abc def\n\t\tgh    ",
        "abc¼☹\x81\uFFFD日本語\x82abc",
}

func TestScanByte(t *testing.T) {
        for n, test := range scanTests {
                buf := strings.NewReader(test)
                s := NewScanner(buf)
                s.Split(ScanBytes)
                var i int
                for i = 0; s.Scan(); i++ {
                        if b := s.Bytes(); len(b) != 1 || b[0] != test[i] {
                                t.Errorf("#%d: %d: expected %q got %q", n, i, test, b)
                        }
                }
                if i != len(test) {
                        t.Errorf("#%d: termination expected at %d; got %d", n, len(test), i)
                }
                err := s.Err()
                if err != nil {
                        t.Errorf("#%d: %v", n, err)
                }
        }
}

// Test that the rune splitter returns same sequence of runes (not bytes) as for range string.
func TestScanRune(t *testing.T) {
        for n, test := range scanTests {
                buf := strings.NewReader(test)
                s := NewScanner(buf)
                s.Split(ScanRunes)
                var i, runeCount int
                var expect rune
                // Use a string range loop to validate the sequence of runes.
                for i, expect = range string(test) {
                        if !s.Scan() {
                                break
                        }
                        runeCount++
                        got, _ := utf8.DecodeRune(s.Bytes())
                        if got != expect {
                                t.Errorf("#%d: %d: expected %q got %q", n, i, expect, got)
                        }
                }
                if s.Scan() {
                        t.Errorf("#%d: scan ran too long, got %q", n, s.Text())
                }
                testRuneCount := utf8.RuneCountInString(test)
                if runeCount != testRuneCount {
                        t.Errorf("#%d: termination expected at %d; got %d", n, testRuneCount, runeCount)
                }
                err := s.Err()
                if err != nil {
                        t.Errorf("#%d: %v", n, err)
                }
        }
}

var wordScanTests = []string{
        "",
        " ",
        "\n",
        "a",
        " a ",
        "abc def",
        " abc def ",
        " abc\tdef\nghi\rjkl\fmno\vpqr\u0085stu\u00a0\n",
}

// Test that the word splitter returns the same data as strings.Fields.
func TestScanWords(t *testing.T) {
        for n, test := range wordScanTests {
                buf := strings.NewReader(test)
                s := NewScanner(buf)
                s.Split(ScanWords)
                words := strings.Fields(test)
                var wordCount int
                for wordCount = 0; wordCount < len(words); wordCount++ {
                        if !s.Scan() {
                                break
                        }
                        got := s.Text()
                        if got != words[wordCount] {
                                t.Errorf("#%d: %d: expected %q got %q", n, wordCount, words[wordCount], got)
                        }
                }
                if s.Scan() {
                        t.Errorf("#%d: scan ran too long, got %q", n, s.Text())
                }
                if wordCount != len(words) {
                        t.Errorf("#%d: termination expected at %d; got %d", n, len(words), wordCount)
                }
                err := s.Err()
                if err != nil {
                        t.Errorf("#%d: %v", n, err)
                }
        }
}

// slowReader is a reader that returns only a few bytes at a time, to test the incremental
// reads in Scanner.Scan.
type slowReader struct {
        max int
        buf io.Reader
}

func (sr *slowReader) Read(p []byte) (n int, err error) {
        if len(p) > sr.max {
                p = p[0:sr.max]
        }
        return sr.buf.Read(p)
}

// genLine writes to buf a predictable but non-trivial line of text of length
// n, including the terminal newline and an occasional carriage return.
// If addNewline is false, the \r and \n are not emitted.
func genLine(buf *bytes.Buffer, lineNum, n int, addNewline bool) {
        buf.Reset()
        doCR := lineNum%5 == 0
        if doCR {
                n--
        }
        for i := 0; i < n-1; i++ { // Stop early for \n.
                c := 'a' + byte(lineNum+i)
                if c == '\n' || c == '\r' { // Don't confuse us.
                        c = 'N'
                }
                buf.WriteByte(c)
        }
        if addNewline {
                if doCR {
                        buf.WriteByte('\r')
                }
                buf.WriteByte('\n')
        }
        return
}

// Test the line splitter, including some carriage returns but no long lines.
func TestScanLongLines(t *testing.T) {
        // Build a buffer of lots of line lengths up to but not exceeding smallMaxTokenSize.
        tmp := new(bytes.Buffer)
        buf := new(bytes.Buffer)
        lineNum := 0
        j := 0
        for i := 0; i < 2*smallMaxTokenSize; i++ {
                genLine(tmp, lineNum, j, true)
                if j < smallMaxTokenSize {
                        j++
                } else {
                        j--
                }
                buf.Write(tmp.Bytes())
                lineNum++
        }
        s := NewScanner(&slowReader{1, buf})
        s.Split(ScanLines)
        s.MaxTokenSize(smallMaxTokenSize)
        j = 0
        for lineNum := 0; s.Scan(); lineNum++ {
                genLine(tmp, lineNum, j, false)
                if j < smallMaxTokenSize {
                        j++
                } else {
                        j--
                }
                line := tmp.String() // We use the string-valued token here, for variety.
                if s.Text() != line {
                        t.Errorf("%d: bad line: %d %d\n%.100q\n%.100q\n", lineNum, len(s.Bytes()), len(line), s.Text(), line)
                }
        }
        err := s.Err()
        if err != nil {
                t.Fatal(err)
        }
}

// Test that the line splitter errors out on a long line.
func TestScanLineTooLong(t *testing.T) {
        const smallMaxTokenSize = 256 // Much smaller for more efficient testing.
        // Build a buffer of lots of line lengths up to but not exceeding smallMaxTokenSize.
        tmp := new(bytes.Buffer)
        buf := new(bytes.Buffer)
        lineNum := 0
        j := 0
        for i := 0; i < 2*smallMaxTokenSize; i++ {
                genLine(tmp, lineNum, j, true)
                j++
                buf.Write(tmp.Bytes())
                lineNum++
        }
        s := NewScanner(&slowReader{3, buf})
        s.Split(ScanLines)
        s.MaxTokenSize(smallMaxTokenSize)
        j = 0
        for lineNum := 0; s.Scan(); lineNum++ {
                genLine(tmp, lineNum, j, false)
                if j < smallMaxTokenSize {
                        j++
                } else {
                        j--
                }
                line := tmp.Bytes()
                if !bytes.Equal(s.Bytes(), line) {
                        t.Errorf("%d: bad line: %d %d\n%.100q\n%.100q\n", lineNum, len(s.Bytes()), len(line), s.Bytes(), line)
                }
        }
        err := s.Err()
        if err != ErrTooLong {
                t.Fatalf("expected ErrTooLong; got %s", err)
        }
}

// Test that the line splitter handles a final line without a newline.
func testNoNewline(text string, lines []string, t *testing.T) {
        buf := strings.NewReader(text)
        s := NewScanner(&slowReader{7, buf})
        s.Split(ScanLines)
        for lineNum := 0; s.Scan(); lineNum++ {
                line := lines[lineNum]
                if s.Text() != line {
                        t.Errorf("%d: bad line: %d %d\n%.100q\n%.100q\n", lineNum, len(s.Bytes()), len(line), s.Bytes(), line)
                }
        }
        err := s.Err()
        if err != nil {
                t.Fatal(err)
        }
}

// Test that the line splitter handles a final line without a newline.
func TestScanLineNoNewline(t *testing.T) {
        const text = "abcdefghijklmn\nopqrstuvwxyz"
        lines := []string{
                "abcdefghijklmn",
                "opqrstuvwxyz",
        }
        testNoNewline(text, lines, t)
}

// Test that the line splitter handles a final line with a carriage return but no newline.
func TestScanLineReturnButNoNewline(t *testing.T) {
        const text = "abcdefghijklmn\nopqrstuvwxyz\r"
        lines := []string{
                "abcdefghijklmn",
                "opqrstuvwxyz",
        }
        testNoNewline(text, lines, t)
}

// Test that the line splitter handles a final empty line.
func TestScanLineEmptyFinalLine(t *testing.T) {
        const text = "abcdefghijklmn\nopqrstuvwxyz\n\n"
        lines := []string{
                "abcdefghijklmn",
                "opqrstuvwxyz",
                "",
        }
        testNoNewline(text, lines, t)
}

// Test that the line splitter handles a final empty line with a carriage return but no newline.
func TestScanLineEmptyFinalLineWithCR(t *testing.T) {
        const text = "abcdefghijklmn\nopqrstuvwxyz\n\r"
        lines := []string{
                "abcdefghijklmn",
                "opqrstuvwxyz",
                "",
        }
        testNoNewline(text, lines, t)
}

var testError = errors.New("testError")

// Test the correct error is returned when the split function errors out.
func TestSplitError(t *testing.T) {
        // Create a split function that delivers a little data, then a predictable error.
        numSplits := 0
        const okCount = 7
        errorSplit := func(data []byte, atEOF bool) (advance int, token []byte, err error) {
                if atEOF {
                        panic("didn't get enough data")
                }
                if numSplits >= okCount {
                        return 0, nil, testError
                }
                numSplits++
                return 1, data[0:1], nil
        }
        // Read the data.
        const text = "abcdefghijklmnopqrstuvwxyz"
        buf := strings.NewReader(text)
        s := NewScanner(&slowReader{1, buf})
        s.Split(errorSplit)
        var i int
        for i = 0; s.Scan(); i++ {
                if len(s.Bytes()) != 1 || text[i] != s.Bytes()[0] {
                        t.Errorf("#%d: expected %q got %q", i, text[i], s.Bytes()[0])
                }
        }
        // Check correct termination location and error.
        if i != okCount {
                t.Errorf("unexpected termination; expected %d tokens got %d", okCount, i)
        }
        err := s.Err()
        if err != testError {
                t.Fatalf("expected %q got %v", testError, err)
        }
}

// Test that an EOF is overridden by a user-generated scan error.
func TestErrAtEOF(t *testing.T) {
        s := NewScanner(strings.NewReader("1 2 33"))
        // This splitter will fail on last entry, after s.err==EOF.
        split := func(data []byte, atEOF bool) (advance int, token []byte, err error) {
                advance, token, err = ScanWords(data, atEOF)
                if len(token) > 1 {
                        if s.ErrOrEOF() != io.EOF {
                                t.Fatal("not testing EOF")
                        }
                        err = testError
                }
                return
        }
        s.Split(split)
        for s.Scan() {
        }
        if s.Err() != testError {
                t.Fatal("wrong error:", s.Err())
        }
}

// Test for issue 5268.
type alwaysError struct{}

func (alwaysError) Read(p []byte) (int, error) {
        return 0, io.ErrUnexpectedEOF
}

func TestNonEOFWithEmptyRead(t *testing.T) {
        scanner := NewScanner(alwaysError{})
        for scanner.Scan() {
                t.Fatal("read should fail")
        }
        err := scanner.Err()
        if err != io.ErrUnexpectedEOF {
                t.Errorf("unexpected error: %v", err)
        }
}

// Test that Scan finishes if we have endless empty reads.
type endlessZeros struct{}

func (endlessZeros) Read(p []byte) (int, error) {
        return 0, nil
}

func TestBadReader(t *testing.T) {
        scanner := NewScanner(endlessZeros{})
        for scanner.Scan() {
                t.Fatal("read should fail")
        }
        err := scanner.Err()
        if err != io.ErrNoProgress {
                t.Errorf("unexpected error: %v", err)
        }
}

func TestScanWordsExcessiveWhiteSpace(t *testing.T) {
        const word = "ipsum"
        s := strings.Repeat(" ", 4*smallMaxTokenSize) + word
        scanner := NewScanner(strings.NewReader(s))
        scanner.MaxTokenSize(smallMaxTokenSize)
        scanner.Split(ScanWords)
        if !scanner.Scan() {
                t.Fatalf("scan failed: %v", scanner.Err())
        }
        if token := scanner.Text(); token != word {
                t.Fatalf("unexpected token: %v", token)
        }
}

// Test that empty tokens, including at end of line or end of file, are found by the scanner.
// Issue 8672: Could miss final empty token.

func commaSplit(data []byte, atEOF bool) (advance int, token []byte, err error) {
        for i := 0; i < len(data); i++ {
                if data[i] == ',' {
                        return i + 1, data[:i], nil
                }
        }
        return 0, data, ErrFinalToken
}

func testEmptyTokens(t *testing.T, text string, values []string) {
        s := NewScanner(strings.NewReader(text))
        s.Split(commaSplit)
        var i int
        for i = 0; s.Scan(); i++ {
                if i >= len(values) {
                        t.Fatalf("got %d fields, expected %d", i+1, len(values))
                }
                if s.Text() != values[i] {
                        t.Errorf("%d: expected %q got %q", i, values[i], s.Text())
                }
        }
        if i != len(values) {
                t.Fatalf("got %d fields, expected %d", i, len(values))
        }
        if err := s.Err(); err != nil {
                t.Fatal(err)
        }
}

func TestEmptyTokens(t *testing.T) {
        testEmptyTokens(t, "1,2,3,", []string{"1", "2", "3", ""})
}

func TestWithNoEmptyTokens(t *testing.T) {
        testEmptyTokens(t, "1,2,3", []string{"1", "2", "3"})
}

func loopAtEOFSplit(data []byte, atEOF bool) (advance int, token []byte, err error) {
        if len(data) > 0 {
                return 1, data[:1], nil
        }
        return 0, data, nil
}

func TestDontLoopForever(t *testing.T) {
        s := NewScanner(strings.NewReader("abc"))
        s.Split(loopAtEOFSplit)
        // Expect a panic
        defer func() {
                err := recover()
                if err == nil {
                        t.Fatal("should have panicked")
                }
                if msg, ok := err.(string); !ok || !strings.Contains(msg, "empty tokens") {
                        panic(err)
                }
        }()
        for count := 0; s.Scan(); count++ {
                if count > 1000 {
                        t.Fatal("looping")
                }
        }
        if s.Err() != nil {
                t.Fatal("after scan:", s.Err())
        }
}

func TestBlankLines(t *testing.T) {
        s := NewScanner(strings.NewReader(strings.Repeat("\n", 1000)))
        for count := 0; s.Scan(); count++ {
                if count > 2000 {
                        t.Fatal("looping")
                }
        }
        if s.Err() != nil {
                t.Fatal("after scan:", s.Err())
        }
}

type countdown int

func (c *countdown) split(data []byte, atEOF bool) (advance int, token []byte, err error) {
        if *c > 0 {
                *c--
                return 1, data[:1], nil
        }
        return 0, nil, nil
}

// Check that the looping-at-EOF check doesn't trigger for merely empty tokens.
func TestEmptyLinesOK(t *testing.T) {
        c := countdown(10000)
        s := NewScanner(strings.NewReader(strings.Repeat("\n", 10000)))
        s.Split(c.split)
        for s.Scan() {
        }
        if s.Err() != nil {
                t.Fatal("after scan:", s.Err())
        }
        if c != 0 {
                t.Fatalf("stopped with %d left to process", c)
        }
}

// Make sure we can read a huge token if a big enough buffer is provided.
func TestHugeBuffer(t *testing.T) {
        text := strings.Repeat("x", 2*MaxScanTokenSize)
        s := NewScanner(strings.NewReader(text + "\n"))
        s.Buffer(make([]byte, 100), 3*MaxScanTokenSize)
        for s.Scan() {
                token := s.Text()
                if token != text {
                        t.Errorf("scan got incorrect token of length %d", len(token))
                }
        }
        if s.Err() != nil {
                t.Fatal("after scan:", s.Err())
        }
}