1 // Copyright 2013 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
14 // Scanner provides a convenient interface for reading data such as
15 // a file of newline-delimited lines of text. Successive calls to
16 // the Scan method will step through the 'tokens' of a file, skipping
17 // the bytes between the tokens. The specification of a token is
18 // defined by a split function of type SplitFunc; the default split
19 // function breaks the input into lines with line termination stripped. Split
20 // functions are defined in this package for scanning a file into
21 // lines, bytes, UTF-8-encoded runes, and space-delimited words. The
22 // client may instead provide a custom split function.
24 // Scanning stops unrecoverably at EOF, the first I/O error, or a token too
25 // large to fit in the buffer. When a scan stops, the reader may have
26 // advanced arbitrarily far past the last token. Programs that need more
27 // control over error handling or large tokens, or must run sequential scans
28 // on a reader, should use bufio.Reader instead.
31 r io
.Reader
// The reader provided by the client.
32 split SplitFunc
// The function to split the tokens.
33 maxTokenSize
int // Maximum size of a token; modified by tests.
34 token
[]byte // Last token returned by split.
35 buf
[]byte // Buffer used as argument to split.
36 start
int // First non-processed byte in buf.
37 end
int // End of data in buf.
38 err error
// Sticky error.
41 // SplitFunc is the signature of the split function used to tokenize the
42 // input. The arguments are an initial substring of the remaining unprocessed
43 // data and a flag, atEOF, that reports whether the Reader has no more data
44 // to give. The return values are the number of bytes to advance the input
45 // and the next token to return to the user, plus an error, if any. If the
46 // data does not yet hold a complete token, for instance if it has no newline
47 // while scanning lines, SplitFunc can return (0, nil, nil) to signal the
48 // Scanner to read more data into the slice and try again with a longer slice
49 // starting at the same point in the input.
51 // If the returned error is non-nil, scanning stops and the error
52 // is returned to the client.
54 // The function is never called with an empty data slice unless atEOF
55 // is true. If atEOF is true, however, data may be non-empty and,
56 // as always, holds unprocessed text.
57 type SplitFunc
func(data
[]byte, atEOF
bool) (advance
int, token
[]byte, err error
)
59 // Errors returned by Scanner.
61 ErrTooLong
= errors
.New("bufio.Scanner: token too long")
62 ErrNegativeAdvance
= errors
.New("bufio.Scanner: SplitFunc returns negative advance count")
63 ErrAdvanceTooFar
= errors
.New("bufio.Scanner: SplitFunc returns advance count beyond input")
67 // Maximum size used to buffer a token. The actual maximum token size
68 // may be smaller as the buffer may need to include, for instance, a newline.
69 MaxScanTokenSize
= 64 * 1024
72 // NewScanner returns a new Scanner to read from r.
73 // The split function defaults to ScanLines.
74 func NewScanner(r io
.Reader
) *Scanner
{
78 maxTokenSize
: MaxScanTokenSize
,
79 buf
: make([]byte, 4096), // Plausible starting size; needn't be large.
83 // Err returns the first non-EOF error that was encountered by the Scanner.
84 func (s
*Scanner
) Err() error
{
91 // Bytes returns the most recent token generated by a call to Scan.
92 // The underlying array may point to data that will be overwritten
93 // by a subsequent call to Scan. It does no allocation.
94 func (s
*Scanner
) Bytes() []byte {
98 // Text returns the most recent token generated by a call to Scan
99 // as a newly allocated string holding its bytes.
100 func (s
*Scanner
) Text() string {
101 return string(s
.token
)
104 // Scan advances the Scanner to the next token, which will then be
105 // available through the Bytes or Text method. It returns false when the
106 // scan stops, either by reaching the end of the input or an error.
107 // After Scan returns false, the Err method will return any error that
108 // occurred during scanning, except that if it was io.EOF, Err
110 func (s
*Scanner
) Scan() bool {
111 // Loop until we have a token.
113 // See if we can get a token with what we already have.
115 advance
, token
, err
:= s
.split(s
.buf
[s
.start
:s
.end
], s
.err
!= nil)
120 if !s
.advance(advance
) {
128 // We cannot generate a token with what we are holding.
129 // If we've already hit EOF or an I/O error, we are done.
136 // Must read more data.
137 // First, shift data to beginning of buffer if there's lots of empty space
138 // or space is needed.
139 if s
.start
> 0 && (s
.end
== len(s
.buf
) || s
.start
> len(s
.buf
)/2) {
140 copy(s
.buf
, s
.buf
[s
.start
:s
.end
])
144 // Is the buffer full? If so, resize.
145 if s
.end
== len(s
.buf
) {
146 if len(s
.buf
) >= s
.maxTokenSize
{
150 newSize
:= len(s
.buf
) * 2
151 if newSize
> s
.maxTokenSize
{
152 newSize
= s
.maxTokenSize
154 newBuf
:= make([]byte, newSize
)
155 copy(newBuf
, s
.buf
[s
.start
:s
.end
])
161 // Finally we can read some input. Make sure we don't get stuck with
162 // a misbehaving Reader. Officially we don't need to do this, but let's
163 // be extra careful: Scanner is for safe, simple jobs.
165 n
, err
:= s
.r
.Read(s
.buf
[s
.end
:len(s
.buf
)])
175 if loop
> maxConsecutiveEmptyReads
{
176 s
.setErr(io
.ErrNoProgress
)
183 // advance consumes n bytes of the buffer. It reports whether the advance was legal.
184 func (s
*Scanner
) advance(n
int) bool {
186 s
.setErr(ErrNegativeAdvance
)
189 if n
> s
.end
-s
.start
{
190 s
.setErr(ErrAdvanceTooFar
)
197 // setErr records the first error encountered.
198 func (s
*Scanner
) setErr(err error
) {
199 if s
.err
== nil || s
.err
== io
.EOF
{
204 // Split sets the split function for the Scanner. If called, it must be
205 // called before Scan. The default split function is ScanLines.
206 func (s
*Scanner
) Split(split SplitFunc
) {
212 // ScanBytes is a split function for a Scanner that returns each byte as a token.
213 func ScanBytes(data
[]byte, atEOF
bool) (advance
int, token
[]byte, err error
) {
214 if atEOF
&& len(data
) == 0 {
217 return 1, data
[0:1], nil
220 var errorRune
= []byte(string(utf8
.RuneError
))
222 // ScanRunes is a split function for a Scanner that returns each
223 // UTF-8-encoded rune as a token. The sequence of runes returned is
224 // equivalent to that from a range loop over the input as a string, which
225 // means that erroneous UTF-8 encodings translate to U+FFFD = "\xef\xbf\xbd".
226 // Because of the Scan interface, this makes it impossible for the client to
227 // distinguish correctly encoded replacement runes from encoding errors.
228 func ScanRunes(data
[]byte, atEOF
bool) (advance
int, token
[]byte, err error
) {
229 if atEOF
&& len(data
) == 0 {
233 // Fast path 1: ASCII.
234 if data
[0] < utf8
.RuneSelf
{
235 return 1, data
[0:1], nil
238 // Fast path 2: Correct UTF-8 decode without error.
239 _
, width
:= utf8
.DecodeRune(data
)
241 // It's a valid encoding. Width cannot be one for a correctly encoded
243 return width
, data
[0:width
], nil
246 // We know it's an error: we have width==1 and implicitly r==utf8.RuneError.
247 // Is the error because there wasn't a full rune to be decoded?
248 // FullRune distinguishes correctly between erroneous and incomplete encodings.
249 if !atEOF
&& !utf8
.FullRune(data
) {
250 // Incomplete; get more bytes.
254 // We have a real UTF-8 encoding error. Return a properly encoded error rune
255 // but advance only one byte. This matches the behavior of a range loop over
256 // an incorrectly encoded string.
257 return 1, errorRune
, nil
260 // dropCR drops a terminal \r from the data.
261 func dropCR(data
[]byte) []byte {
262 if len(data
) > 0 && data
[len(data
)-1] == '\r' {
263 return data
[0 : len(data
)-1]
268 // ScanLines is a split function for a Scanner that returns each line of
269 // text, stripped of any trailing end-of-line marker. The returned line may
270 // be empty. The end-of-line marker is one optional carriage return followed
271 // by one mandatory newline. In regular expression notation, it is `\r?\n`.
272 // The last non-empty line of input will be returned even if it has no
274 func ScanLines(data
[]byte, atEOF
bool) (advance
int, token
[]byte, err error
) {
275 if atEOF
&& len(data
) == 0 {
278 if i
:= bytes
.IndexByte(data
, '\n'); i
>= 0 {
279 // We have a full newline-terminated line.
280 return i
+ 1, dropCR(data
[0:i
]), nil
282 // If we're at EOF, we have a final, non-terminated line. Return it.
284 return len(data
), dropCR(data
), nil
286 // Request more data.
290 // isSpace reports whether the character is a Unicode white space character.
291 // We avoid dependency on the unicode package, but check validity of the implementation
293 func isSpace(r rune
) bool {
295 // Obvious ASCII ones: \t through \r plus space. Plus two Latin-1 oddballs.
297 case ' ', '\t', '\n', '\v', '\f', '\r':
299 case '\u0085', '\u00A0':
305 if '\u2000' <= r
&& r
<= '\u200a' {
309 case '\u1680', '\u2028', '\u2029', '\u202f', '\u205f', '\u3000':
315 // ScanWords is a split function for a Scanner that returns each
316 // space-separated word of text, with surrounding spaces deleted. It will
317 // never return an empty string. The definition of space is set by
319 func ScanWords(data
[]byte, atEOF
bool) (advance
int, token
[]byte, err error
) {
320 // Skip leading spaces.
322 for width
:= 0; start
< len(data
); start
+= width
{
324 r
, width
= utf8
.DecodeRune(data
[start
:])
329 if atEOF
&& len(data
) == 0 {
332 // Scan until space, marking end of word.
333 for width
, i
:= 0, start
; i
< len(data
); i
+= width
{
335 r
, width
= utf8
.DecodeRune(data
[i
:])
337 return i
+ width
, data
[start
:i
], nil
340 // If we're at EOF, we have a final, non-empty, non-terminated word. Return it.
341 if atEOF
&& len(data
) > start
{
342 return len(data
), data
[start
:], nil
344 // Request more data.