libgo/go/strconv/quote.go

   1 // Copyright 2009 The Go Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style
   3 // license that can be found in the LICENSE file.
   4
   5 //go:generate go run makeisprint.go -output isprint.go
   6
   7 package strconv
   8
   9 import (
  10         "unicode/utf8"
  11 )
  12
  13 const (
  14         lowerhex = "0123456789abcdef"
  15         upperhex = "0123456789ABCDEF"
  16 )
  17
  18 // contains reports whether the string contains the byte c.
  19 func contains(s string, c byte) bool {
  20         return index(s, c) != -1
  21 }
  22
  23 func quoteWith(s string, quote byte, ASCIIonly, graphicOnly bool) string {
  24         return string(appendQuotedWith(make([]byte, 0, 3*len(s)/2), s, quote, ASCIIonly, graphicOnly))
  25 }
  26
  27 func quoteRuneWith(r rune, quote byte, ASCIIonly, graphicOnly bool) string {
  28         return string(appendQuotedRuneWith(nil, r, quote, ASCIIonly, graphicOnly))
  29 }
  30
  31 func appendQuotedWith(buf []byte, s string, quote byte, ASCIIonly, graphicOnly bool) []byte {
  32         // Often called with big strings, so preallocate. If there's quoting,
  33         // this is conservative but still helps a lot.
  34         if cap(buf)-len(buf) < len(s) {
  35                 nBuf := make([]byte, len(buf), len(buf)+1+len(s)+1)
  36                 copy(nBuf, buf)
  37                 buf = nBuf
  38         }
  39         buf = append(buf, quote)
  40         for width := 0; len(s) > 0; s = s[width:] {
  41                 r := rune(s[0])
  42                 width = 1
  43                 if r >= utf8.RuneSelf {
  44                         r, width = utf8.DecodeRuneInString(s)
  45                 }
  46                 if width == 1 && r == utf8.RuneError {
  47                         buf = append(buf, `\x`...)
  48                         buf = append(buf, lowerhex[s[0]>>4])
  49                         buf = append(buf, lowerhex[s[0]&0xF])
  50                         continue
  51                 }
  52                 buf = appendEscapedRune(buf, r, quote, ASCIIonly, graphicOnly)
  53         }
  54         buf = append(buf, quote)
  55         return buf
  56 }
  57
  58 func appendQuotedRuneWith(buf []byte, r rune, quote byte, ASCIIonly, graphicOnly bool) []byte {
  59         buf = append(buf, quote)
  60         if !utf8.ValidRune(r) {
  61                 r = utf8.RuneError
  62         }
  63         buf = appendEscapedRune(buf, r, quote, ASCIIonly, graphicOnly)
  64         buf = append(buf, quote)
  65         return buf
  66 }
  67
  68 func appendEscapedRune(buf []byte, r rune, quote byte, ASCIIonly, graphicOnly bool) []byte {
  69         var runeTmp [utf8.UTFMax]byte
  70         if r == rune(quote) || r == '\\' { // always backslashed
  71                 buf = append(buf, '\\')
  72                 buf = append(buf, byte(r))
  73                 return buf
  74         }
  75         if ASCIIonly {
  76                 if r < utf8.RuneSelf && IsPrint(r) {
  77                         buf = append(buf, byte(r))
  78                         return buf
  79                 }
  80         } else if IsPrint(r) || graphicOnly && isInGraphicList(r) {
  81                 n := utf8.EncodeRune(runeTmp[:], r)
  82                 buf = append(buf, runeTmp[:n]...)
  83                 return buf
  84         }
  85         switch r {
  86         case '\a':
  87                 buf = append(buf, `\a`...)
  88         case '\b':
  89                 buf = append(buf, `\b`...)
  90         case '\f':
  91                 buf = append(buf, `\f`...)
  92         case '\n':
  93                 buf = append(buf, `\n`...)
  94         case '\r':
  95                 buf = append(buf, `\r`...)
  96         case '\t':
  97                 buf = append(buf, `\t`...)
  98         case '\v':
  99                 buf = append(buf, `\v`...)
 100         default:
 101                 switch {
 102                 case r < ' ':
 103                         buf = append(buf, `\x`...)
 104                         buf = append(buf, lowerhex[byte(r)>>4])
 105                         buf = append(buf, lowerhex[byte(r)&0xF])
 106                 case !utf8.ValidRune(r):
 107                         r = 0xFFFD
 108                         fallthrough
 109                 case r < 0x10000:
 110                         buf = append(buf, `\u`...)
 111                         for s := 12; s >= 0; s -= 4 {
 112                                 buf = append(buf, lowerhex[r>>uint(s)&0xF])
 113                         }
 114                 default:
 115                         buf = append(buf, `\U`...)
 116                         for s := 28; s >= 0; s -= 4 {
 117                                 buf = append(buf, lowerhex[r>>uint(s)&0xF])
 118                         }
 119                 }
 120         }
 121         return buf
 122 }
 123
 124 // Quote returns a double-quoted Go string literal representing s. The
 125 // returned string uses Go escape sequences (\t, \n, \xFF, \u0100) for
 126 // control characters and non-printable characters as defined by
 127 // IsPrint.
 128 func Quote(s string) string {
 129         return quoteWith(s, '"', false, false)
 130 }
 131
 132 // AppendQuote appends a double-quoted Go string literal representing s,
 133 // as generated by Quote, to dst and returns the extended buffer.
 134 func AppendQuote(dst []byte, s string) []byte {
 135         return appendQuotedWith(dst, s, '"', false, false)
 136 }
 137
 138 // QuoteToASCII returns a double-quoted Go string literal representing s.
 139 // The returned string uses Go escape sequences (\t, \n, \xFF, \u0100) for
 140 // non-ASCII characters and non-printable characters as defined by IsPrint.
 141 func QuoteToASCII(s string) string {
 142         return quoteWith(s, '"', true, false)
 143 }
 144
 145 // AppendQuoteToASCII appends a double-quoted Go string literal representing s,
 146 // as generated by QuoteToASCII, to dst and returns the extended buffer.
 147 func AppendQuoteToASCII(dst []byte, s string) []byte {
 148         return appendQuotedWith(dst, s, '"', true, false)
 149 }
 150
 151 // QuoteToGraphic returns a double-quoted Go string literal representing s.
 152 // The returned string leaves Unicode graphic characters, as defined by
 153 // IsGraphic, unchanged and uses Go escape sequences (\t, \n, \xFF, \u0100)
 154 // for non-graphic characters.
 155 func QuoteToGraphic(s string) string {
 156         return quoteWith(s, '"', false, true)
 157 }
 158
 159 // AppendQuoteToGraphic appends a double-quoted Go string literal representing s,
 160 // as generated by QuoteToGraphic, to dst and returns the extended buffer.
 161 func AppendQuoteToGraphic(dst []byte, s string) []byte {
 162         return appendQuotedWith(dst, s, '"', false, true)
 163 }
 164
 165 // QuoteRune returns a single-quoted Go character literal representing the
 166 // rune. The returned string uses Go escape sequences (\t, \n, \xFF, \u0100)
 167 // for control characters and non-printable characters as defined by IsPrint.
 168 func QuoteRune(r rune) string {
 169         return quoteRuneWith(r, '\'', false, false)
 170 }
 171
 172 // AppendQuoteRune appends a single-quoted Go character literal representing the rune,
 173 // as generated by QuoteRune, to dst and returns the extended buffer.
 174 func AppendQuoteRune(dst []byte, r rune) []byte {
 175         return appendQuotedRuneWith(dst, r, '\'', false, false)
 176 }
 177
 178 // QuoteRuneToASCII returns a single-quoted Go character literal representing
 179 // the rune. The returned string uses Go escape sequences (\t, \n, \xFF,
 180 // \u0100) for non-ASCII characters and non-printable characters as defined
 181 // by IsPrint.
 182 func QuoteRuneToASCII(r rune) string {
 183         return quoteRuneWith(r, '\'', true, false)
 184 }
 185
 186 // AppendQuoteRuneToASCII appends a single-quoted Go character literal representing the rune,
 187 // as generated by QuoteRuneToASCII, to dst and returns the extended buffer.
 188 func AppendQuoteRuneToASCII(dst []byte, r rune) []byte {
 189         return appendQuotedRuneWith(dst, r, '\'', true, false)
 190 }
 191
 192 // QuoteRuneToGraphic returns a single-quoted Go character literal representing
 193 // the rune. If the rune is not a Unicode graphic character,
 194 // as defined by IsGraphic, the returned string will use a Go escape sequence
 195 // (\t, \n, \xFF, \u0100).
 196 func QuoteRuneToGraphic(r rune) string {
 197         return quoteRuneWith(r, '\'', false, true)
 198 }
 199
 200 // AppendQuoteRuneToGraphic appends a single-quoted Go character literal representing the rune,
 201 // as generated by QuoteRuneToGraphic, to dst and returns the extended buffer.
 202 func AppendQuoteRuneToGraphic(dst []byte, r rune) []byte {
 203         return appendQuotedRuneWith(dst, r, '\'', false, true)
 204 }
 205
 206 // CanBackquote reports whether the string s can be represented
 207 // unchanged as a single-line backquoted string without control
 208 // characters other than tab.
 209 func CanBackquote(s string) bool {
 210         for len(s) > 0 {
 211                 r, wid := utf8.DecodeRuneInString(s)
 212                 s = s[wid:]
 213                 if wid > 1 {
 214                         if r == '\ufeff' {
 215                                 return false // BOMs are invisible and should not be quoted.
 216                         }
 217                         continue // All other multibyte runes are correctly encoded and assumed printable.
 218                 }
 219                 if r == utf8.RuneError {
 220                         return false
 221                 }
 222                 if (r < ' ' && r != '\t') || r == '`' || r == '\u007F' {
 223                         return false
 224                 }
 225         }
 226         return true
 227 }
 228
 229 func unhex(b byte) (v rune, ok bool) {
 230         c := rune(b)
 231         switch {
 232         case '0' <= c && c <= '9':
 233                 return c - '0', true
 234         case 'a' <= c && c <= 'f':
 235                 return c - 'a' + 10, true
 236         case 'A' <= c && c <= 'F':
 237                 return c - 'A' + 10, true
 238         }
 239         return
 240 }
 241
 242 // UnquoteChar decodes the first character or byte in the escaped string
 243 // or character literal represented by the string s.
 244 // It returns four values:
 245 //
 246 //      1) value, the decoded Unicode code point or byte value;
 247 //      2) multibyte, a boolean indicating whether the decoded character requires a multibyte UTF-8 representation;
 248 //      3) tail, the remainder of the string after the character; and
 249 //      4) an error that will be nil if the character is syntactically valid.
 250 //
 251 // The second argument, quote, specifies the type of literal being parsed
 252 // and therefore which escaped quote character is permitted.
 253 // If set to a single quote, it permits the sequence \' and disallows unescaped '.
 254 // If set to a double quote, it permits \" and disallows unescaped ".
 255 // If set to zero, it does not permit either escape and allows both quote characters to appear unescaped.
 256 func UnquoteChar(s string, quote byte) (value rune, multibyte bool, tail string, err error) {
 257         // easy cases
 258         if len(s) == 0 {
 259                 err = ErrSyntax
 260                 return
 261         }
 262         switch c := s[0]; {
 263         case c == quote && (quote == '\'' || quote == '"'):
 264                 err = ErrSyntax
 265                 return
 266         case c >= utf8.RuneSelf:
 267                 r, size := utf8.DecodeRuneInString(s)
 268                 return r, true, s[size:], nil
 269         case c != '\\':
 270                 return rune(s[0]), false, s[1:], nil
 271         }
 272
 273         // hard case: c is backslash
 274         if len(s) <= 1 {
 275                 err = ErrSyntax
 276                 return
 277         }
 278         c := s[1]
 279         s = s[2:]
 280
 281         switch c {
 282         case 'a':
 283                 value = '\a'
 284         case 'b':
 285                 value = '\b'
 286         case 'f':
 287                 value = '\f'
 288         case 'n':
 289                 value = '\n'
 290         case 'r':
 291                 value = '\r'
 292         case 't':
 293                 value = '\t'
 294         case 'v':
 295                 value = '\v'
 296         case 'x', 'u', 'U':
 297                 n := 0
 298                 switch c {
 299                 case 'x':
 300                         n = 2
 301                 case 'u':
 302                         n = 4
 303                 case 'U':
 304                         n = 8
 305                 }
 306                 var v rune
 307                 if len(s) < n {
 308                         err = ErrSyntax
 309                         return
 310                 }
 311                 for j := 0; j < n; j++ {
 312                         x, ok := unhex(s[j])
 313                         if !ok {
 314                                 err = ErrSyntax
 315                                 return
 316                         }
 317                         v = v<<4 | x
 318                 }
 319                 s = s[n:]
 320                 if c == 'x' {
 321                         // single-byte string, possibly not UTF-8
 322                         value = v
 323                         break
 324                 }
 325                 if !utf8.ValidRune(v) {
 326                         err = ErrSyntax
 327                         return
 328                 }
 329                 value = v
 330                 multibyte = true
 331         case '0', '1', '2', '3', '4', '5', '6', '7':
 332                 v := rune(c) - '0'
 333                 if len(s) < 2 {
 334                         err = ErrSyntax
 335                         return
 336                 }
 337                 for j := 0; j < 2; j++ { // one digit already; two more
 338                         x := rune(s[j]) - '0'
 339                         if x < 0 || x > 7 {
 340                                 err = ErrSyntax
 341                                 return
 342                         }
 343                         v = (v << 3) | x
 344                 }
 345                 s = s[2:]
 346                 if v > 255 {
 347                         err = ErrSyntax
 348                         return
 349                 }
 350                 value = v
 351         case '\\':
 352                 value = '\\'
 353         case '\'', '"':
 354                 if c != quote {
 355                         err = ErrSyntax
 356                         return
 357                 }
 358                 value = rune(c)
 359         default:
 360                 err = ErrSyntax
 361                 return
 362         }
 363         tail = s
 364         return
 365 }
 366
 367 // QuotedPrefix returns the quoted string (as understood by Unquote) at the prefix of s.
 368 // If s does not start with a valid quoted string, QuotedPrefix returns an error.
 369 func QuotedPrefix(s string) (string, error) {
 370         out, _, err := unquote(s, false)
 371         return out, err
 372 }
 373
 374 // Unquote interprets s as a single-quoted, double-quoted,
 375 // or backquoted Go string literal, returning the string value
 376 // that s quotes.  (If s is single-quoted, it would be a Go
 377 // character literal; Unquote returns the corresponding
 378 // one-character string.)
 379 func Unquote(s string) (string, error) {
 380         out, rem, err := unquote(s, true)
 381         if len(rem) > 0 {
 382                 return "", ErrSyntax
 383         }
 384         return out, err
 385 }
 386
 387 // unquote parses a quoted string at the start of the input,
 388 // returning the parsed prefix, the remaining suffix, and any parse errors.
 389 // If unescape is true, the parsed prefix is unescaped,
 390 // otherwise the input prefix is provided verbatim.
 391 func unquote(in string, unescape bool) (out, rem string, err error) {
 392         // Determine the quote form and optimistically find the terminating quote.
 393         if len(in) < 2 {
 394                 return "", in, ErrSyntax
 395         }
 396         quote := in[0]
 397         end := index(in[1:], quote)
 398         if end < 0 {
 399                 return "", in, ErrSyntax
 400         }
 401         end += 2 // position after terminating quote; may be wrong if escape sequences are present
 402
 403         switch quote {
 404         case '`':
 405                 switch {
 406                 case !unescape:
 407                         out = in[:end] // include quotes
 408                 case !contains(in[:end], '\r'):
 409                         out = in[len("`") : end-len("`")] // exclude quotes
 410                 default:
 411                         // Carriage return characters ('\r') inside raw string literals
 412                         // are discarded from the raw string value.
 413                         buf := make([]byte, 0, end-len("`")-len("\r")-len("`"))
 414                         for i := len("`"); i < end-len("`"); i++ {
 415                                 if in[i] != '\r' {
 416                                         buf = append(buf, in[i])
 417                                 }
 418                         }
 419                         out = string(buf)
 420                 }
 421                 // NOTE: Prior implementations did not verify that raw strings consist
 422                 // of valid UTF-8 characters and we continue to not verify it as such.
 423                 // The Go specification does not explicitly require valid UTF-8,
 424                 // but only mention that it is implicitly valid for Go source code
 425                 // (which must be valid UTF-8).
 426                 return out, in[end:], nil
 427         case '"', '\'':
 428                 // Handle quoted strings without any escape sequences.
 429                 if !contains(in[:end], '\\') && !contains(in[:end], '\n') {
 430                         var valid bool
 431                         switch quote {
 432                         case '"':
 433                                 valid = utf8.ValidString(in[len(`"`) : end-len(`"`)])
 434                         case '\'':
 435                                 r, n := utf8.DecodeRuneInString(in[len("'") : end-len("'")])
 436                                 valid = len("'")+n+len("'") == end && (r != utf8.RuneError || n != 1)
 437                         }
 438                         if valid {
 439                                 out = in[:end]
 440                                 if unescape {
 441                                         out = out[1 : end-1] // exclude quotes
 442                                 }
 443                                 return out, in[end:], nil
 444                         }
 445                 }
 446
 447                 // Handle quoted strings with escape sequences.
 448                 var buf []byte
 449                 in0 := in
 450                 in = in[1:] // skip starting quote
 451                 if unescape {
 452                         buf = make([]byte, 0, 3*end/2) // try to avoid more allocations
 453                 }
 454                 for len(in) > 0 && in[0] != quote {
 455                         // Process the next character,
 456                         // rejecting any unescaped newline characters which are invalid.
 457                         r, multibyte, rem, err := UnquoteChar(in, quote)
 458                         if in[0] == '\n' || err != nil {
 459                                 return "", in0, ErrSyntax
 460                         }
 461                         in = rem
 462
 463                         // Append the character if unescaping the input.
 464                         if unescape {
 465                                 if r < utf8.RuneSelf || !multibyte {
 466                                         buf = append(buf, byte(r))
 467                                 } else {
 468                                         var arr [utf8.UTFMax]byte
 469                                         n := utf8.EncodeRune(arr[:], r)
 470                                         buf = append(buf, arr[:n]...)
 471                                 }
 472                         }
 473
 474                         // Single quoted strings must be a single character.
 475                         if quote == '\'' {
 476                                 break
 477                         }
 478                 }
 479
 480                 // Verify that the string ends with a terminating quote.
 481                 if !(len(in) > 0 && in[0] == quote) {
 482                         return "", in0, ErrSyntax
 483                 }
 484                 in = in[1:] // skip terminating quote
 485
 486                 if unescape {
 487                         return string(buf), in, nil
 488                 }
 489                 return in0[:len(in0)-len(in)], in, nil
 490         default:
 491                 return "", in, ErrSyntax
 492         }
 493 }
 494
 495 // bsearch16 returns the smallest i such that a[i] >= x.
 496 // If there is no such i, bsearch16 returns len(a).
 497 func bsearch16(a []uint16, x uint16) int {
 498         i, j := 0, len(a)
 499         for i < j {
 500                 h := i + (j-i)>>1
 501                 if a[h] < x {
 502                         i = h + 1
 503                 } else {
 504                         j = h
 505                 }
 506         }
 507         return i
 508 }
 509
 510 // bsearch32 returns the smallest i such that a[i] >= x.
 511 // If there is no such i, bsearch32 returns len(a).
 512 func bsearch32(a []uint32, x uint32) int {
 513         i, j := 0, len(a)
 514         for i < j {
 515                 h := i + (j-i)>>1
 516                 if a[h] < x {
 517                         i = h + 1
 518                 } else {
 519                         j = h
 520                 }
 521         }
 522         return i
 523 }
 524
 525 // TODO: IsPrint is a local implementation of unicode.IsPrint, verified by the tests
 526 // to give the same answer. It allows this package not to depend on unicode,
 527 // and therefore not pull in all the Unicode tables. If the linker were better
 528 // at tossing unused tables, we could get rid of this implementation.
 529 // That would be nice.
 530
 531 // IsPrint reports whether the rune is defined as printable by Go, with
 532 // the same definition as unicode.IsPrint: letters, numbers, punctuation,
 533 // symbols and ASCII space.
 534 func IsPrint(r rune) bool {
 535         // Fast check for Latin-1
 536         if r <= 0xFF {
 537                 if 0x20 <= r && r <= 0x7E {
 538                         // All the ASCII is printable from space through DEL-1.
 539                         return true
 540                 }
 541                 if 0xA1 <= r && r <= 0xFF {
 542                         // Similarly for ¡ through ÿ...
 543                         return r != 0xAD // ...except for the bizarre soft hyphen.
 544                 }
 545                 return false
 546         }
 547
 548         // Same algorithm, either on uint16 or uint32 value.
 549         // First, find first i such that isPrint[i] >= x.
 550         // This is the index of either the start or end of a pair that might span x.
 551         // The start is even (isPrint[i&^1]) and the end is odd (isPrint[i|1]).
 552         // If we find x in a range, make sure x is not in isNotPrint list.
 553
 554         if 0 <= r && r < 1<<16 {
 555                 rr, isPrint, isNotPrint := uint16(r), isPrint16, isNotPrint16
 556                 i := bsearch16(isPrint, rr)
 557                 if i >= len(isPrint) || rr < isPrint[i&^1] || isPrint[i|1] < rr {
 558                         return false
 559                 }
 560                 j := bsearch16(isNotPrint, rr)
 561                 return j >= len(isNotPrint) || isNotPrint[j] != rr
 562         }
 563
 564         rr, isPrint, isNotPrint := uint32(r), isPrint32, isNotPrint32
 565         i := bsearch32(isPrint, rr)
 566         if i >= len(isPrint) || rr < isPrint[i&^1] || isPrint[i|1] < rr {
 567                 return false
 568         }
 569         if r >= 0x20000 {
 570                 return true
 571         }
 572         r -= 0x10000
 573         j := bsearch16(isNotPrint, uint16(r))
 574         return j >= len(isNotPrint) || isNotPrint[j] != uint16(r)
 575 }
 576
 577 // IsGraphic reports whether the rune is defined as a Graphic by Unicode. Such
 578 // characters include letters, marks, numbers, punctuation, symbols, and
 579 // spaces, from categories L, M, N, P, S, and Zs.
 580 func IsGraphic(r rune) bool {
 581         if IsPrint(r) {
 582                 return true
 583         }
 584         return isInGraphicList(r)
 585 }
 586
 587 // isInGraphicList reports whether the rune is in the isGraphic list. This separation
 588 // from IsGraphic allows quoteWith to avoid two calls to IsPrint.
 589 // Should be called only if IsPrint fails.
 590 func isInGraphicList(r rune) bool {
 591         // We know r must fit in 16 bits - see makeisprint.go.
 592         if r > 0xFFFF {
 593                 return false
 594         }
 595         rr := uint16(r)
 596         i := bsearch16(isGraphic, rr)
 597         return i < len(isGraphic) && rr == isGraphic[i]
 598 }