token.go

     1  // Copyright 2020 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  //go:build goexperiment.jsonv2
     6  
     7  package jsontext
     8  
     9  import (
    10  	"bytes"
    11  	"errors"
    12  	"math"
    13  	"strconv"
    14  
    15  	"encoding/json/internal/jsonflags"
    16  	"encoding/json/internal/jsonwire"
    17  )
    18  
    19  // NOTE: Token is analogous to v1 json.Token.
    20  
    21  const (
    22  	maxInt64  = math.MaxInt64
    23  	minInt64  = math.MinInt64
    24  	maxUint64 = math.MaxUint64
    25  	minUint64 = 0 // for consistency and readability purposes
    26  
    27  	invalidTokenPanic = "invalid jsontext.Token; it has been voided by a subsequent json.Decoder call"
    28  )
    29  
    30  var errInvalidToken = errors.New("invalid jsontext.Token")
    31  
    32  // Token represents a lexical JSON token, which may be one of the following:
    33  //   - a JSON literal (i.e., null, true, or false)
    34  //   - a JSON string (e.g., "hello, world!")
    35  //   - a JSON number (e.g., 123.456)
    36  //   - a begin or end delimiter for a JSON object (i.e., { or } )
    37  //   - a begin or end delimiter for a JSON array (i.e., [ or ] )
    38  //
    39  // A Token cannot represent entire array or object values, while a [Value] can.
    40  // There is no Token to represent commas and colons since
    41  // these structural tokens can be inferred from the surrounding context.
    42  //
    43  // A Token stores data in one of two forms:
    44  //
    45  //   - As raw JSON text: backed by the internal buffer of the [Decoder]
    46  //     and only ever produced by [Decoder.ReadToken].
    47  //     Such a token is only valid until the next call to any method on that
    48  //     [Decoder] (e.g., [Decoder.PeekKind], [Decoder.ReadToken],
    49  //     [Decoder.ReadValue], or [Decoder.SkipValue]).
    50  //     Call [Token.Clone] to copy the raw text into an independent allocation
    51  //     that persists beyond subsequent [Decoder] calls.
    52  //
    53  //   - As a typed Go value: a self-contained representation produced by
    54  //     the constructor functions (e.g., [String], [Int], [Uint], [Float]).
    55  //     Such tokens are valid indefinitely and do not need to be cloned.
    56  type Token struct {
    57  	nonComparable
    58  
    59  	// Tokens can exist in either a "raw" or an "exact" form.
    60  	// Tokens produced by the Decoder are in the "raw" form.
    61  	// Tokens returned by constructors are usually in the "exact" form.
    62  	// The Encoder accepts Tokens in either the "raw" or "exact" form.
    63  	//
    64  	// The following chart shows the possible values for each Token type:
    65  	//	╔══════════════════╦════════════╤════════════╤════════════╗
    66  	//	║ Token type       ║ raw field  │ str field  │ num field  ║
    67  	//	╠══════════════════╬════════════╪════════════╪════════════╣
    68  	//	║ null   (raw)     ║ "null"     │ ""         │ 0          ║
    69  	//	║ false  (raw)     ║ "false"    │ ""         │ 0          ║
    70  	//	║ true   (raw)     ║ "true"     │ ""         │ 0          ║
    71  	//	║ string (raw)     ║ non-empty  │ ""         │ offset     ║
    72  	//	║ string (string)  ║ nil        │ non-empty  │ 0          ║
    73  	//	║ number (raw)     ║ non-empty  │ ""         │ offset     ║
    74  	//	║ number (float32) ║ nil        │ "F"        │ non-zero   ║
    75  	//	║ number (float64) ║ nil        │ "f"        │ non-zero   ║
    76  	//	║ number (int64)   ║ nil        │ "i"        │ non-zero   ║
    77  	//	║ number (uint64)  ║ nil        │ "u"        │ non-zero   ║
    78  	//	║ object (delim)   ║ "{" or "}" │ ""         │ 0          ║
    79  	//	║ array  (delim)   ║ "[" or "]" │ ""         │ 0          ║
    80  	//	╚══════════════════╩════════════╧════════════╧════════════╝
    81  	//
    82  	// Notes:
    83  	//   - For tokens stored in "raw" form, the num field contains the
    84  	//     absolute offset determined by raw.previousOffsetStart().
    85  	//     The buffer itself is stored in raw.previousBuffer().
    86  	//   - JSON literals and structural characters are always in the "raw" form.
    87  	//   - JSON strings and numbers can be in either "raw" or "exact" forms.
    88  	//   - The exact zero value of JSON strings and numbers in the "exact" forms
    89  	//     have ambiguous representation. Thus, they are always represented
    90  	//     in the "raw" form.
    91  
    92  	// raw contains a reference to the raw decode buffer.
    93  	// If non-nil, then its value takes precedence over str and num.
    94  	// It is only valid if num == raw.previousOffsetStart().
    95  	raw *decodeBuffer
    96  
    97  	// str is the unescaped JSON string if num is zero.
    98  	// Otherwise, it is "F", "f", "i", or "u" if num should be interpreted
    99  	// as a float32, float64, int64, or uint64, respectively.
   100  	str string
   101  
   102  	// num is a float32, float64, int64, or uint64 stored as a uint64 value.
   103  	// For floating-point values, it stores the raw IEEE-754 bit-pattern.
   104  	// It is non-zero for any JSON number in the "exact" form.
   105  	num uint64
   106  }
   107  
   108  // TODO: Does representing 1-byte delimiters as *decodeBuffer cause performance issues?
   109  
   110  var (
   111  	Null  Token = rawToken("null")
   112  	False Token = rawToken("false")
   113  	True  Token = rawToken("true")
   114  
   115  	BeginObject Token = rawToken("{")
   116  	EndObject   Token = rawToken("}")
   117  	BeginArray  Token = rawToken("[")
   118  	EndArray    Token = rawToken("]")
   119  
   120  	zeroString Token = rawToken(`""`)
   121  	zeroNumber Token = rawToken(`0`)
   122  
   123  	nanString  Token = String("NaN")
   124  	pinfString Token = String("Infinity")
   125  	ninfString Token = String("-Infinity")
   126  )
   127  
   128  func rawToken(s string) Token {
   129  	return Token{raw: &decodeBuffer{buf: []byte(s), prevStart: 0, prevEnd: len(s)}}
   130  }
   131  
   132  // Bool constructs a Token representing a JSON boolean.
   133  func Bool(b bool) Token {
   134  	if b {
   135  		return True
   136  	}
   137  	return False
   138  }
   139  
   140  // String constructs a Token representing a JSON string.
   141  // The provided string should contain valid UTF-8, otherwise invalid characters
   142  // may be mangled as the Unicode replacement character.
   143  func String(s string) Token {
   144  	if len(s) == 0 {
   145  		return zeroString
   146  	}
   147  	return Token{str: s}
   148  }
   149  
   150  // Float32 constructs a Token representing a JSON number as
   151  // a 32-bit floating-point number formatted according to
   152  // ECMA-262, 10th edition, section 7.1.12.1,
   153  // with the exception that -0 is still formatted as -0.
   154  // The values NaN, +Inf, and -Inf will be represented
   155  // as a JSON string with the values "NaN", "Infinity", and "-Infinity".
   156  //
   157  // Note that most JSON libraries and standards assume that JSON numbers
   158  // are 64-bit floating-point numbers. Use of 32-bit precision should
   159  // only be used if the corresponding decoder knows that
   160  // this JSON number token is expected to only have 32-bit precision.
   161  // For all other situations, prefer using the [Float] constructor instead.
   162  func Float32(n float32) Token {
   163  	if n != 0 && !math.IsNaN(float64(n)) && !math.IsInf(float64(n), 0) {
   164  		return Token{str: "F", num: uint64(math.Float32bits(n))}
   165  	}
   166  	return Float(float64(n)) // handles ±0, NaN, and ±Inf
   167  }
   168  
   169  // Float constructs a Token representing a JSON number as
   170  // a 64-bit floating-point number formatted according to
   171  // ECMA-262, 10th edition, section 7.1.12.1 and RFC 8785, section 3.2.2.3.
   172  // with the exception that -0 is still formatted as -0.
   173  // The values NaN, +Inf, and -Inf will be represented
   174  // as a JSON string with the values "NaN", "Infinity", and "-Infinity".
   175  func Float(n float64) Token {
   176  	switch {
   177  	case math.Float64bits(n) == 0:
   178  		return zeroNumber
   179  	case math.IsNaN(n):
   180  		return nanString
   181  	case math.IsInf(n, +1):
   182  		return pinfString
   183  	case math.IsInf(n, -1):
   184  		return ninfString
   185  	}
   186  	return Token{str: "f", num: math.Float64bits(n)}
   187  }
   188  
   189  // Int constructs a Token representing a JSON number from an int64.
   190  func Int(n int64) Token {
   191  	if n == 0 {
   192  		return zeroNumber
   193  	}
   194  	return Token{str: "i", num: uint64(n)}
   195  }
   196  
   197  // Uint constructs a Token representing a JSON number from a uint64.
   198  func Uint(n uint64) Token {
   199  	if n == 0 {
   200  		return zeroNumber
   201  	}
   202  	return Token{str: "u", num: uint64(n)}
   203  }
   204  
   205  // Clone returns a copy of the token with a value that is not backed by the
   206  // [Decoder] buffer and therefore remains valid past subsequent [Decoder] calls.
   207  // It has no effect on tokens produced by constructor functions,
   208  // since those are already self-contained.
   209  func (t Token) Clone() Token {
   210  	// TODO: Allow caller to avoid any allocations?
   211  	if raw := t.raw; raw != nil {
   212  		// Avoid copying globals.
   213  		if t.raw.prevStart == 0 {
   214  			switch t.raw {
   215  			case Null.raw:
   216  				return Null
   217  			case False.raw:
   218  				return False
   219  			case True.raw:
   220  				return True
   221  			case BeginObject.raw:
   222  				return BeginObject
   223  			case EndObject.raw:
   224  				return EndObject
   225  			case BeginArray.raw:
   226  				return BeginArray
   227  			case EndArray.raw:
   228  				return EndArray
   229  			}
   230  		}
   231  
   232  		// This only ever needs to clone strings and numbers.
   233  		if uint64(raw.previousOffsetStart()) != t.num {
   234  			panic(invalidTokenPanic)
   235  		}
   236  		buf := bytes.Clone(raw.previousBuffer())
   237  		return Token{raw: &decodeBuffer{buf: buf, prevStart: 0, prevEnd: len(buf)}}
   238  	}
   239  	return t
   240  }
   241  
   242  // Bool returns the value for a JSON boolean.
   243  // It panics if the token kind is not a JSON boolean.
   244  func (t Token) Bool() bool {
   245  	switch t.raw {
   246  	case True.raw:
   247  		return true
   248  	case False.raw:
   249  		return false
   250  	default:
   251  		panic("invalid JSON token kind: " + t.Kind().String())
   252  	}
   253  }
   254  
   255  // appendString appends a JSON string to dst and returns it.
   256  // It panics if t is not a JSON string.
   257  func (t Token) appendString(dst []byte, flags *jsonflags.Flags) ([]byte, error) {
   258  	if raw := t.raw; raw != nil {
   259  		// Handle raw string value.
   260  		buf := raw.previousBuffer()
   261  		if Kind(buf[0]) == '"' {
   262  			if jsonwire.ConsumeSimpleString(buf) == len(buf) {
   263  				return append(dst, buf...), nil
   264  			}
   265  			dst, _, err := jsonwire.ReformatString(dst, buf, flags)
   266  			return dst, err
   267  		}
   268  	} else if len(t.str) != 0 && t.num == 0 {
   269  		// Handle exact string value.
   270  		return jsonwire.AppendQuote(dst, []byte(t.str), flags)
   271  	}
   272  
   273  	panic("invalid JSON token kind: " + t.Kind().String())
   274  }
   275  
   276  // String returns the unescaped string value for a JSON string.
   277  // For other JSON kinds, this returns the raw JSON representation.
   278  func (t Token) String() string {
   279  	// This is inlinable to take advantage of "function outlining".
   280  	// This avoids an allocation for the string(b) conversion
   281  	// if the caller does not use the string in an escaping manner.
   282  	// See https://blog.filippo.io/efficient-go-apis-with-the-inliner/
   283  	s, b := t.string()
   284  	if len(b) > 0 {
   285  		return string(b)
   286  	}
   287  	return s
   288  }
   289  func (t Token) string() (string, []byte) {
   290  	if raw := t.raw; raw != nil {
   291  		if uint64(raw.previousOffsetStart()) != t.num {
   292  			panic(invalidTokenPanic)
   293  		}
   294  		buf := raw.previousBuffer()
   295  		if buf[0] == '"' {
   296  			// TODO: Preserve ValueFlags in Token?
   297  			isVerbatim := jsonwire.ConsumeSimpleString(buf) == len(buf)
   298  			return "", jsonwire.UnquoteMayCopy(buf, isVerbatim)
   299  		}
   300  		// Handle tokens that are not JSON strings for fmt.Stringer.
   301  		return "", buf
   302  	}
   303  	if len(t.str) != 0 && t.num == 0 {
   304  		return t.str, nil
   305  	}
   306  	// Handle tokens that are not JSON strings for fmt.Stringer.
   307  	if t.num > 0 {
   308  		switch t.str[0] {
   309  		case 'F':
   310  			return string(jsonwire.AppendFloat(nil, float64(math.Float32frombits(uint32(t.num))), 32)), nil
   311  		case 'f':
   312  			return string(jsonwire.AppendFloat(nil, float64(math.Float64frombits(uint64(t.num))), 64)), nil
   313  		case 'i':
   314  			return strconv.FormatInt(int64(t.num), 10), nil
   315  		case 'u':
   316  			return strconv.FormatUint(uint64(t.num), 10), nil
   317  		}
   318  	}
   319  	return "<invalid jsontext.Token>", nil
   320  }
   321  
   322  // appendNumber appends a JSON number to dst and returns it.
   323  // It panics if t is not a JSON number.
   324  func (t Token) appendNumber(dst []byte, flags *jsonflags.Flags) ([]byte, error) {
   325  	if raw := t.raw; raw != nil {
   326  		// Handle raw number value.
   327  		buf := raw.previousBuffer()
   328  		if Kind(buf[0]).normalize() == '0' {
   329  			dst, _, err := jsonwire.ReformatNumber(dst, buf, flags)
   330  			return dst, err
   331  		}
   332  	} else if t.num != 0 {
   333  		// Handle exact number value.
   334  		switch t.str[0] {
   335  		case 'F':
   336  			return jsonwire.AppendFloat(dst, float64(math.Float32frombits(uint32(t.num))), 32), nil
   337  		case 'f':
   338  			return jsonwire.AppendFloat(dst, float64(math.Float64frombits(uint64(t.num))), 64), nil
   339  		case 'i':
   340  			return strconv.AppendInt(dst, int64(t.num), 10), nil
   341  		case 'u':
   342  			return strconv.AppendUint(dst, uint64(t.num), 10), nil
   343  		}
   344  	}
   345  
   346  	panic("invalid JSON token kind: " + t.Kind().String())
   347  }
   348  
   349  // Float32 returns the floating-point value for a JSON number
   350  // parsed according to 32 bits of precision.
   351  //
   352  // If the JSON number is outside the representable range of a float32,
   353  // it returns +Inf or -Inf along with an error
   354  // that matches [strconv.ErrRange] according to [errors.Is].
   355  //
   356  // It returns a NaN, +Inf, or -Inf value for any JSON string
   357  // with the values "NaN", "Infinity", or "-Infinity".
   358  //
   359  // It panics if the token kind is not a JSON number
   360  // or a JSON string with the aforementioned values.
   361  //
   362  // Note that most JSON libraries and standards assume that JSON numbers
   363  // are 64-bit floating-point numbers.
   364  // This method should only be used if the caller knows
   365  // from other context that this token is a JSON number
   366  // formatted only to 32 bits of precision (such as being encoded
   367  // using the [Float32] constructor). For all other situations,
   368  // prefer using the [Token.Float] accessor instead.
   369  func (t Token) Float32() (float32, error) {
   370  	f, err := t.float(32)
   371  	return float32(f), err
   372  }
   373  
   374  // Float returns the floating-point value for a JSON number
   375  // parsed according to 64 bits of precision.
   376  //
   377  // If the JSON number is outside the representable range of a float64,
   378  // it returns +Inf or -Inf along with an error
   379  // that matches [strconv.ErrRange] according to [errors.Is].
   380  //
   381  // It returns a NaN, +Inf, or -Inf value for any JSON string
   382  // with the values "NaN", "Infinity", or "-Infinity".
   383  //
   384  // It panics if the token kind is not a JSON number
   385  // or a JSON string with the aforementioned values.
   386  func (t Token) Float() (float64, error) {
   387  	f, err := t.float(64)
   388  	return float64(f), err
   389  }
   390  
   391  func (t Token) float(bits int) (float64, error) {
   392  	if raw := t.raw; raw != nil {
   393  		// Handle raw JSON number value.
   394  		if uint64(raw.previousOffsetStart()) != t.num {
   395  			panic(invalidTokenPanic)
   396  		}
   397  		buf := raw.previousBuffer()
   398  		if Kind(buf[0]).normalize() == '0' {
   399  			fv, err := strconv.ParseFloat(string(buf), bits)
   400  			if err != nil {
   401  				err = &numError{accessor: "Float", value: t.String(), err: errors.Unwrap(err)} // only ever ErrRange
   402  			}
   403  			return fv, err
   404  		}
   405  	} else if t.num != 0 {
   406  		// Handle typed Go number value.
   407  		switch t.str[0] {
   408  		case 'F':
   409  			return float64(math.Float32frombits(uint32(t.num))), nil
   410  		case 'f':
   411  			f64 := float64(math.Float64frombits(uint64(t.num)))
   412  			if bits == 32 && !math.IsInf(f64, 0) && math.IsInf(float64(float32(f64)), 0) {
   413  				return f64, &numError{accessor: "Float", value: t.String(), err: strconv.ErrRange}
   414  			}
   415  			return f64, nil
   416  		case 'i':
   417  			return float64(int64(t.num)), nil // NOTE: This may lead to loss of precision.
   418  		case 'u':
   419  			return float64(uint64(t.num)), nil // NOTE: This may lead to loss of precision.
   420  		}
   421  	}
   422  
   423  	// Handle string values with "NaN", "Infinity", or "-Infinity".
   424  	if t.Kind() == '"' {
   425  		switch t.String() {
   426  		case "NaN":
   427  			return math.NaN(), nil
   428  		case "Infinity":
   429  			return math.Inf(+1), nil
   430  		case "-Infinity":
   431  			return math.Inf(-1), nil
   432  		}
   433  		// TODO: Should this be an error instead of a panic?
   434  		// We can safely switch from a panic to an error in the future.
   435  	}
   436  
   437  	panic("invalid JSON token kind: " + t.Kind().String())
   438  }
   439  
   440  // Int returns the signed integer value for a JSON number.
   441  //
   442  // It reports an error that matches [strconv.ErrSyntax] according to [errors.Is]
   443  // if the JSON number does not match the restricted grammar of just a signed integer.
   444  // It reports an error that matches [strconv.ErrRange] according to [errors.Is]
   445  // if the JSON number is a signed integer, but outside the range of an int64.
   446  // Even if an error is reported, a reasonable value is still returned.
   447  // The fractional component of any number is ignored (truncation toward zero).
   448  // Any number beyond the representation of an int64 will be saturated
   449  // to the closest representable value.
   450  //
   451  // It panics if the token kind is not a JSON number.
   452  func (t Token) Int() (int64, error) {
   453  	if raw := t.raw; raw != nil {
   454  		// Handle raw JSON number value.
   455  		if uint64(raw.previousOffsetStart()) != t.num {
   456  			panic(invalidTokenPanic)
   457  		}
   458  		buf := raw.previousBuffer()
   459  		if len(buf) > 0 && buf[0] == '-' {
   460  			// Prospectively parse a negative integer.
   461  			switch abs, ok := jsonwire.ParseUint(buf[len("-"):]); {
   462  			case abs > -minInt64:
   463  				return minInt64, &numError{accessor: "Int", value: t.String(), err: strconv.ErrRange}
   464  			case ok:
   465  				return -1 * int64(abs), nil
   466  			}
   467  		} else {
   468  			// Prospectively parse a non-negative integer.
   469  			switch abs, ok := jsonwire.ParseUint(buf); {
   470  			case abs > +maxInt64:
   471  				return maxInt64, &numError{accessor: "Int", value: t.String(), err: strconv.ErrRange}
   472  			case ok:
   473  				return +1 * int64(abs), nil
   474  			}
   475  		}
   476  		// This is not a signed integer, which implies ErrSyntax.
   477  		if Kind(buf[0]).normalize() == '0' {
   478  			f64, _ := strconv.ParseFloat(string(buf), 64)
   479  			return f64toi64(f64), &numError{accessor: "Int", value: t.String(), err: strconv.ErrSyntax}
   480  		}
   481  	} else if t.num != 0 {
   482  		// Handle typed Go number value.
   483  		switch t.str[0] {
   484  		case 'i':
   485  			return int64(t.num), nil
   486  		case 'u':
   487  			if t.num > maxInt64 {
   488  				return maxInt64, &numError{accessor: "Int", value: t.String(), err: strconv.ErrRange}
   489  			}
   490  			return int64(t.num), nil
   491  		case 'f', 'F':
   492  			f64 := float64(math.Float64frombits(uint64(t.num)))
   493  			if t.str[0] == 'F' {
   494  				f64 = float64(math.Float32frombits(uint32(t.num)))
   495  			}
   496  			switch i64 := f64toi64(f64); {
   497  			case math.IsNaN(f64), math.Trunc(f64) != f64:
   498  				return i64, &numError{accessor: "Int", value: t.String(), err: strconv.ErrSyntax}
   499  			case (i64 == minInt64 && f64 < minInt64) || (i64 == maxInt64 && f64 > maxInt64):
   500  				return i64, &numError{accessor: "Int", value: t.String(), err: strconv.ErrRange}
   501  			default:
   502  				return i64, nil
   503  			}
   504  		}
   505  	}
   506  
   507  	panic("invalid JSON token kind: " + t.Kind().String())
   508  }
   509  
   510  func f64toi64(f64 float64) int64 {
   511  	switch {
   512  	case math.IsNaN(f64):
   513  		return 0
   514  	case f64 >= maxInt64+1:
   515  		return maxInt64
   516  	case f64 < minInt64:
   517  		return minInt64
   518  	default:
   519  		return int64(f64) // NOTE: This may lead to loss of precision.
   520  	}
   521  }
   522  
   523  // Uint returns the unsigned integer value for a JSON number.
   524  //
   525  // It reports an error that matches [strconv.ErrSyntax] if the JSON number
   526  // does not match the restricted grammar of just an unsigned integer.
   527  // It reports an error that matches [strconv.ErrRange] if the JSON number
   528  // is an unsigned integer, but outside the representable range of a uint64.
   529  // Even if an error is reported, a reasonable value is still returned.
   530  // The fractional component of any number is ignored (truncation toward zero).
   531  // Any number beyond the representation of a uint64 will be saturated
   532  // to the closest representable value.
   533  //
   534  // It panics if the token kind is not a JSON number.
   535  func (t Token) Uint() (uint64, error) {
   536  	// NOTE: This accessor returns 0 for any negative JSON number,
   537  	// which might be surprising, but is at least consistent with the behavior
   538  	// of saturating out-of-bounds numbers to the closest representable number.
   539  	// We report ErrSyntax instead of ErrRange since the grammar for
   540  	// an unsigned integer does not permit a negative sign.
   541  
   542  	if raw := t.raw; raw != nil {
   543  		// Handle raw JSON number value.
   544  		if uint64(raw.previousOffsetStart()) != t.num {
   545  			panic(invalidTokenPanic)
   546  		}
   547  		buf := raw.previousBuffer()
   548  		// Prospectively parse an unsigned integer.
   549  		switch abs, ok := jsonwire.ParseUint(buf); {
   550  		case ok:
   551  			return abs, nil
   552  		case abs == maxUint64: // implies overflows
   553  			return maxUint64, &numError{accessor: "Uint", value: t.String(), err: strconv.ErrRange}
   554  		}
   555  		// This is not an unsigned integer, which implies ErrSyntax.
   556  		if Kind(buf[0]).normalize() == '0' {
   557  			f64, _ := strconv.ParseFloat(string(buf), 64)
   558  			return f64tou64(f64), &numError{accessor: "Uint", value: t.String(), err: strconv.ErrSyntax}
   559  		}
   560  	} else if t.num != 0 {
   561  		// Handle typed Go number value.
   562  		switch t.str[0] {
   563  		case 'u':
   564  			return t.num, nil
   565  		case 'i':
   566  			if int64(t.num) < minUint64 {
   567  				return minUint64, &numError{accessor: "Uint", value: t.String(), err: strconv.ErrSyntax}
   568  			}
   569  			return uint64(int64(t.num)), nil
   570  		case 'f', 'F':
   571  			f64 := float64(math.Float64frombits(uint64(t.num)))
   572  			if t.str[0] == 'F' {
   573  				f64 = float64(math.Float32frombits(uint32(t.num)))
   574  			}
   575  			switch u64 := f64tou64(f64); {
   576  			case math.IsNaN(f64), math.Trunc(f64) != f64, math.Signbit(f64):
   577  				return u64, &numError{accessor: "Uint", value: t.String(), err: strconv.ErrSyntax}
   578  			case (u64 == minUint64 && f64 < minUint64) || (u64 == maxUint64 && f64 > maxUint64):
   579  				return u64, &numError{accessor: "Uint", value: t.String(), err: strconv.ErrRange}
   580  			default:
   581  				return u64, nil
   582  			}
   583  		}
   584  	}
   585  
   586  	panic("invalid JSON token kind: " + t.Kind().String())
   587  }
   588  
   589  func f64tou64(f64 float64) uint64 {
   590  	switch {
   591  	case math.IsNaN(f64):
   592  		return 0
   593  	case f64 >= maxUint64+1:
   594  		return maxUint64
   595  	case f64 < minUint64:
   596  		return minUint64
   597  	default:
   598  		return uint64(f64) // NOTE: This may lead to loss of precision.
   599  	}
   600  }
   601  
   602  // Kind returns the token kind.
   603  func (t Token) Kind() Kind {
   604  	switch {
   605  	case t.raw != nil:
   606  		raw := t.raw
   607  		if uint64(raw.previousOffsetStart()) != t.num {
   608  			panic(invalidTokenPanic)
   609  		}
   610  		return Kind(t.raw.buf[raw.prevStart]).normalize()
   611  	case t.num != 0:
   612  		return '0'
   613  	case len(t.str) != 0:
   614  		return '"'
   615  	default:
   616  		return invalidKind
   617  	}
   618  }
   619  
   620  // A Kind represents the kind of a JSON token.
   621  //
   622  // A Kind is a single byte, which is conveniently the first byte of that
   623  // kind's symbol in the grammar (except for numbers, which are always represented
   624  // with '0').
   625  type Kind byte
   626  
   627  const (
   628  	KindInvalid     Kind = 0   // invalid kind
   629  	KindNull        Kind = 'n' // null
   630  	KindFalse       Kind = 'f' // false
   631  	KindTrue        Kind = 't' // true
   632  	KindString      Kind = '"' // string
   633  	KindNumber      Kind = '0' // number
   634  	KindBeginObject Kind = '{' // begin object
   635  	KindEndObject   Kind = '}' // end object
   636  	KindBeginArray  Kind = '[' // begin array
   637  	KindEndArray    Kind = ']' // end array
   638  )
   639  
   640  const invalidKind Kind = 0
   641  
   642  // String returns a string representation of k.
   643  func (k Kind) String() string {
   644  	switch k {
   645  	case 0:
   646  		return "invalid"
   647  	case 'n':
   648  		return "null"
   649  	case 'f':
   650  		return "false"
   651  	case 't':
   652  		return "true"
   653  	case '"':
   654  		return "string"
   655  	case '0':
   656  		return "number"
   657  	case '{':
   658  		return "{"
   659  	case '}':
   660  		return "}"
   661  	case '[':
   662  		return "["
   663  	case ']':
   664  		return "]"
   665  	default:
   666  		return "<invalid jsontext.Kind: " + jsonwire.QuoteRune([]byte{byte(k)}) + ">"
   667  	}
   668  }
   669  
   670  var normKind = [256]Kind{
   671  	'n': 'n',
   672  	'f': 'f',
   673  	't': 't',
   674  	'"': '"',
   675  	'{': '{',
   676  	'}': '}',
   677  	'[': '[',
   678  	']': ']',
   679  	'-': '0',
   680  	'0': '0',
   681  	'1': '0',
   682  	'2': '0',
   683  	'3': '0',
   684  	'4': '0',
   685  	'5': '0',
   686  	'6': '0',
   687  	'7': '0',
   688  	'8': '0',
   689  	'9': '0',
   690  }
   691  
   692  // normalize coalesces all possible starting characters of a number as just '0',
   693  // and converts all invalid kinds to 0.
   694  func (k Kind) normalize() Kind {
   695  	// A lookup table keeps the inlining cost as low as possible.
   696  	return normKind[k]
   697  }
   698
View as plain text