token_test.go

     1  // Copyright 2020 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  //go:build goexperiment.jsonv2
     6  
     7  package jsontext
     8  
     9  import (
    10  	"errors"
    11  	"internal/testenv"
    12  	"math"
    13  	"reflect"
    14  	"strconv"
    15  	"testing"
    16  )
    17  
    18  func TestTokenStringAllocations(t *testing.T) {
    19  	testenv.SkipIfOptimizationOff(t)
    20  	if testing.CoverMode() != "" {
    21  		t.Skip("coverage mode breaks the compiler optimization this depends on")
    22  	}
    23  
    24  	tok := rawToken(`"hello"`)
    25  	var m map[string]bool
    26  	got := int(testing.AllocsPerRun(10, func() {
    27  		// This function uses tok.String() is a non-escaping manner
    28  		// (i.e., looking it up in a Go map). It should not allocate.
    29  		if m[tok.String()] {
    30  			panic("never executed")
    31  		}
    32  	}))
    33  	if got > 0 {
    34  		t.Errorf("Token.String allocated %d times, want 0", got)
    35  	}
    36  }
    37  
    38  func TestTokenAccessors(t *testing.T) {
    39  	type valueError[T any] struct {
    40  		Value T
    41  		Error error
    42  	}
    43  	type token struct {
    44  		Bool    bool
    45  		String  string
    46  		Float32 valueError[float32]
    47  		Float   valueError[float64]
    48  		Int     valueError[int64]
    49  		Uint    valueError[uint64]
    50  		Kind    Kind
    51  	}
    52  	negZero := math.Copysign(0, -1)
    53  	f32 := func(f32 float32) valueError[float32] { return valueError[float32]{Value: f32} }
    54  	f32er := func(f32 float32) valueError[float32] { return valueError[float32]{Value: f32, Error: strconv.ErrRange} }
    55  	f64 := func(f64 float64) valueError[float64] { return valueError[float64]{Value: f64} }
    56  	f64er := func(f64 float64) valueError[float64] { return valueError[float64]{Value: f64, Error: strconv.ErrRange} }
    57  	i64 := func(i64 int64) valueError[int64] { return valueError[int64]{Value: i64} }
    58  	i64er := func(i64 int64) valueError[int64] { return valueError[int64]{Value: i64, Error: strconv.ErrRange} }
    59  	i64es := func(i64 int64) valueError[int64] { return valueError[int64]{Value: i64, Error: strconv.ErrSyntax} }
    60  	u64 := func(u64 uint64) valueError[uint64] { return valueError[uint64]{Value: u64} }
    61  	u64er := func(u64 uint64) valueError[uint64] { return valueError[uint64]{Value: u64, Error: strconv.ErrRange} }
    62  	u64es := func(u64 uint64) valueError[uint64] { return valueError[uint64]{Value: u64, Error: strconv.ErrSyntax} }
    63  
    64  	tests := []struct {
    65  		in   Token
    66  		want token
    67  	}{
    68  		{Token{}, token{String: "<invalid jsontext.Token>"}},
    69  		{Null, token{String: "null", Kind: 'n'}},
    70  		{False, token{Bool: false, String: "false", Kind: 'f'}},
    71  		{True, token{Bool: true, String: "true", Kind: 't'}},
    72  		{Bool(false), token{Bool: false, String: "false", Kind: 'f'}},
    73  		{Bool(true), token{Bool: true, String: "true", Kind: 't'}},
    74  		{BeginObject, token{String: "{", Kind: '{'}},
    75  		{EndObject, token{String: "}", Kind: '}'}},
    76  		{BeginArray, token{String: "[", Kind: '['}},
    77  		{EndArray, token{String: "]", Kind: ']'}},
    78  		{String(""), token{String: "", Kind: '"'}},
    79  		{String("hello, world!"), token{String: "hello, world!", Kind: '"'}},
    80  		{rawToken(`"hello, world!"`), token{String: "hello, world!", Kind: '"'}},
    81  		{Float32(float32(0)), token{String: "0", Float32: f32(0), Float: f64(0), Int: i64(0), Uint: u64(0), Kind: '0'}},
    82  		{Float32(float32(math.Copysign(0, -1))), token{String: "-0", Float32: f32(float32(negZero)), Float: f64(negZero), Int: i64(0), Uint: u64es(0), Kind: '0'}},
    83  		{Float32(float32(math.NaN())), token{String: "NaN", Float32: f32(float32(math.NaN())), Float: f64(math.NaN()), Kind: '"'}},
    84  		{Float32(float32(math.Inf(+1))), token{String: "Infinity", Float32: f32(float32(math.Inf(+1))), Float: f64(math.Inf(+1)), Kind: '"'}},
    85  		{Float32(float32(math.Inf(-1))), token{String: "-Infinity", Float32: f32(float32(math.Inf(-1))), Float: f64(math.Inf(-1)), Kind: '"'}},
    86  		{Float32(float32(math.Pi)), token{String: "3.1415927", Float32: f32(math.Pi), Float: f64(float64(float32(math.Pi))), Int: i64es(3), Uint: u64es(3), Kind: '0'}},
    87  		{Float32(float32(-1 * math.MaxFloat32)), token{String: "-3.4028235e+38", Float32: f32(float32(-1 * math.MaxFloat32)), Float: f64(-1 * math.MaxFloat32), Int: i64er(minInt64), Uint: u64es(minUint64), Kind: '0'}},
    88  		{Float32(float32(+1 * math.MaxFloat32)), token{String: "3.4028235e+38", Float32: f32(float32(+1 * math.MaxFloat32)), Float: f64(+1 * math.MaxFloat32), Int: i64er(maxInt64), Uint: u64er(maxUint64), Kind: '0'}},
    89  		{Float32(float32(123)), token{String: "123", Float32: f32(123), Float: f64(123), Int: i64(123), Uint: u64(123), Kind: '0'}},
    90  		{Float(0), token{String: "0", Float32: f32(0), Float: f64(0), Int: i64(0), Uint: u64(0), Kind: '0'}},
    91  		{Float(negZero), token{String: "-0", Float32: f32(float32(negZero)), Float: f64(negZero), Int: i64(0), Uint: u64es(0), Kind: '0'}},
    92  		{Float(math.NaN()), token{String: "NaN", Float32: f32(float32(math.NaN())), Float: f64(math.NaN()), Int: i64(0), Uint: u64(0), Kind: '"'}},
    93  		{Float(math.Inf(+1)), token{String: "Infinity", Float32: f32(float32(math.Inf(+1))), Float: f64(math.Inf(+1)), Kind: '"'}},
    94  		{Float(math.Inf(-1)), token{String: "-Infinity", Float32: f32(float32(math.Inf(-1))), Float: f64(math.Inf(-1)), Kind: '"'}},
    95  		{Float(math.Pi), token{String: "3.141592653589793", Float32: f32(math.Pi), Float: f64(math.Pi), Int: i64es(3), Uint: u64es(3), Kind: '0'}},
    96  		{Float(-1 * math.MaxFloat64), token{String: "-1.7976931348623157e+308", Float32: f32er(float32(math.Inf(-1))), Float: f64(-1 * math.MaxFloat64), Int: i64er(minInt64), Uint: u64es(minUint64), Kind: '0'}},
    97  		{Float(+1 * math.MaxFloat64), token{String: "1.7976931348623157e+308", Float32: f32er(float32(math.Inf(+1))), Float: f64(+1 * math.MaxFloat64), Int: i64er(maxInt64), Uint: u64er(maxUint64), Kind: '0'}},
    98  		{Float(123), token{String: "123", Float32: f32(123), Float: f64(123), Int: i64(123), Uint: u64(123), Kind: '0'}},
    99  		{Int(minInt64), token{String: "-9223372036854775808", Float32: f32(minInt64), Float: f64(minInt64), Int: i64(minInt64), Uint: u64es(minUint64), Kind: '0'}},
   100  		{Int(minInt64 + 1), token{String: "-9223372036854775807", Float32: f32(minInt64 + 1), Float: f64(minInt64 + 1), Int: i64(minInt64 + 1), Uint: u64es(minUint64), Kind: '0'}},
   101  		{Int(-1), token{String: "-1", Float32: f32(-1), Float: f64(-1), Int: i64(-1), Uint: u64es(minUint64), Kind: '0'}},
   102  		{Int(0), token{String: "0", Float32: f32(0), Float: f64(0), Int: i64(0), Uint: u64(0), Kind: '0'}},
   103  		{Int(+1), token{String: "1", Float32: f32(+1), Float: f64(+1), Int: i64(+1), Uint: u64(+1), Kind: '0'}},
   104  		{Int(maxInt64 - 1), token{String: "9223372036854775806", Float32: f32(maxInt64 - 1), Float: f64(maxInt64 - 1), Int: i64(maxInt64 - 1), Uint: u64(maxInt64 - 1), Kind: '0'}},
   105  		{Int(maxInt64), token{String: "9223372036854775807", Float32: f32(maxInt64), Float: f64(maxInt64), Int: i64(maxInt64), Uint: u64(maxInt64), Kind: '0'}},
   106  		{Uint(minUint64), token{String: "0", Kind: '0'}},
   107  		{Uint(minUint64 + 1), token{String: "1", Float32: f32(minUint64 + 1), Float: f64(minUint64 + 1), Int: i64(minUint64 + 1), Uint: u64(minUint64 + 1), Kind: '0'}},
   108  		{Uint(maxUint64 - 1), token{String: "18446744073709551614", Float32: f32(maxUint64 - 1), Float: f64(maxUint64 - 1), Int: i64er(maxInt64), Uint: u64(maxUint64 - 1), Kind: '0'}},
   109  		{Uint(maxUint64), token{String: "18446744073709551615", Float32: f32(maxUint64 - 1), Float: f64(maxUint64 - 1), Int: i64er(maxInt64), Uint: u64(maxUint64), Kind: '0'}},
   110  		{rawToken(`-0`), token{String: "-0", Float32: f32(float32(negZero)), Float: f64(negZero), Int: i64(0), Uint: u64es(0), Kind: '0'}},
   111  		{rawToken(`1e1000`), token{String: "1e1000", Float32: f32er(float32(math.Inf(+1))), Float: f64er(float64(math.Inf(+1))), Int: i64es(maxInt64), Uint: u64es(maxUint64), Kind: '0'}},
   112  		{rawToken(`-1e1000`), token{String: "-1e1000", Float32: f32er(float32(math.Inf(-1))), Float: f64er(float64(math.Inf(-1))), Int: i64es(minInt64), Uint: u64es(minUint64), Kind: '0'}},
   113  		{rawToken(`0.1`), token{String: "0.1", Float32: f32(0.1), Float: f64(0.1), Int: i64es(0), Uint: u64es(0), Kind: '0'}},
   114  		{rawToken(`0.5`), token{String: "0.5", Float32: f32(0.5), Float: f64(0.5), Int: i64es(0), Uint: u64es(0), Kind: '0'}},
   115  		{rawToken(`0.9`), token{String: "0.9", Float32: f32(0.9), Float: f64(0.9), Int: i64es(0), Uint: u64es(0), Kind: '0'}},
   116  		{rawToken(`1.0`), token{String: "1.0", Float32: f32(1.0), Float: f64(1.0), Int: i64es(1), Uint: u64es(1), Kind: '0'}},
   117  		{rawToken(`1.1`), token{String: "1.1", Float32: f32(1.1), Float: f64(1.1), Int: i64es(1), Uint: u64es(1), Kind: '0'}},
   118  		{rawToken(`123`), token{String: "123", Float32: f32(123), Float: f64(123), Int: i64(123), Uint: u64(123), Kind: '0'}},
   119  		{rawToken(`-0.1`), token{String: "-0.1", Float32: f32(-0.1), Float: f64(-0.1), Int: i64es(0), Uint: u64es(0), Kind: '0'}},
   120  		{rawToken(`-0.5`), token{String: "-0.5", Float32: f32(-0.5), Float: f64(-0.5), Int: i64es(0), Uint: u64es(0), Kind: '0'}},
   121  		{rawToken(`-0.9`), token{String: "-0.9", Float32: f32(-0.9), Float: f64(-0.9), Int: i64es(0), Uint: u64es(0), Kind: '0'}},
   122  		{rawToken(`-1.0`), token{String: "-1.0", Float32: f32(-1.0), Float: f64(-1.0), Int: i64es(-1), Uint: u64es(0), Kind: '0'}},
   123  		{rawToken(`-1.1`), token{String: "-1.1", Float32: f32(-1.1), Float: f64(-1.1), Int: i64es(-1), Uint: u64es(0), Kind: '0'}},
   124  		{rawToken(`-123`), token{String: "-123", Float32: f32(-123), Float: f64(-123), Int: i64(-123), Uint: u64es(0), Kind: '0'}},
   125  		{rawToken(`99999999999999999999`), token{String: "99999999999999999999", Float32: f32(1e20 - 1), Float: f64(1e20 - 1), Int: i64er(maxInt64), Uint: u64er(maxUint64), Kind: '0'}},
   126  		{rawToken(`-99999999999999999999`), token{String: "-99999999999999999999", Float32: f32(-1e20 - 1), Float: f64(-1e20 - 1), Int: i64er(minInt64), Uint: u64es(minUint64), Kind: '0'}},
   127  		{rawToken(`3.1415927`), token{String: "3.1415927", Float32: f32(math.Pi), Float: f64(3.1415927), Int: i64es(3), Uint: u64es(3), Kind: '0'}},
   128  		{rawToken(`3.141592653589793`), token{String: "3.141592653589793", Float32: f32(math.Pi), Float: f64(math.Pi), Int: i64es(3), Uint: u64es(3), Kind: '0'}},
   129  		{rawToken(`-9223372036854775807`), token{String: "-9223372036854775807", Float32: f32(-1 << 63), Float: f64(-1 << 63), Int: i64(minInt64 + 1), Uint: u64es(minUint64), Kind: '0'}},
   130  		{rawToken(`-9223372036854775808`), token{String: "-9223372036854775808", Float32: f32(-1 << 63), Float: f64(-1 << 63), Int: i64(minInt64), Uint: u64es(minUint64), Kind: '0'}},
   131  		{rawToken(`-9223372036854775809`), token{String: "-9223372036854775809", Float32: f32(-1 << 63), Float: f64(-1 << 63), Int: i64er(minInt64), Uint: u64es(minUint64), Kind: '0'}},
   132  		{rawToken(`9223372036854775806`), token{String: "9223372036854775806", Float32: f32(1 << 63), Float: f64(1 << 63), Int: i64(maxInt64 - 1), Uint: u64(maxInt64 - 1), Kind: '0'}},
   133  		{rawToken(`9223372036854775807`), token{String: "9223372036854775807", Float32: f32(1 << 63), Float: f64(1 << 63), Int: i64(maxInt64), Uint: u64(maxInt64), Kind: '0'}},
   134  		{rawToken(`9223372036854775808`), token{String: "9223372036854775808", Float32: f32(1 << 63), Float: f64(1 << 63), Int: i64er(maxInt64), Uint: u64(maxInt64 + 1), Kind: '0'}},
   135  		{rawToken(`18446744073709551614`), token{String: "18446744073709551614", Float32: f32(1 << 64), Float: f64(1 << 64), Int: i64er(maxInt64), Uint: u64(maxUint64 - 1), Kind: '0'}},
   136  		{rawToken(`18446744073709551615`), token{String: "18446744073709551615", Float32: f32(1 << 64), Float: f64(1 << 64), Int: i64er(maxInt64), Uint: u64(maxUint64), Kind: '0'}},
   137  		{rawToken(`18446744073709551616`), token{String: "18446744073709551616", Float32: f32(1 << 64), Float: f64(1 << 64), Int: i64er(maxInt64), Uint: u64er(maxUint64), Kind: '0'}},
   138  
   139  		// NOTE: There exist many raw JSON numbers where:
   140  		//	float32(ParseFloat(s, 32)) != float32(ParseFloat(s, 64))
   141  		// due to issues with double rounding in opposite directions.
   142  		// This suggests the need for a Token.Float32 accessor.
   143  		{rawToken(`9000000000.0000001`), token{String: "9000000000.0000001", Float32: f32(9000000000.0000001), Float: f64(9000000000.0000001), Int: i64es(9e9), Uint: u64es(9e9), Kind: '0'}},
   144  		// NOTE: ±7.038531e-26 is the only 32-bit precision float where:
   145  		//	f != float32(ParseFloat(FormatFloat(f, 32), 64))
   146  		// assuming FormatFloat uses ECMA-262, 10th edition, section 7.1.12.1.
   147  		{rawToken(`7.038531e-26`), token{String: "7.038531e-26", Float32: f32(7.038531e-26), Float: f64(7.038531e-26), Int: i64es(0), Uint: u64es(0), Kind: '0'}},
   148  		{Float32(7.038531e-26), token{String: "7.038531e-26", Float32: f32(7.038531e-26), Float: f64(7.038530691851209e-26), Int: i64es(0), Uint: u64es(0), Kind: '0'}},
   149  		{Float(7.038531e-26), token{String: "7.038531e-26", Float32: f32(7.0385313e-26), Float: f64(7.038531e-26), Int: i64es(0), Uint: u64es(0), Kind: '0'}},
   150  	}
   151  
   152  	for _, tt := range tests {
   153  		t.Run(tt.in.String(), func(t *testing.T) {
   154  			got := token{
   155  				Bool: func() bool {
   156  					defer func() { recover() }()
   157  					return tt.in.Bool()
   158  				}(),
   159  				String: tt.in.String(),
   160  				Float32: func() valueError[float32] {
   161  					defer func() { recover() }()
   162  					f32, err := tt.in.Float32()
   163  					return valueError[float32]{f32, err}
   164  				}(),
   165  				Float: func() valueError[float64] {
   166  					defer func() { recover() }()
   167  					f64, err := tt.in.Float()
   168  					return valueError[float64]{f64, err}
   169  				}(),
   170  				Int: func() valueError[int64] {
   171  					defer func() { recover() }()
   172  					i64, err := tt.in.Int()
   173  					return valueError[int64]{i64, err}
   174  				}(),
   175  				Uint: func() valueError[uint64] {
   176  					defer func() { recover() }()
   177  					u64, err := tt.in.Uint()
   178  					return valueError[uint64]{u64, err}
   179  				}(),
   180  				Kind: tt.in.Kind(),
   181  			}
   182  
   183  			if got.Bool != tt.want.Bool {
   184  				t.Errorf("Token(%s).Bool() = %v, want %v", tt.in, got.Bool, tt.want.Bool)
   185  			}
   186  			if got.String != tt.want.String {
   187  				t.Errorf("Token(%s).String() = %v, want %v", tt.in, got.String, tt.want.String)
   188  			}
   189  			if math.Float32bits(got.Float32.Value) != math.Float32bits(tt.want.Float32.Value) || !errors.Is(got.Float32.Error, tt.want.Float32.Error) {
   190  				t.Errorf("Token(%s).Float32() = (%v, %v), want (%v, %v)", tt.in, got.Float32.Value, got.Float32.Error, tt.want.Float32.Value, tt.want.Float32.Error)
   191  			}
   192  			if math.Float64bits(got.Float.Value) != math.Float64bits(tt.want.Float.Value) || !errors.Is(got.Float.Error, tt.want.Float.Error) {
   193  				t.Errorf("Token(%s).Float() = (%v, %v), want (%v, %v)", tt.in, got.Float.Value, got.Float.Error, tt.want.Float.Value, tt.want.Float.Error)
   194  			}
   195  			if got.Int.Value != tt.want.Int.Value || !errors.Is(got.Int.Error, tt.want.Int.Error) {
   196  				t.Errorf("Token(%s).Int() = (%v, %v), want (%v, %v)", tt.in, got.Int.Value, got.Int.Error, tt.want.Int.Value, tt.want.Int.Error)
   197  			}
   198  			if got.Uint.Value != tt.want.Uint.Value || !errors.Is(got.Uint.Error, tt.want.Uint.Error) {
   199  				t.Errorf("Token(%s).Uint() = (%v, %v), want (%v, %v)", tt.in, got.Uint.Value, got.Uint.Error, tt.want.Uint.Value, tt.want.Uint.Error)
   200  			}
   201  			if got.Kind != tt.want.Kind {
   202  				t.Errorf("Token(%s).Kind() = %v, want %v", tt.in, got.Kind, tt.want.Kind)
   203  			}
   204  		})
   205  	}
   206  }
   207  
   208  func TestTokenClone(t *testing.T) {
   209  	tests := []struct {
   210  		in           Token
   211  		wantExactRaw bool
   212  	}{
   213  		{Token{}, true},
   214  		{Null, true},
   215  		{False, true},
   216  		{True, true},
   217  		{BeginObject, true},
   218  		{EndObject, true},
   219  		{BeginArray, true},
   220  		{EndArray, true},
   221  		{String("hello, world!"), true},
   222  		{rawToken(`"hello, world!"`), false},
   223  		{Float(3.14159), true},
   224  		{rawToken(`3.14159`), false},
   225  	}
   226  
   227  	for _, tt := range tests {
   228  		t.Run(tt.in.String(), func(t *testing.T) {
   229  			got := tt.in.Clone()
   230  			if !reflect.DeepEqual(got, tt.in) {
   231  				t.Errorf("Token(%s) == Token(%s).Clone() = false, want true", tt.in, tt.in)
   232  			}
   233  			gotExactRaw := got.raw == tt.in.raw
   234  			if gotExactRaw != tt.wantExactRaw {
   235  				t.Errorf("Token(%s).raw == Token(%s).Clone().raw = %v, want %v", tt.in, tt.in, gotExactRaw, tt.wantExactRaw)
   236  			}
   237  		})
   238  	}
   239  }
   240
View as plain text