Source file src/testing/sub_test.go

     1  // Copyright 2016 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package testing
     6  
     7  import (
     8  	"bytes"
     9  	"fmt"
    10  	"regexp"
    11  	"runtime"
    12  	"slices"
    13  	"strings"
    14  	"sync"
    15  	"sync/atomic"
    16  	"time"
    17  )
    18  
    19  func init() {
    20  	// Make benchmark tests run 10x faster.
    21  	benchTime.d = 100 * time.Millisecond
    22  }
    23  
    24  func TestTestState(t *T) {
    25  	const (
    26  		add1 = 0
    27  		done = 1
    28  	)
    29  	type call struct {
    30  		typ int // run or done
    31  		// result from applying the call
    32  		running int
    33  		waiting int
    34  		started bool
    35  	}
    36  	testCases := []struct {
    37  		max int
    38  		run []call
    39  	}{{
    40  		max: 1,
    41  		run: []call{
    42  			{typ: add1, running: 1, waiting: 0, started: true},
    43  			{typ: done, running: 0, waiting: 0, started: false},
    44  		},
    45  	}, {
    46  		max: 1,
    47  		run: []call{
    48  			{typ: add1, running: 1, waiting: 0, started: true},
    49  			{typ: add1, running: 1, waiting: 1, started: false},
    50  			{typ: done, running: 1, waiting: 0, started: true},
    51  			{typ: done, running: 0, waiting: 0, started: false},
    52  			{typ: add1, running: 1, waiting: 0, started: true},
    53  		},
    54  	}, {
    55  		max: 3,
    56  		run: []call{
    57  			{typ: add1, running: 1, waiting: 0, started: true},
    58  			{typ: add1, running: 2, waiting: 0, started: true},
    59  			{typ: add1, running: 3, waiting: 0, started: true},
    60  			{typ: add1, running: 3, waiting: 1, started: false},
    61  			{typ: add1, running: 3, waiting: 2, started: false},
    62  			{typ: add1, running: 3, waiting: 3, started: false},
    63  			{typ: done, running: 3, waiting: 2, started: true},
    64  			{typ: add1, running: 3, waiting: 3, started: false},
    65  			{typ: done, running: 3, waiting: 2, started: true},
    66  			{typ: done, running: 3, waiting: 1, started: true},
    67  			{typ: done, running: 3, waiting: 0, started: true},
    68  			{typ: done, running: 2, waiting: 0, started: false},
    69  			{typ: done, running: 1, waiting: 0, started: false},
    70  			{typ: done, running: 0, waiting: 0, started: false},
    71  		},
    72  	}}
    73  	for i, tc := range testCases {
    74  		tstate := &testState{
    75  			startParallel: make(chan bool),
    76  			maxParallel:   tc.max,
    77  		}
    78  		for j, call := range tc.run {
    79  			doCall := func(f func()) chan bool {
    80  				done := make(chan bool)
    81  				go func() {
    82  					f()
    83  					done <- true
    84  				}()
    85  				return done
    86  			}
    87  			started := false
    88  			switch call.typ {
    89  			case add1:
    90  				signal := doCall(tstate.waitParallel)
    91  				select {
    92  				case <-signal:
    93  					started = true
    94  				case tstate.startParallel <- true:
    95  					<-signal
    96  				}
    97  			case done:
    98  				signal := doCall(tstate.release)
    99  				select {
   100  				case <-signal:
   101  				case <-tstate.startParallel:
   102  					started = true
   103  					<-signal
   104  				}
   105  			}
   106  			if started != call.started {
   107  				t.Errorf("%d:%d:started: got %v; want %v", i, j, started, call.started)
   108  			}
   109  			if tstate.running != call.running {
   110  				t.Errorf("%d:%d:running: got %v; want %v", i, j, tstate.running, call.running)
   111  			}
   112  			if tstate.numWaiting != call.waiting {
   113  				t.Errorf("%d:%d:waiting: got %v; want %v", i, j, tstate.numWaiting, call.waiting)
   114  			}
   115  		}
   116  	}
   117  }
   118  
   119  func TestTRun(t *T) {
   120  	realTest := t
   121  	testCases := []struct {
   122  		desc   string
   123  		ok     bool
   124  		maxPar int
   125  		chatty bool
   126  		json   bool
   127  		output string
   128  		f      func(*T)
   129  	}{{
   130  		desc:   "failnow skips future sequential and parallel tests at same level",
   131  		ok:     false,
   132  		maxPar: 1,
   133  		output: `
   134  --- FAIL: failnow skips future sequential and parallel tests at same level (N.NNs)
   135      --- FAIL: failnow skips future sequential and parallel tests at same level/#00 (N.NNs)
   136      `,
   137  		f: func(t *T) {
   138  			ranSeq := false
   139  			ranPar := false
   140  			t.Run("", func(t *T) {
   141  				t.Run("par", func(t *T) {
   142  					t.Parallel()
   143  					ranPar = true
   144  				})
   145  				t.Run("seq", func(t *T) {
   146  					ranSeq = true
   147  				})
   148  				t.FailNow()
   149  				t.Run("seq", func(t *T) {
   150  					realTest.Error("test must be skipped")
   151  				})
   152  				t.Run("par", func(t *T) {
   153  					t.Parallel()
   154  					realTest.Error("test must be skipped.")
   155  				})
   156  			})
   157  			if !ranPar {
   158  				realTest.Error("parallel test was not run")
   159  			}
   160  			if !ranSeq {
   161  				realTest.Error("sequential test was not run")
   162  			}
   163  		},
   164  	}, {
   165  		desc:   "failure in parallel test propagates upwards",
   166  		ok:     false,
   167  		maxPar: 1,
   168  		output: `
   169  --- FAIL: failure in parallel test propagates upwards (N.NNs)
   170      --- FAIL: failure in parallel test propagates upwards/#00 (N.NNs)
   171          --- FAIL: failure in parallel test propagates upwards/#00/par (N.NNs)
   172          `,
   173  		f: func(t *T) {
   174  			t.Run("", func(t *T) {
   175  				t.Parallel()
   176  				t.Run("par", func(t *T) {
   177  					t.Parallel()
   178  					t.Fail()
   179  				})
   180  			})
   181  		},
   182  	}, {
   183  		desc:   "skipping without message, chatty",
   184  		ok:     true,
   185  		chatty: true,
   186  		output: `
   187  === RUN   skipping without message, chatty
   188  --- SKIP: skipping without message, chatty (N.NNs)`,
   189  		f: func(t *T) { t.SkipNow() },
   190  	}, {
   191  		desc:   "chatty with recursion",
   192  		ok:     true,
   193  		chatty: true,
   194  		output: `
   195  === RUN   chatty with recursion
   196  === RUN   chatty with recursion/#00
   197  === RUN   chatty with recursion/#00/#00
   198  === RUN   chatty with recursion/#00/#01
   199      sub_test.go:NNN: ^V^O^N^[
   200  --- PASS: chatty with recursion (N.NNs)
   201      --- PASS: chatty with recursion/#00 (N.NNs)
   202          --- PASS: chatty with recursion/#00/#00 (N.NNs)
   203          --- PASS: chatty with recursion/#00/#01 (N.NNs)`,
   204  		f: func(t *T) {
   205  			t.Run("", func(t *T) {
   206  				t.Run("", func(t *T) {})
   207  				t.Run("", func(t *T) {
   208  					t.Log(string(markFraming) + string(markErrBegin) + string(markErrEnd) + string(markEscape))
   209  				})
   210  			})
   211  		},
   212  	}, {
   213  		desc:   "chatty with recursion and json",
   214  		ok:     false,
   215  		chatty: true,
   216  		json:   true,
   217  		output: `
   218  ^V=== RUN   chatty with recursion and json
   219  ^V=== RUN   chatty with recursion and json/#00
   220  ^V=== RUN   chatty with recursion and json/#00/#00
   221  ^V--- PASS: chatty with recursion and json/#00/#00 (N.NNs)
   222  ^V=== NAME  chatty with recursion and json/#00
   223  ^V=== RUN   chatty with recursion and json/#00/#01
   224      sub_test.go:NNN: skip
   225  ^V--- SKIP: chatty with recursion and json/#00/#01 (N.NNs)
   226  ^V=== NAME  chatty with recursion and json/#00
   227  ^V=== RUN   chatty with recursion and json/#00/#02
   228  ^O    sub_test.go:NNN: fail^N
   229  ^V--- FAIL: chatty with recursion and json/#00/#02 (N.NNs)
   230  ^V=== NAME  chatty with recursion and json/#00
   231  ^V=== RUN   chatty with recursion and json/#00/#03
   232  ^O    sub_test.go:NNN: ^[^O^[^N^[^[^N
   233  ^V--- FAIL: chatty with recursion and json/#00/#03 (N.NNs)
   234  ^V=== NAME  chatty with recursion and json/#00
   235  ^V--- FAIL: chatty with recursion and json/#00 (N.NNs)
   236  ^V=== NAME  chatty with recursion and json
   237  ^V--- FAIL: chatty with recursion and json (N.NNs)
   238  ^V=== NAME  `,
   239  		f: func(t *T) {
   240  			t.Run("", func(t *T) {
   241  				t.Run("", func(t *T) {})
   242  				t.Run("", func(t *T) { t.Skip("skip") })
   243  				t.Run("", func(t *T) { t.Fatal("fail") })
   244  				t.Run("", func(t *T) { t.Error(string(markErrBegin) + string(markErrEnd) + string(markEscape)) })
   245  			})
   246  		},
   247  	}, {
   248  		desc: "skipping without message, not chatty",
   249  		ok:   true,
   250  		f:    func(t *T) { t.SkipNow() },
   251  	}, {
   252  		desc: "skipping after error",
   253  		output: `
   254  --- FAIL: skipping after error (N.NNs)
   255      sub_test.go:NNN: an error
   256      sub_test.go:NNN: skipped`,
   257  		f: func(t *T) {
   258  			t.Error("an error")
   259  			t.Skip("skipped")
   260  		},
   261  	}, {
   262  		desc:   "use Run to locally synchronize parallelism",
   263  		ok:     true,
   264  		maxPar: 1,
   265  		f: func(t *T) {
   266  			var count uint32
   267  			t.Run("waitGroup", func(t *T) {
   268  				for i := 0; i < 4; i++ {
   269  					t.Run("par", func(t *T) {
   270  						t.Parallel()
   271  						atomic.AddUint32(&count, 1)
   272  					})
   273  				}
   274  			})
   275  			if count != 4 {
   276  				t.Errorf("count was %d; want 4", count)
   277  			}
   278  		},
   279  	}, {
   280  		desc: "alternate sequential and parallel",
   281  		// Sequential tests should partake in the counting of running threads.
   282  		// Otherwise, if one runs parallel subtests in sequential tests that are
   283  		// itself subtests of parallel tests, the counts can get askew.
   284  		ok:     true,
   285  		maxPar: 1,
   286  		f: func(t *T) {
   287  			t.Run("a", func(t *T) {
   288  				t.Parallel()
   289  				t.Run("b", func(t *T) {
   290  					// Sequential: ensure running count is decremented.
   291  					t.Run("c", func(t *T) {
   292  						t.Parallel()
   293  					})
   294  				})
   295  			})
   296  		},
   297  	}, {
   298  		desc: "alternate sequential and parallel 2",
   299  		// Sequential tests should partake in the counting of running threads.
   300  		// Otherwise, if one runs parallel subtests in sequential tests that are
   301  		// itself subtests of parallel tests, the counts can get askew.
   302  		ok:     true,
   303  		maxPar: 2,
   304  		f: func(t *T) {
   305  			for i := 0; i < 2; i++ {
   306  				t.Run("a", func(t *T) {
   307  					t.Parallel()
   308  					time.Sleep(time.Nanosecond)
   309  					for i := 0; i < 2; i++ {
   310  						t.Run("b", func(t *T) {
   311  							time.Sleep(time.Nanosecond)
   312  							for i := 0; i < 2; i++ {
   313  								t.Run("c", func(t *T) {
   314  									t.Parallel()
   315  									time.Sleep(time.Nanosecond)
   316  								})
   317  							}
   318  						})
   319  					}
   320  				})
   321  			}
   322  		},
   323  	}, {
   324  		desc:   "stress test",
   325  		ok:     true,
   326  		maxPar: 4,
   327  		f: func(t *T) {
   328  			t.Parallel()
   329  			for i := 0; i < 12; i++ {
   330  				t.Run("a", func(t *T) {
   331  					t.Parallel()
   332  					time.Sleep(time.Nanosecond)
   333  					for i := 0; i < 12; i++ {
   334  						t.Run("b", func(t *T) {
   335  							time.Sleep(time.Nanosecond)
   336  							for i := 0; i < 12; i++ {
   337  								t.Run("c", func(t *T) {
   338  									t.Parallel()
   339  									time.Sleep(time.Nanosecond)
   340  									t.Run("d1", func(t *T) {})
   341  									t.Run("d2", func(t *T) {})
   342  									t.Run("d3", func(t *T) {})
   343  									t.Run("d4", func(t *T) {})
   344  								})
   345  							}
   346  						})
   347  					}
   348  				})
   349  			}
   350  		},
   351  	}, {
   352  		desc:   "skip output",
   353  		ok:     true,
   354  		maxPar: 4,
   355  		f: func(t *T) {
   356  			t.Skip()
   357  		},
   358  	}, {
   359  		desc: "subtest calls error on parent",
   360  		ok:   false,
   361  		output: `
   362  --- FAIL: subtest calls error on parent (N.NNs)
   363      sub_test.go:NNN: first this
   364      sub_test.go:NNN: and now this!
   365      sub_test.go:NNN: oh, and this too`,
   366  		maxPar: 1,
   367  		f: func(t *T) {
   368  			t.Errorf("first this")
   369  			outer := t
   370  			t.Run("", func(t *T) {
   371  				outer.Errorf("and now this!")
   372  			})
   373  			t.Errorf("oh, and this too")
   374  		},
   375  	}, {
   376  		desc: "subtest calls fatal on parent",
   377  		ok:   false,
   378  		output: `
   379  --- FAIL: subtest calls fatal on parent (N.NNs)
   380      sub_test.go:NNN: first this
   381      sub_test.go:NNN: and now this!
   382      --- FAIL: subtest calls fatal on parent/#00 (N.NNs)
   383          testing.go:NNN: test executed panic(nil) or runtime.Goexit: subtest may have called FailNow on a parent test`,
   384  		maxPar: 1,
   385  		f: func(t *T) {
   386  			outer := t
   387  			t.Errorf("first this")
   388  			t.Run("", func(t *T) {
   389  				outer.Fatalf("and now this!")
   390  			})
   391  			t.Errorf("Should not reach here.")
   392  		},
   393  	}, {
   394  		desc: "subtest calls error on ancestor",
   395  		ok:   false,
   396  		output: `
   397  --- FAIL: subtest calls error on ancestor (N.NNs)
   398      sub_test.go:NNN: Report to ancestor
   399      --- FAIL: subtest calls error on ancestor/#00 (N.NNs)
   400          sub_test.go:NNN: Still do this
   401      sub_test.go:NNN: Also do this`,
   402  		maxPar: 1,
   403  		f: func(t *T) {
   404  			outer := t
   405  			t.Run("", func(t *T) {
   406  				t.Run("", func(t *T) {
   407  					outer.Errorf("Report to ancestor")
   408  				})
   409  				t.Errorf("Still do this")
   410  			})
   411  			t.Errorf("Also do this")
   412  		},
   413  	}, {
   414  		desc: "subtest calls fatal on ancestor",
   415  		ok:   false,
   416  		output: `
   417  --- FAIL: subtest calls fatal on ancestor (N.NNs)
   418      sub_test.go:NNN: Nope`,
   419  		maxPar: 1,
   420  		f: func(t *T) {
   421  			outer := t
   422  			t.Run("", func(t *T) {
   423  				for i := 0; i < 4; i++ {
   424  					t.Run("", func(t *T) {
   425  						outer.Fatalf("Nope")
   426  					})
   427  					t.Errorf("Don't do this")
   428  				}
   429  				t.Errorf("And neither do this")
   430  			})
   431  			t.Errorf("Nor this")
   432  		},
   433  	}, {
   434  		desc:   "panic on goroutine fail after test exit",
   435  		ok:     false,
   436  		maxPar: 4,
   437  		f: func(t *T) {
   438  			ch := make(chan bool)
   439  			t.Run("", func(t *T) {
   440  				go func() {
   441  					<-ch
   442  					defer func() {
   443  						if r := recover(); r == nil {
   444  							realTest.Errorf("expected panic")
   445  						}
   446  						ch <- true
   447  					}()
   448  					t.Errorf("failed after success")
   449  				}()
   450  			})
   451  			ch <- true
   452  			<-ch
   453  		},
   454  	}, {
   455  		desc: "log in finished sub test logs to parent",
   456  		ok:   false,
   457  		output: `
   458  		--- FAIL: log in finished sub test logs to parent (N.NNs)
   459      sub_test.go:NNN: message2
   460      sub_test.go:NNN: message1
   461      sub_test.go:NNN: error`,
   462  		maxPar: 1,
   463  		f: func(t *T) {
   464  			ch := make(chan bool)
   465  			t.Run("sub", func(t2 *T) {
   466  				go func() {
   467  					<-ch
   468  					t2.Log("message1")
   469  					ch <- true
   470  				}()
   471  			})
   472  			t.Log("message2")
   473  			ch <- true
   474  			<-ch
   475  			t.Errorf("error")
   476  		},
   477  	}, {
   478  		// A chatty test should always log with fmt.Print, even if the
   479  		// parent test has completed.
   480  		desc:   "log in finished sub test with chatty",
   481  		ok:     false,
   482  		chatty: true,
   483  		output: `
   484  		--- FAIL: log in finished sub test with chatty (N.NNs)`,
   485  		maxPar: 1,
   486  		f: func(t *T) {
   487  			ch := make(chan bool)
   488  			t.Run("sub", func(t2 *T) {
   489  				go func() {
   490  					<-ch
   491  					t2.Log("message1")
   492  					ch <- true
   493  				}()
   494  			})
   495  			t.Log("message2")
   496  			ch <- true
   497  			<-ch
   498  			t.Errorf("error")
   499  		},
   500  	}, {
   501  		// If a subtest panics we should run cleanups.
   502  		desc:   "cleanup when subtest panics",
   503  		ok:     false,
   504  		chatty: false,
   505  		output: `
   506  --- FAIL: cleanup when subtest panics (N.NNs)
   507      --- FAIL: cleanup when subtest panics/sub (N.NNs)
   508      sub_test.go:NNN: running cleanup`,
   509  		f: func(t *T) {
   510  			t.Cleanup(func() { t.Log("running cleanup") })
   511  			t.Run("sub", func(t2 *T) {
   512  				t2.FailNow()
   513  			})
   514  		},
   515  	}, {
   516  		desc: "buffered output gets flushed at test end",
   517  		ok:   false,
   518  		output: `
   519  --- FAIL: buffered output gets flushed at test end (N.NNs)
   520      --- FAIL: buffered output gets flushed at test end/#00 (N.NNs)
   521          a
   522          b`,
   523  		f: func(t *T) {
   524  			t.Run("", func(t *T) {
   525  				o := t.Output()
   526  				o.Write([]byte("a\n"))
   527  				o.Write([]byte("b"))
   528  				t.Fail()
   529  			})
   530  		},
   531  	}, {
   532  		desc:   "output with chatty",
   533  		ok:     true,
   534  		chatty: true,
   535  		output: `
   536  === RUN   output with chatty
   537  === RUN   output with chatty/#00
   538      a
   539      b
   540  --- PASS: output with chatty (N.NNs)
   541      --- PASS: output with chatty/#00 (N.NNs)`,
   542  		f: func(t *T) {
   543  			t.Run("", func(t *T) {
   544  				o := t.Output()
   545  				o.Write([]byte("a\n"))
   546  				o.Write([]byte("b"))
   547  			})
   548  		},
   549  	}, {
   550  		desc:   "output with chatty and json",
   551  		ok:     true,
   552  		chatty: true,
   553  		json:   true,
   554  		output: `
   555  ^V=== RUN   output with chatty and json
   556  ^V=== RUN   output with chatty and json/#00
   557      a
   558      b
   559  ^V--- PASS: output with chatty and json/#00 (N.NNs)
   560  ^V=== NAME  output with chatty and json
   561  ^V--- PASS: output with chatty and json (N.NNs)
   562  ^V=== NAME
   563  `,
   564  		f: func(t *T) {
   565  			t.Run("", func(t *T) {
   566  				o := t.Output()
   567  				o.Write([]byte("a\n"))
   568  				o.Write([]byte("b"))
   569  			})
   570  		},
   571  	}, {
   572  		desc: "output in finished sub test outputs to parent",
   573  		ok:   false,
   574  		output: `
   575  		--- FAIL: output in finished sub test outputs to parent (N.NNs)
   576      message2
   577      message1
   578      sub_test.go:NNN: error`,
   579  		f: func(t *T) {
   580  			ch := make(chan bool)
   581  			t.Run("sub", func(t2 *T) {
   582  				go func() {
   583  					<-ch
   584  					t2.Output().Write([]byte("message1\n"))
   585  					ch <- true
   586  				}()
   587  			})
   588  			t.Output().Write([]byte("message2\n"))
   589  			ch <- true
   590  			<-ch
   591  			t.Errorf("error")
   592  		},
   593  	}, {
   594  		desc: "newline between buffered log and log",
   595  		ok:   false,
   596  		output: `
   597  --- FAIL: newline between buffered log and log (N.NNs)
   598      --- FAIL: newline between buffered log and log/#00 (N.NNs)
   599          buffered message
   600          sub_test.go:NNN: log`,
   601  		f: func(t *T) {
   602  			t.Run("", func(t *T) {
   603  				o := t.Output()
   604  				o.Write([]byte("buffered message"))
   605  				t.Log("log")
   606  				t.Fail()
   607  			})
   608  		},
   609  	}}
   610  	for _, tc := range testCases {
   611  		t.Run(tc.desc, func(t *T) {
   612  			tstate := newTestState(tc.maxPar, allMatcher())
   613  			buf := &strings.Builder{}
   614  			root := &T{
   615  				common: common{
   616  					signal:  make(chan bool),
   617  					barrier: make(chan bool),
   618  					name:    "",
   619  					w:       buf,
   620  				},
   621  				tstate: tstate,
   622  			}
   623  			if tc.chatty {
   624  				root.chatty = newChattyPrinter(root.w)
   625  				root.chatty.json = tc.json
   626  			}
   627  			ok := root.Run(tc.desc, tc.f)
   628  			tstate.release()
   629  
   630  			if ok != tc.ok {
   631  				t.Errorf("%s:ok: got %v; want %v", tc.desc, ok, tc.ok)
   632  			}
   633  			if ok != !root.Failed() {
   634  				t.Errorf("%s:root failed: got %v; want %v", tc.desc, !ok, root.Failed())
   635  			}
   636  			if tstate.running != 0 || tstate.numWaiting != 0 {
   637  				t.Errorf("%s:running and waiting non-zero: got %d and %d", tc.desc, tstate.running, tstate.numWaiting)
   638  			}
   639  			got := strings.TrimSpace(buf.String())
   640  			want := strings.TrimSpace(tc.output)
   641  			re := makeRegexp(want)
   642  			if ok, err := regexp.MatchString(re, got); !ok || err != nil {
   643  				t.Errorf("%s:output:\ngot:\n%s\nwant:\n%s", tc.desc, notateOutput(got), want)
   644  			}
   645  		})
   646  	}
   647  }
   648  
   649  func TestBRun(t *T) {
   650  	work := func(b *B) {
   651  		for i := 0; i < b.N; i++ {
   652  			time.Sleep(time.Nanosecond)
   653  		}
   654  	}
   655  	testCases := []struct {
   656  		desc   string
   657  		failed bool
   658  		chatty bool
   659  		output string
   660  		f      func(*B)
   661  	}{{
   662  		desc: "simulate sequential run of subbenchmarks.",
   663  		f: func(b *B) {
   664  			b.Run("", func(b *B) { work(b) })
   665  			time1 := b.result.NsPerOp()
   666  			b.Run("", func(b *B) { work(b) })
   667  			time2 := b.result.NsPerOp()
   668  			if time1 >= time2 {
   669  				t.Errorf("no time spent in benchmark t1 >= t2 (%d >= %d)", time1, time2)
   670  			}
   671  		},
   672  	}, {
   673  		desc: "bytes set by all benchmarks",
   674  		f: func(b *B) {
   675  			b.Run("", func(b *B) { b.SetBytes(10); work(b) })
   676  			b.Run("", func(b *B) { b.SetBytes(10); work(b) })
   677  			if b.result.Bytes != 20 {
   678  				t.Errorf("bytes: got: %d; want 20", b.result.Bytes)
   679  			}
   680  		},
   681  	}, {
   682  		desc: "bytes set by some benchmarks",
   683  		// In this case the bytes result is meaningless, so it must be 0.
   684  		f: func(b *B) {
   685  			b.Run("", func(b *B) { b.SetBytes(10); work(b) })
   686  			b.Run("", func(b *B) { work(b) })
   687  			b.Run("", func(b *B) { b.SetBytes(10); work(b) })
   688  			if b.result.Bytes != 0 {
   689  				t.Errorf("bytes: got: %d; want 0", b.result.Bytes)
   690  			}
   691  		},
   692  	}, {
   693  		desc:   "failure carried over to root",
   694  		failed: true,
   695  		output: "--- FAIL: root",
   696  		f:      func(b *B) { b.Fail() },
   697  	}, {
   698  		desc:   "skipping without message, chatty",
   699  		chatty: true,
   700  		output: "--- SKIP: root",
   701  		f:      func(b *B) { b.SkipNow() },
   702  	}, {
   703  		desc:   "chatty with recursion",
   704  		chatty: true,
   705  		f: func(b *B) {
   706  			b.Run("", func(b *B) {
   707  				b.Run("", func(b *B) {})
   708  			})
   709  		},
   710  	}, {
   711  		desc: "skipping without message, not chatty",
   712  		f:    func(b *B) { b.SkipNow() },
   713  	}, {
   714  		desc:   "skipping after error",
   715  		failed: true,
   716  		output: `
   717  --- FAIL: root
   718      sub_test.go:NNN: an error
   719      sub_test.go:NNN: skipped`,
   720  		f: func(b *B) {
   721  			b.Error("an error")
   722  			b.Skip("skipped")
   723  		},
   724  	}, {
   725  		desc: "memory allocation",
   726  		f: func(b *B) {
   727  			const bufSize = 256
   728  			alloc := func(b *B) {
   729  				var buf [bufSize]byte
   730  				for i := 0; i < b.N; i++ {
   731  					_ = append([]byte(nil), buf[:]...)
   732  				}
   733  			}
   734  			b.Run("", func(b *B) {
   735  				alloc(b)
   736  				b.ReportAllocs()
   737  			})
   738  			b.Run("", func(b *B) {
   739  				alloc(b)
   740  				b.ReportAllocs()
   741  			})
   742  			// runtime.MemStats sometimes reports more allocations than the
   743  			// benchmark is responsible for. Luckily the point of this test is
   744  			// to ensure that the results are not underreported, so we can
   745  			// simply verify the lower bound.
   746  			if got := b.result.MemAllocs; got < 2 {
   747  				t.Errorf("MemAllocs was %v; want 2", got)
   748  			}
   749  			if got := b.result.MemBytes; got < 2*bufSize {
   750  				t.Errorf("MemBytes was %v; want %v", got, 2*bufSize)
   751  			}
   752  		},
   753  	}, {
   754  		desc: "cleanup is called",
   755  		f: func(b *B) {
   756  			var calls, cleanups, innerCalls, innerCleanups int
   757  			b.Run("", func(b *B) {
   758  				calls++
   759  				b.Cleanup(func() {
   760  					cleanups++
   761  				})
   762  				b.Run("", func(b *B) {
   763  					b.Cleanup(func() {
   764  						innerCleanups++
   765  					})
   766  					innerCalls++
   767  				})
   768  				work(b)
   769  			})
   770  			if calls == 0 || calls != cleanups {
   771  				t.Errorf("mismatched cleanups; got %d want %d", cleanups, calls)
   772  			}
   773  			if innerCalls == 0 || innerCalls != innerCleanups {
   774  				t.Errorf("mismatched cleanups; got %d want %d", cleanups, calls)
   775  			}
   776  		},
   777  	}, {
   778  		desc:   "cleanup is called on failure",
   779  		failed: true,
   780  		f: func(b *B) {
   781  			var calls, cleanups int
   782  			b.Run("", func(b *B) {
   783  				calls++
   784  				b.Cleanup(func() {
   785  					cleanups++
   786  				})
   787  				b.Fatalf("failure")
   788  			})
   789  			if calls == 0 || calls != cleanups {
   790  				t.Errorf("mismatched cleanups; got %d want %d", cleanups, calls)
   791  			}
   792  		},
   793  	}}
   794  	hideStdoutForTesting = true
   795  	defer func() {
   796  		hideStdoutForTesting = false
   797  	}()
   798  	for _, tc := range testCases {
   799  		t.Run(tc.desc, func(t *T) {
   800  			var ok bool
   801  			buf := &strings.Builder{}
   802  			// This is almost like the Benchmark function, except that we override
   803  			// the benchtime and catch the failure result of the subbenchmark.
   804  			root := &B{
   805  				common: common{
   806  					signal: make(chan bool),
   807  					name:   "root",
   808  					w:      buf,
   809  				},
   810  				benchFunc: func(b *B) { ok = b.Run("test", tc.f) }, // Use Run to catch failure.
   811  				benchTime: durationOrCountFlag{d: 1 * time.Microsecond},
   812  			}
   813  			if tc.chatty {
   814  				root.chatty = newChattyPrinter(root.w)
   815  			}
   816  			root.runN(1)
   817  			if ok != !tc.failed {
   818  				t.Errorf("%s:ok: got %v; want %v", tc.desc, ok, !tc.failed)
   819  			}
   820  			if !ok != root.Failed() {
   821  				t.Errorf("%s:root failed: got %v; want %v", tc.desc, !ok, root.Failed())
   822  			}
   823  			// All tests are run as subtests
   824  			if root.result.N != 1 {
   825  				t.Errorf("%s: N for parent benchmark was %d; want 1", tc.desc, root.result.N)
   826  			}
   827  			got := strings.TrimSpace(buf.String())
   828  			want := strings.TrimSpace(tc.output)
   829  			re := makeRegexp(want)
   830  			if ok, err := regexp.MatchString(re, got); !ok || err != nil {
   831  				t.Errorf("%s:output:\ngot:\n%s\nwant:\n%s", tc.desc, got, want)
   832  			}
   833  		})
   834  	}
   835  }
   836  
   837  // makeRegexp transforms a line in the text notation to a pattern.
   838  func makeRegexp(s string) string {
   839  	s = regexp.QuoteMeta(s)
   840  	s = strings.ReplaceAll(s, "^V", string(markFraming))
   841  	s = strings.ReplaceAll(s, "^O", string(markErrBegin))
   842  	s = strings.ReplaceAll(s, "^N", string(markErrEnd))
   843  	s = strings.ReplaceAll(s, "^\\[", string(markEscape))
   844  	s = strings.ReplaceAll(s, ":NNN:", `:\d\d\d\d?:`)
   845  	s = strings.ReplaceAll(s, "N\\.NNs", `\d*\.\d*s`)
   846  	return s
   847  }
   848  
   849  // notateOutput transforms an output line into something more easily comparable
   850  // to text notation.
   851  func notateOutput(s string) string {
   852  	s = strings.ReplaceAll(s, string(markFraming), "^V")
   853  	s = strings.ReplaceAll(s, string(markErrBegin), "^O")
   854  	s = strings.ReplaceAll(s, string(markErrEnd), "^N")
   855  	s = strings.ReplaceAll(s, string(markEscape), "^[")
   856  	return s
   857  }
   858  
   859  func TestBenchmarkOutput(t *T) {
   860  	// Ensure Benchmark initialized common.w by invoking it with an error and
   861  	// normal case.
   862  	Benchmark(func(b *B) { b.Error("do not print this output") })
   863  	Benchmark(func(b *B) {})
   864  }
   865  
   866  func TestBenchmarkStartsFrom1(t *T) {
   867  	first := true
   868  	Benchmark(func(b *B) {
   869  		if first && b.N != 1 {
   870  			panic(fmt.Sprintf("Benchmark() first N=%v; want 1", b.N))
   871  		}
   872  		first = false
   873  	})
   874  }
   875  
   876  func TestBenchmarkReadMemStatsBeforeFirstRun(t *T) {
   877  	first := true
   878  	Benchmark(func(b *B) {
   879  		if first && (b.startAllocs == 0 || b.startBytes == 0) {
   880  			panic("ReadMemStats not called before first run")
   881  		}
   882  		first = false
   883  	})
   884  }
   885  
   886  type funcWriter struct {
   887  	write func([]byte) (int, error)
   888  }
   889  
   890  func (fw *funcWriter) Write(b []byte) (int, error) {
   891  	return fw.write(b)
   892  }
   893  
   894  func TestRacyOutput(t *T) {
   895  	var runs int32  // The number of running Writes
   896  	var races int32 // Incremented for each race detected
   897  	raceDetector := func(b []byte) (int, error) {
   898  		// Check if some other goroutine is concurrently calling Write.
   899  		if atomic.LoadInt32(&runs) > 0 {
   900  			atomic.AddInt32(&races, 1) // Race detected!
   901  		}
   902  		atomic.AddInt32(&runs, 1)
   903  		defer atomic.AddInt32(&runs, -1)
   904  		runtime.Gosched() // Increase probability of a race
   905  		return len(b), nil
   906  	}
   907  
   908  	root := &T{
   909  		common: common{w: &funcWriter{raceDetector}},
   910  		tstate: newTestState(1, allMatcher()),
   911  	}
   912  	root.chatty = newChattyPrinter(root.w)
   913  	root.Run("", func(t *T) {
   914  		var wg sync.WaitGroup
   915  		for i := 0; i < 100; i++ {
   916  			wg.Add(1)
   917  			go func(i int) {
   918  				defer wg.Done()
   919  				t.Run(fmt.Sprint(i), func(t *T) {
   920  					t.Logf("testing run %d", i)
   921  				})
   922  			}(i)
   923  		}
   924  		wg.Wait()
   925  	})
   926  
   927  	if races > 0 {
   928  		t.Errorf("detected %d racy Writes", races)
   929  	}
   930  }
   931  
   932  // The late log message did not include the test name.  Issue 29388.
   933  func TestLogAfterComplete(t *T) {
   934  	tstate := newTestState(1, allMatcher())
   935  	var buf bytes.Buffer
   936  	t1 := &T{
   937  		common: common{
   938  			// Use a buffered channel so that tRunner can write
   939  			// to it although nothing is reading from it.
   940  			signal: make(chan bool, 1),
   941  			w:      &buf,
   942  		},
   943  		tstate: tstate,
   944  	}
   945  
   946  	c1 := make(chan bool)
   947  	c2 := make(chan string)
   948  	tRunner(t1, func(t *T) {
   949  		t.Run("TestLateLog", func(t *T) {
   950  			go func() {
   951  				defer close(c2)
   952  				defer func() {
   953  					p := recover()
   954  					if p == nil {
   955  						c2 <- "subtest did not panic"
   956  						return
   957  					}
   958  					s, ok := p.(string)
   959  					if !ok {
   960  						c2 <- fmt.Sprintf("subtest panic with unexpected value %v of type %T", p, p)
   961  						return
   962  					}
   963  					const want = "Log in goroutine after TestLateLog has completed: log after test"
   964  					if !strings.Contains(s, want) {
   965  						c2 <- fmt.Sprintf("subtest panic %q does not contain %q", s, want)
   966  					}
   967  				}()
   968  
   969  				<-c1
   970  				t.Log("log after test")
   971  			}()
   972  		})
   973  	})
   974  	close(c1)
   975  
   976  	if s := <-c2; s != "" {
   977  		t.Error(s)
   978  	}
   979  }
   980  
   981  func TestBenchmark(t *T) {
   982  	if Short() {
   983  		t.Skip("skipping in short mode")
   984  	}
   985  	res := Benchmark(func(b *B) {
   986  		for i := 0; i < 5; i++ {
   987  			b.Run("", func(b *B) {
   988  				for i := 0; i < b.N; i++ {
   989  					time.Sleep(time.Millisecond)
   990  				}
   991  			})
   992  		}
   993  	})
   994  	if res.NsPerOp() < 4000000 {
   995  		t.Errorf("want >5ms; got %v", time.Duration(res.NsPerOp()))
   996  	}
   997  }
   998  
   999  func TestCleanup(t *T) {
  1000  	var cleanups []int
  1001  	t.Run("test", func(t *T) {
  1002  		t.Cleanup(func() { cleanups = append(cleanups, 1) })
  1003  		t.Cleanup(func() { cleanups = append(cleanups, 2) })
  1004  	})
  1005  	if got, want := cleanups, []int{2, 1}; !slices.Equal(got, want) {
  1006  		t.Errorf("unexpected cleanup record; got %v want %v", got, want)
  1007  	}
  1008  }
  1009  
  1010  func TestConcurrentCleanup(t *T) {
  1011  	cleanups := 0
  1012  	t.Run("test", func(t *T) {
  1013  		var wg sync.WaitGroup
  1014  		wg.Add(2)
  1015  		for i := 0; i < 2; i++ {
  1016  			go func() {
  1017  				t.Cleanup(func() {
  1018  					// Although the calls to Cleanup are concurrent, the functions passed
  1019  					// to Cleanup should be called sequentially, in some nondeterministic
  1020  					// order based on when the Cleanup calls happened to be scheduled.
  1021  					// So these assignments to the cleanups variable should not race.
  1022  					cleanups |= 1 << i
  1023  				})
  1024  				wg.Done()
  1025  			}()
  1026  		}
  1027  		wg.Wait()
  1028  	})
  1029  	if cleanups != 1|2 {
  1030  		t.Errorf("unexpected cleanup; got %d want 3", cleanups)
  1031  	}
  1032  }
  1033  
  1034  func TestCleanupCalledEvenAfterGoexit(t *T) {
  1035  	cleanups := 0
  1036  	t.Run("test", func(t *T) {
  1037  		t.Cleanup(func() {
  1038  			cleanups++
  1039  		})
  1040  		t.Cleanup(func() {
  1041  			runtime.Goexit()
  1042  		})
  1043  	})
  1044  	if cleanups != 1 {
  1045  		t.Errorf("unexpected cleanup count; got %d want 1", cleanups)
  1046  	}
  1047  }
  1048  
  1049  func TestRunCleanup(t *T) {
  1050  	outerCleanup := 0
  1051  	innerCleanup := 0
  1052  	t.Run("test", func(t *T) {
  1053  		t.Cleanup(func() { outerCleanup++ })
  1054  		t.Run("x", func(t *T) {
  1055  			t.Cleanup(func() { innerCleanup++ })
  1056  		})
  1057  	})
  1058  	if innerCleanup != 1 {
  1059  		t.Errorf("unexpected inner cleanup count; got %d want 1", innerCleanup)
  1060  	}
  1061  	if outerCleanup != 1 {
  1062  		t.Errorf("unexpected outer cleanup count; got %d want 0", outerCleanup)
  1063  	}
  1064  }
  1065  
  1066  func TestCleanupParallelSubtests(t *T) {
  1067  	ranCleanup := 0
  1068  	t.Run("test", func(t *T) {
  1069  		t.Cleanup(func() { ranCleanup++ })
  1070  		t.Run("x", func(t *T) {
  1071  			t.Parallel()
  1072  			if ranCleanup > 0 {
  1073  				t.Error("outer cleanup ran before parallel subtest")
  1074  			}
  1075  		})
  1076  	})
  1077  	if ranCleanup != 1 {
  1078  		t.Errorf("unexpected cleanup count; got %d want 1", ranCleanup)
  1079  	}
  1080  }
  1081  
  1082  func TestNestedCleanup(t *T) {
  1083  	ranCleanup := 0
  1084  	t.Run("test", func(t *T) {
  1085  		t.Cleanup(func() {
  1086  			if ranCleanup != 2 {
  1087  				t.Errorf("unexpected cleanup count in first cleanup: got %d want 2", ranCleanup)
  1088  			}
  1089  			ranCleanup++
  1090  		})
  1091  		t.Cleanup(func() {
  1092  			if ranCleanup != 0 {
  1093  				t.Errorf("unexpected cleanup count in second cleanup: got %d want 0", ranCleanup)
  1094  			}
  1095  			ranCleanup++
  1096  			t.Cleanup(func() {
  1097  				if ranCleanup != 1 {
  1098  					t.Errorf("unexpected cleanup count in nested cleanup: got %d want 1", ranCleanup)
  1099  				}
  1100  				ranCleanup++
  1101  			})
  1102  		})
  1103  	})
  1104  	if ranCleanup != 3 {
  1105  		t.Errorf("unexpected cleanup count: got %d want 3", ranCleanup)
  1106  	}
  1107  }
  1108  
  1109  // TestOutput checks that log messages are written,
  1110  // formatted and buffered as expected by Output. It
  1111  // checks both the chatty and non-chatty cases.
  1112  func TestOutput(t *T) {
  1113  	tstate := newTestState(1, allMatcher())
  1114  	root := &T{
  1115  		tstate: tstate,
  1116  	}
  1117  	root.setOutputWriter()
  1118  	o := root.Output()
  1119  
  1120  	// Chatty case
  1121  	tstateChatty := newTestState(1, allMatcher())
  1122  	bufChatty := &strings.Builder{}
  1123  	rootChatty := &T{
  1124  		common: common{
  1125  			w: bufChatty,
  1126  		},
  1127  		tstate: tstateChatty,
  1128  	}
  1129  	rootChatty.setOutputWriter()
  1130  	rootChatty.chatty = newChattyPrinter(rootChatty.w)
  1131  	oChatty := rootChatty.Output()
  1132  
  1133  	testCases := []struct {
  1134  		in  string
  1135  		out string
  1136  		buf string
  1137  	}{{
  1138  		in:  "a",
  1139  		out: "",
  1140  		buf: "a",
  1141  	}, {
  1142  		in:  "b",
  1143  		out: "",
  1144  		buf: "ab",
  1145  	}, {
  1146  		in:  "\n",
  1147  		out: "    ab\n",
  1148  		buf: "",
  1149  	}, {
  1150  		in:  "\nc",
  1151  		out: "    ab\n    \n",
  1152  		buf: "c",
  1153  	}, {
  1154  		in:  "d",
  1155  		out: "    ab\n    \n",
  1156  		buf: "cd",
  1157  	}}
  1158  	for _, tc := range testCases {
  1159  		o.Write([]byte(tc.in))
  1160  		if string(root.output) != tc.out {
  1161  			t.Errorf("output:\ngot:\n%s\nwant:\n%s", root.output, tc.out)
  1162  		}
  1163  		if string(root.o.partial) != tc.buf {
  1164  			t.Errorf("buffer:\ngot:\n%s\nwant:\n%s", root.o.partial, tc.buf)
  1165  		}
  1166  
  1167  		// Chatty case
  1168  		oChatty.Write([]byte(tc.in))
  1169  		if got := bufChatty.String(); got != tc.out {
  1170  			t.Errorf("output:\ngot:\n%s\nwant:\n%s", got, tc.out)
  1171  		}
  1172  	}
  1173  }
  1174  
  1175  // TestOutputAfterComplete ensures that Output panics
  1176  // if called after a test function returns.
  1177  func TestOutputAfterComplete(t *T) {
  1178  	tstate := newTestState(1, allMatcher())
  1179  	var buf bytes.Buffer
  1180  	t1 := &T{
  1181  		common: common{
  1182  			// Use a buffered channel so that tRunner can write
  1183  			// to it although nothing is reading from it.
  1184  			signal: make(chan bool, 1),
  1185  			w:      &buf,
  1186  		},
  1187  		tstate: tstate,
  1188  	}
  1189  
  1190  	c1 := make(chan bool)
  1191  	c2 := make(chan string)
  1192  	tRunner(t1, func(t *T) {
  1193  		t.Run("TestLateOutput", func(t *T) {
  1194  			go func() {
  1195  				defer close(c2)
  1196  				defer func() {
  1197  					p := recover()
  1198  					if p == nil {
  1199  						c2 <- "subtest did not panic"
  1200  						return
  1201  					}
  1202  					s, ok := p.(string)
  1203  					if !ok {
  1204  						c2 <- fmt.Sprintf("subtest panic with unexpected value %v of type %T", p, p)
  1205  						return
  1206  					}
  1207  					const want = "Output called after TestLateOutput has completed"
  1208  					if !strings.Contains(s, want) {
  1209  						c2 <- fmt.Sprintf("subtest panic %q does not contain %q", s, want)
  1210  					}
  1211  				}()
  1212  
  1213  				<-c1
  1214  				t.Output()
  1215  			}()
  1216  		})
  1217  	})
  1218  	close(c1)
  1219  
  1220  	if s := <-c2; s != "" {
  1221  		t.Error(s)
  1222  	}
  1223  }
  1224  
  1225  // TestOutputWriteAfterComplete ensures that Write panics
  1226  // if called on t.Output() of a finished test t.
  1227  func TestOutputWriteAfterComplete(t *T) {
  1228  	tstate := newTestState(1, allMatcher())
  1229  	var buf bytes.Buffer
  1230  	t1 := &T{
  1231  		common: common{
  1232  			// Use a buffered channel so that tRunner can write
  1233  			// to it although nothing is reading from it.
  1234  			signal: make(chan bool, 1),
  1235  			w:      &buf,
  1236  		},
  1237  		tstate: tstate,
  1238  	}
  1239  
  1240  	c1 := make(chan bool)
  1241  	c2 := make(chan string)
  1242  	tRunner(t1, func(t *T) {
  1243  		t.Run("TestLateWrite", func(t *T) {
  1244  			o := t.Output()
  1245  			go func() {
  1246  				defer close(c2)
  1247  				defer func() {
  1248  					p := recover()
  1249  					if p == nil {
  1250  						c2 <- "subtest did not panic"
  1251  						return
  1252  					}
  1253  					s, ok := p.(string)
  1254  					if !ok {
  1255  						c2 <- fmt.Sprintf("subtest panic with unexpected value %v of type %T", p, p)
  1256  						return
  1257  					}
  1258  					const want = "Write called after TestLateWrite has completed"
  1259  					if !strings.Contains(s, want) {
  1260  						c2 <- fmt.Sprintf("subtest panic %q does not contain %q", s, want)
  1261  					}
  1262  				}()
  1263  
  1264  				<-c1
  1265  				o.Write([]byte("write after test"))
  1266  			}()
  1267  		})
  1268  	})
  1269  	close(c1)
  1270  
  1271  	if s := <-c2; s != "" {
  1272  		t.Error(s)
  1273  	}
  1274  }
  1275  
  1276  // Verify that logging to an inactive top-level testing.T does not panic.
  1277  // These tests can run in either order.
  1278  
  1279  func TestOutputEscape1(t *T) { testOutputEscape(t) }
  1280  func TestOutputEscape2(t *T) { testOutputEscape(t) }
  1281  
  1282  var global *T
  1283  
  1284  func testOutputEscape(t *T) {
  1285  	if global == nil {
  1286  		// Store t in a global, to set up for the second execution.
  1287  		global = t
  1288  	} else {
  1289  		// global is inactive here.
  1290  		global.Log("hello")
  1291  	}
  1292  }
  1293  

View as plain text