1 // Copyright 2016 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
19 // Make benchmark tests run 10* faster.
20 *benchTime
= 100 * time
.Millisecond
23 func TestTestContext(t
*T
) {
28 // After each of the calls are applied to the context, the
30 typ
int // run or done
31 // result from applying the call
36 testCases
:= []struct {
42 {typ
: add1
, running
: 1, waiting
: 0, started
: true},
43 {typ
: done
, running
: 0, waiting
: 0, started
: false},
48 {typ
: add1
, running
: 1, waiting
: 0, started
: true},
49 {typ
: add1
, running
: 1, waiting
: 1, started
: false},
50 {typ
: done
, running
: 1, waiting
: 0, started
: true},
51 {typ
: done
, running
: 0, waiting
: 0, started
: false},
52 {typ
: add1
, running
: 1, waiting
: 0, started
: true},
57 {typ
: add1
, running
: 1, waiting
: 0, started
: true},
58 {typ
: add1
, running
: 2, waiting
: 0, started
: true},
59 {typ
: add1
, running
: 3, waiting
: 0, started
: true},
60 {typ
: add1
, running
: 3, waiting
: 1, started
: false},
61 {typ
: add1
, running
: 3, waiting
: 2, started
: false},
62 {typ
: add1
, running
: 3, waiting
: 3, started
: false},
63 {typ
: done
, running
: 3, waiting
: 2, started
: true},
64 {typ
: add1
, running
: 3, waiting
: 3, started
: false},
65 {typ
: done
, running
: 3, waiting
: 2, started
: true},
66 {typ
: done
, running
: 3, waiting
: 1, started
: true},
67 {typ
: done
, running
: 3, waiting
: 0, started
: true},
68 {typ
: done
, running
: 2, waiting
: 0, started
: false},
69 {typ
: done
, running
: 1, waiting
: 0, started
: false},
70 {typ
: done
, running
: 0, waiting
: 0, started
: false},
73 for i
, tc
:= range testCases
{
75 startParallel
: make(chan bool),
78 for j
, call
:= range tc
.run
{
79 doCall
:= func(f
func()) chan bool {
80 done
:= make(chan bool)
90 signal
:= doCall(ctx
.waitParallel
)
94 case ctx
.startParallel
<- true:
98 signal
:= doCall(ctx
.release
)
101 case <-ctx
.startParallel
:
106 if started
!= call
.started
{
107 t
.Errorf("%d:%d:started: got %v; want %v", i
, j
, started
, call
.started
)
109 if ctx
.running
!= call
.running
{
110 t
.Errorf("%d:%d:running: got %v; want %v", i
, j
, ctx
.running
, call
.running
)
112 if ctx
.numWaiting
!= call
.waiting
{
113 t
.Errorf("%d:%d:waiting: got %v; want %v", i
, j
, ctx
.numWaiting
, call
.waiting
)
119 func TestTRun(t
*T
) {
121 testCases
:= []struct {
129 desc
: "failnow skips future sequential and parallel tests at same level",
133 --- FAIL: failnow skips future sequential and parallel tests at same level (N.NNs)
134 --- FAIL: failnow skips future sequential and parallel tests at same level/#00 (N.NNs)
139 t
.Run("", func(t
*T
) {
140 t
.Run("par", func(t
*T
) {
144 t
.Run("seq", func(t
*T
) {
148 t
.Run("seq", func(t
*T
) {
149 realTest
.Error("test must be skipped")
151 t
.Run("par", func(t
*T
) {
153 realTest
.Error("test must be skipped.")
157 realTest
.Error("parallel test was not run")
160 realTest
.Error("sequential test was not run")
164 desc
: "failure in parallel test propagates upwards",
168 --- FAIL: failure in parallel test propagates upwards (N.NNs)
169 --- FAIL: failure in parallel test propagates upwards/#00 (N.NNs)
170 --- FAIL: failure in parallel test propagates upwards/#00/par (N.NNs)
173 t
.Run("", func(t
*T
) {
175 t
.Run("par", func(t
*T
) {
182 desc
: "skipping without message, chatty",
186 === RUN skipping without message, chatty
187 --- SKIP: skipping without message, chatty (N.NNs)`,
188 f
: func(t
*T
) { t
.SkipNow() },
190 desc
: "chatty with recursion",
194 === RUN chatty with recursion
195 === RUN chatty with recursion/#00
196 === RUN chatty with recursion/#00/#00
197 --- PASS: chatty with recursion (N.NNs)
198 --- PASS: chatty with recursion/#00 (N.NNs)
199 --- PASS: chatty with recursion/#00/#00 (N.NNs)`,
201 t
.Run("", func(t
*T
) {
202 t
.Run("", func(t
*T
) {})
206 desc
: "skipping without message, not chatty",
208 f
: func(t
*T
) { t
.SkipNow() },
210 desc
: "skipping after error",
212 --- FAIL: skipping after error (N.NNs)
213 sub_test.go:NNN: an error
214 sub_test.go:NNN: skipped`,
220 desc
: "use Run to locally synchronize parallelism",
225 t
.Run("waitGroup", func(t
*T
) {
226 for i
:= 0; i
< 4; i
++ {
227 t
.Run("par", func(t
*T
) {
229 atomic
.AddUint32(&count
, 1)
234 t
.Errorf("count was %d; want 4", count
)
238 desc
: "alternate sequential and parallel",
239 // Sequential tests should partake in the counting of running threads.
240 // Otherwise, if one runs parallel subtests in sequential tests that are
241 // itself subtests of parallel tests, the counts can get askew.
245 t
.Run("a", func(t
*T
) {
247 t
.Run("b", func(t
*T
) {
248 // Sequential: ensure running count is decremented.
249 t
.Run("c", func(t
*T
) {
257 desc
: "alternate sequential and parallel 2",
258 // Sequential tests should partake in the counting of running threads.
259 // Otherwise, if one runs parallel subtests in sequential tests that are
260 // itself subtests of parallel tests, the counts can get askew.
264 for i
:= 0; i
< 2; i
++ {
265 t
.Run("a", func(t
*T
) {
267 time
.Sleep(time
.Nanosecond
)
268 for i
:= 0; i
< 2; i
++ {
269 t
.Run("b", func(t
*T
) {
270 time
.Sleep(time
.Nanosecond
)
271 for i
:= 0; i
< 2; i
++ {
272 t
.Run("c", func(t
*T
) {
274 time
.Sleep(time
.Nanosecond
)
289 for i
:= 0; i
< 12; i
++ {
290 t
.Run("a", func(t
*T
) {
292 time
.Sleep(time
.Nanosecond
)
293 for i
:= 0; i
< 12; i
++ {
294 t
.Run("b", func(t
*T
) {
295 time
.Sleep(time
.Nanosecond
)
296 for i
:= 0; i
< 12; i
++ {
297 t
.Run("c", func(t
*T
) {
299 time
.Sleep(time
.Nanosecond
)
300 t
.Run("d1", func(t
*T
) {})
301 t
.Run("d2", func(t
*T
) {})
302 t
.Run("d3", func(t
*T
) {})
303 t
.Run("d4", func(t
*T
) {})
319 desc
: "subtest calls error on parent",
322 --- FAIL: subtest calls error on parent (N.NNs)
323 sub_test.go:NNN: first this
324 sub_test.go:NNN: and now this!
325 sub_test.go:NNN: oh, and this too`,
328 t
.Errorf("first this")
330 t
.Run("", func(t
*T
) {
331 outer
.Errorf("and now this!")
333 t
.Errorf("oh, and this too")
336 desc
: "subtest calls fatal on parent",
339 --- FAIL: subtest calls fatal on parent (N.NNs)
340 sub_test.go:NNN: first this
341 sub_test.go:NNN: and now this!
342 --- FAIL: subtest calls fatal on parent/#00 (N.NNs)
343 testing.go:NNN: test executed panic(nil) or runtime.Goexit: subtest may have called FailNow on a parent test`,
347 t
.Errorf("first this")
348 t
.Run("", func(t
*T
) {
349 outer
.Fatalf("and now this!")
351 t
.Errorf("Should not reach here.")
354 desc
: "subtest calls error on ancestor",
357 --- FAIL: subtest calls error on ancestor (N.NNs)
358 sub_test.go:NNN: Report to ancestor
359 --- FAIL: subtest calls error on ancestor/#00 (N.NNs)
360 sub_test.go:NNN: Still do this
361 sub_test.go:NNN: Also do this`,
365 t
.Run("", func(t
*T
) {
366 t
.Run("", func(t
*T
) {
367 outer
.Errorf("Report to ancestor")
369 t
.Errorf("Still do this")
371 t
.Errorf("Also do this")
374 desc
: "subtest calls fatal on ancestor",
377 --- FAIL: subtest calls fatal on ancestor (N.NNs)
378 sub_test.go:NNN: Nope`,
382 t
.Run("", func(t
*T
) {
383 for i
:= 0; i
< 4; i
++ {
384 t
.Run("", func(t
*T
) {
387 t
.Errorf("Don't do this")
389 t
.Errorf("And neither do this")
394 desc
: "panic on goroutine fail after test exit",
398 ch
:= make(chan bool)
399 t
.Run("", func(t
*T
) {
403 if r
:= recover(); r
== nil {
404 realTest
.Errorf("expected panic")
408 t
.Errorf("failed after success")
415 for _
, tc
:= range testCases
{
416 ctx
:= newTestContext(tc
.maxPar
, newMatcher(regexp
.MatchString
, "", ""))
417 buf
:= &bytes
.Buffer
{}
420 signal
: make(chan bool),
427 ok
:= root
.Run(tc
.desc
, tc
.f
)
431 t
.Errorf("%s:ok: got %v; want %v", tc
.desc
, ok
, tc
.ok
)
433 if ok
!= !root
.Failed() {
434 t
.Errorf("%s:root failed: got %v; want %v", tc
.desc
, !ok
, root
.Failed())
436 if ctx
.running
!= 0 || ctx
.numWaiting
!= 0 {
437 t
.Errorf("%s:running and waiting non-zero: got %d and %d", tc
.desc
, ctx
.running
, ctx
.numWaiting
)
439 got
:= strings
.TrimSpace(buf
.String())
440 want
:= strings
.TrimSpace(tc
.output
)
441 re
:= makeRegexp(want
)
442 if ok
, err
:= regexp
.MatchString(re
, got
); !ok || err
!= nil {
443 t
.Errorf("%s:output:\ngot:\n%s\nwant:\n%s", tc
.desc
, got
, want
)
448 func TestBRun(t
*T
) {
450 for i
:= 0; i
< b
.N
; i
++ {
451 time
.Sleep(time
.Nanosecond
)
454 testCases
:= []struct {
461 desc
: "simulate sequential run of subbenchmarks.",
463 b
.Run("", func(b
*B
) { work(b
) })
464 time1
:= b
.result
.NsPerOp()
465 b
.Run("", func(b
*B
) { work(b
) })
466 time2
:= b
.result
.NsPerOp()
468 t
.Errorf("no time spent in benchmark t1 >= t2 (%d >= %d)", time1
, time2
)
472 desc
: "bytes set by all benchmarks",
474 b
.Run("", func(b
*B
) { b
.SetBytes(10); work(b
) })
475 b
.Run("", func(b
*B
) { b
.SetBytes(10); work(b
) })
476 if b
.result
.Bytes
!= 20 {
477 t
.Errorf("bytes: got: %d; want 20", b
.result
.Bytes
)
481 desc
: "bytes set by some benchmarks",
482 // In this case the bytes result is meaningless, so it must be 0.
484 b
.Run("", func(b
*B
) { b
.SetBytes(10); work(b
) })
485 b
.Run("", func(b
*B
) { work(b
) })
486 b
.Run("", func(b
*B
) { b
.SetBytes(10); work(b
) })
487 if b
.result
.Bytes
!= 0 {
488 t
.Errorf("bytes: got: %d; want 0", b
.result
.Bytes
)
492 desc
: "failure carried over to root",
494 output
: "--- FAIL: root",
495 f
: func(b
*B
) { b
.Fail() },
497 desc
: "skipping without message, chatty",
499 output
: "--- SKIP: root",
500 f
: func(b
*B
) { b
.SkipNow() },
502 desc
: "skipping with message, chatty",
506 sub_test.go:NNN: skipping`,
507 f
: func(b
*B
) { b
.Skip("skipping") },
509 desc
: "chatty with recursion",
512 b
.Run("", func(b
*B
) {
513 b
.Run("", func(b
*B
) {})
517 desc
: "skipping without message, not chatty",
518 f
: func(b
*B
) { b
.SkipNow() },
520 desc
: "skipping after error",
524 sub_test.go:NNN: an error
525 sub_test.go:NNN: skipped`,
531 desc
: "memory allocation",
534 alloc
:= func(b
*B
) {
535 var buf
[bufSize
]byte
536 for i
:= 0; i
< b
.N
; i
++ {
537 _
= append([]byte(nil), buf
[:]...)
540 b
.Run("", func(b
*B
) {
544 b
.Run("", func(b
*B
) {
548 // runtime.MemStats sometimes reports more allocations than the
549 // benchmark is responsible for. Luckily the point of this test is
550 // to ensure that the results are not underreported, so we can
551 // simply verify the lower bound.
552 if got
:= b
.result
.MemAllocs
; got
< 2 {
553 t
.Errorf("MemAllocs was %v; want 2", got
)
555 if got
:= b
.result
.MemBytes
; got
< 2*bufSize
{
556 t
.Errorf("MemBytes was %v; want %v", got
, 2*bufSize
)
560 for _
, tc
:= range testCases
{
562 buf
:= &bytes
.Buffer
{}
563 // This is almost like the Benchmark function, except that we override
564 // the benchtime and catch the failure result of the subbenchmark.
567 signal
: make(chan bool),
572 benchFunc
: func(b
*B
) { ok
= b
.Run("test", tc
.f
) }, // Use Run to catch failure.
573 benchTime
: time
.Microsecond
,
576 if ok
!= !tc
.failed
{
577 t
.Errorf("%s:ok: got %v; want %v", tc
.desc
, ok
, !tc
.failed
)
579 if !ok
!= root
.Failed() {
580 t
.Errorf("%s:root failed: got %v; want %v", tc
.desc
, !ok
, root
.Failed())
582 // All tests are run as subtests
583 if root
.result
.N
!= 1 {
584 t
.Errorf("%s: N for parent benchmark was %d; want 1", tc
.desc
, root
.result
.N
)
586 got
:= strings
.TrimSpace(buf
.String())
587 want
:= strings
.TrimSpace(tc
.output
)
588 re
:= makeRegexp(want
)
589 if ok
, err
:= regexp
.MatchString(re
, got
); !ok || err
!= nil {
590 t
.Errorf("%s:output:\ngot:\n%s\nwant:\n%s", tc
.desc
, got
, want
)
595 func makeRegexp(s
string) string {
596 s
= regexp
.QuoteMeta(s
)
597 s
= strings
.Replace(s
, ":NNN:", `:\d\d\d:`, -1)
598 s
= strings
.Replace(s
, "N\\.NNs", `\d*\.\d*s`, -1)
602 func TestBenchmarkOutput(t
*T
) {
603 // Ensure Benchmark initialized common.w by invoking it with an error and
605 Benchmark(func(b
*B
) { b
.Error("do not print this output") })
606 Benchmark(func(b
*B
) {})
609 func TestBenchmarkStartsFrom1(t
*T
) {
611 Benchmark(func(b
*B
) {
612 if first
&& b
.N
!= 1 {
613 panic(fmt
.Sprintf("Benchmark() first N=%v; want 1", b
.N
))
619 func TestBenchmarkReadMemStatsBeforeFirstRun(t
*T
) {
621 Benchmark(func(b
*B
) {
622 if first
&& (b
.startAllocs
== 0 || b
.startBytes
== 0) {
623 panic(fmt
.Sprintf("ReadMemStats not called before first run"))
629 func TestParallelSub(t
*T
) {
631 block
:= make(chan int)
632 for i
:= 0; i
< 10; i
++ {
635 t
.Run(fmt
.Sprint(i
), func(t
*T
) {})
640 for i
:= 0; i
< 10; i
++ {
645 type funcWriter
func([]byte) (int, error
)
647 func (fw funcWriter
) Write(b
[]byte) (int, error
) { return fw(b
) }
649 func TestRacyOutput(t
*T
) {
650 var runs
int32 // The number of running Writes
651 var races
int32 // Incremented for each race detected
652 raceDetector
:= func(b
[]byte) (int, error
) {
653 // Check if some other goroutine is concurrently calling Write.
654 if atomic
.LoadInt32(&runs
) > 0 {
655 atomic
.AddInt32(&races
, 1) // Race detected!
657 atomic
.AddInt32(&runs
, 1)
658 defer atomic
.AddInt32(&runs
, -1)
659 runtime
.Gosched() // Increase probability of a race
663 var wg sync
.WaitGroup
665 common
: common
{w
: funcWriter(raceDetector
), chatty
: true},
666 context
: newTestContext(1, newMatcher(regexp
.MatchString
, "", "")),
668 root
.Run("", func(t
*T
) {
669 for i
:= 0; i
< 100; i
++ {
673 t
.Run(fmt
.Sprint(i
), func(t
*T
) {
674 t
.Logf("testing run %d", i
)
682 t
.Errorf("detected %d racy Writes", races
)
686 func TestBenchmark(t
*T
) {
687 res
:= Benchmark(func(b
*B
) {
688 for i
:= 0; i
< 5; i
++ {
689 b
.Run("", func(b
*B
) {
690 for i
:= 0; i
< b
.N
; i
++ {
691 time
.Sleep(time
.Millisecond
)
696 if res
.NsPerOp() < 4000000 {
697 t
.Errorf("want >5ms; got %v", time
.Duration(res
.NsPerOp()))