libgo: update to Go 1.11
[official-gcc.git] / libgo / go / testing / sub_test.go
blob9af3909b3556e44c6db6870cb43f15c0e98bf6d2
1 // Copyright 2016 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
5 package testing
7 import (
8 "bytes"
9 "fmt"
10 "regexp"
11 "runtime"
12 "strings"
13 "sync"
14 "sync/atomic"
15 "time"
18 func init() {
19 // Make benchmark tests run 10* faster.
20 *benchTime = 100 * time.Millisecond
23 func TestTestContext(t *T) {
24 const (
25 add1 = 0
26 done = 1
28 // After each of the calls are applied to the context, the
29 type call struct {
30 typ int // run or done
31 // result from applying the call
32 running int
33 waiting int
34 started bool
36 testCases := []struct {
37 max int
38 run []call
39 }{{
40 max: 1,
41 run: []call{
42 {typ: add1, running: 1, waiting: 0, started: true},
43 {typ: done, running: 0, waiting: 0, started: false},
45 }, {
46 max: 1,
47 run: []call{
48 {typ: add1, running: 1, waiting: 0, started: true},
49 {typ: add1, running: 1, waiting: 1, started: false},
50 {typ: done, running: 1, waiting: 0, started: true},
51 {typ: done, running: 0, waiting: 0, started: false},
52 {typ: add1, running: 1, waiting: 0, started: true},
54 }, {
55 max: 3,
56 run: []call{
57 {typ: add1, running: 1, waiting: 0, started: true},
58 {typ: add1, running: 2, waiting: 0, started: true},
59 {typ: add1, running: 3, waiting: 0, started: true},
60 {typ: add1, running: 3, waiting: 1, started: false},
61 {typ: add1, running: 3, waiting: 2, started: false},
62 {typ: add1, running: 3, waiting: 3, started: false},
63 {typ: done, running: 3, waiting: 2, started: true},
64 {typ: add1, running: 3, waiting: 3, started: false},
65 {typ: done, running: 3, waiting: 2, started: true},
66 {typ: done, running: 3, waiting: 1, started: true},
67 {typ: done, running: 3, waiting: 0, started: true},
68 {typ: done, running: 2, waiting: 0, started: false},
69 {typ: done, running: 1, waiting: 0, started: false},
70 {typ: done, running: 0, waiting: 0, started: false},
73 for i, tc := range testCases {
74 ctx := &testContext{
75 startParallel: make(chan bool),
76 maxParallel: tc.max,
78 for j, call := range tc.run {
79 doCall := func(f func()) chan bool {
80 done := make(chan bool)
81 go func() {
82 f()
83 done <- true
84 }()
85 return done
87 started := false
88 switch call.typ {
89 case add1:
90 signal := doCall(ctx.waitParallel)
91 select {
92 case <-signal:
93 started = true
94 case ctx.startParallel <- true:
95 <-signal
97 case done:
98 signal := doCall(ctx.release)
99 select {
100 case <-signal:
101 case <-ctx.startParallel:
102 started = true
103 <-signal
106 if started != call.started {
107 t.Errorf("%d:%d:started: got %v; want %v", i, j, started, call.started)
109 if ctx.running != call.running {
110 t.Errorf("%d:%d:running: got %v; want %v", i, j, ctx.running, call.running)
112 if ctx.numWaiting != call.waiting {
113 t.Errorf("%d:%d:waiting: got %v; want %v", i, j, ctx.numWaiting, call.waiting)
119 func TestTRun(t *T) {
120 realTest := t
121 testCases := []struct {
122 desc string
123 ok bool
124 maxPar int
125 chatty bool
126 output string
127 f func(*T)
129 desc: "failnow skips future sequential and parallel tests at same level",
130 ok: false,
131 maxPar: 1,
132 output: `
133 --- FAIL: failnow skips future sequential and parallel tests at same level (N.NNs)
134 --- FAIL: failnow skips future sequential and parallel tests at same level/#00 (N.NNs)
136 f: func(t *T) {
137 ranSeq := false
138 ranPar := false
139 t.Run("", func(t *T) {
140 t.Run("par", func(t *T) {
141 t.Parallel()
142 ranPar = true
144 t.Run("seq", func(t *T) {
145 ranSeq = true
147 t.FailNow()
148 t.Run("seq", func(t *T) {
149 realTest.Error("test must be skipped")
151 t.Run("par", func(t *T) {
152 t.Parallel()
153 realTest.Error("test must be skipped.")
156 if !ranPar {
157 realTest.Error("parallel test was not run")
159 if !ranSeq {
160 realTest.Error("sequential test was not run")
163 }, {
164 desc: "failure in parallel test propagates upwards",
165 ok: false,
166 maxPar: 1,
167 output: `
168 --- FAIL: failure in parallel test propagates upwards (N.NNs)
169 --- FAIL: failure in parallel test propagates upwards/#00 (N.NNs)
170 --- FAIL: failure in parallel test propagates upwards/#00/par (N.NNs)
172 f: func(t *T) {
173 t.Run("", func(t *T) {
174 t.Parallel()
175 t.Run("par", func(t *T) {
176 t.Parallel()
177 t.Fail()
181 }, {
182 desc: "skipping without message, chatty",
183 ok: true,
184 chatty: true,
185 output: `
186 === RUN skipping without message, chatty
187 --- SKIP: skipping without message, chatty (N.NNs)`,
188 f: func(t *T) { t.SkipNow() },
189 }, {
190 desc: "chatty with recursion",
191 ok: true,
192 chatty: true,
193 output: `
194 === RUN chatty with recursion
195 === RUN chatty with recursion/#00
196 === RUN chatty with recursion/#00/#00
197 --- PASS: chatty with recursion (N.NNs)
198 --- PASS: chatty with recursion/#00 (N.NNs)
199 --- PASS: chatty with recursion/#00/#00 (N.NNs)`,
200 f: func(t *T) {
201 t.Run("", func(t *T) {
202 t.Run("", func(t *T) {})
205 }, {
206 desc: "skipping without message, not chatty",
207 ok: true,
208 f: func(t *T) { t.SkipNow() },
209 }, {
210 desc: "skipping after error",
211 output: `
212 --- FAIL: skipping after error (N.NNs)
213 sub_test.go:NNN: an error
214 sub_test.go:NNN: skipped`,
215 f: func(t *T) {
216 t.Error("an error")
217 t.Skip("skipped")
219 }, {
220 desc: "use Run to locally synchronize parallelism",
221 ok: true,
222 maxPar: 1,
223 f: func(t *T) {
224 var count uint32
225 t.Run("waitGroup", func(t *T) {
226 for i := 0; i < 4; i++ {
227 t.Run("par", func(t *T) {
228 t.Parallel()
229 atomic.AddUint32(&count, 1)
233 if count != 4 {
234 t.Errorf("count was %d; want 4", count)
237 }, {
238 desc: "alternate sequential and parallel",
239 // Sequential tests should partake in the counting of running threads.
240 // Otherwise, if one runs parallel subtests in sequential tests that are
241 // itself subtests of parallel tests, the counts can get askew.
242 ok: true,
243 maxPar: 1,
244 f: func(t *T) {
245 t.Run("a", func(t *T) {
246 t.Parallel()
247 t.Run("b", func(t *T) {
248 // Sequential: ensure running count is decremented.
249 t.Run("c", func(t *T) {
250 t.Parallel()
256 }, {
257 desc: "alternate sequential and parallel 2",
258 // Sequential tests should partake in the counting of running threads.
259 // Otherwise, if one runs parallel subtests in sequential tests that are
260 // itself subtests of parallel tests, the counts can get askew.
261 ok: true,
262 maxPar: 2,
263 f: func(t *T) {
264 for i := 0; i < 2; i++ {
265 t.Run("a", func(t *T) {
266 t.Parallel()
267 time.Sleep(time.Nanosecond)
268 for i := 0; i < 2; i++ {
269 t.Run("b", func(t *T) {
270 time.Sleep(time.Nanosecond)
271 for i := 0; i < 2; i++ {
272 t.Run("c", func(t *T) {
273 t.Parallel()
274 time.Sleep(time.Nanosecond)
283 }, {
284 desc: "stress test",
285 ok: true,
286 maxPar: 4,
287 f: func(t *T) {
288 t.Parallel()
289 for i := 0; i < 12; i++ {
290 t.Run("a", func(t *T) {
291 t.Parallel()
292 time.Sleep(time.Nanosecond)
293 for i := 0; i < 12; i++ {
294 t.Run("b", func(t *T) {
295 time.Sleep(time.Nanosecond)
296 for i := 0; i < 12; i++ {
297 t.Run("c", func(t *T) {
298 t.Parallel()
299 time.Sleep(time.Nanosecond)
300 t.Run("d1", func(t *T) {})
301 t.Run("d2", func(t *T) {})
302 t.Run("d3", func(t *T) {})
303 t.Run("d4", func(t *T) {})
311 }, {
312 desc: "skip output",
313 ok: true,
314 maxPar: 4,
315 f: func(t *T) {
316 t.Skip()
318 }, {
319 desc: "subtest calls error on parent",
320 ok: false,
321 output: `
322 --- FAIL: subtest calls error on parent (N.NNs)
323 sub_test.go:NNN: first this
324 sub_test.go:NNN: and now this!
325 sub_test.go:NNN: oh, and this too`,
326 maxPar: 1,
327 f: func(t *T) {
328 t.Errorf("first this")
329 outer := t
330 t.Run("", func(t *T) {
331 outer.Errorf("and now this!")
333 t.Errorf("oh, and this too")
335 }, {
336 desc: "subtest calls fatal on parent",
337 ok: false,
338 output: `
339 --- FAIL: subtest calls fatal on parent (N.NNs)
340 sub_test.go:NNN: first this
341 sub_test.go:NNN: and now this!
342 --- FAIL: subtest calls fatal on parent/#00 (N.NNs)
343 testing.go:NNN: test executed panic(nil) or runtime.Goexit: subtest may have called FailNow on a parent test`,
344 maxPar: 1,
345 f: func(t *T) {
346 outer := t
347 t.Errorf("first this")
348 t.Run("", func(t *T) {
349 outer.Fatalf("and now this!")
351 t.Errorf("Should not reach here.")
353 }, {
354 desc: "subtest calls error on ancestor",
355 ok: false,
356 output: `
357 --- FAIL: subtest calls error on ancestor (N.NNs)
358 sub_test.go:NNN: Report to ancestor
359 --- FAIL: subtest calls error on ancestor/#00 (N.NNs)
360 sub_test.go:NNN: Still do this
361 sub_test.go:NNN: Also do this`,
362 maxPar: 1,
363 f: func(t *T) {
364 outer := t
365 t.Run("", func(t *T) {
366 t.Run("", func(t *T) {
367 outer.Errorf("Report to ancestor")
369 t.Errorf("Still do this")
371 t.Errorf("Also do this")
373 }, {
374 desc: "subtest calls fatal on ancestor",
375 ok: false,
376 output: `
377 --- FAIL: subtest calls fatal on ancestor (N.NNs)
378 sub_test.go:NNN: Nope`,
379 maxPar: 1,
380 f: func(t *T) {
381 outer := t
382 t.Run("", func(t *T) {
383 for i := 0; i < 4; i++ {
384 t.Run("", func(t *T) {
385 outer.Fatalf("Nope")
387 t.Errorf("Don't do this")
389 t.Errorf("And neither do this")
391 t.Errorf("Nor this")
393 }, {
394 desc: "panic on goroutine fail after test exit",
395 ok: false,
396 maxPar: 4,
397 f: func(t *T) {
398 ch := make(chan bool)
399 t.Run("", func(t *T) {
400 go func() {
401 <-ch
402 defer func() {
403 if r := recover(); r == nil {
404 realTest.Errorf("expected panic")
406 ch <- true
408 t.Errorf("failed after success")
411 ch <- true
412 <-ch
415 for _, tc := range testCases {
416 ctx := newTestContext(tc.maxPar, newMatcher(regexp.MatchString, "", ""))
417 buf := &bytes.Buffer{}
418 root := &T{
419 common: common{
420 signal: make(chan bool),
421 name: "Test",
422 w: buf,
423 chatty: tc.chatty,
425 context: ctx,
427 ok := root.Run(tc.desc, tc.f)
428 ctx.release()
430 if ok != tc.ok {
431 t.Errorf("%s:ok: got %v; want %v", tc.desc, ok, tc.ok)
433 if ok != !root.Failed() {
434 t.Errorf("%s:root failed: got %v; want %v", tc.desc, !ok, root.Failed())
436 if ctx.running != 0 || ctx.numWaiting != 0 {
437 t.Errorf("%s:running and waiting non-zero: got %d and %d", tc.desc, ctx.running, ctx.numWaiting)
439 got := strings.TrimSpace(buf.String())
440 want := strings.TrimSpace(tc.output)
441 re := makeRegexp(want)
442 if ok, err := regexp.MatchString(re, got); !ok || err != nil {
443 t.Errorf("%s:output:\ngot:\n%s\nwant:\n%s", tc.desc, got, want)
448 func TestBRun(t *T) {
449 work := func(b *B) {
450 for i := 0; i < b.N; i++ {
451 time.Sleep(time.Nanosecond)
454 testCases := []struct {
455 desc string
456 failed bool
457 chatty bool
458 output string
459 f func(*B)
461 desc: "simulate sequential run of subbenchmarks.",
462 f: func(b *B) {
463 b.Run("", func(b *B) { work(b) })
464 time1 := b.result.NsPerOp()
465 b.Run("", func(b *B) { work(b) })
466 time2 := b.result.NsPerOp()
467 if time1 >= time2 {
468 t.Errorf("no time spent in benchmark t1 >= t2 (%d >= %d)", time1, time2)
471 }, {
472 desc: "bytes set by all benchmarks",
473 f: func(b *B) {
474 b.Run("", func(b *B) { b.SetBytes(10); work(b) })
475 b.Run("", func(b *B) { b.SetBytes(10); work(b) })
476 if b.result.Bytes != 20 {
477 t.Errorf("bytes: got: %d; want 20", b.result.Bytes)
480 }, {
481 desc: "bytes set by some benchmarks",
482 // In this case the bytes result is meaningless, so it must be 0.
483 f: func(b *B) {
484 b.Run("", func(b *B) { b.SetBytes(10); work(b) })
485 b.Run("", func(b *B) { work(b) })
486 b.Run("", func(b *B) { b.SetBytes(10); work(b) })
487 if b.result.Bytes != 0 {
488 t.Errorf("bytes: got: %d; want 0", b.result.Bytes)
491 }, {
492 desc: "failure carried over to root",
493 failed: true,
494 output: "--- FAIL: root",
495 f: func(b *B) { b.Fail() },
496 }, {
497 desc: "skipping without message, chatty",
498 chatty: true,
499 output: "--- SKIP: root",
500 f: func(b *B) { b.SkipNow() },
501 }, {
502 desc: "skipping with message, chatty",
503 chatty: true,
504 output: `
505 --- SKIP: root
506 sub_test.go:NNN: skipping`,
507 f: func(b *B) { b.Skip("skipping") },
508 }, {
509 desc: "chatty with recursion",
510 chatty: true,
511 f: func(b *B) {
512 b.Run("", func(b *B) {
513 b.Run("", func(b *B) {})
516 }, {
517 desc: "skipping without message, not chatty",
518 f: func(b *B) { b.SkipNow() },
519 }, {
520 desc: "skipping after error",
521 failed: true,
522 output: `
523 --- FAIL: root
524 sub_test.go:NNN: an error
525 sub_test.go:NNN: skipped`,
526 f: func(b *B) {
527 b.Error("an error")
528 b.Skip("skipped")
530 }, {
531 desc: "memory allocation",
532 f: func(b *B) {
533 const bufSize = 256
534 alloc := func(b *B) {
535 var buf [bufSize]byte
536 for i := 0; i < b.N; i++ {
537 _ = append([]byte(nil), buf[:]...)
540 b.Run("", func(b *B) {
541 alloc(b)
542 b.ReportAllocs()
544 b.Run("", func(b *B) {
545 alloc(b)
546 b.ReportAllocs()
548 // runtime.MemStats sometimes reports more allocations than the
549 // benchmark is responsible for. Luckily the point of this test is
550 // to ensure that the results are not underreported, so we can
551 // simply verify the lower bound.
552 if got := b.result.MemAllocs; got < 2 {
553 t.Errorf("MemAllocs was %v; want 2", got)
555 if got := b.result.MemBytes; got < 2*bufSize {
556 t.Errorf("MemBytes was %v; want %v", got, 2*bufSize)
560 for _, tc := range testCases {
561 var ok bool
562 buf := &bytes.Buffer{}
563 // This is almost like the Benchmark function, except that we override
564 // the benchtime and catch the failure result of the subbenchmark.
565 root := &B{
566 common: common{
567 signal: make(chan bool),
568 name: "root",
569 w: buf,
570 chatty: tc.chatty,
572 benchFunc: func(b *B) { ok = b.Run("test", tc.f) }, // Use Run to catch failure.
573 benchTime: time.Microsecond,
575 root.runN(1)
576 if ok != !tc.failed {
577 t.Errorf("%s:ok: got %v; want %v", tc.desc, ok, !tc.failed)
579 if !ok != root.Failed() {
580 t.Errorf("%s:root failed: got %v; want %v", tc.desc, !ok, root.Failed())
582 // All tests are run as subtests
583 if root.result.N != 1 {
584 t.Errorf("%s: N for parent benchmark was %d; want 1", tc.desc, root.result.N)
586 got := strings.TrimSpace(buf.String())
587 want := strings.TrimSpace(tc.output)
588 re := makeRegexp(want)
589 if ok, err := regexp.MatchString(re, got); !ok || err != nil {
590 t.Errorf("%s:output:\ngot:\n%s\nwant:\n%s", tc.desc, got, want)
595 func makeRegexp(s string) string {
596 s = regexp.QuoteMeta(s)
597 s = strings.Replace(s, ":NNN:", `:\d\d\d:`, -1)
598 s = strings.Replace(s, "N\\.NNs", `\d*\.\d*s`, -1)
599 return s
602 func TestBenchmarkOutput(t *T) {
603 // Ensure Benchmark initialized common.w by invoking it with an error and
604 // normal case.
605 Benchmark(func(b *B) { b.Error("do not print this output") })
606 Benchmark(func(b *B) {})
609 func TestBenchmarkStartsFrom1(t *T) {
610 var first = true
611 Benchmark(func(b *B) {
612 if first && b.N != 1 {
613 panic(fmt.Sprintf("Benchmark() first N=%v; want 1", b.N))
615 first = false
619 func TestBenchmarkReadMemStatsBeforeFirstRun(t *T) {
620 var first = true
621 Benchmark(func(b *B) {
622 if first && (b.startAllocs == 0 || b.startBytes == 0) {
623 panic(fmt.Sprintf("ReadMemStats not called before first run"))
625 first = false
629 func TestParallelSub(t *T) {
630 c := make(chan int)
631 block := make(chan int)
632 for i := 0; i < 10; i++ {
633 go func(i int) {
634 <-block
635 t.Run(fmt.Sprint(i), func(t *T) {})
636 c <- 1
637 }(i)
639 close(block)
640 for i := 0; i < 10; i++ {
645 type funcWriter func([]byte) (int, error)
647 func (fw funcWriter) Write(b []byte) (int, error) { return fw(b) }
649 func TestRacyOutput(t *T) {
650 var runs int32 // The number of running Writes
651 var races int32 // Incremented for each race detected
652 raceDetector := func(b []byte) (int, error) {
653 // Check if some other goroutine is concurrently calling Write.
654 if atomic.LoadInt32(&runs) > 0 {
655 atomic.AddInt32(&races, 1) // Race detected!
657 atomic.AddInt32(&runs, 1)
658 defer atomic.AddInt32(&runs, -1)
659 runtime.Gosched() // Increase probability of a race
660 return len(b), nil
663 var wg sync.WaitGroup
664 root := &T{
665 common: common{w: funcWriter(raceDetector), chatty: true},
666 context: newTestContext(1, newMatcher(regexp.MatchString, "", "")),
668 root.Run("", func(t *T) {
669 for i := 0; i < 100; i++ {
670 wg.Add(1)
671 go func(i int) {
672 defer wg.Done()
673 t.Run(fmt.Sprint(i), func(t *T) {
674 t.Logf("testing run %d", i)
676 }(i)
679 wg.Wait()
681 if races > 0 {
682 t.Errorf("detected %d racy Writes", races)
686 func TestBenchmark(t *T) {
687 res := Benchmark(func(b *B) {
688 for i := 0; i < 5; i++ {
689 b.Run("", func(b *B) {
690 for i := 0; i < b.N; i++ {
691 time.Sleep(time.Millisecond)
696 if res.NsPerOp() < 4000000 {
697 t.Errorf("want >5ms; got %v", time.Duration(res.NsPerOp()))