libgo/go/testing/sub_test.go

   1 // Copyright 2016 The Go Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style
   3 // license that can be found in the LICENSE file.
   4
   5 package testing
   6
   7 import (
   8         "bytes"
   9         "fmt"
  10         "regexp"
  11         "runtime"
  12         "strings"
  13         "sync"
  14         "sync/atomic"
  15         "time"
  16 )
  17
  18 func init() {
  19         // Make benchmark tests run 10* faster.
  20         *benchTime = 100 * time.Millisecond
  21 }
  22
  23 func TestTestContext(t *T) {
  24         const (
  25                 add1 = 0
  26                 done = 1
  27         )
  28         // After each of the calls are applied to the context, the
  29         type call struct {
  30                 typ int // run or done
  31                 // result from applying the call
  32                 running int
  33                 waiting int
  34                 started bool
  35         }
  36         testCases := []struct {
  37                 max int
  38                 run []call
  39         }{{
  40                 max: 1,
  41                 run: []call{
  42                         {typ: add1, running: 1, waiting: 0, started: true},
  43                         {typ: done, running: 0, waiting: 0, started: false},
  44                 },
  45         }, {
  46                 max: 1,
  47                 run: []call{
  48                         {typ: add1, running: 1, waiting: 0, started: true},
  49                         {typ: add1, running: 1, waiting: 1, started: false},
  50                         {typ: done, running: 1, waiting: 0, started: true},
  51                         {typ: done, running: 0, waiting: 0, started: false},
  52                         {typ: add1, running: 1, waiting: 0, started: true},
  53                 },
  54         }, {
  55                 max: 3,
  56                 run: []call{
  57                         {typ: add1, running: 1, waiting: 0, started: true},
  58                         {typ: add1, running: 2, waiting: 0, started: true},
  59                         {typ: add1, running: 3, waiting: 0, started: true},
  60                         {typ: add1, running: 3, waiting: 1, started: false},
  61                         {typ: add1, running: 3, waiting: 2, started: false},
  62                         {typ: add1, running: 3, waiting: 3, started: false},
  63                         {typ: done, running: 3, waiting: 2, started: true},
  64                         {typ: add1, running: 3, waiting: 3, started: false},
  65                         {typ: done, running: 3, waiting: 2, started: true},
  66                         {typ: done, running: 3, waiting: 1, started: true},
  67                         {typ: done, running: 3, waiting: 0, started: true},
  68                         {typ: done, running: 2, waiting: 0, started: false},
  69                         {typ: done, running: 1, waiting: 0, started: false},
  70                         {typ: done, running: 0, waiting: 0, started: false},
  71                 },
  72         }}
  73         for i, tc := range testCases {
  74                 ctx := &testContext{
  75                         startParallel: make(chan bool),
  76                         maxParallel:   tc.max,
  77                 }
  78                 for j, call := range tc.run {
  79                         doCall := func(f func()) chan bool {
  80                                 done := make(chan bool)
  81                                 go func() {
  82                                         f()
  83                                         done <- true
  84                                 }()
  85                                 return done
  86                         }
  87                         started := false
  88                         switch call.typ {
  89                         case add1:
  90                                 signal := doCall(ctx.waitParallel)
  91                                 select {
  92                                 case <-signal:
  93                                         started = true
  94                                 case ctx.startParallel <- true:
  95                                         <-signal
  96                                 }
  97                         case done:
  98                                 signal := doCall(ctx.release)
  99                                 select {
 100                                 case <-signal:
 101                                 case <-ctx.startParallel:
 102                                         started = true
 103                                         <-signal
 104                                 }
 105                         }
 106                         if started != call.started {
 107                                 t.Errorf("%d:%d:started: got %v; want %v", i, j, started, call.started)
 108                         }
 109                         if ctx.running != call.running {
 110                                 t.Errorf("%d:%d:running: got %v; want %v", i, j, ctx.running, call.running)
 111                         }
 112                         if ctx.numWaiting != call.waiting {
 113                                 t.Errorf("%d:%d:waiting: got %v; want %v", i, j, ctx.numWaiting, call.waiting)
 114                         }
 115                 }
 116         }
 117 }
 118
 119 func TestTRun(t *T) {
 120         realTest := t
 121         testCases := []struct {
 122                 desc   string
 123                 ok     bool
 124                 maxPar int
 125                 chatty bool
 126                 output string
 127                 f      func(*T)
 128         }{{
 129                 desc:   "failnow skips future sequential and parallel tests at same level",
 130                 ok:     false,
 131                 maxPar: 1,
 132                 output: `
 133 --- FAIL: failnow skips future sequential and parallel tests at same level (N.NNs)
 134     --- FAIL: failnow skips future sequential and parallel tests at same level/#00 (N.NNs)
 135     `,
 136                 f: func(t *T) {
 137                         ranSeq := false
 138                         ranPar := false
 139                         t.Run("", func(t *T) {
 140                                 t.Run("par", func(t *T) {
 141                                         t.Parallel()
 142                                         ranPar = true
 143                                 })
 144                                 t.Run("seq", func(t *T) {
 145                                         ranSeq = true
 146                                 })
 147                                 t.FailNow()
 148                                 t.Run("seq", func(t *T) {
 149                                         realTest.Error("test must be skipped")
 150                                 })
 151                                 t.Run("par", func(t *T) {
 152                                         t.Parallel()
 153                                         realTest.Error("test must be skipped.")
 154                                 })
 155                         })
 156                         if !ranPar {
 157                                 realTest.Error("parallel test was not run")
 158                         }
 159                         if !ranSeq {
 160                                 realTest.Error("sequential test was not run")
 161                         }
 162                 },
 163         }, {
 164                 desc:   "failure in parallel test propagates upwards",
 165                 ok:     false,
 166                 maxPar: 1,
 167                 output: `
 168 --- FAIL: failure in parallel test propagates upwards (N.NNs)
 169     --- FAIL: failure in parallel test propagates upwards/#00 (N.NNs)
 170         --- FAIL: failure in parallel test propagates upwards/#00/par (N.NNs)
 171                 `,
 172                 f: func(t *T) {
 173                         t.Run("", func(t *T) {
 174                                 t.Parallel()
 175                                 t.Run("par", func(t *T) {
 176                                         t.Parallel()
 177                                         t.Fail()
 178                                 })
 179                         })
 180                 },
 181         }, {
 182                 desc:   "skipping without message, chatty",
 183                 ok:     true,
 184                 chatty: true,
 185                 output: `
 186 === RUN   skipping without message, chatty
 187 --- SKIP: skipping without message, chatty (N.NNs)`,
 188                 f: func(t *T) { t.SkipNow() },
 189         }, {
 190                 desc:   "chatty with recursion",
 191                 ok:     true,
 192                 chatty: true,
 193                 output: `
 194 === RUN   chatty with recursion
 195 === RUN   chatty with recursion/#00
 196 === RUN   chatty with recursion/#00/#00
 197 --- PASS: chatty with recursion (N.NNs)
 198     --- PASS: chatty with recursion/#00 (N.NNs)
 199         --- PASS: chatty with recursion/#00/#00 (N.NNs)`,
 200                 f: func(t *T) {
 201                         t.Run("", func(t *T) {
 202                                 t.Run("", func(t *T) {})
 203                         })
 204                 },
 205         }, {
 206                 desc: "skipping without message, not chatty",
 207                 ok:   true,
 208                 f:    func(t *T) { t.SkipNow() },
 209         }, {
 210                 desc: "skipping after error",
 211                 output: `
 212 --- FAIL: skipping after error (N.NNs)
 213         sub_test.go:NNN: an error
 214         sub_test.go:NNN: skipped`,
 215                 f: func(t *T) {
 216                         t.Error("an error")
 217                         t.Skip("skipped")
 218                 },
 219         }, {
 220                 desc:   "use Run to locally synchronize parallelism",
 221                 ok:     true,
 222                 maxPar: 1,
 223                 f: func(t *T) {
 224                         var count uint32
 225                         t.Run("waitGroup", func(t *T) {
 226                                 for i := 0; i < 4; i++ {
 227                                         t.Run("par", func(t *T) {
 228                                                 t.Parallel()
 229                                                 atomic.AddUint32(&count, 1)
 230                                         })
 231                                 }
 232                         })
 233                         if count != 4 {
 234                                 t.Errorf("count was %d; want 4", count)
 235                         }
 236                 },
 237         }, {
 238                 desc: "alternate sequential and parallel",
 239                 // Sequential tests should partake in the counting of running threads.
 240                 // Otherwise, if one runs parallel subtests in sequential tests that are
 241                 // itself subtests of parallel tests, the counts can get askew.
 242                 ok:     true,
 243                 maxPar: 1,
 244                 f: func(t *T) {
 245                         t.Run("a", func(t *T) {
 246                                 t.Parallel()
 247                                 t.Run("b", func(t *T) {
 248                                         // Sequential: ensure running count is decremented.
 249                                         t.Run("c", func(t *T) {
 250                                                 t.Parallel()
 251                                         })
 252
 253                                 })
 254                         })
 255                 },
 256         }, {
 257                 desc: "alternate sequential and parallel 2",
 258                 // Sequential tests should partake in the counting of running threads.
 259                 // Otherwise, if one runs parallel subtests in sequential tests that are
 260                 // itself subtests of parallel tests, the counts can get askew.
 261                 ok:     true,
 262                 maxPar: 2,
 263                 f: func(t *T) {
 264                         for i := 0; i < 2; i++ {
 265                                 t.Run("a", func(t *T) {
 266                                         t.Parallel()
 267                                         time.Sleep(time.Nanosecond)
 268                                         for i := 0; i < 2; i++ {
 269                                                 t.Run("b", func(t *T) {
 270                                                         time.Sleep(time.Nanosecond)
 271                                                         for i := 0; i < 2; i++ {
 272                                                                 t.Run("c", func(t *T) {
 273                                                                         t.Parallel()
 274                                                                         time.Sleep(time.Nanosecond)
 275                                                                 })
 276                                                         }
 277
 278                                                 })
 279                                         }
 280                                 })
 281                         }
 282                 },
 283         }, {
 284                 desc:   "stress test",
 285                 ok:     true,
 286                 maxPar: 4,
 287                 f: func(t *T) {
 288                         t.Parallel()
 289                         for i := 0; i < 12; i++ {
 290                                 t.Run("a", func(t *T) {
 291                                         t.Parallel()
 292                                         time.Sleep(time.Nanosecond)
 293                                         for i := 0; i < 12; i++ {
 294                                                 t.Run("b", func(t *T) {
 295                                                         time.Sleep(time.Nanosecond)
 296                                                         for i := 0; i < 12; i++ {
 297                                                                 t.Run("c", func(t *T) {
 298                                                                         t.Parallel()
 299                                                                         time.Sleep(time.Nanosecond)
 300                                                                         t.Run("d1", func(t *T) {})
 301                                                                         t.Run("d2", func(t *T) {})
 302                                                                         t.Run("d3", func(t *T) {})
 303                                                                         t.Run("d4", func(t *T) {})
 304                                                                 })
 305                                                         }
 306                                                 })
 307                                         }
 308                                 })
 309                         }
 310                 },
 311         }, {
 312                 desc:   "skip output",
 313                 ok:     true,
 314                 maxPar: 4,
 315                 f: func(t *T) {
 316                         t.Skip()
 317                 },
 318         }, {
 319                 desc:   "panic on goroutine fail after test exit",
 320                 ok:     false,
 321                 maxPar: 4,
 322                 f: func(t *T) {
 323                         ch := make(chan bool)
 324                         t.Run("", func(t *T) {
 325                                 go func() {
 326                                         <-ch
 327                                         defer func() {
 328                                                 if r := recover(); r == nil {
 329                                                         realTest.Errorf("expected panic")
 330                                                 }
 331                                                 ch <- true
 332                                         }()
 333                                         t.Errorf("failed after success")
 334                                 }()
 335                         })
 336                         ch <- true
 337                         <-ch
 338                 },
 339         }}
 340         for _, tc := range testCases {
 341                 ctx := newTestContext(tc.maxPar, newMatcher(regexp.MatchString, "", ""))
 342                 buf := &bytes.Buffer{}
 343                 root := &T{
 344                         common: common{
 345                                 signal: make(chan bool),
 346                                 name:   "Test",
 347                                 w:      buf,
 348                                 chatty: tc.chatty,
 349                         },
 350                         context: ctx,
 351                 }
 352                 ok := root.Run(tc.desc, tc.f)
 353                 ctx.release()
 354
 355                 if ok != tc.ok {
 356                         t.Errorf("%s:ok: got %v; want %v", tc.desc, ok, tc.ok)
 357                 }
 358                 if ok != !root.Failed() {
 359                         t.Errorf("%s:root failed: got %v; want %v", tc.desc, !ok, root.Failed())
 360                 }
 361                 if ctx.running != 0 || ctx.numWaiting != 0 {
 362                         t.Errorf("%s:running and waiting non-zero: got %d and %d", tc.desc, ctx.running, ctx.numWaiting)
 363                 }
 364                 got := strings.TrimSpace(buf.String())
 365                 want := strings.TrimSpace(tc.output)
 366                 re := makeRegexp(want)
 367                 if ok, err := regexp.MatchString(re, got); !ok || err != nil {
 368                         t.Errorf("%s:output:\ngot:\n%s\nwant:\n%s", tc.desc, got, want)
 369                 }
 370         }
 371 }
 372
 373 func TestBRun(t *T) {
 374         work := func(b *B) {
 375                 for i := 0; i < b.N; i++ {
 376                         time.Sleep(time.Nanosecond)
 377                 }
 378         }
 379         testCases := []struct {
 380                 desc   string
 381                 failed bool
 382                 chatty bool
 383                 output string
 384                 f      func(*B)
 385         }{{
 386                 desc: "simulate sequential run of subbenchmarks.",
 387                 f: func(b *B) {
 388                         b.Run("", func(b *B) { work(b) })
 389                         time1 := b.result.NsPerOp()
 390                         b.Run("", func(b *B) { work(b) })
 391                         time2 := b.result.NsPerOp()
 392                         if time1 >= time2 {
 393                                 t.Errorf("no time spent in benchmark t1 >= t2 (%d >= %d)", time1, time2)
 394                         }
 395                 },
 396         }, {
 397                 desc: "bytes set by all benchmarks",
 398                 f: func(b *B) {
 399                         b.Run("", func(b *B) { b.SetBytes(10); work(b) })
 400                         b.Run("", func(b *B) { b.SetBytes(10); work(b) })
 401                         if b.result.Bytes != 20 {
 402                                 t.Errorf("bytes: got: %d; want 20", b.result.Bytes)
 403                         }
 404                 },
 405         }, {
 406                 desc: "bytes set by some benchmarks",
 407                 // In this case the bytes result is meaningless, so it must be 0.
 408                 f: func(b *B) {
 409                         b.Run("", func(b *B) { b.SetBytes(10); work(b) })
 410                         b.Run("", func(b *B) { work(b) })
 411                         b.Run("", func(b *B) { b.SetBytes(10); work(b) })
 412                         if b.result.Bytes != 0 {
 413                                 t.Errorf("bytes: got: %d; want 0", b.result.Bytes)
 414                         }
 415                 },
 416         }, {
 417                 desc:   "failure carried over to root",
 418                 failed: true,
 419                 output: "--- FAIL: root",
 420                 f:      func(b *B) { b.Fail() },
 421         }, {
 422                 desc:   "skipping without message, chatty",
 423                 chatty: true,
 424                 output: "--- SKIP: root",
 425                 f:      func(b *B) { b.SkipNow() },
 426         }, {
 427                 desc:   "skipping with message, chatty",
 428                 chatty: true,
 429                 output: `
 430 --- SKIP: root
 431         sub_test.go:NNN: skipping`,
 432                 f: func(b *B) { b.Skip("skipping") },
 433         }, {
 434                 desc:   "chatty with recursion",
 435                 chatty: true,
 436                 f: func(b *B) {
 437                         b.Run("", func(b *B) {
 438                                 b.Run("", func(b *B) {})
 439                         })
 440                 },
 441         }, {
 442                 desc: "skipping without message, not chatty",
 443                 f:    func(b *B) { b.SkipNow() },
 444         }, {
 445                 desc:   "skipping after error",
 446                 failed: true,
 447                 output: `
 448 --- FAIL: root
 449         sub_test.go:NNN: an error
 450         sub_test.go:NNN: skipped`,
 451                 f: func(b *B) {
 452                         b.Error("an error")
 453                         b.Skip("skipped")
 454                 },
 455         }, {
 456                 desc: "memory allocation",
 457                 f: func(b *B) {
 458                         const bufSize = 256
 459                         alloc := func(b *B) {
 460                                 var buf [bufSize]byte
 461                                 for i := 0; i < b.N; i++ {
 462                                         _ = append([]byte(nil), buf[:]...)
 463                                 }
 464                         }
 465                         b.Run("", func(b *B) {
 466                                 alloc(b)
 467                                 b.ReportAllocs()
 468                         })
 469                         b.Run("", func(b *B) {
 470                                 alloc(b)
 471                                 b.ReportAllocs()
 472                         })
 473                         // runtime.MemStats sometimes reports more allocations than the
 474                         // benchmark is responsible for. Luckily the point of this test is
 475                         // to ensure that the results are not underreported, so we can
 476                         // simply verify the lower bound.
 477                         if got := b.result.MemAllocs; got < 2 {
 478                                 t.Errorf("MemAllocs was %v; want 2", got)
 479                         }
 480                         if got := b.result.MemBytes; got < 2*bufSize {
 481                                 t.Errorf("MemBytes was %v; want %v", got, 2*bufSize)
 482                         }
 483                 },
 484         }}
 485         for _, tc := range testCases {
 486                 var ok bool
 487                 buf := &bytes.Buffer{}
 488                 // This is almost like the Benchmark function, except that we override
 489                 // the benchtime and catch the failure result of the subbenchmark.
 490                 root := &B{
 491                         common: common{
 492                                 signal: make(chan bool),
 493                                 name:   "root",
 494                                 w:      buf,
 495                                 chatty: tc.chatty,
 496                         },
 497                         benchFunc: func(b *B) { ok = b.Run("test", tc.f) }, // Use Run to catch failure.
 498                         benchTime: time.Microsecond,
 499                 }
 500                 root.runN(1)
 501                 if ok != !tc.failed {
 502                         t.Errorf("%s:ok: got %v; want %v", tc.desc, ok, !tc.failed)
 503                 }
 504                 if !ok != root.Failed() {
 505                         t.Errorf("%s:root failed: got %v; want %v", tc.desc, !ok, root.Failed())
 506                 }
 507                 // All tests are run as subtests
 508                 if root.result.N != 1 {
 509                         t.Errorf("%s: N for parent benchmark was %d; want 1", tc.desc, root.result.N)
 510                 }
 511                 got := strings.TrimSpace(buf.String())
 512                 want := strings.TrimSpace(tc.output)
 513                 re := makeRegexp(want)
 514                 if ok, err := regexp.MatchString(re, got); !ok || err != nil {
 515                         t.Errorf("%s:output:\ngot:\n%s\nwant:\n%s", tc.desc, got, want)
 516                 }
 517         }
 518 }
 519
 520 func makeRegexp(s string) string {
 521         s = strings.Replace(s, ":NNN:", `:\d\d\d:`, -1)
 522         s = strings.Replace(s, "(N.NNs)", `\(\d*\.\d*s\)`, -1)
 523         return s
 524 }
 525
 526 func TestBenchmarkOutput(t *T) {
 527         // Ensure Benchmark initialized common.w by invoking it with an error and
 528         // normal case.
 529         Benchmark(func(b *B) { b.Error("do not print this output") })
 530         Benchmark(func(b *B) {})
 531 }
 532
 533 func TestBenchmarkStartsFrom1(t *T) {
 534         var first = true
 535         Benchmark(func(b *B) {
 536                 if first && b.N != 1 {
 537                         panic(fmt.Sprintf("Benchmark() first N=%v; want 1", b.N))
 538                 }
 539                 first = false
 540         })
 541 }
 542
 543 func TestBenchmarkReadMemStatsBeforeFirstRun(t *T) {
 544         var first = true
 545         Benchmark(func(b *B) {
 546                 if first && (b.startAllocs == 0 || b.startBytes == 0) {
 547                         panic(fmt.Sprintf("ReadMemStats not called before first run"))
 548                 }
 549                 first = false
 550         })
 551 }
 552
 553 func TestParallelSub(t *T) {
 554         c := make(chan int)
 555         block := make(chan int)
 556         for i := 0; i < 10; i++ {
 557                 go func(i int) {
 558                         <-block
 559                         t.Run(fmt.Sprint(i), func(t *T) {})
 560                         c <- 1
 561                 }(i)
 562         }
 563         close(block)
 564         for i := 0; i < 10; i++ {
 565                 <-c
 566         }
 567 }
 568
 569 type funcWriter func([]byte) (int, error)
 570
 571 func (fw funcWriter) Write(b []byte) (int, error) { return fw(b) }
 572
 573 func TestRacyOutput(t *T) {
 574         var runs int32  // The number of running Writes
 575         var races int32 // Incremented for each race detected
 576         raceDetector := func(b []byte) (int, error) {
 577                 // Check if some other goroutine is concurrently calling Write.
 578                 if atomic.LoadInt32(&runs) > 0 {
 579                         atomic.AddInt32(&races, 1) // Race detected!
 580                 }
 581                 atomic.AddInt32(&runs, 1)
 582                 defer atomic.AddInt32(&runs, -1)
 583                 runtime.Gosched() // Increase probability of a race
 584                 return len(b), nil
 585         }
 586
 587         var wg sync.WaitGroup
 588         root := &T{
 589                 common:  common{w: funcWriter(raceDetector), chatty: true},
 590                 context: newTestContext(1, newMatcher(regexp.MatchString, "", "")),
 591         }
 592         root.Run("", func(t *T) {
 593                 for i := 0; i < 100; i++ {
 594                         wg.Add(1)
 595                         go func(i int) {
 596                                 defer wg.Done()
 597                                 t.Run(fmt.Sprint(i), func(t *T) {
 598                                         t.Logf("testing run %d", i)
 599                                 })
 600                         }(i)
 601                 }
 602         })
 603         wg.Wait()
 604
 605         if races > 0 {
 606                 t.Errorf("detected %d racy Writes", races)
 607         }
 608 }
 609
 610 func TestBenchmark(t *T) {
 611         res := Benchmark(func(b *B) {
 612                 for i := 0; i < 5; i++ {
 613                         b.Run("", func(b *B) {
 614                                 for i := 0; i < b.N; i++ {
 615                                         time.Sleep(time.Millisecond)
 616                                 }
 617                         })
 618                 }
 619         })
 620         if res.NsPerOp() < 4000000 {
 621                 t.Errorf("want >5ms; got %v", time.Duration(res.NsPerOp()))
 622         }
 623 }