go.temporal.io/server/tools/testrunner/testrunner.go
480 LOC · 140 covered · 340 uncovered · 68 ranges · 37 concepts · 35 introducers · 17 tests
File neighbourhood
The centred file is linked to every concept that introduces one of its ranges, every test that runs code from the file, and the gray connector concepts standing between those tests and the file's own introducer concepts. Undirected links join concepts to every file where they introduce source and concepts to the tests they introduce; arrows show specialization between the displayed concepts and bridge only concepts omitted from this view. Concept colors match the source ranges below; connector concepts have no source color and are shown in gray.
Focused file, its introducer and connector concepts, their introduced files, and tests that run code from the file
In the embedded map, ordinary wheel input scrolls the page; use the visible controls to zoom and drag to pan. Open the full-screen map for canvas navigation: wheel pans, Ctrl/Command plus wheel zooms, and arrow keys pan when this region is focused. On touch screens, open the full-screen map to pan or pinch. If JavaScript or WebGL is unavailable, use the related-file, concept, and source links on this page.
Graph controls are ready.
Interactive rendering requires JavaScript and WebGL. Use the related-file, concept, and source links on this page while the interactive map is unavailable.
1
package testrunner
2
3
import (
4
"context"
5
"errors"
6
"fmt"
7
"io"
8
"log"
9
"os"
10
"os/exec"
11
"path/filepath"
12
"regexp"
13
"slices"
14
"strconv"
15
"strings"
16
"time"
17
18
"github.com/google/uuid"
19
)
20
21
const (
22
codeCoverageExtension = ".cover.out"
23
maxAttemptsFlag = "--max-attempts="
24
coverProfileFlag = "-coverprofile="
25
junitReportFlag = "--junitfile="
26
junitGlobFlag = "--junit-glob="
27
summaryOutputDirFlag = "--summary-output-dir="
28
crashReportNameFlag = "--crashreportname="
29
gotestsumPathFlag = "--gotestsum-path="
30
totalTimeoutFlag = "--total-timeout="
31
32
// fullRerunThreshold is the number of test failures above which we do a full
33
// rerun instead of retrying only the failed tests.
34
fullRerunThreshold = 20
35
)
36
37
const (
38
testCommand = "test"
39
crashReportCommand = "report-crash"
40
summaryCommand = "generate-summary"
41
)
42
43
type attempt struct {
44
runner *runner
45
number int
46
exitErr *exec.ExitError
47
junitReport *junitReport
48
coverProfilePath string
49
}
50
51
func (a *attempt) run(ctx context.Context, args []string) (string, error) {
52
for i, arg := range args {
53
if strings.HasPrefix(arg, coverProfileFlag) {
54
args[i] = coverProfileFlag + a.coverProfilePath
55
} else if strings.HasPrefix(arg, junitReportFlag) {
56
args[i] = junitReportFlag + a.junitReport.path
57
}
58
}
59
log.Printf("starting test attempt #%d: %v %v",
60
a.number, a.runner.gotestsumPath, strings.Join(args, " "))
61
cmd := exec.CommandContext(ctx, a.runner.gotestsumPath, args...)
62
var output strings.Builder
63
cmd.Stdout = io.MultiWriter(os.Stdout, &output)
64
cmd.Stderr = io.MultiWriter(os.Stderr, &output)
65
cmd.Stdin = os.Stdin
66
err := cmd.Run()
67
return output.String(), err
68
}
69
70
type runner struct {
71
gotestsumPath string
72
junitOutputPath string
73
coverProfilePath string
74
attempts []*attempt
75
maxAttempts int
76
crashName string
77
junitGlob string
78
summaryOutputDir string
79
alerts []alert
80
totalTimeout time.Duration
81
}
82
84
>
return &runner{
85
>
attempts: make([]*attempt, 0),
86
>
maxAttempts: 1,
87
>
}
88
>
}
89
90
// nolint:revive,cognitive-complexity
91
>
func (r *runner) sanitizeAndParseArgs(command string, args []string) ([]string, error) {
testrunner.go ×1
92
>
var sanitizedArgs []string
93
>
for _, arg := range args {
96
>
r.maxAttempts, err = strconv.Atoi(strings.Split(arg, "=")[1])
97
>
if err != nil {
99
>
}
101
>
return nil, fmt.Errorf("invalid argument %q: must be greater than zero", maxAttemptsFlag)
testrunner.go ×1
102
>
}
104
}
105
108
>
r.totalTimeout, err = time.ParseDuration(strings.TrimPrefix(arg, totalTimeoutFlag))
109
>
if err != nil {
111
>
}
113
return nil, fmt.Errorf("invalid argument %q: must be greater than zero", totalTimeoutFlag)
114
}
116
}
117
120
>
continue
121
}
122
125
>
if r.crashName == "" {
126
return nil, fmt.Errorf("invalid argument %q: must not be empty", crashReportNameFlag)
127
}
129
return nil, fmt.Errorf("argument %q is only valid for command %q", crashReportNameFlag, crashReportCommand)
130
}
132
}
133
136
>
continue
137
}
140
>
if command != summaryCommand {
141
return nil, fmt.Errorf("argument %q is only valid for command %q", summaryOutputDirFlag, summaryCommand)
142
}
144
}
149
>
r.junitOutputPath = strings.Split(arg, "=")[1]
150
>
}
151
153
}
154
157
>
if r.coverProfilePath == "" {
159
>
}
162
>
}
165
>
}
167
>
if r.junitOutputPath == "" {
169
>
}
171
return nil, fmt.Errorf("missing required argument %q", crashReportNameFlag)
172
}
174
>
if r.junitGlob == "" {
176
>
}
178
return nil, fmt.Errorf("missing required argument %q", summaryOutputDirFlag)
179
}
180
default:
181
return nil, fmt.Errorf("unknown command %q", command)
182
}
183
185
}
186
188
>
a := &attempt{
189
>
runner: r,
190
>
number: len(r.attempts) + 1,
191
>
coverProfilePath: fmt.Sprintf(
192
>
"%v_%v%v",
193
>
strings.TrimSuffix(r.coverProfilePath, codeCoverageExtension),
194
>
len(r.attempts),
195
>
codeCoverageExtension),
196
>
junitReport: &junitReport{
197
>
path: filepath.Join(os.TempDir(), fmt.Sprintf("temporalio-temporal-%s-junit.xml", uuid.NewString())),
198
>
},
199
>
}
200
>
r.attempts = append(r.attempts, a)
201
>
return a
202
>
}
203
205
>
var reports []*junitReport
206
>
for _, a := range r.attempts {
207
>
reports = append(reports, a.junitReport)
208
>
}
209
>
return reports
210
}
211
212
// Main is the entry point for the testrunner tool.
213
// nolint:revive,deep-exit
214
func Main() {
215
log.SetPrefix("[testrunner] ")
216
ctx := context.Background()
217
218
if len(os.Args) < 2 {
219
log.Fatalf("expected at least 2 arguments")
220
}
221
r := newRunner()
222
223
command := os.Args[1]
224
args, err := r.sanitizeAndParseArgs(command, os.Args[2:])
225
if err != nil {
226
log.Fatalf("failed to parse command line options: %v", err)
227
}
228
229
if r.totalTimeout > 0 {
230
var cancel context.CancelFunc
231
ctx, cancel = context.WithTimeout(ctx, r.totalTimeout)
232
defer cancel()
233
}
234
235
switch command {
236
case testCommand:
237
r.runTests(ctx, args)
238
case crashReportCommand:
239
r.reportCrash()
240
case summaryCommand:
241
if err := r.generateSummary(); err != nil {
242
log.Fatal(err)
243
}
244
default:
245
log.Fatalf("unknown command %q", command)
246
}
247
}
248
249
// nolint:revive,deep-exit
251
>
jr := generateReport([]string{r.crashName}, "crash", failureTypeCrash)
252
>
jr.path = r.junitOutputPath
253
>
if err := jr.write(); err != nil {
254
log.Fatal(err)
255
}
256
}
257
259
>
paths, err := filepath.Glob(r.junitGlob)
260
>
if err != nil {
261
return fmt.Errorf("failed to expand junit glob %q: %w", r.junitGlob, err)
262
}
264
>
265
>
reports := make([]*junitReport, 0, len(paths))
266
>
for _, path := range paths {
267
>
report := &junitReport{path: path}
268
>
if err := report.read(); err != nil {
269
return fmt.Errorf("failed to read junit report %q: %w", path, err)
270
}
272
}
273
275
>
if len(summary.Rows) == 0 {
277
>
return nil
278
>
}
279
281
>
if err := os.MkdirAll(r.summaryOutputDir, 0o755); err != nil {
282
return fmt.Errorf("failed to create summary output directory: %w", err)
283
}
284
>
if err := os.WriteFile(filepath.Join(r.summaryOutputDir, "test-summary.md"), []byte(markdown), 0o644); err != nil {
testrunner.go ×5
285
return fmt.Errorf("failed to write summary markdown: %w", err)
286
}
287
289
>
if err != nil {
290
return fmt.Errorf("failed to render summary json: %w", err)
291
}
292
>
if err := os.WriteFile(filepath.Join(r.summaryOutputDir, "test-summary.json"), append(content, '\n'), 0o644); err != nil {
testrunner.go ×5
293
return fmt.Errorf("failed to write summary json: %w", err)
294
}
296
}
297
298
// writeCurrentReport writes the merged report from all completed attempts to the
299
// final output path. It is called after each attempt so that partial results
300
// survive if the process is killed externally between attempts.
301
// Reporting errors (e.g. unexpected missing reruns) are intentionally ignored
302
// here; they are only checked for the final write at the end of runTests.
304
>
reports := r.allReports()
305
>
if len(reports) == 0 {
306
return
307
}
309
>
if err != nil {
310
log.Printf("warning: failed to merge reports for intermediate write: %v", err)
311
return
312
}
314
merged.appendAlertsSuite(r.alerts)
315
}
317
>
if err := merged.write(); err != nil {
318
log.Printf("warning: failed to write intermediate report: %v", err)
319
}
320
}
321
322
// nolint:revive,deep-exit
323
func (r *runner) runTests(ctx context.Context, args []string) {
324
var currentAttempt *attempt
325
var totalTimeoutFired bool
326
for a := 1; a <= r.maxAttempts; a++ {
327
currentAttempt = r.newAttempt()
328
329
// Run tests.
330
stdout, err := currentAttempt.run(ctx, args)
331
// Extract prominent alerts from this attempt's output.
332
r.alerts = append(r.alerts, parseAlerts(stdout)...)
333
334
// Check whether our total timeout fired (context deadline exceeded).
335
// This happens when the go test binary hangs and never produces its own
336
// "test timed out" panic. We collect whatever results are available from
337
// completed attempts and from the partially-executed current attempt, then
338
// flush the XML before the external kill arrives.
339
if ctx.Err() != nil {
340
log.Printf("total timeout reached, collecting partial results from %d completed attempt(s)", a-1)
341
totalTimeoutFired = true
342
// Try to read whatever gotestsum managed to write before it was killed.
343
if readErr := currentAttempt.junitReport.read(); readErr != nil {
344
// gotestsum didn't finish writing a JUnit XML. Fall back to parsing
345
// stdout for any "--- FAIL:" lines that completed before the kill.
346
if failedTests := parseFailedTestsFromOutput(stdout); len(failedTests) > 0 {
347
currentAttempt.junitReport = generateReport(failedTests, "total timeout", failureTypeTimeout)
348
}
349
// If no failed tests are found either, the current attempt's report
350
// remains empty and mergeReports will include only prior attempts.
351
}
352
// Without this, a mid-run timeout leaves an empty JUnit and CI shows green.
353
currentAttempt.junitReport.appendSyntheticFailure(
354
"testrunner.TotalTimeout",
355
failureTypeTimeout,
356
fmt.Sprintf("test-runner total timeout (%s) reached before all tests completed", r.totalTimeout),
357
)
358
break
359
}
360
361
if err != nil && !errors.As(err, ¤tAttempt.exitErr) {
362
log.Fatalf("test run failed with an unexpected error: %v", err)
363
}
364
365
stacktrace, timedoutTests := parseTestTimeouts(stdout)
366
if len(timedoutTests) > 0 {
367
// Run timed out and was aborted.
368
// Update JUnit XML output for timed out tests since none will have been generated.
369
currentAttempt.junitReport = generateReport(timedoutTests, "timed out", failureTypeTimeout)
370
log.Print(stacktrace)
371
372
// Don't retry.
373
break
374
}
375
376
// All tests were run, parse JUnit XML output.
377
if err = currentAttempt.junitReport.read(); err != nil {
378
log.Fatal(err)
379
}
380
381
// Write intermediate results so they survive if we are killed externally
382
// between attempts (e.g. a GitHub Actions job timeout fires after this
383
// attempt but before the next one completes).
384
r.writeCurrentReport()
385
386
// If the run completely successfull, no need to retry.
387
if currentAttempt.exitErr == nil {
388
break
389
}
390
391
// Sanity check: make sure failures are reported when the run failed.
392
failures := currentAttempt.junitReport.collectTestCaseFailures()
393
if len(failures) == 0 {
394
log.Fatalf("tests failed but no failures have been detected, not rerunning tests")
395
}
396
397
// Rerun all tests from previous attempt if there are too many failures in a single suite.
398
if len(failures) > fullRerunThreshold && a < r.maxAttempts {
399
log.Printf(
400
"number of failures exceeds configured threshold (%d/%d) for narrowing down tests to retry, retrying with previous attempt's args",
401
len(failures), fullRerunThreshold)
402
continue
403
}
404
args = stripRunFromArgs(args)
405
for i, failure := range failures {
406
failures[i] = goTestNameToRunFlagRegexp(failure)
407
}
408
failureArg := strings.Join(failures, "|")
409
// -args has special semantics in Go.
410
argsIdx := slices.Index(args, "-args")
411
if argsIdx == -1 {
412
args = append(args, "-run", failureArg)
413
} else {
414
args = slices.Insert(args, argsIdx, "-run", failureArg)
415
}
416
}
417
418
// Merge reports from all attempts and write the final JUnit report.
419
mergedReport, err := mergeReports(r.allReports())
420
if err != nil {
421
log.Fatal(err)
422
}
423
// Append ALERTS suite to the merged JUnit if any were found.
424
if len(r.alerts) > 0 {
425
mergedReport.appendAlertsSuite(r.alerts)
426
}
427
mergedReport.path = r.junitOutputPath
428
if err = mergedReport.write(); err != nil {
429
log.Fatal(err)
430
}
431
432
// Skip the strict rerun-coverage check when the total timeout fired: the
433
// in-progress attempt was killed before it could execute all expected tests.
434
435
if len(mergedReport.reportingErrs) > 0 && ctx.Err() == nil {
436
log.Fatal(mergedReport.reportingErrs)
437
}
438
439
// Exit with the exit code of the last attempt.
440
if currentAttempt.exitErr != nil {
441
log.Printf("exiting with failure after running %d attempt(s)", len(r.attempts))
442
os.Exit(currentAttempt.exitErr.ExitCode())
443
}
444
// Without a non-zero exit, a total-timeout makes CI silently green.
445
if totalTimeoutFired {
446
log.Printf("exiting with failure: total timeout (%s) reached", r.totalTimeout)
447
os.Exit(1)
448
}
449
}
450
452
>
var skipNext bool
453
>
for _, arg := range args {
454
>
if skipNext {
456
>
continue
459
>
continue
462
}
464
}
466
}
467
468
func goTestNameToRunFlagRegexp(test string) string {
469
parts := strings.Split(test, "/")
470
var sb strings.Builder
471
for i, p := range parts {
472
if i > 0 {
473
sb.WriteByte('/')
474
}
475
sb.WriteByte('^')
476
sb.WriteString(regexp.QuoteMeta(p))
477
sb.WriteByte('$')
478
}
479
return sb.String()
480
}