main (tinygo test): reduce memory use when compiling a large number of packages

This change adds an additional semaphore to tinygo test that limits the number of tests being processed simultaneously (in addition to the existing limit on build jobs and runs).
When running a large number of tests, this limits the number of copies of per-test data stored in memory (avoiding an OOM in CI).
Этот коммит содержится в:
Nia Waldvogel 2021-12-24 15:41:34 -05:00 коммит произвёл Nia
родитель 12e314ccc9
коммит f1b15db258

42
main.go
Просмотреть файл

@ -1378,17 +1378,42 @@ func main() {
os.Exit(1)
}
// Build and run the tests concurrently, buffering the output.
fail := make(chan struct{}, 1)
var wg sync.WaitGroup
bufs := make([]testOutputBuf, len(pkgNames))
for i := range bufs {
bufs[i].done = make(chan struct{})
}
wg.Add(1)
go func() {
defer wg.Done()
// Flush the output one test at a time.
// This ensures that outputs from different tests are not mixed together.
for i := range bufs {
err := bufs[i].flush(os.Stdout, os.Stderr)
if err != nil {
// There was an error writing to stdout or stderr, so we probbably cannot print this.
select {
case fail <- struct{}{}:
default:
}
}
}
}()
// Build and run the tests concurrently.
// This uses an additional semaphore to reduce the memory usage.
testSema := make(chan struct{}, cap(options.Semaphore))
for i, pkgName := range pkgNames {
pkgName := pkgName
buf := &bufs[i]
buf.done = make(chan struct{})
testSema <- struct{}{}
wg.Add(1)
go func() {
defer wg.Done()
defer func() { <-testSema }()
defer close(buf.done)
stdout := (*testStdout)(buf)
stderr := (*testStderr)(buf)
@ -1407,19 +1432,6 @@ func main() {
}()
}
// Flush the output one test at a time.
// This ensures that outputs from different tests are not mixed together.
for i := range bufs {
err := bufs[i].flush(os.Stdout, os.Stderr)
if err != nil {
// There was an error writing to stdout or stderr, so we probbably cannot print this.
select {
case fail <- struct{}{}:
default:
}
}
}
// Wait for all tests to finish.
wg.Wait()
close(fail)