petru/coder-server
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

chore: cache terraform providers for workspaces terraform tests (#20603)

Browse Source 
Fixes flaky `TestWorkspaceTagsTerraform` and
`TestWorkspaceTemplateParamsChange` tests that were failing with
`connection reset by peer` errors when downloading the coder/coder
provider.

This applies the same caching solution which was done in
https://github.com/coder/coder/pull/17373

1. Extracts provider caching logic into `testutil/terraform_cache.go`
2. Updates TestProvision to use the shared caching helpers
3. Updates enterprise workspace tests to use the shared caching helpers

The cache is persisted at `~/.cache/coderv2-test/` and automatically
cached between CI runs via existing GitHub Actions cache setup.

Closes https://github.com/coder/internal/issues/607
This commit is contained in:
Kacper Sawicki
2025-11-12 09:43:22 +01:00
committed by GitHub
parent e49c917bb0
commit f543a87b78
3 changed files with 196 additions and 211 deletions
Download Patch File Download Diff File Expand all files Collapse all files
enterprise/coderd/workspaces_test.go
+9 -43
View File
@@ -7,8 +7,6 @@ import (
"encoding/json"
"fmt"
"net/http"
"os"
"os/exec"
"path/filepath"
"strings"
"sync/atomic"
@@ -3390,51 +3388,19 @@ func workspaceTagsTerraform(t *testing.T, tc testWorkspaceTagsTerraformCase, dyn
}
}
// downloadProviders is a test helper that creates a temporary file and writes a
// terraform CLI config file with a provider_installation stanza for coder/coder
// using dev_overrides. It also fetches the latest provider release from GitHub
// and extracts the binary to the temporary dir. It is the responsibility of the
// caller to set TF_CLI_CONFIG_FILE.
// downloadProviders is a test helper that caches Terraform providers and returns
// the path to a Terraform CLI config file that uses the cached providers.
// This uses the shared testutil caching infrastructure to avoid re-downloading
// providers on every test run. It is the responsibility of the caller to set
// TF_CLI_CONFIG_FILE.
func downloadProviders(t *testing.T, providersTf string) string {
t.Helper()
// We firstly write a Terraform CLI config file to a temporary directory:
var (
tempDir = t.TempDir()
cacheDir = filepath.Join(tempDir, ".cache")
providersTfPath = filepath.Join(tempDir, "providers.tf")
cliConfigPath = filepath.Join(tempDir, "local.tfrc")
)
// Write files to disk
require.NoError(t, os.MkdirAll(cacheDir, os.ModePerm|os.ModeDir))
require.NoError(t, os.WriteFile(providersTfPath, []byte(providersTf), os.ModePerm)) // nolint:gosec
cliConfigTemplate := `
provider_installation {
filesystem_mirror {
path = %q
include = ["*/*/*"]
}
direct {
exclude = ["*/*/*"]
}
}`
err := os.WriteFile(cliConfigPath, []byte(fmt.Sprintf(cliConfigTemplate, cacheDir)), os.ModePerm) // nolint:gosec
require.NoError(t, err, "failed to write %s", cliConfigPath)
ctx := testutil.Context(t, testutil.WaitLong)
// Run terraform providers mirror to mirror required providers to cacheDir
cmd := exec.CommandContext(ctx, "terraform", "providers", "mirror", cacheDir)
cmd.Env = os.Environ() // without this terraform may complain about path
cmd.Env = append(cmd.Env, "TF_CLI_CONFIG_FILE="+cliConfigPath)
cmd.Dir = tempDir
out, err := cmd.CombinedOutput()
if !assert.NoError(t, err) {
t.Log("failed to download providers:")
t.Log(string(out))
t.FailNow()
}
cacheRootDir := filepath.Join(testutil.PersistentCacheDir(t), "terraform_workspace_tags_test")
templateFiles := map[string]string{"providers.tf": providersTf}
testName := "TestWorkspaceTagsTerraform"
cliConfigPath := testutil.CacheTFProviders(t, cacheRootDir, testName, templateFiles)
t.Logf("Set TF_CLI_CONFIG_FILE=%s", cliConfigPath)
return cliConfigPath
}
provisioner/terraform/provision_test.go
+2 -168
View File
@@ -3,17 +3,13 @@
package terraform_test
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"net"
"net/http"
"os"
"os/exec"
"path/filepath"
"sort"
"strings"
@@ -94,168 +90,6 @@ func configure(ctx context.Context, t *testing.T, client proto.DRPCProvisionerCl
return sess
}
func hashTemplateFilesAndTestName(t *testing.T, testName string, templateFiles map[string]string) string {
t.Helper()
sortedFileNames := make([]string, 0, len(templateFiles))
for fileName := range templateFiles {
sortedFileNames = append(sortedFileNames, fileName)
}
sort.Strings(sortedFileNames)
// Inserting a delimiter between the file name and the file content
// ensures that a file named `ab` with content `cd`
// will not hash to the same value as a file named `abc` with content `d`.
// This can still happen if the file name or content include the delimiter,
// but hopefully they won't.
delimiter := []byte("🎉 🌱 🌷")
hasher := sha256.New()
for _, fileName := range sortedFileNames {
file := templateFiles[fileName]
_, err := hasher.Write([]byte(fileName))
require.NoError(t, err)
_, err = hasher.Write(delimiter)
require.NoError(t, err)
_, err = hasher.Write([]byte(file))
require.NoError(t, err)
}
_, err := hasher.Write(delimiter)
require.NoError(t, err)
_, err = hasher.Write([]byte(testName))
require.NoError(t, err)
return hex.EncodeToString(hasher.Sum(nil))
}
const (
terraformConfigFileName = "terraform.rc"
cacheProvidersDirName = "providers"
cacheTemplateFilesDirName = "files"
)
// Writes a Terraform CLI config file (`terraform.rc`) in `dir` to enforce using the local provider mirror.
// This blocks network access for providers, forcing Terraform to use only what's cached in `dir`.
// Returns the path to the generated config file.
func writeCliConfig(t *testing.T, dir string) string {
t.Helper()
cliConfigPath := filepath.Join(dir, terraformConfigFileName)
require.NoError(t, os.MkdirAll(filepath.Dir(cliConfigPath), 0o700))
content := fmt.Sprintf(`
provider_installation {
filesystem_mirror {
path = "%s"
include = ["*/*"]
}
direct {
exclude = ["*/*"]
}
}
`, filepath.Join(dir, cacheProvidersDirName))
require.NoError(t, os.WriteFile(cliConfigPath, []byte(content), 0o600))
return cliConfigPath
}
func runCmd(t *testing.T, dir string, args ...string) {
t.Helper()
stdout, stderr := bytes.NewBuffer(nil), bytes.NewBuffer(nil)
cmd := exec.Command(args[0], args[1:]...) //#nosec
cmd.Dir = dir
cmd.Stdout = stdout
cmd.Stderr = stderr
if err := cmd.Run(); err != nil {
t.Fatalf("failed to run %s: %s\nstdout: %s\nstderr: %s", strings.Join(args, " "), err, stdout.String(), stderr.String())
}
}
// Each test gets a unique cache dir based on its name and template files.
// This ensures that tests can download providers in parallel and that they
// will redownload providers if the template files change.
func getTestCacheDir(t *testing.T, rootDir string, testName string, templateFiles map[string]string) string {
t.Helper()
hash := hashTemplateFilesAndTestName(t, testName, templateFiles)
dir := filepath.Join(rootDir, hash[:12])
return dir
}
// Ensures Terraform providers are downloaded and cached locally in a unique directory for the test.
// Uses `terraform init` then `mirror` to populate the cache if needed.
// Returns the cache directory path.
func downloadProviders(t *testing.T, rootDir string, testName string, templateFiles map[string]string) string {
t.Helper()
dir := getTestCacheDir(t, rootDir, testName, templateFiles)
if _, err := os.Stat(dir); err == nil {
t.Logf("%s: using cached terraform providers", testName)
return dir
}
filesDir := filepath.Join(dir, cacheTemplateFilesDirName)
defer func() {
// The files dir will contain a copy of terraform providers generated
// by the terraform init command. We don't want to persist them since
// we already have a registry mirror in the providers dir.
if err := os.RemoveAll(filesDir); err != nil {
t.Logf("failed to remove files dir %s: %s", filesDir, err)
}
if !t.Failed() {
return
}
// If `downloadProviders` function failed, clean up the cache dir.
// We don't want to leave it around because it may be incomplete or corrupted.
if err := os.RemoveAll(dir); err != nil {
t.Logf("failed to remove dir %s: %s", dir, err)
}
}()
require.NoError(t, os.MkdirAll(filesDir, 0o700))
for fileName, file := range templateFiles {
filePath := filepath.Join(filesDir, fileName)
require.NoError(t, os.MkdirAll(filepath.Dir(filePath), 0o700))
require.NoError(t, os.WriteFile(filePath, []byte(file), 0o600))
}
providersDir := filepath.Join(dir, cacheProvidersDirName)
require.NoError(t, os.MkdirAll(providersDir, 0o700))
// We need to run init because if a test uses modules in its template,
// the mirror command will fail without it.
runCmd(t, filesDir, "terraform", "init")
// Now, mirror the providers into `providersDir`. We use this explicit mirror
// instead of relying only on the standard Terraform plugin cache.
//
// Why? Because this mirror, when used with the CLI config from `writeCliConfig`,
// prevents Terraform from hitting the network registry during `plan`. This cuts
// down on network calls, making CI tests less flaky.
//
// In contrast, the standard cache *still* contacts the registry for metadata
// during `init`, even if the plugins are already cached locally - see link below.
//
// Ref: https://developer.hashicorp.com/terraform/cli/config/config-file#provider-plugin-cache
// > When a plugin cache directory is enabled, the terraform init command will
// > still use the configured or implied installation methods to obtain metadata
// > about which plugins are available
runCmd(t, filesDir, "terraform", "providers", "mirror", providersDir)
return dir
}
// Caches providers locally and generates a Terraform CLI config to use *only* that cache.
// This setup prevents network access for providers during `terraform init`, improving reliability
// in subsequent test runs.
// Returns the path to the generated CLI config file.
func cacheProviders(t *testing.T, rootDir string, testName string, templateFiles map[string]string) string {
t.Helper()
providersParentDir := downloadProviders(t, rootDir, testName, templateFiles)
cliConfigPath := writeCliConfig(t, providersParentDir)
return cliConfigPath
}
func readProvisionLog(t *testing.T, response proto.DRPCProvisioner_SessionClient) string {
var logBuf strings.Builder
for {
@@ -1177,7 +1011,7 @@ func TestProvision(t *testing.T) {
cacheRootDir := filepath.Join(testutil.PersistentCacheDir(t), "terraform_provision_test")
expectedCacheDirs := make(map[string]bool)
for _, testCase := range testCases {
cacheDir := getTestCacheDir(t, cacheRootDir, testCase.Name, testCase.Files)
cacheDir := testutil.GetTestTFCacheDir(t, cacheRootDir, testCase.Name, testCase.Files)
expectedCacheDirs[cacheDir] = true
}
currentCacheDirs, err := filepath.Glob(filepath.Join(cacheRootDir, "*"))
@@ -1199,7 +1033,7 @@ func TestProvision(t *testing.T) {
cliConfigPath := ""
if !testCase.SkipCacheProviders {
cliConfigPath = cacheProviders(
cliConfigPath = testutil.CacheTFProviders(
t,
cacheRootDir,
testCase.Name,
testutil/terraform_cache.go
+185
View File
@@ -0,0 +1,185 @@
//go:build linux || darwin
package testutil
import (
"bytes"
"crypto/sha256"
"encoding/hex"
"fmt"
"os"
"os/exec"
"path/filepath"
"sort"
"strings"
"testing"
"github.com/stretchr/testify/require"
)
const (
// terraformConfigFileName is the name of the Terraform CLI config file.
terraformConfigFileName = "terraform.rc"
// cacheProvidersDirName is the subdirectory name for the provider mirror.
cacheProvidersDirName = "providers"
// cacheTemplateFilesDirName is the subdirectory name for template files.
cacheTemplateFilesDirName = "files"
)
// hashTemplateFilesAndTestName generates a unique hash based on test name and template files.
func hashTemplateFilesAndTestName(t *testing.T, testName string, templateFiles map[string]string) string {
t.Helper()
sortedFileNames := make([]string, 0, len(templateFiles))
for fileName := range templateFiles {
sortedFileNames = append(sortedFileNames, fileName)
}
sort.Strings(sortedFileNames)
// Inserting a delimiter between the file name and the file content
// ensures that a file named `ab` with content `cd`
// will not hash to the same value as a file named `abc` with content `d`.
// This can still happen if the file name or content include the delimiter,
// but hopefully they won't.
delimiter := []byte("🎉 🌱 🌷")
hasher := sha256.New()
for _, fileName := range sortedFileNames {
file := templateFiles[fileName]
_, err := hasher.Write([]byte(fileName))
require.NoError(t, err)
_, err = hasher.Write(delimiter)
require.NoError(t, err)
_, err = hasher.Write([]byte(file))
require.NoError(t, err)
}
_, err := hasher.Write(delimiter)
require.NoError(t, err)
_, err = hasher.Write([]byte(testName))
require.NoError(t, err)
return hex.EncodeToString(hasher.Sum(nil))
}
// WriteTFCliConfig writes a Terraform CLI config file (`terraform.rc`) in `dir` to enforce using the local provider mirror.
// This blocks network access for providers, forcing Terraform to use only what's cached in `dir`.
// Returns the path to the generated config file.
func WriteTFCliConfig(t *testing.T, dir string) string {
t.Helper()
cliConfigPath := filepath.Join(dir, terraformConfigFileName)
require.NoError(t, os.MkdirAll(filepath.Dir(cliConfigPath), 0o700))
content := fmt.Sprintf(`
provider_installation {
filesystem_mirror {
path = "%s"
include = ["*/*"]
}
direct {
exclude = ["*/*"]
}
}
`, filepath.Join(dir, cacheProvidersDirName))
require.NoError(t, os.WriteFile(cliConfigPath, []byte(content), 0o600))
return cliConfigPath
}
func runCmd(t *testing.T, dir string, args ...string) {
t.Helper()
stdout, stderr := bytes.NewBuffer(nil), bytes.NewBuffer(nil)
cmd := exec.Command(args[0], args[1:]...) //#nosec
cmd.Dir = dir
cmd.Stdout = stdout
cmd.Stderr = stderr
if err := cmd.Run(); err != nil {
t.Fatalf("failed to run %s: %s\nstdout: %s\nstderr: %s", strings.Join(args, " "), err, stdout.String(), stderr.String())
}
}
// GetTestTFCacheDir returns a unique cache directory path based on the test name and template files.
// Each test gets a unique cache dir based on its name and template files.
// This ensures that tests can download providers in parallel and that they
// will redownload providers if the template files change.
func GetTestTFCacheDir(t *testing.T, rootDir string, testName string, templateFiles map[string]string) string {
t.Helper()
hash := hashTemplateFilesAndTestName(t, testName, templateFiles)
dir := filepath.Join(rootDir, hash[:12])
return dir
}
// DownloadTFProviders ensures Terraform providers are downloaded and cached locally in a unique directory for the test.
// Uses `terraform init` then `mirror` to populate the cache if needed.
// Returns the cache directory path.
func DownloadTFProviders(t *testing.T, rootDir string, testName string, templateFiles map[string]string) string {
t.Helper()
dir := GetTestTFCacheDir(t, rootDir, testName, templateFiles)
if _, err := os.Stat(dir); err == nil {
t.Logf("%s: using cached terraform providers", testName)
return dir
}
filesDir := filepath.Join(dir, cacheTemplateFilesDirName)
defer func() {
// The files dir will contain a copy of terraform providers generated
// by the terraform init command. We don't want to persist them since
// we already have a registry mirror in the providers dir.
if err := os.RemoveAll(filesDir); err != nil {
t.Logf("failed to remove files dir %s: %s", filesDir, err)
}
if !t.Failed() {
return
}
// If `DownloadTFProviders` function failed, clean up the cache dir.
// We don't want to leave it around because it may be incomplete or corrupted.
if err := os.RemoveAll(dir); err != nil {
t.Logf("failed to remove dir %s: %s", dir, err)
}
}()
require.NoError(t, os.MkdirAll(filesDir, 0o700))
for fileName, file := range templateFiles {
filePath := filepath.Join(filesDir, fileName)
require.NoError(t, os.MkdirAll(filepath.Dir(filePath), 0o700))
require.NoError(t, os.WriteFile(filePath, []byte(file), 0o600))
}
providersDir := filepath.Join(dir, cacheProvidersDirName)
require.NoError(t, os.MkdirAll(providersDir, 0o700))
// We need to run init because if a test uses modules in its template,
// the mirror command will fail without it.
runCmd(t, filesDir, "terraform", "init")
// Now, mirror the providers into `providersDir`. We use this explicit mirror
// instead of relying only on the standard Terraform plugin cache.
//
// Why? Because this mirror, when used with the CLI config from `WriteCliConfig`,
// prevents Terraform from hitting the network registry during `plan`. This cuts
// down on network calls, making CI tests less flaky.
//
// In contrast, the standard cache *still* contacts the registry for metadata
// during `init`, even if the plugins are already cached locally - see link below.
//
// Ref: https://developer.hashicorp.com/terraform/cli/config/config-file#provider-plugin-cache
// > When a plugin cache directory is enabled, the terraform init command will
// > still use the configured or implied installation methods to obtain metadata
// > about which plugins are available
runCmd(t, filesDir, "terraform", "providers", "mirror", providersDir)
return dir
}
// CacheTFProviders caches providers locally and generates a Terraform CLI config to use *only* that cache.
// This setup prevents network access for providers during `terraform init`, improving reliability
// in subsequent test runs.
// Returns the path to the generated CLI config file.
func CacheTFProviders(t *testing.T, rootDir string, testName string, templateFiles map[string]string) string {
t.Helper()
providersParentDir := DownloadTFProviders(t, rootDir, testName, templateFiles)
cliConfigPath := WriteTFCliConfig(t, providersParentDir)
return cliConfigPath
}