feat(scaletest): add runner for prebuilds (#20571)

Relates to https://github.com/coder/internal/issues/914

Adds a runner for scaletesting prebuilds. The runner uploads a no-op template with prebuilds, watches for the corresponding workspaces to be created, and then does the same to tear them down. I didn't originally plan on having metrics for the teardown, but I figured we might as well as it's still the same prebuilds reconciliation loop mechanism being tested.

enterprise/scaletest/prebuilds/run_test.go

@@ -0,0 +1,141 @@
+package prebuilds_test
+
+import (
+	"io"
+	"strconv"
+	"sync"
+	"testing"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/sync/errgroup"
+
+	"github.com/coder/coder/v2/codersdk"
+	"github.com/coder/coder/v2/enterprise/coderd/coderdenttest"
+	"github.com/coder/coder/v2/enterprise/coderd/license"
+	"github.com/coder/coder/v2/scaletest/prebuilds"
+	"github.com/coder/coder/v2/testutil"
+	"github.com/coder/quartz"
+)
+
+func TestRun(t *testing.T) {
+	t.Parallel()
+
+	t.Skip("This test takes several minutes to run, and is intended as a manual regression test")
+
+	ctx := testutil.Context(t, testutil.WaitSuperLong*3)
+
+	client, user := coderdenttest.New(t, &coderdenttest.Options{
+		LicenseOptions: &coderdenttest.LicenseOptions{
+			Features: license.Features{
+				codersdk.FeatureWorkspacePrebuilds:         1,
+				codersdk.FeatureExternalProvisionerDaemons: 1,
+			},
+		},
+	})
+
+	// This is a real Terraform provisioner
+	_ = coderdenttest.NewExternalProvisionerDaemonTerraform(t, client, user.OrganizationID, nil)
+
+	numTemplates := 2
+	numPresets := 1
+	numPresetPrebuilds := 1
+
+	//nolint:gocritic // It's fine to use the owner user to pause prebuilds
+	err := client.PutPrebuildsSettings(ctx, codersdk.PrebuildsSettings{
+		ReconciliationPaused: true,
+	})
+	require.NoError(t, err)
+
+	setupBarrier := new(sync.WaitGroup)
+	setupBarrier.Add(numTemplates)
+	creationBarrier := new(sync.WaitGroup)
+	creationBarrier.Add(numTemplates)
+	deletionSetupBarrier := new(sync.WaitGroup)
+	deletionSetupBarrier.Add(1)
+	deletionBarrier := new(sync.WaitGroup)
+	deletionBarrier.Add(numTemplates)
+
+	metrics := prebuilds.NewMetrics(prometheus.NewRegistry())
+
+	eg, runCtx := errgroup.WithContext(ctx)
+
+	runners := make([]*prebuilds.Runner, 0, numTemplates)
+	for i := range numTemplates {
+		cfg := prebuilds.Config{
+			OrganizationID:            user.OrganizationID,
+			NumPresets:                numPresets,
+			NumPresetPrebuilds:        numPresetPrebuilds,
+			TemplateVersionJobTimeout: testutil.WaitSuperLong * 2,
+			PrebuildWorkspaceTimeout:  testutil.WaitSuperLong * 2,
+			Metrics:                   metrics,
+			SetupBarrier:              setupBarrier,
+			CreationBarrier:           creationBarrier,
+			DeletionSetupBarrier:      deletionSetupBarrier,
+			DeletionBarrier:           deletionBarrier,
+			Clock:                     quartz.NewReal(),
+		}
+		err := cfg.Validate()
+		require.NoError(t, err)
+
+		runner := prebuilds.NewRunner(client, cfg)
+		runners = append(runners, runner)
+		eg.Go(func() error {
+			return runner.Run(runCtx, strconv.Itoa(i), io.Discard)
+		})
+	}
+
+	// Wait for all runners to reach the setup barrier (templates created)
+	setupBarrier.Wait()
+
+	// Resume prebuilds to trigger prebuild creation
+	err = client.PutPrebuildsSettings(ctx, codersdk.PrebuildsSettings{
+		ReconciliationPaused: false,
+	})
+	require.NoError(t, err)
+
+	// Wait for all runners to reach the creation barrier (prebuilds created)
+	creationBarrier.Wait()
+
+	//nolint:gocritic // Owner user is fine here as we want to view all workspaces
+	workspaces, err := client.Workspaces(ctx, codersdk.WorkspaceFilter{})
+	require.NoError(t, err)
+	expectedWorkspaces := numTemplates * numPresets * numPresetPrebuilds
+	require.Equal(t, workspaces.Count, expectedWorkspaces)
+
+	// Pause prebuilds before deletion setup
+	err = client.PutPrebuildsSettings(ctx, codersdk.PrebuildsSettings{
+		ReconciliationPaused: true,
+	})
+	require.NoError(t, err)
+
+	// Signal runners that prebuilds are paused and they can prepare for deletion
+	deletionSetupBarrier.Done()
+
+	// Wait for all runners to reach the deletion barrier (template versions updated to 0 prebuilds)
+	deletionBarrier.Wait()
+
+	// Resume prebuilds to trigger prebuild deletion
+	err = client.PutPrebuildsSettings(ctx, codersdk.PrebuildsSettings{
+		ReconciliationPaused: false,
+	})
+	require.NoError(t, err)
+
+	err = eg.Wait()
+	require.NoError(t, err)
+
+	//nolint:gocritic // Owner user is fine here as we want to view all workspaces
+	workspaces, err = client.Workspaces(ctx, codersdk.WorkspaceFilter{})
+	require.NoError(t, err)
+	require.Equal(t, workspaces.Count, 0)
+
+	cleanupEg, cleanupCtx := errgroup.WithContext(ctx)
+	for i, runner := range runners {
+		cleanupEg.Go(func() error {
+			return runner.Cleanup(cleanupCtx, strconv.Itoa(i), io.Discard)
+		})
+	}
+
+	err = cleanupEg.Wait()
+	require.NoError(t, err)
+}

scaletest/dynamicparameters/template.go

@@ -1,15 +1,12 @@
 package dynamicparameters
 
 import (
-	"archive/tar"
 	"bytes"
 	"context"
 	_ "embed"
 	"encoding/json"
 	"fmt"
 	"io"
-	"path/filepath"
-	"slices"
 	"strings"
 	"text/template"
 	"time"
@@ -20,6 +17,7 @@ import (
 	"cdr.dev/slog"
 	"github.com/coder/coder/v2/codersdk"
 	"github.com/coder/coder/v2/cryptorand"
+	"github.com/coder/coder/v2/scaletest/loadtestutil"
 	"github.com/coder/quartz"
 )
 
@@ -89,48 +87,6 @@ func GetModuleFiles() map[string][]byte {
 	}
 }
 
-func createTarFromFiles(files map[string][]byte) ([]byte, error) {
-	buf := new(bytes.Buffer)
-	writer := tar.NewWriter(buf)
-	dirs := []string{}
-	for name, content := range files {
-		// We need to add directories before any files that use them. But, we only need to do this
-		// once.
-		dir := filepath.Dir(name)
-		if dir != "." && !slices.Contains(dirs, dir) {
-			dirs = append(dirs, dir)
-			err := writer.WriteHeader(&tar.Header{
-				Name:     dir,
-				Mode:     0o755,
-				Typeflag: tar.TypeDir,
-			})
-			if err != nil {
-				return nil, err
-			}
-		}
-
-		err := writer.WriteHeader(&tar.Header{
-			Name: name,
-			Size: int64(len(content)),
-			Mode: 0o644,
-		})
-		if err != nil {
-			return nil, err
-		}
-
-		_, err = writer.Write(content)
-		if err != nil {
-			return nil, err
-		}
-	}
-	// `writer.Close()` function flushes the writer buffer, and adds extra padding to create a legal tarball.
-	err := writer.Close()
-	if err != nil {
-		return nil, err
-	}
-	return buf.Bytes(), nil
-}
-
 func TemplateTarData() ([]byte, error) {
 	mainTF, err := TemplateContent()
 	if err != nil {
@@ -144,7 +100,7 @@ func TemplateTarData() ([]byte, error) {
 	for k, v := range moduleFiles {
 		files[k] = v
 	}
-	tarData, err := createTarFromFiles(files)
+	tarData, err := loadtestutil.CreateTarFromFiles(files)
 	if err != nil {
 		return nil, xerrors.Errorf("failed to create tarball: %w", err)
 	}

scaletest/loadtestutil/files.go

@@ -0,0 +1,50 @@
+package loadtestutil
+
+import (
+	"archive/tar"
+	"bytes"
+	"path/filepath"
+	"slices"
+)
+
+func CreateTarFromFiles(files map[string][]byte) ([]byte, error) {
+	buf := new(bytes.Buffer)
+	writer := tar.NewWriter(buf)
+	dirs := []string{}
+	for name, content := range files {
+		// We need to add directories before any files that use them. But, we only need to do this
+		// once.
+		dir := filepath.Dir(name)
+		if dir != "." && !slices.Contains(dirs, dir) {
+			dirs = append(dirs, dir)
+			err := writer.WriteHeader(&tar.Header{
+				Name:     dir,
+				Mode:     0o755,
+				Typeflag: tar.TypeDir,
+			})
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		err := writer.WriteHeader(&tar.Header{
+			Name: name,
+			Size: int64(len(content)),
+			Mode: 0o644,
+		})
+		if err != nil {
+			return nil, err
+		}
+
+		_, err = writer.Write(content)
+		if err != nil {
+			return nil, err
+		}
+	}
+	// `writer.Close()` function flushes the writer buffer, and adds extra padding to create a legal tarball.
+	err := writer.Close()
+	if err != nil {
+		return nil, err
+	}
+	return buf.Bytes(), nil
+}

scaletest/prebuilds/config.go

@@ -0,0 +1,86 @@
+package prebuilds
+
+import (
+	"sync"
+	"time"
+
+	"github.com/google/uuid"
+	"golang.org/x/xerrors"
+
+	"github.com/coder/quartz"
+)
+
+type Config struct {
+	// OrganizationID is the ID of the organization to create the prebuilds in.
+	OrganizationID uuid.UUID `json:"organization_id"`
+	// NumPresets is the number of presets the template should have.
+	NumPresets int `json:"num_presets"`
+	// NumPresetPrebuilds is the number of prebuilds per preset.
+	// Total prebuilds = NumPresets * NumPresetPrebuilds
+	NumPresetPrebuilds int `json:"num_preset_prebuilds"`
+
+	// TemplateVersionJobTimeout is how long to wait for template version
+	// provisioning jobs to complete.
+	TemplateVersionJobTimeout time.Duration `json:"template_version_job_timeout"`
+
+	// PrebuildWorkspaceTimeout is how long to wait for all prebuild
+	// workspaces to be created and completed.
+	PrebuildWorkspaceTimeout time.Duration `json:"prebuild_workspace_timeout"`
+
+	Metrics *Metrics `json:"-"`
+
+	// SetupBarrier is used to ensure all templates have been created
+	// before unpausing prebuilds.
+	SetupBarrier *sync.WaitGroup `json:"-"`
+
+	// CreationBarrier is used to ensure all prebuild creation has completed
+	// before pausing prebuilds for deletion.
+	CreationBarrier *sync.WaitGroup `json:"-"`
+
+	// DeletionSetupBarrier is used by the runner owner (CLI/test) to signal when
+	// prebuilds have been paused, allowing runners to create new template versions
+	// with 0 prebuilds. Only the owner calls Done(), runners only Wait().
+	DeletionSetupBarrier *sync.WaitGroup `json:"-"`
+
+	// DeletionBarrier is used to ensure all templates have been updated
+	// with 0 prebuilds before resuming prebuilds.
+	DeletionBarrier *sync.WaitGroup `json:"-"`
+
+	Clock quartz.Clock `json:"-"`
+}
+
+func (c Config) Validate() error {
+	if c.TemplateVersionJobTimeout <= 0 {
+		return xerrors.New("template_version_job_timeout must be greater than 0")
+	}
+
+	if c.PrebuildWorkspaceTimeout <= 0 {
+		return xerrors.New("prebuild_workspace_timeout must be greater than 0")
+	}
+
+	if c.SetupBarrier == nil {
+		return xerrors.New("setup barrier must be set")
+	}
+
+	if c.CreationBarrier == nil {
+		return xerrors.New("creation barrier must be set")
+	}
+
+	if c.DeletionSetupBarrier == nil {
+		return xerrors.New("deletion setup barrier must be set")
+	}
+
+	if c.DeletionBarrier == nil {
+		return xerrors.New("deletion barrier must be set")
+	}
+
+	if c.Metrics == nil {
+		return xerrors.New("metrics must be set")
+	}
+
+	if c.Clock == nil {
+		return xerrors.New("clock must be set")
+	}
+
+	return nil
+}

scaletest/prebuilds/metrics.go

@@ -0,0 +1,125 @@
+package prebuilds
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+type Metrics struct {
+	PrebuildJobsCreated   prometheus.GaugeVec
+	PrebuildJobsRunning   prometheus.GaugeVec
+	PrebuildJobsFailed    prometheus.GaugeVec
+	PrebuildJobsCompleted prometheus.GaugeVec
+
+	PrebuildDeletionJobsCreated   prometheus.GaugeVec
+	PrebuildDeletionJobsRunning   prometheus.GaugeVec
+	PrebuildDeletionJobsFailed    prometheus.GaugeVec
+	PrebuildDeletionJobsCompleted prometheus.GaugeVec
+
+	PrebuildErrorsTotal prometheus.CounterVec
+}
+
+func NewMetrics(reg prometheus.Registerer) *Metrics {
+	m := &Metrics{
+		PrebuildJobsCreated: *prometheus.NewGaugeVec(prometheus.GaugeOpts{
+			Namespace: "coderd",
+			Subsystem: "scaletest",
+			Name:      "prebuild_jobs_created",
+			Help:      "Number of prebuild jobs that have been created.",
+		}, []string{"template_name"}),
+		PrebuildJobsRunning: *prometheus.NewGaugeVec(prometheus.GaugeOpts{
+			Namespace: "coderd",
+			Subsystem: "scaletest",
+			Name:      "prebuild_jobs_running",
+			Help:      "Number of prebuild jobs that are currently running.",
+		}, []string{"template_name"}),
+		PrebuildJobsFailed: *prometheus.NewGaugeVec(prometheus.GaugeOpts{
+			Namespace: "coderd",
+			Subsystem: "scaletest",
+			Name:      "prebuild_jobs_failed",
+			Help:      "Number of prebuild jobs that have failed.",
+		}, []string{"template_name"}),
+		PrebuildJobsCompleted: *prometheus.NewGaugeVec(prometheus.GaugeOpts{
+			Namespace: "coderd",
+			Subsystem: "scaletest",
+			Name:      "prebuild_jobs_completed",
+			Help:      "Number of prebuild jobs that have completed successfully.",
+		}, []string{"template_name"}),
+		PrebuildDeletionJobsCreated: *prometheus.NewGaugeVec(prometheus.GaugeOpts{
+			Namespace: "coderd",
+			Subsystem: "scaletest",
+			Name:      "prebuild_deletion_jobs_created",
+			Help:      "Number of prebuild deletion jobs that have been created.",
+		}, []string{"template_name"}),
+		PrebuildDeletionJobsRunning: *prometheus.NewGaugeVec(prometheus.GaugeOpts{
+			Namespace: "coderd",
+			Subsystem: "scaletest",
+			Name:      "prebuild_deletion_jobs_running",
+			Help:      "Number of prebuild deletion jobs that are currently running.",
+		}, []string{"template_name"}),
+		PrebuildDeletionJobsFailed: *prometheus.NewGaugeVec(prometheus.GaugeOpts{
+			Namespace: "coderd",
+			Subsystem: "scaletest",
+			Name:      "prebuild_deletion_jobs_failed",
+			Help:      "Number of prebuild deletion jobs that have failed.",
+		}, []string{"template_name"}),
+		PrebuildDeletionJobsCompleted: *prometheus.NewGaugeVec(prometheus.GaugeOpts{
+			Namespace: "coderd",
+			Subsystem: "scaletest",
+			Name:      "prebuild_deletion_jobs_completed",
+			Help:      "Number of prebuild deletion jobs that have completed successfully.",
+		}, []string{"template_name"}),
+		PrebuildErrorsTotal: *prometheus.NewCounterVec(prometheus.CounterOpts{
+			Namespace: "coderd",
+			Subsystem: "scaletest",
+			Name:      "prebuild_errors_total",
+			Help:      "Total number of prebuild errors",
+		}, []string{"template_name", "action"}),
+	}
+
+	reg.MustRegister(m.PrebuildJobsCreated)
+	reg.MustRegister(m.PrebuildJobsRunning)
+	reg.MustRegister(m.PrebuildJobsFailed)
+	reg.MustRegister(m.PrebuildJobsCompleted)
+	reg.MustRegister(m.PrebuildDeletionJobsCreated)
+	reg.MustRegister(m.PrebuildDeletionJobsRunning)
+	reg.MustRegister(m.PrebuildDeletionJobsFailed)
+	reg.MustRegister(m.PrebuildDeletionJobsCompleted)
+	reg.MustRegister(m.PrebuildErrorsTotal)
+	return m
+}
+
+func (m *Metrics) SetJobsCreated(count int, templateName string) {
+	m.PrebuildJobsCreated.WithLabelValues(templateName).Set(float64(count))
+}
+
+func (m *Metrics) SetJobsRunning(count int, templateName string) {
+	m.PrebuildJobsRunning.WithLabelValues(templateName).Set(float64(count))
+}
+
+func (m *Metrics) SetJobsFailed(count int, templateName string) {
+	m.PrebuildJobsFailed.WithLabelValues(templateName).Set(float64(count))
+}
+
+func (m *Metrics) SetJobsCompleted(count int, templateName string) {
+	m.PrebuildJobsCompleted.WithLabelValues(templateName).Set(float64(count))
+}
+
+func (m *Metrics) SetDeletionJobsCreated(count int, templateName string) {
+	m.PrebuildDeletionJobsCreated.WithLabelValues(templateName).Set(float64(count))
+}
+
+func (m *Metrics) SetDeletionJobsRunning(count int, templateName string) {
+	m.PrebuildDeletionJobsRunning.WithLabelValues(templateName).Set(float64(count))
+}
+
+func (m *Metrics) SetDeletionJobsFailed(count int, templateName string) {
+	m.PrebuildDeletionJobsFailed.WithLabelValues(templateName).Set(float64(count))
+}
+
+func (m *Metrics) SetDeletionJobsCompleted(count int, templateName string) {
+	m.PrebuildDeletionJobsCompleted.WithLabelValues(templateName).Set(float64(count))
+}
+
+func (m *Metrics) AddError(templateName string, action string) {
+	m.PrebuildErrorsTotal.WithLabelValues(templateName, action).Inc()
+}

scaletest/prebuilds/run.go

@@ -0,0 +1,343 @@
+package prebuilds
+
+import (
+	"bytes"
+	"context"
+	_ "embed"
+	"html/template"
+	"io"
+	"time"
+
+	"golang.org/x/xerrors"
+
+	"github.com/google/uuid"
+
+	"cdr.dev/slog"
+	"cdr.dev/slog/sloggers/sloghuman"
+	"github.com/coder/coder/v2/coderd/tracing"
+	"github.com/coder/coder/v2/codersdk"
+	"github.com/coder/coder/v2/scaletest/harness"
+	"github.com/coder/coder/v2/scaletest/loadtestutil"
+)
+
+type Runner struct {
+	client *codersdk.Client
+	cfg    Config
+
+	template codersdk.Template
+}
+
+var (
+	_ harness.Runnable  = &Runner{}
+	_ harness.Cleanable = &Runner{}
+)
+
+func NewRunner(client *codersdk.Client, cfg Config) *Runner {
+	return &Runner{
+		client: client,
+		cfg:    cfg,
+	}
+}
+
+func (r *Runner) Run(ctx context.Context, id string, logs io.Writer) error {
+	ctx, span := tracing.StartSpan(ctx)
+	defer span.End()
+
+	reachedSetupBarrier := false
+	reachedCreationBarrier := false
+	reachedDeletionBarrier := false
+	defer func() {
+		if !reachedSetupBarrier {
+			r.cfg.SetupBarrier.Done()
+		}
+		if !reachedCreationBarrier {
+			r.cfg.CreationBarrier.Done()
+		}
+		if !reachedDeletionBarrier {
+			r.cfg.DeletionBarrier.Done()
+		}
+	}()
+
+	logs = loadtestutil.NewSyncWriter(logs)
+	logger := slog.Make(sloghuman.Sink(logs)).Leveled(slog.LevelDebug)
+	r.client.SetLogger(logger)
+	r.client.SetLogBodies(true)
+
+	templateName := "scaletest-prebuilds-template-" + id
+
+	version, err := r.createTemplateVersion(ctx, uuid.Nil, r.cfg.NumPresets, r.cfg.NumPresetPrebuilds)
+	if err != nil {
+		r.cfg.Metrics.AddError(templateName, "create_template_version")
+		return err
+	}
+
+	templateReq := codersdk.CreateTemplateRequest{
+		Name:        templateName,
+		Description: "`coder exp scaletest prebuilds` template",
+		VersionID:   version.ID,
+	}
+	templ, err := r.client.CreateTemplate(ctx, r.cfg.OrganizationID, templateReq)
+	if err != nil {
+		r.cfg.Metrics.AddError(templateName, "create_template")
+		return xerrors.Errorf("create template: %w", err)
+	}
+	logger.Info(ctx, "created template", slog.F("template_id", templ.ID))
+
+	r.template = templ
+
+	logger.Info(ctx, "waiting for all runners to reach setup barrier")
+	reachedSetupBarrier = true
+	r.cfg.SetupBarrier.Done()
+	r.cfg.SetupBarrier.Wait()
+	logger.Info(ctx, "all runners reached setup barrier, proceeding with prebuild creation test")
+
+	err = r.measureCreation(ctx, logger)
+	if err != nil {
+		return err
+	}
+
+	logger.Info(ctx, "waiting for all runners to reach creation barrier")
+	reachedCreationBarrier = true
+	r.cfg.CreationBarrier.Done()
+	r.cfg.CreationBarrier.Wait()
+	logger.Info(ctx, "all runners reached creation barrier")
+
+	logger.Info(ctx, "waiting for runner owner to pause prebuilds (deletion setup barrier)")
+	r.cfg.DeletionSetupBarrier.Wait()
+	logger.Info(ctx, "prebuilds paused, preparing for deletion")
+
+	// Now prepare for deletion by creating an empty template version
+	// At this point, prebuilds should be paused by the caller
+	logger.Info(ctx, "creating empty template version for deletion")
+	emptyVersion, err := r.createTemplateVersion(ctx, r.template.ID, 0, 0)
+	if err != nil {
+		r.cfg.Metrics.AddError(r.template.Name, "create_empty_template_version")
+		return xerrors.Errorf("create empty template version for deletion: %w", err)
+	}
+
+	err = r.client.UpdateActiveTemplateVersion(ctx, r.template.ID, codersdk.UpdateActiveTemplateVersion{
+		ID: emptyVersion.ID,
+	})
+	if err != nil {
+		r.cfg.Metrics.AddError(r.template.Name, "update_active_template_version")
+		return xerrors.Errorf("update active template version to empty for deletion: %w", err)
+	}
+
+	logger.Info(ctx, "waiting for all runners to reach deletion barrier")
+	reachedDeletionBarrier = true
+	r.cfg.DeletionBarrier.Done()
+	r.cfg.DeletionBarrier.Wait()
+	logger.Info(ctx, "all runners reached deletion barrier, proceeding with prebuild deletion test")
+
+	err = r.measureDeletion(ctx, logger)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (r *Runner) measureCreation(ctx context.Context, logger slog.Logger) error {
+	testStartTime := time.Now().UTC()
+	const workspacesPollInterval = 500 * time.Millisecond
+
+	targetNumWorkspaces := r.cfg.NumPresets * r.cfg.NumPresetPrebuilds
+
+	workspacesCtx, cancel := context.WithTimeout(ctx, r.cfg.PrebuildWorkspaceTimeout)
+	defer cancel()
+
+	tkr := r.cfg.Clock.TickerFunc(workspacesCtx, workspacesPollInterval, func() error {
+		workspaces, err := r.client.Workspaces(workspacesCtx, codersdk.WorkspaceFilter{
+			Template: r.template.Name,
+		})
+		if err != nil {
+			return xerrors.Errorf("list workspaces: %w", err)
+		}
+
+		createdCount := len(workspaces.Workspaces)
+		runningCount := 0
+		failedCount := 0
+		succeededCount := 0
+
+		for _, ws := range workspaces.Workspaces {
+			switch ws.LatestBuild.Job.Status {
+			case codersdk.ProvisionerJobRunning:
+				runningCount++
+			case codersdk.ProvisionerJobFailed, codersdk.ProvisionerJobCanceled:
+				failedCount++
+			case codersdk.ProvisionerJobSucceeded:
+				succeededCount++
+			}
+		}
+
+		r.cfg.Metrics.SetJobsCreated(createdCount, r.template.Name)
+		r.cfg.Metrics.SetJobsRunning(runningCount, r.template.Name)
+		r.cfg.Metrics.SetJobsFailed(failedCount, r.template.Name)
+		r.cfg.Metrics.SetJobsCompleted(succeededCount, r.template.Name)
+
+		if succeededCount >= targetNumWorkspaces {
+			// All jobs succeeded
+			return errTickerDone
+		}
+
+		return nil
+	}, "waitForPrebuildWorkspaces")
+	err := tkr.Wait()
+	if !xerrors.Is(err, errTickerDone) {
+		r.cfg.Metrics.AddError(r.template.Name, "wait_for_workspaces")
+		return xerrors.Errorf("wait for workspaces: %w", err)
+	}
+
+	logger.Info(ctx, "all prebuild workspaces created successfully", slog.F("template_name", r.template.Name), slog.F("duration", time.Since(testStartTime).String()))
+	return nil
+}
+
+func (r *Runner) measureDeletion(ctx context.Context, logger slog.Logger) error {
+	deletionStartTime := time.Now().UTC()
+	const deletionPollInterval = 500 * time.Millisecond
+
+	targetNumWorkspaces := r.cfg.NumPresets * r.cfg.NumPresetPrebuilds
+
+	deletionCtx, cancel := context.WithTimeout(ctx, r.cfg.PrebuildWorkspaceTimeout)
+	defer cancel()
+
+	tkr := r.cfg.Clock.TickerFunc(deletionCtx, deletionPollInterval, func() error {
+		workspaces, err := r.client.Workspaces(deletionCtx, codersdk.WorkspaceFilter{
+			Template: r.template.Name,
+		})
+		if err != nil {
+			return xerrors.Errorf("list workspaces: %w", err)
+		}
+
+		createdCount := 0
+		runningCount := 0
+		failedCount := 0
+
+		for _, ws := range workspaces.Workspaces {
+			if ws.LatestBuild.Transition == codersdk.WorkspaceTransitionDelete {
+				createdCount++
+				switch ws.LatestBuild.Job.Status {
+				case codersdk.ProvisionerJobRunning:
+					runningCount++
+				case codersdk.ProvisionerJobFailed, codersdk.ProvisionerJobCanceled:
+					failedCount++
+				}
+			}
+		}
+
+		completedCount := targetNumWorkspaces - len(workspaces.Workspaces)
+		createdCount += completedCount
+
+		r.cfg.Metrics.SetDeletionJobsCreated(createdCount, r.template.Name)
+		r.cfg.Metrics.SetDeletionJobsRunning(runningCount, r.template.Name)
+		r.cfg.Metrics.SetDeletionJobsFailed(failedCount, r.template.Name)
+		r.cfg.Metrics.SetDeletionJobsCompleted(completedCount, r.template.Name)
+
+		if len(workspaces.Workspaces) == 0 {
+			return errTickerDone
+		}
+
+		return nil
+	}, "waitForPrebuildWorkspacesDeletion")
+	err := tkr.Wait()
+	if !xerrors.Is(err, errTickerDone) {
+		r.cfg.Metrics.AddError(r.template.Name, "wait_for_workspace_deletion")
+		return xerrors.Errorf("wait for workspace deletion: %w", err)
+	}
+
+	logger.Info(ctx, "all prebuild workspaces deleted successfully", slog.F("template_name", r.template.Name), slog.F("duration", time.Since(deletionStartTime).String()))
+	return nil
+}
+
+func (r *Runner) createTemplateVersion(ctx context.Context, templateID uuid.UUID, numPresets, numPresetPrebuilds int) (codersdk.TemplateVersion, error) {
+	tarData, err := TemplateTarData(numPresets, numPresetPrebuilds)
+	if err != nil {
+		return codersdk.TemplateVersion{}, xerrors.Errorf("create prebuilds template tar: %w", err)
+	}
+	uploadResp, err := r.client.Upload(ctx, codersdk.ContentTypeTar, bytes.NewReader(tarData))
+	if err != nil {
+		return codersdk.TemplateVersion{}, xerrors.Errorf("upload prebuilds template tar: %w", err)
+	}
+
+	versionReq := codersdk.CreateTemplateVersionRequest{
+		TemplateID:    templateID,
+		FileID:        uploadResp.ID,
+		Message:       "Template version for scaletest prebuilds",
+		StorageMethod: codersdk.ProvisionerStorageMethodFile,
+		Provisioner:   codersdk.ProvisionerTypeTerraform,
+	}
+	version, err := r.client.CreateTemplateVersion(ctx, r.cfg.OrganizationID, versionReq)
+	if err != nil {
+		return codersdk.TemplateVersion{}, xerrors.Errorf("create template version: %w", err)
+	}
+	if version.MatchedProvisioners != nil && version.MatchedProvisioners.Count == 0 {
+		return codersdk.TemplateVersion{}, xerrors.Errorf("no provisioners matched for template version")
+	}
+
+	const pollInterval = 2 * time.Second
+	versionCtx, cancel := context.WithTimeout(ctx, r.cfg.TemplateVersionJobTimeout)
+	defer cancel()
+
+	tkr := r.cfg.Clock.TickerFunc(versionCtx, pollInterval, func() error {
+		version, err := r.client.TemplateVersion(versionCtx, version.ID)
+		if err != nil {
+			return xerrors.Errorf("get template version: %w", err)
+		}
+		switch version.Job.Status {
+		case codersdk.ProvisionerJobSucceeded:
+			return errTickerDone
+		case codersdk.ProvisionerJobPending, codersdk.ProvisionerJobRunning:
+			return nil
+		default:
+			return xerrors.Errorf("template version provisioning failed: status %s", version.Job.Status)
+		}
+	})
+	err = tkr.Wait()
+	if !xerrors.Is(err, errTickerDone) {
+		return codersdk.TemplateVersion{}, xerrors.Errorf("wait for template version provisioning: %w", err)
+	}
+	return version, nil
+}
+
+var errTickerDone = xerrors.New("done")
+
+func (r *Runner) Cleanup(ctx context.Context, _ string, logs io.Writer) error {
+	logs = loadtestutil.NewSyncWriter(logs)
+	logger := slog.Make(sloghuman.Sink(logs)).Leveled(slog.LevelDebug)
+
+	logger.Info(ctx, "deleting template", slog.F("template_name", r.template.Name))
+
+	err := r.client.DeleteTemplate(ctx, r.template.ID)
+	if err != nil {
+		return xerrors.Errorf("delete template: %w", err)
+	}
+
+	logger.Info(ctx, "template deleted successfully", slog.F("template_name", r.template.Name))
+	return nil
+}
+
+//go:embed tf/main.tf.tpl
+var templateContent string
+
+func TemplateTarData(numPresets, numPresetPrebuilds int) ([]byte, error) {
+	tmpl, err := template.New("prebuilds-template").Parse(templateContent)
+	if err != nil {
+		return nil, err
+	}
+	result := bytes.Buffer{}
+	err = tmpl.Execute(&result, map[string]int{
+		"NumPresets":         numPresets,
+		"NumPresetPrebuilds": numPresetPrebuilds,
+	})
+	if err != nil {
+		return nil, err
+	}
+	files := map[string][]byte{
+		"main.tf": result.Bytes(),
+	}
+	tarBytes, err := loadtestutil.CreateTarFromFiles(files)
+	if err != nil {
+		return nil, err
+	}
+	return tarBytes, nil
+}

scaletest/prebuilds/tf/main.tf.tpl

@@ -0,0 +1,18 @@
+terraform {
+  required_providers {
+    coder = {
+      source  = "coder/coder"
+      version = "2.5.3"
+    }
+  }
+}
+
+resource "null_resource" "workspace" {}
+
+data "coder_workspace_preset" "presets" {
+  count = {{.NumPresets}}
+  name  = "preset-${count.index + 1}"
+  prebuilds {
+    instances = {{.NumPresetPrebuilds}}
+  }
+}