petru/coder-server
1
0
Fork 0
Code Issues Pull Requests Actions 4 Packages Projects Releases Wiki Activity

chore(coderd/database/dbfake): add support for provisioner job timestamp control (#21944)

Browse Source 
Relates to https://github.com/coder/coder/pull/21922 /
https://github.com/coder/internal/issues/1259

* Adds `dbfake.BuilderOption func(*WorkspaceBuildBuilder)`
* Adds `BuilderOption` methods for setting various provisioner job
related fields on `WorkspaceBuildBuilder`.
* Migrates a number of existing tests that previously dependeded on
provisioner job timing to use these updated methods in the following
packages:
  * `coderd/jobreaper`
  * `coderd/notifications/reports`
  * `enterprise/coderd/schedule`
  * `enterprise/coderd/prebuilds`
  * `scripts/workspace-runtime-audit` 

🤖 Created using Mux (Opus 4.5)

---------

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
This commit is contained in:
Cian Johnston
2026-02-06 09:44:40 +00:00
committed by GitHub
parent fabb0b8344
commit 25a0c807cb
6 changed files with 412 additions and 518 deletions
Download Patch File Download Diff File Expand all files Collapse all files
coderd/database/dbfake/dbfake.go
+140 -11
View File
@@ -58,6 +58,61 @@ type WorkspaceBuildBuilder struct {
jobStatus database.ProvisionerJobStatus
taskAppID uuid.UUID
taskSeed database.TaskTable
// Individual timestamp fields for job customization.
jobCreatedAt time.Time
jobStartedAt time.Time
jobUpdatedAt time.Time
jobCompletedAt time.Time
jobError string // Error message for failed jobs
jobErrorCode string // Error code for failed jobs
}
// BuilderOption is a functional option for customizing job timestamps
// on status methods.
type BuilderOption func(*WorkspaceBuildBuilder)
// WithJobCreatedAt sets the CreatedAt timestamp for the provisioner job.
func WithJobCreatedAt(t time.Time) BuilderOption {
return func(b *WorkspaceBuildBuilder) {
b.jobCreatedAt = t
}
}
// WithJobStartedAt sets the StartedAt timestamp for the provisioner job.
func WithJobStartedAt(t time.Time) BuilderOption {
return func(b *WorkspaceBuildBuilder) {
b.jobStartedAt = t
}
}
// WithJobUpdatedAt sets the UpdatedAt timestamp for the provisioner job.
func WithJobUpdatedAt(t time.Time) BuilderOption {
return func(b *WorkspaceBuildBuilder) {
b.jobUpdatedAt = t
}
}
// WithJobCompletedAt sets the CompletedAt timestamp for the provisioner job.
func WithJobCompletedAt(t time.Time) BuilderOption {
return func(b *WorkspaceBuildBuilder) {
b.jobCompletedAt = t
}
}
// WithJobError sets the error message for the provisioner job.
func WithJobError(msg string) BuilderOption {
return func(b *WorkspaceBuildBuilder) {
b.jobError = msg
}
}
// WithJobErrorCode sets the error code for the provisioner job.
func WithJobErrorCode(code string) BuilderOption {
return func(b *WorkspaceBuildBuilder) {
b.jobErrorCode = code
}
}
// WorkspaceBuild generates a workspace build for the provided workspace.
@@ -141,18 +196,59 @@ func (b WorkspaceBuildBuilder) WithTask(taskSeed database.TaskTable, appSeed *sd
})
}
func (b WorkspaceBuildBuilder) Starting() WorkspaceBuildBuilder {
// Starting sets the job to running status.
func (b WorkspaceBuildBuilder) Starting(opts ...BuilderOption) WorkspaceBuildBuilder {
//nolint: revive // returns modified struct
b.jobStatus = database.ProvisionerJobStatusRunning
for _, opt := range opts {
opt(&b)
}
return b
}
func (b WorkspaceBuildBuilder) Pending() WorkspaceBuildBuilder {
// Pending sets the job to pending status.
func (b WorkspaceBuildBuilder) Pending(opts ...BuilderOption) WorkspaceBuildBuilder {
//nolint: revive // returns modified struct
b.jobStatus = database.ProvisionerJobStatusPending
for _, opt := range opts {
opt(&b)
}
return b
}
func (b WorkspaceBuildBuilder) Canceled() WorkspaceBuildBuilder {
// Canceled sets the job to canceled status.
func (b WorkspaceBuildBuilder) Canceled(opts ...BuilderOption) WorkspaceBuildBuilder {
//nolint: revive // returns modified struct
b.jobStatus = database.ProvisionerJobStatusCanceled
for _, opt := range opts {
opt(&b)
}
return b
}
// Succeeded sets the job to succeeded status.
// This is the default status.
func (b WorkspaceBuildBuilder) Succeeded(opts ...BuilderOption) WorkspaceBuildBuilder {
//nolint: revive // returns modified struct
b.jobStatus = database.ProvisionerJobStatusSucceeded
for _, opt := range opts {
opt(&b)
}
return b
}
// Failed sets the provisioner job to a failed state. Use WithJobError and
// WithJobErrorCode options to set the error message and code. If no error
// message is provided, "failed" is used as the default.
func (b WorkspaceBuildBuilder) Failed(opts ...BuilderOption) WorkspaceBuildBuilder {
//nolint: revive // returns modified struct
b.jobStatus = database.ProvisionerJobStatusFailed
for _, opt := range opts {
opt(&b)
}
if b.jobError == "" {
b.jobError = "failed"
}
return b
}
@@ -267,8 +363,8 @@ func (b WorkspaceBuildBuilder) doInTX() WorkspaceResponse {
job, err := b.db.InsertProvisionerJob(ownerCtx, database.InsertProvisionerJobParams{
ID: jobID,
CreatedAt: dbtime.Now(),
UpdatedAt: dbtime.Now(),
CreatedAt: takeFirstTime(b.jobCreatedAt, b.ws.CreatedAt, dbtime.Now()),
UpdatedAt: takeFirstTime(b.jobCreatedAt, b.ws.CreatedAt, dbtime.Now()),
OrganizationID: b.ws.OrganizationID,
InitiatorID: b.ws.OwnerID,
Provisioner: database.ProvisionerTypeEcho,
@@ -291,11 +387,12 @@ func (b WorkspaceBuildBuilder) doInTX() WorkspaceResponse {
// might need to do this multiple times if we got a template version
// import job as well
b.logger.Debug(context.Background(), "looping to acquire provisioner job")
startedAt := takeFirstTime(b.jobStartedAt, dbtime.Now())
for {
j, err := b.db.AcquireProvisionerJob(ownerCtx, database.AcquireProvisionerJobParams{
OrganizationID: job.OrganizationID,
StartedAt: sql.NullTime{
Time: dbtime.Now(),
Time: startedAt,
Valid: true,
},
WorkerID: uuid.NullUUID{
@@ -311,32 +408,54 @@ func (b WorkspaceBuildBuilder) doInTX() WorkspaceResponse {
break
}
}
if !b.jobUpdatedAt.IsZero() {
err = b.db.UpdateProvisionerJobByID(ownerCtx, database.UpdateProvisionerJobByIDParams{
ID: job.ID,
UpdatedAt: b.jobUpdatedAt,
})
require.NoError(b.t, err, "update job updated_at")
}
case database.ProvisionerJobStatusCanceled:
// Set provisioner job status to 'canceled'
b.logger.Debug(context.Background(), "canceling the provisioner job")
now := dbtime.Now()
completedAt := takeFirstTime(b.jobCompletedAt, dbtime.Now())
err = b.db.UpdateProvisionerJobWithCancelByID(ownerCtx, database.UpdateProvisionerJobWithCancelByIDParams{
ID: jobID,
CanceledAt: sql.NullTime{
Time: now,
Time: completedAt,
Valid: true,
},
CompletedAt: sql.NullTime{
Time: now,
Time: completedAt,
Valid: true,
},
})
require.NoError(b.t, err, "cancel job")
case database.ProvisionerJobStatusFailed:
b.logger.Debug(context.Background(), "failing the provisioner job")
completedAt := takeFirstTime(b.jobCompletedAt, dbtime.Now())
err = b.db.UpdateProvisionerJobWithCompleteByID(ownerCtx, database.UpdateProvisionerJobWithCompleteByIDParams{
ID: job.ID,
UpdatedAt: completedAt,
Error: sql.NullString{String: b.jobError, Valid: b.jobError != ""},
ErrorCode: sql.NullString{String: b.jobErrorCode, Valid: b.jobErrorCode != ""},
CompletedAt: sql.NullTime{
Time: completedAt,
Valid: true,
},
})
require.NoError(b.t, err, "fail job")
default:
// By default, consider jobs in 'succeeded' status
b.logger.Debug(context.Background(), "completing the provisioner job")
completedAt := takeFirstTime(b.jobCompletedAt, dbtime.Now())
err = b.db.UpdateProvisionerJobWithCompleteByID(ownerCtx, database.UpdateProvisionerJobWithCompleteByIDParams{
ID: job.ID,
UpdatedAt: dbtime.Now(),
UpdatedAt: completedAt,
Error: sql.NullString{},
ErrorCode: sql.NullString{},
CompletedAt: sql.NullTime{
Time: dbtime.Now(),
Time: completedAt,
Valid: true,
},
})
@@ -751,6 +870,16 @@ func takeFirst[Value comparable](values ...Value) Value {
})
}
// takeFirstTime returns the first non-zero time.Time.
func takeFirstTime(values ...time.Time) time.Time {
for _, v := range values {
if !v.IsZero() {
return v
}
}
return time.Time{}
}
// mustWorkspaceAppByWorkspaceAndBuildAndAppID finds a workspace app by
// workspace ID, build number, and app ID. It returns the workspace app
// if found, otherwise fails the test.
coderd/jobreaper/detector_test.go
+120 -323
View File
@@ -8,7 +8,6 @@ import (
"testing"
"time"
"github.com/google/uuid"
"github.com/prometheus/client_golang/prometheus"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -18,6 +17,7 @@ import (
"github.com/coder/coder/v2/coderd/coderdtest"
"github.com/coder/coder/v2/coderd/database"
"github.com/coder/coder/v2/coderd/database/dbauthz"
"github.com/coder/coder/v2/coderd/database/dbfake"
"github.com/coder/coder/v2/coderd/database/dbgen"
"github.com/coder/coder/v2/coderd/database/dbtestutil"
"github.com/coder/coder/v2/coderd/jobreaper"
@@ -113,87 +113,33 @@ func TestDetectorHungWorkspaceBuild(t *testing.T) {
)
var (
now = time.Now()
twentyMinAgo = now.Add(-time.Minute * 20)
tenMinAgo = now.Add(-time.Minute * 10)
sixMinAgo = now.Add(-time.Minute * 6)
org = dbgen.Organization(t, db, database.Organization{})
user = dbgen.User(t, db, database.User{})
file = dbgen.File(t, db, database.File{})
template = dbgen.Template(t, db, database.Template{
OrganizationID: org.ID,
CreatedBy: user.ID,
})
templateVersion = dbgen.TemplateVersion(t, db, database.TemplateVersion{
OrganizationID: org.ID,
TemplateID: uuid.NullUUID{
UUID: template.ID,
Valid: true,
},
CreatedBy: user.ID,
})
workspace = dbgen.Workspace(t, db, database.WorkspaceTable{
OwnerID: user.ID,
OrganizationID: org.ID,
TemplateID: template.ID,
})
// Previous build.
now = time.Now()
twentyMinAgo = now.Add(-time.Minute * 20)
tenMinAgo = now.Add(-time.Minute * 10)
sixMinAgo = now.Add(-time.Minute * 6)
org = dbgen.Organization(t, db, database.Organization{})
user = dbgen.User(t, db, database.User{})
expectedWorkspaceBuildState = []byte(`{"dean":"cool","colin":"also cool"}`)
previousWorkspaceBuildJob = dbgen.ProvisionerJob(t, db, pubsub, database.ProvisionerJob{
CreatedAt: twentyMinAgo,
UpdatedAt: twentyMinAgo,
StartedAt: sql.NullTime{
Time: twentyMinAgo,
Valid: true,
},
CompletedAt: sql.NullTime{
Time: twentyMinAgo,
Valid: true,
},
OrganizationID: org.ID,
InitiatorID: user.ID,
Provisioner: database.ProvisionerTypeEcho,
StorageMethod: database.ProvisionerStorageMethodFile,
FileID: file.ID,
Type: database.ProvisionerJobTypeWorkspaceBuild,
Input: []byte("{}"),
})
_ = dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{
WorkspaceID: workspace.ID,
TemplateVersionID: templateVersion.ID,
BuildNumber: 1,
ProvisionerState: expectedWorkspaceBuildState,
JobID: previousWorkspaceBuildJob.ID,
})
// Current build.
currentWorkspaceBuildJob = dbgen.ProvisionerJob(t, db, pubsub, database.ProvisionerJob{
CreatedAt: tenMinAgo,
UpdatedAt: sixMinAgo,
StartedAt: sql.NullTime{
Time: tenMinAgo,
Valid: true,
},
OrganizationID: org.ID,
InitiatorID: user.ID,
Provisioner: database.ProvisionerTypeEcho,
StorageMethod: database.ProvisionerStorageMethodFile,
FileID: file.ID,
Type: database.ProvisionerJobTypeWorkspaceBuild,
Input: []byte("{}"),
})
currentWorkspaceBuild = dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{
WorkspaceID: workspace.ID,
TemplateVersionID: templateVersion.ID,
BuildNumber: 2,
JobID: currentWorkspaceBuildJob.ID,
// No provisioner state.
})
)
t.Log("previous job ID: ", previousWorkspaceBuildJob.ID)
t.Log("current job ID: ", currentWorkspaceBuildJob.ID)
// Previous build (completed successfully).
previousBuild := dbfake.WorkspaceBuild(t, db, database.WorkspaceTable{
OrganizationID: org.ID,
OwnerID: user.ID,
}).Pubsub(pubsub).Seed(database.WorkspaceBuild{
ProvisionerState: expectedWorkspaceBuildState,
}).Succeeded(dbfake.WithJobCompletedAt(twentyMinAgo)).
Do()
// Current build (hung - running job with UpdatedAt > 5 min ago).
currentBuild := dbfake.WorkspaceBuild(t, db, previousBuild.Workspace).
Pubsub(pubsub).
Seed(database.WorkspaceBuild{BuildNumber: 2}).
Starting(dbfake.WithJobStartedAt(tenMinAgo), dbfake.WithJobUpdatedAt(sixMinAgo)).
Do()
t.Log("previous job ID: ", previousBuild.Build.JobID)
t.Log("current job ID: ", currentBuild.Build.JobID)
detector := jobreaper.New(ctx, wrapDBAuthz(db, log), pubsub, log, tickCh).WithStatsChannel(statsCh)
detector.Start()
@@ -202,10 +148,10 @@ func TestDetectorHungWorkspaceBuild(t *testing.T) {
stats := <-statsCh
require.NoError(t, stats.Error)
require.Len(t, stats.TerminatedJobIDs, 1)
require.Equal(t, currentWorkspaceBuildJob.ID, stats.TerminatedJobIDs[0])
require.Equal(t, currentBuild.Build.JobID, stats.TerminatedJobIDs[0])
// Check that the current provisioner job was updated.
job, err := db.GetProvisionerJobByID(ctx, currentWorkspaceBuildJob.ID)
job, err := db.GetProvisionerJobByID(ctx, currentBuild.Build.JobID)
require.NoError(t, err)
require.WithinDuration(t, now, job.UpdatedAt, 30*time.Second)
require.True(t, job.CompletedAt.Valid)
@@ -215,7 +161,7 @@ func TestDetectorHungWorkspaceBuild(t *testing.T) {
require.False(t, job.ErrorCode.Valid)
// Check that the provisioner state was copied.
build, err := db.GetWorkspaceBuildByID(ctx, currentWorkspaceBuild.ID)
build, err := db.GetWorkspaceBuildByID(ctx, currentBuild.Build.ID)
require.NoError(t, err)
require.Equal(t, expectedWorkspaceBuildState, build.ProvisionerState)
@@ -235,88 +181,37 @@ func TestDetectorHungWorkspaceBuildNoOverrideState(t *testing.T) {
)
var (
now = time.Now()
twentyMinAgo = now.Add(-time.Minute * 20)
tenMinAgo = now.Add(-time.Minute * 10)
sixMinAgo = now.Add(-time.Minute * 6)
org = dbgen.Organization(t, db, database.Organization{})
user = dbgen.User(t, db, database.User{})
file = dbgen.File(t, db, database.File{})
template = dbgen.Template(t, db, database.Template{
OrganizationID: org.ID,
CreatedBy: user.ID,
})
templateVersion = dbgen.TemplateVersion(t, db, database.TemplateVersion{
OrganizationID: org.ID,
TemplateID: uuid.NullUUID{
UUID: template.ID,
Valid: true,
},
CreatedBy: user.ID,
})
workspace = dbgen.Workspace(t, db, database.WorkspaceTable{
OwnerID: user.ID,
OrganizationID: org.ID,
TemplateID: template.ID,
})
// Previous build.
previousWorkspaceBuildJob = dbgen.ProvisionerJob(t, db, pubsub, database.ProvisionerJob{
CreatedAt: twentyMinAgo,
UpdatedAt: twentyMinAgo,
StartedAt: sql.NullTime{
Time: twentyMinAgo,
Valid: true,
},
CompletedAt: sql.NullTime{
Time: twentyMinAgo,
Valid: true,
},
OrganizationID: org.ID,
InitiatorID: user.ID,
Provisioner: database.ProvisionerTypeEcho,
StorageMethod: database.ProvisionerStorageMethodFile,
FileID: file.ID,
Type: database.ProvisionerJobTypeWorkspaceBuild,
Input: []byte("{}"),
})
_ = dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{
WorkspaceID: workspace.ID,
TemplateVersionID: templateVersion.ID,
BuildNumber: 1,
ProvisionerState: []byte(`{"dean":"NOT cool","colin":"also NOT cool"}`),
JobID: previousWorkspaceBuildJob.ID,
})
// Current build.
now = time.Now()
twentyMinAgo = now.Add(-time.Minute * 20)
tenMinAgo = now.Add(-time.Minute * 10)
sixMinAgo = now.Add(-time.Minute * 6)
org = dbgen.Organization(t, db, database.Organization{})
user = dbgen.User(t, db, database.User{})
expectedWorkspaceBuildState = []byte(`{"dean":"cool","colin":"also cool"}`)
currentWorkspaceBuildJob = dbgen.ProvisionerJob(t, db, pubsub, database.ProvisionerJob{
CreatedAt: tenMinAgo,
UpdatedAt: sixMinAgo,
StartedAt: sql.NullTime{
Time: tenMinAgo,
Valid: true,
},
OrganizationID: org.ID,
InitiatorID: user.ID,
Provisioner: database.ProvisionerTypeEcho,
StorageMethod: database.ProvisionerStorageMethodFile,
FileID: file.ID,
Type: database.ProvisionerJobTypeWorkspaceBuild,
Input: []byte("{}"),
})
currentWorkspaceBuild = dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{
WorkspaceID: workspace.ID,
TemplateVersionID: templateVersion.ID,
BuildNumber: 2,
JobID: currentWorkspaceBuildJob.ID,
// Should not be overridden.
ProvisionerState: expectedWorkspaceBuildState,
})
)
t.Log("previous job ID: ", previousWorkspaceBuildJob.ID)
t.Log("current job ID: ", currentWorkspaceBuildJob.ID)
// Previous build (completed successfully).
previousBuild := dbfake.WorkspaceBuild(t, db, database.WorkspaceTable{
OrganizationID: org.ID,
OwnerID: user.ID,
}).Pubsub(pubsub).Seed(database.WorkspaceBuild{
ProvisionerState: []byte(`{"dean":"NOT cool","colin":"also NOT cool"}`),
}).Succeeded(dbfake.WithJobCompletedAt(twentyMinAgo)).
Do()
// Current build (hung - running job with UpdatedAt > 5 min ago).
// This build already has provisioner state, which should NOT be overridden.
currentBuild := dbfake.WorkspaceBuild(t, db, previousBuild.Workspace).
Pubsub(pubsub).
Seed(database.WorkspaceBuild{
BuildNumber: 2,
ProvisionerState: expectedWorkspaceBuildState,
}).
Starting(dbfake.WithJobStartedAt(tenMinAgo), dbfake.WithJobUpdatedAt(sixMinAgo)).
Do()
t.Log("previous job ID: ", previousBuild.Build.JobID)
t.Log("current job ID: ", currentBuild.Build.JobID)
detector := jobreaper.New(ctx, wrapDBAuthz(db, log), pubsub, log, tickCh).WithStatsChannel(statsCh)
detector.Start()
@@ -325,10 +220,10 @@ func TestDetectorHungWorkspaceBuildNoOverrideState(t *testing.T) {
stats := <-statsCh
require.NoError(t, stats.Error)
require.Len(t, stats.TerminatedJobIDs, 1)
require.Equal(t, currentWorkspaceBuildJob.ID, stats.TerminatedJobIDs[0])
require.Equal(t, currentBuild.Build.JobID, stats.TerminatedJobIDs[0])
// Check that the current provisioner job was updated.
job, err := db.GetProvisionerJobByID(ctx, currentWorkspaceBuildJob.ID)
job, err := db.GetProvisionerJobByID(ctx, currentBuild.Build.JobID)
require.NoError(t, err)
require.WithinDuration(t, now, job.UpdatedAt, 30*time.Second)
require.True(t, job.CompletedAt.Valid)
@@ -338,7 +233,7 @@ func TestDetectorHungWorkspaceBuildNoOverrideState(t *testing.T) {
require.False(t, job.ErrorCode.Valid)
// Check that the provisioner state was NOT copied.
build, err := db.GetWorkspaceBuildByID(ctx, currentWorkspaceBuild.ID)
build, err := db.GetWorkspaceBuildByID(ctx, currentBuild.Build.ID)
require.NoError(t, err)
require.Equal(t, expectedWorkspaceBuildState, build.ProvisionerState)
@@ -358,58 +253,25 @@ func TestDetectorHungWorkspaceBuildNoOverrideStateIfNoExistingBuild(t *testing.T
)
var (
now = time.Now()
tenMinAgo = now.Add(-time.Minute * 10)
sixMinAgo = now.Add(-time.Minute * 6)
org = dbgen.Organization(t, db, database.Organization{})
user = dbgen.User(t, db, database.User{})
file = dbgen.File(t, db, database.File{})
template = dbgen.Template(t, db, database.Template{
OrganizationID: org.ID,
CreatedBy: user.ID,
})
templateVersion = dbgen.TemplateVersion(t, db, database.TemplateVersion{
OrganizationID: org.ID,
TemplateID: uuid.NullUUID{
UUID: template.ID,
Valid: true,
},
CreatedBy: user.ID,
})
workspace = dbgen.Workspace(t, db, database.WorkspaceTable{
OwnerID: user.ID,
OrganizationID: org.ID,
TemplateID: template.ID,
})
// First build.
now = time.Now()
tenMinAgo = now.Add(-time.Minute * 10)
sixMinAgo = now.Add(-time.Minute * 6)
org = dbgen.Organization(t, db, database.Organization{})
user = dbgen.User(t, db, database.User{})
expectedWorkspaceBuildState = []byte(`{"dean":"cool","colin":"also cool"}`)
currentWorkspaceBuildJob = dbgen.ProvisionerJob(t, db, pubsub, database.ProvisionerJob{
CreatedAt: tenMinAgo,
UpdatedAt: sixMinAgo,
StartedAt: sql.NullTime{
Time: tenMinAgo,
Valid: true,
},
OrganizationID: org.ID,
InitiatorID: user.ID,
Provisioner: database.ProvisionerTypeEcho,
StorageMethod: database.ProvisionerStorageMethodFile,
FileID: file.ID,
Type: database.ProvisionerJobTypeWorkspaceBuild,
Input: []byte("{}"),
})
currentWorkspaceBuild = dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{
WorkspaceID: workspace.ID,
TemplateVersionID: templateVersion.ID,
BuildNumber: 1,
JobID: currentWorkspaceBuildJob.ID,
// Should not be overridden.
ProvisionerState: expectedWorkspaceBuildState,
})
)
t.Log("current job ID: ", currentWorkspaceBuildJob.ID)
// First build (hung - no previous build exists).
// This build has provisioner state, which should NOT be overridden.
currentBuild := dbfake.WorkspaceBuild(t, db, database.WorkspaceTable{
OrganizationID: org.ID,
OwnerID: user.ID,
}).Pubsub(pubsub).Seed(database.WorkspaceBuild{
ProvisionerState: expectedWorkspaceBuildState,
}).Starting(dbfake.WithJobStartedAt(tenMinAgo), dbfake.WithJobUpdatedAt(sixMinAgo)).
Do()
t.Log("current job ID: ", currentBuild.Build.JobID)
detector := jobreaper.New(ctx, wrapDBAuthz(db, log), pubsub, log, tickCh).WithStatsChannel(statsCh)
detector.Start()
@@ -418,10 +280,10 @@ func TestDetectorHungWorkspaceBuildNoOverrideStateIfNoExistingBuild(t *testing.T
stats := <-statsCh
require.NoError(t, stats.Error)
require.Len(t, stats.TerminatedJobIDs, 1)
require.Equal(t, currentWorkspaceBuildJob.ID, stats.TerminatedJobIDs[0])
require.Equal(t, currentBuild.Build.JobID, stats.TerminatedJobIDs[0])
// Check that the current provisioner job was updated.
job, err := db.GetProvisionerJobByID(ctx, currentWorkspaceBuildJob.ID)
job, err := db.GetProvisionerJobByID(ctx, currentBuild.Build.JobID)
require.NoError(t, err)
require.WithinDuration(t, now, job.UpdatedAt, 30*time.Second)
require.True(t, job.CompletedAt.Valid)
@@ -431,7 +293,7 @@ func TestDetectorHungWorkspaceBuildNoOverrideStateIfNoExistingBuild(t *testing.T
require.False(t, job.ErrorCode.Valid)
// Check that the provisioner state was NOT updated.
build, err := db.GetWorkspaceBuildByID(ctx, currentWorkspaceBuild.ID)
build, err := db.GetWorkspaceBuildByID(ctx, currentBuild.Build.ID)
require.NoError(t, err)
require.Equal(t, expectedWorkspaceBuildState, build.ProvisionerState)
@@ -451,57 +313,24 @@ func TestDetectorPendingWorkspaceBuildNoOverrideStateIfNoExistingBuild(t *testin
)
var (
now = time.Now()
thirtyFiveMinAgo = now.Add(-time.Minute * 35)
org = dbgen.Organization(t, db, database.Organization{})
user = dbgen.User(t, db, database.User{})
file = dbgen.File(t, db, database.File{})
template = dbgen.Template(t, db, database.Template{
OrganizationID: org.ID,
CreatedBy: user.ID,
})
templateVersion = dbgen.TemplateVersion(t, db, database.TemplateVersion{
OrganizationID: org.ID,
TemplateID: uuid.NullUUID{
UUID: template.ID,
Valid: true,
},
CreatedBy: user.ID,
})
workspace = dbgen.Workspace(t, db, database.WorkspaceTable{
OwnerID: user.ID,
OrganizationID: org.ID,
TemplateID: template.ID,
})
// First build.
now = time.Now()
thirtyFiveMinAgo = now.Add(-time.Minute * 35)
org = dbgen.Organization(t, db, database.Organization{})
user = dbgen.User(t, db, database.User{})
expectedWorkspaceBuildState = []byte(`{"dean":"cool","colin":"also cool"}`)
currentWorkspaceBuildJob = dbgen.ProvisionerJob(t, db, pubsub, database.ProvisionerJob{
CreatedAt: thirtyFiveMinAgo,
UpdatedAt: thirtyFiveMinAgo,
StartedAt: sql.NullTime{
Time: time.Time{},
Valid: false,
},
OrganizationID: org.ID,
InitiatorID: user.ID,
Provisioner: database.ProvisionerTypeEcho,
StorageMethod: database.ProvisionerStorageMethodFile,
FileID: file.ID,
Type: database.ProvisionerJobTypeWorkspaceBuild,
Input: []byte("{}"),
})
currentWorkspaceBuild = dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{
WorkspaceID: workspace.ID,
TemplateVersionID: templateVersion.ID,
BuildNumber: 1,
JobID: currentWorkspaceBuildJob.ID,
// Should not be overridden.
ProvisionerState: expectedWorkspaceBuildState,
})
)
t.Log("current job ID: ", currentWorkspaceBuildJob.ID)
// First build (hung pending - no previous build exists).
// This build has provisioner state, which should NOT be overridden.
currentBuild := dbfake.WorkspaceBuild(t, db, database.WorkspaceTable{
OrganizationID: org.ID,
OwnerID: user.ID,
}).Pubsub(pubsub).Seed(database.WorkspaceBuild{
ProvisionerState: expectedWorkspaceBuildState,
}).Pending(dbfake.WithJobCreatedAt(thirtyFiveMinAgo), dbfake.WithJobUpdatedAt(thirtyFiveMinAgo)).
Do()
t.Log("current job ID: ", currentBuild.Build.JobID)
detector := jobreaper.New(ctx, wrapDBAuthz(db, log), pubsub, log, tickCh).WithStatsChannel(statsCh)
detector.Start()
@@ -510,10 +339,10 @@ func TestDetectorPendingWorkspaceBuildNoOverrideStateIfNoExistingBuild(t *testin
stats := <-statsCh
require.NoError(t, stats.Error)
require.Len(t, stats.TerminatedJobIDs, 1)
require.Equal(t, currentWorkspaceBuildJob.ID, stats.TerminatedJobIDs[0])
require.Equal(t, currentBuild.Build.JobID, stats.TerminatedJobIDs[0])
// Check that the current provisioner job was updated.
job, err := db.GetProvisionerJobByID(ctx, currentWorkspaceBuildJob.ID)
job, err := db.GetProvisionerJobByID(ctx, currentBuild.Build.JobID)
require.NoError(t, err)
require.WithinDuration(t, now, job.UpdatedAt, 30*time.Second)
require.True(t, job.CompletedAt.Valid)
@@ -525,7 +354,7 @@ func TestDetectorPendingWorkspaceBuildNoOverrideStateIfNoExistingBuild(t *testin
require.False(t, job.ErrorCode.Valid)
// Check that the provisioner state was NOT updated.
build, err := db.GetWorkspaceBuildByID(ctx, currentWorkspaceBuild.ID)
build, err := db.GetWorkspaceBuildByID(ctx, currentBuild.Build.ID)
require.NoError(t, err)
require.Equal(t, expectedWorkspaceBuildState, build.ProvisionerState)
@@ -551,66 +380,34 @@ func TestDetectorWorkspaceBuildForDormantWorkspace(t *testing.T) {
)
var (
now = time.Now()
tenMinAgo = now.Add(-time.Minute * 10)
sixMinAgo = now.Add(-time.Minute * 6)
org = dbgen.Organization(t, db, database.Organization{})
user = dbgen.User(t, db, database.User{})
file = dbgen.File(t, db, database.File{})
template = dbgen.Template(t, db, database.Template{
OrganizationID: org.ID,
CreatedBy: user.ID,
})
templateVersion = dbgen.TemplateVersion(t, db, database.TemplateVersion{
OrganizationID: org.ID,
TemplateID: uuid.NullUUID{
UUID: template.ID,
Valid: true,
},
CreatedBy: user.ID,
})
workspace = dbgen.Workspace(t, db, database.WorkspaceTable{
OwnerID: user.ID,
OrganizationID: org.ID,
TemplateID: template.ID,
DormantAt: sql.NullTime{
Time: now.Add(-time.Hour),
Valid: true,
},
})
// First build.
now = time.Now()
tenMinAgo = now.Add(-time.Minute * 10)
sixMinAgo = now.Add(-time.Minute * 6)
org = dbgen.Organization(t, db, database.Organization{})
user = dbgen.User(t, db, database.User{})
expectedWorkspaceBuildState = []byte(`{"dean":"cool","colin":"also cool"}`)
currentWorkspaceBuildJob = dbgen.ProvisionerJob(t, db, pubsub, database.ProvisionerJob{
CreatedAt: tenMinAgo,
UpdatedAt: sixMinAgo,
StartedAt: sql.NullTime{
Time: tenMinAgo,
Valid: true,
},
OrganizationID: org.ID,
InitiatorID: user.ID,
Provisioner: database.ProvisionerTypeEcho,
StorageMethod: database.ProvisionerStorageMethodFile,
FileID: file.ID,
Type: database.ProvisionerJobTypeWorkspaceBuild,
Input: []byte("{}"),
})
_ = dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{
WorkspaceID: workspace.ID,
TemplateVersionID: templateVersion.ID,
BuildNumber: 1,
JobID: currentWorkspaceBuildJob.ID,
// Should not be overridden.
ProvisionerState: expectedWorkspaceBuildState,
})
)
t.Log("current job ID: ", currentWorkspaceBuildJob.ID)
// First build (hung - running job with UpdatedAt > 5 min ago).
// This build has provisioner state, which should NOT be overridden.
// The workspace is dormant from the start.
currentBuild := dbfake.WorkspaceBuild(t, db, database.WorkspaceTable{
OrganizationID: org.ID,
OwnerID: user.ID,
DormantAt: sql.NullTime{
Time: now.Add(-time.Hour),
Valid: true,
},
}).Pubsub(pubsub).Seed(database.WorkspaceBuild{
ProvisionerState: expectedWorkspaceBuildState,
}).Starting(dbfake.WithJobStartedAt(tenMinAgo), dbfake.WithJobUpdatedAt(sixMinAgo)).
Do()
t.Log("current job ID: ", currentBuild.Build.JobID)
// Ensure the RBAC is the dormant type to ensure we're testing the right
// thing.
require.Equal(t, rbac.ResourceWorkspaceDormant.Type, workspace.RBACObject().Type)
require.Equal(t, rbac.ResourceWorkspaceDormant.Type, currentBuild.Workspace.RBACObject().Type)
detector := jobreaper.New(ctx, wrapDBAuthz(db, log), pubsub, log, tickCh).WithStatsChannel(statsCh)
detector.Start()
@@ -619,10 +416,10 @@ func TestDetectorWorkspaceBuildForDormantWorkspace(t *testing.T) {
stats := <-statsCh
require.NoError(t, stats.Error)
require.Len(t, stats.TerminatedJobIDs, 1)
require.Equal(t, currentWorkspaceBuildJob.ID, stats.TerminatedJobIDs[0])
require.Equal(t, currentBuild.Build.JobID, stats.TerminatedJobIDs[0])
// Check that the current provisioner job was updated.
job, err := db.GetProvisionerJobByID(ctx, currentWorkspaceBuildJob.ID)
job, err := db.GetProvisionerJobByID(ctx, currentBuild.Build.JobID)
require.NoError(t, err)
require.WithinDuration(t, now, job.UpdatedAt, 30*time.Second)
require.True(t, job.CompletedAt.Valid)
coderd/notifications/reports/generator_internal_test.go
+81 -32
View File
@@ -16,6 +16,7 @@ import (
"github.com/coder/coder/v2/coderd/coderdtest"
"github.com/coder/coder/v2/coderd/database"
"github.com/coder/coder/v2/coderd/database/dbauthz"
"github.com/coder/coder/v2/coderd/database/dbfake"
"github.com/coder/coder/v2/coderd/database/dbgen"
"github.com/coder/coder/v2/coderd/database/dbtestutil"
"github.com/coder/coder/v2/coderd/database/pubsub"
@@ -92,8 +93,11 @@ func TestReportFailedWorkspaceBuilds(t *testing.T) {
// Workspaces
w1 := dbgen.Workspace(t, db, database.WorkspaceTable{TemplateID: t1.ID, OwnerID: user1.ID, OrganizationID: org.ID})
w1wb1pj := dbgen.ProvisionerJob(t, db, ps, database.ProvisionerJob{OrganizationID: org.ID, Error: jobError, ErrorCode: jobErrorCode, CompletedAt: sql.NullTime{Time: now.Add(-6 * dayDuration), Valid: true}})
_ = dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{WorkspaceID: w1.ID, BuildNumber: 1, TemplateVersionID: t1v1.ID, JobID: w1wb1pj.ID, CreatedAt: now.Add(-2 * dayDuration), Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator})
_ = dbfake.WorkspaceBuild(t, db, w1).
Pubsub(ps).
Seed(database.WorkspaceBuild{BuildNumber: 1, TemplateVersionID: t1v1.ID, CreatedAt: now.Add(-2 * dayDuration), Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator}).
Failed(dbfake.WithJobError(jobError.String), dbfake.WithJobErrorCode(jobErrorCode.String), dbfake.WithJobCompletedAt(now.Add(-6*dayDuration))).
Do()
// When: first run
notifEnq.Clear()
@@ -178,27 +182,54 @@ func TestReportFailedWorkspaceBuilds(t *testing.T) {
now := clk.Now()
// Workspace builds
w1wb1pj := dbgen.ProvisionerJob(t, db, ps, database.ProvisionerJob{OrganizationID: org.ID, Error: jobError, ErrorCode: jobErrorCode, CompletedAt: sql.NullTime{Time: now.Add(-6 * dayDuration), Valid: true}})
_ = dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{WorkspaceID: w1.ID, BuildNumber: 1, TemplateVersionID: t1v1.ID, JobID: w1wb1pj.ID, CreatedAt: now.Add(-6 * dayDuration), Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator})
w1wb2pj := dbgen.ProvisionerJob(t, db, ps, database.ProvisionerJob{OrganizationID: org.ID, CompletedAt: sql.NullTime{Time: now.Add(-5 * dayDuration), Valid: true}})
_ = dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{WorkspaceID: w1.ID, BuildNumber: 2, TemplateVersionID: t1v2.ID, JobID: w1wb2pj.ID, CreatedAt: now.Add(-5 * dayDuration), Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator})
w1wb3pj := dbgen.ProvisionerJob(t, db, ps, database.ProvisionerJob{OrganizationID: org.ID, Error: jobError, ErrorCode: jobErrorCode, CompletedAt: sql.NullTime{Time: now.Add(-4 * dayDuration), Valid: true}})
_ = dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{WorkspaceID: w1.ID, BuildNumber: 3, TemplateVersionID: t1v2.ID, JobID: w1wb3pj.ID, CreatedAt: now.Add(-4 * dayDuration), Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator})
_ = dbfake.WorkspaceBuild(t, db, w1).
Pubsub(ps).
Seed(database.WorkspaceBuild{BuildNumber: 1, TemplateVersionID: t1v1.ID, CreatedAt: now.Add(-6 * dayDuration), Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator}).
Failed(dbfake.WithJobError(jobError.String), dbfake.WithJobErrorCode(jobErrorCode.String), dbfake.WithJobCompletedAt(now.Add(-6*dayDuration))).
Do()
_ = dbfake.WorkspaceBuild(t, db, w1).
Pubsub(ps).
Seed(database.WorkspaceBuild{BuildNumber: 2, TemplateVersionID: t1v2.ID, CreatedAt: now.Add(-5 * dayDuration), Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator}).
Succeeded(dbfake.WithJobCompletedAt(now.Add(-5 * dayDuration))).
Do()
_ = dbfake.WorkspaceBuild(t, db, w1).
Pubsub(ps).
Seed(database.WorkspaceBuild{BuildNumber: 3, TemplateVersionID: t1v2.ID, CreatedAt: now.Add(-4 * dayDuration), Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator}).
Failed(dbfake.WithJobError(jobError.String), dbfake.WithJobErrorCode(jobErrorCode.String), dbfake.WithJobCompletedAt(now.Add(-4*dayDuration))).
Do()
w2wb1pj := dbgen.ProvisionerJob(t, db, ps, database.ProvisionerJob{OrganizationID: org.ID, CompletedAt: sql.NullTime{Time: now.Add(-5 * dayDuration), Valid: true}})
_ = dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{WorkspaceID: w2.ID, BuildNumber: 4, TemplateVersionID: t2v1.ID, JobID: w2wb1pj.ID, CreatedAt: now.Add(-5 * dayDuration), Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator})
w2wb2pj := dbgen.ProvisionerJob(t, db, ps, database.ProvisionerJob{OrganizationID: org.ID, Error: jobError, ErrorCode: jobErrorCode, CompletedAt: sql.NullTime{Time: now.Add(-4 * dayDuration), Valid: true}})
_ = dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{WorkspaceID: w2.ID, BuildNumber: 5, TemplateVersionID: t2v2.ID, JobID: w2wb2pj.ID, CreatedAt: now.Add(-4 * dayDuration), Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator})
w2wb3pj := dbgen.ProvisionerJob(t, db, ps, database.ProvisionerJob{OrganizationID: org.ID, Error: jobError, ErrorCode: jobErrorCode, CompletedAt: sql.NullTime{Time: now.Add(-3 * dayDuration), Valid: true}})
_ = dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{WorkspaceID: w2.ID, BuildNumber: 6, TemplateVersionID: t2v2.ID, JobID: w2wb3pj.ID, CreatedAt: now.Add(-3 * dayDuration), Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator})
_ = dbfake.WorkspaceBuild(t, db, w2).
Pubsub(ps).
Seed(database.WorkspaceBuild{BuildNumber: 4, TemplateVersionID: t2v1.ID, CreatedAt: now.Add(-5 * dayDuration), Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator}).
Succeeded(dbfake.WithJobCompletedAt(now.Add(-5 * dayDuration))).
Do()
_ = dbfake.WorkspaceBuild(t, db, w2).
Pubsub(ps).
Seed(database.WorkspaceBuild{BuildNumber: 5, TemplateVersionID: t2v2.ID, CreatedAt: now.Add(-4 * dayDuration), Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator}).
Failed(dbfake.WithJobError(jobError.String), dbfake.WithJobErrorCode(jobErrorCode.String), dbfake.WithJobCompletedAt(now.Add(-4*dayDuration))).
Do()
_ = dbfake.WorkspaceBuild(t, db, w2).
Pubsub(ps).
Seed(database.WorkspaceBuild{BuildNumber: 6, TemplateVersionID: t2v2.ID, CreatedAt: now.Add(-3 * dayDuration), Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator}).
Failed(dbfake.WithJobError(jobError.String), dbfake.WithJobErrorCode(jobErrorCode.String), dbfake.WithJobCompletedAt(now.Add(-3*dayDuration))).
Do()
w3wb1pj := dbgen.ProvisionerJob(t, db, ps, database.ProvisionerJob{OrganizationID: org.ID, Error: jobError, ErrorCode: jobErrorCode, CompletedAt: sql.NullTime{Time: now.Add(-3 * dayDuration), Valid: true}})
_ = dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{WorkspaceID: w3.ID, BuildNumber: 7, TemplateVersionID: t1v1.ID, JobID: w3wb1pj.ID, CreatedAt: now.Add(-3 * dayDuration), Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator})
_ = dbfake.WorkspaceBuild(t, db, w3).
Pubsub(ps).
Seed(database.WorkspaceBuild{BuildNumber: 7, TemplateVersionID: t1v1.ID, CreatedAt: now.Add(-3 * dayDuration), Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator}).
Failed(dbfake.WithJobError(jobError.String), dbfake.WithJobErrorCode(jobErrorCode.String), dbfake.WithJobCompletedAt(now.Add(-3*dayDuration))).
Do()
w4wb1pj := dbgen.ProvisionerJob(t, db, ps, database.ProvisionerJob{OrganizationID: org.ID, Error: jobError, ErrorCode: jobErrorCode, CompletedAt: sql.NullTime{Time: now.Add(-6 * dayDuration), Valid: true}})
_ = dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{WorkspaceID: w4.ID, BuildNumber: 8, TemplateVersionID: t2v1.ID, JobID: w4wb1pj.ID, CreatedAt: now.Add(-6 * dayDuration), Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator})
w4wb2pj := dbgen.ProvisionerJob(t, db, ps, database.ProvisionerJob{OrganizationID: org.ID, CompletedAt: sql.NullTime{Time: now.Add(-dayDuration), Valid: true}})
_ = dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{WorkspaceID: w4.ID, BuildNumber: 9, TemplateVersionID: t2v2.ID, JobID: w4wb2pj.ID, CreatedAt: now.Add(-dayDuration), Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator})
_ = dbfake.WorkspaceBuild(t, db, w4).
Pubsub(ps).
Seed(database.WorkspaceBuild{BuildNumber: 8, TemplateVersionID: t2v1.ID, CreatedAt: now.Add(-6 * dayDuration), Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator}).
Failed(dbfake.WithJobError(jobError.String), dbfake.WithJobErrorCode(jobErrorCode.String), dbfake.WithJobCompletedAt(now.Add(-6*dayDuration))).
Do()
_ = dbfake.WorkspaceBuild(t, db, w4).
Pubsub(ps).
Seed(database.WorkspaceBuild{BuildNumber: 9, TemplateVersionID: t2v2.ID, CreatedAt: now.Add(-dayDuration), Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator}).
Succeeded(dbfake.WithJobCompletedAt(now.Add(-dayDuration))).
Do()
// When
notifEnq.Clear()
@@ -275,8 +306,11 @@ func TestReportFailedWorkspaceBuilds(t *testing.T) {
clk.Advance(6 * dayDuration).MustWait(context.Background())
now = clk.Now()
w1wb4pj := dbgen.ProvisionerJob(t, db, ps, database.ProvisionerJob{OrganizationID: org.ID, Error: jobError, ErrorCode: jobErrorCode, CompletedAt: sql.NullTime{Time: now.Add(-dayDuration), Valid: true}})
_ = dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{WorkspaceID: w1.ID, BuildNumber: 77, TemplateVersionID: t1v2.ID, JobID: w1wb4pj.ID, CreatedAt: now.Add(-dayDuration), Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator})
_ = dbfake.WorkspaceBuild(t, db, w1).
Pubsub(ps).
Seed(database.WorkspaceBuild{BuildNumber: 77, TemplateVersionID: t1v2.ID, CreatedAt: now.Add(-dayDuration), Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator}).
Failed(dbfake.WithJobError(jobError.String), dbfake.WithJobErrorCode(jobErrorCode.String), dbfake.WithJobCompletedAt(now.Add(-dayDuration))).
Do()
// When
notifEnq.Clear()
@@ -380,17 +414,26 @@ func TestReportFailedWorkspaceBuilds(t *testing.T) {
now := clk.Now()
// Workspace builds
pj0 := dbgen.ProvisionerJob(t, db, ps, database.ProvisionerJob{OrganizationID: org.ID, CompletedAt: sql.NullTime{Time: now.Add(-24 * time.Hour), Valid: true}})
_ = dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{WorkspaceID: w1.ID, BuildNumber: 777, TemplateVersionID: t1v1.ID, JobID: pj0.ID, CreatedAt: now.Add(-24 * time.Hour), Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator})
_ = dbfake.WorkspaceBuild(t, db, w1).
Pubsub(ps).
Seed(database.WorkspaceBuild{BuildNumber: 777, TemplateVersionID: t1v1.ID, CreatedAt: now.Add(-24 * time.Hour), Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator}).
Succeeded(dbfake.WithJobCompletedAt(now.Add(-24 * time.Hour))).
Do()
for i := 1; i <= 23; i++ {
at := now.Add(-time.Duration(i) * time.Hour)
pj1 := dbgen.ProvisionerJob(t, db, ps, database.ProvisionerJob{OrganizationID: org.ID, Error: jobError, ErrorCode: jobErrorCode, CompletedAt: sql.NullTime{Time: at, Valid: true}})
_ = dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{WorkspaceID: w1.ID, BuildNumber: int32(i), TemplateVersionID: t1v1.ID, JobID: pj1.ID, CreatedAt: at, Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator}) // nolint:gosec
_ = dbfake.WorkspaceBuild(t, db, w1).
Pubsub(ps).
Seed(database.WorkspaceBuild{BuildNumber: int32(i), TemplateVersionID: t1v1.ID, CreatedAt: at, Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator}). // nolint:gosec
Failed(dbfake.WithJobError(jobError.String), dbfake.WithJobErrorCode(jobErrorCode.String), dbfake.WithJobCompletedAt(at)).
Do()
pj2 := dbgen.ProvisionerJob(t, db, ps, database.ProvisionerJob{OrganizationID: org.ID, Error: jobError, ErrorCode: jobErrorCode, CompletedAt: sql.NullTime{Time: at, Valid: true}})
_ = dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{WorkspaceID: w1.ID, BuildNumber: int32(i) + 100, TemplateVersionID: t1v2.ID, JobID: pj2.ID, CreatedAt: at, Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator}) // nolint:gosec
_ = dbfake.WorkspaceBuild(t, db, w1).
Pubsub(ps).
Seed(database.WorkspaceBuild{BuildNumber: int32(i) + 100, TemplateVersionID: t1v2.ID, CreatedAt: at, Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator}). // nolint:gosec
Failed(dbfake.WithJobError(jobError.String), dbfake.WithJobErrorCode(jobErrorCode.String), dbfake.WithJobCompletedAt(at)).
Do()
}
// When
@@ -486,10 +529,16 @@ func TestReportFailedWorkspaceBuilds(t *testing.T) {
now := clk.Now()
// Workspace builds
w1wb1pj := dbgen.ProvisionerJob(t, db, ps, database.ProvisionerJob{OrganizationID: org.ID, CompletedAt: sql.NullTime{Time: now.Add(-6 * dayDuration), Valid: true}})
_ = dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{WorkspaceID: w1.ID, BuildNumber: 1, TemplateVersionID: t1v1.ID, JobID: w1wb1pj.ID, CreatedAt: now.Add(-2 * dayDuration), Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator})
w1wb2pj := dbgen.ProvisionerJob(t, db, ps, database.ProvisionerJob{OrganizationID: org.ID, CompletedAt: sql.NullTime{Time: now.Add(-5 * dayDuration), Valid: true}})
_ = dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{WorkspaceID: w1.ID, BuildNumber: 2, TemplateVersionID: t1v1.ID, JobID: w1wb2pj.ID, CreatedAt: now.Add(-1 * dayDuration), Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator})
_ = dbfake.WorkspaceBuild(t, db, w1).
Pubsub(ps).
Seed(database.WorkspaceBuild{BuildNumber: 1, TemplateVersionID: t1v1.ID, CreatedAt: now.Add(-2 * dayDuration), Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator}).
Succeeded(dbfake.WithJobCompletedAt(now.Add(-6 * dayDuration))).
Do()
_ = dbfake.WorkspaceBuild(t, db, w1).
Pubsub(ps).
Seed(database.WorkspaceBuild{BuildNumber: 2, TemplateVersionID: t1v1.ID, CreatedAt: now.Add(-1 * dayDuration), Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator}).
Succeeded(dbfake.WithJobCompletedAt(now.Add(-5 * dayDuration))).
Do()
// When
notifEnq.Clear()
enterprise/coderd/prebuilds/reconcile_test.go
+19 -46
View File
@@ -1830,25 +1830,27 @@ func TestExpiredPrebuildsMultipleActions(t *testing.T) {
expiredCount++
}
workspace, _ := setupTestDBPrebuild(
t,
clock,
db,
pubSub,
database.WorkspaceTransitionStart,
database.ProvisionerJobStatusSucceeded,
org.ID,
preset,
template.ID,
templateVersionID,
withCreatedAt(clock.Now().Add(createdAt)),
)
jobCreatedAt := clock.Now().Add(createdAt)
resp := dbfake.WorkspaceBuild(t, db, database.WorkspaceTable{
OwnerID: database.PrebuildsSystemUserID,
OrganizationID: org.ID,
TemplateID: template.ID,
CreatedAt: jobCreatedAt,
}).Pubsub(pubSub).Seed(database.WorkspaceBuild{
InitiatorID: database.PrebuildsSystemUserID,
TemplateVersionID: templateVersionID,
TemplateVersionPresetID: uuid.NullUUID{UUID: preset.ID, Valid: true},
Transition: database.WorkspaceTransitionStart,
}).Params(database.WorkspaceBuildParameter{
Name: "test",
Value: "test",
}).Do()
if isExpired {
expiredWorkspaces = append(expiredWorkspaces, workspace)
expiredWorkspaces = append(expiredWorkspaces, resp.Workspace)
} else {
nonExpiredWorkspaces = append(nonExpiredWorkspaces, workspace)
nonExpiredWorkspaces = append(nonExpiredWorkspaces, resp.Workspace)
}
runningWorkspaces[workspace.ID.String()] = workspace
runningWorkspaces[resp.Workspace.ID.String()] = resp.Workspace
}
getJobStatusMap := func(workspaces []database.WorkspaceTable) map[database.ProvisionerJobStatus]int {
@@ -2791,21 +2793,6 @@ func setupTestDBPresetWithScheduling(
return preset
}
// prebuildOptions holds optional parameters for creating a prebuild workspace.
type prebuildOptions struct {
createdAt *time.Time
}
// prebuildOption defines a function type to apply optional settings to prebuildOptions.
type prebuildOption func(*prebuildOptions)
// withCreatedAt returns a prebuildOption that sets the CreatedAt timestamp.
func withCreatedAt(createdAt time.Time) prebuildOption {
return func(opts *prebuildOptions) {
opts.createdAt = &createdAt
}
}
func setupTestDBPrebuild(
t *testing.T,
clock quartz.Clock,
@@ -2817,10 +2804,9 @@ func setupTestDBPrebuild(
preset database.TemplateVersionPreset,
templateID uuid.UUID,
templateVersionID uuid.UUID,
opts ...prebuildOption,
) (database.WorkspaceTable, database.WorkspaceBuild) {
t.Helper()
return setupTestDBWorkspace(t, clock, db, ps, transition, prebuildStatus, orgID, preset, templateID, templateVersionID, database.PrebuildsSystemUserID, database.PrebuildsSystemUserID, opts...)
return setupTestDBWorkspace(t, clock, db, ps, transition, prebuildStatus, orgID, preset, templateID, templateVersionID, database.PrebuildsSystemUserID, database.PrebuildsSystemUserID)
}
func setupTestDBWorkspace(
@@ -2836,7 +2822,6 @@ func setupTestDBWorkspace(
templateVersionID uuid.UUID,
initiatorID uuid.UUID,
ownerID uuid.UUID,
opts ...prebuildOption,
) (database.WorkspaceTable, database.WorkspaceBuild) {
t.Helper()
cancelledAt := sql.NullTime{}
@@ -2864,19 +2849,7 @@ func setupTestDBWorkspace(
default:
}
// Apply all provided prebuild options.
prebuiltOptions := &prebuildOptions{}
for _, opt := range opts {
opt(prebuiltOptions)
}
// Set createdAt to default value if not overridden by options.
createdAt := clock.Now().Add(muchEarlier)
if prebuiltOptions.createdAt != nil {
createdAt = *prebuiltOptions.createdAt
// Ensure startedAt matches createdAt for consistency.
startedAt = sql.NullTime{Time: createdAt, Valid: true}
}
workspace := dbgen.Workspace(t, db, database.WorkspaceTable{
TemplateID: templateID,
enterprise/coderd/schedule/template_test.go
+17 -58
View File
@@ -242,73 +242,32 @@ func TestTemplateUpdateBuildDeadlines(t *testing.T) {
t.Log("newMaxDeadline", c.newMaxDeadline)
t.Log("ttl", c.ttl)
var (
template = dbgen.Template(t, db, database.Template{
OrganizationID: organizationID,
ActiveVersionID: templateVersion.ID,
CreatedBy: user.ID,
})
ws = dbgen.Workspace(t, db, database.WorkspaceTable{
OrganizationID: organizationID,
OwnerID: user.ID,
TemplateID: template.ID,
})
job = dbgen.ProvisionerJob(t, db, nil, database.ProvisionerJob{
OrganizationID: organizationID,
FileID: file.ID,
InitiatorID: user.ID,
Provisioner: database.ProvisionerTypeEcho,
Tags: database.StringMap{
c.name: "yeah",
},
})
wsBuild = dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{
WorkspaceID: ws.ID,
BuildNumber: 1,
JobID: job.ID,
InitiatorID: user.ID,
TemplateVersionID: templateVersion.ID,
ProvisionerState: []byte(must(cryptorand.String(64))),
})
)
template := dbgen.Template(t, db, database.Template{
OrganizationID: organizationID,
ActiveVersionID: templateVersion.ID,
CreatedBy: user.ID,
})
buildResp := dbfake.WorkspaceBuild(t, db, database.WorkspaceTable{
OrganizationID: organizationID,
OwnerID: user.ID,
TemplateID: template.ID,
}).Seed(database.WorkspaceBuild{
TemplateVersionID: templateVersion.ID,
ProvisionerState: []byte(must(cryptorand.String(64))),
}).Succeeded(dbfake.WithJobCompletedAt(buildTime)).Do()
// Assert test invariant: workspace build state must not be empty
require.NotEmpty(t, wsBuild.ProvisionerState, "provisioner state must not be empty")
require.NotEmpty(t, buildResp.Build.ProvisionerState, "provisioner state must not be empty")
acquiredJob, err := db.AcquireProvisionerJob(ctx, database.AcquireProvisionerJobParams{
OrganizationID: job.OrganizationID,
StartedAt: sql.NullTime{
Time: buildTime,
Valid: true,
},
WorkerID: uuid.NullUUID{
UUID: uuid.New(),
Valid: true,
},
Types: []database.ProvisionerType{database.ProvisionerTypeEcho},
ProvisionerTags: json.RawMessage(fmt.Sprintf(`{%q: "yeah"}`, c.name)),
})
require.NoError(t, err)
require.Equal(t, job.ID, acquiredJob.ID)
err = db.UpdateProvisionerJobWithCompleteByID(ctx, database.UpdateProvisionerJobWithCompleteByIDParams{
ID: job.ID,
CompletedAt: sql.NullTime{
Time: buildTime,
Valid: true,
},
UpdatedAt: buildTime,
})
require.NoError(t, err)
err = db.UpdateWorkspaceBuildDeadlineByID(ctx, database.UpdateWorkspaceBuildDeadlineByIDParams{
ID: wsBuild.ID,
err := db.UpdateWorkspaceBuildDeadlineByID(ctx, database.UpdateWorkspaceBuildDeadlineByIDParams{
ID: buildResp.Build.ID,
UpdatedAt: buildTime,
Deadline: c.deadline,
MaxDeadline: c.maxDeadline,
})
require.NoError(t, err)
wsBuild, err = db.GetWorkspaceBuildByID(ctx, wsBuild.ID)
wsBuild, err := db.GetWorkspaceBuildByID(ctx, buildResp.Build.ID)
require.NoError(t, err)
logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}).Leveled(slog.LevelDebug)
scripts/workspace-runtime-audit/runtimeaudit_test.go
+35 -48
View File
@@ -5,7 +5,6 @@
package runtimeaudit_test
import (
"database/sql"
_ "embed"
"math"
"strings"
@@ -258,8 +257,8 @@ func TestRuntimeAudit(t *testing.T) {
name: "canceled_start_does_not_count_usage",
// Only start+succeeded counts; canceled start is ignored.
builds: []workspaceBuildArgs{
{at: decUTC(8, 9, 0), canceled: true, transition: database.WorkspaceTransitionStart, jobStatus: database.ProvisionerJobStatusCanceled},
{at: decUTC(8, 10, 0), canceled: false, transition: database.WorkspaceTransitionStop, jobStatus: database.ProvisionerJobStatusSucceeded},
{at: decUTC(8, 9, 0), transition: database.WorkspaceTransitionStart, jobStatus: database.ProvisionerJobStatusCanceled},
{at: decUTC(8, 10, 0), transition: database.WorkspaceTransitionStop, jobStatus: database.ProvisionerJobStatusSucceeded},
},
expect: func(_ time.Time, _ []workspaceBuildArgs) int { return 0 },
},
@@ -267,8 +266,8 @@ func TestRuntimeAudit(t *testing.T) {
name: "failed_start_does_not_count_even_if_later_stop_occurs",
// Start failed => never turns on => later stop does nothing.
builds: []workspaceBuildArgs{
{at: decUTC(9, 9, 0), canceled: false, transition: database.WorkspaceTransitionStart, jobStatus: database.ProvisionerJobStatusFailed},
{at: decUTC(9, 12, 0), canceled: false, transition: database.WorkspaceTransitionStop, jobStatus: database.ProvisionerJobStatusSucceeded},
{at: decUTC(9, 9, 0), transition: database.WorkspaceTransitionStart, jobStatus: database.ProvisionerJobStatusFailed},
{at: decUTC(9, 12, 0), transition: database.WorkspaceTransitionStop, jobStatus: database.ProvisionerJobStatusSucceeded},
},
expect: func(_ time.Time, _ []workspaceBuildArgs) int { return 0 },
},
@@ -276,8 +275,8 @@ func TestRuntimeAudit(t *testing.T) {
name: "canceled_stop_still_stops_timer_and_counts_time",
// Any non-(start+succeeded) is treated as stop while running, regardless of status/canceled.
builds: []workspaceBuildArgs{
{at: decUTC(10, 9, 0), canceled: false, transition: database.WorkspaceTransitionStart, jobStatus: database.ProvisionerJobStatusSucceeded},
{at: decUTC(10, 9, 40), canceled: true, transition: database.WorkspaceTransitionStop, jobStatus: database.ProvisionerJobStatusCanceled},
{at: decUTC(10, 9, 0), transition: database.WorkspaceTransitionStart, jobStatus: database.ProvisionerJobStatusSucceeded},
{at: decUTC(10, 9, 40), transition: database.WorkspaceTransitionStop, jobStatus: database.ProvisionerJobStatusCanceled},
},
expect: func(_ time.Time, in []workspaceBuildArgs) int {
return roundUpHours(in[1].at, in[0].at)
@@ -287,8 +286,8 @@ func TestRuntimeAudit(t *testing.T) {
name: "failed_stop_still_stops_timer_and_counts_time",
// Same as above: stop is stop even if job failed (ELSE path).
builds: []workspaceBuildArgs{
{at: decUTC(11, 10, 0), canceled: false, transition: database.WorkspaceTransitionStart, jobStatus: database.ProvisionerJobStatusSucceeded},
{at: decUTC(11, 10, 10), canceled: false, transition: database.WorkspaceTransitionStop, jobStatus: database.ProvisionerJobStatusFailed},
{at: decUTC(11, 10, 0), transition: database.WorkspaceTransitionStart, jobStatus: database.ProvisionerJobStatusSucceeded},
{at: decUTC(11, 10, 10), transition: database.WorkspaceTransitionStop, jobStatus: database.ProvisionerJobStatusFailed},
},
expect: func(_ time.Time, in []workspaceBuildArgs) int {
return roundUpHours(in[1].at, in[0].at)
@@ -298,8 +297,8 @@ func TestRuntimeAudit(t *testing.T) {
name: "failed_transition_stops_timer_and_counts_time",
// A failed *non-stop* transition (e.g. delete) still stops if currently on.
builds: []workspaceBuildArgs{
{at: decUTC(12, 8, 0), canceled: false, transition: database.WorkspaceTransitionStart, jobStatus: database.ProvisionerJobStatusSucceeded},
{at: decUTC(12, 8, 5), canceled: false, transition: database.WorkspaceTransitionDelete, jobStatus: database.ProvisionerJobStatusFailed},
{at: decUTC(12, 8, 0), transition: database.WorkspaceTransitionStart, jobStatus: database.ProvisionerJobStatusSucceeded},
{at: decUTC(12, 8, 5), transition: database.WorkspaceTransitionDelete, jobStatus: database.ProvisionerJobStatusFailed},
},
expect: func(_ time.Time, in []workspaceBuildArgs) int {
return roundUpHours(in[1].at, in[0].at)
@@ -310,11 +309,11 @@ func TestRuntimeAudit(t *testing.T) {
// When already on, a subsequent non-(start+succeeded) build triggers stop logic.
// This verifies you *do not* treat start+failed as a "start"; it will stop the running timer.
builds: []workspaceBuildArgs{
{at: decUTC(13, 9, 0), canceled: false, transition: database.WorkspaceTransitionStart, jobStatus: database.ProvisionerJobStatusSucceeded},
{at: decUTC(13, 9, 0), transition: database.WorkspaceTransitionStart, jobStatus: database.ProvisionerJobStatusSucceeded},
// This goes to ELSE branch (because job_status != succeeded) and will stop the timer.
{at: decUTC(13, 9, 30), canceled: false, transition: database.WorkspaceTransitionStart, jobStatus: database.ProvisionerJobStatusFailed},
{at: decUTC(13, 9, 30), transition: database.WorkspaceTransitionStart, jobStatus: database.ProvisionerJobStatusFailed},
// Subsequent stop should not add more time because timer was reset.
{at: decUTC(13, 10, 0), canceled: false, transition: database.WorkspaceTransitionStop, jobStatus: database.ProvisionerJobStatusSucceeded},
{at: decUTC(13, 10, 0), transition: database.WorkspaceTransitionStop, jobStatus: database.ProvisionerJobStatusSucceeded},
},
expect: func(_ time.Time, in []workspaceBuildArgs) int {
// Only counts from first start to failed-start event.
@@ -368,13 +367,12 @@ func initSetup(t *testing.T, db database.Store) *setup {
type workspaceBuildArgs struct {
at time.Time
canceled bool
transition database.WorkspaceTransition
jobStatus database.ProvisionerJobStatus
}
func (s *setup) createWorkspace(t *testing.T, db database.Store, builds []workspaceBuildArgs) database.WorkspaceTable {
// Insert the first build
// Create template version first
tv := dbfake.TemplateVersion(t, db).
Seed(database.TemplateVersion{
OrganizationID: s.org.ID,
@@ -390,39 +388,28 @@ func (s *setup) createWorkspace(t *testing.T, db database.Store, builds []worksp
})
for i, b := range builds {
job := dbgen.ProvisionerJob(t, db, nil, database.ProvisionerJob{
CreatedAt: b.at,
UpdatedAt: b.at,
StartedAt: sql.NullTime{
Time: b.at,
Valid: true,
},
CanceledAt: sql.NullTime{
Time: b.at,
Valid: b.canceled,
},
CompletedAt: sql.NullTime{
Time: b.at,
Valid: true,
},
Error: sql.NullString{},
OrganizationID: s.org.ID,
InitiatorID: s.usr.ID,
Type: database.ProvisionerJobTypeWorkspaceBuild,
JobStatus: b.jobStatus,
})
builder := dbfake.WorkspaceBuild(t, db, wrk).
Seed(database.WorkspaceBuild{
CreatedAt: b.at,
UpdatedAt: b.at,
TemplateVersionID: tv.TemplateVersion.ID,
//nolint:gosec // this will not overflow
BuildNumber: int32(i) + 1,
Transition: b.transition,
InitiatorID: s.usr.ID,
}).
Succeeded(dbfake.WithJobCompletedAt(b.at))
dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{
CreatedAt: b.at,
UpdatedAt: b.at,
WorkspaceID: wrk.ID,
TemplateVersionID: tv.TemplateVersion.ID,
///nolint:gosec // this will not overflow
BuildNumber: int32(i) + 1,
Transition: b.transition,
InitiatorID: s.usr.ID,
JobID: job.ID,
})
// Set job status based on the build args
switch b.jobStatus {
case database.ProvisionerJobStatusCanceled:
builder = builder.Canceled(dbfake.WithJobCompletedAt(b.at))
case database.ProvisionerJobStatusFailed:
builder = builder.Failed(dbfake.WithJobError("fake error"), dbfake.WithJobCompletedAt(b.at))
// default: Succeeded (the builder's default)
}
builder.Do()
}
return wrk