feat: purge expired api keys in dbpurge (#20863) (#21040)

Backport of https://github.com/coder/coder/pull/20863

.claude/docs/DATABASE.md

@@ -0,0 +1,218 @@
+# Database Development Patterns
+
+## Database Work Overview
+
+### Database Generation Process
+
+1. Modify SQL files in `coderd/database/queries/`
+2. Run `make gen`
+3. If errors about audit table, update `enterprise/audit/table.go`
+4. Run `make gen` again
+5. Run `make lint` to catch any remaining issues
+
+## Migration Guidelines
+
+### Creating Migration Files
+
+**Location**: `coderd/database/migrations/`
+**Format**: `{number}_{description}.{up|down}.sql`
+
+- Number must be unique and sequential
+- Always include both up and down migrations
+
+### Helper Scripts
+
+| Script                                                              | Purpose                                 |
+|---------------------------------------------------------------------|-----------------------------------------|
+| `./coderd/database/migrations/create_migration.sh "migration name"` | Creates new migration files             |
+| `./coderd/database/migrations/fix_migration_numbers.sh`             | Renumbers migrations to avoid conflicts |
+| `./coderd/database/migrations/create_fixture.sh "fixture name"`     | Creates test fixtures for migrations    |
+
+### Database Query Organization
+
+- **MUST DO**: Any changes to database - adding queries, modifying queries should be done in the `coderd/database/queries/*.sql` files
+- **MUST DO**: Queries are grouped in files relating to context - e.g. `prebuilds.sql`, `users.sql`, `oauth2.sql`
+- After making changes to any `coderd/database/queries/*.sql` files you must run `make gen` to generate respective ORM changes
+
+## Handling Nullable Fields
+
+Use `sql.NullString`, `sql.NullBool`, etc. for optional database fields:
+
+```go
+CodeChallenge: sql.NullString{
+    String: params.codeChallenge,
+    Valid:  params.codeChallenge != "",
+}
+```
+
+Set `.Valid = true` when providing values.
+
+## Audit Table Updates
+
+If adding fields to auditable types:
+
+1. Update `enterprise/audit/table.go`
+2. Add each new field with appropriate action:
+   - `ActionTrack`: Field should be tracked in audit logs
+   - `ActionIgnore`: Field should be ignored in audit logs
+   - `ActionSecret`: Field contains sensitive data
+3. Run `make gen` to verify no audit errors
+
+## Database Architecture
+
+### Core Components
+
+- **PostgreSQL 13+** recommended for production
+- **Migrations** managed with `migrate`
+- **Database authorization** through `dbauthz` package
+
+### Authorization Patterns
+
+```go
+// Public endpoints needing system access (OAuth2 registration)
+app, err := api.Database.GetOAuth2ProviderAppByClientID(dbauthz.AsSystemRestricted(ctx), clientID)
+
+// Authenticated endpoints with user context
+app, err := api.Database.GetOAuth2ProviderAppByClientID(ctx, clientID)
+
+// System operations in middleware
+roles, err := db.GetAuthorizationUserRoles(dbauthz.AsSystemRestricted(ctx), userID)
+```
+
+## Common Database Issues
+
+### Migration Issues
+
+1. **Migration conflicts**: Use `fix_migration_numbers.sh` to renumber
+2. **Missing down migration**: Always create both up and down files
+3. **Schema inconsistencies**: Verify against existing schema
+
+### Field Handling Issues
+
+1. **Nullable field errors**: Use `sql.Null*` types consistently
+2. **Missing audit entries**: Update `enterprise/audit/table.go`
+
+### Query Issues
+
+1. **Query organization**: Group related queries in appropriate files
+2. **Generated code errors**: Run `make gen` after query changes
+3. **Performance issues**: Add appropriate indexes in migrations
+
+## Database Testing
+
+### Test Database Setup
+
+```go
+func TestDatabaseFunction(t *testing.T) {
+    db := dbtestutil.NewDB(t)
+
+    // Test with real database
+    result, err := db.GetSomething(ctx, param)
+    require.NoError(t, err)
+    require.Equal(t, expected, result)
+}
+```
+
+## Best Practices
+
+### Schema Design
+
+1. **Use appropriate data types**: VARCHAR for strings, TIMESTAMP for times
+2. **Add constraints**: NOT NULL, UNIQUE, FOREIGN KEY as appropriate
+3. **Create indexes**: For frequently queried columns
+4. **Consider performance**: Normalize appropriately but avoid over-normalization
+
+### Query Writing
+
+1. **Use parameterized queries**: Prevent SQL injection
+2. **Handle errors appropriately**: Check for specific error types
+3. **Use transactions**: For related operations that must succeed together
+4. **Optimize queries**: Use EXPLAIN to understand query performance
+
+### Migration Writing
+
+1. **Make migrations reversible**: Always include down migration
+2. **Test migrations**: On copy of production data if possible
+3. **Keep migrations small**: One logical change per migration
+4. **Document complex changes**: Add comments explaining rationale
+
+## Advanced Patterns
+
+### Complex Queries
+
+```sql
+-- Example: Complex join with aggregation
+SELECT
+    u.id,
+    u.username,
+    COUNT(w.id) as workspace_count
+FROM users u
+LEFT JOIN workspaces w ON u.id = w.owner_id
+WHERE u.created_at > $1
+GROUP BY u.id, u.username
+ORDER BY workspace_count DESC;
+```
+
+### Conditional Queries
+
+```sql
+-- Example: Dynamic filtering
+SELECT * FROM oauth2_provider_apps
+WHERE
+    ($1::text IS NULL OR name ILIKE '%' || $1 || '%')
+    AND ($2::uuid IS NULL OR organization_id = $2)
+ORDER BY created_at DESC;
+```
+
+### Audit Patterns
+
+```go
+// Example: Auditable database operation
+func (q *sqlQuerier) UpdateUser(ctx context.Context, arg UpdateUserParams) (User, error) {
+    // Implementation here
+
+    // Audit the change
+    if auditor := audit.FromContext(ctx); auditor != nil {
+        auditor.Record(audit.UserUpdate{
+            UserID: arg.ID,
+            Old:    oldUser,
+            New:    newUser,
+        })
+    }
+
+    return newUser, nil
+}
+```
+
+## Debugging Database Issues
+
+### Common Debug Commands
+
+```bash
+# Check database connection
+make test-postgres
+
+# Run specific database tests
+go test ./coderd/database/... -run TestSpecificFunction
+
+# Check query generation
+make gen
+
+# Verify audit table
+make lint
+```
+
+### Debug Techniques
+
+1. **Enable query logging**: Set appropriate log levels
+2. **Use database tools**: pgAdmin, psql for direct inspection
+3. **Check constraints**: UNIQUE, FOREIGN KEY violations
+4. **Analyze performance**: Use EXPLAIN ANALYZE for slow queries
+
+### Troubleshooting Checklist
+
+- [ ] Migration files exist (both up and down)
+- [ ] `make gen` run after query changes
+- [ ] Audit table updated for new fields
+- [ ] Nullable fields use `sql.Null*` types
+- [ ] Authorization context appropriate for endpoint type

.claude/docs/OAUTH2.md

@@ -0,0 +1,157 @@
+# OAuth2 Development Guide
+
+## RFC Compliance Development
+
+### Implementing Standard Protocols
+
+When implementing standard protocols (OAuth2, OpenID Connect, etc.):
+
+1. **Fetch and Analyze Official RFCs**:
+   - Always read the actual RFC specifications before implementation
+   - Use WebFetch tool to get current RFC content for compliance verification
+   - Document RFC requirements in code comments
+
+2. **Default Values Matter**:
+   - Pay close attention to RFC-specified default values
+   - Example: RFC 7591 specifies `client_secret_basic` as default, not `client_secret_post`
+   - Ensure consistency between database migrations and application code
+
+3. **Security Requirements**:
+   - Follow RFC security considerations precisely
+   - Example: RFC 7592 prohibits returning registration access tokens in GET responses
+   - Implement proper error responses per protocol specifications
+
+4. **Validation Compliance**:
+   - Implement comprehensive validation per RFC requirements
+   - Support protocol-specific features (e.g., custom schemes for native OAuth2 apps)
+   - Test edge cases defined in specifications
+
+## OAuth2 Provider Implementation
+
+### OAuth2 Spec Compliance
+
+1. **Follow RFC 6749 for token responses**
+   - Use `expires_in` (seconds) not `expiry` (timestamp) in token responses
+   - Return proper OAuth2 error format: `{"error": "code", "error_description": "details"}`
+
+2. **Error Response Format**
+   - Create OAuth2-compliant error responses for token endpoint
+   - Use standard error codes: `invalid_client`, `invalid_grant`, `invalid_request`
+   - Avoid generic error responses for OAuth2 endpoints
+
+### PKCE Implementation
+
+- Support both with and without PKCE for backward compatibility
+- Use S256 method for code challenge
+- Properly validate code_verifier against stored code_challenge
+
+### UI Authorization Flow
+
+- Use POST requests for consent, not GET with links
+- Avoid dependency on referer headers for security decisions
+- Support proper state parameter validation
+
+### RFC 8707 Resource Indicators
+
+- Store resource parameters in database for server-side validation (opaque tokens)
+- Validate resource consistency between authorization and token requests
+- Support audience validation in refresh token flows
+- Resource parameter is optional but must be consistent when provided
+
+## OAuth2 Error Handling Pattern
+
+```go
+// Define specific OAuth2 errors
+var (
+    errInvalidPKCE = xerrors.New("invalid code_verifier")
+)
+
+// Use OAuth2-compliant error responses
+type OAuth2Error struct {
+    Error            string `json:"error"`
+    ErrorDescription string `json:"error_description,omitempty"`
+}
+
+// Return proper OAuth2 errors
+if errors.Is(err, errInvalidPKCE) {
+    writeOAuth2Error(ctx, rw, http.StatusBadRequest, "invalid_grant", "The PKCE code verifier is invalid")
+    return
+}
+```
+
+## Testing OAuth2 Features
+
+### Test Scripts
+
+Located in `./scripts/oauth2/`:
+
+- `test-mcp-oauth2.sh` - Full automated test suite
+- `setup-test-app.sh` - Create test OAuth2 app
+- `cleanup-test-app.sh` - Remove test app
+- `generate-pkce.sh` - Generate PKCE parameters
+- `test-manual-flow.sh` - Manual browser testing
+
+Always run the full test suite after OAuth2 changes:
+
+```bash
+./scripts/oauth2/test-mcp-oauth2.sh
+```
+
+### RFC Protocol Testing
+
+1. **Compliance Test Coverage**:
+   - Test all RFC-defined error codes and responses
+   - Validate proper HTTP status codes for different scenarios
+   - Test protocol-specific edge cases (URI formats, token formats, etc.)
+
+2. **Security Boundary Testing**:
+   - Test client isolation and privilege separation
+   - Verify information disclosure protections
+   - Test token security and proper invalidation
+
+## Common OAuth2 Issues
+
+1. **OAuth2 endpoints returning wrong error format** - Ensure OAuth2 endpoints return RFC 6749 compliant errors
+2. **Resource indicator validation failing** - Ensure database stores and retrieves resource parameters correctly
+3. **PKCE tests failing** - Verify both authorization code storage and token exchange handle PKCE fields
+4. **RFC compliance failures** - Verify against actual RFC specifications, not assumptions
+5. **Authorization context errors in public endpoints** - Use `dbauthz.AsSystemRestricted(ctx)` pattern
+6. **Default value mismatches** - Ensure database migrations match application code defaults
+7. **Bearer token authentication issues** - Check token extraction precedence and format validation
+8. **URI validation failures** - Support both standard schemes and custom schemes per protocol requirements
+
+## Authorization Context Patterns
+
+```go
+// Public endpoints needing system access (OAuth2 registration)
+app, err := api.Database.GetOAuth2ProviderAppByClientID(dbauthz.AsSystemRestricted(ctx), clientID)
+
+// Authenticated endpoints with user context
+app, err := api.Database.GetOAuth2ProviderAppByClientID(ctx, clientID)
+
+// System operations in middleware
+roles, err := db.GetAuthorizationUserRoles(dbauthz.AsSystemRestricted(ctx), userID)
+```
+
+## OAuth2/Authentication Work Patterns
+
+- Types go in `codersdk/oauth2.go` or similar
+- Handlers go in `coderd/oauth2.go` or `coderd/identityprovider/`
+- Database fields need migration + audit table updates
+- Always support backward compatibility
+
+## Protocol Implementation Checklist
+
+Before completing OAuth2 or authentication feature work:
+
+- [ ] Verify RFC compliance by reading actual specifications
+- [ ] Implement proper error response formats per protocol
+- [ ] Add comprehensive validation for all protocol fields
+- [ ] Test security boundaries and token handling
+- [ ] Update RBAC permissions for new resources
+- [ ] Add audit logging support if applicable
+- [ ] Create database migrations with proper defaults
+- [ ] Add comprehensive test coverage including edge cases
+- [ ] Verify linting compliance
+- [ ] Test both positive and negative scenarios
+- [ ] Document protocol-specific patterns and requirements

.claude/docs/TESTING.md

@@ -0,0 +1,212 @@
+# Testing Patterns and Best Practices
+
+## Testing Best Practices
+
+### Avoiding Race Conditions
+
+1. **Unique Test Identifiers**:
+   - Never use hardcoded names in concurrent tests
+   - Use `time.Now().UnixNano()` or similar for unique identifiers
+   - Example: `fmt.Sprintf("test-client-%s-%d", t.Name(), time.Now().UnixNano())`
+
+2. **Database Constraint Awareness**:
+   - Understand unique constraints that can cause test conflicts
+   - Generate unique values for all constrained fields
+   - Test name isolation prevents cross-test interference
+
+### Testing Patterns
+
+- Use table-driven tests for comprehensive coverage
+- Mock external dependencies
+- Test both positive and negative cases
+- Use `testutil.WaitLong` for timeouts in tests
+
+### Test Package Naming
+
+- **Test packages**: Use `package_test` naming (e.g., `identityprovider_test`) for black-box testing
+
+## RFC Protocol Testing
+
+### Compliance Test Coverage
+
+1. **Test all RFC-defined error codes and responses**
+2. **Validate proper HTTP status codes for different scenarios**
+3. **Test protocol-specific edge cases** (URI formats, token formats, etc.)
+
+### Security Boundary Testing
+
+1. **Test client isolation and privilege separation**
+2. **Verify information disclosure protections**
+3. **Test token security and proper invalidation**
+
+## Test Organization
+
+### Test File Structure
+
+```
+coderd/
+├── oauth2.go                    # Implementation
+├── oauth2_test.go              # Main tests
+├── oauth2_test_helpers.go      # Test utilities
+└── oauth2_validation.go        # Validation logic
+```
+
+### Test Categories
+
+1. **Unit Tests**: Test individual functions in isolation
+2. **Integration Tests**: Test API endpoints with database
+3. **End-to-End Tests**: Full workflow testing
+4. **Race Tests**: Concurrent access testing
+
+## Test Commands
+
+### Running Tests
+
+| Command | Purpose |
+|---------|---------|
+| `make test` | Run all Go tests |
+| `make test RUN=TestFunctionName` | Run specific test |
+| `go test -v ./path/to/package -run TestFunctionName` | Run test with verbose output |
+| `make test-postgres` | Run tests with Postgres database |
+| `make test-race` | Run tests with Go race detector |
+| `make test-e2e` | Run end-to-end tests |
+
+### Frontend Testing
+
+| Command | Purpose |
+|---------|---------|
+| `pnpm test` | Run frontend tests |
+| `pnpm check` | Run code checks |
+
+## Common Testing Issues
+
+### Database-Related
+
+1. **SQL type errors** - Use `sql.Null*` types for nullable fields
+2. **Race conditions in tests** - Use unique identifiers instead of hardcoded names
+
+### OAuth2 Testing
+
+1. **PKCE tests failing** - Verify both authorization code storage and token exchange handle PKCE fields
+2. **Resource indicator validation failing** - Ensure database stores and retrieves resource parameters correctly
+
+### General Issues
+
+1. **Missing newlines** - Ensure files end with newline character
+2. **Package naming errors** - Use `package_test` naming for test files
+3. **Log message formatting errors** - Use lowercase, descriptive messages without special characters
+
+## Systematic Testing Approach
+
+### Multi-Issue Problem Solving
+
+When facing multiple failing tests or complex integration issues:
+
+1. **Identify Root Causes**:
+   - Run failing tests individually to isolate issues
+   - Use LSP tools to trace through call chains
+   - Check both compilation and runtime errors
+
+2. **Fix in Logical Order**:
+   - Address compilation issues first (imports, syntax)
+   - Fix authorization and RBAC issues next
+   - Resolve business logic and validation issues
+   - Handle edge cases and race conditions last
+
+3. **Verification Strategy**:
+   - Test each fix individually before moving to next issue
+   - Use `make lint` and `make gen` after database changes
+   - Verify RFC compliance with actual specifications
+   - Run comprehensive test suites before considering complete
+
+## Test Data Management
+
+### Unique Test Data
+
+```go
+// Good: Unique identifiers prevent conflicts
+clientName := fmt.Sprintf("test-client-%s-%d", t.Name(), time.Now().UnixNano())
+
+// Bad: Hardcoded names cause race conditions
+clientName := "test-client"
+```
+
+### Test Cleanup
+
+```go
+func TestSomething(t *testing.T) {
+    // Setup
+    client := coderdtest.New(t, nil)
+
+    // Test code here
+
+    // Cleanup happens automatically via t.Cleanup() in coderdtest
+}
+```
+
+## Test Utilities
+
+### Common Test Patterns
+
+```go
+// Table-driven tests
+tests := []struct {
+    name     string
+    input    InputType
+    expected OutputType
+    wantErr  bool
+}{
+    {
+        name:     "valid input",
+        input:    validInput,
+        expected: expectedOutput,
+        wantErr:  false,
+    },
+    // ... more test cases
+}
+
+for _, tt := range tests {
+    t.Run(tt.name, func(t *testing.T) {
+        result, err := functionUnderTest(tt.input)
+        if tt.wantErr {
+            require.Error(t, err)
+            return
+        }
+        require.NoError(t, err)
+        require.Equal(t, tt.expected, result)
+    })
+}
+```
+
+### Test Assertions
+
+```go
+// Use testify/require for assertions
+require.NoError(t, err)
+require.Equal(t, expected, actual)
+require.NotNil(t, result)
+require.True(t, condition)
+```
+
+## Performance Testing
+
+### Load Testing
+
+- Use `scaletest/` directory for load testing scenarios
+- Run `./scaletest/scaletest.sh` for performance testing
+
+### Benchmarking
+
+```go
+func BenchmarkFunction(b *testing.B) {
+    for i := 0; i < b.N; i++ {
+        // Function call to benchmark
+        _ = functionUnderTest(input)
+    }
+}
+```
+
+Run benchmarks with:
+```bash
+go test -bench=. -benchmem ./package/path
+```

.claude/docs/TROUBLESHOOTING.md

@@ -0,0 +1,231 @@
+# Troubleshooting Guide
+
+## Common Issues
+
+### Database Issues
+
+1. **"Audit table entry missing action"**
+   - **Solution**: Update `enterprise/audit/table.go`
+   - Add each new field with appropriate action (ActionTrack, ActionIgnore, ActionSecret)
+   - Run `make gen` to verify no audit errors
+
+2. **SQL type errors**
+   - **Solution**: Use `sql.Null*` types for nullable fields
+   - Set `.Valid = true` when providing values
+   - Example:
+
+     ```go
+     CodeChallenge: sql.NullString{
+         String: params.codeChallenge,
+         Valid:  params.codeChallenge != "",
+     }
+     ```
+
+### Testing Issues
+
+3. **"package should be X_test"**
+   - **Solution**: Use `package_test` naming for test files
+   - Example: `identityprovider_test` for black-box testing
+
+4. **Race conditions in tests**
+   - **Solution**: Use unique identifiers instead of hardcoded names
+   - Example: `fmt.Sprintf("test-client-%s-%d", t.Name(), time.Now().UnixNano())`
+   - Never use hardcoded names in concurrent tests
+
+5. **Missing newlines**
+   - **Solution**: Ensure files end with newline character
+   - Most editors can be configured to add this automatically
+
+### OAuth2 Issues
+
+6. **OAuth2 endpoints returning wrong error format**
+   - **Solution**: Ensure OAuth2 endpoints return RFC 6749 compliant errors
+   - Use standard error codes: `invalid_client`, `invalid_grant`, `invalid_request`
+   - Format: `{"error": "code", "error_description": "details"}`
+
+7. **Resource indicator validation failing**
+   - **Solution**: Ensure database stores and retrieves resource parameters correctly
+   - Check both authorization code storage and token exchange handling
+
+8. **PKCE tests failing**
+    - **Solution**: Verify both authorization code storage and token exchange handle PKCE fields
+    - Check `CodeChallenge` and `CodeChallengeMethod` field handling
+
+### RFC Compliance Issues
+
+9. **RFC compliance failures**
+    - **Solution**: Verify against actual RFC specifications, not assumptions
+    - Use WebFetch tool to get current RFC content for compliance verification
+    - Read the actual RFC specifications before implementation
+
+10. **Default value mismatches**
+    - **Solution**: Ensure database migrations match application code defaults
+    - Example: RFC 7591 specifies `client_secret_basic` as default, not `client_secret_post`
+
+### Authorization Issues
+
+11. **Authorization context errors in public endpoints**
+    - **Solution**: Use `dbauthz.AsSystemRestricted(ctx)` pattern
+    - Example:
+
+      ```go
+      // Public endpoints needing system access
+      app, err := api.Database.GetOAuth2ProviderAppByClientID(dbauthz.AsSystemRestricted(ctx), clientID)
+      ```
+
+### Authentication Issues
+
+12. **Bearer token authentication issues**
+    - **Solution**: Check token extraction precedence and format validation
+    - Ensure proper RFC 6750 Bearer Token Support implementation
+
+13. **URI validation failures**
+    - **Solution**: Support both standard schemes and custom schemes per protocol requirements
+    - Native OAuth2 apps may use custom schemes
+
+### General Development Issues
+
+14. **Log message formatting errors**
+    - **Solution**: Use lowercase, descriptive messages without special characters
+    - Follow Go logging conventions
+
+## Systematic Debugging Approach
+
+### Multi-Issue Problem Solving
+
+When facing multiple failing tests or complex integration issues:
+
+1. **Identify Root Causes**:
+   - Run failing tests individually to isolate issues
+   - Use LSP tools to trace through call chains
+   - Check both compilation and runtime errors
+
+2. **Fix in Logical Order**:
+   - Address compilation issues first (imports, syntax)
+   - Fix authorization and RBAC issues next
+   - Resolve business logic and validation issues
+   - Handle edge cases and race conditions last
+
+3. **Verification Strategy**:
+   - Test each fix individually before moving to next issue
+   - Use `make lint` and `make gen` after database changes
+   - Verify RFC compliance with actual specifications
+   - Run comprehensive test suites before considering complete
+
+## Debug Commands
+
+### Useful Debug Commands
+
+| Command                                      | Purpose                               |
+|----------------------------------------------|---------------------------------------|
+| `make lint`                                  | Run all linters                       |
+| `make gen`                                   | Generate mocks, database queries      |
+| `go test -v ./path/to/package -run TestName` | Run specific test with verbose output |
+| `go test -race ./...`                        | Run tests with race detector          |
+
+### LSP Debugging
+
+#### Go LSP (Backend)
+
+| Command                                            | Purpose                      |
+|----------------------------------------------------|------------------------------|
+| `mcp__go-language-server__definition symbolName`   | Find function definition     |
+| `mcp__go-language-server__references symbolName`   | Find all references          |
+| `mcp__go-language-server__diagnostics filePath`    | Check for compilation errors |
+| `mcp__go-language-server__hover filePath line col` | Get type information         |
+
+#### TypeScript LSP (Frontend)
+
+| Command                                                                    | Purpose                            |
+|----------------------------------------------------------------------------|------------------------------------|
+| `mcp__typescript-language-server__definition symbolName`                   | Find component/function definition |
+| `mcp__typescript-language-server__references symbolName`                   | Find all component/type usages     |
+| `mcp__typescript-language-server__diagnostics filePath`                    | Check for TypeScript errors        |
+| `mcp__typescript-language-server__hover filePath line col`                 | Get type information               |
+| `mcp__typescript-language-server__rename_symbol filePath line col newName` | Rename across codebase             |
+
+## Common Error Messages
+
+### Database Errors
+
+**Error**: `pq: relation "oauth2_provider_app_codes" does not exist`
+
+- **Cause**: Missing database migration
+- **Solution**: Run database migrations, check migration files
+
+**Error**: `audit table entry missing action for field X`
+
+- **Cause**: New field added without audit table update
+- **Solution**: Update `enterprise/audit/table.go`
+
+### Go Compilation Errors
+
+**Error**: `package should be identityprovider_test`
+
+- **Cause**: Test package naming convention violation
+- **Solution**: Use `package_test` naming for black-box tests
+
+**Error**: `cannot use X (type Y) as type Z`
+
+- **Cause**: Type mismatch, often with nullable fields
+- **Solution**: Use appropriate `sql.Null*` types
+
+### OAuth2 Errors
+
+**Error**: `invalid_client` but client exists
+
+- **Cause**: Authorization context issue
+- **Solution**: Use `dbauthz.AsSystemRestricted(ctx)` for public endpoints
+
+**Error**: PKCE validation failing
+
+- **Cause**: Missing PKCE fields in database operations
+- **Solution**: Ensure `CodeChallenge` and `CodeChallengeMethod` are handled
+
+## Prevention Strategies
+
+### Before Making Changes
+
+1. **Read the relevant documentation**
+2. **Check if similar patterns exist in codebase**
+3. **Understand the authorization context requirements**
+4. **Plan database changes carefully**
+
+### During Development
+
+1. **Run tests frequently**: `make test`
+2. **Use LSP tools for navigation**: Avoid manual searching
+3. **Follow RFC specifications precisely**
+4. **Update audit tables when adding database fields**
+
+### Before Committing
+
+1. **Run full test suite**: `make test`
+2. **Check linting**: `make lint`
+3. **Test with race detector**: `make test-race`
+
+## Getting Help
+
+### Internal Resources
+
+- Check existing similar implementations in codebase
+- Use LSP tools to understand code relationships
+  - For Go code: Use `mcp__go-language-server__*` commands
+  - For TypeScript/React code: Use `mcp__typescript-language-server__*` commands
+- Read related test files for expected behavior
+
+### External Resources
+
+- Official RFC specifications for protocol compliance
+- Go documentation for language features
+- PostgreSQL documentation for database issues
+
+### Debug Information Collection
+
+When reporting issues, include:
+
+1. **Exact error message**
+2. **Steps to reproduce**
+3. **Relevant code snippets**
+4. **Test output (if applicable)**
+5. **Environment information** (OS, Go version, etc.)

.claude/docs/WORKFLOWS.md

@@ -0,0 +1,223 @@
+# Development Workflows and Guidelines
+
+## Quick Start Checklist for New Features
+
+### Before Starting
+
+- [ ] Run `git pull` to ensure you're on latest code
+- [ ] Check if feature touches database - you'll need migrations
+- [ ] Check if feature touches audit logs - update `enterprise/audit/table.go`
+
+## Development Server
+
+### Starting Development Mode
+
+- **Use `./scripts/develop.sh` to start Coder in development mode**
+- This automatically builds and runs with `--dev` flag and proper access URL
+- **⚠️ Do NOT manually run `make build && ./coder server --dev` - use the script instead**
+
+### Development Workflow
+
+1. **Always start with the development script**: `./scripts/develop.sh`
+2. **Make changes** to your code
+3. **The script will automatically rebuild** and restart as needed
+4. **Access the development server** at the URL provided by the script
+
+## Code Style Guidelines
+
+### Go Style
+
+- Follow [Effective Go](https://go.dev/doc/effective_go) and [Go's Code Review Comments](https://github.com/golang/go/wiki/CodeReviewComments)
+- Create packages when used during implementation
+- Validate abstractions against implementations
+- **Test packages**: Use `package_test` naming (e.g., `identityprovider_test`) for black-box testing
+
+### Error Handling
+
+- Use descriptive error messages
+- Wrap errors with context
+- Propagate errors appropriately
+- Use proper error types
+- Pattern: `xerrors.Errorf("failed to X: %w", err)`
+
+### Naming Conventions
+
+- Use clear, descriptive names
+- Abbreviate only when obvious
+- Follow Go and TypeScript naming conventions
+
+### Comments
+
+- Document exported functions, types, and non-obvious logic
+- Follow JSDoc format for TypeScript
+- Use godoc format for Go code
+
+## Database Migration Workflows
+
+### Migration Guidelines
+
+1. **Create migration files**:
+   - Location: `coderd/database/migrations/`
+   - Format: `{number}_{description}.{up|down}.sql`
+   - Number must be unique and sequential
+   - Always include both up and down migrations
+
+2. **Use helper scripts**:
+   - `./coderd/database/migrations/create_migration.sh "migration name"` - Creates new migration files
+   - `./coderd/database/migrations/fix_migration_numbers.sh` - Renumbers migrations to avoid conflicts
+   - `./coderd/database/migrations/create_fixture.sh "fixture name"` - Creates test fixtures for migrations
+
+3. **Update database queries**:
+   - **MUST DO**: Any changes to database - adding queries, modifying queries should be done in the `coderd/database/queries/*.sql` files
+   - **MUST DO**: Queries are grouped in files relating to context - e.g. `prebuilds.sql`, `users.sql`, `oauth2.sql`
+   - After making changes to any `coderd/database/queries/*.sql` files you must run `make gen` to generate respective ORM changes
+
+4. **Handle nullable fields**:
+   - Use `sql.NullString`, `sql.NullBool`, etc. for optional database fields
+   - Set `.Valid = true` when providing values
+
+5. **Audit table updates**:
+   - If adding fields to auditable types, update `enterprise/audit/table.go`
+   - Add each new field with appropriate action (ActionTrack, ActionIgnore, ActionSecret)
+   - Run `make gen` to verify no audit errors
+
+### Database Generation Process
+
+1. Modify SQL files in `coderd/database/queries/`
+2. Run `make gen`
+3. If errors about audit table, update `enterprise/audit/table.go`
+4. Run `make gen` again
+5. Run `make lint` to catch any remaining issues
+
+## API Development Workflow
+
+### Adding New API Endpoints
+
+1. **Define types** in `codersdk/` package
+2. **Add handler** in appropriate `coderd/` file
+3. **Register route** in `coderd/coderd.go`
+4. **Add tests** in `coderd/*_test.go` files
+5. **Update OpenAPI** by running `make gen`
+
+## Testing Workflows
+
+### Test Execution
+
+- Run full test suite: `make test`
+- Run specific test: `make test RUN=TestFunctionName`
+- Run with Postgres: `make test-postgres`
+- Run with race detector: `make test-race`
+- Run end-to-end tests: `make test-e2e`
+
+### Test Development
+
+- Use table-driven tests for comprehensive coverage
+- Mock external dependencies
+- Test both positive and negative cases
+- Use `testutil.WaitLong` for timeouts in tests
+- Always use `t.Parallel()` in tests
+
+## Commit Style
+
+- Follow [Conventional Commits 1.0.0](https://www.conventionalcommits.org/en/v1.0.0/)
+- Format: `type(scope): message`
+- Types: `feat`, `fix`, `docs`, `style`, `refactor`, `test`, `chore`
+- Keep message titles concise (~70 characters)
+- Use imperative, present tense in commit titles
+
+## Code Navigation and Investigation
+
+### Using LSP Tools (STRONGLY RECOMMENDED)
+
+**IMPORTANT**: Always use LSP tools for code navigation and understanding. These tools provide accurate, real-time analysis of the codebase and should be your first choice for code investigation.
+
+#### Go LSP Tools (for backend code)
+
+1. **Find function definitions** (USE THIS FREQUENTLY):
+   - `mcp__go-language-server__definition symbolName`
+   - Example: `mcp__go-language-server__definition getOAuth2ProviderAppAuthorize`
+   - Quickly jump to function implementations across packages
+
+2. **Find symbol references** (ESSENTIAL FOR UNDERSTANDING IMPACT):
+   - `mcp__go-language-server__references symbolName`
+   - Locate all usages of functions, types, or variables
+   - Critical for refactoring and understanding data flow
+
+3. **Get symbol information**:
+   - `mcp__go-language-server__hover filePath line column`
+   - Get type information and documentation at specific positions
+
+#### TypeScript LSP Tools (for frontend code in site/)
+
+1. **Find component/function definitions** (USE THIS FREQUENTLY):
+   - `mcp__typescript-language-server__definition symbolName`
+   - Example: `mcp__typescript-language-server__definition LoginPage`
+   - Quickly navigate to React components, hooks, and utility functions
+
+2. **Find symbol references** (ESSENTIAL FOR UNDERSTANDING IMPACT):
+   - `mcp__typescript-language-server__references symbolName`
+   - Locate all usages of components, types, or functions
+   - Critical for refactoring React components and understanding prop usage
+
+3. **Get type information**:
+   - `mcp__typescript-language-server__hover filePath line column`
+   - Get TypeScript type information and JSDoc documentation
+
+4. **Rename symbols safely**:
+   - `mcp__typescript-language-server__rename_symbol filePath line column newName`
+   - Rename components, props, or functions across the entire codebase
+
+5. **Check for TypeScript errors**:
+   - `mcp__typescript-language-server__diagnostics filePath`
+   - Get compilation errors and warnings for a specific file
+
+### Investigation Strategy (LSP-First Approach)
+
+#### Backend Investigation (Go)
+
+1. **Start with route registration** in `coderd/coderd.go` to understand API endpoints
+2. **Use Go LSP `definition` lookup** to trace from route handlers to actual implementations
+3. **Use Go LSP `references`** to understand how functions are called throughout the codebase
+4. **Follow the middleware chain** using LSP tools to understand request processing flow
+5. **Check test files** for expected behavior and error patterns
+
+#### Frontend Investigation (TypeScript/React)
+
+1. **Start with route definitions** in `site/src/App.tsx` or router configuration
+2. **Use TypeScript LSP `definition`** to navigate to React components and hooks
+3. **Use TypeScript LSP `references`** to find all component usages and prop drilling
+4. **Follow the component hierarchy** using LSP tools to understand data flow
+5. **Check for TypeScript errors** with `diagnostics` before making changes
+6. **Examine test files** (`.test.tsx`) for component behavior and expected props
+
+## Troubleshooting Development Issues
+
+### Common Issues
+
+1. **Development server won't start** - Use `./scripts/develop.sh` instead of manual commands
+2. **Database migration errors** - Check migration file format and use helper scripts
+3. **Audit table errors** - Update `enterprise/audit/table.go` with new fields
+4. **OAuth2 compliance issues** - Ensure RFC-compliant error responses
+
+### Debug Commands
+
+- Check linting: `make lint`
+- Generate code: `make gen`
+- Clean build: `make clean`
+
+## Development Environment Setup
+
+### Prerequisites
+
+- Go (version specified in go.mod)
+- Node.js and pnpm for frontend development
+- PostgreSQL for database testing
+- Docker for containerized testing
+
+### First Time Setup
+
+1. Clone the repository
+2. Run `./scripts/develop.sh` to start development server
+3. Access the development URL provided
+4. Create admin user as prompted
+5. Begin development

.claude/scripts/format.sh

@@ -0,0 +1,133 @@
+#!/bin/bash
+
+# Claude Code hook script for file formatting
+# This script integrates with the centralized Makefile formatting targets
+# and supports the Claude Code hooks system for automatic file formatting.
+
+set -euo pipefail
+
+# A variable to memoize the command for canonicalizing paths.
+_CANONICALIZE_CMD=""
+
+# canonicalize_path resolves a path to its absolute, canonical form.
+# It tries 'realpath' and 'readlink -f' in order.
+# The chosen command is memoized to avoid repeated checks.
+# If none of these are available, it returns an empty string.
+canonicalize_path() {
+	local path_to_resolve="$1"
+
+	# If we haven't determined a command yet, find one.
+	if [[ -z "$_CANONICALIZE_CMD" ]]; then
+		if command -v realpath >/dev/null 2>&1; then
+			_CANONICALIZE_CMD="realpath"
+		elif command -v readlink >/dev/null 2>&1 && readlink -f . >/dev/null 2>&1; then
+			_CANONICALIZE_CMD="readlink"
+		else
+			# No command found, so we can't resolve.
+			# We set a "none" value to prevent re-checking.
+			_CANONICALIZE_CMD="none"
+		fi
+	fi
+
+	# Now, execute the command.
+	case "$_CANONICALIZE_CMD" in
+	realpath)
+		realpath "$path_to_resolve" 2>/dev/null
+		;;
+	readlink)
+		readlink -f "$path_to_resolve" 2>/dev/null
+		;;
+	*)
+		# This handles the "none" case or any unexpected error.
+		echo ""
+		;;
+	esac
+}
+
+# Read JSON input from stdin
+input=$(cat)
+
+# Extract the file path from the JSON input
+# Expected format: {"tool_input": {"file_path": "/absolute/path/to/file"}} or {"tool_response": {"filePath": "/absolute/path/to/file"}}
+file_path=$(echo "$input" | jq -r '.tool_input.file_path // .tool_response.filePath // empty')
+
+# Secure path canonicalization to prevent path traversal attacks
+# Resolve repo root to an absolute, canonical path.
+repo_root_raw="$(cd "$(dirname "$0")/../.." && pwd)"
+repo_root="$(canonicalize_path "$repo_root_raw")"
+if [[ -z "$repo_root" ]]; then
+	# Fallback if canonicalization fails
+	repo_root="$repo_root_raw"
+fi
+
+# Resolve the input path to an absolute path
+if [[ "$file_path" = /* ]]; then
+	# Already absolute
+	abs_file_path="$file_path"
+else
+	# Make relative paths absolute from repo root
+	abs_file_path="$repo_root/$file_path"
+fi
+
+# Canonicalize the path (resolve symlinks and ".." segments)
+canonical_file_path="$(canonicalize_path "$abs_file_path")"
+
+# Check if canonicalization failed or if the resolved path is outside the repo
+if [[ -z "$canonical_file_path" ]] || { [[ "$canonical_file_path" != "$repo_root" ]] && [[ "$canonical_file_path" != "$repo_root"/* ]]; }; then
+	echo "Error: File path is outside repository or invalid: $file_path" >&2
+	exit 1
+fi
+
+# Handle the case where the file path is the repository root itself.
+if [[ "$canonical_file_path" == "$repo_root" ]]; then
+	echo "Warning: Formatting the repository root is not a supported operation. Skipping." >&2
+	exit 0
+fi
+
+# Convert back to relative path from repo root for consistency
+file_path="${canonical_file_path#"$repo_root"/}"
+
+if [[ -z "$file_path" ]]; then
+	echo "Error: No file path provided in input" >&2
+	exit 1
+fi
+
+# Check if file exists
+if [[ ! -f "$file_path" ]]; then
+	echo "Error: File does not exist: $file_path" >&2
+	exit 1
+fi
+
+# Get the file extension to determine the appropriate formatter
+file_ext="${file_path##*.}"
+
+# Change to the project root directory (where the Makefile is located)
+cd "$(dirname "$0")/../.."
+
+# Call the appropriate Makefile target based on file extension
+case "$file_ext" in
+go)
+	make fmt/go FILE="$file_path"
+	echo "✓ Formatted Go file: $file_path"
+	;;
+js | jsx | ts | tsx)
+	make fmt/ts FILE="$file_path"
+	echo "✓ Formatted TypeScript/JavaScript file: $file_path"
+	;;
+tf | tfvars)
+	make fmt/terraform FILE="$file_path"
+	echo "✓ Formatted Terraform file: $file_path"
+	;;
+sh)
+	make fmt/shfmt FILE="$file_path"
+	echo "✓ Formatted shell script: $file_path"
+	;;
+md)
+	make fmt/markdown FILE="$file_path"
+	echo "✓ Formatted Markdown file: $file_path"
+	;;
+*)
+	echo "No formatter available for file extension: $file_ext"
+	exit 0
+	;;
+esac

.claude/settings.json

@@ -0,0 +1,15 @@
+{
+	"hooks": {
+		"PostToolUse": [
+			{
+				"matcher": "Edit|Write|MultiEdit",
+				"hooks": [
+					{
+						"type": "command",
+						"command": ".claude/scripts/format.sh"
+					}
+				]
+			}
+		]
+	}
+}

.devcontainer/devcontainer.json

@@ -1,11 +1,16 @@
 {
 	"name": "Development environments on your infrastructure",
 	"image": "codercom/oss-dogfood:latest",
-
 	"features": {
-		// See all possible options here https://github.com/devcontainers/features/tree/main/src/docker-in-docker
 		"ghcr.io/devcontainers/features/docker-in-docker:2": {
 			"moby": "false"
+		},
+		"ghcr.io/coder/devcontainer-features/code-server:1": {
+			"auth": "none",
+			"port": 13337
+		},
+		"./filebrowser": {
+			"folder": "${containerWorkspaceFolder}"
 		}
 	},
 	// SYS_PTRACE to enable go debugging
@@ -13,6 +18,65 @@
 	"customizations": {
 		"vscode": {
 			"extensions": ["biomejs.biome"]
+		},
+		"coder": {
+			"apps": [
+				{
+					"slug": "cursor",
+					"displayName": "Cursor Desktop",
+					"url": "cursor://coder.coder-remote/openDevContainer?owner=${localEnv:CODER_WORKSPACE_OWNER_NAME}&workspace=${localEnv:CODER_WORKSPACE_NAME}&agent=${localEnv:CODER_WORKSPACE_PARENT_AGENT_NAME}&url=${localEnv:CODER_URL}&token=$SESSION_TOKEN&devContainerName=${localEnv:CONTAINER_ID}&devContainerFolder=${containerWorkspaceFolder}&localWorkspaceFolder=${localWorkspaceFolder}",
+					"external": true,
+					"icon": "/icon/cursor.svg",
+					"order": 1
+				},
+				{
+					"slug": "windsurf",
+					"displayName": "Windsurf Editor",
+					"url": "windsurf://coder.coder-remote/openDevContainer?owner=${localEnv:CODER_WORKSPACE_OWNER_NAME}&workspace=${localEnv:CODER_WORKSPACE_NAME}&agent=${localEnv:CODER_WORKSPACE_PARENT_AGENT_NAME}&url=${localEnv:CODER_URL}&token=$SESSION_TOKEN&devContainerName=${localEnv:CONTAINER_ID}&devContainerFolder=${containerWorkspaceFolder}&localWorkspaceFolder=${localWorkspaceFolder}",
+					"external": true,
+					"icon": "/icon/windsurf.svg",
+					"order": 4
+				},
+				{
+					"slug": "zed",
+					"displayName": "Zed Editor",
+					"url": "zed://ssh/${localEnv:CODER_WORKSPACE_AGENT_NAME}.${localEnv:CODER_WORKSPACE_NAME}.${localEnv:CODER_WORKSPACE_OWNER_NAME}.coder${containerWorkspaceFolder}",
+					"external": true,
+					"icon": "/icon/zed.svg",
+					"order": 5
+				},
+				// Reproduce `code-server` app here from the code-server
+				// feature so that we can set the correct folder and order.
+				// Currently, the order cannot be specified via option because
+				// we parse it as a number whereas variable interpolation
+				// results in a string. Additionally we set health check which
+				// is not yet set in the feature.
+				{
+					"slug": "code-server",
+					"displayName": "code-server",
+					"url": "http://${localEnv:FEATURE_CODE_SERVER_OPTION_HOST:127.0.0.1}:${localEnv:FEATURE_CODE_SERVER_OPTION_PORT:8080}/?folder=${containerWorkspaceFolder}",
+					"openIn": "${localEnv:FEATURE_CODE_SERVER_OPTION_APPOPENIN:slim-window}",
+					"share": "${localEnv:FEATURE_CODE_SERVER_OPTION_APPSHARE:owner}",
+					"icon": "/icon/code.svg",
+					"group": "${localEnv:FEATURE_CODE_SERVER_OPTION_APPGROUP:Web Editors}",
+					"order": 3,
+					"healthCheck": {
+						"url": "http://${localEnv:FEATURE_CODE_SERVER_OPTION_HOST:127.0.0.1}:${localEnv:FEATURE_CODE_SERVER_OPTION_PORT:8080}/healthz",
+						"interval": 5,
+						"threshold": 2
+					}
+				}
+			]
 		}
-	}
+	},
+	"mounts": [
+		// Add a volume for the Coder home directory to persist shell history,
+		// and speed up dotfiles init and/or personalization.
+		"source=coder-coder-devcontainer-home,target=/home/coder,type=volume",
+		// Mount the entire home because conditional mounts are not supported.
+		// See: https://github.com/devcontainers/spec/issues/132
+		"source=${localEnv:HOME},target=/mnt/home/coder,type=bind,readonly"
+	],
+	"postCreateCommand": ["./.devcontainer/scripts/post_create.sh"],
+	"postStartCommand": ["./.devcontainer/scripts/post_start.sh"]
 }

.devcontainer/filebrowser/devcontainer-feature.json

@@ -0,0 +1,46 @@
+{
+	"id": "filebrowser",
+	"version": "0.0.1",
+	"name": "File Browser",
+	"description": "A web-based file browser for your development container",
+	"options": {
+		"port": {
+			"type": "string",
+			"default": "13339",
+			"description": "The port to run filebrowser on"
+		},
+		"folder": {
+			"type": "string",
+			"default": "",
+			"description": "The root directory for filebrowser to serve"
+		},
+		"baseUrl": {
+			"type": "string",
+			"default": "",
+			"description": "The base URL for filebrowser (e.g., /filebrowser)"
+		}
+	},
+	"entrypoint": "/usr/local/bin/filebrowser-entrypoint",
+	"dependsOn": {
+		"ghcr.io/devcontainers/features/common-utils:2": {}
+	},
+	"customizations": {
+		"coder": {
+			"apps": [
+				{
+					"slug": "filebrowser",
+					"displayName": "File Browser",
+					"url": "http://localhost:${localEnv:FEATURE_FILEBROWSER_OPTION_PORT:13339}",
+					"icon": "/icon/filebrowser.svg",
+					"order": 3,
+					"subdomain": true,
+					"healthcheck": {
+						"url": "http://localhost:${localEnv:FEATURE_FILEBROWSER_OPTION_PORT:13339}/health",
+						"interval": 5,
+						"threshold": 2
+					}
+				}
+			]
+		}
+	}
+}

.devcontainer/filebrowser/install.sh

@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+BOLD='\033[0;1m'
+
+printf "%sInstalling filebrowser\n\n" "${BOLD}"
+
+# Check if filebrowser is installed.
+if ! command -v filebrowser &>/dev/null; then
+	VERSION="v2.42.1"
+	EXPECTED_HASH="7d83c0f077df10a8ec9bfd9bf6e745da5d172c3c768a322b0e50583a6bc1d3cc"
+
+	curl -fsSL "https://github.com/filebrowser/filebrowser/releases/download/${VERSION}/linux-amd64-filebrowser.tar.gz" -o /tmp/filebrowser.tar.gz
+	echo "${EXPECTED_HASH} /tmp/filebrowser.tar.gz" | sha256sum -c
+	tar -xzf /tmp/filebrowser.tar.gz -C /tmp
+	sudo mv /tmp/filebrowser /usr/local/bin/
+	sudo chmod +x /usr/local/bin/filebrowser
+	rm /tmp/filebrowser.tar.gz
+fi
+
+# Create entrypoint.
+cat >/usr/local/bin/filebrowser-entrypoint <<EOF
+#!/usr/bin/env bash
+
+PORT="${PORT}"
+FOLDER="${FOLDER:-}"
+FOLDER="\${FOLDER:-\$(pwd)}"
+BASEURL="${BASEURL:-}"
+LOG_PATH=/tmp/filebrowser.log
+export FB_DATABASE="\${HOME}/.filebrowser.db"
+
+printf "🛠️ Configuring filebrowser\n\n"
+
+# Check if filebrowser db exists.
+if [[ ! -f "\${FB_DATABASE}" ]]; then
+	filebrowser config init >>\${LOG_PATH} 2>&1
+	filebrowser users add admin "" --perm.admin=true --viewMode=mosaic >>\${LOG_PATH} 2>&1
+fi
+
+filebrowser config set --baseurl=\${BASEURL} --port=\${PORT} --auth.method=noauth --root=\${FOLDER} >>\${LOG_PATH} 2>&1
+
+printf "👷 Starting filebrowser...\n\n"
+
+printf "📂 Serving \${FOLDER} at http://localhost:\${PORT}\n\n"
+
+filebrowser >>\${LOG_PATH} 2>&1 &
+
+printf "📝 Logs at \${LOG_PATH}\n\n"
+EOF
+
+chmod +x /usr/local/bin/filebrowser-entrypoint
+
+printf "🥳 Installation complete!\n\n"

.devcontainer/scripts/post_create.sh

@@ -0,0 +1,59 @@
+#!/bin/sh
+
+install_devcontainer_cli() {
+	npm install -g @devcontainers/cli@0.80.0 --integrity=sha512-w2EaxgjyeVGyzfA/KUEZBhyXqu/5PyWNXcnrXsZOBrt3aN2zyGiHrXoG54TF6K0b5DSCF01Rt5fnIyrCeFzFKw==
+}
+
+install_ssh_config() {
+	echo "🔑 Installing SSH configuration..."
+	rsync -a /mnt/home/coder/.ssh/ ~/.ssh/
+	chmod 0700 ~/.ssh
+}
+
+install_git_config() {
+	echo "📂 Installing Git configuration..."
+	if [ -f /mnt/home/coder/git/config ]; then
+		rsync -a /mnt/home/coder/git/ ~/.config/git/
+	elif [ -d /mnt/home/coder/.gitconfig ]; then
+		rsync -a /mnt/home/coder/.gitconfig ~/.gitconfig
+	else
+		echo "⚠️ Git configuration directory not found."
+	fi
+}
+
+install_dotfiles() {
+	if [ ! -d /mnt/home/coder/.config/coderv2/dotfiles ]; then
+		echo "⚠️ Dotfiles directory not found."
+		return
+	fi
+
+	cd /mnt/home/coder/.config/coderv2/dotfiles || return
+	for script in install.sh install bootstrap.sh bootstrap script/bootstrap setup.sh setup script/setup; do
+		if [ -x $script ]; then
+			echo "📦 Installing dotfiles..."
+			./$script || {
+				echo "❌ Error running $script. Please check the script for issues."
+				return
+			}
+			echo "✅ Dotfiles installed successfully."
+			return
+		fi
+	done
+	echo "⚠️ No install script found in dotfiles directory."
+}
+
+personalize() {
+	# Allow script to continue as Coder dogfood utilizes a hack to
+	# synchronize startup script execution.
+	touch /tmp/.coder-startup-script.done
+
+	if [ -x /mnt/home/coder/personalize ]; then
+		echo "🎨 Personalizing environment..."
+		/mnt/home/coder/personalize
+	fi
+}
+
+install_devcontainer_cli
+install_ssh_config
+install_dotfiles
+personalize

.devcontainer/scripts/post_start.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+
+# Start Docker service if not already running.
+sudo service docker start

.editorconfig

@@ -7,10 +7,22 @@ trim_trailing_whitespace = true
 insert_final_newline = true
 indent_style = tab
 
-[*.{yaml,yml,tf,tfvars,nix}]
+[*.{yaml,yml,tf,tftpl,tfvars,nix}]
+indent_style = space
+indent_size = 2
+
+[*.proto]
 indent_style = space
 indent_size = 2
 
 [coderd/database/dump.sql]
 indent_style = space
 indent_size = 4
+
+[coderd/database/queries/*.sql]
+indent_style = tab
+indent_size = 4
+
+[coderd/database/migrations/*.sql]
+indent_style = tab
+indent_size = 4

.gitattributes

@@ -15,6 +15,8 @@ provisionersdk/proto/*.go linguist-generated=true
 *.tfstate.json linguist-generated=true
 *.tfstate.dot linguist-generated=true
 *.tfplan.dot linguist-generated=true
+site/e2e/google/protobuf/timestampGenerated.ts
 site/e2e/provisionerGenerated.ts linguist-generated=true
+site/src/api/countriesGenerated.tsx linguist-generated=true
+site/src/api/rbacresourcesGenerated.tsx linguist-generated=true
 site/src/api/typesGenerated.ts linguist-generated=true
-site/src/pages/SetupPage/countries.tsx linguist-generated=true

.github/.linkspector.yml

@@ -24,5 +24,7 @@ ignorePatterns:
   - pattern: "mutagen.io"
   - pattern: "docs.github.com"
   - pattern: "claude.ai"
+  - pattern: "splunk.com"
+  - pattern: "stackoverflow.com/questions"
 aliveStatusCodes:
   - 200

.github/ISSUE_TEMPLATE/1-bug.yaml

@@ -2,6 +2,7 @@ name: "🐞 Bug"
 description: "File a bug report."
 title: "bug: "
 labels: ["needs-triage"]
+type: "Bug"
 body:
   - type: checkboxes
     id: existing_issues

.github/actions/embedded-pg-cache/download/action.yml

@@ -0,0 +1,49 @@
+name: "Download Embedded Postgres Cache"
+description: |
+  Downloads the embedded postgres cache and outputs today's cache key.
+  A PR job can use a cache if it was created by its base branch, its current
+  branch, or the default branch.
+  https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/caching-dependencies-to-speed-up-workflows#restrictions-for-accessing-a-cache
+outputs:
+  cache-key:
+    description: "Today's cache key"
+    value: ${{ steps.vars.outputs.cache-key }}
+inputs:
+  key-prefix:
+    description: "Prefix for the cache key"
+    required: true
+  cache-path:
+    description: "Path to the cache directory"
+    required: true
+runs:
+  using: "composite"
+  steps:
+    - name: Get date values and cache key
+      id: vars
+      shell: bash
+      run: |
+        export YEAR_MONTH=$(date +'%Y-%m')
+        export PREV_YEAR_MONTH=$(date -d 'last month' +'%Y-%m')
+        export DAY=$(date +'%d')
+        echo "year-month=$YEAR_MONTH" >> "$GITHUB_OUTPUT"
+        echo "prev-year-month=$PREV_YEAR_MONTH" >> "$GITHUB_OUTPUT"
+        echo "cache-key=${INPUTS_KEY_PREFIX}-${YEAR_MONTH}-${DAY}" >> "$GITHUB_OUTPUT"
+      env:
+        INPUTS_KEY_PREFIX: ${{ inputs.key-prefix }}
+
+    # By default, depot keeps caches for 14 days. This is plenty for embedded
+    # postgres, which changes infrequently.
+    # https://depot.dev/docs/github-actions/overview#cache-retention-policy
+    - name: Download embedded Postgres cache
+      uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+      with:
+        path: ${{ inputs.cache-path }}
+        key: ${{ steps.vars.outputs.cache-key }}
+        # > If there are multiple partial matches for a restore key, the action returns the most recently created cache.
+        # https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/caching-dependencies-to-speed-up-workflows#matching-a-cache-key
+        # The second restore key allows non-main branches to use the cache from the previous month.
+        # This prevents PRs from rebuilding the cache on the first day of the month.
+        # It also makes sure that once a month, the cache is fully reset.
+        restore-keys: |
+          ${{ inputs.key-prefix }}-${{ steps.vars.outputs.year-month }}-
+          ${{ github.ref != 'refs/heads/main' && format('{0}-{1}-', inputs.key-prefix, steps.vars.outputs.prev-year-month) || '' }}

.github/actions/embedded-pg-cache/upload/action.yml

@@ -0,0 +1,18 @@
+name: "Upload Embedded Postgres Cache"
+description: Uploads the embedded Postgres cache. This only runs on the main branch.
+inputs:
+  cache-key:
+    description: "Cache key"
+    required: true
+  cache-path:
+    description: "Path to the cache directory"
+    required: true
+runs:
+  using: "composite"
+  steps:
+    - name: Upload Embedded Postgres cache
+      if: ${{ github.ref == 'refs/heads/main' }}
+      uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+      with:
+        path: ${{ inputs.cache-path }}
+        key: ${{ inputs.cache-key }}

.github/actions/setup-embedded-pg-cache-paths/action.yml

@@ -0,0 +1,33 @@
+name: "Setup Embedded Postgres Cache Paths"
+description: Sets up a path for cached embedded postgres binaries.
+outputs:
+  embedded-pg-cache:
+    description: "Value of EMBEDDED_PG_CACHE_DIR"
+    value: ${{ steps.paths.outputs.embedded-pg-cache }}
+  cached-dirs:
+    description: "directories that should be cached between CI runs"
+    value: ${{ steps.paths.outputs.cached-dirs }}
+runs:
+  using: "composite"
+  steps:
+    - name: Override Go paths
+      id: paths
+      uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7
+      with:
+        script: |
+          const path = require('path');
+
+          // RUNNER_TEMP should be backed by a RAM disk on Windows if
+          // coder/setup-ramdisk-action was used
+          const runnerTemp = process.env.RUNNER_TEMP;
+          const embeddedPgCacheDir = path.join(runnerTemp, 'embedded-pg-cache');
+          core.exportVariable('EMBEDDED_PG_CACHE_DIR', embeddedPgCacheDir);
+          core.setOutput('embedded-pg-cache', embeddedPgCacheDir);
+          const cachedDirs = `${embeddedPgCacheDir}`;
+          core.setOutput('cached-dirs', cachedDirs);
+
+    - name: Create directories
+      shell: bash
+      run: |
+        set -e
+        mkdir -p "$EMBEDDED_PG_CACHE_DIR"

.github/actions/setup-go/action.yaml

@@ -4,7 +4,7 @@ description: |
 inputs:
   version:
     description: "The Go version to use."
-    default: "1.24.2"
+    default: "1.24.10"
   use-preinstalled-go:
     description: "Whether to use preinstalled Go."
     default: "false"

.github/actions/setup-node/action.yaml

@@ -16,7 +16,7 @@ runs:
     - name: Setup Node
       uses: actions/setup-node@0a44ba7841725637a19e28fa30b79a866c81b0a6 # v4.0.4
       with:
-        node-version: 20.16.0
+        node-version: 20.19.4
         # See https://github.com/actions/setup-node#caching-global-packages-data
         cache: "pnpm"
         cache-dependency-path: ${{ inputs.directory }}/pnpm-lock.yaml

.github/actions/setup-tf/action.yaml

@@ -7,5 +7,5 @@ runs:
     - name: Install Terraform
       uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd # v3.1.2
       with:
-        terraform_version: 1.11.4
+        terraform_version: 1.13.0
         terraform_wrapper: false

.github/actions/test-cache/download/action.yml

@@ -27,9 +27,11 @@ runs:
         export YEAR_MONTH=$(date +'%Y-%m')
         export PREV_YEAR_MONTH=$(date -d 'last month' +'%Y-%m')
         export DAY=$(date +'%d')
-        echo "year-month=$YEAR_MONTH" >> $GITHUB_OUTPUT
-        echo "prev-year-month=$PREV_YEAR_MONTH" >> $GITHUB_OUTPUT
-        echo "cache-key=${{ inputs.key-prefix }}-${YEAR_MONTH}-${DAY}" >> $GITHUB_OUTPUT
+        echo "year-month=$YEAR_MONTH" >> "$GITHUB_OUTPUT"
+        echo "prev-year-month=$PREV_YEAR_MONTH" >> "$GITHUB_OUTPUT"
+        echo "cache-key=${INPUTS_KEY_PREFIX}-${YEAR_MONTH}-${DAY}" >> "$GITHUB_OUTPUT"
+      env:
+        INPUTS_KEY_PREFIX: ${{ inputs.key-prefix }}
 
     # TODO: As a cost optimization, we could remove caches that are older than
     # a day or two. By default, depot keeps caches for 14 days, which isn't

.github/actions/upload-datadog/action.yaml

@@ -12,13 +12,12 @@ runs:
       run: |
         set -e
 
-        owner=${{ github.repository_owner	 }}
-        echo "owner: $owner"
-        if [[  $owner != "coder" ]]; then
+        echo "owner: $REPO_OWNER"
+        if [[ "$REPO_OWNER" != "coder" ]]; then
           echo "Not a pull request from the main repo, skipping..."
           exit 0
         fi
-        if [[ -z "${{ inputs.api-key }}" ]]; then
+        if [[ -z "${DATADOG_API_KEY}" ]]; then
           # This can happen for dependabot.
           echo "No API key provided, skipping..."
           exit 0
@@ -31,37 +30,38 @@ runs:
 
         TMP_DIR=$(mktemp -d)
 
-        if [[ "${{ runner.os }}" == "Windows" ]]; then
+        if [[ "${RUNNER_OS}" == "Windows" ]]; then
           BINARY_PATH="${TMP_DIR}/datadog-ci.exe"
           BINARY_URL="https://github.com/DataDog/datadog-ci/releases/download/${BINARY_VERSION}/datadog-ci_win-x64"
-        elif [[ "${{ runner.os }}" == "macOS" ]]; then
+        elif [[ "${RUNNER_OS}" == "macOS" ]]; then
           BINARY_PATH="${TMP_DIR}/datadog-ci"
           BINARY_URL="https://github.com/DataDog/datadog-ci/releases/download/${BINARY_VERSION}/datadog-ci_darwin-arm64"
-        elif [[ "${{ runner.os }}" == "Linux" ]]; then
+        elif [[ "${RUNNER_OS}" == "Linux" ]]; then
           BINARY_PATH="${TMP_DIR}/datadog-ci"
           BINARY_URL="https://github.com/DataDog/datadog-ci/releases/download/${BINARY_VERSION}/datadog-ci_linux-x64"
         else
-          echo "Unsupported OS: ${{ runner.os }}"
+          echo "Unsupported OS: $RUNNER_OS"
           exit 1
         fi
 
-        echo "Downloading DataDog CI binary version ${BINARY_VERSION} for ${{ runner.os }}..."
+        echo "Downloading DataDog CI binary version ${BINARY_VERSION} for $RUNNER_OS..."
         curl -sSL "$BINARY_URL" -o "$BINARY_PATH"
 
-        if [[ "${{ runner.os }}" == "Windows" ]]; then
+        if [[ "${RUNNER_OS}" == "Windows" ]]; then
           echo "$BINARY_HASH_WINDOWS  $BINARY_PATH" | sha256sum --check
-        elif [[ "${{ runner.os }}" == "macOS" ]]; then
+        elif [[ "${RUNNER_OS}" == "macOS" ]]; then
           echo "$BINARY_HASH_MACOS  $BINARY_PATH" | shasum -a 256 --check
-        elif [[ "${{ runner.os }}" == "Linux" ]]; then
+        elif [[ "${RUNNER_OS}" == "Linux" ]]; then
           echo "$BINARY_HASH_LINUX  $BINARY_PATH" | sha256sum --check
         fi
 
         # Make binary executable (not needed for Windows)
-        if [[ "${{ runner.os }}" != "Windows" ]]; then
+        if [[ "${RUNNER_OS}" != "Windows" ]]; then
           chmod +x "$BINARY_PATH"
         fi
 
         "$BINARY_PATH" junit upload --service coder ./gotests.xml \
-          --tags os:${{runner.os}} --tags runner_name:${{runner.name}}
+          --tags "os:${RUNNER_OS}" --tags "runner_name:${RUNNER_NAME}"
       env:
+        REPO_OWNER: ${{ github.repository_owner }}
         DATADOG_API_KEY: ${{ inputs.api-key }}

.github/dependabot.yaml

@@ -33,6 +33,7 @@ updates:
       - dependency-name: "*"
         update-types:
           - version-update:semver-patch
+      - dependency-name: "github.com/mark3labs/mcp-go"
 
   # Update our Dockerfile.
   - package-ecosystem: "docker"

.github/pull_request_template.md

@@ -0,0 +1 @@
+If you have used AI to produce some or all of this PR, please ensure you have read our [AI Contribution guidelines](https://coder.com/docs/about/contributing/AI_CONTRIBUTING) before submitting.

.github/workflows/contrib.yaml

@@ -3,6 +3,7 @@ name: contrib
 on:
   issue_comment:
     types: [created, edited]
+  # zizmor: ignore[dangerous-triggers] We explicitly want to run on pull_request_target.
   pull_request_target:
     types:
       - opened
@@ -42,7 +43,7 @@ jobs:
           # branch should not be protected
           branch: "main"
           # Some users have signed a corporate CLA with Coder so are exempt from signing our community one.
-          allowlist: "coryb,aaronlehmann,dependabot*"
+          allowlist: "coryb,aaronlehmann,dependabot*,blink-so*"
 
   release-labels:
     runs-on: ubuntu-latest

.github/workflows/dependabot.yaml

@@ -15,7 +15,7 @@ jobs:
       github.event_name == 'pull_request' &&
       github.event.action == 'opened' &&
       github.event.pull_request.user.login == 'dependabot[bot]' &&
-      github.actor_id == 49699333 &&
+      github.event.pull_request.user.id == 49699333 &&
       github.repository == 'coder/coder'
     permissions:
       pull-requests: write
@@ -44,10 +44,6 @@ jobs:
           GH_TOKEN: ${{secrets.GITHUB_TOKEN}}
 
       - name: Send Slack notification
-        env:
-          PR_URL: ${{github.event.pull_request.html_url}}
-          PR_TITLE: ${{github.event.pull_request.title}}
-          PR_NUMBER: ${{github.event.pull_request.number}}
         run: |
           curl -X POST -H 'Content-type: application/json' \
           --data '{
@@ -58,7 +54,7 @@ jobs:
                 "type": "header",
                 "text": {
                   "type": "plain_text",
-                  "text": ":pr-merged: Auto merge enabled for Dependabot PR #${{ env.PR_NUMBER }}",
+                  "text": ":pr-merged: Auto merge enabled for Dependabot PR #'"${PR_NUMBER}"'",
                   "emoji": true
                 }
               },
@@ -67,7 +63,7 @@ jobs:
                 "fields": [
                   {
                     "type": "mrkdwn",
-                    "text": "${{ env.PR_TITLE }}"
+                    "text": "'"${PR_TITLE}"'"
                   }
                 ]
               },
@@ -80,9 +76,14 @@ jobs:
                       "type": "plain_text",
                       "text": "View PR"
                     },
-                    "url": "${{ env.PR_URL }}"
+                    "url": "'"${PR_URL}"'"
                   }
                 ]
               }
             ]
-          }' ${{ secrets.DEPENDABOT_PRS_SLACK_WEBHOOK }}
+          }' "${{ secrets.DEPENDABOT_PRS_SLACK_WEBHOOK }}"
+        env:
+          SLACK_WEBHOOK: ${{ secrets.DEPENDABOT_PRS_SLACK_WEBHOOK }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+          PR_TITLE: ${{ github.event.pull_request.title }}
+          PR_URL: ${{ github.event.pull_request.html_url }}

.github/workflows/docker-base.yaml

@@ -38,15 +38,17 @@ jobs:
     if: github.repository_owner == 'coder'
     steps:
       - name: Harden Runner
-        uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
+        uses: step-security/harden-runner@ec9f2d5744a09debf3a187a3f4f675c53b671911 # v2.13.0
         with:
           egress-policy: audit
 
       - name: Checkout
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
 
       - name: Docker login
-        uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
+        uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v3.5.0
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
@@ -60,7 +62,7 @@ jobs:
 
       # This uses OIDC authentication, so no auth variables are required.
       - name: Build base Docker image via depot.dev
-        uses: depot/build-push-action@636daae76684e38c301daa0c5eca1c095b24e780 # v1.14.0
+        uses: depot/build-push-action@2583627a84956d07561420dcc1d0eb1f2af3fac0 # v1.15.0
         with:
           project: wl5hnrrkns
           context: base-build-context

.github/workflows/docs-ci.yaml

@@ -23,12 +23,14 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
 
       - name: Setup Node
         uses: ./.github/actions/setup-node
 
-      - uses: tj-actions/changed-files@3981e4f74104e7a4c67a835e1e5dd5d9eb0f0a57 # v45.0.7
+      - uses: tj-actions/changed-files@f963b3f3562b00b6d2dd25efc390eb04e51ef6c6 # v45.0.7
         id: changed-files
         with:
           files: |
@@ -39,10 +41,16 @@ jobs:
       - name: lint
         if: steps.changed-files.outputs.any_changed == 'true'
         run: |
-          pnpm exec markdownlint-cli2 ${{ steps.changed-files.outputs.all_changed_files }}
+          # shellcheck disable=SC2086
+          pnpm exec markdownlint-cli2 $ALL_CHANGED_FILES
+        env:
+          ALL_CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }}
 
       - name: fmt
         if: steps.changed-files.outputs.any_changed == 'true'
         run: |
           # markdown-table-formatter requires a space separated list of files
-          echo ${{ steps.changed-files.outputs.all_changed_files }} | tr ',' '\n' | pnpm exec markdown-table-formatter --check
+          # shellcheck disable=SC2086
+          echo $ALL_CHANGED_FILES | tr ',' '\n' | pnpm exec markdown-table-formatter --check
+        env:
+          ALL_CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }}

.github/workflows/dogfood.yaml

@@ -18,8 +18,7 @@ on:
   workflow_dispatch:
 
 permissions:
-  # Necessary for GCP authentication (https://github.com/google-github-actions/setup-gcloud#usage)
-  id-token: write
+  contents: read
 
 jobs:
   build_image:
@@ -27,15 +26,21 @@ jobs:
     runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-4' || 'ubuntu-latest' }}
     steps:
       - name: Harden Runner
-        uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
+        uses: step-security/harden-runner@ec9f2d5744a09debf3a187a3f4f675c53b671911 # v2.13.0
         with:
           egress-policy: audit
 
       - name: Checkout
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
 
       - name: Setup Nix
-        uses: nixbuild/nix-quick-install-action@5bb6a3b3abe66fd09bbf250dce8ada94f856a703 # v30
+        uses: nixbuild/nix-quick-install-action@63ca48f939ee3b8d835f4126562537df0fee5b91 # v32
+        with:
+          # Pinning to 2.28 here, as Nix gets a "error: [json.exception.type_error.302] type must be array, but is string"
+          # on version 2.29 and above.
+          nix_version: "2.28.4"
 
       - uses: nix-community/cache-nix-action@135667ec418502fa5a3598af6fb9eb733888ce6a # v6.1.3
         with:
@@ -58,31 +63,32 @@ jobs:
 
       - name: Get branch name
         id: branch-name
-        uses: tj-actions/branch-names@dde14ac574a8b9b1cedc59a1cf312788af43d8d8 # v8.2.1
+        uses: tj-actions/branch-names@5250492686b253f06fa55861556d1027b067aeb5 # v9.0.2
 
       - name: "Branch name to Docker tag name"
         id: docker-tag-name
         run: |
-          tag=${{ steps.branch-name.outputs.current_branch }}
           # Replace / with --, e.g. user/feature => user--feature.
-          tag=${tag//\//--}
-          echo "tag=${tag}" >> $GITHUB_OUTPUT
+          tag=${BRANCH_NAME//\//--}
+          echo "tag=${tag}" >> "$GITHUB_OUTPUT"
+        env:
+          BRANCH_NAME: ${{ steps.branch-name.outputs.current_branch }}
 
       - name: Set up Depot CLI
         uses: depot/setup-action@b0b1ea4f69e92ebf5dea3f8713a1b0c37b2126a5 # v1.6.0
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0
+        uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
 
       - name: Login to DockerHub
         if: github.ref == 'refs/heads/main'
-        uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
+        uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v3.5.0
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_PASSWORD }}
 
       - name: Build and push Non-Nix image
-        uses: depot/build-push-action@636daae76684e38c301daa0c5eca1c095b24e780 # v1.14.0
+        uses: depot/build-push-action@2583627a84956d07561420dcc1d0eb1f2af3fac0 # v1.15.0
         with:
           project: b4q6ltmpzh
           token: ${{ secrets.DEPOT_TOKEN }}
@@ -103,32 +109,39 @@ jobs:
 
           CURRENT_SYSTEM=$(nix eval --impure --raw --expr 'builtins.currentSystem')
 
-          docker image tag codercom/oss-dogfood-nix:latest-$CURRENT_SYSTEM codercom/oss-dogfood-nix:${{ steps.docker-tag-name.outputs.tag }}
-          docker image push codercom/oss-dogfood-nix:${{ steps.docker-tag-name.outputs.tag }}
+          docker image tag "codercom/oss-dogfood-nix:latest-$CURRENT_SYSTEM" "codercom/oss-dogfood-nix:${DOCKER_TAG}"
+          docker image push "codercom/oss-dogfood-nix:${DOCKER_TAG}"
 
-          docker image tag codercom/oss-dogfood-nix:latest-$CURRENT_SYSTEM codercom/oss-dogfood-nix:latest
-          docker image push codercom/oss-dogfood-nix:latest
+          docker image tag "codercom/oss-dogfood-nix:latest-$CURRENT_SYSTEM" "codercom/oss-dogfood-nix:latest"
+          docker image push "codercom/oss-dogfood-nix:latest"
+        env:
+          DOCKER_TAG: ${{ steps.docker-tag-name.outputs.tag }}
 
   deploy_template:
     needs: build_image
     runs-on: ubuntu-latest
+    permissions:
+      # Necessary for GCP authentication (https://github.com/google-github-actions/setup-gcloud#usage)
+      id-token: write
     steps:
       - name: Harden Runner
-        uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
+        uses: step-security/harden-runner@ec9f2d5744a09debf3a187a3f4f675c53b671911 # v2.13.0
         with:
           egress-policy: audit
 
       - name: Checkout
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
 
       - name: Setup Terraform
         uses: ./.github/actions/setup-tf
 
       - name: Authenticate to Google Cloud
-        uses: google-github-actions/auth@ba79af03959ebeac9769e648f473a284504d9193 # v2.1.10
+        uses: google-github-actions/auth@b7593ed2efd1c1617e1b0254da33b86225adb2a5 # v2.1.12
         with:
-          workload_identity_provider: projects/573722524737/locations/global/workloadIdentityPools/github/providers/github
-          service_account: coder-ci@coder-dogfood.iam.gserviceaccount.com
+          workload_identity_provider: ${{ vars.GCP_WORKLOAD_ID_PROVIDER }}
+          service_account: ${{ vars.GCP_SERVICE_ACCOUNT }}
 
       - name: Terraform init and validate
         run: |
@@ -148,12 +161,12 @@ jobs:
       - name: Get short commit SHA
         if: github.ref == 'refs/heads/main'
         id: vars
-        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+        run: echo "sha_short=$(git rev-parse --short HEAD)" >> "$GITHUB_OUTPUT"
 
       - name: Get latest commit title
         if: github.ref == 'refs/heads/main'
         id: message
-        run: echo "pr_title=$(git log --format=%s -n 1 ${{ github.sha }})" >> $GITHUB_OUTPUT
+        run: echo "pr_title=$(git log --format=%s -n 1 ${{ github.sha }})" >> "$GITHUB_OUTPUT"
 
       - name: "Push template"
         if: github.ref == 'refs/heads/main'
@@ -165,6 +178,7 @@ jobs:
           CODER_URL: https://dev.coder.com
           CODER_SESSION_TOKEN: ${{ secrets.CODER_SESSION_TOKEN }}
           # Template source & details
+          TF_VAR_CODER_DOGFOOD_ANTHROPIC_API_KEY: ${{ secrets.CODER_DOGFOOD_ANTHROPIC_API_KEY }}
           TF_VAR_CODER_TEMPLATE_NAME: ${{ secrets.CODER_TEMPLATE_NAME }}
           TF_VAR_CODER_TEMPLATE_VERSION: ${{ steps.vars.outputs.sha_short }}
           TF_VAR_CODER_TEMPLATE_DIR: ./coder

.github/workflows/nightly-gauntlet.yaml

@@ -0,0 +1,219 @@
+# The nightly-gauntlet runs tests that are either too flaky or too slow to block
+# every PR.
+name: nightly-gauntlet
+on:
+  schedule:
+    # Every day at 4AM
+    - cron: "0 4 * * 1-5"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  test-go-pg:
+    # make sure to adjust NUM_PARALLEL_PACKAGES and NUM_PARALLEL_TESTS below
+    # when changing runner sizes
+    runs-on: ${{ matrix.os == 'macos-latest' && github.repository_owner == 'coder' && 'depot-macos-latest' || matrix.os == 'windows-2022' && github.repository_owner == 'coder' && 'depot-windows-2022-16' || matrix.os }}
+    # This timeout must be greater than the timeout set by `go test` in
+    # `make test-postgres` to ensure we receive a trace of running
+    # goroutines. Setting this to the timeout +5m should work quite well
+    # even if some of the preceding steps are slow.
+    timeout-minutes: 25
+    strategy:
+      matrix:
+        os:
+          - macos-latest
+          - windows-2022
+    steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@ec9f2d5744a09debf3a187a3f4f675c53b671911 # v2.13.0
+        with:
+          egress-policy: audit
+
+      # macOS indexes all new files in the background. Our Postgres tests
+      # create and destroy thousands of databases on disk, and Spotlight
+      # tries to index all of them, seriously slowing down the tests.
+      - name: Disable Spotlight Indexing
+        if: runner.os == 'macOS'
+        run: |
+          enabled=$(sudo mdutil -a -s | { grep -Fc "Indexing enabled" || true; })
+          if [ "$enabled" -eq 0 ]; then
+            echo "Spotlight indexing is already disabled"
+            exit 0
+          fi
+          sudo mdutil -a -i off
+          sudo mdutil -X /
+          sudo launchctl bootout system /System/Library/LaunchDaemons/com.apple.metadata.mds.plist
+
+      # Set up RAM disks to speed up the rest of the job. This action is in
+      # a separate repository to allow its use before actions/checkout.
+      - name: Setup RAM Disks
+        if: runner.os == 'Windows'
+        uses: coder/setup-ramdisk-action@e1100847ab2d7bcd9d14bcda8f2d1b0f07b36f1b # v0.1.0
+
+      - name: Checkout
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          fetch-depth: 1
+          persist-credentials: false
+
+      - name: Setup Go
+        uses: ./.github/actions/setup-go
+        with:
+          # Runners have Go baked-in and Go will automatically
+          # download the toolchain configured in go.mod, so we don't
+          # need to reinstall it. It's faster on Windows runners.
+          use-preinstalled-go: ${{ runner.os == 'Windows' }}
+
+      - name: Setup Terraform
+        uses: ./.github/actions/setup-tf
+
+      - name: Setup Embedded Postgres Cache Paths
+        id: embedded-pg-cache
+        uses: ./.github/actions/setup-embedded-pg-cache-paths
+
+      - name: Download Embedded Postgres Cache
+        id: download-embedded-pg-cache
+        uses: ./.github/actions/embedded-pg-cache/download
+        with:
+          key-prefix: embedded-pg-${{ runner.os }}-${{ runner.arch }}
+          cache-path: ${{ steps.embedded-pg-cache.outputs.cached-dirs }}
+
+      - name: Test with PostgreSQL Database
+        env:
+          POSTGRES_VERSION: "13"
+          TS_DEBUG_DISCO: "true"
+          LC_CTYPE: "en_US.UTF-8"
+          LC_ALL: "en_US.UTF-8"
+        shell: bash
+        run: |
+          set -o errexit
+          set -o pipefail
+
+          if [ "${{ runner.os }}" == "Windows" ]; then
+            # Create a temp dir on the R: ramdisk drive for Windows. The default
+            # C: drive is extremely slow: https://github.com/actions/runner-images/issues/8755
+            mkdir -p "R:/temp/embedded-pg"
+            go run scripts/embedded-pg/main.go -path "R:/temp/embedded-pg" -cache "${EMBEDDED_PG_CACHE_DIR}"
+          elif [ "${{ runner.os }}" == "macOS" ]; then
+            # Postgres runs faster on a ramdisk on macOS too
+            mkdir -p /tmp/tmpfs
+            sudo mount_tmpfs -o noowners -s 8g /tmp/tmpfs
+            go run scripts/embedded-pg/main.go -path /tmp/tmpfs/embedded-pg -cache "${EMBEDDED_PG_CACHE_DIR}"
+          elif [ "${{ runner.os }}" == "Linux" ]; then
+            make test-postgres-docker
+          fi
+
+          # if macOS, install google-chrome for scaletests
+          # As another concern, should we really have this kind of external dependency
+          # requirement on standard CI?
+          if [ "${{ matrix.os }}" == "macos-latest" ]; then
+            brew install google-chrome
+          fi
+
+          # macOS will output "The default interactive shell is now zsh"
+          # intermittently in CI...
+          if [ "${{ matrix.os }}" == "macos-latest" ]; then
+            touch ~/.bash_profile && echo "export BASH_SILENCE_DEPRECATION_WARNING=1" >> ~/.bash_profile
+          fi
+
+          if [ "${{ runner.os }}" == "Windows" ]; then
+            # Our Windows runners have 16 cores.
+            # On Windows Postgres chokes up when we have 16x16=256 tests
+            # running in parallel, and dbtestutil.NewDB starts to take more than
+            # 10s to complete sometimes causing test timeouts. With 16x8=128 tests
+            # Postgres tends not to choke.
+            NUM_PARALLEL_PACKAGES=8
+            NUM_PARALLEL_TESTS=16
+          elif [ "${{ runner.os }}" == "macOS" ]; then
+            # Our macOS runners have 8 cores. We set NUM_PARALLEL_TESTS to 16
+            # because the tests complete faster and Postgres doesn't choke. It seems
+            # that macOS's tmpfs is faster than the one on Windows.
+            NUM_PARALLEL_PACKAGES=8
+            NUM_PARALLEL_TESTS=16
+          elif [ "${{ runner.os }}" == "Linux" ]; then
+            # Our Linux runners have 8 cores.
+            NUM_PARALLEL_PACKAGES=8
+            NUM_PARALLEL_TESTS=8
+          fi
+
+          # run tests without cache
+          TESTCOUNT="-count=1"
+
+          DB=ci gotestsum \
+            --format standard-quiet --packages "./..." \
+            -- -timeout=20m -v -p $NUM_PARALLEL_PACKAGES -parallel=$NUM_PARALLEL_TESTS $TESTCOUNT
+
+      - name: Upload Embedded Postgres Cache
+        uses: ./.github/actions/embedded-pg-cache/upload
+        # We only use the embedded Postgres cache on macOS and Windows runners.
+        if: runner.OS == 'macOS' || runner.OS == 'Windows'
+        with:
+          cache-key: ${{ steps.download-embedded-pg-cache.outputs.cache-key }}
+          cache-path: "${{ steps.embedded-pg-cache.outputs.embedded-pg-cache }}"
+
+      - name: Upload test stats to Datadog
+        timeout-minutes: 1
+        continue-on-error: true
+        uses: ./.github/actions/upload-datadog
+        if: success() || failure()
+        with:
+          api-key: ${{ secrets.DATADOG_API_KEY }}
+
+  notify-slack-on-failure:
+    needs:
+      - test-go-pg
+    runs-on: ubuntu-latest
+    if: failure() && github.ref == 'refs/heads/main'
+
+    steps:
+      - name: Send Slack notification
+        run: |
+          curl -X POST -H 'Content-type: application/json' \
+          --data '{
+            "blocks": [
+              {
+                "type": "header",
+                "text": {
+                  "type": "plain_text",
+                  "text": "❌ Nightly gauntlet failed",
+                  "emoji": true
+                }
+              },
+              {
+                "type": "section",
+                "fields": [
+                  {
+                    "type": "mrkdwn",
+                    "text": "*Workflow:*\n'"${GITHUB_WORKFLOW}"'"
+                  },
+                  {
+                    "type": "mrkdwn",
+                    "text": "*Committer:*\n'"${GITHUB_ACTOR}"'"
+                  },
+                  {
+                    "type": "mrkdwn",
+                    "text": "*Commit:*\n'"${GITHUB_SHA}"'"
+                  }
+                ]
+              },
+              {
+                "type": "section",
+                "text": {
+                  "type": "mrkdwn",
+                  "text": "*View failure:* <'"${RUN_URL}"'|Click here>"
+                }
+              },
+              {
+                "type": "section",
+                "text": {
+                  "type": "mrkdwn",
+                  "text": "<@U08TJ4YNCA3> investigate this CI failure. Check logs, search for existing issues, use git blame to find who last modified failing tests, create issue in coder/internal (not public repo), use title format \"flake: TestName\" for flaky tests, and assign to the person from git blame."
+                }
+              }
+            ]
+          }' "${SLACK_WEBHOOK}"
+        env:
+          SLACK_WEBHOOK: ${{ secrets.CI_FAILURE_SLACK_WEBHOOK }}
+          RUN_URL: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"

.github/workflows/pr-auto-assign.yaml

@@ -3,6 +3,7 @@
 name: PR Auto Assign
 
 on:
+  # zizmor: ignore[dangerous-triggers] We explicitly want to run on pull_request_target.
   pull_request_target:
     types: [opened]
 
@@ -14,7 +15,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Harden Runner
-        uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
+        uses: step-security/harden-runner@ec9f2d5744a09debf3a187a3f4f675c53b671911 # v2.13.0
         with:
           egress-policy: audit
 

.github/workflows/pr-cleanup.yaml

@@ -19,7 +19,7 @@ jobs:
       packages: write
     steps:
       - name: Harden Runner
-        uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
+        uses: step-security/harden-runner@ec9f2d5744a09debf3a187a3f4f675c53b671911 # v2.13.0
         with:
           egress-policy: audit
 
@@ -27,10 +27,12 @@ jobs:
         id: pr_number
         run: |
           if [ -n "${{ github.event.pull_request.number }}" ]; then
-            echo "PR_NUMBER=${{ github.event.pull_request.number }}" >> $GITHUB_OUTPUT
+            echo "PR_NUMBER=${{ github.event.pull_request.number }}" >> "$GITHUB_OUTPUT"
           else
-            echo "PR_NUMBER=${{ github.event.inputs.pr_number }}" >> $GITHUB_OUTPUT
+            echo "PR_NUMBER=${PR_NUMBER}" >> "$GITHUB_OUTPUT"
           fi
+        env:
+          PR_NUMBER: ${{ github.event.inputs.pr_number }}
 
       - name: Delete image
         continue-on-error: true
@@ -51,17 +53,21 @@ jobs:
       - name: Delete helm release
         run: |
           set -euo pipefail
-          helm delete --namespace "pr${{ steps.pr_number.outputs.PR_NUMBER }}" "pr${{ steps.pr_number.outputs.PR_NUMBER }}" || echo "helm release not found"
+          helm delete --namespace "pr${PR_NUMBER}" "pr${PR_NUMBER}" || echo "helm release not found"
+        env:
+          PR_NUMBER: ${{ steps.pr_number.outputs.PR_NUMBER }}
 
       - name: "Remove PR namespace"
         run: |
-          kubectl delete namespace "pr${{ steps.pr_number.outputs.PR_NUMBER }}" || echo "namespace not found"
+          kubectl delete namespace "pr${PR_NUMBER}" || echo "namespace not found"
+        env:
+          PR_NUMBER: ${{ steps.pr_number.outputs.PR_NUMBER }}
 
       - name: "Remove DNS records"
         run: |
           set -euo pipefail
           # Get identifier for the record
-          record_id=$(curl -X GET "https://api.cloudflare.com/client/v4/zones/${{ secrets.PR_DEPLOYMENTS_ZONE_ID }}/dns_records?name=%2A.pr${{ steps.pr_number.outputs.PR_NUMBER }}.${{ secrets.PR_DEPLOYMENTS_DOMAIN }}" \
+          record_id=$(curl -X GET "https://api.cloudflare.com/client/v4/zones/${{ secrets.PR_DEPLOYMENTS_ZONE_ID }}/dns_records?name=%2A.pr${PR_NUMBER}.${{ secrets.PR_DEPLOYMENTS_DOMAIN }}" \
           -H "Authorization: Bearer ${{ secrets.PR_DEPLOYMENTS_CLOUDFLARE_API_TOKEN }}" \
           -H "Content-Type:application/json" | jq -r '.result[0].id') || echo "DNS record not found"
 
@@ -73,9 +79,13 @@ jobs:
             -H "Authorization: Bearer ${{ secrets.PR_DEPLOYMENTS_CLOUDFLARE_API_TOKEN }}" \
             -H "Content-Type:application/json" | jq -r '.success'
           ) || echo "DNS record not found"
+        env:
+          PR_NUMBER: ${{ steps.pr_number.outputs.PR_NUMBER }}
 
       - name: "Delete certificate"
         if: ${{ github.event.pull_request.merged == true }}
         run: |
           set -euxo pipefail
-          kubectl delete certificate "pr${{ steps.pr_number.outputs.PR_NUMBER }}-tls" -n pr-deployment-certs || echo "certificate not found"
+          kubectl delete certificate "pr${PR_NUMBER}-tls" -n pr-deployment-certs || echo "certificate not found"
+        env:
+          PR_NUMBER: ${{ steps.pr_number.outputs.PR_NUMBER }}

.github/workflows/pr-deploy.yaml

@@ -39,12 +39,14 @@ jobs:
       PR_OPEN: ${{ steps.check_pr.outputs.pr_open }}
     steps:
       - name: Harden Runner
-        uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
+        uses: step-security/harden-runner@ec9f2d5744a09debf3a187a3f4f675c53b671911 # v2.13.0
         with:
           egress-policy: audit
 
       - name: Checkout
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
 
       - name: Check if PR is open
         id: check_pr
@@ -55,7 +57,7 @@ jobs:
             echo "PR doesn't exist or is closed."
             pr_open=false
           fi
-          echo "pr_open=$pr_open" >> $GITHUB_OUTPUT
+          echo "pr_open=$pr_open" >> "$GITHUB_OUTPUT"
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
@@ -74,14 +76,15 @@ jobs:
     runs-on: "ubuntu-latest"
     steps:
       - name: Harden Runner
-        uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
+        uses: step-security/harden-runner@ec9f2d5744a09debf3a187a3f4f675c53b671911 # v2.13.0
         with:
           egress-policy: audit
 
       - name: Checkout
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
         with:
           fetch-depth: 0
+          persist-credentials: false
 
       - name: Get PR number, title, and branch name
         id: pr_info
@@ -90,9 +93,11 @@ jobs:
           PR_NUMBER=$(gh pr view --json number | jq -r '.number')
           PR_TITLE=$(gh pr view --json title | jq -r '.title')
           PR_URL=$(gh pr view --json url | jq -r '.url')
-          echo "PR_URL=$PR_URL" >> $GITHUB_OUTPUT
-          echo "PR_NUMBER=$PR_NUMBER" >> $GITHUB_OUTPUT
-          echo "PR_TITLE=$PR_TITLE" >> $GITHUB_OUTPUT
+          {
+            echo "PR_URL=$PR_URL"
+            echo "PR_NUMBER=$PR_NUMBER"
+            echo "PR_TITLE=$PR_TITLE"
+          } >> "$GITHUB_OUTPUT"
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
@@ -100,8 +105,8 @@ jobs:
         id: set_tags
         run: |
           set -euo pipefail
-          echo "CODER_BASE_IMAGE_TAG=$CODER_BASE_IMAGE_TAG" >> $GITHUB_OUTPUT
-          echo "CODER_IMAGE_TAG=$CODER_IMAGE_TAG" >> $GITHUB_OUTPUT
+          echo "CODER_BASE_IMAGE_TAG=$CODER_BASE_IMAGE_TAG" >> "$GITHUB_OUTPUT"
+          echo "CODER_IMAGE_TAG=$CODER_IMAGE_TAG" >> "$GITHUB_OUTPUT"
         env:
           CODER_BASE_IMAGE_TAG: ghcr.io/coder/coder-preview-base:pr${{ steps.pr_info.outputs.PR_NUMBER }}
           CODER_IMAGE_TAG: ghcr.io/coder/coder-preview:pr${{ steps.pr_info.outputs.PR_NUMBER }}
@@ -118,14 +123,16 @@ jobs:
         id: check_deployment
         run: |
           set -euo pipefail
-          if helm status "pr${{ steps.pr_info.outputs.PR_NUMBER }}" --namespace "pr${{ steps.pr_info.outputs.PR_NUMBER }}" > /dev/null 2>&1; then
+          if helm status "pr${PR_NUMBER}" --namespace "pr${PR_NUMBER}" > /dev/null 2>&1; then
             echo "Deployment already exists. Skipping deployment."
             NEW=false
           else
             echo "Deployment doesn't exist."
             NEW=true
           fi
-          echo "NEW=$NEW" >> $GITHUB_OUTPUT
+          echo "NEW=$NEW" >> "$GITHUB_OUTPUT"
+        env:
+          PR_NUMBER: ${{ steps.pr_info.outputs.PR_NUMBER }}
 
       - name: Check changed files
         uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2
@@ -154,17 +161,20 @@ jobs:
       - name: Print number of changed files
         run: |
           set -euo pipefail
-          echo "Total number of changed files: ${{ steps.filter.outputs.all_count }}"
-          echo "Number of ignored files: ${{ steps.filter.outputs.ignored_count }}"
+          echo "Total number of changed files: ${ALL_COUNT}"
+          echo "Number of ignored files: ${IGNORED_COUNT}"
+        env:
+          ALL_COUNT: ${{ steps.filter.outputs.all_count }}
+          IGNORED_COUNT: ${{ steps.filter.outputs.ignored_count }}
 
       - name: Build conditionals
         id: build_conditionals
         run: |
           set -euo pipefail
           # build if the workflow is manually triggered and the deployment doesn't exist (first build or force rebuild)
-          echo "first_or_force_build=${{ (github.event_name == 'workflow_dispatch' && steps.check_deployment.outputs.NEW == 'true') || github.event.inputs.build == 'true' }}" >> $GITHUB_OUTPUT
+          echo "first_or_force_build=${{ (github.event_name == 'workflow_dispatch' && steps.check_deployment.outputs.NEW == 'true') || github.event.inputs.build == 'true' }}" >> "$GITHUB_OUTPUT"
           # build if the deployment already exist and there are changes in the files that we care about (automatic updates)
-          echo "automatic_rebuild=${{ steps.check_deployment.outputs.NEW == 'false' && steps.filter.outputs.all_count > steps.filter.outputs.ignored_count }}" >> $GITHUB_OUTPUT
+          echo "automatic_rebuild=${{ steps.check_deployment.outputs.NEW == 'false' && steps.filter.outputs.all_count > steps.filter.outputs.ignored_count }}" >> "$GITHUB_OUTPUT"
 
   comment-pr:
     needs: get_info
@@ -174,7 +184,7 @@ jobs:
       pull-requests: write # needed for commenting on PRs
     steps:
       - name: Harden Runner
-        uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
+        uses: step-security/harden-runner@ec9f2d5744a09debf3a187a3f4f675c53b671911 # v2.13.0
         with:
           egress-policy: audit
 
@@ -218,14 +228,15 @@ jobs:
       CODER_IMAGE_TAG: ${{ needs.get_info.outputs.CODER_IMAGE_TAG }}
     steps:
       - name: Harden Runner
-        uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
+        uses: step-security/harden-runner@ec9f2d5744a09debf3a187a3f4f675c53b671911 # v2.13.0
         with:
           egress-policy: audit
 
       - name: Checkout
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
         with:
           fetch-depth: 0
+          persist-credentials: false
 
       - name: Setup Node
         uses: ./.github/actions/setup-node
@@ -237,7 +248,7 @@ jobs:
         uses: ./.github/actions/setup-sqlc
 
       - name: GHCR Login
-        uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
+        uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v3.5.0
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
@@ -250,12 +261,13 @@ jobs:
           make gen/mark-fresh
           export DOCKER_IMAGE_NO_PREREQUISITES=true
           version="$(./scripts/version.sh)"
-          export CODER_IMAGE_BUILD_BASE_TAG="$(CODER_IMAGE_BASE=coder-base ./scripts/image_tag.sh --version "$version")"
+          CODER_IMAGE_BUILD_BASE_TAG="$(CODER_IMAGE_BASE=coder-base ./scripts/image_tag.sh --version "$version")"
+          export CODER_IMAGE_BUILD_BASE_TAG
           make -j build/coder_linux_amd64
           ./scripts/build_docker.sh \
             --arch amd64 \
-            --target ${{ env.CODER_IMAGE_TAG }} \
-            --version $version \
+            --target "${CODER_IMAGE_TAG}" \
+            --version "$version" \
             --push \
             build/coder_linux_amd64
 
@@ -276,7 +288,7 @@ jobs:
       PR_HOSTNAME: "pr${{ needs.get_info.outputs.PR_NUMBER }}.${{ secrets.PR_DEPLOYMENTS_DOMAIN }}"
     steps:
       - name: Harden Runner
-        uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
+        uses: step-security/harden-runner@ec9f2d5744a09debf3a187a3f4f675c53b671911 # v2.13.0
         with:
           egress-policy: audit
 
@@ -293,13 +305,13 @@ jobs:
           set -euo pipefail
           foundTag=$(
             gh api /orgs/coder/packages/container/coder-preview/versions |
-            jq -r --arg tag "pr${{ env.PR_NUMBER }}" '.[] |
+            jq -r --arg tag "pr${PR_NUMBER}" '.[] |
             select(.metadata.container.tags == [$tag]) |
             .metadata.container.tags[0]'
           )
           if [ -z "$foundTag" ]; then
             echo "Image not found"
-            echo "${{ env.CODER_IMAGE_TAG }} not found in ghcr.io/coder/coder-preview"
+            echo "${CODER_IMAGE_TAG} not found in ghcr.io/coder/coder-preview"
             exit 1
           else
             echo "Image found"
@@ -314,40 +326,42 @@ jobs:
           curl -X POST "https://api.cloudflare.com/client/v4/zones/${{ secrets.PR_DEPLOYMENTS_ZONE_ID }}/dns_records" \
             -H "Authorization: Bearer ${{ secrets.PR_DEPLOYMENTS_CLOUDFLARE_API_TOKEN }}" \
             -H "Content-Type:application/json" \
-            --data '{"type":"CNAME","name":"*.${{ env.PR_HOSTNAME }}","content":"${{ env.PR_HOSTNAME }}","ttl":1,"proxied":false}'
+            --data '{"type":"CNAME","name":"*.'"${PR_HOSTNAME}"'","content":"'"${PR_HOSTNAME}"'","ttl":1,"proxied":false}'
 
       - name: Create PR namespace
         if: needs.get_info.outputs.NEW == 'true' || github.event.inputs.deploy == 'true'
         run: |
           set -euo pipefail
           # try to delete the namespace, but don't fail if it doesn't exist
-          kubectl delete namespace "pr${{ env.PR_NUMBER }}" || true
-          kubectl create namespace "pr${{ env.PR_NUMBER }}"
+          kubectl delete namespace "pr${PR_NUMBER}" || true
+          kubectl create namespace "pr${PR_NUMBER}"
 
       - name: Checkout
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
 
       - name: Check and Create Certificate
         if: needs.get_info.outputs.NEW == 'true' || github.event.inputs.deploy == 'true'
         run: |
           # Using kubectl to check if a Certificate resource already exists
           # we are doing this to avoid letsenrypt rate limits
-          if ! kubectl get certificate pr${{ env.PR_NUMBER }}-tls -n pr-deployment-certs > /dev/null 2>&1; then
+          if ! kubectl get certificate "pr${PR_NUMBER}-tls" -n pr-deployment-certs > /dev/null 2>&1; then
             echo "Certificate doesn't exist. Creating a new one."
             envsubst < ./.github/pr-deployments/certificate.yaml | kubectl apply -f -
           else
             echo "Certificate exists. Skipping certificate creation."
           fi
-          echo "Copy certificate from pr-deployment-certs to pr${{ env.PR_NUMBER }} namespace"
-          until kubectl get secret pr${{ env.PR_NUMBER }}-tls -n pr-deployment-certs &> /dev/null
+          echo "Copy certificate from pr-deployment-certs to pr${PR_NUMBER} namespace"
+          until kubectl get secret "pr${PR_NUMBER}-tls" -n pr-deployment-certs &> /dev/null
           do
-            echo "Waiting for secret pr${{ env.PR_NUMBER }}-tls to be created..."
+            echo "Waiting for secret pr${PR_NUMBER}-tls to be created..."
             sleep 5
           done
           (
-            kubectl get secret pr${{ env.PR_NUMBER }}-tls -n pr-deployment-certs -o json |
+            kubectl get secret "pr${PR_NUMBER}-tls" -n pr-deployment-certs -o json |
             jq 'del(.metadata.namespace,.metadata.creationTimestamp,.metadata.resourceVersion,.metadata.selfLink,.metadata.uid,.metadata.managedFields)' |
-            kubectl -n pr${{ env.PR_NUMBER }} apply -f -
+            kubectl -n "pr${PR_NUMBER}" apply -f -
           )
 
       - name: Set up PostgreSQL database
@@ -355,13 +369,13 @@ jobs:
         run: |
           helm repo add bitnami https://charts.bitnami.com/bitnami
           helm install coder-db bitnami/postgresql \
-            --namespace pr${{ env.PR_NUMBER }} \
+            --namespace "pr${PR_NUMBER}" \
             --set auth.username=coder \
             --set auth.password=coder \
             --set auth.database=coder \
             --set persistence.size=10Gi
-          kubectl create secret generic coder-db-url -n pr${{ env.PR_NUMBER }} \
-            --from-literal=url="postgres://coder:coder@coder-db-postgresql.pr${{ env.PR_NUMBER }}.svc.cluster.local:5432/coder?sslmode=disable"
+          kubectl create secret generic coder-db-url -n "pr${PR_NUMBER}" \
+            --from-literal=url="postgres://coder:coder@coder-db-postgresql.pr${PR_NUMBER}.svc.cluster.local:5432/coder?sslmode=disable"
 
       - name: Create a service account, role, and rolebinding for the PR namespace
         if: needs.get_info.outputs.NEW == 'true' || github.event.inputs.deploy == 'true'
@@ -383,8 +397,8 @@ jobs:
         run: |
           set -euo pipefail
           helm dependency update --skip-refresh ./helm/coder
-          helm upgrade --install "pr${{ env.PR_NUMBER }}" ./helm/coder \
-          --namespace "pr${{ env.PR_NUMBER }}" \
+          helm upgrade --install "pr${PR_NUMBER}" ./helm/coder \
+          --namespace "pr${PR_NUMBER}" \
           --values ./pr-deploy-values.yaml \
           --force
 
@@ -393,8 +407,8 @@ jobs:
         run: |
           helm repo add coder-logstream-kube https://helm.coder.com/logstream-kube
           helm upgrade --install coder-logstream-kube coder-logstream-kube/coder-logstream-kube \
-            --namespace "pr${{ env.PR_NUMBER }}" \
-            --set url="https://${{ env.PR_HOSTNAME }}"
+            --namespace "pr${PR_NUMBER}" \
+            --set url="https://${PR_HOSTNAME}"
 
       - name: Get Coder binary
         if: needs.get_info.outputs.NEW == 'true' || github.event.inputs.deploy == 'true'
@@ -402,16 +416,16 @@ jobs:
           set -euo pipefail
 
           DEST="${HOME}/coder"
-          URL="https://${{ env.PR_HOSTNAME }}/bin/coder-linux-amd64"
+          URL="https://${PR_HOSTNAME}/bin/coder-linux-amd64"
 
-          mkdir -p "$(dirname ${DEST})"
+          mkdir -p "$(dirname "$DEST")"
 
           COUNT=0
-          until $(curl --output /dev/null --silent --head --fail "$URL"); do
+          until curl --output /dev/null --silent --head --fail "$URL"; do
               printf '.'
               sleep 5
               COUNT=$((COUNT+1))
-              if [ $COUNT -ge 60 ]; then
+              if [ "$COUNT" -ge 60 ]; then
                 echo "Timed out waiting for URL to be available"
                 exit 1
               fi
@@ -420,7 +434,7 @@ jobs:
           curl -fsSL "$URL" -o "${DEST}"
           chmod +x "${DEST}"
           "${DEST}" version
-          mv "${DEST}" /usr/local/bin/coder
+          sudo mv "${DEST}" /usr/local/bin/coder
 
       - name: Create first user
         if: needs.get_info.outputs.NEW == 'true' || github.event.inputs.deploy == 'true'
@@ -435,24 +449,24 @@ jobs:
 
           # add mask so that the password is not printed to the logs
           echo "::add-mask::$password"
-          echo "password=$password" >> $GITHUB_OUTPUT
+          echo "password=$password" >> "$GITHUB_OUTPUT"
 
           coder login \
-            --first-user-username pr${{ env.PR_NUMBER }}-admin \
-            --first-user-email pr${{ env.PR_NUMBER }}@coder.com \
-            --first-user-password $password \
+            --first-user-username "pr${PR_NUMBER}-admin" \
+            --first-user-email "pr${PR_NUMBER}@coder.com" \
+            --first-user-password "$password" \
             --first-user-trial=false \
             --use-token-as-session \
-            https://${{ env.PR_HOSTNAME }}
+            "https://${PR_HOSTNAME}"
 
           # Create a user for the github.actor
           # TODO: update once https://github.com/coder/coder/issues/15466 is resolved
           # coder users create \
-          #   --username ${{ github.actor }} \
+          #   --username ${GITHUB_ACTOR} \
           #   --login-type github
 
           # promote the user to admin role
-          # coder org members edit-role ${{ github.actor }} organization-admin
+          # coder org members edit-role ${GITHUB_ACTOR} organization-admin
           # TODO: update once https://github.com/coder/internal/issues/207 is resolved
 
       - name: Send Slack notification
@@ -461,17 +475,19 @@ jobs:
           curl -s -o /dev/null -X POST -H 'Content-type: application/json' \
             -d \
             '{
-              "pr_number": "'"${{ env.PR_NUMBER }}"'",
-              "pr_url": "'"${{ env.PR_URL }}"'",
-              "pr_title": "'"${{ env.PR_TITLE }}"'",
-              "pr_access_url": "'"https://${{ env.PR_HOSTNAME }}"'",
-              "pr_username": "'"pr${{ env.PR_NUMBER }}-admin"'",
-              "pr_email": "'"pr${{ env.PR_NUMBER }}@coder.com"'",
-              "pr_password": "'"${{ steps.setup_deployment.outputs.password }}"'",
-              "pr_actor": "'"${{ github.actor }}"'"
+              "pr_number": "'"${PR_NUMBER}"'",
+              "pr_url": "'"${PR_URL}"'",
+              "pr_title": "'"${PR_TITLE}"'",
+              "pr_access_url": "'"https://${PR_HOSTNAME}"'",
+              "pr_username": "'"pr${PR_NUMBER}-admin"'",
+              "pr_email": "'"pr${PR_NUMBER}@coder.com"'",
+              "pr_password": "'"${PASSWORD}"'",
+              "pr_actor": "'"${GITHUB_ACTOR}"'"
             }' \
             ${{ secrets.PR_DEPLOYMENTS_SLACK_WEBHOOK }}
           echo "Slack notification sent"
+        env:
+          PASSWORD: ${{ steps.setup_deployment.outputs.password }}
 
       - name: Find Comment
         uses: peter-evans/find-comment@3eae4d37986fb5a8592848f6a574fdf654e61f9e # v3.1.0
@@ -504,7 +520,7 @@ jobs:
         run: |
           set -euo pipefail
           cd .github/pr-deployments/template
-          coder templates push -y --variable namespace=pr${{ env.PR_NUMBER }} kubernetes
+          coder templates push -y --variable "namespace=pr${PR_NUMBER}" kubernetes
 
           # Create workspace
           coder create --template="kubernetes" kube --parameter cpu=2 --parameter memory=4 --parameter home_disk_size=2 -y

.github/workflows/release-validation.yaml

@@ -14,7 +14,7 @@ jobs:
 
     steps:
       - name: Harden Runner
-        uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
+        uses: step-security/harden-runner@ec9f2d5744a09debf3a187a3f4f675c53b671911 # v2.13.0
         with:
           egress-policy: audit
 

.github/workflows/release.yaml

@@ -32,15 +32,43 @@ env:
   CODER_RELEASE_NOTES: ${{ inputs.release_notes }}
 
 jobs:
+  # Only allow maintainers/admins to release.
+  check-perms:
+    runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-8' || 'ubuntu-latest' }}
+    steps:
+      - name: Allow only maintainers/admins
+        uses: actions/github-script@v7.0.1
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const {data} = await github.rest.repos.getCollaboratorPermissionLevel({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              username: context.actor
+            });
+            const role = data.role_name || data.user?.role_name || data.permission;
+            const perms = data.user?.permissions || {};
+            core.info(`Actor ${context.actor} permission=${data.permission}, role_name=${role}`);
+
+            const allowed =
+              role === 'admin' ||
+              role === 'maintain' ||
+              perms.admin === true ||
+              perms.maintain === true;
+
+            if (!allowed) core.setFailed('Denied: requires maintain or admin');
+
   # build-dylib is a separate job to build the dylib on macOS.
   build-dylib:
     runs-on: ${{ github.repository_owner == 'coder' && 'depot-macos-latest' || 'macos-latest' }}
+    needs: check-perms
     steps:
       # Harden Runner doesn't work on macOS.
       - name: Checkout
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
         with:
           fetch-depth: 0
+          persist-credentials: false
 
       # If the event that triggered the build was an annotated tag (which our
       # tags are supposed to be), actions/checkout has a bug where the tag in
@@ -53,14 +81,16 @@ jobs:
       - name: Setup build tools
         run: |
           brew install bash gnu-getopt make
-          echo "$(brew --prefix bash)/bin" >> $GITHUB_PATH
-          echo "$(brew --prefix gnu-getopt)/bin" >> $GITHUB_PATH
-          echo "$(brew --prefix make)/libexec/gnubin" >> $GITHUB_PATH
+          {
+            echo "$(brew --prefix bash)/bin"
+            echo "$(brew --prefix gnu-getopt)/bin"
+            echo "$(brew --prefix make)/libexec/gnubin"
+          } >> "$GITHUB_PATH"
 
       - name: Switch XCode Version
         uses: maxim-lobanov/setup-xcode@60606e260d2fc5762a71e64e74b2174e8ea3c8bd # v1.6.0
         with:
-          xcode-version: "16.0.0"
+          xcode-version: "16.1.0"
 
       - name: Setup Go
         uses: ./.github/actions/setup-go
@@ -114,7 +144,7 @@ jobs:
 
   release:
     name: Build and publish
-    needs: build-dylib
+    needs: [build-dylib, check-perms]
     runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-8' || 'ubuntu-latest' }}
     permissions:
       # Required to publish a release
@@ -134,14 +164,15 @@ jobs:
       version: ${{ steps.version.outputs.version }}
     steps:
       - name: Harden Runner
-        uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
+        uses: step-security/harden-runner@ec9f2d5744a09debf3a187a3f4f675c53b671911 # v2.13.0
         with:
           egress-policy: audit
 
       - name: Checkout
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
         with:
           fetch-depth: 0
+          persist-credentials: false
 
       # If the event that triggered the build was an annotated tag (which our
       # tags are supposed to be), actions/checkout has a bug where the tag in
@@ -156,9 +187,9 @@ jobs:
         run: |
           set -euo pipefail
           version="$(./scripts/version.sh)"
-          echo "version=$version" >> $GITHUB_OUTPUT
+          echo "version=$version" >> "$GITHUB_OUTPUT"
           # Speed up future version.sh calls.
-          echo "CODER_FORCE_VERSION=$version" >> $GITHUB_ENV
+          echo "CODER_FORCE_VERSION=$version" >> "$GITHUB_ENV"
           echo "$version"
 
       # Verify that all expectations for a release are met.
@@ -200,7 +231,7 @@ jobs:
 
           release_notes_file="$(mktemp -t release_notes.XXXXXX)"
           echo "$CODER_RELEASE_NOTES" > "$release_notes_file"
-          echo CODER_RELEASE_NOTES_FILE="$release_notes_file" >> $GITHUB_ENV
+          echo CODER_RELEASE_NOTES_FILE="$release_notes_file" >> "$GITHUB_ENV"
 
       - name: Show release notes
         run: |
@@ -208,7 +239,7 @@ jobs:
           cat "$CODER_RELEASE_NOTES_FILE"
 
       - name: Docker Login
-        uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
+        uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v3.5.0
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
@@ -286,17 +317,17 @@ jobs:
       # Setup GCloud for signing Windows binaries.
       - name: Authenticate to Google Cloud
         id: gcloud_auth
-        uses: google-github-actions/auth@ba79af03959ebeac9769e648f473a284504d9193 # v2.1.10
+        uses: google-github-actions/auth@b7593ed2efd1c1617e1b0254da33b86225adb2a5 # v2.1.12
         with:
-          workload_identity_provider: ${{ secrets.GCP_CODE_SIGNING_WORKLOAD_ID_PROVIDER }}
-          service_account: ${{ secrets.GCP_CODE_SIGNING_SERVICE_ACCOUNT }}
+          workload_identity_provider: ${{ vars.GCP_CODE_SIGNING_WORKLOAD_ID_PROVIDER }}
+          service_account: ${{ vars.GCP_CODE_SIGNING_SERVICE_ACCOUNT }}
           token_format: "access_token"
 
       - name: Setup GCloud SDK
-        uses: google-github-actions/setup-gcloud@77e7a554d41e2ee56fc945c52dfd3f33d12def9a # v2.1.4
+        uses: google-github-actions/setup-gcloud@cb1e50a9932213ecece00a606661ae9ca44f3397 # v2.2.0
 
       - name: Download dylibs
-        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
+        uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0
         with:
           name: dylibs
           path: ./build
@@ -323,6 +354,8 @@ jobs:
         env:
           CODER_SIGN_WINDOWS: "1"
           CODER_SIGN_DARWIN: "1"
+          CODER_SIGN_GPG: "1"
+          CODER_GPG_RELEASE_KEY_BASE64: ${{ secrets.GPG_RELEASE_KEY_BASE64 }}
           CODER_WINDOWS_RESOURCES: "1"
           AC_CERTIFICATE_FILE: /tmp/apple_cert.p12
           AC_CERTIFICATE_PASSWORD_FILE: /tmp/apple_cert_password.txt
@@ -348,9 +381,9 @@ jobs:
           set -euo pipefail
           if [[ "${CODER_RELEASE:-}" != *t* ]] || [[ "${CODER_DRY_RUN:-}" == *t* ]]; then
             # Empty value means use the default and avoid building a fresh one.
-            echo "tag=" >> $GITHUB_OUTPUT
+            echo "tag=" >> "$GITHUB_OUTPUT"
           else
-            echo "tag=$(CODER_IMAGE_BASE=ghcr.io/coder/coder-base ./scripts/image_tag.sh)" >> $GITHUB_OUTPUT
+            echo "tag=$(CODER_IMAGE_BASE=ghcr.io/coder/coder-base ./scripts/image_tag.sh)" >> "$GITHUB_OUTPUT"
           fi
 
       - name: Create empty base-build-context directory
@@ -364,7 +397,7 @@ jobs:
       # This uses OIDC authentication, so no auth variables are required.
       - name: Build base Docker image via depot.dev
         if: steps.image-base-tag.outputs.tag != ''
-        uses: depot/build-push-action@636daae76684e38c301daa0c5eca1c095b24e780 # v1.14.0
+        uses: depot/build-push-action@2583627a84956d07561420dcc1d0eb1f2af3fac0 # v1.15.0
         with:
           project: wl5hnrrkns
           context: base-build-context
@@ -385,7 +418,7 @@ jobs:
           # available immediately
           for i in {1..10}; do
             rc=0
-            raw_manifests=$(docker buildx imagetools inspect --raw "${{ steps.image-base-tag.outputs.tag }}") || rc=$?
+            raw_manifests=$(docker buildx imagetools inspect --raw "${IMAGE_TAG}") || rc=$?
             if [[ "$rc" -eq 0 ]]; then
               break
             fi
@@ -407,6 +440,8 @@ jobs:
           echo "$manifests" | grep -q linux/amd64
           echo "$manifests" | grep -q linux/arm64
           echo "$manifests" | grep -q linux/arm/v7
+        env:
+          IMAGE_TAG: ${{ steps.image-base-tag.outputs.tag }}
 
       # GitHub attestation provides SLSA provenance for Docker images, establishing a verifiable
       # record that these images were built in GitHub Actions with specific inputs and environment.
@@ -419,7 +454,7 @@ jobs:
         id: attest_base
         if: ${{ !inputs.dry_run && steps.image-base-tag.outputs.tag != '' }}
         continue-on-error: true
-        uses: actions/attest@afd638254319277bb3d7f0a234478733e2e46a73 # v2.3.0
+        uses: actions/attest@ce27ba3b4a9a139d9a20a4a07d69fabb52f1e5bc # v2.4.0
         with:
           subject-name: ${{ steps.image-base-tag.outputs.tag }}
           predicate-type: "https://slsa.dev/provenance/v1"
@@ -474,7 +509,7 @@ jobs:
 
           # Save multiarch image tag for attestation
           multiarch_image="$(./scripts/image_tag.sh)"
-          echo "multiarch_image=${multiarch_image}" >> $GITHUB_OUTPUT
+          echo "multiarch_image=${multiarch_image}" >> "$GITHUB_OUTPUT"
 
           # For debugging, print all docker image tags
           docker images
@@ -482,16 +517,15 @@ jobs:
           # if the current version is equal to the highest (according to semver)
           # version in the repo, also create a multi-arch image as ":latest" and
           # push it
-          created_latest_tag=false
           if [[ "$(git tag | grep '^v' | grep -vE '(rc|dev|-|\+|\/)' | sort -r --version-sort | head -n1)" == "v$(./scripts/version.sh)" ]]; then
+            # shellcheck disable=SC2046
             ./scripts/build_docker_multiarch.sh \
               --push \
               --target "$(./scripts/image_tag.sh --version latest)" \
               $(cat build/coder_"$version"_linux_{amd64,arm64,armv7}.tag)
-            created_latest_tag=true
-            echo "created_latest_tag=true" >> $GITHUB_OUTPUT
+            echo "created_latest_tag=true" >> "$GITHUB_OUTPUT"
           else
-            echo "created_latest_tag=false" >> $GITHUB_OUTPUT
+            echo "created_latest_tag=false" >> "$GITHUB_OUTPUT"
           fi
         env:
           CODER_BASE_IMAGE_TAG: ${{ steps.image-base-tag.outputs.tag }}
@@ -499,24 +533,27 @@ jobs:
       - name: SBOM Generation and Attestation
         if: ${{ !inputs.dry_run }}
         env:
-          COSIGN_EXPERIMENTAL: "1"
+          COSIGN_EXPERIMENTAL: '1'
+          MULTIARCH_IMAGE: ${{ steps.build_docker.outputs.multiarch_image }}
+          VERSION: ${{ steps.version.outputs.version }}
+          CREATED_LATEST_TAG: ${{ steps.build_docker.outputs.created_latest_tag }}
         run: |
           set -euxo pipefail
 
           # Generate SBOM for multi-arch image with version in filename
-          echo "Generating SBOM for multi-arch image: ${{ steps.build_docker.outputs.multiarch_image }}"
-          syft "${{ steps.build_docker.outputs.multiarch_image }}" -o spdx-json > coder_${{ steps.version.outputs.version }}_sbom.spdx.json
+          echo "Generating SBOM for multi-arch image: ${MULTIARCH_IMAGE}"
+          syft "${MULTIARCH_IMAGE}" -o spdx-json > "coder_${VERSION}_sbom.spdx.json"
 
           # Attest SBOM to multi-arch image
-          echo "Attesting SBOM to multi-arch image: ${{ steps.build_docker.outputs.multiarch_image }}"
-          cosign clean --force=true "${{ steps.build_docker.outputs.multiarch_image }}"
+          echo "Attesting SBOM to multi-arch image: ${MULTIARCH_IMAGE}"
+          cosign clean --force=true "${MULTIARCH_IMAGE}"
           cosign attest --type spdxjson \
-            --predicate coder_${{ steps.version.outputs.version }}_sbom.spdx.json \
+            --predicate "coder_${VERSION}_sbom.spdx.json" \
             --yes \
-            "${{ steps.build_docker.outputs.multiarch_image }}"
+            "${MULTIARCH_IMAGE}"
 
           # If latest tag was created, also attest it
-          if [[ "${{ steps.build_docker.outputs.created_latest_tag }}" == "true" ]]; then
+          if [[ "${CREATED_LATEST_TAG}" == "true" ]]; then
             latest_tag="$(./scripts/image_tag.sh --version latest)"
             echo "Generating SBOM for latest image: ${latest_tag}"
             syft "${latest_tag}" -o spdx-json > coder_latest_sbom.spdx.json
@@ -533,7 +570,7 @@ jobs:
         id: attest_main
         if: ${{ !inputs.dry_run }}
         continue-on-error: true
-        uses: actions/attest@afd638254319277bb3d7f0a234478733e2e46a73 # v2.3.0
+        uses: actions/attest@ce27ba3b4a9a139d9a20a4a07d69fabb52f1e5bc # v2.4.0
         with:
           subject-name: ${{ steps.build_docker.outputs.multiarch_image }}
           predicate-type: "https://slsa.dev/provenance/v1"
@@ -570,14 +607,14 @@ jobs:
       - name: Get latest tag name
         id: latest_tag
         if: ${{ !inputs.dry_run && steps.build_docker.outputs.created_latest_tag == 'true' }}
-        run: echo "tag=$(./scripts/image_tag.sh --version latest)" >> $GITHUB_OUTPUT
+        run: echo "tag=$(./scripts/image_tag.sh --version latest)" >> "$GITHUB_OUTPUT"
 
       # If this is the highest version according to semver, also attest the "latest" tag
       - name: GitHub Attestation for "latest" Docker image
         id: attest_latest
         if: ${{ !inputs.dry_run && steps.build_docker.outputs.created_latest_tag == 'true' }}
         continue-on-error: true
-        uses: actions/attest@afd638254319277bb3d7f0a234478733e2e46a73 # v2.3.0
+        uses: actions/attest@ce27ba3b4a9a139d9a20a4a07d69fabb52f1e5bc # v2.4.0
         with:
           subject-name: ${{ steps.latest_tag.outputs.tag }}
           predicate-type: "https://slsa.dev/provenance/v1"
@@ -613,7 +650,7 @@ jobs:
       # Report attestation failures but don't fail the workflow
       - name: Check attestation status
         if: ${{ !inputs.dry_run }}
-        run: |
+        run: | # zizmor: ignore[template-injection] We're just reading steps.attest_x.outcome here, no risk of injection
           if [[ "${{ steps.attest_base.outcome }}" == "failure" && "${{ steps.attest_base.conclusion }}" != "skipped" ]]; then
             echo "::warning::GitHub attestation for base image failed"
           fi
@@ -632,6 +669,30 @@ jobs:
       - name: ls build
         run: ls -lh build
 
+      - name: Publish Coder CLI binaries and detached signatures to GCS
+        if: ${{ !inputs.dry_run }}
+        run: |
+          set -euxo pipefail
+
+          version="$(./scripts/version.sh)"
+
+          # Source array of slim binaries
+          declare -A binaries
+          binaries["coder-darwin-amd64"]="coder-slim_${version}_darwin_amd64"
+          binaries["coder-darwin-arm64"]="coder-slim_${version}_darwin_arm64"
+          binaries["coder-linux-amd64"]="coder-slim_${version}_linux_amd64"
+          binaries["coder-linux-arm64"]="coder-slim_${version}_linux_arm64"
+          binaries["coder-linux-armv7"]="coder-slim_${version}_linux_armv7"
+          binaries["coder-windows-amd64.exe"]="coder-slim_${version}_windows_amd64.exe"
+          binaries["coder-windows-arm64.exe"]="coder-slim_${version}_windows_arm64.exe"
+
+          for cli_name in "${!binaries[@]}"; do
+            slim_binary="${binaries[$cli_name]}"
+            detached_signature="${slim_binary}.asc"
+            gcloud storage cp "./build/${slim_binary}" "gs://releases.coder.com/coder-cli/${version}/${cli_name}"
+            gcloud storage cp "./build/${detached_signature}" "gs://releases.coder.com/coder-cli/${version}/${cli_name}.asc"
+          done
+
       - name: Publish release
         run: |
           set -euo pipefail
@@ -654,11 +715,11 @@ jobs:
             ./build/*.apk
             ./build/*.deb
             ./build/*.rpm
-            ./coder_${{ steps.version.outputs.version }}_sbom.spdx.json
+            "./coder_${VERSION}_sbom.spdx.json"
           )
 
           # Only include the latest SBOM file if it was created
-          if [[ "${{ steps.build_docker.outputs.created_latest_tag }}" == "true" ]]; then
+          if [[ "${CREATED_LATEST_TAG}" == "true" ]]; then
             files+=(./coder_latest_sbom.spdx.json)
           fi
 
@@ -669,15 +730,17 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           CODER_GPG_RELEASE_KEY_BASE64: ${{ secrets.GPG_RELEASE_KEY_BASE64 }}
+          VERSION: ${{ steps.version.outputs.version }}
+          CREATED_LATEST_TAG: ${{ steps.build_docker.outputs.created_latest_tag }}
 
       - name: Authenticate to Google Cloud
-        uses: google-github-actions/auth@ba79af03959ebeac9769e648f473a284504d9193 # v2.1.10
+        uses: google-github-actions/auth@b7593ed2efd1c1617e1b0254da33b86225adb2a5 # v2.1.12
         with:
-          workload_identity_provider: ${{ secrets.GCP_WORKLOAD_ID_PROVIDER }}
-          service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }}
+          workload_identity_provider: ${{ vars.GCP_WORKLOAD_ID_PROVIDER }}
+          service_account: ${{ vars.GCP_SERVICE_ACCOUNT }}
 
       - name: Setup GCloud SDK
-        uses: google-github-actions/setup-gcloud@77e7a554d41e2ee56fc945c52dfd3f33d12def9a # 2.1.4
+        uses: google-github-actions/setup-gcloud@cb1e50a9932213ecece00a606661ae9ca44f3397 # 2.2.0
 
       - name: Publish Helm Chart
         if: ${{ !inputs.dry_run }}
@@ -689,10 +752,12 @@ jobs:
           cp "build/provisioner_helm_${version}.tgz" build/helm
           gsutil cp gs://helm.coder.com/v2/index.yaml build/helm/index.yaml
           helm repo index build/helm --url https://helm.coder.com/v2 --merge build/helm/index.yaml
-          gsutil -h "Cache-Control:no-cache,max-age=0" cp build/helm/coder_helm_${version}.tgz gs://helm.coder.com/v2
-          gsutil -h "Cache-Control:no-cache,max-age=0" cp build/helm/provisioner_helm_${version}.tgz gs://helm.coder.com/v2
-          gsutil -h "Cache-Control:no-cache,max-age=0" cp build/helm/index.yaml gs://helm.coder.com/v2
-          gsutil -h "Cache-Control:no-cache,max-age=0" cp helm/artifacthub-repo.yml gs://helm.coder.com/v2
+          gsutil -h "Cache-Control:no-cache,max-age=0" cp "build/helm/coder_helm_${version}.tgz" gs://helm.coder.com/v2
+          gsutil -h "Cache-Control:no-cache,max-age=0" cp "build/helm/provisioner_helm_${version}.tgz" gs://helm.coder.com/v2
+          gsutil -h "Cache-Control:no-cache,max-age=0" cp "build/helm/index.yaml" gs://helm.coder.com/v2
+          gsutil -h "Cache-Control:no-cache,max-age=0" cp "helm/artifacthub-repo.yml" gs://helm.coder.com/v2
+          helm push "build/coder_helm_${version}.tgz" oci://ghcr.io/coder/chart
+          helm push "build/provisioner_helm_${version}.tgz" oci://ghcr.io/coder/chart
 
       - name: Upload artifacts to actions (if dry-run)
         if: ${{ inputs.dry_run }}
@@ -737,18 +802,18 @@ jobs:
       # TODO: skip this if it's not a new release (i.e. a backport). This is
       #       fine right now because it just makes a PR that we can close.
       - name: Harden Runner
-        uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
+        uses: step-security/harden-runner@ec9f2d5744a09debf3a187a3f4f675c53b671911 # v2.13.0
         with:
           egress-policy: audit
 
       - name: Update homebrew
         env:
-          # Variables used by the `gh` command
           GH_REPO: coder/homebrew-coder
           GH_TOKEN: ${{ secrets.CDRCI_GITHUB_TOKEN }}
+          VERSION: ${{ needs.release.outputs.version }}
         run: |
           # Keep version number around for reference, removing any potential leading v
-          coder_version="$(echo "${{ needs.release.outputs.version }}" | tr -d v)"
+          coder_version="$(echo "${VERSION}" | tr -d v)"
 
           set -euxo pipefail
 
@@ -767,9 +832,9 @@ jobs:
           wget "$checksums_url" -O checksums.txt
 
           # Get the SHAs
-          darwin_arm_sha="$(cat checksums.txt | grep "darwin_arm64.zip" | awk '{ print $1 }')"
-          darwin_intel_sha="$(cat checksums.txt | grep "darwin_amd64.zip" | awk '{ print $1 }')"
-          linux_sha="$(cat checksums.txt | grep "linux_amd64.tar.gz" | awk '{ print $1 }')"
+          darwin_arm_sha="$(grep "darwin_arm64.zip" checksums.txt | awk '{ print $1 }')"
+          darwin_intel_sha="$(grep "darwin_amd64.zip" checksums.txt | awk '{ print $1 }')"
+          linux_sha="$(grep "linux_amd64.tar.gz" checksums.txt | awk '{ print $1 }')"
 
           echo "macOS arm64: $darwin_arm_sha"
           echo "macOS amd64: $darwin_intel_sha"
@@ -782,7 +847,7 @@ jobs:
 
           # Check if a PR already exists.
           pr_count="$(gh pr list --search "head:$brew_branch" --json id,closed | jq -r ".[] | select(.closed == false) | .id" | wc -l)"
-          if [[ "$pr_count" > 0 ]]; then
+          if [ "$pr_count" -gt 0 ]; then
             echo "Bailing out as PR already exists" 2>&1
             exit 0
           fi
@@ -801,8 +866,8 @@ jobs:
             -B master -H "$brew_branch" \
             -t "coder $coder_version" \
             -b "" \
-            -r "${{ github.actor }}" \
-            -a "${{ github.actor }}" \
+            -r "${GITHUB_ACTOR}" \
+            -a "${GITHUB_ACTOR}" \
             -b "This automatic PR was triggered by the release of Coder v$coder_version"
 
   publish-winget:
@@ -813,7 +878,7 @@ jobs:
 
     steps:
       - name: Harden Runner
-        uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
+        uses: step-security/harden-runner@ec9f2d5744a09debf3a187a3f4f675c53b671911 # v2.13.0
         with:
           egress-policy: audit
 
@@ -823,9 +888,10 @@ jobs:
           GH_TOKEN: ${{ secrets.CDRCI_GITHUB_TOKEN }}
 
       - name: Checkout
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
         with:
           fetch-depth: 0
+          persist-credentials: false
 
       # If the event that triggered the build was an annotated tag (which our
       # tags are supposed to be), actions/checkout has a bug where the tag in
@@ -844,7 +910,7 @@ jobs:
           # The package version is the same as the tag minus the leading "v".
           # The version in this output already has the leading "v" removed but
           # we do it again to be safe.
-          $version = "${{ needs.release.outputs.version }}".Trim('v')
+          $version = $env:VERSION.Trim('v')
 
           $release_assets = gh release view --repo coder/coder "v${version}" --json assets | `
             ConvertFrom-Json
@@ -876,13 +942,14 @@ jobs:
           # For wingetcreate. We need a real token since we're pushing a commit
           # to GitHub and then making a PR in a different repo.
           WINGET_GH_TOKEN: ${{ secrets.CDRCI_GITHUB_TOKEN }}
+          VERSION: ${{ needs.release.outputs.version }}
 
       - name: Comment on PR
         run: |
           # wait 30 seconds
           Start-Sleep -Seconds 30.0
           # Find the PR that wingetcreate just made.
-          $version = "${{ needs.release.outputs.version }}".Trim('v')
+          $version = $env:VERSION.Trim('v')
           $pr_list = gh pr list --repo microsoft/winget-pkgs --search "author:cdrci Coder.Coder version ${version}" --limit 1 --json number | `
             ConvertFrom-Json
           $pr_number = $pr_list[0].number
@@ -893,6 +960,7 @@ jobs:
           # For gh CLI. We need a real token since we're commenting on a PR in a
           # different repo.
           GH_TOKEN: ${{ secrets.CDRCI_GITHUB_TOKEN }}
+          VERSION: ${{ needs.release.outputs.version }}
 
   # publish-sqlc pushes the latest schema to sqlc cloud.
   # At present these pushes cannot be tagged, so the last push is always the latest.
@@ -903,14 +971,15 @@ jobs:
     if: ${{ !inputs.dry_run }}
     steps:
       - name: Harden Runner
-        uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
+        uses: step-security/harden-runner@ec9f2d5744a09debf3a187a3f4f675c53b671911 # v2.13.0
         with:
           egress-policy: audit
 
       - name: Checkout
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
         with:
           fetch-depth: 1
+          persist-credentials: false
 
       # We need golang to run the migration main.go
       - name: Setup Go

.github/workflows/scorecard.yml

@@ -20,17 +20,17 @@ jobs:
 
     steps:
       - name: Harden Runner
-        uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
+        uses: step-security/harden-runner@ec9f2d5744a09debf3a187a3f4f675c53b671911 # v2.13.0
         with:
           egress-policy: audit
 
       - name: "Checkout code"
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
         with:
           persist-credentials: false
 
       - name: "Run analysis"
-        uses: ossf/scorecard-action@f49aabe0b5af0936a0987cfb85d86b75731b0186 # v2.4.1
+        uses: ossf/scorecard-action@05b42c624433fc40578a4040d5cf5e36ddca8cde # v2.4.2
         with:
           results_file: results.sarif
           results_format: sarif
@@ -47,6 +47,6 @@ jobs:
 
       # Upload the results to GitHub's code scanning dashboard.
       - name: "Upload to code-scanning"
-        uses: github/codeql-action/upload-sarif@ff0a06e83cb2de871e5a09832bc6a81e7276941f # v3.28.18
+        uses: github/codeql-action/upload-sarif@76621b61decf072c1cee8dd1ce2d2a82d33c17ed # v3.29.5
         with:
           sarif_file: results.sarif

.github/workflows/security.yaml

@@ -27,18 +27,20 @@ jobs:
     runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-8' || 'ubuntu-latest' }}
     steps:
       - name: Harden Runner
-        uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
+        uses: step-security/harden-runner@ec9f2d5744a09debf3a187a3f4f675c53b671911 # v2.13.0
         with:
           egress-policy: audit
 
       - name: Checkout
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
 
       - name: Setup Go
         uses: ./.github/actions/setup-go
 
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@ff0a06e83cb2de871e5a09832bc6a81e7276941f # v3.28.18
+        uses: github/codeql-action/init@76621b61decf072c1cee8dd1ce2d2a82d33c17ed # v3.29.5
         with:
           languages: go, javascript
 
@@ -48,7 +50,7 @@ jobs:
           rm Makefile
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@ff0a06e83cb2de871e5a09832bc6a81e7276941f # v3.28.18
+        uses: github/codeql-action/analyze@76621b61decf072c1cee8dd1ce2d2a82d33c17ed # v3.29.5
 
       - name: Send Slack notification on failure
         if: ${{ failure() }}
@@ -67,14 +69,15 @@ jobs:
     runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-8' || 'ubuntu-latest' }}
     steps:
       - name: Harden Runner
-        uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
+        uses: step-security/harden-runner@ec9f2d5744a09debf3a187a3f4f675c53b671911 # v2.13.0
         with:
           egress-policy: audit
 
       - name: Checkout
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
         with:
           fetch-depth: 0
+          persist-credentials: false
 
       - name: Setup Go
         uses: ./.github/actions/setup-go
@@ -134,15 +137,16 @@ jobs:
           # This environment variables forces scripts/build_docker.sh to build
           # the base image tag locally instead of using the cached version from
           # the registry.
-          export CODER_IMAGE_BUILD_BASE_TAG="$(CODER_IMAGE_BASE=coder-base ./scripts/image_tag.sh --version "$version")"
+          CODER_IMAGE_BUILD_BASE_TAG="$(CODER_IMAGE_BASE=coder-base ./scripts/image_tag.sh --version "$version")"
+          export CODER_IMAGE_BUILD_BASE_TAG
 
           # We would like to use make -j here, but it doesn't work with the some recent additions
           # to our code generation.
           make "$image_job"
-          echo "image=$(cat "$image_job")" >> $GITHUB_OUTPUT
+          echo "image=$(cat "$image_job")" >> "$GITHUB_OUTPUT"
 
       - name: Run Trivy vulnerability scanner
-        uses: aquasecurity/trivy-action@6c175e9c4083a92bbca2f9724c8a5e33bc2d97a5
+        uses: aquasecurity/trivy-action@dc5a429b52fcf669ce959baa2c2dd26090d2a6c4
         with:
           image-ref: ${{ steps.build.outputs.image }}
           format: sarif
@@ -150,7 +154,7 @@ jobs:
           severity: "CRITICAL,HIGH"
 
       - name: Upload Trivy scan results to GitHub Security tab
-        uses: github/codeql-action/upload-sarif@ff0a06e83cb2de871e5a09832bc6a81e7276941f # v3.28.18
+        uses: github/codeql-action/upload-sarif@76621b61decf072c1cee8dd1ce2d2a82d33c17ed # v3.29.5
         with:
           sarif_file: trivy-results.sarif
           category: "Trivy"

.github/workflows/stale.yaml

@@ -18,7 +18,7 @@ jobs:
       pull-requests: write
     steps:
       - name: Harden Runner
-        uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
+        uses: step-security/harden-runner@ec9f2d5744a09debf3a187a3f4f675c53b671911 # v2.13.0
         with:
           egress-policy: audit
 
@@ -96,12 +96,14 @@ jobs:
       contents: write
     steps:
       - name: Harden Runner
-        uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
+        uses: step-security/harden-runner@ec9f2d5744a09debf3a187a3f4f675c53b671911 # v2.13.0
         with:
           egress-policy: audit
 
       - name: Checkout repository
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
       - name: Run delete-old-branches-action
         uses: beatlabs/delete-old-branches-action@4eeeb8740ff8b3cb310296ddd6b43c3387734588 # v0.0.11
         with:
@@ -118,7 +120,7 @@ jobs:
       actions: write
     steps:
       - name: Harden Runner
-        uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
+        uses: step-security/harden-runner@ec9f2d5744a09debf3a187a3f4f675c53b671911 # v2.13.0
         with:
           egress-policy: audit
 

.github/workflows/start-workspace.yaml

@@ -19,7 +19,7 @@ jobs:
     timeout-minutes: 5
     steps:
       - name: Start Coder workspace
-        uses: coder/start-workspace-action@35a4608cefc7e8cc56573cae7c3b85304575cb72
+        uses: coder/start-workspace-action@f97a681b4cc7985c9eef9963750c7cc6ebc93a19
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           github-username: >-

.github/workflows/typos.toml

@@ -28,6 +28,7 @@ HELO = "HELO"
 LKE = "LKE"
 byt = "byt"
 typ = "typ"
+Inferrable = "Inferrable"
 
 [files]
 extend-exclude = [
@@ -47,5 +48,5 @@ extend-exclude = [
 	"provisioner/terraform/testdata/**",
 	# notifications' golden files confuse the detector because of quoted-printable encoding
 	"coderd/notifications/testdata/**",
-	"agent/agentcontainers/testdata/devcontainercli/**"
+	"agent/agentcontainers/testdata/devcontainercli/**",
 ]

.github/workflows/weekly-docs.yaml

@@ -21,15 +21,17 @@ jobs:
       pull-requests: write # required to post PR review comments by the action
     steps:
       - name: Harden Runner
-        uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
+        uses: step-security/harden-runner@ec9f2d5744a09debf3a187a3f4f675c53b671911 # v2.13.0
         with:
           egress-policy: audit
 
       - name: Checkout
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
 
       - name: Check Markdown links
-        uses: umbrelladocs/action-linkspector@a0567ce1c7c13de4a2358587492ed43cab5d0102 # v1.3.4
+        uses: umbrelladocs/action-linkspector@874d01cae9fd488e3077b08952093235bd626977 # v1.3.7
         id: markdown-link-check
         # checks all markdown files from /docs including all subfolders
         with:
@@ -41,7 +43,10 @@ jobs:
       - name: Send Slack notification
         if: failure() && github.event_name == 'schedule'
         run: |
-          curl -X POST -H 'Content-type: application/json' -d '{"msg":"Broken links found in the documentation. Please check the logs at ${{ env.LOGS_URL }}"}' ${{ secrets.DOCS_LINK_SLACK_WEBHOOK }}
+          curl \
+            -X POST \
+            -H 'Content-type: application/json' \
+            -d '{"msg":"Broken links found in the documentation. Please check the logs at '"${LOGS_URL}"'"}' "${{ secrets.DOCS_LINK_SLACK_WEBHOOK }}"
           echo "Sent Slack notification"
         env:
           LOGS_URL: https://github.com/coder/coder/actions/runs/${{ github.run_id }}

.golangci.yaml

@@ -181,7 +181,6 @@ linters-settings:
 
 issues:
   exclude-dirs:
-    - coderd/database/dbmem
     - node_modules
     - .git
 

.mcp.json

@@ -0,0 +1,36 @@
+{
+  "mcpServers": {
+    "go-language-server": {
+      "type": "stdio",
+      "command": "go",
+      "args": [
+        "run",
+        "github.com/isaacphi/mcp-language-server@latest",
+        "-workspace",
+        "./",
+        "-lsp",
+        "go",
+        "--",
+        "run",
+        "golang.org/x/tools/gopls@latest"
+      ],
+      "env": {}
+    },
+    "typescript-language-server": {
+      "type": "stdio",
+      "command": "go",
+      "args": [
+        "run",
+        "github.com/isaacphi/mcp-language-server@latest",
+        "-workspace",
+        "./site/",
+        "-lsp",
+        "pnpx",
+        "--",
+        "typescript-language-server",
+        "--stdio"
+      ],
+      "env": {}
+    }
+  }
+}

.vscode/settings.json

@@ -49,7 +49,7 @@
 	"[javascript][javascriptreact][json][jsonc][typescript][typescriptreact]": {
 		"editor.defaultFormatter": "biomejs.biome",
 		"editor.codeActionsOnSave": {
-			"quickfix.biome": "explicit"
+			"source.fixAll.biome": "explicit"
 			//   "source.organizeImports.biome": "explicit"
 		}
 	},

AGENTS.md

@@ -0,0 +1 @@
+CLAUDE.md

CLAUDE.md

@@ -1,21 +1,25 @@
 # Coder Development Guidelines
 
-Read [cursor rules](.cursorrules).
+@.claude/docs/WORKFLOWS.md
+@.cursorrules
+@README.md
+@package.json
 
-## Build/Test/Lint Commands
+## 🚀 Essential Commands
 
-### Main Commands
-
-- `make build` or `make build-fat` - Build all "fat" binaries (includes "server" functionality)
-- `make build-slim` - Build "slim" binaries
-- `make test` - Run Go tests
-- `make test RUN=TestFunctionName` or `go test -v ./path/to/package -run TestFunctionName` - Test single
-- `make test-postgres` - Run tests with Postgres database
-- `make test-race` - Run tests with Go race detector
-- `make test-e2e` - Run end-to-end tests
-- `make lint` - Run all linters
-- `make fmt` - Format all code
-- `make gen` - Generates mocks, database queries and other auto-generated files
+| Task              | Command                  | Notes                            |
+|-------------------|--------------------------|----------------------------------|
+| **Development**   | `./scripts/develop.sh`   | ⚠️ Don't use manual build        |
+| **Build**         | `make build`             | Fat binaries (includes server)   |
+| **Build Slim**    | `make build-slim`        | Slim binaries                    |
+| **Test**          | `make test`              | Full test suite                  |
+| **Test Single**   | `make test RUN=TestName` | Faster than full suite           |
+| **Test Postgres** | `make test-postgres`     | Run tests with Postgres database |
+| **Test Race**     | `make test-race`         | Run tests with Go race detector  |
+| **Lint**          | `make lint`              | Always run after changes         |
+| **Generate**      | `make gen`               | After database changes           |
+| **Format**        | `make fmt`               | Auto-format code                 |
+| **Clean**         | `make clean`             | Clean build artifacts            |
 
 ### Frontend Commands (site directory)
 
@@ -26,79 +30,109 @@ Read [cursor rules](.cursorrules).
 - `pnpm lint` - Lint frontend code
 - `pnpm test` - Run frontend tests
 
-## Code Style Guidelines
+### Documentation Commands
 
-### Go
+- `pnpm run format-docs` - Format markdown tables in docs
+- `pnpm run lint-docs` - Lint and fix markdown files
+- `pnpm run storybook` - Run Storybook (from site directory)
 
-- Follow [Effective Go](https://go.dev/doc/effective_go) and [Go's Code Review Comments](https://github.com/golang/go/wiki/CodeReviewComments)
-- Use `gofumpt` for formatting
-- Create packages when used during implementation
-- Validate abstractions against implementations
+## 🔧 Critical Patterns
 
-### Error Handling
+### Database Changes (ALWAYS FOLLOW)
 
-- Use descriptive error messages
-- Wrap errors with context
-- Propagate errors appropriately
-- Use proper error types
-- (`xerrors.Errorf("failed to X: %w", err)`)
+1. Modify `coderd/database/queries/*.sql` files
+2. Run `make gen`
+3. If audit errors: update `enterprise/audit/table.go`
+4. Run `make gen` again
 
-### Naming
+### LSP Navigation (USE FIRST)
 
-- Use clear, descriptive names
-- Abbreviate only when obvious
-- Follow Go and TypeScript naming conventions
+#### Go LSP (for backend code)
 
-### Comments
+- **Find definitions**: `mcp__go-language-server__definition symbolName`
+- **Find references**: `mcp__go-language-server__references symbolName`
+- **Get type info**: `mcp__go-language-server__hover filePath line column`
+- **Rename symbol**: `mcp__go-language-server__rename_symbol filePath line column newName`
 
-- Document exported functions, types, and non-obvious logic
-- Follow JSDoc format for TypeScript
-- Use godoc format for Go code
+#### TypeScript LSP (for frontend code in site/)
 
-## Commit Style
+- **Find definitions**: `mcp__typescript-language-server__definition symbolName`
+- **Find references**: `mcp__typescript-language-server__references symbolName`
+- **Get type info**: `mcp__typescript-language-server__hover filePath line column`
+- **Rename symbol**: `mcp__typescript-language-server__rename_symbol filePath line column newName`
 
-- Follow [Conventional Commits 1.0.0](https://www.conventionalcommits.org/en/v1.0.0/)
-- Format: `type(scope): message`
-- Types: `feat`, `fix`, `docs`, `style`, `refactor`, `test`, `chore`
-- Keep message titles concise (~70 characters)
-- Use imperative, present tense in commit titles
+### OAuth2 Error Handling
 
-## Database queries
+```go
+// OAuth2-compliant error responses
+writeOAuth2Error(ctx, rw, http.StatusBadRequest, "invalid_grant", "description")
+```
 
-- MUST DO! Any changes to database - adding queries, modifying queries should be done in the  `coderd\database\queries\*.sql` files. Use `make gen` to generate necessary changes after.
-- MUST DO! Queries are grouped in files relating to context - e.g. `prebuilds.sql`, `users.sql`, `provisionerjobs.sql`.
-- After making changes to any `coderd\database\queries\*.sql` files you must run `make gen` to generate respective ORM changes.
+### Authorization Context
 
-## Architecture
+```go
+// Public endpoints needing system access
+app, err := api.Database.GetOAuth2ProviderAppByClientID(dbauthz.AsSystemRestricted(ctx), clientID)
 
-### Core Components
+// Authenticated endpoints with user context
+app, err := api.Database.GetOAuth2ProviderAppByClientID(ctx, clientID)
+```
 
-- **coderd**: Main API service connecting workspaces, provisioners, and users
-- **provisionerd**: Execution context for infrastructure-modifying providers
-- **Agents**: Services in remote workspaces providing features like SSH and port forwarding
-- **Workspaces**: Cloud resources defined by Terraform
+## 📋 Quick Reference
 
-## Sub-modules
+### Full workflows available in imported WORKFLOWS.md
 
-### Template System
+### New Feature Checklist
 
-- Templates define infrastructure for workspaces using Terraform
-- Environment variables pass context between Coder and templates
-- Official modules extend development environments
+- [ ] Run `git pull` to ensure latest code
+- [ ] Check if feature touches database - you'll need migrations
+- [ ] Check if feature touches audit logs - update `enterprise/audit/table.go`
 
-### RBAC System
+## 🏗️ Architecture
 
-- Permissions defined at site, organization, and user levels
-- Object-Action model protects resources
-- Built-in roles: owner, member, auditor, templateAdmin
-- Permission format: `<sign>?<level>.<object>.<id>.<action>`
+- **coderd**: Main API service
+- **provisionerd**: Infrastructure provisioning
+- **Agents**: Workspace services (SSH, port forwarding)
+- **Database**: PostgreSQL with `dbauthz` authorization
 
-### Database
+## 🧪 Testing
 
-- PostgreSQL 13+ recommended for production
-- Migrations managed with `migrate`
-- Database authorization through `dbauthz` package
+### Race Condition Prevention
 
-## Frontend
+- Use unique identifiers: `fmt.Sprintf("test-client-%s-%d", t.Name(), time.Now().UnixNano())`
+- Never use hardcoded names in concurrent tests
 
-For building Frontend refer to [this document](docs/contributing/frontend.md)
+### OAuth2 Testing
+
+- Full suite: `./scripts/oauth2/test-mcp-oauth2.sh`
+- Manual testing: `./scripts/oauth2/test-manual-flow.sh`
+
+### Timing Issues
+
+NEVER use `time.Sleep` to mitigate timing issues. If an issue
+seems like it should use `time.Sleep`, read through https://github.com/coder/quartz and specifically the [README](https://github.com/coder/quartz/blob/main/README.md) to better understand how to handle timing issues.
+
+## 🎯 Code Style
+
+### Detailed guidelines in imported WORKFLOWS.md
+
+- Follow [Uber Go Style Guide](https://github.com/uber-go/guide/blob/master/style.md)
+- Commit format: `type(scope): message`
+
+## 📚 Detailed Development Guides
+
+@.claude/docs/OAUTH2.md
+@.claude/docs/TESTING.md
+@.claude/docs/TROUBLESHOOTING.md
+@.claude/docs/DATABASE.md
+
+## 🚨 Common Pitfalls
+
+1. **Audit table errors** → Update `enterprise/audit/table.go`
+2. **OAuth2 errors** → Return RFC-compliant format
+3. **Race conditions** → Use unique test identifiers
+4. **Missing newlines** → Ensure files end with newline
+
+---
+
+*This file stays lean and actionable. Detailed workflows and explanations are imported automatically.*

CODEOWNERS

@@ -1,8 +1,41 @@
-# These APIs are versioned, so any changes need to be carefully reviewed for whether
-# to bump API major or minor versions.
+# These APIs are versioned, so any changes need to be carefully reviewed for
+# whether to bump API major or minor versions.
 agent/proto/ @spikecurtis @johnstcn
+provisionerd/proto/ @spikecurtis @johnstcn
+provisionersdk/proto/ @spikecurtis @johnstcn
 tailnet/proto/ @spikecurtis @johnstcn
 vpn/vpn.proto @spikecurtis @johnstcn
 vpn/version.go @spikecurtis @johnstcn
-provisionerd/proto/ @spikecurtis @johnstcn
-provisionersdk/proto/ @spikecurtis @johnstcn
+
+# This caching code is particularly tricky, and one must be very careful when
+# altering it.
+coderd/files/ @aslilac
+
+coderd/dynamicparameters/ @Emyrk
+coderd/rbac/ @Emyrk
+
+# Mainly dependent on coder/guts, which is maintained by @Emyrk
+scripts/apitypings/ @Emyrk
+scripts/gensite/ @aslilac
+
+site/ @aslilac @Parkreiner
+site/src/hooks/ @Parkreiner
+# These rules intentionally do not specify any owners. More specific rules
+# override less specific rules, so these files are "ignored" by the site/ rule.
+site/e2e/google/protobuf/timestampGenerated.ts
+site/e2e/provisionerGenerated.ts
+site/src/api/countriesGenerated.ts
+site/src/api/rbacresourcesGenerated.ts
+site/src/api/typesGenerated.ts
+site/src/testHelpers/entities.ts
+site/CLAUDE.md
+
+# The blood and guts of the autostop algorithm, which is quite complex and
+# requires elite ball knowledge of most of the scheduling code to make changes
+# without inadvertently affecting other parts of the codebase.
+coderd/schedule/autostop.go @deansheather @DanielleMaywood
+
+# Usage tracking code requires intimate knowledge of Tallyman and Metronome, as
+# well as guidance from revenue.
+coderd/usage/ @deansheather @spikecurtis
+enterprise/coderd/usage/ @deansheather @spikecurtis

CODE_OF_CONDUCT.md

@@ -1,2 +1,2 @@
 <!-- markdownlint-disable MD041 -->
-[https://coder.com/docs/contributing/CODE_OF_CONDUCT](https://coder.com/docs/contributing/CODE_OF_CONDUCT)
+[https://coder.com/docs/about/contributing/CODE_OF_CONDUCT](https://coder.com/docs/about/contributing/CODE_OF_CONDUCT)

Makefile

@@ -36,7 +36,9 @@ GOOS         := $(shell go env GOOS)
 GOARCH       := $(shell go env GOARCH)
 GOOS_BIN_EXT := $(if $(filter windows, $(GOOS)),.exe,)
 VERSION      := $(shell ./scripts/version.sh)
-POSTGRES_VERSION ?= 16
+
+POSTGRES_VERSION ?= 17
+POSTGRES_IMAGE   ?= us-docker.pkg.dev/coder-v2-images-public/public/postgres:$(POSTGRES_VERSION)
 
 # Use the highest ZSTD compression level in CI.
 ifdef CI
@@ -250,6 +252,10 @@ $(CODER_ALL_BINARIES): go.mod go.sum \
 		fi
 
 		cp "$@" "./site/out/bin/coder-$$os-$$arch$$dot_ext"
+
+		if [[ "$${CODER_SIGN_GPG:-0}" == "1" ]]; then
+			cp "$@.asc" "./site/out/bin/coder-$$os-$$arch$$dot_ext.asc"
+		fi
 	fi
 
 # This task builds Coder Desktop dylibs
@@ -454,16 +460,31 @@ fmt: fmt/ts fmt/go fmt/terraform fmt/shfmt fmt/biome fmt/markdown
 .PHONY: fmt
 
 fmt/go:
+ifdef FILE
+	# Format single file
+	if [[ -f "$(FILE)" ]] && [[ "$(FILE)" == *.go ]] && ! grep -q "DO NOT EDIT" "$(FILE)"; then \
+		echo "$(GREEN)==>$(RESET) $(BOLD)fmt/go$(RESET) $(FILE)"; \
+		go run mvdan.cc/gofumpt@v0.8.0 -w -l "$(FILE)"; \
+	fi
+else
 	go mod tidy
 	echo "$(GREEN)==>$(RESET) $(BOLD)fmt/go$(RESET)"
 	# VS Code users should check out
 	# https://github.com/mvdan/gofumpt#visual-studio-code
 	find . $(FIND_EXCLUSIONS) -type f -name '*.go' -print0 | \
-		xargs -0 grep --null -L "DO NOT EDIT" | \
-		xargs -0 go run mvdan.cc/gofumpt@v0.4.0 -w -l
+		xargs -0 grep -E --null -L '^// Code generated .* DO NOT EDIT\.$$' | \
+		xargs -0 go run mvdan.cc/gofumpt@v0.8.0 -w -l
+endif
 .PHONY: fmt/go
 
 fmt/ts: site/node_modules/.installed
+ifdef FILE
+	# Format single TypeScript/JavaScript file
+	if [[ -f "$(FILE)" ]] && [[ "$(FILE)" == *.ts ]] || [[ "$(FILE)" == *.tsx ]] || [[ "$(FILE)" == *.js ]] || [[ "$(FILE)" == *.jsx ]]; then \
+		echo "$(GREEN)==>$(RESET) $(BOLD)fmt/ts$(RESET) $(FILE)"; \
+		(cd site/ && pnpm exec biome format --write "../$(FILE)"); \
+	fi
+else
 	echo "$(GREEN)==>$(RESET) $(BOLD)fmt/ts$(RESET)"
 	cd site
 # Avoid writing files in CI to reduce file write activity
@@ -472,9 +493,17 @@ ifdef CI
 else
 	pnpm run check:fix
 endif
+endif
 .PHONY: fmt/ts
 
 fmt/biome: site/node_modules/.installed
+ifdef FILE
+	# Format single file with biome
+	if [[ -f "$(FILE)" ]] && [[ "$(FILE)" == *.ts ]] || [[ "$(FILE)" == *.tsx ]] || [[ "$(FILE)" == *.js ]] || [[ "$(FILE)" == *.jsx ]]; then \
+		echo "$(GREEN)==>$(RESET) $(BOLD)fmt/biome$(RESET) $(FILE)"; \
+		(cd site/ && pnpm exec biome format --write "../$(FILE)"); \
+	fi
+else
 	echo "$(GREEN)==>$(RESET) $(BOLD)fmt/biome$(RESET)"
 	cd site/
 # Avoid writing files in CI to reduce file write activity
@@ -483,14 +512,30 @@ ifdef CI
 else
 	pnpm run format
 endif
+endif
 .PHONY: fmt/biome
 
 fmt/terraform: $(wildcard *.tf)
+ifdef FILE
+	# Format single Terraform file
+	if [[ -f "$(FILE)" ]] && [[ "$(FILE)" == *.tf ]] || [[ "$(FILE)" == *.tfvars ]]; then \
+		echo "$(GREEN)==>$(RESET) $(BOLD)fmt/terraform$(RESET) $(FILE)"; \
+		terraform fmt "$(FILE)"; \
+	fi
+else
 	echo "$(GREEN)==>$(RESET) $(BOLD)fmt/terraform$(RESET)"
 	terraform fmt -recursive
+endif
 .PHONY: fmt/terraform
 
 fmt/shfmt: $(SHELL_SRC_FILES)
+ifdef FILE
+	# Format single shell script
+	if [[ -f "$(FILE)" ]] && [[ "$(FILE)" == *.sh ]]; then \
+		echo "$(GREEN)==>$(RESET) $(BOLD)fmt/shfmt$(RESET) $(FILE)"; \
+		shfmt -w "$(FILE)"; \
+	fi
+else
 	echo "$(GREEN)==>$(RESET) $(BOLD)fmt/shfmt$(RESET)"
 # Only do diff check in CI, errors on diff.
 ifdef CI
@@ -498,14 +543,25 @@ ifdef CI
 else
 	shfmt -w $(SHELL_SRC_FILES)
 endif
+endif
 .PHONY: fmt/shfmt
 
 fmt/markdown: node_modules/.installed
+ifdef FILE
+	# Format single markdown file
+	if [[ -f "$(FILE)" ]] && [[ "$(FILE)" == *.md ]]; then \
+		echo "$(GREEN)==>$(RESET) $(BOLD)fmt/markdown$(RESET) $(FILE)"; \
+		pnpm exec markdown-table-formatter "$(FILE)"; \
+	fi
+else
 	echo "$(GREEN)==>$(RESET) $(BOLD)fmt/markdown$(RESET)"
 	pnpm format-docs
+endif
 .PHONY: fmt/markdown
 
-lint: lint/shellcheck lint/go lint/ts lint/examples lint/helm lint/site-icons lint/markdown
+# Note: we don't run zizmor in the lint target because it takes a while. CI
+#       runs it explicitly.
+lint: lint/shellcheck lint/go lint/ts lint/examples lint/helm lint/site-icons lint/markdown lint/actions/actionlint
 .PHONY: lint
 
 lint/site-icons:
@@ -522,6 +578,7 @@ lint/go:
 	./scripts/check_codersdk_imports.sh
 	linter_ver=$(shell egrep -o 'GOLANGCI_LINT_VERSION=\S+' dogfood/coder/Dockerfile | cut -d '=' -f 2)
 	go run github.com/golangci/golangci-lint/cmd/golangci-lint@v$$linter_ver run
+	go run github.com/coder/paralleltestctx/cmd/paralleltestctx@v0.0.1 -custom-funcs="testutil.Context" ./...
 .PHONY: lint/go
 
 lint/examples:
@@ -543,13 +600,26 @@ lint/markdown: node_modules/.installed
 	pnpm lint-docs
 .PHONY: lint/markdown
 
+lint/actions: lint/actions/actionlint lint/actions/zizmor
+.PHONY: lint/actions
+
+lint/actions/actionlint:
+	go run github.com/rhysd/actionlint/cmd/actionlint@v1.7.7
+.PHONY: lint/actions/actionlint
+
+lint/actions/zizmor:
+	./scripts/zizmor.sh \
+		--strict-collection \
+		--persona=regular \
+		.
+.PHONY: lint/actions/zizmor
+
 # All files generated by the database should be added here, and this can be used
 # as a target for jobs that need to run after the database is generated.
 DB_GEN_FILES := \
 	coderd/database/dump.sql \
 	coderd/database/querier.go \
 	coderd/database/unique_constraint.go \
-	coderd/database/dbmem/dbmem.go \
 	coderd/database/dbmetrics/dbmetrics.go \
 	coderd/database/dbauthz/dbauthz.go \
 	coderd/database/dbmock/dbmock.go
@@ -582,7 +652,8 @@ GEN_FILES := \
 	coderd/database/pubsub/psmock/psmock.go \
 	agent/agentcontainers/acmock/acmock.go \
 	agent/agentcontainers/dcspec/dcspec_gen.go \
-	coderd/httpmw/loggermw/loggermock/loggermock.go
+	coderd/httpmw/loggermw/loggermock/loggermock.go \
+	codersdk/workspacesdk/agentconnmock/agentconnmock.go
 
 # all gen targets should be added here and to gen/mark-fresh
 gen: gen/db gen/golden-files $(GEN_FILES)
@@ -632,6 +703,7 @@ gen/mark-fresh:
 		agent/agentcontainers/acmock/acmock.go \
 		agent/agentcontainers/dcspec/dcspec_gen.go \
 		coderd/httpmw/loggermw/loggermock/loggermock.go \
+		codersdk/workspacesdk/agentconnmock/agentconnmock.go \
 		"
 
 	for file in $$files; do
@@ -675,6 +747,10 @@ coderd/httpmw/loggermw/loggermock/loggermock.go: coderd/httpmw/loggermw/logger.g
 	go generate ./coderd/httpmw/loggermw/loggermock/
 	touch "$@"
 
+codersdk/workspacesdk/agentconnmock/agentconnmock.go: codersdk/workspacesdk/agentconn.go
+	go generate ./codersdk/workspacesdk/agentconnmock/
+	touch "$@"
+
 agent/agentcontainers/dcspec/dcspec_gen.go: \
 	node_modules/.installed \
 	agent/agentcontainers/dcspec/devContainer.base.schema.json \
@@ -882,12 +958,31 @@ else
 GOTESTSUM_RETRY_FLAGS :=
 endif
 
+# default to 8x8 parallelism to avoid overwhelming our workspaces. Hopefully we can remove these defaults
+# when we get our test suite's resource utilization under control.
+GOTEST_FLAGS := -v -p $(or $(TEST_NUM_PARALLEL_PACKAGES),"8") -parallel=$(or $(TEST_NUM_PARALLEL_TESTS),"8")
+
+# The most common use is to set TEST_COUNT=1 to avoid Go's test cache.
+ifdef TEST_COUNT
+GOTEST_FLAGS += -count=$(TEST_COUNT)
+endif
+
+ifdef TEST_SHORT
+GOTEST_FLAGS += -short
+endif
+
+ifdef RUN
+GOTEST_FLAGS += -run $(RUN)
+endif
+
+TEST_PACKAGES ?= ./...
+
 test:
-	$(GIT_FLAGS) gotestsum --format standard-quiet $(GOTESTSUM_RETRY_FLAGS) --packages="./..." -- -v -short -count=1 $(if $(RUN),-run $(RUN))
+	$(GIT_FLAGS) gotestsum --format standard-quiet $(GOTESTSUM_RETRY_FLAGS) --packages="$(TEST_PACKAGES)" -- $(GOTEST_FLAGS)
 .PHONY: test
 
 test-cli:
-	$(GIT_FLAGS) gotestsum --format standard-quiet $(GOTESTSUM_RETRY_FLAGS) --packages="./cli/..." -- -v -short -count=1
+	$(MAKE) test TEST_PACKAGES="./cli..."
 .PHONY: test-cli
 
 # sqlc-cloud-is-setup will fail if no SQLc auth token is set. Use this as a
@@ -923,7 +1018,7 @@ sqlc-vet: test-postgres-docker
 test-postgres: test-postgres-docker
 	# The postgres test is prone to failure, so we limit parallelism for
 	# more consistent execution.
-	$(GIT_FLAGS)  DB=ci gotestsum \
+	$(GIT_FLAGS)  gotestsum \
 		--junitfile="gotests.xml" \
 		--jsonfile="gotests.json" \
 		$(GOTESTSUM_RETRY_FLAGS) \
@@ -949,12 +1044,12 @@ test-postgres-docker:
 	docker rm -f test-postgres-docker-${POSTGRES_VERSION} || true
 
 	# Try pulling up to three times to avoid CI flakes.
-	docker pull gcr.io/coder-dev-1/postgres:${POSTGRES_VERSION} || {
+	docker pull ${POSTGRES_IMAGE} || {
 		retries=2
 		for try in $(seq 1 ${retries}); do
 			echo "Failed to pull image, retrying (${try}/${retries})..."
 			sleep 1
-			if docker pull gcr.io/coder-dev-1/postgres:${POSTGRES_VERSION}; then
+			if docker pull ${POSTGRES_IMAGE}; then
 				break
 			fi
 		done
@@ -982,7 +1077,7 @@ test-postgres-docker:
 		--restart no \
 		--detach \
 		--memory 16GB \
-		gcr.io/coder-dev-1/postgres:${POSTGRES_VERSION} \
+		${POSTGRES_IMAGE} \
 		-c shared_buffers=2GB \
 		-c effective_cache_size=1GB \
 		-c work_mem=8MB \

README.md

@@ -109,9 +109,10 @@ We are always working on new integrations. Please feel free to open an issue and
 ### Official
 
 - [**VS Code Extension**](https://marketplace.visualstudio.com/items?itemName=coder.coder-remote): Open any Coder workspace in VS Code with a single click
-- [**JetBrains Gateway Extension**](https://plugins.jetbrains.com/plugin/19620-coder): Open any Coder workspace in JetBrains Gateway with a single click
+- [**JetBrains Toolbox Plugin**](https://plugins.jetbrains.com/plugin/26968-coder): Open any Coder workspace from JetBrains Toolbox with a single click
+- [**JetBrains Gateway Plugin**](https://plugins.jetbrains.com/plugin/19620-coder): Open any Coder workspace in JetBrains Gateway with a single click
 - [**Dev Container Builder**](https://github.com/coder/envbuilder): Build development environments using `devcontainer.json` on Docker, Kubernetes, and OpenShift
-- [**Module Registry**](https://registry.coder.com): Extend development environments with common use-cases
+- [**Coder Registry**](https://registry.coder.com): Build and extend development environments with common use-cases
 - [**Kubernetes Log Stream**](https://github.com/coder/coder-logstream-kube): Stream Kubernetes Pod events to the Coder startup logs
 - [**Self-Hosted VS Code Extension Marketplace**](https://github.com/coder/code-marketplace): A private extension marketplace that works in restricted or airgapped networks integrating with [code-server](https://github.com/coder/code-server).
 - [**Setup Coder**](https://github.com/marketplace/actions/setup-coder): An action to setup coder CLI in GitHub workflows.

agent/agent.go

@@ -89,16 +89,16 @@ type Options struct {
 	ServiceBannerRefreshInterval time.Duration
 	BlockFileTransfer            bool
 	Execer                       agentexec.Execer
-
-	ExperimentalDevcontainersEnabled bool
-	ContainerAPIOptions              []agentcontainers.Option // Enable ExperimentalDevcontainersEnabled for these to be effective.
+	Devcontainers                bool
+	DevcontainerAPIOptions       []agentcontainers.Option // Enable Devcontainers for these to be effective.
+	Clock                        quartz.Clock
 }
 
 type Client interface {
-	ConnectRPC25(ctx context.Context) (
-		proto.DRPCAgentClient25, tailnetproto.DRPCTailnetClient25, error,
+	ConnectRPC26(ctx context.Context) (
+		proto.DRPCAgentClient26, tailnetproto.DRPCTailnetClient26, error,
 	)
-	RewriteDERPMap(derpMap *tailcfg.DERPMap)
+	tailnet.DERPMapRewriter
 }
 
 type Agent interface {
@@ -145,6 +145,9 @@ func New(options Options) Agent {
 	if options.PortCacheDuration == 0 {
 		options.PortCacheDuration = 1 * time.Second
 	}
+	if options.Clock == nil {
+		options.Clock = quartz.NewReal()
+	}
 
 	prometheusRegistry := options.PrometheusRegistry
 	if prometheusRegistry == nil {
@@ -158,6 +161,7 @@ func New(options Options) Agent {
 	hardCtx, hardCancel := context.WithCancel(context.Background())
 	gracefulCtx, gracefulCancel := context.WithCancel(hardCtx)
 	a := &agent{
+		clock:                              options.Clock,
 		tailnetListenPort:                  options.TailnetListenPort,
 		reconnectingPTYTimeout:             options.ReconnectingPTYTimeout,
 		logger:                             options.Logger,
@@ -190,8 +194,8 @@ func New(options Options) Agent {
 		metrics:            newAgentMetrics(prometheusRegistry),
 		execer:             options.Execer,
 
-		experimentalDevcontainersEnabled: options.ExperimentalDevcontainersEnabled,
-		containerAPIOptions:              options.ContainerAPIOptions,
+		devcontainers:       options.Devcontainers,
+		containerAPIOptions: options.DevcontainerAPIOptions,
 	}
 	// Initially, we have a closed channel, reflecting the fact that we are not initially connected.
 	// Each time we connect we replace the channel (while holding the closeMutex) with a new one
@@ -205,6 +209,7 @@ func New(options Options) Agent {
 }
 
 type agent struct {
+	clock             quartz.Clock
 	logger            slog.Logger
 	client            Client
 	exchangeToken     func(ctx context.Context) (string, error)
@@ -272,9 +277,9 @@ type agent struct {
 	metrics *agentMetrics
 	execer  agentexec.Execer
 
-	experimentalDevcontainersEnabled bool
-	containerAPIOptions              []agentcontainers.Option
-	containerAPI                     atomic.Pointer[agentcontainers.API] // Set by apiHandler.
+	devcontainers       bool
+	containerAPIOptions []agentcontainers.Option
+	containerAPI        *agentcontainers.API
 }
 
 func (a *agent) TailnetConn() *tailnet.Conn {
@@ -311,7 +316,7 @@ func (a *agent) init() {
 			return a.reportConnection(id, connectionType, ip)
 		},
 
-		ExperimentalDevContainersEnabled: a.experimentalDevcontainersEnabled,
+		ExperimentalContainers: a.devcontainers,
 	})
 	if err != nil {
 		panic(err)
@@ -331,6 +336,17 @@ func (a *agent) init() {
 	// will not report anywhere.
 	a.scriptRunner.RegisterMetrics(a.prometheusRegistry)
 
+	containerAPIOpts := []agentcontainers.Option{
+		agentcontainers.WithExecer(a.execer),
+		agentcontainers.WithCommandEnv(a.sshServer.CommandEnv),
+		agentcontainers.WithScriptLogger(func(logSourceID uuid.UUID) agentcontainers.ScriptLogger {
+			return a.logSender.GetScriptLogger(logSourceID)
+		}),
+	}
+	containerAPIOpts = append(containerAPIOpts, a.containerAPIOptions...)
+
+	a.containerAPI = agentcontainers.NewAPI(a.logger.Named("containers"), containerAPIOpts...)
+
 	a.reconnectingPTYServer = reconnectingpty.NewServer(
 		a.logger.Named("reconnecting-pty"),
 		a.sshServer,
@@ -340,7 +356,7 @@ func (a *agent) init() {
 		a.metrics.connectionsTotal, a.metrics.reconnectingPTYErrors,
 		a.reconnectingPTYTimeout,
 		func(s *reconnectingpty.Server) {
-			s.ExperimentalDevcontainersEnabled = a.experimentalDevcontainersEnabled
+			s.ExperimentalContainers = a.devcontainers
 		},
 	)
 	go a.runLoop()
@@ -456,7 +472,7 @@ func (t *trySingleflight) Do(key string, fn func()) {
 	fn()
 }
 
-func (a *agent) reportMetadata(ctx context.Context, aAPI proto.DRPCAgentClient24) error {
+func (a *agent) reportMetadata(ctx context.Context, aAPI proto.DRPCAgentClient26) error {
 	tickerDone := make(chan struct{})
 	collectDone := make(chan struct{})
 	ctx, cancel := context.WithCancel(ctx)
@@ -547,7 +563,6 @@ func (a *agent) reportMetadata(ctx context.Context, aAPI proto.DRPCAgentClient24
 			// channel to synchronize the results and avoid both messy
 			// mutex logic and overloading the API.
 			for _, md := range manifest.Metadata {
-				md := md
 				// We send the result to the channel in the goroutine to avoid
 				// sending the same result multiple times. So, we don't care about
 				// the return values.
@@ -672,7 +687,7 @@ func (a *agent) reportMetadata(ctx context.Context, aAPI proto.DRPCAgentClient24
 
 // reportLifecycle reports the current lifecycle state once. All state
 // changes are reported in order.
-func (a *agent) reportLifecycle(ctx context.Context, aAPI proto.DRPCAgentClient24) error {
+func (a *agent) reportLifecycle(ctx context.Context, aAPI proto.DRPCAgentClient26) error {
 	for {
 		select {
 		case <-a.lifecycleUpdate:
@@ -752,7 +767,7 @@ func (a *agent) setLifecycle(state codersdk.WorkspaceAgentLifecycle) {
 }
 
 // reportConnectionsLoop reports connections to the agent for auditing.
-func (a *agent) reportConnectionsLoop(ctx context.Context, aAPI proto.DRPCAgentClient24) error {
+func (a *agent) reportConnectionsLoop(ctx context.Context, aAPI proto.DRPCAgentClient26) error {
 	for {
 		select {
 		case <-a.reportConnectionsUpdate:
@@ -775,11 +790,15 @@ func (a *agent) reportConnectionsLoop(ctx context.Context, aAPI proto.DRPCAgentC
 			logger.Debug(ctx, "reporting connection")
 			_, err := aAPI.ReportConnection(ctx, payload)
 			if err != nil {
-				return xerrors.Errorf("failed to report connection: %w", err)
+				// Do not fail the loop if we fail to report a connection, just
+				// log a warning.
+				// Related to https://github.com/coder/coder/issues/20194
+				logger.Warn(ctx, "failed to report connection to server", slog.Error(err))
+				// no continue here, we still need to remove it from the slice
+			} else {
+				logger.Debug(ctx, "successfully reported connection")
 			}
 
-			logger.Debug(ctx, "successfully reported connection")
-
 			// Remove the payload we sent.
 			a.reportConnectionsMu.Lock()
 			a.reportConnections[0] = nil // Release the pointer from the underlying array.
@@ -810,6 +829,13 @@ func (a *agent) reportConnection(id uuid.UUID, connectionType proto.Connection_T
 		ip = host
 	}
 
+	// If the IP is "localhost" (which it can be in some cases), set it to
+	// 127.0.0.1 instead.
+	// Related to https://github.com/coder/coder/issues/20194
+	if ip == "localhost" {
+		ip = "127.0.0.1"
+	}
+
 	a.reportConnectionsMu.Lock()
 	defer a.reportConnectionsMu.Unlock()
 
@@ -872,7 +898,7 @@ func (a *agent) reportConnection(id uuid.UUID, connectionType proto.Connection_T
 // fetchServiceBannerLoop fetches the service banner on an interval.  It will
 // not be fetched immediately; the expectation is that it is primed elsewhere
 // (and must be done before the session actually starts).
-func (a *agent) fetchServiceBannerLoop(ctx context.Context, aAPI proto.DRPCAgentClient24) error {
+func (a *agent) fetchServiceBannerLoop(ctx context.Context, aAPI proto.DRPCAgentClient26) error {
 	ticker := time.NewTicker(a.announcementBannersRefreshInterval)
 	defer ticker.Stop()
 	for {
@@ -908,7 +934,7 @@ func (a *agent) run() (retErr error) {
 	a.sessionToken.Store(&sessionToken)
 
 	// ConnectRPC returns the dRPC connection we use for the Agent and Tailnet v2+ APIs
-	aAPI, tAPI, err := a.client.ConnectRPC25(a.hardCtx)
+	aAPI, tAPI, err := a.client.ConnectRPC26(a.hardCtx)
 	if err != nil {
 		return err
 	}
@@ -925,7 +951,7 @@ func (a *agent) run() (retErr error) {
 	connMan := newAPIConnRoutineManager(a.gracefulCtx, a.hardCtx, a.logger, aAPI, tAPI)
 
 	connMan.startAgentAPI("init notification banners", gracefulShutdownBehaviorStop,
-		func(ctx context.Context, aAPI proto.DRPCAgentClient24) error {
+		func(ctx context.Context, aAPI proto.DRPCAgentClient26) error {
 			bannersProto, err := aAPI.GetAnnouncementBanners(ctx, &proto.GetAnnouncementBannersRequest{})
 			if err != nil {
 				return xerrors.Errorf("fetch service banner: %w", err)
@@ -942,7 +968,7 @@ func (a *agent) run() (retErr error) {
 	// sending logs gets gracefulShutdownBehaviorRemain because we want to send logs generated by
 	// shutdown scripts.
 	connMan.startAgentAPI("send logs", gracefulShutdownBehaviorRemain,
-		func(ctx context.Context, aAPI proto.DRPCAgentClient24) error {
+		func(ctx context.Context, aAPI proto.DRPCAgentClient26) error {
 			err := a.logSender.SendLoop(ctx, aAPI)
 			if xerrors.Is(err, agentsdk.ErrLogLimitExceeded) {
 				// we don't want this error to tear down the API connection and propagate to the
@@ -961,7 +987,7 @@ func (a *agent) run() (retErr error) {
 	connMan.startAgentAPI("report metadata", gracefulShutdownBehaviorStop, a.reportMetadata)
 
 	// resources monitor can cease as soon as we start gracefully shutting down.
-	connMan.startAgentAPI("resources monitor", gracefulShutdownBehaviorStop, func(ctx context.Context, aAPI proto.DRPCAgentClient24) error {
+	connMan.startAgentAPI("resources monitor", gracefulShutdownBehaviorStop, func(ctx context.Context, aAPI proto.DRPCAgentClient26) error {
 		logger := a.logger.Named("resources_monitor")
 		clk := quartz.NewReal()
 		config, err := aAPI.GetResourcesMonitoringConfiguration(ctx, &proto.GetResourcesMonitoringConfigurationRequest{})
@@ -1008,7 +1034,7 @@ func (a *agent) run() (retErr error) {
 	connMan.startAgentAPI("handle manifest", gracefulShutdownBehaviorStop, a.handleManifest(manifestOK))
 
 	connMan.startAgentAPI("app health reporter", gracefulShutdownBehaviorStop,
-		func(ctx context.Context, aAPI proto.DRPCAgentClient24) error {
+		func(ctx context.Context, aAPI proto.DRPCAgentClient26) error {
 			if err := manifestOK.wait(ctx); err != nil {
 				return xerrors.Errorf("no manifest: %w", err)
 			}
@@ -1041,7 +1067,7 @@ func (a *agent) run() (retErr error) {
 
 	connMan.startAgentAPI("fetch service banner loop", gracefulShutdownBehaviorStop, a.fetchServiceBannerLoop)
 
-	connMan.startAgentAPI("stats report loop", gracefulShutdownBehaviorStop, func(ctx context.Context, aAPI proto.DRPCAgentClient24) error {
+	connMan.startAgentAPI("stats report loop", gracefulShutdownBehaviorStop, func(ctx context.Context, aAPI proto.DRPCAgentClient26) error {
 		if err := networkOK.wait(ctx); err != nil {
 			return xerrors.Errorf("no network: %w", err)
 		}
@@ -1056,8 +1082,8 @@ func (a *agent) run() (retErr error) {
 }
 
 // handleManifest returns a function that fetches and processes the manifest
-func (a *agent) handleManifest(manifestOK *checkpoint) func(ctx context.Context, aAPI proto.DRPCAgentClient24) error {
-	return func(ctx context.Context, aAPI proto.DRPCAgentClient24) error {
+func (a *agent) handleManifest(manifestOK *checkpoint) func(ctx context.Context, aAPI proto.DRPCAgentClient26) error {
+	return func(ctx context.Context, aAPI proto.DRPCAgentClient26) error {
 		var (
 			sentResult = false
 			err        error
@@ -1071,7 +1097,7 @@ func (a *agent) handleManifest(manifestOK *checkpoint) func(ctx context.Context,
 		if err != nil {
 			return xerrors.Errorf("fetch metadata: %w", err)
 		}
-		a.logger.Info(ctx, "fetched manifest", slog.F("manifest", mp))
+		a.logger.Info(ctx, "fetched manifest")
 		manifest, err := agentsdk.ManifestFromProto(mp)
 		if err != nil {
 			a.logger.Critical(ctx, "failed to convert manifest", slog.F("manifest", mp), slog.Error(err))
@@ -1080,6 +1106,18 @@ func (a *agent) handleManifest(manifestOK *checkpoint) func(ctx context.Context,
 		if manifest.AgentID == uuid.Nil {
 			return xerrors.New("nil agentID returned by manifest")
 		}
+		if manifest.ParentID != uuid.Nil {
+			// This is a sub agent, disable all the features that should not
+			// be used by sub agents.
+			a.logger.Debug(ctx, "sub agent detected, disabling features",
+				slog.F("parent_id", manifest.ParentID),
+				slog.F("agent_id", manifest.AgentID),
+			)
+			if a.devcontainers {
+				a.logger.Info(ctx, "devcontainers are not supported on sub agents, disabling feature")
+				a.devcontainers = false
+			}
+		}
 		a.client.RewriteDERPMap(manifest.DERPMap)
 
 		// Expand the directory and send it back to coderd so external
@@ -1130,17 +1168,27 @@ func (a *agent) handleManifest(manifestOK *checkpoint) func(ctx context.Context,
 			}
 
 			var (
-				scripts          = manifest.Scripts
-				scriptRunnerOpts []agentscripts.InitOption
+				scripts             = manifest.Scripts
+				devcontainerScripts map[uuid.UUID]codersdk.WorkspaceAgentScript
 			)
-			if a.experimentalDevcontainersEnabled {
-				var dcScripts []codersdk.WorkspaceAgentScript
-				scripts, dcScripts = agentcontainers.ExtractAndInitializeDevcontainerScripts(manifest.Devcontainers, scripts)
-				// See ExtractAndInitializeDevcontainerScripts for motivation
-				// behind running dcScripts as post start scripts.
-				scriptRunnerOpts = append(scriptRunnerOpts, agentscripts.WithPostStartScripts(dcScripts...))
+			if a.devcontainers {
+				// Init the container API with the manifest and client so that
+				// we can start accepting requests. The final start of the API
+				// happens after the startup scripts have been executed to
+				// ensure the presence of required tools. This means we can
+				// return existing devcontainers but actual container detection
+				// and creation will be deferred.
+				a.containerAPI.Init(
+					agentcontainers.WithManifestInfo(manifest.OwnerName, manifest.WorkspaceName, manifest.AgentName, manifest.Directory),
+					agentcontainers.WithDevcontainers(manifest.Devcontainers, manifest.Scripts),
+					agentcontainers.WithSubAgentClient(agentcontainers.NewSubAgentClientFromAPI(a.logger, aAPI)),
+				)
+
+				// Since devcontainer are enabled, remove devcontainer scripts
+				// from the main scripts list to avoid showing an error.
+				scripts, devcontainerScripts = agentcontainers.ExtractDevcontainerScripts(manifest.Devcontainers, scripts)
 			}
-			err = a.scriptRunner.Init(scripts, aAPI.ScriptCompleted, scriptRunnerOpts...)
+			err = a.scriptRunner.Init(scripts, aAPI.ScriptCompleted)
 			if err != nil {
 				return xerrors.Errorf("init script runner: %w", err)
 			}
@@ -1157,7 +1205,18 @@ func (a *agent) handleManifest(manifestOK *checkpoint) func(ctx context.Context,
 				// finished (both start and post start). For instance, an
 				// autostarted devcontainer will be included in this time.
 				err := a.scriptRunner.Execute(a.gracefulCtx, agentscripts.ExecuteStartScripts)
-				err = errors.Join(err, a.scriptRunner.Execute(a.gracefulCtx, agentscripts.ExecutePostStartScripts))
+
+				if a.devcontainers {
+					// Start the container API after the startup scripts have
+					// been executed to ensure that the required tools can be
+					// installed.
+					a.containerAPI.Start()
+					for _, dc := range manifest.Devcontainers {
+						cErr := a.createDevcontainer(ctx, aAPI, dc, devcontainerScripts[dc.ID])
+						err = errors.Join(err, cErr)
+					}
+				}
+
 				dur := time.Since(start).Seconds()
 				if err != nil {
 					a.logger.Warn(ctx, "startup script(s) failed", slog.Error(err))
@@ -1185,10 +1244,42 @@ func (a *agent) handleManifest(manifestOK *checkpoint) func(ctx context.Context,
 	}
 }
 
+func (a *agent) createDevcontainer(
+	ctx context.Context,
+	aAPI proto.DRPCAgentClient26,
+	dc codersdk.WorkspaceAgentDevcontainer,
+	script codersdk.WorkspaceAgentScript,
+) (err error) {
+	var (
+		exitCode  = int32(0)
+		startTime = a.clock.Now()
+		status    = proto.Timing_OK
+	)
+	if err = a.containerAPI.CreateDevcontainer(dc.WorkspaceFolder, dc.ConfigPath); err != nil {
+		exitCode = 1
+		status = proto.Timing_EXIT_FAILURE
+	}
+	endTime := a.clock.Now()
+
+	if _, scriptErr := aAPI.ScriptCompleted(ctx, &proto.WorkspaceAgentScriptCompletedRequest{
+		Timing: &proto.Timing{
+			ScriptId: script.ID[:],
+			Start:    timestamppb.New(startTime),
+			End:      timestamppb.New(endTime),
+			ExitCode: exitCode,
+			Stage:    proto.Timing_START,
+			Status:   status,
+		},
+	}); scriptErr != nil {
+		a.logger.Warn(ctx, "reporting script completed failed", slog.Error(scriptErr))
+	}
+	return err
+}
+
 // createOrUpdateNetwork waits for the manifest to be set using manifestOK, then creates or updates
 // the tailnet using the information in the manifest
-func (a *agent) createOrUpdateNetwork(manifestOK, networkOK *checkpoint) func(context.Context, proto.DRPCAgentClient24) error {
-	return func(ctx context.Context, _ proto.DRPCAgentClient24) (retErr error) {
+func (a *agent) createOrUpdateNetwork(manifestOK, networkOK *checkpoint) func(context.Context, proto.DRPCAgentClient26) error {
+	return func(ctx context.Context, aAPI proto.DRPCAgentClient26) (retErr error) {
 		if err := manifestOK.wait(ctx); err != nil {
 			return xerrors.Errorf("no manifest: %w", err)
 		}
@@ -1240,6 +1331,12 @@ func (a *agent) createOrUpdateNetwork(manifestOK, networkOK *checkpoint) func(co
 			network.SetDERPMap(manifest.DERPMap)
 			network.SetDERPForceWebSockets(manifest.DERPForceWebSockets)
 			network.SetBlockEndpoints(manifest.DisableDirectConnections)
+
+			// Update the subagent client if the container API is available.
+			if a.containerAPI != nil {
+				client := agentcontainers.NewSubAgentClientFromAPI(a.logger, aAPI)
+				a.containerAPI.UpdateSubAgentClient(client)
+			}
 		}
 		return nil
 	}
@@ -1270,6 +1367,7 @@ func (a *agent) updateCommandEnv(current []string) (updated []string, err error)
 		"CODER":                      "true",
 		"CODER_WORKSPACE_NAME":       manifest.WorkspaceName,
 		"CODER_WORKSPACE_AGENT_NAME": manifest.AgentName,
+		"CODER_WORKSPACE_OWNER_NAME": manifest.OwnerName,
 
 		// Specific Coder subcommands require the agent token exposed!
 		"CODER_AGENT_TOKEN": *a.sessionToken.Load(),
@@ -1487,10 +1585,7 @@ func (a *agent) createTailnet(
 	}()
 	if err = a.trackGoroutine(func() {
 		defer apiListener.Close()
-		apiHandler, closeAPIHAndler := a.apiHandler()
-		defer func() {
-			_ = closeAPIHAndler()
-		}()
+		apiHandler := a.apiHandler()
 		server := &http.Server{
 			BaseContext:       func(net.Listener) context.Context { return ctx },
 			Handler:           apiHandler,
@@ -1504,7 +1599,6 @@ func (a *agent) createTailnet(
 			case <-ctx.Done():
 			case <-a.hardCtx.Done():
 			}
-			_ = closeAPIHAndler()
 			_ = server.Close()
 		}()
 
@@ -1843,6 +1937,10 @@ func (a *agent) Close() error {
 		a.logger.Error(a.hardCtx, "script runner close", slog.Error(err))
 	}
 
+	if err := a.containerAPI.Close(); err != nil {
+		a.logger.Error(a.hardCtx, "container API close", slog.Error(err))
+	}
+
 	// Wait for the graceful shutdown to complete, but don't wait forever so
 	// that we don't break user expectations.
 	go func() {
@@ -1960,7 +2058,7 @@ const (
 
 type apiConnRoutineManager struct {
 	logger    slog.Logger
-	aAPI      proto.DRPCAgentClient24
+	aAPI      proto.DRPCAgentClient26
 	tAPI      tailnetproto.DRPCTailnetClient24
 	eg        *errgroup.Group
 	stopCtx   context.Context
@@ -1969,7 +2067,7 @@ type apiConnRoutineManager struct {
 
 func newAPIConnRoutineManager(
 	gracefulCtx, hardCtx context.Context, logger slog.Logger,
-	aAPI proto.DRPCAgentClient24, tAPI tailnetproto.DRPCTailnetClient24,
+	aAPI proto.DRPCAgentClient26, tAPI tailnetproto.DRPCTailnetClient24,
 ) *apiConnRoutineManager {
 	// routines that remain in operation during graceful shutdown use the remainCtx.  They'll still
 	// exit if the errgroup hits an error, which usually means a problem with the conn.
@@ -2002,7 +2100,7 @@ func newAPIConnRoutineManager(
 // but for Tailnet.
 func (a *apiConnRoutineManager) startAgentAPI(
 	name string, behavior gracefulShutdownBehavior,
-	f func(context.Context, proto.DRPCAgentClient24) error,
+	f func(context.Context, proto.DRPCAgentClient26) error,
 ) {
 	logger := a.logger.With(slog.F("name", name))
 	var ctx context.Context

agent/agent_test.go

@@ -48,6 +48,7 @@ import (
 	"cdr.dev/slog/sloggers/slogtest"
 
 	"github.com/coder/coder/v2/agent"
+	"github.com/coder/coder/v2/agent/agentcontainers"
 	"github.com/coder/coder/v2/agent/agentssh"
 	"github.com/coder/coder/v2/agent/agenttest"
 	"github.com/coder/coder/v2/agent/proto"
@@ -60,9 +61,16 @@ import (
 	"github.com/coder/coder/v2/tailnet"
 	"github.com/coder/coder/v2/tailnet/tailnettest"
 	"github.com/coder/coder/v2/testutil"
+	"github.com/coder/quartz"
 )
 
 func TestMain(m *testing.M) {
+	if os.Getenv("CODER_TEST_RUN_SUB_AGENT_MAIN") == "1" {
+		// If we're running as a subagent, we don't want to run the main tests.
+		// Instead, we just run the subagent tests.
+		exit := runSubAgentMain()
+		os.Exit(exit)
+	}
 	goleak.VerifyTestMain(m, testutil.GoleakOptions...)
 }
 
@@ -122,7 +130,6 @@ func TestAgent_Stats_SSH(t *testing.T) {
 	t.Parallel()
 
 	for _, port := range sshPorts {
-		port := port
 		t.Run(fmt.Sprintf("(:%d)", port), func(t *testing.T) {
 			t.Parallel()
 
@@ -334,7 +341,6 @@ func TestAgent_SessionExec(t *testing.T) {
 	t.Parallel()
 
 	for _, port := range sshPorts {
-		port := port
 		t.Run(fmt.Sprintf("(:%d)", port), func(t *testing.T) {
 			t.Parallel()
 
@@ -450,8 +456,6 @@ func TestAgent_GitSSH(t *testing.T) {
 
 func TestAgent_SessionTTYShell(t *testing.T) {
 	t.Parallel()
-	ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
-	t.Cleanup(cancel)
 	if runtime.GOOS == "windows" {
 		// This might be our implementation, or ConPTY itself.
 		// It's difficult to find extensive tests for it, so
@@ -460,9 +464,9 @@ func TestAgent_SessionTTYShell(t *testing.T) {
 	}
 
 	for _, port := range sshPorts {
-		port := port
 		t.Run(fmt.Sprintf("(%d)", port), func(t *testing.T) {
 			t.Parallel()
+			ctx := testutil.Context(t, testutil.WaitShort)
 
 			session := setupSSHSessionOnPort(t, agentsdk.Manifest{}, codersdk.ServiceBannerConfig{}, nil, port)
 			command := "sh"
@@ -603,7 +607,6 @@ func TestAgent_Session_TTY_MOTD(t *testing.T) {
 	}
 
 	for _, test := range tests {
-		test := test
 		t.Run(test.name, func(t *testing.T) {
 			t.Parallel()
 			session := setupSSHSession(t, test.manifest, test.banner, func(fs afero.Fs) {
@@ -680,8 +683,6 @@ func TestAgent_Session_TTY_MOTD_Update(t *testing.T) {
 
 	//nolint:paralleltest // These tests need to swap the banner func.
 	for _, port := range sshPorts {
-		port := port
-
 		sshClient, err := conn.SSHClientOnPort(ctx, port)
 		require.NoError(t, err)
 		t.Cleanup(func() {
@@ -689,7 +690,6 @@ func TestAgent_Session_TTY_MOTD_Update(t *testing.T) {
 		})
 
 		for i, test := range tests {
-			test := test
 			t.Run(fmt.Sprintf("(:%d)/%d", port, i), func(t *testing.T) {
 				// Set new banner func and wait for the agent to call it to update the
 				// banner.
@@ -1201,8 +1201,7 @@ func TestAgent_EnvironmentVariableExpansion(t *testing.T) {
 func TestAgent_CoderEnvVars(t *testing.T) {
 	t.Parallel()
 
-	for _, key := range []string{"CODER", "CODER_WORKSPACE_NAME", "CODER_WORKSPACE_AGENT_NAME"} {
-		key := key
+	for _, key := range []string{"CODER", "CODER_WORKSPACE_NAME", "CODER_WORKSPACE_OWNER_NAME", "CODER_WORKSPACE_AGENT_NAME"} {
 		t.Run(key, func(t *testing.T) {
 			t.Parallel()
 
@@ -1225,7 +1224,6 @@ func TestAgent_SSHConnectionEnvVars(t *testing.T) {
 	// For some reason this test produces a TTY locally and a non-TTY in CI
 	// so we don't test for the absence of SSH_TTY.
 	for _, key := range []string{"SSH_CONNECTION", "SSH_CLIENT"} {
-		key := key
 		t.Run(key, func(t *testing.T) {
 			t.Parallel()
 
@@ -1268,7 +1266,6 @@ func TestAgent_SSHConnectionLoginVars(t *testing.T) {
 		},
 	}
 	for _, tt := range tests {
-		tt := tt
 		t.Run(tt.key, func(t *testing.T) {
 			t.Parallel()
 
@@ -1788,7 +1785,6 @@ func TestAgent_ReconnectingPTY(t *testing.T) {
 	t.Setenv("LANG", "C")
 
 	for _, backendType := range backends {
-		backendType := backendType
 		t.Run(backendType, func(t *testing.T) {
 			if backendType == "Screen" {
 				if runtime.GOOS != "linux" {
@@ -1930,6 +1926,9 @@ func TestAgent_ReconnectingPTYContainer(t *testing.T) {
 	if os.Getenv("CODER_TEST_USE_DOCKER") != "1" {
 		t.Skip("Set CODER_TEST_USE_DOCKER=1 to run this test")
 	}
+	if _, err := exec.LookPath("devcontainer"); err != nil {
+		t.Skip("This test requires the devcontainer CLI: npm install -g @devcontainers/cli")
+	}
 
 	pool, err := dockertest.NewPool("")
 	require.NoError(t, err, "Could not connect to docker")
@@ -1954,7 +1953,10 @@ func TestAgent_ReconnectingPTYContainer(t *testing.T) {
 
 	// nolint: dogsled
 	conn, _, _, _, _ := setupAgent(t, agentsdk.Manifest{}, 0, func(_ *agenttest.Client, o *agent.Options) {
-		o.ExperimentalDevcontainersEnabled = true
+		o.Devcontainers = true
+		o.DevcontainerAPIOptions = append(o.DevcontainerAPIOptions,
+			agentcontainers.WithContainerLabelIncludeFilter("this.label.does.not.exist.ignore.devcontainers", "true"),
+		)
 	})
 	ctx := testutil.Context(t, testutil.WaitLong)
 	ac, err := conn.ReconnectingPTY(ctx, uuid.New(), 80, 80, "/bin/sh", func(arp *workspacesdk.AgentReconnectingPTYInit) {
@@ -1986,6 +1988,60 @@ func TestAgent_ReconnectingPTYContainer(t *testing.T) {
 	require.ErrorIs(t, tr.ReadUntil(ctx, nil), io.EOF)
 }
 
+type subAgentRequestPayload struct {
+	Token     string `json:"token"`
+	Directory string `json:"directory"`
+}
+
+// runSubAgentMain is the main function for the sub-agent that connects
+// to the control plane. It reads the CODER_AGENT_URL and
+// CODER_AGENT_TOKEN environment variables, sends the token, and exits
+// with a status code based on the response.
+func runSubAgentMain() int {
+	url := os.Getenv("CODER_AGENT_URL")
+	token := os.Getenv("CODER_AGENT_TOKEN")
+	if url == "" || token == "" {
+		_, _ = fmt.Fprintln(os.Stderr, "CODER_AGENT_URL and CODER_AGENT_TOKEN must be set")
+		return 10
+	}
+
+	dir, err := os.Getwd()
+	if err != nil {
+		_, _ = fmt.Fprintf(os.Stderr, "failed to get current working directory: %v\n", err)
+		return 1
+	}
+	payload := subAgentRequestPayload{
+		Token:     token,
+		Directory: dir,
+	}
+	b, err := json.Marshal(payload)
+	if err != nil {
+		_, _ = fmt.Fprintf(os.Stderr, "failed to marshal payload: %v\n", err)
+		return 1
+	}
+
+	req, err := http.NewRequest("POST", url, bytes.NewReader(b))
+	if err != nil {
+		_, _ = fmt.Fprintf(os.Stderr, "failed to create request: %v\n", err)
+		return 1
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
+	defer cancel()
+	req = req.WithContext(ctx)
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		_, _ = fmt.Fprintf(os.Stderr, "agent connection failed: %v\n", err)
+		return 11
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		_, _ = fmt.Fprintf(os.Stderr, "agent exiting with non-zero exit code %d\n", resp.StatusCode)
+		return 12
+	}
+	_, _ = fmt.Println("sub-agent connected successfully")
+	return 0
+}
+
 // This tests end-to-end functionality of auto-starting a devcontainer.
 // It runs "devcontainer up" which creates a real Docker container. As
 // such, it does not run by default in CI.
@@ -1999,6 +2055,60 @@ func TestAgent_DevcontainerAutostart(t *testing.T) {
 	if os.Getenv("CODER_TEST_USE_DOCKER") != "1" {
 		t.Skip("Set CODER_TEST_USE_DOCKER=1 to run this test")
 	}
+	if _, err := exec.LookPath("devcontainer"); err != nil {
+		t.Skip("This test requires the devcontainer CLI: npm install -g @devcontainers/cli")
+	}
+
+	// This HTTP handler handles requests from runSubAgentMain which
+	// acts as a fake sub-agent. We want to verify that the sub-agent
+	// connects and sends its token. We use a channel to signal
+	// that the sub-agent has connected successfully and then we wait
+	// until we receive another signal to return from the handler. This
+	// keeps the agent "alive" for as long as we want.
+	subAgentConnected := make(chan subAgentRequestPayload, 1)
+	subAgentReady := make(chan struct{}, 1)
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.Method == http.MethodGet && strings.HasPrefix(r.URL.Path, "/api/v2/workspaceagents/me/") {
+			return
+		}
+
+		t.Logf("Sub-agent request received: %s %s", r.Method, r.URL.Path)
+
+		if r.Method != http.MethodPost {
+			http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
+			return
+		}
+
+		// Read the token from the request body.
+		var payload subAgentRequestPayload
+		if err := json.NewDecoder(r.Body).Decode(&payload); err != nil {
+			http.Error(w, "Failed to read token", http.StatusBadRequest)
+			t.Logf("Failed to read token: %v", err)
+			return
+		}
+		defer r.Body.Close()
+
+		t.Logf("Sub-agent request payload received: %+v", payload)
+
+		// Signal that the sub-agent has connected successfully.
+		select {
+		case <-t.Context().Done():
+			t.Logf("Test context done, not processing sub-agent request")
+			return
+		case subAgentConnected <- payload:
+		}
+
+		// Wait for the signal to return from the handler.
+		select {
+		case <-t.Context().Done():
+			t.Logf("Test context done, not waiting for sub-agent ready")
+			return
+		case <-subAgentReady:
+		}
+
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer srv.Close()
 
 	pool, err := dockertest.NewPool("")
 	require.NoError(t, err, "Could not connect to docker")
@@ -2016,9 +2126,10 @@ func TestAgent_DevcontainerAutostart(t *testing.T) {
 	require.NoError(t, err, "create devcontainer directory")
 	devcontainerFile := filepath.Join(devcontainerPath, "devcontainer.json")
 	err = os.WriteFile(devcontainerFile, []byte(`{
-        "name": "mywork",
-        "image": "busybox:latest",
-        "cmd": ["sleep", "infinity"]
+		"name": "mywork",
+		"image": "ubuntu:latest",
+		"cmd": ["sleep", "infinity"],
+		"runArgs": ["--network=host", "--label=`+agentcontainers.DevcontainerIsTestRunLabel+`=true"]
     }`), 0o600)
 	require.NoError(t, err, "write devcontainer.json")
 
@@ -2043,9 +2154,25 @@ func TestAgent_DevcontainerAutostart(t *testing.T) {
 			},
 		},
 	}
+	mClock := quartz.NewMock(t)
+	mClock.Set(time.Now())
+	tickerFuncTrap := mClock.Trap().TickerFunc("agentcontainers")
+
 	//nolint:dogsled
-	conn, _, _, _, _ := setupAgent(t, manifest, 0, func(_ *agenttest.Client, o *agent.Options) {
-		o.ExperimentalDevcontainersEnabled = true
+	_, agentClient, _, _, _ := setupAgent(t, manifest, 0, func(_ *agenttest.Client, o *agent.Options) {
+		o.Devcontainers = true
+		o.DevcontainerAPIOptions = append(
+			o.DevcontainerAPIOptions,
+			// Only match this specific dev container.
+			agentcontainers.WithClock(mClock),
+			agentcontainers.WithContainerLabelIncludeFilter("devcontainer.local_folder", tempWorkspaceFolder),
+			agentcontainers.WithContainerLabelIncludeFilter(agentcontainers.DevcontainerIsTestRunLabel, "true"),
+			agentcontainers.WithSubAgentURL(srv.URL),
+			// The agent will copy "itself", but in the case of this test, the
+			// agent is actually this test binary. So we'll tell the test binary
+			// to execute the sub-agent main function via this env.
+			agentcontainers.WithSubAgentEnv("CODER_TEST_RUN_SUB_AGENT_MAIN=1"),
+		)
 	})
 
 	t.Logf("Waiting for container with label: devcontainer.local_folder=%s", tempWorkspaceFolder)
@@ -2089,32 +2216,45 @@ func TestAgent_DevcontainerAutostart(t *testing.T) {
 
 	ctx := testutil.Context(t, testutil.WaitLong)
 
-	ac, err := conn.ReconnectingPTY(ctx, uuid.New(), 80, 80, "", func(opts *workspacesdk.AgentReconnectingPTYInit) {
-		opts.Container = container.ID
-	})
-	require.NoError(t, err, "failed to create ReconnectingPTY")
-	defer ac.Close()
+	// Ensure the container update routine runs.
+	tickerFuncTrap.MustWait(ctx).MustRelease(ctx)
+	tickerFuncTrap.Close()
 
-	// Use terminal reader so we can see output in case somethin goes wrong.
-	tr := testutil.NewTerminalReader(t, ac)
+	// Since the agent does RefreshContainers, and the ticker function
+	// is set to skip instead of queue, we must advance the clock
+	// multiple times to ensure that the sub-agent is created.
+	var subAgents []*proto.SubAgent
+	for {
+		_, next := mClock.AdvanceNext()
+		next.MustWait(ctx)
 
-	require.NoError(t, tr.ReadUntil(ctx, func(line string) bool {
-		return strings.Contains(line, "#") || strings.Contains(line, "$")
-	}), "find prompt")
+		// Verify that a subagent was created.
+		subAgents = agentClient.GetSubAgents()
+		if len(subAgents) > 0 {
+			t.Logf("Found sub-agents: %d", len(subAgents))
+			break
+		}
+	}
+	require.Len(t, subAgents, 1, "expected one sub agent")
 
-	wantFileName := "file-from-devcontainer"
-	wantFile := filepath.Join(tempWorkspaceFolder, wantFileName)
+	subAgent := subAgents[0]
+	subAgentID, err := uuid.FromBytes(subAgent.GetId())
+	require.NoError(t, err, "failed to parse sub-agent ID")
+	t.Logf("Connecting to sub-agent: %s (ID: %s)", subAgent.Name, subAgentID)
 
-	require.NoError(t, json.NewEncoder(ac).Encode(workspacesdk.ReconnectingPTYRequest{
-		// NOTE(mafredri): We must use absolute path here for some reason.
-		Data: fmt.Sprintf("touch /workspaces/mywork/%s; exit\r", wantFileName),
-	}), "create file inside devcontainer")
+	gotDir, err := agentClient.GetSubAgentDirectory(subAgentID)
+	require.NoError(t, err, "failed to get sub-agent directory")
+	require.Equal(t, "/workspaces/mywork", gotDir, "sub-agent directory should match")
 
-	// Wait for the connection to close to ensure the touch was executed.
-	require.ErrorIs(t, tr.ReadUntil(ctx, nil), io.EOF)
+	subAgentToken, err := uuid.FromBytes(subAgent.GetAuthToken())
+	require.NoError(t, err, "failed to parse sub-agent token")
 
-	_, err = os.Stat(wantFile)
-	require.NoError(t, err, "file should exist outside devcontainer")
+	payload := testutil.RequireReceive(ctx, t, subAgentConnected)
+	require.Equal(t, subAgentToken.String(), payload.Token, "sub-agent token should match")
+	require.Equal(t, "/workspaces/mywork", payload.Directory, "sub-agent directory should match")
+
+	// Allow the subagent to exit.
+	close(subAgentReady)
 }
 
 // TestAgent_DevcontainerRecreate tests that RecreateDevcontainer
@@ -2148,7 +2288,8 @@ func TestAgent_DevcontainerRecreate(t *testing.T) {
 	err = os.WriteFile(devcontainerFile, []byte(`{
         "name": "mywork",
         "image": "busybox:latest",
-        "cmd": ["sleep", "infinity"]
+        "cmd": ["sleep", "infinity"],
+		"runArgs": ["--label=`+agentcontainers.DevcontainerIsTestRunLabel+`=true"]
     }`), 0o600)
 	require.NoError(t, err, "write devcontainer.json")
 
@@ -2172,7 +2313,11 @@ func TestAgent_DevcontainerRecreate(t *testing.T) {
 
 	//nolint:dogsled
 	conn, client, _, _, _ := setupAgent(t, manifest, 0, func(_ *agenttest.Client, o *agent.Options) {
-		o.ExperimentalDevcontainersEnabled = true
+		o.Devcontainers = true
+		o.DevcontainerAPIOptions = append(o.DevcontainerAPIOptions,
+			agentcontainers.WithContainerLabelIncludeFilter("devcontainer.local_folder", workspaceFolder),
+			agentcontainers.WithContainerLabelIncludeFilter(agentcontainers.DevcontainerIsTestRunLabel, "true"),
+		)
 	})
 
 	ctx := testutil.Context(t, testutil.WaitLong)
@@ -2226,7 +2371,7 @@ func TestAgent_DevcontainerRecreate(t *testing.T) {
 	// devcontainer, we do it in a goroutine so we can process logs
 	// concurrently.
 	go func(container codersdk.WorkspaceAgentContainer) {
-		_, err := conn.RecreateDevcontainer(ctx, container.ID)
+		_, err := conn.RecreateDevcontainer(ctx, devcontainerID.String())
 		assert.NoError(t, err, "recreate devcontainer should succeed")
 	}(container)
 
@@ -2284,16 +2429,250 @@ waitForOutcomeLoop:
 	}(container)
 }
 
+func TestAgent_DevcontainersDisabledForSubAgent(t *testing.T) {
+	t.Parallel()
+
+	// Create a manifest with a ParentID to make this a sub agent.
+	manifest := agentsdk.Manifest{
+		AgentID:  uuid.New(),
+		ParentID: uuid.New(),
+	}
+
+	// Setup the agent with devcontainers enabled initially.
+	//nolint:dogsled
+	conn, _, _, _, _ := setupAgent(t, manifest, 0, func(_ *agenttest.Client, o *agent.Options) {
+		o.Devcontainers = true
+	})
+
+	// Query the containers API endpoint. This should fail because
+	// devcontainers have been disabled for the sub agent.
+	ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitMedium)
+	defer cancel()
+
+	_, err := conn.ListContainers(ctx)
+	require.Error(t, err)
+
+	// Verify the error message contains the expected text.
+	require.Contains(t, err.Error(), "Dev Container feature not supported.")
+	require.Contains(t, err.Error(), "Dev Container integration inside other Dev Containers is explicitly not supported.")
+}
+
+// TestAgent_DevcontainerPrebuildClaim tests that we correctly handle
+// the claiming process for running devcontainers.
+//
+// You can run it manually as follows:
+//
+// CODER_TEST_USE_DOCKER=1 go test -count=1 ./agent -run TestAgent_DevcontainerPrebuildClaim
+//
+//nolint:paralleltest // This test sets an environment variable.
+func TestAgent_DevcontainerPrebuildClaim(t *testing.T) {
+	if os.Getenv("CODER_TEST_USE_DOCKER") != "1" {
+		t.Skip("Set CODER_TEST_USE_DOCKER=1 to run this test")
+	}
+	if _, err := exec.LookPath("devcontainer"); err != nil {
+		t.Skip("This test requires the devcontainer CLI: npm install -g @devcontainers/cli")
+	}
+
+	pool, err := dockertest.NewPool("")
+	require.NoError(t, err, "Could not connect to docker")
+
+	var (
+		ctx = testutil.Context(t, testutil.WaitShort)
+
+		devcontainerID          = uuid.New()
+		devcontainerLogSourceID = uuid.New()
+
+		workspaceFolder    = filepath.Join(t.TempDir(), "project")
+		devcontainerPath   = filepath.Join(workspaceFolder, ".devcontainer")
+		devcontainerConfig = filepath.Join(devcontainerPath, "devcontainer.json")
+	)
+
+	// Given: A devcontainer project.
+	t.Logf("Workspace folder: %s", workspaceFolder)
+
+	err = os.MkdirAll(devcontainerPath, 0o755)
+	require.NoError(t, err, "create dev container directory")
+
+	// Given: This devcontainer project specifies an app that uses the owner name and workspace name.
+	err = os.WriteFile(devcontainerConfig, []byte(`{
+	    "name": "project",
+		"image": "busybox:latest",
+	    "cmd": ["sleep", "infinity"],
+		"runArgs": ["--label=`+agentcontainers.DevcontainerIsTestRunLabel+`=true"],
+		"customizations": {
+		 	"coder": {
+				"apps": [{
+					"slug": "zed",
+					"url": "zed://ssh/${localEnv:CODER_WORKSPACE_AGENT_NAME}.${localEnv:CODER_WORKSPACE_NAME}.${localEnv:CODER_WORKSPACE_OWNER_NAME}.coder${containerWorkspaceFolder}"
+				}]
+			}
+		}
+	}`), 0o600)
+	require.NoError(t, err, "write devcontainer config")
+
+	// Given: A manifest with a prebuild username and workspace name.
+	manifest := agentsdk.Manifest{
+		OwnerName:     "prebuilds",
+		WorkspaceName: "prebuilds-xyz-123",
+
+		Devcontainers: []codersdk.WorkspaceAgentDevcontainer{
+			{ID: devcontainerID, Name: "test", WorkspaceFolder: workspaceFolder},
+		},
+		Scripts: []codersdk.WorkspaceAgentScript{
+			{ID: devcontainerID, LogSourceID: devcontainerLogSourceID},
+		},
+	}
+
+	// When: We create an agent with devcontainers enabled.
+	//nolint:dogsled
+	conn, client, _, _, _ := setupAgent(t, manifest, 0, func(_ *agenttest.Client, o *agent.Options) {
+		o.Devcontainers = true
+		o.DevcontainerAPIOptions = append(o.DevcontainerAPIOptions,
+			agentcontainers.WithContainerLabelIncludeFilter(agentcontainers.DevcontainerLocalFolderLabel, workspaceFolder),
+			agentcontainers.WithContainerLabelIncludeFilter(agentcontainers.DevcontainerIsTestRunLabel, "true"),
+		)
+	})
+
+	testutil.Eventually(ctx, t, func(ctx context.Context) bool {
+		return slices.Contains(client.GetLifecycleStates(), codersdk.WorkspaceAgentLifecycleReady)
+	}, testutil.IntervalMedium, "agent not ready")
+
+	var dcPrebuild codersdk.WorkspaceAgentDevcontainer
+	testutil.Eventually(ctx, t, func(ctx context.Context) bool {
+		resp, err := conn.ListContainers(ctx)
+		require.NoError(t, err)
+
+		for _, dc := range resp.Devcontainers {
+			if dc.Container == nil {
+				continue
+			}
+
+			v, ok := dc.Container.Labels[agentcontainers.DevcontainerLocalFolderLabel]
+			if ok && v == workspaceFolder {
+				dcPrebuild = dc
+				return true
+			}
+		}
+
+		return false
+	}, testutil.IntervalMedium, "devcontainer not found")
+	defer func() {
+		pool.Client.RemoveContainer(docker.RemoveContainerOptions{
+			ID:            dcPrebuild.Container.ID,
+			RemoveVolumes: true,
+			Force:         true,
+		})
+	}()
+
+	// Then: We expect a sub agent to have been created.
+	subAgents := client.GetSubAgents()
+	require.Len(t, subAgents, 1)
+
+	subAgent := subAgents[0]
+	subAgentID, err := uuid.FromBytes(subAgent.GetId())
+	require.NoError(t, err)
+
+	// And: We expect there to be 1 app.
+	subAgentApps, err := client.GetSubAgentApps(subAgentID)
+	require.NoError(t, err)
+	require.Len(t, subAgentApps, 1)
+
+	// And: This app should contain the prebuild workspace name and owner name.
+	subAgentApp := subAgentApps[0]
+	require.Equal(t, "zed://ssh/project.prebuilds-xyz-123.prebuilds.coder/workspaces/project", subAgentApp.GetUrl())
+
+	// Given: We close the client and connection
+	client.Close()
+	conn.Close()
+
+	// Given: A new manifest with a regular user owner name and workspace name.
+	manifest = agentsdk.Manifest{
+		OwnerName:     "user",
+		WorkspaceName: "user-workspace",
+
+		Devcontainers: []codersdk.WorkspaceAgentDevcontainer{
+			{ID: devcontainerID, Name: "test", WorkspaceFolder: workspaceFolder},
+		},
+		Scripts: []codersdk.WorkspaceAgentScript{
+			{ID: devcontainerID, LogSourceID: devcontainerLogSourceID},
+		},
+	}
+
+	// When: We create an agent with devcontainers enabled.
+	//nolint:dogsled
+	conn, client, _, _, _ = setupAgent(t, manifest, 0, func(_ *agenttest.Client, o *agent.Options) {
+		o.Devcontainers = true
+		o.DevcontainerAPIOptions = append(o.DevcontainerAPIOptions,
+			agentcontainers.WithContainerLabelIncludeFilter(agentcontainers.DevcontainerLocalFolderLabel, workspaceFolder),
+			agentcontainers.WithContainerLabelIncludeFilter(agentcontainers.DevcontainerIsTestRunLabel, "true"),
+		)
+	})
+
+	testutil.Eventually(ctx, t, func(ctx context.Context) bool {
+		return slices.Contains(client.GetLifecycleStates(), codersdk.WorkspaceAgentLifecycleReady)
+	}, testutil.IntervalMedium, "agent not ready")
+
+	var dcClaimed codersdk.WorkspaceAgentDevcontainer
+	testutil.Eventually(ctx, t, func(ctx context.Context) bool {
+		resp, err := conn.ListContainers(ctx)
+		require.NoError(t, err)
+
+		for _, dc := range resp.Devcontainers {
+			if dc.Container == nil {
+				continue
+			}
+
+			v, ok := dc.Container.Labels[agentcontainers.DevcontainerLocalFolderLabel]
+			if ok && v == workspaceFolder {
+				dcClaimed = dc
+				return true
+			}
+		}
+
+		return false
+	}, testutil.IntervalMedium, "devcontainer not found")
+	defer func() {
+		if dcClaimed.Container.ID != dcPrebuild.Container.ID {
+			pool.Client.RemoveContainer(docker.RemoveContainerOptions{
+				ID:            dcClaimed.Container.ID,
+				RemoveVolumes: true,
+				Force:         true,
+			})
+		}
+	}()
+
+	// Then: We expect the claimed devcontainer and prebuild devcontainer
+	// to be using the same underlying container.
+	require.Equal(t, dcPrebuild.Container.ID, dcClaimed.Container.ID)
+
+	// And: We expect there to be a sub agent created.
+	subAgents = client.GetSubAgents()
+	require.Len(t, subAgents, 1)
+
+	subAgent = subAgents[0]
+	subAgentID, err = uuid.FromBytes(subAgent.GetId())
+	require.NoError(t, err)
+
+	// And: We expect there to be an app.
+	subAgentApps, err = client.GetSubAgentApps(subAgentID)
+	require.NoError(t, err)
+	require.Len(t, subAgentApps, 1)
+
+	// And: We expect this app to have the user's owner name and workspace name.
+	subAgentApp = subAgentApps[0]
+	require.Equal(t, "zed://ssh/project.user-workspace.user.coder/workspaces/project", subAgentApp.GetUrl())
+}
+
 func TestAgent_Dial(t *testing.T) {
 	t.Parallel()
 
 	cases := []struct {
 		name  string
-		setup func(t *testing.T) net.Listener
+		setup func(t testing.TB) net.Listener
 	}{
 		{
 			name: "TCP",
-			setup: func(t *testing.T) net.Listener {
+			setup: func(t testing.TB) net.Listener {
 				l, err := net.Listen("tcp", "127.0.0.1:0")
 				require.NoError(t, err, "create TCP listener")
 				return l
@@ -2301,7 +2680,7 @@ func TestAgent_Dial(t *testing.T) {
 		},
 		{
 			name: "UDP",
-			setup: func(t *testing.T) net.Listener {
+			setup: func(t testing.TB) net.Listener {
 				addr := net.UDPAddr{
 					IP:   net.ParseIP("127.0.0.1"),
 					Port: 0,
@@ -2314,63 +2693,74 @@ func TestAgent_Dial(t *testing.T) {
 	}
 
 	for _, c := range cases {
-		c := c
 		t.Run(c.name, func(t *testing.T) {
 			t.Parallel()
 
 			// The purpose of this test is to ensure that a client can dial a
 			// listener in the workspace over tailnet.
-			l := c.setup(t)
-			done := make(chan struct{})
-			defer func() {
-				l.Close()
-				<-done
-			}()
+			//
+			// The OS sometimes drops packets if the system can't keep up with
+			// them. For TCP packets, it's typically fine due to
+			// retransmissions, but for UDP packets, it can fail this test.
+			//
+			// The OS gets involved for the Wireguard traffic (either via DERP
+			// or direct UDP), and also for the traffic between the agent and
+			// the listener in the "workspace".
+			//
+			// To avoid this, we'll retry this test up to 3 times.
+			//nolint:gocritic // This test is flaky due to uncontrollable OS packet drops under heavy load.
+			testutil.RunRetry(t, 3, func(t testing.TB) {
+				ctx := testutil.Context(t, testutil.WaitLong)
 
-			ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
-			defer cancel()
+				l := c.setup(t)
+				done := make(chan struct{})
+				defer func() {
+					l.Close()
+					<-done
+				}()
 
-			go func() {
-				defer close(done)
-				for range 2 {
-					c, err := l.Accept()
-					if assert.NoError(t, err, "accept connection") {
-						testAccept(ctx, t, c)
-						_ = c.Close()
+				go func() {
+					defer close(done)
+					for range 2 {
+						c, err := l.Accept()
+						if assert.NoError(t, err, "accept connection") {
+							testAccept(ctx, t, c)
+							_ = c.Close()
+						}
 					}
+				}()
+
+				agentID := uuid.UUID{0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8}
+				//nolint:dogsled
+				agentConn, _, _, _, _ := setupAgent(t, agentsdk.Manifest{
+					AgentID: agentID,
+				}, 0)
+				require.True(t, agentConn.AwaitReachable(ctx))
+				conn, err := agentConn.DialContext(ctx, l.Addr().Network(), l.Addr().String())
+				require.NoError(t, err)
+				testDial(ctx, t, conn)
+				err = conn.Close()
+				require.NoError(t, err)
+
+				// also connect via the CoderServicePrefix, to test that we can reach the agent on this
+				// IP. This will be required for CoderVPN.
+				_, rawPort, _ := net.SplitHostPort(l.Addr().String())
+				port, _ := strconv.ParseUint(rawPort, 10, 16)
+				ipp := netip.AddrPortFrom(tailnet.CoderServicePrefix.AddrFromUUID(agentID), uint16(port))
+
+				switch l.Addr().Network() {
+				case "tcp":
+					conn, err = agentConn.TailnetConn().DialContextTCP(ctx, ipp)
+				case "udp":
+					conn, err = agentConn.TailnetConn().DialContextUDP(ctx, ipp)
+				default:
+					t.Fatalf("unknown network: %s", l.Addr().Network())
 				}
-			}()
-
-			agentID := uuid.UUID{0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8}
-			//nolint:dogsled
-			agentConn, _, _, _, _ := setupAgent(t, agentsdk.Manifest{
-				AgentID: agentID,
-			}, 0)
-			require.True(t, agentConn.AwaitReachable(ctx))
-			conn, err := agentConn.DialContext(ctx, l.Addr().Network(), l.Addr().String())
-			require.NoError(t, err)
-			testDial(ctx, t, conn)
-			err = conn.Close()
-			require.NoError(t, err)
-
-			// also connect via the CoderServicePrefix, to test that we can reach the agent on this
-			// IP. This will be required for CoderVPN.
-			_, rawPort, _ := net.SplitHostPort(l.Addr().String())
-			port, _ := strconv.ParseUint(rawPort, 10, 16)
-			ipp := netip.AddrPortFrom(tailnet.CoderServicePrefix.AddrFromUUID(agentID), uint16(port))
-
-			switch l.Addr().Network() {
-			case "tcp":
-				conn, err = agentConn.Conn.DialContextTCP(ctx, ipp)
-			case "udp":
-				conn, err = agentConn.Conn.DialContextUDP(ctx, ipp)
-			default:
-				t.Fatalf("unknown network: %s", l.Addr().Network())
-			}
-			require.NoError(t, err)
-			testDial(ctx, t, conn)
-			err = conn.Close()
-			require.NoError(t, err)
+				require.NoError(t, err)
+				testDial(ctx, t, conn)
+				err = conn.Close()
+				require.NoError(t, err)
+			})
 		})
 	}
 }
@@ -2421,7 +2811,7 @@ func TestAgent_UpdatedDERP(t *testing.T) {
 	})
 
 	// Setup a client connection.
-	newClientConn := func(derpMap *tailcfg.DERPMap, name string) *workspacesdk.AgentConn {
+	newClientConn := func(derpMap *tailcfg.DERPMap, name string) workspacesdk.AgentConn {
 		conn, err := tailnet.NewConn(&tailnet.Options{
 			Addresses: []netip.Prefix{tailnet.TailscaleServicePrefix.RandomPrefix()},
 			DERPMap:   derpMap,
@@ -2501,13 +2891,13 @@ func TestAgent_UpdatedDERP(t *testing.T) {
 
 	// Connect from a second client and make sure it uses the new DERP map.
 	conn2 := newClientConn(newDerpMap, "client2")
-	require.Equal(t, []int{2}, conn2.DERPMap().RegionIDs())
+	require.Equal(t, []int{2}, conn2.TailnetConn().DERPMap().RegionIDs())
 	t.Log("conn2 got the new DERPMap")
 
 	// If the first client gets a DERP map update, it should be able to
 	// reconnect just fine.
-	conn1.SetDERPMap(newDerpMap)
-	require.Equal(t, []int{2}, conn1.DERPMap().RegionIDs())
+	conn1.TailnetConn().SetDERPMap(newDerpMap)
+	require.Equal(t, []int{2}, conn1.TailnetConn().DERPMap().RegionIDs())
 	t.Log("set the new DERPMap on conn1")
 	ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
 	defer cancel()
@@ -2873,8 +3263,8 @@ func setupSSHSessionOnPort(
 	return session
 }
 
-func setupAgent(t *testing.T, metadata agentsdk.Manifest, ptyTimeout time.Duration, opts ...func(*agenttest.Client, *agent.Options)) (
-	*workspacesdk.AgentConn,
+func setupAgent(t testing.TB, metadata agentsdk.Manifest, ptyTimeout time.Duration, opts ...func(*agenttest.Client, *agent.Options)) (
+	workspacesdk.AgentConn,
 	*agenttest.Client,
 	<-chan *proto.Stats,
 	afero.Fs,
@@ -2897,6 +3287,9 @@ func setupAgent(t *testing.T, metadata agentsdk.Manifest, ptyTimeout time.Durati
 	if metadata.WorkspaceName == "" {
 		metadata.WorkspaceName = "test-workspace"
 	}
+	if metadata.OwnerName == "" {
+		metadata.OwnerName = "test-user"
+	}
 	if metadata.WorkspaceID == uuid.Nil {
 		metadata.WorkspaceID = uuid.New()
 	}
@@ -2968,7 +3361,7 @@ func setupAgent(t *testing.T, metadata agentsdk.Manifest, ptyTimeout time.Durati
 
 var dialTestPayload = []byte("dean-was-here123")
 
-func testDial(ctx context.Context, t *testing.T, c net.Conn) {
+func testDial(ctx context.Context, t testing.TB, c net.Conn) {
 	t.Helper()
 
 	if deadline, ok := ctx.Deadline(); ok {
@@ -2984,7 +3377,7 @@ func testDial(ctx context.Context, t *testing.T, c net.Conn) {
 	assertReadPayload(t, c, dialTestPayload)
 }
 
-func testAccept(ctx context.Context, t *testing.T, c net.Conn) {
+func testAccept(ctx context.Context, t testing.TB, c net.Conn) {
 	t.Helper()
 	defer c.Close()
 
@@ -3001,7 +3394,7 @@ func testAccept(ctx context.Context, t *testing.T, c net.Conn) {
 	assertWritePayload(t, c, dialTestPayload)
 }
 
-func assertReadPayload(t *testing.T, r io.Reader, payload []byte) {
+func assertReadPayload(t testing.TB, r io.Reader, payload []byte) {
 	t.Helper()
 	b := make([]byte, len(payload)+16)
 	n, err := r.Read(b)
@@ -3010,11 +3403,11 @@ func assertReadPayload(t *testing.T, r io.Reader, payload []byte) {
 	assert.Equal(t, payload, b[:n])
 }
 
-func assertWritePayload(t *testing.T, w io.Writer, payload []byte) {
+func assertWritePayload(t testing.TB, w io.Writer, payload []byte) {
 	t.Helper()
 	n, err := w.Write(payload)
 	assert.NoError(t, err, "write payload")
-	assert.Equal(t, len(payload), n, "payload length does not match")
+	assert.Equal(t, len(payload), n, "written payload length does not match")
 }
 
 func testSessionOutput(t *testing.T, session *ssh.Session, expected, unexpected []string, expectedRe *regexp.Regexp) {
@@ -3077,7 +3470,11 @@ func TestAgent_Metrics_SSH(t *testing.T) {
 	registry := prometheus.NewRegistry()
 
 	//nolint:dogsled
-	conn, _, _, _, _ := setupAgent(t, agentsdk.Manifest{}, 0, func(_ *agenttest.Client, o *agent.Options) {
+	conn, _, _, _, _ := setupAgent(t, agentsdk.Manifest{
+		// Make sure we always get a DERP connection for
+		// currently_reachable_peers.
+		DisableDirectConnections: true,
+	}, 0, func(_ *agenttest.Client, o *agent.Options) {
 		o.PrometheusRegistry = registry
 	})
 
@@ -3131,7 +3528,7 @@ func TestAgent_Metrics_SSH(t *testing.T) {
 		{
 			Name:  "coderd_agentstats_currently_reachable_peers",
 			Type:  proto.Stats_Metric_GAUGE,
-			Value: 0,
+			Value: 1,
 			Labels: []*proto.Stats_Metric_Label{
 				{
 					Name:  "connection_type",
@@ -3142,7 +3539,7 @@ func TestAgent_Metrics_SSH(t *testing.T) {
 		{
 			Name:  "coderd_agentstats_currently_reachable_peers",
 			Type:  proto.Stats_Metric_GAUGE,
-			Value: 1,
+			Value: 0,
 			Labels: []*proto.Stats_Metric_Label{
 				{
 					Name:  "connection_type",

agent/agentcontainers/acmock/acmock.go

@@ -1,9 +1,9 @@
 // Code generated by MockGen. DO NOT EDIT.
-// Source: .. (interfaces: Lister,DevcontainerCLI)
+// Source: .. (interfaces: ContainerCLI,DevcontainerCLI)
 //
 // Generated by this command:
 //
-//	mockgen -destination ./acmock.go -package acmock .. Lister,DevcontainerCLI
+//	mockgen -destination ./acmock.go -package acmock .. ContainerCLI,DevcontainerCLI
 //
 
 // Package acmock is a generated GoMock package.
@@ -18,32 +18,81 @@ import (
 	gomock "go.uber.org/mock/gomock"
 )
 
-// MockLister is a mock of Lister interface.
-type MockLister struct {
+// MockContainerCLI is a mock of ContainerCLI interface.
+type MockContainerCLI struct {
 	ctrl     *gomock.Controller
-	recorder *MockListerMockRecorder
+	recorder *MockContainerCLIMockRecorder
 	isgomock struct{}
 }
 
-// MockListerMockRecorder is the mock recorder for MockLister.
-type MockListerMockRecorder struct {
-	mock *MockLister
+// MockContainerCLIMockRecorder is the mock recorder for MockContainerCLI.
+type MockContainerCLIMockRecorder struct {
+	mock *MockContainerCLI
 }
 
-// NewMockLister creates a new mock instance.
-func NewMockLister(ctrl *gomock.Controller) *MockLister {
-	mock := &MockLister{ctrl: ctrl}
-	mock.recorder = &MockListerMockRecorder{mock}
+// NewMockContainerCLI creates a new mock instance.
+func NewMockContainerCLI(ctrl *gomock.Controller) *MockContainerCLI {
+	mock := &MockContainerCLI{ctrl: ctrl}
+	mock.recorder = &MockContainerCLIMockRecorder{mock}
 	return mock
 }
 
 // EXPECT returns an object that allows the caller to indicate expected use.
-func (m *MockLister) EXPECT() *MockListerMockRecorder {
+func (m *MockContainerCLI) EXPECT() *MockContainerCLIMockRecorder {
 	return m.recorder
 }
 
+// Copy mocks base method.
+func (m *MockContainerCLI) Copy(ctx context.Context, containerName, src, dst string) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Copy", ctx, containerName, src, dst)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// Copy indicates an expected call of Copy.
+func (mr *MockContainerCLIMockRecorder) Copy(ctx, containerName, src, dst any) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Copy", reflect.TypeOf((*MockContainerCLI)(nil).Copy), ctx, containerName, src, dst)
+}
+
+// DetectArchitecture mocks base method.
+func (m *MockContainerCLI) DetectArchitecture(ctx context.Context, containerName string) (string, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "DetectArchitecture", ctx, containerName)
+	ret0, _ := ret[0].(string)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// DetectArchitecture indicates an expected call of DetectArchitecture.
+func (mr *MockContainerCLIMockRecorder) DetectArchitecture(ctx, containerName any) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DetectArchitecture", reflect.TypeOf((*MockContainerCLI)(nil).DetectArchitecture), ctx, containerName)
+}
+
+// ExecAs mocks base method.
+func (m *MockContainerCLI) ExecAs(ctx context.Context, containerName, user string, args ...string) ([]byte, error) {
+	m.ctrl.T.Helper()
+	varargs := []any{ctx, containerName, user}
+	for _, a := range args {
+		varargs = append(varargs, a)
+	}
+	ret := m.ctrl.Call(m, "ExecAs", varargs...)
+	ret0, _ := ret[0].([]byte)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// ExecAs indicates an expected call of ExecAs.
+func (mr *MockContainerCLIMockRecorder) ExecAs(ctx, containerName, user any, args ...any) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	varargs := append([]any{ctx, containerName, user}, args...)
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ExecAs", reflect.TypeOf((*MockContainerCLI)(nil).ExecAs), varargs...)
+}
+
 // List mocks base method.
-func (m *MockLister) List(ctx context.Context) (codersdk.WorkspaceAgentListContainersResponse, error) {
+func (m *MockContainerCLI) List(ctx context.Context) (codersdk.WorkspaceAgentListContainersResponse, error) {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "List", ctx)
 	ret0, _ := ret[0].(codersdk.WorkspaceAgentListContainersResponse)
@@ -52,9 +101,9 @@ func (m *MockLister) List(ctx context.Context) (codersdk.WorkspaceAgentListConta
 }
 
 // List indicates an expected call of List.
-func (mr *MockListerMockRecorder) List(ctx any) *gomock.Call {
+func (mr *MockContainerCLIMockRecorder) List(ctx any) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "List", reflect.TypeOf((*MockLister)(nil).List), ctx)
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "List", reflect.TypeOf((*MockContainerCLI)(nil).List), ctx)
 }
 
 // MockDevcontainerCLI is a mock of DevcontainerCLI interface.
@@ -81,6 +130,45 @@ func (m *MockDevcontainerCLI) EXPECT() *MockDevcontainerCLIMockRecorder {
 	return m.recorder
 }
 
+// Exec mocks base method.
+func (m *MockDevcontainerCLI) Exec(ctx context.Context, workspaceFolder, configPath, cmd string, cmdArgs []string, opts ...agentcontainers.DevcontainerCLIExecOptions) error {
+	m.ctrl.T.Helper()
+	varargs := []any{ctx, workspaceFolder, configPath, cmd, cmdArgs}
+	for _, a := range opts {
+		varargs = append(varargs, a)
+	}
+	ret := m.ctrl.Call(m, "Exec", varargs...)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// Exec indicates an expected call of Exec.
+func (mr *MockDevcontainerCLIMockRecorder) Exec(ctx, workspaceFolder, configPath, cmd, cmdArgs any, opts ...any) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	varargs := append([]any{ctx, workspaceFolder, configPath, cmd, cmdArgs}, opts...)
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Exec", reflect.TypeOf((*MockDevcontainerCLI)(nil).Exec), varargs...)
+}
+
+// ReadConfig mocks base method.
+func (m *MockDevcontainerCLI) ReadConfig(ctx context.Context, workspaceFolder, configPath string, env []string, opts ...agentcontainers.DevcontainerCLIReadConfigOptions) (agentcontainers.DevcontainerConfig, error) {
+	m.ctrl.T.Helper()
+	varargs := []any{ctx, workspaceFolder, configPath, env}
+	for _, a := range opts {
+		varargs = append(varargs, a)
+	}
+	ret := m.ctrl.Call(m, "ReadConfig", varargs...)
+	ret0, _ := ret[0].(agentcontainers.DevcontainerConfig)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// ReadConfig indicates an expected call of ReadConfig.
+func (mr *MockDevcontainerCLIMockRecorder) ReadConfig(ctx, workspaceFolder, configPath, env any, opts ...any) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	varargs := append([]any{ctx, workspaceFolder, configPath, env}, opts...)
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReadConfig", reflect.TypeOf((*MockDevcontainerCLI)(nil).ReadConfig), varargs...)
+}
+
 // Up mocks base method.
 func (m *MockDevcontainerCLI) Up(ctx context.Context, workspaceFolder, configPath string, opts ...agentcontainers.DevcontainerCLIUpOptions) (string, error) {
 	m.ctrl.T.Helper()

agent/agentcontainers/acmock/doc.go

@@ -1,4 +1,4 @@
 // Package acmock contains a mock implementation of agentcontainers.Lister for use in tests.
 package acmock
 
-//go:generate mockgen -destination ./acmock.go -package acmock .. Lister,DevcontainerCLI
+//go:generate mockgen -destination ./acmock.go -package acmock .. ContainerCLI,DevcontainerCLI

agent/agentcontainers/api_internal_test.go

@@ -0,0 +1,358 @@
+package agentcontainers
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/coder/coder/v2/provisioner"
+)
+
+func TestSafeAgentName(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name       string
+		folderName string
+		expected   string
+		fallback   bool
+	}{
+		// Basic valid names
+		{
+			folderName: "simple",
+			expected:   "simple",
+		},
+		{
+			folderName: "with-hyphens",
+			expected:   "with-hyphens",
+		},
+		{
+			folderName: "123numbers",
+			expected:   "123numbers",
+		},
+		{
+			folderName: "mixed123",
+			expected:   "mixed123",
+		},
+
+		// Names that need transformation
+		{
+			folderName: "With_Underscores",
+			expected:   "with-underscores",
+		},
+		{
+			folderName: "With Spaces",
+			expected:   "with-spaces",
+		},
+		{
+			folderName: "UPPERCASE",
+			expected:   "uppercase",
+		},
+		{
+			folderName: "Mixed_Case-Name",
+			expected:   "mixed-case-name",
+		},
+
+		// Names with special characters that get replaced
+		{
+			folderName: "special@#$chars",
+			expected:   "special-chars",
+		},
+		{
+			folderName: "dots.and.more",
+			expected:   "dots-and-more",
+		},
+		{
+			folderName: "multiple___underscores",
+			expected:   "multiple-underscores",
+		},
+		{
+			folderName: "multiple---hyphens",
+			expected:   "multiple-hyphens",
+		},
+
+		// Edge cases with leading/trailing special chars
+		{
+			folderName: "-leading-hyphen",
+			expected:   "leading-hyphen",
+		},
+		{
+			folderName: "trailing-hyphen-",
+			expected:   "trailing-hyphen",
+		},
+		{
+			folderName: "_leading_underscore",
+			expected:   "leading-underscore",
+		},
+		{
+			folderName: "trailing_underscore_",
+			expected:   "trailing-underscore",
+		},
+		{
+			folderName: "---multiple-leading",
+			expected:   "multiple-leading",
+		},
+		{
+			folderName: "trailing-multiple---",
+			expected:   "trailing-multiple",
+		},
+
+		// Complex transformation cases
+		{
+			folderName: "___very---complex@@@name___",
+			expected:   "very-complex-name",
+		},
+		{
+			folderName: "my.project-folder_v2",
+			expected:   "my-project-folder-v2",
+		},
+
+		// Empty and fallback cases - now correctly uses friendlyName fallback
+		{
+			folderName: "",
+			expected:   "friendly-fallback",
+			fallback:   true,
+		},
+		{
+			folderName: "---",
+			expected:   "friendly-fallback",
+			fallback:   true,
+		},
+		{
+			folderName: "___",
+			expected:   "friendly-fallback",
+			fallback:   true,
+		},
+		{
+			folderName: "@#$",
+			expected:   "friendly-fallback",
+			fallback:   true,
+		},
+
+		// Additional edge cases
+		{
+			folderName: "a",
+			expected:   "a",
+		},
+		{
+			folderName: "1",
+			expected:   "1",
+		},
+		{
+			folderName: "a1b2c3",
+			expected:   "a1b2c3",
+		},
+		{
+			folderName: "CamelCase",
+			expected:   "camelcase",
+		},
+		{
+			folderName: "snake_case_name",
+			expected:   "snake-case-name",
+		},
+		{
+			folderName: "kebab-case-name",
+			expected:   "kebab-case-name",
+		},
+		{
+			folderName: "mix3d_C4s3-N4m3",
+			expected:   "mix3d-c4s3-n4m3",
+		},
+		{
+			folderName: "123-456-789",
+			expected:   "123-456-789",
+		},
+		{
+			folderName: "abc123def456",
+			expected:   "abc123def456",
+		},
+		{
+			folderName: "   spaces   everywhere   ",
+			expected:   "spaces-everywhere",
+		},
+		{
+			folderName: "unicode-café-naïve",
+			expected:   "unicode-caf-na-ve",
+		},
+		{
+			folderName: "path/with/slashes",
+			expected:   "path-with-slashes",
+		},
+		{
+			folderName: "file.tar.gz",
+			expected:   "file-tar-gz",
+		},
+		{
+			folderName: "version-1.2.3-alpha",
+			expected:   "version-1-2-3-alpha",
+		},
+
+		// Truncation test for names exceeding 64 characters
+		{
+			folderName: "this-is-a-very-long-folder-name-that-exceeds-sixty-four-characters-limit-and-should-be-truncated",
+			expected:   "this-is-a-very-long-folder-name-that-exceeds-sixty-four-characte",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.folderName, func(t *testing.T) {
+			t.Parallel()
+			name, usingWorkspaceFolder := safeAgentName(tt.folderName, "friendly-fallback")
+
+			assert.Equal(t, tt.expected, name)
+			assert.True(t, provisioner.AgentNameRegex.Match([]byte(name)))
+			assert.Equal(t, tt.fallback, !usingWorkspaceFolder)
+		})
+	}
+}
+
+func TestExpandedAgentName(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name            string
+		workspaceFolder string
+		friendlyName    string
+		depth           int
+		expected        string
+		fallback        bool
+	}{
+		{
+			name:            "simple path depth 1",
+			workspaceFolder: "/home/coder/project",
+			friendlyName:    "friendly-fallback",
+			depth:           0,
+			expected:        "project",
+		},
+		{
+			name:            "simple path depth 2",
+			workspaceFolder: "/home/coder/project",
+			friendlyName:    "friendly-fallback",
+			depth:           1,
+			expected:        "coder-project",
+		},
+		{
+			name:            "simple path depth 3",
+			workspaceFolder: "/home/coder/project",
+			friendlyName:    "friendly-fallback",
+			depth:           2,
+			expected:        "home-coder-project",
+		},
+		{
+			name:            "simple path depth exceeds available",
+			workspaceFolder: "/home/coder/project",
+			friendlyName:    "friendly-fallback",
+			depth:           9,
+			expected:        "home-coder-project",
+		},
+		// Cases with special characters that need sanitization
+		{
+			name:            "path with spaces and special chars",
+			workspaceFolder: "/home/coder/My Project_v2",
+			friendlyName:    "friendly-fallback",
+			depth:           1,
+			expected:        "coder-my-project-v2",
+		},
+		{
+			name:            "path with dots and underscores",
+			workspaceFolder: "/home/user.name/project_folder.git",
+			friendlyName:    "friendly-fallback",
+			depth:           1,
+			expected:        "user-name-project-folder-git",
+		},
+		// Edge cases
+		{
+			name:            "empty path",
+			workspaceFolder: "",
+			friendlyName:    "friendly-fallback",
+			depth:           0,
+			expected:        "friendly-fallback",
+			fallback:        true,
+		},
+		{
+			name:            "root path",
+			workspaceFolder: "/",
+			friendlyName:    "friendly-fallback",
+			depth:           0,
+			expected:        "friendly-fallback",
+			fallback:        true,
+		},
+		{
+			name:            "single component",
+			workspaceFolder: "project",
+			friendlyName:    "friendly-fallback",
+			depth:           0,
+			expected:        "project",
+		},
+		{
+			name:            "single component with depth 2",
+			workspaceFolder: "project",
+			friendlyName:    "friendly-fallback",
+			depth:           1,
+			expected:        "project",
+		},
+		// Collision simulation cases
+		{
+			name:            "foo/project depth 1",
+			workspaceFolder: "/home/coder/foo/project",
+			friendlyName:    "friendly-fallback",
+			depth:           0,
+			expected:        "project",
+		},
+		{
+			name:            "foo/project depth 2",
+			workspaceFolder: "/home/coder/foo/project",
+			friendlyName:    "friendly-fallback",
+			depth:           1,
+			expected:        "foo-project",
+		},
+		{
+			name:            "bar/project depth 1",
+			workspaceFolder: "/home/coder/bar/project",
+			friendlyName:    "friendly-fallback",
+			depth:           0,
+			expected:        "project",
+		},
+		{
+			name:            "bar/project depth 2",
+			workspaceFolder: "/home/coder/bar/project",
+			friendlyName:    "friendly-fallback",
+			depth:           1,
+			expected:        "bar-project",
+		},
+		// Path with trailing slashes
+		{
+			name:            "path with trailing slash",
+			workspaceFolder: "/home/coder/project/",
+			friendlyName:    "friendly-fallback",
+			depth:           1,
+			expected:        "coder-project",
+		},
+		{
+			name:            "path with multiple trailing slashes",
+			workspaceFolder: "/home/coder/project///",
+			friendlyName:    "friendly-fallback",
+			depth:           1,
+			expected:        "coder-project",
+		},
+		// Path with leading slashes
+		{
+			name:            "path with multiple leading slashes",
+			workspaceFolder: "///home/coder/project",
+			friendlyName:    "friendly-fallback",
+			depth:           1,
+			expected:        "coder-project",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+			name, usingWorkspaceFolder := expandedAgentName(tt.workspaceFolder, tt.friendlyName, tt.depth)
+
+			assert.Equal(t, tt.expected, name)
+			assert.True(t, provisioner.AgentNameRegex.Match([]byte(name)))
+			assert.Equal(t, tt.fallback, !usingWorkspaceFolder)
+		})
+	}
+}

agent/agentcontainers/containers.go

@@ -6,19 +6,32 @@ import (
 	"github.com/coder/coder/v2/codersdk"
 )
 
-// Lister is an interface for listing containers visible to the
-// workspace agent.
-type Lister interface {
+// ContainerCLI is an interface for interacting with containers in a workspace.
+type ContainerCLI interface {
 	// List returns a list of containers visible to the workspace agent.
 	// This should include running and stopped containers.
 	List(ctx context.Context) (codersdk.WorkspaceAgentListContainersResponse, error)
+	// DetectArchitecture detects the architecture of a container.
+	DetectArchitecture(ctx context.Context, containerName string) (string, error)
+	// Copy copies a file from the host to a container.
+	Copy(ctx context.Context, containerName, src, dst string) error
+	// ExecAs executes a command in a container as a specific user.
+	ExecAs(ctx context.Context, containerName, user string, args ...string) ([]byte, error)
 }
 
-// NoopLister is a Lister interface that never returns any containers.
-type NoopLister struct{}
+// noopContainerCLI is a ContainerCLI that does nothing.
+type noopContainerCLI struct{}
 
-var _ Lister = NoopLister{}
+var _ ContainerCLI = noopContainerCLI{}
 
-func (NoopLister) List(_ context.Context) (codersdk.WorkspaceAgentListContainersResponse, error) {
+func (noopContainerCLI) List(_ context.Context) (codersdk.WorkspaceAgentListContainersResponse, error) {
 	return codersdk.WorkspaceAgentListContainersResponse{}, nil
 }
+
+func (noopContainerCLI) DetectArchitecture(_ context.Context, _ string) (string, error) {
+	return "<none>", nil
+}
+func (noopContainerCLI) Copy(_ context.Context, _ string, _ string, _ string) error { return nil }
+func (noopContainerCLI) ExecAs(_ context.Context, _ string, _ string, _ ...string) ([]byte, error) {
+	return nil, nil
+}

agent/agentcontainers/containers_dockercli.go

@@ -228,23 +228,23 @@ func run(ctx context.Context, execer agentexec.Execer, cmd string, args ...strin
 	return stdout, stderr, err
 }
 
-// DockerCLILister is a ContainerLister that lists containers using the docker CLI
-type DockerCLILister struct {
+// dockerCLI is an implementation for Docker CLI that lists containers.
+type dockerCLI struct {
 	execer agentexec.Execer
 }
 
-var _ Lister = &DockerCLILister{}
+var _ ContainerCLI = (*dockerCLI)(nil)
 
-func NewDocker(execer agentexec.Execer) Lister {
-	return &DockerCLILister{
-		execer: agentexec.DefaultExecer,
+func NewDockerCLI(execer agentexec.Execer) ContainerCLI {
+	return &dockerCLI{
+		execer: execer,
 	}
 }
 
-func (dcl *DockerCLILister) List(ctx context.Context) (codersdk.WorkspaceAgentListContainersResponse, error) {
+func (dcli *dockerCLI) List(ctx context.Context) (codersdk.WorkspaceAgentListContainersResponse, error) {
 	var stdoutBuf, stderrBuf bytes.Buffer
 	// List all container IDs, one per line, with no truncation
-	cmd := dcl.execer.CommandContext(ctx, "docker", "ps", "--all", "--quiet", "--no-trunc")
+	cmd := dcli.execer.CommandContext(ctx, "docker", "ps", "--all", "--quiet", "--no-trunc")
 	cmd.Stdout = &stdoutBuf
 	cmd.Stderr = &stderrBuf
 	if err := cmd.Run(); err != nil {
@@ -288,7 +288,7 @@ func (dcl *DockerCLILister) List(ctx context.Context) (codersdk.WorkspaceAgentLi
 	// will still contain valid JSON. We will just end up missing
 	// information about the removed container. We could potentially
 	// log this error, but I'm not sure it's worth it.
-	dockerInspectStdout, dockerInspectStderr, err := runDockerInspect(ctx, dcl.execer, ids...)
+	dockerInspectStdout, dockerInspectStderr, err := runDockerInspect(ctx, dcli.execer, ids...)
 	if err != nil {
 		return codersdk.WorkspaceAgentListContainersResponse{}, xerrors.Errorf("run docker inspect: %w: %s", err, dockerInspectStderr)
 	}
@@ -311,6 +311,10 @@ func (dcl *DockerCLILister) List(ctx context.Context) (codersdk.WorkspaceAgentLi
 // container IDs and returns the parsed output.
 // The stderr output is also returned for logging purposes.
 func runDockerInspect(ctx context.Context, execer agentexec.Execer, ids ...string) (stdout, stderr []byte, err error) {
+	if ctx.Err() != nil {
+		// If the context is done, we don't want to run the command.
+		return []byte{}, []byte{}, ctx.Err()
+	}
 	var stdoutBuf, stderrBuf bytes.Buffer
 	cmd := execer.CommandContext(ctx, "docker", append([]string{"inspect"}, ids...)...)
 	cmd.Stdout = &stdoutBuf
@@ -319,6 +323,12 @@ func runDockerInspect(ctx context.Context, execer agentexec.Execer, ids ...strin
 	stdout = bytes.TrimSpace(stdoutBuf.Bytes())
 	stderr = bytes.TrimSpace(stderrBuf.Bytes())
 	if err != nil {
+		if ctx.Err() != nil {
+			// If the context was canceled while running the command,
+			// return the context error instead of the command error,
+			// which is likely to be "signal: killed".
+			return stdout, stderr, ctx.Err()
+		}
 		if bytes.Contains(stderr, []byte("No such object:")) {
 			// This can happen if a container is deleted between the time we check for its existence and the time we inspect it.
 			return stdout, stderr, nil
@@ -517,3 +527,71 @@ func isLoopbackOrUnspecified(ips string) bool {
 	}
 	return nip.IsLoopback() || nip.IsUnspecified()
 }
+
+// DetectArchitecture detects the architecture of a container by inspecting its
+// image.
+func (dcli *dockerCLI) DetectArchitecture(ctx context.Context, containerName string) (string, error) {
+	// Inspect the container to get the image name, which contains the architecture.
+	stdout, stderr, err := runCmd(ctx, dcli.execer, "docker", "inspect", "--format", "{{.Config.Image}}", containerName)
+	if err != nil {
+		return "", xerrors.Errorf("inspect container %s: %w: %s", containerName, err, stderr)
+	}
+	imageName := string(stdout)
+	if imageName == "" {
+		return "", xerrors.Errorf("no image found for container %s", containerName)
+	}
+
+	stdout, stderr, err = runCmd(ctx, dcli.execer, "docker", "inspect", "--format", "{{.Architecture}}", imageName)
+	if err != nil {
+		return "", xerrors.Errorf("inspect image %s: %w: %s", imageName, err, stderr)
+	}
+	arch := string(stdout)
+	if arch == "" {
+		return "", xerrors.Errorf("no architecture found for image %s", imageName)
+	}
+	return arch, nil
+}
+
+// Copy copies a file from the host to a container.
+func (dcli *dockerCLI) Copy(ctx context.Context, containerName, src, dst string) error {
+	_, stderr, err := runCmd(ctx, dcli.execer, "docker", "cp", src, containerName+":"+dst)
+	if err != nil {
+		return xerrors.Errorf("copy %s to %s:%s: %w: %s", src, containerName, dst, err, stderr)
+	}
+	return nil
+}
+
+// ExecAs executes a command in a container as a specific user.
+func (dcli *dockerCLI) ExecAs(ctx context.Context, containerName, uid string, args ...string) ([]byte, error) {
+	execArgs := []string{"exec"}
+	if uid != "" {
+		altUID := uid
+		if uid == "root" {
+			// UID 0 is more portable than the name root, so we use that
+			// because  some containers may not have a user named "root".
+			altUID = "0"
+		}
+		execArgs = append(execArgs, "--user", altUID)
+	}
+	execArgs = append(execArgs, containerName)
+	execArgs = append(execArgs, args...)
+
+	stdout, stderr, err := runCmd(ctx, dcli.execer, "docker", execArgs...)
+	if err != nil {
+		return nil, xerrors.Errorf("exec in container %s as user %s: %w: %s", containerName, uid, err, stderr)
+	}
+	return stdout, nil
+}
+
+// runCmd is a helper function that runs a command with the given
+// arguments and returns the stdout and stderr output.
+func runCmd(ctx context.Context, execer agentexec.Execer, cmd string, args ...string) (stdout, stderr []byte, err error) {
+	var stdoutBuf, stderrBuf bytes.Buffer
+	c := execer.CommandContext(ctx, cmd, args...)
+	c.Stdout = &stdoutBuf
+	c.Stderr = &stderrBuf
+	err = c.Run()
+	stdout = bytes.TrimSpace(stdoutBuf.Bytes())
+	stderr = bytes.TrimSpace(stderrBuf.Bytes())
+	return stdout, stderr, err
+}

agent/agentcontainers/containers_dockercli_test.go

@@ -0,0 +1,128 @@
+package agentcontainers_test
+
+import (
+	"os"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"testing"
+
+	"github.com/ory/dockertest/v3"
+	"github.com/ory/dockertest/v3/docker"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/coder/coder/v2/agent/agentcontainers"
+	"github.com/coder/coder/v2/agent/agentexec"
+	"github.com/coder/coder/v2/testutil"
+)
+
+// TestIntegrationDockerCLI tests the DetectArchitecture, Copy, and
+// ExecAs methods using a real Docker container. All tests share a
+// single container to avoid setup overhead.
+//
+// Run manually with: CODER_TEST_USE_DOCKER=1 go test ./agent/agentcontainers -run TestIntegrationDockerCLI
+//
+//nolint:tparallel,paralleltest // Docker integration tests don't run in parallel to avoid flakiness.
+func TestIntegrationDockerCLI(t *testing.T) {
+	if ctud, ok := os.LookupEnv("CODER_TEST_USE_DOCKER"); !ok || ctud != "1" {
+		t.Skip("Set CODER_TEST_USE_DOCKER=1 to run this test")
+	}
+
+	pool, err := dockertest.NewPool("")
+	require.NoError(t, err, "Could not connect to docker")
+
+	// Start a simple busybox container for all subtests to share.
+	ct, err := pool.RunWithOptions(&dockertest.RunOptions{
+		Repository: "busybox",
+		Tag:        "latest",
+		Cmd:        []string{"sleep", "infinity"},
+	}, func(config *docker.HostConfig) {
+		config.AutoRemove = true
+		config.RestartPolicy = docker.RestartPolicy{Name: "no"}
+	})
+	require.NoError(t, err, "Could not start test docker container")
+	t.Logf("Created container %q", ct.Container.Name)
+	t.Cleanup(func() {
+		assert.NoError(t, pool.Purge(ct), "Could not purge resource %q", ct.Container.Name)
+		t.Logf("Purged container %q", ct.Container.Name)
+	})
+
+	// Wait for container to start.
+	require.Eventually(t, func() bool {
+		ct, ok := pool.ContainerByName(ct.Container.Name)
+		return ok && ct.Container.State.Running
+	}, testutil.WaitShort, testutil.IntervalSlow, "Container did not start in time")
+
+	dcli := agentcontainers.NewDockerCLI(agentexec.DefaultExecer)
+	containerName := strings.TrimPrefix(ct.Container.Name, "/")
+
+	t.Run("DetectArchitecture", func(t *testing.T) {
+		t.Parallel()
+		ctx := testutil.Context(t, testutil.WaitShort)
+
+		arch, err := dcli.DetectArchitecture(ctx, containerName)
+		require.NoError(t, err, "DetectArchitecture failed")
+		require.NotEmpty(t, arch, "arch has no content")
+		require.Equal(t, runtime.GOARCH, arch, "architecture does not match runtime, did you run this test with a remote Docker socket?")
+
+		t.Logf("Detected architecture: %s", arch)
+	})
+
+	t.Run("Copy", func(t *testing.T) {
+		t.Parallel()
+		ctx := testutil.Context(t, testutil.WaitShort)
+
+		want := "Help, I'm trapped!"
+		tempFile := filepath.Join(t.TempDir(), "test-file.txt")
+		err := os.WriteFile(tempFile, []byte(want), 0o600)
+		require.NoError(t, err, "create test file failed")
+
+		destPath := "/tmp/copied-file.txt"
+		err = dcli.Copy(ctx, containerName, tempFile, destPath)
+		require.NoError(t, err, "Copy failed")
+
+		got, err := dcli.ExecAs(ctx, containerName, "", "cat", destPath)
+		require.NoError(t, err, "ExecAs failed after Copy")
+		require.Equal(t, want, string(got), "copied file content did not match original")
+
+		t.Logf("Successfully copied file from %s to container %s:%s", tempFile, containerName, destPath)
+	})
+
+	t.Run("ExecAs", func(t *testing.T) {
+		t.Parallel()
+		ctx := testutil.Context(t, testutil.WaitShort)
+
+		// Test ExecAs without specifying user (should use container's default).
+		want := "root"
+		got, err := dcli.ExecAs(ctx, containerName, "", "whoami")
+		require.NoError(t, err, "ExecAs without user should succeed")
+		require.Equal(t, want, string(got), "ExecAs without user should output expected string")
+
+		// Test ExecAs with numeric UID (non root).
+		want = "1000"
+		_, err = dcli.ExecAs(ctx, containerName, want, "whoami")
+		require.Error(t, err, "ExecAs with UID 1000 should fail as user does not exist in busybox")
+		require.Contains(t, err.Error(), "whoami: unknown uid 1000", "ExecAs with UID 1000 should return 'unknown uid' error")
+
+		// Test ExecAs with root user (should convert "root" to "0", which still outputs root due to passwd).
+		want = "root"
+		got, err = dcli.ExecAs(ctx, containerName, "root", "whoami")
+		require.NoError(t, err, "ExecAs with root user should succeed")
+		require.Equal(t, want, string(got), "ExecAs with root user should output expected string")
+
+		// Test ExecAs with numeric UID.
+		want = "root"
+		got, err = dcli.ExecAs(ctx, containerName, "0", "whoami")
+		require.NoError(t, err, "ExecAs with UID 0 should succeed")
+		require.Equal(t, want, string(got), "ExecAs with UID 0 should output expected string")
+
+		// Test ExecAs with multiple arguments.
+		want = "multiple args test"
+		got, err = dcli.ExecAs(ctx, containerName, "", "sh", "-c", "echo '"+want+"'")
+		require.NoError(t, err, "ExecAs with multiple arguments should succeed")
+		require.Equal(t, want, string(got), "ExecAs with multiple arguments should output expected string")
+
+		t.Logf("Successfully executed commands in container %s", containerName)
+	})
+}

agent/agentcontainers/containers_internal_test.go

@@ -41,7 +41,6 @@ func TestWrapDockerExec(t *testing.T) {
 		},
 	}
 	for _, tt := range tests {
-		tt := tt // appease the linter even though this isn't needed anymore
 		t.Run(tt.name, func(t *testing.T) {
 			t.Parallel()
 			actualCmd, actualArgs := wrapDockerExec("my-container", tt.containerUser, tt.cmdArgs[0], tt.cmdArgs[1:]...)
@@ -54,7 +53,6 @@ func TestWrapDockerExec(t *testing.T) {
 func TestConvertDockerPort(t *testing.T) {
 	t.Parallel()
 
-	//nolint:paralleltest // variable recapture no longer required
 	for _, tc := range []struct {
 		name          string
 		in            string
@@ -101,7 +99,6 @@ func TestConvertDockerPort(t *testing.T) {
 			expectError: "invalid port",
 		},
 	} {
-		//nolint: paralleltest // variable recapture no longer required
 		t.Run(tc.name, func(t *testing.T) {
 			t.Parallel()
 			actualPort, actualNetwork, actualErr := convertDockerPort(tc.in)
@@ -151,7 +148,6 @@ func TestConvertDockerVolume(t *testing.T) {
 			expectError: "invalid volume",
 		},
 	} {
-		tc := tc
 		t.Run(tc.name, func(t *testing.T) {
 			t.Parallel()
 		})

agent/agentcontainers/containers_test.go

@@ -78,7 +78,7 @@ func TestIntegrationDocker(t *testing.T) {
 		return ok && ct.Container.State.Running
 	}, testutil.WaitShort, testutil.IntervalSlow, "Container did not start in time")
 
-	dcl := agentcontainers.NewDocker(agentexec.DefaultExecer)
+	dcl := agentcontainers.NewDockerCLI(agentexec.DefaultExecer)
 	ctx := testutil.Context(t, testutil.WaitShort)
 	actual, err := dcl.List(ctx)
 	require.NoError(t, err, "Could not list containers")

agent/agentcontainers/dcspec/gen.sh

@@ -61,7 +61,7 @@ fi
 exec 3>&-
 
 # Format the generated code.
-go run mvdan.cc/gofumpt@v0.4.0 -w -l "${TMPDIR}/${DEST_FILENAME}"
+go run mvdan.cc/gofumpt@v0.8.0 -w -l "${TMPDIR}/${DEST_FILENAME}"
 
 # Add a header so that Go recognizes this as a generated file.
 if grep -q -- "\[-i extension\]" < <(sed -h 2>&1); then

agent/agentcontainers/devcontainer.go

@@ -2,10 +2,10 @@ package agentcontainers
 
 import (
 	"context"
-	"fmt"
 	"os"
 	"path/filepath"
-	"strings"
+
+	"github.com/google/uuid"
 
 	"cdr.dev/slog"
 	"github.com/coder/coder/v2/codersdk"
@@ -18,35 +18,25 @@ const (
 	// DevcontainerConfigFileLabel is the label that contains the path to
 	// the devcontainer.json configuration file.
 	DevcontainerConfigFileLabel = "devcontainer.config_file"
+	// DevcontainerIsTestRunLabel is set if the devcontainer is part of a test
+	// and should be excluded.
+	DevcontainerIsTestRunLabel = "devcontainer.is_test_run"
+	// The default workspace folder inside the devcontainer.
+	DevcontainerDefaultContainerWorkspaceFolder = "/workspaces"
 )
 
-const devcontainerUpScriptTemplate = `
-if ! which devcontainer > /dev/null 2>&1; then
-  echo "ERROR: Unable to start devcontainer, @devcontainers/cli is not installed or not found in \$PATH." 1>&2
-  echo "Please install @devcontainers/cli by running \"npm install -g @devcontainers/cli\" or by using the \"devcontainers-cli\" Coder module." 1>&2
-  exit 1
-fi
-devcontainer up %s
-`
-
-// ExtractAndInitializeDevcontainerScripts extracts devcontainer scripts from
-// the given scripts and devcontainers. The devcontainer scripts are removed
-// from the returned scripts so that they can be run separately.
-//
-// Dev Containers have an inherent dependency on start scripts, since they
-// initialize the workspace (e.g. git clone, npm install, etc). This is
-// important if e.g. a Coder module to install @devcontainer/cli is used.
-func ExtractAndInitializeDevcontainerScripts(
+func ExtractDevcontainerScripts(
 	devcontainers []codersdk.WorkspaceAgentDevcontainer,
 	scripts []codersdk.WorkspaceAgentScript,
-) (filteredScripts []codersdk.WorkspaceAgentScript, devcontainerScripts []codersdk.WorkspaceAgentScript) {
+) (filteredScripts []codersdk.WorkspaceAgentScript, devcontainerScripts map[uuid.UUID]codersdk.WorkspaceAgentScript) {
+	devcontainerScripts = make(map[uuid.UUID]codersdk.WorkspaceAgentScript)
 ScriptLoop:
 	for _, script := range scripts {
 		for _, dc := range devcontainers {
 			// The devcontainer scripts match the devcontainer ID for
 			// identification.
 			if script.ID == dc.ID {
-				devcontainerScripts = append(devcontainerScripts, devcontainerStartupScript(dc, script))
+				devcontainerScripts[dc.ID] = script
 				continue ScriptLoop
 			}
 		}
@@ -57,24 +47,6 @@ ScriptLoop:
 	return filteredScripts, devcontainerScripts
 }
 
-func devcontainerStartupScript(dc codersdk.WorkspaceAgentDevcontainer, script codersdk.WorkspaceAgentScript) codersdk.WorkspaceAgentScript {
-	args := []string{
-		"--log-format json",
-		fmt.Sprintf("--workspace-folder %q", dc.WorkspaceFolder),
-	}
-	if dc.ConfigPath != "" {
-		args = append(args, fmt.Sprintf("--config %q", dc.ConfigPath))
-	}
-	cmd := fmt.Sprintf(devcontainerUpScriptTemplate, strings.Join(args, " "))
-	// Force the script to run in /bin/sh, since some shells (e.g. fish)
-	// don't support the script.
-	script.Script = fmt.Sprintf("/bin/sh -c '%s'", cmd)
-	// Disable RunOnStart, scripts have this set so that when devcontainers
-	// have not been enabled, a warning will be surfaced in the agent logs.
-	script.RunOnStart = false
-	return script
-}
-
 // ExpandAllDevcontainerPaths expands all devcontainer paths in the given
 // devcontainers. This is required by the devcontainer CLI, which requires
 // absolute paths for the workspace folder and config path.

agent/agentcontainers/devcontainer_test.go

@@ -1,274 +0,0 @@
-package agentcontainers_test
-
-import (
-	"path/filepath"
-	"strings"
-	"testing"
-
-	"github.com/google/go-cmp/cmp"
-	"github.com/google/go-cmp/cmp/cmpopts"
-	"github.com/google/uuid"
-	"github.com/stretchr/testify/require"
-
-	"cdr.dev/slog/sloggers/slogtest"
-	"github.com/coder/coder/v2/agent/agentcontainers"
-	"github.com/coder/coder/v2/codersdk"
-)
-
-func TestExtractAndInitializeDevcontainerScripts(t *testing.T) {
-	t.Parallel()
-
-	scriptIDs := []uuid.UUID{uuid.New(), uuid.New()}
-	devcontainerIDs := []uuid.UUID{uuid.New(), uuid.New()}
-
-	type args struct {
-		expandPath    func(string) (string, error)
-		devcontainers []codersdk.WorkspaceAgentDevcontainer
-		scripts       []codersdk.WorkspaceAgentScript
-	}
-	tests := []struct {
-		name                    string
-		args                    args
-		wantFilteredScripts     []codersdk.WorkspaceAgentScript
-		wantDevcontainerScripts []codersdk.WorkspaceAgentScript
-
-		skipOnWindowsDueToPathSeparator bool
-	}{
-		{
-			name: "no scripts",
-			args: args{
-				expandPath:    nil,
-				devcontainers: nil,
-				scripts:       nil,
-			},
-			wantFilteredScripts:     nil,
-			wantDevcontainerScripts: nil,
-		},
-		{
-			name: "no devcontainers",
-			args: args{
-				expandPath:    nil,
-				devcontainers: nil,
-				scripts: []codersdk.WorkspaceAgentScript{
-					{ID: scriptIDs[0]},
-					{ID: scriptIDs[1]},
-				},
-			},
-			wantFilteredScripts: []codersdk.WorkspaceAgentScript{
-				{ID: scriptIDs[0]},
-				{ID: scriptIDs[1]},
-			},
-			wantDevcontainerScripts: nil,
-		},
-		{
-			name: "no scripts match devcontainers",
-			args: args{
-				expandPath: nil,
-				devcontainers: []codersdk.WorkspaceAgentDevcontainer{
-					{ID: devcontainerIDs[0]},
-					{ID: devcontainerIDs[1]},
-				},
-				scripts: []codersdk.WorkspaceAgentScript{
-					{ID: scriptIDs[0]},
-					{ID: scriptIDs[1]},
-				},
-			},
-			wantFilteredScripts: []codersdk.WorkspaceAgentScript{
-				{ID: scriptIDs[0]},
-				{ID: scriptIDs[1]},
-			},
-			wantDevcontainerScripts: nil,
-		},
-		{
-			name: "scripts match devcontainers and sets RunOnStart=false",
-			args: args{
-				expandPath: nil,
-				devcontainers: []codersdk.WorkspaceAgentDevcontainer{
-					{ID: devcontainerIDs[0], WorkspaceFolder: "workspace1"},
-					{ID: devcontainerIDs[1], WorkspaceFolder: "workspace2"},
-				},
-				scripts: []codersdk.WorkspaceAgentScript{
-					{ID: scriptIDs[0], RunOnStart: true},
-					{ID: scriptIDs[1], RunOnStart: true},
-					{ID: devcontainerIDs[0], RunOnStart: true},
-					{ID: devcontainerIDs[1], RunOnStart: true},
-				},
-			},
-			wantFilteredScripts: []codersdk.WorkspaceAgentScript{
-				{ID: scriptIDs[0], RunOnStart: true},
-				{ID: scriptIDs[1], RunOnStart: true},
-			},
-			wantDevcontainerScripts: []codersdk.WorkspaceAgentScript{
-				{
-					ID:         devcontainerIDs[0],
-					Script:     "devcontainer up --log-format json --workspace-folder \"workspace1\"",
-					RunOnStart: false,
-				},
-				{
-					ID:         devcontainerIDs[1],
-					Script:     "devcontainer up --log-format json --workspace-folder \"workspace2\"",
-					RunOnStart: false,
-				},
-			},
-		},
-		{
-			name: "scripts match devcontainers with config path",
-			args: args{
-				expandPath: nil,
-				devcontainers: []codersdk.WorkspaceAgentDevcontainer{
-					{
-						ID:              devcontainerIDs[0],
-						WorkspaceFolder: "workspace1",
-						ConfigPath:      "config1",
-					},
-					{
-						ID:              devcontainerIDs[1],
-						WorkspaceFolder: "workspace2",
-						ConfigPath:      "config2",
-					},
-				},
-				scripts: []codersdk.WorkspaceAgentScript{
-					{ID: devcontainerIDs[0]},
-					{ID: devcontainerIDs[1]},
-				},
-			},
-			wantFilteredScripts: []codersdk.WorkspaceAgentScript{},
-			wantDevcontainerScripts: []codersdk.WorkspaceAgentScript{
-				{
-					ID:         devcontainerIDs[0],
-					Script:     "devcontainer up --log-format json --workspace-folder \"workspace1\" --config \"workspace1/config1\"",
-					RunOnStart: false,
-				},
-				{
-					ID:         devcontainerIDs[1],
-					Script:     "devcontainer up --log-format json --workspace-folder \"workspace2\" --config \"workspace2/config2\"",
-					RunOnStart: false,
-				},
-			},
-			skipOnWindowsDueToPathSeparator: true,
-		},
-		{
-			name: "scripts match devcontainers with expand path",
-			args: args{
-				expandPath: func(s string) (string, error) {
-					return "/home/" + s, nil
-				},
-				devcontainers: []codersdk.WorkspaceAgentDevcontainer{
-					{
-						ID:              devcontainerIDs[0],
-						WorkspaceFolder: "workspace1",
-						ConfigPath:      "config1",
-					},
-					{
-						ID:              devcontainerIDs[1],
-						WorkspaceFolder: "workspace2",
-						ConfigPath:      "config2",
-					},
-				},
-				scripts: []codersdk.WorkspaceAgentScript{
-					{ID: devcontainerIDs[0], RunOnStart: true},
-					{ID: devcontainerIDs[1], RunOnStart: true},
-				},
-			},
-			wantFilteredScripts: []codersdk.WorkspaceAgentScript{},
-			wantDevcontainerScripts: []codersdk.WorkspaceAgentScript{
-				{
-					ID:         devcontainerIDs[0],
-					Script:     "devcontainer up --log-format json --workspace-folder \"/home/workspace1\" --config \"/home/workspace1/config1\"",
-					RunOnStart: false,
-				},
-				{
-					ID:         devcontainerIDs[1],
-					Script:     "devcontainer up --log-format json --workspace-folder \"/home/workspace2\" --config \"/home/workspace2/config2\"",
-					RunOnStart: false,
-				},
-			},
-			skipOnWindowsDueToPathSeparator: true,
-		},
-		{
-			name: "expand config path when ~",
-			args: args{
-				expandPath: func(s string) (string, error) {
-					s = strings.Replace(s, "~/", "", 1)
-					if filepath.IsAbs(s) {
-						return s, nil
-					}
-					return "/home/" + s, nil
-				},
-				devcontainers: []codersdk.WorkspaceAgentDevcontainer{
-					{
-						ID:              devcontainerIDs[0],
-						WorkspaceFolder: "workspace1",
-						ConfigPath:      "~/config1",
-					},
-					{
-						ID:              devcontainerIDs[1],
-						WorkspaceFolder: "workspace2",
-						ConfigPath:      "/config2",
-					},
-				},
-				scripts: []codersdk.WorkspaceAgentScript{
-					{ID: devcontainerIDs[0], RunOnStart: true},
-					{ID: devcontainerIDs[1], RunOnStart: true},
-				},
-			},
-			wantFilteredScripts: []codersdk.WorkspaceAgentScript{},
-			wantDevcontainerScripts: []codersdk.WorkspaceAgentScript{
-				{
-					ID:         devcontainerIDs[0],
-					Script:     "devcontainer up --log-format json --workspace-folder \"/home/workspace1\" --config \"/home/config1\"",
-					RunOnStart: false,
-				},
-				{
-					ID:         devcontainerIDs[1],
-					Script:     "devcontainer up --log-format json --workspace-folder \"/home/workspace2\" --config \"/config2\"",
-					RunOnStart: false,
-				},
-			},
-			skipOnWindowsDueToPathSeparator: true,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			t.Parallel()
-			if tt.skipOnWindowsDueToPathSeparator && filepath.Separator == '\\' {
-				t.Skip("Skipping test on Windows due to path separator difference.")
-			}
-
-			logger := slogtest.Make(t, nil)
-			if tt.args.expandPath == nil {
-				tt.args.expandPath = func(s string) (string, error) {
-					return s, nil
-				}
-			}
-			gotFilteredScripts, gotDevcontainerScripts := agentcontainers.ExtractAndInitializeDevcontainerScripts(
-				agentcontainers.ExpandAllDevcontainerPaths(logger, tt.args.expandPath, tt.args.devcontainers),
-				tt.args.scripts,
-			)
-
-			if diff := cmp.Diff(tt.wantFilteredScripts, gotFilteredScripts, cmpopts.EquateEmpty()); diff != "" {
-				t.Errorf("ExtractAndInitializeDevcontainerScripts() gotFilteredScripts mismatch (-want +got):\n%s", diff)
-			}
-
-			// Preprocess the devcontainer scripts to remove scripting part.
-			for i := range gotDevcontainerScripts {
-				gotDevcontainerScripts[i].Script = textGrep("devcontainer up", gotDevcontainerScripts[i].Script)
-				require.NotEmpty(t, gotDevcontainerScripts[i].Script, "devcontainer up script not found")
-			}
-			if diff := cmp.Diff(tt.wantDevcontainerScripts, gotDevcontainerScripts); diff != "" {
-				t.Errorf("ExtractAndInitializeDevcontainerScripts() gotDevcontainerScripts mismatch (-want +got):\n%s", diff)
-			}
-		})
-	}
-}
-
-// textGrep returns matching lines from multiline string.
-func textGrep(want, got string) (filtered string) {
-	var lines []string
-	for _, line := range strings.Split(got, "\n") {
-		if strings.Contains(line, want) {
-			lines = append(lines, line)
-		}
-	}
-	return strings.Join(lines, "\n")
-}

agent/agentcontainers/devcontainercli.go

@@ -6,49 +6,208 @@ import (
 	"context"
 	"encoding/json"
 	"errors"
+	"fmt"
 	"io"
+	"slices"
+	"strings"
 
 	"golang.org/x/xerrors"
 
 	"cdr.dev/slog"
 	"github.com/coder/coder/v2/agent/agentexec"
+	"github.com/coder/coder/v2/codersdk"
 )
 
+// DevcontainerConfig is a wrapper around the output from `read-configuration`.
+// Unfortunately we cannot make use of `dcspec` as the output doesn't appear to
+// match.
+type DevcontainerConfig struct {
+	MergedConfiguration DevcontainerMergedConfiguration `json:"mergedConfiguration"`
+	Configuration       DevcontainerConfiguration       `json:"configuration"`
+	Workspace           DevcontainerWorkspace           `json:"workspace"`
+}
+
+type DevcontainerMergedConfiguration struct {
+	Customizations DevcontainerMergedCustomizations `json:"customizations,omitempty"`
+	Features       DevcontainerFeatures             `json:"features,omitempty"`
+}
+
+type DevcontainerMergedCustomizations struct {
+	Coder []CoderCustomization `json:"coder,omitempty"`
+}
+
+type DevcontainerFeatures map[string]any
+
+// OptionsAsEnvs converts the DevcontainerFeatures into a list of
+// environment variables that can be used to set feature options.
+// The format is FEATURE_<FEATURE_NAME>_OPTION_<OPTION_NAME>=<value>.
+// For example, if the feature is:
+//
+//		"ghcr.io/coder/devcontainer-features/code-server:1": {
+//	   "port": 9090,
+//	 }
+//
+// It will produce:
+//
+//	FEATURE_CODE_SERVER_OPTION_PORT=9090
+//
+// Note that the feature name is derived from the last part of the key,
+// so "ghcr.io/coder/devcontainer-features/code-server:1" becomes
+// "CODE_SERVER". The version part (e.g. ":1") is removed, and dashes in
+// the feature and option names are replaced with underscores.
+func (f DevcontainerFeatures) OptionsAsEnvs() []string {
+	var env []string
+	for k, v := range f {
+		vv, ok := v.(map[string]any)
+		if !ok {
+			continue
+		}
+		// Take the last part of the key as the feature name/path.
+		k = k[strings.LastIndex(k, "/")+1:]
+		// Remove ":" and anything following it.
+		if idx := strings.Index(k, ":"); idx != -1 {
+			k = k[:idx]
+		}
+		k = strings.ReplaceAll(k, "-", "_")
+		for k2, v2 := range vv {
+			k2 = strings.ReplaceAll(k2, "-", "_")
+			env = append(env, fmt.Sprintf("FEATURE_%s_OPTION_%s=%s", strings.ToUpper(k), strings.ToUpper(k2), fmt.Sprintf("%v", v2)))
+		}
+	}
+	slices.Sort(env)
+	return env
+}
+
+type DevcontainerConfiguration struct {
+	Customizations DevcontainerCustomizations `json:"customizations,omitempty"`
+}
+
+type DevcontainerCustomizations struct {
+	Coder CoderCustomization `json:"coder,omitempty"`
+}
+
+type CoderCustomization struct {
+	DisplayApps map[codersdk.DisplayApp]bool `json:"displayApps,omitempty"`
+	Apps        []SubAgentApp                `json:"apps,omitempty"`
+	Name        string                       `json:"name,omitempty"`
+	Ignore      bool                         `json:"ignore,omitempty"`
+	AutoStart   bool                         `json:"autoStart,omitempty"`
+}
+
+type DevcontainerWorkspace struct {
+	WorkspaceFolder string `json:"workspaceFolder"`
+}
+
 // DevcontainerCLI is an interface for the devcontainer CLI.
 type DevcontainerCLI interface {
 	Up(ctx context.Context, workspaceFolder, configPath string, opts ...DevcontainerCLIUpOptions) (id string, err error)
+	Exec(ctx context.Context, workspaceFolder, configPath string, cmd string, cmdArgs []string, opts ...DevcontainerCLIExecOptions) error
+	ReadConfig(ctx context.Context, workspaceFolder, configPath string, env []string, opts ...DevcontainerCLIReadConfigOptions) (DevcontainerConfig, error)
 }
 
-// DevcontainerCLIUpOptions are options for the devcontainer CLI up
+// DevcontainerCLIUpOptions are options for the devcontainer CLI Up
 // command.
-type DevcontainerCLIUpOptions func(*devcontainerCLIUpConfig)
+type DevcontainerCLIUpOptions func(*DevcontainerCLIUpConfig)
+
+type DevcontainerCLIUpConfig struct {
+	Args   []string // Additional arguments for the Up command.
+	Stdout io.Writer
+	Stderr io.Writer
+}
 
 // WithRemoveExistingContainer is an option to remove the existing
 // container.
 func WithRemoveExistingContainer() DevcontainerCLIUpOptions {
-	return func(o *devcontainerCLIUpConfig) {
-		o.removeExistingContainer = true
+	return func(o *DevcontainerCLIUpConfig) {
+		o.Args = append(o.Args, "--remove-existing-container")
 	}
 }
 
-// WithOutput sets stdout and stderr writers for Up command logs.
-func WithOutput(stdout, stderr io.Writer) DevcontainerCLIUpOptions {
-	return func(o *devcontainerCLIUpConfig) {
+// WithUpOutput sets additional stdout and stderr writers for logs
+// during Up operations.
+func WithUpOutput(stdout, stderr io.Writer) DevcontainerCLIUpOptions {
+	return func(o *DevcontainerCLIUpConfig) {
+		o.Stdout = stdout
+		o.Stderr = stderr
+	}
+}
+
+// DevcontainerCLIExecOptions are options for the devcontainer CLI Exec
+// command.
+type DevcontainerCLIExecOptions func(*DevcontainerCLIExecConfig)
+
+type DevcontainerCLIExecConfig struct {
+	Args   []string // Additional arguments for the Exec command.
+	Stdout io.Writer
+	Stderr io.Writer
+}
+
+// WithExecOutput sets additional stdout and stderr writers for logs
+// during Exec operations.
+func WithExecOutput(stdout, stderr io.Writer) DevcontainerCLIExecOptions {
+	return func(o *DevcontainerCLIExecConfig) {
+		o.Stdout = stdout
+		o.Stderr = stderr
+	}
+}
+
+// WithExecContainerID sets the container ID to target a specific
+// container.
+func WithExecContainerID(id string) DevcontainerCLIExecOptions {
+	return func(o *DevcontainerCLIExecConfig) {
+		o.Args = append(o.Args, "--container-id", id)
+	}
+}
+
+// WithRemoteEnv sets environment variables for the Exec command.
+func WithRemoteEnv(env ...string) DevcontainerCLIExecOptions {
+	return func(o *DevcontainerCLIExecConfig) {
+		for _, e := range env {
+			o.Args = append(o.Args, "--remote-env", e)
+		}
+	}
+}
+
+// DevcontainerCLIExecOptions are options for the devcontainer CLI ReadConfig
+// command.
+type DevcontainerCLIReadConfigOptions func(*devcontainerCLIReadConfigConfig)
+
+type devcontainerCLIReadConfigConfig struct {
+	stdout io.Writer
+	stderr io.Writer
+}
+
+// WithReadConfigOutput sets additional stdout and stderr writers for logs
+// during ReadConfig operations.
+func WithReadConfigOutput(stdout, stderr io.Writer) DevcontainerCLIReadConfigOptions {
+	return func(o *devcontainerCLIReadConfigConfig) {
 		o.stdout = stdout
 		o.stderr = stderr
 	}
 }
 
-type devcontainerCLIUpConfig struct {
-	removeExistingContainer bool
-	stdout                  io.Writer
-	stderr                  io.Writer
+func applyDevcontainerCLIUpOptions(opts []DevcontainerCLIUpOptions) DevcontainerCLIUpConfig {
+	conf := DevcontainerCLIUpConfig{Stdout: io.Discard, Stderr: io.Discard}
+	for _, opt := range opts {
+		if opt != nil {
+			opt(&conf)
+		}
+	}
+	return conf
 }
 
-func applyDevcontainerCLIUpOptions(opts []DevcontainerCLIUpOptions) devcontainerCLIUpConfig {
-	conf := devcontainerCLIUpConfig{
-		removeExistingContainer: false,
+func applyDevcontainerCLIExecOptions(opts []DevcontainerCLIExecOptions) DevcontainerCLIExecConfig {
+	conf := DevcontainerCLIExecConfig{Stdout: io.Discard, Stderr: io.Discard}
+	for _, opt := range opts {
+		if opt != nil {
+			opt(&conf)
+		}
 	}
+	return conf
+}
+
+func applyDevcontainerCLIReadConfigOptions(opts []DevcontainerCLIReadConfigOptions) devcontainerCLIReadConfigConfig {
+	conf := devcontainerCLIReadConfigConfig{stdout: io.Discard, stderr: io.Discard}
 	for _, opt := range opts {
 		if opt != nil {
 			opt(&conf)
@@ -73,7 +232,7 @@ func NewDevcontainerCLI(logger slog.Logger, execer agentexec.Execer) Devcontaine
 
 func (d *devcontainerCLI) Up(ctx context.Context, workspaceFolder, configPath string, opts ...DevcontainerCLIUpOptions) (string, error) {
 	conf := applyDevcontainerCLIUpOptions(opts)
-	logger := d.logger.With(slog.F("workspace_folder", workspaceFolder), slog.F("config_path", configPath), slog.F("recreate", conf.removeExistingContainer))
+	logger := d.logger.With(slog.F("workspace_folder", workspaceFolder), slog.F("config_path", configPath))
 
 	args := []string{
 		"up",
@@ -83,33 +242,35 @@ func (d *devcontainerCLI) Up(ctx context.Context, workspaceFolder, configPath st
 	if configPath != "" {
 		args = append(args, "--config", configPath)
 	}
-	if conf.removeExistingContainer {
-		args = append(args, "--remove-existing-container")
-	}
+	args = append(args, conf.Args...)
 	cmd := d.execer.CommandContext(ctx, "devcontainer", args...)
 
 	// Capture stdout for parsing and stream logs for both default and provided writers.
 	var stdoutBuf bytes.Buffer
-	stdoutWriters := []io.Writer{&stdoutBuf, &devcontainerCLILogWriter{ctx: ctx, logger: logger.With(slog.F("stdout", true))}}
-	if conf.stdout != nil {
-		stdoutWriters = append(stdoutWriters, conf.stdout)
-	}
-	cmd.Stdout = io.MultiWriter(stdoutWriters...)
+	cmd.Stdout = io.MultiWriter(
+		&stdoutBuf,
+		&devcontainerCLILogWriter{
+			ctx:    ctx,
+			logger: logger.With(slog.F("stdout", true)),
+			writer: conf.Stdout,
+		},
+	)
 	// Stream stderr logs and provided writer if any.
-	stderrWriters := []io.Writer{&devcontainerCLILogWriter{ctx: ctx, logger: logger.With(slog.F("stderr", true))}}
-	if conf.stderr != nil {
-		stderrWriters = append(stderrWriters, conf.stderr)
+	cmd.Stderr = &devcontainerCLILogWriter{
+		ctx:    ctx,
+		logger: logger.With(slog.F("stderr", true)),
+		writer: conf.Stderr,
 	}
-	cmd.Stderr = io.MultiWriter(stderrWriters...)
 
 	if err := cmd.Run(); err != nil {
-		if _, err2 := parseDevcontainerCLILastLine(ctx, logger, stdoutBuf.Bytes()); err2 != nil {
+		_, err2 := parseDevcontainerCLILastLine[devcontainerCLIResult](ctx, logger, stdoutBuf.Bytes())
+		if err2 != nil {
 			err = errors.Join(err, err2)
 		}
 		return "", err
 	}
 
-	result, err := parseDevcontainerCLILastLine(ctx, logger, stdoutBuf.Bytes())
+	result, err := parseDevcontainerCLILastLine[devcontainerCLIResult](ctx, logger, stdoutBuf.Bytes())
 	if err != nil {
 		return "", err
 	}
@@ -117,9 +278,92 @@ func (d *devcontainerCLI) Up(ctx context.Context, workspaceFolder, configPath st
 	return result.ContainerID, nil
 }
 
+func (d *devcontainerCLI) Exec(ctx context.Context, workspaceFolder, configPath string, cmd string, cmdArgs []string, opts ...DevcontainerCLIExecOptions) error {
+	conf := applyDevcontainerCLIExecOptions(opts)
+	logger := d.logger.With(slog.F("workspace_folder", workspaceFolder), slog.F("config_path", configPath))
+
+	args := []string{"exec"}
+	// For now, always set workspace folder even if --container-id is provided.
+	// Otherwise the environment of exec will be incomplete, like `pwd` will be
+	// /home/coder instead of /workspaces/coder. The downside is that the local
+	// `devcontainer.json` config will overwrite settings serialized in the
+	// container label.
+	if workspaceFolder != "" {
+		args = append(args, "--workspace-folder", workspaceFolder)
+	}
+	if configPath != "" {
+		args = append(args, "--config", configPath)
+	}
+	args = append(args, conf.Args...)
+	args = append(args, cmd)
+	args = append(args, cmdArgs...)
+	c := d.execer.CommandContext(ctx, "devcontainer", args...)
+
+	c.Stdout = io.MultiWriter(conf.Stdout, &devcontainerCLILogWriter{
+		ctx:    ctx,
+		logger: logger.With(slog.F("stdout", true)),
+		writer: io.Discard,
+	})
+	c.Stderr = io.MultiWriter(conf.Stderr, &devcontainerCLILogWriter{
+		ctx:    ctx,
+		logger: logger.With(slog.F("stderr", true)),
+		writer: io.Discard,
+	})
+
+	if err := c.Run(); err != nil {
+		return xerrors.Errorf("devcontainer exec failed: %w", err)
+	}
+
+	return nil
+}
+
+func (d *devcontainerCLI) ReadConfig(ctx context.Context, workspaceFolder, configPath string, env []string, opts ...DevcontainerCLIReadConfigOptions) (DevcontainerConfig, error) {
+	conf := applyDevcontainerCLIReadConfigOptions(opts)
+	logger := d.logger.With(slog.F("workspace_folder", workspaceFolder), slog.F("config_path", configPath))
+
+	args := []string{"read-configuration", "--include-merged-configuration"}
+	if workspaceFolder != "" {
+		args = append(args, "--workspace-folder", workspaceFolder)
+	}
+	if configPath != "" {
+		args = append(args, "--config", configPath)
+	}
+
+	c := d.execer.CommandContext(ctx, "devcontainer", args...)
+	c.Env = append(c.Env, env...)
+
+	var stdoutBuf bytes.Buffer
+	c.Stdout = io.MultiWriter(
+		&stdoutBuf,
+		&devcontainerCLILogWriter{
+			ctx:    ctx,
+			logger: logger.With(slog.F("stdout", true)),
+			writer: conf.stdout,
+		},
+	)
+	c.Stderr = &devcontainerCLILogWriter{
+		ctx:    ctx,
+		logger: logger.With(slog.F("stderr", true)),
+		writer: conf.stderr,
+	}
+
+	if err := c.Run(); err != nil {
+		return DevcontainerConfig{}, xerrors.Errorf("devcontainer read-configuration failed: %w", err)
+	}
+
+	config, err := parseDevcontainerCLILastLine[DevcontainerConfig](ctx, logger, stdoutBuf.Bytes())
+	if err != nil {
+		return DevcontainerConfig{}, err
+	}
+
+	return config, nil
+}
+
 // parseDevcontainerCLILastLine parses the last line of the devcontainer CLI output
 // which is a JSON object.
-func parseDevcontainerCLILastLine(ctx context.Context, logger slog.Logger, p []byte) (result devcontainerCLIResult, err error) {
+func parseDevcontainerCLILastLine[T any](ctx context.Context, logger slog.Logger, p []byte) (T, error) {
+	var result T
+
 	s := bufio.NewScanner(bytes.NewReader(p))
 	var lastLine []byte
 	for s.Scan() {
@@ -129,19 +373,19 @@ func parseDevcontainerCLILastLine(ctx context.Context, logger slog.Logger, p []b
 		}
 		lastLine = b
 	}
-	if err = s.Err(); err != nil {
+	if err := s.Err(); err != nil {
 		return result, err
 	}
 	if len(lastLine) == 0 || lastLine[0] != '{' {
 		logger.Error(ctx, "devcontainer result is not json", slog.F("result", string(lastLine)))
 		return result, xerrors.Errorf("devcontainer result is not json: %q", string(lastLine))
 	}
-	if err = json.Unmarshal(lastLine, &result); err != nil {
+	if err := json.Unmarshal(lastLine, &result); err != nil {
 		logger.Error(ctx, "parse devcontainer result failed", slog.Error(err), slog.F("result", string(lastLine)))
 		return result, err
 	}
 
-	return result, result.Err()
+	return result, nil
 }
 
 // devcontainerCLIResult is the result of the devcontainer CLI command.
@@ -160,6 +404,18 @@ type devcontainerCLIResult struct {
 	Description string `json:"description"`
 }
 
+func (r *devcontainerCLIResult) UnmarshalJSON(data []byte) error {
+	type wrapperResult devcontainerCLIResult
+
+	var wrappedResult wrapperResult
+	if err := json.Unmarshal(data, &wrappedResult); err != nil {
+		return err
+	}
+
+	*r = devcontainerCLIResult(wrappedResult)
+	return r.Err()
+}
+
 func (r devcontainerCLIResult) Err() error {
 	if r.Outcome == "success" {
 		return nil
@@ -182,6 +438,7 @@ type devcontainerCLIJSONLogLine struct {
 type devcontainerCLILogWriter struct {
 	ctx    context.Context
 	logger slog.Logger
+	writer io.Writer
 }
 
 func (l *devcontainerCLILogWriter) Write(p []byte) (n int, err error) {
@@ -202,8 +459,20 @@ func (l *devcontainerCLILogWriter) Write(p []byte) (n int, err error) {
 		}
 		if logLine.Level >= 3 {
 			l.logger.Info(l.ctx, "@devcontainer/cli", slog.F("line", string(line)))
+			_, _ = l.writer.Write([]byte(strings.TrimSpace(logLine.Text) + "\n"))
 			continue
 		}
+		// If we've successfully parsed the final log line, it will successfully parse
+		// but will not fill out any of the fields for `logLine`. In this scenario we
+		// assume it is the final log line, unmarshal it as that, and check if the
+		// outcome is a non-empty string.
+		if logLine.Level == 0 {
+			var lastLine devcontainerCLIResult
+			if err := json.Unmarshal(line, &lastLine); err == nil && lastLine.Outcome != "" {
+				_, _ = l.writer.Write(line)
+				_, _ = l.writer.Write([]byte{'\n'})
+			}
+		}
 		l.logger.Debug(l.ctx, "@devcontainer/cli", slog.F("line", string(line)))
 	}
 	if err := s.Err(); err != nil {

agent/agentcontainers/devcontainercli_test.go

@@ -3,6 +3,7 @@ package agentcontainers_test
 import (
 	"bytes"
 	"context"
+	"encoding/json"
 	"errors"
 	"flag"
 	"fmt"
@@ -10,9 +11,11 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"runtime"
 	"strings"
 	"testing"
 
+	"github.com/google/go-cmp/cmp"
 	"github.com/ory/dockertest/v3"
 	"github.com/ory/dockertest/v3/docker"
 	"github.com/stretchr/testify/assert"
@@ -22,6 +25,7 @@ import (
 	"cdr.dev/slog/sloggers/slogtest"
 	"github.com/coder/coder/v2/agent/agentcontainers"
 	"github.com/coder/coder/v2/agent/agentexec"
+	"github.com/coder/coder/v2/codersdk"
 	"github.com/coder/coder/v2/pty"
 	"github.com/coder/coder/v2/testutil"
 )
@@ -126,9 +130,209 @@ func TestDevcontainerCLI_ArgsAndParsing(t *testing.T) {
 			})
 		}
 	})
+
+	t.Run("Exec", func(t *testing.T) {
+		t.Parallel()
+
+		tests := []struct {
+			name            string
+			workspaceFolder string
+			configPath      string
+			cmd             string
+			cmdArgs         []string
+			opts            []agentcontainers.DevcontainerCLIExecOptions
+			wantArgs        string
+			wantError       bool
+		}{
+			{
+				name:            "simple command",
+				workspaceFolder: "/test/workspace",
+				configPath:      "",
+				cmd:             "echo",
+				cmdArgs:         []string{"hello"},
+				wantArgs:        "exec --workspace-folder /test/workspace echo hello",
+				wantError:       false,
+			},
+			{
+				name:            "command with multiple args",
+				workspaceFolder: "/test/workspace",
+				configPath:      "/test/config.json",
+				cmd:             "ls",
+				cmdArgs:         []string{"-la", "/workspace"},
+				wantArgs:        "exec --workspace-folder /test/workspace --config /test/config.json ls -la /workspace",
+				wantError:       false,
+			},
+			{
+				name:            "empty command args",
+				workspaceFolder: "/test/workspace",
+				configPath:      "",
+				cmd:             "bash",
+				cmdArgs:         nil,
+				wantArgs:        "exec --workspace-folder /test/workspace bash",
+				wantError:       false,
+			},
+			{
+				name:            "workspace not found",
+				workspaceFolder: "/nonexistent/workspace",
+				configPath:      "",
+				cmd:             "echo",
+				cmdArgs:         []string{"test"},
+				wantArgs:        "exec --workspace-folder /nonexistent/workspace echo test",
+				wantError:       true,
+			},
+			{
+				name:            "with container ID",
+				workspaceFolder: "/test/workspace",
+				configPath:      "",
+				cmd:             "echo",
+				cmdArgs:         []string{"hello"},
+				opts:            []agentcontainers.DevcontainerCLIExecOptions{agentcontainers.WithExecContainerID("test-container-123")},
+				wantArgs:        "exec --workspace-folder /test/workspace --container-id test-container-123 echo hello",
+				wantError:       false,
+			},
+			{
+				name:            "with container ID and config",
+				workspaceFolder: "/test/workspace",
+				configPath:      "/test/config.json",
+				cmd:             "bash",
+				cmdArgs:         []string{"-c", "ls -la"},
+				opts:            []agentcontainers.DevcontainerCLIExecOptions{agentcontainers.WithExecContainerID("my-container")},
+				wantArgs:        "exec --workspace-folder /test/workspace --config /test/config.json --container-id my-container bash -c ls -la",
+				wantError:       false,
+			},
+			{
+				name:            "with container ID and output capture",
+				workspaceFolder: "/test/workspace",
+				configPath:      "",
+				cmd:             "cat",
+				cmdArgs:         []string{"/etc/hostname"},
+				opts: []agentcontainers.DevcontainerCLIExecOptions{
+					agentcontainers.WithExecContainerID("test-container-789"),
+				},
+				wantArgs:  "exec --workspace-folder /test/workspace --container-id test-container-789 cat /etc/hostname",
+				wantError: false,
+			},
+		}
+
+		for _, tt := range tests {
+			t.Run(tt.name, func(t *testing.T) {
+				t.Parallel()
+
+				ctx := testutil.Context(t, testutil.WaitMedium)
+
+				testExecer := &testDevcontainerExecer{
+					testExePath: testExePath,
+					wantArgs:    tt.wantArgs,
+					wantError:   tt.wantError,
+					logFile:     "", // Exec doesn't need log file parsing
+				}
+
+				dccli := agentcontainers.NewDevcontainerCLI(logger, testExecer)
+				err := dccli.Exec(ctx, tt.workspaceFolder, tt.configPath, tt.cmd, tt.cmdArgs, tt.opts...)
+				if tt.wantError {
+					assert.Error(t, err, "want error")
+				} else {
+					assert.NoError(t, err, "want no error")
+				}
+			})
+		}
+	})
+
+	t.Run("ReadConfig", func(t *testing.T) {
+		t.Parallel()
+
+		tests := []struct {
+			name            string
+			logFile         string
+			workspaceFolder string
+			configPath      string
+			opts            []agentcontainers.DevcontainerCLIReadConfigOptions
+			wantArgs        string
+			wantError       bool
+			wantConfig      agentcontainers.DevcontainerConfig
+		}{
+			{
+				name:            "WithCoderCustomization",
+				logFile:         "read-config-with-coder-customization.log",
+				workspaceFolder: "/test/workspace",
+				configPath:      "",
+				wantArgs:        "read-configuration --include-merged-configuration --workspace-folder /test/workspace",
+				wantError:       false,
+				wantConfig: agentcontainers.DevcontainerConfig{
+					MergedConfiguration: agentcontainers.DevcontainerMergedConfiguration{
+						Customizations: agentcontainers.DevcontainerMergedCustomizations{
+							Coder: []agentcontainers.CoderCustomization{
+								{
+									DisplayApps: map[codersdk.DisplayApp]bool{
+										codersdk.DisplayAppVSCodeDesktop: true,
+										codersdk.DisplayAppWebTerminal:   true,
+									},
+								},
+								{
+									DisplayApps: map[codersdk.DisplayApp]bool{
+										codersdk.DisplayAppVSCodeInsiders: true,
+										codersdk.DisplayAppWebTerminal:    false,
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+			{
+				name:            "WithoutCoderCustomization",
+				logFile:         "read-config-without-coder-customization.log",
+				workspaceFolder: "/test/workspace",
+				configPath:      "/test/config.json",
+				wantArgs:        "read-configuration --include-merged-configuration --workspace-folder /test/workspace --config /test/config.json",
+				wantError:       false,
+				wantConfig: agentcontainers.DevcontainerConfig{
+					MergedConfiguration: agentcontainers.DevcontainerMergedConfiguration{
+						Customizations: agentcontainers.DevcontainerMergedCustomizations{
+							Coder: nil,
+						},
+					},
+				},
+			},
+			{
+				name:            "FileNotFound",
+				logFile:         "read-config-error-not-found.log",
+				workspaceFolder: "/nonexistent/workspace",
+				configPath:      "",
+				wantArgs:        "read-configuration --include-merged-configuration --workspace-folder /nonexistent/workspace",
+				wantError:       true,
+				wantConfig:      agentcontainers.DevcontainerConfig{},
+			},
+		}
+
+		for _, tt := range tests {
+			t.Run(tt.name, func(t *testing.T) {
+				t.Parallel()
+
+				ctx := testutil.Context(t, testutil.WaitMedium)
+
+				testExecer := &testDevcontainerExecer{
+					testExePath: testExePath,
+					wantArgs:    tt.wantArgs,
+					wantError:   tt.wantError,
+					logFile:     filepath.Join("testdata", "devcontainercli", "readconfig", tt.logFile),
+				}
+
+				dccli := agentcontainers.NewDevcontainerCLI(logger, testExecer)
+				config, err := dccli.ReadConfig(ctx, tt.workspaceFolder, tt.configPath, []string{}, tt.opts...)
+				if tt.wantError {
+					assert.Error(t, err, "want error")
+					assert.Equal(t, agentcontainers.DevcontainerConfig{}, config, "expected empty config on error")
+				} else {
+					assert.NoError(t, err, "want no error")
+					assert.Equal(t, tt.wantConfig, config, "expected config to match")
+				}
+			})
+		}
+	})
 }
 
-// TestDevcontainerCLI_WithOutput tests that WithOutput captures CLI
+// TestDevcontainerCLI_WithOutput tests that WithUpOutput and WithExecOutput capture CLI
 // logs to provided writers.
 func TestDevcontainerCLI_WithOutput(t *testing.T) {
 	t.Parallel()
@@ -136,35 +340,81 @@ func TestDevcontainerCLI_WithOutput(t *testing.T) {
 	// Prepare test executable and logger.
 	testExePath, err := os.Executable()
 	require.NoError(t, err, "get test executable path")
-	logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}).Leveled(slog.LevelDebug)
-	ctx := testutil.Context(t, testutil.WaitMedium)
 
-	// Buffers to capture stdout and stderr.
-	outBuf := &bytes.Buffer{}
-	errBuf := &bytes.Buffer{}
+	t.Run("Up", func(t *testing.T) {
+		t.Parallel()
 
-	// Simulate CLI execution with a standard up.log file.
-	wantArgs := "up --log-format json --workspace-folder /test/workspace"
-	testExecer := &testDevcontainerExecer{
-		testExePath: testExePath,
-		wantArgs:    wantArgs,
-		wantError:   false,
-		logFile:     filepath.Join("testdata", "devcontainercli", "parse", "up.log"),
-	}
-	dccli := agentcontainers.NewDevcontainerCLI(logger, testExecer)
+		if runtime.GOOS == "windows" {
+			t.Skip("Windows uses CRLF line endings, golden file is LF")
+		}
 
-	// Call Up with WithOutput to capture CLI logs.
-	containerID, err := dccli.Up(ctx, "/test/workspace", "", agentcontainers.WithOutput(outBuf, errBuf))
-	require.NoError(t, err, "Up should succeed")
-	require.NotEmpty(t, containerID, "expected non-empty container ID")
+		// Buffers to capture stdout and stderr.
+		outBuf := &bytes.Buffer{}
+		errBuf := &bytes.Buffer{}
 
-	// Read expected log content.
-	expLog, err := os.ReadFile(filepath.Join("testdata", "devcontainercli", "parse", "up.log"))
-	require.NoError(t, err, "reading expected log file")
+		// Simulate CLI execution with a standard up.log file.
+		wantArgs := "up --log-format json --workspace-folder /test/workspace"
+		testExecer := &testDevcontainerExecer{
+			testExePath: testExePath,
+			wantArgs:    wantArgs,
+			wantError:   false,
+			logFile:     filepath.Join("testdata", "devcontainercli", "parse", "up.log"),
+		}
+		logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}).Leveled(slog.LevelDebug)
+		dccli := agentcontainers.NewDevcontainerCLI(logger, testExecer)
 
-	// Verify stdout buffer contains the CLI logs and stderr is empty.
-	assert.Equal(t, string(expLog), outBuf.String(), "stdout buffer should match CLI logs")
-	assert.Empty(t, errBuf.String(), "stderr buffer should be empty on success")
+		// Call Up with WithUpOutput to capture CLI logs.
+		ctx := testutil.Context(t, testutil.WaitMedium)
+		containerID, err := dccli.Up(ctx, "/test/workspace", "", agentcontainers.WithUpOutput(outBuf, errBuf))
+		require.NoError(t, err, "Up should succeed")
+		require.NotEmpty(t, containerID, "expected non-empty container ID")
+
+		// Read expected log content.
+		expLog, err := os.ReadFile(filepath.Join("testdata", "devcontainercli", "parse", "up.golden"))
+		require.NoError(t, err, "reading expected log file")
+
+		// Verify stdout buffer contains the CLI logs and stderr is empty.
+		assert.Equal(t, string(expLog), outBuf.String(), "stdout buffer should match CLI logs")
+		assert.Empty(t, errBuf.String(), "stderr buffer should be empty on success")
+	})
+
+	t.Run("Exec", func(t *testing.T) {
+		t.Parallel()
+
+		logFile := filepath.Join(t.TempDir(), "exec.log")
+		f, err := os.Create(logFile)
+		require.NoError(t, err, "create exec log file")
+		_, err = f.WriteString("exec command log\n")
+		require.NoError(t, err, "write to exec log file")
+		err = f.Close()
+		require.NoError(t, err, "close exec log file")
+
+		// Buffers to capture stdout and stderr.
+		outBuf := &bytes.Buffer{}
+		errBuf := &bytes.Buffer{}
+
+		// Simulate CLI execution for exec command with container ID.
+		wantArgs := "exec --workspace-folder /test/workspace --container-id test-container-456 echo hello"
+		testExecer := &testDevcontainerExecer{
+			testExePath: testExePath,
+			wantArgs:    wantArgs,
+			wantError:   false,
+			logFile:     logFile,
+		}
+		logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}).Leveled(slog.LevelDebug)
+		dccli := agentcontainers.NewDevcontainerCLI(logger, testExecer)
+
+		// Call Exec with WithExecOutput and WithContainerID to capture any command output.
+		ctx := testutil.Context(t, testutil.WaitMedium)
+		err = dccli.Exec(ctx, "/test/workspace", "", "echo", []string{"hello"},
+			agentcontainers.WithExecContainerID("test-container-456"),
+			agentcontainers.WithExecOutput(outBuf, errBuf),
+		)
+		require.NoError(t, err, "Exec should succeed")
+
+		assert.NotEmpty(t, outBuf.String(), "stdout buffer should not be empty for exec with log file")
+		assert.Empty(t, errBuf.String(), "stderr buffer should be empty")
+	})
 }
 
 // testDevcontainerExecer implements the agentexec.Execer interface for testing.
@@ -243,13 +493,16 @@ func TestDevcontainerHelperProcess(t *testing.T) {
 	}
 
 	logFilePath := os.Getenv("TEST_DEVCONTAINER_LOG_FILE")
-	output, err := os.ReadFile(logFilePath)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "Reading log file %s failed: %v\n", logFilePath, err)
-		os.Exit(2)
+	if logFilePath != "" {
+		// Read and output log file for commands that need it (like "up")
+		output, err := os.ReadFile(logFilePath)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Reading log file %s failed: %v\n", logFilePath, err)
+			os.Exit(2)
+		}
+		_, _ = io.Copy(os.Stdout, bytes.NewReader(output))
 	}
 
-	_, _ = io.Copy(os.Stdout, bytes.NewReader(output))
 	if os.Getenv("TEST_DEVCONTAINER_WANT_ERROR") == "true" {
 		os.Exit(1)
 	}
@@ -340,7 +593,7 @@ func setupDevcontainerWorkspace(t *testing.T, workspaceFolder string) string {
 	"containerEnv": {
 		"TEST_CONTAINER": "true"
 	},
-	"runArgs": ["--label", "com.coder.test=devcontainercli"]
+	"runArgs": ["--label=com.coder.test=devcontainercli", "--label=` + agentcontainers.DevcontainerIsTestRunLabel + `=true"]
 }`
 	err = os.WriteFile(configPath, []byte(content), 0o600)
 	require.NoError(t, err, "create devcontainer.json file")
@@ -391,3 +644,107 @@ func removeDevcontainerByID(t *testing.T, pool *dockertest.Pool, id string) {
 		assert.NoError(t, err, "remove container failed")
 	}
 }
+
+func TestDevcontainerFeatures_OptionsAsEnvs(t *testing.T) {
+	t.Parallel()
+
+	realConfigJSON := `{
+		"mergedConfiguration": {
+			"features": {
+				"./code-server": {
+					"port": 9090
+				},
+				"ghcr.io/devcontainers/features/docker-in-docker:2": {
+					"moby": "false"
+				}
+			}
+		}
+	}`
+	var realConfig agentcontainers.DevcontainerConfig
+	err := json.Unmarshal([]byte(realConfigJSON), &realConfig)
+	require.NoError(t, err, "unmarshal JSON payload")
+
+	tests := []struct {
+		name     string
+		features agentcontainers.DevcontainerFeatures
+		want     []string
+	}{
+		{
+			name: "code-server feature",
+			features: agentcontainers.DevcontainerFeatures{
+				"./code-server": map[string]any{
+					"port": 9090,
+				},
+			},
+			want: []string{
+				"FEATURE_CODE_SERVER_OPTION_PORT=9090",
+			},
+		},
+		{
+			name: "docker-in-docker feature",
+			features: agentcontainers.DevcontainerFeatures{
+				"ghcr.io/devcontainers/features/docker-in-docker:2": map[string]any{
+					"moby": "false",
+				},
+			},
+			want: []string{
+				"FEATURE_DOCKER_IN_DOCKER_OPTION_MOBY=false",
+			},
+		},
+		{
+			name: "multiple features with multiple options",
+			features: agentcontainers.DevcontainerFeatures{
+				"./code-server": map[string]any{
+					"port":     9090,
+					"password": "secret",
+				},
+				"ghcr.io/devcontainers/features/docker-in-docker:2": map[string]any{
+					"moby":                        "false",
+					"docker-dash-compose-version": "v2",
+				},
+			},
+			want: []string{
+				"FEATURE_CODE_SERVER_OPTION_PASSWORD=secret",
+				"FEATURE_CODE_SERVER_OPTION_PORT=9090",
+				"FEATURE_DOCKER_IN_DOCKER_OPTION_DOCKER_DASH_COMPOSE_VERSION=v2",
+				"FEATURE_DOCKER_IN_DOCKER_OPTION_MOBY=false",
+			},
+		},
+		{
+			name: "feature with non-map value (should be ignored)",
+			features: agentcontainers.DevcontainerFeatures{
+				"./code-server": map[string]any{
+					"port": 9090,
+				},
+				"./invalid-feature": "not-a-map",
+			},
+			want: []string{
+				"FEATURE_CODE_SERVER_OPTION_PORT=9090",
+			},
+		},
+		{
+			name:     "real config example",
+			features: realConfig.MergedConfiguration.Features,
+			want: []string{
+				"FEATURE_CODE_SERVER_OPTION_PORT=9090",
+				"FEATURE_DOCKER_IN_DOCKER_OPTION_MOBY=false",
+			},
+		},
+		{
+			name:     "empty features",
+			features: agentcontainers.DevcontainerFeatures{},
+			want:     nil,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			got := tt.features.OptionsAsEnvs()
+			if diff := cmp.Diff(tt.want, got); diff != "" {
+				require.Failf(t, "OptionsAsEnvs() mismatch (-want +got):\n%s", diff)
+			}
+		})
+	}
+}

agent/agentcontainers/execer.go

@@ -0,0 +1,80 @@
+package agentcontainers
+
+import (
+	"context"
+	"fmt"
+	"os/exec"
+	"runtime"
+	"strings"
+
+	"cdr.dev/slog"
+	"github.com/coder/coder/v2/agent/agentexec"
+	"github.com/coder/coder/v2/agent/usershell"
+	"github.com/coder/coder/v2/pty"
+)
+
+// CommandEnv is a function that returns the shell, working directory,
+// and environment variables to use when executing a command. It takes
+// an EnvInfoer and a pre-existing environment slice as arguments.
+// This signature matches agentssh.Server.CommandEnv.
+type CommandEnv func(ei usershell.EnvInfoer, addEnv []string) (shell, dir string, env []string, err error)
+
+// commandEnvExecer is an agentexec.Execer that uses a CommandEnv to
+// determine the shell, working directory, and environment variables
+// for commands. It wraps another agentexec.Execer to provide the
+// necessary context.
+type commandEnvExecer struct {
+	logger     slog.Logger
+	commandEnv CommandEnv
+	execer     agentexec.Execer
+}
+
+func newCommandEnvExecer(
+	logger slog.Logger,
+	commandEnv CommandEnv,
+	execer agentexec.Execer,
+) *commandEnvExecer {
+	return &commandEnvExecer{
+		logger:     logger,
+		commandEnv: commandEnv,
+		execer:     execer,
+	}
+}
+
+// Ensure commandEnvExecer implements agentexec.Execer.
+var _ agentexec.Execer = (*commandEnvExecer)(nil)
+
+func (e *commandEnvExecer) prepare(ctx context.Context, inName string, inArgs ...string) (name string, args []string, dir string, env []string) {
+	shell, dir, env, err := e.commandEnv(nil, nil)
+	if err != nil {
+		e.logger.Error(ctx, "get command environment failed", slog.Error(err))
+		return inName, inArgs, "", nil
+	}
+
+	caller := "-c"
+	if runtime.GOOS == "windows" {
+		caller = "/c"
+	}
+	name = shell
+	for _, arg := range append([]string{inName}, inArgs...) {
+		args = append(args, fmt.Sprintf("%q", arg))
+	}
+	args = []string{caller, strings.Join(args, " ")}
+	return name, args, dir, env
+}
+
+func (e *commandEnvExecer) CommandContext(ctx context.Context, cmd string, args ...string) *exec.Cmd {
+	name, args, dir, env := e.prepare(ctx, cmd, args...)
+	c := e.execer.CommandContext(ctx, name, args...)
+	c.Dir = dir
+	c.Env = env
+	return c
+}
+
+func (e *commandEnvExecer) PTYCommandContext(ctx context.Context, cmd string, args ...string) *pty.Cmd {
+	name, args, dir, env := e.prepare(ctx, cmd, args...)
+	c := e.execer.PTYCommandContext(ctx, name, args...)
+	c.Dir = dir
+	c.Env = env
+	return c
+}

agent/agentcontainers/ignore/dir.go

@@ -0,0 +1,124 @@
+package ignore
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/go-git/go-git/v5/plumbing/format/config"
+	"github.com/go-git/go-git/v5/plumbing/format/gitignore"
+	"github.com/spf13/afero"
+	"golang.org/x/xerrors"
+
+	"cdr.dev/slog"
+)
+
+const (
+	gitconfigFile      = ".gitconfig"
+	gitignoreFile      = ".gitignore"
+	gitInfoExcludeFile = ".git/info/exclude"
+)
+
+func FilePathToParts(path string) []string {
+	components := []string{}
+
+	if path == "" {
+		return components
+	}
+
+	for segment := range strings.SplitSeq(filepath.Clean(path), string(filepath.Separator)) {
+		if segment != "" {
+			components = append(components, segment)
+		}
+	}
+
+	return components
+}
+
+func readIgnoreFile(fileSystem afero.Fs, path, ignore string) ([]gitignore.Pattern, error) {
+	var ps []gitignore.Pattern
+
+	data, err := afero.ReadFile(fileSystem, filepath.Join(path, ignore))
+	if err != nil && !errors.Is(err, os.ErrNotExist) {
+		return nil, err
+	}
+
+	for s := range strings.SplitSeq(string(data), "\n") {
+		if !strings.HasPrefix(s, "#") && len(strings.TrimSpace(s)) > 0 {
+			ps = append(ps, gitignore.ParsePattern(s, FilePathToParts(path)))
+		}
+	}
+
+	return ps, nil
+}
+
+func ReadPatterns(ctx context.Context, logger slog.Logger, fileSystem afero.Fs, path string) ([]gitignore.Pattern, error) {
+	var ps []gitignore.Pattern
+
+	subPs, err := readIgnoreFile(fileSystem, path, gitInfoExcludeFile)
+	if err != nil {
+		return nil, err
+	}
+
+	ps = append(ps, subPs...)
+
+	if err := afero.Walk(fileSystem, path, func(path string, info fs.FileInfo, err error) error {
+		if err != nil {
+			logger.Error(ctx, "encountered error while walking for git ignore files",
+				slog.F("path", path),
+				slog.Error(err))
+			return nil
+		}
+
+		if !info.IsDir() {
+			return nil
+		}
+
+		subPs, err := readIgnoreFile(fileSystem, path, gitignoreFile)
+		if err != nil {
+			return err
+		}
+
+		ps = append(ps, subPs...)
+
+		return nil
+	}); err != nil {
+		return nil, err
+	}
+
+	return ps, nil
+}
+
+func loadPatterns(fileSystem afero.Fs, path string) ([]gitignore.Pattern, error) {
+	data, err := afero.ReadFile(fileSystem, path)
+	if err != nil && !errors.Is(err, os.ErrNotExist) {
+		return nil, err
+	}
+
+	decoder := config.NewDecoder(bytes.NewBuffer(data))
+
+	conf := config.New()
+	if err := decoder.Decode(conf); err != nil {
+		return nil, xerrors.Errorf("decode config: %w", err)
+	}
+
+	excludes := conf.Section("core").Options.Get("excludesfile")
+	if excludes == "" {
+		return nil, nil
+	}
+
+	return readIgnoreFile(fileSystem, "", excludes)
+}
+
+func LoadGlobalPatterns(fileSystem afero.Fs) ([]gitignore.Pattern, error) {
+	home, err := os.UserHomeDir()
+	if err != nil {
+		return nil, err
+	}
+
+	return loadPatterns(fileSystem, filepath.Join(home, gitconfigFile))
+}

agent/agentcontainers/ignore/dir_test.go

@@ -0,0 +1,38 @@
+package ignore_test
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/coder/coder/v2/agent/agentcontainers/ignore"
+)
+
+func TestFilePathToParts(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		path     string
+		expected []string
+	}{
+		{"", []string{}},
+		{"/", []string{}},
+		{"foo", []string{"foo"}},
+		{"/foo", []string{"foo"}},
+		{"./foo/bar", []string{"foo", "bar"}},
+		{"../foo/bar", []string{"..", "foo", "bar"}},
+		{"foo/bar/baz", []string{"foo", "bar", "baz"}},
+		{"/foo/bar/baz", []string{"foo", "bar", "baz"}},
+		{"foo/../bar", []string{"bar"}},
+	}
+
+	for _, tt := range tests {
+		t.Run(fmt.Sprintf("`%s`", tt.path), func(t *testing.T) {
+			t.Parallel()
+
+			parts := ignore.FilePathToParts(tt.path)
+			require.Equal(t, tt.expected, parts)
+		})
+	}
+}

agent/agentcontainers/subagent.go

@@ -0,0 +1,294 @@
+package agentcontainers
+
+import (
+	"context"
+	"slices"
+
+	"github.com/google/uuid"
+	"golang.org/x/xerrors"
+
+	"cdr.dev/slog"
+
+	agentproto "github.com/coder/coder/v2/agent/proto"
+	"github.com/coder/coder/v2/codersdk"
+)
+
+// SubAgent represents an agent running in a dev container.
+type SubAgent struct {
+	ID              uuid.UUID
+	Name            string
+	AuthToken       uuid.UUID
+	Directory       string
+	Architecture    string
+	OperatingSystem string
+	Apps            []SubAgentApp
+	DisplayApps     []codersdk.DisplayApp
+}
+
+// CloneConfig makes a copy of SubAgent without ID and AuthToken. The
+// name is inherited from the devcontainer.
+func (s SubAgent) CloneConfig(dc codersdk.WorkspaceAgentDevcontainer) SubAgent {
+	return SubAgent{
+		Name:            dc.Name,
+		Directory:       s.Directory,
+		Architecture:    s.Architecture,
+		OperatingSystem: s.OperatingSystem,
+		DisplayApps:     slices.Clone(s.DisplayApps),
+		Apps:            slices.Clone(s.Apps),
+	}
+}
+
+func (s SubAgent) EqualConfig(other SubAgent) bool {
+	return s.Name == other.Name &&
+		s.Directory == other.Directory &&
+		s.Architecture == other.Architecture &&
+		s.OperatingSystem == other.OperatingSystem &&
+		slices.Equal(s.DisplayApps, other.DisplayApps) &&
+		slices.Equal(s.Apps, other.Apps)
+}
+
+type SubAgentApp struct {
+	Slug        string                            `json:"slug"`
+	Command     string                            `json:"command"`
+	DisplayName string                            `json:"displayName"`
+	External    bool                              `json:"external"`
+	Group       string                            `json:"group"`
+	HealthCheck SubAgentHealthCheck               `json:"healthCheck"`
+	Hidden      bool                              `json:"hidden"`
+	Icon        string                            `json:"icon"`
+	OpenIn      codersdk.WorkspaceAppOpenIn       `json:"openIn"`
+	Order       int32                             `json:"order"`
+	Share       codersdk.WorkspaceAppSharingLevel `json:"share"`
+	Subdomain   bool                              `json:"subdomain"`
+	URL         string                            `json:"url"`
+}
+
+func (app SubAgentApp) ToProtoApp() (*agentproto.CreateSubAgentRequest_App, error) {
+	proto := agentproto.CreateSubAgentRequest_App{
+		Slug:      app.Slug,
+		External:  &app.External,
+		Hidden:    &app.Hidden,
+		Order:     &app.Order,
+		Subdomain: &app.Subdomain,
+	}
+
+	if app.Command != "" {
+		proto.Command = &app.Command
+	}
+	if app.DisplayName != "" {
+		proto.DisplayName = &app.DisplayName
+	}
+	if app.Group != "" {
+		proto.Group = &app.Group
+	}
+	if app.Icon != "" {
+		proto.Icon = &app.Icon
+	}
+	if app.URL != "" {
+		proto.Url = &app.URL
+	}
+
+	if app.HealthCheck.URL != "" {
+		proto.Healthcheck = &agentproto.CreateSubAgentRequest_App_Healthcheck{
+			Interval:  app.HealthCheck.Interval,
+			Threshold: app.HealthCheck.Threshold,
+			Url:       app.HealthCheck.URL,
+		}
+	}
+
+	if app.OpenIn != "" {
+		switch app.OpenIn {
+		case codersdk.WorkspaceAppOpenInSlimWindow:
+			proto.OpenIn = agentproto.CreateSubAgentRequest_App_SLIM_WINDOW.Enum()
+		case codersdk.WorkspaceAppOpenInTab:
+			proto.OpenIn = agentproto.CreateSubAgentRequest_App_TAB.Enum()
+		default:
+			return nil, xerrors.Errorf("unexpected codersdk.WorkspaceAppOpenIn: %#v", app.OpenIn)
+		}
+	}
+
+	if app.Share != "" {
+		switch app.Share {
+		case codersdk.WorkspaceAppSharingLevelAuthenticated:
+			proto.Share = agentproto.CreateSubAgentRequest_App_AUTHENTICATED.Enum()
+		case codersdk.WorkspaceAppSharingLevelOwner:
+			proto.Share = agentproto.CreateSubAgentRequest_App_OWNER.Enum()
+		case codersdk.WorkspaceAppSharingLevelPublic:
+			proto.Share = agentproto.CreateSubAgentRequest_App_PUBLIC.Enum()
+		case codersdk.WorkspaceAppSharingLevelOrganization:
+			proto.Share = agentproto.CreateSubAgentRequest_App_ORGANIZATION.Enum()
+		default:
+			return nil, xerrors.Errorf("unexpected codersdk.WorkspaceAppSharingLevel: %#v", app.Share)
+		}
+	}
+
+	return &proto, nil
+}
+
+type SubAgentHealthCheck struct {
+	Interval  int32  `json:"interval"`
+	Threshold int32  `json:"threshold"`
+	URL       string `json:"url"`
+}
+
+// SubAgentClient is an interface for managing sub agents and allows
+// changing the implementation without having to deal with the
+// agentproto package directly.
+type SubAgentClient interface {
+	// List returns a list of all agents.
+	List(ctx context.Context) ([]SubAgent, error)
+	// Create adds a new agent.
+	Create(ctx context.Context, agent SubAgent) (SubAgent, error)
+	// Delete removes an agent by its ID.
+	Delete(ctx context.Context, id uuid.UUID) error
+}
+
+// NewSubAgentClient returns a SubAgentClient that uses the provided
+// agent API client.
+type subAgentAPIClient struct {
+	logger slog.Logger
+	api    agentproto.DRPCAgentClient26
+}
+
+var _ SubAgentClient = (*subAgentAPIClient)(nil)
+
+func NewSubAgentClientFromAPI(logger slog.Logger, agentAPI agentproto.DRPCAgentClient26) SubAgentClient {
+	if agentAPI == nil {
+		panic("developer error: agentAPI cannot be nil")
+	}
+	return &subAgentAPIClient{
+		logger: logger.Named("subagentclient"),
+		api:    agentAPI,
+	}
+}
+
+func (a *subAgentAPIClient) List(ctx context.Context) ([]SubAgent, error) {
+	a.logger.Debug(ctx, "listing sub agents")
+	resp, err := a.api.ListSubAgents(ctx, &agentproto.ListSubAgentsRequest{})
+	if err != nil {
+		return nil, err
+	}
+
+	agents := make([]SubAgent, len(resp.Agents))
+	for i, agent := range resp.Agents {
+		id, err := uuid.FromBytes(agent.GetId())
+		if err != nil {
+			return nil, err
+		}
+		authToken, err := uuid.FromBytes(agent.GetAuthToken())
+		if err != nil {
+			return nil, err
+		}
+		agents[i] = SubAgent{
+			ID:        id,
+			Name:      agent.GetName(),
+			AuthToken: authToken,
+		}
+	}
+	return agents, nil
+}
+
+func (a *subAgentAPIClient) Create(ctx context.Context, agent SubAgent) (_ SubAgent, err error) {
+	a.logger.Debug(ctx, "creating sub agent", slog.F("name", agent.Name), slog.F("directory", agent.Directory))
+
+	displayApps := make([]agentproto.CreateSubAgentRequest_DisplayApp, 0, len(agent.DisplayApps))
+	for _, displayApp := range agent.DisplayApps {
+		var app agentproto.CreateSubAgentRequest_DisplayApp
+		switch displayApp {
+		case codersdk.DisplayAppPortForward:
+			app = agentproto.CreateSubAgentRequest_PORT_FORWARDING_HELPER
+		case codersdk.DisplayAppSSH:
+			app = agentproto.CreateSubAgentRequest_SSH_HELPER
+		case codersdk.DisplayAppVSCodeDesktop:
+			app = agentproto.CreateSubAgentRequest_VSCODE
+		case codersdk.DisplayAppVSCodeInsiders:
+			app = agentproto.CreateSubAgentRequest_VSCODE_INSIDERS
+		case codersdk.DisplayAppWebTerminal:
+			app = agentproto.CreateSubAgentRequest_WEB_TERMINAL
+		default:
+			return SubAgent{}, xerrors.Errorf("unexpected codersdk.DisplayApp: %#v", displayApp)
+		}
+
+		displayApps = append(displayApps, app)
+	}
+
+	apps := make([]*agentproto.CreateSubAgentRequest_App, 0, len(agent.Apps))
+	for _, app := range agent.Apps {
+		protoApp, err := app.ToProtoApp()
+		if err != nil {
+			return SubAgent{}, xerrors.Errorf("convert app: %w", err)
+		}
+
+		apps = append(apps, protoApp)
+	}
+
+	resp, err := a.api.CreateSubAgent(ctx, &agentproto.CreateSubAgentRequest{
+		Name:            agent.Name,
+		Directory:       agent.Directory,
+		Architecture:    agent.Architecture,
+		OperatingSystem: agent.OperatingSystem,
+		DisplayApps:     displayApps,
+		Apps:            apps,
+	})
+	if err != nil {
+		return SubAgent{}, err
+	}
+	defer func() {
+		if err != nil {
+			// Best effort.
+			_, _ = a.api.DeleteSubAgent(ctx, &agentproto.DeleteSubAgentRequest{
+				Id: resp.GetAgent().GetId(),
+			})
+		}
+	}()
+
+	agent.Name = resp.GetAgent().GetName()
+	agent.ID, err = uuid.FromBytes(resp.GetAgent().GetId())
+	if err != nil {
+		return SubAgent{}, err
+	}
+	agent.AuthToken, err = uuid.FromBytes(resp.GetAgent().GetAuthToken())
+	if err != nil {
+		return SubAgent{}, err
+	}
+
+	for _, appError := range resp.GetAppCreationErrors() {
+		app := apps[appError.GetIndex()]
+
+		a.logger.Warn(ctx, "unable to create app",
+			slog.F("agent_name", agent.Name),
+			slog.F("agent_id", agent.ID),
+			slog.F("directory", agent.Directory),
+			slog.F("app_slug", app.Slug),
+			slog.F("field", appError.GetField()),
+			slog.F("error", appError.GetError()),
+		)
+	}
+
+	return agent, nil
+}
+
+func (a *subAgentAPIClient) Delete(ctx context.Context, id uuid.UUID) error {
+	a.logger.Debug(ctx, "deleting sub agent", slog.F("id", id.String()))
+	_, err := a.api.DeleteSubAgent(ctx, &agentproto.DeleteSubAgentRequest{
+		Id: id[:],
+	})
+	return err
+}
+
+// noopSubAgentClient is a SubAgentClient that does nothing.
+type noopSubAgentClient struct{}
+
+var _ SubAgentClient = noopSubAgentClient{}
+
+func (noopSubAgentClient) List(_ context.Context) ([]SubAgent, error) {
+	return nil, nil
+}
+
+func (noopSubAgentClient) Create(_ context.Context, _ SubAgent) (SubAgent, error) {
+	return SubAgent{}, xerrors.New("noopSubAgentClient does not support creating sub agents")
+}
+
+func (noopSubAgentClient) Delete(_ context.Context, _ uuid.UUID) error {
+	return xerrors.New("noopSubAgentClient does not support deleting sub agents")
+}

agent/agentcontainers/subagent_test.go

@@ -0,0 +1,308 @@
+package agentcontainers_test
+
+import (
+	"testing"
+
+	"github.com/google/uuid"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/coder/coder/v2/agent/agentcontainers"
+	"github.com/coder/coder/v2/agent/agenttest"
+	agentproto "github.com/coder/coder/v2/agent/proto"
+	"github.com/coder/coder/v2/coderd/util/ptr"
+	"github.com/coder/coder/v2/codersdk"
+	"github.com/coder/coder/v2/codersdk/agentsdk"
+	"github.com/coder/coder/v2/tailnet"
+	"github.com/coder/coder/v2/testutil"
+)
+
+func TestSubAgentClient_CreateWithDisplayApps(t *testing.T) {
+	t.Parallel()
+
+	t.Run("CreateWithDisplayApps", func(t *testing.T) {
+		t.Parallel()
+
+		tests := []struct {
+			name         string
+			displayApps  []codersdk.DisplayApp
+			expectedApps []agentproto.CreateSubAgentRequest_DisplayApp
+		}{
+			{
+				name:        "single display app",
+				displayApps: []codersdk.DisplayApp{codersdk.DisplayAppVSCodeDesktop},
+				expectedApps: []agentproto.CreateSubAgentRequest_DisplayApp{
+					agentproto.CreateSubAgentRequest_VSCODE,
+				},
+			},
+			{
+				name: "multiple display apps",
+				displayApps: []codersdk.DisplayApp{
+					codersdk.DisplayAppVSCodeDesktop,
+					codersdk.DisplayAppSSH,
+					codersdk.DisplayAppPortForward,
+				},
+				expectedApps: []agentproto.CreateSubAgentRequest_DisplayApp{
+					agentproto.CreateSubAgentRequest_VSCODE,
+					agentproto.CreateSubAgentRequest_SSH_HELPER,
+					agentproto.CreateSubAgentRequest_PORT_FORWARDING_HELPER,
+				},
+			},
+			{
+				name: "all display apps",
+				displayApps: []codersdk.DisplayApp{
+					codersdk.DisplayAppPortForward,
+					codersdk.DisplayAppSSH,
+					codersdk.DisplayAppVSCodeDesktop,
+					codersdk.DisplayAppVSCodeInsiders,
+					codersdk.DisplayAppWebTerminal,
+				},
+				expectedApps: []agentproto.CreateSubAgentRequest_DisplayApp{
+					agentproto.CreateSubAgentRequest_PORT_FORWARDING_HELPER,
+					agentproto.CreateSubAgentRequest_SSH_HELPER,
+					agentproto.CreateSubAgentRequest_VSCODE,
+					agentproto.CreateSubAgentRequest_VSCODE_INSIDERS,
+					agentproto.CreateSubAgentRequest_WEB_TERMINAL,
+				},
+			},
+			{
+				name:        "no display apps",
+				displayApps: []codersdk.DisplayApp{},
+			},
+		}
+
+		for _, tt := range tests {
+			t.Run(tt.name, func(t *testing.T) {
+				t.Parallel()
+
+				ctx := testutil.Context(t, testutil.WaitShort)
+				logger := testutil.Logger(t)
+				statsCh := make(chan *agentproto.Stats)
+
+				agentAPI := agenttest.NewClient(t, logger, uuid.New(), agentsdk.Manifest{}, statsCh, tailnet.NewCoordinator(logger))
+
+				agentClient, _, err := agentAPI.ConnectRPC26(ctx)
+				require.NoError(t, err)
+
+				subAgentClient := agentcontainers.NewSubAgentClientFromAPI(logger, agentClient)
+
+				// When: We create a sub agent with display apps.
+				subAgent, err := subAgentClient.Create(ctx, agentcontainers.SubAgent{
+					Name:            "sub-agent-" + tt.name,
+					Directory:       "/workspaces/coder",
+					Architecture:    "amd64",
+					OperatingSystem: "linux",
+					DisplayApps:     tt.displayApps,
+				})
+				require.NoError(t, err)
+
+				displayApps, err := agentAPI.GetSubAgentDisplayApps(subAgent.ID)
+				require.NoError(t, err)
+
+				// Then: We expect the apps to be created.
+				require.Equal(t, tt.expectedApps, displayApps)
+			})
+		}
+	})
+
+	t.Run("CreateWithApps", func(t *testing.T) {
+		t.Parallel()
+
+		tests := []struct {
+			name         string
+			apps         []agentcontainers.SubAgentApp
+			expectedApps []*agentproto.CreateSubAgentRequest_App
+		}{
+			{
+				name: "SlugOnly",
+				apps: []agentcontainers.SubAgentApp{
+					{
+						Slug: "code-server",
+					},
+				},
+				expectedApps: []*agentproto.CreateSubAgentRequest_App{
+					{
+						Slug: "code-server",
+					},
+				},
+			},
+			{
+				name: "AllFields",
+				apps: []agentcontainers.SubAgentApp{
+					{
+						Slug:        "jupyter",
+						Command:     "jupyter lab --port=8888",
+						DisplayName: "Jupyter Lab",
+						External:    false,
+						Group:       "Development",
+						HealthCheck: agentcontainers.SubAgentHealthCheck{
+							Interval:  30,
+							Threshold: 3,
+							URL:       "http://localhost:8888/api",
+						},
+						Hidden:    false,
+						Icon:      "/icon/jupyter.svg",
+						OpenIn:    codersdk.WorkspaceAppOpenInTab,
+						Order:     int32(1),
+						Share:     codersdk.WorkspaceAppSharingLevelAuthenticated,
+						Subdomain: true,
+						URL:       "http://localhost:8888",
+					},
+				},
+				expectedApps: []*agentproto.CreateSubAgentRequest_App{
+					{
+						Slug:        "jupyter",
+						Command:     ptr.Ref("jupyter lab --port=8888"),
+						DisplayName: ptr.Ref("Jupyter Lab"),
+						External:    ptr.Ref(false),
+						Group:       ptr.Ref("Development"),
+						Healthcheck: &agentproto.CreateSubAgentRequest_App_Healthcheck{
+							Interval:  30,
+							Threshold: 3,
+							Url:       "http://localhost:8888/api",
+						},
+						Hidden:    ptr.Ref(false),
+						Icon:      ptr.Ref("/icon/jupyter.svg"),
+						OpenIn:    agentproto.CreateSubAgentRequest_App_TAB.Enum(),
+						Order:     ptr.Ref(int32(1)),
+						Share:     agentproto.CreateSubAgentRequest_App_AUTHENTICATED.Enum(),
+						Subdomain: ptr.Ref(true),
+						Url:       ptr.Ref("http://localhost:8888"),
+					},
+				},
+			},
+			{
+				name: "AllSharingLevels",
+				apps: []agentcontainers.SubAgentApp{
+					{
+						Slug:  "owner-app",
+						Share: codersdk.WorkspaceAppSharingLevelOwner,
+					},
+					{
+						Slug:  "authenticated-app",
+						Share: codersdk.WorkspaceAppSharingLevelAuthenticated,
+					},
+					{
+						Slug:  "public-app",
+						Share: codersdk.WorkspaceAppSharingLevelPublic,
+					},
+					{
+						Slug:  "organization-app",
+						Share: codersdk.WorkspaceAppSharingLevelOrganization,
+					},
+				},
+				expectedApps: []*agentproto.CreateSubAgentRequest_App{
+					{
+						Slug:  "owner-app",
+						Share: agentproto.CreateSubAgentRequest_App_OWNER.Enum(),
+					},
+					{
+						Slug:  "authenticated-app",
+						Share: agentproto.CreateSubAgentRequest_App_AUTHENTICATED.Enum(),
+					},
+					{
+						Slug:  "public-app",
+						Share: agentproto.CreateSubAgentRequest_App_PUBLIC.Enum(),
+					},
+					{
+						Slug:  "organization-app",
+						Share: agentproto.CreateSubAgentRequest_App_ORGANIZATION.Enum(),
+					},
+				},
+			},
+			{
+				name: "WithHealthCheck",
+				apps: []agentcontainers.SubAgentApp{
+					{
+						Slug: "health-app",
+						HealthCheck: agentcontainers.SubAgentHealthCheck{
+							Interval:  60,
+							Threshold: 5,
+							URL:       "http://localhost:3000/health",
+						},
+					},
+				},
+				expectedApps: []*agentproto.CreateSubAgentRequest_App{
+					{
+						Slug: "health-app",
+						Healthcheck: &agentproto.CreateSubAgentRequest_App_Healthcheck{
+							Interval:  60,
+							Threshold: 5,
+							Url:       "http://localhost:3000/health",
+						},
+					},
+				},
+			},
+		}
+
+		for _, tt := range tests {
+			t.Run(tt.name, func(t *testing.T) {
+				t.Parallel()
+
+				ctx := testutil.Context(t, testutil.WaitShort)
+				logger := testutil.Logger(t)
+				statsCh := make(chan *agentproto.Stats)
+
+				agentAPI := agenttest.NewClient(t, logger, uuid.New(), agentsdk.Manifest{}, statsCh, tailnet.NewCoordinator(logger))
+
+				agentClient, _, err := agentAPI.ConnectRPC26(ctx)
+				require.NoError(t, err)
+
+				subAgentClient := agentcontainers.NewSubAgentClientFromAPI(logger, agentClient)
+
+				// When: We create a sub agent with display apps.
+				subAgent, err := subAgentClient.Create(ctx, agentcontainers.SubAgent{
+					Name:            "sub-agent-" + tt.name,
+					Directory:       "/workspaces/coder",
+					Architecture:    "amd64",
+					OperatingSystem: "linux",
+					Apps:            tt.apps,
+				})
+				require.NoError(t, err)
+
+				apps, err := agentAPI.GetSubAgentApps(subAgent.ID)
+				require.NoError(t, err)
+
+				// Then: We expect the apps to be created.
+				require.Len(t, apps, len(tt.expectedApps))
+				for i, expectedApp := range tt.expectedApps {
+					actualApp := apps[i]
+
+					assert.Equal(t, expectedApp.Slug, actualApp.Slug)
+					assert.Equal(t, expectedApp.Command, actualApp.Command)
+					assert.Equal(t, expectedApp.DisplayName, actualApp.DisplayName)
+					assert.Equal(t, ptr.NilToEmpty(expectedApp.External), ptr.NilToEmpty(actualApp.External))
+					assert.Equal(t, expectedApp.Group, actualApp.Group)
+					assert.Equal(t, ptr.NilToEmpty(expectedApp.Hidden), ptr.NilToEmpty(actualApp.Hidden))
+					assert.Equal(t, expectedApp.Icon, actualApp.Icon)
+					assert.Equal(t, ptr.NilToEmpty(expectedApp.Order), ptr.NilToEmpty(actualApp.Order))
+					assert.Equal(t, ptr.NilToEmpty(expectedApp.Subdomain), ptr.NilToEmpty(actualApp.Subdomain))
+					assert.Equal(t, expectedApp.Url, actualApp.Url)
+
+					if expectedApp.OpenIn != nil {
+						require.NotNil(t, actualApp.OpenIn)
+						assert.Equal(t, *expectedApp.OpenIn, *actualApp.OpenIn)
+					} else {
+						assert.Equal(t, expectedApp.OpenIn, actualApp.OpenIn)
+					}
+
+					if expectedApp.Share != nil {
+						require.NotNil(t, actualApp.Share)
+						assert.Equal(t, *expectedApp.Share, *actualApp.Share)
+					} else {
+						assert.Equal(t, expectedApp.Share, actualApp.Share)
+					}
+
+					if expectedApp.Healthcheck != nil {
+						require.NotNil(t, expectedApp.Healthcheck)
+						assert.Equal(t, expectedApp.Healthcheck.Interval, actualApp.Healthcheck.Interval)
+						assert.Equal(t, expectedApp.Healthcheck.Threshold, actualApp.Healthcheck.Threshold)
+						assert.Equal(t, expectedApp.Healthcheck.Url, actualApp.Healthcheck.Url)
+					} else {
+						assert.Equal(t, expectedApp.Healthcheck, actualApp.Healthcheck)
+					}
+				}
+			})
+		}
+	})
+}

agent/agentcontainers/testdata/devcontainercli/parse/up.golden

@@ -0,0 +1,64 @@
+@devcontainers/cli 0.75.0. Node.js v23.9.0. darwin 24.4.0 arm64.
+Resolving Feature dependencies for 'ghcr.io/devcontainers/features/docker-in-docker:2'...
+Soft-dependency 'ghcr.io/devcontainers/features/common-utils' is not required.  Removing from installation order...
+Files to omit: ''
+Run: docker buildx build --load --build-context dev_containers_feature_content_source=/var/folders/1y/cm8mblxd7_x9cljwl_jvfprh0000gn/T/devcontainercli/container-features/0.75.0-1744102171193 --build-arg _DEV_CONTAINERS_BASE_IMAGE=mcr.microsoft.com/devcontainers/javascript-node:1-18-bullseye --build-arg _DEV_CONTAINERS_IMAGE_USER=root --build-arg _DEV_CONTAINERS_FEATURE_CONTENT_SOURCE=dev_container_feature_content_temp --target dev_containers_target_stage -f /var/folders/1y/cm8mblxd7_x9cljwl_jvfprh0000gn/T/devcontainercli/container-features/0.75.0-1744102171193/Dockerfile.extended -t vsc-devcontainers-template-starter-81d8f17e32abef6d434cbb5a37fe05e5c8a6f8ccede47a61197f002dcbf60566-features /var/folders/1y/cm8mblxd7_x9cljwl_jvfprh0000gn/T/devcontainercli/empty-folder
+#0 building with "orbstack" instance using docker driver
+
+#1 [internal] load build definition from Dockerfile.extended
+#1 transferring dockerfile: 3.09kB done
+#1 DONE 0.0s
+
+#2 resolve image config for docker-image://docker.io/docker/dockerfile:1.4
+#2 DONE 1.3s
+#3 docker-image://docker.io/docker/dockerfile:1.4@sha256:9ba7531bd80fb0a858632727cf7a112fbfd19b17e94c4e84ced81e24ef1a0dbc
+#3 CACHED
+
+#4 [internal] load .dockerignore
+#4 transferring context: 2B done
+#4 DONE 0.0s
+
+#5 [internal] load metadata for mcr.microsoft.com/devcontainers/javascript-node:1-18-bullseye
+#5 DONE 0.0s
+
+#6 [context dev_containers_feature_content_source] load .dockerignore
+#6 transferring dev_containers_feature_content_source: 2B done
+#6 DONE 0.0s
+
+#7 [dev_containers_feature_content_normalize 1/3] FROM mcr.microsoft.com/devcontainers/javascript-node:1-18-bullseye
+#7 DONE 0.0s
+
+#8 [context dev_containers_feature_content_source] load from client
+#8 transferring dev_containers_feature_content_source: 82.11kB 0.0s done
+#8 DONE 0.0s
+
+#9 [dev_containers_feature_content_normalize 2/3] COPY --from=dev_containers_feature_content_source devcontainer-features.builtin.env /tmp/build-features/
+#9 CACHED
+
+#10 [dev_containers_target_stage 2/5] RUN mkdir -p /tmp/dev-container-features
+#10 CACHED
+
+#11 [dev_containers_target_stage 3/5] COPY --from=dev_containers_feature_content_normalize /tmp/build-features/ /tmp/dev-container-features
+#11 CACHED
+
+#12 [dev_containers_target_stage 4/5] RUN echo "_CONTAINER_USER_HOME=$( (command -v getent >/dev/null 2>&1 && getent passwd 'root' || grep -E '^root|^[^:]*:[^:]*:root:' /etc/passwd || true) | cut -d: -f6)" >> /tmp/dev-container-features/devcontainer-features.builtin.env && echo "_REMOTE_USER_HOME=$( (command -v getent >/dev/null 2>&1 && getent passwd 'node' || grep -E '^node|^[^:]*:[^:]*:node:' /etc/passwd || true) | cut -d: -f6)" >> /tmp/dev-container-features/devcontainer-features.builtin.env
+#12 CACHED
+
+#13 [dev_containers_feature_content_normalize 3/3] RUN chmod -R 0755 /tmp/build-features/
+#13 CACHED
+
+#14 [dev_containers_target_stage 5/5] RUN --mount=type=bind,from=dev_containers_feature_content_source,source=docker-in-docker_0,target=/tmp/build-features-src/docker-in-docker_0     cp -ar /tmp/build-features-src/docker-in-docker_0 /tmp/dev-container-features  && chmod -R 0755 /tmp/dev-container-features/docker-in-docker_0  && cd /tmp/dev-container-features/docker-in-docker_0  && chmod +x ./devcontainer-features-install.sh  && ./devcontainer-features-install.sh  && rm -rf /tmp/dev-container-features/docker-in-docker_0
+#14 CACHED
+
+#15 exporting to image
+#15 exporting layers done
+#15 writing image sha256:275dc193c905d448ef3945e3fc86220cc315fe0cb41013988d6ff9f8d6ef2357 done
+#15 naming to docker.io/library/vsc-devcontainers-template-starter-81d8f17e32abef6d434cbb5a37fe05e5c8a6f8ccede47a61197f002dcbf60566-features done
+#15 DONE 0.0s
+Run: docker buildx build --load --build-context dev_containers_feature_content_source=/var/folders/1y/cm8mblxd7_x9cljwl_jvfprh0000gn/T/devcontainercli/container-features/0.75.0-1744102171193 --build-arg _DEV_CONTAINERS_BASE_IMAGE=mcr.microsoft.com/devcontainers/javascript-node:1-18-bullseye --build-arg _DEV_CONTAINERS_IMAGE_USER=root --build-arg _DEV_CONTAINERS_FEATURE_CONTENT_SOURCE=dev_container_feature_content_temp --target dev_containers_target_stage -f /var/folders/1y/cm8mblxd7_x9cljwl_jvfprh0000gn/T/devcontainercli/container-features/0.75.0-1744102171193/Dockerfile.extended -t vsc-devcontainers-template-starter-81d8f17e32abef6d434cbb5a37fe05e5c8a6f8ccede47a61197f002dcbf60566-features /var/folders/1y/cm8mblxd7_x9cljwl_jvfprh0000gn/T/devcontainercli/empty-folder
+Run: docker run --sig-proxy=false -a STDOUT -a STDERR --mount type=bind,source=/code/devcontainers-template-starter,target=/workspaces/devcontainers-template-starter,consistency=cached --mount type=volume,src=dind-var-lib-docker-0pctifo8bbg3pd06g3j5s9ae8j7lp5qfcd67m25kuahurel7v7jm,dst=/var/lib/docker -l devcontainer.local_folder=/code/devcontainers-template-starter -l devcontainer.config_file=/code/devcontainers-template-starter/.devcontainer/devcontainer.json --privileged --entrypoint /bin/sh vsc-devcontainers-template-starter-81d8f17e32abef6d434cbb5a37fe05e5c8a6f8ccede47a61197f002dcbf60566-features -c echo Container started
+Container started
+Not setting dockerd DNS manually.
+Running the postCreateCommand from devcontainer.json...
+added 1 package in 784ms
+{"outcome":"success","containerId":"bc72db8d0c4c4e941bd9ffc341aee64a18d3397fd45b87cd93d4746150967ba8","remoteUser":"node","remoteWorkspaceFolder":"/workspaces/devcontainers-template-starter"}

agent/agentcontainers/testdata/devcontainercli/readconfig/read-config-error-not-found.log

@@ -0,0 +1,2 @@
+{"type":"text","level":3,"timestamp":1749557935646,"text":"@devcontainers/cli 0.75.0. Node.js v20.16.0. linux 6.8.0-60-generic x64."}
+{"type":"text","level":2,"timestamp":1749557935646,"text":"Error: Dev container config (/home/coder/.devcontainer/devcontainer.json) not found.\n    at v7 (/usr/local/nvm/versions/node/v20.16.0/lib/node_modules/@devcontainers/cli/dist/spec-node/devContainersSpecCLI.js:668:6918)\n    at async /usr/local/nvm/versions/node/v20.16.0/lib/node_modules/@devcontainers/cli/dist/spec-node/devContainersSpecCLI.js:484:1188"}

agent/agentcontainers/testdata/devcontainercli/readconfig/read-config-with-coder-customization.log

@@ -0,0 +1,8 @@
+{"type":"text","level":3,"timestamp":1749557820014,"text":"@devcontainers/cli 0.75.0. Node.js v20.16.0. linux 6.8.0-60-generic x64."}
+{"type":"start","level":2,"timestamp":1749557820014,"text":"Run: git rev-parse --show-cdup"}
+{"type":"stop","level":2,"timestamp":1749557820023,"text":"Run: git rev-parse --show-cdup","startTimestamp":1749557820014}
+{"type":"start","level":2,"timestamp":1749557820023,"text":"Run: docker ps -q -a --filter label=devcontainer.local_folder=/home/coder/coder --filter label=devcontainer.config_file=/home/coder/coder/.devcontainer/devcontainer.json"}
+{"type":"stop","level":2,"timestamp":1749557820039,"text":"Run: docker ps -q -a --filter label=devcontainer.local_folder=/home/coder/coder --filter label=devcontainer.config_file=/home/coder/coder/.devcontainer/devcontainer.json","startTimestamp":1749557820023}
+{"type":"start","level":2,"timestamp":1749557820039,"text":"Run: docker ps -q -a --filter label=devcontainer.local_folder=/home/coder/coder"}
+{"type":"stop","level":2,"timestamp":1749557820054,"text":"Run: docker ps -q -a --filter label=devcontainer.local_folder=/home/coder/coder","startTimestamp":1749557820039}
+{"mergedConfiguration":{"customizations":{"coder":[{"displayApps":{"vscode":true,"web_terminal":true}},{"displayApps":{"vscode_insiders":true,"web_terminal":false}}]}}}

agent/agentcontainers/testdata/devcontainercli/readconfig/read-config-without-coder-customization.log

@@ -0,0 +1,8 @@
+{"type":"text","level":3,"timestamp":1749557820014,"text":"@devcontainers/cli 0.75.0. Node.js v20.16.0. linux 6.8.0-60-generic x64."}
+{"type":"start","level":2,"timestamp":1749557820014,"text":"Run: git rev-parse --show-cdup"}
+{"type":"stop","level":2,"timestamp":1749557820023,"text":"Run: git rev-parse --show-cdup","startTimestamp":1749557820014}
+{"type":"start","level":2,"timestamp":1749557820023,"text":"Run: docker ps -q -a --filter label=devcontainer.local_folder=/home/coder/coder --filter label=devcontainer.config_file=/home/coder/coder/.devcontainer/devcontainer.json"}
+{"type":"stop","level":2,"timestamp":1749557820039,"text":"Run: docker ps -q -a --filter label=devcontainer.local_folder=/home/coder/coder --filter label=devcontainer.config_file=/home/coder/coder/.devcontainer/devcontainer.json","startTimestamp":1749557820023}
+{"type":"start","level":2,"timestamp":1749557820039,"text":"Run: docker ps -q -a --filter label=devcontainer.local_folder=/home/coder/coder"}
+{"type":"stop","level":2,"timestamp":1749557820054,"text":"Run: docker ps -q -a --filter label=devcontainer.local_folder=/home/coder/coder","startTimestamp":1749557820039}
+{"mergedConfiguration":{"customizations":{}}}

agent/agentcontainers/watcher/watcher_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"os"
 	"path/filepath"
+	"runtime"
 	"testing"
 
 	"github.com/fsnotify/fsnotify"
@@ -88,24 +89,34 @@ func TestFSNotifyWatcher(t *testing.T) {
 		break
 	}
 
-	err = os.WriteFile(testFile+".atomic", []byte(`{"test": "atomic"}`), 0o600)
-	require.NoError(t, err, "write new atomic test file failed")
+	// TODO(DanielleMaywood):
+	// Unfortunately it appears this atomic-rename phase of the test is flakey on macOS.
+	//
+	// This test flake could be indicative of an issue that may present itself
+	// in a running environment. Fortunately, we only use this (as of 2025-07-29)
+	// for our dev container integration. We do not expect the host workspace
+	// (where this is used), to ever be run on macOS, as containers are a linux
+	// paradigm.
+	if runtime.GOOS != "darwin" {
+		err = os.WriteFile(testFile+".atomic", []byte(`{"test": "atomic"}`), 0o600)
+		require.NoError(t, err, "write new atomic test file failed")
 
-	err = os.Rename(testFile+".atomic", testFile)
-	require.NoError(t, err, "rename atomic test file failed")
+		err = os.Rename(testFile+".atomic", testFile)
+		require.NoError(t, err, "rename atomic test file failed")
 
-	// Verify that we receive the event we want.
-	for {
-		event, err := wut.Next(ctx)
-		require.NoError(t, err, "next event failed")
-		require.NotNil(t, event, "want non-nil event")
-		if !event.Has(fsnotify.Create) {
-			t.Logf("Ignoring event: %s", event)
-			continue
+		// Verify that we receive the event we want.
+		for {
+			event, err := wut.Next(ctx)
+			require.NoError(t, err, "next event failed")
+			require.NotNil(t, event, "want non-nil event")
+			if !event.Has(fsnotify.Create) {
+				t.Logf("Ignoring event: %s", event)
+				continue
+			}
+			require.Truef(t, event.Has(fsnotify.Create), "want create event: %s", event.String())
+			require.Equal(t, event.Name, testFile, "want event for test file")
+			break
 		}
-		require.Truef(t, event.Has(fsnotify.Create), "want create event: %s", event.String())
-		require.Equal(t, event.Name, testFile, "want event for test file")
-		break
 	}
 
 	// Test removing the file from the watcher.

agent/agentscripts/agentscripts.go

@@ -79,21 +79,6 @@ func New(opts Options) *Runner {
 
 type ScriptCompletedFunc func(context.Context, *proto.WorkspaceAgentScriptCompletedRequest) (*proto.WorkspaceAgentScriptCompletedResponse, error)
 
-type runnerScript struct {
-	runOnPostStart bool
-	codersdk.WorkspaceAgentScript
-}
-
-func toRunnerScript(scripts ...codersdk.WorkspaceAgentScript) []runnerScript {
-	var rs []runnerScript
-	for _, s := range scripts {
-		rs = append(rs, runnerScript{
-			WorkspaceAgentScript: s,
-		})
-	}
-	return rs
-}
-
 type Runner struct {
 	Options
 
@@ -103,7 +88,7 @@ type Runner struct {
 	closed          chan struct{}
 	closeMutex      sync.Mutex
 	cron            *cron.Cron
-	scripts         []runnerScript
+	scripts         []codersdk.WorkspaceAgentScript
 	dataDir         string
 	scriptCompleted ScriptCompletedFunc
 
@@ -138,19 +123,6 @@ func (r *Runner) RegisterMetrics(reg prometheus.Registerer) {
 // InitOption describes an option for the runner initialization.
 type InitOption func(*Runner)
 
-// WithPostStartScripts adds scripts that should be run after the workspace
-// start scripts but before the workspace is marked as started.
-func WithPostStartScripts(scripts ...codersdk.WorkspaceAgentScript) InitOption {
-	return func(r *Runner) {
-		for _, s := range scripts {
-			r.scripts = append(r.scripts, runnerScript{
-				runOnPostStart:       true,
-				WorkspaceAgentScript: s,
-			})
-		}
-	}
-}
-
 // Init initializes the runner with the provided scripts.
 // It also schedules any scripts that have a schedule.
 // This function must be called before Execute.
@@ -161,7 +133,7 @@ func (r *Runner) Init(scripts []codersdk.WorkspaceAgentScript, scriptCompleted S
 		return xerrors.New("init: already initialized")
 	}
 	r.initialized = true
-	r.scripts = toRunnerScript(scripts...)
+	r.scripts = scripts
 	r.scriptCompleted = scriptCompleted
 	for _, opt := range opts {
 		opt(r)
@@ -177,9 +149,8 @@ func (r *Runner) Init(scripts []codersdk.WorkspaceAgentScript, scriptCompleted S
 		if script.Cron == "" {
 			continue
 		}
-		script := script
 		_, err := r.cron.AddFunc(script.Cron, func() {
-			err := r.trackRun(r.cronCtx, script.WorkspaceAgentScript, ExecuteCronScripts)
+			err := r.trackRun(r.cronCtx, script, ExecuteCronScripts)
 			if err != nil {
 				r.Logger.Warn(context.Background(), "run agent script on schedule", slog.Error(err))
 			}
@@ -223,7 +194,6 @@ type ExecuteOption int
 const (
 	ExecuteAllScripts ExecuteOption = iota
 	ExecuteStartScripts
-	ExecutePostStartScripts
 	ExecuteStopScripts
 	ExecuteCronScripts
 )
@@ -246,7 +216,6 @@ func (r *Runner) Execute(ctx context.Context, option ExecuteOption) error {
 	for _, script := range r.scripts {
 		runScript := (option == ExecuteStartScripts && script.RunOnStart) ||
 			(option == ExecuteStopScripts && script.RunOnStop) ||
-			(option == ExecutePostStartScripts && script.runOnPostStart) ||
 			(option == ExecuteCronScripts && script.Cron != "") ||
 			option == ExecuteAllScripts
 
@@ -254,9 +223,8 @@ func (r *Runner) Execute(ctx context.Context, option ExecuteOption) error {
 			continue
 		}
 
-		script := script
 		eg.Go(func() error {
-			err := r.trackRun(ctx, script.WorkspaceAgentScript, option)
+			err := r.trackRun(ctx, script, option)
 			if err != nil {
 				return xerrors.Errorf("run agent script %q: %w", script.LogSourceID, err)
 			}

agent/agentscripts/agentscripts_test.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"path/filepath"
 	"runtime"
-	"slices"
 	"sync"
 	"testing"
 	"time"
@@ -144,6 +143,12 @@ func TestScriptReportsTiming(t *testing.T) {
 
 	timing := timings[0]
 	require.Equal(t, int32(0), timing.ExitCode)
+	if assert.True(t, timing.Start.IsValid(), "start time should be valid") {
+		require.NotZero(t, timing.Start.AsTime(), "start time should not be zero")
+	}
+	if assert.True(t, timing.End.IsValid(), "end time should be valid") {
+		require.NotZero(t, timing.End.AsTime(), "end time should not be zero")
+	}
 	require.GreaterOrEqual(t, timing.End.AsTime(), timing.Start.AsTime())
 }
 
@@ -171,11 +176,6 @@ func TestExecuteOptions(t *testing.T) {
 		Script:      "echo stop",
 		RunOnStop:   true,
 	}
-	postStartScript := codersdk.WorkspaceAgentScript{
-		ID:          uuid.New(),
-		LogSourceID: uuid.New(),
-		Script:      "echo poststart",
-	}
 	regularScript := codersdk.WorkspaceAgentScript{
 		ID:          uuid.New(),
 		LogSourceID: uuid.New(),
@@ -187,10 +187,9 @@ func TestExecuteOptions(t *testing.T) {
 		stopScript,
 		regularScript,
 	}
-	allScripts := append(slices.Clone(scripts), postStartScript)
 
 	scriptByID := func(t *testing.T, id uuid.UUID) codersdk.WorkspaceAgentScript {
-		for _, script := range allScripts {
+		for _, script := range scripts {
 			if script.ID == id {
 				return script
 			}
@@ -200,10 +199,9 @@ func TestExecuteOptions(t *testing.T) {
 	}
 
 	wantOutput := map[uuid.UUID]string{
-		startScript.ID:     "start",
-		stopScript.ID:      "stop",
-		postStartScript.ID: "poststart",
-		regularScript.ID:   "regular",
+		startScript.ID:   "start",
+		stopScript.ID:    "stop",
+		regularScript.ID: "regular",
 	}
 
 	testCases := []struct {
@@ -214,18 +212,13 @@ func TestExecuteOptions(t *testing.T) {
 		{
 			name:    "ExecuteAllScripts",
 			option:  agentscripts.ExecuteAllScripts,
-			wantRun: []uuid.UUID{startScript.ID, stopScript.ID, regularScript.ID, postStartScript.ID},
+			wantRun: []uuid.UUID{startScript.ID, stopScript.ID, regularScript.ID},
 		},
 		{
 			name:    "ExecuteStartScripts",
 			option:  agentscripts.ExecuteStartScripts,
 			wantRun: []uuid.UUID{startScript.ID},
 		},
-		{
-			name:    "ExecutePostStartScripts",
-			option:  agentscripts.ExecutePostStartScripts,
-			wantRun: []uuid.UUID{postStartScript.ID},
-		},
 		{
 			name:    "ExecuteStopScripts",
 			option:  agentscripts.ExecuteStopScripts,
@@ -254,7 +247,6 @@ func TestExecuteOptions(t *testing.T) {
 			err := runner.Init(
 				scripts,
 				aAPI.ScriptCompleted,
-				agentscripts.WithPostStartScripts(postStartScript),
 			)
 			require.NoError(t, err)
 
@@ -268,7 +260,7 @@ func TestExecuteOptions(t *testing.T) {
 					"script %s should have run when using filter %s", scriptByID(t, id).Script, tc.name)
 			}
 
-			for _, script := range allScripts {
+			for _, script := range scripts {
 				if _, ok := gotRun[script.ID]; ok {
 					continue
 				}

agent/agentssh/agentssh.go

@@ -46,6 +46,8 @@ const (
 	// MagicProcessCmdlineJetBrains is a string in a process's command line that
 	// uniquely identifies it as JetBrains software.
 	MagicProcessCmdlineJetBrains = "idea.vendor.name=JetBrains"
+	MagicProcessCmdlineToolbox   = "com.jetbrains.toolbox"
+	MagicProcessCmdlineGateway   = "remote-dev-server"
 
 	// BlockedFileTransferErrorCode indicates that SSH server restricted the raw command from performing
 	// the file transfer.
@@ -113,9 +115,14 @@ type Config struct {
 	BlockFileTransfer bool
 	// ReportConnection.
 	ReportConnection reportConnectionFunc
-	// Experimental: allow connecting to running containers if
-	// CODER_AGENT_DEVCONTAINERS_ENABLE=true.
-	ExperimentalDevContainersEnabled bool
+	// Experimental: allow connecting to running containers via Docker exec.
+	// Note that this is different from the devcontainers feature, which uses
+	// subagents.
+	ExperimentalContainers bool
+	// X11Net allows overriding the networking implementation used for X11
+	// forwarding listeners. When nil, a default implementation backed by the
+	// standard library networking package is used.
+	X11Net X11Network
 }
 
 type Server struct {
@@ -124,14 +131,16 @@ type Server struct {
 	listeners map[net.Listener]struct{}
 	conns     map[net.Conn]struct{}
 	sessions  map[ssh.Session]struct{}
+	processes map[*os.Process]struct{}
 	closing   chan struct{}
 	// Wait for goroutines to exit, waited without
 	// a lock on mu but protected by closing.
 	wg sync.WaitGroup
 
-	Execer agentexec.Execer
-	logger slog.Logger
-	srv    *ssh.Server
+	Execer       agentexec.Execer
+	logger       slog.Logger
+	srv          *ssh.Server
+	x11Forwarder *x11Forwarder
 
 	config *Config
 
@@ -182,11 +191,26 @@ func NewServer(ctx context.Context, logger slog.Logger, prometheusRegistry *prom
 		fs:        fs,
 		conns:     make(map[net.Conn]struct{}),
 		sessions:  make(map[ssh.Session]struct{}),
+		processes: make(map[*os.Process]struct{}),
 		logger:    logger,
 
 		config: config,
 
 		metrics: metrics,
+		x11Forwarder: &x11Forwarder{
+			logger:           logger,
+			x11HandlerErrors: metrics.x11HandlerErrors,
+			fs:               fs,
+			displayOffset:    *config.X11DisplayOffset,
+			sessions:         make(map[*x11Session]struct{}),
+			connections:      make(map[net.Conn]struct{}),
+			network: func() X11Network {
+				if config.X11Net != nil {
+					return config.X11Net
+				}
+				return osNet{}
+			}(),
+		},
 	}
 
 	srv := &ssh.Server{
@@ -435,7 +459,7 @@ func (s *Server) sessionHandler(session ssh.Session) {
 	switch ss := session.Subsystem(); ss {
 	case "":
 	case "sftp":
-		if s.config.ExperimentalDevContainersEnabled && container != "" {
+		if s.config.ExperimentalContainers && container != "" {
 			closeCause("sftp not yet supported with containers")
 			_ = session.Exit(1)
 			return
@@ -454,7 +478,7 @@ func (s *Server) sessionHandler(session ssh.Session) {
 
 	x11, hasX11 := session.X11()
 	if hasX11 {
-		display, handled := s.x11Handler(session.Context(), x11)
+		display, handled := s.x11Forwarder.x11Handler(ctx, session)
 		if !handled {
 			logger.Error(ctx, "x11 handler failed")
 			closeCause("x11 handler failed")
@@ -549,7 +573,7 @@ func (s *Server) sessionStart(logger slog.Logger, session ssh.Session, env []str
 
 	var ei usershell.EnvInfoer
 	var err error
-	if s.config.ExperimentalDevContainersEnabled && container != "" {
+	if s.config.ExperimentalContainers && container != "" {
 		ei, err = agentcontainers.EnvInfo(ctx, s.Execer, container, containerUser)
 		if err != nil {
 			s.metrics.sessionErrors.WithLabelValues(magicTypeLabel, ptyLabel, "container_env_info").Add(1)
@@ -586,7 +610,12 @@ func (s *Server) startNonPTYSession(logger slog.Logger, session ssh.Session, mag
 	// otherwise context cancellation will not propagate properly
 	// and SSH server close may be delayed.
 	cmd.SysProcAttr = cmdSysProcAttr()
-	cmd.Cancel = cmdCancel(session.Context(), logger, cmd)
+
+	// to match OpenSSH, we don't actually tear a non-TTY command down, even if the session ends. OpenSSH closes the
+	// pipes to the process when the session ends; which is what happens here since we wire the command up to the
+	// session for I/O.
+	// c.f. https://github.com/coder/coder/issues/18519#issuecomment-3019118271
+	cmd.Cancel = nil
 
 	cmd.Stdout = session
 	cmd.Stderr = session.Stderr()
@@ -609,6 +638,16 @@ func (s *Server) startNonPTYSession(logger slog.Logger, session ssh.Session, mag
 		s.metrics.sessionErrors.WithLabelValues(magicTypeLabel, "no", "start_command").Add(1)
 		return xerrors.Errorf("start: %w", err)
 	}
+
+	// Since we don't cancel the process when the session stops, we still need to tear it down if we are closing. So
+	// track it here.
+	if !s.trackProcess(cmd.Process, true) {
+		// must be closing
+		err = cmdCancel(logger, cmd.Process)
+		return xerrors.Errorf("failed to track process: %w", err)
+	}
+	defer s.trackProcess(cmd.Process, false)
+
 	sigs := make(chan ssh.Signal, 1)
 	session.Signals(sigs)
 	defer func() {
@@ -816,6 +855,49 @@ func (s *Server) sftpHandler(logger slog.Logger, session ssh.Session) error {
 	return xerrors.Errorf("sftp server closed with error: %w", err)
 }
 
+func (s *Server) CommandEnv(ei usershell.EnvInfoer, addEnv []string) (shell, dir string, env []string, err error) {
+	if ei == nil {
+		ei = &usershell.SystemEnvInfo{}
+	}
+
+	currentUser, err := ei.User()
+	if err != nil {
+		return "", "", nil, xerrors.Errorf("get current user: %w", err)
+	}
+	username := currentUser.Username
+
+	shell, err = ei.Shell(username)
+	if err != nil {
+		return "", "", nil, xerrors.Errorf("get user shell: %w", err)
+	}
+
+	dir = s.config.WorkingDirectory()
+
+	// If the metadata directory doesn't exist, we run the command
+	// in the users home directory.
+	_, err = os.Stat(dir)
+	if dir == "" || err != nil {
+		// Default to user home if a directory is not set.
+		homedir, err := ei.HomeDir()
+		if err != nil {
+			return "", "", nil, xerrors.Errorf("get home dir: %w", err)
+		}
+		dir = homedir
+	}
+	env = append(ei.Environ(), addEnv...)
+	// Set login variables (see `man login`).
+	env = append(env, fmt.Sprintf("USER=%s", username))
+	env = append(env, fmt.Sprintf("LOGNAME=%s", username))
+	env = append(env, fmt.Sprintf("SHELL=%s", shell))
+
+	env, err = s.config.UpdateEnv(env)
+	if err != nil {
+		return "", "", nil, xerrors.Errorf("apply env: %w", err)
+	}
+
+	return shell, dir, env, nil
+}
+
 // CreateCommand processes raw command input with OpenSSH-like behavior.
 // If the script provided is empty, it will default to the users shell.
 // This injects environment variables specified by the user at launch too.
@@ -827,15 +909,10 @@ func (s *Server) CreateCommand(ctx context.Context, script string, env []string,
 	if ei == nil {
 		ei = &usershell.SystemEnvInfo{}
 	}
-	currentUser, err := ei.User()
-	if err != nil {
-		return nil, xerrors.Errorf("get current user: %w", err)
-	}
-	username := currentUser.Username
 
-	shell, err := ei.Shell(username)
+	shell, dir, env, err := s.CommandEnv(ei, env)
 	if err != nil {
-		return nil, xerrors.Errorf("get user shell: %w", err)
+		return nil, xerrors.Errorf("prepare command env: %w", err)
 	}
 
 	// OpenSSH executes all commands with the users current shell.
@@ -893,24 +970,8 @@ func (s *Server) CreateCommand(ctx context.Context, script string, env []string,
 		)
 	}
 	cmd := s.Execer.PTYCommandContext(ctx, modifiedName, modifiedArgs...)
-	cmd.Dir = s.config.WorkingDirectory()
-
-	// If the metadata directory doesn't exist, we run the command
-	// in the users home directory.
-	_, err = os.Stat(cmd.Dir)
-	if cmd.Dir == "" || err != nil {
-		// Default to user home if a directory is not set.
-		homedir, err := ei.HomeDir()
-		if err != nil {
-			return nil, xerrors.Errorf("get home dir: %w", err)
-		}
-		cmd.Dir = homedir
-	}
-	cmd.Env = append(ei.Environ(), env...)
-	// Set login variables (see `man login`).
-	cmd.Env = append(cmd.Env, fmt.Sprintf("USER=%s", username))
-	cmd.Env = append(cmd.Env, fmt.Sprintf("LOGNAME=%s", username))
-	cmd.Env = append(cmd.Env, fmt.Sprintf("SHELL=%s", shell))
+	cmd.Dir = dir
+	cmd.Env = env
 
 	// Set SSH connection environment variables (these are also set by OpenSSH
 	// and thus expected to be present by SSH clients). Since the agent does
@@ -921,11 +982,6 @@ func (s *Server) CreateCommand(ctx context.Context, script string, env []string,
 	cmd.Env = append(cmd.Env, fmt.Sprintf("SSH_CLIENT=%s %s %s", srcAddr, srcPort, dstPort))
 	cmd.Env = append(cmd.Env, fmt.Sprintf("SSH_CONNECTION=%s %s %s %s", srcAddr, srcPort, dstAddr, dstPort))
 
-	cmd.Env, err = s.config.UpdateEnv(cmd.Env)
-	if err != nil {
-		return nil, xerrors.Errorf("apply env: %w", err)
-	}
-
 	return cmd, nil
 }
 
@@ -973,7 +1029,7 @@ func (s *Server) handleConn(l net.Listener, c net.Conn) {
 		return
 	}
 	defer s.trackConn(l, c, false)
-	logger.Info(context.Background(), "started serving connection")
+	logger.Info(context.Background(), "started serving ssh connection")
 	// note: srv.ConnectionCompleteCallback logs completion of the connection
 	s.srv.HandleConn(c)
 }
@@ -1052,6 +1108,27 @@ func (s *Server) trackSession(ss ssh.Session, add bool) (ok bool) {
 	return true
 }
 
+// trackCommand registers the process with the server. If the server is
+// closing, the process is not registered and should be closed.
+//
+//nolint:revive
+func (s *Server) trackProcess(p *os.Process, add bool) (ok bool) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if add {
+		if s.closing != nil {
+			// Server closed.
+			return false
+		}
+		s.wg.Add(1)
+		s.processes[p] = struct{}{}
+		return true
+	}
+	s.wg.Done()
+	delete(s.processes, p)
+	return true
+}
+
 // Close the server and all active connections. Server can be re-used
 // after Close is done.
 func (s *Server) Close() error {
@@ -1091,11 +1168,18 @@ func (s *Server) Close() error {
 		_ = c.Close()
 	}
 
+	for p := range s.processes {
+		_ = cmdCancel(s.logger, p)
+	}
+
 	s.logger.Debug(ctx, "closing SSH server")
 	err := s.srv.Close()
 
 	s.mu.Unlock()
 
+	s.logger.Debug(ctx, "closing X11 forwarding")
+	_ = s.x11Forwarder.Close()
+
 	s.logger.Debug(ctx, "waiting for all goroutines to exit")
 	s.wg.Wait() // Wait for all goroutines to exit.
 

agent/agentssh/agentssh_test.go

@@ -8,7 +8,9 @@ import (
 	"context"
 	"fmt"
 	"net"
+	"os"
 	"os/user"
+	"path/filepath"
 	"runtime"
 	"strings"
 	"sync"
@@ -403,6 +405,92 @@ func TestNewServer_Signal(t *testing.T) {
 	})
 }
 
+func TestSSHServer_ClosesStdin(t *testing.T) {
+	t.Parallel()
+	if runtime.GOOS == "windows" {
+		t.Skip("bash doesn't exist on Windows")
+	}
+
+	ctx := testutil.Context(t, testutil.WaitMedium)
+	logger := testutil.Logger(t)
+	s, err := agentssh.NewServer(ctx, logger.Named("ssh-server"), prometheus.NewRegistry(), afero.NewMemMapFs(), agentexec.DefaultExecer, nil)
+	require.NoError(t, err)
+	logger = logger.Named("test")
+	defer s.Close()
+	err = s.UpdateHostSigner(42)
+	assert.NoError(t, err)
+
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	require.NoError(t, err)
+
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		err := s.Serve(ln)
+		assert.Error(t, err) // Server is closed.
+	}()
+	defer func() {
+		err := s.Close()
+		require.NoError(t, err)
+		<-done
+	}()
+
+	c := sshClient(t, ln.Addr().String())
+
+	sess, err := c.NewSession()
+	require.NoError(t, err)
+	stdout, err := sess.StdoutPipe()
+	require.NoError(t, err)
+	stdin, err := sess.StdinPipe()
+	require.NoError(t, err)
+	defer stdin.Close()
+
+	dir := t.TempDir()
+	err = os.MkdirAll(dir, 0o755)
+	require.NoError(t, err)
+	filePath := filepath.Join(dir, "result.txt")
+
+	// the shell command `read` will block until data is written to stdin, or closed. It will return
+	// exit code 1 if it hits EOF, which is what we want to test.
+	cmdErrCh := make(chan error, 1)
+	go func() {
+		cmdErrCh <- sess.Start(fmt.Sprintf(`echo started; echo "read exit code: $(read && echo 0 || echo 1)" > %s`, filePath))
+	}()
+
+	cmdErr := testutil.RequireReceive(ctx, t, cmdErrCh)
+	require.NoError(t, cmdErr)
+
+	readCh := make(chan error, 1)
+	go func() {
+		buf := make([]byte, 8)
+		_, err := stdout.Read(buf)
+		assert.Equal(t, "started\n", string(buf))
+		readCh <- err
+	}()
+	err = testutil.RequireReceive(ctx, t, readCh)
+	require.NoError(t, err)
+
+	err = sess.Close()
+	require.NoError(t, err)
+
+	var content []byte
+	expected := []byte("read exit code: 1\n")
+	testutil.Eventually(ctx, t, func(_ context.Context) bool {
+		content, err = os.ReadFile(filePath)
+		if err != nil {
+			logger.Debug(ctx, "failed to read file; will retry", slog.Error(err))
+			return false
+		}
+		if len(content) != len(expected) {
+			logger.Debug(ctx, "file is partially written", slog.F("content", content))
+			return false
+		}
+		return true
+	}, testutil.IntervalFast)
+	require.NoError(t, err)
+	require.Equal(t, string(expected), string(content))
+}
+
 func sshClient(t *testing.T, addr string) *ssh.Client {
 	conn, err := net.Dial("tcp", addr)
 	require.NoError(t, err)

agent/agentssh/exec_other.go

@@ -4,7 +4,7 @@ package agentssh
 
 import (
 	"context"
-	"os/exec"
+	"os"
 	"syscall"
 
 	"cdr.dev/slog"
@@ -16,9 +16,7 @@ func cmdSysProcAttr() *syscall.SysProcAttr {
 	}
 }
 
-func cmdCancel(ctx context.Context, logger slog.Logger, cmd *exec.Cmd) func() error {
-	return func() error {
-		logger.Debug(ctx, "cmdCancel: sending SIGHUP to process and children", slog.F("pid", cmd.Process.Pid))
-		return syscall.Kill(-cmd.Process.Pid, syscall.SIGHUP)
-	}
+func cmdCancel(logger slog.Logger, p *os.Process) error {
+	logger.Debug(context.Background(), "cmdCancel: sending SIGHUP to process and children", slog.F("pid", p.Pid))
+	return syscall.Kill(-p.Pid, syscall.SIGHUP)
 }

agent/agentssh/exec_windows.go

@@ -2,7 +2,7 @@ package agentssh
 
 import (
 	"context"
-	"os/exec"
+	"os"
 	"syscall"
 
 	"cdr.dev/slog"
@@ -12,14 +12,12 @@ func cmdSysProcAttr() *syscall.SysProcAttr {
 	return &syscall.SysProcAttr{}
 }
 
-func cmdCancel(ctx context.Context, logger slog.Logger, cmd *exec.Cmd) func() error {
-	return func() error {
-		logger.Debug(ctx, "cmdCancel: killing process", slog.F("pid", cmd.Process.Pid))
-		// Windows doesn't support sending signals to process groups, so we
-		// have to kill the process directly. In the future, we may want to
-		// implement a more sophisticated solution for process groups on
-		// Windows, but for now, this is a simple way to ensure that the
-		// process is terminated when the context is cancelled.
-		return cmd.Process.Kill()
-	}
+func cmdCancel(logger slog.Logger, p *os.Process) error {
+	logger.Debug(context.Background(), "cmdCancel: killing process", slog.F("pid", p.Pid))
+	// Windows doesn't support sending signals to process groups, so we
+	// have to kill the process directly. In the future, we may want to
+	// implement a more sophisticated solution for process groups on
+	// Windows, but for now, this is a simple way to ensure that the
+	// process is terminated when the context is cancelled.
+	return p.Kill()
 }

agent/agentssh/jetbrainstrack.go

@@ -53,7 +53,7 @@ func NewJetbrainsChannelWatcher(ctx ssh.Context, logger slog.Logger, reportConne
 
 	// If this is not JetBrains, then we do not need to do anything special.  We
 	// attempt to match on something that appears unique to JetBrains software.
-	if !strings.Contains(strings.ToLower(cmdline), strings.ToLower(MagicProcessCmdlineJetBrains)) {
+	if !isJetbrainsProcess(cmdline) {
 		return newChannel
 	}
 
@@ -104,3 +104,18 @@ func (c *ChannelOnClose) Close() error {
 	c.once.Do(c.done)
 	return c.Channel.Close()
 }
+
+func isJetbrainsProcess(cmdline string) bool {
+	opts := []string{
+		MagicProcessCmdlineJetBrains,
+		MagicProcessCmdlineToolbox,
+		MagicProcessCmdlineGateway,
+	}
+
+	for _, opt := range opts {
+		if strings.Contains(strings.ToLower(cmdline), strings.ToLower(opt)) {
+			return true
+		}
+	}
+	return false
+}

agent/agentssh/x11.go

@@ -7,15 +7,16 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"math"
 	"net"
 	"os"
 	"path/filepath"
 	"strconv"
+	"sync"
 	"time"
 
 	"github.com/gliderlabs/ssh"
 	"github.com/gofrs/flock"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/spf13/afero"
 	gossh "golang.org/x/crypto/ssh"
 	"golang.org/x/xerrors"
@@ -29,8 +30,51 @@ const (
 	X11StartPort = 6000
 	// X11DefaultDisplayOffset is the default offset for X11 forwarding.
 	X11DefaultDisplayOffset = 10
+	X11MaxDisplays          = 200
+	// X11MaxPort is the highest port we will ever use for X11 forwarding. This limits the total number of TCP sockets
+	// we will create. It seems more useful to have a maximum port number than a direct limit on sockets with no max
+	// port because we'd like to be able to tell users the exact range of ports the Agent might use.
+	X11MaxPort = X11StartPort + X11MaxDisplays
 )
 
+// X11Network abstracts the creation of network listeners for X11 forwarding.
+// It is intended mainly for testing; production code uses the default
+// implementation backed by the operating system networking stack.
+type X11Network interface {
+	Listen(network, address string) (net.Listener, error)
+}
+
+// osNet is the default X11Network implementation that uses the standard
+// library network stack.
+type osNet struct{}
+
+func (osNet) Listen(network, address string) (net.Listener, error) {
+	return net.Listen(network, address)
+}
+
+type x11Forwarder struct {
+	logger           slog.Logger
+	x11HandlerErrors *prometheus.CounterVec
+	fs               afero.Fs
+	displayOffset    int
+
+	// network creates X11 listener sockets. Defaults to osNet{}.
+	network X11Network
+
+	mu          sync.Mutex
+	sessions    map[*x11Session]struct{}
+	connections map[net.Conn]struct{}
+	closing     bool
+	wg          sync.WaitGroup
+}
+
+type x11Session struct {
+	session  ssh.Session
+	display  int
+	listener net.Listener
+	usedAt   time.Time
+}
+
 // x11Callback is called when the client requests X11 forwarding.
 func (*Server) x11Callback(_ ssh.Context, _ ssh.X11) bool {
 	// Always allow.
@@ -39,115 +83,243 @@ func (*Server) x11Callback(_ ssh.Context, _ ssh.X11) bool {
 
 // x11Handler is called when a session has requested X11 forwarding.
 // It listens for X11 connections and forwards them to the client.
-func (s *Server) x11Handler(ctx ssh.Context, x11 ssh.X11) (displayNumber int, handled bool) {
-	serverConn, valid := ctx.Value(ssh.ContextKeyConn).(*gossh.ServerConn)
-	if !valid {
-		s.logger.Warn(ctx, "failed to get server connection")
+func (x *x11Forwarder) x11Handler(sshCtx ssh.Context, sshSession ssh.Session) (displayNumber int, handled bool) {
+	x11, hasX11 := sshSession.X11()
+	if !hasX11 {
 		return -1, false
 	}
+	serverConn, valid := sshCtx.Value(ssh.ContextKeyConn).(*gossh.ServerConn)
+	if !valid {
+		x.logger.Warn(sshCtx, "failed to get server connection")
+		return -1, false
+	}
+	ctx := slog.With(sshCtx, slog.F("session_id", fmt.Sprintf("%x", serverConn.SessionID())))
 
 	hostname, err := os.Hostname()
 	if err != nil {
-		s.logger.Warn(ctx, "failed to get hostname", slog.Error(err))
-		s.metrics.x11HandlerErrors.WithLabelValues("hostname").Add(1)
+		x.logger.Warn(ctx, "failed to get hostname", slog.Error(err))
+		x.x11HandlerErrors.WithLabelValues("hostname").Add(1)
 		return -1, false
 	}
 
-	ln, display, err := createX11Listener(ctx, *s.config.X11DisplayOffset)
+	x11session, err := x.createX11Session(ctx, sshSession)
 	if err != nil {
-		s.logger.Warn(ctx, "failed to create X11 listener", slog.Error(err))
-		s.metrics.x11HandlerErrors.WithLabelValues("listen").Add(1)
+		x.logger.Warn(ctx, "failed to create X11 listener", slog.Error(err))
+		x.x11HandlerErrors.WithLabelValues("listen").Add(1)
 		return -1, false
 	}
-	s.trackListener(ln, true)
 	defer func() {
 		if !handled {
-			s.trackListener(ln, false)
-			_ = ln.Close()
+			x.closeAndRemoveSession(x11session)
 		}
 	}()
 
-	err = addXauthEntry(ctx, s.fs, hostname, strconv.Itoa(display), x11.AuthProtocol, x11.AuthCookie)
+	err = addXauthEntry(ctx, x.fs, hostname, strconv.Itoa(x11session.display), x11.AuthProtocol, x11.AuthCookie)
 	if err != nil {
-		s.logger.Warn(ctx, "failed to add Xauthority entry", slog.Error(err))
-		s.metrics.x11HandlerErrors.WithLabelValues("xauthority").Add(1)
+		x.logger.Warn(ctx, "failed to add Xauthority entry", slog.Error(err))
+		x.x11HandlerErrors.WithLabelValues("xauthority").Add(1)
 		return -1, false
 	}
 
+	// clean up the X11 session if the SSH session completes.
 	go func() {
-		// Don't leave the listener open after the session is gone.
 		<-ctx.Done()
-		_ = ln.Close()
+		x.closeAndRemoveSession(x11session)
 	}()
 
-	go func() {
-		defer ln.Close()
-		defer s.trackListener(ln, false)
+	go x.listenForConnections(ctx, x11session, serverConn, x11)
+	x.logger.Debug(ctx, "X11 forwarding started", slog.F("display", x11session.display))
 
-		for {
-			conn, err := ln.Accept()
-			if err != nil {
-				if errors.Is(err, net.ErrClosed) {
-					return
-				}
-				s.logger.Warn(ctx, "failed to accept X11 connection", slog.Error(err))
+	return x11session.display, true
+}
+
+func (x *x11Forwarder) trackGoroutine() (closing bool, done func()) {
+	x.mu.Lock()
+	defer x.mu.Unlock()
+	if !x.closing {
+		x.wg.Add(1)
+		return false, func() { x.wg.Done() }
+	}
+	return true, func() {}
+}
+
+func (x *x11Forwarder) listenForConnections(
+	ctx context.Context, session *x11Session, serverConn *gossh.ServerConn, x11 ssh.X11,
+) {
+	defer x.closeAndRemoveSession(session)
+	if closing, done := x.trackGoroutine(); closing {
+		return
+	} else { // nolint: revive
+		defer done()
+	}
+
+	for {
+		conn, err := session.listener.Accept()
+		if err != nil {
+			if errors.Is(err, net.ErrClosed) {
 				return
 			}
-			if x11.SingleConnection {
-				s.logger.Debug(ctx, "single connection requested, closing X11 listener")
-				_ = ln.Close()
-			}
-
-			tcpConn, ok := conn.(*net.TCPConn)
-			if !ok {
-				s.logger.Warn(ctx, fmt.Sprintf("failed to cast connection to TCPConn. got: %T", conn))
-				_ = conn.Close()
-				continue
-			}
-			tcpAddr, ok := tcpConn.LocalAddr().(*net.TCPAddr)
-			if !ok {
-				s.logger.Warn(ctx, fmt.Sprintf("failed to cast local address to TCPAddr. got: %T", tcpConn.LocalAddr()))
-				_ = conn.Close()
-				continue
-			}
-
-			channel, reqs, err := serverConn.OpenChannel("x11", gossh.Marshal(struct {
-				OriginatorAddress string
-				OriginatorPort    uint32
-			}{
-				OriginatorAddress: tcpAddr.IP.String(),
-				// #nosec G115 - Safe conversion as TCP port numbers are within uint32 range (0-65535)
-				OriginatorPort: uint32(tcpAddr.Port),
-			}))
-			if err != nil {
-				s.logger.Warn(ctx, "failed to open X11 channel", slog.Error(err))
-				_ = conn.Close()
-				continue
-			}
-			go gossh.DiscardRequests(reqs)
-
-			if !s.trackConn(ln, conn, true) {
-				s.logger.Warn(ctx, "failed to track X11 connection")
-				_ = conn.Close()
-				continue
-			}
-			go func() {
-				defer s.trackConn(ln, conn, false)
-				Bicopy(ctx, conn, channel)
-			}()
+			x.logger.Warn(ctx, "failed to accept X11 connection", slog.Error(err))
+			return
 		}
-	}()
 
-	return display, true
+		// Update session usage time since a new X11 connection was forwarded.
+		x.mu.Lock()
+		session.usedAt = time.Now()
+		x.mu.Unlock()
+		if x11.SingleConnection {
+			x.logger.Debug(ctx, "single connection requested, closing X11 listener")
+			x.closeAndRemoveSession(session)
+		}
+
+		var originAddr string
+		var originPort uint32
+
+		if tcpConn, ok := conn.(*net.TCPConn); ok {
+			if tcpAddr, ok := tcpConn.LocalAddr().(*net.TCPAddr); ok {
+				originAddr = tcpAddr.IP.String()
+				// #nosec G115 - Safe conversion as TCP port numbers are within uint32 range (0-65535)
+				originPort = uint32(tcpAddr.Port)
+			}
+		}
+		// Fallback values for in-memory or non-TCP connections.
+		if originAddr == "" {
+			originAddr = "127.0.0.1"
+		}
+
+		channel, reqs, err := serverConn.OpenChannel("x11", gossh.Marshal(struct {
+			OriginatorAddress string
+			OriginatorPort    uint32
+		}{
+			OriginatorAddress: originAddr,
+			OriginatorPort:    originPort,
+		}))
+		if err != nil {
+			x.logger.Warn(ctx, "failed to open X11 channel", slog.Error(err))
+			_ = conn.Close()
+			continue
+		}
+		go gossh.DiscardRequests(reqs)
+
+		if !x.trackConn(conn, true) {
+			x.logger.Warn(ctx, "failed to track X11 connection")
+			_ = conn.Close()
+			continue
+		}
+		go func() {
+			defer x.trackConn(conn, false)
+			Bicopy(ctx, conn, channel)
+		}()
+	}
+}
+
+// closeAndRemoveSession closes and removes the session.
+func (x *x11Forwarder) closeAndRemoveSession(x11session *x11Session) {
+	_ = x11session.listener.Close()
+	x.mu.Lock()
+	delete(x.sessions, x11session)
+	x.mu.Unlock()
+}
+
+// createX11Session creates an X11 forwarding session.
+func (x *x11Forwarder) createX11Session(ctx context.Context, sshSession ssh.Session) (*x11Session, error) {
+	var (
+		ln      net.Listener
+		display int
+		err     error
+	)
+	// retry listener creation after evictions. Limit to 10 retries to prevent pathological cases looping forever.
+	const maxRetries = 10
+	for try := range maxRetries {
+		ln, display, err = x.createX11Listener(ctx)
+		if err == nil {
+			break
+		}
+		if try == maxRetries-1 {
+			return nil, xerrors.New("max retries exceeded while creating X11 session")
+		}
+		x.logger.Warn(ctx, "failed to create X11 listener; will evict an X11 forwarding session",
+			slog.F("num_current_sessions", x.numSessions()),
+			slog.Error(err))
+		x.evictLeastRecentlyUsedSession()
+	}
+	x.mu.Lock()
+	defer x.mu.Unlock()
+	if x.closing {
+		closeErr := ln.Close()
+		if closeErr != nil {
+			x.logger.Error(ctx, "error closing X11 listener", slog.Error(closeErr))
+		}
+		return nil, xerrors.New("server is closing")
+	}
+	x11Sess := &x11Session{
+		session:  sshSession,
+		display:  display,
+		listener: ln,
+		usedAt:   time.Now(),
+	}
+	x.sessions[x11Sess] = struct{}{}
+	return x11Sess, nil
+}
+
+func (x *x11Forwarder) numSessions() int {
+	x.mu.Lock()
+	defer x.mu.Unlock()
+	return len(x.sessions)
+}
+
+func (x *x11Forwarder) popLeastRecentlyUsedSession() *x11Session {
+	x.mu.Lock()
+	defer x.mu.Unlock()
+	var lru *x11Session
+	for s := range x.sessions {
+		if lru == nil {
+			lru = s
+			continue
+		}
+		if s.usedAt.Before(lru.usedAt) {
+			lru = s
+			continue
+		}
+	}
+	if lru == nil {
+		x.logger.Debug(context.Background(), "tried to pop from empty set of X11 sessions")
+		return nil
+	}
+	delete(x.sessions, lru)
+	return lru
+}
+
+func (x *x11Forwarder) evictLeastRecentlyUsedSession() {
+	lru := x.popLeastRecentlyUsedSession()
+	if lru == nil {
+		return
+	}
+	err := lru.listener.Close()
+	if err != nil {
+		x.logger.Error(context.Background(), "failed to close evicted X11 session listener", slog.Error(err))
+	}
+	// when we evict, we also want to force the SSH session to be closed as well. This is because we intend to reuse
+	// the X11 TCP listener port for a new X11 forwarding session. If we left the SSH session up, then graphical apps
+	// started in that session could potentially connect to an unintended X11 Server (i.e. the display on a different
+	// computer than the one that started the SSH session). Most likely, this session is a zombie anyway if we've
+	// reached the maximum number of X11 forwarding sessions.
+	err = lru.session.Close()
+	if err != nil {
+		x.logger.Error(context.Background(), "failed to close evicted X11 SSH session", slog.Error(err))
+	}
 }
 
 // createX11Listener creates a listener for X11 forwarding, it will use
 // the next available port starting from X11StartPort and displayOffset.
-func createX11Listener(ctx context.Context, displayOffset int) (ln net.Listener, display int, err error) {
-	var lc net.ListenConfig
+func (x *x11Forwarder) createX11Listener(ctx context.Context) (ln net.Listener, display int, err error) {
 	// Look for an open port to listen on.
-	for port := X11StartPort + displayOffset; port < math.MaxUint16; port++ {
-		ln, err = lc.Listen(ctx, "tcp", fmt.Sprintf("localhost:%d", port))
+	for port := X11StartPort + x.displayOffset; port <= X11MaxPort; port++ {
+		if ctx.Err() != nil {
+			return nil, -1, ctx.Err()
+		}
+
+		ln, err = x.network.Listen("tcp", fmt.Sprintf("localhost:%d", port))
 		if err == nil {
 			display = port - X11StartPort
 			return ln, display, nil
@@ -156,6 +328,49 @@ func createX11Listener(ctx context.Context, displayOffset int) (ln net.Listener,
 	return nil, -1, xerrors.Errorf("failed to find open port for X11 listener: %w", err)
 }
 
+// trackConn registers the connection with the x11Forwarder. If the server is
+// closed, the connection is not registered and should be closed.
+//
+//nolint:revive
+func (x *x11Forwarder) trackConn(c net.Conn, add bool) (ok bool) {
+	x.mu.Lock()
+	defer x.mu.Unlock()
+	if add {
+		if x.closing {
+			// Server or listener closed.
+			return false
+		}
+		x.wg.Add(1)
+		x.connections[c] = struct{}{}
+		return true
+	}
+	x.wg.Done()
+	delete(x.connections, c)
+	return true
+}
+
+func (x *x11Forwarder) Close() error {
+	x.mu.Lock()
+	x.closing = true
+
+	for s := range x.sessions {
+		sErr := s.listener.Close()
+		if sErr != nil {
+			x.logger.Debug(context.Background(), "failed to close X11 listener", slog.Error(sErr))
+		}
+	}
+	for c := range x.connections {
+		cErr := c.Close()
+		if cErr != nil {
+			x.logger.Debug(context.Background(), "failed to close X11 connection", slog.Error(cErr))
+		}
+	}
+
+	x.mu.Unlock()
+	x.wg.Wait()
+	return nil
+}
+
 // addXauthEntry adds an Xauthority entry to the Xauthority file.
 // The Xauthority file is located at ~/.Xauthority.
 func addXauthEntry(ctx context.Context, fs afero.Fs, host string, display string, authProtocol string, authCookie string) error {

agent/agentssh/x11_internal_test.go

@@ -228,7 +228,6 @@ func Test_addXauthEntry(t *testing.T) {
 	require.NoError(t, err)
 
 	for _, tt := range tests {
-		tt := tt
 		t.Run(tt.name, func(t *testing.T) {
 			t.Parallel()
 

agent/agentssh/x11_test.go

@@ -3,9 +3,9 @@ package agentssh_test
 import (
 	"bufio"
 	"bytes"
-	"context"
 	"encoding/hex"
 	"fmt"
+	"io"
 	"net"
 	"os"
 	"path/filepath"
@@ -32,10 +32,19 @@ func TestServer_X11(t *testing.T) {
 		t.Skip("X11 forwarding is only supported on Linux")
 	}
 
-	ctx := context.Background()
+	ctx := testutil.Context(t, testutil.WaitShort)
 	logger := testutil.Logger(t)
-	fs := afero.NewOsFs()
-	s, err := agentssh.NewServer(ctx, logger, prometheus.NewRegistry(), fs, agentexec.DefaultExecer, &agentssh.Config{})
+	fs := afero.NewMemMapFs()
+
+	// Use in-process networking for X11 forwarding.
+	inproc := testutil.NewInProcNet()
+
+	// Create server config with custom X11 listener.
+	cfg := &agentssh.Config{
+		X11Net: inproc,
+	}
+
+	s, err := agentssh.NewServer(ctx, logger, prometheus.NewRegistry(), fs, agentexec.DefaultExecer, cfg)
 	require.NoError(t, err)
 	defer s.Close()
 	err = s.UpdateHostSigner(42)
@@ -93,17 +102,15 @@ func TestServer_X11(t *testing.T) {
 
 	x11Chans := c.HandleChannelOpen("x11")
 	payload := "hello world"
-	require.Eventually(t, func() bool {
-		conn, err := net.Dial("tcp", fmt.Sprintf("localhost:%d", agentssh.X11StartPort+displayNumber))
-		if err == nil {
-			_, err = conn.Write([]byte(payload))
-			assert.NoError(t, err)
-			_ = conn.Close()
-		}
-		return err == nil
-	}, testutil.WaitShort, testutil.IntervalFast)
+	go func() {
+		conn, err := inproc.Dial(ctx, testutil.NewAddr("tcp", fmt.Sprintf("localhost:%d", agentssh.X11StartPort+displayNumber)))
+		assert.NoError(t, err)
+		_, err = conn.Write([]byte(payload))
+		assert.NoError(t, err)
+		_ = conn.Close()
+	}()
 
-	x11 := <-x11Chans
+	x11 := testutil.RequireReceive(ctx, t, x11Chans)
 	ch, reqs, err := x11.Accept()
 	require.NoError(t, err)
 	go gossh.DiscardRequests(reqs)
@@ -121,3 +128,211 @@ func TestServer_X11(t *testing.T) {
 	_, err = fs.Stat(filepath.Join(home, ".Xauthority"))
 	require.NoError(t, err)
 }
+
+func TestServer_X11_EvictionLRU(t *testing.T) {
+	t.Parallel()
+	if runtime.GOOS != "linux" {
+		t.Skip("X11 forwarding is only supported on Linux")
+	}
+
+	ctx := testutil.Context(t, testutil.WaitSuperLong)
+	logger := testutil.Logger(t)
+	fs := afero.NewMemMapFs()
+
+	// Use in-process networking for X11 forwarding.
+	inproc := testutil.NewInProcNet()
+
+	cfg := &agentssh.Config{
+		X11Net: inproc,
+	}
+
+	s, err := agentssh.NewServer(ctx, logger, prometheus.NewRegistry(), fs, agentexec.DefaultExecer, cfg)
+	require.NoError(t, err)
+	defer s.Close()
+	err = s.UpdateHostSigner(42)
+	require.NoError(t, err)
+
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	require.NoError(t, err)
+
+	done := testutil.Go(t, func() {
+		err := s.Serve(ln)
+		assert.Error(t, err)
+	})
+
+	c := sshClient(t, ln.Addr().String())
+
+	// block off one port to test x11Forwarder evicts at highest port, not number of listeners.
+	externalListener, err := inproc.Listen("tcp",
+		fmt.Sprintf("localhost:%d", agentssh.X11StartPort+agentssh.X11DefaultDisplayOffset+1))
+	require.NoError(t, err)
+	defer externalListener.Close()
+
+	// Calculate how many simultaneous X11 sessions we can create given the
+	// configured port range.
+
+	startPort := agentssh.X11StartPort + agentssh.X11DefaultDisplayOffset
+	maxSessions := agentssh.X11MaxPort - startPort + 1 - 1 // -1 for the blocked port
+	require.Greater(t, maxSessions, 0, "expected a positive maxSessions value")
+
+	// shellSession holds references to the session and its standard streams so
+	// that the test can keep them open (and optionally interact with them) for
+	// the lifetime of the test. If we don't start the Shell with pipes in place,
+	// the session will be torn down asynchronously during the test.
+	type shellSession struct {
+		sess   *gossh.Session
+		stdin  io.WriteCloser
+		stdout io.Reader
+		stderr io.Reader
+		// scanner is used to read the output of the session, line by line.
+		scanner *bufio.Scanner
+	}
+
+	sessions := make([]shellSession, 0, maxSessions)
+	for i := 0; i < maxSessions; i++ {
+		sess, err := c.NewSession()
+		require.NoError(t, err)
+
+		_, err = sess.SendRequest("x11-req", true, gossh.Marshal(ssh.X11{
+			AuthProtocol: "MIT-MAGIC-COOKIE-1",
+			AuthCookie:   hex.EncodeToString([]byte(fmt.Sprintf("cookie%d", i))),
+			ScreenNumber: uint32(0),
+		}))
+		require.NoError(t, err)
+
+		stdin, err := sess.StdinPipe()
+		require.NoError(t, err)
+		stdout, err := sess.StdoutPipe()
+		require.NoError(t, err)
+		stderr, err := sess.StderrPipe()
+		require.NoError(t, err)
+		require.NoError(t, sess.Shell())
+
+		// The SSH server lazily starts the session. We need to write a command
+		// and read back to ensure the X11 forwarding is started.
+		scanner := bufio.NewScanner(stdout)
+		msg := fmt.Sprintf("ready-%d", i)
+		_, err = stdin.Write([]byte("echo " + msg + "\n"))
+		require.NoError(t, err)
+		// Read until we get the message (first token may be empty due to shell prompt)
+		for scanner.Scan() {
+			line := strings.TrimSpace(scanner.Text())
+			if strings.Contains(line, msg) {
+				break
+			}
+		}
+		require.NoError(t, scanner.Err())
+
+		sessions = append(sessions, shellSession{
+			sess:    sess,
+			stdin:   stdin,
+			stdout:  stdout,
+			stderr:  stderr,
+			scanner: scanner,
+		})
+	}
+
+	// Connect X11 forwarding to the first session. This is used to test that
+	// connecting counts as a use of the display.
+	x11Chans := c.HandleChannelOpen("x11")
+	payload := "hello world"
+	go func() {
+		conn, err := inproc.Dial(ctx, testutil.NewAddr("tcp", fmt.Sprintf("localhost:%d", agentssh.X11StartPort+agentssh.X11DefaultDisplayOffset)))
+		if !assert.NoError(t, err) {
+			return
+		}
+		_, err = conn.Write([]byte(payload))
+		assert.NoError(t, err)
+		_ = conn.Close()
+	}()
+
+	x11 := testutil.RequireReceive(ctx, t, x11Chans)
+	ch, reqs, err := x11.Accept()
+	require.NoError(t, err)
+	go gossh.DiscardRequests(reqs)
+	got := make([]byte, len(payload))
+	_, err = ch.Read(got)
+	require.NoError(t, err)
+	assert.Equal(t, payload, string(got))
+	_ = ch.Close()
+
+	// Create one more session which should evict a session and reuse the display.
+	// The first session was used to connect X11 forwarding, so it should not be evicted.
+	// Therefore, the second session should be evicted and its display reused.
+	extraSess, err := c.NewSession()
+	require.NoError(t, err)
+
+	_, err = extraSess.SendRequest("x11-req", true, gossh.Marshal(ssh.X11{
+		AuthProtocol: "MIT-MAGIC-COOKIE-1",
+		AuthCookie:   hex.EncodeToString([]byte("extra")),
+		ScreenNumber: uint32(0),
+	}))
+	require.NoError(t, err)
+
+	// Ask the remote side for the DISPLAY value so we can extract the display
+	// number that was assigned to this session.
+	out, err := extraSess.Output("echo DISPLAY=$DISPLAY")
+	require.NoError(t, err)
+
+	// Example output line: "DISPLAY=localhost:10.0".
+	var newDisplayNumber int
+	{
+		sc := bufio.NewScanner(bytes.NewReader(out))
+		for sc.Scan() {
+			line := strings.TrimSpace(sc.Text())
+			if strings.HasPrefix(line, "DISPLAY=") {
+				parts := strings.SplitN(line, ":", 2)
+				require.Len(t, parts, 2)
+				displayPart := parts[1]
+				if strings.Contains(displayPart, ".") {
+					displayPart = strings.SplitN(displayPart, ".", 2)[0]
+				}
+				var convErr error
+				newDisplayNumber, convErr = strconv.Atoi(displayPart)
+				require.NoError(t, convErr)
+				break
+			}
+		}
+		require.NoError(t, sc.Err())
+	}
+
+	// The display number reused should correspond to the SECOND session (display offset 12)
+	expectedDisplay := agentssh.X11DefaultDisplayOffset + 2 // +1 was blocked port
+	assert.Equal(t, expectedDisplay, newDisplayNumber, "second session should have been evicted and its display reused")
+
+	// First session should still be alive: send cmd and read output.
+	msgFirst := "still-alive"
+	_, err = sessions[0].stdin.Write([]byte("echo " + msgFirst + "\n"))
+	require.NoError(t, err)
+	for sessions[0].scanner.Scan() {
+		line := strings.TrimSpace(sessions[0].scanner.Text())
+		if strings.Contains(line, msgFirst) {
+			break
+		}
+	}
+	require.NoError(t, sessions[0].scanner.Err())
+
+	// Second session should now be closed.
+	_, err = sessions[1].stdin.Write([]byte("echo dead\n"))
+	require.ErrorIs(t, err, io.EOF)
+	err = sessions[1].sess.Wait()
+	require.Error(t, err)
+
+	// Cleanup.
+	for i, sh := range sessions {
+		if i == 1 {
+			// already closed
+			continue
+		}
+		err = sh.stdin.Close()
+		require.NoError(t, err)
+		err = sh.sess.Wait()
+		require.NoError(t, err)
+	}
+	err = extraSess.Close()
+	require.ErrorIs(t, err, io.EOF)
+
+	err = s.Close()
+	require.NoError(t, err)
+	_ = testutil.TryReceive(ctx, t, done)
+}

agent/agenttest/client.go

@@ -98,8 +98,8 @@ func (c *Client) Close() {
 	c.derpMapOnce.Do(func() { close(c.derpMapUpdates) })
 }
 
-func (c *Client) ConnectRPC25(ctx context.Context) (
-	agentproto.DRPCAgentClient25, proto.DRPCTailnetClient25, error,
+func (c *Client) ConnectRPC26(ctx context.Context) (
+	agentproto.DRPCAgentClient26, proto.DRPCTailnetClient26, error,
 ) {
 	conn, lis := drpcsdk.MemTransportPipe()
 	c.LastWorkspaceAgent = func() {
@@ -163,20 +163,40 @@ func (c *Client) GetConnectionReports() []*agentproto.ReportConnectionRequest {
 	return c.fakeAgentAPI.GetConnectionReports()
 }
 
+func (c *Client) GetSubAgents() []*agentproto.SubAgent {
+	return c.fakeAgentAPI.GetSubAgents()
+}
+
+func (c *Client) GetSubAgentDirectory(id uuid.UUID) (string, error) {
+	return c.fakeAgentAPI.GetSubAgentDirectory(id)
+}
+
+func (c *Client) GetSubAgentDisplayApps(id uuid.UUID) ([]agentproto.CreateSubAgentRequest_DisplayApp, error) {
+	return c.fakeAgentAPI.GetSubAgentDisplayApps(id)
+}
+
+func (c *Client) GetSubAgentApps(id uuid.UUID) ([]*agentproto.CreateSubAgentRequest_App, error) {
+	return c.fakeAgentAPI.GetSubAgentApps(id)
+}
+
 type FakeAgentAPI struct {
 	sync.Mutex
 	t      testing.TB
 	logger slog.Logger
 
-	manifest          *agentproto.Manifest
-	startupCh         chan *agentproto.Startup
-	statsCh           chan *agentproto.Stats
-	appHealthCh       chan *agentproto.BatchUpdateAppHealthRequest
-	logsCh            chan<- *agentproto.BatchCreateLogsRequest
-	lifecycleStates   []codersdk.WorkspaceAgentLifecycle
-	metadata          map[string]agentsdk.Metadata
-	timings           []*agentproto.Timing
-	connectionReports []*agentproto.ReportConnectionRequest
+	manifest            *agentproto.Manifest
+	startupCh           chan *agentproto.Startup
+	statsCh             chan *agentproto.Stats
+	appHealthCh         chan *agentproto.BatchUpdateAppHealthRequest
+	logsCh              chan<- *agentproto.BatchCreateLogsRequest
+	lifecycleStates     []codersdk.WorkspaceAgentLifecycle
+	metadata            map[string]agentsdk.Metadata
+	timings             []*agentproto.Timing
+	connectionReports   []*agentproto.ReportConnectionRequest
+	subAgents           map[uuid.UUID]*agentproto.SubAgent
+	subAgentDirs        map[uuid.UUID]string
+	subAgentDisplayApps map[uuid.UUID][]agentproto.CreateSubAgentRequest_DisplayApp
+	subAgentApps        map[uuid.UUID][]*agentproto.CreateSubAgentRequest_App
 
 	getAnnouncementBannersFunc              func() ([]codersdk.BannerConfig, error)
 	getResourcesMonitoringConfigurationFunc func() (*agentproto.GetResourcesMonitoringConfigurationResponse, error)
@@ -365,6 +385,148 @@ func (f *FakeAgentAPI) GetConnectionReports() []*agentproto.ReportConnectionRequ
 	return slices.Clone(f.connectionReports)
 }
 
+func (f *FakeAgentAPI) CreateSubAgent(ctx context.Context, req *agentproto.CreateSubAgentRequest) (*agentproto.CreateSubAgentResponse, error) {
+	f.Lock()
+	defer f.Unlock()
+
+	f.logger.Debug(ctx, "create sub agent called", slog.F("req", req))
+
+	// Generate IDs for the new sub-agent.
+	subAgentID := uuid.New()
+	authToken := uuid.New()
+
+	// Create the sub-agent proto object.
+	subAgent := &agentproto.SubAgent{
+		Id:        subAgentID[:],
+		Name:      req.Name,
+		AuthToken: authToken[:],
+	}
+
+	// Store the sub-agent in our map.
+	if f.subAgents == nil {
+		f.subAgents = make(map[uuid.UUID]*agentproto.SubAgent)
+	}
+	f.subAgents[subAgentID] = subAgent
+	if f.subAgentDirs == nil {
+		f.subAgentDirs = make(map[uuid.UUID]string)
+	}
+	f.subAgentDirs[subAgentID] = req.GetDirectory()
+	if f.subAgentDisplayApps == nil {
+		f.subAgentDisplayApps = make(map[uuid.UUID][]agentproto.CreateSubAgentRequest_DisplayApp)
+	}
+	f.subAgentDisplayApps[subAgentID] = req.GetDisplayApps()
+	if f.subAgentApps == nil {
+		f.subAgentApps = make(map[uuid.UUID][]*agentproto.CreateSubAgentRequest_App)
+	}
+	f.subAgentApps[subAgentID] = req.GetApps()
+
+	// For a fake implementation, we don't create workspace apps.
+	// Real implementations would handle req.Apps here.
+	return &agentproto.CreateSubAgentResponse{
+		Agent:             subAgent,
+		AppCreationErrors: nil,
+	}, nil
+}
+
+func (f *FakeAgentAPI) DeleteSubAgent(ctx context.Context, req *agentproto.DeleteSubAgentRequest) (*agentproto.DeleteSubAgentResponse, error) {
+	f.Lock()
+	defer f.Unlock()
+
+	f.logger.Debug(ctx, "delete sub agent called", slog.F("req", req))
+
+	subAgentID, err := uuid.FromBytes(req.Id)
+	if err != nil {
+		return nil, err
+	}
+
+	// Remove the sub-agent from our map.
+	if f.subAgents != nil {
+		delete(f.subAgents, subAgentID)
+	}
+
+	return &agentproto.DeleteSubAgentResponse{}, nil
+}
+
+func (f *FakeAgentAPI) ListSubAgents(ctx context.Context, req *agentproto.ListSubAgentsRequest) (*agentproto.ListSubAgentsResponse, error) {
+	f.Lock()
+	defer f.Unlock()
+
+	f.logger.Debug(ctx, "list sub agents called", slog.F("req", req))
+
+	var agents []*agentproto.SubAgent
+	if f.subAgents != nil {
+		agents = make([]*agentproto.SubAgent, 0, len(f.subAgents))
+		for _, agent := range f.subAgents {
+			agents = append(agents, agent)
+		}
+	}
+
+	return &agentproto.ListSubAgentsResponse{
+		Agents: agents,
+	}, nil
+}
+
+func (f *FakeAgentAPI) GetSubAgents() []*agentproto.SubAgent {
+	f.Lock()
+	defer f.Unlock()
+	var agents []*agentproto.SubAgent
+	if f.subAgents != nil {
+		agents = make([]*agentproto.SubAgent, 0, len(f.subAgents))
+		for _, agent := range f.subAgents {
+			agents = append(agents, agent)
+		}
+	}
+	return agents
+}
+
+func (f *FakeAgentAPI) GetSubAgentDirectory(id uuid.UUID) (string, error) {
+	f.Lock()
+	defer f.Unlock()
+
+	if f.subAgentDirs == nil {
+		return "", xerrors.New("no sub-agent directories available")
+	}
+
+	dir, ok := f.subAgentDirs[id]
+	if !ok {
+		return "", xerrors.New("sub-agent directory not found")
+	}
+
+	return dir, nil
+}
+
+func (f *FakeAgentAPI) GetSubAgentDisplayApps(id uuid.UUID) ([]agentproto.CreateSubAgentRequest_DisplayApp, error) {
+	f.Lock()
+	defer f.Unlock()
+
+	if f.subAgentDisplayApps == nil {
+		return nil, xerrors.New("no sub-agent display apps available")
+	}
+
+	displayApps, ok := f.subAgentDisplayApps[id]
+	if !ok {
+		return nil, xerrors.New("sub-agent display apps not found")
+	}
+
+	return displayApps, nil
+}
+
+func (f *FakeAgentAPI) GetSubAgentApps(id uuid.UUID) ([]*agentproto.CreateSubAgentRequest_App, error) {
+	f.Lock()
+	defer f.Unlock()
+
+	if f.subAgentApps == nil {
+		return nil, xerrors.New("no sub-agent apps available")
+	}
+
+	apps, ok := f.subAgentApps[id]
+	if !ok {
+		return nil, xerrors.New("sub-agent apps not found")
+	}
+
+	return apps, nil
+}
+
 func NewFakeAgentAPI(t testing.TB, logger slog.Logger, manifest *agentproto.Manifest, statsCh chan *agentproto.Stats) *FakeAgentAPI {
 	return &FakeAgentAPI{
 		t:           t,

agent/api.go

@@ -6,15 +6,13 @@ import (
 	"time"
 
 	"github.com/go-chi/chi/v5"
-
 	"github.com/google/uuid"
 
-	"github.com/coder/coder/v2/agent/agentcontainers"
 	"github.com/coder/coder/v2/coderd/httpapi"
 	"github.com/coder/coder/v2/codersdk"
 )
 
-func (a *agent) apiHandler() (http.Handler, func() error) {
+func (a *agent) apiHandler() http.Handler {
 	r := chi.NewRouter()
 	r.Get("/", func(rw http.ResponseWriter, r *http.Request) {
 		httpapi.Write(r.Context(), rw, http.StatusOK, codersdk.Response{
@@ -39,31 +37,19 @@ func (a *agent) apiHandler() (http.Handler, func() error) {
 		cacheDuration: cacheDuration,
 	}
 
-	if a.experimentalDevcontainersEnabled {
-		containerAPIOpts := []agentcontainers.Option{
-			agentcontainers.WithExecer(a.execer),
-			agentcontainers.WithScriptLogger(func(logSourceID uuid.UUID) agentcontainers.ScriptLogger {
-				return a.logSender.GetScriptLogger(logSourceID)
-			}),
-		}
-		manifest := a.manifest.Load()
-		if manifest != nil && len(manifest.Devcontainers) > 0 {
-			containerAPIOpts = append(
-				containerAPIOpts,
-				agentcontainers.WithDevcontainers(manifest.Devcontainers, manifest.Scripts),
-			)
-		}
-
-		// Append after to allow the agent options to override the default options.
-		containerAPIOpts = append(containerAPIOpts, a.containerAPIOptions...)
-
-		containerAPI := agentcontainers.NewAPI(a.logger.Named("containers"), containerAPIOpts...)
-		r.Mount("/api/v0/containers", containerAPI.Routes())
-		a.containerAPI.Store(containerAPI)
+	if a.devcontainers {
+		r.Mount("/api/v0/containers", a.containerAPI.Routes())
+	} else if manifest := a.manifest.Load(); manifest != nil && manifest.ParentID != uuid.Nil {
+		r.HandleFunc("/api/v0/containers", func(w http.ResponseWriter, r *http.Request) {
+			httpapi.Write(r.Context(), w, http.StatusForbidden, codersdk.Response{
+				Message: "Dev Container feature not supported.",
+				Detail:  "Dev Container integration inside other Dev Containers is explicitly not supported.",
+			})
+		})
 	} else {
 		r.HandleFunc("/api/v0/containers", func(w http.ResponseWriter, r *http.Request) {
 			httpapi.Write(r.Context(), w, http.StatusForbidden, codersdk.Response{
-				Message: "The agent dev containers feature is experimental and not enabled by default.",
+				Message: "Dev Container feature not enabled.",
 				Detail:  "To enable this feature, set CODER_AGENT_DEVCONTAINERS_ENABLE=true in your template.",
 			})
 		})
@@ -80,12 +66,7 @@ func (a *agent) apiHandler() (http.Handler, func() error) {
 	r.Get("/debug/manifest", a.HandleHTTPDebugManifest)
 	r.Get("/debug/prometheus", promHandler.ServeHTTP)
 
-	return r, func() error {
-		if containerAPI := a.containerAPI.Load(); containerAPI != nil {
-			return containerAPI.Close()
-		}
-		return nil
-	}
+	return r
 }
 
 type listeningPortsHandler struct {

agent/proto/agent.proto

@@ -24,6 +24,7 @@ message WorkspaceApp {
 		OWNER = 1;
 		AUTHENTICATED = 2;
 		PUBLIC = 3;
+		ORGANIZATION = 4;
 	}
 	SharingLevel sharing_level = 10;
 
@@ -377,6 +378,88 @@ message ReportConnectionRequest {
 	Connection connection = 1;
 }
 
+message SubAgent {
+	string name = 1;
+	bytes id = 2;
+	bytes auth_token = 3;
+}
+
+message CreateSubAgentRequest {
+	string name = 1;
+	string directory = 2;
+	string architecture = 3;
+	string operating_system = 4;
+
+	message App {
+		message Healthcheck {
+			int32 interval = 1;
+			int32 threshold = 2;
+			string url = 3;
+		}
+
+		enum OpenIn {
+			SLIM_WINDOW = 0;
+			TAB = 1;
+		}
+
+		enum SharingLevel {
+			OWNER = 0;
+			AUTHENTICATED = 1;
+			PUBLIC = 2;
+			ORGANIZATION = 3;
+		}
+
+		string slug = 1;
+		optional string command = 2;
+		optional string display_name = 3;
+		optional bool external = 4;
+		optional string group = 5;
+		optional Healthcheck healthcheck = 6;
+		optional bool hidden = 7;
+		optional string icon = 8;
+		optional OpenIn open_in = 9;
+		optional int32 order = 10;
+		optional SharingLevel share = 11;
+		optional bool subdomain = 12;
+		optional string url = 13;
+	}
+
+	repeated App apps = 5;
+
+	enum DisplayApp {
+		VSCODE = 0;
+		VSCODE_INSIDERS = 1;
+		WEB_TERMINAL = 2;
+		SSH_HELPER = 3;
+		PORT_FORWARDING_HELPER = 4;
+	}
+
+	repeated DisplayApp display_apps = 6;
+}
+
+message CreateSubAgentResponse {
+	message AppCreationError {
+		int32 index = 1;
+		optional string field = 2;
+		string error = 3;
+	}
+
+	SubAgent agent = 1;
+	repeated AppCreationError app_creation_errors = 2;
+}
+
+message DeleteSubAgentRequest {
+	bytes id = 1;
+}
+
+message DeleteSubAgentResponse {}
+
+message ListSubAgentsRequest {}
+
+message ListSubAgentsResponse {
+	repeated SubAgent agents = 1;
+}
+
 service Agent {
 	rpc GetManifest(GetManifestRequest) returns (Manifest);
 	rpc GetServiceBanner(GetServiceBannerRequest) returns (ServiceBanner);
@@ -391,4 +474,7 @@ service Agent {
 	rpc GetResourcesMonitoringConfiguration(GetResourcesMonitoringConfigurationRequest) returns (GetResourcesMonitoringConfigurationResponse);
 	rpc PushResourcesMonitoringUsage(PushResourcesMonitoringUsageRequest) returns (PushResourcesMonitoringUsageResponse);
 	rpc ReportConnection(ReportConnectionRequest) returns (google.protobuf.Empty);
+	rpc CreateSubAgent(CreateSubAgentRequest) returns (CreateSubAgentResponse);
+	rpc DeleteSubAgent(DeleteSubAgentRequest) returns (DeleteSubAgentResponse);
+	rpc ListSubAgents(ListSubAgentsRequest) returns (ListSubAgentsResponse);
 }

agent/proto/agent_drpc.pb.go

@@ -52,6 +52,9 @@ type DRPCAgentClient interface {
 	GetResourcesMonitoringConfiguration(ctx context.Context, in *GetResourcesMonitoringConfigurationRequest) (*GetResourcesMonitoringConfigurationResponse, error)
 	PushResourcesMonitoringUsage(ctx context.Context, in *PushResourcesMonitoringUsageRequest) (*PushResourcesMonitoringUsageResponse, error)
 	ReportConnection(ctx context.Context, in *ReportConnectionRequest) (*emptypb.Empty, error)
+	CreateSubAgent(ctx context.Context, in *CreateSubAgentRequest) (*CreateSubAgentResponse, error)
+	DeleteSubAgent(ctx context.Context, in *DeleteSubAgentRequest) (*DeleteSubAgentResponse, error)
+	ListSubAgents(ctx context.Context, in *ListSubAgentsRequest) (*ListSubAgentsResponse, error)
 }
 
 type drpcAgentClient struct {
@@ -181,6 +184,33 @@ func (c *drpcAgentClient) ReportConnection(ctx context.Context, in *ReportConnec
 	return out, nil
 }
 
+func (c *drpcAgentClient) CreateSubAgent(ctx context.Context, in *CreateSubAgentRequest) (*CreateSubAgentResponse, error) {
+	out := new(CreateSubAgentResponse)
+	err := c.cc.Invoke(ctx, "/coder.agent.v2.Agent/CreateSubAgent", drpcEncoding_File_agent_proto_agent_proto{}, in, out)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *drpcAgentClient) DeleteSubAgent(ctx context.Context, in *DeleteSubAgentRequest) (*DeleteSubAgentResponse, error) {
+	out := new(DeleteSubAgentResponse)
+	err := c.cc.Invoke(ctx, "/coder.agent.v2.Agent/DeleteSubAgent", drpcEncoding_File_agent_proto_agent_proto{}, in, out)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *drpcAgentClient) ListSubAgents(ctx context.Context, in *ListSubAgentsRequest) (*ListSubAgentsResponse, error) {
+	out := new(ListSubAgentsResponse)
+	err := c.cc.Invoke(ctx, "/coder.agent.v2.Agent/ListSubAgents", drpcEncoding_File_agent_proto_agent_proto{}, in, out)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
 type DRPCAgentServer interface {
 	GetManifest(context.Context, *GetManifestRequest) (*Manifest, error)
 	GetServiceBanner(context.Context, *GetServiceBannerRequest) (*ServiceBanner, error)
@@ -195,6 +225,9 @@ type DRPCAgentServer interface {
 	GetResourcesMonitoringConfiguration(context.Context, *GetResourcesMonitoringConfigurationRequest) (*GetResourcesMonitoringConfigurationResponse, error)
 	PushResourcesMonitoringUsage(context.Context, *PushResourcesMonitoringUsageRequest) (*PushResourcesMonitoringUsageResponse, error)
 	ReportConnection(context.Context, *ReportConnectionRequest) (*emptypb.Empty, error)
+	CreateSubAgent(context.Context, *CreateSubAgentRequest) (*CreateSubAgentResponse, error)
+	DeleteSubAgent(context.Context, *DeleteSubAgentRequest) (*DeleteSubAgentResponse, error)
+	ListSubAgents(context.Context, *ListSubAgentsRequest) (*ListSubAgentsResponse, error)
 }
 
 type DRPCAgentUnimplementedServer struct{}
@@ -251,9 +284,21 @@ func (s *DRPCAgentUnimplementedServer) ReportConnection(context.Context, *Report
 	return nil, drpcerr.WithCode(errors.New("Unimplemented"), drpcerr.Unimplemented)
 }
 
+func (s *DRPCAgentUnimplementedServer) CreateSubAgent(context.Context, *CreateSubAgentRequest) (*CreateSubAgentResponse, error) {
+	return nil, drpcerr.WithCode(errors.New("Unimplemented"), drpcerr.Unimplemented)
+}
+
+func (s *DRPCAgentUnimplementedServer) DeleteSubAgent(context.Context, *DeleteSubAgentRequest) (*DeleteSubAgentResponse, error) {
+	return nil, drpcerr.WithCode(errors.New("Unimplemented"), drpcerr.Unimplemented)
+}
+
+func (s *DRPCAgentUnimplementedServer) ListSubAgents(context.Context, *ListSubAgentsRequest) (*ListSubAgentsResponse, error) {
+	return nil, drpcerr.WithCode(errors.New("Unimplemented"), drpcerr.Unimplemented)
+}
+
 type DRPCAgentDescription struct{}
 
-func (DRPCAgentDescription) NumMethods() int { return 13 }
+func (DRPCAgentDescription) NumMethods() int { return 16 }
 
 func (DRPCAgentDescription) Method(n int) (string, drpc.Encoding, drpc.Receiver, interface{}, bool) {
 	switch n {
@@ -374,6 +419,33 @@ func (DRPCAgentDescription) Method(n int) (string, drpc.Encoding, drpc.Receiver,
 						in1.(*ReportConnectionRequest),
 					)
 			}, DRPCAgentServer.ReportConnection, true
+	case 13:
+		return "/coder.agent.v2.Agent/CreateSubAgent", drpcEncoding_File_agent_proto_agent_proto{},
+			func(srv interface{}, ctx context.Context, in1, in2 interface{}) (drpc.Message, error) {
+				return srv.(DRPCAgentServer).
+					CreateSubAgent(
+						ctx,
+						in1.(*CreateSubAgentRequest),
+					)
+			}, DRPCAgentServer.CreateSubAgent, true
+	case 14:
+		return "/coder.agent.v2.Agent/DeleteSubAgent", drpcEncoding_File_agent_proto_agent_proto{},
+			func(srv interface{}, ctx context.Context, in1, in2 interface{}) (drpc.Message, error) {
+				return srv.(DRPCAgentServer).
+					DeleteSubAgent(
+						ctx,
+						in1.(*DeleteSubAgentRequest),
+					)
+			}, DRPCAgentServer.DeleteSubAgent, true
+	case 15:
+		return "/coder.agent.v2.Agent/ListSubAgents", drpcEncoding_File_agent_proto_agent_proto{},
+			func(srv interface{}, ctx context.Context, in1, in2 interface{}) (drpc.Message, error) {
+				return srv.(DRPCAgentServer).
+					ListSubAgents(
+						ctx,
+						in1.(*ListSubAgentsRequest),
+					)
+			}, DRPCAgentServer.ListSubAgents, true
 	default:
 		return "", nil, nil, nil, false
 	}
@@ -590,3 +662,51 @@ func (x *drpcAgent_ReportConnectionStream) SendAndClose(m *emptypb.Empty) error
 	}
 	return x.CloseSend()
 }
+
+type DRPCAgent_CreateSubAgentStream interface {
+	drpc.Stream
+	SendAndClose(*CreateSubAgentResponse) error
+}
+
+type drpcAgent_CreateSubAgentStream struct {
+	drpc.Stream
+}
+
+func (x *drpcAgent_CreateSubAgentStream) SendAndClose(m *CreateSubAgentResponse) error {
+	if err := x.MsgSend(m, drpcEncoding_File_agent_proto_agent_proto{}); err != nil {
+		return err
+	}
+	return x.CloseSend()
+}
+
+type DRPCAgent_DeleteSubAgentStream interface {
+	drpc.Stream
+	SendAndClose(*DeleteSubAgentResponse) error
+}
+
+type drpcAgent_DeleteSubAgentStream struct {
+	drpc.Stream
+}
+
+func (x *drpcAgent_DeleteSubAgentStream) SendAndClose(m *DeleteSubAgentResponse) error {
+	if err := x.MsgSend(m, drpcEncoding_File_agent_proto_agent_proto{}); err != nil {
+		return err
+	}
+	return x.CloseSend()
+}
+
+type DRPCAgent_ListSubAgentsStream interface {
+	drpc.Stream
+	SendAndClose(*ListSubAgentsResponse) error
+}
+
+type drpcAgent_ListSubAgentsStream struct {
+	drpc.Stream
+}
+
+func (x *drpcAgent_ListSubAgentsStream) SendAndClose(m *ListSubAgentsResponse) error {
+	if err := x.MsgSend(m, drpcEncoding_File_agent_proto_agent_proto{}); err != nil {
+		return err
+	}
+	return x.CloseSend()
+}

agent/proto/agent_drpc_old.go

@@ -51,7 +51,17 @@ type DRPCAgentClient24 interface {
 	ReportConnection(ctx context.Context, in *ReportConnectionRequest) (*emptypb.Empty, error)
 }
 
-// DRPCAgentClient25 is the Agent API at v2.5.
+// DRPCAgentClient25 is the Agent API at v2.5. It adds a ParentId field to the
+// agent manifest response. Compatible with Coder v2.23+
 type DRPCAgentClient25 interface {
 	DRPCAgentClient24
 }
+
+// DRPCAgentClient26 is the Agent API at v2.6. It adds the CreateSubAgent,
+// DeleteSubAgent and ListSubAgents RPCs. Compatible with Coder v2.24+
+type DRPCAgentClient26 interface {
+	DRPCAgentClient25
+	CreateSubAgent(ctx context.Context, in *CreateSubAgentRequest) (*CreateSubAgentResponse, error)
+	DeleteSubAgent(ctx context.Context, in *DeleteSubAgentRequest) (*DeleteSubAgentResponse, error)
+	ListSubAgents(ctx context.Context, in *ListSubAgentsRequest) (*ListSubAgentsResponse, error)
+}

agent/proto/compare_test.go

@@ -67,7 +67,6 @@ func TestLabelsEqual(t *testing.T) {
 			eq: false,
 		},
 	} {
-		tc := tc
 		t.Run(tc.name, func(t *testing.T) {
 			t.Parallel()
 			require.Equal(t, tc.eq, proto.LabelsEqual(tc.a, tc.b))

agent/proto/resourcesmonitor/queue_test.go

@@ -65,8 +65,6 @@ func TestResourceMonitorQueue(t *testing.T) {
 	}
 
 	for _, tt := range tests {
-		tt := tt
-
 		t.Run(tt.name, func(t *testing.T) {
 			t.Parallel()
 			queue := resourcesmonitor.NewQueue(20)

agent/proto/resourcesmonitor/resources_monitor_test.go

@@ -195,7 +195,6 @@ func TestPushResourcesMonitoringWithConfig(t *testing.T) {
 	}
 
 	for _, tt := range tests {
-		tt := tt
 		t.Run(tt.name, func(t *testing.T) {
 			t.Parallel()
 

agent/reconnectingpty/server.go

@@ -31,8 +31,10 @@ type Server struct {
 	connCount        atomic.Int64
 	reconnectingPTYs sync.Map
 	timeout          time.Duration
-
-	ExperimentalDevcontainersEnabled bool
+	// Experimental: allow connecting to running containers via Docker exec.
+	// Note that this is different from the devcontainers feature, which uses
+	// subagents.
+	ExperimentalContainers bool
 }
 
 // NewServer returns a new ReconnectingPTY server
@@ -187,7 +189,7 @@ func (s *Server) handleConn(ctx context.Context, logger slog.Logger, conn net.Co
 		}()
 
 		var ei usershell.EnvInfoer
-		if s.ExperimentalDevcontainersEnabled && msg.Container != "" {
+		if s.ExperimentalContainers && msg.Container != "" {
 			dei, err := agentcontainers.EnvInfo(ctx, s.commandCreator.Execer, msg.Container, msg.ContainerUser)
 			if err != nil {
 				return xerrors.Errorf("get container env info: %w", err)