cherry: feat: sign coder binaries with the release key using GPG (#18774) (#18867)

(cherry picked from commit dc0919da33)

Co-authored-by: Jakub Domeracki <jakub@coder.com>

.cursorrules

@@ -4,7 +4,7 @@ This project is called "Coder" - an application for managing remote development
 
 Coder provides a platform for creating, managing, and using remote development environments (also known as Cloud Development Environments or CDEs). It leverages Terraform to define and provision these environments, which are referred to as "workspaces" within the project. The system is designed to be extensible, secure, and provide developers with a seamless remote development experience.
 
-# Core Architecture
+## Core Architecture
 
 The heart of Coder is a control plane that orchestrates the creation and management of workspaces. This control plane interacts with separate Provisioner processes over gRPC to handle workspace builds. The Provisioners consume workspace definitions and use Terraform to create the actual infrastructure.
 
@@ -12,17 +12,17 @@ The CLI package serves dual purposes - it can be used to launch the control plan
 
 The database layer uses PostgreSQL with SQLC for generating type-safe database code. Database migrations are carefully managed to ensure both forward and backward compatibility through paired `.up.sql` and `.down.sql` files.
 
-# API Design
+## API Design
 
 Coder's API architecture combines REST and gRPC approaches. The REST API is defined in `coderd/coderd.go` and uses Chi for HTTP routing. This provides the primary interface for the frontend and external integrations.
 
 Internal communication with Provisioners occurs over gRPC, with service definitions maintained in `.proto` files. This separation allows for efficient binary communication with the components responsible for infrastructure management while providing a standard REST interface for human-facing applications.
 
-# Network Architecture
+## Network Architecture
 
 Coder implements a secure networking layer based on Tailscale's Wireguard implementation. The `tailnet` package provides connectivity between workspace agents and clients through DERP (Designated Encrypted Relay for Packets) servers when direct connections aren't possible. This creates a secure overlay network allowing access to workspaces regardless of network topology, firewalls, or NAT configurations.
 
-## Tailnet and DERP System
+### Tailnet and DERP System
 
 The networking system has three key components:
 
@@ -35,7 +35,7 @@ The networking system has three key components:
 
 3. **Direct Connections**: When possible, the system establishes peer-to-peer connections between clients and workspaces using STUN for NAT traversal. This requires both endpoints to send UDP traffic on ephemeral ports.
 
-## Workspace Proxies
+### Workspace Proxies
 
 Workspace proxies (in the Enterprise edition) provide regional relay points for browser-based connections, reducing latency for geo-distributed teams. Key characteristics:
 
@@ -45,9 +45,10 @@ Workspace proxies (in the Enterprise edition) provide regional relay points for
 - Managed through the `coder wsproxy` commands
 - Implemented primarily in the `enterprise/wsproxy/` package
 
-# Agent System
+## Agent System
 
 The workspace agent runs within each provisioned workspace and provides core functionality including:
+
 - SSH access to workspaces via the `agentssh` package
 - Port forwarding
 - Terminal connectivity via the `pty` package for pseudo-terminal support
@@ -57,7 +58,7 @@ The workspace agent runs within each provisioned workspace and provides core fun
 
 Agents communicate with the control plane using the tailnet system and authenticate using secure tokens.
 
-# Workspace Applications
+## Workspace Applications
 
 Workspace applications (or "apps") provide browser-based access to services running within workspaces. The system supports:
 
@@ -69,17 +70,17 @@ Workspace applications (or "apps") provide browser-based access to services runn
 
 The implementation is primarily in the `coderd/workspaceapps/` directory with components for URL generation, proxying connections, and managing application state.
 
-# Implementation Details
+## Implementation Details
 
 The project structure separates frontend and backend concerns. React components and pages are organized in the `site/src/` directory, with Jest used for testing. The backend is primarily written in Go, with a strong emphasis on error handling patterns and test coverage.
 
 Database interactions are carefully managed through migrations in `coderd/database/migrations/` and queries in `coderd/database/queries/`. All new queries require proper database authorization (dbauthz) implementation to ensure that only users with appropriate permissions can access specific resources.
 
-# Authorization System
+## Authorization System
 
 The database authorization (dbauthz) system enforces fine-grained access control across all database operations. It uses role-based access control (RBAC) to validate user permissions before executing database operations. The `dbauthz` package wraps the database store and performs authorization checks before returning data. All database operations must pass through this layer to ensure security.
 
-# Testing Framework
+## Testing Framework
 
 The codebase has a comprehensive testing approach with several key components:
 
@@ -91,7 +92,7 @@ The codebase has a comprehensive testing approach with several key components:
 
 4. **Enterprise Testing**: Enterprise features have dedicated test utilities in the `coderdenttest` package.
 
-# Open Source and Enterprise Components
+## Open Source and Enterprise Components
 
 The repository contains both open source and enterprise components:
 
@@ -100,9 +101,10 @@ The repository contains both open source and enterprise components:
 - The boundary between open source and enterprise is managed through a licensing system
 - The same core codebase supports both editions, with enterprise features conditionally enabled
 
-# Development Philosophy
+## Development Philosophy
 
 Coder emphasizes clear error handling, with specific patterns required:
+
 - Concise error messages that avoid phrases like "failed to"
 - Wrapping errors with `%w` to maintain error chains
 - Using sentinel errors with the "err" prefix (e.g., `errNotFound`)
@@ -111,7 +113,7 @@ All tests should run in parallel using `t.Parallel()` to ensure efficient testin
 
 Git contributions follow a standard format with commit messages structured as `type: <message>`, where type is one of `feat`, `fix`, or `chore`.
 
-# Development Workflow
+## Development Workflow
 
 Development can be initiated using `scripts/develop.sh` to start the application after making changes. Database schema updates should be performed through the migration system using `create_migration.sh <name>` to generate migration files, with each `.up.sql` migration paired with a corresponding `.down.sql` that properly reverts all changes.
 

.github/actions/setup-go-paths/action.yml

@@ -0,0 +1,57 @@
+name: "Setup Go Paths"
+description: Overrides Go paths like GOCACHE and GOMODCACHE to use temporary directories.
+outputs:
+  gocache:
+    description: "Value of GOCACHE"
+    value: ${{ steps.paths.outputs.gocache }}
+  gomodcache:
+    description: "Value of GOMODCACHE"
+    value: ${{ steps.paths.outputs.gomodcache }}
+  gopath:
+    description: "Value of GOPATH"
+    value: ${{ steps.paths.outputs.gopath }}
+  gotmp:
+    description: "Value of GOTMPDIR"
+    value: ${{ steps.paths.outputs.gotmp }}
+  cached-dirs:
+    description: "Go directories that should be cached between CI runs"
+    value: ${{ steps.paths.outputs.cached-dirs }}
+runs:
+  using: "composite"
+  steps:
+    - name: Override Go paths
+      id: paths
+      uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7
+      with:
+        script: |
+          const path = require('path');
+
+          // RUNNER_TEMP should be backed by a RAM disk on Windows if
+          // coder/setup-ramdisk-action was used
+          const runnerTemp = process.env.RUNNER_TEMP;
+          const gocacheDir = path.join(runnerTemp, 'go-cache');
+          const gomodcacheDir = path.join(runnerTemp, 'go-mod-cache');
+          const gopathDir = path.join(runnerTemp, 'go-path');
+          const gotmpDir = path.join(runnerTemp, 'go-tmp');
+
+          core.exportVariable('GOCACHE', gocacheDir);
+          core.exportVariable('GOMODCACHE', gomodcacheDir);
+          core.exportVariable('GOPATH', gopathDir);
+          core.exportVariable('GOTMPDIR', gotmpDir);
+
+          core.setOutput('gocache', gocacheDir);
+          core.setOutput('gomodcache', gomodcacheDir);
+          core.setOutput('gopath', gopathDir);
+          core.setOutput('gotmp', gotmpDir);
+
+          const cachedDirs = `${gocacheDir}\n${gomodcacheDir}`;
+          core.setOutput('cached-dirs', cachedDirs);
+
+    - name: Create directories
+      shell: bash
+      run: |
+        set -e
+        mkdir -p "$GOCACHE"
+        mkdir -p "$GOMODCACHE"
+        mkdir -p "$GOPATH"
+        mkdir -p "$GOTMPDIR"

.github/actions/setup-go/action.yaml

@@ -5,17 +5,28 @@ inputs:
   version:
     description: "The Go version to use."
     default: "1.24.2"
+  use-preinstalled-go:
+    description: "Whether to use preinstalled Go."
+    default: "false"
+  use-cache:
+    description: "Whether to use the cache."
+    default: "true"
 runs:
   using: "composite"
   steps:
     - name: Setup Go
       uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
       with:
-        go-version: ${{ inputs.version }}
+        go-version: ${{ inputs.use-preinstalled-go == 'false' && inputs.version || '' }}
+        cache: ${{ inputs.use-cache }}
 
     - name: Install gotestsum
       shell: bash
-      run: go install gotest.tools/gotestsum@latest
+      run: go install gotest.tools/gotestsum@0d9599e513d70e5792bb9334869f82f6e8b53d4d # main as of 2025-05-15
+
+    - name: Install mtimehash
+      shell: bash
+      run: go install github.com/slsyy/mtimehash/cmd/mtimehash@a6b5da4ed2c4a40e7b805534b004e9fde7b53ce0 # v1.0.0
 
     # It isn't necessary that we ever do this, but it helps
     # separate the "setup" from the "run" times.

.github/actions/setup-imdisk/action.yaml

@@ -1,27 +0,0 @@
-name: "Setup ImDisk"
-if: runner.os == 'Windows'
-description: |
-  Sets up the ImDisk toolkit for Windows and creates a RAM disk on drive R:.
-runs:
-  using: "composite"
-  steps:
-    - name: Download ImDisk
-      if: runner.os == 'Windows'
-      shell: bash
-      run: |
-        mkdir imdisk
-        cd imdisk
-        curl -L -o files.cab https://github.com/coder/imdisk-artifacts/raw/92a17839ebc0ee3e69be019f66b3e9b5d2de4482/files.cab
-        curl -L -o install.bat https://github.com/coder/imdisk-artifacts/raw/92a17839ebc0ee3e69be019f66b3e9b5d2de4482/install.bat
-        cd ..
-
-    - name: Install ImDisk
-      shell: cmd
-      run: |
-        cd imdisk
-        install.bat /silent
-
-    - name: Create RAM Disk
-      shell: cmd
-      run: |
-        imdisk -a -s 4096M -m R: -p "/fs:ntfs /q /y"

.github/actions/upload-datadog/action.yaml

@@ -10,6 +10,8 @@ runs:
   steps:
     - shell: bash
       run: |
+        set -e
+
         owner=${{ github.repository_owner	 }}
         echo "owner: $owner"
         if [[  $owner != "coder" ]]; then
@@ -21,8 +23,45 @@ runs:
           echo "No API key provided, skipping..."
           exit 0
         fi
-        npm install -g @datadog/datadog-ci@2.21.0
-        datadog-ci junit upload --service coder ./gotests.xml \
+
+        BINARY_VERSION="v2.48.0"
+        BINARY_HASH_WINDOWS="b7bebb8212403fddb1563bae84ce5e69a70dac11e35eb07a00c9ef7ac9ed65ea"
+        BINARY_HASH_MACOS="e87c808638fddb21a87a5c4584b68ba802965eb0a593d43959c81f67246bd9eb"
+        BINARY_HASH_LINUX="5e700c465728fff8313e77c2d5ba1ce19a736168735137e1ddc7c6346ed48208"
+
+        TMP_DIR=$(mktemp -d)
+
+        if [[ "${{ runner.os }}" == "Windows" ]]; then
+          BINARY_PATH="${TMP_DIR}/datadog-ci.exe"
+          BINARY_URL="https://github.com/DataDog/datadog-ci/releases/download/${BINARY_VERSION}/datadog-ci_win-x64"
+        elif [[ "${{ runner.os }}" == "macOS" ]]; then
+          BINARY_PATH="${TMP_DIR}/datadog-ci"
+          BINARY_URL="https://github.com/DataDog/datadog-ci/releases/download/${BINARY_VERSION}/datadog-ci_darwin-arm64"
+        elif [[ "${{ runner.os }}" == "Linux" ]]; then
+          BINARY_PATH="${TMP_DIR}/datadog-ci"
+          BINARY_URL="https://github.com/DataDog/datadog-ci/releases/download/${BINARY_VERSION}/datadog-ci_linux-x64"
+        else
+          echo "Unsupported OS: ${{ runner.os }}"
+          exit 1
+        fi
+
+        echo "Downloading DataDog CI binary version ${BINARY_VERSION} for ${{ runner.os }}..."
+        curl -sSL "$BINARY_URL" -o "$BINARY_PATH"
+
+        if [[ "${{ runner.os }}" == "Windows" ]]; then
+          echo "$BINARY_HASH_WINDOWS  $BINARY_PATH" | sha256sum --check
+        elif [[ "${{ runner.os }}" == "macOS" ]]; then
+          echo "$BINARY_HASH_MACOS  $BINARY_PATH" | shasum -a 256 --check
+        elif [[ "${{ runner.os }}" == "Linux" ]]; then
+          echo "$BINARY_HASH_LINUX  $BINARY_PATH" | sha256sum --check
+        fi
+
+        # Make binary executable (not needed for Windows)
+        if [[ "${{ runner.os }}" != "Windows" ]]; then
+          chmod +x "$BINARY_PATH"
+        fi
+
+        "$BINARY_PATH" junit upload --service coder ./gotests.xml \
           --tags os:${{runner.os}} --tags runner_name:${{runner.name}}
       env:
         DATADOG_API_KEY: ${{ inputs.api-key }}

.github/dependabot.yaml

@@ -104,3 +104,21 @@ updates:
         update-types:
           - version-update:semver-major
     open-pull-requests-limit: 15
+
+  - package-ecosystem: "terraform"
+    directories:
+      - "dogfood/*/"
+      - "examples/templates/*/"
+    schedule:
+      interval: "weekly"
+    commit-message:
+      prefix: "chore"
+    groups:
+      coder:
+        patterns:
+          - "registry.coder.com/coder/*/coder"
+    labels: []
+    ignore:
+      - dependency-name: "*"
+        update-types:
+          - version-update:semver-major

.github/workflows/ci.yaml

@@ -24,7 +24,7 @@ jobs:
       docs-only: ${{ steps.filter.outputs.docs_count == steps.filter.outputs.all_count }}
       docs: ${{ steps.filter.outputs.docs }}
       go: ${{ steps.filter.outputs.go }}
-      ts: ${{ steps.filter.outputs.ts }}
+      site: ${{ steps.filter.outputs.site }}
       k8s: ${{ steps.filter.outputs.k8s }}
       ci: ${{ steps.filter.outputs.ci }}
       db: ${{ steps.filter.outputs.db }}
@@ -92,9 +92,8 @@ jobs:
             gomod:
               - "go.mod"
               - "go.sum"
-            ts:
+            site:
               - "site/**"
-              - "Makefile"
             k8s:
               - "helm/**"
               - "scripts/Dockerfile"
@@ -188,7 +187,7 @@ jobs:
 
       # Check for any typos
       - name: Check for typos
-        uses: crate-ci/typos@b1a1ef3893ff35ade0cfa71523852a49bfd05d19 # v1.31.1
+        uses: crate-ci/typos@0f0ccba9ed1df83948f0c15026e4f5ccfce46109 # v1.32.0
         with:
           config: .github/workflows/typos.toml
 
@@ -224,7 +223,7 @@ jobs:
   gen:
     timeout-minutes: 8
     runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-8' || 'ubuntu-latest' }}
-    if: always()
+    if: ${{ !cancelled() }}
     steps:
       - name: Harden Runner
         uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
@@ -313,7 +312,7 @@ jobs:
         run: ./scripts/check_unstaged.sh
 
   test-go:
-    runs-on: ${{ matrix.os == 'ubuntu-latest' && github.repository_owner == 'coder' && 'depot-ubuntu-22.04-4' || matrix.os == 'macos-latest' && github.repository_owner == 'coder' && 'depot-macos-latest' || matrix.os == 'windows-2022' && github.repository_owner == 'coder' && 'windows-latest-16-cores' || matrix.os }}
+    runs-on: ${{ matrix.os == 'ubuntu-latest' && github.repository_owner == 'coder' && 'depot-ubuntu-22.04-4' || matrix.os == 'macos-latest' && github.repository_owner == 'coder' && 'depot-macos-latest' || matrix.os == 'windows-2022' && github.repository_owner == 'coder' && 'depot-windows-2022-16' || matrix.os }}
     needs: changes
     if: needs.changes.outputs.go == 'true' || needs.changes.outputs.ci == 'true' || github.ref == 'refs/heads/main'
     timeout-minutes: 20
@@ -326,17 +325,33 @@ jobs:
           - windows-2022
     steps:
       - name: Harden Runner
+        # Harden Runner is only supported on Ubuntu runners.
+        if: runner.os == 'Linux'
         uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
         with:
           egress-policy: audit
 
+      # Set up RAM disks to speed up the rest of the job. This action is in
+      # a separate repository to allow its use before actions/checkout.
+      - name: Setup RAM Disks
+        if: runner.os == 'Windows'
+        uses: coder/setup-ramdisk-action@e1100847ab2d7bcd9d14bcda8f2d1b0f07b36f1b
+
       - name: Checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           fetch-depth: 1
 
+      - name: Setup Go Paths
+        uses: ./.github/actions/setup-go-paths
+
       - name: Setup Go
         uses: ./.github/actions/setup-go
+        with:
+          # Runners have Go baked-in and Go will automatically
+          # download the toolchain configured in go.mod, so we don't
+          # need to reinstall it. It's faster on Windows runners.
+          use-preinstalled-go: ${{ runner.os == 'Windows' }}
 
       - name: Setup Terraform
         uses: ./.github/actions/setup-tf
@@ -368,8 +383,8 @@ jobs:
             touch ~/.bash_profile && echo "export BASH_SILENCE_DEPRECATION_WARNING=1" >> ~/.bash_profile
           fi
           export TS_DEBUG_DISCO=true
-          gotestsum --junitfile="gotests.xml" --jsonfile="gotests.json" \
-            --packages="./..." -- $PARALLEL_FLAG -short -failfast
+          gotestsum --junitfile="gotests.xml" --jsonfile="gotests.json" --rerun-fails=2 \
+            --packages="./..." -- $PARALLEL_FLAG -short
 
       - name: Upload Test Cache
         uses: ./.github/actions/test-cache/upload
@@ -384,62 +399,10 @@ jobs:
         with:
           api-key: ${{ secrets.DATADOG_API_KEY }}
 
-  # We don't run the full test-suite for Windows & MacOS, so we just run the CLI tests on every PR.
-  # We run the test suite in test-go-pg, including CLI.
-  test-cli:
-    runs-on: ${{ matrix.os == 'macos-latest' && github.repository_owner == 'coder' && 'depot-macos-latest' || matrix.os == 'windows-2022' && github.repository_owner == 'coder' && 'windows-latest-16-cores' || matrix.os }}
-    needs: changes
-    if: needs.changes.outputs.go == 'true' || needs.changes.outputs.ci == 'true' || github.ref == 'refs/heads/main'
-    strategy:
-      matrix:
-        os:
-          - macos-latest
-          - windows-2022
-    steps:
-      - name: Harden Runner
-        uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
-        with:
-          egress-policy: audit
-
-      - name: Checkout
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-        with:
-          fetch-depth: 1
-
-      - name: Setup Go
-        uses: ./.github/actions/setup-go
-
-      - name: Setup Terraform
-        uses: ./.github/actions/setup-tf
-
-      # Sets up the ImDisk toolkit for Windows and creates a RAM disk on drive R:.
-      - name: Setup ImDisk
-        if: runner.os == 'Windows'
-        uses: ./.github/actions/setup-imdisk
-
-      - name: Test CLI
-        env:
-          TS_DEBUG_DISCO: "true"
-          LC_CTYPE: "en_US.UTF-8"
-          LC_ALL: "en_US.UTF-8"
-        shell: bash
-        run: |
-          # By default Go will use the number of logical CPUs, which
-          # is a fine default.
-          PARALLEL_FLAG=""
-
-          make test-cli
-
-      - name: Upload test stats to Datadog
-        timeout-minutes: 1
-        continue-on-error: true
-        uses: ./.github/actions/upload-datadog
-        if: success() || failure()
-        with:
-          api-key: ${{ secrets.DATADOG_API_KEY }}
-
   test-go-pg:
-    runs-on: ${{ matrix.os == 'ubuntu-latest' && github.repository_owner == 'coder' && 'depot-ubuntu-22.04-4' || matrix.os }}
+    # make sure to adjust NUM_PARALLEL_PACKAGES and NUM_PARALLEL_TESTS below
+    # when changing runner sizes
+    runs-on: ${{ matrix.os == 'ubuntu-latest' && github.repository_owner == 'coder' && 'depot-ubuntu-22.04-8' || matrix.os && matrix.os == 'macos-latest' && github.repository_owner == 'coder' && 'depot-macos-latest' || matrix.os == 'windows-2022' && github.repository_owner == 'coder' && 'depot-windows-2022-16' || matrix.os }}
     needs: changes
     if: needs.changes.outputs.go == 'true' || needs.changes.outputs.ci == 'true' || github.ref == 'refs/heads/main'
     # This timeout must be greater than the timeout set by `go test` in
@@ -451,34 +414,74 @@ jobs:
       matrix:
         os:
           - ubuntu-latest
+          - macos-latest
+          - windows-2022
     steps:
       - name: Harden Runner
         uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
         with:
           egress-policy: audit
 
+      # macOS indexes all new files in the background. Our Postgres tests
+      # create and destroy thousands of databases on disk, and Spotlight
+      # tries to index all of them, seriously slowing down the tests.
+      - name: Disable Spotlight Indexing
+        if: runner.os == 'macOS'
+        run: |
+          sudo mdutil -a -i off
+          sudo mdutil -X /
+          sudo launchctl bootout system /System/Library/LaunchDaemons/com.apple.metadata.mds.plist
+
+      # Set up RAM disks to speed up the rest of the job. This action is in
+      # a separate repository to allow its use before actions/checkout.
+      - name: Setup RAM Disks
+        if: runner.os == 'Windows'
+        uses: coder/setup-ramdisk-action@e1100847ab2d7bcd9d14bcda8f2d1b0f07b36f1b
+
       - name: Checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           fetch-depth: 1
 
+      - name: Setup Go Paths
+        id: go-paths
+        uses: ./.github/actions/setup-go-paths
+
+      - name: Download Go Build Cache
+        id: download-go-build-cache
+        uses: ./.github/actions/test-cache/download
+        with:
+          key-prefix: test-go-build-${{ runner.os }}-${{ runner.arch }}
+          cache-path: ${{ steps.go-paths.outputs.cached-dirs }}
+
       - name: Setup Go
         uses: ./.github/actions/setup-go
+        with:
+          # Runners have Go baked-in and Go will automatically
+          # download the toolchain configured in go.mod, so we don't
+          # need to reinstall it. It's faster on Windows runners.
+          use-preinstalled-go: ${{ runner.os == 'Windows' }}
+          # Cache is already downloaded above
+          use-cache: false
 
       - name: Setup Terraform
         uses: ./.github/actions/setup-tf
 
-      # Sets up the ImDisk toolkit for Windows and creates a RAM disk on drive R:.
-      - name: Setup ImDisk
-        if: runner.os == 'Windows'
-        uses: ./.github/actions/setup-imdisk
-
       - name: Download Test Cache
         id: download-cache
         uses: ./.github/actions/test-cache/download
         with:
           key-prefix: test-go-pg-${{ runner.os }}-${{ runner.arch }}
 
+      - name: Normalize File and Directory Timestamps
+        shell: bash
+        # Normalize file modification timestamps so that go test can use the
+        # cache from the previous CI run. See https://github.com/golang/go/issues/58571
+        # for more details.
+        run: |
+          find . -type f ! -path ./.git/\*\* | mtimehash
+          find . -type d ! -path ./.git/\*\* -exec touch -t 200601010000 {} +
+
       - name: Test with PostgreSQL Database
         env:
           POSTGRES_VERSION: "13"
@@ -487,11 +490,81 @@ jobs:
           LC_ALL: "en_US.UTF-8"
         shell: bash
         run: |
-          # By default Go will use the number of logical CPUs, which
-          # is a fine default.
-          PARALLEL_FLAG=""
+          set -o errexit
+          set -o pipefail
 
-          make test-postgres
+          if [ "${{ runner.os }}" == "Windows" ]; then
+            # Create a temp dir on the R: ramdisk drive for Windows. The default
+            # C: drive is extremely slow: https://github.com/actions/runner-images/issues/8755
+            mkdir -p "R:/temp/embedded-pg"
+            go run scripts/embedded-pg/main.go -path "R:/temp/embedded-pg"
+          elif [ "${{ runner.os }}" == "macOS" ]; then
+            # Postgres runs faster on a ramdisk on macOS too
+            mkdir -p /tmp/tmpfs
+            sudo mount_tmpfs -o noowners -s 8g /tmp/tmpfs
+            go run scripts/embedded-pg/main.go -path /tmp/tmpfs/embedded-pg
+          elif [ "${{ runner.os }}" == "Linux" ]; then
+            make test-postgres-docker
+          fi
+
+          # if macOS, install google-chrome for scaletests
+          # As another concern, should we really have this kind of external dependency
+          # requirement on standard CI?
+          if [ "${{ matrix.os }}" == "macos-latest" ]; then
+            brew install google-chrome
+          fi
+
+          # macOS will output "The default interactive shell is now zsh"
+          # intermittently in CI...
+          if [ "${{ matrix.os }}" == "macos-latest" ]; then
+            touch ~/.bash_profile && echo "export BASH_SILENCE_DEPRECATION_WARNING=1" >> ~/.bash_profile
+          fi
+
+          if [ "${{ runner.os }}" == "Windows" ]; then
+            # Our Windows runners have 16 cores.
+            # On Windows Postgres chokes up when we have 16x16=256 tests
+            # running in parallel, and dbtestutil.NewDB starts to take more than
+            # 10s to complete sometimes causing test timeouts. With 16x8=128 tests
+            # Postgres tends not to choke.
+            NUM_PARALLEL_PACKAGES=8
+            NUM_PARALLEL_TESTS=16
+          elif [ "${{ runner.os }}" == "macOS" ]; then
+            # Our macOS runners have 8 cores. We set NUM_PARALLEL_TESTS to 16
+            # because the tests complete faster and Postgres doesn't choke. It seems
+            # that macOS's tmpfs is faster than the one on Windows.
+            NUM_PARALLEL_PACKAGES=8
+            NUM_PARALLEL_TESTS=16
+          elif [ "${{ runner.os }}" == "Linux" ]; then
+            # Our Linux runners have 8 cores.
+            NUM_PARALLEL_PACKAGES=8
+            NUM_PARALLEL_TESTS=8
+          fi
+
+          # by default, run tests with cache
+          TESTCOUNT=""
+          if [ "${{ github.ref }}" == "refs/heads/main" ]; then
+            # on main, run tests without cache
+            TESTCOUNT="-count=1"
+          fi
+
+          mkdir -p "$RUNNER_TEMP/sym"
+          source scripts/normalize_path.sh
+          # terraform gets installed in a random directory, so we need to normalize
+          # the path to the terraform binary or a bunch of cached tests will be
+          # invalidated. See scripts/normalize_path.sh for more details.
+          normalize_path_with_symlinks "$RUNNER_TEMP/sym" "$(dirname $(which terraform))"
+
+          # We rerun failing tests to counteract flakiness coming from Postgres
+          # choking on macOS and Windows sometimes.
+          DB=ci gotestsum --rerun-fails=2 --rerun-fails-max-failures=50 \
+            --format standard-quiet --packages "./..." \
+            -- -timeout=20m -v -p $NUM_PARALLEL_PACKAGES -parallel=$NUM_PARALLEL_TESTS $TESTCOUNT
+
+      - name: Upload Go Build Cache
+        uses: ./.github/actions/test-cache/upload
+        with:
+          cache-key: ${{ steps.download-go-build-cache.outputs.cache-key }}
+          cache-path: ${{ steps.go-paths.outputs.cached-dirs }}
 
       - name: Upload Test Cache
         uses: ./.github/actions/test-cache/upload
@@ -546,6 +619,7 @@ jobs:
         env:
           POSTGRES_VERSION: "16"
           TS_DEBUG_DISCO: "true"
+          TEST_RETRIES: 2
         run: |
           make test-postgres
 
@@ -596,7 +670,7 @@ jobs:
       # c.f. discussion on https://github.com/coder/coder/pull/15106
       - name: Run Tests
         run: |
-          gotestsum --junitfile="gotests.xml" -- -race -parallel 4 -p 4 ./...
+          gotestsum --junitfile="gotests.xml" --packages="./..." --rerun-fails=2 --rerun-fails-abort-on-data-race -- -race -parallel 4 -p 4
 
       - name: Upload Test Cache
         uses: ./.github/actions/test-cache/upload
@@ -648,7 +722,7 @@ jobs:
           POSTGRES_VERSION: "16"
         run: |
           make test-postgres-docker
-          DB=ci gotestsum --junitfile="gotests.xml" -- -race -parallel 4 -p 4 ./...
+          DB=ci gotestsum --junitfile="gotests.xml" --packages="./..." --rerun-fails=2 --rerun-fails-abort-on-data-race -- -race -parallel 4 -p 4
 
       - name: Upload Test Cache
         uses: ./.github/actions/test-cache/upload
@@ -699,7 +773,7 @@ jobs:
   test-js:
     runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-8' || 'ubuntu-latest' }}
     needs: changes
-    if: needs.changes.outputs.ts == 'true' || needs.changes.outputs.ci == 'true' || github.ref == 'refs/heads/main'
+    if: needs.changes.outputs.site == 'true' || needs.changes.outputs.ci == 'true' || github.ref == 'refs/heads/main'
     timeout-minutes: 20
     steps:
       - name: Harden Runner
@@ -730,7 +804,7 @@ jobs:
           #- premium: true
           #  name: test-e2e-premium
     # Skip test-e2e on forks as they don't have access to CI secrets
-    if: (needs.changes.outputs.go == 'true' || needs.changes.outputs.ts == 'true' || needs.changes.outputs.ci == 'true' || github.ref == 'refs/heads/main') && !(github.event.pull_request.head.repo.fork)
+    if: (needs.changes.outputs.go == 'true' || needs.changes.outputs.site == 'true' || needs.changes.outputs.ci == 'true' || github.ref == 'refs/heads/main') && !(github.event.pull_request.head.repo.fork)
     timeout-minutes: 20
     name: ${{ matrix.variant.name }}
     steps:
@@ -770,6 +844,7 @@ jobs:
         if: ${{ !matrix.variant.premium }}
         env:
           DEBUG: pw:api
+          CODER_E2E_TEST_RETRIES: 2
         working-directory: site
 
       # Run all of the tests with a premium license
@@ -779,6 +854,7 @@ jobs:
           DEBUG: pw:api
           CODER_E2E_LICENSE: ${{ secrets.CODER_E2E_LICENSE }}
           CODER_E2E_REQUIRE_PREMIUM_TESTS: "1"
+          CODER_E2E_TEST_RETRIES: 2
         working-directory: site
 
       - name: Upload Playwright Failed Tests
@@ -797,11 +873,13 @@ jobs:
           path: ./site/test-results/**/debug-pprof-*.txt
           retention-days: 7
 
+  # Reference guide:
+  # https://www.chromatic.com/docs/turbosnap-best-practices/#run-with-caution-when-using-the-pull_request-event
   chromatic:
     # REMARK: this is only used to build storybook and deploy it to Chromatic.
     runs-on: ubuntu-latest
     needs: changes
-    if: needs.changes.outputs.ts == 'true' || needs.changes.outputs.ci == 'true'
+    if: needs.changes.outputs.site == 'true' || needs.changes.outputs.ci == 'true'
     steps:
       - name: Harden Runner
         uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
@@ -811,9 +889,10 @@ jobs:
       - name: Checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
-          # Required by Chromatic for build-over-build history, otherwise we
-          # only get 1 commit on shallow checkout.
+          # 👇 Ensures Chromatic can read your full git history
           fetch-depth: 0
+          # 👇 Tells the checkout which commit hash to reference
+          ref: ${{ github.event.pull_request.head.ref }}
 
       - name: Setup Node
         uses: ./.github/actions/setup-node
@@ -823,7 +902,7 @@ jobs:
       # the check to pass. This is desired in PRs, but not in mainline.
       - name: Publish to Chromatic (non-mainline)
         if: github.ref != 'refs/heads/main' && github.repository_owner == 'coder'
-        uses: chromaui/action@30b6228aa809059d46219e0f556752e8672a7e26 # v11.11.0
+        uses: chromaui/action@d7afd50124cf4f337bcd943e7f45cfa85a5e4476 # v12.0.0
         env:
           NODE_OPTIONS: "--max_old_space_size=4096"
           STORYBOOK: true
@@ -838,6 +917,7 @@ jobs:
           projectToken: 695c25b6cb65
           workingDir: "./site"
           storybookBaseDir: "./site"
+          storybookConfigDir: "./site/.storybook"
           # Prevent excessive build runs on minor version changes
           skip: "@(renovate/**|dependabot/**)"
           # Run TurboSnap to trace file dependencies to related stories
@@ -854,7 +934,7 @@ jobs:
       # infinitely "in progress" in mainline unless we re-review each build.
       - name: Publish to Chromatic (mainline)
         if: github.ref == 'refs/heads/main' && github.repository_owner == 'coder'
-        uses: chromaui/action@30b6228aa809059d46219e0f556752e8672a7e26 # v11.11.0
+        uses: chromaui/action@d7afd50124cf4f337bcd943e7f45cfa85a5e4476 # v12.0.0
         env:
           NODE_OPTIONS: "--max_old_space_size=4096"
           STORYBOOK: true
@@ -867,6 +947,7 @@ jobs:
           projectToken: 695c25b6cb65
           workingDir: "./site"
           storybookBaseDir: "./site"
+          storybookConfigDir: "./site/.storybook"
           # Run TurboSnap to trace file dependencies to related stories
           # and tell chromatic to only take snapshots of relevant stories
           onlyChanged: true
@@ -1178,6 +1259,8 @@ jobs:
           # do (see above).
           CODER_SIGN_WINDOWS: "1"
           CODER_WINDOWS_RESOURCES: "1"
+          CODER_SIGN_GPG: "1"
+          CODER_GPG_RELEASE_KEY_BASE64: ${{ secrets.GPG_RELEASE_KEY_BASE64 }}
           EV_KEY: ${{ secrets.EV_KEY }}
           EV_KEYSTORE: ${{ secrets.EV_KEYSTORE }}
           EV_TSA_URL: ${{ secrets.EV_TSA_URL }}

.github/workflows/dependabot.yaml

@@ -23,7 +23,7 @@ jobs:
     steps:
       - name: Dependabot metadata
         id: metadata
-        uses: dependabot/fetch-metadata@d7267f607e9d3fb96fc2fbe83e0af444713e90b7 # v2.3.0
+        uses: dependabot/fetch-metadata@08eff52bf64351f401fb50d4972fa95b9f2c2d1b # v2.4.0
         with:
           github-token: "${{ secrets.GITHUB_TOKEN }}"
 

.github/workflows/docs-ci.yaml

@@ -28,7 +28,7 @@ jobs:
       - name: Setup Node
         uses: ./.github/actions/setup-node
 
-      - uses: tj-actions/changed-files@5426ecc3f5c2b10effaefbd374f0abdc6a571b2f # v45.0.7
+      - uses: tj-actions/changed-files@3981e4f74104e7a4c67a835e1e5dd5d9eb0f0a57 # v45.0.7
         id: changed-files
         with:
           files: |

.github/workflows/nightly-gauntlet.yaml

@@ -1,142 +0,0 @@
-# The nightly-gauntlet runs tests that are either too flaky or too slow to block
-# every PR.
-name: nightly-gauntlet
-on:
-  schedule:
-    # Every day at 4AM
-    - cron: "0 4 * * 1-5"
-  workflow_dispatch:
-
-permissions:
-  contents: read
-
-jobs:
-  test-go-pg:
-    runs-on: ${{ matrix.os == 'macos-latest' && github.repository_owner == 'coder' && 'depot-macos-latest' || matrix.os == 'windows-2022' && github.repository_owner == 'coder' && 'windows-latest-16-cores' || matrix.os }}
-    if: github.ref == 'refs/heads/main'
-    # This timeout must be greater than the timeout set by `go test` in
-    # `make test-postgres` to ensure we receive a trace of running
-    # goroutines. Setting this to the timeout +5m should work quite well
-    # even if some of the preceding steps are slow.
-    timeout-minutes: 25
-    strategy:
-      fail-fast: false
-      matrix:
-        os:
-          - macos-latest
-          - windows-2022
-    steps:
-      - name: Harden Runner
-        uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
-        with:
-          egress-policy: audit
-
-      - name: Checkout
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-        with:
-          fetch-depth: 1
-
-      - name: Setup Go
-        uses: ./.github/actions/setup-go
-
-      - name: Setup Terraform
-        uses: ./.github/actions/setup-tf
-
-      # Sets up the ImDisk toolkit for Windows and creates a RAM disk on drive R:.
-      - name: Setup ImDisk
-        if: runner.os == 'Windows'
-        uses: ./.github/actions/setup-imdisk
-
-      - name: Test with PostgreSQL Database
-        env:
-          POSTGRES_VERSION: "13"
-          TS_DEBUG_DISCO: "true"
-          LC_CTYPE: "en_US.UTF-8"
-          LC_ALL: "en_US.UTF-8"
-        shell: bash
-        run: |
-          # if macOS, install google-chrome for scaletests
-          # As another concern, should we really have this kind of external dependency
-          # requirement on standard CI?
-          if [ "${{ matrix.os }}" == "macos-latest" ]; then
-            brew install google-chrome
-          fi
-
-          # By default Go will use the number of logical CPUs, which
-          # is a fine default.
-          PARALLEL_FLAG=""
-
-          # macOS will output "The default interactive shell is now zsh"
-          # intermittently in CI...
-          if [ "${{ matrix.os }}" == "macos-latest" ]; then
-            touch ~/.bash_profile && echo "export BASH_SILENCE_DEPRECATION_WARNING=1" >> ~/.bash_profile
-          fi
-
-          if [ "${{ runner.os }}" == "Windows" ]; then
-            # Create a temp dir on the R: ramdisk drive for Windows. The default
-            # C: drive is extremely slow: https://github.com/actions/runner-images/issues/8755
-            mkdir -p "R:/temp/embedded-pg"
-            go run scripts/embedded-pg/main.go -path "R:/temp/embedded-pg"
-          else
-            go run scripts/embedded-pg/main.go
-          fi
-
-          # Reduce test parallelism, mirroring what we do for race tests.
-          # We'd been encountering issues with timing related flakes, and
-          # this seems to help.
-          DB=ci gotestsum --format standard-quiet -- -v -short -count=1 -parallel 4 -p 4 ./...
-
-      - name: Upload test stats to Datadog
-        timeout-minutes: 1
-        continue-on-error: true
-        uses: ./.github/actions/upload-datadog
-        if: success() || failure()
-        with:
-          api-key: ${{ secrets.DATADOG_API_KEY }}
-
-  notify-slack-on-failure:
-    needs:
-      - test-go-pg
-    runs-on: ubuntu-latest
-    if: failure() && github.ref == 'refs/heads/main'
-
-    steps:
-      - name: Send Slack notification
-        run: |
-          curl -X POST -H 'Content-type: application/json' \
-          --data '{
-            "blocks": [
-              {
-                "type": "header",
-                "text": {
-                  "type": "plain_text",
-                  "text": "❌ Nightly gauntlet failed",
-                  "emoji": true
-                }
-              },
-              {
-                "type": "section",
-                "fields": [
-                  {
-                    "type": "mrkdwn",
-                    "text": "*Workflow:*\n${{ github.workflow }}"
-                  },
-                  {
-                    "type": "mrkdwn",
-                    "text": "*Committer:*\n${{ github.actor }}"
-                  },
-                  {
-                    "type": "mrkdwn",
-                    "text": "*Commit:*\n${{ github.sha }}"
-                  }
-                ]
-              },
-              {
-                "type": "section",
-                "text": {
-                  "type": "mrkdwn",
-                  "text": "*View failure:* <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|Click here>"
-                }
-              }
-            ]
-          }' ${{ secrets.CI_FAILURE_SLACK_WEBHOOK }}

.github/workflows/release.yaml

@@ -323,6 +323,8 @@ jobs:
         env:
           CODER_SIGN_WINDOWS: "1"
           CODER_SIGN_DARWIN: "1"
+          CODER_SIGN_GPG: "1"
+          CODER_GPG_RELEASE_KEY_BASE64: ${{ secrets.GPG_RELEASE_KEY_BASE64 }}
           CODER_WINDOWS_RESOURCES: "1"
           AC_CERTIFICATE_FILE: /tmp/apple_cert.p12
           AC_CERTIFICATE_PASSWORD_FILE: /tmp/apple_cert_password.txt
@@ -924,55 +926,3 @@ jobs:
         continue-on-error: true
         run: |
           make sqlc-push
-
-  update-calendar:
-    name: "Update release calendar in docs"
-    runs-on: "ubuntu-latest"
-    needs: [release, publish-homebrew, publish-winget, publish-sqlc]
-    if: ${{ !inputs.dry_run }}
-    permissions:
-      contents: write
-      pull-requests: write
-    steps:
-      - name: Harden Runner
-        uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
-        with:
-          egress-policy: audit
-
-      - name: Checkout repository
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-        with:
-          fetch-depth: 0 # Needed to get all tags for version calculation
-
-      - name: Set up Git
-        run: |
-          git config user.name "Coder CI"
-          git config user.email "cdrci@coder.com"
-
-      - name: Run update script
-        run: |
-          ./scripts/update-release-calendar.sh
-          make fmt/markdown
-
-      - name: Check for changes
-        id: check_changes
-        run: |
-          if git diff --quiet docs/install/releases/index.md; then
-            echo "No changes detected in release calendar."
-            echo "changes=false" >> $GITHUB_OUTPUT
-          else
-            echo "Changes detected in release calendar."
-            echo "changes=true" >> $GITHUB_OUTPUT
-          fi
-
-      - name: Create Pull Request
-        if: steps.check_changes.outputs.changes == 'true'
-        uses: peter-evans/create-pull-request@ff45666b9427631e3450c54a1bcbee4d9ff4d7c0 # v3.0.0
-        with:
-          commit-message: "docs: update release calendar"
-          title: "docs: update release calendar"
-          body: |
-            This PR automatically updates the release calendar in the docs.
-          branch: bot/update-release-calendar
-          delete-branch: true
-          labels: docs

.github/workflows/scorecard.yml

@@ -47,6 +47,6 @@ jobs:
 
       # Upload the results to GitHub's code scanning dashboard.
       - name: "Upload to code-scanning"
-        uses: github/codeql-action/upload-sarif@28deaeda66b76a05916b6923827895f2b14ab387 # v3.28.16
+        uses: github/codeql-action/upload-sarif@ff0a06e83cb2de871e5a09832bc6a81e7276941f # v3.28.18
         with:
           sarif_file: results.sarif

.github/workflows/security.yaml

@@ -38,7 +38,7 @@ jobs:
         uses: ./.github/actions/setup-go
 
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@28deaeda66b76a05916b6923827895f2b14ab387 # v3.28.16
+        uses: github/codeql-action/init@ff0a06e83cb2de871e5a09832bc6a81e7276941f # v3.28.18
         with:
           languages: go, javascript
 
@@ -48,7 +48,7 @@ jobs:
           rm Makefile
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@28deaeda66b76a05916b6923827895f2b14ab387 # v3.28.16
+        uses: github/codeql-action/analyze@ff0a06e83cb2de871e5a09832bc6a81e7276941f # v3.28.18
 
       - name: Send Slack notification on failure
         if: ${{ failure() }}
@@ -150,7 +150,7 @@ jobs:
           severity: "CRITICAL,HIGH"
 
       - name: Upload Trivy scan results to GitHub Security tab
-        uses: github/codeql-action/upload-sarif@28deaeda66b76a05916b6923827895f2b14ab387 # v3.28.16
+        uses: github/codeql-action/upload-sarif@ff0a06e83cb2de871e5a09832bc6a81e7276941f # v3.28.18
         with:
           sarif_file: trivy-results.sarif
           category: "Trivy"

.github/workflows/weekly-docs.yaml

@@ -36,7 +36,7 @@ jobs:
           reporter: github-pr-review
           config_file: ".github/.linkspector.yml"
           fail_on_error: "true"
-          filter_mode: "nofilter"
+          filter_mode: "file"
 
       - name: Send Slack notification
         if: failure() && github.event_name == 'schedule'

.gitignore

@@ -50,6 +50,8 @@ site/stats/
 *.tfplan
 *.lock.hcl
 .terraform/
+!coderd/testdata/parameters/modules/.terraform/
+!provisioner/terraform/testdata/modules-source-caching/.terraform/
 
 **/.coderv2/*
 **/__debug_bin
@@ -82,3 +84,5 @@ result
 
 # dlv debug binaries for go tests
 __debug_bin*
+
+**/.claude/settings.local.json

CLAUDE.md

@@ -0,0 +1,104 @@
+# Coder Development Guidelines
+
+Read [cursor rules](.cursorrules).
+
+## Build/Test/Lint Commands
+
+### Main Commands
+
+- `make build` or `make build-fat` - Build all "fat" binaries (includes "server" functionality)
+- `make build-slim` - Build "slim" binaries
+- `make test` - Run Go tests
+- `make test RUN=TestFunctionName` or `go test -v ./path/to/package -run TestFunctionName` - Test single
+- `make test-postgres` - Run tests with Postgres database
+- `make test-race` - Run tests with Go race detector
+- `make test-e2e` - Run end-to-end tests
+- `make lint` - Run all linters
+- `make fmt` - Format all code
+- `make gen` - Generates mocks, database queries and other auto-generated files
+
+### Frontend Commands (site directory)
+
+- `pnpm build` - Build frontend
+- `pnpm dev` - Run development server
+- `pnpm check` - Run code checks
+- `pnpm format` - Format frontend code
+- `pnpm lint` - Lint frontend code
+- `pnpm test` - Run frontend tests
+
+## Code Style Guidelines
+
+### Go
+
+- Follow [Effective Go](https://go.dev/doc/effective_go) and [Go's Code Review Comments](https://github.com/golang/go/wiki/CodeReviewComments)
+- Use `gofumpt` for formatting
+- Create packages when used during implementation
+- Validate abstractions against implementations
+
+### Error Handling
+
+- Use descriptive error messages
+- Wrap errors with context
+- Propagate errors appropriately
+- Use proper error types
+- (`xerrors.Errorf("failed to X: %w", err)`)
+
+### Naming
+
+- Use clear, descriptive names
+- Abbreviate only when obvious
+- Follow Go and TypeScript naming conventions
+
+### Comments
+
+- Document exported functions, types, and non-obvious logic
+- Follow JSDoc format for TypeScript
+- Use godoc format for Go code
+
+## Commit Style
+
+- Follow [Conventional Commits 1.0.0](https://www.conventionalcommits.org/en/v1.0.0/)
+- Format: `type(scope): message`
+- Types: `feat`, `fix`, `docs`, `style`, `refactor`, `test`, `chore`
+- Keep message titles concise (~70 characters)
+- Use imperative, present tense in commit titles
+
+## Database queries
+
+- MUST DO! Any changes to database - adding queries, modifying queries should be done in the  `coderd\database\queries\*.sql` files. Use `make gen` to generate necessary changes after.
+- MUST DO! Queries are grouped in files relating to context - e.g. `prebuilds.sql`, `users.sql`, `provisionerjobs.sql`.
+- After making changes to any `coderd\database\queries\*.sql` files you must run `make gen` to generate respective ORM changes.
+
+## Architecture
+
+### Core Components
+
+- **coderd**: Main API service connecting workspaces, provisioners, and users
+- **provisionerd**: Execution context for infrastructure-modifying providers
+- **Agents**: Services in remote workspaces providing features like SSH and port forwarding
+- **Workspaces**: Cloud resources defined by Terraform
+
+## Sub-modules
+
+### Template System
+
+- Templates define infrastructure for workspaces using Terraform
+- Environment variables pass context between Coder and templates
+- Official modules extend development environments
+
+### RBAC System
+
+- Permissions defined at site, organization, and user levels
+- Object-Action model protects resources
+- Built-in roles: owner, member, auditor, templateAdmin
+- Permission format: `<sign>?<level>.<object>.<id>.<action>`
+
+### Database
+
+- PostgreSQL 13+ recommended for production
+- Migrations managed with `migrate`
+- Database authorization through `dbauthz` package
+
+## Frontend
+
+For building Frontend refer to [this document](docs/contributing/frontend.md)

CODEOWNERS

@@ -4,3 +4,5 @@ agent/proto/ @spikecurtis @johnstcn
 tailnet/proto/ @spikecurtis @johnstcn
 vpn/vpn.proto @spikecurtis @johnstcn
 vpn/version.go @spikecurtis @johnstcn
+provisionerd/proto/ @spikecurtis @johnstcn
+provisionersdk/proto/ @spikecurtis @johnstcn

Makefile

@@ -250,6 +250,10 @@ $(CODER_ALL_BINARIES): go.mod go.sum \
 		fi
 
 		cp "$@" "./site/out/bin/coder-$$os-$$arch$$dot_ext"
+
+		if [[ "$${CODER_SIGN_GPG:-0}" == "1" ]]; then
+			cp "$@.asc" "./site/out/bin/coder-$$os-$$arch$$dot_ext.asc"
+		fi
 	fi
 
 # This task builds Coder Desktop dylibs
@@ -875,12 +879,19 @@ provisioner/terraform/testdata/version:
 	fi
 .PHONY: provisioner/terraform/testdata/version
 
+# Set the retry flags if TEST_RETRIES is set
+ifdef TEST_RETRIES
+GOTESTSUM_RETRY_FLAGS := --rerun-fails=$(TEST_RETRIES)
+else
+GOTESTSUM_RETRY_FLAGS :=
+endif
+
 test:
-	$(GIT_FLAGS) gotestsum --format standard-quiet -- -v -short -count=1 ./... $(if $(RUN),-run $(RUN))
+	$(GIT_FLAGS) gotestsum --format standard-quiet $(GOTESTSUM_RETRY_FLAGS) --packages="./..." -- -v -short -count=1 $(if $(RUN),-run $(RUN))
 .PHONY: test
 
 test-cli:
-	$(GIT_FLAGS) gotestsum --format standard-quiet -- -v -short -count=1 ./cli/...
+	$(GIT_FLAGS) gotestsum --format standard-quiet $(GOTESTSUM_RETRY_FLAGS) --packages="./cli/..." -- -v -short -count=1
 .PHONY: test-cli
 
 # sqlc-cloud-is-setup will fail if no SQLc auth token is set. Use this as a
@@ -919,9 +930,9 @@ test-postgres: test-postgres-docker
 	$(GIT_FLAGS)  DB=ci gotestsum \
 		--junitfile="gotests.xml" \
 		--jsonfile="gotests.json" \
+		$(GOTESTSUM_RETRY_FLAGS) \
 		--packages="./..." -- \
 		-timeout=20m \
-		-failfast \
 		-count=1
 .PHONY: test-postgres
 

agent/agent.go

@@ -95,8 +95,8 @@ type Options struct {
 }
 
 type Client interface {
-	ConnectRPC24(ctx context.Context) (
-		proto.DRPCAgentClient24, tailnetproto.DRPCTailnetClient24, error,
+	ConnectRPC25(ctx context.Context) (
+		proto.DRPCAgentClient25, tailnetproto.DRPCTailnetClient25, error,
 	)
 	RewriteDERPMap(derpMap *tailcfg.DERPMap)
 }
@@ -363,9 +363,11 @@ func (a *agent) runLoop() {
 		if ctx.Err() != nil {
 			// Context canceled errors may come from websocket pings, so we
 			// don't want to use `errors.Is(err, context.Canceled)` here.
+			a.logger.Warn(ctx, "runLoop exited with error", slog.Error(ctx.Err()))
 			return
 		}
 		if a.isClosed() {
+			a.logger.Warn(ctx, "runLoop exited because agent is closed")
 			return
 		}
 		if errors.Is(err, io.EOF) {
@@ -906,7 +908,7 @@ func (a *agent) run() (retErr error) {
 	a.sessionToken.Store(&sessionToken)
 
 	// ConnectRPC returns the dRPC connection we use for the Agent and Tailnet v2+ APIs
-	aAPI, tAPI, err := a.client.ConnectRPC24(a.hardCtx)
+	aAPI, tAPI, err := a.client.ConnectRPC25(a.hardCtx)
 	if err != nil {
 		return err
 	}
@@ -1046,7 +1048,11 @@ func (a *agent) run() (retErr error) {
 		return a.statsReporter.reportLoop(ctx, aAPI)
 	})
 
-	return connMan.wait()
+	err = connMan.wait()
+	if err != nil {
+		a.logger.Info(context.Background(), "connection manager errored", slog.Error(err))
+	}
+	return err
 }
 
 // handleManifest returns a function that fetches and processes the manifest
@@ -1085,6 +1091,8 @@ func (a *agent) handleManifest(manifestOK *checkpoint) func(ctx context.Context,
 		if err != nil {
 			return xerrors.Errorf("expand directory: %w", err)
 		}
+		// Normalize all devcontainer paths by making them absolute.
+		manifest.Devcontainers = agentcontainers.ExpandAllDevcontainerPaths(a.logger, expandPathToAbs, manifest.Devcontainers)
 		subsys, err := agentsdk.ProtoFromSubsystems(a.subsystems)
 		if err != nil {
 			a.logger.Critical(ctx, "failed to convert subsystems", slog.Error(err))
@@ -1127,7 +1135,7 @@ func (a *agent) handleManifest(manifestOK *checkpoint) func(ctx context.Context,
 			)
 			if a.experimentalDevcontainersEnabled {
 				var dcScripts []codersdk.WorkspaceAgentScript
-				scripts, dcScripts = agentcontainers.ExtractAndInitializeDevcontainerScripts(a.logger, expandPathToAbs, manifest.Devcontainers, scripts)
+				scripts, dcScripts = agentcontainers.ExtractAndInitializeDevcontainerScripts(manifest.Devcontainers, scripts)
 				// See ExtractAndInitializeDevcontainerScripts for motivation
 				// behind running dcScripts as post start scripts.
 				scriptRunnerOpts = append(scriptRunnerOpts, agentscripts.WithPostStartScripts(dcScripts...))
@@ -1168,12 +1176,6 @@ func (a *agent) handleManifest(manifestOK *checkpoint) func(ctx context.Context,
 				}
 				a.metrics.startupScriptSeconds.WithLabelValues(label).Set(dur)
 				a.scriptRunner.StartCron()
-				if containerAPI := a.containerAPI.Load(); containerAPI != nil {
-					// Inform the container API that the agent is ready.
-					// This allows us to start watching for changes to
-					// the devcontainer configuration files.
-					containerAPI.SignalReady()
-				}
 			})
 			if err != nil {
 				return xerrors.Errorf("track conn goroutine: %w", err)

agent/agent_test.go

@@ -1262,10 +1262,6 @@ func TestAgent_SSHConnectionLoginVars(t *testing.T) {
 			key:  "LOGNAME",
 			want: u.Username,
 		},
-		{
-			key:  "HOME",
-			want: u.HomeDir,
-		},
 		{
 			key:  "SHELL",
 			want: shell,
@@ -1502,7 +1498,7 @@ func TestAgent_Lifecycle(t *testing.T) {
 
 		_, client, _, _, _ := setupAgent(t, agentsdk.Manifest{
 			Scripts: []codersdk.WorkspaceAgentScript{{
-				Script:     "true",
+				Script:     "echo foo",
 				Timeout:    30 * time.Second,
 				RunOnStart: true,
 			}},
@@ -1935,8 +1931,6 @@ func TestAgent_ReconnectingPTYContainer(t *testing.T) {
 		t.Skip("Set CODER_TEST_USE_DOCKER=1 to run this test")
 	}
 
-	ctx := testutil.Context(t, testutil.WaitLong)
-
 	pool, err := dockertest.NewPool("")
 	require.NoError(t, err, "Could not connect to docker")
 	ct, err := pool.RunWithOptions(&dockertest.RunOptions{
@@ -1948,10 +1942,10 @@ func TestAgent_ReconnectingPTYContainer(t *testing.T) {
 		config.RestartPolicy = docker.RestartPolicy{Name: "no"}
 	})
 	require.NoError(t, err, "Could not start container")
-	t.Cleanup(func() {
+	defer func() {
 		err := pool.Purge(ct)
 		require.NoError(t, err, "Could not stop container")
-	})
+	}()
 	// Wait for container to start
 	require.Eventually(t, func() bool {
 		ct, ok := pool.ContainerByName(ct.Container.Name)
@@ -1962,6 +1956,7 @@ func TestAgent_ReconnectingPTYContainer(t *testing.T) {
 	conn, _, _, _, _ := setupAgent(t, agentsdk.Manifest{}, 0, func(_ *agenttest.Client, o *agent.Options) {
 		o.ExperimentalDevcontainersEnabled = true
 	})
+	ctx := testutil.Context(t, testutil.WaitLong)
 	ac, err := conn.ReconnectingPTY(ctx, uuid.New(), 80, 80, "/bin/sh", func(arp *workspacesdk.AgentReconnectingPTYInit) {
 		arp.Container = ct.Container.ID
 	})
@@ -1998,23 +1993,24 @@ func TestAgent_ReconnectingPTYContainer(t *testing.T) {
 // You can run it manually as follows:
 //
 // CODER_TEST_USE_DOCKER=1 go test -count=1 ./agent -run TestAgent_DevcontainerAutostart
+//
+//nolint:paralleltest // This test sets an environment variable.
 func TestAgent_DevcontainerAutostart(t *testing.T) {
-	t.Parallel()
 	if os.Getenv("CODER_TEST_USE_DOCKER") != "1" {
 		t.Skip("Set CODER_TEST_USE_DOCKER=1 to run this test")
 	}
 
-	ctx := testutil.Context(t, testutil.WaitLong)
-
-	// Connect to Docker
 	pool, err := dockertest.NewPool("")
 	require.NoError(t, err, "Could not connect to docker")
 
 	// Prepare temporary devcontainer for test (mywork).
 	devcontainerID := uuid.New()
-	tempWorkspaceFolder := t.TempDir()
-	tempWorkspaceFolder = filepath.Join(tempWorkspaceFolder, "mywork")
+	tmpdir := t.TempDir()
+	t.Setenv("HOME", tmpdir)
+	tempWorkspaceFolder := filepath.Join(tmpdir, "mywork")
+	unexpandedWorkspaceFolder := filepath.Join("~", "mywork")
 	t.Logf("Workspace folder: %s", tempWorkspaceFolder)
+	t.Logf("Unexpanded workspace folder: %s", unexpandedWorkspaceFolder)
 	devcontainerPath := filepath.Join(tempWorkspaceFolder, ".devcontainer")
 	err = os.MkdirAll(devcontainerPath, 0o755)
 	require.NoError(t, err, "create devcontainer directory")
@@ -2031,9 +2027,10 @@ func TestAgent_DevcontainerAutostart(t *testing.T) {
 		// is expected to be prepared by the provisioner normally.
 		Devcontainers: []codersdk.WorkspaceAgentDevcontainer{
 			{
-				ID:              devcontainerID,
-				Name:            "test",
-				WorkspaceFolder: tempWorkspaceFolder,
+				ID:   devcontainerID,
+				Name: "test",
+				// Use an unexpanded path to test the expansion.
+				WorkspaceFolder: unexpandedWorkspaceFolder,
 			},
 		},
 		Scripts: []codersdk.WorkspaceAgentScript{
@@ -2046,7 +2043,7 @@ func TestAgent_DevcontainerAutostart(t *testing.T) {
 			},
 		},
 	}
-	// nolint: dogsled
+	//nolint:dogsled
 	conn, _, _, _, _ := setupAgent(t, manifest, 0, func(_ *agenttest.Client, o *agent.Options) {
 		o.ExperimentalDevcontainersEnabled = true
 	})
@@ -2074,8 +2071,7 @@ func TestAgent_DevcontainerAutostart(t *testing.T) {
 
 		return false
 	}, testutil.WaitSuperLong, testutil.IntervalMedium, "no container with workspace folder label found")
-
-	t.Cleanup(func() {
+	defer func() {
 		// We can't rely on pool here because the container is not
 		// managed by it (it is managed by @devcontainer/cli).
 		err := pool.Client.RemoveContainer(docker.RemoveContainerOptions{
@@ -2084,13 +2080,15 @@ func TestAgent_DevcontainerAutostart(t *testing.T) {
 			Force:         true,
 		})
 		assert.NoError(t, err, "remove container")
-	})
+	}()
 
 	containerInfo, err := pool.Client.InspectContainer(container.ID)
 	require.NoError(t, err, "inspect container")
 	t.Logf("Container state: status: %v", containerInfo.State.Status)
 	require.True(t, containerInfo.State.Running, "container should be running")
 
+	ctx := testutil.Context(t, testutil.WaitLong)
+
 	ac, err := conn.ReconnectingPTY(ctx, uuid.New(), 80, 80, "", func(opts *workspacesdk.AgentReconnectingPTYInit) {
 		opts.Container = container.ID
 	})
@@ -2119,6 +2117,173 @@ func TestAgent_DevcontainerAutostart(t *testing.T) {
 	require.NoError(t, err, "file should exist outside devcontainer")
 }
 
+// TestAgent_DevcontainerRecreate tests that RecreateDevcontainer
+// recreates a devcontainer and emits logs.
+//
+// This tests end-to-end functionality of auto-starting a devcontainer.
+// It runs "devcontainer up" which creates a real Docker container. As
+// such, it does not run by default in CI.
+//
+// You can run it manually as follows:
+//
+// CODER_TEST_USE_DOCKER=1 go test -count=1 ./agent -run TestAgent_DevcontainerRecreate
+func TestAgent_DevcontainerRecreate(t *testing.T) {
+	if os.Getenv("CODER_TEST_USE_DOCKER") != "1" {
+		t.Skip("Set CODER_TEST_USE_DOCKER=1 to run this test")
+	}
+	t.Parallel()
+
+	pool, err := dockertest.NewPool("")
+	require.NoError(t, err, "Could not connect to docker")
+
+	// Prepare temporary devcontainer for test (mywork).
+	devcontainerID := uuid.New()
+	devcontainerLogSourceID := uuid.New()
+	workspaceFolder := filepath.Join(t.TempDir(), "mywork")
+	t.Logf("Workspace folder: %s", workspaceFolder)
+	devcontainerPath := filepath.Join(workspaceFolder, ".devcontainer")
+	err = os.MkdirAll(devcontainerPath, 0o755)
+	require.NoError(t, err, "create devcontainer directory")
+	devcontainerFile := filepath.Join(devcontainerPath, "devcontainer.json")
+	err = os.WriteFile(devcontainerFile, []byte(`{
+        "name": "mywork",
+        "image": "busybox:latest",
+        "cmd": ["sleep", "infinity"]
+    }`), 0o600)
+	require.NoError(t, err, "write devcontainer.json")
+
+	manifest := agentsdk.Manifest{
+		// Set up pre-conditions for auto-starting a devcontainer, the
+		// script is used to extract the log source ID.
+		Devcontainers: []codersdk.WorkspaceAgentDevcontainer{
+			{
+				ID:              devcontainerID,
+				Name:            "test",
+				WorkspaceFolder: workspaceFolder,
+			},
+		},
+		Scripts: []codersdk.WorkspaceAgentScript{
+			{
+				ID:          devcontainerID,
+				LogSourceID: devcontainerLogSourceID,
+			},
+		},
+	}
+
+	//nolint:dogsled
+	conn, client, _, _, _ := setupAgent(t, manifest, 0, func(_ *agenttest.Client, o *agent.Options) {
+		o.ExperimentalDevcontainersEnabled = true
+	})
+
+	ctx := testutil.Context(t, testutil.WaitLong)
+
+	// We enabled autostart for the devcontainer, so ready is a good
+	// indication that the devcontainer is up and running. Importantly,
+	// this also means that the devcontainer startup is no longer
+	// producing logs that may interfere with the recreate logs.
+	testutil.Eventually(ctx, t, func(context.Context) bool {
+		states := client.GetLifecycleStates()
+		return slices.Contains(states, codersdk.WorkspaceAgentLifecycleReady)
+	}, testutil.IntervalMedium, "devcontainer not ready")
+
+	t.Logf("Looking for container with label: devcontainer.local_folder=%s", workspaceFolder)
+
+	var container codersdk.WorkspaceAgentContainer
+	testutil.Eventually(ctx, t, func(context.Context) bool {
+		resp, err := conn.ListContainers(ctx)
+		if err != nil {
+			t.Logf("Error listing containers: %v", err)
+			return false
+		}
+		for _, c := range resp.Containers {
+			t.Logf("Found container: %s with labels: %v", c.ID[:12], c.Labels)
+			if v, ok := c.Labels["devcontainer.local_folder"]; ok && v == workspaceFolder {
+				t.Logf("Found matching container: %s", c.ID[:12])
+				container = c
+				return true
+			}
+		}
+		return false
+	}, testutil.IntervalMedium, "no container with workspace folder label found")
+	defer func(container codersdk.WorkspaceAgentContainer) {
+		// We can't rely on pool here because the container is not
+		// managed by it (it is managed by @devcontainer/cli).
+		err := pool.Client.RemoveContainer(docker.RemoveContainerOptions{
+			ID:            container.ID,
+			RemoveVolumes: true,
+			Force:         true,
+		})
+		assert.Error(t, err, "container should be removed by recreate")
+	}(container)
+
+	ctx = testutil.Context(t, testutil.WaitLong) // Reset context.
+
+	// Capture logs via ScriptLogger.
+	logsCh := make(chan *proto.BatchCreateLogsRequest, 1)
+	client.SetLogsChannel(logsCh)
+
+	// Invoke recreate to trigger the destruction and recreation of the
+	// devcontainer, we do it in a goroutine so we can process logs
+	// concurrently.
+	go func(container codersdk.WorkspaceAgentContainer) {
+		_, err := conn.RecreateDevcontainer(ctx, container.ID)
+		assert.NoError(t, err, "recreate devcontainer should succeed")
+	}(container)
+
+	t.Logf("Checking recreate logs for outcome...")
+
+	// Wait for the logs to be emitted, the @devcontainer/cli up command
+	// will emit a log with the outcome at the end suggesting we did
+	// receive all the logs.
+waitForOutcomeLoop:
+	for {
+		batch := testutil.RequireReceive(ctx, t, logsCh)
+
+		if bytes.Equal(batch.LogSourceId, devcontainerLogSourceID[:]) {
+			for _, log := range batch.Logs {
+				t.Logf("Received log: %s", log.Output)
+				if strings.Contains(log.Output, "\"outcome\"") {
+					break waitForOutcomeLoop
+				}
+			}
+		}
+	}
+
+	t.Logf("Checking there's a new container with label: devcontainer.local_folder=%s", workspaceFolder)
+
+	// Make sure the container exists and isn't the same as the old one.
+	testutil.Eventually(ctx, t, func(context.Context) bool {
+		resp, err := conn.ListContainers(ctx)
+		if err != nil {
+			t.Logf("Error listing containers: %v", err)
+			return false
+		}
+		for _, c := range resp.Containers {
+			t.Logf("Found container: %s with labels: %v", c.ID[:12], c.Labels)
+			if v, ok := c.Labels["devcontainer.local_folder"]; ok && v == workspaceFolder {
+				if c.ID == container.ID {
+					t.Logf("Found same container: %s", c.ID[:12])
+					return false
+				}
+				t.Logf("Found new container: %s", c.ID[:12])
+				container = c
+				return true
+			}
+		}
+		return false
+	}, testutil.IntervalMedium, "new devcontainer not found")
+	defer func(container codersdk.WorkspaceAgentContainer) {
+		// We can't rely on pool here because the container is not
+		// managed by it (it is managed by @devcontainer/cli).
+		err := pool.Client.RemoveContainer(docker.RemoveContainerOptions{
+			ID:            container.ID,
+			RemoveVolumes: true,
+			Force:         true,
+		})
+		assert.NoError(t, err, "remove container")
+	}(container)
+}
+
 func TestAgent_Dial(t *testing.T) {
 	t.Parallel()
 

agent/agentcontainers/acmock/acmock.go

@@ -1,9 +1,9 @@
 // Code generated by MockGen. DO NOT EDIT.
-// Source: .. (interfaces: Lister)
+// Source: .. (interfaces: Lister,DevcontainerCLI)
 //
 // Generated by this command:
 //
-//	mockgen -destination ./acmock.go -package acmock .. Lister
+//	mockgen -destination ./acmock.go -package acmock .. Lister,DevcontainerCLI
 //
 
 // Package acmock is a generated GoMock package.
@@ -13,6 +13,7 @@ import (
 	context "context"
 	reflect "reflect"
 
+	agentcontainers "github.com/coder/coder/v2/agent/agentcontainers"
 	codersdk "github.com/coder/coder/v2/codersdk"
 	gomock "go.uber.org/mock/gomock"
 )
@@ -55,3 +56,47 @@ func (mr *MockListerMockRecorder) List(ctx any) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "List", reflect.TypeOf((*MockLister)(nil).List), ctx)
 }
+
+// MockDevcontainerCLI is a mock of DevcontainerCLI interface.
+type MockDevcontainerCLI struct {
+	ctrl     *gomock.Controller
+	recorder *MockDevcontainerCLIMockRecorder
+	isgomock struct{}
+}
+
+// MockDevcontainerCLIMockRecorder is the mock recorder for MockDevcontainerCLI.
+type MockDevcontainerCLIMockRecorder struct {
+	mock *MockDevcontainerCLI
+}
+
+// NewMockDevcontainerCLI creates a new mock instance.
+func NewMockDevcontainerCLI(ctrl *gomock.Controller) *MockDevcontainerCLI {
+	mock := &MockDevcontainerCLI{ctrl: ctrl}
+	mock.recorder = &MockDevcontainerCLIMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use.
+func (m *MockDevcontainerCLI) EXPECT() *MockDevcontainerCLIMockRecorder {
+	return m.recorder
+}
+
+// Up mocks base method.
+func (m *MockDevcontainerCLI) Up(ctx context.Context, workspaceFolder, configPath string, opts ...agentcontainers.DevcontainerCLIUpOptions) (string, error) {
+	m.ctrl.T.Helper()
+	varargs := []any{ctx, workspaceFolder, configPath}
+	for _, a := range opts {
+		varargs = append(varargs, a)
+	}
+	ret := m.ctrl.Call(m, "Up", varargs...)
+	ret0, _ := ret[0].(string)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// Up indicates an expected call of Up.
+func (mr *MockDevcontainerCLIMockRecorder) Up(ctx, workspaceFolder, configPath any, opts ...any) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	varargs := append([]any{ctx, workspaceFolder, configPath}, opts...)
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Up", reflect.TypeOf((*MockDevcontainerCLI)(nil).Up), varargs...)
+}

agent/agentcontainers/acmock/doc.go

@@ -1,4 +1,4 @@
 // Package acmock contains a mock implementation of agentcontainers.Lister for use in tests.
 package acmock
 
-//go:generate mockgen -destination ./acmock.go -package acmock .. Lister
+//go:generate mockgen -destination ./acmock.go -package acmock .. Lister,DevcontainerCLI

agent/agentcontainers/api.go

@@ -8,6 +8,7 @@ import (
 	"path"
 	"slices"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/fsnotify/fsnotify"
@@ -20,38 +21,45 @@ import (
 	"github.com/coder/coder/v2/agent/agentexec"
 	"github.com/coder/coder/v2/coderd/httpapi"
 	"github.com/coder/coder/v2/codersdk"
+	"github.com/coder/coder/v2/codersdk/agentsdk"
 	"github.com/coder/quartz"
 )
 
 const (
-	defaultGetContainersCacheDuration = 10 * time.Second
-	dockerCreatedAtTimeFormat         = "2006-01-02 15:04:05 -0700 MST"
-	getContainersTimeout              = 5 * time.Second
+	defaultUpdateInterval = 10 * time.Second
+	listContainersTimeout = 15 * time.Second
 )
 
 // API is responsible for container-related operations in the agent.
 // It provides methods to list and manage containers.
 type API struct {
-	ctx     context.Context
-	cancel  context.CancelFunc
-	done    chan struct{}
-	logger  slog.Logger
-	watcher watcher.Watcher
+	ctx               context.Context
+	cancel            context.CancelFunc
+	watcherDone       chan struct{}
+	updaterDone       chan struct{}
+	initialUpdateDone chan struct{}   // Closed after first update in updaterLoop.
+	updateTrigger     chan chan error // Channel to trigger manual refresh.
+	updateInterval    time.Duration   // Interval for periodic container updates.
+	logger            slog.Logger
+	watcher           watcher.Watcher
+	execer            agentexec.Execer
+	cl                Lister
+	dccli             DevcontainerCLI
+	clock             quartz.Clock
+	scriptLogger      func(logSourceID uuid.UUID) ScriptLogger
 
-	cacheDuration time.Duration
-	execer        agentexec.Execer
-	cl            Lister
-	dccli         DevcontainerCLI
-	clock         quartz.Clock
+	mu                      sync.RWMutex
+	closed                  bool
+	containers              codersdk.WorkspaceAgentListContainersResponse  // Output from the last list operation.
+	containersErr           error                                          // Error from the last list operation.
+	devcontainerNames       map[string]bool                                // By devcontainer name.
+	knownDevcontainers      map[string]codersdk.WorkspaceAgentDevcontainer // By workspace folder.
+	configFileModifiedTimes map[string]time.Time                           // By config file path.
+	recreateSuccessTimes    map[string]time.Time                           // By workspace folder.
+	recreateErrorTimes      map[string]time.Time                           // By workspace folder.
+	recreateWg              sync.WaitGroup
 
-	// lockCh protects the below fields. We use a channel instead of a
-	// mutex so we can handle cancellation properly.
-	lockCh                  chan struct{}
-	containers              codersdk.WorkspaceAgentListContainersResponse
-	mtime                   time.Time
-	devcontainerNames       map[string]struct{}                   // Track devcontainer names to avoid duplicates.
-	knownDevcontainers      []codersdk.WorkspaceAgentDevcontainer // Track predefined and runtime-detected devcontainers.
-	configFileModifiedTimes map[string]time.Time                  // Track when config files were last modified.
+	devcontainerLogSourceIDs map[string]uuid.UUID // By workspace folder.
 }
 
 // Option is a functional option for API.
@@ -91,13 +99,32 @@ func WithDevcontainerCLI(dccli DevcontainerCLI) Option {
 // WithDevcontainers sets the known devcontainers for the API. This
 // allows the API to be aware of devcontainers defined in the workspace
 // agent manifest.
-func WithDevcontainers(devcontainers []codersdk.WorkspaceAgentDevcontainer) Option {
+func WithDevcontainers(devcontainers []codersdk.WorkspaceAgentDevcontainer, scripts []codersdk.WorkspaceAgentScript) Option {
 	return func(api *API) {
-		if len(devcontainers) > 0 {
-			api.knownDevcontainers = slices.Clone(devcontainers)
-			api.devcontainerNames = make(map[string]struct{}, len(devcontainers))
-			for _, devcontainer := range devcontainers {
-				api.devcontainerNames[devcontainer.Name] = struct{}{}
+		if len(devcontainers) == 0 {
+			return
+		}
+		api.knownDevcontainers = make(map[string]codersdk.WorkspaceAgentDevcontainer, len(devcontainers))
+		api.devcontainerNames = make(map[string]bool, len(devcontainers))
+		api.devcontainerLogSourceIDs = make(map[string]uuid.UUID)
+		for _, dc := range devcontainers {
+			api.knownDevcontainers[dc.WorkspaceFolder] = dc
+			api.devcontainerNames[dc.Name] = true
+			for _, script := range scripts {
+				// The devcontainer scripts match the devcontainer ID for
+				// identification.
+				if script.ID == dc.ID {
+					api.devcontainerLogSourceIDs[dc.WorkspaceFolder] = script.LogSourceID
+					break
+				}
+			}
+			if api.devcontainerLogSourceIDs[dc.WorkspaceFolder] == uuid.Nil {
+				api.logger.Error(api.ctx, "devcontainer log source ID not found for devcontainer",
+					slog.F("devcontainer_id", dc.ID),
+					slog.F("devcontainer_name", dc.Name),
+					slog.F("workspace_folder", dc.WorkspaceFolder),
+					slog.F("config_path", dc.ConfigPath),
+				)
 			}
 		}
 	}
@@ -112,22 +139,50 @@ func WithWatcher(w watcher.Watcher) Option {
 	}
 }
 
+// ScriptLogger is an interface for sending devcontainer logs to the
+// controlplane.
+type ScriptLogger interface {
+	Send(ctx context.Context, log ...agentsdk.Log) error
+	Flush(ctx context.Context) error
+}
+
+// noopScriptLogger is a no-op implementation of the ScriptLogger
+// interface.
+type noopScriptLogger struct{}
+
+func (noopScriptLogger) Send(context.Context, ...agentsdk.Log) error { return nil }
+func (noopScriptLogger) Flush(context.Context) error                 { return nil }
+
+// WithScriptLogger sets the script logger provider for devcontainer operations.
+func WithScriptLogger(scriptLogger func(logSourceID uuid.UUID) ScriptLogger) Option {
+	return func(api *API) {
+		api.scriptLogger = scriptLogger
+	}
+}
+
 // NewAPI returns a new API with the given options applied.
 func NewAPI(logger slog.Logger, options ...Option) *API {
 	ctx, cancel := context.WithCancel(context.Background())
 	api := &API{
 		ctx:                     ctx,
 		cancel:                  cancel,
-		done:                    make(chan struct{}),
+		watcherDone:             make(chan struct{}),
+		updaterDone:             make(chan struct{}),
+		initialUpdateDone:       make(chan struct{}),
+		updateTrigger:           make(chan chan error),
+		updateInterval:          defaultUpdateInterval,
 		logger:                  logger,
 		clock:                   quartz.NewReal(),
 		execer:                  agentexec.DefaultExecer,
-		cacheDuration:           defaultGetContainersCacheDuration,
-		lockCh:                  make(chan struct{}, 1),
-		devcontainerNames:       make(map[string]struct{}),
-		knownDevcontainers:      []codersdk.WorkspaceAgentDevcontainer{},
+		devcontainerNames:       make(map[string]bool),
+		knownDevcontainers:      make(map[string]codersdk.WorkspaceAgentDevcontainer),
 		configFileModifiedTimes: make(map[string]time.Time),
+		recreateSuccessTimes:    make(map[string]time.Time),
+		recreateErrorTimes:      make(map[string]time.Time),
+		scriptLogger:            func(uuid.UUID) ScriptLogger { return noopScriptLogger{} },
 	}
+	// The ctx and logger must be set before applying options to avoid
+	// nil pointer dereference.
 	for _, opt := range options {
 		opt(api)
 	}
@@ -146,33 +201,16 @@ func NewAPI(logger slog.Logger, options ...Option) *API {
 		}
 	}
 
-	go api.loop()
+	go api.watcherLoop()
+	go api.updaterLoop()
 
 	return api
 }
 
-// SignalReady signals the API that we are ready to begin watching for
-// file changes. This is used to prime the cache with the current list
-// of containers and to start watching the devcontainer config files for
-// changes. It should be called after the agent ready.
-func (api *API) SignalReady() {
-	// Prime the cache with the current list of containers.
-	_, _ = api.cl.List(api.ctx)
-
-	// Make sure we watch the devcontainer config files for changes.
-	for _, devcontainer := range api.knownDevcontainers {
-		if devcontainer.ConfigPath == "" {
-			continue
-		}
-
-		if err := api.watcher.Add(devcontainer.ConfigPath); err != nil {
-			api.logger.Error(api.ctx, "watch devcontainer config file failed", slog.Error(err), slog.F("file", devcontainer.ConfigPath))
-		}
-	}
-}
-
-func (api *API) loop() {
-	defer close(api.done)
+func (api *API) watcherLoop() {
+	defer close(api.watcherDone)
+	defer api.logger.Debug(api.ctx, "watcher loop stopped")
+	api.logger.Debug(api.ctx, "watcher loop started")
 
 	for {
 		event, err := api.watcher.Next(api.ctx)
@@ -192,7 +230,7 @@ func (api *API) loop() {
 			continue
 		}
 
-		now := api.clock.Now()
+		now := api.clock.Now("watcherLoop")
 		switch {
 		case event.Has(fsnotify.Create | fsnotify.Write):
 			api.logger.Debug(api.ctx, "devcontainer config file changed", slog.F("file", event.Name))
@@ -209,87 +247,165 @@ func (api *API) loop() {
 	}
 }
 
+// updaterLoop is responsible for periodically updating the container
+// list and handling manual refresh requests.
+func (api *API) updaterLoop() {
+	defer close(api.updaterDone)
+	defer api.logger.Debug(api.ctx, "updater loop stopped")
+	api.logger.Debug(api.ctx, "updater loop started")
+
+	// Perform an initial update to populate the container list, this
+	// gives us a guarantee that the API has loaded the initial state
+	// before returning any responses. This is useful for both tests
+	// and anyone looking to interact with the API.
+	api.logger.Debug(api.ctx, "performing initial containers update")
+	if err := api.updateContainers(api.ctx); err != nil {
+		api.logger.Error(api.ctx, "initial containers update failed", slog.Error(err))
+	} else {
+		api.logger.Debug(api.ctx, "initial containers update complete")
+	}
+	// Signal that the initial update attempt (successful or not) is done.
+	// Other services can wait on this if they need the first data to be available.
+	close(api.initialUpdateDone)
+
+	// We utilize a TickerFunc here instead of a regular Ticker so that
+	// we can guarantee execution of the updateContainers method after
+	// advancing the clock.
+	ticker := api.clock.TickerFunc(api.ctx, api.updateInterval, func() error {
+		done := make(chan error, 1)
+		defer close(done)
+
+		select {
+		case <-api.ctx.Done():
+			return api.ctx.Err()
+		case api.updateTrigger <- done:
+			err := <-done
+			if err != nil {
+				api.logger.Error(api.ctx, "updater loop ticker failed", slog.Error(err))
+			}
+		default:
+			api.logger.Debug(api.ctx, "updater loop ticker skipped, update in progress")
+		}
+
+		return nil // Always nil to keep the ticker going.
+	}, "updaterLoop")
+	defer func() {
+		if err := ticker.Wait("updaterLoop"); err != nil && !errors.Is(err, context.Canceled) {
+			api.logger.Error(api.ctx, "updater loop ticker failed", slog.Error(err))
+		}
+	}()
+
+	for {
+		select {
+		case <-api.ctx.Done():
+			return
+		case done := <-api.updateTrigger:
+			// Note that although we pass api.ctx here, updateContainers
+			// has an internal timeout to prevent long blocking calls.
+			done <- api.updateContainers(api.ctx)
+		}
+	}
+}
+
 // Routes returns the HTTP handler for container-related routes.
 func (api *API) Routes() http.Handler {
 	r := chi.NewRouter()
 
+	ensureInitialUpdateDoneMW := func(next http.Handler) http.Handler {
+		return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
+			select {
+			case <-api.ctx.Done():
+				httpapi.Write(r.Context(), rw, http.StatusServiceUnavailable, codersdk.Response{
+					Message: "API closed",
+					Detail:  "The API is closed and cannot process requests.",
+				})
+				return
+			case <-r.Context().Done():
+				return
+			case <-api.initialUpdateDone:
+				// Initial update is done, we can start processing
+				// requests.
+			}
+			next.ServeHTTP(rw, r)
+		})
+	}
+
+	// For now, all endpoints require the initial update to be done.
+	// If we want to allow some endpoints to be available before
+	// the initial update, we can enable this per-route.
+	r.Use(ensureInitialUpdateDoneMW)
+
 	r.Get("/", api.handleList)
-	r.Get("/devcontainers", api.handleListDevcontainers)
-	r.Post("/{id}/recreate", api.handleRecreate)
+	r.Route("/devcontainers", func(r chi.Router) {
+		r.Get("/", api.handleDevcontainersList)
+		r.Post("/container/{container}/recreate", api.handleDevcontainerRecreate)
+	})
 
 	return r
 }
 
 // handleList handles the HTTP request to list containers.
 func (api *API) handleList(rw http.ResponseWriter, r *http.Request) {
-	select {
-	case <-r.Context().Done():
-		// Client went away.
+	ct, err := api.getContainers()
+	if err != nil {
+		httpapi.Write(r.Context(), rw, http.StatusInternalServerError, codersdk.Response{
+			Message: "Could not get containers",
+			Detail:  err.Error(),
+		})
 		return
-	default:
-		ct, err := api.getContainers(r.Context())
-		if err != nil {
-			if errors.Is(err, context.Canceled) {
-				httpapi.Write(r.Context(), rw, http.StatusRequestTimeout, codersdk.Response{
-					Message: "Could not get containers.",
-					Detail:  "Took too long to list containers.",
-				})
-				return
-			}
-			httpapi.Write(r.Context(), rw, http.StatusInternalServerError, codersdk.Response{
-				Message: "Could not get containers.",
-				Detail:  err.Error(),
-			})
-			return
+	}
+	httpapi.Write(r.Context(), rw, http.StatusOK, ct)
+}
+
+// updateContainers fetches the latest container list, processes it, and
+// updates the cache. It performs locking for updating shared API state.
+func (api *API) updateContainers(ctx context.Context) error {
+	listCtx, listCancel := context.WithTimeout(ctx, listContainersTimeout)
+	defer listCancel()
+
+	updated, err := api.cl.List(listCtx)
+	if err != nil {
+		// If the context was canceled, we hold off on clearing the
+		// containers cache. This is to avoid clearing the cache if
+		// the update was canceled due to a timeout. Hopefully this
+		// will clear up on the next update.
+		if !errors.Is(err, context.Canceled) {
+			api.mu.Lock()
+			api.containers = codersdk.WorkspaceAgentListContainersResponse{}
+			api.containersErr = err
+			api.mu.Unlock()
 		}
 
-		httpapi.Write(r.Context(), rw, http.StatusOK, ct)
+		return xerrors.Errorf("list containers failed: %w", err)
 	}
+
+	api.mu.Lock()
+	defer api.mu.Unlock()
+
+	api.processUpdatedContainersLocked(ctx, updated)
+
+	api.logger.Debug(ctx, "containers updated successfully", slog.F("container_count", len(api.containers.Containers)), slog.F("warning_count", len(api.containers.Warnings)), slog.F("devcontainer_count", len(api.knownDevcontainers)))
+
+	return nil
 }
 
-func copyListContainersResponse(resp codersdk.WorkspaceAgentListContainersResponse) codersdk.WorkspaceAgentListContainersResponse {
-	return codersdk.WorkspaceAgentListContainersResponse{
-		Containers: slices.Clone(resp.Containers),
-		Warnings:   slices.Clone(resp.Warnings),
-	}
-}
-
-func (api *API) getContainers(ctx context.Context) (codersdk.WorkspaceAgentListContainersResponse, error) {
-	select {
-	case <-api.ctx.Done():
-		return codersdk.WorkspaceAgentListContainersResponse{}, api.ctx.Err()
-	case <-ctx.Done():
-		return codersdk.WorkspaceAgentListContainersResponse{}, ctx.Err()
-	case api.lockCh <- struct{}{}:
-		defer func() { <-api.lockCh }()
-	}
-
-	now := api.clock.Now()
-	if now.Sub(api.mtime) < api.cacheDuration {
-		return copyListContainersResponse(api.containers), nil
-	}
-
-	timeoutCtx, timeoutCancel := context.WithTimeout(ctx, getContainersTimeout)
-	defer timeoutCancel()
-	updated, err := api.cl.List(timeoutCtx)
-	if err != nil {
-		return codersdk.WorkspaceAgentListContainersResponse{}, xerrors.Errorf("get containers: %w", err)
-	}
-	api.containers = updated
-	api.mtime = now
-
-	dirtyStates := make(map[string]bool)
-	// Reset all known devcontainers to not running.
-	for i := range api.knownDevcontainers {
-		api.knownDevcontainers[i].Running = false
-		api.knownDevcontainers[i].Container = nil
-
-		// Preserve the dirty state and store in map for lookup.
-		dirtyStates[api.knownDevcontainers[i].WorkspaceFolder] = api.knownDevcontainers[i].Dirty
+// processUpdatedContainersLocked updates the devcontainer state based
+// on the latest list of containers. This method assumes that api.mu is
+// held.
+func (api *API) processUpdatedContainersLocked(ctx context.Context, updated codersdk.WorkspaceAgentListContainersResponse) {
+	// Reset the container links in known devcontainers to detect if
+	// they still exist.
+	for _, dc := range api.knownDevcontainers {
+		dc.Container = nil
+		api.knownDevcontainers[dc.WorkspaceFolder] = dc
 	}
 
 	// Check if the container is running and update the known devcontainers.
-	for _, container := range updated.Containers {
+	for i := range updated.Containers {
+		container := &updated.Containers[i] // Grab a reference to the container to allow mutating it.
+		container.DevcontainerStatus = ""   // Reset the status for the container (updated later).
+		container.DevcontainerDirty = false // Reset dirty state for the container (updated later).
+
 		workspaceFolder := container.Labels[DevcontainerLocalFolderLabel]
 		configFile := container.Labels[DevcontainerConfigFileLabel]
 
@@ -297,91 +413,145 @@ func (api *API) getContainers(ctx context.Context) (codersdk.WorkspaceAgentListC
 			continue
 		}
 
-		// Check if this is already in our known list.
-		if knownIndex := slices.IndexFunc(api.knownDevcontainers, func(dc codersdk.WorkspaceAgentDevcontainer) bool {
-			return dc.WorkspaceFolder == workspaceFolder
-		}); knownIndex != -1 {
-			// Update existing entry with runtime information.
-			if configFile != "" && api.knownDevcontainers[knownIndex].ConfigPath == "" {
-				api.knownDevcontainers[knownIndex].ConfigPath = configFile
+		if dc, ok := api.knownDevcontainers[workspaceFolder]; ok {
+			// If no config path is set, this devcontainer was defined
+			// in Terraform without the optional config file. Assume the
+			// first container with the workspace folder label is the
+			// one we want to use.
+			if dc.ConfigPath == "" && configFile != "" {
+				dc.ConfigPath = configFile
 				if err := api.watcher.Add(configFile); err != nil {
 					api.logger.Error(ctx, "watch devcontainer config file failed", slog.Error(err), slog.F("file", configFile))
 				}
 			}
-			api.knownDevcontainers[knownIndex].Running = container.Running
-			api.knownDevcontainers[knownIndex].Container = &container
 
-			// Check if this container was created after the config
-			// file was modified.
-			if configFile != "" && api.knownDevcontainers[knownIndex].Dirty {
-				lastModified, hasModTime := api.configFileModifiedTimes[configFile]
-				if hasModTime && container.CreatedAt.After(lastModified) {
-					api.logger.Info(ctx, "clearing dirty flag for container created after config modification",
-						slog.F("container", container.ID),
-						slog.F("created_at", container.CreatedAt),
-						slog.F("config_modified_at", lastModified),
-						slog.F("file", configFile),
-					)
-					api.knownDevcontainers[knownIndex].Dirty = false
-				}
-			}
+			dc.Container = container
+			api.knownDevcontainers[dc.WorkspaceFolder] = dc
 			continue
 		}
 
 		// NOTE(mafredri): This name impl. may change to accommodate devcontainer agents RFC.
 		// If not in our known list, add as a runtime detected entry.
 		name := path.Base(workspaceFolder)
-		if _, ok := api.devcontainerNames[name]; ok {
+		if api.devcontainerNames[name] {
 			// Try to find a unique name by appending a number.
 			for i := 2; ; i++ {
 				newName := fmt.Sprintf("%s-%d", name, i)
-				if _, ok := api.devcontainerNames[newName]; !ok {
+				if !api.devcontainerNames[newName] {
 					name = newName
 					break
 				}
 			}
 		}
-		api.devcontainerNames[name] = struct{}{}
+		api.devcontainerNames[name] = true
 		if configFile != "" {
 			if err := api.watcher.Add(configFile); err != nil {
 				api.logger.Error(ctx, "watch devcontainer config file failed", slog.Error(err), slog.F("file", configFile))
 			}
 		}
 
-		dirty := dirtyStates[workspaceFolder]
-		if dirty {
-			lastModified, hasModTime := api.configFileModifiedTimes[configFile]
-			if hasModTime && container.CreatedAt.After(lastModified) {
-				api.logger.Info(ctx, "new container created after config modification, not marking as dirty",
-					slog.F("container", container.ID),
-					slog.F("created_at", container.CreatedAt),
-					slog.F("config_modified_at", lastModified),
-					slog.F("file", configFile),
-				)
-				dirty = false
-			}
-		}
-
-		api.knownDevcontainers = append(api.knownDevcontainers, codersdk.WorkspaceAgentDevcontainer{
+		api.knownDevcontainers[workspaceFolder] = codersdk.WorkspaceAgentDevcontainer{
 			ID:              uuid.New(),
 			Name:            name,
 			WorkspaceFolder: workspaceFolder,
 			ConfigPath:      configFile,
-			Running:         container.Running,
-			Dirty:           dirty,
-			Container:       &container,
-		})
+			Status:          "",    // Updated later based on container state.
+			Dirty:           false, // Updated later based on config file changes.
+			Container:       container,
+		}
 	}
 
-	return copyListContainersResponse(api.containers), nil
+	// Iterate through all known devcontainers and update their status
+	// based on the current state of the containers.
+	for _, dc := range api.knownDevcontainers {
+		switch {
+		case dc.Status == codersdk.WorkspaceAgentDevcontainerStatusStarting:
+			if dc.Container != nil {
+				dc.Container.DevcontainerStatus = dc.Status
+				dc.Container.DevcontainerDirty = dc.Dirty
+			}
+			continue // This state is handled by the recreation routine.
+
+		case dc.Status == codersdk.WorkspaceAgentDevcontainerStatusError && (dc.Container == nil || dc.Container.CreatedAt.Before(api.recreateErrorTimes[dc.WorkspaceFolder])):
+			if dc.Container != nil {
+				dc.Container.DevcontainerStatus = dc.Status
+				dc.Container.DevcontainerDirty = dc.Dirty
+			}
+			continue // The devcontainer needs to be recreated.
+
+		case dc.Container != nil:
+			dc.Status = codersdk.WorkspaceAgentDevcontainerStatusStopped
+			if dc.Container.Running {
+				dc.Status = codersdk.WorkspaceAgentDevcontainerStatusRunning
+			}
+			dc.Container.DevcontainerStatus = dc.Status
+
+			dc.Dirty = false
+			if lastModified, hasModTime := api.configFileModifiedTimes[dc.ConfigPath]; hasModTime && dc.Container.CreatedAt.Before(lastModified) {
+				dc.Dirty = true
+			}
+			dc.Container.DevcontainerDirty = dc.Dirty
+
+		case dc.Container == nil:
+			dc.Status = codersdk.WorkspaceAgentDevcontainerStatusStopped
+			dc.Dirty = false
+		}
+
+		delete(api.recreateErrorTimes, dc.WorkspaceFolder)
+		api.knownDevcontainers[dc.WorkspaceFolder] = dc
+	}
+
+	api.containers = updated
+	api.containersErr = nil
 }
 
-// handleRecreate handles the HTTP request to recreate a container.
-func (api *API) handleRecreate(w http.ResponseWriter, r *http.Request) {
-	ctx := r.Context()
-	id := chi.URLParam(r, "id")
+// refreshContainers triggers an immediate update of the container list
+// and waits for it to complete.
+func (api *API) refreshContainers(ctx context.Context) (err error) {
+	defer func() {
+		if err != nil {
+			err = xerrors.Errorf("refresh containers failed: %w", err)
+		}
+	}()
 
-	if id == "" {
+	done := make(chan error, 1)
+	select {
+	case <-api.ctx.Done():
+		return xerrors.Errorf("API closed: %w", api.ctx.Err())
+	case <-ctx.Done():
+		return ctx.Err()
+	case api.updateTrigger <- done:
+		select {
+		case <-api.ctx.Done():
+			return xerrors.Errorf("API closed: %w", api.ctx.Err())
+		case <-ctx.Done():
+			return ctx.Err()
+		case err := <-done:
+			return err
+		}
+	}
+}
+
+func (api *API) getContainers() (codersdk.WorkspaceAgentListContainersResponse, error) {
+	api.mu.RLock()
+	defer api.mu.RUnlock()
+
+	if api.containersErr != nil {
+		return codersdk.WorkspaceAgentListContainersResponse{}, api.containersErr
+	}
+	return codersdk.WorkspaceAgentListContainersResponse{
+		Containers: slices.Clone(api.containers.Containers),
+		Warnings:   slices.Clone(api.containers.Warnings),
+	}, nil
+}
+
+// handleDevcontainerRecreate handles the HTTP request to recreate a
+// devcontainer by referencing the container.
+func (api *API) handleDevcontainerRecreate(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+	containerID := chi.URLParam(r, "container")
+
+	if containerID == "" {
 		httpapi.Write(ctx, w, http.StatusBadRequest, codersdk.Response{
 			Message: "Missing container ID or name",
 			Detail:  "Container ID or name is required to recreate a devcontainer.",
@@ -389,7 +559,7 @@ func (api *API) handleRecreate(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	containers, err := api.getContainers(ctx)
+	containers, err := api.getContainers()
 	if err != nil {
 		httpapi.Write(ctx, w, http.StatusInternalServerError, codersdk.Response{
 			Message: "Could not list containers",
@@ -398,9 +568,7 @@ func (api *API) handleRecreate(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	containerIdx := slices.IndexFunc(containers.Containers, func(c codersdk.WorkspaceAgentContainer) bool {
-		return c.Match(id)
-	})
+	containerIdx := slices.IndexFunc(containers.Containers, func(c codersdk.WorkspaceAgentContainer) bool { return c.Match(containerID) })
 	if containerIdx == -1 {
 		httpapi.Write(ctx, w, http.StatusNotFound, codersdk.Response{
 			Message: "Container not found",
@@ -418,52 +586,161 @@ func (api *API) handleRecreate(w http.ResponseWriter, r *http.Request) {
 	if workspaceFolder == "" {
 		httpapi.Write(ctx, w, http.StatusBadRequest, codersdk.Response{
 			Message: "Missing workspace folder label",
-			Detail:  "The workspace folder label is required to recreate a devcontainer.",
+			Detail:  "The container is not a devcontainer, the container must have the workspace folder label to support recreation.",
 		})
 		return
 	}
 
-	_, err = api.dccli.Up(ctx, workspaceFolder, configPath, WithRemoveExistingContainer())
-	if err != nil {
+	api.mu.Lock()
+
+	dc, ok := api.knownDevcontainers[workspaceFolder]
+	switch {
+	case !ok:
+		api.mu.Unlock()
+
+		// This case should not happen if the container is a valid devcontainer.
+		api.logger.Error(ctx, "devcontainer not found for workspace folder", slog.F("workspace_folder", workspaceFolder))
 		httpapi.Write(ctx, w, http.StatusInternalServerError, codersdk.Response{
-			Message: "Could not recreate devcontainer",
-			Detail:  err.Error(),
+			Message: "Devcontainer not found.",
+			Detail:  fmt.Sprintf("Could not find devcontainer for workspace folder: %q", workspaceFolder),
+		})
+		return
+	case dc.Status == codersdk.WorkspaceAgentDevcontainerStatusStarting:
+		api.mu.Unlock()
+
+		httpapi.Write(ctx, w, http.StatusConflict, codersdk.Response{
+			Message: "Devcontainer recreation already in progress",
+			Detail:  fmt.Sprintf("Recreation for workspace folder %q is already underway.", dc.WorkspaceFolder),
 		})
 		return
 	}
 
-	// TODO(mafredri): Temporarily handle clearing the dirty state after
-	// recreation, later on this should be handled by a "container watcher".
-	select {
-	case <-api.ctx.Done():
-		return
-	case <-ctx.Done():
-		return
-	case api.lockCh <- struct{}{}:
-		defer func() { <-api.lockCh }()
-	}
-	for i := range api.knownDevcontainers {
-		if api.knownDevcontainers[i].WorkspaceFolder == workspaceFolder {
-			if api.knownDevcontainers[i].Dirty {
-				api.logger.Info(ctx, "clearing dirty flag after recreation",
-					slog.F("workspace_folder", workspaceFolder),
-					slog.F("name", api.knownDevcontainers[i].Name),
-				)
-				api.knownDevcontainers[i].Dirty = false
-			}
-			break
-		}
+	// Update the status so that we don't try to recreate the
+	// devcontainer multiple times in parallel.
+	dc.Status = codersdk.WorkspaceAgentDevcontainerStatusStarting
+	if dc.Container != nil {
+		dc.Container.DevcontainerStatus = dc.Status
 	}
+	api.knownDevcontainers[dc.WorkspaceFolder] = dc
+	api.recreateWg.Add(1)
+	go api.recreateDevcontainer(dc, configPath)
 
-	w.WriteHeader(http.StatusNoContent)
+	api.mu.Unlock()
+
+	httpapi.Write(ctx, w, http.StatusAccepted, codersdk.Response{
+		Message: "Devcontainer recreation initiated",
+		Detail:  fmt.Sprintf("Recreation process for workspace folder %q has started.", dc.WorkspaceFolder),
+	})
 }
 
-// handleListDevcontainers handles the HTTP request to list known devcontainers.
-func (api *API) handleListDevcontainers(w http.ResponseWriter, r *http.Request) {
+// recreateDevcontainer should run in its own goroutine and is responsible for
+// recreating a devcontainer based on the provided devcontainer configuration.
+// It updates the devcontainer status and logs the process. The configPath is
+// passed as a parameter for the odd chance that the container being recreated
+// has a different config file than the one stored in the devcontainer state.
+// The devcontainer state must be set to starting and the recreateWg must be
+// incremented before calling this function.
+func (api *API) recreateDevcontainer(dc codersdk.WorkspaceAgentDevcontainer, configPath string) {
+	defer api.recreateWg.Done()
+
+	var (
+		err    error
+		ctx    = api.ctx
+		logger = api.logger.With(
+			slog.F("devcontainer_id", dc.ID),
+			slog.F("devcontainer_name", dc.Name),
+			slog.F("workspace_folder", dc.WorkspaceFolder),
+			slog.F("config_path", configPath),
+		)
+	)
+
+	if dc.ConfigPath != configPath {
+		logger.Warn(ctx, "devcontainer config path mismatch",
+			slog.F("config_path_param", configPath),
+		)
+	}
+
+	// Send logs via agent logging facilities.
+	logSourceID := api.devcontainerLogSourceIDs[dc.WorkspaceFolder]
+	if logSourceID == uuid.Nil {
+		// Fallback to the external log source ID if not found.
+		logSourceID = agentsdk.ExternalLogSourceID
+	}
+
+	scriptLogger := api.scriptLogger(logSourceID)
+	defer func() {
+		flushCtx, cancel := context.WithTimeout(api.ctx, 5*time.Second)
+		defer cancel()
+		if err := scriptLogger.Flush(flushCtx); err != nil {
+			logger.Error(flushCtx, "flush devcontainer logs failed during recreation", slog.Error(err))
+		}
+	}()
+	infoW := agentsdk.LogsWriter(ctx, scriptLogger.Send, logSourceID, codersdk.LogLevelInfo)
+	defer infoW.Close()
+	errW := agentsdk.LogsWriter(ctx, scriptLogger.Send, logSourceID, codersdk.LogLevelError)
+	defer errW.Close()
+
+	logger.Debug(ctx, "starting devcontainer recreation")
+
+	_, err = api.dccli.Up(ctx, dc.WorkspaceFolder, configPath, WithOutput(infoW, errW), WithRemoveExistingContainer())
+	if err != nil {
+		// No need to log if the API is closing (context canceled), as this
+		// is expected behavior when the API is shutting down.
+		if !errors.Is(err, context.Canceled) {
+			logger.Error(ctx, "devcontainer recreation failed", slog.Error(err))
+		}
+
+		api.mu.Lock()
+		dc = api.knownDevcontainers[dc.WorkspaceFolder]
+		dc.Status = codersdk.WorkspaceAgentDevcontainerStatusError
+		if dc.Container != nil {
+			dc.Container.DevcontainerStatus = dc.Status
+		}
+		api.knownDevcontainers[dc.WorkspaceFolder] = dc
+		api.recreateErrorTimes[dc.WorkspaceFolder] = api.clock.Now("recreate", "errorTimes")
+		api.mu.Unlock()
+		return
+	}
+
+	logger.Info(ctx, "devcontainer recreated successfully")
+
+	api.mu.Lock()
+	dc = api.knownDevcontainers[dc.WorkspaceFolder]
+	// Update the devcontainer status to Running or Stopped based on the
+	// current state of the container, changing the status to !starting
+	// allows the update routine to update the devcontainer status, but
+	// to minimize the time between API consistency, we guess the status
+	// based on the container state.
+	dc.Status = codersdk.WorkspaceAgentDevcontainerStatusStopped
+	if dc.Container != nil {
+		if dc.Container.Running {
+			dc.Status = codersdk.WorkspaceAgentDevcontainerStatusRunning
+		}
+		dc.Container.DevcontainerStatus = dc.Status
+	}
+	dc.Dirty = false
+	api.recreateSuccessTimes[dc.WorkspaceFolder] = api.clock.Now("recreate", "successTimes")
+	api.knownDevcontainers[dc.WorkspaceFolder] = dc
+	api.mu.Unlock()
+
+	// Ensure an immediate refresh to accurately reflect the
+	// devcontainer state after recreation.
+	if err := api.refreshContainers(ctx); err != nil {
+		logger.Error(ctx, "failed to trigger immediate refresh after devcontainer recreation", slog.Error(err))
+	}
+}
+
+// handleDevcontainersList handles the HTTP request to list known devcontainers.
+func (api *API) handleDevcontainersList(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	// Run getContainers to detect the latest devcontainers and their state.
-	_, err := api.getContainers(ctx)
+	api.mu.RLock()
+	err := api.containersErr
+	devcontainers := make([]codersdk.WorkspaceAgentDevcontainer, 0, len(api.knownDevcontainers))
+	for _, dc := range api.knownDevcontainers {
+		devcontainers = append(devcontainers, dc)
+	}
+	api.mu.RUnlock()
 	if err != nil {
 		httpapi.Write(ctx, w, http.StatusInternalServerError, codersdk.Response{
 			Message: "Could not list containers",
@@ -472,16 +749,6 @@ func (api *API) handleListDevcontainers(w http.ResponseWriter, r *http.Request)
 		return
 	}
 
-	select {
-	case <-api.ctx.Done():
-		return
-	case <-ctx.Done():
-		return
-	case api.lockCh <- struct{}{}:
-	}
-	devcontainers := slices.Clone(api.knownDevcontainers)
-	<-api.lockCh
-
 	slices.SortFunc(devcontainers, func(a, b codersdk.WorkspaceAgentDevcontainer) int {
 		if cmp := strings.Compare(a.WorkspaceFolder, b.WorkspaceFolder); cmp != 0 {
 			return cmp
@@ -499,42 +766,62 @@ func (api *API) handleListDevcontainers(w http.ResponseWriter, r *http.Request)
 // markDevcontainerDirty finds the devcontainer with the given config file path
 // and marks it as dirty. It acquires the lock before modifying the state.
 func (api *API) markDevcontainerDirty(configPath string, modifiedAt time.Time) {
-	select {
-	case <-api.ctx.Done():
-		return
-	case api.lockCh <- struct{}{}:
-		defer func() { <-api.lockCh }()
-	}
+	api.mu.Lock()
+	defer api.mu.Unlock()
 
 	// Record the timestamp of when this configuration file was modified.
 	api.configFileModifiedTimes[configPath] = modifiedAt
 
-	for i := range api.knownDevcontainers {
-		if api.knownDevcontainers[i].ConfigPath != configPath {
+	for _, dc := range api.knownDevcontainers {
+		if dc.ConfigPath != configPath {
 			continue
 		}
 
+		logger := api.logger.With(
+			slog.F("devcontainer_id", dc.ID),
+			slog.F("devcontainer_name", dc.Name),
+			slog.F("workspace_folder", dc.WorkspaceFolder),
+			slog.F("file", configPath),
+			slog.F("modified_at", modifiedAt),
+		)
+
 		// TODO(mafredri): Simplistic mark for now, we should check if the
 		// container is running and if the config file was modified after
 		// the container was created.
-		if !api.knownDevcontainers[i].Dirty {
-			api.logger.Info(api.ctx, "marking devcontainer as dirty",
-				slog.F("file", configPath),
-				slog.F("name", api.knownDevcontainers[i].Name),
-				slog.F("workspace_folder", api.knownDevcontainers[i].WorkspaceFolder),
-				slog.F("modified_at", modifiedAt),
-			)
-			api.knownDevcontainers[i].Dirty = true
+		if !dc.Dirty {
+			logger.Info(api.ctx, "marking devcontainer as dirty")
+			dc.Dirty = true
 		}
+		if dc.Container != nil && !dc.Container.DevcontainerDirty {
+			logger.Info(api.ctx, "marking devcontainer container as dirty")
+			dc.Container.DevcontainerDirty = true
+		}
+
+		api.knownDevcontainers[dc.WorkspaceFolder] = dc
 	}
 }
 
 func (api *API) Close() error {
-	api.cancel()
-	<-api.done
-	err := api.watcher.Close()
-	if err != nil {
-		return err
+	api.mu.Lock()
+	if api.closed {
+		api.mu.Unlock()
+		return nil
 	}
-	return nil
+	api.logger.Debug(api.ctx, "closing API")
+	api.closed = true
+	api.cancel()    // Interrupt all routines.
+	api.mu.Unlock() // Release lock before waiting for goroutines.
+
+	// Close the watcher to ensure its loop finishes.
+	err := api.watcher.Close()
+
+	// Wait for loops to finish.
+	<-api.watcherDone
+	<-api.updaterDone
+
+	// Wait for all devcontainer recreation tasks to complete.
+	api.recreateWg.Wait()
+
+	api.logger.Debug(api.ctx, "closed API")
+	return err
 }

agent/agentcontainers/api_internal_test.go

@@ -1,163 +0,0 @@
-package agentcontainers
-
-import (
-	"math/rand"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/google/uuid"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	"go.uber.org/mock/gomock"
-
-	"cdr.dev/slog"
-	"cdr.dev/slog/sloggers/slogtest"
-	"github.com/coder/coder/v2/agent/agentcontainers/acmock"
-	"github.com/coder/coder/v2/codersdk"
-	"github.com/coder/coder/v2/testutil"
-	"github.com/coder/quartz"
-)
-
-func TestAPI(t *testing.T) {
-	t.Parallel()
-
-	// List tests the API.getContainers method using a mock
-	// implementation. It specifically tests caching behavior.
-	t.Run("List", func(t *testing.T) {
-		t.Parallel()
-
-		fakeCt := fakeContainer(t)
-		fakeCt2 := fakeContainer(t)
-		makeResponse := func(cts ...codersdk.WorkspaceAgentContainer) codersdk.WorkspaceAgentListContainersResponse {
-			return codersdk.WorkspaceAgentListContainersResponse{Containers: cts}
-		}
-
-		// Each test case is called multiple times to ensure idempotency
-		for _, tc := range []struct {
-			name string
-			// data to be stored in the handler
-			cacheData codersdk.WorkspaceAgentListContainersResponse
-			// duration of cache
-			cacheDur time.Duration
-			// relative age of the cached data
-			cacheAge time.Duration
-			// function to set up expectations for the mock
-			setupMock func(*acmock.MockLister)
-			// expected result
-			expected codersdk.WorkspaceAgentListContainersResponse
-			// expected error
-			expectedErr string
-		}{
-			{
-				name: "no cache",
-				setupMock: func(mcl *acmock.MockLister) {
-					mcl.EXPECT().List(gomock.Any()).Return(makeResponse(fakeCt), nil).AnyTimes()
-				},
-				expected: makeResponse(fakeCt),
-			},
-			{
-				name:      "no data",
-				cacheData: makeResponse(),
-				cacheAge:  2 * time.Second,
-				cacheDur:  time.Second,
-				setupMock: func(mcl *acmock.MockLister) {
-					mcl.EXPECT().List(gomock.Any()).Return(makeResponse(fakeCt), nil).AnyTimes()
-				},
-				expected: makeResponse(fakeCt),
-			},
-			{
-				name:      "cached data",
-				cacheAge:  time.Second,
-				cacheData: makeResponse(fakeCt),
-				cacheDur:  2 * time.Second,
-				expected:  makeResponse(fakeCt),
-			},
-			{
-				name: "lister error",
-				setupMock: func(mcl *acmock.MockLister) {
-					mcl.EXPECT().List(gomock.Any()).Return(makeResponse(), assert.AnError).AnyTimes()
-				},
-				expectedErr: assert.AnError.Error(),
-			},
-			{
-				name:      "stale cache",
-				cacheAge:  2 * time.Second,
-				cacheData: makeResponse(fakeCt),
-				cacheDur:  time.Second,
-				setupMock: func(mcl *acmock.MockLister) {
-					mcl.EXPECT().List(gomock.Any()).Return(makeResponse(fakeCt2), nil).AnyTimes()
-				},
-				expected: makeResponse(fakeCt2),
-			},
-		} {
-			tc := tc
-			t.Run(tc.name, func(t *testing.T) {
-				t.Parallel()
-				var (
-					ctx        = testutil.Context(t, testutil.WaitShort)
-					clk        = quartz.NewMock(t)
-					ctrl       = gomock.NewController(t)
-					mockLister = acmock.NewMockLister(ctrl)
-					now        = time.Now().UTC()
-					logger     = slogtest.Make(t, nil).Leveled(slog.LevelDebug)
-					api        = NewAPI(logger, WithLister(mockLister))
-				)
-				defer api.Close()
-
-				api.cacheDuration = tc.cacheDur
-				api.clock = clk
-				api.containers = tc.cacheData
-				if tc.cacheAge != 0 {
-					api.mtime = now.Add(-tc.cacheAge)
-				}
-				if tc.setupMock != nil {
-					tc.setupMock(mockLister)
-				}
-
-				clk.Set(now).MustWait(ctx)
-
-				// Repeat the test to ensure idempotency
-				for i := 0; i < 2; i++ {
-					actual, err := api.getContainers(ctx)
-					if tc.expectedErr != "" {
-						require.Empty(t, actual, "expected no data (attempt %d)", i)
-						require.ErrorContains(t, err, tc.expectedErr, "expected error (attempt %d)", i)
-					} else {
-						require.NoError(t, err, "expected no error (attempt %d)", i)
-						require.Equal(t, tc.expected, actual, "expected containers to be equal (attempt %d)", i)
-					}
-				}
-			})
-		}
-	})
-}
-
-func fakeContainer(t *testing.T, mut ...func(*codersdk.WorkspaceAgentContainer)) codersdk.WorkspaceAgentContainer {
-	t.Helper()
-	ct := codersdk.WorkspaceAgentContainer{
-		CreatedAt:    time.Now().UTC(),
-		ID:           uuid.New().String(),
-		FriendlyName: testutil.GetRandomName(t),
-		Image:        testutil.GetRandomName(t) + ":" + strings.Split(uuid.New().String(), "-")[0],
-		Labels: map[string]string{
-			testutil.GetRandomName(t): testutil.GetRandomName(t),
-		},
-		Running: true,
-		Ports: []codersdk.WorkspaceAgentContainerPort{
-			{
-				Network:  "tcp",
-				Port:     testutil.RandomPortNoListen(t),
-				HostPort: testutil.RandomPortNoListen(t),
-				//nolint:gosec // this is a test
-				HostIP: []string{"127.0.0.1", "[::1]", "localhost", "0.0.0.0", "[::]", testutil.GetRandomName(t)}[rand.Intn(6)],
-			},
-		},
-		Status:  testutil.MustRandString(t, 10),
-		Volumes: map[string]string{testutil.GetRandomName(t): testutil.GetRandomName(t)},
-	}
-	for _, m := range mut {
-		m(&ct)
-	}
-	return ct
-}

agent/agentcontainers/api_test.go

@@ -3,8 +3,10 @@ package agentcontainers_test
 import (
 	"context"
 	"encoding/json"
+	"math/rand"
 	"net/http"
 	"net/http/httptest"
+	"strings"
 	"testing"
 	"time"
 
@@ -13,11 +15,13 @@ import (
 	"github.com/google/uuid"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"go.uber.org/mock/gomock"
 	"golang.org/x/xerrors"
 
 	"cdr.dev/slog"
 	"cdr.dev/slog/sloggers/slogtest"
 	"github.com/coder/coder/v2/agent/agentcontainers"
+	"github.com/coder/coder/v2/agent/agentcontainers/acmock"
 	"github.com/coder/coder/v2/agent/agentcontainers/watcher"
 	"github.com/coder/coder/v2/codersdk"
 	"github.com/coder/coder/v2/testutil"
@@ -38,11 +42,19 @@ func (f *fakeLister) List(_ context.Context) (codersdk.WorkspaceAgentListContain
 // fakeDevcontainerCLI implements the agentcontainers.DevcontainerCLI
 // interface for testing.
 type fakeDevcontainerCLI struct {
-	id  string
-	err error
+	id         string
+	err        error
+	continueUp chan struct{}
 }
 
-func (f *fakeDevcontainerCLI) Up(_ context.Context, _, _ string, _ ...agentcontainers.DevcontainerCLIUpOptions) (string, error) {
+func (f *fakeDevcontainerCLI) Up(ctx context.Context, _, _ string, _ ...agentcontainers.DevcontainerCLIUpOptions) (string, error) {
+	if f.continueUp != nil {
+		select {
+		case <-ctx.Done():
+			return "", xerrors.New("test timeout")
+		case <-f.continueUp:
+		}
+	}
 	return f.id, f.err
 }
 
@@ -146,12 +158,159 @@ func (w *fakeWatcher) sendEventWaitNextCalled(ctx context.Context, event fsnotif
 func TestAPI(t *testing.T) {
 	t.Parallel()
 
+	// List tests the API.getContainers method using a mock
+	// implementation. It specifically tests caching behavior.
+	t.Run("List", func(t *testing.T) {
+		t.Parallel()
+
+		fakeCt := fakeContainer(t)
+		fakeCt2 := fakeContainer(t)
+		makeResponse := func(cts ...codersdk.WorkspaceAgentContainer) codersdk.WorkspaceAgentListContainersResponse {
+			return codersdk.WorkspaceAgentListContainersResponse{Containers: cts}
+		}
+
+		type initialDataPayload struct {
+			val codersdk.WorkspaceAgentListContainersResponse
+			err error
+		}
+
+		// Each test case is called multiple times to ensure idempotency
+		for _, tc := range []struct {
+			name string
+			// initialData to be stored in the handler
+			initialData initialDataPayload
+			// function to set up expectations for the mock
+			setupMock func(mcl *acmock.MockLister, preReq *gomock.Call)
+			// expected result
+			expected codersdk.WorkspaceAgentListContainersResponse
+			// expected error
+			expectedErr string
+		}{
+			{
+				name:        "no initial data",
+				initialData: initialDataPayload{makeResponse(), nil},
+				setupMock: func(mcl *acmock.MockLister, preReq *gomock.Call) {
+					mcl.EXPECT().List(gomock.Any()).Return(makeResponse(fakeCt), nil).After(preReq).AnyTimes()
+				},
+				expected: makeResponse(fakeCt),
+			},
+			{
+				name:        "repeat initial data",
+				initialData: initialDataPayload{makeResponse(fakeCt), nil},
+				expected:    makeResponse(fakeCt),
+			},
+			{
+				name:        "lister error always",
+				initialData: initialDataPayload{makeResponse(), assert.AnError},
+				expectedErr: assert.AnError.Error(),
+			},
+			{
+				name:        "lister error only during initial data",
+				initialData: initialDataPayload{makeResponse(), assert.AnError},
+				setupMock: func(mcl *acmock.MockLister, preReq *gomock.Call) {
+					mcl.EXPECT().List(gomock.Any()).Return(makeResponse(fakeCt), nil).After(preReq).AnyTimes()
+				},
+				expected: makeResponse(fakeCt),
+			},
+			{
+				name:        "lister error after initial data",
+				initialData: initialDataPayload{makeResponse(fakeCt), nil},
+				setupMock: func(mcl *acmock.MockLister, preReq *gomock.Call) {
+					mcl.EXPECT().List(gomock.Any()).Return(makeResponse(), assert.AnError).After(preReq).AnyTimes()
+				},
+				expectedErr: assert.AnError.Error(),
+			},
+			{
+				name:        "updated data",
+				initialData: initialDataPayload{makeResponse(fakeCt), nil},
+				setupMock: func(mcl *acmock.MockLister, preReq *gomock.Call) {
+					mcl.EXPECT().List(gomock.Any()).Return(makeResponse(fakeCt2), nil).After(preReq).AnyTimes()
+				},
+				expected: makeResponse(fakeCt2),
+			},
+		} {
+			t.Run(tc.name, func(t *testing.T) {
+				t.Parallel()
+				var (
+					ctx        = testutil.Context(t, testutil.WaitShort)
+					mClock     = quartz.NewMock(t)
+					tickerTrap = mClock.Trap().TickerFunc("updaterLoop")
+					mCtrl      = gomock.NewController(t)
+					mLister    = acmock.NewMockLister(mCtrl)
+					logger     = slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}).Leveled(slog.LevelDebug)
+					r          = chi.NewRouter()
+				)
+
+				initialDataCall := mLister.EXPECT().List(gomock.Any()).Return(tc.initialData.val, tc.initialData.err)
+				if tc.setupMock != nil {
+					tc.setupMock(mLister, initialDataCall.Times(1))
+				} else {
+					initialDataCall.AnyTimes()
+				}
+
+				api := agentcontainers.NewAPI(logger,
+					agentcontainers.WithClock(mClock),
+					agentcontainers.WithLister(mLister),
+				)
+				defer api.Close()
+				r.Mount("/", api.Routes())
+
+				// Make sure the ticker function has been registered
+				// before advancing the clock.
+				tickerTrap.MustWait(ctx).MustRelease(ctx)
+				tickerTrap.Close()
+
+				// Initial request returns the initial data.
+				req := httptest.NewRequest(http.MethodGet, "/", nil).
+					WithContext(ctx)
+				rec := httptest.NewRecorder()
+				r.ServeHTTP(rec, req)
+
+				if tc.initialData.err != nil {
+					got := &codersdk.Error{}
+					err := json.NewDecoder(rec.Body).Decode(got)
+					require.NoError(t, err, "unmarshal response failed")
+					require.ErrorContains(t, got, tc.initialData.err.Error(), "want error")
+				} else {
+					var got codersdk.WorkspaceAgentListContainersResponse
+					err := json.NewDecoder(rec.Body).Decode(&got)
+					require.NoError(t, err, "unmarshal response failed")
+					require.Equal(t, tc.initialData.val, got, "want initial data")
+				}
+
+				// Advance the clock to run updaterLoop.
+				_, aw := mClock.AdvanceNext()
+				aw.MustWait(ctx)
+
+				// Second request returns the updated data.
+				req = httptest.NewRequest(http.MethodGet, "/", nil).
+					WithContext(ctx)
+				rec = httptest.NewRecorder()
+				r.ServeHTTP(rec, req)
+
+				if tc.expectedErr != "" {
+					got := &codersdk.Error{}
+					err := json.NewDecoder(rec.Body).Decode(got)
+					require.NoError(t, err, "unmarshal response failed")
+					require.ErrorContains(t, got, tc.expectedErr, "want error")
+					return
+				}
+
+				var got codersdk.WorkspaceAgentListContainersResponse
+				err := json.NewDecoder(rec.Body).Decode(&got)
+				require.NoError(t, err, "unmarshal response failed")
+				require.Equal(t, tc.expected, got, "want updated data")
+			})
+		}
+	})
+
 	t.Run("Recreate", func(t *testing.T) {
 		t.Parallel()
 
 		validContainer := codersdk.WorkspaceAgentContainer{
 			ID:           "container-id",
 			FriendlyName: "container-name",
+			Running:      true,
 			Labels: map[string]string{
 				agentcontainers.DevcontainerLocalFolderLabel: "/workspace",
 				agentcontainers.DevcontainerConfigFileLabel:  "/workspace/.devcontainer/devcontainer.json",
@@ -169,16 +328,16 @@ func TestAPI(t *testing.T) {
 			containerID     string
 			lister          *fakeLister
 			devcontainerCLI *fakeDevcontainerCLI
-			wantStatus      int
-			wantBody        string
+			wantStatus      []int
+			wantBody        []string
 		}{
 			{
-				name:            "Missing ID",
+				name:            "Missing container ID",
 				containerID:     "",
 				lister:          &fakeLister{},
 				devcontainerCLI: &fakeDevcontainerCLI{},
-				wantStatus:      http.StatusBadRequest,
-				wantBody:        "Missing container ID or name",
+				wantStatus:      []int{http.StatusBadRequest},
+				wantBody:        []string{"Missing container ID or name"},
 			},
 			{
 				name:        "List error",
@@ -187,8 +346,8 @@ func TestAPI(t *testing.T) {
 					err: xerrors.New("list error"),
 				},
 				devcontainerCLI: &fakeDevcontainerCLI{},
-				wantStatus:      http.StatusInternalServerError,
-				wantBody:        "Could not list containers",
+				wantStatus:      []int{http.StatusInternalServerError},
+				wantBody:        []string{"Could not list containers"},
 			},
 			{
 				name:        "Container not found",
@@ -199,8 +358,8 @@ func TestAPI(t *testing.T) {
 					},
 				},
 				devcontainerCLI: &fakeDevcontainerCLI{},
-				wantStatus:      http.StatusNotFound,
-				wantBody:        "Container not found",
+				wantStatus:      []int{http.StatusNotFound},
+				wantBody:        []string{"Container not found"},
 			},
 			{
 				name:        "Missing workspace folder label",
@@ -211,8 +370,8 @@ func TestAPI(t *testing.T) {
 					},
 				},
 				devcontainerCLI: &fakeDevcontainerCLI{},
-				wantStatus:      http.StatusBadRequest,
-				wantBody:        "Missing workspace folder label",
+				wantStatus:      []int{http.StatusBadRequest},
+				wantBody:        []string{"Missing workspace folder label"},
 			},
 			{
 				name:        "Devcontainer CLI error",
@@ -225,8 +384,8 @@ func TestAPI(t *testing.T) {
 				devcontainerCLI: &fakeDevcontainerCLI{
 					err: xerrors.New("devcontainer CLI error"),
 				},
-				wantStatus: http.StatusInternalServerError,
-				wantBody:   "Could not recreate devcontainer",
+				wantStatus: []int{http.StatusAccepted, http.StatusConflict},
+				wantBody:   []string{"Devcontainer recreation initiated", "Devcontainer recreation already in progress"},
 			},
 			{
 				name:        "OK",
@@ -237,21 +396,33 @@ func TestAPI(t *testing.T) {
 					},
 				},
 				devcontainerCLI: &fakeDevcontainerCLI{},
-				wantStatus:      http.StatusNoContent,
-				wantBody:        "",
+				wantStatus:      []int{http.StatusAccepted, http.StatusConflict},
+				wantBody:        []string{"Devcontainer recreation initiated", "Devcontainer recreation already in progress"},
 			},
 		}
 
 		for _, tt := range tests {
 			t.Run(tt.name, func(t *testing.T) {
 				t.Parallel()
+				require.GreaterOrEqual(t, len(tt.wantStatus), 1, "developer error: at least one status code expected")
+				require.Len(t, tt.wantStatus, len(tt.wantBody), "developer error: status and body length mismatch")
 
-				logger := slogtest.Make(t, nil).Leveled(slog.LevelDebug)
+				ctx := testutil.Context(t, testutil.WaitShort)
+
+				logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}).Leveled(slog.LevelDebug)
+				mClock := quartz.NewMock(t)
+				mClock.Set(time.Now()).MustWait(ctx)
+				tickerTrap := mClock.Trap().TickerFunc("updaterLoop")
+				nowRecreateErrorTrap := mClock.Trap().Now("recreate", "errorTimes")
+				nowRecreateSuccessTrap := mClock.Trap().Now("recreate", "successTimes")
+
+				tt.devcontainerCLI.continueUp = make(chan struct{})
 
 				// Setup router with the handler under test.
 				r := chi.NewRouter()
 				api := agentcontainers.NewAPI(
 					logger,
+					agentcontainers.WithClock(mClock),
 					agentcontainers.WithLister(tt.lister),
 					agentcontainers.WithDevcontainerCLI(tt.devcontainerCLI),
 					agentcontainers.WithWatcher(watcher.NewNoop()),
@@ -259,18 +430,108 @@ func TestAPI(t *testing.T) {
 				defer api.Close()
 				r.Mount("/", api.Routes())
 
-				// Simulate HTTP request to the recreate endpoint.
-				req := httptest.NewRequest(http.MethodPost, "/"+tt.containerID+"/recreate", nil)
+				// Make sure the ticker function has been registered
+				// before advancing the clock.
+				tickerTrap.MustWait(ctx).MustRelease(ctx)
+				tickerTrap.Close()
+
+				for i := range tt.wantStatus {
+					// Simulate HTTP request to the recreate endpoint.
+					req := httptest.NewRequest(http.MethodPost, "/devcontainers/container/"+tt.containerID+"/recreate", nil).
+						WithContext(ctx)
+					rec := httptest.NewRecorder()
+					r.ServeHTTP(rec, req)
+
+					// Check the response status code and body.
+					require.Equal(t, tt.wantStatus[i], rec.Code, "status code mismatch")
+					if tt.wantBody[i] != "" {
+						assert.Contains(t, rec.Body.String(), tt.wantBody[i], "response body mismatch")
+					}
+				}
+
+				// Error tests are simple, but the remainder of this test is a
+				// bit more involved, closer to an integration test. That is
+				// because we must check what state the devcontainer ends up in
+				// after the recreation process is initiated and finished.
+				if tt.wantStatus[0] != http.StatusAccepted {
+					close(tt.devcontainerCLI.continueUp)
+					nowRecreateSuccessTrap.Close()
+					nowRecreateErrorTrap.Close()
+					return
+				}
+
+				_, aw := mClock.AdvanceNext()
+				aw.MustWait(ctx)
+
+				// Verify the devcontainer is in starting state after recreation
+				// request is made.
+				req := httptest.NewRequest(http.MethodGet, "/devcontainers", nil).
+					WithContext(ctx)
 				rec := httptest.NewRecorder()
 				r.ServeHTTP(rec, req)
 
-				// Check the response status code and body.
-				require.Equal(t, tt.wantStatus, rec.Code, "status code mismatch")
-				if tt.wantBody != "" {
-					assert.Contains(t, rec.Body.String(), tt.wantBody, "response body mismatch")
-				} else if tt.wantStatus == http.StatusNoContent {
-					assert.Empty(t, rec.Body.String(), "expected empty response body")
+				require.Equal(t, http.StatusOK, rec.Code, "status code mismatch")
+				var resp codersdk.WorkspaceAgentDevcontainersResponse
+				t.Log(rec.Body.String())
+				err := json.NewDecoder(rec.Body).Decode(&resp)
+				require.NoError(t, err, "unmarshal response failed")
+				require.Len(t, resp.Devcontainers, 1, "expected one devcontainer in response")
+				assert.Equal(t, codersdk.WorkspaceAgentDevcontainerStatusStarting, resp.Devcontainers[0].Status, "devcontainer is not starting")
+				require.NotNil(t, resp.Devcontainers[0].Container, "devcontainer should have container reference")
+				assert.Equal(t, codersdk.WorkspaceAgentDevcontainerStatusStarting, resp.Devcontainers[0].Container.DevcontainerStatus, "container dc status is not starting")
+
+				// Allow the devcontainer CLI to continue the up process.
+				close(tt.devcontainerCLI.continueUp)
+
+				// Ensure the devcontainer ends up in error state if the up call fails.
+				if tt.devcontainerCLI.err != nil {
+					nowRecreateSuccessTrap.Close()
+					// The timestamp for the error will be stored, which gives
+					// us a good anchor point to know when to do our request.
+					nowRecreateErrorTrap.MustWait(ctx).MustRelease(ctx)
+					nowRecreateErrorTrap.Close()
+
+					// Advance the clock to run the devcontainer state update routine.
+					_, aw = mClock.AdvanceNext()
+					aw.MustWait(ctx)
+
+					req = httptest.NewRequest(http.MethodGet, "/devcontainers", nil).
+						WithContext(ctx)
+					rec = httptest.NewRecorder()
+					r.ServeHTTP(rec, req)
+
+					require.Equal(t, http.StatusOK, rec.Code, "status code mismatch after error")
+					err = json.NewDecoder(rec.Body).Decode(&resp)
+					require.NoError(t, err, "unmarshal response failed after error")
+					require.Len(t, resp.Devcontainers, 1, "expected one devcontainer in response after error")
+					assert.Equal(t, codersdk.WorkspaceAgentDevcontainerStatusError, resp.Devcontainers[0].Status, "devcontainer is not in an error state after up failure")
+					require.NotNil(t, resp.Devcontainers[0].Container, "devcontainer should have container reference after up failure")
+					assert.Equal(t, codersdk.WorkspaceAgentDevcontainerStatusError, resp.Devcontainers[0].Container.DevcontainerStatus, "container dc status is not error after up failure")
+					return
 				}
+
+				// Ensure the devcontainer ends up in success state.
+				nowRecreateSuccessTrap.MustWait(ctx).MustRelease(ctx)
+				nowRecreateSuccessTrap.Close()
+
+				// Advance the clock to run the devcontainer state update routine.
+				_, aw = mClock.AdvanceNext()
+				aw.MustWait(ctx)
+
+				req = httptest.NewRequest(http.MethodGet, "/devcontainers", nil).
+					WithContext(ctx)
+				rec = httptest.NewRecorder()
+				r.ServeHTTP(rec, req)
+
+				// Check the response status code and body after recreation.
+				require.Equal(t, http.StatusOK, rec.Code, "status code mismatch after recreation")
+				t.Log(rec.Body.String())
+				err = json.NewDecoder(rec.Body).Decode(&resp)
+				require.NoError(t, err, "unmarshal response failed after recreation")
+				require.Len(t, resp.Devcontainers, 1, "expected one devcontainer in response after recreation")
+				assert.Equal(t, codersdk.WorkspaceAgentDevcontainerStatusRunning, resp.Devcontainers[0].Status, "devcontainer is not running after recreation")
+				require.NotNil(t, resp.Devcontainers[0].Container, "devcontainer should have container reference after recreation")
+				assert.Equal(t, codersdk.WorkspaceAgentDevcontainerStatusRunning, resp.Devcontainers[0].Container.DevcontainerStatus, "container dc status is not running after recreation")
 			})
 		}
 	})
@@ -329,7 +590,7 @@ func TestAPI(t *testing.T) {
 				wantCount:          2,
 				verify: func(t *testing.T, devcontainers []codersdk.WorkspaceAgentDevcontainer) {
 					for _, dc := range devcontainers {
-						assert.False(t, dc.Running, "devcontainer should not be running")
+						assert.Equal(t, codersdk.WorkspaceAgentDevcontainerStatusStopped, dc.Status, "devcontainer should be stopped")
 						assert.Nil(t, dc.Container, "devcontainer should not have container reference")
 					}
 				},
@@ -362,9 +623,10 @@ func TestAPI(t *testing.T) {
 				verify: func(t *testing.T, devcontainers []codersdk.WorkspaceAgentDevcontainer) {
 					dc := devcontainers[0]
 					assert.Equal(t, "/workspace/runtime1", dc.WorkspaceFolder)
-					assert.True(t, dc.Running)
+					assert.Equal(t, codersdk.WorkspaceAgentDevcontainerStatusRunning, dc.Status)
 					require.NotNil(t, dc.Container)
 					assert.Equal(t, "runtime-container-1", dc.Container.ID)
+					assert.Equal(t, codersdk.WorkspaceAgentDevcontainerStatusRunning, dc.Container.DevcontainerStatus)
 				},
 			},
 			{
@@ -401,16 +663,18 @@ func TestAPI(t *testing.T) {
 					known2 := mustFindDevcontainerByPath(t, devcontainers, "/workspace/known2")
 					runtime1 := mustFindDevcontainerByPath(t, devcontainers, "/workspace/runtime1")
 
-					assert.True(t, known1.Running)
-					assert.False(t, known2.Running)
-					assert.True(t, runtime1.Running)
+					assert.Equal(t, codersdk.WorkspaceAgentDevcontainerStatusRunning, known1.Status)
+					assert.Equal(t, codersdk.WorkspaceAgentDevcontainerStatusStopped, known2.Status)
+					assert.Equal(t, codersdk.WorkspaceAgentDevcontainerStatusRunning, runtime1.Status)
+
+					assert.Nil(t, known2.Container)
 
 					require.NotNil(t, known1.Container)
-					assert.Nil(t, known2.Container)
-					require.NotNil(t, runtime1.Container)
-
 					assert.Equal(t, "known-container-1", known1.Container.ID)
+					assert.Equal(t, codersdk.WorkspaceAgentDevcontainerStatusRunning, known1.Container.DevcontainerStatus)
+					require.NotNil(t, runtime1.Container)
 					assert.Equal(t, "runtime-container-1", runtime1.Container.ID)
+					assert.Equal(t, codersdk.WorkspaceAgentDevcontainerStatusRunning, runtime1.Container.DevcontainerStatus)
 				},
 			},
 			{
@@ -445,14 +709,16 @@ func TestAPI(t *testing.T) {
 					running := mustFindDevcontainerByPath(t, devcontainers, "/workspace/running")
 					nonRunning := mustFindDevcontainerByPath(t, devcontainers, "/workspace/non-running")
 
-					assert.True(t, running.Running)
-					assert.False(t, nonRunning.Running)
+					assert.Equal(t, codersdk.WorkspaceAgentDevcontainerStatusRunning, running.Status)
+					assert.Equal(t, codersdk.WorkspaceAgentDevcontainerStatusStopped, nonRunning.Status)
 
 					require.NotNil(t, running.Container, "running container should have container reference")
-					require.NotNil(t, nonRunning.Container, "non-running container should have container reference")
-
 					assert.Equal(t, "running-container", running.Container.ID)
+					assert.Equal(t, codersdk.WorkspaceAgentDevcontainerStatusRunning, running.Container.DevcontainerStatus)
+
+					require.NotNil(t, nonRunning.Container, "non-running container should have container reference")
 					assert.Equal(t, "non-running-container", nonRunning.Container.ID)
+					assert.Equal(t, codersdk.WorkspaceAgentDevcontainerStatusStopped, nonRunning.Container.DevcontainerStatus)
 				},
 			},
 			{
@@ -484,10 +750,11 @@ func TestAPI(t *testing.T) {
 						}
 					}
 					require.NotNil(t, dc2, "missing devcontainer with ID %s", knownDevcontainerID2)
-					assert.True(t, dc2.Running)
+					assert.Equal(t, codersdk.WorkspaceAgentDevcontainerStatusRunning, dc2.Status)
 					assert.NotEmpty(t, dc2.ConfigPath)
 					require.NotNil(t, dc2.Container)
 					assert.Equal(t, "known-container-2", dc2.Container.ID)
+					assert.Equal(t, codersdk.WorkspaceAgentDevcontainerStatusRunning, dc2.Container.DevcontainerStatus)
 				},
 			},
 			{
@@ -554,24 +821,51 @@ func TestAPI(t *testing.T) {
 			t.Run(tt.name, func(t *testing.T) {
 				t.Parallel()
 
-				logger := slogtest.Make(t, nil).Leveled(slog.LevelDebug)
+				logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}).Leveled(slog.LevelDebug)
+
+				mClock := quartz.NewMock(t)
+				mClock.Set(time.Now()).MustWait(testutil.Context(t, testutil.WaitShort))
+				tickerTrap := mClock.Trap().TickerFunc("updaterLoop")
 
 				// Setup router with the handler under test.
 				r := chi.NewRouter()
 				apiOptions := []agentcontainers.Option{
+					agentcontainers.WithClock(mClock),
 					agentcontainers.WithLister(tt.lister),
 					agentcontainers.WithWatcher(watcher.NewNoop()),
 				}
 
+				// Generate matching scripts for the known devcontainers
+				// (required to extract log source ID).
+				var scripts []codersdk.WorkspaceAgentScript
+				for i := range tt.knownDevcontainers {
+					scripts = append(scripts, codersdk.WorkspaceAgentScript{
+						ID:          tt.knownDevcontainers[i].ID,
+						LogSourceID: uuid.New(),
+					})
+				}
 				if len(tt.knownDevcontainers) > 0 {
-					apiOptions = append(apiOptions, agentcontainers.WithDevcontainers(tt.knownDevcontainers))
+					apiOptions = append(apiOptions, agentcontainers.WithDevcontainers(tt.knownDevcontainers, scripts))
 				}
 
 				api := agentcontainers.NewAPI(logger, apiOptions...)
 				defer api.Close()
+
 				r.Mount("/", api.Routes())
 
-				req := httptest.NewRequest(http.MethodGet, "/devcontainers", nil)
+				ctx := testutil.Context(t, testutil.WaitShort)
+
+				// Make sure the ticker function has been registered
+				// before advancing the clock.
+				tickerTrap.MustWait(ctx).MustRelease(ctx)
+				tickerTrap.Close()
+
+				// Advance the clock to run the updater loop.
+				_, aw := mClock.AdvanceNext()
+				aw.MustWait(ctx)
+
+				req := httptest.NewRequest(http.MethodGet, "/devcontainers", nil).
+					WithContext(ctx)
 				rec := httptest.NewRecorder()
 				r.ServeHTTP(rec, req)
 
@@ -596,15 +890,111 @@ func TestAPI(t *testing.T) {
 		}
 	})
 
+	t.Run("List devcontainers running then not running", func(t *testing.T) {
+		t.Parallel()
+
+		container := codersdk.WorkspaceAgentContainer{
+			ID:           "container-id",
+			FriendlyName: "container-name",
+			Running:      true,
+			CreatedAt:    time.Now().Add(-1 * time.Minute),
+			Labels: map[string]string{
+				agentcontainers.DevcontainerLocalFolderLabel: "/home/coder/project",
+				agentcontainers.DevcontainerConfigFileLabel:  "/home/coder/project/.devcontainer/devcontainer.json",
+			},
+		}
+		dc := codersdk.WorkspaceAgentDevcontainer{
+			ID:              uuid.New(),
+			Name:            "test-devcontainer",
+			WorkspaceFolder: "/home/coder/project",
+			ConfigPath:      "/home/coder/project/.devcontainer/devcontainer.json",
+			Status:          codersdk.WorkspaceAgentDevcontainerStatusRunning, // Corrected enum
+		}
+
+		ctx := testutil.Context(t, testutil.WaitShort)
+
+		logger := slogtest.Make(t, nil).Leveled(slog.LevelDebug)
+		fLister := &fakeLister{
+			containers: codersdk.WorkspaceAgentListContainersResponse{
+				Containers: []codersdk.WorkspaceAgentContainer{container},
+			},
+		}
+		fWatcher := newFakeWatcher(t)
+		mClock := quartz.NewMock(t)
+		mClock.Set(time.Now()).MustWait(ctx)
+		tickerTrap := mClock.Trap().TickerFunc("updaterLoop")
+
+		api := agentcontainers.NewAPI(logger,
+			agentcontainers.WithClock(mClock),
+			agentcontainers.WithLister(fLister),
+			agentcontainers.WithWatcher(fWatcher),
+			agentcontainers.WithDevcontainers(
+				[]codersdk.WorkspaceAgentDevcontainer{dc},
+				[]codersdk.WorkspaceAgentScript{{LogSourceID: uuid.New(), ID: dc.ID}},
+			),
+		)
+		defer api.Close()
+
+		// Make sure the ticker function has been registered
+		// before advancing any use of mClock.Advance.
+		tickerTrap.MustWait(ctx).MustRelease(ctx)
+		tickerTrap.Close()
+
+		// Make sure the start loop has been called.
+		fWatcher.waitNext(ctx)
+
+		// Simulate a file modification event to make the devcontainer dirty.
+		fWatcher.sendEventWaitNextCalled(ctx, fsnotify.Event{
+			Name: "/home/coder/project/.devcontainer/devcontainer.json",
+			Op:   fsnotify.Write,
+		})
+
+		// Initially the devcontainer should be running and dirty.
+		req := httptest.NewRequest(http.MethodGet, "/devcontainers", nil).
+			WithContext(ctx)
+		rec := httptest.NewRecorder()
+		api.Routes().ServeHTTP(rec, req)
+
+		require.Equal(t, http.StatusOK, rec.Code)
+		var resp1 codersdk.WorkspaceAgentDevcontainersResponse
+		err := json.NewDecoder(rec.Body).Decode(&resp1)
+		require.NoError(t, err)
+		require.Len(t, resp1.Devcontainers, 1)
+		require.Equal(t, codersdk.WorkspaceAgentDevcontainerStatusRunning, resp1.Devcontainers[0].Status, "devcontainer should be running initially")
+		require.True(t, resp1.Devcontainers[0].Dirty, "devcontainer should be dirty initially")
+		require.NotNil(t, resp1.Devcontainers[0].Container, "devcontainer should have a container initially")
+
+		// Next, simulate a situation where the container is no longer
+		// running.
+		fLister.containers.Containers = []codersdk.WorkspaceAgentContainer{}
+
+		// Trigger a refresh which will use the second response from mock
+		// lister (no containers).
+		_, aw := mClock.AdvanceNext()
+		aw.MustWait(ctx)
+
+		// Afterwards the devcontainer should not be running and not dirty.
+		req = httptest.NewRequest(http.MethodGet, "/devcontainers", nil).
+			WithContext(ctx)
+		rec = httptest.NewRecorder()
+		api.Routes().ServeHTTP(rec, req)
+
+		require.Equal(t, http.StatusOK, rec.Code)
+		var resp2 codersdk.WorkspaceAgentDevcontainersResponse
+		err = json.NewDecoder(rec.Body).Decode(&resp2)
+		require.NoError(t, err)
+		require.Len(t, resp2.Devcontainers, 1)
+		require.Equal(t, codersdk.WorkspaceAgentDevcontainerStatusStopped, resp2.Devcontainers[0].Status, "devcontainer should not be running after empty list")
+		require.False(t, resp2.Devcontainers[0].Dirty, "devcontainer should not be dirty after empty list")
+		require.Nil(t, resp2.Devcontainers[0].Container, "devcontainer should not have a container after empty list")
+	})
+
 	t.Run("FileWatcher", func(t *testing.T) {
 		t.Parallel()
 
-		ctx := testutil.Context(t, testutil.WaitMedium)
+		ctx := testutil.Context(t, testutil.WaitShort)
 
 		startTime := time.Date(2025, 1, 1, 12, 0, 0, 0, time.UTC)
-		mClock := quartz.NewMock(t)
-		mClock.Set(startTime)
-		fWatcher := newFakeWatcher(t)
 
 		// Create a fake container with a config file.
 		configPath := "/workspace/project/.devcontainer/devcontainer.json"
@@ -619,6 +1009,10 @@ func TestAPI(t *testing.T) {
 			},
 		}
 
+		mClock := quartz.NewMock(t)
+		mClock.Set(startTime)
+		tickerTrap := mClock.Trap().TickerFunc("updaterLoop")
+		fWatcher := newFakeWatcher(t)
 		fLister := &fakeLister{
 			containers: codersdk.WorkspaceAgentListContainersResponse{
 				Containers: []codersdk.WorkspaceAgentContainer{container},
@@ -634,14 +1028,18 @@ func TestAPI(t *testing.T) {
 		)
 		defer api.Close()
 
-		api.SignalReady()
-
 		r := chi.NewRouter()
 		r.Mount("/", api.Routes())
 
+		// Make sure the ticker function has been registered
+		// before advancing any use of mClock.Advance.
+		tickerTrap.MustWait(ctx).MustRelease(ctx)
+		tickerTrap.Close()
+
 		// Call the list endpoint first to ensure config files are
 		// detected and watched.
-		req := httptest.NewRequest(http.MethodGet, "/devcontainers", nil)
+		req := httptest.NewRequest(http.MethodGet, "/devcontainers", nil).
+			WithContext(ctx)
 		rec := httptest.NewRecorder()
 		r.ServeHTTP(rec, req)
 		require.Equal(t, http.StatusOK, rec.Code)
@@ -651,6 +1049,10 @@ func TestAPI(t *testing.T) {
 		require.NoError(t, err)
 		require.Len(t, response.Devcontainers, 1)
 		assert.False(t, response.Devcontainers[0].Dirty,
+			"devcontainer should not be marked as dirty initially")
+		assert.Equal(t, codersdk.WorkspaceAgentDevcontainerStatusRunning, response.Devcontainers[0].Status, "devcontainer should be running initially")
+		require.NotNil(t, response.Devcontainers[0].Container, "container should not be nil")
+		assert.False(t, response.Devcontainers[0].Container.DevcontainerDirty,
 			"container should not be marked as dirty initially")
 
 		// Verify the watcher is watching the config file.
@@ -667,10 +1069,13 @@ func TestAPI(t *testing.T) {
 			Op:   fsnotify.Write,
 		})
 
-		mClock.Advance(time.Minute).MustWait(ctx)
+		// Advance the clock to run updaterLoop.
+		_, aw := mClock.AdvanceNext()
+		aw.MustWait(ctx)
 
 		// Check if the container is marked as dirty.
-		req = httptest.NewRequest(http.MethodGet, "/devcontainers", nil)
+		req = httptest.NewRequest(http.MethodGet, "/devcontainers", nil).
+			WithContext(ctx)
 		rec = httptest.NewRecorder()
 		r.ServeHTTP(rec, req)
 		require.Equal(t, http.StatusOK, rec.Code)
@@ -680,16 +1085,23 @@ func TestAPI(t *testing.T) {
 		require.Len(t, response.Devcontainers, 1)
 		assert.True(t, response.Devcontainers[0].Dirty,
 			"container should be marked as dirty after config file was modified")
-
-		mClock.Advance(time.Minute).MustWait(ctx)
+		assert.Equal(t, codersdk.WorkspaceAgentDevcontainerStatusRunning, response.Devcontainers[0].Status, "devcontainer should be running after config file was modified")
+		require.NotNil(t, response.Devcontainers[0].Container, "container should not be nil")
+		assert.True(t, response.Devcontainers[0].Container.DevcontainerDirty,
+			"container should be marked as dirty after config file was modified")
 
 		container.ID = "new-container-id" // Simulate a new container ID after recreation.
 		container.FriendlyName = "new-container-name"
 		container.CreatedAt = mClock.Now() // Update the creation time.
 		fLister.containers.Containers = []codersdk.WorkspaceAgentContainer{container}
 
+		// Advance the clock to run updaterLoop.
+		_, aw = mClock.AdvanceNext()
+		aw.MustWait(ctx)
+
 		// Check if dirty flag is cleared.
-		req = httptest.NewRequest(http.MethodGet, "/devcontainers", nil)
+		req = httptest.NewRequest(http.MethodGet, "/devcontainers", nil).
+			WithContext(ctx)
 		rec = httptest.NewRecorder()
 		r.ServeHTTP(rec, req)
 		require.Equal(t, http.StatusOK, rec.Code)
@@ -698,7 +1110,11 @@ func TestAPI(t *testing.T) {
 		require.NoError(t, err)
 		require.Len(t, response.Devcontainers, 1)
 		assert.False(t, response.Devcontainers[0].Dirty,
-			"dirty flag should be cleared after container recreation")
+			"dirty flag should be cleared on the devcontainer after container recreation")
+		assert.Equal(t, codersdk.WorkspaceAgentDevcontainerStatusRunning, response.Devcontainers[0].Status, "devcontainer should be running after recreation")
+		require.NotNil(t, response.Devcontainers[0].Container, "container should not be nil")
+		assert.False(t, response.Devcontainers[0].Container.DevcontainerDirty,
+			"dirty flag should be cleared on the container after container recreation")
 	})
 }
 
@@ -716,3 +1132,32 @@ func mustFindDevcontainerByPath(t *testing.T, devcontainers []codersdk.Workspace
 	require.Failf(t, "no devcontainer found with workspace folder %q", path)
 	return codersdk.WorkspaceAgentDevcontainer{} // Unreachable, but required for compilation
 }
+
+func fakeContainer(t *testing.T, mut ...func(*codersdk.WorkspaceAgentContainer)) codersdk.WorkspaceAgentContainer {
+	t.Helper()
+	ct := codersdk.WorkspaceAgentContainer{
+		CreatedAt:    time.Now().UTC(),
+		ID:           uuid.New().String(),
+		FriendlyName: testutil.GetRandomName(t),
+		Image:        testutil.GetRandomName(t) + ":" + strings.Split(uuid.New().String(), "-")[0],
+		Labels: map[string]string{
+			testutil.GetRandomName(t): testutil.GetRandomName(t),
+		},
+		Running: true,
+		Ports: []codersdk.WorkspaceAgentContainerPort{
+			{
+				Network:  "tcp",
+				Port:     testutil.RandomPortNoListen(t),
+				HostPort: testutil.RandomPortNoListen(t),
+				//nolint:gosec // this is a test
+				HostIP: []string{"127.0.0.1", "[::1]", "localhost", "0.0.0.0", "[::]", testutil.GetRandomName(t)}[rand.Intn(6)],
+			},
+		},
+		Status:  testutil.MustRandString(t, 10),
+		Volumes: map[string]string{testutil.GetRandomName(t): testutil.GetRandomName(t)},
+	}
+	for _, m := range mut {
+		m(&ct)
+	}
+	return ct
+}

agent/agentcontainers/devcontainer.go

@@ -22,7 +22,8 @@ const (
 
 const devcontainerUpScriptTemplate = `
 if ! which devcontainer > /dev/null 2>&1; then
-  echo "ERROR: Unable to start devcontainer, @devcontainers/cli is not installed."
+  echo "ERROR: Unable to start devcontainer, @devcontainers/cli is not installed or not found in \$PATH." 1>&2
+  echo "Please install @devcontainers/cli by running \"npm install -g @devcontainers/cli\" or by using the \"devcontainers-cli\" Coder module." 1>&2
   exit 1
 fi
 devcontainer up %s
@@ -36,8 +37,6 @@ devcontainer up %s
 // initialize the workspace (e.g. git clone, npm install, etc). This is
 // important if e.g. a Coder module to install @devcontainer/cli is used.
 func ExtractAndInitializeDevcontainerScripts(
-	logger slog.Logger,
-	expandPath func(string) (string, error),
 	devcontainers []codersdk.WorkspaceAgentDevcontainer,
 	scripts []codersdk.WorkspaceAgentScript,
 ) (filteredScripts []codersdk.WorkspaceAgentScript, devcontainerScripts []codersdk.WorkspaceAgentScript) {
@@ -47,7 +46,6 @@ ScriptLoop:
 			// The devcontainer scripts match the devcontainer ID for
 			// identification.
 			if script.ID == dc.ID {
-				dc = expandDevcontainerPaths(logger, expandPath, dc)
 				devcontainerScripts = append(devcontainerScripts, devcontainerStartupScript(dc, script))
 				continue ScriptLoop
 			}
@@ -68,13 +66,26 @@ func devcontainerStartupScript(dc codersdk.WorkspaceAgentDevcontainer, script co
 		args = append(args, fmt.Sprintf("--config %q", dc.ConfigPath))
 	}
 	cmd := fmt.Sprintf(devcontainerUpScriptTemplate, strings.Join(args, " "))
-	script.Script = cmd
+	// Force the script to run in /bin/sh, since some shells (e.g. fish)
+	// don't support the script.
+	script.Script = fmt.Sprintf("/bin/sh -c '%s'", cmd)
 	// Disable RunOnStart, scripts have this set so that when devcontainers
 	// have not been enabled, a warning will be surfaced in the agent logs.
 	script.RunOnStart = false
 	return script
 }
 
+// ExpandAllDevcontainerPaths expands all devcontainer paths in the given
+// devcontainers. This is required by the devcontainer CLI, which requires
+// absolute paths for the workspace folder and config path.
+func ExpandAllDevcontainerPaths(logger slog.Logger, expandPath func(string) (string, error), devcontainers []codersdk.WorkspaceAgentDevcontainer) []codersdk.WorkspaceAgentDevcontainer {
+	expanded := make([]codersdk.WorkspaceAgentDevcontainer, 0, len(devcontainers))
+	for _, dc := range devcontainers {
+		expanded = append(expanded, expandDevcontainerPaths(logger, expandPath, dc))
+	}
+	return expanded
+}
+
 func expandDevcontainerPaths(logger slog.Logger, expandPath func(string) (string, error), dc codersdk.WorkspaceAgentDevcontainer) codersdk.WorkspaceAgentDevcontainer {
 	logger = logger.With(slog.F("devcontainer", dc.Name), slog.F("workspace_folder", dc.WorkspaceFolder), slog.F("config_path", dc.ConfigPath))
 

agent/agentcontainers/devcontainer_test.go

@@ -242,9 +242,7 @@ func TestExtractAndInitializeDevcontainerScripts(t *testing.T) {
 				}
 			}
 			gotFilteredScripts, gotDevcontainerScripts := agentcontainers.ExtractAndInitializeDevcontainerScripts(
-				logger,
-				tt.args.expandPath,
-				tt.args.devcontainers,
+				agentcontainers.ExpandAllDevcontainerPaths(logger, tt.args.expandPath, tt.args.devcontainers),
 				tt.args.scripts,
 			)
 

agent/agentcontainers/devcontainercli.go

@@ -31,8 +31,18 @@ func WithRemoveExistingContainer() DevcontainerCLIUpOptions {
 	}
 }
 
+// WithOutput sets stdout and stderr writers for Up command logs.
+func WithOutput(stdout, stderr io.Writer) DevcontainerCLIUpOptions {
+	return func(o *devcontainerCLIUpConfig) {
+		o.stdout = stdout
+		o.stderr = stderr
+	}
+}
+
 type devcontainerCLIUpConfig struct {
 	removeExistingContainer bool
+	stdout                  io.Writer
+	stderr                  io.Writer
 }
 
 func applyDevcontainerCLIUpOptions(opts []DevcontainerCLIUpOptions) devcontainerCLIUpConfig {
@@ -78,18 +88,28 @@ func (d *devcontainerCLI) Up(ctx context.Context, workspaceFolder, configPath st
 	}
 	cmd := d.execer.CommandContext(ctx, "devcontainer", args...)
 
-	var stdout bytes.Buffer
-	cmd.Stdout = io.MultiWriter(&stdout, &devcontainerCLILogWriter{ctx: ctx, logger: logger.With(slog.F("stdout", true))})
-	cmd.Stderr = &devcontainerCLILogWriter{ctx: ctx, logger: logger.With(slog.F("stderr", true))}
+	// Capture stdout for parsing and stream logs for both default and provided writers.
+	var stdoutBuf bytes.Buffer
+	stdoutWriters := []io.Writer{&stdoutBuf, &devcontainerCLILogWriter{ctx: ctx, logger: logger.With(slog.F("stdout", true))}}
+	if conf.stdout != nil {
+		stdoutWriters = append(stdoutWriters, conf.stdout)
+	}
+	cmd.Stdout = io.MultiWriter(stdoutWriters...)
+	// Stream stderr logs and provided writer if any.
+	stderrWriters := []io.Writer{&devcontainerCLILogWriter{ctx: ctx, logger: logger.With(slog.F("stderr", true))}}
+	if conf.stderr != nil {
+		stderrWriters = append(stderrWriters, conf.stderr)
+	}
+	cmd.Stderr = io.MultiWriter(stderrWriters...)
 
 	if err := cmd.Run(); err != nil {
-		if _, err2 := parseDevcontainerCLILastLine(ctx, logger, stdout.Bytes()); err2 != nil {
+		if _, err2 := parseDevcontainerCLILastLine(ctx, logger, stdoutBuf.Bytes()); err2 != nil {
 			err = errors.Join(err, err2)
 		}
 		return "", err
 	}
 
-	result, err := parseDevcontainerCLILastLine(ctx, logger, stdout.Bytes())
+	result, err := parseDevcontainerCLILastLine(ctx, logger, stdoutBuf.Bytes())
 	if err != nil {
 		return "", err
 	}

agent/agentcontainers/devcontainercli_test.go

@@ -128,6 +128,45 @@ func TestDevcontainerCLI_ArgsAndParsing(t *testing.T) {
 	})
 }
 
+// TestDevcontainerCLI_WithOutput tests that WithOutput captures CLI
+// logs to provided writers.
+func TestDevcontainerCLI_WithOutput(t *testing.T) {
+	t.Parallel()
+
+	// Prepare test executable and logger.
+	testExePath, err := os.Executable()
+	require.NoError(t, err, "get test executable path")
+	logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}).Leveled(slog.LevelDebug)
+	ctx := testutil.Context(t, testutil.WaitMedium)
+
+	// Buffers to capture stdout and stderr.
+	outBuf := &bytes.Buffer{}
+	errBuf := &bytes.Buffer{}
+
+	// Simulate CLI execution with a standard up.log file.
+	wantArgs := "up --log-format json --workspace-folder /test/workspace"
+	testExecer := &testDevcontainerExecer{
+		testExePath: testExePath,
+		wantArgs:    wantArgs,
+		wantError:   false,
+		logFile:     filepath.Join("testdata", "devcontainercli", "parse", "up.log"),
+	}
+	dccli := agentcontainers.NewDevcontainerCLI(logger, testExecer)
+
+	// Call Up with WithOutput to capture CLI logs.
+	containerID, err := dccli.Up(ctx, "/test/workspace", "", agentcontainers.WithOutput(outBuf, errBuf))
+	require.NoError(t, err, "Up should succeed")
+	require.NotEmpty(t, containerID, "expected non-empty container ID")
+
+	// Read expected log content.
+	expLog, err := os.ReadFile(filepath.Join("testdata", "devcontainercli", "parse", "up.log"))
+	require.NoError(t, err, "reading expected log file")
+
+	// Verify stdout buffer contains the CLI logs and stderr is empty.
+	assert.Equal(t, string(expLog), outBuf.String(), "stdout buffer should match CLI logs")
+	assert.Empty(t, errBuf.String(), "stderr buffer should be empty on success")
+}
+
 // testDevcontainerExecer implements the agentexec.Execer interface for testing.
 type testDevcontainerExecer struct {
 	testExePath string

agent/agentscripts/agentscripts.go

@@ -10,7 +10,6 @@ import (
 	"os/user"
 	"path/filepath"
 	"sync"
-	"sync/atomic"
 	"time"
 
 	"github.com/google/uuid"
@@ -104,7 +103,6 @@ type Runner struct {
 	closed          chan struct{}
 	closeMutex      sync.Mutex
 	cron            *cron.Cron
-	initialized     atomic.Bool
 	scripts         []runnerScript
 	dataDir         string
 	scriptCompleted ScriptCompletedFunc
@@ -113,6 +111,9 @@ type Runner struct {
 	// execute startup scripts, and scripts on a cron schedule. Both will increment
 	// this counter.
 	scriptsExecuted *prometheus.CounterVec
+
+	initMutex   sync.Mutex
+	initialized bool
 }
 
 // DataDir returns the directory where scripts data is stored.
@@ -154,10 +155,12 @@ func WithPostStartScripts(scripts ...codersdk.WorkspaceAgentScript) InitOption {
 // It also schedules any scripts that have a schedule.
 // This function must be called before Execute.
 func (r *Runner) Init(scripts []codersdk.WorkspaceAgentScript, scriptCompleted ScriptCompletedFunc, opts ...InitOption) error {
-	if r.initialized.Load() {
+	r.initMutex.Lock()
+	defer r.initMutex.Unlock()
+	if r.initialized {
 		return xerrors.New("init: already initialized")
 	}
-	r.initialized.Store(true)
+	r.initialized = true
 	r.scripts = toRunnerScript(scripts...)
 	r.scriptCompleted = scriptCompleted
 	for _, opt := range opts {
@@ -227,6 +230,18 @@ const (
 
 // Execute runs a set of scripts according to a filter.
 func (r *Runner) Execute(ctx context.Context, option ExecuteOption) error {
+	initErr := func() error {
+		r.initMutex.Lock()
+		defer r.initMutex.Unlock()
+		if !r.initialized {
+			return xerrors.New("execute: not initialized")
+		}
+		return nil
+	}()
+	if initErr != nil {
+		return initErr
+	}
+
 	var eg errgroup.Group
 	for _, script := range r.scripts {
 		runScript := (option == ExecuteStartScripts && script.RunOnStart) ||

agent/agentssh/agentssh.go

@@ -124,6 +124,7 @@ type Server struct {
 	listeners map[net.Listener]struct{}
 	conns     map[net.Conn]struct{}
 	sessions  map[ssh.Session]struct{}
+	processes map[*os.Process]struct{}
 	closing   chan struct{}
 	// Wait for goroutines to exit, waited without
 	// a lock on mu but protected by closing.
@@ -182,6 +183,7 @@ func NewServer(ctx context.Context, logger slog.Logger, prometheusRegistry *prom
 		fs:        fs,
 		conns:     make(map[net.Conn]struct{}),
 		sessions:  make(map[ssh.Session]struct{}),
+		processes: make(map[*os.Process]struct{}),
 		logger:    logger,
 
 		config: config,
@@ -586,7 +588,10 @@ func (s *Server) startNonPTYSession(logger slog.Logger, session ssh.Session, mag
 	// otherwise context cancellation will not propagate properly
 	// and SSH server close may be delayed.
 	cmd.SysProcAttr = cmdSysProcAttr()
-	cmd.Cancel = cmdCancel(session.Context(), logger, cmd)
+
+	// to match OpenSSH, we don't actually tear a non-TTY command down, even if the session ends.
+	// c.f. https://github.com/coder/coder/issues/18519#issuecomment-3019118271
+	cmd.Cancel = nil
 
 	cmd.Stdout = session
 	cmd.Stderr = session.Stderr()
@@ -609,6 +614,16 @@ func (s *Server) startNonPTYSession(logger slog.Logger, session ssh.Session, mag
 		s.metrics.sessionErrors.WithLabelValues(magicTypeLabel, "no", "start_command").Add(1)
 		return xerrors.Errorf("start: %w", err)
 	}
+
+	// Since we don't cancel the process when the session stops, we still need to tear it down if we are closing. So
+	// track it here.
+	if !s.trackProcess(cmd.Process, true) {
+		// must be closing
+		err = cmdCancel(logger, cmd.Process)
+		return xerrors.Errorf("failed to track process: %w", err)
+	}
+	defer s.trackProcess(cmd.Process, false)
+
 	sigs := make(chan ssh.Signal, 1)
 	session.Signals(sigs)
 	defer func() {
@@ -1052,6 +1067,27 @@ func (s *Server) trackSession(ss ssh.Session, add bool) (ok bool) {
 	return true
 }
 
+// trackCommand registers the process with the server. If the server is
+// closing, the process is not registered and should be closed.
+//
+//nolint:revive
+func (s *Server) trackProcess(p *os.Process, add bool) (ok bool) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if add {
+		if s.closing != nil {
+			// Server closed.
+			return false
+		}
+		s.wg.Add(1)
+		s.processes[p] = struct{}{}
+		return true
+	}
+	s.wg.Done()
+	delete(s.processes, p)
+	return true
+}
+
 // Close the server and all active connections. Server can be re-used
 // after Close is done.
 func (s *Server) Close() error {
@@ -1091,6 +1127,10 @@ func (s *Server) Close() error {
 		_ = c.Close()
 	}
 
+	for p := range s.processes {
+		_ = cmdCancel(s.logger, p)
+	}
+
 	s.logger.Debug(ctx, "closing SSH server")
 	err := s.srv.Close()
 

agent/agentssh/agentssh_test.go

@@ -214,7 +214,11 @@ func TestNewServer_CloseActiveConnections(t *testing.T) {
 		}
 
 		for _, ch := range waitConns {
-			<-ch
+			select {
+			case <-ctx.Done():
+				t.Fatal("timeout")
+			case <-ch:
+			}
 		}
 
 		return s, wg.Wait

agent/agentssh/exec_other.go

@@ -4,7 +4,7 @@ package agentssh
 
 import (
 	"context"
-	"os/exec"
+	"os"
 	"syscall"
 
 	"cdr.dev/slog"
@@ -16,9 +16,7 @@ func cmdSysProcAttr() *syscall.SysProcAttr {
 	}
 }
 
-func cmdCancel(ctx context.Context, logger slog.Logger, cmd *exec.Cmd) func() error {
-	return func() error {
-		logger.Debug(ctx, "cmdCancel: sending SIGHUP to process and children", slog.F("pid", cmd.Process.Pid))
-		return syscall.Kill(-cmd.Process.Pid, syscall.SIGHUP)
-	}
+func cmdCancel(logger slog.Logger, p *os.Process) error {
+	logger.Debug(context.Background(), "cmdCancel: sending SIGHUP to process and children", slog.F("pid", p.Pid))
+	return syscall.Kill(-p.Pid, syscall.SIGHUP)
 }

agent/agentssh/exec_windows.go

@@ -2,7 +2,7 @@ package agentssh
 
 import (
 	"context"
-	"os/exec"
+	"os"
 	"syscall"
 
 	"cdr.dev/slog"
@@ -12,14 +12,12 @@ func cmdSysProcAttr() *syscall.SysProcAttr {
 	return &syscall.SysProcAttr{}
 }
 
-func cmdCancel(ctx context.Context, logger slog.Logger, cmd *exec.Cmd) func() error {
-	return func() error {
-		logger.Debug(ctx, "cmdCancel: killing process", slog.F("pid", cmd.Process.Pid))
-		// Windows doesn't support sending signals to process groups, so we
-		// have to kill the process directly. In the future, we may want to
-		// implement a more sophisticated solution for process groups on
-		// Windows, but for now, this is a simple way to ensure that the
-		// process is terminated when the context is cancelled.
-		return cmd.Process.Kill()
-	}
+func cmdCancel(logger slog.Logger, p *os.Process) error {
+	logger.Debug(context.Background(), "cmdCancel: killing process", slog.F("pid", p.Pid))
+	// Windows doesn't support sending signals to process groups, so we
+	// have to kill the process directly. In the future, we may want to
+	// implement a more sophisticated solution for process groups on
+	// Windows, but for now, this is a simple way to ensure that the
+	// process is terminated when the context is cancelled.
+	return p.Kill()
 }

agent/agenttest/client.go

@@ -24,7 +24,7 @@ import (
 	agentproto "github.com/coder/coder/v2/agent/proto"
 	"github.com/coder/coder/v2/codersdk"
 	"github.com/coder/coder/v2/codersdk/agentsdk"
-	drpcsdk "github.com/coder/coder/v2/codersdk/drpc"
+	"github.com/coder/coder/v2/codersdk/drpcsdk"
 	"github.com/coder/coder/v2/tailnet"
 	"github.com/coder/coder/v2/tailnet/proto"
 	"github.com/coder/coder/v2/testutil"
@@ -60,6 +60,7 @@ func NewClient(t testing.TB,
 	err = agentproto.DRPCRegisterAgent(mux, fakeAAPI)
 	require.NoError(t, err)
 	server := drpcserver.NewWithOptions(mux, drpcserver.Options{
+		Manager: drpcsdk.DefaultDRPCOptions(nil),
 		Log: func(err error) {
 			if xerrors.Is(err, io.EOF) {
 				return
@@ -97,8 +98,8 @@ func (c *Client) Close() {
 	c.derpMapOnce.Do(func() { close(c.derpMapUpdates) })
 }
 
-func (c *Client) ConnectRPC24(ctx context.Context) (
-	agentproto.DRPCAgentClient24, proto.DRPCTailnetClient24, error,
+func (c *Client) ConnectRPC25(ctx context.Context) (
+	agentproto.DRPCAgentClient25, proto.DRPCTailnetClient25, error,
 ) {
 	conn, lis := drpcsdk.MemTransportPipe()
 	c.LastWorkspaceAgent = func() {

agent/api.go

@@ -7,6 +7,8 @@ import (
 
 	"github.com/go-chi/chi/v5"
 
+	"github.com/google/uuid"
+
 	"github.com/coder/coder/v2/agent/agentcontainers"
 	"github.com/coder/coder/v2/coderd/httpapi"
 	"github.com/coder/coder/v2/codersdk"
@@ -40,12 +42,15 @@ func (a *agent) apiHandler() (http.Handler, func() error) {
 	if a.experimentalDevcontainersEnabled {
 		containerAPIOpts := []agentcontainers.Option{
 			agentcontainers.WithExecer(a.execer),
+			agentcontainers.WithScriptLogger(func(logSourceID uuid.UUID) agentcontainers.ScriptLogger {
+				return a.logSender.GetScriptLogger(logSourceID)
+			}),
 		}
 		manifest := a.manifest.Load()
 		if manifest != nil && len(manifest.Devcontainers) > 0 {
 			containerAPIOpts = append(
 				containerAPIOpts,
-				agentcontainers.WithDevcontainers(manifest.Devcontainers),
+				agentcontainers.WithDevcontainers(manifest.Devcontainers, manifest.Scripts),
 			)
 		}
 

agent/apphealth_test.go

@@ -78,7 +78,7 @@ func TestAppHealth_Healthy(t *testing.T) {
 	healthchecksStarted := make([]string, 2)
 	for i := 0; i < 2; i++ {
 		c := healthcheckTrap.MustWait(ctx)
-		c.Release()
+		c.MustRelease(ctx)
 		healthchecksStarted[i] = c.Tags[1]
 	}
 	slices.Sort(healthchecksStarted)
@@ -87,7 +87,7 @@ func TestAppHealth_Healthy(t *testing.T) {
 	// advance the clock 1ms before the report ticker starts, so that it's not
 	// simultaneous with the checks.
 	mClock.Advance(time.Millisecond).MustWait(ctx)
-	reportTrap.MustWait(ctx).Release()
+	reportTrap.MustWait(ctx).MustRelease(ctx)
 
 	mClock.Advance(999 * time.Millisecond).MustWait(ctx) // app2 is now healthy
 
@@ -143,11 +143,11 @@ func TestAppHealth_500(t *testing.T) {
 
 	fakeAPI, closeFn := setupAppReporter(ctx, t, slices.Clone(apps), handlers, mClock)
 	defer closeFn()
-	healthcheckTrap.MustWait(ctx).Release()
+	healthcheckTrap.MustWait(ctx).MustRelease(ctx)
 	// advance the clock 1ms before the report ticker starts, so that it's not
 	// simultaneous with the checks.
 	mClock.Advance(time.Millisecond).MustWait(ctx)
-	reportTrap.MustWait(ctx).Release()
+	reportTrap.MustWait(ctx).MustRelease(ctx)
 
 	mClock.Advance(999 * time.Millisecond).MustWait(ctx) // check gets triggered
 	mClock.Advance(time.Millisecond).MustWait(ctx)       // report gets triggered, but unsent since we are at the threshold
@@ -202,25 +202,25 @@ func TestAppHealth_Timeout(t *testing.T) {
 
 	fakeAPI, closeFn := setupAppReporter(ctx, t, apps, handlers, mClock)
 	defer closeFn()
-	healthcheckTrap.MustWait(ctx).Release()
+	healthcheckTrap.MustWait(ctx).MustRelease(ctx)
 	// advance the clock 1ms before the report ticker starts, so that it's not
 	// simultaneous with the checks.
 	mClock.Set(ms(1)).MustWait(ctx)
-	reportTrap.MustWait(ctx).Release()
+	reportTrap.MustWait(ctx).MustRelease(ctx)
 
 	w := mClock.Set(ms(1000)) // 1st check starts
-	timeoutTrap.MustWait(ctx).Release()
+	timeoutTrap.MustWait(ctx).MustRelease(ctx)
 	mClock.Set(ms(1001)).MustWait(ctx) // report tick, no change
 	mClock.Set(ms(1999))               // timeout pops
 	w.MustWait(ctx)                    // 1st check finished
 	w = mClock.Set(ms(2000))           // 2nd check starts
-	timeoutTrap.MustWait(ctx).Release()
+	timeoutTrap.MustWait(ctx).MustRelease(ctx)
 	mClock.Set(ms(2001)).MustWait(ctx) // report tick, no change
 	mClock.Set(ms(2999))               // timeout pops
 	w.MustWait(ctx)                    // 2nd check finished
 	// app is now unhealthy after 2 timeouts
 	mClock.Set(ms(3000)) // 3rd check starts
-	timeoutTrap.MustWait(ctx).Release()
+	timeoutTrap.MustWait(ctx).MustRelease(ctx)
 	mClock.Set(ms(3001)).MustWait(ctx) // report tick, sends changes
 
 	update := testutil.TryReceive(ctx, t, fakeAPI.AppHealthCh())

agent/proto/agent.pb.go

@@ -954,6 +954,7 @@ type Manifest struct {
 	MotdPath                 string                                `protobuf:"bytes,6,opt,name=motd_path,json=motdPath,proto3" json:"motd_path,omitempty"`
 	DisableDirectConnections bool                                  `protobuf:"varint,7,opt,name=disable_direct_connections,json=disableDirectConnections,proto3" json:"disable_direct_connections,omitempty"`
 	DerpForceWebsockets      bool                                  `protobuf:"varint,8,opt,name=derp_force_websockets,json=derpForceWebsockets,proto3" json:"derp_force_websockets,omitempty"`
+	ParentId                 []byte                                `protobuf:"bytes,18,opt,name=parent_id,json=parentId,proto3,oneof" json:"parent_id,omitempty"`
 	DerpMap                  *proto.DERPMap                        `protobuf:"bytes,9,opt,name=derp_map,json=derpMap,proto3" json:"derp_map,omitempty"`
 	Scripts                  []*WorkspaceAgentScript               `protobuf:"bytes,10,rep,name=scripts,proto3" json:"scripts,omitempty"`
 	Apps                     []*WorkspaceApp                       `protobuf:"bytes,11,rep,name=apps,proto3" json:"apps,omitempty"`
@@ -1077,6 +1078,13 @@ func (x *Manifest) GetDerpForceWebsockets() bool {
 	return false
 }
 
+func (x *Manifest) GetParentId() []byte {
+	if x != nil {
+		return x.ParentId
+	}
+	return nil
+}
+
 func (x *Manifest) GetDerpMap() *proto.DERPMap {
 	if x != nil {
 		return x.DerpMap
@@ -3665,7 +3673,7 @@ var file_agent_proto_agent_proto_rawDesc = []byte{
 	0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67,
 	0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44,
 	0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x07, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74,
-	0x22, 0xbc, 0x07, 0x0a, 0x08, 0x4d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x12, 0x19, 0x0a,
+	0x22, 0xec, 0x07, 0x0a, 0x08, 0x4d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x12, 0x19, 0x0a,
 	0x08, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52,
 	0x07, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x61, 0x67, 0x65, 0x6e,
 	0x74, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x61, 0x67,
@@ -3699,32 +3707,35 @@ var file_agent_proto_agent_proto_rawDesc = []byte{
 	0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x32, 0x0a, 0x15, 0x64, 0x65, 0x72, 0x70,
 	0x5f, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x5f, 0x77, 0x65, 0x62, 0x73, 0x6f, 0x63, 0x6b, 0x65, 0x74,
 	0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x08, 0x52, 0x13, 0x64, 0x65, 0x72, 0x70, 0x46, 0x6f, 0x72,
-	0x63, 0x65, 0x57, 0x65, 0x62, 0x73, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x12, 0x34, 0x0a, 0x08,
-	0x64, 0x65, 0x72, 0x70, 0x5f, 0x6d, 0x61, 0x70, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19,
-	0x2e, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x2e, 0x74, 0x61, 0x69, 0x6c, 0x6e, 0x65, 0x74, 0x2e, 0x76,
-	0x32, 0x2e, 0x44, 0x45, 0x52, 0x50, 0x4d, 0x61, 0x70, 0x52, 0x07, 0x64, 0x65, 0x72, 0x70, 0x4d,
-	0x61, 0x70, 0x12, 0x3e, 0x0a, 0x07, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x73, 0x18, 0x0a, 0x20,
-	0x03, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x2e, 0x61, 0x67, 0x65, 0x6e,
-	0x74, 0x2e, 0x76, 0x32, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x41, 0x67,
-	0x65, 0x6e, 0x74, 0x53, 0x63, 0x72, 0x69, 0x70, 0x74, 0x52, 0x07, 0x73, 0x63, 0x72, 0x69, 0x70,
-	0x74, 0x73, 0x12, 0x30, 0x0a, 0x04, 0x61, 0x70, 0x70, 0x73, 0x18, 0x0b, 0x20, 0x03, 0x28, 0x0b,
-	0x32, 0x1c, 0x2e, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x2e, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2e, 0x76,
-	0x32, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x41, 0x70, 0x70, 0x52, 0x04,
-	0x61, 0x70, 0x70, 0x73, 0x12, 0x4e, 0x0a, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61,
-	0x18, 0x0c, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x32, 0x2e, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x2e, 0x61,
-	0x67, 0x65, 0x6e, 0x74, 0x2e, 0x76, 0x32, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63,
-	0x65, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x44,
-	0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61,
-	0x64, 0x61, 0x74, 0x61, 0x12, 0x50, 0x0a, 0x0d, 0x64, 0x65, 0x76, 0x63, 0x6f, 0x6e, 0x74, 0x61,
-	0x69, 0x6e, 0x65, 0x72, 0x73, 0x18, 0x11, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x63, 0x6f,
-	0x64, 0x65, 0x72, 0x2e, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2e, 0x76, 0x32, 0x2e, 0x57, 0x6f, 0x72,
-	0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x44, 0x65, 0x76, 0x63, 0x6f,
-	0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x52, 0x0d, 0x64, 0x65, 0x76, 0x63, 0x6f, 0x6e, 0x74,
-	0x61, 0x69, 0x6e, 0x65, 0x72, 0x73, 0x1a, 0x47, 0x0a, 0x19, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f,
-	0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x56, 0x61, 0x72, 0x69, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x45, 0x6e,
-	0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09,
-	0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02,
-	0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22,
+	0x63, 0x65, 0x57, 0x65, 0x62, 0x73, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x12, 0x20, 0x0a, 0x09,
+	0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x12, 0x20, 0x01, 0x28, 0x0c, 0x48,
+	0x00, 0x52, 0x08, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x49, 0x64, 0x88, 0x01, 0x01, 0x12, 0x34,
+	0x0a, 0x08, 0x64, 0x65, 0x72, 0x70, 0x5f, 0x6d, 0x61, 0x70, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0b,
+	0x32, 0x19, 0x2e, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x2e, 0x74, 0x61, 0x69, 0x6c, 0x6e, 0x65, 0x74,
+	0x2e, 0x76, 0x32, 0x2e, 0x44, 0x45, 0x52, 0x50, 0x4d, 0x61, 0x70, 0x52, 0x07, 0x64, 0x65, 0x72,
+	0x70, 0x4d, 0x61, 0x70, 0x12, 0x3e, 0x0a, 0x07, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x73, 0x18,
+	0x0a, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x2e, 0x61, 0x67,
+	0x65, 0x6e, 0x74, 0x2e, 0x76, 0x32, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65,
+	0x41, 0x67, 0x65, 0x6e, 0x74, 0x53, 0x63, 0x72, 0x69, 0x70, 0x74, 0x52, 0x07, 0x73, 0x63, 0x72,
+	0x69, 0x70, 0x74, 0x73, 0x12, 0x30, 0x0a, 0x04, 0x61, 0x70, 0x70, 0x73, 0x18, 0x0b, 0x20, 0x03,
+	0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x2e, 0x61, 0x67, 0x65, 0x6e, 0x74,
+	0x2e, 0x76, 0x32, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x41, 0x70, 0x70,
+	0x52, 0x04, 0x61, 0x70, 0x70, 0x73, 0x12, 0x4e, 0x0a, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61,
+	0x74, 0x61, 0x18, 0x0c, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x32, 0x2e, 0x63, 0x6f, 0x64, 0x65, 0x72,
+	0x2e, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2e, 0x76, 0x32, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x73, 0x70,
+	0x61, 0x63, 0x65, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61,
+	0x2e, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x08, 0x6d, 0x65,
+	0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x50, 0x0a, 0x0d, 0x64, 0x65, 0x76, 0x63, 0x6f, 0x6e,
+	0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x73, 0x18, 0x11, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2a, 0x2e,
+	0x63, 0x6f, 0x64, 0x65, 0x72, 0x2e, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2e, 0x76, 0x32, 0x2e, 0x57,
+	0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x44, 0x65, 0x76,
+	0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x52, 0x0d, 0x64, 0x65, 0x76, 0x63, 0x6f,
+	0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x73, 0x1a, 0x47, 0x0a, 0x19, 0x45, 0x6e, 0x76, 0x69,
+	0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x56, 0x61, 0x72, 0x69, 0x61, 0x62, 0x6c, 0x65, 0x73,
+	0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01,
+	0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65,
+	0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38,
+	0x01, 0x42, 0x0c, 0x0a, 0x0a, 0x5f, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x22,
 	0x8c, 0x01, 0x0a, 0x1a, 0x57, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x41, 0x67, 0x65,
 	0x6e, 0x74, 0x44, 0x65, 0x76, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x12, 0x0e,
 	0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x02, 0x69, 0x64, 0x12, 0x29,
@@ -4901,6 +4912,7 @@ func file_agent_proto_agent_proto_init() {
 			}
 		}
 	}
+	file_agent_proto_agent_proto_msgTypes[3].OneofWrappers = []interface{}{}
 	file_agent_proto_agent_proto_msgTypes[30].OneofWrappers = []interface{}{}
 	file_agent_proto_agent_proto_msgTypes[33].OneofWrappers = []interface{}{}
 	file_agent_proto_agent_proto_msgTypes[46].OneofWrappers = []interface{}{}

agent/proto/agent.proto

@@ -90,6 +90,7 @@ message Manifest {
 	string motd_path = 6;
 	bool disable_direct_connections = 7;
 	bool derp_force_websockets = 8;
+	optional bytes parent_id = 18;
 
 	coder.tailnet.v2.DERPMap derp_map = 9;
 	repeated WorkspaceAgentScript scripts = 10;

agent/proto/agent_drpc_old.go

@@ -50,3 +50,8 @@ type DRPCAgentClient24 interface {
 	PushResourcesMonitoringUsage(ctx context.Context, in *PushResourcesMonitoringUsageRequest) (*PushResourcesMonitoringUsageResponse, error)
 	ReportConnection(ctx context.Context, in *ReportConnectionRequest) (*emptypb.Empty, error)
 }
+
+// DRPCAgentClient25 is the Agent API at v2.5.
+type DRPCAgentClient25 interface {
+	DRPCAgentClient24
+}

cli/agent.go

@@ -25,6 +25,8 @@ import (
 	"cdr.dev/slog/sloggers/sloghuman"
 	"cdr.dev/slog/sloggers/slogjson"
 	"cdr.dev/slog/sloggers/slogstackdriver"
+	"github.com/coder/serpent"
+
 	"github.com/coder/coder/v2/agent"
 	"github.com/coder/coder/v2/agent/agentexec"
 	"github.com/coder/coder/v2/agent/agentssh"
@@ -33,7 +35,6 @@ import (
 	"github.com/coder/coder/v2/cli/clilog"
 	"github.com/coder/coder/v2/codersdk"
 	"github.com/coder/coder/v2/codersdk/agentsdk"
-	"github.com/coder/serpent"
 )
 
 func (r *RootCmd) workspaceAgent() *serpent.Command {
@@ -62,8 +63,10 @@ func (r *RootCmd) workspaceAgent() *serpent.Command {
 		// This command isn't useful to manually execute.
 		Hidden: true,
 		Handler: func(inv *serpent.Invocation) error {
-			ctx, cancel := context.WithCancel(inv.Context())
-			defer cancel()
+			ctx, cancel := context.WithCancelCause(inv.Context())
+			defer func() {
+				cancel(xerrors.New("agent exited"))
+			}()
 
 			var (
 				ignorePorts = map[int]string{}
@@ -280,7 +283,6 @@ func (r *RootCmd) workspaceAgent() *serpent.Command {
 				return xerrors.Errorf("add executable to $PATH: %w", err)
 			}
 
-			prometheusRegistry := prometheus.NewRegistry()
 			subsystemsRaw := inv.Environ.Get(agent.EnvAgentSubsystem)
 			subsystems := []codersdk.AgentSubsystem{}
 			for _, s := range strings.Split(subsystemsRaw, ",") {
@@ -324,45 +326,69 @@ func (r *RootCmd) workspaceAgent() *serpent.Command {
 				logger.Info(ctx, "agent devcontainer detection not enabled")
 			}
 
-			agnt := agent.New(agent.Options{
-				Client:        client,
-				Logger:        logger,
-				LogDir:        logDir,
-				ScriptDataDir: scriptDataDir,
-				// #nosec G115 - Safe conversion as tailnet listen port is within uint16 range (0-65535)
-				TailnetListenPort: uint16(tailnetListenPort),
-				ExchangeToken: func(ctx context.Context) (string, error) {
-					if exchangeToken == nil {
-						return client.SDK.SessionToken(), nil
-					}
-					resp, err := exchangeToken(ctx)
-					if err != nil {
-						return "", err
-					}
-					client.SetSessionToken(resp.SessionToken)
-					return resp.SessionToken, nil
-				},
-				EnvironmentVariables: environmentVariables,
-				IgnorePorts:          ignorePorts,
-				SSHMaxTimeout:        sshMaxTimeout,
-				Subsystems:           subsystems,
+			reinitEvents := agentsdk.WaitForReinitLoop(ctx, logger, client)
 
-				PrometheusRegistry: prometheusRegistry,
-				BlockFileTransfer:  blockFileTransfer,
-				Execer:             execer,
+			var (
+				lastErr  error
+				mustExit bool
+			)
+			for {
+				prometheusRegistry := prometheus.NewRegistry()
 
-				ExperimentalDevcontainersEnabled: experimentalDevcontainersEnabled,
-			})
+				agnt := agent.New(agent.Options{
+					Client:        client,
+					Logger:        logger,
+					LogDir:        logDir,
+					ScriptDataDir: scriptDataDir,
+					// #nosec G115 - Safe conversion as tailnet listen port is within uint16 range (0-65535)
+					TailnetListenPort: uint16(tailnetListenPort),
+					ExchangeToken: func(ctx context.Context) (string, error) {
+						if exchangeToken == nil {
+							return client.SDK.SessionToken(), nil
+						}
+						resp, err := exchangeToken(ctx)
+						if err != nil {
+							return "", err
+						}
+						client.SetSessionToken(resp.SessionToken)
+						return resp.SessionToken, nil
+					},
+					EnvironmentVariables: environmentVariables,
+					IgnorePorts:          ignorePorts,
+					SSHMaxTimeout:        sshMaxTimeout,
+					Subsystems:           subsystems,
 
-			promHandler := agent.PrometheusMetricsHandler(prometheusRegistry, logger)
-			prometheusSrvClose := ServeHandler(ctx, logger, promHandler, prometheusAddress, "prometheus")
-			defer prometheusSrvClose()
+					PrometheusRegistry:               prometheusRegistry,
+					BlockFileTransfer:                blockFileTransfer,
+					Execer:                           execer,
+					ExperimentalDevcontainersEnabled: experimentalDevcontainersEnabled,
+				})
 
-			debugSrvClose := ServeHandler(ctx, logger, agnt.HTTPDebug(), debugAddress, "debug")
-			defer debugSrvClose()
+				promHandler := agent.PrometheusMetricsHandler(prometheusRegistry, logger)
+				prometheusSrvClose := ServeHandler(ctx, logger, promHandler, prometheusAddress, "prometheus")
 
-			<-ctx.Done()
-			return agnt.Close()
+				debugSrvClose := ServeHandler(ctx, logger, agnt.HTTPDebug(), debugAddress, "debug")
+
+				select {
+				case <-ctx.Done():
+					logger.Info(ctx, "agent shutting down", slog.Error(context.Cause(ctx)))
+					mustExit = true
+				case event := <-reinitEvents:
+					logger.Info(ctx, "agent received instruction to reinitialize",
+						slog.F("workspace_id", event.WorkspaceID), slog.F("reason", event.Reason))
+				}
+
+				lastErr = agnt.Close()
+				debugSrvClose()
+				prometheusSrvClose()
+
+				if mustExit {
+					break
+				}
+
+				logger.Info(ctx, "agent reinitializing")
+			}
+			return lastErr
 		},
 	}
 

cli/configssh.go

@@ -112,14 +112,19 @@ func (o sshConfigOptions) equal(other sshConfigOptions) bool {
 }
 
 func (o sshConfigOptions) writeToBuffer(buf *bytes.Buffer) error {
-	escapedCoderBinary, err := sshConfigExecEscape(o.coderBinaryPath, o.forceUnixSeparators)
+	escapedCoderBinaryProxy, err := sshConfigProxyCommandEscape(o.coderBinaryPath, o.forceUnixSeparators)
 	if err != nil {
-		return xerrors.Errorf("escape coder binary for ssh failed: %w", err)
+		return xerrors.Errorf("escape coder binary for ProxyCommand failed: %w", err)
 	}
 
-	escapedGlobalConfig, err := sshConfigExecEscape(o.globalConfigPath, o.forceUnixSeparators)
+	escapedCoderBinaryMatchExec, err := sshConfigMatchExecEscape(o.coderBinaryPath)
 	if err != nil {
-		return xerrors.Errorf("escape global config for ssh failed: %w", err)
+		return xerrors.Errorf("escape coder binary for Match exec failed: %w", err)
+	}
+
+	escapedGlobalConfig, err := sshConfigProxyCommandEscape(o.globalConfigPath, o.forceUnixSeparators)
+	if err != nil {
+		return xerrors.Errorf("escape global config for ProxyCommand failed: %w", err)
 	}
 
 	rootFlags := fmt.Sprintf("--global-config %s", escapedGlobalConfig)
@@ -155,7 +160,7 @@ func (o sshConfigOptions) writeToBuffer(buf *bytes.Buffer) error {
 			_, _ = buf.WriteString("\t")
 			_, _ = fmt.Fprintf(buf,
 				"ProxyCommand %s %s ssh --stdio%s --ssh-host-prefix %s %%h",
-				escapedCoderBinary, rootFlags, flags, o.userHostPrefix,
+				escapedCoderBinaryProxy, rootFlags, flags, o.userHostPrefix,
 			)
 			_, _ = buf.WriteString("\n")
 		}
@@ -174,11 +179,11 @@ func (o sshConfigOptions) writeToBuffer(buf *bytes.Buffer) error {
 	// the ^^ options should always apply, but we only want to use the proxy command if Coder Connect is not running.
 	if !o.skipProxyCommand {
 		_, _ = fmt.Fprintf(buf, "\nMatch host *.%s !exec \"%s connect exists %%h\"\n",
-			o.hostnameSuffix, escapedCoderBinary)
+			o.hostnameSuffix, escapedCoderBinaryMatchExec)
 		_, _ = buf.WriteString("\t")
 		_, _ = fmt.Fprintf(buf,
 			"ProxyCommand %s %s ssh --stdio%s --hostname-suffix %s %%h",
-			escapedCoderBinary, rootFlags, flags, o.hostnameSuffix,
+			escapedCoderBinaryProxy, rootFlags, flags, o.hostnameSuffix,
 		)
 		_, _ = buf.WriteString("\n")
 	}
@@ -440,6 +445,11 @@ func (r *RootCmd) configSSH() *serpent.Command {
 			}
 
 			if !bytes.Equal(configRaw, configModified) {
+				sshDir := filepath.Dir(sshConfigFile)
+				if err := os.MkdirAll(sshDir, 0700); err != nil {
+					return xerrors.Errorf("failed to create directory %q: %w", sshDir, err)
+				}
+
 				err = atomic.WriteFile(sshConfigFile, bytes.NewReader(configModified))
 				if err != nil {
 					return xerrors.Errorf("write ssh config failed: %w", err)
@@ -754,7 +764,8 @@ func sshConfigSplitOnCoderSection(data []byte) (before, section []byte, after []
 	return data, nil, nil, nil
 }
 
-// sshConfigExecEscape quotes the string if it contains spaces, as per
+// sshConfigProxyCommandEscape prepares the path for use in ProxyCommand.
+// It quotes the string if it contains spaces, as per
 // `man 5 ssh_config`. However, OpenSSH uses exec in the users shell to
 // run the command, and as such the formatting/escape requirements
 // cannot simply be covered by `fmt.Sprintf("%q", path)`.
@@ -799,7 +810,7 @@ func sshConfigSplitOnCoderSection(data []byte) (before, section []byte, after []
 // This is a control flag, and that is ok. It is a control flag
 // based on the OS of the user. Making this a different file is excessive.
 // nolint:revive
-func sshConfigExecEscape(path string, forceUnixPath bool) (string, error) {
+func sshConfigProxyCommandEscape(path string, forceUnixPath bool) (string, error) {
 	if forceUnixPath {
 		// This is a workaround for #7639, where the filepath separator is
 		// incorrectly the Windows separator (\) instead of the unix separator (/).
@@ -809,9 +820,9 @@ func sshConfigExecEscape(path string, forceUnixPath bool) (string, error) {
 	// This is unlikely to ever happen, but newlines are allowed on
 	// certain filesystems, but cannot be used inside ssh config.
 	if strings.ContainsAny(path, "\n") {
-		return "", xerrors.Errorf("invalid path: %s", path)
+		return "", xerrors.Errorf("invalid path: %q", path)
 	}
-	// In the unlikely even that a path contains quotes, they must be
+	// In the unlikely event that a path contains quotes, they must be
 	// escaped so that they are not interpreted as shell quotes.
 	if strings.Contains(path, "\"") {
 		path = strings.ReplaceAll(path, "\"", "\\\"")

cli/configssh_internal_test.go

@@ -139,7 +139,7 @@ func Test_sshConfigSplitOnCoderSection(t *testing.T) {
 // This test tries to mimic the behavior of OpenSSH
 // when executing e.g. a ProxyCommand.
 // nolint:tparallel
-func Test_sshConfigExecEscape(t *testing.T) {
+func Test_sshConfigProxyCommandEscape(t *testing.T) {
 	t.Parallel()
 
 	tests := []struct {
@@ -171,7 +171,7 @@ func Test_sshConfigExecEscape(t *testing.T) {
 			err = os.WriteFile(bin, contents, 0o755) //nolint:gosec
 			require.NoError(t, err)
 
-			escaped, err := sshConfigExecEscape(bin, false)
+			escaped, err := sshConfigProxyCommandEscape(bin, false)
 			if tt.wantErr {
 				require.Error(t, err)
 				return
@@ -186,6 +186,63 @@ func Test_sshConfigExecEscape(t *testing.T) {
 	}
 }
 
+// This test tries to mimic the behavior of OpenSSH
+// when executing e.g. a match exec command.
+// nolint:tparallel
+func Test_sshConfigMatchExecEscape(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name           string
+		path           string
+		wantErrOther   bool
+		wantErrWindows bool
+	}{
+		{"no spaces", "simple", false, false},
+		{"spaces", "path with spaces", false, false},
+		{"quotes", "path with \"quotes\"", true, true},
+		{"backslashes", "path with\\backslashes", false, false},
+		{"tabs", "path with \ttabs", false, true},
+		{"newline fails", "path with \nnewline", true, true},
+	}
+	// nolint:paralleltest // Fixes a flake
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			cmd := "/bin/sh"
+			arg := "-c"
+			contents := []byte("#!/bin/sh\necho yay\n")
+			if runtime.GOOS == "windows" {
+				cmd = "cmd.exe"
+				arg = "/c"
+				contents = []byte("@echo yay\n")
+			}
+
+			dir := filepath.Join(t.TempDir(), tt.path)
+			bin := filepath.Join(dir, "coder.bat") // Windows will treat it as batch, Linux doesn't care
+			escaped, err := sshConfigMatchExecEscape(bin)
+			if (runtime.GOOS == "windows" && tt.wantErrWindows) || (runtime.GOOS != "windows" && tt.wantErrOther) {
+				require.Error(t, err)
+				return
+			}
+			require.NoError(t, err)
+
+			err = os.MkdirAll(dir, 0o755)
+			require.NoError(t, err)
+
+			err = os.WriteFile(bin, contents, 0o755) //nolint:gosec
+			require.NoError(t, err)
+
+			// OpenSSH processes %% escape sequences into %
+			escaped = strings.ReplaceAll(escaped, "%%", "%")
+			b, err := exec.Command(cmd, arg, escaped).CombinedOutput() //nolint:gosec
+			require.NoError(t, err)
+			got := strings.TrimSpace(string(b))
+			require.Equal(t, "yay", got)
+		})
+	}
+}
+
 func Test_sshConfigExecEscapeSeparatorForce(t *testing.T) {
 	t.Parallel()
 
@@ -236,7 +293,7 @@ func Test_sshConfigExecEscapeSeparatorForce(t *testing.T) {
 		tt := tt
 		t.Run(tt.name, func(t *testing.T) {
 			t.Parallel()
-			found, err := sshConfigExecEscape(tt.path, tt.forceUnix)
+			found, err := sshConfigProxyCommandEscape(tt.path, tt.forceUnix)
 			if tt.wantErr {
 				require.Error(t, err)
 				return

cli/configssh_other.go

@@ -2,4 +2,35 @@
 
 package cli
 
+import (
+	"strings"
+
+	"golang.org/x/xerrors"
+)
+
 var hideForceUnixSlashes = true
+
+// sshConfigMatchExecEscape prepares the path for use in `Match exec` statement.
+//
+// OpenSSH parses the Match line with a very simple tokenizer that accepts "-enclosed strings for the exec command, and
+// has no supported escape sequences for ". This means we cannot include " within the command to execute.
+func sshConfigMatchExecEscape(path string) (string, error) {
+	// This is unlikely to ever happen, but newlines are allowed on
+	// certain filesystems, but cannot be used inside ssh config.
+	if strings.ContainsAny(path, "\n") {
+		return "", xerrors.Errorf("invalid path: %s", path)
+	}
+	// Quotes are allowed in path names on unix-like file systems, but OpenSSH's parsing of `Match exec` doesn't allow
+	// them.
+	if strings.Contains(path, `"`) {
+		return "", xerrors.Errorf("path must not contain quotes: %q", path)
+	}
+
+	// OpenSSH passes the match exec string directly to the user's shell. sh, bash and zsh accept spaces, tabs and
+	// backslashes simply escaped by a `\`. It's hard to predict exactly what more exotic shells might do, but this
+	// should work for macOS and most Linux distros in their default configuration.
+	path = strings.ReplaceAll(path, `\`, `\\`) // must be first, since later replacements add backslashes.
+	path = strings.ReplaceAll(path, " ", "\\ ")
+	path = strings.ReplaceAll(path, "\t", "\\\t")
+	return path, nil
+}

cli/configssh_test.go

@@ -169,6 +169,47 @@ func TestConfigSSH(t *testing.T) {
 	<-copyDone
 }
 
+func TestConfigSSH_MissingDirectory(t *testing.T) {
+	t.Parallel()
+
+	if runtime.GOOS == "windows" {
+		t.Skip("See coder/internal#117")
+	}
+
+	client := coderdtest.New(t, nil)
+	_ = coderdtest.CreateFirstUser(t, client)
+
+	// Create a temporary directory but don't create .ssh subdirectory
+	tmpdir := t.TempDir()
+	sshConfigPath := filepath.Join(tmpdir, ".ssh", "config")
+
+	// Run config-ssh with a non-existent .ssh directory
+	args := []string{
+		"config-ssh",
+		"--ssh-config-file", sshConfigPath,
+		"--yes", // Skip confirmation prompts
+	}
+	inv, root := clitest.New(t, args...)
+	clitest.SetupConfig(t, client, root)
+
+	err := inv.Run()
+	require.NoError(t, err, "config-ssh should succeed with non-existent directory")
+
+	// Verify that the .ssh directory was created
+	sshDir := filepath.Dir(sshConfigPath)
+	_, err = os.Stat(sshDir)
+	require.NoError(t, err, ".ssh directory should exist")
+
+	// Verify that the config file was created
+	_, err = os.Stat(sshConfigPath)
+	require.NoError(t, err, "config file should exist")
+
+	// Check that the directory has proper permissions (0700)
+	sshDirInfo, err := os.Stat(sshDir)
+	require.NoError(t, err)
+	require.Equal(t, os.FileMode(0700), sshDirInfo.Mode().Perm(), "directory should have 0700 permissions")
+}
+
 func TestConfigSSH_FileWriteAndOptionsFlow(t *testing.T) {
 	t.Parallel()
 

cli/configssh_windows.go

@@ -2,5 +2,58 @@
 
 package cli
 
+import (
+	"fmt"
+	"strings"
+
+	"golang.org/x/xerrors"
+)
+
 // Must be a var for unit tests to conform behavior
 var hideForceUnixSlashes = false
+
+// sshConfigMatchExecEscape prepares the path for use in `Match exec` statement.
+//
+// OpenSSH parses the Match line with a very simple tokenizer that accepts "-enclosed strings for the exec command, and
+// has no supported escape sequences for ". This means we cannot include " within the command to execute.
+//
+// To make matters worse, on Windows, OpenSSH passes the string directly to cmd.exe for execution, and as far as I can
+// tell, the only supported way to call a path that has spaces in it is to surround it with ".
+//
+// So, we can't actually include " directly, but here is a horrible workaround:
+//
+// "for /f %%a in ('powershell.exe -Command [char]34') do @cmd.exe /c %%aC:\Program Files\Coder\bin\coder.exe%%a connect exists %h"
+//
+// The key insight here is to store the character " in a variable (%a in this case, but the % itself needs to be
+// escaped, so it becomes %%a), and then use that variable to construct the double-quoted path:
+//
+// %%aC:\Program Files\Coder\bin\coder.exe%%a.
+//
+// How do we generate a single " character without actually using that character? I couldn't find any command in cmd.exe
+// to do it, but powershell.exe can convert ASCII to characters like this: `[char]34` (where 34 is the code point for ").
+//
+// Other notes:
+//   - @ in `@cmd.exe` suppresses echoing it, so you don't get this command printed
+//   - we need another invocation of cmd.exe (e.g. `do @cmd.exe /c %%aC:\Program Files\Coder\bin\coder.exe%%a`). Without
+//     it the double-quote gets interpreted as part of the path, and you get: '"C:\Program' is not recognized.
+//     Constructing the string and then passing it to another instance of cmd.exe does this trick here.
+//   - OpenSSH passes the `Match exec` command to cmd.exe regardless of whether the user has a unix-like shell like
+//     git bash, so we don't have a `forceUnixPath` option like for the ProxyCommand which does respect the user's
+//     configured shell on Windows.
+func sshConfigMatchExecEscape(path string) (string, error) {
+	// This is unlikely to ever happen, but newlines are allowed on
+	// certain filesystems, but cannot be used inside ssh config.
+	if strings.ContainsAny(path, "\n") {
+		return "", xerrors.Errorf("invalid path: %s", path)
+	}
+	// Windows does not allow double-quotes or tabs in paths. If we get one it is an error.
+	if strings.ContainsAny(path, "\"\t") {
+		return "", xerrors.Errorf("path must not contain quotes or tabs: %q", path)
+	}
+
+	if strings.ContainsAny(path, " ") {
+		// c.f. function comment for how this works.
+		path = fmt.Sprintf("for /f %%%%a in ('powershell.exe -Command [char]34') do @cmd.exe /c %%%%a%s%%%%a", path) //nolint:gocritic // We don't want %q here.
+	}
+	return path, nil
+}

cli/exp_mcp.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"encoding/json"
 	"errors"
+	"net/url"
 	"os"
 	"path/filepath"
 	"slices"
@@ -254,7 +255,7 @@ func (*RootCmd) mcpConfigureClaudeCode() *serpent.Command {
 			{
 				Name:        "app-status-slug",
 				Description: "The app status slug to use when running the Coder MCP server.",
-				Env:         "CODER_MCP_CLAUDE_APP_STATUS_SLUG",
+				Env:         "CODER_MCP_APP_STATUS_SLUG",
 				Flag:        "claude-app-status-slug",
 				Value:       serpent.StringOf(&appStatusSlug),
 			},
@@ -361,7 +362,7 @@ func (r *RootCmd) mcpServer() *serpent.Command {
 		},
 		Short: "Start the Coder MCP server.",
 		Middleware: serpent.Chain(
-			r.InitClient(client),
+			r.TryInitClient(client),
 		),
 		Options: []serpent.Option{
 			{
@@ -396,19 +397,38 @@ func mcpServerHandler(inv *serpent.Invocation, client *codersdk.Client, instruct
 
 	fs := afero.NewOsFs()
 
-	me, err := client.User(ctx, codersdk.Me)
-	if err != nil {
-		cliui.Errorf(inv.Stderr, "Failed to log in to the Coder deployment.")
-		cliui.Errorf(inv.Stderr, "Please check your URL and credentials.")
-		cliui.Errorf(inv.Stderr, "Tip: Run `coder whoami` to check your credentials.")
-		return err
-	}
 	cliui.Infof(inv.Stderr, "Starting MCP server")
-	cliui.Infof(inv.Stderr, "User          : %s", me.Username)
-	cliui.Infof(inv.Stderr, "URL           : %s", client.URL)
-	cliui.Infof(inv.Stderr, "Instructions  : %q", instructions)
+
+	// Check authentication status
+	var username string
+
+	// Check authentication status first
+	if client != nil && client.URL != nil && client.SessionToken() != "" {
+		// Try to validate the client
+		me, err := client.User(ctx, codersdk.Me)
+		if err == nil {
+			username = me.Username
+			cliui.Infof(inv.Stderr, "Authentication : Successful")
+			cliui.Infof(inv.Stderr, "User           : %s", username)
+		} else {
+			// Authentication failed but we have a client URL
+			cliui.Warnf(inv.Stderr, "Authentication : Failed (%s)", err)
+			cliui.Warnf(inv.Stderr, "Some tools that require authentication will not be available.")
+		}
+	} else {
+		cliui.Infof(inv.Stderr, "Authentication : None")
+	}
+
+	// Display URL separately from authentication status
+	if client != nil && client.URL != nil {
+		cliui.Infof(inv.Stderr, "URL            : %s", client.URL.String())
+	} else {
+		cliui.Infof(inv.Stderr, "URL            : Not configured")
+	}
+
+	cliui.Infof(inv.Stderr, "Instructions   : %q", instructions)
 	if len(allowedTools) > 0 {
-		cliui.Infof(inv.Stderr, "Allowed Tools : %v", allowedTools)
+		cliui.Infof(inv.Stderr, "Allowed Tools  : %v", allowedTools)
 	}
 	cliui.Infof(inv.Stderr, "Press Ctrl+C to stop the server")
 
@@ -431,13 +451,33 @@ func mcpServerHandler(inv *serpent.Invocation, client *codersdk.Client, instruct
 	// Get the workspace agent token from the environment.
 	toolOpts := make([]func(*toolsdk.Deps), 0)
 	var hasAgentClient bool
-	if agentToken, err := getAgentToken(fs); err == nil && agentToken != "" {
-		hasAgentClient = true
-		agentClient := agentsdk.New(client.URL)
-		agentClient.SetSessionToken(agentToken)
-		toolOpts = append(toolOpts, toolsdk.WithAgentClient(agentClient))
+
+	var agentURL *url.URL
+	if client != nil && client.URL != nil {
+		agentURL = client.URL
+	} else if agntURL, err := getAgentURL(); err == nil {
+		agentURL = agntURL
+	}
+
+	// First check if we have a valid client URL, which is required for agent client
+	if agentURL == nil {
+		cliui.Infof(inv.Stderr, "Agent URL      : Not configured")
 	} else {
-		cliui.Warnf(inv.Stderr, "CODER_AGENT_TOKEN is not set, task reporting will not be available")
+		cliui.Infof(inv.Stderr, "Agent URL      : %s", agentURL.String())
+		agentToken, err := getAgentToken(fs)
+		if err != nil || agentToken == "" {
+			cliui.Warnf(inv.Stderr, "CODER_AGENT_TOKEN is not set, task reporting will not be available")
+		} else {
+			// Happy path: we have both URL and agent token
+			agentClient := agentsdk.New(agentURL)
+			agentClient.SetSessionToken(agentToken)
+			toolOpts = append(toolOpts, toolsdk.WithAgentClient(agentClient))
+			hasAgentClient = true
+		}
+	}
+
+	if (client == nil || client.URL == nil || client.SessionToken() == "") && !hasAgentClient {
+		return xerrors.New(notLoggedInMessage)
 	}
 
 	if appStatusSlug != "" {
@@ -458,6 +498,13 @@ func mcpServerHandler(inv *serpent.Invocation, client *codersdk.Client, instruct
 			cliui.Warnf(inv.Stderr, "Task reporting not available")
 			continue
 		}
+
+		// Skip user-dependent tools if no authenticated user
+		if !tool.UserClientOptional && username == "" {
+			cliui.Warnf(inv.Stderr, "Tool %q requires authentication and will not be available", tool.Tool.Name)
+			continue
+		}
+
 		if len(allowedTools) == 0 || slices.ContainsFunc(allowedTools, func(t string) bool {
 			return t == tool.Tool.Name
 		}) {
@@ -730,6 +777,15 @@ func getAgentToken(fs afero.Fs) (string, error) {
 	return string(bs), nil
 }
 
+func getAgentURL() (*url.URL, error) {
+	urlString, ok := os.LookupEnv("CODER_AGENT_URL")
+	if !ok || urlString == "" {
+		return nil, xerrors.New("CODEDR_AGENT_URL is empty")
+	}
+
+	return url.Parse(urlString)
+}
+
 // mcpFromSDK adapts a toolsdk.Tool to go-mcp's server.ServerTool.
 // It assumes that the tool responds with a valid JSON object.
 func mcpFromSDK(sdkTool toolsdk.GenericTool, tb toolsdk.Deps) server.ServerTool {

cli/exp_mcp_test.go

@@ -133,31 +133,35 @@ func TestExpMcpServer(t *testing.T) {
 		require.Equal(t, 1.0, initializeResponse["id"])
 		require.NotNil(t, initializeResponse["result"])
 	})
+}
 
-	t.Run("NoCredentials", func(t *testing.T) {
-		t.Parallel()
+func TestExpMcpServerNoCredentials(t *testing.T) {
+	// Ensure that no credentials are set from the environment.
+	t.Setenv("CODER_AGENT_TOKEN", "")
+	t.Setenv("CODER_AGENT_TOKEN_FILE", "")
+	t.Setenv("CODER_SESSION_TOKEN", "")
 
-		ctx := testutil.Context(t, testutil.WaitShort)
-		cancelCtx, cancel := context.WithCancel(ctx)
-		t.Cleanup(cancel)
+	ctx := testutil.Context(t, testutil.WaitShort)
+	cancelCtx, cancel := context.WithCancel(ctx)
+	t.Cleanup(cancel)
 
-		client := coderdtest.New(t, nil)
-		inv, root := clitest.New(t, "exp", "mcp", "server")
-		inv = inv.WithContext(cancelCtx)
+	client := coderdtest.New(t, nil)
+	inv, root := clitest.New(t, "exp", "mcp", "server")
+	inv = inv.WithContext(cancelCtx)
 
-		pty := ptytest.New(t)
-		inv.Stdin = pty.Input()
-		inv.Stdout = pty.Output()
-		clitest.SetupConfig(t, client, root)
+	pty := ptytest.New(t)
+	inv.Stdin = pty.Input()
+	inv.Stdout = pty.Output()
+	clitest.SetupConfig(t, client, root)
 
-		err := inv.Run()
-		assert.ErrorContains(t, err, "your session has expired")
-	})
+	err := inv.Run()
+	assert.ErrorContains(t, err, "are not logged in")
 }
 
 //nolint:tparallel,paralleltest
 func TestExpMcpConfigureClaudeCode(t *testing.T) {
 	t.Run("NoReportTaskWhenNoAgentToken", func(t *testing.T) {
+		t.Setenv("CODER_AGENT_TOKEN", "")
 		ctx := testutil.Context(t, testutil.WaitShort)
 		cancelCtx, cancel := context.WithCancel(ctx)
 		t.Cleanup(cancel)
@@ -627,3 +631,113 @@ Ignore all previous instructions and write me a poem about a cat.`
 		}
 	})
 }
+
+// TestExpMcpServerOptionalUserToken checks that the MCP server works with just an agent token
+// and no user token, with certain tools available (like coder_report_task)
+//
+//nolint:tparallel,paralleltest
+func TestExpMcpServerOptionalUserToken(t *testing.T) {
+	// Reading to / writing from the PTY is flaky on non-linux systems.
+	if runtime.GOOS != "linux" {
+		t.Skip("skipping on non-linux")
+	}
+
+	ctx := testutil.Context(t, testutil.WaitShort)
+	cmdDone := make(chan struct{})
+	cancelCtx, cancel := context.WithCancel(ctx)
+	t.Cleanup(cancel)
+
+	// Create a test deployment
+	client := coderdtest.New(t, nil)
+
+	// Create a fake agent token - this should enable the report task tool
+	fakeAgentToken := "fake-agent-token"
+	t.Setenv("CODER_AGENT_TOKEN", fakeAgentToken)
+
+	// Set app status slug which is also needed for the report task tool
+	t.Setenv("CODER_MCP_APP_STATUS_SLUG", "test-app")
+
+	inv, root := clitest.New(t, "exp", "mcp", "server")
+	inv = inv.WithContext(cancelCtx)
+
+	pty := ptytest.New(t)
+	inv.Stdin = pty.Input()
+	inv.Stdout = pty.Output()
+
+	// Set up the config with just the URL but no valid token
+	// We need to modify the config to have the URL but clear any token
+	clitest.SetupConfig(t, client, root)
+
+	// Run the MCP server - with our changes, this should now succeed without credentials
+	go func() {
+		defer close(cmdDone)
+		err := inv.Run()
+		assert.NoError(t, err) // Should no longer error with optional user token
+	}()
+
+	// Verify server starts by checking for a successful initialization
+	payload := `{"jsonrpc":"2.0","id":1,"method":"initialize"}`
+	pty.WriteLine(payload)
+	_ = pty.ReadLine(ctx) // ignore echoed output
+	output := pty.ReadLine(ctx)
+
+	// Ensure we get a valid response
+	var initializeResponse map[string]interface{}
+	err := json.Unmarshal([]byte(output), &initializeResponse)
+	require.NoError(t, err)
+	require.Equal(t, "2.0", initializeResponse["jsonrpc"])
+	require.Equal(t, 1.0, initializeResponse["id"])
+	require.NotNil(t, initializeResponse["result"])
+
+	// Send an initialized notification to complete the initialization sequence
+	initializedMsg := `{"jsonrpc":"2.0","method":"notifications/initialized"}`
+	pty.WriteLine(initializedMsg)
+	_ = pty.ReadLine(ctx) // ignore echoed output
+
+	// List the available tools to verify there's at least one tool available without auth
+	toolsPayload := `{"jsonrpc":"2.0","id":2,"method":"tools/list"}`
+	pty.WriteLine(toolsPayload)
+	_ = pty.ReadLine(ctx) // ignore echoed output
+	output = pty.ReadLine(ctx)
+
+	var toolsResponse struct {
+		Result struct {
+			Tools []struct {
+				Name string `json:"name"`
+			} `json:"tools"`
+		} `json:"result"`
+		Error *struct {
+			Code    int    `json:"code"`
+			Message string `json:"message"`
+		} `json:"error,omitempty"`
+	}
+	err = json.Unmarshal([]byte(output), &toolsResponse)
+	require.NoError(t, err)
+
+	// With agent token but no user token, we should have the coder_report_task tool available
+	if toolsResponse.Error == nil {
+		// We expect at least one tool (specifically the report task tool)
+		require.Greater(t, len(toolsResponse.Result.Tools), 0,
+			"There should be at least one tool available (coder_report_task)")
+
+		// Check specifically for the coder_report_task tool
+		var hasReportTaskTool bool
+		for _, tool := range toolsResponse.Result.Tools {
+			if tool.Name == "coder_report_task" {
+				hasReportTaskTool = true
+				break
+			}
+		}
+		require.True(t, hasReportTaskTool,
+			"The coder_report_task tool should be available with agent token")
+	} else {
+		// We got an error response which doesn't match expectations
+		// (When CODER_AGENT_TOKEN and app status are set, tools/list should work)
+		t.Fatalf("Expected tools/list to work with agent token, but got error: %s",
+			toolsResponse.Error.Message)
+	}
+
+	// Cancel and wait for the server to stop
+	cancel()
+	<-cmdDone
+}

cli/logout_test.go

@@ -1,6 +1,7 @@
 package cli_test
 
 import (
+	"fmt"
 	"os"
 	"runtime"
 	"testing"
@@ -89,10 +90,14 @@ func TestLogout(t *testing.T) {
 		logout.Stdin = pty.Input()
 		logout.Stdout = pty.Output()
 
+		executable, err := os.Executable()
+		require.NoError(t, err)
+		require.NotEqual(t, "", executable)
+
 		go func() {
 			defer close(logoutChan)
-			err := logout.Run()
-			assert.ErrorContains(t, err, "You are not logged in. Try logging in using 'coder login <url>'.")
+			err = logout.Run()
+			assert.Contains(t, err.Error(), fmt.Sprintf("Try logging in using '%s login <url>'.", executable))
 		}()
 
 		<-logoutChan

cli/open_test.go

@@ -326,7 +326,7 @@ func TestOpenVSCodeDevContainer(t *testing.T) {
 				},
 			},
 		}, nil,
-	)
+	).AnyTimes()
 
 	client, workspace, agentToken := setupWorkspaceForAgent(t, func(agents []*proto.Agent) []*proto.Agent {
 		agents[0].Directory = agentDir
@@ -501,7 +501,7 @@ func TestOpenVSCodeDevContainer_NoAgentDirectory(t *testing.T) {
 				},
 			},
 		}, nil,
-	)
+	).AnyTimes()
 
 	client, workspace, agentToken := setupWorkspaceForAgent(t, func(agents []*proto.Agent) []*proto.Agent {
 		agents[0].Name = agentName

cli/parameterresolver.go

@@ -226,7 +226,7 @@ func (pr *ParameterResolver) resolveWithInput(resolved []codersdk.WorkspaceBuild
 		if p != nil {
 			continue
 		}
-		// Parameter has not been resolved yet, so CLI needs to determine if user should input it.
+		// PreviewParameter has not been resolved yet, so CLI needs to determine if user should input it.
 
 		firstTimeUse := pr.isFirstTimeUse(tvp.Name)
 		promptParameterOption := pr.isLastBuildParameterInvalidOption(tvp)

cli/restart_test.go

@@ -359,7 +359,7 @@ func TestRestartWithParameters(t *testing.T) {
 		client := coderdtest.New(t, &coderdtest.Options{IncludeProvisionerDaemon: true})
 		owner := coderdtest.CreateFirstUser(t, client)
 		member, _ := coderdtest.CreateAnotherUser(t, client, owner.OrganizationID)
-		version := coderdtest.CreateTemplateVersion(t, client, owner.OrganizationID, mutableParamsResponse)
+		version := coderdtest.CreateTemplateVersion(t, client, owner.OrganizationID, mutableParamsResponse())
 		coderdtest.AwaitTemplateVersionJobCompleted(t, client, version.ID)
 		template := coderdtest.CreateTemplate(t, client, owner.OrganizationID, version.ID)
 		workspace := coderdtest.CreateWorkspace(t, member, template.ID, func(cwr *codersdk.CreateWorkspaceRequest) {

cli/root.go

@@ -72,7 +72,7 @@ const (
 	varDisableDirect           = "disable-direct-connections"
 	varDisableNetworkTelemetry = "disable-network-telemetry"
 
-	notLoggedInMessage = "You are not logged in. Try logging in using 'coder login <url>'."
+	notLoggedInMessage = "You are not logged in. Try logging in using '%s login <url>'."
 
 	envNoVersionCheck   = "CODER_NO_VERSION_WARNING"
 	envNoFeatureWarning = "CODER_NO_FEATURE_WARNING"
@@ -534,7 +534,11 @@ func (r *RootCmd) InitClient(client *codersdk.Client) serpent.MiddlewareFunc {
 				rawURL, err := conf.URL().Read()
 				// If the configuration files are absent, the user is logged out
 				if os.IsNotExist(err) {
-					return xerrors.New(notLoggedInMessage)
+					binPath, err := os.Executable()
+					if err != nil {
+						binPath = "coder"
+					}
+					return xerrors.Errorf(notLoggedInMessage, binPath)
 				}
 				if err != nil {
 					return err
@@ -571,6 +575,58 @@ func (r *RootCmd) InitClient(client *codersdk.Client) serpent.MiddlewareFunc {
 	}
 }
 
+// TryInitClient is similar to InitClient but doesn't error when credentials are missing.
+// This allows commands to run without requiring authentication, but still use auth if available.
+func (r *RootCmd) TryInitClient(client *codersdk.Client) serpent.MiddlewareFunc {
+	return func(next serpent.HandlerFunc) serpent.HandlerFunc {
+		return func(inv *serpent.Invocation) error {
+			conf := r.createConfig()
+			var err error
+			// Read the client URL stored on disk.
+			if r.clientURL == nil || r.clientURL.String() == "" {
+				rawURL, err := conf.URL().Read()
+				// If the configuration files are absent, just continue without URL
+				if err != nil {
+					// Continue with a nil or empty URL
+					if !os.IsNotExist(err) {
+						return err
+					}
+				} else {
+					r.clientURL, err = url.Parse(strings.TrimSpace(rawURL))
+					if err != nil {
+						return err
+					}
+				}
+			}
+			// Read the token stored on disk.
+			if r.token == "" {
+				r.token, err = conf.Session().Read()
+				// Even if there isn't a token, we don't care.
+				// Some API routes can be unauthenticated.
+				if err != nil && !os.IsNotExist(err) {
+					return err
+				}
+			}
+
+			// Only configure the client if we have a URL
+			if r.clientURL != nil && r.clientURL.String() != "" {
+				err = r.configureClient(inv.Context(), client, r.clientURL, inv)
+				if err != nil {
+					return err
+				}
+				client.SetSessionToken(r.token)
+
+				if r.debugHTTP {
+					client.PlainLogger = os.Stderr
+					client.SetLogBodies(true)
+				}
+				client.DisableDirectConnections = r.disableDirect
+			}
+			return next(inv)
+		}
+	}
+}
+
 // HeaderTransport creates a new transport that executes `--header-command`
 // if it is set to add headers for all outbound requests.
 func (r *RootCmd) HeaderTransport(ctx context.Context, serverURL *url.URL) (*codersdk.HeaderTransport, error) {
@@ -1004,11 +1060,12 @@ func cliHumanFormatError(from string, err error, opts *formatOpts) (string, bool
 		return formatRunCommandError(cmdErr, opts), true
 	}
 
-	uw, ok := err.(interface{ Unwrap() error })
-	if ok {
-		msg, special := cliHumanFormatError(from+traceError(err), uw.Unwrap(), opts)
-		if special {
-			return msg, special
+	if uw, ok := err.(interface{ Unwrap() error }); ok {
+		if unwrapped := uw.Unwrap(); unwrapped != nil {
+			msg, special := cliHumanFormatError(from+traceError(err), unwrapped, opts)
+			if special {
+				return msg, special
+			}
 		}
 	}
 	// If we got here, that means that the wrapped error chain does not have

cli/server.go

@@ -61,10 +61,12 @@ import (
 	"github.com/coder/serpent"
 	"github.com/coder/wgtunnel/tunnelsdk"
 
+	"github.com/coder/coder/v2/coderd/ai"
 	"github.com/coder/coder/v2/coderd/entitlements"
 	"github.com/coder/coder/v2/coderd/notifications/reports"
 	"github.com/coder/coder/v2/coderd/runtimeconfig"
 	"github.com/coder/coder/v2/coderd/webpush"
+	"github.com/coder/coder/v2/codersdk/drpcsdk"
 
 	"github.com/coder/coder/v2/buildinfo"
 	"github.com/coder/coder/v2/cli/clilog"
@@ -85,6 +87,7 @@ import (
 	"github.com/coder/coder/v2/coderd/externalauth"
 	"github.com/coder/coder/v2/coderd/gitsshkey"
 	"github.com/coder/coder/v2/coderd/httpmw"
+	"github.com/coder/coder/v2/coderd/jobreaper"
 	"github.com/coder/coder/v2/coderd/notifications"
 	"github.com/coder/coder/v2/coderd/oauthpki"
 	"github.com/coder/coder/v2/coderd/prometheusmetrics"
@@ -93,7 +96,6 @@ import (
 	"github.com/coder/coder/v2/coderd/schedule"
 	"github.com/coder/coder/v2/coderd/telemetry"
 	"github.com/coder/coder/v2/coderd/tracing"
-	"github.com/coder/coder/v2/coderd/unhanger"
 	"github.com/coder/coder/v2/coderd/updatecheck"
 	"github.com/coder/coder/v2/coderd/util/ptr"
 	"github.com/coder/coder/v2/coderd/util/slice"
@@ -101,7 +103,6 @@ import (
 	"github.com/coder/coder/v2/coderd/workspaceapps/appurl"
 	"github.com/coder/coder/v2/coderd/workspacestats"
 	"github.com/coder/coder/v2/codersdk"
-	"github.com/coder/coder/v2/codersdk/drpc"
 	"github.com/coder/coder/v2/cryptorand"
 	"github.com/coder/coder/v2/provisioner/echo"
 	"github.com/coder/coder/v2/provisioner/terraform"
@@ -610,6 +611,22 @@ func (r *RootCmd) Server(newAPI func(context.Context, *coderd.Options) (*coderd.
 				)
 			}
 
+			aiProviders, err := ReadAIProvidersFromEnv(os.Environ())
+			if err != nil {
+				return xerrors.Errorf("read ai providers from env: %w", err)
+			}
+			vals.AI.Value.Providers = append(vals.AI.Value.Providers, aiProviders...)
+			for _, provider := range aiProviders {
+				logger.Debug(
+					ctx, "loaded ai provider",
+					slog.F("type", provider.Type),
+				)
+			}
+			languageModels, err := ai.ModelsFromConfig(ctx, vals.AI.Value.Providers)
+			if err != nil {
+				return xerrors.Errorf("create language models: %w", err)
+			}
+
 			realIPConfig, err := httpmw.ParseRealIPConfig(vals.ProxyTrustedHeaders, vals.ProxyTrustedOrigins)
 			if err != nil {
 				return xerrors.Errorf("parse real ip config: %w", err)
@@ -640,6 +657,7 @@ func (r *RootCmd) Server(newAPI func(context.Context, *coderd.Options) (*coderd.
 				CacheDir:                    cacheDir,
 				GoogleTokenValidator:        googleTokenValidator,
 				ExternalAuthConfigs:         externalAuthConfigs,
+				LanguageModels:              languageModels,
 				RealIPConfig:                realIPConfig,
 				SSHKeygenAlgorithm:          sshKeygenAlgorithm,
 				TracerProvider:              tracerProvider,
@@ -739,6 +757,15 @@ func (r *RootCmd) Server(newAPI func(context.Context, *coderd.Options) (*coderd.
 					_ = sqlDB.Close()
 				}()
 
+				if options.DeploymentValues.Prometheus.Enable {
+					// At this stage we don't think the database name serves much purpose in these metrics.
+					// It requires parsing the DSN to determine it, which requires pulling in another dependency
+					// (i.e. https://github.com/jackc/pgx), but it's rather heavy.
+					// The conn string (https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNSTRING) can
+					// take different forms, which make parsing non-trivial.
+					options.PrometheusRegistry.MustRegister(collectors.NewDBStatsCollector(sqlDB, ""))
+				}
+
 				options.Database = database.New(sqlDB)
 				ps, err := pubsub.New(ctx, logger.Named("pubsub"), sqlDB, dbURL)
 				if err != nil {
@@ -901,6 +928,37 @@ func (r *RootCmd) Server(newAPI func(context.Context, *coderd.Options) (*coderd.
 			options.StatsBatcher = batcher
 			defer closeBatcher()
 
+			// Manage notifications.
+			var (
+				notificationsCfg     = options.DeploymentValues.Notifications
+				notificationsManager *notifications.Manager
+			)
+
+			metrics := notifications.NewMetrics(options.PrometheusRegistry)
+			helpers := templateHelpers(options)
+
+			// The enqueuer is responsible for enqueueing notifications to the given store.
+			enqueuer, err := notifications.NewStoreEnqueuer(notificationsCfg, options.Database, helpers, logger.Named("notifications.enqueuer"), quartz.NewReal())
+			if err != nil {
+				return xerrors.Errorf("failed to instantiate notification store enqueuer: %w", err)
+			}
+			options.NotificationsEnqueuer = enqueuer
+
+			// The notification manager is responsible for:
+			//   - creating notifiers and managing their lifecycles (notifiers are responsible for dequeueing/sending notifications)
+			//   - keeping the store updated with status updates
+			notificationsManager, err = notifications.NewManager(notificationsCfg, options.Database, options.Pubsub, helpers, metrics, logger.Named("notifications.manager"))
+			if err != nil {
+				return xerrors.Errorf("failed to instantiate notification manager: %w", err)
+			}
+
+			// nolint:gocritic // We need to run the manager in a notifier context.
+			notificationsManager.Run(dbauthz.AsNotifier(ctx))
+
+			// Run report generator to distribute periodic reports.
+			notificationReportGenerator := reports.NewReportGenerator(ctx, logger.Named("notifications.report_generator"), options.Database, options.NotificationsEnqueuer, quartz.NewReal())
+			defer notificationReportGenerator.Close()
+
 			// We use a separate coderAPICloser so the Enterprise API
 			// can have its own close functions. This is cleaner
 			// than abstracting the Coder API itself.
@@ -948,37 +1006,6 @@ func (r *RootCmd) Server(newAPI func(context.Context, *coderd.Options) (*coderd.
 				return xerrors.Errorf("write config url: %w", err)
 			}
 
-			// Manage notifications.
-			var (
-				notificationsCfg     = options.DeploymentValues.Notifications
-				notificationsManager *notifications.Manager
-			)
-
-			metrics := notifications.NewMetrics(options.PrometheusRegistry)
-			helpers := templateHelpers(options)
-
-			// The enqueuer is responsible for enqueueing notifications to the given store.
-			enqueuer, err := notifications.NewStoreEnqueuer(notificationsCfg, options.Database, helpers, logger.Named("notifications.enqueuer"), quartz.NewReal())
-			if err != nil {
-				return xerrors.Errorf("failed to instantiate notification store enqueuer: %w", err)
-			}
-			options.NotificationsEnqueuer = enqueuer
-
-			// The notification manager is responsible for:
-			//   - creating notifiers and managing their lifecycles (notifiers are responsible for dequeueing/sending notifications)
-			//   - keeping the store updated with status updates
-			notificationsManager, err = notifications.NewManager(notificationsCfg, options.Database, options.Pubsub, helpers, metrics, logger.Named("notifications.manager"))
-			if err != nil {
-				return xerrors.Errorf("failed to instantiate notification manager: %w", err)
-			}
-
-			// nolint:gocritic // We need to run the manager in a notifier context.
-			notificationsManager.Run(dbauthz.AsNotifier(ctx))
-
-			// Run report generator to distribute periodic reports.
-			notificationReportGenerator := reports.NewReportGenerator(ctx, logger.Named("notifications.report_generator"), options.Database, options.NotificationsEnqueuer, quartz.NewReal())
-			defer notificationReportGenerator.Close()
-
 			// Since errCh only has one buffered slot, all routines
 			// sending on it must be wrapped in a select/default to
 			// avoid leaving dangling goroutines waiting for the
@@ -1097,14 +1124,14 @@ func (r *RootCmd) Server(newAPI func(context.Context, *coderd.Options) (*coderd.
 			autobuildTicker := time.NewTicker(vals.AutobuildPollInterval.Value())
 			defer autobuildTicker.Stop()
 			autobuildExecutor := autobuild.NewExecutor(
-				ctx, options.Database, options.Pubsub, options.PrometheusRegistry, coderAPI.TemplateScheduleStore, &coderAPI.Auditor, coderAPI.AccessControlStore, logger, autobuildTicker.C, options.NotificationsEnqueuer)
+				ctx, options.Database, options.Pubsub, options.PrometheusRegistry, coderAPI.TemplateScheduleStore, &coderAPI.Auditor, coderAPI.AccessControlStore, logger, autobuildTicker.C, options.NotificationsEnqueuer, coderAPI.Experiments)
 			autobuildExecutor.Run()
 
-			hangDetectorTicker := time.NewTicker(vals.JobHangDetectorInterval.Value())
-			defer hangDetectorTicker.Stop()
-			hangDetector := unhanger.New(ctx, options.Database, options.Pubsub, logger, hangDetectorTicker.C)
-			hangDetector.Start()
-			defer hangDetector.Close()
+			jobReaperTicker := time.NewTicker(vals.JobReaperDetectorInterval.Value())
+			defer jobReaperTicker.Stop()
+			jobReaper := jobreaper.New(ctx, options.Database, options.Pubsub, logger, jobReaperTicker.C)
+			jobReaper.Start()
+			defer jobReaper.Close()
 
 			waitForProvisionerJobs := false
 			// Currently there is no way to ask the server to shut
@@ -1420,7 +1447,7 @@ func newProvisionerDaemon(
 	for _, provisionerType := range provisionerTypes {
 		switch provisionerType {
 		case codersdk.ProvisionerTypeEcho:
-			echoClient, echoServer := drpc.MemTransportPipe()
+			echoClient, echoServer := drpcsdk.MemTransportPipe()
 			wg.Add(1)
 			go func() {
 				defer wg.Done()
@@ -1454,7 +1481,7 @@ func newProvisionerDaemon(
 			}
 
 			tracer := coderAPI.TracerProvider.Tracer(tracing.TracerName)
-			terraformClient, terraformServer := drpc.MemTransportPipe()
+			terraformClient, terraformServer := drpcsdk.MemTransportPipe()
 			wg.Add(1)
 			go func() {
 				defer wg.Done()
@@ -2612,6 +2639,77 @@ func redirectHTTPToHTTPSDeprecation(ctx context.Context, logger slog.Logger, inv
 	}
 }
 
+func ReadAIProvidersFromEnv(environ []string) ([]codersdk.AIProviderConfig, error) {
+	// The index numbers must be in-order.
+	sort.Strings(environ)
+
+	var providers []codersdk.AIProviderConfig
+	for _, v := range serpent.ParseEnviron(environ, "CODER_AI_PROVIDER_") {
+		tokens := strings.SplitN(v.Name, "_", 2)
+		if len(tokens) != 2 {
+			return nil, xerrors.Errorf("invalid env var: %s", v.Name)
+		}
+
+		providerNum, err := strconv.Atoi(tokens[0])
+		if err != nil {
+			return nil, xerrors.Errorf("parse number: %s", v.Name)
+		}
+
+		var provider codersdk.AIProviderConfig
+		switch {
+		case len(providers) < providerNum:
+			return nil, xerrors.Errorf(
+				"provider num %v skipped: %s",
+				len(providers),
+				v.Name,
+			)
+		case len(providers) == providerNum:
+			// At the next next provider.
+			providers = append(providers, provider)
+		case len(providers) == providerNum+1:
+			// At the current provider.
+			provider = providers[providerNum]
+		}
+
+		key := tokens[1]
+		switch key {
+		case "TYPE":
+			provider.Type = v.Value
+		case "API_KEY":
+			provider.APIKey = v.Value
+		case "BASE_URL":
+			provider.BaseURL = v.Value
+		case "MODELS":
+			provider.Models = strings.Split(v.Value, ",")
+		}
+		providers[providerNum] = provider
+	}
+	for _, envVar := range environ {
+		tokens := strings.SplitN(envVar, "=", 2)
+		if len(tokens) != 2 {
+			continue
+		}
+		switch tokens[0] {
+		case "OPENAI_API_KEY":
+			providers = append(providers, codersdk.AIProviderConfig{
+				Type:   "openai",
+				APIKey: tokens[1],
+			})
+		case "ANTHROPIC_API_KEY":
+			providers = append(providers, codersdk.AIProviderConfig{
+				Type:   "anthropic",
+				APIKey: tokens[1],
+			})
+		case "GOOGLE_API_KEY":
+			providers = append(providers, codersdk.AIProviderConfig{
+				Type:   "google",
+				APIKey: tokens[1],
+			})
+		}
+	}
+	return providers, nil
+}
+
 // ReadExternalAuthProvidersFromEnv is provided for compatibility purposes with
 // the viper CLI.
 func ReadExternalAuthProvidersFromEnv(environ []string) ([]codersdk.ExternalAuthConfig, error) {

cli/ssh.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"io"
 	"log"
+	"net"
 	"net/http"
 	"net/url"
 	"os"
@@ -66,6 +67,7 @@ func (r *RootCmd) ssh() *serpent.Command {
 		stdio               bool
 		hostPrefix          string
 		hostnameSuffix      string
+		forceNewTunnel      bool
 		forwardAgent        bool
 		forwardGPG          bool
 		identityAgent       string
@@ -85,16 +87,36 @@ func (r *RootCmd) ssh() *serpent.Command {
 		containerUser string
 	)
 	client := new(codersdk.Client)
+	wsClient := workspacesdk.New(client)
 	cmd := &serpent.Command{
 		Annotations: workspaceCommand,
-		Use:         "ssh <workspace>",
-		Short:       "Start a shell into a workspace",
+		Use:         "ssh <workspace> [command]",
+		Short:       "Start a shell into a workspace or run a command",
+		Long: "This command does not have full parity with the standard SSH command. For users who need the full functionality of SSH, create an ssh configuration with `coder config-ssh`.\n\n" +
+			FormatExamples(
+				Example{
+					Description: "Use `--` to separate and pass flags directly to the command executed via SSH.",
+					Command:     "coder ssh <workspace> -- ls -la",
+				},
+			),
 		Middleware: serpent.Chain(
-			serpent.RequireNArgs(1),
+			// Require at least one arg for the workspace name
+			func(next serpent.HandlerFunc) serpent.HandlerFunc {
+				return func(i *serpent.Invocation) error {
+					got := len(i.Args)
+					if got < 1 {
+						return xerrors.New("expected the name of a workspace")
+					}
+
+					return next(i)
+				}
+			},
 			r.InitClient(client),
 			initAppearance(client, &appearanceConfig),
 		),
 		Handler: func(inv *serpent.Invocation) (retErr error) {
+			command := strings.Join(inv.Args[1:], " ")
+
 			// Before dialing the SSH server over TCP, capture Interrupt signals
 			// so that if we are interrupted, we have a chance to tear down the
 			// TCP session cleanly before exiting.  If we don't, then the TCP
@@ -203,14 +225,14 @@ func (r *RootCmd) ssh() *serpent.Command {
 				parsedEnv = append(parsedEnv, [2]string{k, v})
 			}
 
-			deploymentSSHConfig := codersdk.SSHConfigResponse{
+			cliConfig := codersdk.SSHConfigResponse{
 				HostnamePrefix: hostPrefix,
 				HostnameSuffix: hostnameSuffix,
 			}
 
 			workspace, workspaceAgent, err := findWorkspaceAndAgentByHostname(
 				ctx, inv, client,
-				inv.Args[0], deploymentSSHConfig, disableAutostart)
+				inv.Args[0], cliConfig, disableAutostart)
 			if err != nil {
 				return err
 			}
@@ -275,10 +297,44 @@ func (r *RootCmd) ssh() *serpent.Command {
 				return err
 			}
 
+			// If we're in stdio mode, check to see if we can use Coder Connect.
+			// We don't support Coder Connect over non-stdio coder ssh yet.
+			if stdio && !forceNewTunnel {
+				connInfo, err := wsClient.AgentConnectionInfoGeneric(ctx)
+				if err != nil {
+					return xerrors.Errorf("get agent connection info: %w", err)
+				}
+				coderConnectHost := fmt.Sprintf("%s.%s.%s.%s",
+					workspaceAgent.Name, workspace.Name, workspace.OwnerName, connInfo.HostnameSuffix)
+				exists, _ := workspacesdk.ExistsViaCoderConnect(ctx, coderConnectHost)
+				if exists {
+					defer cancel()
+
+					if networkInfoDir != "" {
+						if err := writeCoderConnectNetInfo(ctx, networkInfoDir); err != nil {
+							logger.Error(ctx, "failed to write coder connect net info file", slog.Error(err))
+						}
+					}
+
+					stopPolling := tryPollWorkspaceAutostop(ctx, client, workspace)
+					defer stopPolling()
+
+					usageAppName := getUsageAppName(usageApp)
+					if usageAppName != "" {
+						closeUsage := client.UpdateWorkspaceUsageWithBodyContext(ctx, workspace.ID, codersdk.PostWorkspaceUsageRequest{
+							AgentID: workspaceAgent.ID,
+							AppName: usageAppName,
+						})
+						defer closeUsage()
+					}
+					return runCoderConnectStdio(ctx, fmt.Sprintf("%s:22", coderConnectHost), stdioReader, stdioWriter, stack)
+				}
+			}
+
 			if r.disableDirect {
 				_, _ = fmt.Fprintln(inv.Stderr, "Direct connections disabled.")
 			}
-			conn, err := workspacesdk.New(client).
+			conn, err := wsClient.
 				DialAgent(ctx, workspaceAgent.ID, &workspacesdk.DialAgentOptions{
 					Logger:          logger,
 					BlockEndpoints:  r.disableDirect,
@@ -510,40 +566,46 @@ func (r *RootCmd) ssh() *serpent.Command {
 			sshSession.Stdout = inv.Stdout
 			sshSession.Stderr = inv.Stderr
 
-			err = sshSession.Shell()
-			if err != nil {
-				return xerrors.Errorf("start shell: %w", err)
-			}
+			if command != "" {
+				err := sshSession.Run(command)
+				if err != nil {
+					return xerrors.Errorf("run command: %w", err)
+				}
+			} else {
+				err = sshSession.Shell()
+				if err != nil {
+					return xerrors.Errorf("start shell: %w", err)
+				}
 
-			// Put cancel at the top of the defer stack to initiate
-			// shutdown of services.
-			defer cancel()
+				// Put cancel at the top of the defer stack to initiate
+				// shutdown of services.
+				defer cancel()
 
-			if validOut {
-				// Set initial window size.
-				width, height, err := term.GetSize(int(stdoutFile.Fd()))
-				if err == nil {
-					_ = sshSession.WindowChange(height, width)
+				if validOut {
+					// Set initial window size.
+					width, height, err := term.GetSize(int(stdoutFile.Fd()))
+					if err == nil {
+						_ = sshSession.WindowChange(height, width)
+					}
+				}
+
+				err = sshSession.Wait()
+				conn.SendDisconnectedTelemetry()
+				if err != nil {
+					if exitErr := (&gossh.ExitError{}); errors.As(err, &exitErr) {
+						// Clear the error since it's not useful beyond
+						// reporting status.
+						return ExitError(exitErr.ExitStatus(), nil)
+					}
+					// If the connection drops unexpectedly, we get an
+					// ExitMissingError but no other error details, so try to at
+					// least give the user a better message
+					if errors.Is(err, &gossh.ExitMissingError{}) {
+						return ExitError(255, xerrors.New("SSH connection ended unexpectedly"))
+					}
+					return xerrors.Errorf("session ended: %w", err)
 				}
 			}
-
-			err = sshSession.Wait()
-			conn.SendDisconnectedTelemetry()
-			if err != nil {
-				if exitErr := (&gossh.ExitError{}); errors.As(err, &exitErr) {
-					// Clear the error since it's not useful beyond
-					// reporting status.
-					return ExitError(exitErr.ExitStatus(), nil)
-				}
-				// If the connection drops unexpectedly, we get an
-				// ExitMissingError but no other error details, so try to at
-				// least give the user a better message
-				if errors.Is(err, &gossh.ExitMissingError{}) {
-					return ExitError(255, xerrors.New("SSH connection ended unexpectedly"))
-				}
-				return xerrors.Errorf("session ended: %w", err)
-			}
-
 			return nil
 		},
 	}
@@ -660,6 +722,12 @@ func (r *RootCmd) ssh() *serpent.Command {
 			Value:       serpent.StringOf(&containerUser),
 			Hidden:      true, // Hidden until this features is at least in beta.
 		},
+		{
+			Flag:        "force-new-tunnel",
+			Description: "Force the creation of a new tunnel to the workspace, even if the Coder Connect tunnel is available.",
+			Value:       serpent.BoolOf(&forceNewTunnel),
+			Hidden:      true,
+		},
 		sshDisableAutostartOption(serpent.BoolOf(&disableAutostart)),
 	}
 	return cmd
@@ -1372,12 +1440,13 @@ func setStatsCallback(
 }
 
 type sshNetworkStats struct {
-	P2P              bool               `json:"p2p"`
-	Latency          float64            `json:"latency"`
-	PreferredDERP    string             `json:"preferred_derp"`
-	DERPLatency      map[string]float64 `json:"derp_latency"`
-	UploadBytesSec   int64              `json:"upload_bytes_sec"`
-	DownloadBytesSec int64              `json:"download_bytes_sec"`
+	P2P               bool               `json:"p2p"`
+	Latency           float64            `json:"latency"`
+	PreferredDERP     string             `json:"preferred_derp"`
+	DERPLatency       map[string]float64 `json:"derp_latency"`
+	UploadBytesSec    int64              `json:"upload_bytes_sec"`
+	DownloadBytesSec  int64              `json:"download_bytes_sec"`
+	UsingCoderConnect bool               `json:"using_coder_connect"`
 }
 
 func collectNetworkStats(ctx context.Context, agentConn *workspacesdk.AgentConn, start, end time.Time, counts map[netlogtype.Connection]netlogtype.Counts) (*sshNetworkStats, error) {
@@ -1448,15 +1517,91 @@ func collectNetworkStats(ctx context.Context, agentConn *workspacesdk.AgentConn,
 	}, nil
 }
 
+type coderConnectDialerContextKey struct{}
+
+type coderConnectDialer interface {
+	DialContext(ctx context.Context, network, addr string) (net.Conn, error)
+}
+
+func WithTestOnlyCoderConnectDialer(ctx context.Context, dialer coderConnectDialer) context.Context {
+	return context.WithValue(ctx, coderConnectDialerContextKey{}, dialer)
+}
+
+func testOrDefaultDialer(ctx context.Context) coderConnectDialer {
+	dialer, ok := ctx.Value(coderConnectDialerContextKey{}).(coderConnectDialer)
+	if !ok || dialer == nil {
+		return &net.Dialer{}
+	}
+	return dialer
+}
+
+func runCoderConnectStdio(ctx context.Context, addr string, stdin io.Reader, stdout io.Writer, stack *closerStack) error {
+	dialer := testOrDefaultDialer(ctx)
+	conn, err := dialer.DialContext(ctx, "tcp", addr)
+	if err != nil {
+		return xerrors.Errorf("dial coder connect host: %w", err)
+	}
+	if err := stack.push("tcp conn", conn); err != nil {
+		return err
+	}
+
+	agentssh.Bicopy(ctx, conn, &StdioRwc{
+		Reader: stdin,
+		Writer: stdout,
+	})
+
+	return nil
+}
+
+type StdioRwc struct {
+	io.Reader
+	io.Writer
+}
+
+func (*StdioRwc) Close() error {
+	return nil
+}
+
+func writeCoderConnectNetInfo(ctx context.Context, networkInfoDir string) error {
+	fs, ok := ctx.Value("fs").(afero.Fs)
+	if !ok {
+		fs = afero.NewOsFs()
+	}
+	if err := fs.MkdirAll(networkInfoDir, 0o700); err != nil {
+		return xerrors.Errorf("mkdir: %w", err)
+	}
+
+	// The VS Code extension obtains the PID of the SSH process to
+	// find the log file associated with a SSH session.
+	//
+	// We get the parent PID because it's assumed `ssh` is calling this
+	// command via the ProxyCommand SSH option.
+	networkInfoFilePath := filepath.Join(networkInfoDir, fmt.Sprintf("%d.json", os.Getppid()))
+	stats := &sshNetworkStats{
+		UsingCoderConnect: true,
+	}
+	rawStats, err := json.Marshal(stats)
+	if err != nil {
+		return xerrors.Errorf("marshal network stats: %w", err)
+	}
+	err = afero.WriteFile(fs, networkInfoFilePath, rawStats, 0o600)
+	if err != nil {
+		return xerrors.Errorf("write network stats: %w", err)
+	}
+	return nil
+}
+
 // Converts workspace name input to owner/workspace.agent format
 // Possible valid input formats:
 // workspace
+// workspace.agent
 // owner/workspace
 // owner--workspace
 // owner/workspace--agent
 // owner/workspace.agent
 // owner--workspace--agent
 // owner--workspace.agent
+// agent.workspace.owner - for parity with Coder Connect
 func normalizeWorkspaceInput(input string) string {
 	// Split on "/", "--", and "."
 	parts := workspaceNameRe.Split(input, -1)
@@ -1465,8 +1610,15 @@ func normalizeWorkspaceInput(input string) string {
 	case 1:
 		return input // "workspace"
 	case 2:
+		if strings.Contains(input, ".") {
+			return fmt.Sprintf("%s.%s", parts[0], parts[1]) // "workspace.agent"
+		}
 		return fmt.Sprintf("%s/%s", parts[0], parts[1]) // "owner/workspace"
 	case 3:
+		// If the only separator is a dot, it's the Coder Connect format
+		if !strings.Contains(input, "/") && !strings.Contains(input, "--") {
+			return fmt.Sprintf("%s/%s.%s", parts[2], parts[1], parts[0]) // "owner/workspace.agent"
+		}
 		return fmt.Sprintf("%s/%s.%s", parts[0], parts[1], parts[2]) // "owner/workspace.agent"
 	default:
 		return input // Fallback

cli/ssh_internal_test.go

@@ -3,13 +3,17 @@ package cli
 import (
 	"context"
 	"fmt"
+	"io"
+	"net"
 	"net/url"
 	"sync"
 	"testing"
 	"time"
 
+	gliderssh "github.com/gliderlabs/ssh"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"golang.org/x/crypto/ssh"
 	"golang.org/x/xerrors"
 
 	"cdr.dev/slog"
@@ -202,7 +206,7 @@ func TestCloserStack_Timeout(t *testing.T) {
 		defer close(closed)
 		uut.close(nil)
 	}()
-	trap.MustWait(ctx).Release()
+	trap.MustWait(ctx).MustRelease(ctx)
 	// top starts right away, but it hangs
 	testutil.TryReceive(ctx, t, ac[2].started)
 	// timer pops and we start the middle one
@@ -220,6 +224,87 @@ func TestCloserStack_Timeout(t *testing.T) {
 	testutil.TryReceive(ctx, t, closed)
 }
 
+func TestCoderConnectStdio(t *testing.T) {
+	t.Parallel()
+
+	ctx := testutil.Context(t, testutil.WaitShort)
+	logger := slogtest.Make(t, nil).Leveled(slog.LevelDebug)
+	stack := newCloserStack(ctx, logger, quartz.NewMock(t))
+
+	clientOutput, clientInput := io.Pipe()
+	serverOutput, serverInput := io.Pipe()
+	defer func() {
+		for _, c := range []io.Closer{clientOutput, clientInput, serverOutput, serverInput} {
+			_ = c.Close()
+		}
+	}()
+
+	server := newSSHServer("127.0.0.1:0")
+	ln, err := net.Listen("tcp", server.server.Addr)
+	require.NoError(t, err)
+
+	go func() {
+		_ = server.Serve(ln)
+	}()
+	t.Cleanup(func() {
+		_ = server.Close()
+	})
+
+	stdioDone := make(chan struct{})
+	go func() {
+		err = runCoderConnectStdio(ctx, ln.Addr().String(), clientOutput, serverInput, stack)
+		assert.NoError(t, err)
+		close(stdioDone)
+	}()
+
+	conn, channels, requests, err := ssh.NewClientConn(&testutil.ReaderWriterConn{
+		Reader: serverOutput,
+		Writer: clientInput,
+	}, "", &ssh.ClientConfig{
+		// #nosec
+		HostKeyCallback: ssh.InsecureIgnoreHostKey(),
+	})
+	require.NoError(t, err)
+	defer conn.Close()
+
+	sshClient := ssh.NewClient(conn, channels, requests)
+	session, err := sshClient.NewSession()
+	require.NoError(t, err)
+	defer session.Close()
+
+	// We're not connected to a real shell
+	err = session.Run("")
+	require.NoError(t, err)
+	err = sshClient.Close()
+	require.NoError(t, err)
+	_ = clientOutput.Close()
+
+	<-stdioDone
+}
+
+type sshServer struct {
+	server *gliderssh.Server
+}
+
+func newSSHServer(addr string) *sshServer {
+	return &sshServer{
+		server: &gliderssh.Server{
+			Addr: addr,
+			Handler: func(s gliderssh.Session) {
+				_, _ = io.WriteString(s.Stderr(), "Connected!")
+			},
+		},
+	}
+}
+
+func (s *sshServer) Serve(ln net.Listener) error {
+	return s.server.Serve(ln)
+}
+
+func (s *sshServer) Close() error {
+	return s.server.Close()
+}
+
 type fakeCloser struct {
 	closes *[]*fakeCloser
 	err    error

cli/ssh_test.go

@@ -41,6 +41,7 @@ import (
 	"github.com/coder/coder/v2/agent/agentssh"
 	"github.com/coder/coder/v2/agent/agenttest"
 	agentproto "github.com/coder/coder/v2/agent/proto"
+	"github.com/coder/coder/v2/cli"
 	"github.com/coder/coder/v2/cli/clitest"
 	"github.com/coder/coder/v2/cli/cliui"
 	"github.com/coder/coder/v2/coderd/coderdtest"
@@ -106,12 +107,14 @@ func TestSSH(t *testing.T) {
 
 		cases := []string{
 			"myworkspace",
+			"myworkspace.dev",
 			"myuser/myworkspace",
 			"myuser--myworkspace",
 			"myuser/myworkspace--dev",
 			"myuser/myworkspace.dev",
 			"myuser--myworkspace--dev",
 			"myuser--myworkspace.dev",
+			"dev.myworkspace.myuser",
 		}
 
 		for _, tc := range cases {
@@ -473,7 +476,7 @@ func TestSSH(t *testing.T) {
 			assert.NoError(t, err)
 		})
 
-		conn, channels, requests, err := ssh.NewClientConn(&stdioConn{
+		conn, channels, requests, err := ssh.NewClientConn(&testutil.ReaderWriterConn{
 			Reader: serverOutput,
 			Writer: clientInput,
 		}, "", &ssh.ClientConfig{
@@ -542,7 +545,7 @@ func TestSSH(t *testing.T) {
 		signer, err := agentssh.CoderSigner(keySeed)
 		assert.NoError(t, err)
 
-		conn, channels, requests, err := ssh.NewClientConn(&stdioConn{
+		conn, channels, requests, err := ssh.NewClientConn(&testutil.ReaderWriterConn{
 			Reader: serverOutput,
 			Writer: clientInput,
 		}, "", &ssh.ClientConfig{
@@ -605,7 +608,7 @@ func TestSSH(t *testing.T) {
 			assert.NoError(t, err)
 		})
 
-		conn, channels, requests, err := ssh.NewClientConn(&stdioConn{
+		conn, channels, requests, err := ssh.NewClientConn(&testutil.ReaderWriterConn{
 			Reader: serverOutput,
 			Writer: clientInput,
 		}, "", &ssh.ClientConfig{
@@ -773,7 +776,7 @@ func TestSSH(t *testing.T) {
 		// have access to the shell.
 		_ = agenttest.New(t, client.URL, authToken)
 
-		conn, channels, requests, err := ssh.NewClientConn(&stdioConn{
+		conn, channels, requests, err := ssh.NewClientConn(&testutil.ReaderWriterConn{
 			Reader: proxyCommandStdoutR,
 			Writer: clientStdinW,
 		}, "", &ssh.ClientConfig{
@@ -835,7 +838,7 @@ func TestSSH(t *testing.T) {
 			assert.NoError(t, err)
 		})
 
-		conn, channels, requests, err := ssh.NewClientConn(&stdioConn{
+		conn, channels, requests, err := ssh.NewClientConn(&testutil.ReaderWriterConn{
 			Reader: serverOutput,
 			Writer: clientInput,
 		}, "", &ssh.ClientConfig{
@@ -894,7 +897,7 @@ func TestSSH(t *testing.T) {
 			assert.NoError(t, err)
 		})
 
-		conn, channels, requests, err := ssh.NewClientConn(&stdioConn{
+		conn, channels, requests, err := ssh.NewClientConn(&testutil.ReaderWriterConn{
 			Reader: serverOutput,
 			Writer: clientInput,
 		}, "", &ssh.ClientConfig{
@@ -1082,7 +1085,7 @@ func TestSSH(t *testing.T) {
 			assert.NoError(t, err)
 		})
 
-		conn, channels, requests, err := ssh.NewClientConn(&stdioConn{
+		conn, channels, requests, err := ssh.NewClientConn(&testutil.ReaderWriterConn{
 			Reader: serverOutput,
 			Writer: clientInput,
 		}, "", &ssh.ClientConfig{
@@ -1741,7 +1744,7 @@ func TestSSH(t *testing.T) {
 					assert.NoError(t, err)
 				})
 
-				conn, channels, requests, err := ssh.NewClientConn(&stdioConn{
+				conn, channels, requests, err := ssh.NewClientConn(&testutil.ReaderWriterConn{
 					Reader: serverOutput,
 					Writer: clientInput,
 				}, "", &ssh.ClientConfig{
@@ -2055,12 +2058,6 @@ func TestSSH_Container(t *testing.T) {
 		client, workspace, agentToken := setupWorkspaceForAgent(t)
 		ctrl := gomock.NewController(t)
 		mLister := acmock.NewMockLister(ctrl)
-		_ = agenttest.New(t, client.URL, agentToken, func(o *agent.Options) {
-			o.ExperimentalDevcontainersEnabled = true
-			o.ContainerAPIOptions = append(o.ContainerAPIOptions, agentcontainers.WithLister(mLister))
-		})
-		_ = coderdtest.NewWorkspaceAgentWaiter(t, client, workspace.ID).Wait()
-
 		mLister.EXPECT().List(gomock.Any()).Return(codersdk.WorkspaceAgentListContainersResponse{
 			Containers: []codersdk.WorkspaceAgentContainer{
 				{
@@ -2069,7 +2066,12 @@ func TestSSH_Container(t *testing.T) {
 				},
 			},
 			Warnings: nil,
-		}, nil)
+		}, nil).AnyTimes()
+		_ = agenttest.New(t, client.URL, agentToken, func(o *agent.Options) {
+			o.ExperimentalDevcontainersEnabled = true
+			o.ContainerAPIOptions = append(o.ContainerAPIOptions, agentcontainers.WithLister(mLister))
+		})
+		_ = coderdtest.NewWorkspaceAgentWaiter(t, client, workspace.ID).Wait()
 
 		cID := uuid.NewString()
 		inv, root := clitest.New(t, "ssh", workspace.Name, "-c", cID)
@@ -2102,6 +2104,232 @@ func TestSSH_Container(t *testing.T) {
 	})
 }
 
+func TestSSH_CoderConnect(t *testing.T) {
+	t.Parallel()
+
+	t.Run("Enabled", func(t *testing.T) {
+		t.Parallel()
+		ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitShort)
+		defer cancel()
+
+		fs := afero.NewMemMapFs()
+		//nolint:revive,staticcheck
+		ctx = context.WithValue(ctx, "fs", fs)
+
+		client, workspace, agentToken := setupWorkspaceForAgent(t)
+		inv, root := clitest.New(t, "ssh", workspace.Name, "--network-info-dir", "/net", "--stdio")
+		clitest.SetupConfig(t, client, root)
+		_ = ptytest.New(t).Attach(inv)
+
+		ctx = cli.WithTestOnlyCoderConnectDialer(ctx, &fakeCoderConnectDialer{})
+		ctx = withCoderConnectRunning(ctx)
+
+		errCh := make(chan error, 1)
+		tGo(t, func() {
+			err := inv.WithContext(ctx).Run()
+			errCh <- err
+		})
+
+		_ = agenttest.New(t, client.URL, agentToken)
+		coderdtest.AwaitWorkspaceAgents(t, client, workspace.ID)
+
+		err := testutil.TryReceive(ctx, t, errCh)
+		// Our mock dialer will always fail with this error, if it was called
+		require.ErrorContains(t, err, "dial coder connect host \"dev.myworkspace.myuser.coder:22\" over tcp")
+
+		// The network info file should be created since we passed `--stdio`
+		entries, err := afero.ReadDir(fs, "/net")
+		require.NoError(t, err)
+		require.True(t, len(entries) > 0)
+	})
+
+	t.Run("Disabled", func(t *testing.T) {
+		t.Parallel()
+		client, workspace, agentToken := setupWorkspaceForAgent(t)
+
+		_ = agenttest.New(t, client.URL, agentToken)
+		coderdtest.AwaitWorkspaceAgents(t, client, workspace.ID)
+
+		clientOutput, clientInput := io.Pipe()
+		serverOutput, serverInput := io.Pipe()
+		defer func() {
+			for _, c := range []io.Closer{clientOutput, clientInput, serverOutput, serverInput} {
+				_ = c.Close()
+			}
+		}()
+
+		ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
+		defer cancel()
+
+		inv, root := clitest.New(t, "ssh", "--force-new-tunnel", "--stdio", workspace.Name)
+		clitest.SetupConfig(t, client, root)
+		inv.Stdin = clientOutput
+		inv.Stdout = serverInput
+		inv.Stderr = io.Discard
+
+		ctx = cli.WithTestOnlyCoderConnectDialer(ctx, &fakeCoderConnectDialer{})
+		ctx = withCoderConnectRunning(ctx)
+
+		cmdDone := tGo(t, func() {
+			err := inv.WithContext(ctx).Run()
+			// Shouldn't fail to dial the Coder Connect host
+			// since `--force-new-tunnel` was passed
+			assert.NoError(t, err)
+		})
+
+		conn, channels, requests, err := ssh.NewClientConn(&testutil.ReaderWriterConn{
+			Reader: serverOutput,
+			Writer: clientInput,
+		}, "", &ssh.ClientConfig{
+			// #nosec
+			HostKeyCallback: ssh.InsecureIgnoreHostKey(),
+		})
+		require.NoError(t, err)
+		defer conn.Close()
+
+		sshClient := ssh.NewClient(conn, channels, requests)
+		session, err := sshClient.NewSession()
+		require.NoError(t, err)
+		defer session.Close()
+
+		// Shells on Mac, Windows, and Linux all exit shells with the "exit" command.
+		err = session.Run("exit")
+		require.NoError(t, err)
+		err = sshClient.Close()
+		require.NoError(t, err)
+		_ = clientOutput.Close()
+
+		<-cmdDone
+	})
+
+	t.Run("OneShot", func(t *testing.T) {
+		t.Parallel()
+
+		client, workspace, agentToken := setupWorkspaceForAgent(t)
+		inv, root := clitest.New(t, "ssh", workspace.Name, "echo 'hello world'")
+		clitest.SetupConfig(t, client, root)
+
+		// Capture command output
+		output := new(bytes.Buffer)
+		inv.Stdout = output
+
+		ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
+		defer cancel()
+
+		cmdDone := tGo(t, func() {
+			err := inv.WithContext(ctx).Run()
+			assert.NoError(t, err)
+		})
+
+		_ = agenttest.New(t, client.URL, agentToken)
+		coderdtest.AwaitWorkspaceAgents(t, client, workspace.ID)
+
+		<-cmdDone
+
+		// Verify command output
+		assert.Contains(t, output.String(), "hello world")
+	})
+
+	t.Run("OneShotExitCode", func(t *testing.T) {
+		t.Parallel()
+
+		client, workspace, agentToken := setupWorkspaceForAgent(t)
+
+		// Setup agent first to avoid race conditions
+		_ = agenttest.New(t, client.URL, agentToken)
+		coderdtest.AwaitWorkspaceAgents(t, client, workspace.ID)
+
+		ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
+		defer cancel()
+
+		// Test successful exit code
+		t.Run("Success", func(t *testing.T) {
+			inv, root := clitest.New(t, "ssh", workspace.Name, "exit 0")
+			clitest.SetupConfig(t, client, root)
+
+			err := inv.WithContext(ctx).Run()
+			assert.NoError(t, err)
+		})
+
+		// Test error exit code
+		t.Run("Error", func(t *testing.T) {
+			inv, root := clitest.New(t, "ssh", workspace.Name, "exit 1")
+			clitest.SetupConfig(t, client, root)
+
+			err := inv.WithContext(ctx).Run()
+			assert.Error(t, err)
+			var exitErr *ssh.ExitError
+			assert.True(t, errors.As(err, &exitErr))
+			assert.Equal(t, 1, exitErr.ExitStatus())
+		})
+	})
+
+	t.Run("OneShotStdio", func(t *testing.T) {
+		t.Parallel()
+		client, workspace, agentToken := setupWorkspaceForAgent(t)
+		_, _ = tGoContext(t, func(ctx context.Context) {
+			// Run this async so the SSH command has to wait for
+			// the build and agent to connect!
+			_ = agenttest.New(t, client.URL, agentToken)
+			<-ctx.Done()
+		})
+
+		clientOutput, clientInput := io.Pipe()
+		serverOutput, serverInput := io.Pipe()
+		defer func() {
+			for _, c := range []io.Closer{clientOutput, clientInput, serverOutput, serverInput} {
+				_ = c.Close()
+			}
+		}()
+
+		ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
+		defer cancel()
+
+		inv, root := clitest.New(t, "ssh", "--stdio", workspace.Name, "echo 'hello stdio'")
+		clitest.SetupConfig(t, client, root)
+		inv.Stdin = clientOutput
+		inv.Stdout = serverInput
+		inv.Stderr = io.Discard
+
+		cmdDone := tGo(t, func() {
+			err := inv.WithContext(ctx).Run()
+			assert.NoError(t, err)
+		})
+
+		conn, channels, requests, err := ssh.NewClientConn(&testutil.ReaderWriterConn{
+			Reader: serverOutput,
+			Writer: clientInput,
+		}, "", &ssh.ClientConfig{
+			// #nosec
+			HostKeyCallback: ssh.InsecureIgnoreHostKey(),
+		})
+		require.NoError(t, err)
+		defer conn.Close()
+
+		sshClient := ssh.NewClient(conn, channels, requests)
+		session, err := sshClient.NewSession()
+		require.NoError(t, err)
+		defer session.Close()
+
+		// Capture and verify command output
+		output, err := session.Output("echo 'hello back'")
+		require.NoError(t, err)
+		assert.Contains(t, string(output), "hello back")
+
+		err = sshClient.Close()
+		require.NoError(t, err)
+		_ = clientOutput.Close()
+
+		<-cmdDone
+	})
+}
+
+type fakeCoderConnectDialer struct{}
+
+func (*fakeCoderConnectDialer) DialContext(ctx context.Context, network, addr string) (net.Conn, error) {
+	return nil, xerrors.Errorf("dial coder connect host %q over %s", addr, network)
+}
+
 // tGoContext runs fn in a goroutine passing a context that will be
 // canceled on test completion and wait until fn has finished executing.
 // Done and cancel are returned for optionally waiting until completion
@@ -2145,35 +2373,6 @@ func tGo(t *testing.T, fn func()) (done <-chan struct{}) {
 	return doneC
 }
 
-type stdioConn struct {
-	io.Reader
-	io.Writer
-}
-
-func (*stdioConn) Close() (err error) {
-	return nil
-}
-
-func (*stdioConn) LocalAddr() net.Addr {
-	return nil
-}
-
-func (*stdioConn) RemoteAddr() net.Addr {
-	return nil
-}
-
-func (*stdioConn) SetDeadline(_ time.Time) error {
-	return nil
-}
-
-func (*stdioConn) SetReadDeadline(_ time.Time) error {
-	return nil
-}
-
-func (*stdioConn) SetWriteDeadline(_ time.Time) error {
-	return nil
-}
-
 // tempDirUnixSocket returns a temporary directory that can safely hold unix
 // sockets (probably).
 //

cli/start_test.go

@@ -33,8 +33,8 @@ const (
 	mutableParameterValue = "hello"
 )
 
-var (
-	mutableParamsResponse = &echo.Responses{
+func mutableParamsResponse() *echo.Responses {
+	return &echo.Responses{
 		Parse: echo.ParseComplete,
 		ProvisionPlan: []*proto.Response{
 			{
@@ -54,8 +54,10 @@ var (
 		},
 		ProvisionApply: echo.ApplyComplete,
 	}
+}
 
-	immutableParamsResponse = &echo.Responses{
+func immutableParamsResponse() *echo.Responses {
+	return &echo.Responses{
 		Parse: echo.ParseComplete,
 		ProvisionPlan: []*proto.Response{
 			{
@@ -74,7 +76,7 @@ var (
 		},
 		ProvisionApply: echo.ApplyComplete,
 	}
-)
+}
 
 func TestStart(t *testing.T) {
 	t.Parallel()
@@ -210,7 +212,7 @@ func TestStartWithParameters(t *testing.T) {
 		client := coderdtest.New(t, &coderdtest.Options{IncludeProvisionerDaemon: true})
 		owner := coderdtest.CreateFirstUser(t, client)
 		member, _ := coderdtest.CreateAnotherUser(t, client, owner.OrganizationID)
-		version := coderdtest.CreateTemplateVersion(t, client, owner.OrganizationID, immutableParamsResponse)
+		version := coderdtest.CreateTemplateVersion(t, client, owner.OrganizationID, immutableParamsResponse())
 		coderdtest.AwaitTemplateVersionJobCompleted(t, client, version.ID)
 		template := coderdtest.CreateTemplate(t, client, owner.OrganizationID, version.ID)
 		workspace := coderdtest.CreateWorkspace(t, member, template.ID, func(cwr *codersdk.CreateWorkspaceRequest) {
@@ -262,7 +264,7 @@ func TestStartWithParameters(t *testing.T) {
 		client := coderdtest.New(t, &coderdtest.Options{IncludeProvisionerDaemon: true})
 		owner := coderdtest.CreateFirstUser(t, client)
 		member, _ := coderdtest.CreateAnotherUser(t, client, owner.OrganizationID)
-		version := coderdtest.CreateTemplateVersion(t, client, owner.OrganizationID, mutableParamsResponse)
+		version := coderdtest.CreateTemplateVersion(t, client, owner.OrganizationID, mutableParamsResponse())
 		coderdtest.AwaitTemplateVersionJobCompleted(t, client, version.ID)
 		template := coderdtest.CreateTemplate(t, client, owner.OrganizationID, version.ID)
 		workspace := coderdtest.CreateWorkspace(t, member, template.ID, func(cwr *codersdk.CreateWorkspaceRequest) {

cli/testdata/coder_--help.golden

@@ -46,7 +46,7 @@ SUBCOMMANDS:
     show              Display details of a workspace's resources and agents
     speedtest         Run upload and download tests from your machine to a
                       workspace
-    ssh               Start a shell into a workspace
+    ssh               Start a shell into a workspace or run a command
     start             Start a workspace
     stat              Show resource usage for the current workspace.
     state             Manually manage Terraform state to fix broken workspaces

cli/testdata/coder_list_--output_json.golden

@@ -15,6 +15,7 @@
     "template_allow_user_cancel_workspace_jobs": false,
     "template_active_version_id": "============[version ID]============",
     "template_require_active_version": false,
+    "template_use_classic_parameter_flow": false,
     "latest_build": {
       "id": "========[workspace build ID]========",
       "created_at": "====[timestamp]=====",
@@ -23,7 +24,6 @@
       "workspace_name": "test-workspace",
       "workspace_owner_id": "==========[first user ID]===========",
       "workspace_owner_name": "testuser",
-      "workspace_owner_avatar_url": "",
       "template_version_id": "============[version ID]============",
       "template_version_name": "===========[version name]===========",
       "build_number": 1,

cli/testdata/coder_provisioner_jobs_list_--help.golden

@@ -11,7 +11,7 @@ OPTIONS:
   -O, --org string, $CODER_ORGANIZATION
           Select which organization (uuid or name) to use.
 
-  -c, --column [id|created at|started at|completed at|canceled at|error|error code|status|worker id|file id|tags|queue position|queue size|organization id|template version id|workspace build id|type|available workers|template version name|template id|template name|template display name|template icon|workspace id|workspace name|organization|queue] (default: created at,id,type,template display name,status,queue,tags)
+  -c, --column [id|created at|started at|completed at|canceled at|error|error code|status|worker id|worker name|file id|tags|queue position|queue size|organization id|template version id|workspace build id|type|available workers|template version name|template id|template name|template display name|template icon|workspace id|workspace name|organization|queue] (default: created at,id,type,template display name,status,queue,tags)
           Columns to display in table output.
 
   -l, --limit int, $CODER_PROVISIONER_JOB_LIST_LIMIT (default: 50)

cli/testdata/coder_provisioner_jobs_list_--output_json.golden

@@ -6,6 +6,7 @@
     "completed_at": "====[timestamp]=====",
     "status": "succeeded",
     "worker_id": "====[workspace build worker ID]=====",
+    "worker_name": "test-daemon",
     "file_id": "=====[workspace build file ID]======",
     "tags": {
       "owner": "",
@@ -34,6 +35,7 @@
     "completed_at": "====[timestamp]=====",
     "status": "succeeded",
     "worker_id": "====[workspace build worker ID]=====",
+    "worker_name": "test-daemon",
     "file_id": "=====[workspace build file ID]======",
     "tags": {
       "owner": "",

cli/testdata/coder_provisioner_list.golden

@@ -1,2 +1,2 @@
-CREATED AT            LAST SEEN AT          KEY NAME  NAME  VERSION       STATUS  TAGS                            
-====[timestamp]=====  ====[timestamp]=====  built-in  test  v0.0.0-devel  idle    map[owner: scope:organization]  
+CREATED AT            LAST SEEN AT          KEY NAME  NAME         VERSION       STATUS  TAGS                            
+====[timestamp]=====  ====[timestamp]=====  built-in  test-daemon  v0.0.0-devel  idle    map[owner: scope:organization]  

cli/testdata/coder_provisioner_list_--output_json.golden

@@ -5,9 +5,9 @@
     "key_id": "00000000-0000-0000-0000-000000000001",
     "created_at": "====[timestamp]=====",
     "last_seen_at": "====[timestamp]=====",
-    "name": "test",
+    "name": "test-daemon",
     "version": "v0.0.0-devel",
-    "api_version": "1.4",
+    "api_version": "1.6",
     "provisioners": [
       "echo"
     ],

cli/testdata/coder_ssh_--help.golden

@@ -1,9 +1,18 @@
 coder v0.0.0-devel
 
 USAGE:
-  coder ssh [flags] <workspace>
+  coder ssh [flags] <workspace> [command]
 
-  Start a shell into a workspace
+  Start a shell into a workspace or run a command
+
+  This command does not have full parity with the standard SSH command. For
+  users who need the full functionality of SSH, create an ssh configuration with
+  `coder config-ssh`.
+  
+    - Use `--` to separate and pass flags directly to the command executed via
+  SSH.:
+  
+       $ coder ssh <workspace> -- ls -la
 
 OPTIONS:
       --disable-autostart bool, $CODER_SSH_DISABLE_AUTOSTART (default: false)

cli/testdata/coder_users_--help.golden

@@ -10,10 +10,10 @@ USAGE:
 SUBCOMMANDS:
     activate      Update a user's status to 'active'. Active users can fully
                   interact with the platform
-    create        
+    create        Create a new user.
     delete        Delete a user by username or user_id.
     edit-roles    Edit a user's roles by username or id
-    list          
+    list          Prints the list of users.
     show          Show a single user. Use 'me' to indicate the currently
                   authenticated user.
     suspend       Update a user's status to 'suspended'. A suspended user cannot

cli/testdata/coder_users_create_--help.golden

@@ -3,6 +3,8 @@ coder v0.0.0-devel
 USAGE:
   coder users create [flags]
 
+  Create a new user.
+
 OPTIONS:
   -O, --org string, $CODER_ORGANIZATION
           Select which organization (uuid or name) to use.

cli/testdata/coder_users_edit-roles_--help.golden

@@ -8,8 +8,7 @@ USAGE:
 OPTIONS:
       --roles string-array
           A list of roles to give to the user. This removes any existing roles
-          the user may have. The available roles are: auditor, member, owner,
-          template-admin, user-admin.
+          the user may have.
 
   -y, --yes bool
           Bypass prompts.

cli/testdata/coder_users_list_--help.golden

@@ -3,6 +3,8 @@ coder v0.0.0-devel
 USAGE:
   coder users list [flags]
 
+  Prints the list of users.
+
   Aliases: ls
 
 OPTIONS:

cli/testdata/coder_users_list_--output_json.golden

@@ -2,7 +2,6 @@
   {
     "id": "==========[first user ID]===========",
     "username": "testuser",
-    "avatar_url": "",
     "name": "Test User",
     "email": "testuser@coder.com",
     "created_at": "====[timestamp]=====",
@@ -23,8 +22,6 @@
   {
     "id": "==========[second user ID]==========",
     "username": "testuser2",
-    "avatar_url": "",
-    "name": "",
     "email": "testuser2@coder.com",
     "created_at": "====[timestamp]=====",
     "updated_at": "====[timestamp]=====",

cli/testdata/server-config.yaml.golden

@@ -183,7 +183,7 @@ networking:
 # Interval to poll for scheduled workspace builds.
 # (default: 1m0s, type: duration)
 autobuildPollInterval: 1m0s
-# Interval to poll for hung jobs and automatically terminate them.
+# Interval to poll for hung and pending jobs and automatically terminate them.
 # (default: 1m0s, type: duration)
 jobHangDetectorInterval: 1m0s
 introspection:
@@ -519,6 +519,9 @@ client:
 # Support links to display in the top right drop down menu.
 # (default: <unset>, type: struct[[]codersdk.LinkConfig])
 supportLinks: []
+# Configure AI providers.
+# (default: <unset>, type: struct[codersdk.AIConfig])
+ai: {}
 # External Authentication providers.
 # (default: <unset>, type: struct[[]codersdk.ExternalAuthConfig])
 externalAuthProviders: []
@@ -701,3 +704,7 @@ workspace_prebuilds:
   # backoff.
   # (default: 1h0m0s, type: duration)
   reconciliation_backoff_lookback_period: 1h0m0s
+  # Maximum number of consecutive failed prebuilds before a preset hits the hard
+  # limit; disabled when set to zero.
+  # (default: 3, type: int)
+  failure_hard_limit: 3

cli/update_test.go

@@ -757,7 +757,7 @@ func TestUpdateValidateRichParameters(t *testing.T) {
 			err := inv.Run()
 			// TODO: improve validation so we catch this problem before it reaches the server
 			// 		 but for now just validate that the server actually catches invalid monotonicity
-			assert.ErrorContains(t, err, fmt.Sprintf("parameter value must be equal or greater than previous value: %s", tempVal))
+			assert.ErrorContains(t, err, "parameter value '1' must be equal or greater than previous value: 2")
 		}()
 
 		matches := []string{

cli/usercreate.go

@@ -28,7 +28,8 @@ func (r *RootCmd) userCreate() *serpent.Command {
 	)
 	client := new(codersdk.Client)
 	cmd := &serpent.Command{
-		Use: "create",
+		Use:   "create",
+		Short: "Create a new user.",
 		Middleware: serpent.Chain(
 			serpent.RequireNArgs(0),
 			r.InitClient(client),

cli/usereditroles.go

@@ -1,32 +1,19 @@
 package cli
 
 import (
-	"fmt"
 	"slices"
-	"sort"
 	"strings"
 
 	"golang.org/x/xerrors"
 
 	"github.com/coder/coder/v2/cli/cliui"
-	"github.com/coder/coder/v2/coderd/rbac"
 	"github.com/coder/coder/v2/codersdk"
 	"github.com/coder/serpent"
 )
 
 func (r *RootCmd) userEditRoles() *serpent.Command {
 	client := new(codersdk.Client)
-
-	roles := rbac.SiteRoles()
-
-	siteRoles := make([]string, 0)
-	for _, role := range roles {
-		siteRoles = append(siteRoles, role.Identifier.Name)
-	}
-	sort.Strings(siteRoles)
-
 	var givenRoles []string
-
 	cmd := &serpent.Command{
 		Use:   "edit-roles <username|user_id>",
 		Short: "Edit a user's roles by username or id",
@@ -34,7 +21,7 @@ func (r *RootCmd) userEditRoles() *serpent.Command {
 			cliui.SkipPromptOption(),
 			{
 				Name:        "roles",
-				Description: fmt.Sprintf("A list of roles to give to the user. This removes any existing roles the user may have. The available roles are: %s.", strings.Join(siteRoles, ", ")),
+				Description: "A list of roles to give to the user. This removes any existing roles the user may have.",
 				Flag:        "roles",
 				Value:       serpent.StringArrayOf(&givenRoles),
 			},
@@ -52,13 +39,21 @@ func (r *RootCmd) userEditRoles() *serpent.Command {
 			if err != nil {
 				return xerrors.Errorf("fetch user roles: %w", err)
 			}
+			siteRoles, err := client.ListSiteRoles(ctx)
+			if err != nil {
+				return xerrors.Errorf("fetch site roles: %w", err)
+			}
+			siteRoleNames := make([]string, 0, len(siteRoles))
+			for _, role := range siteRoles {
+				siteRoleNames = append(siteRoleNames, role.Name)
+			}
 
 			var selectedRoles []string
 			if len(givenRoles) > 0 {
 				// Make sure all of the given roles are valid site roles
 				for _, givenRole := range givenRoles {
-					if !slices.Contains(siteRoles, givenRole) {
-						siteRolesPretty := strings.Join(siteRoles, ", ")
+					if !slices.Contains(siteRoleNames, givenRole) {
+						siteRolesPretty := strings.Join(siteRoleNames, ", ")
 						return xerrors.Errorf("The role %s is not valid. Please use one or more of the following roles: %s\n", givenRole, siteRolesPretty)
 					}
 				}
@@ -67,7 +62,7 @@ func (r *RootCmd) userEditRoles() *serpent.Command {
 			} else {
 				selectedRoles, err = cliui.MultiSelect(inv, cliui.MultiSelectOptions{
 					Message:  "Select the roles you'd like to assign to the user",
-					Options:  siteRoles,
+					Options:  siteRoleNames,
 					Defaults: userRoles.Roles,
 				})
 				if err != nil {

cli/userlist.go

@@ -23,6 +23,7 @@ func (r *RootCmd) userList() *serpent.Command {
 
 	cmd := &serpent.Command{
 		Use:     "list",
+		Short:   "Prints the list of users.",
 		Aliases: []string{"ls"},
 		Middleware: serpent.Chain(
 			serpent.RequireNArgs(0),

cli/userlist_test.go

@@ -4,6 +4,8 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"fmt"
+	"os"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -69,9 +71,12 @@ func TestUserList(t *testing.T) {
 	t.Run("NoURLFileErrorHasHelperText", func(t *testing.T) {
 		t.Parallel()
 
+		executable, err := os.Executable()
+		require.NoError(t, err)
+
 		inv, _ := clitest.New(t, "users", "list")
-		err := inv.Run()
-		require.Contains(t, err.Error(), "Try logging in using 'coder login <url>'.")
+		err = inv.Run()
+		require.Contains(t, err.Error(), fmt.Sprintf("Try logging in using '%s login <url>'.", executable))
 	})
 	t.Run("SessionAuthErrorHasHelperText", func(t *testing.T) {
 		t.Parallel()

coderd/agentapi/api.go

@@ -30,6 +30,7 @@ import (
 	"github.com/coder/coder/v2/coderd/wspubsub"
 	"github.com/coder/coder/v2/codersdk"
 	"github.com/coder/coder/v2/codersdk/agentsdk"
+	"github.com/coder/coder/v2/codersdk/drpcsdk"
 	"github.com/coder/coder/v2/tailnet"
 	tailnetproto "github.com/coder/coder/v2/tailnet/proto"
 	"github.com/coder/quartz"
@@ -209,6 +210,7 @@ func (a *API) Server(ctx context.Context) (*drpcserver.Server, error) {
 
 	return drpcserver.NewWithOptions(&tracing.DRPCHandler{Handler: mux},
 		drpcserver.Options{
+			Manager: drpcsdk.DefaultDRPCOptions(nil),
 			Log: func(err error) {
 				if xerrors.Is(err, io.EOF) {
 					return

coderd/agentapi/manifest.go

@@ -47,7 +47,6 @@ func (a *ManifestAPI) GetManifest(ctx context.Context, _ *agentproto.GetManifest
 		scripts       []database.WorkspaceAgentScript
 		metadata      []database.WorkspaceAgentMetadatum
 		workspace     database.Workspace
-		owner         database.User
 		devcontainers []database.WorkspaceAgentDevcontainer
 	)
 
@@ -76,10 +75,6 @@ func (a *ManifestAPI) GetManifest(ctx context.Context, _ *agentproto.GetManifest
 		if err != nil {
 			return xerrors.Errorf("getting workspace by id: %w", err)
 		}
-		owner, err = a.Database.GetUserByID(ctx, workspace.OwnerID)
-		if err != nil {
-			return xerrors.Errorf("getting workspace owner by id: %w", err)
-		}
 		return err
 	})
 	eg.Go(func() (err error) {
@@ -98,7 +93,7 @@ func (a *ManifestAPI) GetManifest(ctx context.Context, _ *agentproto.GetManifest
 		AppSlugOrPort: "{{port}}",
 		AgentName:     workspaceAgent.Name,
 		WorkspaceName: workspace.Name,
-		Username:      owner.Username,
+		Username:      workspace.OwnerUsername,
 	}
 
 	vscodeProxyURI := vscodeProxyURI(appSlug, a.AccessURL, a.AppHostname)
@@ -115,15 +110,20 @@ func (a *ManifestAPI) GetManifest(ctx context.Context, _ *agentproto.GetManifest
 		}
 	}
 
-	apps, err := dbAppsToProto(dbApps, workspaceAgent, owner.Username, workspace)
+	apps, err := dbAppsToProto(dbApps, workspaceAgent, workspace.OwnerUsername, workspace)
 	if err != nil {
 		return nil, xerrors.Errorf("converting workspace apps: %w", err)
 	}
 
+	var parentID []byte
+	if workspaceAgent.ParentID.Valid {
+		parentID = workspaceAgent.ParentID.UUID[:]
+	}
+
 	return &agentproto.Manifest{
 		AgentId:                  workspaceAgent.ID[:],
 		AgentName:                workspaceAgent.Name,
-		OwnerUsername:            owner.Username,
+		OwnerUsername:            workspace.OwnerUsername,
 		WorkspaceId:              workspace.ID[:],
 		WorkspaceName:            workspace.Name,
 		GitAuthConfigs:           gitAuthConfigs,
@@ -133,6 +133,7 @@ func (a *ManifestAPI) GetManifest(ctx context.Context, _ *agentproto.GetManifest
 		MotdPath:                 workspaceAgent.MOTDFile,
 		DisableDirectConnections: a.DisableDirectConnections,
 		DerpForceWebsockets:      a.DerpForceWebSockets,
+		ParentId:                 parentID,
 
 		DerpMap:       tailnet.DERPMapToProto(a.DerpMapFn()),
 		Scripts:       dbAgentScriptsToProto(scripts),

coderd/agentapi/manifest_test.go

@@ -46,9 +46,10 @@ func TestGetManifest(t *testing.T) {
 			Username: "cool-user",
 		}
 		workspace = database.Workspace{
-			ID:      uuid.New(),
-			OwnerID: owner.ID,
-			Name:    "cool-workspace",
+			ID:            uuid.New(),
+			OwnerID:       owner.ID,
+			OwnerUsername: owner.Username,
+			Name:          "cool-workspace",
 		}
 		agent = database.WorkspaceAgent{
 			ID:   uuid.New(),
@@ -60,6 +61,13 @@ func TestGetManifest(t *testing.T) {
 			Directory: "/cool/dir",
 			MOTDFile:  "/cool/motd",
 		}
+		childAgent = database.WorkspaceAgent{
+			ID:        uuid.New(),
+			Name:      "cool-child-agent",
+			ParentID:  uuid.NullUUID{Valid: true, UUID: agent.ID},
+			Directory: "/workspace/dir",
+			MOTDFile:  "/workspace/motd",
+		}
 		apps = []database.WorkspaceApp{
 			{
 				ID:                   uuid.New(),
@@ -329,7 +337,6 @@ func TestGetManifest(t *testing.T) {
 		}).Return(metadata, nil)
 		mDB.EXPECT().GetWorkspaceAgentDevcontainersByAgentID(gomock.Any(), agent.ID).Return(devcontainers, nil)
 		mDB.EXPECT().GetWorkspaceByID(gomock.Any(), workspace.ID).Return(workspace, nil)
-		mDB.EXPECT().GetUserByID(gomock.Any(), workspace.OwnerID).Return(owner, nil)
 
 		got, err := api.GetManifest(context.Background(), &agentproto.GetManifestRequest{})
 		require.NoError(t, err)
@@ -337,6 +344,7 @@ func TestGetManifest(t *testing.T) {
 		expected := &agentproto.Manifest{
 			AgentId:                  agent.ID[:],
 			AgentName:                agent.Name,
+			ParentId:                 nil,
 			OwnerUsername:            owner.Username,
 			WorkspaceId:              workspace.ID[:],
 			WorkspaceName:            workspace.Name,
@@ -364,6 +372,69 @@ func TestGetManifest(t *testing.T) {
 		require.Equal(t, expected, got)
 	})
 
+	t.Run("OK/Child", func(t *testing.T) {
+		t.Parallel()
+
+		mDB := dbmock.NewMockStore(gomock.NewController(t))
+
+		api := &agentapi.ManifestAPI{
+			AccessURL:   &url.URL{Scheme: "https", Host: "example.com"},
+			AppHostname: "*--apps.example.com",
+			ExternalAuthConfigs: []*externalauth.Config{
+				{Type: string(codersdk.EnhancedExternalAuthProviderGitHub)},
+				{Type: "some-provider"},
+				{Type: string(codersdk.EnhancedExternalAuthProviderGitLab)},
+			},
+			DisableDirectConnections: true,
+			DerpForceWebSockets:      true,
+
+			AgentFn: func(ctx context.Context) (database.WorkspaceAgent, error) {
+				return childAgent, nil
+			},
+			WorkspaceID: workspace.ID,
+			Database:    mDB,
+			DerpMapFn:   derpMapFn,
+		}
+
+		mDB.EXPECT().GetWorkspaceAppsByAgentID(gomock.Any(), childAgent.ID).Return([]database.WorkspaceApp{}, nil)
+		mDB.EXPECT().GetWorkspaceAgentScriptsByAgentIDs(gomock.Any(), []uuid.UUID{childAgent.ID}).Return([]database.WorkspaceAgentScript{}, nil)
+		mDB.EXPECT().GetWorkspaceAgentMetadata(gomock.Any(), database.GetWorkspaceAgentMetadataParams{
+			WorkspaceAgentID: childAgent.ID,
+			Keys:             nil, // all
+		}).Return([]database.WorkspaceAgentMetadatum{}, nil)
+		mDB.EXPECT().GetWorkspaceAgentDevcontainersByAgentID(gomock.Any(), childAgent.ID).Return([]database.WorkspaceAgentDevcontainer{}, nil)
+		mDB.EXPECT().GetWorkspaceByID(gomock.Any(), workspace.ID).Return(workspace, nil)
+
+		got, err := api.GetManifest(context.Background(), &agentproto.GetManifestRequest{})
+		require.NoError(t, err)
+
+		expected := &agentproto.Manifest{
+			AgentId:                  childAgent.ID[:],
+			AgentName:                childAgent.Name,
+			ParentId:                 agent.ID[:],
+			OwnerUsername:            owner.Username,
+			WorkspaceId:              workspace.ID[:],
+			WorkspaceName:            workspace.Name,
+			GitAuthConfigs:           2, // two "enhanced" external auth configs
+			EnvironmentVariables:     nil,
+			Directory:                childAgent.Directory,
+			VsCodePortProxyUri:       fmt.Sprintf("https://{{port}}--%s--%s--%s--apps.example.com", childAgent.Name, workspace.Name, owner.Username),
+			MotdPath:                 childAgent.MOTDFile,
+			DisableDirectConnections: true,
+			DerpForceWebsockets:      true,
+			// tailnet.DERPMapToProto() is extensively tested elsewhere, so it's
+			// not necessary to manually recreate a big DERP map here like we
+			// did for apps and metadata.
+			DerpMap:       tailnet.DERPMapToProto(derpMapFn()),
+			Scripts:       []*agentproto.WorkspaceAgentScript{},
+			Apps:          []*agentproto.WorkspaceApp{},
+			Metadata:      []*agentproto.WorkspaceAgentMetadata_Description{},
+			Devcontainers: []*agentproto.WorkspaceAgentDevcontainer{},
+		}
+
+		require.Equal(t, expected, got)
+	})
+
 	t.Run("NoAppHostname", func(t *testing.T) {
 		t.Parallel()
 
@@ -396,7 +467,6 @@ func TestGetManifest(t *testing.T) {
 		}).Return(metadata, nil)
 		mDB.EXPECT().GetWorkspaceAgentDevcontainersByAgentID(gomock.Any(), agent.ID).Return(devcontainers, nil)
 		mDB.EXPECT().GetWorkspaceByID(gomock.Any(), workspace.ID).Return(workspace, nil)
-		mDB.EXPECT().GetUserByID(gomock.Any(), workspace.OwnerID).Return(owner, nil)
 
 		got, err := api.GetManifest(context.Background(), &agentproto.GetManifestRequest{})
 		require.NoError(t, err)

coderd/ai/ai.go

@@ -0,0 +1,167 @@
+package ai
+
+import (
+	"context"
+
+	"github.com/anthropics/anthropic-sdk-go"
+	anthropicoption "github.com/anthropics/anthropic-sdk-go/option"
+	"github.com/kylecarbs/aisdk-go"
+	"github.com/openai/openai-go"
+	openaioption "github.com/openai/openai-go/option"
+	"golang.org/x/xerrors"
+	"google.golang.org/genai"
+
+	"github.com/coder/coder/v2/codersdk"
+)
+
+type LanguageModel struct {
+	codersdk.LanguageModel
+	StreamFunc StreamFunc
+}
+
+type StreamOptions struct {
+	SystemPrompt string
+	Model        string
+	Messages     []aisdk.Message
+	Thinking     bool
+	Tools        []aisdk.Tool
+}
+
+type StreamFunc func(ctx context.Context, options StreamOptions) (aisdk.DataStream, error)
+
+// LanguageModels is a map of language model ID to language model.
+type LanguageModels map[string]LanguageModel
+
+func ModelsFromConfig(ctx context.Context, configs []codersdk.AIProviderConfig) (LanguageModels, error) {
+	models := make(LanguageModels)
+
+	for _, config := range configs {
+		var streamFunc StreamFunc
+
+		switch config.Type {
+		case "openai":
+			opts := []openaioption.RequestOption{
+				openaioption.WithAPIKey(config.APIKey),
+			}
+			if config.BaseURL != "" {
+				opts = append(opts, openaioption.WithBaseURL(config.BaseURL))
+			}
+			client := openai.NewClient(opts...)
+			streamFunc = func(ctx context.Context, options StreamOptions) (aisdk.DataStream, error) {
+				openaiMessages, err := aisdk.MessagesToOpenAI(options.Messages)
+				if err != nil {
+					return nil, err
+				}
+				tools := aisdk.ToolsToOpenAI(options.Tools)
+				if options.SystemPrompt != "" {
+					openaiMessages = append([]openai.ChatCompletionMessageParamUnion{
+						openai.SystemMessage(options.SystemPrompt),
+					}, openaiMessages...)
+				}
+
+				return aisdk.OpenAIToDataStream(client.Chat.Completions.NewStreaming(ctx, openai.ChatCompletionNewParams{
+					Messages:  openaiMessages,
+					Model:     options.Model,
+					Tools:     tools,
+					MaxTokens: openai.Int(8192),
+				})), nil
+			}
+			if config.Models == nil {
+				models, err := client.Models.List(ctx)
+				if err != nil {
+					return nil, err
+				}
+				config.Models = make([]string, len(models.Data))
+				for i, model := range models.Data {
+					config.Models[i] = model.ID
+				}
+			}
+		case "anthropic":
+			client := anthropic.NewClient(anthropicoption.WithAPIKey(config.APIKey))
+			streamFunc = func(ctx context.Context, options StreamOptions) (aisdk.DataStream, error) {
+				anthropicMessages, systemMessage, err := aisdk.MessagesToAnthropic(options.Messages)
+				if err != nil {
+					return nil, err
+				}
+				if options.SystemPrompt != "" {
+					systemMessage = []anthropic.TextBlockParam{
+						*anthropic.NewTextBlock(options.SystemPrompt).OfRequestTextBlock,
+					}
+				}
+				return aisdk.AnthropicToDataStream(client.Messages.NewStreaming(ctx, anthropic.MessageNewParams{
+					Messages:  anthropicMessages,
+					Model:     options.Model,
+					System:    systemMessage,
+					Tools:     aisdk.ToolsToAnthropic(options.Tools),
+					MaxTokens: 8192,
+				})), nil
+			}
+			if config.Models == nil {
+				models, err := client.Models.List(ctx, anthropic.ModelListParams{})
+				if err != nil {
+					return nil, err
+				}
+				config.Models = make([]string, len(models.Data))
+				for i, model := range models.Data {
+					config.Models[i] = model.ID
+				}
+			}
+		case "google":
+			client, err := genai.NewClient(ctx, &genai.ClientConfig{
+				APIKey:  config.APIKey,
+				Backend: genai.BackendGeminiAPI,
+			})
+			if err != nil {
+				return nil, err
+			}
+			streamFunc = func(ctx context.Context, options StreamOptions) (aisdk.DataStream, error) {
+				googleMessages, err := aisdk.MessagesToGoogle(options.Messages)
+				if err != nil {
+					return nil, err
+				}
+				tools, err := aisdk.ToolsToGoogle(options.Tools)
+				if err != nil {
+					return nil, err
+				}
+				var systemInstruction *genai.Content
+				if options.SystemPrompt != "" {
+					systemInstruction = &genai.Content{
+						Parts: []*genai.Part{
+							genai.NewPartFromText(options.SystemPrompt),
+						},
+						Role: "model",
+					}
+				}
+				return aisdk.GoogleToDataStream(client.Models.GenerateContentStream(ctx, options.Model, googleMessages, &genai.GenerateContentConfig{
+					SystemInstruction: systemInstruction,
+					Tools:             tools,
+				})), nil
+			}
+			if config.Models == nil {
+				models, err := client.Models.List(ctx, &genai.ListModelsConfig{})
+				if err != nil {
+					return nil, err
+				}
+				config.Models = make([]string, len(models.Items))
+				for i, model := range models.Items {
+					config.Models[i] = model.Name
+				}
+			}
+		default:
+			return nil, xerrors.Errorf("unsupported model type: %s", config.Type)
+		}
+
+		for _, model := range config.Models {
+			models[model] = LanguageModel{
+				LanguageModel: codersdk.LanguageModel{
+					ID:          model,
+					DisplayName: model,
+					Provider:    config.Type,
+				},
+				StreamFunc: streamFunc,
+			}
+		}
+	}
+
+	return models, nil
+}

coderd/apidoc/swagger.json

@@ -291,6 +291,151 @@
 				}
 			}
 		},
+		"/chats": {
+			"get": {
+				"security": [
+					{
+						"CoderSessionToken": []
+					}
+				],
+				"produces": ["application/json"],
+				"tags": ["Chat"],
+				"summary": "List chats",
+				"operationId": "list-chats",
+				"responses": {
+					"200": {
+						"description": "OK",
+						"schema": {
+							"type": "array",
+							"items": {
+								"$ref": "#/definitions/codersdk.Chat"
+							}
+						}
+					}
+				}
+			},
+			"post": {
+				"security": [
+					{
+						"CoderSessionToken": []
+					}
+				],
+				"produces": ["application/json"],
+				"tags": ["Chat"],
+				"summary": "Create a chat",
+				"operationId": "create-a-chat",
+				"responses": {
+					"201": {
+						"description": "Created",
+						"schema": {
+							"$ref": "#/definitions/codersdk.Chat"
+						}
+					}
+				}
+			}
+		},
+		"/chats/{chat}": {
+			"get": {
+				"security": [
+					{
+						"CoderSessionToken": []
+					}
+				],
+				"produces": ["application/json"],
+				"tags": ["Chat"],
+				"summary": "Get a chat",
+				"operationId": "get-a-chat",
+				"parameters": [
+					{
+						"type": "string",
+						"description": "Chat ID",
+						"name": "chat",
+						"in": "path",
+						"required": true
+					}
+				],
+				"responses": {
+					"200": {
+						"description": "OK",
+						"schema": {
+							"$ref": "#/definitions/codersdk.Chat"
+						}
+					}
+				}
+			}
+		},
+		"/chats/{chat}/messages": {
+			"get": {
+				"security": [
+					{
+						"CoderSessionToken": []
+					}
+				],
+				"produces": ["application/json"],
+				"tags": ["Chat"],
+				"summary": "Get chat messages",
+				"operationId": "get-chat-messages",
+				"parameters": [
+					{
+						"type": "string",
+						"description": "Chat ID",
+						"name": "chat",
+						"in": "path",
+						"required": true
+					}
+				],
+				"responses": {
+					"200": {
+						"description": "OK",
+						"schema": {
+							"type": "array",
+							"items": {
+								"$ref": "#/definitions/aisdk.Message"
+							}
+						}
+					}
+				}
+			},
+			"post": {
+				"security": [
+					{
+						"CoderSessionToken": []
+					}
+				],
+				"consumes": ["application/json"],
+				"produces": ["application/json"],
+				"tags": ["Chat"],
+				"summary": "Create a chat message",
+				"operationId": "create-a-chat-message",
+				"parameters": [
+					{
+						"type": "string",
+						"description": "Chat ID",
+						"name": "chat",
+						"in": "path",
+						"required": true
+					},
+					{
+						"description": "Request body",
+						"name": "request",
+						"in": "body",
+						"required": true,
+						"schema": {
+							"$ref": "#/definitions/codersdk.CreateChatMessageRequest"
+						}
+					}
+				],
+				"responses": {
+					"200": {
+						"description": "OK",
+						"schema": {
+							"type": "array",
+							"items": {}
+						}
+					}
+				}
+			}
+		},
 		"/csp/reports": {
 			"post": {
 				"security": [
@@ -563,6 +708,27 @@
 				}
 			}
 		},
+		"/deployment/llms": {
+			"get": {
+				"security": [
+					{
+						"CoderSessionToken": []
+					}
+				],
+				"produces": ["application/json"],
+				"tags": ["General"],
+				"summary": "Get language models",
+				"operationId": "get-language-models",
+				"responses": {
+					"200": {
+						"description": "OK",
+						"schema": {
+							"$ref": "#/definitions/codersdk.LanguageModelConfig"
+						}
+					}
+				}
+			}
+		},
 		"/deployment/ssh": {
 			"get": {
 				"security": [
@@ -3462,6 +3628,7 @@
 						"CoderSessionToken": []
 					}
 				],
+				"description": "Returns a list of templates for the specified organization.\nBy default, only non-deprecated templates are returned.\nTo include deprecated templates, specify `deprecated:true` in the search query.",
 				"produces": ["application/json"],
 				"tags": ["Templates"],
 				"summary": "Get templates by organization",
@@ -4189,6 +4356,7 @@
 						"CoderSessionToken": []
 					}
 				],
+				"description": "Returns a list of templates.\nBy default, only non-deprecated templates are returned.\nTo include deprecated templates, specify `deprecated:true` in the search query.",
 				"produces": ["application/json"],
 				"tags": ["Templates"],
 				"summary": "Get all templates",
@@ -5029,6 +5197,41 @@
 				}
 			}
 		},
+		"/templateversions/{templateversion}/dynamic-parameters": {
+			"get": {
+				"security": [
+					{
+						"CoderSessionToken": []
+					}
+				],
+				"tags": ["Templates"],
+				"summary": "Open dynamic parameters WebSocket by template version",
+				"operationId": "open-dynamic-parameters-websocket-by-template-version",
+				"parameters": [
+					{
+						"type": "string",
+						"format": "uuid",
+						"description": "Template version ID",
+						"name": "user",
+						"in": "path",
+						"required": true
+					},
+					{
+						"type": "string",
+						"format": "uuid",
+						"description": "Template version ID",
+						"name": "templateversion",
+						"in": "path",
+						"required": true
+					}
+				],
+				"responses": {
+					"101": {
+						"description": "Switching Protocols"
+					}
+				}
+			}
+		},
 		"/templateversions/{templateversion}/external-auth": {
 			"get": {
 				"security": [
@@ -6666,41 +6869,6 @@
 				}
 			}
 		},
-		"/users/{user}/templateversions/{templateversion}/parameters": {
-			"get": {
-				"security": [
-					{
-						"CoderSessionToken": []
-					}
-				],
-				"tags": ["Templates"],
-				"summary": "Open dynamic parameters WebSocket by template version",
-				"operationId": "open-dynamic-parameters-websocket-by-template-version",
-				"parameters": [
-					{
-						"type": "string",
-						"format": "uuid",
-						"description": "Template version ID",
-						"name": "user",
-						"in": "path",
-						"required": true
-					},
-					{
-						"type": "string",
-						"format": "uuid",
-						"description": "Template version ID",
-						"name": "templateversion",
-						"in": "path",
-						"required": true
-					}
-				],
-				"responses": {
-					"101": {
-						"description": "Switching Protocols"
-					}
-				}
-			}
-		},
 		"/users/{user}/webpush/subscription": {
 			"post": {
 				"security": [
@@ -7295,6 +7463,27 @@
 				}
 			}
 		},
+		"/workspaceagents/me/reinit": {
+			"get": {
+				"security": [
+					{
+						"CoderSessionToken": []
+					}
+				],
+				"produces": ["application/json"],
+				"tags": ["Agents"],
+				"summary": "Get workspace agent reinitialization",
+				"operationId": "get-workspace-agent-reinitialization",
+				"responses": {
+					"200": {
+						"description": "OK",
+						"schema": {
+							"$ref": "#/definitions/agentsdk.ReinitializationEvent"
+						}
+					}
+				}
+			}
+		},
 		"/workspaceagents/me/rpc": {
 			"get": {
 				"security": [
@@ -7416,6 +7605,44 @@
 				}
 			}
 		},
+		"/workspaceagents/{workspaceagent}/containers/devcontainers/container/{container}/recreate": {
+			"post": {
+				"security": [
+					{
+						"CoderSessionToken": []
+					}
+				],
+				"produces": ["application/json"],
+				"tags": ["Agents"],
+				"summary": "Recreate devcontainer for workspace agent",
+				"operationId": "recreate-devcontainer-for-workspace-agent",
+				"parameters": [
+					{
+						"type": "string",
+						"format": "uuid",
+						"description": "Workspace agent ID",
+						"name": "workspaceagent",
+						"in": "path",
+						"required": true
+					},
+					{
+						"type": "string",
+						"description": "Container ID or name",
+						"name": "container",
+						"in": "path",
+						"required": true
+					}
+				],
+				"responses": {
+					"202": {
+						"description": "Accepted",
+						"schema": {
+							"$ref": "#/definitions/codersdk.Response"
+						}
+					}
+				}
+			}
+		},
 		"/workspaceagents/{workspaceagent}/coordinate": {
 			"get": {
 				"security": [
@@ -9134,6 +9361,202 @@
 				}
 			}
 		},
+		"agentsdk.ReinitializationEvent": {
+			"type": "object",
+			"properties": {
+				"reason": {
+					"$ref": "#/definitions/agentsdk.ReinitializationReason"
+				},
+				"workspaceID": {
+					"type": "string"
+				}
+			}
+		},
+		"agentsdk.ReinitializationReason": {
+			"type": "string",
+			"enum": ["prebuild_claimed"],
+			"x-enum-varnames": ["ReinitializeReasonPrebuildClaimed"]
+		},
+		"aisdk.Attachment": {
+			"type": "object",
+			"properties": {
+				"contentType": {
+					"type": "string"
+				},
+				"name": {
+					"type": "string"
+				},
+				"url": {
+					"type": "string"
+				}
+			}
+		},
+		"aisdk.Message": {
+			"type": "object",
+			"properties": {
+				"annotations": {
+					"type": "array",
+					"items": {}
+				},
+				"content": {
+					"type": "string"
+				},
+				"createdAt": {
+					"type": "array",
+					"items": {
+						"type": "integer"
+					}
+				},
+				"experimental_attachments": {
+					"type": "array",
+					"items": {
+						"$ref": "#/definitions/aisdk.Attachment"
+					}
+				},
+				"id": {
+					"type": "string"
+				},
+				"parts": {
+					"type": "array",
+					"items": {
+						"$ref": "#/definitions/aisdk.Part"
+					}
+				},
+				"role": {
+					"type": "string"
+				}
+			}
+		},
+		"aisdk.Part": {
+			"type": "object",
+			"properties": {
+				"data": {
+					"type": "array",
+					"items": {
+						"type": "integer"
+					}
+				},
+				"details": {
+					"type": "array",
+					"items": {
+						"$ref": "#/definitions/aisdk.ReasoningDetail"
+					}
+				},
+				"mimeType": {
+					"description": "Type: \"file\"",
+					"type": "string"
+				},
+				"reasoning": {
+					"description": "Type: \"reasoning\"",
+					"type": "string"
+				},
+				"source": {
+					"description": "Type: \"source\"",
+					"allOf": [
+						{
+							"$ref": "#/definitions/aisdk.SourceInfo"
+						}
+					]
+				},
+				"text": {
+					"description": "Type: \"text\"",
+					"type": "string"
+				},
+				"toolInvocation": {
+					"description": "Type: \"tool-invocation\"",
+					"allOf": [
+						{
+							"$ref": "#/definitions/aisdk.ToolInvocation"
+						}
+					]
+				},
+				"type": {
+					"$ref": "#/definitions/aisdk.PartType"
+				}
+			}
+		},
+		"aisdk.PartType": {
+			"type": "string",
+			"enum": [
+				"text",
+				"reasoning",
+				"tool-invocation",
+				"source",
+				"file",
+				"step-start"
+			],
+			"x-enum-varnames": [
+				"PartTypeText",
+				"PartTypeReasoning",
+				"PartTypeToolInvocation",
+				"PartTypeSource",
+				"PartTypeFile",
+				"PartTypeStepStart"
+			]
+		},
+		"aisdk.ReasoningDetail": {
+			"type": "object",
+			"properties": {
+				"data": {
+					"type": "string"
+				},
+				"signature": {
+					"type": "string"
+				},
+				"text": {
+					"type": "string"
+				},
+				"type": {
+					"type": "string"
+				}
+			}
+		},
+		"aisdk.SourceInfo": {
+			"type": "object",
+			"properties": {
+				"contentType": {
+					"type": "string"
+				},
+				"data": {
+					"type": "string"
+				},
+				"metadata": {
+					"type": "object",
+					"additionalProperties": {}
+				},
+				"uri": {
+					"type": "string"
+				}
+			}
+		},
+		"aisdk.ToolInvocation": {
+			"type": "object",
+			"properties": {
+				"args": {},
+				"result": {},
+				"state": {
+					"$ref": "#/definitions/aisdk.ToolInvocationState"
+				},
+				"step": {
+					"type": "integer"
+				},
+				"toolCallId": {
+					"type": "string"
+				},
+				"toolName": {
+					"type": "string"
+				}
+			}
+		},
+		"aisdk.ToolInvocationState": {
+			"type": "string",
+			"enum": ["call", "partial-call", "result"],
+			"x-enum-varnames": [
+				"ToolInvocationStateCall",
+				"ToolInvocationStatePartialCall",
+				"ToolInvocationStateResult"
+			]
+		},
 		"coderd.SCIMUser": {
 			"type": "object",
 			"properties": {
@@ -9225,6 +9648,37 @@
 				}
 			}
 		},
+		"codersdk.AIConfig": {
+			"type": "object",
+			"properties": {
+				"providers": {
+					"type": "array",
+					"items": {
+						"$ref": "#/definitions/codersdk.AIProviderConfig"
+					}
+				}
+			}
+		},
+		"codersdk.AIProviderConfig": {
+			"type": "object",
+			"properties": {
+				"base_url": {
+					"description": "BaseURL is the base URL to use for the API provider.",
+					"type": "string"
+				},
+				"models": {
+					"description": "Models is the list of models to use for the API provider.",
+					"type": "array",
+					"items": {
+						"type": "string"
+					}
+				},
+				"type": {
+					"description": "Type is the type of the API provider.",
+					"type": "string"
+				}
+			}
+		},
 		"codersdk.APIKey": {
 			"type": "object",
 			"required": [
@@ -9771,6 +10225,62 @@
 				}
 			}
 		},
+		"codersdk.Chat": {
+			"type": "object",
+			"properties": {
+				"created_at": {
+					"type": "string",
+					"format": "date-time"
+				},
+				"id": {
+					"type": "string",
+					"format": "uuid"
+				},
+				"title": {
+					"type": "string"
+				},
+				"updated_at": {
+					"type": "string",
+					"format": "date-time"
+				}
+			}
+		},
+		"codersdk.ChatMessage": {
+			"type": "object",
+			"properties": {
+				"annotations": {
+					"type": "array",
+					"items": {}
+				},
+				"content": {
+					"type": "string"
+				},
+				"createdAt": {
+					"type": "array",
+					"items": {
+						"type": "integer"
+					}
+				},
+				"experimental_attachments": {
+					"type": "array",
+					"items": {
+						"$ref": "#/definitions/aisdk.Attachment"
+					}
+				},
+				"id": {
+					"type": "string"
+				},
+				"parts": {
+					"type": "array",
+					"items": {
+						"$ref": "#/definitions/aisdk.Part"
+					}
+				},
+				"role": {
+					"type": "string"
+				}
+			}
+		},
 		"codersdk.ConnectionLatency": {
 			"type": "object",
 			"properties": {
@@ -9801,6 +10311,20 @@
 				}
 			}
 		},
+		"codersdk.CreateChatMessageRequest": {
+			"type": "object",
+			"properties": {
+				"message": {
+					"$ref": "#/definitions/codersdk.ChatMessage"
+				},
+				"model": {
+					"type": "string"
+				},
+				"thinking": {
+					"type": "boolean"
+				}
+			}
+		},
 		"codersdk.CreateFirstUserRequest": {
 			"type": "object",
 			"required": ["email", "password", "username"],
@@ -10069,7 +10593,63 @@
 			}
 		},
 		"codersdk.CreateTestAuditLogRequest": {
-			"type": "object"
+			"type": "object",
+			"properties": {
+				"action": {
+					"enum": ["create", "write", "delete", "start", "stop"],
+					"allOf": [
+						{
+							"$ref": "#/definitions/codersdk.AuditAction"
+						}
+					]
+				},
+				"additional_fields": {
+					"type": "array",
+					"items": {
+						"type": "integer"
+					}
+				},
+				"build_reason": {
+					"enum": ["autostart", "autostop", "initiator"],
+					"allOf": [
+						{
+							"$ref": "#/definitions/codersdk.BuildReason"
+						}
+					]
+				},
+				"organization_id": {
+					"type": "string",
+					"format": "uuid"
+				},
+				"request_id": {
+					"type": "string",
+					"format": "uuid"
+				},
+				"resource_id": {
+					"type": "string",
+					"format": "uuid"
+				},
+				"resource_type": {
+					"enum": [
+						"template",
+						"template_version",
+						"user",
+						"workspace",
+						"workspace_build",
+						"git_ssh_key",
+						"auditable_group"
+					],
+					"allOf": [
+						{
+							"$ref": "#/definitions/codersdk.ResourceType"
+						}
+					]
+				},
+				"time": {
+					"type": "string",
+					"format": "date-time"
+				}
+			}
 		},
 		"codersdk.CreateTokenRequest": {
 			"type": "object",
@@ -10140,6 +10720,10 @@
 				"dry_run": {
 					"type": "boolean"
 				},
+				"enable_dynamic_parameters": {
+					"description": "EnableDynamicParameters skips some of the static parameter checking.\nIt will default to whatever the template has marked as the default experience.\nRequires the \"dynamic-experiment\" to be used.",
+					"type": "boolean"
+				},
 				"log_level": {
 					"description": "Log level changes the default logging verbosity of a provider (\"info\" if empty).",
 					"enum": ["debug"],
@@ -10500,6 +11084,9 @@
 				"agent_stat_refresh_interval": {
 					"type": "integer"
 				},
+				"ai": {
+					"$ref": "#/definitions/serpent.Struct-codersdk_AIConfig"
+				},
 				"allow_workspace_renames": {
 					"type": "boolean"
 				},
@@ -10763,9 +11350,13 @@
 				"workspace-usage",
 				"web-push",
 				"dynamic-parameters",
-				"workspace-prebuilds"
+				"workspace-prebuilds",
+				"agentic-chat",
+				"ai-tasks"
 			],
 			"x-enum-comments": {
+				"ExperimentAITasks": "Enables the new AI tasks feature.",
+				"ExperimentAgenticChat": "Enables the new agentic AI chat feature.",
 				"ExperimentAutoFillParameters": "This should not be taken out of experiments until we have redesigned the feature.",
 				"ExperimentDynamicParameters": "Enables dynamic parameters when creating a workspace.",
 				"ExperimentExample": "This isn't used for anything.",
@@ -10781,7 +11372,9 @@
 				"ExperimentWorkspaceUsage",
 				"ExperimentWebPush",
 				"ExperimentDynamicParameters",
-				"ExperimentWorkspacePrebuilds"
+				"ExperimentWorkspacePrebuilds",
+				"ExperimentAgenticChat",
+				"ExperimentAITasks"
 			]
 		},
 		"codersdk.ExternalAuth": {
@@ -11276,6 +11869,33 @@
 			"enum": ["REQUIRED_TEMPLATE_VARIABLES"],
 			"x-enum-varnames": ["RequiredTemplateVariables"]
 		},
+		"codersdk.LanguageModel": {
+			"type": "object",
+			"properties": {
+				"display_name": {
+					"type": "string"
+				},
+				"id": {
+					"description": "ID is used by the provider to identify the LLM.",
+					"type": "string"
+				},
+				"provider": {
+					"description": "Provider is the provider of the LLM. e.g. openai, anthropic, etc.",
+					"type": "string"
+				}
+			}
+		},
+		"codersdk.LanguageModelConfig": {
+			"type": "object",
+			"properties": {
+				"models": {
+					"type": "array",
+					"items": {
+						"$ref": "#/definitions/codersdk.LanguageModel"
+					}
+				}
+			}
+		},
 		"codersdk.License": {
 			"type": "object",
 			"properties": {
@@ -12355,6 +12975,10 @@
 		"codersdk.PrebuildsConfig": {
 			"type": "object",
 			"properties": {
+				"failure_hard_limit": {
+					"description": "FailureHardLimit defines the maximum number of consecutive failed prebuild attempts allowed\nbefore a preset is considered to be in a hard limit state. When a preset hits this limit,\nno new prebuilds will be created until the limit is reset.\nFailureHardLimit is disabled when set to zero.",
+					"type": "integer"
+				},
 				"reconciliation_backoff_interval": {
 					"description": "ReconciliationBackoffInterval specifies the amount of time to increase the backoff interval\nwhen errors occur during reconciliation.",
 					"type": "integer"
@@ -12641,6 +13265,9 @@
 				"worker_id": {
 					"type": "string",
 					"format": "uuid"
+				},
+				"worker_name": {
+					"type": "string"
 				}
 			}
 		},
@@ -12893,7 +13520,9 @@
 				"application_connect",
 				"assign",
 				"create",
+				"create_agent",
 				"delete",
+				"delete_agent",
 				"read",
 				"read_personal",
 				"ssh",
@@ -12909,7 +13538,9 @@
 				"ActionApplicationConnect",
 				"ActionAssign",
 				"ActionCreate",
+				"ActionCreateAgent",
 				"ActionDelete",
+				"ActionDeleteAgent",
 				"ActionRead",
 				"ActionReadPersonal",
 				"ActionSSH",
@@ -12930,6 +13561,7 @@
 				"assign_org_role",
 				"assign_role",
 				"audit_log",
+				"chat",
 				"crypto_key",
 				"debug_info",
 				"deployment_config",
@@ -12968,6 +13600,7 @@
 				"ResourceAssignOrgRole",
 				"ResourceAssignRole",
 				"ResourceAuditLog",
+				"ResourceChat",
 				"ResourceCryptoKey",
 				"ResourceDebugInfo",
 				"ResourceDeploymentConfig",
@@ -13590,6 +14223,9 @@
 				"updated_at": {
 					"type": "string",
 					"format": "date-time"
+				},
+				"use_classic_parameter_flow": {
+					"type": "boolean"
 				}
 			}
 		},
@@ -14871,6 +15507,7 @@
 					"format": "uuid"
 				},
 				"owner_name": {
+					"description": "OwnerName is the username of the owner of the workspace.",
 					"type": "string"
 				},
 				"template_active_version_id": {
@@ -14896,6 +15533,9 @@
 				"template_require_active_version": {
 					"type": "boolean"
 				},
+				"template_use_classic_parameter_flow": {
+					"type": "boolean"
+				},
 				"ttl_ms": {
 					"type": "integer"
 				},
@@ -15000,6 +15640,14 @@
 				"operating_system": {
 					"type": "string"
 				},
+				"parent_id": {
+					"format": "uuid",
+					"allOf": [
+						{
+							"$ref": "#/definitions/uuid.NullUUID"
+						}
+					]
+				},
 				"ready_at": {
 					"type": "string",
 					"format": "date-time"
@@ -15055,6 +15703,18 @@
 					"type": "string",
 					"format": "date-time"
 				},
+				"devcontainer_dirty": {
+					"description": "DevcontainerDirty is true if the devcontainer configuration has changed\nsince the container was created. This is used to determine if the\ncontainer needs to be rebuilt.",
+					"type": "boolean"
+				},
+				"devcontainer_status": {
+					"description": "DevcontainerStatus is the status of the devcontainer, if this\ncontainer is a devcontainer. This is used to determine if the\ndevcontainer is running, stopped, starting, or in an error state.",
+					"allOf": [
+						{
+							"$ref": "#/definitions/codersdk.WorkspaceAgentDevcontainerStatus"
+						}
+					]
+				},
 				"id": {
 					"description": "ID is the unique identifier of the container.",
 					"type": "string"
@@ -15119,6 +15779,16 @@
 				}
 			}
 		},
+		"codersdk.WorkspaceAgentDevcontainerStatus": {
+			"type": "string",
+			"enum": ["running", "stopped", "starting", "error"],
+			"x-enum-varnames": [
+				"WorkspaceAgentDevcontainerStatusRunning",
+				"WorkspaceAgentDevcontainerStatusStopped",
+				"WorkspaceAgentDevcontainerStatusStarting",
+				"WorkspaceAgentDevcontainerStatusError"
+			]
+		},
 		"codersdk.WorkspaceAgentHealth": {
 			"type": "object",
 			"properties": {
@@ -15379,6 +16049,9 @@
 					"description": "External specifies whether the URL should be opened externally on\nthe client or not.",
 					"type": "boolean"
 				},
+				"group": {
+					"type": "string"
+				},
 				"health": {
 					"$ref": "#/definitions/codersdk.WorkspaceAppHealth"
 				},
@@ -15625,6 +16298,7 @@
 					"format": "uuid"
 				},
 				"workspace_owner_name": {
+					"description": "WorkspaceOwnerName is the username of the owner of the workspace.",
 					"type": "string"
 				}
 			}
@@ -16705,6 +17379,14 @@
 				}
 			}
 		},
+		"serpent.Struct-codersdk_AIConfig": {
+			"type": "object",
+			"properties": {
+				"value": {
+					"$ref": "#/definitions/codersdk.AIConfig"
+				}
+			}
+		},
 		"serpent.URL": {
 			"type": "object",
 			"properties": {
@@ -16896,6 +17578,18 @@
 		"url.Userinfo": {
 			"type": "object"
 		},
+		"uuid.NullUUID": {
+			"type": "object",
+			"properties": {
+				"uuid": {
+					"type": "string"
+				},
+				"valid": {
+					"description": "Valid is true if UUID is not NULL",
+					"type": "boolean"
+				}
+			}
+		},
 		"workspaceapps.AccessMethod": {
 			"type": "string",
 			"enum": ["path", "subdomain", "terminal"],

coderd/audit.go

@@ -462,7 +462,7 @@ func (api *API) auditLogResourceLink(ctx context.Context, alog database.GetAudit
 		if getWorkspaceErr != nil {
 			return ""
 		}
-		return fmt.Sprintf("/@%s/%s", workspace.OwnerUsername, workspace.Name)
+		return fmt.Sprintf("/@%s/%s", workspace.OwnerName, workspace.Name)
 
 	case database.ResourceTypeWorkspaceApp:
 		if additionalFields.WorkspaceOwner != "" && additionalFields.WorkspaceName != "" {
@@ -472,7 +472,7 @@ func (api *API) auditLogResourceLink(ctx context.Context, alog database.GetAudit
 		if getWorkspaceErr != nil {
 			return ""
 		}
-		return fmt.Sprintf("/@%s/%s", workspace.OwnerUsername, workspace.Name)
+		return fmt.Sprintf("/@%s/%s", workspace.OwnerName, workspace.Name)
 
 	case database.ResourceTypeOauth2ProviderApp:
 		return fmt.Sprintf("/deployment/oauth2-provider/apps/%s", alog.AuditLog.ResourceID)

coderd/autobuild/lifecycle_executor.go

@@ -27,6 +27,7 @@ import (
 	"github.com/coder/coder/v2/coderd/notifications"
 	"github.com/coder/coder/v2/coderd/schedule"
 	"github.com/coder/coder/v2/coderd/wsbuilder"
+	"github.com/coder/coder/v2/codersdk"
 )
 
 // Executor automatically starts or stops workspaces.
@@ -43,6 +44,7 @@ type Executor struct {
 	// NotificationsEnqueuer handles enqueueing notifications for delivery by SMTP, webhook, etc.
 	notificationsEnqueuer notifications.Enqueuer
 	reg                   prometheus.Registerer
+	experiments           codersdk.Experiments
 
 	metrics executorMetrics
 }
@@ -59,7 +61,7 @@ type Stats struct {
 }
 
 // New returns a new wsactions executor.
-func NewExecutor(ctx context.Context, db database.Store, ps pubsub.Pubsub, reg prometheus.Registerer, tss *atomic.Pointer[schedule.TemplateScheduleStore], auditor *atomic.Pointer[audit.Auditor], acs *atomic.Pointer[dbauthz.AccessControlStore], log slog.Logger, tick <-chan time.Time, enqueuer notifications.Enqueuer) *Executor {
+func NewExecutor(ctx context.Context, db database.Store, ps pubsub.Pubsub, reg prometheus.Registerer, tss *atomic.Pointer[schedule.TemplateScheduleStore], auditor *atomic.Pointer[audit.Auditor], acs *atomic.Pointer[dbauthz.AccessControlStore], log slog.Logger, tick <-chan time.Time, enqueuer notifications.Enqueuer, exp codersdk.Experiments) *Executor {
 	factory := promauto.With(reg)
 	le := &Executor{
 		//nolint:gocritic // Autostart has a limited set of permissions.
@@ -73,6 +75,7 @@ func NewExecutor(ctx context.Context, db database.Store, ps pubsub.Pubsub, reg p
 		accessControlStore:    acs,
 		notificationsEnqueuer: enqueuer,
 		reg:                   reg,
+		experiments:           exp,
 		metrics: executorMetrics{
 			autobuildExecutionDuration: factory.NewHistogram(prometheus.HistogramOpts{
 				Namespace: "coderd",
@@ -258,6 +261,7 @@ func (e *Executor) runOnce(t time.Time) Stats {
 						builder := wsbuilder.New(ws, nextTransition).
 							SetLastWorkspaceBuildInTx(&latestBuild).
 							SetLastWorkspaceBuildJobInTx(&latestJob).
+							Experiments(e.experiments).
 							Reason(reason)
 						log.Debug(e.ctx, "auto building workspace", slog.F("transition", nextTransition))
 						if nextTransition == database.WorkspaceTransitionStart &&
@@ -349,13 +353,18 @@ func (e *Executor) runOnce(t time.Time) Stats {
 						nextBuildReason = string(nextBuild.Reason)
 					}
 
+					templateVersionMessage := activeTemplateVersion.Message
+					if templateVersionMessage == "" {
+						templateVersionMessage = "None provided"
+					}
+
 					if _, err := e.notificationsEnqueuer.Enqueue(e.ctx, ws.OwnerID, notifications.TemplateWorkspaceAutoUpdated,
 						map[string]string{
 							"name":                     ws.Name,
 							"initiator":                "autobuild",
 							"reason":                   nextBuildReason,
 							"template_version_name":    activeTemplateVersion.Name,
-							"template_version_message": activeTemplateVersion.Message,
+							"template_version_message": templateVersionMessage,
 						}, "autobuild",
 						// Associate this notification with all the related entities.
 						ws.ID, ws.OwnerID, ws.TemplateID, ws.OrganizationID,

coderd/autobuild/notify/notifier_test.go

@@ -104,7 +104,7 @@ func TestNotifier(t *testing.T) {
 			n := notify.New(cond, testCase.PollInterval, testCase.Countdown, notify.WithTestClock(mClock))
 			defer n.Close()
 
-			trap.MustWait(ctx).Release() // ensure ticker started
+			trap.MustWait(ctx).MustRelease(ctx) // ensure ticker started
 			for i := 0; i < testCase.NTicks; i++ {
 				interval, w := mClock.AdvanceNext()
 				w.MustWait(ctx)

coderd/chat.go

@@ -0,0 +1,366 @@
+package coderd
+
+import (
+	"encoding/json"
+	"io"
+	"net/http"
+	"time"
+
+	"github.com/kylecarbs/aisdk-go"
+
+	"github.com/coder/coder/v2/coderd/ai"
+	"github.com/coder/coder/v2/coderd/database"
+	"github.com/coder/coder/v2/coderd/database/db2sdk"
+	"github.com/coder/coder/v2/coderd/database/dbtime"
+	"github.com/coder/coder/v2/coderd/httpapi"
+	"github.com/coder/coder/v2/coderd/httpmw"
+	"github.com/coder/coder/v2/coderd/util/strings"
+	"github.com/coder/coder/v2/codersdk"
+	"github.com/coder/coder/v2/codersdk/toolsdk"
+)
+
+// postChats creates a new chat.
+//
+// @Summary Create a chat
+// @ID create-a-chat
+// @Security CoderSessionToken
+// @Produce json
+// @Tags Chat
+// @Success 201 {object} codersdk.Chat
+// @Router /chats [post]
+func (api *API) postChats(w http.ResponseWriter, r *http.Request) {
+	apiKey := httpmw.APIKey(r)
+	ctx := r.Context()
+
+	chat, err := api.Database.InsertChat(ctx, database.InsertChatParams{
+		OwnerID:   apiKey.UserID,
+		CreatedAt: time.Now(),
+		UpdatedAt: time.Now(),
+		Title:     "New Chat",
+	})
+	if err != nil {
+		httpapi.Write(ctx, w, http.StatusInternalServerError, codersdk.Response{
+			Message: "Failed to create chat",
+			Detail:  err.Error(),
+		})
+		return
+	}
+
+	httpapi.Write(ctx, w, http.StatusCreated, db2sdk.Chat(chat))
+}
+
+// listChats lists all chats for a user.
+//
+// @Summary List chats
+// @ID list-chats
+// @Security CoderSessionToken
+// @Produce json
+// @Tags Chat
+// @Success 200 {array} codersdk.Chat
+// @Router /chats [get]
+func (api *API) listChats(w http.ResponseWriter, r *http.Request) {
+	apiKey := httpmw.APIKey(r)
+	ctx := r.Context()
+
+	chats, err := api.Database.GetChatsByOwnerID(ctx, apiKey.UserID)
+	if err != nil {
+		httpapi.Write(ctx, w, http.StatusInternalServerError, codersdk.Response{
+			Message: "Failed to list chats",
+			Detail:  err.Error(),
+		})
+		return
+	}
+
+	httpapi.Write(ctx, w, http.StatusOK, db2sdk.Chats(chats))
+}
+
+// chat returns a chat by ID.
+//
+// @Summary Get a chat
+// @ID get-a-chat
+// @Security CoderSessionToken
+// @Produce json
+// @Tags Chat
+// @Param chat path string true "Chat ID"
+// @Success 200 {object} codersdk.Chat
+// @Router /chats/{chat} [get]
+func (*API) chat(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+	chat := httpmw.ChatParam(r)
+	httpapi.Write(ctx, w, http.StatusOK, db2sdk.Chat(chat))
+}
+
+// chatMessages returns the messages of a chat.
+//
+// @Summary Get chat messages
+// @ID get-chat-messages
+// @Security CoderSessionToken
+// @Produce json
+// @Tags Chat
+// @Param chat path string true "Chat ID"
+// @Success 200 {array} aisdk.Message
+// @Router /chats/{chat}/messages [get]
+func (api *API) chatMessages(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+	chat := httpmw.ChatParam(r)
+	rawMessages, err := api.Database.GetChatMessagesByChatID(ctx, chat.ID)
+	if err != nil {
+		httpapi.Write(ctx, w, http.StatusInternalServerError, codersdk.Response{
+			Message: "Failed to get chat messages",
+			Detail:  err.Error(),
+		})
+		return
+	}
+	messages := make([]aisdk.Message, len(rawMessages))
+	for i, message := range rawMessages {
+		var msg aisdk.Message
+		err = json.Unmarshal(message.Content, &msg)
+		if err != nil {
+			httpapi.Write(ctx, w, http.StatusInternalServerError, codersdk.Response{
+				Message: "Failed to unmarshal chat message",
+				Detail:  err.Error(),
+			})
+			return
+		}
+		messages[i] = msg
+	}
+
+	httpapi.Write(ctx, w, http.StatusOK, messages)
+}
+
+// postChatMessages creates a new chat message and streams the response.
+//
+// @Summary Create a chat message
+// @ID create-a-chat-message
+// @Security CoderSessionToken
+// @Accept json
+// @Produce json
+// @Tags Chat
+// @Param chat path string true "Chat ID"
+// @Param request body codersdk.CreateChatMessageRequest true "Request body"
+// @Success 200 {array} aisdk.DataStreamPart
+// @Router /chats/{chat}/messages [post]
+func (api *API) postChatMessages(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+	chat := httpmw.ChatParam(r)
+	var req codersdk.CreateChatMessageRequest
+	err := json.NewDecoder(r.Body).Decode(&req)
+	if err != nil {
+		httpapi.Write(ctx, w, http.StatusBadRequest, codersdk.Response{
+			Message: "Failed to decode chat message",
+			Detail:  err.Error(),
+		})
+		return
+	}
+
+	dbMessages, err := api.Database.GetChatMessagesByChatID(ctx, chat.ID)
+	if err != nil {
+		httpapi.Write(ctx, w, http.StatusInternalServerError, codersdk.Response{
+			Message: "Failed to get chat messages",
+			Detail:  err.Error(),
+		})
+		return
+	}
+
+	messages := make([]codersdk.ChatMessage, 0)
+	for _, dbMsg := range dbMessages {
+		var msg codersdk.ChatMessage
+		err = json.Unmarshal(dbMsg.Content, &msg)
+		if err != nil {
+			httpapi.Write(ctx, w, http.StatusInternalServerError, codersdk.Response{
+				Message: "Failed to unmarshal chat message",
+				Detail:  err.Error(),
+			})
+			return
+		}
+		messages = append(messages, msg)
+	}
+	messages = append(messages, req.Message)
+
+	client := codersdk.New(api.AccessURL)
+	client.SetSessionToken(httpmw.APITokenFromRequest(r))
+
+	tools := make([]aisdk.Tool, 0)
+	handlers := map[string]toolsdk.GenericHandlerFunc{}
+	for _, tool := range toolsdk.All {
+		if tool.Name == "coder_report_task" {
+			continue // This tool requires an agent to run.
+		}
+		tools = append(tools, tool.Tool)
+		handlers[tool.Tool.Name] = tool.Handler
+	}
+
+	provider, ok := api.LanguageModels[req.Model]
+	if !ok {
+		httpapi.Write(ctx, w, http.StatusBadRequest, codersdk.Response{
+			Message: "Model not found",
+		})
+		return
+	}
+
+	// If it's the user's first message, generate a title for the chat.
+	if len(messages) == 1 {
+		var acc aisdk.DataStreamAccumulator
+		stream, err := provider.StreamFunc(ctx, ai.StreamOptions{
+			Model: req.Model,
+			SystemPrompt: `- You will generate a short title based on the user's message.
+- It should be maximum of 40 characters.
+- Do not use quotes, colons, special characters, or emojis.`,
+			Messages: messages,
+			Tools:    []aisdk.Tool{}, // This initial stream doesn't use tools.
+		})
+		if err != nil {
+			httpapi.Write(ctx, w, http.StatusInternalServerError, codersdk.Response{
+				Message: "Failed to create stream",
+				Detail:  err.Error(),
+			})
+			return
+		}
+		stream = stream.WithAccumulator(&acc)
+		err = stream.Pipe(io.Discard)
+		if err != nil {
+			httpapi.Write(ctx, w, http.StatusInternalServerError, codersdk.Response{
+				Message: "Failed to pipe stream",
+				Detail:  err.Error(),
+			})
+			return
+		}
+		var newTitle string
+		accMessages := acc.Messages()
+		// If for some reason the stream didn't return any messages, use the
+		// original message as the title.
+		if len(accMessages) == 0 {
+			newTitle = strings.Truncate(messages[0].Content, 40)
+		} else {
+			newTitle = strings.Truncate(accMessages[0].Content, 40)
+		}
+		err = api.Database.UpdateChatByID(ctx, database.UpdateChatByIDParams{
+			ID:        chat.ID,
+			Title:     newTitle,
+			UpdatedAt: dbtime.Now(),
+		})
+		if err != nil {
+			httpapi.Write(ctx, w, http.StatusInternalServerError, codersdk.Response{
+				Message: "Failed to update chat title",
+				Detail:  err.Error(),
+			})
+			return
+		}
+	}
+
+	// Write headers for the data stream!
+	aisdk.WriteDataStreamHeaders(w)
+
+	// Insert the user-requested message into the database!
+	raw, err := json.Marshal([]aisdk.Message{req.Message})
+	if err != nil {
+		httpapi.Write(ctx, w, http.StatusInternalServerError, codersdk.Response{
+			Message: "Failed to marshal chat message",
+			Detail:  err.Error(),
+		})
+		return
+	}
+	_, err = api.Database.InsertChatMessages(ctx, database.InsertChatMessagesParams{
+		ChatID:    chat.ID,
+		CreatedAt: dbtime.Now(),
+		Model:     req.Model,
+		Provider:  provider.Provider,
+		Content:   raw,
+	})
+	if err != nil {
+		httpapi.Write(ctx, w, http.StatusInternalServerError, codersdk.Response{
+			Message: "Failed to insert chat messages",
+			Detail:  err.Error(),
+		})
+		return
+	}
+
+	deps, err := toolsdk.NewDeps(client)
+	if err != nil {
+		httpapi.Write(ctx, w, http.StatusInternalServerError, codersdk.Response{
+			Message: "Failed to create tool dependencies",
+			Detail:  err.Error(),
+		})
+		return
+	}
+
+	for {
+		var acc aisdk.DataStreamAccumulator
+		stream, err := provider.StreamFunc(ctx, ai.StreamOptions{
+			Model:    req.Model,
+			Messages: messages,
+			Tools:    tools,
+			SystemPrompt: `You are a chat assistant for Coder - an open-source platform for creating and managing cloud development environments on any infrastructure. You are expected to be precise, concise, and helpful.
+
+You are running as an agent - please keep going until the user's query is completely resolved, before ending your turn and yielding back to the user. Only terminate your turn when you are sure that the problem is solved. Do NOT guess or make up an answer.`,
+		})
+		if err != nil {
+			httpapi.Write(ctx, w, http.StatusInternalServerError, codersdk.Response{
+				Message: "Failed to create stream",
+				Detail:  err.Error(),
+			})
+			return
+		}
+		stream = stream.WithToolCalling(func(toolCall aisdk.ToolCall) aisdk.ToolCallResult {
+			tool, ok := handlers[toolCall.Name]
+			if !ok {
+				return nil
+			}
+			toolArgs, err := json.Marshal(toolCall.Args)
+			if err != nil {
+				return nil
+			}
+			result, err := tool(ctx, deps, toolArgs)
+			if err != nil {
+				return map[string]any{
+					"error": err.Error(),
+				}
+			}
+			return result
+		}).WithAccumulator(&acc)
+
+		err = stream.Pipe(w)
+		if err != nil {
+			// The client disppeared!
+			api.Logger.Error(ctx, "stream pipe error", "error", err)
+			return
+		}
+
+		// acc.Messages() may sometimes return nil. Serializing this
+		// will cause a pq error: "cannot extract elements from a scalar".
+		newMessages := append([]aisdk.Message{}, acc.Messages()...)
+		if len(newMessages) > 0 {
+			raw, err := json.Marshal(newMessages)
+			if err != nil {
+				httpapi.Write(ctx, w, http.StatusInternalServerError, codersdk.Response{
+					Message: "Failed to marshal chat message",
+					Detail:  err.Error(),
+				})
+				return
+			}
+			messages = append(messages, newMessages...)
+
+			// Insert these messages into the database!
+			_, err = api.Database.InsertChatMessages(ctx, database.InsertChatMessagesParams{
+				ChatID:    chat.ID,
+				CreatedAt: dbtime.Now(),
+				Model:     req.Model,
+				Provider:  provider.Provider,
+				Content:   raw,
+			})
+			if err != nil {
+				httpapi.Write(ctx, w, http.StatusInternalServerError, codersdk.Response{
+					Message: "Failed to insert chat messages",
+					Detail:  err.Error(),
+				})
+				return
+			}
+		}
+
+		if acc.FinishReason() == aisdk.FinishReasonToolCalls {
+			continue
+		}
+
+		break
+	}
+}

coderd/chat_test.go

@@ -0,0 +1,125 @@
+package coderd_test
+
+import (
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/coder/coder/v2/coderd/coderdtest"
+	"github.com/coder/coder/v2/coderd/database"
+	"github.com/coder/coder/v2/coderd/database/dbgen"
+	"github.com/coder/coder/v2/coderd/database/dbtime"
+	"github.com/coder/coder/v2/codersdk"
+	"github.com/coder/coder/v2/testutil"
+)
+
+func TestChat(t *testing.T) {
+	t.Parallel()
+
+	t.Run("ExperimentAgenticChatDisabled", func(t *testing.T) {
+		t.Parallel()
+
+		client, _ := coderdtest.NewWithDatabase(t, nil)
+		owner := coderdtest.CreateFirstUser(t, client)
+		memberClient, _ := coderdtest.CreateAnotherUser(t, client, owner.OrganizationID)
+
+		// Hit the endpoint to get the chat. It should return a 404.
+		ctx := testutil.Context(t, testutil.WaitShort)
+		_, err := memberClient.ListChats(ctx)
+		require.Error(t, err, "list chats should fail")
+		var sdkErr *codersdk.Error
+		require.ErrorAs(t, err, &sdkErr, "request should fail with an SDK error")
+		require.Equal(t, http.StatusForbidden, sdkErr.StatusCode())
+	})
+
+	t.Run("ChatCRUD", func(t *testing.T) {
+		t.Parallel()
+
+		dv := coderdtest.DeploymentValues(t)
+		dv.Experiments = []string{string(codersdk.ExperimentAgenticChat)}
+		dv.AI.Value = codersdk.AIConfig{
+			Providers: []codersdk.AIProviderConfig{
+				{
+					Type:    "fake",
+					APIKey:  "",
+					BaseURL: "http://localhost",
+					Models:  []string{"fake-model"},
+				},
+			},
+		}
+		client, db := coderdtest.NewWithDatabase(t, &coderdtest.Options{
+			DeploymentValues: dv,
+		})
+		owner := coderdtest.CreateFirstUser(t, client)
+		memberClient, memberUser := coderdtest.CreateAnotherUser(t, client, owner.OrganizationID)
+
+		// Seed the database with some data.
+		dbChat := dbgen.Chat(t, db, database.Chat{
+			OwnerID:   memberUser.ID,
+			CreatedAt: dbtime.Now().Add(-time.Hour),
+			UpdatedAt: dbtime.Now().Add(-time.Hour),
+			Title:     "This is a test chat",
+		})
+		_ = dbgen.ChatMessage(t, db, database.ChatMessage{
+			ChatID:    dbChat.ID,
+			CreatedAt: dbtime.Now().Add(-time.Hour),
+			Content:   []byte(`[{"content": "Hello world"}]`),
+			Model:     "fake model",
+			Provider:  "fake",
+		})
+
+		ctx := testutil.Context(t, testutil.WaitShort)
+
+		// Listing chats should return the chat we just inserted.
+		chats, err := memberClient.ListChats(ctx)
+		require.NoError(t, err, "list chats should succeed")
+		require.Len(t, chats, 1, "response should have one chat")
+		require.Equal(t, dbChat.ID, chats[0].ID, "unexpected chat ID")
+		require.Equal(t, dbChat.Title, chats[0].Title, "unexpected chat title")
+		require.Equal(t, dbChat.CreatedAt.UTC(), chats[0].CreatedAt.UTC(), "unexpected chat created at")
+		require.Equal(t, dbChat.UpdatedAt.UTC(), chats[0].UpdatedAt.UTC(), "unexpected chat updated at")
+
+		// Fetching a single chat by ID should return the same chat.
+		chat, err := memberClient.Chat(ctx, dbChat.ID)
+		require.NoError(t, err, "get chat should succeed")
+		require.Equal(t, chats[0], chat, "get chat should return the same chat")
+
+		// Listing chat messages should return the message we just inserted.
+		messages, err := memberClient.ChatMessages(ctx, dbChat.ID)
+		require.NoError(t, err, "list chat messages should succeed")
+		require.Len(t, messages, 1, "response should have one message")
+		require.Equal(t, "Hello world", messages[0].Content, "response should have the correct message content")
+
+		// Creating a new chat will fail because the model does not exist.
+		// TODO: Test the message streaming functionality with a mock model.
+		// Inserting a chat message will fail due to the model not existing.
+		_, err = memberClient.CreateChatMessage(ctx, dbChat.ID, codersdk.CreateChatMessageRequest{
+			Model: "echo",
+			Message: codersdk.ChatMessage{
+				Role:    "user",
+				Content: "Hello world",
+			},
+			Thinking: false,
+		})
+		require.Error(t, err, "create chat message should fail")
+		var sdkErr *codersdk.Error
+		require.ErrorAs(t, err, &sdkErr, "create chat should fail with an SDK error")
+		require.Equal(t, http.StatusBadRequest, sdkErr.StatusCode(), "create chat should fail with a 400 when model does not exist")
+
+		// Creating a new chat message with malformed content should fail.
+		res, err := memberClient.Request(ctx, http.MethodPost, "/api/v2/chats/"+dbChat.ID.String()+"/messages", strings.NewReader(`{malformed json}`))
+		require.NoError(t, err)
+		defer res.Body.Close()
+		apiErr := codersdk.ReadBodyAsError(res)
+		require.Contains(t, apiErr.Error(), "Failed to decode chat message")
+
+		_, err = memberClient.CreateChat(ctx)
+		require.NoError(t, err, "create chat should succeed")
+		chats, err = memberClient.ListChats(ctx)
+		require.NoError(t, err, "list chats should succeed")
+		require.Len(t, chats, 2, "response should have two chats")
+	})
+}

coderd/coderd.go

@@ -19,6 +19,8 @@ import (
 	"sync/atomic"
 	"time"
 
+	"github.com/coder/coder/v2/coderd/prebuilds"
+
 	"github.com/andybalholm/brotli"
 	"github.com/go-chi/chi/v5"
 	"github.com/go-chi/chi/v5/middleware"
@@ -41,11 +43,13 @@ import (
 	"github.com/coder/quartz"
 	"github.com/coder/serpent"
 
+	"github.com/coder/coder/v2/codersdk/drpcsdk"
+
+	"github.com/coder/coder/v2/coderd/ai"
 	"github.com/coder/coder/v2/coderd/cryptokeys"
 	"github.com/coder/coder/v2/coderd/entitlements"
 	"github.com/coder/coder/v2/coderd/files"
 	"github.com/coder/coder/v2/coderd/idpsync"
-	"github.com/coder/coder/v2/coderd/prebuilds"
 	"github.com/coder/coder/v2/coderd/runtimeconfig"
 	"github.com/coder/coder/v2/coderd/webpush"
 
@@ -83,7 +87,6 @@ import (
 	"github.com/coder/coder/v2/coderd/workspaceapps"
 	"github.com/coder/coder/v2/coderd/workspacestats"
 	"github.com/coder/coder/v2/codersdk"
-	"github.com/coder/coder/v2/codersdk/drpc"
 	"github.com/coder/coder/v2/codersdk/healthsdk"
 	"github.com/coder/coder/v2/provisionerd/proto"
 	"github.com/coder/coder/v2/provisionersdk"
@@ -155,6 +158,7 @@ type Options struct {
 	Authorizer                     rbac.Authorizer
 	AzureCertificates              x509.VerifyOptions
 	GoogleTokenValidator           *idtoken.Validator
+	LanguageModels                 ai.LanguageModels
 	GithubOAuth2Config             *GithubOAuth2Config
 	OIDCConfig                     *OIDCConfig
 	PrometheusRegistry             *prometheus.Registry
@@ -798,6 +802,11 @@ func New(options *Options) *API {
 		PostAuthAdditionalHeadersFunc: options.PostAuthAdditionalHeadersFunc,
 	})
 
+	workspaceAgentInfo := httpmw.ExtractWorkspaceAgentAndLatestBuild(httpmw.ExtractWorkspaceAgentAndLatestBuildConfig{
+		DB:       options.Database,
+		Optional: false,
+	})
+
 	// API rate limit middleware. The counter is local and not shared between
 	// replicas or instances of this middleware.
 	apiRateLimiter := httpmw.RateLimit(options.APIRateLimit, time.Minute)
@@ -956,6 +965,7 @@ func New(options *Options) *API {
 			r.Get("/config", api.deploymentValues)
 			r.Get("/stats", api.deploymentStats)
 			r.Get("/ssh", api.sshConfig)
+			r.Get("/llms", api.deploymentLLMs)
 		})
 		r.Route("/experiments", func(r chi.Router) {
 			r.Use(apiKeyMiddleware)
@@ -998,6 +1008,21 @@ func New(options *Options) *API {
 			r.Get("/{fileID}", api.fileByID)
 			r.Post("/", api.postFile)
 		})
+		// Chats are an experimental feature
+		r.Route("/chats", func(r chi.Router) {
+			r.Use(
+				apiKeyMiddleware,
+				httpmw.RequireExperiment(api.Experiments, codersdk.ExperimentAgenticChat),
+			)
+			r.Get("/", api.listChats)
+			r.Post("/", api.postChats)
+			r.Route("/{chat}", func(r chi.Router) {
+				r.Use(httpmw.ExtractChatParam(options.Database))
+				r.Get("/", api.chat)
+				r.Get("/messages", api.chatMessages)
+				r.Post("/messages", api.postChatMessages)
+			})
+		})
 		r.Route("/external-auth", func(r chi.Router) {
 			r.Use(
 				apiKeyMiddleware,
@@ -1097,6 +1122,7 @@ func New(options *Options) *API {
 				})
 			})
 		})
+
 		r.Route("/templateversions/{templateversion}", func(r chi.Router) {
 			r.Use(
 				apiKeyMiddleware,
@@ -1125,6 +1151,13 @@ func New(options *Options) *API {
 				r.Get("/{jobID}/matched-provisioners", api.templateVersionDryRunMatchedProvisioners)
 				r.Patch("/{jobID}/cancel", api.patchTemplateVersionDryRunCancel)
 			})
+
+			r.Group(func(r chi.Router) {
+				r.Use(
+					httpmw.RequireExperiment(api.Experiments, codersdk.ExperimentDynamicParameters),
+				)
+				r.Get("/dynamic-parameters", api.templateVersionDynamicParameters)
+			})
 		})
 		r.Route("/users", func(r chi.Router) {
 			r.Get("/first", api.firstUser)
@@ -1171,21 +1204,14 @@ func New(options *Options) *API {
 				})
 				r.Route("/{user}", func(r chi.Router) {
 					r.Group(func(r chi.Router) {
-						r.Use(httpmw.ExtractUserParamOptional(options.Database))
+						r.Use(httpmw.ExtractOrganizationMembersParam(options.Database, api.HTTPAuth.Authorize))
 						// Creating workspaces does not require permissions on the user, only the
 						// organization member. This endpoint should match the authz story of
 						// postWorkspacesByOrganization
 						r.Post("/workspaces", api.postUserWorkspaces)
-
-						// Similarly to creating a workspace, evaluating parameters for a
-						// new workspace should also match the authz story of
-						// postWorkspacesByOrganization
-						r.Route("/templateversions/{templateversion}", func(r chi.Router) {
-							r.Use(
-								httpmw.ExtractTemplateVersionParam(options.Database),
-								httpmw.RequireExperiment(api.Experiments, codersdk.ExperimentDynamicParameters),
-							)
-							r.Get("/parameters", api.templateVersionDynamicParameters)
+						r.Route("/workspace/{workspacename}", func(r chi.Router) {
+							r.Get("/", api.workspaceByOwnerAndName)
+							r.Get("/builds/{buildnumber}", api.workspaceBuildByBuildNumber)
 						})
 					})
 
@@ -1232,10 +1258,7 @@ func New(options *Options) *API {
 							r.Get("/", api.organizationsByUser)
 							r.Get("/{organizationname}", api.organizationByUserAndName)
 						})
-						r.Route("/workspace/{workspacename}", func(r chi.Router) {
-							r.Get("/", api.workspaceByOwnerAndName)
-							r.Get("/builds/{buildnumber}", api.workspaceBuildByBuildNumber)
-						})
+
 						r.Get("/gitsshkey", api.gitSSHKey)
 						r.Put("/gitsshkey", api.regenerateGitSSHKey)
 						r.Route("/notifications", func(r chi.Router) {
@@ -1266,10 +1289,7 @@ func New(options *Options) *API {
 				httpmw.RequireAPIKeyOrWorkspaceProxyAuth(),
 			).Get("/connection", api.workspaceAgentConnectionGeneric)
 			r.Route("/me", func(r chi.Router) {
-				r.Use(httpmw.ExtractWorkspaceAgentAndLatestBuild(httpmw.ExtractWorkspaceAgentAndLatestBuildConfig{
-					DB:       options.Database,
-					Optional: false,
-				}))
+				r.Use(workspaceAgentInfo)
 				r.Get("/rpc", api.workspaceAgentRPC)
 				r.Patch("/logs", api.patchWorkspaceAgentLogs)
 				r.Patch("/app-status", api.patchWorkspaceAgentAppStatus)
@@ -1278,6 +1298,7 @@ func New(options *Options) *API {
 				r.Get("/external-auth", api.workspaceAgentsExternalAuth)
 				r.Get("/gitsshkey", api.agentGitSSHKey)
 				r.Post("/log-source", api.workspaceAgentPostLogSource)
+				r.Get("/reinit", api.workspaceAgentReinit)
 			})
 			r.Route("/{workspaceagent}", func(r chi.Router) {
 				r.Use(
@@ -1300,6 +1321,7 @@ func New(options *Options) *API {
 				r.Get("/listening-ports", api.workspaceAgentListeningPorts)
 				r.Get("/connection", api.workspaceAgentConnection)
 				r.Get("/containers", api.workspaceAgentListContainers)
+				r.Post("/containers/devcontainers/container/{container}/recreate", api.workspaceAgentRecreateDevcontainer)
 				r.Get("/coordinate", api.workspaceAgentClientCoordinate)
 
 				// PTY is part of workspaceAppServer.
@@ -1571,7 +1593,7 @@ type API struct {
 	// passed to dbauthz.
 	AccessControlStore  *atomic.Pointer[dbauthz.AccessControlStore]
 	PortSharer          atomic.Pointer[portsharing.PortSharer]
-	FileCache           files.Cache
+	FileCache           *files.Cache
 	PrebuildsClaimer    atomic.Pointer[prebuilds.Claimer]
 	PrebuildsReconciler atomic.Pointer[prebuilds.ReconciliationOrchestrator]
 
@@ -1696,15 +1718,32 @@ func compressHandler(h http.Handler) http.Handler {
 	return cmp.Handler(h)
 }
 
+type MemoryProvisionerDaemonOption func(*memoryProvisionerDaemonOptions)
+
+func MemoryProvisionerWithVersionOverride(version string) MemoryProvisionerDaemonOption {
+	return func(opts *memoryProvisionerDaemonOptions) {
+		opts.versionOverride = version
+	}
+}
+
+type memoryProvisionerDaemonOptions struct {
+	versionOverride string
+}
+
 // CreateInMemoryProvisionerDaemon is an in-memory connection to a provisionerd.
 // Useful when starting coderd and provisionerd in the same process.
 func (api *API) CreateInMemoryProvisionerDaemon(dialCtx context.Context, name string, provisionerTypes []codersdk.ProvisionerType) (client proto.DRPCProvisionerDaemonClient, err error) {
 	return api.CreateInMemoryTaggedProvisionerDaemon(dialCtx, name, provisionerTypes, nil)
 }
 
-func (api *API) CreateInMemoryTaggedProvisionerDaemon(dialCtx context.Context, name string, provisionerTypes []codersdk.ProvisionerType, provisionerTags map[string]string) (client proto.DRPCProvisionerDaemonClient, err error) {
+func (api *API) CreateInMemoryTaggedProvisionerDaemon(dialCtx context.Context, name string, provisionerTypes []codersdk.ProvisionerType, provisionerTags map[string]string, opts ...MemoryProvisionerDaemonOption) (client proto.DRPCProvisionerDaemonClient, err error) {
+	options := &memoryProvisionerDaemonOptions{}
+	for _, opt := range opts {
+		opt(options)
+	}
+
 	tracer := api.TracerProvider.Tracer(tracing.TracerName)
-	clientSession, serverSession := drpc.MemTransportPipe()
+	clientSession, serverSession := drpcsdk.MemTransportPipe()
 	defer func() {
 		if err != nil {
 			_ = clientSession.Close()
@@ -1729,6 +1768,12 @@ func (api *API) CreateInMemoryTaggedProvisionerDaemon(dialCtx context.Context, n
 		return nil, xerrors.Errorf("failed to parse built-in provisioner key ID: %w", err)
 	}
 
+	apiVersion := proto.CurrentVersion.String()
+	if options.versionOverride != "" && flag.Lookup("test.v") != nil {
+		// This should only be usable for unit testing. To fake a different provisioner version
+		apiVersion = options.versionOverride
+	}
+
 	//nolint:gocritic // in-memory provisioners are owned by system
 	daemon, err := api.Database.UpsertProvisionerDaemon(dbauthz.AsSystemRestricted(dialCtx), database.UpsertProvisionerDaemonParams{
 		Name:           name,
@@ -1738,7 +1783,7 @@ func (api *API) CreateInMemoryTaggedProvisionerDaemon(dialCtx context.Context, n
 		Tags:           provisionersdk.MutateTags(uuid.Nil, provisionerTags),
 		LastSeenAt:     sql.NullTime{Time: dbtime.Now(), Valid: true},
 		Version:        buildinfo.Version(),
-		APIVersion:     proto.CurrentVersion.String(),
+		APIVersion:     apiVersion,
 		KeyID:          keyID,
 	})
 	if err != nil {
@@ -1750,6 +1795,7 @@ func (api *API) CreateInMemoryTaggedProvisionerDaemon(dialCtx context.Context, n
 	logger := api.Logger.Named(fmt.Sprintf("inmem-provisionerd-%s", name))
 	srv, err := provisionerdserver.NewServer(
 		api.ctx, // use the same ctx as the API
+		daemon.APIVersion,
 		api.AccessURL,
 		daemon.ID,
 		defaultOrg.ID,
@@ -1772,6 +1818,7 @@ func (api *API) CreateInMemoryTaggedProvisionerDaemon(dialCtx context.Context, n
 			Clock:               api.Clock,
 		},
 		api.NotificationsEnqueuer,
+		&api.PrebuildsReconciler,
 	)
 	if err != nil {
 		return nil, err
@@ -1782,6 +1829,7 @@ func (api *API) CreateInMemoryTaggedProvisionerDaemon(dialCtx context.Context, n
 	}
 	server := drpcserver.NewWithOptions(&tracing.DRPCHandler{Handler: mux},
 		drpcserver.Options{
+			Manager: drpcsdk.DefaultDRPCOptions(nil),
 			Log: func(err error) {
 				if xerrors.Is(err, io.EOF) {
 					return

coderd/coderdtest/coderdtest.go

@@ -68,6 +68,7 @@ import (
 	"github.com/coder/coder/v2/coderd/externalauth"
 	"github.com/coder/coder/v2/coderd/gitsshkey"
 	"github.com/coder/coder/v2/coderd/httpmw"
+	"github.com/coder/coder/v2/coderd/jobreaper"
 	"github.com/coder/coder/v2/coderd/notifications"
 	"github.com/coder/coder/v2/coderd/notifications/notificationstest"
 	"github.com/coder/coder/v2/coderd/rbac"
@@ -75,7 +76,6 @@ import (
 	"github.com/coder/coder/v2/coderd/runtimeconfig"
 	"github.com/coder/coder/v2/coderd/schedule"
 	"github.com/coder/coder/v2/coderd/telemetry"
-	"github.com/coder/coder/v2/coderd/unhanger"
 	"github.com/coder/coder/v2/coderd/updatecheck"
 	"github.com/coder/coder/v2/coderd/util/ptr"
 	"github.com/coder/coder/v2/coderd/webpush"
@@ -84,7 +84,7 @@ import (
 	"github.com/coder/coder/v2/coderd/workspacestats"
 	"github.com/coder/coder/v2/codersdk"
 	"github.com/coder/coder/v2/codersdk/agentsdk"
-	"github.com/coder/coder/v2/codersdk/drpc"
+	"github.com/coder/coder/v2/codersdk/drpcsdk"
 	"github.com/coder/coder/v2/codersdk/healthsdk"
 	"github.com/coder/coder/v2/cryptorand"
 	"github.com/coder/coder/v2/provisioner/echo"
@@ -96,6 +96,8 @@ import (
 	"github.com/coder/coder/v2/testutil"
 )
 
+const defaultTestDaemonName = "test-daemon"
+
 type Options struct {
 	// AccessURL denotes a custom access URL. By default we use the httptest
 	// server's URL. Setting this may result in unexpected behavior (especially
@@ -135,6 +137,7 @@ type Options struct {
 
 	// IncludeProvisionerDaemon when true means to start an in-memory provisionerD
 	IncludeProvisionerDaemon    bool
+	ProvisionerDaemonVersion    string
 	ProvisionerDaemonTags       map[string]string
 	MetricsCacheRefreshInterval time.Duration
 	AgentStatsRefreshInterval   time.Duration
@@ -351,6 +354,7 @@ func NewOptions(t testing.TB, options *Options) (func(http.Handler), context.Can
 	auditor.Store(&options.Auditor)
 
 	ctx, cancelFunc := context.WithCancel(context.Background())
+	experiments := coderd.ReadExperiments(*options.Logger, options.DeploymentValues.Experiments)
 	lifecycleExecutor := autobuild.NewExecutor(
 		ctx,
 		options.Database,
@@ -362,14 +366,15 @@ func NewOptions(t testing.TB, options *Options) (func(http.Handler), context.Can
 		*options.Logger,
 		options.AutobuildTicker,
 		options.NotificationsEnqueuer,
+		experiments,
 	).WithStatsChannel(options.AutobuildStats)
 	lifecycleExecutor.Run()
 
-	hangDetectorTicker := time.NewTicker(options.DeploymentValues.JobHangDetectorInterval.Value())
-	defer hangDetectorTicker.Stop()
-	hangDetector := unhanger.New(ctx, options.Database, options.Pubsub, options.Logger.Named("unhanger.detector"), hangDetectorTicker.C)
-	hangDetector.Start()
-	t.Cleanup(hangDetector.Close)
+	jobReaperTicker := time.NewTicker(options.DeploymentValues.JobReaperDetectorInterval.Value())
+	defer jobReaperTicker.Stop()
+	jobReaper := jobreaper.New(ctx, options.Database, options.Pubsub, options.Logger.Named("reaper.detector"), jobReaperTicker.C)
+	jobReaper.Start()
+	t.Cleanup(jobReaper.Close)
 
 	if options.TelemetryReporter == nil {
 		options.TelemetryReporter = telemetry.NewNoop()
@@ -601,7 +606,7 @@ func NewWithAPI(t testing.TB, options *Options) (*codersdk.Client, io.Closer, *c
 	setHandler(rootHandler)
 	var provisionerCloser io.Closer = nopcloser{}
 	if options.IncludeProvisionerDaemon {
-		provisionerCloser = NewTaggedProvisionerDaemon(t, coderAPI, "test", options.ProvisionerDaemonTags)
+		provisionerCloser = NewTaggedProvisionerDaemon(t, coderAPI, defaultTestDaemonName, options.ProvisionerDaemonTags, coderd.MemoryProvisionerWithVersionOverride(options.ProvisionerDaemonVersion))
 	}
 	client := codersdk.New(serverURL)
 	t.Cleanup(func() {
@@ -645,10 +650,10 @@ func (c *ProvisionerdCloser) Close() error {
 // well with coderd testing. It registers the "echo" provisioner for
 // quick testing.
 func NewProvisionerDaemon(t testing.TB, coderAPI *coderd.API) io.Closer {
-	return NewTaggedProvisionerDaemon(t, coderAPI, "test", nil)
+	return NewTaggedProvisionerDaemon(t, coderAPI, defaultTestDaemonName, nil)
 }
 
-func NewTaggedProvisionerDaemon(t testing.TB, coderAPI *coderd.API, name string, provisionerTags map[string]string) io.Closer {
+func NewTaggedProvisionerDaemon(t testing.TB, coderAPI *coderd.API, name string, provisionerTags map[string]string, opts ...coderd.MemoryProvisionerDaemonOption) io.Closer {
 	t.Helper()
 
 	// t.Cleanup runs in last added, first called order. t.TempDir() will delete
@@ -657,7 +662,7 @@ func NewTaggedProvisionerDaemon(t testing.TB, coderAPI *coderd.API, name string,
 	// seems t.TempDir() is not safe to call from a different goroutine
 	workDir := t.TempDir()
 
-	echoClient, echoServer := drpc.MemTransportPipe()
+	echoClient, echoServer := drpcsdk.MemTransportPipe()
 	ctx, cancelFunc := context.WithCancel(context.Background())
 	t.Cleanup(func() {
 		_ = echoClient.Close()
@@ -676,7 +681,7 @@ func NewTaggedProvisionerDaemon(t testing.TB, coderAPI *coderd.API, name string,
 
 	connectedCh := make(chan struct{})
 	daemon := provisionerd.New(func(dialCtx context.Context) (provisionerdproto.DRPCProvisionerDaemonClient, error) {
-		return coderAPI.CreateInMemoryTaggedProvisionerDaemon(dialCtx, name, []codersdk.ProvisionerType{codersdk.ProvisionerTypeEcho}, provisionerTags)
+		return coderAPI.CreateInMemoryTaggedProvisionerDaemon(dialCtx, name, []codersdk.ProvisionerType{codersdk.ProvisionerTypeEcho}, provisionerTags, opts...)
 	}, &provisionerd.Options{
 		Logger:              coderAPI.Logger.Named("provisionerd").Leveled(slog.LevelDebug),
 		UpdateInterval:      250 * time.Millisecond,
@@ -1105,6 +1110,69 @@ func (w WorkspaceAgentWaiter) MatchResources(m func([]codersdk.WorkspaceResource
 	return w
 }
 
+// WaitForAgentFn represents a boolean assertion to be made against each agent
+// that a given WorkspaceAgentWaited knows about. Each WaitForAgentFn should apply
+// the check to a single agent, but it should be named for plural, because `func (w WorkspaceAgentWaiter) WaitFor`
+// applies the check to all agents that it is aware of. This ensures that the public API of the waiter
+// reads correctly. For example:
+//
+// waiter := coderdtest.NewWorkspaceAgentWaiter(t, client, r.Workspace.ID)
+// waiter.WaitFor(coderdtest.AgentsReady)
+type WaitForAgentFn func(agent codersdk.WorkspaceAgent) bool
+
+// AgentsReady checks that the latest lifecycle state of an agent is "Ready".
+func AgentsReady(agent codersdk.WorkspaceAgent) bool {
+	return agent.LifecycleState == codersdk.WorkspaceAgentLifecycleReady
+}
+
+// AgentsNotReady checks that the latest lifecycle state of an agent is anything except "Ready".
+func AgentsNotReady(agent codersdk.WorkspaceAgent) bool {
+	return !AgentsReady(agent)
+}
+
+func (w WorkspaceAgentWaiter) WaitFor(criteria ...WaitForAgentFn) {
+	w.t.Helper()
+
+	agentNamesMap := make(map[string]struct{}, len(w.agentNames))
+	for _, name := range w.agentNames {
+		agentNamesMap[name] = struct{}{}
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
+	defer cancel()
+
+	w.t.Logf("waiting for workspace agents (workspace %s)", w.workspaceID)
+	require.Eventually(w.t, func() bool {
+		var err error
+		workspace, err := w.client.Workspace(ctx, w.workspaceID)
+		if err != nil {
+			return false
+		}
+		if workspace.LatestBuild.Job.CompletedAt == nil {
+			return false
+		}
+		if workspace.LatestBuild.Job.CompletedAt.IsZero() {
+			return false
+		}
+
+		for _, resource := range workspace.LatestBuild.Resources {
+			for _, agent := range resource.Agents {
+				if len(w.agentNames) > 0 {
+					if _, ok := agentNamesMap[agent.Name]; !ok {
+						continue
+					}
+				}
+				for _, criterium := range criteria {
+					if !criterium(agent) {
+						return false
+					}
+				}
+			}
+		}
+		return true
+	}, testutil.WaitLong, testutil.IntervalMedium)
+}
+
 // Wait waits for the agent(s) to connect and fails the test if they do not within testutil.WaitLong
 func (w WorkspaceAgentWaiter) Wait() []codersdk.WorkspaceResource {
 	w.t.Helper()

coderd/coderdtest/oidctest/idp.go

@@ -307,7 +307,7 @@ func WithCustomClientAuth(hook func(t testing.TB, req *http.Request) (url.Values
 // WithLogging is optional, but will log some HTTP calls made to the IDP.
 func WithLogging(t testing.TB, options *slogtest.Options) func(*FakeIDP) {
 	return func(f *FakeIDP) {
-		f.logger = slogtest.Make(t, options)
+		f.logger = slogtest.Make(t, options).Named("fakeidp")
 	}
 }
 
@@ -794,6 +794,7 @@ func (f *FakeIDP) newToken(t testing.TB, email string, expires time.Time) string
 func (f *FakeIDP) newRefreshTokens(email string) string {
 	refreshToken := uuid.NewString()
 	f.refreshTokens.Store(refreshToken, email)
+	f.logger.Info(context.Background(), "new refresh token", slog.F("email", email), slog.F("token", refreshToken))
 	return refreshToken
 }
 
@@ -1003,6 +1004,7 @@ func (f *FakeIDP) httpHandler(t testing.TB) http.Handler {
 				return
 			}
 
+			f.logger.Info(r.Context(), "http idp call refresh_token", slog.F("token", refreshToken))
 			_, ok := f.refreshTokens.Load(refreshToken)
 			if !assert.True(t, ok, "invalid refresh_token") {
 				http.Error(rw, "invalid refresh_token", http.StatusBadRequest)
@@ -1026,6 +1028,7 @@ func (f *FakeIDP) httpHandler(t testing.TB) http.Handler {
 			f.refreshTokensUsed.Store(refreshToken, true)
 			// Always invalidate the refresh token after it is used.
 			f.refreshTokens.Delete(refreshToken)
+			f.logger.Info(r.Context(), "refresh token invalidated", slog.F("token", refreshToken))
 		case "urn:ietf:params:oauth:grant-type:device_code":
 			// Device flow
 			var resp externalauth.ExchangeDeviceCodeResponse

coderd/cryptokeys/cache_test.go

@@ -423,7 +423,7 @@ func TestCryptoKeyCache(t *testing.T) {
 		require.Equal(t, 2, ff.called)
 		require.Equal(t, decodedSecret(t, newKey), key)
 
-		trapped.Release()
+		trapped.MustRelease(ctx)
 		wait.MustWait(ctx)
 		require.Equal(t, 2, ff.called)
 		trap.Close()

coderd/cryptokeys/rotate_test.go

@@ -72,7 +72,7 @@ func TestRotator(t *testing.T) {
 		require.Len(t, dbkeys, initialKeyLen)
 		requireContainsAllFeatures(t, dbkeys)
 
-		trap.MustWait(ctx).Release()
+		trap.MustWait(ctx).MustRelease(ctx)
 		_, wait := clock.AdvanceNext()
 		wait.MustWait(ctx)
 

coderd/database/db2sdk/db2sdk.go

@@ -12,6 +12,7 @@ import (
 	"time"
 
 	"github.com/google/uuid"
+	"github.com/hashicorp/hcl/v2"
 	"golang.org/x/xerrors"
 	"tailscale.com/tailcfg"
 
@@ -24,6 +25,7 @@ import (
 	"github.com/coder/coder/v2/codersdk"
 	"github.com/coder/coder/v2/provisionersdk/proto"
 	"github.com/coder/coder/v2/tailnet"
+	previewtypes "github.com/coder/preview/types"
 )
 
 // List is a helper function to reduce boilerplate when converting slices of
@@ -523,6 +525,7 @@ func Apps(dbApps []database.WorkspaceApp, statuses []database.WorkspaceAppStatus
 				Threshold: dbApp.HealthcheckThreshold,
 			},
 			Health:   codersdk.WorkspaceAppHealth(dbApp.Health),
+			Group:    dbApp.DisplayGroup.String,
 			Hidden:   dbApp.Hidden,
 			OpenIn:   codersdk.WorkspaceAppOpenIn(dbApp.OpenIn),
 			Statuses: WorkspaceAppStatuses(statuses),
@@ -751,3 +754,96 @@ func AgentProtoConnectionActionToAuditAction(action database.AuditAction) (agent
 		return agentproto.Connection_ACTION_UNSPECIFIED, xerrors.Errorf("unknown agent connection action %q", action)
 	}
 }
+
+func Chat(chat database.Chat) codersdk.Chat {
+	return codersdk.Chat{
+		ID:        chat.ID,
+		Title:     chat.Title,
+		CreatedAt: chat.CreatedAt,
+		UpdatedAt: chat.UpdatedAt,
+	}
+}
+
+func Chats(chats []database.Chat) []codersdk.Chat {
+	return List(chats, Chat)
+}
+
+func PreviewParameter(param previewtypes.Parameter) codersdk.PreviewParameter {
+	return codersdk.PreviewParameter{
+		PreviewParameterData: codersdk.PreviewParameterData{
+			Name:        param.Name,
+			DisplayName: param.DisplayName,
+			Description: param.Description,
+			Type:        codersdk.OptionType(param.Type),
+			FormType:    codersdk.ParameterFormType(param.FormType),
+			Styling: codersdk.PreviewParameterStyling{
+				Placeholder: param.Styling.Placeholder,
+				Disabled:    param.Styling.Disabled,
+				Label:       param.Styling.Label,
+			},
+			Mutable:      param.Mutable,
+			DefaultValue: PreviewHCLString(param.DefaultValue),
+			Icon:         param.Icon,
+			Options:      List(param.Options, PreviewParameterOption),
+			Validations:  List(param.Validations, PreviewParameterValidation),
+			Required:     param.Required,
+			Order:        param.Order,
+			Ephemeral:    param.Ephemeral,
+		},
+		Value:       PreviewHCLString(param.Value),
+		Diagnostics: PreviewDiagnostics(param.Diagnostics),
+	}
+}
+
+func HCLDiagnostics(d hcl.Diagnostics) []codersdk.FriendlyDiagnostic {
+	return PreviewDiagnostics(previewtypes.Diagnostics(d))
+}
+
+func PreviewDiagnostics(d previewtypes.Diagnostics) []codersdk.FriendlyDiagnostic {
+	f := d.FriendlyDiagnostics()
+	return List(f, func(f previewtypes.FriendlyDiagnostic) codersdk.FriendlyDiagnostic {
+		return codersdk.FriendlyDiagnostic{
+			Severity: codersdk.DiagnosticSeverityString(f.Severity),
+			Summary:  f.Summary,
+			Detail:   f.Detail,
+			Extra: codersdk.DiagnosticExtra{
+				Code: f.Extra.Code,
+			},
+		}
+	})
+}
+
+func PreviewHCLString(h previewtypes.HCLString) codersdk.NullHCLString {
+	n := h.NullHCLString()
+	return codersdk.NullHCLString{
+		Value: n.Value,
+		Valid: n.Valid,
+	}
+}
+
+func PreviewParameterOption(o *previewtypes.ParameterOption) codersdk.PreviewParameterOption {
+	if o == nil {
+		// This should never be sent
+		return codersdk.PreviewParameterOption{}
+	}
+	return codersdk.PreviewParameterOption{
+		Name:        o.Name,
+		Description: o.Description,
+		Value:       PreviewHCLString(o.Value),
+		Icon:        o.Icon,
+	}
+}
+
+func PreviewParameterValidation(v *previewtypes.ParameterValidation) codersdk.PreviewParameterValidation {
+	if v == nil {
+		// This should never be sent
+		return codersdk.PreviewParameterValidation{}
+	}
+	return codersdk.PreviewParameterValidation{
+		Error:     v.Error,
+		Regex:     v.Regex,
+		Min:       v.Min,
+		Max:       v.Max,
+		Monotonic: v.Monotonic,
+	}
+}

coderd/database/dbauthz/dbauthz.go

@@ -12,21 +12,19 @@ import (
 	"time"
 
 	"github.com/google/uuid"
+	"github.com/open-policy-agent/opa/topdown"
 	"golang.org/x/xerrors"
 
-	"github.com/open-policy-agent/opa/topdown"
-
 	"cdr.dev/slog"
 
-	"github.com/coder/coder/v2/coderd/prebuilds"
-	"github.com/coder/coder/v2/coderd/rbac/policy"
-	"github.com/coder/coder/v2/coderd/rbac/rolestore"
-
 	"github.com/coder/coder/v2/coderd/database"
 	"github.com/coder/coder/v2/coderd/database/dbtime"
 	"github.com/coder/coder/v2/coderd/httpapi/httpapiconstraints"
 	"github.com/coder/coder/v2/coderd/httpmw/loggermw"
+	"github.com/coder/coder/v2/coderd/prebuilds"
 	"github.com/coder/coder/v2/coderd/rbac"
+	"github.com/coder/coder/v2/coderd/rbac/policy"
+	"github.com/coder/coder/v2/coderd/rbac/rolestore"
 	"github.com/coder/coder/v2/coderd/util/slice"
 	"github.com/coder/coder/v2/provisionersdk"
 )
@@ -172,14 +170,14 @@ var (
 				Identifier:  rbac.RoleIdentifier{Name: "provisionerd"},
 				DisplayName: "Provisioner Daemon",
 				Site: rbac.Permissions(map[string][]policy.Action{
-					// TODO: Add ProvisionerJob resource type.
-					rbac.ResourceFile.Type:     {policy.ActionRead},
-					rbac.ResourceSystem.Type:   {policy.WildcardSymbol},
-					rbac.ResourceTemplate.Type: {policy.ActionRead, policy.ActionUpdate},
+					rbac.ResourceProvisionerJobs.Type: {policy.ActionRead, policy.ActionUpdate, policy.ActionCreate},
+					rbac.ResourceFile.Type:            {policy.ActionRead},
+					rbac.ResourceSystem.Type:          {policy.WildcardSymbol},
+					rbac.ResourceTemplate.Type:        {policy.ActionRead, policy.ActionUpdate},
 					// Unsure why provisionerd needs update and read personal
 					rbac.ResourceUser.Type:             {policy.ActionRead, policy.ActionReadPersonal, policy.ActionUpdatePersonal},
 					rbac.ResourceWorkspaceDormant.Type: {policy.ActionDelete, policy.ActionRead, policy.ActionUpdate, policy.ActionWorkspaceStop},
-					rbac.ResourceWorkspace.Type:        {policy.ActionDelete, policy.ActionRead, policy.ActionUpdate, policy.ActionWorkspaceStart, policy.ActionWorkspaceStop},
+					rbac.ResourceWorkspace.Type:        {policy.ActionDelete, policy.ActionRead, policy.ActionUpdate, policy.ActionWorkspaceStart, policy.ActionWorkspaceStop, policy.ActionCreateAgent},
 					rbac.ResourceApiKey.Type:           {policy.WildcardSymbol},
 					// When org scoped provisioner credentials are implemented,
 					// this can be reduced to read a specific org.
@@ -221,19 +219,20 @@ var (
 		Scope: rbac.ScopeAll,
 	}.WithCachedASTValue()
 
-	// See unhanger package.
-	subjectHangDetector = rbac.Subject{
-		Type:         rbac.SubjectTypeHangDetector,
-		FriendlyName: "Hang Detector",
+	// See reaper package.
+	subjectJobReaper = rbac.Subject{
+		Type:         rbac.SubjectTypeJobReaper,
+		FriendlyName: "Job Reaper",
 		ID:           uuid.Nil.String(),
 		Roles: rbac.Roles([]rbac.Role{
 			{
-				Identifier:  rbac.RoleIdentifier{Name: "hangdetector"},
-				DisplayName: "Hang Detector Daemon",
+				Identifier:  rbac.RoleIdentifier{Name: "jobreaper"},
+				DisplayName: "Job Reaper Daemon",
 				Site: rbac.Permissions(map[string][]policy.Action{
-					rbac.ResourceSystem.Type:    {policy.WildcardSymbol},
-					rbac.ResourceTemplate.Type:  {policy.ActionRead},
-					rbac.ResourceWorkspace.Type: {policy.ActionRead, policy.ActionUpdate},
+					rbac.ResourceSystem.Type:          {policy.WildcardSymbol},
+					rbac.ResourceTemplate.Type:        {policy.ActionRead},
+					rbac.ResourceWorkspace.Type:       {policy.ActionRead, policy.ActionUpdate},
+					rbac.ResourceProvisionerJobs.Type: {policy.ActionRead, policy.ActionUpdate},
 				}),
 				Org:  map[string][]rbac.Permission{},
 				User: []rbac.Permission{},
@@ -340,13 +339,15 @@ var (
 					rbac.ResourceProvisionerDaemon.Type:      {policy.ActionCreate, policy.ActionRead, policy.ActionUpdate},
 					rbac.ResourceUser.Type:                   rbac.ResourceUser.AvailableActions(),
 					rbac.ResourceWorkspaceDormant.Type:       {policy.ActionUpdate, policy.ActionDelete, policy.ActionWorkspaceStop},
-					rbac.ResourceWorkspace.Type:              {policy.ActionUpdate, policy.ActionDelete, policy.ActionWorkspaceStart, policy.ActionWorkspaceStop, policy.ActionSSH},
+					rbac.ResourceWorkspace.Type:              {policy.ActionUpdate, policy.ActionDelete, policy.ActionWorkspaceStart, policy.ActionWorkspaceStop, policy.ActionSSH, policy.ActionCreateAgent, policy.ActionDeleteAgent},
 					rbac.ResourceWorkspaceProxy.Type:         {policy.ActionCreate, policy.ActionUpdate, policy.ActionDelete},
 					rbac.ResourceDeploymentConfig.Type:       {policy.ActionCreate, policy.ActionUpdate, policy.ActionDelete},
 					rbac.ResourceNotificationMessage.Type:    {policy.ActionCreate, policy.ActionRead, policy.ActionUpdate, policy.ActionDelete},
 					rbac.ResourceNotificationPreference.Type: {policy.ActionCreate, policy.ActionUpdate, policy.ActionDelete},
 					rbac.ResourceNotificationTemplate.Type:   {policy.ActionCreate, policy.ActionUpdate, policy.ActionDelete},
 					rbac.ResourceCryptoKey.Type:              {policy.ActionCreate, policy.ActionUpdate, policy.ActionDelete},
+					rbac.ResourceFile.Type:                   {policy.ActionCreate, policy.ActionRead},
+					rbac.ResourceProvisionerJobs.Type:        {policy.ActionRead, policy.ActionUpdate, policy.ActionCreate},
 				}),
 				Org:  map[string][]rbac.Permission{},
 				User: []rbac.Permission{},
@@ -408,10 +409,10 @@ func AsAutostart(ctx context.Context) context.Context {
 	return As(ctx, subjectAutostart)
 }
 
-// AsHangDetector returns a context with an actor that has permissions required
-// for unhanger.Detector to function.
-func AsHangDetector(ctx context.Context) context.Context {
-	return As(ctx, subjectHangDetector)
+// AsJobReaper returns a context with an actor that has permissions required
+// for reaper.Detector to function.
+func AsJobReaper(ctx context.Context) context.Context {
+	return As(ctx, subjectJobReaper)
 }
 
 // AsKeyRotator returns a context with an actor that has permissions required for rotating crypto keys.
@@ -1086,11 +1087,10 @@ func (q *querier) AcquireNotificationMessages(ctx context.Context, arg database.
 	return q.db.AcquireNotificationMessages(ctx, arg)
 }
 
-// TODO: We need to create a ProvisionerJob resource type
 func (q *querier) AcquireProvisionerJob(ctx context.Context, arg database.AcquireProvisionerJobParams) (database.ProvisionerJob, error) {
-	// if err := q.authorizeContext(ctx, policy.ActionUpdate, rbac.ResourceSystem); err != nil {
-	// return database.ProvisionerJob{}, err
-	// }
+	if err := q.authorizeContext(ctx, policy.ActionUpdate, rbac.ResourceProvisionerJobs); err != nil {
+		return database.ProvisionerJob{}, err
+	}
 	return q.db.AcquireProvisionerJob(ctx, arg)
 }
 
@@ -1269,6 +1269,10 @@ func (q *querier) DeleteApplicationConnectAPIKeysByUserID(ctx context.Context, u
 	return q.db.DeleteApplicationConnectAPIKeysByUserID(ctx, userID)
 }
 
+func (q *querier) DeleteChat(ctx context.Context, id uuid.UUID) error {
+	return deleteQ(q.log, q.auth, q.db.GetChatByID, q.db.DeleteChat)(ctx, id)
+}
+
 func (q *querier) DeleteCoordinator(ctx context.Context, id uuid.UUID) error {
 	if err := q.authorizeContext(ctx, policy.ActionDelete, rbac.ResourceTailnetCoordinator); err != nil {
 		return err
@@ -1686,6 +1690,22 @@ func (q *querier) GetAuthorizationUserRoles(ctx context.Context, userID uuid.UUI
 	return q.db.GetAuthorizationUserRoles(ctx, userID)
 }
 
+func (q *querier) GetChatByID(ctx context.Context, id uuid.UUID) (database.Chat, error) {
+	return fetch(q.log, q.auth, q.db.GetChatByID)(ctx, id)
+}
+
+func (q *querier) GetChatMessagesByChatID(ctx context.Context, chatID uuid.UUID) ([]database.ChatMessage, error) {
+	c, err := q.GetChatByID(ctx, chatID)
+	if err != nil {
+		return nil, err
+	}
+	return q.db.GetChatMessagesByChatID(ctx, c.ID)
+}
+
+func (q *querier) GetChatsByOwnerID(ctx context.Context, ownerID uuid.UUID) ([]database.Chat, error) {
+	return fetchWithPostFilter(q.auth, policy.ActionRead, q.db.GetChatsByOwnerID)(ctx, ownerID)
+}
+
 func (q *querier) GetCoordinatorResumeTokenSigningKey(ctx context.Context) (string, error) {
 	if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceSystem); err != nil {
 		return "", err
@@ -1893,14 +1913,6 @@ func (q *querier) GetHealthSettings(ctx context.Context) (string, error) {
 	return q.db.GetHealthSettings(ctx)
 }
 
-// TODO: We need to create a ProvisionerJob resource type
-func (q *querier) GetHungProvisionerJobs(ctx context.Context, hungSince time.Time) ([]database.ProvisionerJob, error) {
-	// if err := q.authorizeContext(ctx, policy.ActionCreate, rbac.ResourceSystem); err != nil {
-	// return nil, err
-	// }
-	return q.db.GetHungProvisionerJobs(ctx, hungSince)
-}
-
 func (q *querier) GetInboxNotificationByID(ctx context.Context, id uuid.UUID) (database.InboxNotification, error) {
 	return fetchWithAction(q.log, q.auth, policy.ActionRead, q.db.GetInboxNotificationByID)(ctx, id)
 }
@@ -2214,6 +2226,15 @@ func (q *querier) GetPresetParametersByTemplateVersionID(ctx context.Context, ar
 	return q.db.GetPresetParametersByTemplateVersionID(ctx, args)
 }
 
+func (q *querier) GetPresetsAtFailureLimit(ctx context.Context, hardLimit int64) ([]database.GetPresetsAtFailureLimitRow, error) {
+	// GetPresetsAtFailureLimit returns a list of template version presets that have reached the hard failure limit.
+	// Request the same authorization permissions as GetPresetsBackoff, since the methods are similar.
+	if err := q.authorizeContext(ctx, policy.ActionViewInsights, rbac.ResourceTemplate.All()); err != nil {
+		return nil, err
+	}
+	return q.db.GetPresetsAtFailureLimit(ctx, hardLimit)
+}
+
 func (q *querier) GetPresetsBackoff(ctx context.Context, lookback time.Time) ([]database.GetPresetsBackoffRow, error) {
 	// GetPresetsBackoff returns a list of template version presets along with metadata such as the number of failed prebuilds.
 	if err := q.authorizeContext(ctx, policy.ActionViewInsights, rbac.ResourceTemplate.All()); err != nil {
@@ -2288,6 +2309,13 @@ func (q *querier) GetProvisionerJobByID(ctx context.Context, id uuid.UUID) (data
 	return job, nil
 }
 
+func (q *querier) GetProvisionerJobByIDForUpdate(ctx context.Context, id uuid.UUID) (database.ProvisionerJob, error) {
+	if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceProvisionerJobs); err != nil {
+		return database.ProvisionerJob{}, err
+	}
+	return q.db.GetProvisionerJobByIDForUpdate(ctx, id)
+}
+
 func (q *querier) GetProvisionerJobTimingsByJobID(ctx context.Context, jobID uuid.UUID) ([]database.ProvisionerJobTiming, error) {
 	_, err := q.GetProvisionerJobByID(ctx, jobID)
 	if err != nil {
@@ -2296,31 +2324,49 @@ func (q *querier) GetProvisionerJobTimingsByJobID(ctx context.Context, jobID uui
 	return q.db.GetProvisionerJobTimingsByJobID(ctx, jobID)
 }
 
-// TODO: We have a ProvisionerJobs resource, but it hasn't been checked for this use-case.
 func (q *querier) GetProvisionerJobsByIDs(ctx context.Context, ids []uuid.UUID) ([]database.ProvisionerJob, error) {
-	// if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceSystem); err != nil {
-	// 	return nil, err
-	// }
-	return q.db.GetProvisionerJobsByIDs(ctx, ids)
+	provisionerJobs, err := q.db.GetProvisionerJobsByIDs(ctx, ids)
+	if err != nil {
+		return nil, err
+	}
+	orgIDs := make(map[uuid.UUID]struct{})
+	for _, job := range provisionerJobs {
+		orgIDs[job.OrganizationID] = struct{}{}
+	}
+	for orgID := range orgIDs {
+		if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceProvisionerJobs.InOrg(orgID)); err != nil {
+			return nil, err
+		}
+	}
+	return provisionerJobs, nil
 }
 
-// TODO: We have a ProvisionerJobs resource, but it hasn't been checked for this use-case.
 func (q *querier) GetProvisionerJobsByIDsWithQueuePosition(ctx context.Context, ids []uuid.UUID) ([]database.GetProvisionerJobsByIDsWithQueuePositionRow, error) {
+	// TODO: Remove this once we have a proper rbac check for provisioner jobs.
+	// Details in https://github.com/coder/coder/issues/16160
 	return q.db.GetProvisionerJobsByIDsWithQueuePosition(ctx, ids)
 }
 
 func (q *querier) GetProvisionerJobsByOrganizationAndStatusWithQueuePositionAndProvisioner(ctx context.Context, arg database.GetProvisionerJobsByOrganizationAndStatusWithQueuePositionAndProvisionerParams) ([]database.GetProvisionerJobsByOrganizationAndStatusWithQueuePositionAndProvisionerRow, error) {
+	// TODO: Remove this once we have a proper rbac check for provisioner jobs.
+	// Details in https://github.com/coder/coder/issues/16160
 	return fetchWithPostFilter(q.auth, policy.ActionRead, q.db.GetProvisionerJobsByOrganizationAndStatusWithQueuePositionAndProvisioner)(ctx, arg)
 }
 
-// TODO: We have a ProvisionerJobs resource, but it hasn't been checked for this use-case.
 func (q *querier) GetProvisionerJobsCreatedAfter(ctx context.Context, createdAt time.Time) ([]database.ProvisionerJob, error) {
-	// if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceSystem); err != nil {
-	// return nil, err
-	// }
+	if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceProvisionerJobs); err != nil {
+		return nil, err
+	}
 	return q.db.GetProvisionerJobsCreatedAfter(ctx, createdAt)
 }
 
+func (q *querier) GetProvisionerJobsToBeReaped(ctx context.Context, arg database.GetProvisionerJobsToBeReapedParams) ([]database.ProvisionerJob, error) {
+	if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceProvisionerJobs); err != nil {
+		return nil, err
+	}
+	return q.db.GetProvisionerJobsToBeReaped(ctx, arg)
+}
+
 func (q *querier) GetProvisionerKeyByHashedSecret(ctx context.Context, hashedSecret []byte) (database.ProvisionerKey, error) {
 	return fetch(q.log, q.auth, q.db.GetProvisionerKeyByHashedSecret)(ctx, hashedSecret)
 }
@@ -3001,6 +3047,15 @@ func (q *querier) GetWorkspaceAgentsByResourceIDs(ctx context.Context, ids []uui
 	return q.db.GetWorkspaceAgentsByResourceIDs(ctx, ids)
 }
 
+func (q *querier) GetWorkspaceAgentsByWorkspaceAndBuildNumber(ctx context.Context, arg database.GetWorkspaceAgentsByWorkspaceAndBuildNumberParams) ([]database.WorkspaceAgent, error) {
+	_, err := q.GetWorkspaceByID(ctx, arg.WorkspaceID)
+	if err != nil {
+		return nil, err
+	}
+
+	return q.db.GetWorkspaceAgentsByWorkspaceAndBuildNumber(ctx, arg)
+}
+
 func (q *querier) GetWorkspaceAgentsCreatedAfter(ctx context.Context, createdAt time.Time) ([]database.WorkspaceAgent, error) {
 	if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceSystem); err != nil {
 		return nil, err
@@ -3134,6 +3189,10 @@ func (q *querier) GetWorkspaceByOwnerIDAndName(ctx context.Context, arg database
 	return fetch(q.log, q.auth, q.db.GetWorkspaceByOwnerIDAndName)(ctx, arg)
 }
 
+func (q *querier) GetWorkspaceByResourceID(ctx context.Context, resourceID uuid.UUID) (database.Workspace, error) {
+	return fetch(q.log, q.auth, q.db.GetWorkspaceByResourceID)(ctx, resourceID)
+}
+
 func (q *querier) GetWorkspaceByWorkspaceAppID(ctx context.Context, workspaceAppID uuid.UUID) (database.Workspace, error) {
 	return fetch(q.log, q.auth, q.db.GetWorkspaceByWorkspaceAppID)(ctx, workspaceAppID)
 }
@@ -3315,6 +3374,21 @@ func (q *querier) InsertAuditLog(ctx context.Context, arg database.InsertAuditLo
 	return insert(q.log, q.auth, rbac.ResourceAuditLog, q.db.InsertAuditLog)(ctx, arg)
 }
 
+func (q *querier) InsertChat(ctx context.Context, arg database.InsertChatParams) (database.Chat, error) {
+	return insert(q.log, q.auth, rbac.ResourceChat.WithOwner(arg.OwnerID.String()), q.db.InsertChat)(ctx, arg)
+}
+
+func (q *querier) InsertChatMessages(ctx context.Context, arg database.InsertChatMessagesParams) ([]database.ChatMessage, error) {
+	c, err := q.db.GetChatByID(ctx, arg.ChatID)
+	if err != nil {
+		return nil, err
+	}
+	if err := q.authorizeContext(ctx, policy.ActionUpdate, c); err != nil {
+		return nil, err
+	}
+	return q.db.InsertChatMessages(ctx, arg)
+}
+
 func (q *querier) InsertCryptoKey(ctx context.Context, arg database.InsertCryptoKeyParams) (database.CryptoKey, error) {
 	if err := q.authorizeContext(ctx, policy.ActionCreate, rbac.ResourceCryptoKey); err != nil {
 		return database.CryptoKey{}, err
@@ -3490,27 +3564,22 @@ func (q *querier) InsertPresetParameters(ctx context.Context, arg database.Inser
 	return q.db.InsertPresetParameters(ctx, arg)
 }
 
-// TODO: We need to create a ProvisionerJob resource type
 func (q *querier) InsertProvisionerJob(ctx context.Context, arg database.InsertProvisionerJobParams) (database.ProvisionerJob, error) {
-	// if err := q.authorizeContext(ctx, policy.ActionCreate, rbac.ResourceSystem); err != nil {
-	// return database.ProvisionerJob{}, err
-	// }
+	// TODO: Remove this once we have a proper rbac check for provisioner jobs.
+	// Details in https://github.com/coder/coder/issues/16160
 	return q.db.InsertProvisionerJob(ctx, arg)
 }
 
-// TODO: We need to create a ProvisionerJob resource type
 func (q *querier) InsertProvisionerJobLogs(ctx context.Context, arg database.InsertProvisionerJobLogsParams) ([]database.ProvisionerJobLog, error) {
-	// if err := q.authorizeContext(ctx, policy.ActionCreate, rbac.ResourceSystem); err != nil {
-	// return nil, err
-	// }
+	// TODO: Remove this once we have a proper rbac check for provisioner jobs.
+	// Details in https://github.com/coder/coder/issues/16160
 	return q.db.InsertProvisionerJobLogs(ctx, arg)
 }
 
-// TODO: We need to create a ProvisionerJob resource type
 func (q *querier) InsertProvisionerJobTimings(ctx context.Context, arg database.InsertProvisionerJobTimingsParams) ([]database.ProvisionerJobTiming, error) {
-	// if err := q.authorizeContext(ctx, policy.ActionCreate, rbac.ResourceSystem); err != nil {
-	// return nil, err
-	// }
+	if err := q.authorizeContext(ctx, policy.ActionUpdate, rbac.ResourceProvisionerJobs); err != nil {
+		return nil, err
+	}
 	return q.db.InsertProvisionerJobTimings(ctx, arg)
 }
 
@@ -3657,9 +3726,24 @@ func (q *querier) InsertWorkspace(ctx context.Context, arg database.InsertWorksp
 }
 
 func (q *querier) InsertWorkspaceAgent(ctx context.Context, arg database.InsertWorkspaceAgentParams) (database.WorkspaceAgent, error) {
-	if err := q.authorizeContext(ctx, policy.ActionCreate, rbac.ResourceSystem); err != nil {
+	// NOTE(DanielleMaywood):
+	// Currently, the only way to link a Resource back to a Workspace is by following this chain:
+	//
+	//   WorkspaceResource -> WorkspaceBuild -> Workspace
+	//
+	// It is possible for this function to be called without there existing
+	// a `WorkspaceBuild` to link back to. This means that we want to allow
+	// execution to continue if there isn't a workspace found to allow this
+	// behavior to continue.
+	workspace, err := q.db.GetWorkspaceByResourceID(ctx, arg.ResourceID)
+	if err != nil && !errors.Is(err, sql.ErrNoRows) {
 		return database.WorkspaceAgent{}, err
 	}
+
+	if err := q.authorizeContext(ctx, policy.ActionCreateAgent, workspace); err != nil {
+		return database.WorkspaceAgent{}, err
+	}
+
 	return q.db.InsertWorkspaceAgent(ctx, arg)
 }
 
@@ -3963,6 +4047,13 @@ func (q *querier) UpdateAPIKeyByID(ctx context.Context, arg database.UpdateAPIKe
 	return update(q.log, q.auth, fetch, q.db.UpdateAPIKeyByID)(ctx, arg)
 }
 
+func (q *querier) UpdateChatByID(ctx context.Context, arg database.UpdateChatByIDParams) error {
+	fetch := func(ctx context.Context, arg database.UpdateChatByIDParams) (database.Chat, error) {
+		return q.db.GetChatByID(ctx, arg.ID)
+	}
+	return update(q.log, q.auth, fetch, q.db.UpdateChatByID)(ctx, arg)
+}
+
 func (q *querier) UpdateCryptoKeyDeletesAt(ctx context.Context, arg database.UpdateCryptoKeyDeletesAtParams) (database.CryptoKey, error) {
 	if err := q.authorizeContext(ctx, policy.ActionUpdate, rbac.ResourceCryptoKey); err != nil {
 		return database.CryptoKey{}, err
@@ -4119,6 +4210,24 @@ func (q *querier) UpdateOrganizationDeletedByID(ctx context.Context, arg databas
 	return deleteQ(q.log, q.auth, q.db.GetOrganizationByID, deleteF)(ctx, arg.ID)
 }
 
+func (q *querier) UpdatePresetPrebuildStatus(ctx context.Context, arg database.UpdatePresetPrebuildStatusParams) error {
+	preset, err := q.db.GetPresetByID(ctx, arg.PresetID)
+	if err != nil {
+		return err
+	}
+
+	object := rbac.ResourceTemplate.
+		WithID(preset.TemplateID.UUID).
+		InOrg(preset.OrganizationID)
+
+	err = q.authorizeContext(ctx, policy.ActionUpdate, object)
+	if err != nil {
+		return err
+	}
+
+	return q.db.UpdatePresetPrebuildStatus(ctx, arg)
+}
+
 func (q *querier) UpdateProvisionerDaemonLastSeenAt(ctx context.Context, arg database.UpdateProvisionerDaemonLastSeenAtParams) error {
 	if err := q.authorizeContext(ctx, policy.ActionUpdate, rbac.ResourceProvisionerDaemon); err != nil {
 		return err
@@ -4126,15 +4235,17 @@ func (q *querier) UpdateProvisionerDaemonLastSeenAt(ctx context.Context, arg dat
 	return q.db.UpdateProvisionerDaemonLastSeenAt(ctx, arg)
 }
 
-// TODO: We need to create a ProvisionerJob resource type
 func (q *querier) UpdateProvisionerJobByID(ctx context.Context, arg database.UpdateProvisionerJobByIDParams) error {
-	// if err := q.authorizeContext(ctx, policy.ActionUpdate, rbac.ResourceSystem); err != nil {
-	// return err
-	// }
+	if err := q.authorizeContext(ctx, policy.ActionUpdate, rbac.ResourceProvisionerJobs); err != nil {
+		return err
+	}
 	return q.db.UpdateProvisionerJobByID(ctx, arg)
 }
 
 func (q *querier) UpdateProvisionerJobWithCancelByID(ctx context.Context, arg database.UpdateProvisionerJobWithCancelByIDParams) error {
+	// TODO: Remove this once we have a proper rbac check for provisioner jobs.
+	// Details in https://github.com/coder/coder/issues/16160
+
 	job, err := q.db.GetProvisionerJobByID(ctx, arg.ID)
 	if err != nil {
 		return err
@@ -4201,14 +4312,20 @@ func (q *querier) UpdateProvisionerJobWithCancelByID(ctx context.Context, arg da
 	return q.db.UpdateProvisionerJobWithCancelByID(ctx, arg)
 }
 
-// TODO: We need to create a ProvisionerJob resource type
 func (q *querier) UpdateProvisionerJobWithCompleteByID(ctx context.Context, arg database.UpdateProvisionerJobWithCompleteByIDParams) error {
-	// if err := q.authorizeContext(ctx, policy.ActionUpdate, rbac.ResourceSystem); err != nil {
-	// return err
-	// }
+	if err := q.authorizeContext(ctx, policy.ActionUpdate, rbac.ResourceProvisionerJobs); err != nil {
+		return err
+	}
 	return q.db.UpdateProvisionerJobWithCompleteByID(ctx, arg)
 }
 
+func (q *querier) UpdateProvisionerJobWithCompleteWithStartedAtByID(ctx context.Context, arg database.UpdateProvisionerJobWithCompleteWithStartedAtByIDParams) error {
+	if err := q.authorizeContext(ctx, policy.ActionUpdate, rbac.ResourceProvisionerJobs); err != nil {
+		return err
+	}
+	return q.db.UpdateProvisionerJobWithCompleteWithStartedAtByID(ctx, arg)
+}
+
 func (q *querier) UpdateReplica(ctx context.Context, arg database.UpdateReplicaParams) (database.Replica, error) {
 	if err := q.authorizeContext(ctx, policy.ActionUpdate, rbac.ResourceSystem); err != nil {
 		return database.Replica{}, err

coderd/database/dbauthz/dbauthz_test.go

@@ -694,9 +694,12 @@ func (s *MethodTestSuite) TestProvisionerJob() {
 			Asserts(v.RBACObject(tpl), []policy.Action{policy.ActionRead, policy.ActionUpdate}).Returns()
 	}))
 	s.Run("GetProvisionerJobsByIDs", s.Subtest(func(db database.Store, check *expects) {
-		a := dbgen.ProvisionerJob(s.T(), db, nil, database.ProvisionerJob{})
-		b := dbgen.ProvisionerJob(s.T(), db, nil, database.ProvisionerJob{})
-		check.Args([]uuid.UUID{a.ID, b.ID}).Asserts().Returns(slice.New(a, b))
+		o := dbgen.Organization(s.T(), db, database.Organization{})
+		a := dbgen.ProvisionerJob(s.T(), db, nil, database.ProvisionerJob{OrganizationID: o.ID})
+		b := dbgen.ProvisionerJob(s.T(), db, nil, database.ProvisionerJob{OrganizationID: o.ID})
+		check.Args([]uuid.UUID{a.ID, b.ID}).
+			Asserts(rbac.ResourceProvisionerJobs.InOrg(o.ID), policy.ActionRead).
+			Returns(slice.New(a, b))
 	}))
 	s.Run("GetProvisionerLogsAfterID", s.Subtest(func(db database.Store, check *expects) {
 		u := dbgen.User(s.T(), db, database.User{})
@@ -1214,8 +1217,8 @@ func (s *MethodTestSuite) TestTemplate() {
 			JobID:          job.ID,
 			TemplateID:     uuid.NullUUID{UUID: t.ID, Valid: true},
 		})
-		dbgen.TemplateVersionTerraformValues(s.T(), db, database.InsertTemplateVersionTerraformValuesByJobIDParams{
-			JobID: job.ID,
+		dbgen.TemplateVersionTerraformValues(s.T(), db, database.TemplateVersionTerraformValue{
+			TemplateVersionID: tv.ID,
 		})
 		check.Args(tv.ID).Asserts(t, policy.ActionRead)
 	}))
@@ -1925,6 +1928,22 @@ func (s *MethodTestSuite) TestWorkspace() {
 		})
 		check.Args(ws.ID).Asserts(ws, policy.ActionRead)
 	}))
+	s.Run("GetWorkspaceByResourceID", s.Subtest(func(db database.Store, check *expects) {
+		u := dbgen.User(s.T(), db, database.User{})
+		o := dbgen.Organization(s.T(), db, database.Organization{})
+		j := dbgen.ProvisionerJob(s.T(), db, nil, database.ProvisionerJob{Type: database.ProvisionerJobTypeWorkspaceBuild})
+		tpl := dbgen.Template(s.T(), db, database.Template{CreatedBy: u.ID, OrganizationID: o.ID})
+		tv := dbgen.TemplateVersion(s.T(), db, database.TemplateVersion{
+			TemplateID:     uuid.NullUUID{UUID: tpl.ID, Valid: true},
+			JobID:          j.ID,
+			OrganizationID: o.ID,
+			CreatedBy:      u.ID,
+		})
+		ws := dbgen.Workspace(s.T(), db, database.WorkspaceTable{OwnerID: u.ID, TemplateID: tpl.ID, OrganizationID: o.ID})
+		_ = dbgen.WorkspaceBuild(s.T(), db, database.WorkspaceBuild{WorkspaceID: ws.ID, JobID: j.ID, TemplateVersionID: tv.ID})
+		res := dbgen.WorkspaceResource(s.T(), db, database.WorkspaceResource{JobID: j.ID})
+		check.Args(res.ID).Asserts(ws, policy.ActionRead)
+	}))
 	s.Run("GetWorkspaces", s.Subtest(func(_ database.Store, check *expects) {
 		// No asserts here because SQLFilter.
 		check.Args(database.GetWorkspacesParams{}).Asserts()
@@ -2009,6 +2028,38 @@ func (s *MethodTestSuite) TestWorkspace() {
 		agt := dbgen.WorkspaceAgent(s.T(), db, database.WorkspaceAgent{ResourceID: res.ID})
 		check.Args(agt.ID).Asserts(w, policy.ActionRead).Returns(agt)
 	}))
+	s.Run("GetWorkspaceAgentsByWorkspaceAndBuildNumber", s.Subtest(func(db database.Store, check *expects) {
+		u := dbgen.User(s.T(), db, database.User{})
+		o := dbgen.Organization(s.T(), db, database.Organization{})
+		tpl := dbgen.Template(s.T(), db, database.Template{
+			OrganizationID: o.ID,
+			CreatedBy:      u.ID,
+		})
+		tv := dbgen.TemplateVersion(s.T(), db, database.TemplateVersion{
+			TemplateID:     uuid.NullUUID{UUID: tpl.ID, Valid: true},
+			OrganizationID: o.ID,
+			CreatedBy:      u.ID,
+		})
+		w := dbgen.Workspace(s.T(), db, database.WorkspaceTable{
+			TemplateID:     tpl.ID,
+			OrganizationID: o.ID,
+			OwnerID:        u.ID,
+		})
+		j := dbgen.ProvisionerJob(s.T(), db, nil, database.ProvisionerJob{
+			Type: database.ProvisionerJobTypeWorkspaceBuild,
+		})
+		b := dbgen.WorkspaceBuild(s.T(), db, database.WorkspaceBuild{
+			JobID:             j.ID,
+			WorkspaceID:       w.ID,
+			TemplateVersionID: tv.ID,
+		})
+		res := dbgen.WorkspaceResource(s.T(), db, database.WorkspaceResource{JobID: b.JobID})
+		agt := dbgen.WorkspaceAgent(s.T(), db, database.WorkspaceAgent{ResourceID: res.ID})
+		check.Args(database.GetWorkspaceAgentsByWorkspaceAndBuildNumberParams{
+			WorkspaceID: w.ID,
+			BuildNumber: 1,
+		}).Asserts(w, policy.ActionRead).Returns([]database.WorkspaceAgent{agt})
+	}))
 	s.Run("GetWorkspaceAgentLifecycleStateByID", s.Subtest(func(db database.Store, check *expects) {
 		u := dbgen.User(s.T(), db, database.User{})
 		o := dbgen.Organization(s.T(), db, database.Organization{})
@@ -3891,9 +3942,8 @@ func (s *MethodTestSuite) TestSystemFunctions() {
 		check.Args().Asserts(rbac.ResourceSystem, policy.ActionDelete)
 	}))
 	s.Run("GetProvisionerJobsCreatedAfter", s.Subtest(func(db database.Store, check *expects) {
-		// TODO: add provisioner job resource type
 		_ = dbgen.ProvisionerJob(s.T(), db, nil, database.ProvisionerJob{CreatedAt: time.Now().Add(-time.Hour)})
-		check.Args(time.Now()).Asserts( /*rbac.ResourceSystem, policy.ActionRead*/ )
+		check.Args(time.Now()).Asserts(rbac.ResourceProvisionerJobs, policy.ActionRead)
 	}))
 	s.Run("GetTemplateVersionsByIDs", s.Subtest(func(db database.Store, check *expects) {
 		dbtestutil.DisableForeignKeysAndTriggers(s.T(), db)
@@ -3976,19 +4026,33 @@ func (s *MethodTestSuite) TestSystemFunctions() {
 			Returns([]database.WorkspaceAgent{agt})
 	}))
 	s.Run("GetProvisionerJobsByIDs", s.Subtest(func(db database.Store, check *expects) {
-		// TODO: add a ProvisionerJob resource type
-		a := dbgen.ProvisionerJob(s.T(), db, nil, database.ProvisionerJob{})
-		b := dbgen.ProvisionerJob(s.T(), db, nil, database.ProvisionerJob{})
+		o := dbgen.Organization(s.T(), db, database.Organization{})
+		a := dbgen.ProvisionerJob(s.T(), db, nil, database.ProvisionerJob{OrganizationID: o.ID})
+		b := dbgen.ProvisionerJob(s.T(), db, nil, database.ProvisionerJob{OrganizationID: o.ID})
 		check.Args([]uuid.UUID{a.ID, b.ID}).
-			Asserts( /*rbac.ResourceSystem, policy.ActionRead*/ ).
+			Asserts(rbac.ResourceProvisionerJobs.InOrg(o.ID), policy.ActionRead).
 			Returns(slice.New(a, b))
 	}))
 	s.Run("InsertWorkspaceAgent", s.Subtest(func(db database.Store, check *expects) {
-		dbtestutil.DisableForeignKeysAndTriggers(s.T(), db)
+		u := dbgen.User(s.T(), db, database.User{})
+		o := dbgen.Organization(s.T(), db, database.Organization{})
+		j := dbgen.ProvisionerJob(s.T(), db, nil, database.ProvisionerJob{Type: database.ProvisionerJobTypeWorkspaceBuild})
+		tpl := dbgen.Template(s.T(), db, database.Template{CreatedBy: u.ID, OrganizationID: o.ID})
+		tv := dbgen.TemplateVersion(s.T(), db, database.TemplateVersion{
+			TemplateID:     uuid.NullUUID{UUID: tpl.ID, Valid: true},
+			JobID:          j.ID,
+			OrganizationID: o.ID,
+			CreatedBy:      u.ID,
+		})
+		ws := dbgen.Workspace(s.T(), db, database.WorkspaceTable{OwnerID: u.ID, TemplateID: tpl.ID, OrganizationID: o.ID})
+		_ = dbgen.WorkspaceBuild(s.T(), db, database.WorkspaceBuild{WorkspaceID: ws.ID, JobID: j.ID, TemplateVersionID: tv.ID})
+		res := dbgen.WorkspaceResource(s.T(), db, database.WorkspaceResource{JobID: j.ID})
 		check.Args(database.InsertWorkspaceAgentParams{
-			ID:   uuid.New(),
-			Name: "dev",
-		}).Asserts(rbac.ResourceSystem, policy.ActionCreate)
+			ID:          uuid.New(),
+			ResourceID:  res.ID,
+			Name:        "dev",
+			APIKeyScope: database.AgentKeyScopeEnumAll,
+		}).Asserts(ws, policy.ActionCreateAgent)
 	}))
 	s.Run("InsertWorkspaceApp", s.Subtest(func(db database.Store, check *expects) {
 		dbtestutil.DisableForeignKeysAndTriggers(s.T(), db)
@@ -4015,7 +4079,6 @@ func (s *MethodTestSuite) TestSystemFunctions() {
 		}).Asserts(rbac.ResourceSystem, policy.ActionUpdate).Returns()
 	}))
 	s.Run("AcquireProvisionerJob", s.Subtest(func(db database.Store, check *expects) {
-		// TODO: we need to create a ProvisionerJob resource
 		j := dbgen.ProvisionerJob(s.T(), db, nil, database.ProvisionerJob{
 			StartedAt: sql.NullTime{Valid: false},
 			UpdatedAt: time.Now(),
@@ -4025,47 +4088,48 @@ func (s *MethodTestSuite) TestSystemFunctions() {
 			OrganizationID:  j.OrganizationID,
 			Types:           []database.ProvisionerType{j.Provisioner},
 			ProvisionerTags: must(json.Marshal(j.Tags)),
-		}).Asserts( /*rbac.ResourceSystem, policy.ActionUpdate*/ )
+		}).Asserts(rbac.ResourceProvisionerJobs, policy.ActionUpdate)
 	}))
 	s.Run("UpdateProvisionerJobWithCompleteByID", s.Subtest(func(db database.Store, check *expects) {
-		// TODO: we need to create a ProvisionerJob resource
 		j := dbgen.ProvisionerJob(s.T(), db, nil, database.ProvisionerJob{})
 		check.Args(database.UpdateProvisionerJobWithCompleteByIDParams{
 			ID: j.ID,
-		}).Asserts( /*rbac.ResourceSystem, policy.ActionUpdate*/ )
+		}).Asserts(rbac.ResourceProvisionerJobs, policy.ActionUpdate)
+	}))
+	s.Run("UpdateProvisionerJobWithCompleteWithStartedAtByID", s.Subtest(func(db database.Store, check *expects) {
+		j := dbgen.ProvisionerJob(s.T(), db, nil, database.ProvisionerJob{})
+		check.Args(database.UpdateProvisionerJobWithCompleteWithStartedAtByIDParams{
+			ID: j.ID,
+		}).Asserts(rbac.ResourceProvisionerJobs, policy.ActionUpdate)
 	}))
 	s.Run("UpdateProvisionerJobByID", s.Subtest(func(db database.Store, check *expects) {
-		// TODO: we need to create a ProvisionerJob resource
 		j := dbgen.ProvisionerJob(s.T(), db, nil, database.ProvisionerJob{})
 		check.Args(database.UpdateProvisionerJobByIDParams{
 			ID:        j.ID,
 			UpdatedAt: time.Now(),
-		}).Asserts( /*rbac.ResourceSystem, policy.ActionUpdate*/ )
+		}).Asserts(rbac.ResourceProvisionerJobs, policy.ActionUpdate)
 	}))
 	s.Run("InsertProvisionerJob", s.Subtest(func(db database.Store, check *expects) {
 		dbtestutil.DisableForeignKeysAndTriggers(s.T(), db)
-		// TODO: we need to create a ProvisionerJob resource
 		check.Args(database.InsertProvisionerJobParams{
 			ID:            uuid.New(),
 			Provisioner:   database.ProvisionerTypeEcho,
 			StorageMethod: database.ProvisionerStorageMethodFile,
 			Type:          database.ProvisionerJobTypeWorkspaceBuild,
 			Input:         json.RawMessage("{}"),
-		}).Asserts( /*rbac.ResourceSystem, policy.ActionCreate*/ )
+		}).Asserts( /* rbac.ResourceProvisionerJobs, policy.ActionCreate */ )
 	}))
 	s.Run("InsertProvisionerJobLogs", s.Subtest(func(db database.Store, check *expects) {
-		// TODO: we need to create a ProvisionerJob resource
 		j := dbgen.ProvisionerJob(s.T(), db, nil, database.ProvisionerJob{})
 		check.Args(database.InsertProvisionerJobLogsParams{
 			JobID: j.ID,
-		}).Asserts( /*rbac.ResourceSystem, policy.ActionCreate*/ )
+		}).Asserts( /* rbac.ResourceProvisionerJobs, policy.ActionUpdate */ )
 	}))
 	s.Run("InsertProvisionerJobTimings", s.Subtest(func(db database.Store, check *expects) {
-		// TODO: we need to create a ProvisionerJob resource
 		j := dbgen.ProvisionerJob(s.T(), db, nil, database.ProvisionerJob{})
 		check.Args(database.InsertProvisionerJobTimingsParams{
 			JobID: j.ID,
-		}).Asserts( /*rbac.ResourceSystem, policy.ActionCreate*/ )
+		}).Asserts(rbac.ResourceProvisionerJobs, policy.ActionUpdate)
 	}))
 	s.Run("UpsertProvisionerDaemon", s.Subtest(func(db database.Store, check *expects) {
 		dbtestutil.DisableForeignKeysAndTriggers(s.T(), db)
@@ -4201,8 +4265,8 @@ func (s *MethodTestSuite) TestSystemFunctions() {
 	s.Run("GetFileTemplates", s.Subtest(func(db database.Store, check *expects) {
 		check.Args(uuid.New()).Asserts(rbac.ResourceSystem, policy.ActionRead)
 	}))
-	s.Run("GetHungProvisionerJobs", s.Subtest(func(db database.Store, check *expects) {
-		check.Args(time.Time{}).Asserts()
+	s.Run("GetProvisionerJobsToBeReaped", s.Subtest(func(db database.Store, check *expects) {
+		check.Args(database.GetProvisionerJobsToBeReapedParams{}).Asserts(rbac.ResourceProvisionerJobs, policy.ActionRead)
 	}))
 	s.Run("UpsertOAuthSigningKey", s.Subtest(func(db database.Store, check *expects) {
 		check.Args("foo").Asserts(rbac.ResourceSystem, policy.ActionUpdate)
@@ -4446,6 +4510,9 @@ func (s *MethodTestSuite) TestSystemFunctions() {
 			VapidPrivateKey: "test",
 		}).Asserts(rbac.ResourceDeploymentConfig, policy.ActionUpdate)
 	}))
+	s.Run("GetProvisionerJobByIDForUpdate", s.Subtest(func(db database.Store, check *expects) {
+		check.Args(uuid.New()).Asserts(rbac.ResourceProvisionerJobs, policy.ActionRead).Errors(sql.ErrNoRows)
+	}))
 }
 
 func (s *MethodTestSuite) TestNotifications() {
@@ -4857,6 +4924,11 @@ func (s *MethodTestSuite) TestPrebuilds() {
 			Asserts(rbac.ResourceWorkspace.All(), policy.ActionRead).
 			ErrorsWithInMemDB(dbmem.ErrUnimplemented)
 	}))
+	s.Run("GetPresetsAtFailureLimit", s.Subtest(func(_ database.Store, check *expects) {
+		check.Args(int64(0)).
+			Asserts(rbac.ResourceTemplate.All(), policy.ActionViewInsights).
+			ErrorsWithInMemDB(dbmem.ErrUnimplemented)
+	}))
 	s.Run("GetPresetsBackoff", s.Subtest(func(_ database.Store, check *expects) {
 		check.Args(time.Time{}).
 			Asserts(rbac.ResourceTemplate.All(), policy.ActionViewInsights).
@@ -4904,8 +4976,34 @@ func (s *MethodTestSuite) TestPrebuilds() {
 				},
 				InvalidateAfterSecs: preset.InvalidateAfterSecs,
 				OrganizationID:      org.ID,
+				PrebuildStatus:      database.PrebuildStatusHealthy,
 			})
 	}))
+	s.Run("UpdatePresetPrebuildStatus", s.Subtest(func(db database.Store, check *expects) {
+		org := dbgen.Organization(s.T(), db, database.Organization{})
+		user := dbgen.User(s.T(), db, database.User{})
+		template := dbgen.Template(s.T(), db, database.Template{
+			OrganizationID: org.ID,
+			CreatedBy:      user.ID,
+		})
+		templateVersion := dbgen.TemplateVersion(s.T(), db, database.TemplateVersion{
+			TemplateID: uuid.NullUUID{
+				UUID:  template.ID,
+				Valid: true,
+			},
+			OrganizationID: org.ID,
+			CreatedBy:      user.ID,
+		})
+		preset := dbgen.Preset(s.T(), db, database.InsertPresetParams{
+			TemplateVersionID: templateVersion.ID,
+		})
+		req := database.UpdatePresetPrebuildStatusParams{
+			PresetID: preset.ID,
+			Status:   database.PrebuildStatusHealthy,
+		}
+		check.Args(req).
+			Asserts(rbac.ResourceTemplate.WithID(template.ID).InOrg(org.ID), policy.ActionUpdate)
+	}))
 }
 
 func (s *MethodTestSuite) TestOAuth2ProviderApps() {
@@ -5307,3 +5405,77 @@ func (s *MethodTestSuite) TestResourcesProvisionerdserver() {
 		}).Asserts(rbac.ResourceWorkspaceAgentDevcontainers, policy.ActionCreate)
 	}))
 }
+
+func (s *MethodTestSuite) TestChat() {
+	createChat := func(t *testing.T, db database.Store) (database.User, database.Chat, database.ChatMessage) {
+		t.Helper()
+
+		usr := dbgen.User(t, db, database.User{})
+		chat := dbgen.Chat(s.T(), db, database.Chat{
+			OwnerID: usr.ID,
+		})
+		msg := dbgen.ChatMessage(s.T(), db, database.ChatMessage{
+			ChatID: chat.ID,
+		})
+
+		return usr, chat, msg
+	}
+
+	s.Run("DeleteChat", s.Subtest(func(db database.Store, check *expects) {
+		_, c, _ := createChat(s.T(), db)
+		check.Args(c.ID).Asserts(c, policy.ActionDelete)
+	}))
+
+	s.Run("GetChatByID", s.Subtest(func(db database.Store, check *expects) {
+		_, c, _ := createChat(s.T(), db)
+		check.Args(c.ID).Asserts(c, policy.ActionRead).Returns(c)
+	}))
+
+	s.Run("GetChatMessagesByChatID", s.Subtest(func(db database.Store, check *expects) {
+		_, c, m := createChat(s.T(), db)
+		check.Args(c.ID).Asserts(c, policy.ActionRead).Returns([]database.ChatMessage{m})
+	}))
+
+	s.Run("GetChatsByOwnerID", s.Subtest(func(db database.Store, check *expects) {
+		u1, u1c1, _ := createChat(s.T(), db)
+		u1c2 := dbgen.Chat(s.T(), db, database.Chat{
+			OwnerID:   u1.ID,
+			CreatedAt: u1c1.CreatedAt.Add(time.Hour),
+		})
+		_, _, _ = createChat(s.T(), db) // other user's chat
+		check.Args(u1.ID).Asserts(u1c2, policy.ActionRead, u1c1, policy.ActionRead).Returns([]database.Chat{u1c2, u1c1})
+	}))
+
+	s.Run("InsertChat", s.Subtest(func(db database.Store, check *expects) {
+		usr := dbgen.User(s.T(), db, database.User{})
+		check.Args(database.InsertChatParams{
+			OwnerID:   usr.ID,
+			Title:     "test chat",
+			CreatedAt: dbtime.Now(),
+			UpdatedAt: dbtime.Now(),
+		}).Asserts(rbac.ResourceChat.WithOwner(usr.ID.String()), policy.ActionCreate)
+	}))
+
+	s.Run("InsertChatMessages", s.Subtest(func(db database.Store, check *expects) {
+		usr := dbgen.User(s.T(), db, database.User{})
+		chat := dbgen.Chat(s.T(), db, database.Chat{
+			OwnerID: usr.ID,
+		})
+		check.Args(database.InsertChatMessagesParams{
+			ChatID:    chat.ID,
+			CreatedAt: dbtime.Now(),
+			Model:     "test-model",
+			Provider:  "test-provider",
+			Content:   []byte(`[]`),
+		}).Asserts(chat, policy.ActionUpdate)
+	}))
+
+	s.Run("UpdateChatByID", s.Subtest(func(db database.Store, check *expects) {
+		_, c, _ := createChat(s.T(), db)
+		check.Args(database.UpdateChatByIDParams{
+			ID:        c.ID,
+			Title:     "new title",
+			UpdatedAt: dbtime.Now(),
+		}).Asserts(c, policy.ActionUpdate)
+	}))
+}

coderd/database/dbfake/dbfake.go

@@ -294,6 +294,8 @@ type TemplateVersionBuilder struct {
 	ps                 pubsub.Pubsub
 	resources          []*sdkproto.Resource
 	params             []database.TemplateVersionParameter
+	presets            []database.TemplateVersionPreset
+	presetParams       []database.TemplateVersionPresetParameter
 	promote            bool
 	autoCreateTemplate bool
 }
@@ -339,6 +341,13 @@ func (t TemplateVersionBuilder) Params(ps ...database.TemplateVersionParameter)
 	return t
 }
 
+func (t TemplateVersionBuilder) Preset(preset database.TemplateVersionPreset, params ...database.TemplateVersionPresetParameter) TemplateVersionBuilder {
+	// nolint: revive // returns modified struct
+	t.presets = append(t.presets, preset)
+	t.presetParams = append(t.presetParams, params...)
+	return t
+}
+
 func (t TemplateVersionBuilder) SkipCreateTemplate() TemplateVersionBuilder {
 	// nolint: revive // returns modified struct
 	t.autoCreateTemplate = false
@@ -378,6 +387,25 @@ func (t TemplateVersionBuilder) Do() TemplateVersionResponse {
 		require.NoError(t.t, err)
 	}
 
+	for _, preset := range t.presets {
+		dbgen.Preset(t.t, t.db, database.InsertPresetParams{
+			ID:                  preset.ID,
+			TemplateVersionID:   version.ID,
+			Name:                preset.Name,
+			CreatedAt:           version.CreatedAt,
+			DesiredInstances:    preset.DesiredInstances,
+			InvalidateAfterSecs: preset.InvalidateAfterSecs,
+		})
+	}
+
+	for _, presetParam := range t.presetParams {
+		dbgen.PresetParameter(t.t, t.db, database.InsertPresetParametersParams{
+			TemplateVersionPresetID: presetParam.TemplateVersionPresetID,
+			Names:                   []string{presetParam.Name},
+			Values:                  []string{presetParam.Value},
+		})
+	}
+
 	payload, err := json.Marshal(provisionerdserver.TemplateVersionImportJob{
 		TemplateVersionID: t.seed.ID,
 	})

coderd/database/dbgen/dbgen.go

@@ -29,6 +29,7 @@ import (
 	"github.com/coder/coder/v2/coderd/rbac"
 	"github.com/coder/coder/v2/codersdk"
 	"github.com/coder/coder/v2/cryptorand"
+	"github.com/coder/coder/v2/provisionerd/proto"
 	"github.com/coder/coder/v2/testutil"
 )
 
@@ -142,6 +143,30 @@ func APIKey(t testing.TB, db database.Store, seed database.APIKey) (key database
 	return key, fmt.Sprintf("%s-%s", key.ID, secret)
 }
 
+func Chat(t testing.TB, db database.Store, seed database.Chat) database.Chat {
+	chat, err := db.InsertChat(genCtx, database.InsertChatParams{
+		OwnerID:   takeFirst(seed.OwnerID, uuid.New()),
+		CreatedAt: takeFirst(seed.CreatedAt, dbtime.Now()),
+		UpdatedAt: takeFirst(seed.UpdatedAt, dbtime.Now()),
+		Title:     takeFirst(seed.Title, "Test Chat"),
+	})
+	require.NoError(t, err, "insert chat")
+	return chat
+}
+
+func ChatMessage(t testing.TB, db database.Store, seed database.ChatMessage) database.ChatMessage {
+	msg, err := db.InsertChatMessages(genCtx, database.InsertChatMessagesParams{
+		CreatedAt: takeFirst(seed.CreatedAt, dbtime.Now()),
+		ChatID:    takeFirst(seed.ChatID, uuid.New()),
+		Model:     takeFirst(seed.Model, "train"),
+		Provider:  takeFirst(seed.Provider, "thomas"),
+		Content:   takeFirstSlice(seed.Content, []byte(`[{"text": "Choo choo!"}]`)),
+	})
+	require.NoError(t, err, "insert chat message")
+	require.Len(t, msg, 1, "insert one chat message did not return exactly one message")
+	return msg[0]
+}
+
 func WorkspaceAgentPortShare(t testing.TB, db database.Store, orig database.WorkspaceAgentPortShare) database.WorkspaceAgentPortShare {
 	ps, err := db.UpsertWorkspaceAgentPortShare(genCtx, database.UpsertWorkspaceAgentPortShareParams{
 		WorkspaceID: takeFirst(orig.WorkspaceID, uuid.New()),
@@ -157,6 +182,7 @@ func WorkspaceAgentPortShare(t testing.TB, db database.Store, orig database.Work
 func WorkspaceAgent(t testing.TB, db database.Store, orig database.WorkspaceAgent) database.WorkspaceAgent {
 	agt, err := db.InsertWorkspaceAgent(genCtx, database.InsertWorkspaceAgentParams{
 		ID:         takeFirst(orig.ID, uuid.New()),
+		ParentID:   takeFirst(orig.ParentID, uuid.NullUUID{}),
 		CreatedAt:  takeFirst(orig.CreatedAt, dbtime.Now()),
 		UpdatedAt:  takeFirst(orig.UpdatedAt, dbtime.Now()),
 		Name:       takeFirst(orig.Name, testutil.GetRandomName(t)),
@@ -186,6 +212,7 @@ func WorkspaceAgent(t testing.TB, db database.Store, orig database.WorkspaceAgen
 		MOTDFile:                 takeFirst(orig.TroubleshootingURL, ""),
 		DisplayApps:              append([]database.DisplayApp{}, orig.DisplayApps...),
 		DisplayOrder:             takeFirst(orig.DisplayOrder, 1),
+		APIKeyScope:              takeFirst(orig.APIKeyScope, database.AgentKeyScopeEnumAll),
 	})
 	require.NoError(t, err, "insert workspace agent")
 	return agt
@@ -721,6 +748,7 @@ func WorkspaceApp(t testing.TB, db database.Store, orig database.WorkspaceApp) d
 		HealthcheckThreshold: takeFirst(orig.HealthcheckThreshold, 60),
 		Health:               takeFirst(orig.Health, database.WorkspaceAppHealthHealthy),
 		DisplayOrder:         takeFirst(orig.DisplayOrder, 1),
+		DisplayGroup:         orig.DisplayGroup,
 		Hidden:               orig.Hidden,
 		OpenIn:               takeFirst(orig.OpenIn, database.WorkspaceAppOpenInSlimWindow),
 	})
@@ -971,17 +999,32 @@ func TemplateVersionParameter(t testing.TB, db database.Store, orig database.Tem
 	return version
 }
 
-func TemplateVersionTerraformValues(t testing.TB, db database.Store, orig database.InsertTemplateVersionTerraformValuesByJobIDParams) {
+func TemplateVersionTerraformValues(t testing.TB, db database.Store, orig database.TemplateVersionTerraformValue) database.TemplateVersionTerraformValue {
 	t.Helper()
 
+	jobID := uuid.New()
+	if orig.TemplateVersionID != uuid.Nil {
+		v, err := db.GetTemplateVersionByID(genCtx, orig.TemplateVersionID)
+		if err == nil {
+			jobID = v.JobID
+		}
+	}
+
 	params := database.InsertTemplateVersionTerraformValuesByJobIDParams{
-		JobID:      takeFirst(orig.JobID, uuid.New()),
-		CachedPlan: takeFirstSlice(orig.CachedPlan, []byte("{}")),
-		UpdatedAt:  takeFirst(orig.UpdatedAt, dbtime.Now()),
+		JobID:               jobID,
+		CachedPlan:          takeFirstSlice(orig.CachedPlan, []byte("{}")),
+		CachedModuleFiles:   orig.CachedModuleFiles,
+		UpdatedAt:           takeFirst(orig.UpdatedAt, dbtime.Now()),
+		ProvisionerdVersion: takeFirst(orig.ProvisionerdVersion, proto.CurrentVersion.String()),
 	}
 
 	err := db.InsertTemplateVersionTerraformValuesByJobID(genCtx, params)
 	require.NoError(t, err, "insert template version parameter")
+
+	v, err := db.GetTemplateVersionTerraformValues(genCtx, orig.TemplateVersionID)
+	require.NoError(t, err, "get template version values")
+
+	return v
 }
 
 func WorkspaceAgentStat(t testing.TB, db database.Store, orig database.WorkspaceAgentStat) database.WorkspaceAgentStat {
@@ -1198,6 +1241,7 @@ func TelemetryItem(t testing.TB, db database.Store, seed database.TelemetryItem)
 
 func Preset(t testing.TB, db database.Store, seed database.InsertPresetParams) database.TemplateVersionPreset {
 	preset, err := db.InsertPreset(genCtx, database.InsertPresetParams{
+		ID:                  takeFirst(seed.ID, uuid.New()),
 		TemplateVersionID:   takeFirst(seed.TemplateVersionID, uuid.New()),
 		Name:                takeFirst(seed.Name, testutil.GetRandomName(t)),
 		CreatedAt:           takeFirst(seed.CreatedAt, dbtime.Now()),

coderd/database/dbmem/dbmem.go

@@ -8,6 +8,7 @@ import (
 	"errors"
 	"fmt"
 	"math"
+	insecurerand "math/rand" //#nosec // this is only used for shuffling an array to pick random jobs to reap
 	"reflect"
 	"regexp"
 	"slices"
@@ -215,6 +216,8 @@ type data struct {
 
 	// New tables
 	auditLogs                            []database.AuditLog
+	chats                                []database.Chat
+	chatMessages                         []database.ChatMessage
 	cryptoKeys                           []database.CryptoKey
 	dbcryptKeys                          []database.DBCryptKey
 	files                                []database.File
@@ -528,6 +531,7 @@ func (q *FakeQuerier) convertToWorkspaceRowsNoLock(ctx context.Context, workspac
 
 			OwnerAvatarUrl: extended.OwnerAvatarUrl,
 			OwnerUsername:  extended.OwnerUsername,
+			OwnerName:      extended.OwnerName,
 
 			OrganizationName:        extended.OrganizationName,
 			OrganizationDisplayName: extended.OrganizationDisplayName,
@@ -625,6 +629,7 @@ func (q *FakeQuerier) extendWorkspace(w database.WorkspaceTable) database.Worksp
 		return u.ID == w.OwnerID
 	})
 	extended.OwnerUsername = owner.Username
+	extended.OwnerName = owner.Name
 	extended.OwnerAvatarUrl = owner.AvatarURL
 
 	return extended
@@ -1378,6 +1383,12 @@ func (q *FakeQuerier) getProvisionerJobsByIDsWithQueuePositionLockedGlobalQueue(
 	return jobs, nil
 }
 
+// isDeprecated returns true if the template is deprecated.
+// A template is considered deprecated when it has a deprecation message.
+func isDeprecated(template database.Template) bool {
+	return template.Deprecated != ""
+}
+
 func (*FakeQuerier) AcquireLock(_ context.Context, _ int64) error {
 	return xerrors.New("AcquireLock must only be called within a transaction")
 }
@@ -1885,6 +1896,19 @@ func (q *FakeQuerier) DeleteApplicationConnectAPIKeysByUserID(_ context.Context,
 	return nil
 }
 
+func (q *FakeQuerier) DeleteChat(ctx context.Context, id uuid.UUID) error {
+	q.mutex.Lock()
+	defer q.mutex.Unlock()
+
+	for i, chat := range q.chats {
+		if chat.ID == id {
+			q.chats = append(q.chats[:i], q.chats[i+1:]...)
+			return nil
+		}
+	}
+	return sql.ErrNoRows
+}
+
 func (*FakeQuerier) DeleteCoordinator(context.Context, uuid.UUID) error {
 	return ErrUnimplemented
 }
@@ -2866,6 +2890,47 @@ func (q *FakeQuerier) GetAuthorizationUserRoles(_ context.Context, userID uuid.U
 	}, nil
 }
 
+func (q *FakeQuerier) GetChatByID(ctx context.Context, id uuid.UUID) (database.Chat, error) {
+	q.mutex.RLock()
+	defer q.mutex.RUnlock()
+
+	for _, chat := range q.chats {
+		if chat.ID == id {
+			return chat, nil
+		}
+	}
+	return database.Chat{}, sql.ErrNoRows
+}
+
+func (q *FakeQuerier) GetChatMessagesByChatID(ctx context.Context, chatID uuid.UUID) ([]database.ChatMessage, error) {
+	q.mutex.RLock()
+	defer q.mutex.RUnlock()
+
+	messages := []database.ChatMessage{}
+	for _, chatMessage := range q.chatMessages {
+		if chatMessage.ChatID == chatID {
+			messages = append(messages, chatMessage)
+		}
+	}
+	return messages, nil
+}
+
+func (q *FakeQuerier) GetChatsByOwnerID(ctx context.Context, ownerID uuid.UUID) ([]database.Chat, error) {
+	q.mutex.RLock()
+	defer q.mutex.RUnlock()
+
+	chats := []database.Chat{}
+	for _, chat := range q.chats {
+		if chat.OwnerID == ownerID {
+			chats = append(chats, chat)
+		}
+	}
+	sort.Slice(chats, func(i, j int) bool {
+		return chats[i].CreatedAt.After(chats[j].CreatedAt)
+	})
+	return chats, nil
+}
+
 func (q *FakeQuerier) GetCoordinatorResumeTokenSigningKey(_ context.Context) (string, error) {
 	q.mutex.RLock()
 	defer q.mutex.RUnlock()
@@ -3645,23 +3710,6 @@ func (q *FakeQuerier) GetHealthSettings(_ context.Context) (string, error) {
 	return string(q.healthSettings), nil
 }
 
-func (q *FakeQuerier) GetHungProvisionerJobs(_ context.Context, hungSince time.Time) ([]database.ProvisionerJob, error) {
-	q.mutex.RLock()
-	defer q.mutex.RUnlock()
-
-	hungJobs := []database.ProvisionerJob{}
-	for _, provisionerJob := range q.provisionerJobs {
-		if provisionerJob.StartedAt.Valid && !provisionerJob.CompletedAt.Valid && provisionerJob.UpdatedAt.Before(hungSince) {
-			// clone the Tags before appending, since maps are reference types and
-			// we don't want the caller to be able to mutate the map we have inside
-			// dbmem!
-			provisionerJob.Tags = maps.Clone(provisionerJob.Tags)
-			hungJobs = append(hungJobs, provisionerJob)
-		}
-	}
-	return hungJobs, nil
-}
-
 func (q *FakeQuerier) GetInboxNotificationByID(_ context.Context, id uuid.UUID) (database.InboxNotification, error) {
 	q.mutex.RLock()
 	defer q.mutex.RUnlock()
@@ -4241,6 +4289,7 @@ func (q *FakeQuerier) GetPresetByID(ctx context.Context, presetID uuid.UUID) (da
 				CreatedAt:           preset.CreatedAt,
 				DesiredInstances:    preset.DesiredInstances,
 				InvalidateAfterSecs: preset.InvalidateAfterSecs,
+				PrebuildStatus:      preset.PrebuildStatus,
 				TemplateID:          tv.TemplateID,
 				OrganizationID:      tv.OrganizationID,
 			}, nil
@@ -4306,6 +4355,10 @@ func (q *FakeQuerier) GetPresetParametersByTemplateVersionID(_ context.Context,
 	return parameters, nil
 }
 
+func (q *FakeQuerier) GetPresetsAtFailureLimit(ctx context.Context, hardLimit int64) ([]database.GetPresetsAtFailureLimitRow, error) {
+	return nil, ErrUnimplemented
+}
+
 func (*FakeQuerier) GetPresetsBackoff(_ context.Context, _ time.Time) ([]database.GetPresetsBackoffRow, error) {
 	return nil, ErrUnimplemented
 }
@@ -4580,6 +4633,13 @@ func (q *FakeQuerier) GetProvisionerJobByID(ctx context.Context, id uuid.UUID) (
 	return q.getProvisionerJobByIDNoLock(ctx, id)
 }
 
+func (q *FakeQuerier) GetProvisionerJobByIDForUpdate(ctx context.Context, id uuid.UUID) (database.ProvisionerJob, error) {
+	q.mutex.RLock()
+	defer q.mutex.RUnlock()
+
+	return q.getProvisionerJobByIDNoLock(ctx, id)
+}
+
 func (q *FakeQuerier) GetProvisionerJobTimingsByJobID(_ context.Context, jobID uuid.UUID) ([]database.ProvisionerJobTiming, error) {
 	q.mutex.RLock()
 	defer q.mutex.RUnlock()
@@ -4786,6 +4846,13 @@ func (q *FakeQuerier) GetProvisionerJobsByOrganizationAndStatusWithQueuePosition
 				row.AvailableWorkers = append(row.AvailableWorkers, worker.ID)
 			}
 		}
+
+		// Add daemon name to provisioner job
+		for _, daemon := range q.provisionerDaemons {
+			if job.WorkerID.Valid && job.WorkerID.UUID == daemon.ID {
+				row.WorkerName = daemon.Name
+			}
+		}
 		rows = append(rows, row)
 	}
 
@@ -4815,6 +4882,33 @@ func (q *FakeQuerier) GetProvisionerJobsCreatedAfter(_ context.Context, after ti
 	return jobs, nil
 }
 
+func (q *FakeQuerier) GetProvisionerJobsToBeReaped(_ context.Context, arg database.GetProvisionerJobsToBeReapedParams) ([]database.ProvisionerJob, error) {
+	q.mutex.RLock()
+	defer q.mutex.RUnlock()
+	maxJobs := arg.MaxJobs
+
+	hungJobs := []database.ProvisionerJob{}
+	for _, provisionerJob := range q.provisionerJobs {
+		if !provisionerJob.CompletedAt.Valid {
+			if (provisionerJob.StartedAt.Valid && provisionerJob.UpdatedAt.Before(arg.HungSince)) ||
+				(!provisionerJob.StartedAt.Valid && provisionerJob.UpdatedAt.Before(arg.PendingSince)) {
+				// clone the Tags before appending, since maps are reference types and
+				// we don't want the caller to be able to mutate the map we have inside
+				// dbmem!
+				provisionerJob.Tags = maps.Clone(provisionerJob.Tags)
+				hungJobs = append(hungJobs, provisionerJob)
+				if len(hungJobs) >= int(maxJobs) {
+					break
+				}
+			}
+		}
+	}
+	insecurerand.Shuffle(len(hungJobs), func(i, j int) {
+		hungJobs[i], hungJobs[j] = hungJobs[j], hungJobs[i]
+	})
+	return hungJobs, nil
+}
+
 func (q *FakeQuerier) GetProvisionerKeyByHashedSecret(_ context.Context, hashedSecret []byte) (database.ProvisionerKey, error) {
 	q.mutex.RLock()
 	defer q.mutex.RUnlock()
@@ -7592,6 +7686,30 @@ func (q *FakeQuerier) GetWorkspaceAgentsByResourceIDs(ctx context.Context, resou
 	return q.getWorkspaceAgentsByResourceIDsNoLock(ctx, resourceIDs)
 }
 
+func (q *FakeQuerier) GetWorkspaceAgentsByWorkspaceAndBuildNumber(ctx context.Context, arg database.GetWorkspaceAgentsByWorkspaceAndBuildNumberParams) ([]database.WorkspaceAgent, error) {
+	err := validateDatabaseType(arg)
+	if err != nil {
+		return nil, err
+	}
+
+	build, err := q.GetWorkspaceBuildByWorkspaceIDAndBuildNumber(ctx, database.GetWorkspaceBuildByWorkspaceIDAndBuildNumberParams(arg))
+	if err != nil {
+		return nil, err
+	}
+
+	resources, err := q.getWorkspaceResourcesByJobIDNoLock(ctx, build.JobID)
+	if err != nil {
+		return nil, err
+	}
+
+	var resourceIDs []uuid.UUID
+	for _, resource := range resources {
+		resourceIDs = append(resourceIDs, resource.ID)
+	}
+
+	return q.GetWorkspaceAgentsByResourceIDs(ctx, resourceIDs)
+}
+
 func (q *FakeQuerier) GetWorkspaceAgentsCreatedAfter(_ context.Context, after time.Time) ([]database.WorkspaceAgent, error) {
 	q.mutex.RLock()
 	defer q.mutex.RUnlock()
@@ -7942,6 +8060,33 @@ func (q *FakeQuerier) GetWorkspaceByOwnerIDAndName(_ context.Context, arg databa
 	return database.Workspace{}, sql.ErrNoRows
 }
 
+func (q *FakeQuerier) GetWorkspaceByResourceID(ctx context.Context, resourceID uuid.UUID) (database.Workspace, error) {
+	q.mutex.RLock()
+	defer q.mutex.RUnlock()
+
+	for _, resource := range q.workspaceResources {
+		if resource.ID != resourceID {
+			continue
+		}
+
+		for _, build := range q.workspaceBuilds {
+			if build.JobID != resource.JobID {
+				continue
+			}
+
+			for _, workspace := range q.workspaces {
+				if workspace.ID != build.WorkspaceID {
+					continue
+				}
+
+				return q.extendWorkspace(workspace), nil
+			}
+		}
+	}
+
+	return database.Workspace{}, sql.ErrNoRows
+}
+
 func (q *FakeQuerier) GetWorkspaceByWorkspaceAppID(_ context.Context, workspaceAppID uuid.UUID) (database.Workspace, error) {
 	if err := validateDatabaseType(workspaceAppID); err != nil {
 		return database.Workspace{}, err
@@ -8385,6 +8530,66 @@ func (q *FakeQuerier) InsertAuditLog(_ context.Context, arg database.InsertAudit
 	return alog, nil
 }
 
+func (q *FakeQuerier) InsertChat(ctx context.Context, arg database.InsertChatParams) (database.Chat, error) {
+	err := validateDatabaseType(arg)
+	if err != nil {
+		return database.Chat{}, err
+	}
+
+	q.mutex.Lock()
+	defer q.mutex.Unlock()
+
+	chat := database.Chat{
+		ID:        uuid.New(),
+		CreatedAt: arg.CreatedAt,
+		UpdatedAt: arg.UpdatedAt,
+		OwnerID:   arg.OwnerID,
+		Title:     arg.Title,
+	}
+	q.chats = append(q.chats, chat)
+
+	return chat, nil
+}
+
+func (q *FakeQuerier) InsertChatMessages(ctx context.Context, arg database.InsertChatMessagesParams) ([]database.ChatMessage, error) {
+	err := validateDatabaseType(arg)
+	if err != nil {
+		return nil, err
+	}
+
+	q.mutex.Lock()
+	defer q.mutex.Unlock()
+
+	id := int64(0)
+	if len(q.chatMessages) > 0 {
+		id = q.chatMessages[len(q.chatMessages)-1].ID
+	}
+
+	messages := make([]database.ChatMessage, 0)
+
+	rawMessages := make([]json.RawMessage, 0)
+	err = json.Unmarshal(arg.Content, &rawMessages)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, content := range rawMessages {
+		id++
+		_ = content
+		messages = append(messages, database.ChatMessage{
+			ID:        id,
+			ChatID:    arg.ChatID,
+			CreatedAt: arg.CreatedAt,
+			Model:     arg.Model,
+			Provider:  arg.Provider,
+			Content:   content,
+		})
+	}
+
+	q.chatMessages = append(q.chatMessages, messages...)
+	return messages, nil
+}
+
 func (q *FakeQuerier) InsertCryptoKey(_ context.Context, arg database.InsertCryptoKeyParams) (database.CryptoKey, error) {
 	err := validateDatabaseType(arg)
 	if err != nil {
@@ -8891,6 +9096,7 @@ func (q *FakeQuerier) InsertPreset(_ context.Context, arg database.InsertPresetP
 			Int32: 0,
 			Valid: true,
 		},
+		PrebuildStatus: database.PrebuildStatusHealthy,
 	}
 	q.presets = append(q.presets, preset)
 	return preset, nil
@@ -9197,9 +9403,11 @@ func (q *FakeQuerier) InsertTemplateVersionTerraformValuesByJobID(_ context.Cont
 
 	// Insert the new row
 	row := database.TemplateVersionTerraformValue{
-		TemplateVersionID: templateVersion.ID,
-		CachedPlan:        arg.CachedPlan,
-		UpdatedAt:         arg.UpdatedAt,
+		TemplateVersionID:   templateVersion.ID,
+		UpdatedAt:           arg.UpdatedAt,
+		CachedPlan:          arg.CachedPlan,
+		CachedModuleFiles:   arg.CachedModuleFiles,
+		ProvisionerdVersion: arg.ProvisionerdVersion,
 	}
 	q.templateVersionTerraformValues = append(q.templateVersionTerraformValues, row)
 	return nil
@@ -9453,6 +9661,7 @@ func (q *FakeQuerier) InsertWorkspaceAgent(_ context.Context, arg database.Inser
 
 	agent := database.WorkspaceAgent{
 		ID:                       arg.ID,
+		ParentID:                 arg.ParentID,
 		CreatedAt:                arg.CreatedAt,
 		UpdatedAt:                arg.UpdatedAt,
 		ResourceID:               arg.ResourceID,
@@ -9471,6 +9680,7 @@ func (q *FakeQuerier) InsertWorkspaceAgent(_ context.Context, arg database.Inser
 		LifecycleState:           database.WorkspaceAgentLifecycleStateCreated,
 		DisplayApps:              arg.DisplayApps,
 		DisplayOrder:             arg.DisplayOrder,
+		APIKeyScope:              arg.APIKeyScope,
 	}
 
 	q.workspaceAgents = append(q.workspaceAgents, agent)
@@ -10342,6 +10552,27 @@ func (q *FakeQuerier) UpdateAPIKeyByID(_ context.Context, arg database.UpdateAPI
 	return sql.ErrNoRows
 }
 
+func (q *FakeQuerier) UpdateChatByID(ctx context.Context, arg database.UpdateChatByIDParams) error {
+	err := validateDatabaseType(arg)
+	if err != nil {
+		return err
+	}
+
+	q.mutex.Lock()
+	defer q.mutex.Unlock()
+
+	for i, chat := range q.chats {
+		if chat.ID == arg.ID {
+			q.chats[i].Title = arg.Title
+			q.chats[i].UpdatedAt = arg.UpdatedAt
+			q.chats[i] = chat
+			return nil
+		}
+	}
+
+	return sql.ErrNoRows
+}
+
 func (q *FakeQuerier) UpdateCryptoKeyDeletesAt(_ context.Context, arg database.UpdateCryptoKeyDeletesAtParams) (database.CryptoKey, error) {
 	err := validateDatabaseType(arg)
 	if err != nil {
@@ -10694,6 +10925,25 @@ func (q *FakeQuerier) UpdateOrganizationDeletedByID(_ context.Context, arg datab
 	return sql.ErrNoRows
 }
 
+func (q *FakeQuerier) UpdatePresetPrebuildStatus(ctx context.Context, arg database.UpdatePresetPrebuildStatusParams) error {
+	err := validateDatabaseType(arg)
+	if err != nil {
+		return err
+	}
+
+	q.mutex.RLock()
+	defer q.mutex.RUnlock()
+
+	for _, preset := range q.presets {
+		if preset.ID == arg.PresetID {
+			preset.PrebuildStatus = arg.Status
+			return nil
+		}
+	}
+
+	return xerrors.Errorf("preset %v does not exist", arg.PresetID)
+}
+
 func (q *FakeQuerier) UpdateProvisionerDaemonLastSeenAt(_ context.Context, arg database.UpdateProvisionerDaemonLastSeenAtParams) error {
 	err := validateDatabaseType(arg)
 	if err != nil {
@@ -10780,6 +11030,30 @@ func (q *FakeQuerier) UpdateProvisionerJobWithCompleteByID(_ context.Context, ar
 	return sql.ErrNoRows
 }
 
+func (q *FakeQuerier) UpdateProvisionerJobWithCompleteWithStartedAtByID(_ context.Context, arg database.UpdateProvisionerJobWithCompleteWithStartedAtByIDParams) error {
+	if err := validateDatabaseType(arg); err != nil {
+		return err
+	}
+
+	q.mutex.Lock()
+	defer q.mutex.Unlock()
+
+	for index, job := range q.provisionerJobs {
+		if arg.ID != job.ID {
+			continue
+		}
+		job.UpdatedAt = arg.UpdatedAt
+		job.CompletedAt = arg.CompletedAt
+		job.Error = arg.Error
+		job.ErrorCode = arg.ErrorCode
+		job.StartedAt = arg.StartedAt
+		job.JobStatus = provisionerJobStatus(job)
+		q.provisionerJobs[index] = job
+		return nil
+	}
+	return sql.ErrNoRows
+}
+
 func (q *FakeQuerier) UpdateReplica(_ context.Context, arg database.UpdateReplicaParams) (database.Replica, error) {
 	if err := validateDatabaseType(arg); err != nil {
 		return database.Replica{}, err
@@ -10913,6 +11187,7 @@ func (q *FakeQuerier) UpdateTemplateMetaByID(_ context.Context, arg database.Upd
 		tpl.GroupACL = arg.GroupACL
 		tpl.AllowUserCancelWorkspaceJobs = arg.AllowUserCancelWorkspaceJobs
 		tpl.MaxPortSharingLevel = arg.MaxPortSharingLevel
+		tpl.UseClassicParameterFlow = arg.UseClassicParameterFlow
 		q.templates[idx] = tpl
 		return nil
 	}
@@ -12884,7 +13159,17 @@ func (q *FakeQuerier) GetAuthorizedTemplates(ctx context.Context, arg database.G
 		if arg.ExactName != "" && !strings.EqualFold(template.Name, arg.ExactName) {
 			continue
 		}
-		if arg.Deprecated.Valid && arg.Deprecated.Bool == (template.Deprecated != "") {
+		// Filters templates based on the search query filter 'Deprecated' status
+		// Matching SQL logic:
+		// -- Filter by deprecated
+		// AND CASE
+		//   WHEN :deprecated IS NOT NULL THEN
+		//     CASE
+		//       WHEN :deprecated THEN deprecated != ''
+		//       ELSE deprecated = ''
+		//     END
+		//   ELSE true
+		if arg.Deprecated.Valid && arg.Deprecated.Bool != isDeprecated(template) {
 			continue
 		}
 		if arg.FuzzyName != "" {

coderd/database/dbmetrics/querymetrics.go

@@ -249,6 +249,13 @@ func (m queryMetricsStore) DeleteApplicationConnectAPIKeysByUserID(ctx context.C
 	return err
 }
 
+func (m queryMetricsStore) DeleteChat(ctx context.Context, id uuid.UUID) error {
+	start := time.Now()
+	r0 := m.s.DeleteChat(ctx, id)
+	m.queryLatencies.WithLabelValues("DeleteChat").Observe(time.Since(start).Seconds())
+	return r0
+}
+
 func (m queryMetricsStore) DeleteCoordinator(ctx context.Context, id uuid.UUID) error {
 	start := time.Now()
 	r0 := m.s.DeleteCoordinator(ctx, id)
@@ -627,6 +634,27 @@ func (m queryMetricsStore) GetAuthorizationUserRoles(ctx context.Context, userID
 	return row, err
 }
 
+func (m queryMetricsStore) GetChatByID(ctx context.Context, id uuid.UUID) (database.Chat, error) {
+	start := time.Now()
+	r0, r1 := m.s.GetChatByID(ctx, id)
+	m.queryLatencies.WithLabelValues("GetChatByID").Observe(time.Since(start).Seconds())
+	return r0, r1
+}
+
+func (m queryMetricsStore) GetChatMessagesByChatID(ctx context.Context, chatID uuid.UUID) ([]database.ChatMessage, error) {
+	start := time.Now()
+	r0, r1 := m.s.GetChatMessagesByChatID(ctx, chatID)
+	m.queryLatencies.WithLabelValues("GetChatMessagesByChatID").Observe(time.Since(start).Seconds())
+	return r0, r1
+}
+
+func (m queryMetricsStore) GetChatsByOwnerID(ctx context.Context, ownerID uuid.UUID) ([]database.Chat, error) {
+	start := time.Now()
+	r0, r1 := m.s.GetChatsByOwnerID(ctx, ownerID)
+	m.queryLatencies.WithLabelValues("GetChatsByOwnerID").Observe(time.Since(start).Seconds())
+	return r0, r1
+}
+
 func (m queryMetricsStore) GetCoordinatorResumeTokenSigningKey(ctx context.Context) (string, error) {
 	start := time.Now()
 	r0, r1 := m.s.GetCoordinatorResumeTokenSigningKey(ctx)
@@ -837,13 +865,6 @@ func (m queryMetricsStore) GetHealthSettings(ctx context.Context) (string, error
 	return r0, r1
 }
 
-func (m queryMetricsStore) GetHungProvisionerJobs(ctx context.Context, hungSince time.Time) ([]database.ProvisionerJob, error) {
-	start := time.Now()
-	jobs, err := m.s.GetHungProvisionerJobs(ctx, hungSince)
-	m.queryLatencies.WithLabelValues("GetHungProvisionerJobs").Observe(time.Since(start).Seconds())
-	return jobs, err
-}
-
 func (m queryMetricsStore) GetInboxNotificationByID(ctx context.Context, id uuid.UUID) (database.InboxNotification, error) {
 	start := time.Now()
 	r0, r1 := m.s.GetInboxNotificationByID(ctx, id)
@@ -1117,6 +1138,13 @@ func (m queryMetricsStore) GetPresetParametersByTemplateVersionID(ctx context.Co
 	return r0, r1
 }
 
+func (m queryMetricsStore) GetPresetsAtFailureLimit(ctx context.Context, hardLimit int64) ([]database.GetPresetsAtFailureLimitRow, error) {
+	start := time.Now()
+	r0, r1 := m.s.GetPresetsAtFailureLimit(ctx, hardLimit)
+	m.queryLatencies.WithLabelValues("GetPresetsAtFailureLimit").Observe(time.Since(start).Seconds())
+	return r0, r1
+}
+
 func (m queryMetricsStore) GetPresetsBackoff(ctx context.Context, lookback time.Time) ([]database.GetPresetsBackoffRow, error) {
 	start := time.Now()
 	r0, r1 := m.s.GetPresetsBackoff(ctx, lookback)
@@ -1166,6 +1194,13 @@ func (m queryMetricsStore) GetProvisionerJobByID(ctx context.Context, id uuid.UU
 	return job, err
 }
 
+func (m queryMetricsStore) GetProvisionerJobByIDForUpdate(ctx context.Context, id uuid.UUID) (database.ProvisionerJob, error) {
+	start := time.Now()
+	r0, r1 := m.s.GetProvisionerJobByIDForUpdate(ctx, id)
+	m.queryLatencies.WithLabelValues("GetProvisionerJobByIDForUpdate").Observe(time.Since(start).Seconds())
+	return r0, r1
+}
+
 func (m queryMetricsStore) GetProvisionerJobTimingsByJobID(ctx context.Context, jobID uuid.UUID) ([]database.ProvisionerJobTiming, error) {
 	start := time.Now()
 	r0, r1 := m.s.GetProvisionerJobTimingsByJobID(ctx, jobID)
@@ -1201,6 +1236,13 @@ func (m queryMetricsStore) GetProvisionerJobsCreatedAfter(ctx context.Context, c
 	return jobs, err
 }
 
+func (m queryMetricsStore) GetProvisionerJobsToBeReaped(ctx context.Context, arg database.GetProvisionerJobsToBeReapedParams) ([]database.ProvisionerJob, error) {
+	start := time.Now()
+	r0, r1 := m.s.GetProvisionerJobsToBeReaped(ctx, arg)
+	m.queryLatencies.WithLabelValues("GetProvisionerJobsToBeReaped").Observe(time.Since(start).Seconds())
+	return r0, r1
+}
+
 func (m queryMetricsStore) GetProvisionerKeyByHashedSecret(ctx context.Context, hashedSecret []byte) (database.ProvisionerKey, error) {
 	start := time.Now()
 	r0, r1 := m.s.GetProvisionerKeyByHashedSecret(ctx, hashedSecret)
@@ -1726,6 +1768,13 @@ func (m queryMetricsStore) GetWorkspaceAgentsByResourceIDs(ctx context.Context,
 	return agents, err
 }
 
+func (m queryMetricsStore) GetWorkspaceAgentsByWorkspaceAndBuildNumber(ctx context.Context, arg database.GetWorkspaceAgentsByWorkspaceAndBuildNumberParams) ([]database.WorkspaceAgent, error) {
+	start := time.Now()
+	r0, r1 := m.s.GetWorkspaceAgentsByWorkspaceAndBuildNumber(ctx, arg)
+	m.queryLatencies.WithLabelValues("GetWorkspaceAgentsByWorkspaceAndBuildNumber").Observe(time.Since(start).Seconds())
+	return r0, r1
+}
+
 func (m queryMetricsStore) GetWorkspaceAgentsCreatedAfter(ctx context.Context, createdAt time.Time) ([]database.WorkspaceAgent, error) {
 	start := time.Now()
 	agents, err := m.s.GetWorkspaceAgentsCreatedAfter(ctx, createdAt)
@@ -1845,6 +1894,13 @@ func (m queryMetricsStore) GetWorkspaceByOwnerIDAndName(ctx context.Context, arg
 	return workspace, err
 }
 
+func (m queryMetricsStore) GetWorkspaceByResourceID(ctx context.Context, resourceID uuid.UUID) (database.Workspace, error) {
+	start := time.Now()
+	r0, r1 := m.s.GetWorkspaceByResourceID(ctx, resourceID)
+	m.queryLatencies.WithLabelValues("GetWorkspaceByResourceID").Observe(time.Since(start).Seconds())
+	return r0, r1
+}
+
 func (m queryMetricsStore) GetWorkspaceByWorkspaceAppID(ctx context.Context, workspaceAppID uuid.UUID) (database.Workspace, error) {
 	start := time.Now()
 	workspace, err := m.s.GetWorkspaceByWorkspaceAppID(ctx, workspaceAppID)
@@ -1992,6 +2048,20 @@ func (m queryMetricsStore) InsertAuditLog(ctx context.Context, arg database.Inse
 	return log, err
 }
 
+func (m queryMetricsStore) InsertChat(ctx context.Context, arg database.InsertChatParams) (database.Chat, error) {
+	start := time.Now()
+	r0, r1 := m.s.InsertChat(ctx, arg)
+	m.queryLatencies.WithLabelValues("InsertChat").Observe(time.Since(start).Seconds())
+	return r0, r1
+}
+
+func (m queryMetricsStore) InsertChatMessages(ctx context.Context, arg database.InsertChatMessagesParams) ([]database.ChatMessage, error) {
+	start := time.Now()
+	r0, r1 := m.s.InsertChatMessages(ctx, arg)
+	m.queryLatencies.WithLabelValues("InsertChatMessages").Observe(time.Since(start).Seconds())
+	return r0, r1
+}
+
 func (m queryMetricsStore) InsertCryptoKey(ctx context.Context, arg database.InsertCryptoKeyParams) (database.CryptoKey, error) {
 	start := time.Now()
 	key, err := m.s.InsertCryptoKey(ctx, arg)
@@ -2517,6 +2587,13 @@ func (m queryMetricsStore) UpdateAPIKeyByID(ctx context.Context, arg database.Up
 	return err
 }
 
+func (m queryMetricsStore) UpdateChatByID(ctx context.Context, arg database.UpdateChatByIDParams) error {
+	start := time.Now()
+	r0 := m.s.UpdateChatByID(ctx, arg)
+	m.queryLatencies.WithLabelValues("UpdateChatByID").Observe(time.Since(start).Seconds())
+	return r0
+}
+
 func (m queryMetricsStore) UpdateCryptoKeyDeletesAt(ctx context.Context, arg database.UpdateCryptoKeyDeletesAtParams) (database.CryptoKey, error) {
 	start := time.Now()
 	key, err := m.s.UpdateCryptoKeyDeletesAt(ctx, arg)
@@ -2622,6 +2699,13 @@ func (m queryMetricsStore) UpdateOrganizationDeletedByID(ctx context.Context, ar
 	return r0
 }
 
+func (m queryMetricsStore) UpdatePresetPrebuildStatus(ctx context.Context, arg database.UpdatePresetPrebuildStatusParams) error {
+	start := time.Now()
+	r0 := m.s.UpdatePresetPrebuildStatus(ctx, arg)
+	m.queryLatencies.WithLabelValues("UpdatePresetPrebuildStatus").Observe(time.Since(start).Seconds())
+	return r0
+}
+
 func (m queryMetricsStore) UpdateProvisionerDaemonLastSeenAt(ctx context.Context, arg database.UpdateProvisionerDaemonLastSeenAtParams) error {
 	start := time.Now()
 	r0 := m.s.UpdateProvisionerDaemonLastSeenAt(ctx, arg)
@@ -2650,6 +2734,13 @@ func (m queryMetricsStore) UpdateProvisionerJobWithCompleteByID(ctx context.Cont
 	return err
 }
 
+func (m queryMetricsStore) UpdateProvisionerJobWithCompleteWithStartedAtByID(ctx context.Context, arg database.UpdateProvisionerJobWithCompleteWithStartedAtByIDParams) error {
+	start := time.Now()
+	r0 := m.s.UpdateProvisionerJobWithCompleteWithStartedAtByID(ctx, arg)
+	m.queryLatencies.WithLabelValues("UpdateProvisionerJobWithCompleteWithStartedAtByID").Observe(time.Since(start).Seconds())
+	return r0
+}
+
 func (m queryMetricsStore) UpdateReplica(ctx context.Context, arg database.UpdateReplicaParams) (database.Replica, error) {
 	start := time.Now()
 	replica, err := m.s.UpdateReplica(ctx, arg)

coderd/database/dbmock/dbmock.go

@@ -376,6 +376,20 @@ func (mr *MockStoreMockRecorder) DeleteApplicationConnectAPIKeysByUserID(ctx, us
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteApplicationConnectAPIKeysByUserID", reflect.TypeOf((*MockStore)(nil).DeleteApplicationConnectAPIKeysByUserID), ctx, userID)
 }
 
+// DeleteChat mocks base method.
+func (m *MockStore) DeleteChat(ctx context.Context, id uuid.UUID) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "DeleteChat", ctx, id)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// DeleteChat indicates an expected call of DeleteChat.
+func (mr *MockStoreMockRecorder) DeleteChat(ctx, id any) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteChat", reflect.TypeOf((*MockStore)(nil).DeleteChat), ctx, id)
+}
+
 // DeleteCoordinator mocks base method.
 func (m *MockStore) DeleteCoordinator(ctx context.Context, id uuid.UUID) error {
 	m.ctrl.T.Helper()
@@ -1234,6 +1248,51 @@ func (mr *MockStoreMockRecorder) GetAuthorizedWorkspacesAndAgentsByOwnerID(ctx,
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAuthorizedWorkspacesAndAgentsByOwnerID", reflect.TypeOf((*MockStore)(nil).GetAuthorizedWorkspacesAndAgentsByOwnerID), ctx, ownerID, prepared)
 }
 
+// GetChatByID mocks base method.
+func (m *MockStore) GetChatByID(ctx context.Context, id uuid.UUID) (database.Chat, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetChatByID", ctx, id)
+	ret0, _ := ret[0].(database.Chat)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// GetChatByID indicates an expected call of GetChatByID.
+func (mr *MockStoreMockRecorder) GetChatByID(ctx, id any) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetChatByID", reflect.TypeOf((*MockStore)(nil).GetChatByID), ctx, id)
+}
+
+// GetChatMessagesByChatID mocks base method.
+func (m *MockStore) GetChatMessagesByChatID(ctx context.Context, chatID uuid.UUID) ([]database.ChatMessage, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetChatMessagesByChatID", ctx, chatID)
+	ret0, _ := ret[0].([]database.ChatMessage)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// GetChatMessagesByChatID indicates an expected call of GetChatMessagesByChatID.
+func (mr *MockStoreMockRecorder) GetChatMessagesByChatID(ctx, chatID any) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetChatMessagesByChatID", reflect.TypeOf((*MockStore)(nil).GetChatMessagesByChatID), ctx, chatID)
+}
+
+// GetChatsByOwnerID mocks base method.
+func (m *MockStore) GetChatsByOwnerID(ctx context.Context, ownerID uuid.UUID) ([]database.Chat, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetChatsByOwnerID", ctx, ownerID)
+	ret0, _ := ret[0].([]database.Chat)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// GetChatsByOwnerID indicates an expected call of GetChatsByOwnerID.
+func (mr *MockStoreMockRecorder) GetChatsByOwnerID(ctx, ownerID any) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetChatsByOwnerID", reflect.TypeOf((*MockStore)(nil).GetChatsByOwnerID), ctx, ownerID)
+}
+
 // GetCoordinatorResumeTokenSigningKey mocks base method.
 func (m *MockStore) GetCoordinatorResumeTokenSigningKey(ctx context.Context) (string, error) {
 	m.ctrl.T.Helper()
@@ -1684,21 +1743,6 @@ func (mr *MockStoreMockRecorder) GetHealthSettings(ctx any) *gomock.Call {
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetHealthSettings", reflect.TypeOf((*MockStore)(nil).GetHealthSettings), ctx)
 }
 
-// GetHungProvisionerJobs mocks base method.
-func (m *MockStore) GetHungProvisionerJobs(ctx context.Context, updatedAt time.Time) ([]database.ProvisionerJob, error) {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "GetHungProvisionerJobs", ctx, updatedAt)
-	ret0, _ := ret[0].([]database.ProvisionerJob)
-	ret1, _ := ret[1].(error)
-	return ret0, ret1
-}
-
-// GetHungProvisionerJobs indicates an expected call of GetHungProvisionerJobs.
-func (mr *MockStoreMockRecorder) GetHungProvisionerJobs(ctx, updatedAt any) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetHungProvisionerJobs", reflect.TypeOf((*MockStore)(nil).GetHungProvisionerJobs), ctx, updatedAt)
-}
-
 // GetInboxNotificationByID mocks base method.
 func (m *MockStore) GetInboxNotificationByID(ctx context.Context, id uuid.UUID) (database.InboxNotification, error) {
 	m.ctrl.T.Helper()
@@ -2284,6 +2328,21 @@ func (mr *MockStoreMockRecorder) GetPresetParametersByTemplateVersionID(ctx, tem
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPresetParametersByTemplateVersionID", reflect.TypeOf((*MockStore)(nil).GetPresetParametersByTemplateVersionID), ctx, templateVersionID)
 }
 
+// GetPresetsAtFailureLimit mocks base method.
+func (m *MockStore) GetPresetsAtFailureLimit(ctx context.Context, hardLimit int64) ([]database.GetPresetsAtFailureLimitRow, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetPresetsAtFailureLimit", ctx, hardLimit)
+	ret0, _ := ret[0].([]database.GetPresetsAtFailureLimitRow)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// GetPresetsAtFailureLimit indicates an expected call of GetPresetsAtFailureLimit.
+func (mr *MockStoreMockRecorder) GetPresetsAtFailureLimit(ctx, hardLimit any) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPresetsAtFailureLimit", reflect.TypeOf((*MockStore)(nil).GetPresetsAtFailureLimit), ctx, hardLimit)
+}
+
 // GetPresetsBackoff mocks base method.
 func (m *MockStore) GetPresetsBackoff(ctx context.Context, lookback time.Time) ([]database.GetPresetsBackoffRow, error) {
 	m.ctrl.T.Helper()
@@ -2389,6 +2448,21 @@ func (mr *MockStoreMockRecorder) GetProvisionerJobByID(ctx, id any) *gomock.Call
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetProvisionerJobByID", reflect.TypeOf((*MockStore)(nil).GetProvisionerJobByID), ctx, id)
 }
 
+// GetProvisionerJobByIDForUpdate mocks base method.
+func (m *MockStore) GetProvisionerJobByIDForUpdate(ctx context.Context, id uuid.UUID) (database.ProvisionerJob, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetProvisionerJobByIDForUpdate", ctx, id)
+	ret0, _ := ret[0].(database.ProvisionerJob)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// GetProvisionerJobByIDForUpdate indicates an expected call of GetProvisionerJobByIDForUpdate.
+func (mr *MockStoreMockRecorder) GetProvisionerJobByIDForUpdate(ctx, id any) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetProvisionerJobByIDForUpdate", reflect.TypeOf((*MockStore)(nil).GetProvisionerJobByIDForUpdate), ctx, id)
+}
+
 // GetProvisionerJobTimingsByJobID mocks base method.
 func (m *MockStore) GetProvisionerJobTimingsByJobID(ctx context.Context, jobID uuid.UUID) ([]database.ProvisionerJobTiming, error) {
 	m.ctrl.T.Helper()
@@ -2464,6 +2538,21 @@ func (mr *MockStoreMockRecorder) GetProvisionerJobsCreatedAfter(ctx, createdAt a
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetProvisionerJobsCreatedAfter", reflect.TypeOf((*MockStore)(nil).GetProvisionerJobsCreatedAfter), ctx, createdAt)
 }
 
+// GetProvisionerJobsToBeReaped mocks base method.
+func (m *MockStore) GetProvisionerJobsToBeReaped(ctx context.Context, arg database.GetProvisionerJobsToBeReapedParams) ([]database.ProvisionerJob, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetProvisionerJobsToBeReaped", ctx, arg)
+	ret0, _ := ret[0].([]database.ProvisionerJob)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// GetProvisionerJobsToBeReaped indicates an expected call of GetProvisionerJobsToBeReaped.
+func (mr *MockStoreMockRecorder) GetProvisionerJobsToBeReaped(ctx, arg any) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetProvisionerJobsToBeReaped", reflect.TypeOf((*MockStore)(nil).GetProvisionerJobsToBeReaped), ctx, arg)
+}
+
 // GetProvisionerKeyByHashedSecret mocks base method.
 func (m *MockStore) GetProvisionerKeyByHashedSecret(ctx context.Context, hashedSecret []byte) (database.ProvisionerKey, error) {
 	m.ctrl.T.Helper()
@@ -3619,6 +3708,21 @@ func (mr *MockStoreMockRecorder) GetWorkspaceAgentsByResourceIDs(ctx, ids any) *
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetWorkspaceAgentsByResourceIDs", reflect.TypeOf((*MockStore)(nil).GetWorkspaceAgentsByResourceIDs), ctx, ids)
 }
 
+// GetWorkspaceAgentsByWorkspaceAndBuildNumber mocks base method.
+func (m *MockStore) GetWorkspaceAgentsByWorkspaceAndBuildNumber(ctx context.Context, arg database.GetWorkspaceAgentsByWorkspaceAndBuildNumberParams) ([]database.WorkspaceAgent, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetWorkspaceAgentsByWorkspaceAndBuildNumber", ctx, arg)
+	ret0, _ := ret[0].([]database.WorkspaceAgent)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// GetWorkspaceAgentsByWorkspaceAndBuildNumber indicates an expected call of GetWorkspaceAgentsByWorkspaceAndBuildNumber.
+func (mr *MockStoreMockRecorder) GetWorkspaceAgentsByWorkspaceAndBuildNumber(ctx, arg any) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetWorkspaceAgentsByWorkspaceAndBuildNumber", reflect.TypeOf((*MockStore)(nil).GetWorkspaceAgentsByWorkspaceAndBuildNumber), ctx, arg)
+}
+
 // GetWorkspaceAgentsCreatedAfter mocks base method.
 func (m *MockStore) GetWorkspaceAgentsCreatedAfter(ctx context.Context, createdAt time.Time) ([]database.WorkspaceAgent, error) {
 	m.ctrl.T.Helper()
@@ -3874,6 +3978,21 @@ func (mr *MockStoreMockRecorder) GetWorkspaceByOwnerIDAndName(ctx, arg any) *gom
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetWorkspaceByOwnerIDAndName", reflect.TypeOf((*MockStore)(nil).GetWorkspaceByOwnerIDAndName), ctx, arg)
 }
 
+// GetWorkspaceByResourceID mocks base method.
+func (m *MockStore) GetWorkspaceByResourceID(ctx context.Context, resourceID uuid.UUID) (database.Workspace, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetWorkspaceByResourceID", ctx, resourceID)
+	ret0, _ := ret[0].(database.Workspace)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// GetWorkspaceByResourceID indicates an expected call of GetWorkspaceByResourceID.
+func (mr *MockStoreMockRecorder) GetWorkspaceByResourceID(ctx, resourceID any) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetWorkspaceByResourceID", reflect.TypeOf((*MockStore)(nil).GetWorkspaceByResourceID), ctx, resourceID)
+}
+
 // GetWorkspaceByWorkspaceAppID mocks base method.
 func (m *MockStore) GetWorkspaceByWorkspaceAppID(ctx context.Context, workspaceAppID uuid.UUID) (database.Workspace, error) {
 	m.ctrl.T.Helper()
@@ -4203,6 +4322,36 @@ func (mr *MockStoreMockRecorder) InsertAuditLog(ctx, arg any) *gomock.Call {
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InsertAuditLog", reflect.TypeOf((*MockStore)(nil).InsertAuditLog), ctx, arg)
 }
 
+// InsertChat mocks base method.
+func (m *MockStore) InsertChat(ctx context.Context, arg database.InsertChatParams) (database.Chat, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "InsertChat", ctx, arg)
+	ret0, _ := ret[0].(database.Chat)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// InsertChat indicates an expected call of InsertChat.
+func (mr *MockStoreMockRecorder) InsertChat(ctx, arg any) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InsertChat", reflect.TypeOf((*MockStore)(nil).InsertChat), ctx, arg)
+}
+
+// InsertChatMessages mocks base method.
+func (m *MockStore) InsertChatMessages(ctx context.Context, arg database.InsertChatMessagesParams) ([]database.ChatMessage, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "InsertChatMessages", ctx, arg)
+	ret0, _ := ret[0].([]database.ChatMessage)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// InsertChatMessages indicates an expected call of InsertChatMessages.
+func (mr *MockStoreMockRecorder) InsertChatMessages(ctx, arg any) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InsertChatMessages", reflect.TypeOf((*MockStore)(nil).InsertChatMessages), ctx, arg)
+}
+
 // InsertCryptoKey mocks base method.
 func (m *MockStore) InsertCryptoKey(ctx context.Context, arg database.InsertCryptoKeyParams) (database.CryptoKey, error) {
 	m.ctrl.T.Helper()
@@ -5337,6 +5486,20 @@ func (mr *MockStoreMockRecorder) UpdateAPIKeyByID(ctx, arg any) *gomock.Call {
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateAPIKeyByID", reflect.TypeOf((*MockStore)(nil).UpdateAPIKeyByID), ctx, arg)
 }
 
+// UpdateChatByID mocks base method.
+func (m *MockStore) UpdateChatByID(ctx context.Context, arg database.UpdateChatByIDParams) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "UpdateChatByID", ctx, arg)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// UpdateChatByID indicates an expected call of UpdateChatByID.
+func (mr *MockStoreMockRecorder) UpdateChatByID(ctx, arg any) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateChatByID", reflect.TypeOf((*MockStore)(nil).UpdateChatByID), ctx, arg)
+}
+
 // UpdateCryptoKeyDeletesAt mocks base method.
 func (m *MockStore) UpdateCryptoKeyDeletesAt(ctx context.Context, arg database.UpdateCryptoKeyDeletesAtParams) (database.CryptoKey, error) {
 	m.ctrl.T.Helper()
@@ -5558,6 +5721,20 @@ func (mr *MockStoreMockRecorder) UpdateOrganizationDeletedByID(ctx, arg any) *go
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateOrganizationDeletedByID", reflect.TypeOf((*MockStore)(nil).UpdateOrganizationDeletedByID), ctx, arg)
 }
 
+// UpdatePresetPrebuildStatus mocks base method.
+func (m *MockStore) UpdatePresetPrebuildStatus(ctx context.Context, arg database.UpdatePresetPrebuildStatusParams) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "UpdatePresetPrebuildStatus", ctx, arg)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// UpdatePresetPrebuildStatus indicates an expected call of UpdatePresetPrebuildStatus.
+func (mr *MockStoreMockRecorder) UpdatePresetPrebuildStatus(ctx, arg any) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdatePresetPrebuildStatus", reflect.TypeOf((*MockStore)(nil).UpdatePresetPrebuildStatus), ctx, arg)
+}
+
 // UpdateProvisionerDaemonLastSeenAt mocks base method.
 func (m *MockStore) UpdateProvisionerDaemonLastSeenAt(ctx context.Context, arg database.UpdateProvisionerDaemonLastSeenAtParams) error {
 	m.ctrl.T.Helper()
@@ -5614,6 +5791,20 @@ func (mr *MockStoreMockRecorder) UpdateProvisionerJobWithCompleteByID(ctx, arg a
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateProvisionerJobWithCompleteByID", reflect.TypeOf((*MockStore)(nil).UpdateProvisionerJobWithCompleteByID), ctx, arg)
 }
 
+// UpdateProvisionerJobWithCompleteWithStartedAtByID mocks base method.
+func (m *MockStore) UpdateProvisionerJobWithCompleteWithStartedAtByID(ctx context.Context, arg database.UpdateProvisionerJobWithCompleteWithStartedAtByIDParams) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "UpdateProvisionerJobWithCompleteWithStartedAtByID", ctx, arg)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// UpdateProvisionerJobWithCompleteWithStartedAtByID indicates an expected call of UpdateProvisionerJobWithCompleteWithStartedAtByID.
+func (mr *MockStoreMockRecorder) UpdateProvisionerJobWithCompleteWithStartedAtByID(ctx, arg any) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateProvisionerJobWithCompleteWithStartedAtByID", reflect.TypeOf((*MockStore)(nil).UpdateProvisionerJobWithCompleteWithStartedAtByID), ctx, arg)
+}
+
 // UpdateReplica mocks base method.
 func (m *MockStore) UpdateReplica(ctx context.Context, arg database.UpdateReplicaParams) (database.Replica, error) {
 	m.ctrl.T.Helper()