chore: revert CLI binary publishing for releases.coder.com (#19234)

THIS IS A SECURITY FIX - cherry picks #19214 

upgrade to go 1.24.6 to avoid https://github.com/golang/go/issues/74831
(CVE-2025-47907)

Also points to a new version of our lib/pq fork that worked around the
Go issue, which should restore better performance.

---------

Co-authored-by: Cian Johnston <cian@coder.com>
Co-authored-by: Ethan <39577870+ethanndickson@users.noreply.github.com>

.cursorrules

@@ -0,0 +1,124 @@
+# Cursor Rules
+
+This project is called "Coder" - an application for managing remote development environments.
+
+Coder provides a platform for creating, managing, and using remote development environments (also known as Cloud Development Environments or CDEs). It leverages Terraform to define and provision these environments, which are referred to as "workspaces" within the project. The system is designed to be extensible, secure, and provide developers with a seamless remote development experience.
+
+## Core Architecture
+
+The heart of Coder is a control plane that orchestrates the creation and management of workspaces. This control plane interacts with separate Provisioner processes over gRPC to handle workspace builds. The Provisioners consume workspace definitions and use Terraform to create the actual infrastructure.
+
+The CLI package serves dual purposes - it can be used to launch the control plane itself and also provides client functionality for users to interact with an existing control plane instance. All user-facing frontend code is developed in TypeScript using React and lives in the `site/` directory.
+
+The database layer uses PostgreSQL with SQLC for generating type-safe database code. Database migrations are carefully managed to ensure both forward and backward compatibility through paired `.up.sql` and `.down.sql` files.
+
+## API Design
+
+Coder's API architecture combines REST and gRPC approaches. The REST API is defined in `coderd/coderd.go` and uses Chi for HTTP routing. This provides the primary interface for the frontend and external integrations.
+
+Internal communication with Provisioners occurs over gRPC, with service definitions maintained in `.proto` files. This separation allows for efficient binary communication with the components responsible for infrastructure management while providing a standard REST interface for human-facing applications.
+
+## Network Architecture
+
+Coder implements a secure networking layer based on Tailscale's Wireguard implementation. The `tailnet` package provides connectivity between workspace agents and clients through DERP (Designated Encrypted Relay for Packets) servers when direct connections aren't possible. This creates a secure overlay network allowing access to workspaces regardless of network topology, firewalls, or NAT configurations.
+
+### Tailnet and DERP System
+
+The networking system has three key components:
+
+1. **Tailnet**: An overlay network implemented in the `tailnet` package that provides secure, end-to-end encrypted connections between clients, the Coder server, and workspace agents.
+
+2. **DERP Servers**: These relay traffic when direct connections aren't possible. Coder provides several options:
+   - A built-in DERP server that runs on the Coder control plane
+   - Integration with Tailscale's global DERP infrastructure
+   - Support for custom DERP servers for lower latency or offline deployments
+
+3. **Direct Connections**: When possible, the system establishes peer-to-peer connections between clients and workspaces using STUN for NAT traversal. This requires both endpoints to send UDP traffic on ephemeral ports.
+
+### Workspace Proxies
+
+Workspace proxies (in the Enterprise edition) provide regional relay points for browser-based connections, reducing latency for geo-distributed teams. Key characteristics:
+
+- Deployed as independent servers that authenticate with the Coder control plane
+- Relay connections for SSH, workspace apps, port forwarding, and web terminals
+- Do not make direct database connections
+- Managed through the `coder wsproxy` commands
+- Implemented primarily in the `enterprise/wsproxy/` package
+
+## Agent System
+
+The workspace agent runs within each provisioned workspace and provides core functionality including:
+
+- SSH access to workspaces via the `agentssh` package
+- Port forwarding
+- Terminal connectivity via the `pty` package for pseudo-terminal support
+- Application serving
+- Healthcheck monitoring
+- Resource usage reporting
+
+Agents communicate with the control plane using the tailnet system and authenticate using secure tokens.
+
+## Workspace Applications
+
+Workspace applications (or "apps") provide browser-based access to services running within workspaces. The system supports:
+
+- HTTP(S) and WebSocket connections
+- Path-based or subdomain-based access URLs
+- Health checks to monitor application availability
+- Different sharing levels (owner-only, authenticated users, or public)
+- Custom icons and display settings
+
+The implementation is primarily in the `coderd/workspaceapps/` directory with components for URL generation, proxying connections, and managing application state.
+
+## Implementation Details
+
+The project structure separates frontend and backend concerns. React components and pages are organized in the `site/src/` directory, with Jest used for testing. The backend is primarily written in Go, with a strong emphasis on error handling patterns and test coverage.
+
+Database interactions are carefully managed through migrations in `coderd/database/migrations/` and queries in `coderd/database/queries/`. All new queries require proper database authorization (dbauthz) implementation to ensure that only users with appropriate permissions can access specific resources.
+
+## Authorization System
+
+The database authorization (dbauthz) system enforces fine-grained access control across all database operations. It uses role-based access control (RBAC) to validate user permissions before executing database operations. The `dbauthz` package wraps the database store and performs authorization checks before returning data. All database operations must pass through this layer to ensure security.
+
+## Testing Framework
+
+The codebase has a comprehensive testing approach with several key components:
+
+1. **Parallel Testing**: All tests must use `t.Parallel()` to run concurrently, which improves test suite performance and helps identify race conditions.
+
+2. **coderdtest Package**: This package in `coderd/coderdtest/` provides utilities for creating test instances of the Coder server, setting up test users and workspaces, and mocking external components.
+
+3. **Integration Tests**: Tests often span multiple components to verify system behavior, such as template creation, workspace provisioning, and agent connectivity.
+
+4. **Enterprise Testing**: Enterprise features have dedicated test utilities in the `coderdenttest` package.
+
+## Open Source and Enterprise Components
+
+The repository contains both open source and enterprise components:
+
+- Enterprise code lives primarily in the `enterprise/` directory
+- Enterprise features focus on governance, scalability (high availability), and advanced deployment options like workspace proxies
+- The boundary between open source and enterprise is managed through a licensing system
+- The same core codebase supports both editions, with enterprise features conditionally enabled
+
+## Development Philosophy
+
+Coder emphasizes clear error handling, with specific patterns required:
+
+- Concise error messages that avoid phrases like "failed to"
+- Wrapping errors with `%w` to maintain error chains
+- Using sentinel errors with the "err" prefix (e.g., `errNotFound`)
+
+All tests should run in parallel using `t.Parallel()` to ensure efficient testing and expose potential race conditions. The codebase is rigorously linted with golangci-lint to maintain consistent code quality.
+
+Git contributions follow a standard format with commit messages structured as `type: <message>`, where type is one of `feat`, `fix`, or `chore`.
+
+## Development Workflow
+
+Development can be initiated using `scripts/develop.sh` to start the application after making changes. Database schema updates should be performed through the migration system using `create_migration.sh <name>` to generate migration files, with each `.up.sql` migration paired with a corresponding `.down.sql` that properly reverts all changes.
+
+If the development database gets into a bad state, it can be completely reset by removing the PostgreSQL data directory with `rm -rf .coderv2/postgres`. This will destroy all data in the development database, requiring you to recreate any test users, templates, or workspaces after restarting the application.
+
+Code generation for the database layer uses `coderd/database/generate.sh`, and developers should refer to `sqlc.yaml` for the appropriate style and patterns to follow when creating new queries or tables.
+
+The focus should always be on maintaining security through proper database authorization, clean error handling, and comprehensive test coverage to ensure the platform remains robust and reliable.

.gitattributes

@@ -1,5 +1,7 @@
 # Generated files
 agent/agentcontainers/acmock/acmock.go linguist-generated=true
+agent/agentcontainers/dcspec/dcspec_gen.go linguist-generated=true
+agent/agentcontainers/testdata/devcontainercli/*/*.log linguist-generated=true
 coderd/apidoc/docs.go linguist-generated=true
 docs/reference/api/*.md linguist-generated=true
 docs/reference/cli/*.md linguist-generated=true

.github/.linkspector.yml

@@ -21,5 +21,8 @@ ignorePatterns:
   - pattern: "linux.die.net/man"
   - pattern: "www.gnu.org"
   - pattern: "wiki.ubuntu.com"
+  - pattern: "mutagen.io"
+  - pattern: "docs.github.com"
+  - pattern: "claude.ai"
 aliveStatusCodes:
   - 200

.github/ISSUE_TEMPLATE/1-bug.yaml

@@ -1,6 +1,6 @@
 name: "🐞 Bug"
 description: "File a bug report."
-title: "<title>"
+title: "bug: "
 labels: ["needs-triage"]
 body:
   - type: checkboxes

.github/actions/install-cosign/action.yaml

@@ -0,0 +1,10 @@
+name: "Install cosign"
+description: |
+  Cosign Github Action.
+runs:
+  using: "composite"
+  steps:
+    - name: Install cosign
+      uses: sigstore/cosign-installer@d7d6bc7722e3daa8354c50bcb52f4837da5e9b6a # v3.8.1
+      with:
+        cosign-release: "v2.4.3"

.github/actions/install-syft/action.yaml

@@ -0,0 +1,10 @@
+name: "Install syft"
+description: |
+  Downloads Syft to the Action tool cache and provides a reference.
+runs:
+  using: "composite"
+  steps:
+    - name: Install syft
+      uses: anchore/sbom-action/download-syft@f325610c9f50a54015d37c8d16cb3b0e2c8f4de0 # v0.18.0
+      with:
+        syft-version: "v1.20.0"

.github/actions/setup-go-paths/action.yml

@@ -0,0 +1,57 @@
+name: "Setup Go Paths"
+description: Overrides Go paths like GOCACHE and GOMODCACHE to use temporary directories.
+outputs:
+  gocache:
+    description: "Value of GOCACHE"
+    value: ${{ steps.paths.outputs.gocache }}
+  gomodcache:
+    description: "Value of GOMODCACHE"
+    value: ${{ steps.paths.outputs.gomodcache }}
+  gopath:
+    description: "Value of GOPATH"
+    value: ${{ steps.paths.outputs.gopath }}
+  gotmp:
+    description: "Value of GOTMPDIR"
+    value: ${{ steps.paths.outputs.gotmp }}
+  cached-dirs:
+    description: "Go directories that should be cached between CI runs"
+    value: ${{ steps.paths.outputs.cached-dirs }}
+runs:
+  using: "composite"
+  steps:
+    - name: Override Go paths
+      id: paths
+      uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7
+      with:
+        script: |
+          const path = require('path');
+
+          // RUNNER_TEMP should be backed by a RAM disk on Windows if
+          // coder/setup-ramdisk-action was used
+          const runnerTemp = process.env.RUNNER_TEMP;
+          const gocacheDir = path.join(runnerTemp, 'go-cache');
+          const gomodcacheDir = path.join(runnerTemp, 'go-mod-cache');
+          const gopathDir = path.join(runnerTemp, 'go-path');
+          const gotmpDir = path.join(runnerTemp, 'go-tmp');
+
+          core.exportVariable('GOCACHE', gocacheDir);
+          core.exportVariable('GOMODCACHE', gomodcacheDir);
+          core.exportVariable('GOPATH', gopathDir);
+          core.exportVariable('GOTMPDIR', gotmpDir);
+
+          core.setOutput('gocache', gocacheDir);
+          core.setOutput('gomodcache', gomodcacheDir);
+          core.setOutput('gopath', gopathDir);
+          core.setOutput('gotmp', gotmpDir);
+
+          const cachedDirs = `${gocacheDir}\n${gomodcacheDir}`;
+          core.setOutput('cached-dirs', cachedDirs);
+
+    - name: Create directories
+      shell: bash
+      run: |
+        set -e
+        mkdir -p "$GOCACHE"
+        mkdir -p "$GOMODCACHE"
+        mkdir -p "$GOPATH"
+        mkdir -p "$GOTMPDIR"

.github/actions/setup-go-tools/action.yaml

@@ -0,0 +1,14 @@
+name: "Setup Go tools"
+description: |
+  Set up tools for `make gen`, `offlinedocs` and Schmoder CI.
+runs:
+  using: "composite"
+  steps:
+    - name: go install tools
+      shell: bash
+      run: |
+          go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.30
+          go install storj.io/drpc/cmd/protoc-gen-go-drpc@v0.0.34
+          go install golang.org/x/tools/cmd/goimports@v0.31.0
+          go install github.com/mikefarah/yq/v4@v4.44.3
+          go install go.uber.org/mock/mockgen@v0.5.0

.github/actions/setup-go/action.yaml

@@ -4,18 +4,29 @@ description: |
 inputs:
   version:
     description: "The Go version to use."
-    default: "1.22.8"
+    default: "1.24.6"
+  use-preinstalled-go:
+    description: "Whether to use preinstalled Go."
+    default: "false"
+  use-cache:
+    description: "Whether to use the cache."
+    default: "true"
 runs:
   using: "composite"
   steps:
     - name: Setup Go
       uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
       with:
-        go-version: ${{ inputs.version }}
+        go-version: ${{ inputs.use-preinstalled-go == 'false' && inputs.version || '' }}
+        cache: ${{ inputs.use-cache }}
 
     - name: Install gotestsum
       shell: bash
-      run: go install gotest.tools/gotestsum@latest
+      run: go install gotest.tools/gotestsum@0d9599e513d70e5792bb9334869f82f6e8b53d4d # main as of 2025-05-15
+
+    - name: Install mtimehash
+      shell: bash
+      run: go install github.com/slsyy/mtimehash/cmd/mtimehash@a6b5da4ed2c4a40e7b805534b004e9fde7b53ce0 # v1.0.0
 
     # It isn't necessary that we ever do this, but it helps
     # separate the "setup" from the "run" times.

.github/actions/setup-imdisk/action.yaml

@@ -1,27 +0,0 @@
-name: "Setup ImDisk"
-if: runner.os == 'Windows'
-description: |
-  Sets up the ImDisk toolkit for Windows and creates a RAM disk on drive R:.
-runs:
-  using: "composite"
-  steps:
-    - name: Download ImDisk
-      if: runner.os == 'Windows'
-      shell: bash
-      run: |
-        mkdir imdisk
-        cd imdisk
-        curl -L -o files.cab https://github.com/coder/imdisk-artifacts/raw/92a17839ebc0ee3e69be019f66b3e9b5d2de4482/files.cab
-        curl -L -o install.bat https://github.com/coder/imdisk-artifacts/raw/92a17839ebc0ee3e69be019f66b3e9b5d2de4482/install.bat
-        cd ..
-
-    - name: Install ImDisk
-      shell: cmd
-      run: |
-        cd imdisk
-        install.bat /silent
-
-    - name: Create RAM Disk
-      shell: cmd
-      run: |
-        imdisk -a -s 4096M -m R: -p "/fs:ntfs /q /y"

.github/actions/setup-tf/action.yaml

@@ -7,5 +7,5 @@ runs:
     - name: Install Terraform
       uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd # v3.1.2
       with:
-        terraform_version: 1.10.5
+        terraform_version: 1.11.4
         terraform_wrapper: false

.github/actions/test-cache/download/action.yml

@@ -0,0 +1,50 @@
+name: "Download Test Cache"
+description: |
+  Downloads the test cache and outputs today's cache key.
+  A PR job can use a cache if it was created by its base branch, its current
+  branch, or the default branch.
+  https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/caching-dependencies-to-speed-up-workflows#restrictions-for-accessing-a-cache
+outputs:
+  cache-key:
+    description: "Today's cache key"
+    value: ${{ steps.vars.outputs.cache-key }}
+inputs:
+  key-prefix:
+    description: "Prefix for the cache key"
+    required: true
+  cache-path:
+    description: "Path to the cache directory"
+    required: true
+    # This path is defined in testutil/cache.go
+    default: "~/.cache/coderv2-test"
+runs:
+  using: "composite"
+  steps:
+    - name: Get date values and cache key
+      id: vars
+      shell: bash
+      run: |
+        export YEAR_MONTH=$(date +'%Y-%m')
+        export PREV_YEAR_MONTH=$(date -d 'last month' +'%Y-%m')
+        export DAY=$(date +'%d')
+        echo "year-month=$YEAR_MONTH" >> $GITHUB_OUTPUT
+        echo "prev-year-month=$PREV_YEAR_MONTH" >> $GITHUB_OUTPUT
+        echo "cache-key=${{ inputs.key-prefix }}-${YEAR_MONTH}-${DAY}" >> $GITHUB_OUTPUT
+
+    # TODO: As a cost optimization, we could remove caches that are older than
+    # a day or two. By default, depot keeps caches for 14 days, which isn't
+    # necessary for the test cache.
+    # https://depot.dev/docs/github-actions/overview#cache-retention-policy
+    - name: Download test cache
+      uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+      with:
+        path: ${{ inputs.cache-path }}
+        key: ${{ steps.vars.outputs.cache-key }}
+        # > If there are multiple partial matches for a restore key, the action returns the most recently created cache.
+        # https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/caching-dependencies-to-speed-up-workflows#matching-a-cache-key
+        # The second restore key allows non-main branches to use the cache from the previous month.
+        # This prevents PRs from rebuilding the cache on the first day of the month.
+        # It also makes sure that once a month, the cache is fully reset.
+        restore-keys: |
+          ${{ inputs.key-prefix }}-${{ steps.vars.outputs.year-month }}-
+          ${{ github.ref != 'refs/heads/main' && format('{0}-{1}-', inputs.key-prefix, steps.vars.outputs.prev-year-month) || '' }}

.github/actions/test-cache/upload/action.yml

@@ -0,0 +1,20 @@
+name: "Upload Test Cache"
+description: Uploads the test cache. Only works on the main branch.
+inputs:
+  cache-key:
+    description: "Cache key"
+    required: true
+  cache-path:
+    description: "Path to the cache directory"
+    required: true
+    # This path is defined in testutil/cache.go
+    default: "~/.cache/coderv2-test"
+runs:
+  using: "composite"
+  steps:
+    - name: Upload test cache
+      if: ${{ github.ref == 'refs/heads/main' }}
+      uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+      with:
+        path: ${{ inputs.cache-path }}
+        key: ${{ inputs.cache-key }}

.github/actions/upload-datadog/action.yaml

@@ -10,6 +10,8 @@ runs:
   steps:
     - shell: bash
       run: |
+        set -e
+
         owner=${{ github.repository_owner	 }}
         echo "owner: $owner"
         if [[  $owner != "coder" ]]; then
@@ -21,8 +23,45 @@ runs:
           echo "No API key provided, skipping..."
           exit 0
         fi
-        npm install -g @datadog/datadog-ci@2.21.0
-        datadog-ci junit upload --service coder ./gotests.xml \
+
+        BINARY_VERSION="v2.48.0"
+        BINARY_HASH_WINDOWS="b7bebb8212403fddb1563bae84ce5e69a70dac11e35eb07a00c9ef7ac9ed65ea"
+        BINARY_HASH_MACOS="e87c808638fddb21a87a5c4584b68ba802965eb0a593d43959c81f67246bd9eb"
+        BINARY_HASH_LINUX="5e700c465728fff8313e77c2d5ba1ce19a736168735137e1ddc7c6346ed48208"
+
+        TMP_DIR=$(mktemp -d)
+
+        if [[ "${{ runner.os }}" == "Windows" ]]; then
+          BINARY_PATH="${TMP_DIR}/datadog-ci.exe"
+          BINARY_URL="https://github.com/DataDog/datadog-ci/releases/download/${BINARY_VERSION}/datadog-ci_win-x64"
+        elif [[ "${{ runner.os }}" == "macOS" ]]; then
+          BINARY_PATH="${TMP_DIR}/datadog-ci"
+          BINARY_URL="https://github.com/DataDog/datadog-ci/releases/download/${BINARY_VERSION}/datadog-ci_darwin-arm64"
+        elif [[ "${{ runner.os }}" == "Linux" ]]; then
+          BINARY_PATH="${TMP_DIR}/datadog-ci"
+          BINARY_URL="https://github.com/DataDog/datadog-ci/releases/download/${BINARY_VERSION}/datadog-ci_linux-x64"
+        else
+          echo "Unsupported OS: ${{ runner.os }}"
+          exit 1
+        fi
+
+        echo "Downloading DataDog CI binary version ${BINARY_VERSION} for ${{ runner.os }}..."
+        curl -sSL "$BINARY_URL" -o "$BINARY_PATH"
+
+        if [[ "${{ runner.os }}" == "Windows" ]]; then
+          echo "$BINARY_HASH_WINDOWS  $BINARY_PATH" | sha256sum --check
+        elif [[ "${{ runner.os }}" == "macOS" ]]; then
+          echo "$BINARY_HASH_MACOS  $BINARY_PATH" | shasum -a 256 --check
+        elif [[ "${{ runner.os }}" == "Linux" ]]; then
+          echo "$BINARY_HASH_LINUX  $BINARY_PATH" | sha256sum --check
+        fi
+
+        # Make binary executable (not needed for Windows)
+        if [[ "${{ runner.os }}" != "Windows" ]]; then
+          chmod +x "$BINARY_PATH"
+        fi
+
+        "$BINARY_PATH" junit upload --service coder ./gotests.xml \
           --tags os:${{runner.os}} --tags runner_name:${{runner.name}}
       env:
         DATADOG_API_KEY: ${{ inputs.api-key }}

.github/dependabot.yaml

@@ -37,7 +37,8 @@ updates:
   # Update our Dockerfile.
   - package-ecosystem: "docker"
     directories:
-      - "/dogfood/contents"
+      - "/dogfood/coder"
+      - "/dogfood/coder-envbuilder"
       - "/scripts"
       - "/examples/templates/docker/build"
       - "/examples/parameters/build"
@@ -103,3 +104,21 @@ updates:
         update-types:
           - version-update:semver-major
     open-pull-requests-limit: 15
+
+  - package-ecosystem: "terraform"
+    directories:
+      - "dogfood/*/"
+      - "examples/templates/*/"
+    schedule:
+      interval: "weekly"
+    commit-message:
+      prefix: "chore"
+    groups:
+      coder:
+        patterns:
+          - "registry.coder.com/coder/*/coder"
+    labels: []
+    ignore:
+      - dependency-name: "*"
+        update-types:
+          - version-update:semver-major

.github/workflows/ci.yaml

@@ -24,7 +24,7 @@ jobs:
       docs-only: ${{ steps.filter.outputs.docs_count == steps.filter.outputs.all_count }}
       docs: ${{ steps.filter.outputs.docs }}
       go: ${{ steps.filter.outputs.go }}
-      ts: ${{ steps.filter.outputs.ts }}
+      site: ${{ steps.filter.outputs.site }}
       k8s: ${{ steps.filter.outputs.k8s }}
       ci: ${{ steps.filter.outputs.ci }}
       db: ${{ steps.filter.outputs.db }}
@@ -34,7 +34,7 @@ jobs:
       tailnet-integration: ${{ steps.filter.outputs.tailnet-integration }}
     steps:
       - name: Harden Runner
-        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
         with:
           egress-policy: audit
 
@@ -92,9 +92,8 @@ jobs:
             gomod:
               - "go.mod"
               - "go.sum"
-            ts:
+            site:
               - "site/**"
-              - "Makefile"
             k8s:
               - "helm/**"
               - "scripts/Dockerfile"
@@ -155,7 +154,7 @@ jobs:
     runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-8' || 'ubuntu-latest' }}
     steps:
       - name: Harden Runner
-        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
         with:
           egress-policy: audit
 
@@ -172,13 +171,13 @@ jobs:
 
       - name: Get golangci-lint cache dir
         run: |
-          linter_ver=$(egrep -o 'GOLANGCI_LINT_VERSION=\S+' dogfood/contents/Dockerfile | cut -d '=' -f 2)
+          linter_ver=$(egrep -o 'GOLANGCI_LINT_VERSION=\S+' dogfood/coder/Dockerfile | cut -d '=' -f 2)
           go install github.com/golangci/golangci-lint/cmd/golangci-lint@v$linter_ver
           dir=$(golangci-lint cache status | awk '/Dir/ { print $2 }')
           echo "LINT_CACHE_DIR=$dir" >> $GITHUB_ENV
 
       - name: golangci-lint cache
-        uses: actions/cache@0c907a75c2c80ebcb7f088228285e798b750cf8f # v4.2.1
+        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
         with:
           path: |
             ${{ env.LINT_CACHE_DIR }}
@@ -188,7 +187,7 @@ jobs:
 
       # Check for any typos
       - name: Check for typos
-        uses: crate-ci/typos@212923e4ff05b7fc2294a204405eec047b807138 # v1.29.9
+        uses: crate-ci/typos@0f0ccba9ed1df83948f0c15026e4f5ccfce46109 # v1.32.0
         with:
           config: .github/workflows/typos.toml
 
@@ -224,10 +223,10 @@ jobs:
   gen:
     timeout-minutes: 8
     runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-8' || 'ubuntu-latest' }}
-    if: always()
+    if: ${{ !cancelled() }}
     steps:
       - name: Harden Runner
-        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
         with:
           egress-policy: audit
 
@@ -249,12 +248,7 @@ jobs:
         uses: ./.github/actions/setup-tf
 
       - name: go install tools
-        run: |
-          go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.30
-          go install storj.io/drpc/cmd/protoc-gen-go-drpc@v0.0.34
-          go install golang.org/x/tools/cmd/goimports@latest
-          go install github.com/mikefarah/yq/v4@v4.44.3
-          go install go.uber.org/mock/mockgen@v0.5.0
+        uses: ./.github/actions/setup-go-tools
 
       - name: Install Protoc
         run: |
@@ -267,18 +261,15 @@ jobs:
           popd
 
       - name: make gen
-        # no `-j` flag as `make` fails with:
-        # coderd/rbac/object_gen.go:1:1: syntax error: package statement must be first
-        run: "make --output-sync -B gen"
-
-      - name: make update-golden-files
         run: |
+          # Remove golden files to detect discrepancy in generated files.
           make clean/golden-files
           # Notifications require DB, we could start a DB instance here but
           # let's just restore for now.
           git checkout -- coderd/notifications/testdata/rendered-templates
-          # As above, skip `-j` flag.
-          make --output-sync -B update-golden-files
+          # no `-j` flag as `make` fails with:
+          # coderd/rbac/object_gen.go:1:1: syntax error: package statement must be first
+          make --output-sync -B gen
 
       - name: Check for unstaged files
         run: ./scripts/check_unstaged.sh
@@ -290,7 +281,7 @@ jobs:
     timeout-minutes: 7
     steps:
       - name: Harden Runner
-        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
         with:
           egress-policy: audit
 
@@ -302,6 +293,9 @@ jobs:
       - name: Setup Node
         uses: ./.github/actions/setup-node
 
+      - name: Check Go version
+        run: IGNORE_NIX=true ./scripts/check_go_versions.sh
+
       # Use default Go version
       - name: Setup Go
         uses: ./.github/actions/setup-go
@@ -318,7 +312,7 @@ jobs:
         run: ./scripts/check_unstaged.sh
 
   test-go:
-    runs-on: ${{ matrix.os == 'ubuntu-latest' && github.repository_owner == 'coder' && 'depot-ubuntu-22.04-4' || matrix.os == 'macos-latest' && github.repository_owner == 'coder' && 'depot-macos-latest' || matrix.os == 'windows-2022' && github.repository_owner == 'coder' && 'windows-latest-16-cores' || matrix.os }}
+    runs-on: ${{ matrix.os == 'ubuntu-latest' && github.repository_owner == 'coder' && 'depot-ubuntu-22.04-4' || matrix.os == 'macos-latest' && github.repository_owner == 'coder' && 'depot-macos-latest' || matrix.os == 'windows-2022' && github.repository_owner == 'coder' && 'depot-windows-2022-16' || matrix.os }}
     needs: changes
     if: needs.changes.outputs.go == 'true' || needs.changes.outputs.ci == 'true' || github.ref == 'refs/heads/main'
     timeout-minutes: 20
@@ -331,21 +325,43 @@ jobs:
           - windows-2022
     steps:
       - name: Harden Runner
-        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        # Harden Runner is only supported on Ubuntu runners.
+        if: runner.os == 'Linux'
+        uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
         with:
           egress-policy: audit
 
+      # Set up RAM disks to speed up the rest of the job. This action is in
+      # a separate repository to allow its use before actions/checkout.
+      - name: Setup RAM Disks
+        if: runner.os == 'Windows'
+        uses: coder/setup-ramdisk-action@e1100847ab2d7bcd9d14bcda8f2d1b0f07b36f1b
+
       - name: Checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           fetch-depth: 1
 
+      - name: Setup Go Paths
+        uses: ./.github/actions/setup-go-paths
+
       - name: Setup Go
         uses: ./.github/actions/setup-go
+        with:
+          # Runners have Go baked-in and Go will automatically
+          # download the toolchain configured in go.mod, so we don't
+          # need to reinstall it. It's faster on Windows runners.
+          use-preinstalled-go: ${{ runner.os == 'Windows' }}
 
       - name: Setup Terraform
         uses: ./.github/actions/setup-tf
 
+      - name: Download Test Cache
+        id: download-cache
+        uses: ./.github/actions/test-cache/download
+        with:
+          key-prefix: test-go-${{ runner.os }}-${{ runner.arch }}
+
       - name: Test with Mock Database
         id: test
         shell: bash
@@ -367,62 +383,13 @@ jobs:
             touch ~/.bash_profile && echo "export BASH_SILENCE_DEPRECATION_WARNING=1" >> ~/.bash_profile
           fi
           export TS_DEBUG_DISCO=true
-          gotestsum --junitfile="gotests.xml" --jsonfile="gotests.json" \
-            --packages="./..." -- $PARALLEL_FLAG -short -failfast
+          gotestsum --junitfile="gotests.xml" --jsonfile="gotests.json" --rerun-fails=2 \
+            --packages="./..." -- $PARALLEL_FLAG -short
 
-      - name: Upload test stats to Datadog
-        timeout-minutes: 1
-        continue-on-error: true
-        uses: ./.github/actions/upload-datadog
-        if: success() || failure()
+      - name: Upload Test Cache
+        uses: ./.github/actions/test-cache/upload
         with:
-          api-key: ${{ secrets.DATADOG_API_KEY }}
-
-  # We don't run the full test-suite for Windows & MacOS, so we just run the CLI tests on every PR.
-  # We run the test suite in test-go-pg, including CLI.
-  test-cli:
-    runs-on: ${{ matrix.os == 'macos-latest' && github.repository_owner == 'coder' && 'depot-macos-latest' || matrix.os == 'windows-2022' && github.repository_owner == 'coder' && 'windows-latest-16-cores' || matrix.os }}
-    needs: changes
-    if: needs.changes.outputs.go == 'true' || needs.changes.outputs.ci == 'true' || github.ref == 'refs/heads/main'
-    strategy:
-      matrix:
-        os:
-          - macos-latest
-          - windows-2022
-    steps:
-      - name: Harden Runner
-        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
-        with:
-          egress-policy: audit
-
-      - name: Checkout
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-        with:
-          fetch-depth: 1
-
-      - name: Setup Go
-        uses: ./.github/actions/setup-go
-
-      - name: Setup Terraform
-        uses: ./.github/actions/setup-tf
-
-      # Sets up the ImDisk toolkit for Windows and creates a RAM disk on drive R:.
-      - name: Setup ImDisk
-        if: runner.os == 'Windows'
-        uses: ./.github/actions/setup-imdisk
-
-      - name: Test CLI
-        env:
-          TS_DEBUG_DISCO: "true"
-          LC_CTYPE: "en_US.UTF-8"
-          LC_ALL: "en_US.UTF-8"
-        shell: bash
-        run: |
-          # By default Go will use the number of logical CPUs, which
-          # is a fine default.
-          PARALLEL_FLAG=""
-
-          make test-cli
+          cache-key: ${{ steps.download-cache.outputs.cache-key }}
 
       - name: Upload test stats to Datadog
         timeout-minutes: 1
@@ -433,7 +400,9 @@ jobs:
           api-key: ${{ secrets.DATADOG_API_KEY }}
 
   test-go-pg:
-    runs-on: ${{ matrix.os == 'ubuntu-latest' && github.repository_owner == 'coder' && 'depot-ubuntu-22.04-4' || matrix.os }}
+    # make sure to adjust NUM_PARALLEL_PACKAGES and NUM_PARALLEL_TESTS below
+    # when changing runner sizes
+    runs-on: ${{ matrix.os == 'ubuntu-latest' && github.repository_owner == 'coder' && 'depot-ubuntu-22.04-8' || matrix.os && matrix.os == 'macos-latest' && github.repository_owner == 'coder' && 'depot-macos-latest' || matrix.os == 'windows-2022' && github.repository_owner == 'coder' && 'depot-windows-2022-16' || matrix.os }}
     needs: changes
     if: needs.changes.outputs.go == 'true' || needs.changes.outputs.ci == 'true' || github.ref == 'refs/heads/main'
     # This timeout must be greater than the timeout set by `go test` in
@@ -445,27 +414,78 @@ jobs:
       matrix:
         os:
           - ubuntu-latest
+          - macos-latest
+          - windows-2022
     steps:
       - name: Harden Runner
-        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
         with:
           egress-policy: audit
 
+      # macOS indexes all new files in the background. Our Postgres tests
+      # create and destroy thousands of databases on disk, and Spotlight
+      # tries to index all of them, seriously slowing down the tests.
+      - name: Disable Spotlight Indexing
+        if: runner.os == 'macOS'
+        run: |
+          enabled=$(sudo mdutil -a -s | grep "Indexing enabled" | wc -l)
+          if [ $enabled -eq 0 ]; then
+            echo "Spotlight indexing is already disabled"
+            exit 0
+          fi
+          sudo mdutil -a -i off
+          sudo mdutil -X /
+          sudo launchctl bootout system /System/Library/LaunchDaemons/com.apple.metadata.mds.plist
+
+      # Set up RAM disks to speed up the rest of the job. This action is in
+      # a separate repository to allow its use before actions/checkout.
+      - name: Setup RAM Disks
+        if: runner.os == 'Windows'
+        uses: coder/setup-ramdisk-action@e1100847ab2d7bcd9d14bcda8f2d1b0f07b36f1b
+
       - name: Checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           fetch-depth: 1
 
+      - name: Setup Go Paths
+        id: go-paths
+        uses: ./.github/actions/setup-go-paths
+
+      - name: Download Go Build Cache
+        id: download-go-build-cache
+        uses: ./.github/actions/test-cache/download
+        with:
+          key-prefix: test-go-build-${{ runner.os }}-${{ runner.arch }}
+          cache-path: ${{ steps.go-paths.outputs.cached-dirs }}
+
       - name: Setup Go
         uses: ./.github/actions/setup-go
+        with:
+          # Runners have Go baked-in and Go will automatically
+          # download the toolchain configured in go.mod, so we don't
+          # need to reinstall it. It's faster on Windows runners.
+          use-preinstalled-go: ${{ runner.os == 'Windows' }}
+          # Cache is already downloaded above
+          use-cache: false
 
       - name: Setup Terraform
         uses: ./.github/actions/setup-tf
 
-      # Sets up the ImDisk toolkit for Windows and creates a RAM disk on drive R:.
-      - name: Setup ImDisk
-        if: runner.os == 'Windows'
-        uses: ./.github/actions/setup-imdisk
+      - name: Download Test Cache
+        id: download-cache
+        uses: ./.github/actions/test-cache/download
+        with:
+          key-prefix: test-go-pg-${{ runner.os }}-${{ runner.arch }}
+
+      - name: Normalize File and Directory Timestamps
+        shell: bash
+        # Normalize file modification timestamps so that go test can use the
+        # cache from the previous CI run. See https://github.com/golang/go/issues/58571
+        # for more details.
+        run: |
+          find . -type f ! -path ./.git/\*\* | mtimehash
+          find . -type d ! -path ./.git/\*\* -exec touch -t 200601010000 {} +
 
       - name: Test with PostgreSQL Database
         env:
@@ -475,11 +495,86 @@ jobs:
           LC_ALL: "en_US.UTF-8"
         shell: bash
         run: |
-          # By default Go will use the number of logical CPUs, which
-          # is a fine default.
-          PARALLEL_FLAG=""
+          set -o errexit
+          set -o pipefail
 
-          make test-postgres
+          if [ "${{ runner.os }}" == "Windows" ]; then
+            # Create a temp dir on the R: ramdisk drive for Windows. The default
+            # C: drive is extremely slow: https://github.com/actions/runner-images/issues/8755
+            mkdir -p "R:/temp/embedded-pg"
+            go run scripts/embedded-pg/main.go -path "R:/temp/embedded-pg"
+          elif [ "${{ runner.os }}" == "macOS" ]; then
+            # Postgres runs faster on a ramdisk on macOS too
+            mkdir -p /tmp/tmpfs
+            sudo mount_tmpfs -o noowners -s 8g /tmp/tmpfs
+            go run scripts/embedded-pg/main.go -path /tmp/tmpfs/embedded-pg
+          elif [ "${{ runner.os }}" == "Linux" ]; then
+            make test-postgres-docker
+          fi
+
+          # if macOS, install google-chrome for scaletests
+          # As another concern, should we really have this kind of external dependency
+          # requirement on standard CI?
+          if [ "${{ matrix.os }}" == "macos-latest" ]; then
+            brew install google-chrome
+          fi
+
+          # macOS will output "The default interactive shell is now zsh"
+          # intermittently in CI...
+          if [ "${{ matrix.os }}" == "macos-latest" ]; then
+            touch ~/.bash_profile && echo "export BASH_SILENCE_DEPRECATION_WARNING=1" >> ~/.bash_profile
+          fi
+
+          if [ "${{ runner.os }}" == "Windows" ]; then
+            # Our Windows runners have 16 cores.
+            # On Windows Postgres chokes up when we have 16x16=256 tests
+            # running in parallel, and dbtestutil.NewDB starts to take more than
+            # 10s to complete sometimes causing test timeouts. With 16x8=128 tests
+            # Postgres tends not to choke.
+            NUM_PARALLEL_PACKAGES=8
+            NUM_PARALLEL_TESTS=16
+          elif [ "${{ runner.os }}" == "macOS" ]; then
+            # Our macOS runners have 8 cores. We set NUM_PARALLEL_TESTS to 16
+            # because the tests complete faster and Postgres doesn't choke. It seems
+            # that macOS's tmpfs is faster than the one on Windows.
+            NUM_PARALLEL_PACKAGES=8
+            NUM_PARALLEL_TESTS=16
+          elif [ "${{ runner.os }}" == "Linux" ]; then
+            # Our Linux runners have 8 cores.
+            NUM_PARALLEL_PACKAGES=8
+            NUM_PARALLEL_TESTS=8
+          fi
+
+          # by default, run tests with cache
+          TESTCOUNT=""
+          if [ "${{ github.ref }}" == "refs/heads/main" ]; then
+            # on main, run tests without cache
+            TESTCOUNT="-count=1"
+          fi
+
+          mkdir -p "$RUNNER_TEMP/sym"
+          source scripts/normalize_path.sh
+          # terraform gets installed in a random directory, so we need to normalize
+          # the path to the terraform binary or a bunch of cached tests will be
+          # invalidated. See scripts/normalize_path.sh for more details.
+          normalize_path_with_symlinks "$RUNNER_TEMP/sym" "$(dirname $(which terraform))"
+
+          # We rerun failing tests to counteract flakiness coming from Postgres
+          # choking on macOS and Windows sometimes.
+          DB=ci gotestsum --rerun-fails=2 --rerun-fails-max-failures=50 \
+            --format standard-quiet --packages "./..." \
+            -- -timeout=20m -v -p $NUM_PARALLEL_PACKAGES -parallel=$NUM_PARALLEL_TESTS $TESTCOUNT
+
+      - name: Upload Go Build Cache
+        uses: ./.github/actions/test-cache/upload
+        with:
+          cache-key: ${{ steps.download-go-build-cache.outputs.cache-key }}
+          cache-path: ${{ steps.go-paths.outputs.cached-dirs }}
+
+      - name: Upload Test Cache
+        uses: ./.github/actions/test-cache/upload
+        with:
+          cache-key: ${{ steps.download-cache.outputs.cache-key }}
 
       - name: Upload test stats to Datadog
         timeout-minutes: 1
@@ -504,7 +599,7 @@ jobs:
     timeout-minutes: 25
     steps:
       - name: Harden Runner
-        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
         with:
           egress-policy: audit
 
@@ -519,13 +614,25 @@ jobs:
       - name: Setup Terraform
         uses: ./.github/actions/setup-tf
 
+      - name: Download Test Cache
+        id: download-cache
+        uses: ./.github/actions/test-cache/download
+        with:
+          key-prefix: test-go-pg-16-${{ runner.os }}-${{ runner.arch }}
+
       - name: Test with PostgreSQL Database
         env:
           POSTGRES_VERSION: "16"
           TS_DEBUG_DISCO: "true"
+          TEST_RETRIES: 2
         run: |
           make test-postgres
 
+      - name: Upload Test Cache
+        uses: ./.github/actions/test-cache/upload
+        with:
+          cache-key: ${{ steps.download-cache.outputs.cache-key }}
+
       - name: Upload test stats to Datadog
         timeout-minutes: 1
         continue-on-error: true
@@ -541,7 +648,7 @@ jobs:
     timeout-minutes: 25
     steps:
       - name: Harden Runner
-        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
         with:
           egress-policy: audit
 
@@ -556,13 +663,24 @@ jobs:
       - name: Setup Terraform
         uses: ./.github/actions/setup-tf
 
+      - name: Download Test Cache
+        id: download-cache
+        uses: ./.github/actions/test-cache/download
+        with:
+          key-prefix: test-go-race-${{ runner.os }}-${{ runner.arch }}
+
       # We run race tests with reduced parallelism because they use more CPU and we were finding
       # instances where tests appear to hang for multiple seconds, resulting in flaky tests when
       # short timeouts are used.
       # c.f. discussion on https://github.com/coder/coder/pull/15106
       - name: Run Tests
         run: |
-          gotestsum --junitfile="gotests.xml" -- -race -parallel 4 -p 4 ./...
+          gotestsum --junitfile="gotests.xml" --packages="./..." --rerun-fails=2 --rerun-fails-abort-on-data-race -- -race -parallel 4 -p 4
+
+      - name: Upload Test Cache
+        uses: ./.github/actions/test-cache/upload
+        with:
+          cache-key: ${{ steps.download-cache.outputs.cache-key }}
 
       - name: Upload test stats to Datadog
         timeout-minutes: 1
@@ -579,7 +697,7 @@ jobs:
     timeout-minutes: 25
     steps:
       - name: Harden Runner
-        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
         with:
           egress-policy: audit
 
@@ -594,6 +712,12 @@ jobs:
       - name: Setup Terraform
         uses: ./.github/actions/setup-tf
 
+      - name: Download Test Cache
+        id: download-cache
+        uses: ./.github/actions/test-cache/download
+        with:
+          key-prefix: test-go-race-pg-${{ runner.os }}-${{ runner.arch }}
+
       # We run race tests with reduced parallelism because they use more CPU and we were finding
       # instances where tests appear to hang for multiple seconds, resulting in flaky tests when
       # short timeouts are used.
@@ -603,7 +727,12 @@ jobs:
           POSTGRES_VERSION: "16"
         run: |
           make test-postgres-docker
-          DB=ci gotestsum --junitfile="gotests.xml" -- -race -parallel 4 -p 4 ./...
+          DB=ci gotestsum --junitfile="gotests.xml" --packages="./..." --rerun-fails=2 --rerun-fails-abort-on-data-race -- -race -parallel 4 -p 4
+
+      - name: Upload Test Cache
+        uses: ./.github/actions/test-cache/upload
+        with:
+          cache-key: ${{ steps.download-cache.outputs.cache-key }}
 
       - name: Upload test stats to Datadog
         timeout-minutes: 1
@@ -627,7 +756,7 @@ jobs:
     timeout-minutes: 20
     steps:
       - name: Harden Runner
-        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
         with:
           egress-policy: audit
 
@@ -649,11 +778,11 @@ jobs:
   test-js:
     runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-8' || 'ubuntu-latest' }}
     needs: changes
-    if: needs.changes.outputs.ts == 'true' || needs.changes.outputs.ci == 'true' || github.ref == 'refs/heads/main'
+    if: needs.changes.outputs.site == 'true' || needs.changes.outputs.ci == 'true' || github.ref == 'refs/heads/main'
     timeout-minutes: 20
     steps:
       - name: Harden Runner
-        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
         with:
           egress-policy: audit
 
@@ -677,15 +806,15 @@ jobs:
         variant:
           - premium: false
             name: test-e2e
-          - premium: true
-            name: test-e2e-premium
+          #- premium: true
+          #  name: test-e2e-premium
     # Skip test-e2e on forks as they don't have access to CI secrets
-    if: (needs.changes.outputs.go == 'true' || needs.changes.outputs.ts == 'true' || needs.changes.outputs.ci == 'true' || github.ref == 'refs/heads/main') && !(github.event.pull_request.head.repo.fork)
+    if: (needs.changes.outputs.go == 'true' || needs.changes.outputs.site == 'true' || needs.changes.outputs.ci == 'true' || github.ref == 'refs/heads/main') && !(github.event.pull_request.head.repo.fork)
     timeout-minutes: 20
     name: ${{ matrix.variant.name }}
     steps:
       - name: Harden Runner
-        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
         with:
           egress-policy: audit
 
@@ -720,6 +849,7 @@ jobs:
         if: ${{ !matrix.variant.premium }}
         env:
           DEBUG: pw:api
+          CODER_E2E_TEST_RETRIES: 2
         working-directory: site
 
       # Run all of the tests with a premium license
@@ -729,11 +859,12 @@ jobs:
           DEBUG: pw:api
           CODER_E2E_LICENSE: ${{ secrets.CODER_E2E_LICENSE }}
           CODER_E2E_REQUIRE_PREMIUM_TESTS: "1"
+          CODER_E2E_TEST_RETRIES: 2
         working-directory: site
 
       - name: Upload Playwright Failed Tests
         if: always() && github.actor != 'dependabot[bot]' && runner.os == 'Linux' && !github.event.pull_request.head.repo.fork
-        uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
           name: failed-test-videos${{ matrix.variant.premium && '-premium' || '' }}
           path: ./site/test-results/**/*.webm
@@ -741,29 +872,32 @@ jobs:
 
       - name: Upload pprof dumps
         if: always() && github.actor != 'dependabot[bot]' && runner.os == 'Linux' && !github.event.pull_request.head.repo.fork
-        uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
           name: debug-pprof-dumps${{ matrix.variant.premium && '-premium' || ''  }}
           path: ./site/test-results/**/debug-pprof-*.txt
           retention-days: 7
 
+  # Reference guide:
+  # https://www.chromatic.com/docs/turbosnap-best-practices/#run-with-caution-when-using-the-pull_request-event
   chromatic:
     # REMARK: this is only used to build storybook and deploy it to Chromatic.
     runs-on: ubuntu-latest
     needs: changes
-    if: needs.changes.outputs.ts == 'true' || needs.changes.outputs.ci == 'true'
+    if: needs.changes.outputs.site == 'true' || needs.changes.outputs.ci == 'true'
     steps:
       - name: Harden Runner
-        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
         with:
           egress-policy: audit
 
       - name: Checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
-          # Required by Chromatic for build-over-build history, otherwise we
-          # only get 1 commit on shallow checkout.
+          # 👇 Ensures Chromatic can read your full git history
           fetch-depth: 0
+          # 👇 Tells the checkout which commit hash to reference
+          ref: ${{ github.event.pull_request.head.ref }}
 
       - name: Setup Node
         uses: ./.github/actions/setup-node
@@ -773,7 +907,7 @@ jobs:
       # the check to pass. This is desired in PRs, but not in mainline.
       - name: Publish to Chromatic (non-mainline)
         if: github.ref != 'refs/heads/main' && github.repository_owner == 'coder'
-        uses: chromaui/action@30b6228aa809059d46219e0f556752e8672a7e26 # v11.11.0
+        uses: chromaui/action@d7afd50124cf4f337bcd943e7f45cfa85a5e4476 # v12.0.0
         env:
           NODE_OPTIONS: "--max_old_space_size=4096"
           STORYBOOK: true
@@ -788,6 +922,7 @@ jobs:
           projectToken: 695c25b6cb65
           workingDir: "./site"
           storybookBaseDir: "./site"
+          storybookConfigDir: "./site/.storybook"
           # Prevent excessive build runs on minor version changes
           skip: "@(renovate/**|dependabot/**)"
           # Run TurboSnap to trace file dependencies to related stories
@@ -804,7 +939,7 @@ jobs:
       # infinitely "in progress" in mainline unless we re-review each build.
       - name: Publish to Chromatic (mainline)
         if: github.ref == 'refs/heads/main' && github.repository_owner == 'coder'
-        uses: chromaui/action@30b6228aa809059d46219e0f556752e8672a7e26 # v11.11.0
+        uses: chromaui/action@d7afd50124cf4f337bcd943e7f45cfa85a5e4476 # v12.0.0
         env:
           NODE_OPTIONS: "--max_old_space_size=4096"
           STORYBOOK: true
@@ -817,6 +952,7 @@ jobs:
           projectToken: 695c25b6cb65
           workingDir: "./site"
           storybookBaseDir: "./site"
+          storybookConfigDir: "./site/.storybook"
           # Run TurboSnap to trace file dependencies to related stories
           # and tell chromatic to only take snapshots of relevant stories
           onlyChanged: true
@@ -831,7 +967,7 @@ jobs:
 
     steps:
       - name: Harden Runner
-        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
         with:
           egress-policy: audit
 
@@ -860,12 +996,7 @@ jobs:
         uses: ./.github/actions/setup-go
 
       - name: Install go tools
-        run: |
-          go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.30
-          go install storj.io/drpc/cmd/protoc-gen-go-drpc@v0.0.34
-          go install golang.org/x/tools/cmd/goimports@latest
-          go install github.com/mikefarah/yq/v4@v4.44.3
-          go install go.uber.org/mock/mockgen@v0.5.0
+        uses: ./.github/actions/setup-go-tools
 
       - name: Setup sqlc
         uses: ./.github/actions/setup-sqlc
@@ -905,7 +1036,7 @@ jobs:
     if: always()
     steps:
       - name: Harden Runner
-        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
         with:
           egress-policy: audit
 
@@ -956,7 +1087,7 @@ jobs:
       - name: Switch XCode Version
         uses: maxim-lobanov/setup-xcode@60606e260d2fc5762a71e64e74b2174e8ea3c8bd # v1.6.0
         with:
-          xcode-version: "16.0.0"
+          xcode-version: "16.1.0"
 
       - name: Setup Go
         uses: ./.github/actions/setup-go
@@ -1000,7 +1131,7 @@ jobs:
 
       - name: Upload build artifacts
         if: ${{ github.repository_owner == 'coder' && github.ref == 'refs/heads/main' }}
-        uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
           name: dylibs
           path: |
@@ -1024,14 +1155,18 @@ jobs:
       # Necessary to push docker images to ghcr.io.
       packages: write
       # Necessary for GCP authentication (https://github.com/google-github-actions/setup-gcloud#usage)
+      # Also necessary for keyless cosign (https://docs.sigstore.dev/cosign/signing/overview/)
+      # And for GitHub Actions attestation
       id-token: write
+      # Required for GitHub Actions attestation
+      attestations: write
     env:
       DOCKER_CLI_EXPERIMENTAL: "enabled"
     outputs:
       IMAGE: ghcr.io/coder/coder-preview:${{ steps.build-docker.outputs.tag }}
     steps:
       - name: Harden Runner
-        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
         with:
           egress-policy: audit
 
@@ -1041,7 +1176,7 @@ jobs:
           fetch-depth: 0
 
       - name: GHCR Login
-        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
+        uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
@@ -1055,7 +1190,7 @@ jobs:
 
       # Necessary for signing Windows binaries.
       - name: Setup Java
-        uses: actions/setup-java@3a4f6e1af504cf6a31855fa899c6aa5355ba6c12 # v4.7.0
+        uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4.7.1
         with:
           distribution: "zulu"
           java-version: "11.0"
@@ -1069,6 +1204,12 @@ jobs:
       - name: Install zstd
         run: sudo apt-get install -y zstd
 
+      - name: Install cosign
+        uses: ./.github/actions/install-cosign
+
+      - name: Install syft
+        uses: ./.github/actions/install-syft
+
       - name: Setup Windows EV Signing Certificate
         run: |
           set -euo pipefail
@@ -1082,7 +1223,7 @@ jobs:
       # Setup GCloud for signing Windows binaries.
       - name: Authenticate to Google Cloud
         id: gcloud_auth
-        uses: google-github-actions/auth@71f986410dfbc7added4569d411d040a91dc6935 # v2.1.8
+        uses: google-github-actions/auth@ba79af03959ebeac9769e648f473a284504d9193 # v2.1.10
         with:
           workload_identity_provider: ${{ secrets.GCP_CODE_SIGNING_WORKLOAD_ID_PROVIDER }}
           service_account: ${{ secrets.GCP_CODE_SIGNING_SERVICE_ACCOUNT }}
@@ -1092,7 +1233,7 @@ jobs:
         uses: google-github-actions/setup-gcloud@77e7a554d41e2ee56fc945c52dfd3f33d12def9a # v2.1.4
 
       - name: Download dylibs
-        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
         with:
           name: dylibs
           path: ./build
@@ -1123,6 +1264,8 @@ jobs:
           # do (see above).
           CODER_SIGN_WINDOWS: "1"
           CODER_WINDOWS_RESOURCES: "1"
+          CODER_SIGN_GPG: "1"
+          CODER_GPG_RELEASE_KEY_BASE64: ${{ secrets.GPG_RELEASE_KEY_BASE64 }}
           EV_KEY: ${{ secrets.EV_KEY }}
           EV_KEYSTORE: ${{ secrets.EV_KEYSTORE }}
           EV_TSA_URL: ${{ secrets.EV_TSA_URL }}
@@ -1170,6 +1313,166 @@ jobs:
             done
           fi
 
+      - name: SBOM Generation and Attestation
+        if: github.ref == 'refs/heads/main'
+        continue-on-error: true
+        env:
+          COSIGN_EXPERIMENTAL: 1
+        run: |
+          set -euxo pipefail
+
+          # Define image base and tags
+          IMAGE_BASE="ghcr.io/coder/coder-preview"
+          TAGS=("${{ steps.build-docker.outputs.tag }}" "main" "latest")
+
+          # Generate and attest SBOM for each tag
+          for tag in "${TAGS[@]}"; do
+            IMAGE="${IMAGE_BASE}:${tag}"
+            SBOM_FILE="coder_sbom_${tag//[:\/]/_}.spdx.json"
+
+            echo "Generating SBOM for image: ${IMAGE}"
+            syft "${IMAGE}" -o spdx-json > "${SBOM_FILE}"
+
+            echo "Attesting SBOM to image: ${IMAGE}"
+            cosign clean --force=true "${IMAGE}"
+            cosign attest --type spdxjson \
+              --predicate "${SBOM_FILE}" \
+              --yes \
+              "${IMAGE}"
+          done
+
+      # GitHub attestation provides SLSA provenance for the Docker images, establishing a verifiable
+      # record that these images were built in GitHub Actions with specific inputs and environment.
+      # This complements our existing cosign attestations which focus on SBOMs.
+      #
+      # We attest each tag separately to ensure all tags have proper provenance records.
+      # TODO: Consider refactoring these steps to use a matrix strategy or composite action to reduce duplication
+      # while maintaining the required functionality for each tag.
+      - name: GitHub Attestation for Docker image
+        id: attest_main
+        if: github.ref == 'refs/heads/main'
+        continue-on-error: true
+        uses: actions/attest@afd638254319277bb3d7f0a234478733e2e46a73 # v2.3.0
+        with:
+          subject-name: "ghcr.io/coder/coder-preview:main"
+          predicate-type: "https://slsa.dev/provenance/v1"
+          predicate: |
+            {
+              "buildType": "https://github.com/actions/runner-images/",
+              "builder": {
+                "id": "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+              },
+              "invocation": {
+                "configSource": {
+                  "uri": "git+https://github.com/${{ github.repository }}@${{ github.ref }}",
+                  "digest": {
+                    "sha1": "${{ github.sha }}"
+                  },
+                  "entryPoint": ".github/workflows/ci.yaml"
+                },
+                "environment": {
+                  "github_workflow": "${{ github.workflow }}",
+                  "github_run_id": "${{ github.run_id }}"
+                }
+              },
+              "metadata": {
+                "buildInvocationID": "${{ github.run_id }}",
+                "completeness": {
+                  "environment": true,
+                  "materials": true
+                }
+              }
+            }
+          push-to-registry: true
+
+      - name: GitHub Attestation for Docker image (latest tag)
+        id: attest_latest
+        if: github.ref == 'refs/heads/main'
+        continue-on-error: true
+        uses: actions/attest@afd638254319277bb3d7f0a234478733e2e46a73 # v2.3.0
+        with:
+          subject-name: "ghcr.io/coder/coder-preview:latest"
+          predicate-type: "https://slsa.dev/provenance/v1"
+          predicate: |
+            {
+              "buildType": "https://github.com/actions/runner-images/",
+              "builder": {
+                "id": "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+              },
+              "invocation": {
+                "configSource": {
+                  "uri": "git+https://github.com/${{ github.repository }}@${{ github.ref }}",
+                  "digest": {
+                    "sha1": "${{ github.sha }}"
+                  },
+                  "entryPoint": ".github/workflows/ci.yaml"
+                },
+                "environment": {
+                  "github_workflow": "${{ github.workflow }}",
+                  "github_run_id": "${{ github.run_id }}"
+                }
+              },
+              "metadata": {
+                "buildInvocationID": "${{ github.run_id }}",
+                "completeness": {
+                  "environment": true,
+                  "materials": true
+                }
+              }
+            }
+          push-to-registry: true
+
+      - name: GitHub Attestation for version-specific Docker image
+        id: attest_version
+        if: github.ref == 'refs/heads/main'
+        continue-on-error: true
+        uses: actions/attest@afd638254319277bb3d7f0a234478733e2e46a73 # v2.3.0
+        with:
+          subject-name: "ghcr.io/coder/coder-preview:${{ steps.build-docker.outputs.tag }}"
+          predicate-type: "https://slsa.dev/provenance/v1"
+          predicate: |
+            {
+              "buildType": "https://github.com/actions/runner-images/",
+              "builder": {
+                "id": "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+              },
+              "invocation": {
+                "configSource": {
+                  "uri": "git+https://github.com/${{ github.repository }}@${{ github.ref }}",
+                  "digest": {
+                    "sha1": "${{ github.sha }}"
+                  },
+                  "entryPoint": ".github/workflows/ci.yaml"
+                },
+                "environment": {
+                  "github_workflow": "${{ github.workflow }}",
+                  "github_run_id": "${{ github.run_id }}"
+                }
+              },
+              "metadata": {
+                "buildInvocationID": "${{ github.run_id }}",
+                "completeness": {
+                  "environment": true,
+                  "materials": true
+                }
+              }
+            }
+          push-to-registry: true
+
+      # Report attestation failures but don't fail the workflow
+      - name: Check attestation status
+        if: github.ref == 'refs/heads/main'
+        run: |
+          if [[ "${{ steps.attest_main.outcome }}" == "failure" ]]; then
+            echo "::warning::GitHub attestation for main tag failed"
+          fi
+          if [[ "${{ steps.attest_latest.outcome }}" == "failure" ]]; then
+            echo "::warning::GitHub attestation for latest tag failed"
+          fi
+          if [[ "${{ steps.attest_version.outcome }}" == "failure" ]]; then
+            echo "::warning::GitHub attestation for version-specific tag failed"
+          fi
+
       - name: Prune old images
         if: github.ref == 'refs/heads/main'
         uses: vlaurin/action-ghcr-prune@0cf7d39f88546edd31965acba78cdcb0be14d641 # v0.6.0
@@ -1187,7 +1490,7 @@ jobs:
 
       - name: Upload build artifacts
         if: github.ref == 'refs/heads/main'
-        uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
           name: coder
           path: |
@@ -1211,7 +1514,7 @@ jobs:
       id-token: write
     steps:
       - name: Harden Runner
-        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
         with:
           egress-policy: audit
 
@@ -1221,7 +1524,7 @@ jobs:
           fetch-depth: 0
 
       - name: Authenticate to Google Cloud
-        uses: google-github-actions/auth@71f986410dfbc7added4569d411d040a91dc6935 # v2.1.8
+        uses: google-github-actions/auth@ba79af03959ebeac9769e648f473a284504d9193 # v2.1.10
         with:
           workload_identity_provider: projects/573722524737/locations/global/workloadIdentityPools/github/providers/github
           service_account: coder-ci@coder-dogfood.iam.gserviceaccount.com
@@ -1236,7 +1539,7 @@ jobs:
           version: "2.5.1"
 
       - name: Get Cluster Credentials
-        uses: google-github-actions/get-gke-credentials@7a108e64ed8546fe38316b4086e91da13f4785e1 # v2.3.1
+        uses: google-github-actions/get-gke-credentials@d0cee45012069b163a631894b98904a9e6723729 # v2.3.3
         with:
           cluster_name: dogfood-v2
           location: us-central1-a
@@ -1266,6 +1569,8 @@ jobs:
           kubectl --namespace coder rollout status deployment/coder
           kubectl --namespace coder rollout restart deployment/coder-provisioner
           kubectl --namespace coder rollout status deployment/coder-provisioner
+          kubectl --namespace coder rollout restart deployment/coder-provisioner-tagged
+          kubectl --namespace coder rollout status deployment/coder-provisioner-tagged
 
   deploy-wsproxies:
     runs-on: ubuntu-latest
@@ -1273,7 +1578,7 @@ jobs:
     if: github.ref == 'refs/heads/main' && !github.event.pull_request.head.repo.fork
     steps:
       - name: Harden Runner
-        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
         with:
           egress-policy: audit
 
@@ -1308,7 +1613,7 @@ jobs:
     if: needs.changes.outputs.db == 'true' || needs.changes.outputs.ci == 'true' || github.ref == 'refs/heads/main'
     steps:
       - name: Harden Runner
-        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
         with:
           egress-policy: audit
 

.github/workflows/dependabot.yaml

@@ -23,7 +23,7 @@ jobs:
     steps:
       - name: Dependabot metadata
         id: metadata
-        uses: dependabot/fetch-metadata@d7267f607e9d3fb96fc2fbe83e0af444713e90b7 # v2.3.0
+        uses: dependabot/fetch-metadata@08eff52bf64351f401fb50d4972fa95b9f2c2d1b # v2.4.0
         with:
           github-token: "${{ secrets.GITHUB_TOKEN }}"
 

.github/workflows/docker-base.yaml

@@ -38,7 +38,7 @@ jobs:
     if: github.repository_owner == 'coder'
     steps:
       - name: Harden Runner
-        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
         with:
           egress-policy: audit
 
@@ -46,7 +46,7 @@ jobs:
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
       - name: Docker login
-        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
+        uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
         with:
           registry: ghcr.io
           username: ${{ github.actor }}

.github/workflows/docs-ci.yaml

@@ -28,7 +28,7 @@ jobs:
       - name: Setup Node
         uses: ./.github/actions/setup-node
 
-      - uses: tj-actions/changed-files@dcc7a0cba800f454d79fff4b993e8c3555bcc0a8 # v45.0.7
+      - uses: tj-actions/changed-files@3981e4f74104e7a4c67a835e1e5dd5d9eb0f0a57 # v45.0.7
         id: changed-files
         with:
           files: |

.github/workflows/dogfood.yaml

@@ -27,7 +27,7 @@ jobs:
     runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-4' || 'ubuntu-latest' }}
     steps:
       - name: Harden Runner
-        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
         with:
           egress-policy: audit
 
@@ -35,11 +35,30 @@ jobs:
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
       - name: Setup Nix
-        uses: DeterminateSystems/nix-installer-action@e50d5f73bfe71c2dd0aa4218de8f4afa59f8f81d # v16
+        uses: nixbuild/nix-quick-install-action@5bb6a3b3abe66fd09bbf250dce8ada94f856a703 # v30
+
+      - uses: nix-community/cache-nix-action@135667ec418502fa5a3598af6fb9eb733888ce6a # v6.1.3
+        with:
+          # restore and save a cache using this key
+          primary-key: nix-${{ runner.os }}-${{ hashFiles('**/*.nix', '**/flake.lock') }}
+          # if there's no cache hit, restore a cache by this prefix
+          restore-prefixes-first-match: nix-${{ runner.os }}-
+          # collect garbage until Nix store size (in bytes) is at most this number
+          # before trying to save a new cache
+          # 1G = 1073741824
+          gc-max-store-size-linux: 5G
+          # do purge caches
+          purge: true
+          # purge all versions of the cache
+          purge-prefixes: nix-${{ runner.os }}-
+          # created more than this number of seconds ago relative to the start of the `Post Restore` phase
+          purge-created: 0
+          # except the version with the `primary-key`, if it exists
+          purge-primary-key: never
 
       - name: Get branch name
         id: branch-name
-        uses: tj-actions/branch-names@6871f53176ad61624f978536bbf089c574dc19a2 # v8.0.1
+        uses: tj-actions/branch-names@dde14ac574a8b9b1cedc59a1cf312788af43d8d8 # v8.2.1
 
       - name: "Branch name to Docker tag name"
         id: docker-tag-name
@@ -53,11 +72,11 @@ jobs:
         uses: depot/setup-action@b0b1ea4f69e92ebf5dea3f8713a1b0c37b2126a5 # v1.6.0
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@f7ce87c1d6bead3e36075b2ce75da1f6cc28aaca # v3.9.0
+        uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0
 
       - name: Login to DockerHub
         if: github.ref == 'refs/heads/main'
-        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
+        uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_PASSWORD }}
@@ -68,7 +87,7 @@ jobs:
           project: b4q6ltmpzh
           token: ${{ secrets.DEPOT_TOKEN }}
           buildx-fallback: true
-          context: "{{defaultContext}}:dogfood/contents"
+          context: "{{defaultContext}}:dogfood/coder"
           pull: true
           save: true
           push: ${{ github.ref == 'refs/heads/main' }}
@@ -95,7 +114,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Harden Runner
-        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
         with:
           egress-policy: audit
 
@@ -106,19 +125,25 @@ jobs:
         uses: ./.github/actions/setup-tf
 
       - name: Authenticate to Google Cloud
-        uses: google-github-actions/auth@71f986410dfbc7added4569d411d040a91dc6935 # v2.1.8
+        uses: google-github-actions/auth@ba79af03959ebeac9769e648f473a284504d9193 # v2.1.10
         with:
           workload_identity_provider: projects/573722524737/locations/global/workloadIdentityPools/github/providers/github
           service_account: coder-ci@coder-dogfood.iam.gserviceaccount.com
 
       - name: Terraform init and validate
         run: |
-          cd dogfood
-          terraform init -upgrade
+          pushd dogfood/
+          terraform init
           terraform validate
-          cd contents
-          terraform init -upgrade
+          popd
+          pushd dogfood/coder
+          terraform init
           terraform validate
+          popd
+          pushd dogfood/coder-envbuilder
+          terraform init
+          terraform validate
+          popd
 
       - name: Get short commit SHA
         if: github.ref == 'refs/heads/main'
@@ -142,6 +167,6 @@ jobs:
           # Template source & details
           TF_VAR_CODER_TEMPLATE_NAME: ${{ secrets.CODER_TEMPLATE_NAME }}
           TF_VAR_CODER_TEMPLATE_VERSION: ${{ steps.vars.outputs.sha_short }}
-          TF_VAR_CODER_TEMPLATE_DIR: ./contents
+          TF_VAR_CODER_TEMPLATE_DIR: ./coder
           TF_VAR_CODER_TEMPLATE_MESSAGE: ${{ steps.message.outputs.pr_title }}
           TF_LOG: info

.github/workflows/nightly-gauntlet.yaml

@@ -1,141 +0,0 @@
-# The nightly-gauntlet runs tests that are either too flaky or too slow to block
-# every PR.
-name: nightly-gauntlet
-on:
-  schedule:
-    # Every day at 4AM
-    - cron: "0 4 * * 1-5"
-  workflow_dispatch:
-
-permissions:
-  contents: read
-
-jobs:
-  test-go-pg:
-    runs-on: ${{ matrix.os == 'macos-latest' && github.repository_owner == 'coder' && 'depot-macos-latest' || matrix.os == 'windows-2022' && github.repository_owner == 'coder' && 'windows-latest-16-cores' || matrix.os }}
-    if: github.ref == 'refs/heads/main'
-    # This timeout must be greater than the timeout set by `go test` in
-    # `make test-postgres` to ensure we receive a trace of running
-    # goroutines. Setting this to the timeout +5m should work quite well
-    # even if some of the preceding steps are slow.
-    timeout-minutes: 25
-    strategy:
-      matrix:
-        os:
-          - macos-latest
-          - windows-2022
-    steps:
-      - name: Harden Runner
-        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
-        with:
-          egress-policy: audit
-
-      - name: Checkout
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-        with:
-          fetch-depth: 1
-
-      - name: Setup Go
-        uses: ./.github/actions/setup-go
-
-      - name: Setup Terraform
-        uses: ./.github/actions/setup-tf
-
-      # Sets up the ImDisk toolkit for Windows and creates a RAM disk on drive R:.
-      - name: Setup ImDisk
-        if: runner.os == 'Windows'
-        uses: ./.github/actions/setup-imdisk
-
-      - name: Test with PostgreSQL Database
-        env:
-          POSTGRES_VERSION: "13"
-          TS_DEBUG_DISCO: "true"
-          LC_CTYPE: "en_US.UTF-8"
-          LC_ALL: "en_US.UTF-8"
-        shell: bash
-        run: |
-          # if macOS, install google-chrome for scaletests
-          # As another concern, should we really have this kind of external dependency
-          # requirement on standard CI?
-          if [ "${{ matrix.os }}" == "macos-latest" ]; then
-            brew install google-chrome
-          fi
-
-          # By default Go will use the number of logical CPUs, which
-          # is a fine default.
-          PARALLEL_FLAG=""
-
-          # macOS will output "The default interactive shell is now zsh"
-          # intermittently in CI...
-          if [ "${{ matrix.os }}" == "macos-latest" ]; then
-            touch ~/.bash_profile && echo "export BASH_SILENCE_DEPRECATION_WARNING=1" >> ~/.bash_profile
-          fi
-
-          if [ "${{ runner.os }}" == "Windows" ]; then
-            # Create a temp dir on the R: ramdisk drive for Windows. The default
-            # C: drive is extremely slow: https://github.com/actions/runner-images/issues/8755
-            mkdir -p "R:/temp/embedded-pg"
-            go run scripts/embedded-pg/main.go -path "R:/temp/embedded-pg"
-          else
-            go run scripts/embedded-pg/main.go
-          fi
-
-          # Reduce test parallelism, mirroring what we do for race tests.
-          # We'd been encountering issues with timing related flakes, and
-          # this seems to help.
-          DB=ci gotestsum --format standard-quiet -- -v -short -count=1 -parallel 4 -p 4 ./...
-
-      - name: Upload test stats to Datadog
-        timeout-minutes: 1
-        continue-on-error: true
-        uses: ./.github/actions/upload-datadog
-        if: success() || failure()
-        with:
-          api-key: ${{ secrets.DATADOG_API_KEY }}
-
-  notify-slack-on-failure:
-    needs:
-      - test-go-pg
-    runs-on: ubuntu-latest
-    if: failure() && github.ref == 'refs/heads/main'
-
-    steps:
-      - name: Send Slack notification
-        run: |
-          curl -X POST -H 'Content-type: application/json' \
-          --data '{
-            "blocks": [
-              {
-                "type": "header",
-                "text": {
-                  "type": "plain_text",
-                  "text": "❌ Nightly gauntlet failed",
-                  "emoji": true
-                }
-              },
-              {
-                "type": "section",
-                "fields": [
-                  {
-                    "type": "mrkdwn",
-                    "text": "*Workflow:*\n${{ github.workflow }}"
-                  },
-                  {
-                    "type": "mrkdwn",
-                    "text": "*Committer:*\n${{ github.actor }}"
-                  },
-                  {
-                    "type": "mrkdwn",
-                    "text": "*Commit:*\n${{ github.sha }}"
-                  }
-                ]
-              },
-              {
-                "type": "section",
-                "text": {
-                  "type": "mrkdwn",
-                  "text": "*View failure:* <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|Click here>"
-                }
-              }
-            ]
-          }' ${{ secrets.CI_FAILURE_SLACK_WEBHOOK }}

.github/workflows/pr-auto-assign.yaml

@@ -14,7 +14,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Harden Runner
-        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
         with:
           egress-policy: audit
 

.github/workflows/pr-cleanup.yaml

@@ -19,7 +19,7 @@ jobs:
       packages: write
     steps:
       - name: Harden Runner
-        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
         with:
           egress-policy: audit
 

.github/workflows/pr-deploy.yaml

@@ -39,7 +39,7 @@ jobs:
       PR_OPEN: ${{ steps.check_pr.outputs.pr_open }}
     steps:
       - name: Harden Runner
-        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
         with:
           egress-policy: audit
 
@@ -74,7 +74,7 @@ jobs:
     runs-on: "ubuntu-latest"
     steps:
       - name: Harden Runner
-        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
         with:
           egress-policy: audit
 
@@ -174,7 +174,7 @@ jobs:
       pull-requests: write # needed for commenting on PRs
     steps:
       - name: Harden Runner
-        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
         with:
           egress-policy: audit
 
@@ -218,7 +218,7 @@ jobs:
       CODER_IMAGE_TAG: ${{ needs.get_info.outputs.CODER_IMAGE_TAG }}
     steps:
       - name: Harden Runner
-        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
         with:
           egress-policy: audit
 
@@ -237,7 +237,7 @@ jobs:
         uses: ./.github/actions/setup-sqlc
 
       - name: GHCR Login
-        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
+        uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
@@ -276,7 +276,7 @@ jobs:
       PR_HOSTNAME: "pr${{ needs.get_info.outputs.PR_NUMBER }}.${{ secrets.PR_DEPLOYMENTS_DOMAIN }}"
     steps:
       - name: Harden Runner
-        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
         with:
           egress-policy: audit
 

.github/workflows/release-validation.yaml

@@ -14,7 +14,7 @@ jobs:
 
     steps:
       - name: Harden Runner
-        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
         with:
           egress-policy: audit
 

.github/workflows/release.yaml

@@ -60,7 +60,7 @@ jobs:
       - name: Switch XCode Version
         uses: maxim-lobanov/setup-xcode@60606e260d2fc5762a71e64e74b2174e8ea3c8bd # v1.6.0
         with:
-          xcode-version: "16.0.0"
+          xcode-version: "16.1.0"
 
       - name: Setup Go
         uses: ./.github/actions/setup-go
@@ -101,7 +101,7 @@ jobs:
           AC_CERTIFICATE_PASSWORD_FILE: /tmp/apple_cert_password.txt
 
       - name: Upload build artifacts
-        uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
           name: dylibs
           path: |
@@ -122,7 +122,11 @@ jobs:
       # Necessary to push docker images to ghcr.io.
       packages: write
       # Necessary for GCP authentication (https://github.com/google-github-actions/setup-gcloud#usage)
+      # Also necessary for keyless cosign (https://docs.sigstore.dev/cosign/signing/overview/)
+      # And for GitHub Actions attestation
       id-token: write
+      # Required for GitHub Actions attestation
+      attestations: write
     env:
       # Necessary for Docker manifest
       DOCKER_CLI_EXPERIMENTAL: "enabled"
@@ -130,7 +134,7 @@ jobs:
       version: ${{ steps.version.outputs.version }}
     steps:
       - name: Harden Runner
-        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
         with:
           egress-policy: audit
 
@@ -204,7 +208,7 @@ jobs:
           cat "$CODER_RELEASE_NOTES_FILE"
 
       - name: Docker Login
-        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
+        uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
@@ -218,7 +222,7 @@ jobs:
 
       # Necessary for signing Windows binaries.
       - name: Setup Java
-        uses: actions/setup-java@3a4f6e1af504cf6a31855fa899c6aa5355ba6c12 # v4.7.0
+        uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4.7.1
         with:
           distribution: "zulu"
           java-version: "11.0"
@@ -246,6 +250,12 @@ jobs:
             apple-codesign-0.22.0-x86_64-unknown-linux-musl/rcodesign
           rm /tmp/rcodesign.tar.gz
 
+      - name: Install cosign
+        uses: ./.github/actions/install-cosign
+
+      - name: Install syft
+        uses: ./.github/actions/install-syft
+
       - name: Setup Apple Developer certificate and API key
         run: |
           set -euo pipefail
@@ -276,7 +286,7 @@ jobs:
       # Setup GCloud for signing Windows binaries.
       - name: Authenticate to Google Cloud
         id: gcloud_auth
-        uses: google-github-actions/auth@71f986410dfbc7added4569d411d040a91dc6935 # v2.1.8
+        uses: google-github-actions/auth@ba79af03959ebeac9769e648f473a284504d9193 # v2.1.10
         with:
           workload_identity_provider: ${{ secrets.GCP_CODE_SIGNING_WORKLOAD_ID_PROVIDER }}
           service_account: ${{ secrets.GCP_CODE_SIGNING_SERVICE_ACCOUNT }}
@@ -286,7 +296,7 @@ jobs:
         uses: google-github-actions/setup-gcloud@77e7a554d41e2ee56fc945c52dfd3f33d12def9a # v2.1.4
 
       - name: Download dylibs
-        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
         with:
           name: dylibs
           path: ./build
@@ -313,6 +323,8 @@ jobs:
         env:
           CODER_SIGN_WINDOWS: "1"
           CODER_SIGN_DARWIN: "1"
+          CODER_SIGN_GPG: "1"
+          CODER_GPG_RELEASE_KEY_BASE64: ${{ secrets.GPG_RELEASE_KEY_BASE64 }}
           CODER_WINDOWS_RESOURCES: "1"
           AC_CERTIFICATE_FILE: /tmp/apple_cert.p12
           AC_CERTIFICATE_PASSWORD_FILE: /tmp/apple_cert_password.txt
@@ -361,6 +373,7 @@ jobs:
           file: scripts/Dockerfile.base
           platforms: linux/amd64,linux/arm64,linux/arm/v7
           provenance: true
+          sbom: true
           pull: true
           no-cache: true
           push: true
@@ -397,7 +410,52 @@ jobs:
           echo "$manifests" | grep -q linux/arm64
           echo "$manifests" | grep -q linux/arm/v7
 
+      # GitHub attestation provides SLSA provenance for Docker images, establishing a verifiable
+      # record that these images were built in GitHub Actions with specific inputs and environment.
+      # This complements our existing cosign attestations (which focus on SBOMs) by adding
+      # GitHub-specific build provenance to enhance our supply chain security.
+      #
+      # TODO: Consider refactoring these attestation steps to use a matrix strategy or composite action
+      # to reduce duplication while maintaining the required functionality for each distinct image tag.
+      - name: GitHub Attestation for Base Docker image
+        id: attest_base
+        if: ${{ !inputs.dry_run && steps.image-base-tag.outputs.tag != '' }}
+        continue-on-error: true
+        uses: actions/attest@afd638254319277bb3d7f0a234478733e2e46a73 # v2.3.0
+        with:
+          subject-name: ${{ steps.image-base-tag.outputs.tag }}
+          predicate-type: "https://slsa.dev/provenance/v1"
+          predicate: |
+            {
+              "buildType": "https://github.com/actions/runner-images/",
+              "builder": {
+                "id": "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+              },
+              "invocation": {
+                "configSource": {
+                  "uri": "git+https://github.com/${{ github.repository }}@${{ github.ref }}",
+                  "digest": {
+                    "sha1": "${{ github.sha }}"
+                  },
+                  "entryPoint": ".github/workflows/release.yaml"
+                },
+                "environment": {
+                  "github_workflow": "${{ github.workflow }}",
+                  "github_run_id": "${{ github.run_id }}"
+                }
+              },
+              "metadata": {
+                "buildInvocationID": "${{ github.run_id }}",
+                "completeness": {
+                  "environment": true,
+                  "materials": true
+                }
+              }
+            }
+          push-to-registry: true
+
       - name: Build Linux Docker images
+        id: build_docker
         run: |
           set -euxo pipefail
 
@@ -416,18 +474,158 @@ jobs:
           # being pushed so will automatically push them.
           make push/build/coder_"$version"_linux.tag
 
+          # Save multiarch image tag for attestation
+          multiarch_image="$(./scripts/image_tag.sh)"
+          echo "multiarch_image=${multiarch_image}" >> $GITHUB_OUTPUT
+
+          # For debugging, print all docker image tags
+          docker images
+
           # if the current version is equal to the highest (according to semver)
           # version in the repo, also create a multi-arch image as ":latest" and
           # push it
+          created_latest_tag=false
           if [[ "$(git tag | grep '^v' | grep -vE '(rc|dev|-|\+|\/)' | sort -r --version-sort | head -n1)" == "v$(./scripts/version.sh)" ]]; then
             ./scripts/build_docker_multiarch.sh \
               --push \
               --target "$(./scripts/image_tag.sh --version latest)" \
               $(cat build/coder_"$version"_linux_{amd64,arm64,armv7}.tag)
+            created_latest_tag=true
+            echo "created_latest_tag=true" >> $GITHUB_OUTPUT
+          else
+            echo "created_latest_tag=false" >> $GITHUB_OUTPUT
           fi
         env:
           CODER_BASE_IMAGE_TAG: ${{ steps.image-base-tag.outputs.tag }}
 
+      - name: SBOM Generation and Attestation
+        if: ${{ !inputs.dry_run }}
+        env:
+          COSIGN_EXPERIMENTAL: "1"
+        run: |
+          set -euxo pipefail
+
+          # Generate SBOM for multi-arch image with version in filename
+          echo "Generating SBOM for multi-arch image: ${{ steps.build_docker.outputs.multiarch_image }}"
+          syft "${{ steps.build_docker.outputs.multiarch_image }}" -o spdx-json > coder_${{ steps.version.outputs.version }}_sbom.spdx.json
+
+          # Attest SBOM to multi-arch image
+          echo "Attesting SBOM to multi-arch image: ${{ steps.build_docker.outputs.multiarch_image }}"
+          cosign clean --force=true "${{ steps.build_docker.outputs.multiarch_image }}"
+          cosign attest --type spdxjson \
+            --predicate coder_${{ steps.version.outputs.version }}_sbom.spdx.json \
+            --yes \
+            "${{ steps.build_docker.outputs.multiarch_image }}"
+
+          # If latest tag was created, also attest it
+          if [[ "${{ steps.build_docker.outputs.created_latest_tag }}" == "true" ]]; then
+            latest_tag="$(./scripts/image_tag.sh --version latest)"
+            echo "Generating SBOM for latest image: ${latest_tag}"
+            syft "${latest_tag}" -o spdx-json > coder_latest_sbom.spdx.json
+
+            echo "Attesting SBOM to latest image: ${latest_tag}"
+            cosign clean --force=true "${latest_tag}"
+            cosign attest --type spdxjson \
+              --predicate coder_latest_sbom.spdx.json \
+              --yes \
+              "${latest_tag}"
+          fi
+
+      - name: GitHub Attestation for Docker image
+        id: attest_main
+        if: ${{ !inputs.dry_run }}
+        continue-on-error: true
+        uses: actions/attest@afd638254319277bb3d7f0a234478733e2e46a73 # v2.3.0
+        with:
+          subject-name: ${{ steps.build_docker.outputs.multiarch_image }}
+          predicate-type: "https://slsa.dev/provenance/v1"
+          predicate: |
+            {
+              "buildType": "https://github.com/actions/runner-images/",
+              "builder": {
+                "id": "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+              },
+              "invocation": {
+                "configSource": {
+                  "uri": "git+https://github.com/${{ github.repository }}@${{ github.ref }}",
+                  "digest": {
+                    "sha1": "${{ github.sha }}"
+                  },
+                  "entryPoint": ".github/workflows/release.yaml"
+                },
+                "environment": {
+                  "github_workflow": "${{ github.workflow }}",
+                  "github_run_id": "${{ github.run_id }}"
+                }
+              },
+              "metadata": {
+                "buildInvocationID": "${{ github.run_id }}",
+                "completeness": {
+                  "environment": true,
+                  "materials": true
+                }
+              }
+            }
+          push-to-registry: true
+
+      # Get the latest tag name for attestation
+      - name: Get latest tag name
+        id: latest_tag
+        if: ${{ !inputs.dry_run && steps.build_docker.outputs.created_latest_tag == 'true' }}
+        run: echo "tag=$(./scripts/image_tag.sh --version latest)" >> $GITHUB_OUTPUT
+
+      # If this is the highest version according to semver, also attest the "latest" tag
+      - name: GitHub Attestation for "latest" Docker image
+        id: attest_latest
+        if: ${{ !inputs.dry_run && steps.build_docker.outputs.created_latest_tag == 'true' }}
+        continue-on-error: true
+        uses: actions/attest@afd638254319277bb3d7f0a234478733e2e46a73 # v2.3.0
+        with:
+          subject-name: ${{ steps.latest_tag.outputs.tag }}
+          predicate-type: "https://slsa.dev/provenance/v1"
+          predicate: |
+            {
+              "buildType": "https://github.com/actions/runner-images/",
+              "builder": {
+                "id": "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+              },
+              "invocation": {
+                "configSource": {
+                  "uri": "git+https://github.com/${{ github.repository }}@${{ github.ref }}",
+                  "digest": {
+                    "sha1": "${{ github.sha }}"
+                  },
+                  "entryPoint": ".github/workflows/release.yaml"
+                },
+                "environment": {
+                  "github_workflow": "${{ github.workflow }}",
+                  "github_run_id": "${{ github.run_id }}"
+                }
+              },
+              "metadata": {
+                "buildInvocationID": "${{ github.run_id }}",
+                "completeness": {
+                  "environment": true,
+                  "materials": true
+                }
+              }
+            }
+          push-to-registry: true
+
+      # Report attestation failures but don't fail the workflow
+      - name: Check attestation status
+        if: ${{ !inputs.dry_run }}
+        run: |
+          if [[ "${{ steps.attest_base.outcome }}" == "failure" && "${{ steps.attest_base.conclusion }}" != "skipped" ]]; then
+            echo "::warning::GitHub attestation for base image failed"
+          fi
+          if [[ "${{ steps.attest_main.outcome }}" == "failure" ]]; then
+            echo "::warning::GitHub attestation for main image failed"
+          fi
+          if [[ "${{ steps.attest_latest.outcome }}" == "failure" && "${{ steps.attest_latest.conclusion }}" != "skipped" ]]; then
+            echo "::warning::GitHub attestation for latest image failed"
+          fi
+
       - name: Generate offline docs
         run: |
           version="$(./scripts/version.sh)"
@@ -436,6 +634,29 @@ jobs:
       - name: ls build
         run: ls -lh build
 
+      - name: Publish Coder CLI binaries and detached signatures to GCS
+        if: ${{ !inputs.dry_run && github.ref == 'refs/heads/main' && github.repository_owner == 'coder'}}
+        run: |
+          set -euxo pipefail
+
+          version="$(./scripts/version.sh)"
+
+          binaries=(
+              "coder-darwin-amd64"
+              "coder-darwin-arm64"
+              "coder-linux-amd64"
+              "coder-linux-arm64"
+              "coder-linux-armv7"
+              "coder-windows-amd64.exe"
+              "coder-windows-arm64.exe"
+          )
+
+          for binary in "${binaries[@]}"; do
+            detached_signature="${binary}.asc"
+            gcloud storage cp "./site/out/bin/${binary}" "gs://releases.coder.com/coder-cli/${version}/${binary}"
+            gcloud storage cp "./site/out/bin/${detached_signature}" "gs://releases.coder.com/coder-cli/${version}/${detached_signature}"
+          done
+
       - name: Publish release
         run: |
           set -euo pipefail
@@ -449,22 +670,33 @@ jobs:
           fi
           declare -p publish_args
 
+          # Build the list of files to publish
+          files=(
+            ./build/*_installer.exe
+            ./build/*.zip
+            ./build/*.tar.gz
+            ./build/*.tgz
+            ./build/*.apk
+            ./build/*.deb
+            ./build/*.rpm
+            ./coder_${{ steps.version.outputs.version }}_sbom.spdx.json
+          )
+
+          # Only include the latest SBOM file if it was created
+          if [[ "${{ steps.build_docker.outputs.created_latest_tag }}" == "true" ]]; then
+            files+=(./coder_latest_sbom.spdx.json)
+          fi
+
           ./scripts/release/publish.sh \
             "${publish_args[@]}" \
             --release-notes-file "$CODER_RELEASE_NOTES_FILE" \
-            ./build/*_installer.exe \
-            ./build/*.zip \
-            ./build/*.tar.gz \
-            ./build/*.tgz \
-            ./build/*.apk \
-            ./build/*.deb \
-            ./build/*.rpm
+            "${files[@]}"
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           CODER_GPG_RELEASE_KEY_BASE64: ${{ secrets.GPG_RELEASE_KEY_BASE64 }}
 
       - name: Authenticate to Google Cloud
-        uses: google-github-actions/auth@71f986410dfbc7added4569d411d040a91dc6935 # v2.1.8
+        uses: google-github-actions/auth@ba79af03959ebeac9769e648f473a284504d9193 # v2.1.10
         with:
           workload_identity_provider: ${{ secrets.GCP_WORKLOAD_ID_PROVIDER }}
           service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }}
@@ -489,7 +721,7 @@ jobs:
 
       - name: Upload artifacts to actions (if dry-run)
         if: ${{ inputs.dry_run }}
-        uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
           name: release-artifacts
           path: |
@@ -500,6 +732,15 @@ jobs:
             ./build/*.apk
             ./build/*.deb
             ./build/*.rpm
+            ./coder_${{ steps.version.outputs.version }}_sbom.spdx.json
+          retention-days: 7
+
+      - name: Upload latest sbom artifact to actions (if dry-run)
+        if: inputs.dry_run && steps.build_docker.outputs.created_latest_tag == 'true'
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: latest-sbom-artifact
+          path: ./coder_latest_sbom.spdx.json
           retention-days: 7
 
       - name: Send repository-dispatch event
@@ -521,7 +762,7 @@ jobs:
       # TODO: skip this if it's not a new release (i.e. a backport). This is
       #       fine right now because it just makes a PR that we can close.
       - name: Harden Runner
-        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
         with:
           egress-policy: audit
 
@@ -597,7 +838,7 @@ jobs:
 
     steps:
       - name: Harden Runner
-        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
         with:
           egress-policy: audit
 
@@ -687,7 +928,7 @@ jobs:
     if: ${{ !inputs.dry_run }}
     steps:
       - name: Harden Runner
-        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
         with:
           egress-policy: audit
 

.github/workflows/scorecard.yml

@@ -20,7 +20,7 @@ jobs:
 
     steps:
       - name: Harden Runner
-        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
         with:
           egress-policy: audit
 
@@ -39,7 +39,7 @@ jobs:
 
       # Upload the results as artifacts.
       - name: "Upload artifact"
-        uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
           name: SARIF file
           path: results.sarif
@@ -47,6 +47,6 @@ jobs:
 
       # Upload the results to GitHub's code scanning dashboard.
       - name: "Upload to code-scanning"
-        uses: github/codeql-action/upload-sarif@b56ba49b26e50535fa1e7f7db0f4f7b4bf65d80d # v3.28.10
+        uses: github/codeql-action/upload-sarif@ff0a06e83cb2de871e5a09832bc6a81e7276941f # v3.28.18
         with:
           sarif_file: results.sarif

.github/workflows/security.yaml

@@ -27,7 +27,7 @@ jobs:
     runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-8' || 'ubuntu-latest' }}
     steps:
       - name: Harden Runner
-        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
         with:
           egress-policy: audit
 
@@ -38,7 +38,7 @@ jobs:
         uses: ./.github/actions/setup-go
 
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@b56ba49b26e50535fa1e7f7db0f4f7b4bf65d80d # v3.28.10
+        uses: github/codeql-action/init@ff0a06e83cb2de871e5a09832bc6a81e7276941f # v3.28.18
         with:
           languages: go, javascript
 
@@ -48,7 +48,7 @@ jobs:
           rm Makefile
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@b56ba49b26e50535fa1e7f7db0f4f7b4bf65d80d # v3.28.10
+        uses: github/codeql-action/analyze@ff0a06e83cb2de871e5a09832bc6a81e7276941f # v3.28.18
 
       - name: Send Slack notification on failure
         if: ${{ failure() }}
@@ -67,7 +67,7 @@ jobs:
     runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-8' || 'ubuntu-latest' }}
     steps:
       - name: Harden Runner
-        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
         with:
           egress-policy: audit
 
@@ -85,6 +85,12 @@ jobs:
       - name: Setup sqlc
         uses: ./.github/actions/setup-sqlc
 
+      - name: Install cosign
+        uses: ./.github/actions/install-cosign
+
+      - name: Install syft
+        uses: ./.github/actions/install-syft
+
       - name: Install yq
         run: go run github.com/mikefarah/yq/v4@v4.44.3
       - name: Install mockgen
@@ -99,7 +105,7 @@ jobs:
           # version in the comments will differ. This is also defined in
           # ci.yaml.
           set -euxo pipefail
-          cd dogfood/contents
+          cd dogfood/coder
           mkdir -p /usr/local/bin
           mkdir -p /usr/local/include
 
@@ -136,7 +142,7 @@ jobs:
           echo "image=$(cat "$image_job")" >> $GITHUB_OUTPUT
 
       - name: Run Trivy vulnerability scanner
-        uses: aquasecurity/trivy-action@18f2510ee396bbf400402947b394f2dd8c87dbb0
+        uses: aquasecurity/trivy-action@6c175e9c4083a92bbca2f9724c8a5e33bc2d97a5
         with:
           image-ref: ${{ steps.build.outputs.image }}
           format: sarif
@@ -144,13 +150,13 @@ jobs:
           severity: "CRITICAL,HIGH"
 
       - name: Upload Trivy scan results to GitHub Security tab
-        uses: github/codeql-action/upload-sarif@b56ba49b26e50535fa1e7f7db0f4f7b4bf65d80d # v3.28.10
+        uses: github/codeql-action/upload-sarif@ff0a06e83cb2de871e5a09832bc6a81e7276941f # v3.28.18
         with:
           sarif_file: trivy-results.sarif
           category: "Trivy"
 
       - name: Upload Trivy scan results as an artifact
-        uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
           name: trivy
           path: trivy-results.sarif

.github/workflows/stale.yaml

@@ -18,7 +18,7 @@ jobs:
       pull-requests: write
     steps:
       - name: Harden Runner
-        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
         with:
           egress-policy: audit
 
@@ -96,14 +96,14 @@ jobs:
       contents: write
     steps:
       - name: Harden Runner
-        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
         with:
           egress-policy: audit
 
       - name: Checkout repository
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - name: Run delete-old-branches-action
-        uses: beatlabs/delete-old-branches-action@6e94df089372a619c01ae2c2f666bf474f890911 # v0.0.10
+        uses: beatlabs/delete-old-branches-action@4eeeb8740ff8b3cb310296ddd6b43c3387734588 # v0.0.11
         with:
           repo_token: ${{ github.token }}
           date: "6 months ago"
@@ -118,7 +118,7 @@ jobs:
       actions: write
     steps:
       - name: Harden Runner
-        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
         with:
           egress-policy: audit
 

.github/workflows/start-workspace.yaml

@@ -0,0 +1,35 @@
+name: Start Workspace On Issue Creation or Comment
+
+on:
+  issues:
+    types: [opened]
+  issue_comment:
+    types: [created]
+
+permissions:
+  issues: write
+
+jobs:
+  comment:
+    runs-on: ubuntu-latest
+    if: >-
+      (github.event_name == 'issue_comment' && contains(github.event.comment.body, '@coder')) || 
+      (github.event_name == 'issues' && contains(github.event.issue.body, '@coder'))
+    environment: dev.coder.com
+    timeout-minutes: 5
+    steps:
+      - name: Start Coder workspace
+        uses: coder/start-workspace-action@35a4608cefc7e8cc56573cae7c3b85304575cb72
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          github-username: >-
+            ${{
+              (github.event_name == 'issue_comment' && github.event.comment.user.login) || 
+              (github.event_name == 'issues' && github.event.issue.user.login)
+            }}
+          coder-url: ${{ secrets.CODER_URL }}
+          coder-token: ${{ secrets.CODER_TOKEN }}
+          template-name: ${{ secrets.CODER_TEMPLATE_NAME }}
+          parameters: |-
+            AI Prompt: "Use the gh CLI tool to read the details of issue https://github.com/${{ github.repository }}/issues/${{ github.event.issue.number }} and then address it."
+            Region: us-pittsburgh

.github/workflows/typos.toml

@@ -1,3 +1,6 @@
+[default]
+extend-ignore-identifiers-re = ["gho_.*"]
+
 [default.extend-identifiers]
 alog = "alog"
 Jetbrains = "JetBrains"
@@ -24,6 +27,7 @@ EDE = "EDE"
 HELO = "HELO"
 LKE = "LKE"
 byt = "byt"
+typ = "typ"
 
 [files]
 extend-exclude = [
@@ -42,5 +46,6 @@ extend-exclude = [
 	"site/src/pages/SetupPage/countries.tsx",
 	"provisioner/terraform/testdata/**",
 	# notifications' golden files confuse the detector because of quoted-printable encoding
-	"coderd/notifications/testdata/**"
+	"coderd/notifications/testdata/**",
+	"agent/agentcontainers/testdata/devcontainercli/**"
 ]

.github/workflows/weekly-docs.yaml

@@ -21,7 +21,7 @@ jobs:
       pull-requests: write # required to post PR review comments by the action
     steps:
       - name: Harden Runner
-        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
         with:
           egress-policy: audit
 
@@ -29,14 +29,14 @@ jobs:
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
       - name: Check Markdown links
-        uses: umbrelladocs/action-linkspector@de84085e0f51452a470558693d7d308fbb2fa261 # v1.2.5
+        uses: umbrelladocs/action-linkspector@a0567ce1c7c13de4a2358587492ed43cab5d0102 # v1.3.4
         id: markdown-link-check
         # checks all markdown files from /docs including all subfolders
         with:
           reporter: github-pr-review
           config_file: ".github/.linkspector.yml"
           fail_on_error: "true"
-          filter_mode: "nofilter"
+          filter_mode: "file"
 
       - name: Send Slack notification
         if: failure() && github.event_name == 'schedule'

.gitignore

@@ -32,7 +32,8 @@ site/e2e/.auth.json
 site/playwright-report/*
 site/.swc
 
-# Make target for updating golden files (any dir).
+# Make target for updating generated/golden files (any dir).
+.gen
 .gen-golden
 
 # Build
@@ -49,6 +50,8 @@ site/stats/
 *.tfplan
 *.lock.hcl
 .terraform/
+!coderd/testdata/parameters/modules/.terraform/
+!provisioner/terraform/testdata/modules-source-caching/.terraform/
 
 **/.coderv2/*
 **/__debug_bin
@@ -78,3 +81,8 @@ result
 
 # Zed
 .zed_server
+
+# dlv debug binaries for go tests
+__debug_bin*
+
+**/.claude/settings.local.json

.golangci.yaml

@@ -24,30 +24,19 @@ linters-settings:
     enabled-checks:
       # - appendAssign
       # - appendCombine
-      - argOrder
       # - assignOp
       # - badCall
-      - badCond
       - badLock
       - badRegexp
       - boolExprSimplify
       # - builtinShadow
       - builtinShadowDecl
-      - captLocal
-      - caseOrder
-      - codegenComment
       # - commentedOutCode
       - commentedOutImport
-      - commentFormatting
-      - defaultCaseOrder
       - deferUnlambda
       # - deprecatedComment
       # - docStub
-      - dupArg
-      - dupBranchBody
-      - dupCase
       - dupImport
-      - dupSubExpr
       # - elseif
       - emptyFallthrough
       # - emptyStringTest
@@ -56,8 +45,6 @@ linters-settings:
       # - exitAfterDefer
       # - exposedSyncMutex
       # - filepathJoin
-      - flagDeref
-      - flagName
       - hexLiteral
       # - httpNoBody
       # - hugeParam
@@ -65,47 +52,36 @@ linters-settings:
       # - importShadow
       - indexAlloc
       - initClause
-      - mapKey
       - methodExprCall
       # - nestingReduce
-      - newDeref
       - nilValReturn
       # - octalLiteral
-      - offBy1
       # - paramTypeCombine
       # - preferStringWriter
       # - preferWriteByte
       # - ptrToRefParam
       # - rangeExprCopy
       # - rangeValCopy
-      - regexpMust
       - regexpPattern
       # - regexpSimplify
       - ruleguard
-      - singleCaseSwitch
-      - sloppyLen
       # - sloppyReassign
-      - sloppyTypeAssert
       - sortSlice
       - sprintfQuotedString
       - sqlQuery
       # - stringConcatSimplify
       # - stringXbytes
       # - suspiciousSorting
-      - switchTrue
       - truncateCmp
       - typeAssertChain
       # - typeDefFirst
-      - typeSwitchVar
       # - typeUnparen
-      - underef
       # - unlabelStmt
       # - unlambda
       # - unnamedResult
       # - unnecessaryBlock
       # - unnecessaryDefer
       # - unslice
-      - valSwap
       - weakCond
       # - whyNoLint
       # - wrapperFunc
@@ -188,6 +164,7 @@ linters-settings:
       - name: unnecessary-stmt
       - name: unreachable-code
       - name: unused-parameter
+        exclude: "**/*_test.go"
       - name: unused-receiver
       - name: var-declaration
       - name: var-naming
@@ -203,6 +180,14 @@ linters-settings:
       - G601
 
 issues:
+  exclude-dirs:
+    - coderd/database/dbmem
+    - node_modules
+    - .git
+
+  exclude-files:
+    - scripts/rules.go
+
   # Rules listed here: https://github.com/securego/gosec#available-rules
   exclude-rules:
     - path: _test\.go
@@ -214,17 +199,15 @@ issues:
     - path: scripts/*
       linters:
         - exhaustruct
+    - path: scripts/rules.go
+      linters:
+        - ALL
 
   fix: true
   max-issues-per-linter: 0
   max-same-issues: 0
 
 run:
-  skip-dirs:
-    - node_modules
-    - .git
-  skip-files:
-    - scripts/rules.go
   timeout: 10m
 
 # Over time, add more and more linters from

.vscode/markdown.code-snippets

@@ -1,14 +1,14 @@
 {
 	// For info about snippets, visit https://code.visualstudio.com/docs/editor/userdefinedsnippets
+    // https://docs.github.com/en/get-started/writing-on-github/getting-started-with-writing-and-formatting-on-github/basic-writing-and-formatting-syntax#alerts
 
-    "admonition": {
-        "prefix": "#callout",
+	"alert": {
+        "prefix": "#alert",
         "body": [
-            "<blockquote class=\"admonition ${1|caution,important,note,tip,warning|}\">\n",
-            "${TM_SELECTED_TEXT:${2:add info here}}\n",
-            "</blockquote>\n"
+            "> [!${1|CAUTION,IMPORTANT,NOTE,TIP,WARNING|}]",
+            "> ${TM_SELECTED_TEXT:${2:add info here}}\n"
         ],
-        "description": "callout admonition caution info note tip warning"
+        "description": "callout admonition caution important note tip warning"
     },
 	"fenced code block": {
 		"prefix": "#codeblock",
@@ -23,9 +23,8 @@
 	"premium-feature": {
 		"prefix": "#premium-feature",
 		"body": [
-			"<blockquote class=\"info\">\n",
-			"${1:feature} ${2|is,are|} an Enterprise and Premium feature. [Learn more](https://coder.com/pricing#compare-plans).\n",
-			"</blockquote>"
+			"> [!NOTE]\n",
+			"> ${1:feature} ${2|is,are|} an Enterprise and Premium feature. [Learn more](https://coder.com/pricing#compare-plans).\n"
 		]
 	},
 	"tabs": {

.vscode/settings.json

@@ -57,5 +57,8 @@
 	"[css][html][markdown][yaml]": {
 		"editor.defaultFormatter": "esbenp.prettier-vscode"
 	},
-	"typos.config": ".github/workflows/typos.toml"
+	"typos.config": ".github/workflows/typos.toml",
+	"[markdown]": {
+		"editor.defaultFormatter": "DavidAnson.vscode-markdownlint"
+	}
 }

CLAUDE.md

@@ -0,0 +1,104 @@
+# Coder Development Guidelines
+
+Read [cursor rules](.cursorrules).
+
+## Build/Test/Lint Commands
+
+### Main Commands
+
+- `make build` or `make build-fat` - Build all "fat" binaries (includes "server" functionality)
+- `make build-slim` - Build "slim" binaries
+- `make test` - Run Go tests
+- `make test RUN=TestFunctionName` or `go test -v ./path/to/package -run TestFunctionName` - Test single
+- `make test-postgres` - Run tests with Postgres database
+- `make test-race` - Run tests with Go race detector
+- `make test-e2e` - Run end-to-end tests
+- `make lint` - Run all linters
+- `make fmt` - Format all code
+- `make gen` - Generates mocks, database queries and other auto-generated files
+
+### Frontend Commands (site directory)
+
+- `pnpm build` - Build frontend
+- `pnpm dev` - Run development server
+- `pnpm check` - Run code checks
+- `pnpm format` - Format frontend code
+- `pnpm lint` - Lint frontend code
+- `pnpm test` - Run frontend tests
+
+## Code Style Guidelines
+
+### Go
+
+- Follow [Effective Go](https://go.dev/doc/effective_go) and [Go's Code Review Comments](https://github.com/golang/go/wiki/CodeReviewComments)
+- Use `gofumpt` for formatting
+- Create packages when used during implementation
+- Validate abstractions against implementations
+
+### Error Handling
+
+- Use descriptive error messages
+- Wrap errors with context
+- Propagate errors appropriately
+- Use proper error types
+- (`xerrors.Errorf("failed to X: %w", err)`)
+
+### Naming
+
+- Use clear, descriptive names
+- Abbreviate only when obvious
+- Follow Go and TypeScript naming conventions
+
+### Comments
+
+- Document exported functions, types, and non-obvious logic
+- Follow JSDoc format for TypeScript
+- Use godoc format for Go code
+
+## Commit Style
+
+- Follow [Conventional Commits 1.0.0](https://www.conventionalcommits.org/en/v1.0.0/)
+- Format: `type(scope): message`
+- Types: `feat`, `fix`, `docs`, `style`, `refactor`, `test`, `chore`
+- Keep message titles concise (~70 characters)
+- Use imperative, present tense in commit titles
+
+## Database queries
+
+- MUST DO! Any changes to database - adding queries, modifying queries should be done in the  `coderd\database\queries\*.sql` files. Use `make gen` to generate necessary changes after.
+- MUST DO! Queries are grouped in files relating to context - e.g. `prebuilds.sql`, `users.sql`, `provisionerjobs.sql`.
+- After making changes to any `coderd\database\queries\*.sql` files you must run `make gen` to generate respective ORM changes.
+
+## Architecture
+
+### Core Components
+
+- **coderd**: Main API service connecting workspaces, provisioners, and users
+- **provisionerd**: Execution context for infrastructure-modifying providers
+- **Agents**: Services in remote workspaces providing features like SSH and port forwarding
+- **Workspaces**: Cloud resources defined by Terraform
+
+## Sub-modules
+
+### Template System
+
+- Templates define infrastructure for workspaces using Terraform
+- Environment variables pass context between Coder and templates
+- Official modules extend development environments
+
+### RBAC System
+
+- Permissions defined at site, organization, and user levels
+- Object-Action model protects resources
+- Built-in roles: owner, member, auditor, templateAdmin
+- Permission format: `<sign>?<level>.<object>.<id>.<action>`
+
+### Database
+
+- PostgreSQL 13+ recommended for production
+- Migrations managed with `migrate`
+- Database authorization through `dbauthz` package
+
+## Frontend
+
+For building Frontend refer to [this document](docs/contributing/frontend.md)

CODEOWNERS

@@ -4,3 +4,5 @@ agent/proto/ @spikecurtis @johnstcn
 tailnet/proto/ @spikecurtis @johnstcn
 vpn/vpn.proto @spikecurtis @johnstcn
 vpn/version.go @spikecurtis @johnstcn
+provisionerd/proto/ @spikecurtis @johnstcn
+provisionersdk/proto/ @spikecurtis @johnstcn

Makefile

@@ -54,6 +54,16 @@ FIND_EXCLUSIONS= \
 	-not \( \( -path '*/.git/*' -o -path './build/*' -o -path './vendor/*' -o -path './.coderv2/*' -o -path '*/node_modules/*' -o -path '*/out/*' -o -path './coderd/apidoc/*' -o -path '*/.next/*' -o -path '*/.terraform/*' \) -prune \)
 # Source files used for make targets, evaluated on use.
 GO_SRC_FILES := $(shell find . $(FIND_EXCLUSIONS) -type f -name '*.go' -not -name '*_test.go')
+# Same as GO_SRC_FILES but excluding certain files that have problematic
+# Makefile dependencies (e.g. pnpm).
+MOST_GO_SRC_FILES := $(shell \
+	find . \
+		$(FIND_EXCLUSIONS) \
+		-type f \
+		-name '*.go' \
+		-not -name '*_test.go' \
+		-not -wholename './agent/agentcontainers/dcspec/dcspec_gen.go' \
+)
 # All the shell files in the repo, excluding ignored files.
 SHELL_SRC_FILES := $(shell find . $(FIND_EXCLUSIONS) -type f -name '*.sh')
 
@@ -240,10 +250,14 @@ $(CODER_ALL_BINARIES): go.mod go.sum \
 		fi
 
 		cp "$@" "./site/out/bin/coder-$$os-$$arch$$dot_ext"
+
+		if [[ "$${CODER_SIGN_GPG:-0}" == "1" ]]; then
+			cp "$@.asc" "./site/out/bin/coder-$$os-$$arch$$dot_ext.asc"
+		fi
 	fi
 
 # This task builds Coder Desktop dylibs
-$(CODER_DYLIBS): go.mod go.sum $(GO_SRC_FILES)
+$(CODER_DYLIBS): go.mod go.sum $(MOST_GO_SRC_FILES)
 	@if [ "$(shell uname)" = "Darwin" ]; then
 		$(get-mode-os-arch-ext)
 		./scripts/build_go.sh \
@@ -388,16 +402,21 @@ $(foreach chart,$(charts),build/$(chart)_helm_$(VERSION).tgz): build/%_helm_$(VE
 		--chart $* \
 		--output "$@"
 
-node_modules/.installed: package.json
+node_modules/.installed: package.json pnpm-lock.yaml
 	./scripts/pnpm_install.sh
+	touch "$@"
 
-offlinedocs/node_modules/.installed: offlinedocs/package.json
-	cd offlinedocs/
-	../scripts/pnpm_install.sh
+offlinedocs/node_modules/.installed: offlinedocs/package.json offlinedocs/pnpm-lock.yaml
+	(cd offlinedocs/ && ../scripts/pnpm_install.sh)
+	touch "$@"
 
-site/node_modules/.installed: site/package.json
-	cd site/
-	../scripts/pnpm_install.sh
+site/node_modules/.installed: site/package.json site/pnpm-lock.yaml
+	(cd site/ && ../scripts/pnpm_install.sh)
+	touch "$@"
+
+scripts/apidocgen/node_modules/.installed: scripts/apidocgen/package.json scripts/apidocgen/pnpm-lock.yaml
+	(cd scripts/apidocgen && ../../scripts/pnpm_install.sh)
+	touch "$@"
 
 SITE_GEN_FILES := \
 	site/src/api/typesGenerated.ts \
@@ -505,7 +524,7 @@ lint/ts: site/node_modules/.installed
 lint/go:
 	./scripts/check_enterprise_imports.sh
 	./scripts/check_codersdk_imports.sh
-	linter_ver=$(shell egrep -o 'GOLANGCI_LINT_VERSION=\S+' dogfood/contents/Dockerfile | cut -d '=' -f 2)
+	linter_ver=$(shell egrep -o 'GOLANGCI_LINT_VERSION=\S+' dogfood/coder/Dockerfile | cut -d '=' -f 2)
 	go run github.com/golangci/golangci-lint/cmd/golangci-lint@v$$linter_ver run
 .PHONY: lint/go
 
@@ -559,21 +578,35 @@ GEN_FILES := \
 	docs/reference/cli/index.md \
 	docs/admin/security/audit-logs.md \
 	coderd/apidoc/swagger.json \
+	docs/manifest.json \
 	provisioner/terraform/testdata/version \
 	site/e2e/provisionerGenerated.ts \
 	examples/examples.gen.json \
 	$(TAILNETTEST_MOCKS) \
 	coderd/database/pubsub/psmock/psmock.go \
-	agent/agentcontainers/acmock/acmock.go
-
+	agent/agentcontainers/acmock/acmock.go \
+	agent/agentcontainers/dcspec/dcspec_gen.go \
+	coderd/httpmw/loggermw/loggermock/loggermock.go
 
 # all gen targets should be added here and to gen/mark-fresh
-gen: gen/db $(GEN_FILES)
+gen: gen/db gen/golden-files $(GEN_FILES)
 .PHONY: gen
 
 gen/db: $(DB_GEN_FILES)
 .PHONY: gen/db
 
+gen/golden-files: \
+	cli/testdata/.gen-golden \
+	coderd/.gen-golden \
+	coderd/notifications/.gen-golden \
+	enterprise/cli/testdata/.gen-golden \
+	enterprise/tailnet/testdata/.gen-golden \
+	helm/coder/tests/testdata/.gen-golden \
+	helm/provisioner/tests/testdata/.gen-golden \
+	provisioner/terraform/testdata/.gen-golden \
+	tailnet/testdata/.gen-golden
+.PHONY: gen/golden-files
+
 # Mark all generated files as fresh so make thinks they're up-to-date. This is
 # used during releases so we don't run generation scripts.
 gen/mark-fresh:
@@ -594,12 +627,15 @@ gen/mark-fresh:
 		docs/reference/cli/index.md \
 		docs/admin/security/audit-logs.md \
 		coderd/apidoc/swagger.json \
+		docs/manifest.json \
 		site/e2e/provisionerGenerated.ts \
 		site/src/theme/icons.json \
 		examples/examples.gen.json \
 		$(TAILNETTEST_MOCKS) \
 		coderd/database/pubsub/psmock/psmock.go \
 		agent/agentcontainers/acmock/acmock.go \
+		agent/agentcontainers/dcspec/dcspec_gen.go \
+		coderd/httpmw/loggermw/loggermock/loggermock.go \
 		"
 
 	for file in $$files; do
@@ -618,24 +654,42 @@ gen/mark-fresh:
 # applied.
 coderd/database/dump.sql: coderd/database/gen/dump/main.go $(wildcard coderd/database/migrations/*.sql)
 	go run ./coderd/database/gen/dump/main.go
+	touch "$@"
 
 # Generates Go code for querying the database.
 # coderd/database/queries.sql.go
 # coderd/database/models.go
 coderd/database/querier.go: coderd/database/sqlc.yaml coderd/database/dump.sql $(wildcard coderd/database/queries/*.sql)
 	./coderd/database/generate.sh
+	touch "$@"
 
 coderd/database/dbmock/dbmock.go: coderd/database/db.go coderd/database/querier.go
 	go generate ./coderd/database/dbmock/
+	touch "$@"
 
 coderd/database/pubsub/psmock/psmock.go: coderd/database/pubsub/pubsub.go
 	go generate ./coderd/database/pubsub/psmock
+	touch "$@"
 
 agent/agentcontainers/acmock/acmock.go: agent/agentcontainers/containers.go
 	go generate ./agent/agentcontainers/acmock/
+	touch "$@"
+
+coderd/httpmw/loggermw/loggermock/loggermock.go: coderd/httpmw/loggermw/logger.go
+	go generate ./coderd/httpmw/loggermw/loggermock/
+	touch "$@"
+
+agent/agentcontainers/dcspec/dcspec_gen.go: \
+	node_modules/.installed \
+	agent/agentcontainers/dcspec/devContainer.base.schema.json \
+	agent/agentcontainers/dcspec/gen.sh \
+	agent/agentcontainers/dcspec/doc.go
+	DCSPEC_QUIET=true go generate ./agent/agentcontainers/dcspec/
+	touch "$@"
 
 $(TAILNETTEST_MOCKS): tailnet/coordinator.go tailnet/service.go
 	go generate ./tailnet/tailnettest/
+	touch "$@"
 
 tailnet/proto/tailnet.pb.go: tailnet/proto/tailnet.proto
 	protoc \
@@ -678,77 +732,94 @@ vpn/vpn.pb.go: vpn/vpn.proto
 site/src/api/typesGenerated.ts: site/node_modules/.installed $(wildcard scripts/apitypings/*) $(shell find ./codersdk $(FIND_EXCLUSIONS) -type f -name '*.go')
 	# -C sets the directory for the go run command
 	go run -C ./scripts/apitypings main.go > $@
-	cd site/
-	pnpm exec biome format --write src/api/typesGenerated.ts
+	(cd site/ && pnpm exec biome format --write src/api/typesGenerated.ts)
+	touch "$@"
 
 site/e2e/provisionerGenerated.ts: site/node_modules/.installed provisionerd/proto/provisionerd.pb.go provisionersdk/proto/provisioner.pb.go
-	cd site/
-	pnpm run gen:provisioner
+	(cd site/ && pnpm run gen:provisioner)
+	touch "$@"
 
 site/src/theme/icons.json: site/node_modules/.installed $(wildcard scripts/gensite/*) $(wildcard site/static/icon/*)
 	go run ./scripts/gensite/ -icons "$@"
-	cd site/
-	pnpm exec biome format --write src/theme/icons.json
+	(cd site/ && pnpm exec biome format --write src/theme/icons.json)
+	touch "$@"
 
 examples/examples.gen.json: scripts/examplegen/main.go examples/examples.go $(shell find ./examples/templates)
 	go run ./scripts/examplegen/main.go > examples/examples.gen.json
+	touch "$@"
 
 coderd/rbac/object_gen.go: scripts/typegen/rbacobject.gotmpl scripts/typegen/main.go coderd/rbac/object.go coderd/rbac/policy/policy.go
 	tempdir=$(shell mktemp -d /tmp/typegen_rbac_object.XXXXXX)
 	go run ./scripts/typegen/main.go rbac object > "$$tempdir/object_gen.go"
 	mv -v "$$tempdir/object_gen.go" coderd/rbac/object_gen.go
 	rmdir -v "$$tempdir"
+	touch "$@"
 
 codersdk/rbacresources_gen.go: scripts/typegen/codersdk.gotmpl scripts/typegen/main.go coderd/rbac/object.go coderd/rbac/policy/policy.go
 	# Do no overwrite codersdk/rbacresources_gen.go directly, as it would make the file empty, breaking
  	# the `codersdk` package and any parallel build targets.
 	go run scripts/typegen/main.go rbac codersdk > /tmp/rbacresources_gen.go
 	mv /tmp/rbacresources_gen.go codersdk/rbacresources_gen.go
+	touch "$@"
 
 site/src/api/rbacresourcesGenerated.ts: site/node_modules/.installed scripts/typegen/codersdk.gotmpl scripts/typegen/main.go coderd/rbac/object.go coderd/rbac/policy/policy.go
 	go run scripts/typegen/main.go rbac typescript > "$@"
-	cd site/
-	pnpm exec biome format --write src/api/rbacresourcesGenerated.ts
+	(cd site/ && pnpm exec biome format --write src/api/rbacresourcesGenerated.ts)
+	touch "$@"
 
 site/src/api/countriesGenerated.ts: site/node_modules/.installed scripts/typegen/countries.tstmpl scripts/typegen/main.go codersdk/countries.go
 	go run scripts/typegen/main.go countries > "$@"
-	cd site/
-	pnpm exec biome format --write src/api/countriesGenerated.ts
+	(cd site/ && pnpm exec biome format --write src/api/countriesGenerated.ts)
+	touch "$@"
 
 docs/admin/integrations/prometheus.md: node_modules/.installed scripts/metricsdocgen/main.go scripts/metricsdocgen/metrics
 	go run scripts/metricsdocgen/main.go
 	pnpm exec markdownlint-cli2 --fix ./docs/admin/integrations/prometheus.md
 	pnpm exec markdown-table-formatter ./docs/admin/integrations/prometheus.md
+	touch "$@"
 
-docs/reference/cli/index.md: node_modules/.installed site/node_modules/.installed scripts/clidocgen/main.go examples/examples.gen.json $(GO_SRC_FILES)
+docs/reference/cli/index.md: node_modules/.installed scripts/clidocgen/main.go examples/examples.gen.json $(GO_SRC_FILES)
 	CI=true BASE_PATH="." go run ./scripts/clidocgen
 	pnpm exec markdownlint-cli2 --fix ./docs/reference/cli/*.md
 	pnpm exec markdown-table-formatter ./docs/reference/cli/*.md
-	cd site/
-	pnpm exec biome format --write ../docs/manifest.json
+	touch "$@"
 
 docs/admin/security/audit-logs.md: node_modules/.installed coderd/database/querier.go scripts/auditdocgen/main.go enterprise/audit/table.go coderd/rbac/object_gen.go
 	go run scripts/auditdocgen/main.go
 	pnpm exec markdownlint-cli2 --fix ./docs/admin/security/audit-logs.md
 	pnpm exec markdown-table-formatter ./docs/admin/security/audit-logs.md
+	touch "$@"
 
-coderd/apidoc/swagger.json: node_modules/.installed site/node_modules/.installed $(shell find ./scripts/apidocgen $(FIND_EXCLUSIONS) -type f) $(wildcard coderd/*.go) $(wildcard enterprise/coderd/*.go) $(wildcard codersdk/*.go) $(wildcard enterprise/wsproxy/wsproxysdk/*.go) $(DB_GEN_FILES) .swaggo docs/manifest.json coderd/rbac/object_gen.go
+coderd/apidoc/.gen: \
+	node_modules/.installed \
+	scripts/apidocgen/node_modules/.installed \
+	$(wildcard coderd/*.go) \
+	$(wildcard enterprise/coderd/*.go) \
+	$(wildcard codersdk/*.go) \
+	$(wildcard enterprise/wsproxy/wsproxysdk/*.go) \
+	$(DB_GEN_FILES) \
+	coderd/rbac/object_gen.go \
+	.swaggo \
+	scripts/apidocgen/generate.sh \
+	$(wildcard scripts/apidocgen/postprocess/*) \
+	$(wildcard scripts/apidocgen/markdown-template/*)
 	./scripts/apidocgen/generate.sh
 	pnpm exec markdownlint-cli2 --fix ./docs/reference/api/*.md
 	pnpm exec markdown-table-formatter ./docs/reference/api/*.md
-	cd site/
-	pnpm exec biome format --write ../docs/manifest.json ../coderd/apidoc/swagger.json
+	touch "$@"
 
-update-golden-files: \
-	cli/testdata/.gen-golden \
-	coderd/.gen-golden \
-	coderd/notifications/.gen-golden \
-	enterprise/cli/testdata/.gen-golden \
-	enterprise/tailnet/testdata/.gen-golden \
-	helm/coder/tests/testdata/.gen-golden \
-	helm/provisioner/tests/testdata/.gen-golden \
-	provisioner/terraform/testdata/.gen-golden \
-	tailnet/testdata/.gen-golden
+docs/manifest.json: site/node_modules/.installed coderd/apidoc/.gen docs/reference/cli/index.md
+	(cd site/ && pnpm exec biome format --write ../docs/manifest.json)
+	touch "$@"
+
+coderd/apidoc/swagger.json: site/node_modules/.installed coderd/apidoc/.gen
+	(cd site/ && pnpm exec biome format --write ../coderd/apidoc/swagger.json)
+	touch "$@"
+
+update-golden-files:
+	echo 'WARNING: This target is deprecated. Use "make gen/golden-files" instead.' >&2
+	echo 'Running "make gen/golden-files"' >&2
+	make gen/golden-files
 .PHONY: update-golden-files
 
 clean/golden-files:
@@ -767,39 +838,39 @@ clean/golden-files:
 .PHONY: clean/golden-files
 
 cli/testdata/.gen-golden: $(wildcard cli/testdata/*.golden) $(wildcard cli/*.tpl) $(GO_SRC_FILES) $(wildcard cli/*_test.go)
-	go test ./cli -run="Test(CommandHelp|ServerYAML|ErrorExamples|.*Golden)" -update
+	TZ=UTC go test ./cli -run="Test(CommandHelp|ServerYAML|ErrorExamples|.*Golden)" -update
 	touch "$@"
 
 enterprise/cli/testdata/.gen-golden: $(wildcard enterprise/cli/testdata/*.golden) $(wildcard cli/*.tpl) $(GO_SRC_FILES) $(wildcard enterprise/cli/*_test.go)
-	go test ./enterprise/cli -run="TestEnterpriseCommandHelp" -update
+	TZ=UTC go test ./enterprise/cli -run="TestEnterpriseCommandHelp" -update
 	touch "$@"
 
 tailnet/testdata/.gen-golden: $(wildcard tailnet/testdata/*.golden.html) $(GO_SRC_FILES) $(wildcard tailnet/*_test.go)
-	go test ./tailnet -run="TestDebugTemplate" -update
+	TZ=UTC go test ./tailnet -run="TestDebugTemplate" -update
 	touch "$@"
 
 enterprise/tailnet/testdata/.gen-golden: $(wildcard enterprise/tailnet/testdata/*.golden.html) $(GO_SRC_FILES) $(wildcard enterprise/tailnet/*_test.go)
-	go test ./enterprise/tailnet -run="TestDebugTemplate" -update
+	TZ=UTC go test ./enterprise/tailnet -run="TestDebugTemplate" -update
 	touch "$@"
 
 helm/coder/tests/testdata/.gen-golden: $(wildcard helm/coder/tests/testdata/*.yaml) $(wildcard helm/coder/tests/testdata/*.golden) $(GO_SRC_FILES) $(wildcard helm/coder/tests/*_test.go)
-	go test ./helm/coder/tests -run=TestUpdateGoldenFiles -update
+	TZ=UTC go test ./helm/coder/tests -run=TestUpdateGoldenFiles -update
 	touch "$@"
 
 helm/provisioner/tests/testdata/.gen-golden: $(wildcard helm/provisioner/tests/testdata/*.yaml) $(wildcard helm/provisioner/tests/testdata/*.golden) $(GO_SRC_FILES) $(wildcard helm/provisioner/tests/*_test.go)
-	go test ./helm/provisioner/tests -run=TestUpdateGoldenFiles -update
+	TZ=UTC go test ./helm/provisioner/tests -run=TestUpdateGoldenFiles -update
 	touch "$@"
 
 coderd/.gen-golden: $(wildcard coderd/testdata/*/*.golden) $(GO_SRC_FILES) $(wildcard coderd/*_test.go)
-	go test ./coderd -run="Test.*Golden$$" -update
+	TZ=UTC go test ./coderd -run="Test.*Golden$$" -update
 	touch "$@"
 
 coderd/notifications/.gen-golden: $(wildcard coderd/notifications/testdata/*/*.golden) $(GO_SRC_FILES) $(wildcard coderd/notifications/*_test.go)
-	go test ./coderd/notifications -run="Test.*Golden$$" -update
+	TZ=UTC go test ./coderd/notifications -run="Test.*Golden$$" -update
 	touch "$@"
 
 provisioner/terraform/testdata/.gen-golden: $(wildcard provisioner/terraform/testdata/*/*.golden) $(GO_SRC_FILES) $(wildcard provisioner/terraform/*_test.go)
-	go test ./provisioner/terraform -run="Test.*Golden$$" -update
+	TZ=UTC go test ./provisioner/terraform -run="Test.*Golden$$" -update
 	touch "$@"
 
 provisioner/terraform/testdata/version:
@@ -808,12 +879,19 @@ provisioner/terraform/testdata/version:
 	fi
 .PHONY: provisioner/terraform/testdata/version
 
+# Set the retry flags if TEST_RETRIES is set
+ifdef TEST_RETRIES
+GOTESTSUM_RETRY_FLAGS := --rerun-fails=$(TEST_RETRIES)
+else
+GOTESTSUM_RETRY_FLAGS :=
+endif
+
 test:
-	$(GIT_FLAGS) gotestsum --format standard-quiet -- -v -short -count=1 ./... $(if $(RUN),-run $(RUN))
+	$(GIT_FLAGS) gotestsum --format standard-quiet $(GOTESTSUM_RETRY_FLAGS) --packages="./..." -- -v -short -count=1 $(if $(RUN),-run $(RUN))
 .PHONY: test
 
 test-cli:
-	$(GIT_FLAGS) gotestsum --format standard-quiet -- -v -short -count=1 ./cli/...
+	$(GIT_FLAGS) gotestsum --format standard-quiet $(GOTESTSUM_RETRY_FLAGS) --packages="./cli/..." -- -v -short -count=1
 .PHONY: test-cli
 
 # sqlc-cloud-is-setup will fail if no SQLc auth token is set. Use this as a
@@ -852,9 +930,9 @@ test-postgres: test-postgres-docker
 	$(GIT_FLAGS)  DB=ci gotestsum \
 		--junitfile="gotests.xml" \
 		--jsonfile="gotests.json" \
+		$(GOTESTSUM_RETRY_FLAGS) \
 		--packages="./..." -- \
 		-timeout=20m \
-		-failfast \
 		-count=1
 .PHONY: test-postgres
 
@@ -963,5 +1041,5 @@ else
 endif
 .PHONY: test-e2e
 
-dogfood/contents/nix.hash: flake.nix flake.lock
-	sha256sum flake.nix flake.lock >./dogfood/contents/nix.hash
+dogfood/coder/nix.hash: flake.nix flake.lock
+	sha256sum flake.nix flake.lock >./dogfood/coder/nix.hash

agent/agent.go

@@ -8,11 +8,13 @@ import (
 	"fmt"
 	"hash/fnv"
 	"io"
+	"net"
 	"net/http"
 	"net/netip"
 	"os"
 	"os/user"
 	"path/filepath"
+	"slices"
 	"sort"
 	"strconv"
 	"strings"
@@ -25,15 +27,16 @@ import (
 	"github.com/prometheus/common/expfmt"
 	"github.com/spf13/afero"
 	"go.uber.org/atomic"
-	"golang.org/x/exp/slices"
 	"golang.org/x/sync/errgroup"
 	"golang.org/x/xerrors"
+	"google.golang.org/protobuf/types/known/timestamppb"
 	"tailscale.com/net/speedtest"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/netlogtype"
 	"tailscale.com/util/clientmetric"
 
 	"cdr.dev/slog"
+	"github.com/coder/clistat"
 	"github.com/coder/coder/v2/agent/agentcontainers"
 	"github.com/coder/coder/v2/agent/agentexec"
 	"github.com/coder/coder/v2/agent/agentscripts"
@@ -42,7 +45,6 @@ import (
 	"github.com/coder/coder/v2/agent/proto/resourcesmonitor"
 	"github.com/coder/coder/v2/agent/reconnectingpty"
 	"github.com/coder/coder/v2/buildinfo"
-	"github.com/coder/coder/v2/cli/clistat"
 	"github.com/coder/coder/v2/cli/gitauth"
 	"github.com/coder/coder/v2/coderd/database/dbtime"
 	"github.com/coder/coder/v2/codersdk"
@@ -87,12 +89,14 @@ type Options struct {
 	ServiceBannerRefreshInterval time.Duration
 	BlockFileTransfer            bool
 	Execer                       agentexec.Execer
-	ContainerLister              agentcontainers.Lister
+
+	ExperimentalDevcontainersEnabled bool
+	ContainerAPIOptions              []agentcontainers.Option // Enable ExperimentalDevcontainersEnabled for these to be effective.
 }
 
 type Client interface {
-	ConnectRPC24(ctx context.Context) (
-		proto.DRPCAgentClient24, tailnetproto.DRPCTailnetClient24, error,
+	ConnectRPC25(ctx context.Context) (
+		proto.DRPCAgentClient25, tailnetproto.DRPCTailnetClient25, error,
 	)
 	RewriteDERPMap(derpMap *tailcfg.DERPMap)
 }
@@ -150,9 +154,6 @@ func New(options Options) Agent {
 	if options.Execer == nil {
 		options.Execer = agentexec.DefaultExecer
 	}
-	if options.ContainerLister == nil {
-		options.ContainerLister = agentcontainers.NewDocker(options.Execer)
-	}
 
 	hardCtx, hardCancel := context.WithCancel(context.Background())
 	gracefulCtx, gracefulCancel := context.WithCancel(hardCtx)
@@ -175,6 +176,7 @@ func New(options Options) Agent {
 		lifecycleUpdate:                    make(chan struct{}, 1),
 		lifecycleReported:                  make(chan codersdk.WorkspaceAgentLifecycle, 1),
 		lifecycleStates:                    []agentsdk.PostLifecycleRequest{{State: codersdk.WorkspaceAgentLifecycleCreated}},
+		reportConnectionsUpdate:            make(chan struct{}, 1),
 		ignorePorts:                        options.IgnorePorts,
 		portCacheDuration:                  options.PortCacheDuration,
 		reportMetadataInterval:             options.ReportMetadataInterval,
@@ -187,7 +189,9 @@ func New(options Options) Agent {
 		prometheusRegistry: prometheusRegistry,
 		metrics:            newAgentMetrics(prometheusRegistry),
 		execer:             options.Execer,
-		lister:             options.ContainerLister,
+
+		experimentalDevcontainersEnabled: options.ExperimentalDevcontainersEnabled,
+		containerAPIOptions:              options.ContainerAPIOptions,
 	}
 	// Initially, we have a closed channel, reflecting the fact that we are not initially connected.
 	// Each time we connect we replace the channel (while holding the closeMutex) with a new one
@@ -222,13 +226,21 @@ type agent struct {
 	// we track 2 contexts and associated cancel functions: "graceful" which is Done when it is time
 	// to start gracefully shutting down and "hard" which is Done when it is time to close
 	// everything down (regardless of whether graceful shutdown completed).
-	gracefulCtx       context.Context
-	gracefulCancel    context.CancelFunc
-	hardCtx           context.Context
-	hardCancel        context.CancelFunc
-	closeWaitGroup    sync.WaitGroup
+	gracefulCtx    context.Context
+	gracefulCancel context.CancelFunc
+	hardCtx        context.Context
+	hardCancel     context.CancelFunc
+
+	// closeMutex protects the following:
 	closeMutex        sync.Mutex
+	closeWaitGroup    sync.WaitGroup
 	coordDisconnected chan struct{}
+	closing           bool
+	// note that once the network is set to non-nil, it is never modified, as with the statsReporter. So, routines
+	// that run after createOrUpdateNetwork and check the networkOK checkpoint do not need to hold the lock to use them.
+	network       *tailnet.Conn
+	statsReporter *statsReporter
+	// end fields protected by closeMutex
 
 	environmentVariables map[string]string
 
@@ -248,19 +260,26 @@ type agent struct {
 	lifecycleStates            []agentsdk.PostLifecycleRequest
 	lifecycleLastReportedIndex int // Keeps track of the last lifecycle state we successfully reported.
 
-	network       *tailnet.Conn
-	statsReporter *statsReporter
-	logSender     *agentsdk.LogSender
+	reportConnectionsUpdate chan struct{}
+	reportConnectionsMu     sync.Mutex
+	reportConnections       []*proto.ReportConnectionRequest
+
+	logSender *agentsdk.LogSender
 
 	prometheusRegistry *prometheus.Registry
 	// metrics are prometheus registered metrics that will be collected and
 	// labeled in Coder with the agent + workspace.
 	metrics *agentMetrics
 	execer  agentexec.Execer
-	lister  agentcontainers.Lister
+
+	experimentalDevcontainersEnabled bool
+	containerAPIOptions              []agentcontainers.Option
+	containerAPI                     atomic.Pointer[agentcontainers.API] // Set by apiHandler.
 }
 
 func (a *agent) TailnetConn() *tailnet.Conn {
+	a.closeMutex.Lock()
+	defer a.closeMutex.Unlock()
 	return a.network
 }
 
@@ -273,6 +292,26 @@ func (a *agent) init() {
 		UpdateEnv:           a.updateCommandEnv,
 		WorkingDirectory:    func() string { return a.manifest.Load().Directory },
 		BlockFileTransfer:   a.blockFileTransfer,
+		ReportConnection: func(id uuid.UUID, magicType agentssh.MagicSessionType, ip string) func(code int, reason string) {
+			var connectionType proto.Connection_Type
+			switch magicType {
+			case agentssh.MagicSessionTypeSSH:
+				connectionType = proto.Connection_SSH
+			case agentssh.MagicSessionTypeVSCode:
+				connectionType = proto.Connection_VSCODE
+			case agentssh.MagicSessionTypeJetBrains:
+				connectionType = proto.Connection_JETBRAINS
+			case agentssh.MagicSessionTypeUnknown:
+				connectionType = proto.Connection_TYPE_UNSPECIFIED
+			default:
+				a.logger.Error(a.hardCtx, "unhandled magic session type when reporting connection", slog.F("magic_type", magicType))
+				connectionType = proto.Connection_TYPE_UNSPECIFIED
+			}
+
+			return a.reportConnection(id, connectionType, ip)
+		},
+
+		ExperimentalDevContainersEnabled: a.experimentalDevcontainersEnabled,
 	})
 	if err != nil {
 		panic(err)
@@ -295,8 +334,14 @@ func (a *agent) init() {
 	a.reconnectingPTYServer = reconnectingpty.NewServer(
 		a.logger.Named("reconnecting-pty"),
 		a.sshServer,
+		func(id uuid.UUID, ip string) func(code int, reason string) {
+			return a.reportConnection(id, proto.Connection_RECONNECTING_PTY, ip)
+		},
 		a.metrics.connectionsTotal, a.metrics.reconnectingPTYErrors,
 		a.reconnectingPTYTimeout,
+		func(s *reconnectingpty.Server) {
+			s.ExperimentalDevcontainersEnabled = a.experimentalDevcontainersEnabled
+		},
 	)
 	go a.runLoop()
 }
@@ -318,9 +363,11 @@ func (a *agent) runLoop() {
 		if ctx.Err() != nil {
 			// Context canceled errors may come from websocket pings, so we
 			// don't want to use `errors.Is(err, context.Canceled)` here.
+			a.logger.Warn(ctx, "runLoop exited with error", slog.Error(ctx.Err()))
 			return
 		}
 		if a.isClosed() {
+			a.logger.Warn(ctx, "runLoop exited because agent is closed")
 			return
 		}
 		if errors.Is(err, io.EOF) {
@@ -704,6 +751,124 @@ func (a *agent) setLifecycle(state codersdk.WorkspaceAgentLifecycle) {
 	}
 }
 
+// reportConnectionsLoop reports connections to the agent for auditing.
+func (a *agent) reportConnectionsLoop(ctx context.Context, aAPI proto.DRPCAgentClient24) error {
+	for {
+		select {
+		case <-a.reportConnectionsUpdate:
+		case <-ctx.Done():
+			return ctx.Err()
+		}
+
+		for {
+			a.reportConnectionsMu.Lock()
+			if len(a.reportConnections) == 0 {
+				a.reportConnectionsMu.Unlock()
+				break
+			}
+			payload := a.reportConnections[0]
+			// Release lock while we send the payload, this is safe
+			// since we only append to the slice.
+			a.reportConnectionsMu.Unlock()
+
+			logger := a.logger.With(slog.F("payload", payload))
+			logger.Debug(ctx, "reporting connection")
+			_, err := aAPI.ReportConnection(ctx, payload)
+			if err != nil {
+				return xerrors.Errorf("failed to report connection: %w", err)
+			}
+
+			logger.Debug(ctx, "successfully reported connection")
+
+			// Remove the payload we sent.
+			a.reportConnectionsMu.Lock()
+			a.reportConnections[0] = nil // Release the pointer from the underlying array.
+			a.reportConnections = a.reportConnections[1:]
+			a.reportConnectionsMu.Unlock()
+		}
+	}
+}
+
+const (
+	// reportConnectionBufferLimit limits the number of connection reports we
+	// buffer to avoid growing the buffer indefinitely. This should not happen
+	// unless the agent has lost connection to coderd for a long time or if
+	// the agent is being spammed with connections.
+	//
+	// If we assume ~150 byte per connection report, this would be around 300KB
+	// of memory which seems acceptable. We could reduce this if necessary by
+	// not using the proto struct directly.
+	reportConnectionBufferLimit = 2048
+)
+
+func (a *agent) reportConnection(id uuid.UUID, connectionType proto.Connection_Type, ip string) (disconnected func(code int, reason string)) {
+	// Remove the port from the IP because ports are not supported in coderd.
+	if host, _, err := net.SplitHostPort(ip); err != nil {
+		a.logger.Error(a.hardCtx, "split host and port for connection report failed", slog.F("ip", ip), slog.Error(err))
+	} else {
+		// Best effort.
+		ip = host
+	}
+
+	a.reportConnectionsMu.Lock()
+	defer a.reportConnectionsMu.Unlock()
+
+	if len(a.reportConnections) >= reportConnectionBufferLimit {
+		a.logger.Warn(a.hardCtx, "connection report buffer limit reached, dropping connect",
+			slog.F("limit", reportConnectionBufferLimit),
+			slog.F("connection_id", id),
+			slog.F("connection_type", connectionType),
+			slog.F("ip", ip),
+		)
+	} else {
+		a.reportConnections = append(a.reportConnections, &proto.ReportConnectionRequest{
+			Connection: &proto.Connection{
+				Id:         id[:],
+				Action:     proto.Connection_CONNECT,
+				Type:       connectionType,
+				Timestamp:  timestamppb.New(time.Now()),
+				Ip:         ip,
+				StatusCode: 0,
+				Reason:     nil,
+			},
+		})
+		select {
+		case a.reportConnectionsUpdate <- struct{}{}:
+		default:
+		}
+	}
+
+	return func(code int, reason string) {
+		a.reportConnectionsMu.Lock()
+		defer a.reportConnectionsMu.Unlock()
+		if len(a.reportConnections) >= reportConnectionBufferLimit {
+			a.logger.Warn(a.hardCtx, "connection report buffer limit reached, dropping disconnect",
+				slog.F("limit", reportConnectionBufferLimit),
+				slog.F("connection_id", id),
+				slog.F("connection_type", connectionType),
+				slog.F("ip", ip),
+			)
+			return
+		}
+
+		a.reportConnections = append(a.reportConnections, &proto.ReportConnectionRequest{
+			Connection: &proto.Connection{
+				Id:         id[:],
+				Action:     proto.Connection_DISCONNECT,
+				Type:       connectionType,
+				Timestamp:  timestamppb.New(time.Now()),
+				Ip:         ip,
+				StatusCode: int32(code), //nolint:gosec
+				Reason:     &reason,
+			},
+		})
+		select {
+		case a.reportConnectionsUpdate <- struct{}{}:
+		default:
+		}
+	}
+}
+
 // fetchServiceBannerLoop fetches the service banner on an interval.  It will
 // not be fetched immediately; the expectation is that it is primed elsewhere
 // (and must be done before the session actually starts).
@@ -743,14 +908,14 @@ func (a *agent) run() (retErr error) {
 	a.sessionToken.Store(&sessionToken)
 
 	// ConnectRPC returns the dRPC connection we use for the Agent and Tailnet v2+ APIs
-	aAPI, tAPI, err := a.client.ConnectRPC24(a.hardCtx)
+	aAPI, tAPI, err := a.client.ConnectRPC25(a.hardCtx)
 	if err != nil {
 		return err
 	}
 	defer func() {
 		cErr := aAPI.DRPCConn().Close()
 		if cErr != nil {
-			a.logger.Debug(a.hardCtx, "error closing drpc connection", slog.Error(err))
+			a.logger.Debug(a.hardCtx, "error closing drpc connection", slog.Error(cErr))
 		}
 	}()
 
@@ -779,7 +944,7 @@ func (a *agent) run() (retErr error) {
 	connMan.startAgentAPI("send logs", gracefulShutdownBehaviorRemain,
 		func(ctx context.Context, aAPI proto.DRPCAgentClient24) error {
 			err := a.logSender.SendLoop(ctx, aAPI)
-			if xerrors.Is(err, agentsdk.LogLimitExceededError) {
+			if xerrors.Is(err, agentsdk.ErrLogLimitExceeded) {
 				// we don't want this error to tear down the API connection and propagate to the
 				// other routines that use the API.  The LogSender has already dropped a warning
 				// log, so just return nil here.
@@ -808,12 +973,19 @@ func (a *agent) run() (retErr error) {
 		if err != nil {
 			return xerrors.Errorf("failed to create resources fetcher: %w", err)
 		}
-		resourcesFetcher := resourcesmonitor.NewFetcher(statfetcher)
+		resourcesFetcher, err := resourcesmonitor.NewFetcher(statfetcher)
+		if err != nil {
+			return xerrors.Errorf("new resource fetcher: %w", err)
+		}
 
 		resourcesmonitor := resourcesmonitor.NewResourcesMonitor(logger, clk, config, resourcesFetcher, aAPI)
 		return resourcesmonitor.Start(ctx)
 	})
 
+	// Connection reports are part of auditing, we should keep sending them via
+	// gracefulShutdownBehaviorRemain.
+	connMan.startAgentAPI("report connections", gracefulShutdownBehaviorRemain, a.reportConnectionsLoop)
+
 	// channels to sync goroutines below
 	//  handle manifest
 	//       |
@@ -876,7 +1048,11 @@ func (a *agent) run() (retErr error) {
 		return a.statsReporter.reportLoop(ctx, aAPI)
 	})
 
-	return connMan.wait()
+	err = connMan.wait()
+	if err != nil {
+		a.logger.Info(context.Background(), "connection manager errored", slog.Error(err))
+	}
+	return err
 }
 
 // handleManifest returns a function that fetches and processes the manifest
@@ -911,10 +1087,12 @@ func (a *agent) handleManifest(manifestOK *checkpoint) func(ctx context.Context,
 		//
 		// An example is VS Code Remote, which must know the directory
 		// before initializing a connection.
-		manifest.Directory, err = expandDirectory(manifest.Directory)
+		manifest.Directory, err = expandPathToAbs(manifest.Directory)
 		if err != nil {
 			return xerrors.Errorf("expand directory: %w", err)
 		}
+		// Normalize all devcontainer paths by making them absolute.
+		manifest.Devcontainers = agentcontainers.ExpandAllDevcontainerPaths(a.logger, expandPathToAbs, manifest.Devcontainers)
 		subsys, err := agentsdk.ProtoFromSubsystems(a.subsystems)
 		if err != nil {
 			a.logger.Critical(ctx, "failed to convert subsystems", slog.Error(err))
@@ -951,16 +1129,35 @@ func (a *agent) handleManifest(manifestOK *checkpoint) func(ctx context.Context,
 				}
 			}
 
-			err = a.scriptRunner.Init(manifest.Scripts, aAPI.ScriptCompleted)
+			var (
+				scripts          = manifest.Scripts
+				scriptRunnerOpts []agentscripts.InitOption
+			)
+			if a.experimentalDevcontainersEnabled {
+				var dcScripts []codersdk.WorkspaceAgentScript
+				scripts, dcScripts = agentcontainers.ExtractAndInitializeDevcontainerScripts(manifest.Devcontainers, scripts)
+				// See ExtractAndInitializeDevcontainerScripts for motivation
+				// behind running dcScripts as post start scripts.
+				scriptRunnerOpts = append(scriptRunnerOpts, agentscripts.WithPostStartScripts(dcScripts...))
+			}
+			err = a.scriptRunner.Init(scripts, aAPI.ScriptCompleted, scriptRunnerOpts...)
 			if err != nil {
 				return xerrors.Errorf("init script runner: %w", err)
 			}
 			err = a.trackGoroutine(func() {
 				start := time.Now()
-				// here we use the graceful context because the script runner is not directly tied
-				// to the agent API.
+				// Here we use the graceful context because the script runner is
+				// not directly tied to the agent API.
+				//
+				// First we run the start scripts to ensure the workspace has
+				// been initialized and then the post start scripts which may
+				// depend on the workspace start scripts.
+				//
+				// Measure the time immediately after the start scripts have
+				// finished (both start and post start). For instance, an
+				// autostarted devcontainer will be included in this time.
 				err := a.scriptRunner.Execute(a.gracefulCtx, agentscripts.ExecuteStartScripts)
-				// Measure the time immediately after the script has finished
+				err = errors.Join(err, a.scriptRunner.Execute(a.gracefulCtx, agentscripts.ExecutePostStartScripts))
 				dur := time.Since(start).Seconds()
 				if err != nil {
 					a.logger.Warn(ctx, "startup script(s) failed", slog.Error(err))
@@ -1003,9 +1200,9 @@ func (a *agent) createOrUpdateNetwork(manifestOK, networkOK *checkpoint) func(co
 		network := a.network
 		a.closeMutex.Unlock()
 		if network == nil {
-			keySeed, err := WorkspaceKeySeed(manifest.WorkspaceID, manifest.AgentName)
+			keySeed, err := SSHKeySeed(manifest.OwnerName, manifest.WorkspaceName, manifest.AgentName)
 			if err != nil {
-				return xerrors.Errorf("generate seed from workspace id: %w", err)
+				return xerrors.Errorf("generate SSH key seed: %w", err)
 			}
 			// use the graceful context here, because creating the tailnet is not itself tied to the
 			// agent API.
@@ -1022,15 +1219,15 @@ func (a *agent) createOrUpdateNetwork(manifestOK, networkOK *checkpoint) func(co
 			}
 			a.closeMutex.Lock()
 			// Re-check if agent was closed while initializing the network.
-			closed := a.isClosed()
-			if !closed {
+			closing := a.closing
+			if !closing {
 				a.network = network
 				a.statsReporter = newStatsReporter(a.logger, network, a)
 			}
 			a.closeMutex.Unlock()
-			if closed {
+			if closing {
 				_ = network.Close()
-				return xerrors.New("agent is closed")
+				return xerrors.New("agent is closing")
 			}
 		} else {
 			// Update the wireguard IPs if the agent ID changed.
@@ -1145,8 +1342,8 @@ func (*agent) wireguardAddresses(agentID uuid.UUID) []netip.Prefix {
 func (a *agent) trackGoroutine(fn func()) error {
 	a.closeMutex.Lock()
 	defer a.closeMutex.Unlock()
-	if a.isClosed() {
-		return xerrors.New("track conn goroutine: agent is closed")
+	if a.closing {
+		return xerrors.New("track conn goroutine: agent is closing")
 	}
 	a.closeWaitGroup.Add(1)
 	go func() {
@@ -1225,7 +1422,7 @@ func (a *agent) createTailnet(
 		if rPTYServeErr != nil &&
 			a.gracefulCtx.Err() == nil &&
 			!strings.Contains(rPTYServeErr.Error(), "use of closed network connection") {
-			a.logger.Error(ctx, "error serving reconnecting PTY", slog.Error(err))
+			a.logger.Error(ctx, "error serving reconnecting PTY", slog.Error(rPTYServeErr))
 		}
 	}); err != nil {
 		return nil, err
@@ -1290,8 +1487,13 @@ func (a *agent) createTailnet(
 	}()
 	if err = a.trackGoroutine(func() {
 		defer apiListener.Close()
+		apiHandler, closeAPIHAndler := a.apiHandler()
+		defer func() {
+			_ = closeAPIHAndler()
+		}()
 		server := &http.Server{
-			Handler:           a.apiHandler(),
+			BaseContext:       func(net.Listener) context.Context { return ctx },
+			Handler:           apiHandler,
 			ReadTimeout:       20 * time.Second,
 			ReadHeaderTimeout: 20 * time.Second,
 			WriteTimeout:      20 * time.Second,
@@ -1302,6 +1504,7 @@ func (a *agent) createTailnet(
 			case <-ctx.Done():
 			case <-a.hardCtx.Done():
 			}
+			_ = closeAPIHAndler()
 			_ = server.Close()
 		}()
 
@@ -1335,14 +1538,11 @@ func (a *agent) runCoordinator(ctx context.Context, tClient tailnetproto.DRPCTai
 	a.logger.Info(ctx, "connected to coordination RPC")
 
 	// This allows the Close() routine to wait for the coordinator to gracefully disconnect.
-	a.closeMutex.Lock()
-	if a.isClosed() {
-		return nil
+	disconnected := a.setCoordDisconnected()
+	if disconnected == nil {
+		return nil // already closed by something else
 	}
-	disconnected := make(chan struct{})
-	a.coordDisconnected = disconnected
 	defer close(disconnected)
-	a.closeMutex.Unlock()
 
 	ctrl := tailnet.NewAgentCoordinationController(a.logger, network)
 	coordination := ctrl.New(coordinate)
@@ -1364,6 +1564,17 @@ func (a *agent) runCoordinator(ctx context.Context, tClient tailnetproto.DRPCTai
 	return <-errCh
 }
 
+func (a *agent) setCoordDisconnected() chan struct{} {
+	a.closeMutex.Lock()
+	defer a.closeMutex.Unlock()
+	if a.closing {
+		return nil
+	}
+	disconnected := make(chan struct{})
+	a.coordDisconnected = disconnected
+	return disconnected
+}
+
 // runDERPMapSubscriber runs a coordinator and returns if a reconnect should occur.
 func (a *agent) runDERPMapSubscriber(ctx context.Context, tClient tailnetproto.DRPCTailnetClient24, network *tailnet.Conn) error {
 	defer a.logger.Debug(ctx, "disconnected from derp map RPC")
@@ -1400,9 +1611,13 @@ func (a *agent) Collect(ctx context.Context, networkStats map[netlogtype.Connect
 	}
 	for conn, counts := range networkStats {
 		stats.ConnectionsByProto[conn.Proto.String()]++
+		// #nosec G115 - Safe conversions for network statistics which we expect to be within int64 range
 		stats.RxBytes += int64(counts.RxBytes)
+		// #nosec G115 - Safe conversions for network statistics which we expect to be within int64 range
 		stats.RxPackets += int64(counts.RxPackets)
+		// #nosec G115 - Safe conversions for network statistics which we expect to be within int64 range
 		stats.TxBytes += int64(counts.TxBytes)
+		// #nosec G115 - Safe conversions for network statistics which we expect to be within int64 range
 		stats.TxPackets += int64(counts.TxPackets)
 	}
 
@@ -1455,11 +1670,12 @@ func (a *agent) Collect(ctx context.Context, networkStats map[netlogtype.Connect
 	wg.Wait()
 	sort.Float64s(durations)
 	durationsLength := len(durations)
-	if durationsLength == 0 {
+	switch {
+	case durationsLength == 0:
 		stats.ConnectionMedianLatencyMs = -1
-	} else if durationsLength%2 == 0 {
+	case durationsLength%2 == 0:
 		stats.ConnectionMedianLatencyMs = (durations[durationsLength/2-1] + durations[durationsLength/2]) / 2
-	} else {
+	default:
 		stats.ConnectionMedianLatencyMs = durations[durationsLength/2]
 	}
 	// Convert from microseconds to milliseconds.
@@ -1566,7 +1782,7 @@ func (a *agent) HTTPDebug() http.Handler {
 	r.Get("/debug/magicsock", a.HandleHTTPDebugMagicsock)
 	r.Get("/debug/magicsock/debug-logging/{state}", a.HandleHTTPMagicsockDebugLoggingState)
 	r.Get("/debug/manifest", a.HandleHTTPDebugManifest)
-	r.NotFound(func(w http.ResponseWriter, r *http.Request) {
+	r.NotFound(func(w http.ResponseWriter, _ *http.Request) {
 		w.WriteHeader(http.StatusNotFound)
 		_, _ = w.Write([]byte("404 not found"))
 	})
@@ -1576,7 +1792,10 @@ func (a *agent) HTTPDebug() http.Handler {
 
 func (a *agent) Close() error {
 	a.closeMutex.Lock()
-	defer a.closeMutex.Unlock()
+	network := a.network
+	coordDisconnected := a.coordDisconnected
+	a.closing = true
+	a.closeMutex.Unlock()
 	if a.isClosed() {
 		return nil
 	}
@@ -1585,15 +1804,22 @@ func (a *agent) Close() error {
 	a.setLifecycle(codersdk.WorkspaceAgentLifecycleShuttingDown)
 
 	// Attempt to gracefully shut down all active SSH connections and
-	// stop accepting new ones.
-	err := a.sshServer.Shutdown(a.hardCtx)
+	// stop accepting new ones. If all processes have not exited after 5
+	// seconds, we just log it and move on as it's more important to run
+	// the shutdown scripts. A typical shutdown time for containers is
+	// 10 seconds, so this still leaves a bit of time to run the
+	// shutdown scripts in the worst-case.
+	sshShutdownCtx, sshShutdownCancel := context.WithTimeout(a.hardCtx, 5*time.Second)
+	defer sshShutdownCancel()
+	err := a.sshServer.Shutdown(sshShutdownCtx)
 	if err != nil {
-		a.logger.Error(a.hardCtx, "ssh server shutdown", slog.Error(err))
-	}
-	err = a.sshServer.Close()
-	if err != nil {
-		a.logger.Error(a.hardCtx, "ssh server close", slog.Error(err))
+		if errors.Is(err, context.DeadlineExceeded) {
+			a.logger.Warn(sshShutdownCtx, "ssh server shutdown timeout", slog.Error(err))
+		} else {
+			a.logger.Error(sshShutdownCtx, "ssh server shutdown", slog.Error(err))
+		}
 	}
+
 	// wait for SSH to shut down before the general graceful cancel, because
 	// this triggers a disconnect in the tailnet layer, telling all clients to
 	// shut down their wireguard tunnels to us. If SSH sessions are still up,
@@ -1646,7 +1872,7 @@ lifecycleWaitLoop:
 	select {
 	case <-a.hardCtx.Done():
 		a.logger.Warn(context.Background(), "timed out waiting for Coordinator RPC disconnect")
-	case <-a.coordDisconnected:
+	case <-coordDisconnected:
 		a.logger.Debug(context.Background(), "coordinator RPC disconnected")
 	}
 
@@ -1657,8 +1883,8 @@ lifecycleWaitLoop:
 	}
 
 	a.hardCancel()
-	if a.network != nil {
-		_ = a.network.Close()
+	if network != nil {
+		_ = network.Close()
 	}
 	a.closeWaitGroup.Wait()
 
@@ -1682,30 +1908,29 @@ func userHomeDir() (string, error) {
 	return u.HomeDir, nil
 }
 
-// expandDirectory converts a directory path to an absolute path.
-// It primarily resolves the home directory and any environment
-// variables that may be set
-func expandDirectory(dir string) (string, error) {
-	if dir == "" {
+// expandPathToAbs converts a path to an absolute path. It primarily resolves
+// the home directory and any environment variables that may be set.
+func expandPathToAbs(path string) (string, error) {
+	if path == "" {
 		return "", nil
 	}
-	if dir[0] == '~' {
+	if path[0] == '~' {
 		home, err := userHomeDir()
 		if err != nil {
 			return "", err
 		}
-		dir = filepath.Join(home, dir[1:])
+		path = filepath.Join(home, path[1:])
 	}
-	dir = os.ExpandEnv(dir)
+	path = os.ExpandEnv(path)
 
-	if !filepath.IsAbs(dir) {
+	if !filepath.IsAbs(path) {
 		home, err := userHomeDir()
 		if err != nil {
 			return "", err
 		}
-		dir = filepath.Join(home, dir)
+		path = filepath.Join(home, path)
 	}
-	return dir, nil
+	return path, nil
 }
 
 // EnvAgentSubsystem is the environment variable used to denote the
@@ -1852,7 +2077,7 @@ func (a *apiConnRoutineManager) wait() error {
 }
 
 func PrometheusMetricsHandler(prometheusRegistry *prometheus.Registry, logger slog.Logger) http.Handler {
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+	return http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
 		w.Header().Set("Content-Type", "text/plain")
 
 		// Based on: https://github.com/tailscale/tailscale/blob/280255acae604796a1113861f5a84e6fa2dc6121/ipn/localapi/localapi.go#L489
@@ -1874,12 +2099,31 @@ func PrometheusMetricsHandler(prometheusRegistry *prometheus.Registry, logger sl
 	})
 }
 
-// WorkspaceKeySeed converts a WorkspaceID UUID and agent name to an int64 hash.
+// SSHKeySeed converts an owner userName, workspaceName and agentName to an int64 hash.
 // This uses the FNV-1a hash algorithm which provides decent distribution and collision
 // resistance for string inputs.
-func WorkspaceKeySeed(workspaceID uuid.UUID, agentName string) (int64, error) {
+//
+// Why owner username, workspace name, and agent name? These are the components that are used in hostnames for the
+// workspace over SSH, and so we want the workspace to have a stable key with respect to these.  We don't use the
+// respective UUIDs.  The workspace UUID would be different if you delete and recreate a workspace with the same name.
+// The agent UUID is regenerated on each build. Since Coder's Tailnet networking is handling the authentication, we
+// should not be showing users warnings about host SSH keys.
+func SSHKeySeed(userName, workspaceName, agentName string) (int64, error) {
 	h := fnv.New64a()
-	_, err := h.Write(workspaceID[:])
+	_, err := h.Write([]byte(userName))
+	if err != nil {
+		return 42, err
+	}
+	// null separators between strings so that (dog, foodstuff) is distinct from (dogfood, stuff)
+	_, err = h.Write([]byte{0})
+	if err != nil {
+		return 42, err
+	}
+	_, err = h.Write([]byte(workspaceName))
+	if err != nil {
+		return 42, err
+	}
+	_, err = h.Write([]byte{0})
 	if err != nil {
 		return 42, err
 	}
@@ -1888,5 +2132,6 @@ func WorkspaceKeySeed(workspaceID uuid.UUID, agentName string) (int64, error) {
 		return 42, err
 	}
 
+	// #nosec G115 - Safe conversion to generate int64 hash from Sum64, data loss acceptable
 	return int64(h.Sum64()), nil
 }

agent/agent_test.go

@@ -19,14 +19,21 @@ import (
 	"path/filepath"
 	"regexp"
 	"runtime"
+	"slices"
 	"strconv"
 	"strings"
 	"sync/atomic"
 	"testing"
 	"time"
 
+	"go.uber.org/goleak"
+	"tailscale.com/net/speedtest"
+	"tailscale.com/tailcfg"
+
 	"github.com/bramvdbogaerde/go-scp"
 	"github.com/google/uuid"
+	"github.com/ory/dockertest/v3"
+	"github.com/ory/dockertest/v3/docker"
 	"github.com/pion/udp"
 	"github.com/pkg/sftp"
 	"github.com/prometheus/client_golang/prometheus"
@@ -34,19 +41,17 @@ import (
 	"github.com/spf13/afero"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-	"go.uber.org/goleak"
 	"golang.org/x/crypto/ssh"
-	"golang.org/x/exp/slices"
 	"golang.org/x/xerrors"
-	"tailscale.com/net/speedtest"
-	"tailscale.com/tailcfg"
 
 	"cdr.dev/slog"
 	"cdr.dev/slog/sloggers/slogtest"
+
 	"github.com/coder/coder/v2/agent"
 	"github.com/coder/coder/v2/agent/agentssh"
 	"github.com/coder/coder/v2/agent/agenttest"
 	"github.com/coder/coder/v2/agent/proto"
+	"github.com/coder/coder/v2/agent/usershell"
 	"github.com/coder/coder/v2/codersdk"
 	"github.com/coder/coder/v2/codersdk/agentsdk"
 	"github.com/coder/coder/v2/codersdk/workspacesdk"
@@ -63,6 +68,54 @@ func TestMain(m *testing.M) {
 
 var sshPorts = []uint16{workspacesdk.AgentSSHPort, workspacesdk.AgentStandardSSHPort}
 
+// TestAgent_CloseWhileStarting is a regression test for https://github.com/coder/coder/issues/17328
+func TestAgent_ImmediateClose(t *testing.T) {
+	t.Parallel()
+
+	ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
+	defer cancel()
+
+	logger := slogtest.Make(t, &slogtest.Options{
+		// Agent can drop errors when shutting down, and some, like the
+		// fasthttplistener connection closed error, are unexported.
+		IgnoreErrors: true,
+	}).Leveled(slog.LevelDebug)
+	manifest := agentsdk.Manifest{
+		AgentID:       uuid.New(),
+		AgentName:     "test-agent",
+		WorkspaceName: "test-workspace",
+		WorkspaceID:   uuid.New(),
+	}
+
+	coordinator := tailnet.NewCoordinator(logger)
+	t.Cleanup(func() {
+		_ = coordinator.Close()
+	})
+	statsCh := make(chan *proto.Stats, 50)
+	fs := afero.NewMemMapFs()
+	client := agenttest.NewClient(t, logger.Named("agenttest"), manifest.AgentID, manifest, statsCh, coordinator)
+	t.Cleanup(client.Close)
+
+	options := agent.Options{
+		Client:                 client,
+		Filesystem:             fs,
+		Logger:                 logger.Named("agent"),
+		ReconnectingPTYTimeout: 0,
+		EnvironmentVariables:   map[string]string{},
+	}
+
+	agentUnderTest := agent.New(options)
+	t.Cleanup(func() {
+		_ = agentUnderTest.Close()
+	})
+
+	// wait until the agent has connected and is starting to find races in the startup code
+	_ = testutil.TryReceive(ctx, t, client.GetStartup())
+	t.Log("Closing Agent")
+	err := agentUnderTest.Close()
+	require.NoError(t, err)
+}
+
 // NOTE: These tests only work when your default shell is bash for some reason.
 
 func TestAgent_Stats_SSH(t *testing.T) {
@@ -169,7 +222,7 @@ func TestAgent_Stats_Magic(t *testing.T) {
 		ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
 		defer cancel()
 		//nolint:dogsled
-		conn, _, stats, _, _ := setupAgent(t, agentsdk.Manifest{}, 0)
+		conn, agentClient, stats, _, _ := setupAgent(t, agentsdk.Manifest{}, 0)
 		sshClient, err := conn.SSHClient(ctx)
 		require.NoError(t, err)
 		defer sshClient.Close()
@@ -185,7 +238,7 @@ func TestAgent_Stats_Magic(t *testing.T) {
 			s, ok := <-stats
 			t.Logf("got stats: ok=%t, ConnectionCount=%d, RxBytes=%d, TxBytes=%d, SessionCountVSCode=%d, ConnectionMedianLatencyMS=%f",
 				ok, s.ConnectionCount, s.RxBytes, s.TxBytes, s.SessionCountVscode, s.ConnectionMedianLatencyMs)
-			return ok && s.ConnectionCount > 0 && s.RxBytes > 0 && s.TxBytes > 0 &&
+			return ok &&
 				// Ensure that the connection didn't count as a "normal" SSH session.
 				// This was a special one, so it should be labeled specially in the stats!
 				s.SessionCountVscode == 1 &&
@@ -199,6 +252,8 @@ func TestAgent_Stats_Magic(t *testing.T) {
 		_ = stdin.Close()
 		err = session.Wait()
 		require.NoError(t, err)
+
+		assertConnectionReport(t, agentClient, proto.Connection_VSCODE, 0, "")
 	})
 
 	t.Run("TracksJetBrains", func(t *testing.T) {
@@ -235,7 +290,7 @@ func TestAgent_Stats_Magic(t *testing.T) {
 		remotePort := sc.Text()
 
 		//nolint:dogsled
-		conn, _, stats, _, _ := setupAgent(t, agentsdk.Manifest{}, 0)
+		conn, agentClient, stats, _, _ := setupAgent(t, agentsdk.Manifest{}, 0)
 		sshClient, err := conn.SSHClient(ctx)
 		require.NoError(t, err)
 
@@ -251,8 +306,7 @@ func TestAgent_Stats_Magic(t *testing.T) {
 			s, ok := <-stats
 			t.Logf("got stats with conn open: ok=%t, ConnectionCount=%d, SessionCountJetBrains=%d",
 				ok, s.ConnectionCount, s.SessionCountJetbrains)
-			return ok && s.ConnectionCount > 0 &&
-				s.SessionCountJetbrains == 1
+			return ok && s.SessionCountJetbrains == 1
 		}, testutil.WaitLong, testutil.IntervalFast,
 			"never saw stats with conn open",
 		)
@@ -271,6 +325,8 @@ func TestAgent_Stats_Magic(t *testing.T) {
 		}, testutil.WaitLong, testutil.IntervalFast,
 			"never saw stats after conn closes",
 		)
+
+		assertConnectionReport(t, agentClient, proto.Connection_JETBRAINS, 0, "")
 	})
 }
 
@@ -948,7 +1004,7 @@ func TestAgent_SFTP(t *testing.T) {
 		home = "/" + strings.ReplaceAll(home, "\\", "/")
 	}
 	//nolint:dogsled
-	conn, _, _, _, _ := setupAgent(t, agentsdk.Manifest{}, 0)
+	conn, agentClient, _, _, _ := setupAgent(t, agentsdk.Manifest{}, 0)
 	sshClient, err := conn.SSHClient(ctx)
 	require.NoError(t, err)
 	defer sshClient.Close()
@@ -971,6 +1027,10 @@ func TestAgent_SFTP(t *testing.T) {
 	require.NoError(t, err)
 	_, err = os.Stat(tempFile)
 	require.NoError(t, err)
+
+	// Close the client to trigger disconnect event.
+	_ = client.Close()
+	assertConnectionReport(t, agentClient, proto.Connection_SSH, 0, "")
 }
 
 func TestAgent_SCP(t *testing.T) {
@@ -980,7 +1040,7 @@ func TestAgent_SCP(t *testing.T) {
 	defer cancel()
 
 	//nolint:dogsled
-	conn, _, _, _, _ := setupAgent(t, agentsdk.Manifest{}, 0)
+	conn, agentClient, _, _, _ := setupAgent(t, agentsdk.Manifest{}, 0)
 	sshClient, err := conn.SSHClient(ctx)
 	require.NoError(t, err)
 	defer sshClient.Close()
@@ -993,6 +1053,10 @@ func TestAgent_SCP(t *testing.T) {
 	require.NoError(t, err)
 	_, err = os.Stat(tempFile)
 	require.NoError(t, err)
+
+	// Close the client to trigger disconnect event.
+	scpClient.Close()
+	assertConnectionReport(t, agentClient, proto.Connection_SSH, 0, "")
 }
 
 func TestAgent_FileTransferBlocked(t *testing.T) {
@@ -1017,7 +1081,7 @@ func TestAgent_FileTransferBlocked(t *testing.T) {
 		defer cancel()
 
 		//nolint:dogsled
-		conn, _, _, _, _ := setupAgent(t, agentsdk.Manifest{}, 0, func(_ *agenttest.Client, o *agent.Options) {
+		conn, agentClient, _, _, _ := setupAgent(t, agentsdk.Manifest{}, 0, func(_ *agenttest.Client, o *agent.Options) {
 			o.BlockFileTransfer = true
 		})
 		sshClient, err := conn.SSHClient(ctx)
@@ -1026,6 +1090,8 @@ func TestAgent_FileTransferBlocked(t *testing.T) {
 		_, err = sftp.NewClient(sshClient)
 		require.Error(t, err)
 		assertFileTransferBlocked(t, err.Error())
+
+		assertConnectionReport(t, agentClient, proto.Connection_SSH, agentssh.BlockedFileTransferErrorCode, "")
 	})
 
 	t.Run("SCP with go-scp package", func(t *testing.T) {
@@ -1035,7 +1101,7 @@ func TestAgent_FileTransferBlocked(t *testing.T) {
 		defer cancel()
 
 		//nolint:dogsled
-		conn, _, _, _, _ := setupAgent(t, agentsdk.Manifest{}, 0, func(_ *agenttest.Client, o *agent.Options) {
+		conn, agentClient, _, _, _ := setupAgent(t, agentsdk.Manifest{}, 0, func(_ *agenttest.Client, o *agent.Options) {
 			o.BlockFileTransfer = true
 		})
 		sshClient, err := conn.SSHClient(ctx)
@@ -1048,6 +1114,8 @@ func TestAgent_FileTransferBlocked(t *testing.T) {
 		err = scpClient.CopyFile(context.Background(), strings.NewReader("hello world"), tempFile, "0755")
 		require.Error(t, err)
 		assertFileTransferBlocked(t, err.Error())
+
+		assertConnectionReport(t, agentClient, proto.Connection_SSH, agentssh.BlockedFileTransferErrorCode, "")
 	})
 
 	t.Run("Forbidden commands", func(t *testing.T) {
@@ -1061,7 +1129,7 @@ func TestAgent_FileTransferBlocked(t *testing.T) {
 				defer cancel()
 
 				//nolint:dogsled
-				conn, _, _, _, _ := setupAgent(t, agentsdk.Manifest{}, 0, func(_ *agenttest.Client, o *agent.Options) {
+				conn, agentClient, _, _, _ := setupAgent(t, agentsdk.Manifest{}, 0, func(_ *agenttest.Client, o *agent.Options) {
 					o.BlockFileTransfer = true
 				})
 				sshClient, err := conn.SSHClient(ctx)
@@ -1083,6 +1151,8 @@ func TestAgent_FileTransferBlocked(t *testing.T) {
 				msg, err := io.ReadAll(stdout)
 				require.NoError(t, err)
 				assertFileTransferBlocked(t, string(msg))
+
+				assertConnectionReport(t, agentClient, proto.Connection_SSH, agentssh.BlockedFileTransferErrorCode, "")
 			})
 		}
 	})
@@ -1171,6 +1241,49 @@ func TestAgent_SSHConnectionEnvVars(t *testing.T) {
 	}
 }
 
+func TestAgent_SSHConnectionLoginVars(t *testing.T) {
+	t.Parallel()
+
+	envInfo := usershell.SystemEnvInfo{}
+	u, err := envInfo.User()
+	require.NoError(t, err, "get current user")
+	shell, err := envInfo.Shell(u.Username)
+	require.NoError(t, err, "get current shell")
+
+	tests := []struct {
+		key  string
+		want string
+	}{
+		{
+			key:  "USER",
+			want: u.Username,
+		},
+		{
+			key:  "LOGNAME",
+			want: u.Username,
+		},
+		{
+			key:  "SHELL",
+			want: shell,
+		},
+	}
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.key, func(t *testing.T) {
+			t.Parallel()
+
+			session := setupSSHSession(t, agentsdk.Manifest{}, codersdk.ServiceBannerConfig{}, nil)
+			command := "sh -c 'echo $" + tt.key + "'"
+			if runtime.GOOS == "windows" {
+				command = "cmd.exe /c echo %" + tt.key + "%"
+			}
+			output, err := session.Output(command)
+			require.NoError(t, err)
+			require.Equal(t, tt.want, strings.TrimSpace(string(output)))
+		})
+	}
+}
+
 func TestAgent_Metadata(t *testing.T) {
 	t.Parallel()
 
@@ -1385,7 +1498,7 @@ func TestAgent_Lifecycle(t *testing.T) {
 
 		_, client, _, _, _ := setupAgent(t, agentsdk.Manifest{
 			Scripts: []codersdk.WorkspaceAgentScript{{
-				Script:     "true",
+				Script:     "echo foo",
 				Timeout:    30 * time.Second,
 				RunOnStart: true,
 			}},
@@ -1533,8 +1646,10 @@ func TestAgent_Lifecycle(t *testing.T) {
 	t.Run("ShutdownScriptOnce", func(t *testing.T) {
 		t.Parallel()
 		logger := testutil.Logger(t)
+		ctx := testutil.Context(t, testutil.WaitMedium)
 		expected := "this-is-shutdown"
 		derpMap, _ := tailnettest.RunDERPAndSTUN(t)
+		statsCh := make(chan *proto.Stats, 50)
 
 		client := agenttest.NewClient(t,
 			logger,
@@ -1553,7 +1668,7 @@ func TestAgent_Lifecycle(t *testing.T) {
 					RunOnStop: true,
 				}},
 			},
-			make(chan *proto.Stats, 50),
+			statsCh,
 			tailnet.NewCoordinator(logger),
 		)
 		defer client.Close()
@@ -1578,6 +1693,11 @@ func TestAgent_Lifecycle(t *testing.T) {
 			return len(content) > 0 // something is in the startup log file
 		}, testutil.WaitShort, testutil.IntervalMedium)
 
+		// In order to avoid shutting down the agent before it is fully started and triggering
+		// errors, we'll wait until the agent is fully up. It's a bit hokey, but among the last things the agent starts
+		// is the stats reporting, so getting a stats report is a good indication the agent is fully up.
+		_ = testutil.TryReceive(ctx, t, statsCh)
+
 		err := agent.Close()
 		require.NoError(t, err, "agent should be closed successfully")
 
@@ -1606,7 +1726,7 @@ func TestAgent_Startup(t *testing.T) {
 		_, client, _, _, _ := setupAgent(t, agentsdk.Manifest{
 			Directory: "",
 		}, 0)
-		startup := testutil.RequireRecvCtx(ctx, t, client.GetStartup())
+		startup := testutil.TryReceive(ctx, t, client.GetStartup())
 		require.Equal(t, "", startup.GetExpandedDirectory())
 	})
 
@@ -1617,7 +1737,7 @@ func TestAgent_Startup(t *testing.T) {
 		_, client, _, _, _ := setupAgent(t, agentsdk.Manifest{
 			Directory: "~",
 		}, 0)
-		startup := testutil.RequireRecvCtx(ctx, t, client.GetStartup())
+		startup := testutil.TryReceive(ctx, t, client.GetStartup())
 		homeDir, err := os.UserHomeDir()
 		require.NoError(t, err)
 		require.Equal(t, homeDir, startup.GetExpandedDirectory())
@@ -1630,7 +1750,7 @@ func TestAgent_Startup(t *testing.T) {
 		_, client, _, _, _ := setupAgent(t, agentsdk.Manifest{
 			Directory: "coder/coder",
 		}, 0)
-		startup := testutil.RequireRecvCtx(ctx, t, client.GetStartup())
+		startup := testutil.TryReceive(ctx, t, client.GetStartup())
 		homeDir, err := os.UserHomeDir()
 		require.NoError(t, err)
 		require.Equal(t, filepath.Join(homeDir, "coder/coder"), startup.GetExpandedDirectory())
@@ -1643,7 +1763,7 @@ func TestAgent_Startup(t *testing.T) {
 		_, client, _, _, _ := setupAgent(t, agentsdk.Manifest{
 			Directory: "$HOME",
 		}, 0)
-		startup := testutil.RequireRecvCtx(ctx, t, client.GetStartup())
+		startup := testutil.TryReceive(ctx, t, client.GetStartup())
 		homeDir, err := os.UserHomeDir()
 		require.NoError(t, err)
 		require.Equal(t, homeDir, startup.GetExpandedDirectory())
@@ -1691,8 +1811,16 @@ func TestAgent_ReconnectingPTY(t *testing.T) {
 			defer cancel()
 
 			//nolint:dogsled
-			conn, _, _, _, _ := setupAgent(t, agentsdk.Manifest{}, 0)
+			conn, agentClient, _, _, _ := setupAgent(t, agentsdk.Manifest{}, 0)
 			id := uuid.New()
+
+			// Test that the connection is reported. This must be tested in the
+			// first connection because we care about verifying all of these.
+			netConn0, err := conn.ReconnectingPTY(ctx, id, 80, 80, "bash --norc")
+			require.NoError(t, err)
+			_ = netConn0.Close()
+			assertConnectionReport(t, agentClient, proto.Connection_RECONNECTING_PTY, 0, "")
+
 			// --norc disables executing .bashrc, which is often used to customize the bash prompt
 			netConn1, err := conn.ReconnectingPTY(ctx, id, 80, 80, "bash --norc")
 			require.NoError(t, err)
@@ -1791,6 +1919,371 @@ func TestAgent_ReconnectingPTY(t *testing.T) {
 	}
 }
 
+// This tests end-to-end functionality of connecting to a running container
+// and executing a command. It creates a real Docker container and runs a
+// command. As such, it does not run by default in CI.
+// You can run it manually as follows:
+//
+// CODER_TEST_USE_DOCKER=1 go test -count=1 ./agent -run TestAgent_ReconnectingPTYContainer
+func TestAgent_ReconnectingPTYContainer(t *testing.T) {
+	t.Parallel()
+	if os.Getenv("CODER_TEST_USE_DOCKER") != "1" {
+		t.Skip("Set CODER_TEST_USE_DOCKER=1 to run this test")
+	}
+
+	pool, err := dockertest.NewPool("")
+	require.NoError(t, err, "Could not connect to docker")
+	ct, err := pool.RunWithOptions(&dockertest.RunOptions{
+		Repository: "busybox",
+		Tag:        "latest",
+		Cmd:        []string{"sleep", "infnity"},
+	}, func(config *docker.HostConfig) {
+		config.AutoRemove = true
+		config.RestartPolicy = docker.RestartPolicy{Name: "no"}
+	})
+	require.NoError(t, err, "Could not start container")
+	defer func() {
+		err := pool.Purge(ct)
+		require.NoError(t, err, "Could not stop container")
+	}()
+	// Wait for container to start
+	require.Eventually(t, func() bool {
+		ct, ok := pool.ContainerByName(ct.Container.Name)
+		return ok && ct.Container.State.Running
+	}, testutil.WaitShort, testutil.IntervalSlow, "Container did not start in time")
+
+	// nolint: dogsled
+	conn, _, _, _, _ := setupAgent(t, agentsdk.Manifest{}, 0, func(_ *agenttest.Client, o *agent.Options) {
+		o.ExperimentalDevcontainersEnabled = true
+	})
+	ctx := testutil.Context(t, testutil.WaitLong)
+	ac, err := conn.ReconnectingPTY(ctx, uuid.New(), 80, 80, "/bin/sh", func(arp *workspacesdk.AgentReconnectingPTYInit) {
+		arp.Container = ct.Container.ID
+	})
+	require.NoError(t, err, "failed to create ReconnectingPTY")
+	defer ac.Close()
+	tr := testutil.NewTerminalReader(t, ac)
+
+	require.NoError(t, tr.ReadUntil(ctx, func(line string) bool {
+		return strings.Contains(line, "#") || strings.Contains(line, "$")
+	}), "find prompt")
+
+	require.NoError(t, json.NewEncoder(ac).Encode(workspacesdk.ReconnectingPTYRequest{
+		Data: "hostname\r",
+	}), "write hostname")
+	require.NoError(t, tr.ReadUntil(ctx, func(line string) bool {
+		return strings.Contains(line, "hostname")
+	}), "find hostname command")
+
+	require.NoError(t, tr.ReadUntil(ctx, func(line string) bool {
+		return strings.Contains(line, ct.Container.Config.Hostname)
+	}), "find hostname output")
+	require.NoError(t, json.NewEncoder(ac).Encode(workspacesdk.ReconnectingPTYRequest{
+		Data: "exit\r",
+	}), "write exit command")
+
+	// Wait for the connection to close.
+	require.ErrorIs(t, tr.ReadUntil(ctx, nil), io.EOF)
+}
+
+// This tests end-to-end functionality of auto-starting a devcontainer.
+// It runs "devcontainer up" which creates a real Docker container. As
+// such, it does not run by default in CI.
+//
+// You can run it manually as follows:
+//
+// CODER_TEST_USE_DOCKER=1 go test -count=1 ./agent -run TestAgent_DevcontainerAutostart
+//
+//nolint:paralleltest // This test sets an environment variable.
+func TestAgent_DevcontainerAutostart(t *testing.T) {
+	if os.Getenv("CODER_TEST_USE_DOCKER") != "1" {
+		t.Skip("Set CODER_TEST_USE_DOCKER=1 to run this test")
+	}
+
+	pool, err := dockertest.NewPool("")
+	require.NoError(t, err, "Could not connect to docker")
+
+	// Prepare temporary devcontainer for test (mywork).
+	devcontainerID := uuid.New()
+	tmpdir := t.TempDir()
+	t.Setenv("HOME", tmpdir)
+	tempWorkspaceFolder := filepath.Join(tmpdir, "mywork")
+	unexpandedWorkspaceFolder := filepath.Join("~", "mywork")
+	t.Logf("Workspace folder: %s", tempWorkspaceFolder)
+	t.Logf("Unexpanded workspace folder: %s", unexpandedWorkspaceFolder)
+	devcontainerPath := filepath.Join(tempWorkspaceFolder, ".devcontainer")
+	err = os.MkdirAll(devcontainerPath, 0o755)
+	require.NoError(t, err, "create devcontainer directory")
+	devcontainerFile := filepath.Join(devcontainerPath, "devcontainer.json")
+	err = os.WriteFile(devcontainerFile, []byte(`{
+        "name": "mywork",
+        "image": "busybox:latest",
+        "cmd": ["sleep", "infinity"]
+    }`), 0o600)
+	require.NoError(t, err, "write devcontainer.json")
+
+	manifest := agentsdk.Manifest{
+		// Set up pre-conditions for auto-starting a devcontainer, the script
+		// is expected to be prepared by the provisioner normally.
+		Devcontainers: []codersdk.WorkspaceAgentDevcontainer{
+			{
+				ID:   devcontainerID,
+				Name: "test",
+				// Use an unexpanded path to test the expansion.
+				WorkspaceFolder: unexpandedWorkspaceFolder,
+			},
+		},
+		Scripts: []codersdk.WorkspaceAgentScript{
+			{
+				ID:          devcontainerID,
+				LogSourceID: agentsdk.ExternalLogSourceID,
+				RunOnStart:  true,
+				Script:      "echo this-will-be-replaced",
+				DisplayName: "Dev Container (test)",
+			},
+		},
+	}
+	//nolint:dogsled
+	conn, _, _, _, _ := setupAgent(t, manifest, 0, func(_ *agenttest.Client, o *agent.Options) {
+		o.ExperimentalDevcontainersEnabled = true
+	})
+
+	t.Logf("Waiting for container with label: devcontainer.local_folder=%s", tempWorkspaceFolder)
+
+	var container docker.APIContainers
+	require.Eventually(t, func() bool {
+		containers, err := pool.Client.ListContainers(docker.ListContainersOptions{All: true})
+		if err != nil {
+			t.Logf("Error listing containers: %v", err)
+			return false
+		}
+
+		for _, c := range containers {
+			t.Logf("Found container: %s with labels: %v", c.ID[:12], c.Labels)
+			if labelValue, ok := c.Labels["devcontainer.local_folder"]; ok {
+				if labelValue == tempWorkspaceFolder {
+					t.Logf("Found matching container: %s", c.ID[:12])
+					container = c
+					return true
+				}
+			}
+		}
+
+		return false
+	}, testutil.WaitSuperLong, testutil.IntervalMedium, "no container with workspace folder label found")
+	defer func() {
+		// We can't rely on pool here because the container is not
+		// managed by it (it is managed by @devcontainer/cli).
+		err := pool.Client.RemoveContainer(docker.RemoveContainerOptions{
+			ID:            container.ID,
+			RemoveVolumes: true,
+			Force:         true,
+		})
+		assert.NoError(t, err, "remove container")
+	}()
+
+	containerInfo, err := pool.Client.InspectContainer(container.ID)
+	require.NoError(t, err, "inspect container")
+	t.Logf("Container state: status: %v", containerInfo.State.Status)
+	require.True(t, containerInfo.State.Running, "container should be running")
+
+	ctx := testutil.Context(t, testutil.WaitLong)
+
+	ac, err := conn.ReconnectingPTY(ctx, uuid.New(), 80, 80, "", func(opts *workspacesdk.AgentReconnectingPTYInit) {
+		opts.Container = container.ID
+	})
+	require.NoError(t, err, "failed to create ReconnectingPTY")
+	defer ac.Close()
+
+	// Use terminal reader so we can see output in case somethin goes wrong.
+	tr := testutil.NewTerminalReader(t, ac)
+
+	require.NoError(t, tr.ReadUntil(ctx, func(line string) bool {
+		return strings.Contains(line, "#") || strings.Contains(line, "$")
+	}), "find prompt")
+
+	wantFileName := "file-from-devcontainer"
+	wantFile := filepath.Join(tempWorkspaceFolder, wantFileName)
+
+	require.NoError(t, json.NewEncoder(ac).Encode(workspacesdk.ReconnectingPTYRequest{
+		// NOTE(mafredri): We must use absolute path here for some reason.
+		Data: fmt.Sprintf("touch /workspaces/mywork/%s; exit\r", wantFileName),
+	}), "create file inside devcontainer")
+
+	// Wait for the connection to close to ensure the touch was executed.
+	require.ErrorIs(t, tr.ReadUntil(ctx, nil), io.EOF)
+
+	_, err = os.Stat(wantFile)
+	require.NoError(t, err, "file should exist outside devcontainer")
+}
+
+// TestAgent_DevcontainerRecreate tests that RecreateDevcontainer
+// recreates a devcontainer and emits logs.
+//
+// This tests end-to-end functionality of auto-starting a devcontainer.
+// It runs "devcontainer up" which creates a real Docker container. As
+// such, it does not run by default in CI.
+//
+// You can run it manually as follows:
+//
+// CODER_TEST_USE_DOCKER=1 go test -count=1 ./agent -run TestAgent_DevcontainerRecreate
+func TestAgent_DevcontainerRecreate(t *testing.T) {
+	if os.Getenv("CODER_TEST_USE_DOCKER") != "1" {
+		t.Skip("Set CODER_TEST_USE_DOCKER=1 to run this test")
+	}
+	t.Parallel()
+
+	pool, err := dockertest.NewPool("")
+	require.NoError(t, err, "Could not connect to docker")
+
+	// Prepare temporary devcontainer for test (mywork).
+	devcontainerID := uuid.New()
+	devcontainerLogSourceID := uuid.New()
+	workspaceFolder := filepath.Join(t.TempDir(), "mywork")
+	t.Logf("Workspace folder: %s", workspaceFolder)
+	devcontainerPath := filepath.Join(workspaceFolder, ".devcontainer")
+	err = os.MkdirAll(devcontainerPath, 0o755)
+	require.NoError(t, err, "create devcontainer directory")
+	devcontainerFile := filepath.Join(devcontainerPath, "devcontainer.json")
+	err = os.WriteFile(devcontainerFile, []byte(`{
+        "name": "mywork",
+        "image": "busybox:latest",
+        "cmd": ["sleep", "infinity"]
+    }`), 0o600)
+	require.NoError(t, err, "write devcontainer.json")
+
+	manifest := agentsdk.Manifest{
+		// Set up pre-conditions for auto-starting a devcontainer, the
+		// script is used to extract the log source ID.
+		Devcontainers: []codersdk.WorkspaceAgentDevcontainer{
+			{
+				ID:              devcontainerID,
+				Name:            "test",
+				WorkspaceFolder: workspaceFolder,
+			},
+		},
+		Scripts: []codersdk.WorkspaceAgentScript{
+			{
+				ID:          devcontainerID,
+				LogSourceID: devcontainerLogSourceID,
+			},
+		},
+	}
+
+	//nolint:dogsled
+	conn, client, _, _, _ := setupAgent(t, manifest, 0, func(_ *agenttest.Client, o *agent.Options) {
+		o.ExperimentalDevcontainersEnabled = true
+	})
+
+	ctx := testutil.Context(t, testutil.WaitLong)
+
+	// We enabled autostart for the devcontainer, so ready is a good
+	// indication that the devcontainer is up and running. Importantly,
+	// this also means that the devcontainer startup is no longer
+	// producing logs that may interfere with the recreate logs.
+	testutil.Eventually(ctx, t, func(context.Context) bool {
+		states := client.GetLifecycleStates()
+		return slices.Contains(states, codersdk.WorkspaceAgentLifecycleReady)
+	}, testutil.IntervalMedium, "devcontainer not ready")
+
+	t.Logf("Looking for container with label: devcontainer.local_folder=%s", workspaceFolder)
+
+	var container codersdk.WorkspaceAgentContainer
+	testutil.Eventually(ctx, t, func(context.Context) bool {
+		resp, err := conn.ListContainers(ctx)
+		if err != nil {
+			t.Logf("Error listing containers: %v", err)
+			return false
+		}
+		for _, c := range resp.Containers {
+			t.Logf("Found container: %s with labels: %v", c.ID[:12], c.Labels)
+			if v, ok := c.Labels["devcontainer.local_folder"]; ok && v == workspaceFolder {
+				t.Logf("Found matching container: %s", c.ID[:12])
+				container = c
+				return true
+			}
+		}
+		return false
+	}, testutil.IntervalMedium, "no container with workspace folder label found")
+	defer func(container codersdk.WorkspaceAgentContainer) {
+		// We can't rely on pool here because the container is not
+		// managed by it (it is managed by @devcontainer/cli).
+		err := pool.Client.RemoveContainer(docker.RemoveContainerOptions{
+			ID:            container.ID,
+			RemoveVolumes: true,
+			Force:         true,
+		})
+		assert.Error(t, err, "container should be removed by recreate")
+	}(container)
+
+	ctx = testutil.Context(t, testutil.WaitLong) // Reset context.
+
+	// Capture logs via ScriptLogger.
+	logsCh := make(chan *proto.BatchCreateLogsRequest, 1)
+	client.SetLogsChannel(logsCh)
+
+	// Invoke recreate to trigger the destruction and recreation of the
+	// devcontainer, we do it in a goroutine so we can process logs
+	// concurrently.
+	go func(container codersdk.WorkspaceAgentContainer) {
+		_, err := conn.RecreateDevcontainer(ctx, container.ID)
+		assert.NoError(t, err, "recreate devcontainer should succeed")
+	}(container)
+
+	t.Logf("Checking recreate logs for outcome...")
+
+	// Wait for the logs to be emitted, the @devcontainer/cli up command
+	// will emit a log with the outcome at the end suggesting we did
+	// receive all the logs.
+waitForOutcomeLoop:
+	for {
+		batch := testutil.RequireReceive(ctx, t, logsCh)
+
+		if bytes.Equal(batch.LogSourceId, devcontainerLogSourceID[:]) {
+			for _, log := range batch.Logs {
+				t.Logf("Received log: %s", log.Output)
+				if strings.Contains(log.Output, "\"outcome\"") {
+					break waitForOutcomeLoop
+				}
+			}
+		}
+	}
+
+	t.Logf("Checking there's a new container with label: devcontainer.local_folder=%s", workspaceFolder)
+
+	// Make sure the container exists and isn't the same as the old one.
+	testutil.Eventually(ctx, t, func(context.Context) bool {
+		resp, err := conn.ListContainers(ctx)
+		if err != nil {
+			t.Logf("Error listing containers: %v", err)
+			return false
+		}
+		for _, c := range resp.Containers {
+			t.Logf("Found container: %s with labels: %v", c.ID[:12], c.Labels)
+			if v, ok := c.Labels["devcontainer.local_folder"]; ok && v == workspaceFolder {
+				if c.ID == container.ID {
+					t.Logf("Found same container: %s", c.ID[:12])
+					return false
+				}
+				t.Logf("Found new container: %s", c.ID[:12])
+				container = c
+				return true
+			}
+		}
+		return false
+	}, testutil.IntervalMedium, "new devcontainer not found")
+	defer func(container codersdk.WorkspaceAgentContainer) {
+		// We can't rely on pool here because the container is not
+		// managed by it (it is managed by @devcontainer/cli).
+		err := pool.Client.RemoveContainer(docker.RemoveContainerOptions{
+			ID:            container.ID,
+			RemoveVolumes: true,
+			Force:         true,
+		})
+		assert.NoError(t, err, "remove container")
+	}(container)
+}
+
 func TestAgent_Dial(t *testing.T) {
 	t.Parallel()
 
@@ -2304,7 +2797,7 @@ done
 
 	n := 1
 	for n <= 5 {
-		logs := testutil.RequireRecvCtx(ctx, t, logsCh)
+		logs := testutil.TryReceive(ctx, t, logsCh)
 		require.NotNil(t, logs)
 		for _, l := range logs.GetLogs() {
 			require.Equal(t, fmt.Sprintf("start %d", n), l.GetOutput())
@@ -2317,7 +2810,7 @@ done
 
 	n = 1
 	for n <= 3000 {
-		logs := testutil.RequireRecvCtx(ctx, t, logsCh)
+		logs := testutil.TryReceive(ctx, t, logsCh)
 		require.NotNil(t, logs)
 		for _, l := range logs.GetLogs() {
 			require.Equal(t, fmt.Sprintf("stop %d", n), l.GetOutput())
@@ -2732,3 +3225,35 @@ func requireEcho(t *testing.T, conn net.Conn) {
 	require.NoError(t, err)
 	require.Equal(t, "test", string(b))
 }
+
+func assertConnectionReport(t testing.TB, agentClient *agenttest.Client, connectionType proto.Connection_Type, status int, reason string) {
+	t.Helper()
+
+	var reports []*proto.ReportConnectionRequest
+	if !assert.Eventually(t, func() bool {
+		reports = agentClient.GetConnectionReports()
+		return len(reports) >= 2
+	}, testutil.WaitMedium, testutil.IntervalFast, "waiting for 2 connection reports or more; got %d", len(reports)) {
+		return
+	}
+
+	assert.Len(t, reports, 2, "want 2 connection reports")
+
+	assert.Equal(t, proto.Connection_CONNECT, reports[0].GetConnection().GetAction(), "first report should be connect")
+	assert.Equal(t, proto.Connection_DISCONNECT, reports[1].GetConnection().GetAction(), "second report should be disconnect")
+	assert.Equal(t, connectionType, reports[0].GetConnection().GetType(), "connect type should be %s", connectionType)
+	assert.Equal(t, connectionType, reports[1].GetConnection().GetType(), "disconnect type should be %s", connectionType)
+	t1 := reports[0].GetConnection().GetTimestamp().AsTime()
+	t2 := reports[1].GetConnection().GetTimestamp().AsTime()
+	assert.True(t, t1.Before(t2) || t1.Equal(t2), "connect timestamp should be before or equal to disconnect timestamp")
+	assert.NotEmpty(t, reports[0].GetConnection().GetIp(), "connect ip should not be empty")
+	assert.NotEmpty(t, reports[1].GetConnection().GetIp(), "disconnect ip should not be empty")
+	assert.Equal(t, 0, int(reports[0].GetConnection().GetStatusCode()), "connect status code should be 0")
+	assert.Equal(t, status, int(reports[1].GetConnection().GetStatusCode()), "disconnect status code should be %d", status)
+	assert.Equal(t, "", reports[0].GetConnection().GetReason(), "connect reason should be empty")
+	if reason != "" {
+		assert.Contains(t, reports[1].GetConnection().GetReason(), reason, "disconnect reason should contain %s", reason)
+	} else {
+		t.Logf("connection report disconnect reason: %s", reports[1].GetConnection().GetReason())
+	}
+}

agent/agentcontainers/acmock/acmock.go

@@ -1,9 +1,9 @@
 // Code generated by MockGen. DO NOT EDIT.
-// Source: .. (interfaces: Lister)
+// Source: .. (interfaces: Lister,DevcontainerCLI)
 //
 // Generated by this command:
 //
-//	mockgen -destination ./acmock.go -package acmock .. Lister
+//	mockgen -destination ./acmock.go -package acmock .. Lister,DevcontainerCLI
 //
 
 // Package acmock is a generated GoMock package.
@@ -13,6 +13,7 @@ import (
 	context "context"
 	reflect "reflect"
 
+	agentcontainers "github.com/coder/coder/v2/agent/agentcontainers"
 	codersdk "github.com/coder/coder/v2/codersdk"
 	gomock "go.uber.org/mock/gomock"
 )
@@ -55,3 +56,47 @@ func (mr *MockListerMockRecorder) List(ctx any) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "List", reflect.TypeOf((*MockLister)(nil).List), ctx)
 }
+
+// MockDevcontainerCLI is a mock of DevcontainerCLI interface.
+type MockDevcontainerCLI struct {
+	ctrl     *gomock.Controller
+	recorder *MockDevcontainerCLIMockRecorder
+	isgomock struct{}
+}
+
+// MockDevcontainerCLIMockRecorder is the mock recorder for MockDevcontainerCLI.
+type MockDevcontainerCLIMockRecorder struct {
+	mock *MockDevcontainerCLI
+}
+
+// NewMockDevcontainerCLI creates a new mock instance.
+func NewMockDevcontainerCLI(ctrl *gomock.Controller) *MockDevcontainerCLI {
+	mock := &MockDevcontainerCLI{ctrl: ctrl}
+	mock.recorder = &MockDevcontainerCLIMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use.
+func (m *MockDevcontainerCLI) EXPECT() *MockDevcontainerCLIMockRecorder {
+	return m.recorder
+}
+
+// Up mocks base method.
+func (m *MockDevcontainerCLI) Up(ctx context.Context, workspaceFolder, configPath string, opts ...agentcontainers.DevcontainerCLIUpOptions) (string, error) {
+	m.ctrl.T.Helper()
+	varargs := []any{ctx, workspaceFolder, configPath}
+	for _, a := range opts {
+		varargs = append(varargs, a)
+	}
+	ret := m.ctrl.Call(m, "Up", varargs...)
+	ret0, _ := ret[0].(string)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// Up indicates an expected call of Up.
+func (mr *MockDevcontainerCLIMockRecorder) Up(ctx, workspaceFolder, configPath any, opts ...any) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	varargs := append([]any{ctx, workspaceFolder, configPath}, opts...)
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Up", reflect.TypeOf((*MockDevcontainerCLI)(nil).Up), varargs...)
+}

agent/agentcontainers/acmock/doc.go

@@ -1,4 +1,4 @@
 // Package acmock contains a mock implementation of agentcontainers.Lister for use in tests.
 package acmock
 
-//go:generate mockgen -destination ./acmock.go -package acmock .. Lister
+//go:generate mockgen -destination ./acmock.go -package acmock .. Lister,DevcontainerCLI

agent/agentcontainers/api.go

@@ -0,0 +1,827 @@
+package agentcontainers
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net/http"
+	"path"
+	"slices"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/fsnotify/fsnotify"
+	"github.com/go-chi/chi/v5"
+	"github.com/google/uuid"
+	"golang.org/x/xerrors"
+
+	"cdr.dev/slog"
+	"github.com/coder/coder/v2/agent/agentcontainers/watcher"
+	"github.com/coder/coder/v2/agent/agentexec"
+	"github.com/coder/coder/v2/coderd/httpapi"
+	"github.com/coder/coder/v2/codersdk"
+	"github.com/coder/coder/v2/codersdk/agentsdk"
+	"github.com/coder/quartz"
+)
+
+const (
+	defaultUpdateInterval = 10 * time.Second
+	listContainersTimeout = 15 * time.Second
+)
+
+// API is responsible for container-related operations in the agent.
+// It provides methods to list and manage containers.
+type API struct {
+	ctx               context.Context
+	cancel            context.CancelFunc
+	watcherDone       chan struct{}
+	updaterDone       chan struct{}
+	initialUpdateDone chan struct{}   // Closed after first update in updaterLoop.
+	updateTrigger     chan chan error // Channel to trigger manual refresh.
+	updateInterval    time.Duration   // Interval for periodic container updates.
+	logger            slog.Logger
+	watcher           watcher.Watcher
+	execer            agentexec.Execer
+	cl                Lister
+	dccli             DevcontainerCLI
+	clock             quartz.Clock
+	scriptLogger      func(logSourceID uuid.UUID) ScriptLogger
+
+	mu                      sync.RWMutex
+	closed                  bool
+	containers              codersdk.WorkspaceAgentListContainersResponse  // Output from the last list operation.
+	containersErr           error                                          // Error from the last list operation.
+	devcontainerNames       map[string]bool                                // By devcontainer name.
+	knownDevcontainers      map[string]codersdk.WorkspaceAgentDevcontainer // By workspace folder.
+	configFileModifiedTimes map[string]time.Time                           // By config file path.
+	recreateSuccessTimes    map[string]time.Time                           // By workspace folder.
+	recreateErrorTimes      map[string]time.Time                           // By workspace folder.
+	recreateWg              sync.WaitGroup
+
+	devcontainerLogSourceIDs map[string]uuid.UUID // By workspace folder.
+}
+
+// Option is a functional option for API.
+type Option func(*API)
+
+// WithClock sets the quartz.Clock implementation to use.
+// This is primarily used for testing to control time.
+func WithClock(clock quartz.Clock) Option {
+	return func(api *API) {
+		api.clock = clock
+	}
+}
+
+// WithExecer sets the agentexec.Execer implementation to use.
+func WithExecer(execer agentexec.Execer) Option {
+	return func(api *API) {
+		api.execer = execer
+	}
+}
+
+// WithLister sets the agentcontainers.Lister implementation to use.
+// The default implementation uses the Docker CLI to list containers.
+func WithLister(cl Lister) Option {
+	return func(api *API) {
+		api.cl = cl
+	}
+}
+
+// WithDevcontainerCLI sets the DevcontainerCLI implementation to use.
+// This can be used in tests to modify @devcontainer/cli behavior.
+func WithDevcontainerCLI(dccli DevcontainerCLI) Option {
+	return func(api *API) {
+		api.dccli = dccli
+	}
+}
+
+// WithDevcontainers sets the known devcontainers for the API. This
+// allows the API to be aware of devcontainers defined in the workspace
+// agent manifest.
+func WithDevcontainers(devcontainers []codersdk.WorkspaceAgentDevcontainer, scripts []codersdk.WorkspaceAgentScript) Option {
+	return func(api *API) {
+		if len(devcontainers) == 0 {
+			return
+		}
+		api.knownDevcontainers = make(map[string]codersdk.WorkspaceAgentDevcontainer, len(devcontainers))
+		api.devcontainerNames = make(map[string]bool, len(devcontainers))
+		api.devcontainerLogSourceIDs = make(map[string]uuid.UUID)
+		for _, dc := range devcontainers {
+			api.knownDevcontainers[dc.WorkspaceFolder] = dc
+			api.devcontainerNames[dc.Name] = true
+			for _, script := range scripts {
+				// The devcontainer scripts match the devcontainer ID for
+				// identification.
+				if script.ID == dc.ID {
+					api.devcontainerLogSourceIDs[dc.WorkspaceFolder] = script.LogSourceID
+					break
+				}
+			}
+			if api.devcontainerLogSourceIDs[dc.WorkspaceFolder] == uuid.Nil {
+				api.logger.Error(api.ctx, "devcontainer log source ID not found for devcontainer",
+					slog.F("devcontainer_id", dc.ID),
+					slog.F("devcontainer_name", dc.Name),
+					slog.F("workspace_folder", dc.WorkspaceFolder),
+					slog.F("config_path", dc.ConfigPath),
+				)
+			}
+		}
+	}
+}
+
+// WithWatcher sets the file watcher implementation to use. By default a
+// noop watcher is used. This can be used in tests to modify the watcher
+// behavior or to use an actual file watcher (e.g. fsnotify).
+func WithWatcher(w watcher.Watcher) Option {
+	return func(api *API) {
+		api.watcher = w
+	}
+}
+
+// ScriptLogger is an interface for sending devcontainer logs to the
+// controlplane.
+type ScriptLogger interface {
+	Send(ctx context.Context, log ...agentsdk.Log) error
+	Flush(ctx context.Context) error
+}
+
+// noopScriptLogger is a no-op implementation of the ScriptLogger
+// interface.
+type noopScriptLogger struct{}
+
+func (noopScriptLogger) Send(context.Context, ...agentsdk.Log) error { return nil }
+func (noopScriptLogger) Flush(context.Context) error                 { return nil }
+
+// WithScriptLogger sets the script logger provider for devcontainer operations.
+func WithScriptLogger(scriptLogger func(logSourceID uuid.UUID) ScriptLogger) Option {
+	return func(api *API) {
+		api.scriptLogger = scriptLogger
+	}
+}
+
+// NewAPI returns a new API with the given options applied.
+func NewAPI(logger slog.Logger, options ...Option) *API {
+	ctx, cancel := context.WithCancel(context.Background())
+	api := &API{
+		ctx:                     ctx,
+		cancel:                  cancel,
+		watcherDone:             make(chan struct{}),
+		updaterDone:             make(chan struct{}),
+		initialUpdateDone:       make(chan struct{}),
+		updateTrigger:           make(chan chan error),
+		updateInterval:          defaultUpdateInterval,
+		logger:                  logger,
+		clock:                   quartz.NewReal(),
+		execer:                  agentexec.DefaultExecer,
+		devcontainerNames:       make(map[string]bool),
+		knownDevcontainers:      make(map[string]codersdk.WorkspaceAgentDevcontainer),
+		configFileModifiedTimes: make(map[string]time.Time),
+		recreateSuccessTimes:    make(map[string]time.Time),
+		recreateErrorTimes:      make(map[string]time.Time),
+		scriptLogger:            func(uuid.UUID) ScriptLogger { return noopScriptLogger{} },
+	}
+	// The ctx and logger must be set before applying options to avoid
+	// nil pointer dereference.
+	for _, opt := range options {
+		opt(api)
+	}
+	if api.cl == nil {
+		api.cl = NewDocker(api.execer)
+	}
+	if api.dccli == nil {
+		api.dccli = NewDevcontainerCLI(logger.Named("devcontainer-cli"), api.execer)
+	}
+	if api.watcher == nil {
+		var err error
+		api.watcher, err = watcher.NewFSNotify()
+		if err != nil {
+			logger.Error(ctx, "create file watcher service failed", slog.Error(err))
+			api.watcher = watcher.NewNoop()
+		}
+	}
+
+	go api.watcherLoop()
+	go api.updaterLoop()
+
+	return api
+}
+
+func (api *API) watcherLoop() {
+	defer close(api.watcherDone)
+	defer api.logger.Debug(api.ctx, "watcher loop stopped")
+	api.logger.Debug(api.ctx, "watcher loop started")
+
+	for {
+		event, err := api.watcher.Next(api.ctx)
+		if err != nil {
+			if errors.Is(err, watcher.ErrClosed) {
+				api.logger.Debug(api.ctx, "watcher closed")
+				return
+			}
+			if api.ctx.Err() != nil {
+				api.logger.Debug(api.ctx, "api context canceled")
+				return
+			}
+			api.logger.Error(api.ctx, "watcher error waiting for next event", slog.Error(err))
+			continue
+		}
+		if event == nil {
+			continue
+		}
+
+		now := api.clock.Now("watcherLoop")
+		switch {
+		case event.Has(fsnotify.Create | fsnotify.Write):
+			api.logger.Debug(api.ctx, "devcontainer config file changed", slog.F("file", event.Name))
+			api.markDevcontainerDirty(event.Name, now)
+		case event.Has(fsnotify.Remove):
+			api.logger.Debug(api.ctx, "devcontainer config file removed", slog.F("file", event.Name))
+			api.markDevcontainerDirty(event.Name, now)
+		case event.Has(fsnotify.Rename):
+			api.logger.Debug(api.ctx, "devcontainer config file renamed", slog.F("file", event.Name))
+			api.markDevcontainerDirty(event.Name, now)
+		default:
+			api.logger.Debug(api.ctx, "devcontainer config file event ignored", slog.F("file", event.Name), slog.F("event", event))
+		}
+	}
+}
+
+// updaterLoop is responsible for periodically updating the container
+// list and handling manual refresh requests.
+func (api *API) updaterLoop() {
+	defer close(api.updaterDone)
+	defer api.logger.Debug(api.ctx, "updater loop stopped")
+	api.logger.Debug(api.ctx, "updater loop started")
+
+	// Perform an initial update to populate the container list, this
+	// gives us a guarantee that the API has loaded the initial state
+	// before returning any responses. This is useful for both tests
+	// and anyone looking to interact with the API.
+	api.logger.Debug(api.ctx, "performing initial containers update")
+	if err := api.updateContainers(api.ctx); err != nil {
+		api.logger.Error(api.ctx, "initial containers update failed", slog.Error(err))
+	} else {
+		api.logger.Debug(api.ctx, "initial containers update complete")
+	}
+	// Signal that the initial update attempt (successful or not) is done.
+	// Other services can wait on this if they need the first data to be available.
+	close(api.initialUpdateDone)
+
+	// We utilize a TickerFunc here instead of a regular Ticker so that
+	// we can guarantee execution of the updateContainers method after
+	// advancing the clock.
+	ticker := api.clock.TickerFunc(api.ctx, api.updateInterval, func() error {
+		done := make(chan error, 1)
+		defer close(done)
+
+		select {
+		case <-api.ctx.Done():
+			return api.ctx.Err()
+		case api.updateTrigger <- done:
+			err := <-done
+			if err != nil {
+				api.logger.Error(api.ctx, "updater loop ticker failed", slog.Error(err))
+			}
+		default:
+			api.logger.Debug(api.ctx, "updater loop ticker skipped, update in progress")
+		}
+
+		return nil // Always nil to keep the ticker going.
+	}, "updaterLoop")
+	defer func() {
+		if err := ticker.Wait("updaterLoop"); err != nil && !errors.Is(err, context.Canceled) {
+			api.logger.Error(api.ctx, "updater loop ticker failed", slog.Error(err))
+		}
+	}()
+
+	for {
+		select {
+		case <-api.ctx.Done():
+			return
+		case done := <-api.updateTrigger:
+			// Note that although we pass api.ctx here, updateContainers
+			// has an internal timeout to prevent long blocking calls.
+			done <- api.updateContainers(api.ctx)
+		}
+	}
+}
+
+// Routes returns the HTTP handler for container-related routes.
+func (api *API) Routes() http.Handler {
+	r := chi.NewRouter()
+
+	ensureInitialUpdateDoneMW := func(next http.Handler) http.Handler {
+		return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
+			select {
+			case <-api.ctx.Done():
+				httpapi.Write(r.Context(), rw, http.StatusServiceUnavailable, codersdk.Response{
+					Message: "API closed",
+					Detail:  "The API is closed and cannot process requests.",
+				})
+				return
+			case <-r.Context().Done():
+				return
+			case <-api.initialUpdateDone:
+				// Initial update is done, we can start processing
+				// requests.
+			}
+			next.ServeHTTP(rw, r)
+		})
+	}
+
+	// For now, all endpoints require the initial update to be done.
+	// If we want to allow some endpoints to be available before
+	// the initial update, we can enable this per-route.
+	r.Use(ensureInitialUpdateDoneMW)
+
+	r.Get("/", api.handleList)
+	r.Route("/devcontainers", func(r chi.Router) {
+		r.Get("/", api.handleDevcontainersList)
+		r.Post("/container/{container}/recreate", api.handleDevcontainerRecreate)
+	})
+
+	return r
+}
+
+// handleList handles the HTTP request to list containers.
+func (api *API) handleList(rw http.ResponseWriter, r *http.Request) {
+	ct, err := api.getContainers()
+	if err != nil {
+		httpapi.Write(r.Context(), rw, http.StatusInternalServerError, codersdk.Response{
+			Message: "Could not get containers",
+			Detail:  err.Error(),
+		})
+		return
+	}
+	httpapi.Write(r.Context(), rw, http.StatusOK, ct)
+}
+
+// updateContainers fetches the latest container list, processes it, and
+// updates the cache. It performs locking for updating shared API state.
+func (api *API) updateContainers(ctx context.Context) error {
+	listCtx, listCancel := context.WithTimeout(ctx, listContainersTimeout)
+	defer listCancel()
+
+	updated, err := api.cl.List(listCtx)
+	if err != nil {
+		// If the context was canceled, we hold off on clearing the
+		// containers cache. This is to avoid clearing the cache if
+		// the update was canceled due to a timeout. Hopefully this
+		// will clear up on the next update.
+		if !errors.Is(err, context.Canceled) {
+			api.mu.Lock()
+			api.containers = codersdk.WorkspaceAgentListContainersResponse{}
+			api.containersErr = err
+			api.mu.Unlock()
+		}
+
+		return xerrors.Errorf("list containers failed: %w", err)
+	}
+
+	api.mu.Lock()
+	defer api.mu.Unlock()
+
+	api.processUpdatedContainersLocked(ctx, updated)
+
+	api.logger.Debug(ctx, "containers updated successfully", slog.F("container_count", len(api.containers.Containers)), slog.F("warning_count", len(api.containers.Warnings)), slog.F("devcontainer_count", len(api.knownDevcontainers)))
+
+	return nil
+}
+
+// processUpdatedContainersLocked updates the devcontainer state based
+// on the latest list of containers. This method assumes that api.mu is
+// held.
+func (api *API) processUpdatedContainersLocked(ctx context.Context, updated codersdk.WorkspaceAgentListContainersResponse) {
+	// Reset the container links in known devcontainers to detect if
+	// they still exist.
+	for _, dc := range api.knownDevcontainers {
+		dc.Container = nil
+		api.knownDevcontainers[dc.WorkspaceFolder] = dc
+	}
+
+	// Check if the container is running and update the known devcontainers.
+	for i := range updated.Containers {
+		container := &updated.Containers[i] // Grab a reference to the container to allow mutating it.
+		container.DevcontainerStatus = ""   // Reset the status for the container (updated later).
+		container.DevcontainerDirty = false // Reset dirty state for the container (updated later).
+
+		workspaceFolder := container.Labels[DevcontainerLocalFolderLabel]
+		configFile := container.Labels[DevcontainerConfigFileLabel]
+
+		if workspaceFolder == "" {
+			continue
+		}
+
+		if dc, ok := api.knownDevcontainers[workspaceFolder]; ok {
+			// If no config path is set, this devcontainer was defined
+			// in Terraform without the optional config file. Assume the
+			// first container with the workspace folder label is the
+			// one we want to use.
+			if dc.ConfigPath == "" && configFile != "" {
+				dc.ConfigPath = configFile
+				if err := api.watcher.Add(configFile); err != nil {
+					api.logger.Error(ctx, "watch devcontainer config file failed", slog.Error(err), slog.F("file", configFile))
+				}
+			}
+
+			dc.Container = container
+			api.knownDevcontainers[dc.WorkspaceFolder] = dc
+			continue
+		}
+
+		// NOTE(mafredri): This name impl. may change to accommodate devcontainer agents RFC.
+		// If not in our known list, add as a runtime detected entry.
+		name := path.Base(workspaceFolder)
+		if api.devcontainerNames[name] {
+			// Try to find a unique name by appending a number.
+			for i := 2; ; i++ {
+				newName := fmt.Sprintf("%s-%d", name, i)
+				if !api.devcontainerNames[newName] {
+					name = newName
+					break
+				}
+			}
+		}
+		api.devcontainerNames[name] = true
+		if configFile != "" {
+			if err := api.watcher.Add(configFile); err != nil {
+				api.logger.Error(ctx, "watch devcontainer config file failed", slog.Error(err), slog.F("file", configFile))
+			}
+		}
+
+		api.knownDevcontainers[workspaceFolder] = codersdk.WorkspaceAgentDevcontainer{
+			ID:              uuid.New(),
+			Name:            name,
+			WorkspaceFolder: workspaceFolder,
+			ConfigPath:      configFile,
+			Status:          "",    // Updated later based on container state.
+			Dirty:           false, // Updated later based on config file changes.
+			Container:       container,
+		}
+	}
+
+	// Iterate through all known devcontainers and update their status
+	// based on the current state of the containers.
+	for _, dc := range api.knownDevcontainers {
+		switch {
+		case dc.Status == codersdk.WorkspaceAgentDevcontainerStatusStarting:
+			if dc.Container != nil {
+				dc.Container.DevcontainerStatus = dc.Status
+				dc.Container.DevcontainerDirty = dc.Dirty
+			}
+			continue // This state is handled by the recreation routine.
+
+		case dc.Status == codersdk.WorkspaceAgentDevcontainerStatusError && (dc.Container == nil || dc.Container.CreatedAt.Before(api.recreateErrorTimes[dc.WorkspaceFolder])):
+			if dc.Container != nil {
+				dc.Container.DevcontainerStatus = dc.Status
+				dc.Container.DevcontainerDirty = dc.Dirty
+			}
+			continue // The devcontainer needs to be recreated.
+
+		case dc.Container != nil:
+			dc.Status = codersdk.WorkspaceAgentDevcontainerStatusStopped
+			if dc.Container.Running {
+				dc.Status = codersdk.WorkspaceAgentDevcontainerStatusRunning
+			}
+			dc.Container.DevcontainerStatus = dc.Status
+
+			dc.Dirty = false
+			if lastModified, hasModTime := api.configFileModifiedTimes[dc.ConfigPath]; hasModTime && dc.Container.CreatedAt.Before(lastModified) {
+				dc.Dirty = true
+			}
+			dc.Container.DevcontainerDirty = dc.Dirty
+
+		case dc.Container == nil:
+			dc.Status = codersdk.WorkspaceAgentDevcontainerStatusStopped
+			dc.Dirty = false
+		}
+
+		delete(api.recreateErrorTimes, dc.WorkspaceFolder)
+		api.knownDevcontainers[dc.WorkspaceFolder] = dc
+	}
+
+	api.containers = updated
+	api.containersErr = nil
+}
+
+// refreshContainers triggers an immediate update of the container list
+// and waits for it to complete.
+func (api *API) refreshContainers(ctx context.Context) (err error) {
+	defer func() {
+		if err != nil {
+			err = xerrors.Errorf("refresh containers failed: %w", err)
+		}
+	}()
+
+	done := make(chan error, 1)
+	select {
+	case <-api.ctx.Done():
+		return xerrors.Errorf("API closed: %w", api.ctx.Err())
+	case <-ctx.Done():
+		return ctx.Err()
+	case api.updateTrigger <- done:
+		select {
+		case <-api.ctx.Done():
+			return xerrors.Errorf("API closed: %w", api.ctx.Err())
+		case <-ctx.Done():
+			return ctx.Err()
+		case err := <-done:
+			return err
+		}
+	}
+}
+
+func (api *API) getContainers() (codersdk.WorkspaceAgentListContainersResponse, error) {
+	api.mu.RLock()
+	defer api.mu.RUnlock()
+
+	if api.containersErr != nil {
+		return codersdk.WorkspaceAgentListContainersResponse{}, api.containersErr
+	}
+	return codersdk.WorkspaceAgentListContainersResponse{
+		Containers: slices.Clone(api.containers.Containers),
+		Warnings:   slices.Clone(api.containers.Warnings),
+	}, nil
+}
+
+// handleDevcontainerRecreate handles the HTTP request to recreate a
+// devcontainer by referencing the container.
+func (api *API) handleDevcontainerRecreate(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+	containerID := chi.URLParam(r, "container")
+
+	if containerID == "" {
+		httpapi.Write(ctx, w, http.StatusBadRequest, codersdk.Response{
+			Message: "Missing container ID or name",
+			Detail:  "Container ID or name is required to recreate a devcontainer.",
+		})
+		return
+	}
+
+	containers, err := api.getContainers()
+	if err != nil {
+		httpapi.Write(ctx, w, http.StatusInternalServerError, codersdk.Response{
+			Message: "Could not list containers",
+			Detail:  err.Error(),
+		})
+		return
+	}
+
+	containerIdx := slices.IndexFunc(containers.Containers, func(c codersdk.WorkspaceAgentContainer) bool { return c.Match(containerID) })
+	if containerIdx == -1 {
+		httpapi.Write(ctx, w, http.StatusNotFound, codersdk.Response{
+			Message: "Container not found",
+			Detail:  "Container ID or name not found in the list of containers.",
+		})
+		return
+	}
+
+	container := containers.Containers[containerIdx]
+	workspaceFolder := container.Labels[DevcontainerLocalFolderLabel]
+	configPath := container.Labels[DevcontainerConfigFileLabel]
+
+	// Workspace folder is required to recreate a container, we don't verify
+	// the config path here because it's optional.
+	if workspaceFolder == "" {
+		httpapi.Write(ctx, w, http.StatusBadRequest, codersdk.Response{
+			Message: "Missing workspace folder label",
+			Detail:  "The container is not a devcontainer, the container must have the workspace folder label to support recreation.",
+		})
+		return
+	}
+
+	api.mu.Lock()
+
+	dc, ok := api.knownDevcontainers[workspaceFolder]
+	switch {
+	case !ok:
+		api.mu.Unlock()
+
+		// This case should not happen if the container is a valid devcontainer.
+		api.logger.Error(ctx, "devcontainer not found for workspace folder", slog.F("workspace_folder", workspaceFolder))
+		httpapi.Write(ctx, w, http.StatusInternalServerError, codersdk.Response{
+			Message: "Devcontainer not found.",
+			Detail:  fmt.Sprintf("Could not find devcontainer for workspace folder: %q", workspaceFolder),
+		})
+		return
+	case dc.Status == codersdk.WorkspaceAgentDevcontainerStatusStarting:
+		api.mu.Unlock()
+
+		httpapi.Write(ctx, w, http.StatusConflict, codersdk.Response{
+			Message: "Devcontainer recreation already in progress",
+			Detail:  fmt.Sprintf("Recreation for workspace folder %q is already underway.", dc.WorkspaceFolder),
+		})
+		return
+	}
+
+	// Update the status so that we don't try to recreate the
+	// devcontainer multiple times in parallel.
+	dc.Status = codersdk.WorkspaceAgentDevcontainerStatusStarting
+	if dc.Container != nil {
+		dc.Container.DevcontainerStatus = dc.Status
+	}
+	api.knownDevcontainers[dc.WorkspaceFolder] = dc
+	api.recreateWg.Add(1)
+	go api.recreateDevcontainer(dc, configPath)
+
+	api.mu.Unlock()
+
+	httpapi.Write(ctx, w, http.StatusAccepted, codersdk.Response{
+		Message: "Devcontainer recreation initiated",
+		Detail:  fmt.Sprintf("Recreation process for workspace folder %q has started.", dc.WorkspaceFolder),
+	})
+}
+
+// recreateDevcontainer should run in its own goroutine and is responsible for
+// recreating a devcontainer based on the provided devcontainer configuration.
+// It updates the devcontainer status and logs the process. The configPath is
+// passed as a parameter for the odd chance that the container being recreated
+// has a different config file than the one stored in the devcontainer state.
+// The devcontainer state must be set to starting and the recreateWg must be
+// incremented before calling this function.
+func (api *API) recreateDevcontainer(dc codersdk.WorkspaceAgentDevcontainer, configPath string) {
+	defer api.recreateWg.Done()
+
+	var (
+		err    error
+		ctx    = api.ctx
+		logger = api.logger.With(
+			slog.F("devcontainer_id", dc.ID),
+			slog.F("devcontainer_name", dc.Name),
+			slog.F("workspace_folder", dc.WorkspaceFolder),
+			slog.F("config_path", configPath),
+		)
+	)
+
+	if dc.ConfigPath != configPath {
+		logger.Warn(ctx, "devcontainer config path mismatch",
+			slog.F("config_path_param", configPath),
+		)
+	}
+
+	// Send logs via agent logging facilities.
+	logSourceID := api.devcontainerLogSourceIDs[dc.WorkspaceFolder]
+	if logSourceID == uuid.Nil {
+		// Fallback to the external log source ID if not found.
+		logSourceID = agentsdk.ExternalLogSourceID
+	}
+
+	scriptLogger := api.scriptLogger(logSourceID)
+	defer func() {
+		flushCtx, cancel := context.WithTimeout(api.ctx, 5*time.Second)
+		defer cancel()
+		if err := scriptLogger.Flush(flushCtx); err != nil {
+			logger.Error(flushCtx, "flush devcontainer logs failed during recreation", slog.Error(err))
+		}
+	}()
+	infoW := agentsdk.LogsWriter(ctx, scriptLogger.Send, logSourceID, codersdk.LogLevelInfo)
+	defer infoW.Close()
+	errW := agentsdk.LogsWriter(ctx, scriptLogger.Send, logSourceID, codersdk.LogLevelError)
+	defer errW.Close()
+
+	logger.Debug(ctx, "starting devcontainer recreation")
+
+	_, err = api.dccli.Up(ctx, dc.WorkspaceFolder, configPath, WithOutput(infoW, errW), WithRemoveExistingContainer())
+	if err != nil {
+		// No need to log if the API is closing (context canceled), as this
+		// is expected behavior when the API is shutting down.
+		if !errors.Is(err, context.Canceled) {
+			logger.Error(ctx, "devcontainer recreation failed", slog.Error(err))
+		}
+
+		api.mu.Lock()
+		dc = api.knownDevcontainers[dc.WorkspaceFolder]
+		dc.Status = codersdk.WorkspaceAgentDevcontainerStatusError
+		if dc.Container != nil {
+			dc.Container.DevcontainerStatus = dc.Status
+		}
+		api.knownDevcontainers[dc.WorkspaceFolder] = dc
+		api.recreateErrorTimes[dc.WorkspaceFolder] = api.clock.Now("recreate", "errorTimes")
+		api.mu.Unlock()
+		return
+	}
+
+	logger.Info(ctx, "devcontainer recreated successfully")
+
+	api.mu.Lock()
+	dc = api.knownDevcontainers[dc.WorkspaceFolder]
+	// Update the devcontainer status to Running or Stopped based on the
+	// current state of the container, changing the status to !starting
+	// allows the update routine to update the devcontainer status, but
+	// to minimize the time between API consistency, we guess the status
+	// based on the container state.
+	dc.Status = codersdk.WorkspaceAgentDevcontainerStatusStopped
+	if dc.Container != nil {
+		if dc.Container.Running {
+			dc.Status = codersdk.WorkspaceAgentDevcontainerStatusRunning
+		}
+		dc.Container.DevcontainerStatus = dc.Status
+	}
+	dc.Dirty = false
+	api.recreateSuccessTimes[dc.WorkspaceFolder] = api.clock.Now("recreate", "successTimes")
+	api.knownDevcontainers[dc.WorkspaceFolder] = dc
+	api.mu.Unlock()
+
+	// Ensure an immediate refresh to accurately reflect the
+	// devcontainer state after recreation.
+	if err := api.refreshContainers(ctx); err != nil {
+		logger.Error(ctx, "failed to trigger immediate refresh after devcontainer recreation", slog.Error(err))
+	}
+}
+
+// handleDevcontainersList handles the HTTP request to list known devcontainers.
+func (api *API) handleDevcontainersList(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+
+	api.mu.RLock()
+	err := api.containersErr
+	devcontainers := make([]codersdk.WorkspaceAgentDevcontainer, 0, len(api.knownDevcontainers))
+	for _, dc := range api.knownDevcontainers {
+		devcontainers = append(devcontainers, dc)
+	}
+	api.mu.RUnlock()
+	if err != nil {
+		httpapi.Write(ctx, w, http.StatusInternalServerError, codersdk.Response{
+			Message: "Could not list containers",
+			Detail:  err.Error(),
+		})
+		return
+	}
+
+	slices.SortFunc(devcontainers, func(a, b codersdk.WorkspaceAgentDevcontainer) int {
+		if cmp := strings.Compare(a.WorkspaceFolder, b.WorkspaceFolder); cmp != 0 {
+			return cmp
+		}
+		return strings.Compare(a.ConfigPath, b.ConfigPath)
+	})
+
+	response := codersdk.WorkspaceAgentDevcontainersResponse{
+		Devcontainers: devcontainers,
+	}
+
+	httpapi.Write(ctx, w, http.StatusOK, response)
+}
+
+// markDevcontainerDirty finds the devcontainer with the given config file path
+// and marks it as dirty. It acquires the lock before modifying the state.
+func (api *API) markDevcontainerDirty(configPath string, modifiedAt time.Time) {
+	api.mu.Lock()
+	defer api.mu.Unlock()
+
+	// Record the timestamp of when this configuration file was modified.
+	api.configFileModifiedTimes[configPath] = modifiedAt
+
+	for _, dc := range api.knownDevcontainers {
+		if dc.ConfigPath != configPath {
+			continue
+		}
+
+		logger := api.logger.With(
+			slog.F("devcontainer_id", dc.ID),
+			slog.F("devcontainer_name", dc.Name),
+			slog.F("workspace_folder", dc.WorkspaceFolder),
+			slog.F("file", configPath),
+			slog.F("modified_at", modifiedAt),
+		)
+
+		// TODO(mafredri): Simplistic mark for now, we should check if the
+		// container is running and if the config file was modified after
+		// the container was created.
+		if !dc.Dirty {
+			logger.Info(api.ctx, "marking devcontainer as dirty")
+			dc.Dirty = true
+		}
+		if dc.Container != nil && !dc.Container.DevcontainerDirty {
+			logger.Info(api.ctx, "marking devcontainer container as dirty")
+			dc.Container.DevcontainerDirty = true
+		}
+
+		api.knownDevcontainers[dc.WorkspaceFolder] = dc
+	}
+}
+
+func (api *API) Close() error {
+	api.mu.Lock()
+	if api.closed {
+		api.mu.Unlock()
+		return nil
+	}
+	api.logger.Debug(api.ctx, "closing API")
+	api.closed = true
+	api.cancel()    // Interrupt all routines.
+	api.mu.Unlock() // Release lock before waiting for goroutines.
+
+	// Close the watcher to ensure its loop finishes.
+	err := api.watcher.Close()
+
+	// Wait for loops to finish.
+	<-api.watcherDone
+	<-api.updaterDone
+
+	// Wait for all devcontainer recreation tasks to complete.
+	api.recreateWg.Wait()
+
+	api.logger.Debug(api.ctx, "closed API")
+	return err
+}

agent/agentcontainers/containers.go

@@ -2,137 +2,10 @@ package agentcontainers
 
 import (
 	"context"
-	"errors"
-	"net/http"
-	"slices"
-	"time"
 
-	"golang.org/x/xerrors"
-
-	"github.com/coder/coder/v2/coderd/httpapi"
 	"github.com/coder/coder/v2/codersdk"
-	"github.com/coder/quartz"
 )
 
-const (
-	defaultGetContainersCacheDuration = 10 * time.Second
-	dockerCreatedAtTimeFormat         = "2006-01-02 15:04:05 -0700 MST"
-	getContainersTimeout              = 5 * time.Second
-)
-
-type devcontainersHandler struct {
-	cacheDuration time.Duration
-	cl            Lister
-	clock         quartz.Clock
-
-	// lockCh protects the below fields. We use a channel instead of a mutex so we
-	// can handle cancellation properly.
-	lockCh     chan struct{}
-	containers *codersdk.WorkspaceAgentListContainersResponse
-	mtime      time.Time
-}
-
-// Option is a functional option for devcontainersHandler.
-type Option func(*devcontainersHandler)
-
-// WithLister sets the agentcontainers.Lister implementation to use.
-// The default implementation uses the Docker CLI to list containers.
-func WithLister(cl Lister) Option {
-	return func(ch *devcontainersHandler) {
-		ch.cl = cl
-	}
-}
-
-// New returns a new devcontainersHandler with the given options applied.
-func New(options ...Option) http.Handler {
-	ch := &devcontainersHandler{
-		lockCh: make(chan struct{}, 1),
-	}
-	for _, opt := range options {
-		opt(ch)
-	}
-	return ch
-}
-
-func (ch *devcontainersHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
-	select {
-	case <-r.Context().Done():
-		// Client went away.
-		return
-	default:
-		ct, err := ch.getContainers(r.Context())
-		if err != nil {
-			if errors.Is(err, context.Canceled) {
-				httpapi.Write(r.Context(), rw, http.StatusRequestTimeout, codersdk.Response{
-					Message: "Could not get containers.",
-					Detail:  "Took too long to list containers.",
-				})
-				return
-			}
-			httpapi.Write(r.Context(), rw, http.StatusInternalServerError, codersdk.Response{
-				Message: "Could not get containers.",
-				Detail:  err.Error(),
-			})
-			return
-		}
-
-		httpapi.Write(r.Context(), rw, http.StatusOK, ct)
-	}
-}
-
-func (ch *devcontainersHandler) getContainers(ctx context.Context) (codersdk.WorkspaceAgentListContainersResponse, error) {
-	select {
-	case <-ctx.Done():
-		return codersdk.WorkspaceAgentListContainersResponse{}, ctx.Err()
-	default:
-		ch.lockCh <- struct{}{}
-	}
-	defer func() {
-		<-ch.lockCh
-	}()
-
-	// make zero-value usable
-	if ch.cacheDuration == 0 {
-		ch.cacheDuration = defaultGetContainersCacheDuration
-	}
-	if ch.cl == nil {
-		ch.cl = &DockerCLILister{}
-	}
-	if ch.containers == nil {
-		ch.containers = &codersdk.WorkspaceAgentListContainersResponse{}
-	}
-	if ch.clock == nil {
-		ch.clock = quartz.NewReal()
-	}
-
-	now := ch.clock.Now()
-	if now.Sub(ch.mtime) < ch.cacheDuration {
-		// Return a copy of the cached data to avoid accidental modification by the caller.
-		cpy := codersdk.WorkspaceAgentListContainersResponse{
-			Containers: slices.Clone(ch.containers.Containers),
-			Warnings:   slices.Clone(ch.containers.Warnings),
-		}
-		return cpy, nil
-	}
-
-	timeoutCtx, timeoutCancel := context.WithTimeout(ctx, getContainersTimeout)
-	defer timeoutCancel()
-	updated, err := ch.cl.List(timeoutCtx)
-	if err != nil {
-		return codersdk.WorkspaceAgentListContainersResponse{}, xerrors.Errorf("get containers: %w", err)
-	}
-	ch.containers = &updated
-	ch.mtime = now
-
-	// Return a copy of the cached data to avoid accidental modification by the
-	// caller.
-	cpy := codersdk.WorkspaceAgentListContainersResponse{
-		Containers: slices.Clone(ch.containers.Containers),
-		Warnings:   slices.Clone(ch.containers.Warnings),
-	}
-	return cpy, nil
-}
-
 // Lister is an interface for listing containers visible to the
 // workspace agent.
 type Lister interface {

agent/agentcontainers/containers_dockercli.go

@@ -6,7 +6,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"os"
+	"net"
 	"os/user"
 	"slices"
 	"sort"
@@ -14,29 +14,20 @@ import (
 	"strings"
 	"time"
 
-	"github.com/coder/coder/v2/agent/agentexec"
-	"github.com/coder/coder/v2/codersdk"
-
 	"golang.org/x/exp/maps"
 	"golang.org/x/xerrors"
+
+	"github.com/coder/coder/v2/agent/agentcontainers/dcspec"
+	"github.com/coder/coder/v2/agent/agentexec"
+	"github.com/coder/coder/v2/agent/usershell"
+	"github.com/coder/coder/v2/coderd/util/ptr"
+	"github.com/coder/coder/v2/codersdk"
 )
 
-// DockerCLILister is a ContainerLister that lists containers using the docker CLI
-type DockerCLILister struct {
-	execer agentexec.Execer
-}
-
-var _ Lister = &DockerCLILister{}
-
-func NewDocker(execer agentexec.Execer) Lister {
-	return &DockerCLILister{
-		execer: agentexec.DefaultExecer,
-	}
-}
-
 // DockerEnvInfoer is an implementation of agentssh.EnvInfoer that returns
 // information about a container.
 type DockerEnvInfoer struct {
+	usershell.SystemEnvInfo
 	container string
 	user      *user.User
 	userShell string
@@ -122,26 +113,13 @@ func EnvInfo(ctx context.Context, execer agentexec.Execer, container, containerU
 	return &dei, nil
 }
 
-func (dei *DockerEnvInfoer) CurrentUser() (*user.User, error) {
+func (dei *DockerEnvInfoer) User() (*user.User, error) {
 	// Clone the user so that the caller can't modify it
 	u := *dei.user
 	return &u, nil
 }
 
-func (*DockerEnvInfoer) Environ() []string {
-	// Return a clone of the environment so that the caller can't modify it
-	return os.Environ()
-}
-
-func (*DockerEnvInfoer) UserHomeDir() (string, error) {
-	// We default the working directory of the command to the user's home
-	// directory. Since this came from inside the container, we cannot guarantee
-	// that this exists on the host. Return the "real" home directory of the user
-	// instead.
-	return os.UserHomeDir()
-}
-
-func (dei *DockerEnvInfoer) UserShell(string) (string, error) {
+func (dei *DockerEnvInfoer) Shell(string) (string, error) {
 	return dei.userShell, nil
 }
 
@@ -174,37 +152,49 @@ func (dei *DockerEnvInfoer) ModifyCommand(cmd string, args ...string) (string, [
 // devcontainerEnv is a helper function that inspects the container labels to
 // find the required environment variables for running a command in the container.
 func devcontainerEnv(ctx context.Context, execer agentexec.Execer, container string) ([]string, error) {
-	ins, stderr, err := runDockerInspect(ctx, execer, container)
+	stdout, stderr, err := runDockerInspect(ctx, execer, container)
 	if err != nil {
 		return nil, xerrors.Errorf("inspect container: %w: %q", err, stderr)
 	}
 
+	ins, _, err := convertDockerInspect(stdout)
+	if err != nil {
+		return nil, xerrors.Errorf("inspect container: %w", err)
+	}
+
 	if len(ins) != 1 {
 		return nil, xerrors.Errorf("inspect container: expected 1 container, got %d", len(ins))
 	}
 
 	in := ins[0]
-	if in.Config.Labels == nil {
+	if in.Labels == nil {
 		return nil, nil
 	}
 
 	// We want to look for the devcontainer metadata, which is in the
 	// value of the label `devcontainer.metadata`.
-	rawMeta, ok := in.Config.Labels["devcontainer.metadata"]
+	rawMeta, ok := in.Labels["devcontainer.metadata"]
 	if !ok {
 		return nil, nil
 	}
-	meta := struct {
-		RemoteEnv map[string]string `json:"remoteEnv"`
-	}{}
+
+	meta := make([]dcspec.DevContainer, 0)
 	if err := json.Unmarshal([]byte(rawMeta), &meta); err != nil {
 		return nil, xerrors.Errorf("unmarshal devcontainer.metadata: %w", err)
 	}
 
 	// The environment variables are stored in the `remoteEnv` key.
-	env := make([]string, 0, len(meta.RemoteEnv))
-	for k, v := range meta.RemoteEnv {
-		env = append(env, fmt.Sprintf("%s=%s", k, v))
+	env := make([]string, 0)
+	for _, m := range meta {
+		for k, v := range m.RemoteEnv {
+			if v == nil { // *string per spec
+				// devcontainer-cli will set this to the string "null" if the value is
+				// not set. Explicitly setting to an empty string here as this would be
+				// more expected here.
+				v = ptr.Ref("")
+			}
+			env = append(env, fmt.Sprintf("%s=%s", k, *v))
+		}
 	}
 	slices.Sort(env)
 	return env, nil
@@ -238,6 +228,19 @@ func run(ctx context.Context, execer agentexec.Execer, cmd string, args ...strin
 	return stdout, stderr, err
 }
 
+// DockerCLILister is a ContainerLister that lists containers using the docker CLI
+type DockerCLILister struct {
+	execer agentexec.Execer
+}
+
+var _ Lister = &DockerCLILister{}
+
+func NewDocker(execer agentexec.Execer) Lister {
+	return &DockerCLILister{
+		execer: agentexec.DefaultExecer,
+	}
+}
+
 func (dcl *DockerCLILister) List(ctx context.Context) (codersdk.WorkspaceAgentListContainersResponse, error) {
 	var stdoutBuf, stderrBuf bytes.Buffer
 	// List all container IDs, one per line, with no truncation
@@ -265,11 +268,16 @@ func (dcl *DockerCLILister) List(ctx context.Context) (codersdk.WorkspaceAgentLi
 		return codersdk.WorkspaceAgentListContainersResponse{}, xerrors.Errorf("scan docker ps output: %w", err)
 	}
 
+	res := codersdk.WorkspaceAgentListContainersResponse{
+		Containers: make([]codersdk.WorkspaceAgentContainer, 0, len(ids)),
+		Warnings:   make([]string, 0),
+	}
 	dockerPsStderr := strings.TrimSpace(stderrBuf.String())
+	if dockerPsStderr != "" {
+		res.Warnings = append(res.Warnings, dockerPsStderr)
+	}
 	if len(ids) == 0 {
-		return codersdk.WorkspaceAgentListContainersResponse{
-			Warnings: []string{dockerPsStderr},
-		}, nil
+		return res, nil
 	}
 
 	// now we can get the detailed information for each container
@@ -280,26 +288,21 @@ func (dcl *DockerCLILister) List(ctx context.Context) (codersdk.WorkspaceAgentLi
 	// will still contain valid JSON. We will just end up missing
 	// information about the removed container. We could potentially
 	// log this error, but I'm not sure it's worth it.
-	ins, dockerInspectStderr, err := runDockerInspect(ctx, dcl.execer, ids...)
+	dockerInspectStdout, dockerInspectStderr, err := runDockerInspect(ctx, dcl.execer, ids...)
 	if err != nil {
-		return codersdk.WorkspaceAgentListContainersResponse{}, xerrors.Errorf("run docker inspect: %w", err)
+		return codersdk.WorkspaceAgentListContainersResponse{}, xerrors.Errorf("run docker inspect: %w: %s", err, dockerInspectStderr)
 	}
 
-	res := codersdk.WorkspaceAgentListContainersResponse{
-		Containers: make([]codersdk.WorkspaceAgentDevcontainer, len(ins)),
-	}
-	for idx, in := range ins {
-		out, warns := convertDockerInspect(in)
-		res.Warnings = append(res.Warnings, warns...)
-		res.Containers[idx] = out
+	if len(dockerInspectStderr) > 0 {
+		res.Warnings = append(res.Warnings, string(dockerInspectStderr))
 	}
 
-	if dockerPsStderr != "" {
-		res.Warnings = append(res.Warnings, dockerPsStderr)
-	}
-	if dockerInspectStderr != "" {
-		res.Warnings = append(res.Warnings, dockerInspectStderr)
+	outs, warns, err := convertDockerInspect(dockerInspectStdout)
+	if err != nil {
+		return codersdk.WorkspaceAgentListContainersResponse{}, xerrors.Errorf("convert docker inspect output: %w", err)
 	}
+	res.Warnings = append(res.Warnings, warns...)
+	res.Containers = append(res.Containers, outs...)
 
 	return res, nil
 }
@@ -307,35 +310,34 @@ func (dcl *DockerCLILister) List(ctx context.Context) (codersdk.WorkspaceAgentLi
 // runDockerInspect is a helper function that runs `docker inspect` on the given
 // container IDs and returns the parsed output.
 // The stderr output is also returned for logging purposes.
-func runDockerInspect(ctx context.Context, execer agentexec.Execer, ids ...string) ([]dockerInspect, string, error) {
+func runDockerInspect(ctx context.Context, execer agentexec.Execer, ids ...string) (stdout, stderr []byte, err error) {
 	var stdoutBuf, stderrBuf bytes.Buffer
 	cmd := execer.CommandContext(ctx, "docker", append([]string{"inspect"}, ids...)...)
 	cmd.Stdout = &stdoutBuf
 	cmd.Stderr = &stderrBuf
-	err := cmd.Run()
-	stderr := strings.TrimSpace(stderrBuf.String())
+	err = cmd.Run()
+	stdout = bytes.TrimSpace(stdoutBuf.Bytes())
+	stderr = bytes.TrimSpace(stderrBuf.Bytes())
 	if err != nil {
-		return nil, stderr, err
+		if bytes.Contains(stderr, []byte("No such object:")) {
+			// This can happen if a container is deleted between the time we check for its existence and the time we inspect it.
+			return stdout, stderr, nil
+		}
+		return stdout, stderr, err
 	}
-
-	var ins []dockerInspect
-	if err := json.NewDecoder(&stdoutBuf).Decode(&ins); err != nil {
-		return nil, stderr, xerrors.Errorf("decode docker inspect output: %w", err)
-	}
-
-	return ins, stderr, nil
+	return stdout, stderr, nil
 }
 
 // To avoid a direct dependency on the Docker API, we use the docker CLI
 // to fetch information about containers.
 type dockerInspect struct {
-	ID         string                  `json:"Id"`
-	Created    time.Time               `json:"Created"`
-	Config     dockerInspectConfig     `json:"Config"`
-	HostConfig dockerInspectHostConfig `json:"HostConfig"`
-	Name       string                  `json:"Name"`
-	Mounts     []dockerInspectMount    `json:"Mounts"`
-	State      dockerInspectState      `json:"State"`
+	ID              string                       `json:"Id"`
+	Created         time.Time                    `json:"Created"`
+	Config          dockerInspectConfig          `json:"Config"`
+	Name            string                       `json:"Name"`
+	Mounts          []dockerInspectMount         `json:"Mounts"`
+	State           dockerInspectState           `json:"State"`
+	NetworkSettings dockerInspectNetworkSettings `json:"NetworkSettings"`
 }
 
 type dockerInspectConfig struct {
@@ -343,8 +345,9 @@ type dockerInspectConfig struct {
 	Labels map[string]string `json:"Labels"`
 }
 
-type dockerInspectHostConfig struct {
-	PortBindings map[string]any `json:"PortBindings"`
+type dockerInspectPort struct {
+	HostIP   string `json:"HostIp"`
+	HostPort string `json:"HostPort"`
 }
 
 type dockerInspectMount struct {
@@ -359,6 +362,10 @@ type dockerInspectState struct {
 	Error    string `json:"Error"`
 }
 
+type dockerInspectNetworkSettings struct {
+	Ports map[string][]dockerInspectPort `json:"Ports"`
+}
+
 func (dis dockerInspectState) String() string {
 	if dis.Running {
 		return "running"
@@ -376,50 +383,109 @@ func (dis dockerInspectState) String() string {
 	return sb.String()
 }
 
-func convertDockerInspect(in dockerInspect) (codersdk.WorkspaceAgentDevcontainer, []string) {
+func convertDockerInspect(raw []byte) ([]codersdk.WorkspaceAgentContainer, []string, error) {
 	var warns []string
-	out := codersdk.WorkspaceAgentDevcontainer{
-		CreatedAt: in.Created,
-		// Remove the leading slash from the container name
-		FriendlyName: strings.TrimPrefix(in.Name, "/"),
-		ID:           in.ID,
-		Image:        in.Config.Image,
-		Labels:       in.Config.Labels,
-		Ports:        make([]codersdk.WorkspaceAgentListeningPort, 0),
-		Running:      in.State.Running,
-		Status:       in.State.String(),
-		Volumes:      make(map[string]string, len(in.Mounts)),
+	var ins []dockerInspect
+	if err := json.NewDecoder(bytes.NewReader(raw)).Decode(&ins); err != nil {
+		return nil, nil, xerrors.Errorf("decode docker inspect output: %w", err)
+	}
+	outs := make([]codersdk.WorkspaceAgentContainer, 0, len(ins))
+
+	// Say you have two containers:
+	//  - Container A with Host IP 127.0.0.1:8000 mapped to container port 8001
+	//  - Container B with Host IP [::1]:8000 mapped to container port 8001
+	// A request to localhost:8000 may be routed to either container.
+	// We don't know which one for sure, so we need to surface this to the user.
+	// Keep track of all host ports we see. If we see the same host port
+	// mapped to multiple containers on different host IPs, we need to
+	// warn the user about this.
+	// Note that we only do this for loopback or unspecified IPs.
+	// We'll assume that the user knows what they're doing if they bind to
+	// a specific IP address.
+	hostPortContainers := make(map[int][]string)
+
+	for _, in := range ins {
+		out := codersdk.WorkspaceAgentContainer{
+			CreatedAt: in.Created,
+			// Remove the leading slash from the container name
+			FriendlyName: strings.TrimPrefix(in.Name, "/"),
+			ID:           in.ID,
+			Image:        in.Config.Image,
+			Labels:       in.Config.Labels,
+			Ports:        make([]codersdk.WorkspaceAgentContainerPort, 0),
+			Running:      in.State.Running,
+			Status:       in.State.String(),
+			Volumes:      make(map[string]string, len(in.Mounts)),
+		}
+
+		if in.NetworkSettings.Ports == nil {
+			in.NetworkSettings.Ports = make(map[string][]dockerInspectPort)
+		}
+		portKeys := maps.Keys(in.NetworkSettings.Ports)
+		// Sort the ports for deterministic output.
+		sort.Strings(portKeys)
+		// If we see the same port bound to both ipv4 and ipv6 loopback or unspecified
+		// interfaces to the same container port, there is no point in adding it multiple times.
+		loopbackHostPortContainerPorts := make(map[int]uint16, 0)
+		for _, pk := range portKeys {
+			for _, p := range in.NetworkSettings.Ports[pk] {
+				cp, network, err := convertDockerPort(pk)
+				if err != nil {
+					warns = append(warns, fmt.Sprintf("convert docker port: %s", err.Error()))
+					// Default network to "tcp" if we can't parse it.
+					network = "tcp"
+				}
+				hp, err := strconv.Atoi(p.HostPort)
+				if err != nil {
+					warns = append(warns, fmt.Sprintf("convert docker host port: %s", err.Error()))
+					continue
+				}
+				if hp > 65535 || hp < 1 { // invalid port
+					warns = append(warns, fmt.Sprintf("convert docker host port: invalid host port %d", hp))
+					continue
+				}
+
+				// Deduplicate host ports for loopback and unspecified IPs.
+				if isLoopbackOrUnspecified(p.HostIP) {
+					if found, ok := loopbackHostPortContainerPorts[hp]; ok && found == cp {
+						// We've already seen this port, so skip it.
+						continue
+					}
+					loopbackHostPortContainerPorts[hp] = cp
+					// Also keep track of the host port and the container ID.
+					hostPortContainers[hp] = append(hostPortContainers[hp], in.ID)
+				}
+				out.Ports = append(out.Ports, codersdk.WorkspaceAgentContainerPort{
+					Network: network,
+					Port:    cp,
+					// #nosec G115 - Safe conversion since Docker ports are limited to uint16 range
+					HostPort: uint16(hp),
+					HostIP:   p.HostIP,
+				})
+			}
+		}
+
+		if in.Mounts == nil {
+			in.Mounts = []dockerInspectMount{}
+		}
+		// Sort the mounts for deterministic output.
+		sort.Slice(in.Mounts, func(i, j int) bool {
+			return in.Mounts[i].Source < in.Mounts[j].Source
+		})
+		for _, k := range in.Mounts {
+			out.Volumes[k.Source] = k.Destination
+		}
+		outs = append(outs, out)
 	}
 
-	if in.HostConfig.PortBindings == nil {
-		in.HostConfig.PortBindings = make(map[string]any)
-	}
-	portKeys := maps.Keys(in.HostConfig.PortBindings)
-	// Sort the ports for deterministic output.
-	sort.Strings(portKeys)
-	for _, p := range portKeys {
-		if port, network, err := convertDockerPort(p); err != nil {
-			warns = append(warns, err.Error())
-		} else {
-			out.Ports = append(out.Ports, codersdk.WorkspaceAgentListeningPort{
-				Network: network,
-				Port:    port,
-			})
+	// Check if any host ports are mapped to multiple containers.
+	for hp, ids := range hostPortContainers {
+		if len(ids) > 1 {
+			warns = append(warns, fmt.Sprintf("host port %d is mapped to multiple containers on different interfaces: %s", hp, strings.Join(ids, ", ")))
 		}
 	}
 
-	if in.Mounts == nil {
-		in.Mounts = []dockerInspectMount{}
-	}
-	// Sort the mounts for deterministic output.
-	sort.Slice(in.Mounts, func(i, j int) bool {
-		return in.Mounts[i].Source < in.Mounts[j].Source
-	})
-	for _, k := range in.Mounts {
-		out.Volumes[k.Source] = k.Destination
-	}
-
-	return out, warns
+	return outs, warns, nil
 }
 
 // convertDockerPort converts a Docker port string to a port number and network
@@ -428,21 +494,26 @@ func convertDockerInspect(in dockerInspect) (codersdk.WorkspaceAgentDevcontainer
 //	"8080" -> 8080, "tcp"
 func convertDockerPort(in string) (uint16, string, error) {
 	parts := strings.Split(in, "/")
+	p, err := strconv.ParseUint(parts[0], 10, 16)
+	if err != nil {
+		return 0, "", xerrors.Errorf("invalid port format: %s", in)
+	}
 	switch len(parts) {
 	case 1:
 		// assume it's a TCP port
-		p, err := strconv.Atoi(parts[0])
-		if err != nil {
-			return 0, "", xerrors.Errorf("invalid port format: %s", in)
-		}
 		return uint16(p), "tcp", nil
 	case 2:
-		p, err := strconv.Atoi(parts[0])
-		if err != nil {
-			return 0, "", xerrors.Errorf("invalid port format: %s", in)
-		}
 		return uint16(p), parts[1], nil
 	default:
 		return 0, "", xerrors.Errorf("invalid port format: %s", in)
 	}
 }
+
+// convenience function to check if an IP address is loopback or unspecified
+func isLoopbackOrUnspecified(ips string) bool {
+	nip := net.ParseIP(ips)
+	if nip == nil {
+		return false // technically correct, I suppose
+	}
+	return nip.IsLoopback() || nip.IsUnspecified()
+}

agent/agentcontainers/containers_internal_test.go

@@ -1,159 +1,18 @@
 package agentcontainers
 
 import (
-	"fmt"
 	"os"
-	"slices"
-	"strconv"
-	"strings"
+	"path/filepath"
 	"testing"
 	"time"
 
-	"go.uber.org/mock/gomock"
-
-	"github.com/google/uuid"
-	"github.com/ory/dockertest/v3"
-	"github.com/ory/dockertest/v3/docker"
+	"github.com/google/go-cmp/cmp"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
-	"github.com/coder/coder/v2/agent/agentcontainers/acmock"
-	"github.com/coder/coder/v2/agent/agentexec"
 	"github.com/coder/coder/v2/codersdk"
-	"github.com/coder/coder/v2/pty"
-	"github.com/coder/coder/v2/testutil"
-	"github.com/coder/quartz"
 )
 
-// TestIntegrationDocker tests agentcontainers functionality using a real
-// Docker container. It starts a container with a known
-// label, lists the containers, and verifies that the expected container is
-// returned. It also executes a sample command inside the container.
-// The container is deleted after the test is complete.
-// As this test creates containers, it is skipped by default.
-// It can be run manually as follows:
-//
-// CODER_TEST_USE_DOCKER=1 go test ./agent/agentcontainers -run TestDockerCLIContainerLister
-func TestIntegrationDocker(t *testing.T) {
-	t.Parallel()
-	if ctud, ok := os.LookupEnv("CODER_TEST_USE_DOCKER"); !ok || ctud != "1" {
-		t.Skip("Set CODER_TEST_USE_DOCKER=1 to run this test")
-	}
-
-	pool, err := dockertest.NewPool("")
-	require.NoError(t, err, "Could not connect to docker")
-	testLabelValue := uuid.New().String()
-	// Create a temporary directory to validate that we surface mounts correctly.
-	testTempDir := t.TempDir()
-	// Pick a random port to expose for testing port bindings.
-	testRandPort := testutil.RandomPortNoListen(t)
-	ct, err := pool.RunWithOptions(&dockertest.RunOptions{
-		Repository: "busybox",
-		Tag:        "latest",
-		Cmd:        []string{"sleep", "infnity"},
-		Labels: map[string]string{
-			"com.coder.test":        testLabelValue,
-			"devcontainer.metadata": `{"remoteEnv": {"FOO": "bar", "MULTILINE": "foo\nbar\nbaz"}}`,
-		},
-		Mounts:       []string{testTempDir + ":" + testTempDir},
-		ExposedPorts: []string{fmt.Sprintf("%d/tcp", testRandPort)},
-		PortBindings: map[docker.Port][]docker.PortBinding{
-			docker.Port(fmt.Sprintf("%d/tcp", testRandPort)): {
-				{
-					HostIP:   "0.0.0.0",
-					HostPort: strconv.FormatInt(int64(testRandPort), 10),
-				},
-			},
-		},
-	}, func(config *docker.HostConfig) {
-		config.AutoRemove = true
-		config.RestartPolicy = docker.RestartPolicy{Name: "no"}
-	})
-	require.NoError(t, err, "Could not start test docker container")
-	t.Logf("Created container %q", ct.Container.Name)
-	t.Cleanup(func() {
-		assert.NoError(t, pool.Purge(ct), "Could not purge resource %q", ct.Container.Name)
-		t.Logf("Purged container %q", ct.Container.Name)
-	})
-	// Wait for container to start
-	require.Eventually(t, func() bool {
-		ct, ok := pool.ContainerByName(ct.Container.Name)
-		return ok && ct.Container.State.Running
-	}, testutil.WaitShort, testutil.IntervalSlow, "Container did not start in time")
-
-	dcl := NewDocker(agentexec.DefaultExecer)
-	ctx := testutil.Context(t, testutil.WaitShort)
-	actual, err := dcl.List(ctx)
-	require.NoError(t, err, "Could not list containers")
-	require.Empty(t, actual.Warnings, "Expected no warnings")
-	var found bool
-	for _, foundContainer := range actual.Containers {
-		if foundContainer.ID == ct.Container.ID {
-			found = true
-			assert.Equal(t, ct.Container.Created, foundContainer.CreatedAt)
-			// ory/dockertest pre-pends a forward slash to the container name.
-			assert.Equal(t, strings.TrimPrefix(ct.Container.Name, "/"), foundContainer.FriendlyName)
-			// ory/dockertest returns the sha256 digest of the image.
-			assert.Equal(t, "busybox:latest", foundContainer.Image)
-			assert.Equal(t, ct.Container.Config.Labels, foundContainer.Labels)
-			assert.True(t, foundContainer.Running)
-			assert.Equal(t, "running", foundContainer.Status)
-			if assert.Len(t, foundContainer.Ports, 1) {
-				assert.Equal(t, testRandPort, foundContainer.Ports[0].Port)
-				assert.Equal(t, "tcp", foundContainer.Ports[0].Network)
-			}
-			if assert.Len(t, foundContainer.Volumes, 1) {
-				assert.Equal(t, testTempDir, foundContainer.Volumes[testTempDir])
-			}
-			// Test that EnvInfo is able to correctly modify a command to be
-			// executed inside the container.
-			dei, err := EnvInfo(ctx, agentexec.DefaultExecer, ct.Container.ID, "")
-			require.NoError(t, err, "Expected no error from DockerEnvInfo()")
-			ptyWrappedCmd, ptyWrappedArgs := dei.ModifyCommand("/bin/sh", "--norc")
-			ptyCmd, ptyPs, err := pty.Start(agentexec.DefaultExecer.PTYCommandContext(ctx, ptyWrappedCmd, ptyWrappedArgs...))
-			require.NoError(t, err, "failed to start pty command")
-			t.Cleanup(func() {
-				_ = ptyPs.Kill()
-				_ = ptyCmd.Close()
-			})
-			tr := testutil.NewTerminalReader(t, ptyCmd.OutputReader())
-			matchPrompt := func(line string) bool {
-				return strings.Contains(line, "#")
-			}
-			matchHostnameCmd := func(line string) bool {
-				return strings.Contains(strings.TrimSpace(line), "hostname")
-			}
-			matchHostnameOuput := func(line string) bool {
-				return strings.Contains(strings.TrimSpace(line), ct.Container.Config.Hostname)
-			}
-			matchEnvCmd := func(line string) bool {
-				return strings.Contains(strings.TrimSpace(line), "env")
-			}
-			matchEnvOutput := func(line string) bool {
-				return strings.Contains(line, "FOO=bar") || strings.Contains(line, "MULTILINE=foo")
-			}
-			require.NoError(t, tr.ReadUntil(ctx, matchPrompt), "failed to match prompt")
-			t.Logf("Matched prompt")
-			_, err = ptyCmd.InputWriter().Write([]byte("hostname\r\n"))
-			require.NoError(t, err, "failed to write to pty")
-			t.Logf("Wrote hostname command")
-			require.NoError(t, tr.ReadUntil(ctx, matchHostnameCmd), "failed to match hostname command")
-			t.Logf("Matched hostname command")
-			require.NoError(t, tr.ReadUntil(ctx, matchHostnameOuput), "failed to match hostname output")
-			t.Logf("Matched hostname output")
-			_, err = ptyCmd.InputWriter().Write([]byte("env\r\n"))
-			require.NoError(t, err, "failed to write to pty")
-			t.Logf("Wrote env command")
-			require.NoError(t, tr.ReadUntil(ctx, matchEnvCmd), "failed to match env command")
-			t.Logf("Matched env command")
-			require.NoError(t, tr.ReadUntil(ctx, matchEnvOutput), "failed to match env output")
-			t.Logf("Matched env output")
-			break
-		}
-	}
-	assert.True(t, found, "Expected to find container with label 'com.coder.test=%s'", testLabelValue)
-}
-
 func TestWrapDockerExec(t *testing.T) {
 	t.Parallel()
 	tests := []struct {
@@ -192,123 +51,10 @@ func TestWrapDockerExec(t *testing.T) {
 	}
 }
 
-// TestContainersHandler tests the containersHandler.getContainers method using
-// a mock implementation. It specifically tests caching behavior.
-func TestContainersHandler(t *testing.T) {
-	t.Parallel()
-
-	t.Run("list", func(t *testing.T) {
-		t.Parallel()
-
-		fakeCt := fakeContainer(t)
-		fakeCt2 := fakeContainer(t)
-		makeResponse := func(cts ...codersdk.WorkspaceAgentDevcontainer) codersdk.WorkspaceAgentListContainersResponse {
-			return codersdk.WorkspaceAgentListContainersResponse{Containers: cts}
-		}
-
-		// Each test case is called multiple times to ensure idempotency
-		for _, tc := range []struct {
-			name string
-			// data to be stored in the handler
-			cacheData codersdk.WorkspaceAgentListContainersResponse
-			// duration of cache
-			cacheDur time.Duration
-			// relative age of the cached data
-			cacheAge time.Duration
-			// function to set up expectations for the mock
-			setupMock func(*acmock.MockLister)
-			// expected result
-			expected codersdk.WorkspaceAgentListContainersResponse
-			// expected error
-			expectedErr string
-		}{
-			{
-				name: "no cache",
-				setupMock: func(mcl *acmock.MockLister) {
-					mcl.EXPECT().List(gomock.Any()).Return(makeResponse(fakeCt), nil).AnyTimes()
-				},
-				expected: makeResponse(fakeCt),
-			},
-			{
-				name:      "no data",
-				cacheData: makeResponse(),
-				cacheAge:  2 * time.Second,
-				cacheDur:  time.Second,
-				setupMock: func(mcl *acmock.MockLister) {
-					mcl.EXPECT().List(gomock.Any()).Return(makeResponse(fakeCt), nil).AnyTimes()
-				},
-				expected: makeResponse(fakeCt),
-			},
-			{
-				name:      "cached data",
-				cacheAge:  time.Second,
-				cacheData: makeResponse(fakeCt),
-				cacheDur:  2 * time.Second,
-				expected:  makeResponse(fakeCt),
-			},
-			{
-				name: "lister error",
-				setupMock: func(mcl *acmock.MockLister) {
-					mcl.EXPECT().List(gomock.Any()).Return(makeResponse(), assert.AnError).AnyTimes()
-				},
-				expectedErr: assert.AnError.Error(),
-			},
-			{
-				name:      "stale cache",
-				cacheAge:  2 * time.Second,
-				cacheData: makeResponse(fakeCt),
-				cacheDur:  time.Second,
-				setupMock: func(mcl *acmock.MockLister) {
-					mcl.EXPECT().List(gomock.Any()).Return(makeResponse(fakeCt2), nil).AnyTimes()
-				},
-				expected: makeResponse(fakeCt2),
-			},
-		} {
-			tc := tc
-			t.Run(tc.name, func(t *testing.T) {
-				t.Parallel()
-				var (
-					ctx        = testutil.Context(t, testutil.WaitShort)
-					clk        = quartz.NewMock(t)
-					ctrl       = gomock.NewController(t)
-					mockLister = acmock.NewMockLister(ctrl)
-					now        = time.Now().UTC()
-					ch         = devcontainersHandler{
-						cacheDuration: tc.cacheDur,
-						cl:            mockLister,
-						clock:         clk,
-						containers:    &tc.cacheData,
-						lockCh:        make(chan struct{}, 1),
-					}
-				)
-				if tc.cacheAge != 0 {
-					ch.mtime = now.Add(-tc.cacheAge)
-				}
-				if tc.setupMock != nil {
-					tc.setupMock(mockLister)
-				}
-
-				clk.Set(now).MustWait(ctx)
-
-				// Repeat the test to ensure idempotency
-				for i := 0; i < 2; i++ {
-					actual, err := ch.getContainers(ctx)
-					if tc.expectedErr != "" {
-						require.Empty(t, actual, "expected no data (attempt %d)", i)
-						require.ErrorContains(t, err, tc.expectedErr, "expected error (attempt %d)", i)
-					} else {
-						require.NoError(t, err, "expected no error (attempt %d)", i)
-						require.Equal(t, tc.expected, actual, "expected containers to be equal (attempt %d)", i)
-					}
-				}
-			})
-		}
-	})
-}
-
 func TestConvertDockerPort(t *testing.T) {
 	t.Parallel()
 
+	//nolint:paralleltest // variable recapture no longer required
 	for _, tc := range []struct {
 		name          string
 		in            string
@@ -355,7 +101,7 @@ func TestConvertDockerPort(t *testing.T) {
 			expectError: "invalid port",
 		},
 	} {
-		tc := tc // not needed anymore but makes the linter happy
+		//nolint: paralleltest // variable recapture no longer required
 		t.Run(tc.name, func(t *testing.T) {
 			t.Parallel()
 			actualPort, actualNetwork, actualErr := convertDockerPort(tc.in)
@@ -412,153 +158,261 @@ func TestConvertDockerVolume(t *testing.T) {
 	}
 }
 
-// TestDockerEnvInfoer tests the ability of EnvInfo to extract information from
-// running containers. Containers are deleted after the test is complete.
-// As this test creates containers, it is skipped by default.
-// It can be run manually as follows:
-//
-// CODER_TEST_USE_DOCKER=1 go test ./agent/agentcontainers -run TestDockerEnvInfoer
-func TestDockerEnvInfoer(t *testing.T) {
+// TestConvertDockerInspect tests the convertDockerInspect function using
+// fixtures from ./testdata.
+func TestConvertDockerInspect(t *testing.T) {
 	t.Parallel()
-	if ctud, ok := os.LookupEnv("CODER_TEST_USE_DOCKER"); !ok || ctud != "1" {
-		t.Skip("Set CODER_TEST_USE_DOCKER=1 to run this test")
-	}
 
-	pool, err := dockertest.NewPool("")
-	require.NoError(t, err, "Could not connect to docker")
-	// nolint:paralleltest // variable recapture no longer required
-	for idx, tt := range []struct {
-		image             string
-		labels            map[string]string
-		expectedEnv       []string
-		containerUser     string
-		expectedUsername  string
-		expectedUserShell string
+	//nolint:paralleltest // variable recapture no longer required
+	for _, tt := range []struct {
+		name        string
+		expect      []codersdk.WorkspaceAgentContainer
+		expectWarns []string
+		expectError string
 	}{
 		{
-			image:  "busybox:latest",
-			labels: map[string]string{`devcontainer.metadata`: `{"remoteEnv": {"FOO": "bar", "MULTILINE": "foo\nbar\nbaz"}}`},
-
-			expectedEnv:       []string{"FOO=bar", "MULTILINE=foo\nbar\nbaz"},
-			expectedUsername:  "root",
-			expectedUserShell: "/bin/sh",
+			name: "container_simple",
+			expect: []codersdk.WorkspaceAgentContainer{
+				{
+					CreatedAt:    time.Date(2025, 3, 11, 17, 55, 58, 91280203, time.UTC),
+					ID:           "6b539b8c60f5230b8b0fde2502cd2332d31c0d526a3e6eb6eef1cc39439b3286",
+					FriendlyName: "eloquent_kowalevski",
+					Image:        "debian:bookworm",
+					Labels:       map[string]string{},
+					Running:      true,
+					Status:       "running",
+					Ports:        []codersdk.WorkspaceAgentContainerPort{},
+					Volumes:      map[string]string{},
+				},
+			},
 		},
 		{
-			image:             "busybox:latest",
-			labels:            map[string]string{`devcontainer.metadata`: `{"remoteEnv": {"FOO": "bar", "MULTILINE": "foo\nbar\nbaz"}}`},
-			expectedEnv:       []string{"FOO=bar", "MULTILINE=foo\nbar\nbaz"},
-			containerUser:     "root",
-			expectedUsername:  "root",
-			expectedUserShell: "/bin/sh",
+			name: "container_labels",
+			expect: []codersdk.WorkspaceAgentContainer{
+				{
+					CreatedAt:    time.Date(2025, 3, 11, 20, 3, 28, 71706536, time.UTC),
+					ID:           "bd8818e670230fc6f36145b21cf8d6d35580355662aa4d9fe5ae1b188a4c905f",
+					FriendlyName: "fervent_bardeen",
+					Image:        "debian:bookworm",
+					Labels:       map[string]string{"baz": "zap", "foo": "bar"},
+					Running:      true,
+					Status:       "running",
+					Ports:        []codersdk.WorkspaceAgentContainerPort{},
+					Volumes:      map[string]string{},
+				},
+			},
 		},
 		{
-			image:             "codercom/enterprise-minimal:ubuntu",
-			labels:            map[string]string{`devcontainer.metadata`: `{"remoteEnv": {"FOO": "bar", "MULTILINE": "foo\nbar\nbaz"}}`},
-			expectedEnv:       []string{"FOO=bar", "MULTILINE=foo\nbar\nbaz"},
-			expectedUsername:  "coder",
-			expectedUserShell: "/bin/bash",
+			name: "container_binds",
+			expect: []codersdk.WorkspaceAgentContainer{
+				{
+					CreatedAt:    time.Date(2025, 3, 11, 17, 58, 43, 522505027, time.UTC),
+					ID:           "fdc75ebefdc0243c0fce959e7685931691ac7aede278664a0e2c23af8a1e8d6a",
+					FriendlyName: "silly_beaver",
+					Image:        "debian:bookworm",
+					Labels:       map[string]string{},
+					Running:      true,
+					Status:       "running",
+					Ports:        []codersdk.WorkspaceAgentContainerPort{},
+					Volumes: map[string]string{
+						"/tmp/test/a": "/var/coder/a",
+						"/tmp/test/b": "/var/coder/b",
+					},
+				},
+			},
 		},
 		{
-			image:             "codercom/enterprise-minimal:ubuntu",
-			labels:            map[string]string{`devcontainer.metadata`: `{"remoteEnv": {"FOO": "bar", "MULTILINE": "foo\nbar\nbaz"}}`},
-			expectedEnv:       []string{"FOO=bar", "MULTILINE=foo\nbar\nbaz"},
-			containerUser:     "coder",
-			expectedUsername:  "coder",
-			expectedUserShell: "/bin/bash",
+			name: "container_sameport",
+			expect: []codersdk.WorkspaceAgentContainer{
+				{
+					CreatedAt:    time.Date(2025, 3, 11, 17, 56, 34, 842164541, time.UTC),
+					ID:           "4eac5ce199d27b2329d0ff0ce1a6fc595612ced48eba3669aadb6c57ebef3fa2",
+					FriendlyName: "modest_varahamihira",
+					Image:        "debian:bookworm",
+					Labels:       map[string]string{},
+					Running:      true,
+					Status:       "running",
+					Ports: []codersdk.WorkspaceAgentContainerPort{
+						{
+							Network:  "tcp",
+							Port:     12345,
+							HostPort: 12345,
+							HostIP:   "0.0.0.0",
+						},
+					},
+					Volumes: map[string]string{},
+				},
+			},
 		},
 		{
-			image:             "codercom/enterprise-minimal:ubuntu",
-			labels:            map[string]string{`devcontainer.metadata`: `{"remoteEnv": {"FOO": "bar", "MULTILINE": "foo\nbar\nbaz"}}`},
-			expectedEnv:       []string{"FOO=bar", "MULTILINE=foo\nbar\nbaz"},
-			containerUser:     "root",
-			expectedUsername:  "root",
-			expectedUserShell: "/bin/bash",
+			name: "container_differentport",
+			expect: []codersdk.WorkspaceAgentContainer{
+				{
+					CreatedAt:    time.Date(2025, 3, 11, 17, 57, 8, 862545133, time.UTC),
+					ID:           "3090de8b72b1224758a94a11b827c82ba2b09c45524f1263dc4a2d83e19625ea",
+					FriendlyName: "boring_ellis",
+					Image:        "debian:bookworm",
+					Labels:       map[string]string{},
+					Running:      true,
+					Status:       "running",
+					Ports: []codersdk.WorkspaceAgentContainerPort{
+						{
+							Network:  "tcp",
+							Port:     23456,
+							HostPort: 12345,
+							HostIP:   "0.0.0.0",
+						},
+					},
+					Volumes: map[string]string{},
+				},
+			},
+		},
+		{
+			name: "container_sameportdiffip",
+			expect: []codersdk.WorkspaceAgentContainer{
+				{
+					CreatedAt:    time.Date(2025, 3, 11, 17, 56, 34, 842164541, time.UTC),
+					ID:           "a",
+					FriendlyName: "a",
+					Image:        "debian:bookworm",
+					Labels:       map[string]string{},
+					Running:      true,
+					Status:       "running",
+					Ports: []codersdk.WorkspaceAgentContainerPort{
+						{
+							Network:  "tcp",
+							Port:     8001,
+							HostPort: 8000,
+							HostIP:   "0.0.0.0",
+						},
+					},
+					Volumes: map[string]string{},
+				},
+				{
+					CreatedAt:    time.Date(2025, 3, 11, 17, 56, 34, 842164541, time.UTC),
+					ID:           "b",
+					FriendlyName: "b",
+					Image:        "debian:bookworm",
+					Labels:       map[string]string{},
+					Running:      true,
+					Status:       "running",
+					Ports: []codersdk.WorkspaceAgentContainerPort{
+						{
+							Network:  "tcp",
+							Port:     8001,
+							HostPort: 8000,
+							HostIP:   "::",
+						},
+					},
+					Volumes: map[string]string{},
+				},
+			},
+			expectWarns: []string{"host port 8000 is mapped to multiple containers on different interfaces: a, b"},
+		},
+		{
+			name: "container_volume",
+			expect: []codersdk.WorkspaceAgentContainer{
+				{
+					CreatedAt:    time.Date(2025, 3, 11, 17, 59, 42, 39484134, time.UTC),
+					ID:           "b3688d98c007f53402a55e46d803f2f3ba9181d8e3f71a2eb19b392cf0377b4e",
+					FriendlyName: "upbeat_carver",
+					Image:        "debian:bookworm",
+					Labels:       map[string]string{},
+					Running:      true,
+					Status:       "running",
+					Ports:        []codersdk.WorkspaceAgentContainerPort{},
+					Volumes: map[string]string{
+						"/var/lib/docker/volumes/testvol/_data": "/testvol",
+					},
+				},
+			},
+		},
+		{
+			name: "devcontainer_simple",
+			expect: []codersdk.WorkspaceAgentContainer{
+				{
+					CreatedAt:    time.Date(2025, 3, 11, 17, 1, 5, 751972661, time.UTC),
+					ID:           "0b2a9fcf5727d9562943ce47d445019f4520e37a2aa7c6d9346d01af4f4f9aed",
+					FriendlyName: "optimistic_hopper",
+					Image:        "debian:bookworm",
+					Labels: map[string]string{
+						"devcontainer.config_file": "/home/coder/src/coder/coder/agent/agentcontainers/testdata/devcontainer_simple.json",
+						"devcontainer.metadata":    "[]",
+					},
+					Running: true,
+					Status:  "running",
+					Ports:   []codersdk.WorkspaceAgentContainerPort{},
+					Volumes: map[string]string{},
+				},
+			},
+		},
+		{
+			name: "devcontainer_forwardport",
+			expect: []codersdk.WorkspaceAgentContainer{
+				{
+					CreatedAt:    time.Date(2025, 3, 11, 17, 3, 55, 22053072, time.UTC),
+					ID:           "4a16af2293fb75dc827a6949a3905dd57ea28cc008823218ce24fab1cb66c067",
+					FriendlyName: "serene_khayyam",
+					Image:        "debian:bookworm",
+					Labels: map[string]string{
+						"devcontainer.config_file": "/home/coder/src/coder/coder/agent/agentcontainers/testdata/devcontainer_forwardport.json",
+						"devcontainer.metadata":    "[]",
+					},
+					Running: true,
+					Status:  "running",
+					Ports:   []codersdk.WorkspaceAgentContainerPort{},
+					Volumes: map[string]string{},
+				},
+			},
+		},
+		{
+			name: "devcontainer_appport",
+			expect: []codersdk.WorkspaceAgentContainer{
+				{
+					CreatedAt:    time.Date(2025, 3, 11, 17, 2, 42, 613747761, time.UTC),
+					ID:           "52d23691f4b954d083f117358ea763e20f69af584e1c08f479c5752629ee0be3",
+					FriendlyName: "suspicious_margulis",
+					Image:        "debian:bookworm",
+					Labels: map[string]string{
+						"devcontainer.config_file": "/home/coder/src/coder/coder/agent/agentcontainers/testdata/devcontainer_appport.json",
+						"devcontainer.metadata":    "[]",
+					},
+					Running: true,
+					Status:  "running",
+					Ports: []codersdk.WorkspaceAgentContainerPort{
+						{
+							Network:  "tcp",
+							Port:     8080,
+							HostPort: 32768,
+							HostIP:   "0.0.0.0",
+						},
+					},
+					Volumes: map[string]string{},
+				},
+			},
 		},
 	} {
-		t.Run(fmt.Sprintf("#%d", idx), func(t *testing.T) {
+		// nolint:paralleltest // variable recapture no longer required
+		t.Run(tt.name, func(t *testing.T) {
 			t.Parallel()
-
-			// Start a container with the given image
-			// and environment variables
-			image := strings.Split(tt.image, ":")[0]
-			tag := strings.Split(tt.image, ":")[1]
-			ct, err := pool.RunWithOptions(&dockertest.RunOptions{
-				Repository: image,
-				Tag:        tag,
-				Cmd:        []string{"sleep", "infinity"},
-				Labels:     tt.labels,
-			}, func(config *docker.HostConfig) {
-				config.AutoRemove = true
-				config.RestartPolicy = docker.RestartPolicy{Name: "no"}
-			})
-			require.NoError(t, err, "Could not start test docker container")
-			t.Logf("Created container %q", ct.Container.Name)
-			t.Cleanup(func() {
-				assert.NoError(t, pool.Purge(ct), "Could not purge resource %q", ct.Container.Name)
-				t.Logf("Purged container %q", ct.Container.Name)
-			})
-
-			ctx := testutil.Context(t, testutil.WaitShort)
-			dei, err := EnvInfo(ctx, agentexec.DefaultExecer, ct.Container.ID, tt.containerUser)
-			require.NoError(t, err, "Expected no error from DockerEnvInfo()")
-
-			u, err := dei.CurrentUser()
-			require.NoError(t, err, "Expected no error from CurrentUser()")
-			require.Equal(t, tt.expectedUsername, u.Username, "Expected username to match")
-
-			hd, err := dei.UserHomeDir()
-			require.NoError(t, err, "Expected no error from UserHomeDir()")
-			require.NotEmpty(t, hd, "Expected user homedir to be non-empty")
-
-			sh, err := dei.UserShell(tt.containerUser)
-			require.NoError(t, err, "Expected no error from UserShell()")
-			require.Equal(t, tt.expectedUserShell, sh, "Expected user shell to match")
-
-			// We don't need to test the actual environment variables here.
-			environ := dei.Environ()
-			require.NotEmpty(t, environ, "Expected environ to be non-empty")
-
-			// Test that the environment variables are present in modified command
-			// output.
-			envCmd, envArgs := dei.ModifyCommand("env")
-			for _, env := range tt.expectedEnv {
-				require.Subset(t, envArgs, []string{"--env", env})
+			bs, err := os.ReadFile(filepath.Join("testdata", tt.name, "docker_inspect.json"))
+			require.NoError(t, err, "failed to read testdata file")
+			actual, warns, err := convertDockerInspect(bs)
+			if len(tt.expectWarns) > 0 {
+				assert.Len(t, warns, len(tt.expectWarns), "expected warnings")
+				for _, warn := range tt.expectWarns {
+					assert.Contains(t, warns, warn)
+				}
 			}
-			// Run the command in the container and check the output
-			// HACK: we remove the --tty argument because we're not running in a tty
-			envArgs = slices.DeleteFunc(envArgs, func(s string) bool { return s == "--tty" })
-			stdout, stderr, err := run(ctx, agentexec.DefaultExecer, envCmd, envArgs...)
-			require.Empty(t, stderr, "Expected no stderr output")
-			require.NoError(t, err, "Expected no error from running command")
-			for _, env := range tt.expectedEnv {
-				require.Contains(t, stdout, env)
+			if tt.expectError != "" {
+				assert.Empty(t, actual, "expected no data")
+				assert.ErrorContains(t, err, tt.expectError)
+				return
+			}
+			require.NoError(t, err, "expected no error")
+			if diff := cmp.Diff(tt.expect, actual); diff != "" {
+				t.Errorf("unexpected diff (-want +got):\n%s", diff)
 			}
 		})
 	}
 }
-
-func fakeContainer(t *testing.T, mut ...func(*codersdk.WorkspaceAgentDevcontainer)) codersdk.WorkspaceAgentDevcontainer {
-	t.Helper()
-	ct := codersdk.WorkspaceAgentDevcontainer{
-		CreatedAt:    time.Now().UTC(),
-		ID:           uuid.New().String(),
-		FriendlyName: testutil.GetRandomName(t),
-		Image:        testutil.GetRandomName(t) + ":" + strings.Split(uuid.New().String(), "-")[0],
-		Labels: map[string]string{
-			testutil.GetRandomName(t): testutil.GetRandomName(t),
-		},
-		Running: true,
-		Ports: []codersdk.WorkspaceAgentListeningPort{
-			{
-				Network: "tcp",
-				Port:    testutil.RandomPortNoListen(t),
-			},
-		},
-		Status:  testutil.MustRandString(t, 10),
-		Volumes: map[string]string{testutil.GetRandomName(t): testutil.GetRandomName(t)},
-	}
-	for _, m := range mut {
-		m(&ct)
-	}
-	return ct
-}

agent/agentcontainers/containers_test.go

@@ -0,0 +1,296 @@
+package agentcontainers_test
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"slices"
+	"strconv"
+	"strings"
+	"testing"
+
+	"github.com/google/uuid"
+	"github.com/ory/dockertest/v3"
+	"github.com/ory/dockertest/v3/docker"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/coder/coder/v2/agent/agentcontainers"
+	"github.com/coder/coder/v2/agent/agentexec"
+	"github.com/coder/coder/v2/pty"
+	"github.com/coder/coder/v2/testutil"
+)
+
+// TestIntegrationDocker tests agentcontainers functionality using a real
+// Docker container. It starts a container with a known
+// label, lists the containers, and verifies that the expected container is
+// returned. It also executes a sample command inside the container.
+// The container is deleted after the test is complete.
+// As this test creates containers, it is skipped by default.
+// It can be run manually as follows:
+//
+// CODER_TEST_USE_DOCKER=1 go test ./agent/agentcontainers -run TestDockerCLIContainerLister
+//
+//nolint:paralleltest // This test tends to flake when lots of containers start and stop in parallel.
+func TestIntegrationDocker(t *testing.T) {
+	if ctud, ok := os.LookupEnv("CODER_TEST_USE_DOCKER"); !ok || ctud != "1" {
+		t.Skip("Set CODER_TEST_USE_DOCKER=1 to run this test")
+	}
+
+	pool, err := dockertest.NewPool("")
+	require.NoError(t, err, "Could not connect to docker")
+	testLabelValue := uuid.New().String()
+	// Create a temporary directory to validate that we surface mounts correctly.
+	testTempDir := t.TempDir()
+	// Pick a random port to expose for testing port bindings.
+	testRandPort := testutil.RandomPortNoListen(t)
+	ct, err := pool.RunWithOptions(&dockertest.RunOptions{
+		Repository: "busybox",
+		Tag:        "latest",
+		Cmd:        []string{"sleep", "infnity"},
+		Labels: map[string]string{
+			"com.coder.test":        testLabelValue,
+			"devcontainer.metadata": `[{"remoteEnv": {"FOO": "bar", "MULTILINE": "foo\nbar\nbaz"}}]`,
+		},
+		Mounts:       []string{testTempDir + ":" + testTempDir},
+		ExposedPorts: []string{fmt.Sprintf("%d/tcp", testRandPort)},
+		PortBindings: map[docker.Port][]docker.PortBinding{
+			docker.Port(fmt.Sprintf("%d/tcp", testRandPort)): {
+				{
+					HostIP:   "0.0.0.0",
+					HostPort: strconv.FormatInt(int64(testRandPort), 10),
+				},
+			},
+		},
+	}, func(config *docker.HostConfig) {
+		config.AutoRemove = true
+		config.RestartPolicy = docker.RestartPolicy{Name: "no"}
+	})
+	require.NoError(t, err, "Could not start test docker container")
+	t.Logf("Created container %q", ct.Container.Name)
+	t.Cleanup(func() {
+		assert.NoError(t, pool.Purge(ct), "Could not purge resource %q", ct.Container.Name)
+		t.Logf("Purged container %q", ct.Container.Name)
+	})
+	// Wait for container to start
+	require.Eventually(t, func() bool {
+		ct, ok := pool.ContainerByName(ct.Container.Name)
+		return ok && ct.Container.State.Running
+	}, testutil.WaitShort, testutil.IntervalSlow, "Container did not start in time")
+
+	dcl := agentcontainers.NewDocker(agentexec.DefaultExecer)
+	ctx := testutil.Context(t, testutil.WaitShort)
+	actual, err := dcl.List(ctx)
+	require.NoError(t, err, "Could not list containers")
+	require.Empty(t, actual.Warnings, "Expected no warnings")
+	var found bool
+	for _, foundContainer := range actual.Containers {
+		if foundContainer.ID == ct.Container.ID {
+			found = true
+			assert.Equal(t, ct.Container.Created, foundContainer.CreatedAt)
+			// ory/dockertest pre-pends a forward slash to the container name.
+			assert.Equal(t, strings.TrimPrefix(ct.Container.Name, "/"), foundContainer.FriendlyName)
+			// ory/dockertest returns the sha256 digest of the image.
+			assert.Equal(t, "busybox:latest", foundContainer.Image)
+			assert.Equal(t, ct.Container.Config.Labels, foundContainer.Labels)
+			assert.True(t, foundContainer.Running)
+			assert.Equal(t, "running", foundContainer.Status)
+			if assert.Len(t, foundContainer.Ports, 1) {
+				assert.Equal(t, testRandPort, foundContainer.Ports[0].Port)
+				assert.Equal(t, "tcp", foundContainer.Ports[0].Network)
+			}
+			if assert.Len(t, foundContainer.Volumes, 1) {
+				assert.Equal(t, testTempDir, foundContainer.Volumes[testTempDir])
+			}
+			// Test that EnvInfo is able to correctly modify a command to be
+			// executed inside the container.
+			dei, err := agentcontainers.EnvInfo(ctx, agentexec.DefaultExecer, ct.Container.ID, "")
+			require.NoError(t, err, "Expected no error from DockerEnvInfo()")
+			ptyWrappedCmd, ptyWrappedArgs := dei.ModifyCommand("/bin/sh", "--norc")
+			ptyCmd, ptyPs, err := pty.Start(agentexec.DefaultExecer.PTYCommandContext(ctx, ptyWrappedCmd, ptyWrappedArgs...))
+			require.NoError(t, err, "failed to start pty command")
+			t.Cleanup(func() {
+				_ = ptyPs.Kill()
+				_ = ptyCmd.Close()
+			})
+			tr := testutil.NewTerminalReader(t, ptyCmd.OutputReader())
+			matchPrompt := func(line string) bool {
+				return strings.Contains(line, "#")
+			}
+			matchHostnameCmd := func(line string) bool {
+				return strings.Contains(strings.TrimSpace(line), "hostname")
+			}
+			matchHostnameOuput := func(line string) bool {
+				return strings.Contains(strings.TrimSpace(line), ct.Container.Config.Hostname)
+			}
+			matchEnvCmd := func(line string) bool {
+				return strings.Contains(strings.TrimSpace(line), "env")
+			}
+			matchEnvOutput := func(line string) bool {
+				return strings.Contains(line, "FOO=bar") || strings.Contains(line, "MULTILINE=foo")
+			}
+			require.NoError(t, tr.ReadUntil(ctx, matchPrompt), "failed to match prompt")
+			t.Logf("Matched prompt")
+			_, err = ptyCmd.InputWriter().Write([]byte("hostname\r\n"))
+			require.NoError(t, err, "failed to write to pty")
+			t.Logf("Wrote hostname command")
+			require.NoError(t, tr.ReadUntil(ctx, matchHostnameCmd), "failed to match hostname command")
+			t.Logf("Matched hostname command")
+			require.NoError(t, tr.ReadUntil(ctx, matchHostnameOuput), "failed to match hostname output")
+			t.Logf("Matched hostname output")
+			_, err = ptyCmd.InputWriter().Write([]byte("env\r\n"))
+			require.NoError(t, err, "failed to write to pty")
+			t.Logf("Wrote env command")
+			require.NoError(t, tr.ReadUntil(ctx, matchEnvCmd), "failed to match env command")
+			t.Logf("Matched env command")
+			require.NoError(t, tr.ReadUntil(ctx, matchEnvOutput), "failed to match env output")
+			t.Logf("Matched env output")
+			break
+		}
+	}
+	assert.True(t, found, "Expected to find container with label 'com.coder.test=%s'", testLabelValue)
+}
+
+// TestDockerEnvInfoer tests the ability of EnvInfo to extract information from
+// running containers. Containers are deleted after the test is complete.
+// As this test creates containers, it is skipped by default.
+// It can be run manually as follows:
+//
+// CODER_TEST_USE_DOCKER=1 go test ./agent/agentcontainers -run TestDockerEnvInfoer
+//
+//nolint:paralleltest // This test tends to flake when lots of containers start and stop in parallel.
+func TestDockerEnvInfoer(t *testing.T) {
+	if ctud, ok := os.LookupEnv("CODER_TEST_USE_DOCKER"); !ok || ctud != "1" {
+		t.Skip("Set CODER_TEST_USE_DOCKER=1 to run this test")
+	}
+
+	pool, err := dockertest.NewPool("")
+	require.NoError(t, err, "Could not connect to docker")
+	// nolint:paralleltest // variable recapture no longer required
+	for idx, tt := range []struct {
+		image             string
+		labels            map[string]string
+		expectedEnv       []string
+		containerUser     string
+		expectedUsername  string
+		expectedUserShell string
+	}{
+		{
+			image:  "busybox:latest",
+			labels: map[string]string{`devcontainer.metadata`: `[{"remoteEnv": {"FOO": "bar", "MULTILINE": "foo\nbar\nbaz"}}]`},
+
+			expectedEnv:       []string{"FOO=bar", "MULTILINE=foo\nbar\nbaz"},
+			expectedUsername:  "root",
+			expectedUserShell: "/bin/sh",
+		},
+		{
+			image:             "busybox:latest",
+			labels:            map[string]string{`devcontainer.metadata`: `[{"remoteEnv": {"FOO": "bar", "MULTILINE": "foo\nbar\nbaz"}}]`},
+			expectedEnv:       []string{"FOO=bar", "MULTILINE=foo\nbar\nbaz"},
+			containerUser:     "root",
+			expectedUsername:  "root",
+			expectedUserShell: "/bin/sh",
+		},
+		{
+			image:             "codercom/enterprise-minimal:ubuntu",
+			labels:            map[string]string{`devcontainer.metadata`: `[{"remoteEnv": {"FOO": "bar", "MULTILINE": "foo\nbar\nbaz"}}]`},
+			expectedEnv:       []string{"FOO=bar", "MULTILINE=foo\nbar\nbaz"},
+			expectedUsername:  "coder",
+			expectedUserShell: "/bin/bash",
+		},
+		{
+			image:             "codercom/enterprise-minimal:ubuntu",
+			labels:            map[string]string{`devcontainer.metadata`: `[{"remoteEnv": {"FOO": "bar", "MULTILINE": "foo\nbar\nbaz"}}]`},
+			expectedEnv:       []string{"FOO=bar", "MULTILINE=foo\nbar\nbaz"},
+			containerUser:     "coder",
+			expectedUsername:  "coder",
+			expectedUserShell: "/bin/bash",
+		},
+		{
+			image:             "codercom/enterprise-minimal:ubuntu",
+			labels:            map[string]string{`devcontainer.metadata`: `[{"remoteEnv": {"FOO": "bar", "MULTILINE": "foo\nbar\nbaz"}}]`},
+			expectedEnv:       []string{"FOO=bar", "MULTILINE=foo\nbar\nbaz"},
+			containerUser:     "root",
+			expectedUsername:  "root",
+			expectedUserShell: "/bin/bash",
+		},
+		{
+			image:             "codercom/enterprise-minimal:ubuntu",
+			labels:            map[string]string{`devcontainer.metadata`: `[{"remoteEnv": {"FOO": "bar"}},{"remoteEnv": {"MULTILINE": "foo\nbar\nbaz"}}]`},
+			expectedEnv:       []string{"FOO=bar", "MULTILINE=foo\nbar\nbaz"},
+			containerUser:     "root",
+			expectedUsername:  "root",
+			expectedUserShell: "/bin/bash",
+		},
+	} {
+		//nolint:paralleltest // variable recapture no longer required
+		t.Run(fmt.Sprintf("#%d", idx), func(t *testing.T) {
+			// Start a container with the given image
+			// and environment variables
+			image := strings.Split(tt.image, ":")[0]
+			tag := strings.Split(tt.image, ":")[1]
+			ct, err := pool.RunWithOptions(&dockertest.RunOptions{
+				Repository: image,
+				Tag:        tag,
+				Cmd:        []string{"sleep", "infinity"},
+				Labels:     tt.labels,
+			}, func(config *docker.HostConfig) {
+				config.AutoRemove = true
+				config.RestartPolicy = docker.RestartPolicy{Name: "no"}
+			})
+			require.NoError(t, err, "Could not start test docker container")
+			t.Logf("Created container %q", ct.Container.Name)
+			t.Cleanup(func() {
+				assert.NoError(t, pool.Purge(ct), "Could not purge resource %q", ct.Container.Name)
+				t.Logf("Purged container %q", ct.Container.Name)
+			})
+
+			ctx := testutil.Context(t, testutil.WaitShort)
+			dei, err := agentcontainers.EnvInfo(ctx, agentexec.DefaultExecer, ct.Container.ID, tt.containerUser)
+			require.NoError(t, err, "Expected no error from DockerEnvInfo()")
+
+			u, err := dei.User()
+			require.NoError(t, err, "Expected no error from CurrentUser()")
+			require.Equal(t, tt.expectedUsername, u.Username, "Expected username to match")
+
+			hd, err := dei.HomeDir()
+			require.NoError(t, err, "Expected no error from UserHomeDir()")
+			require.NotEmpty(t, hd, "Expected user homedir to be non-empty")
+
+			sh, err := dei.Shell(tt.containerUser)
+			require.NoError(t, err, "Expected no error from UserShell()")
+			require.Equal(t, tt.expectedUserShell, sh, "Expected user shell to match")
+
+			// We don't need to test the actual environment variables here.
+			environ := dei.Environ()
+			require.NotEmpty(t, environ, "Expected environ to be non-empty")
+
+			// Test that the environment variables are present in modified command
+			// output.
+			envCmd, envArgs := dei.ModifyCommand("env")
+			for _, env := range tt.expectedEnv {
+				require.Subset(t, envArgs, []string{"--env", env})
+			}
+			// Run the command in the container and check the output
+			// HACK: we remove the --tty argument because we're not running in a tty
+			envArgs = slices.DeleteFunc(envArgs, func(s string) bool { return s == "--tty" })
+			stdout, stderr, err := run(ctx, agentexec.DefaultExecer, envCmd, envArgs...)
+			require.Empty(t, stderr, "Expected no stderr output")
+			require.NoError(t, err, "Expected no error from running command")
+			for _, env := range tt.expectedEnv {
+				require.Contains(t, stdout, env)
+			}
+		})
+	}
+}
+
+func run(ctx context.Context, execer agentexec.Execer, cmd string, args ...string) (stdout, stderr string, err error) {
+	var stdoutBuf, stderrBuf strings.Builder
+	execCmd := execer.CommandContext(ctx, cmd, args...)
+	execCmd.Stdout = &stdoutBuf
+	execCmd.Stderr = &stderrBuf
+	err = execCmd.Run()
+	stdout = strings.TrimSpace(stdoutBuf.String())
+	stderr = strings.TrimSpace(stderrBuf.String())
+	return stdout, stderr, err
+}

agent/agentcontainers/dcspec/dcspec_gen.go

@@ -0,0 +1,601 @@
+// Code generated by dcspec/gen.sh. DO NOT EDIT.
+//
+// This file was generated from JSON Schema using quicktype, do not modify it directly.
+// To parse and unparse this JSON data, add this code to your project and do:
+//
+//    devContainer, err := UnmarshalDevContainer(bytes)
+//    bytes, err = devContainer.Marshal()
+
+package dcspec
+
+import (
+	"bytes"
+	"errors"
+)
+
+import "encoding/json"
+
+func UnmarshalDevContainer(data []byte) (DevContainer, error) {
+	var r DevContainer
+	err := json.Unmarshal(data, &r)
+	return r, err
+}
+
+func (r *DevContainer) Marshal() ([]byte, error) {
+	return json.Marshal(r)
+}
+
+// Defines a dev container
+type DevContainer struct {
+	// Docker build-related options.
+	Build *BuildOptions `json:"build,omitempty"`
+	// The location of the context folder for building the Docker image. The path is relative to
+	// the folder containing the `devcontainer.json` file.
+	Context *string `json:"context,omitempty"`
+	// The location of the Dockerfile that defines the contents of the container. The path is
+	// relative to the folder containing the `devcontainer.json` file.
+	DockerFile *string `json:"dockerFile,omitempty"`
+	// The docker image that will be used to create the container.
+	Image *string `json:"image,omitempty"`
+	// Application ports that are exposed by the container. This can be a single port or an
+	// array of ports. Each port can be a number or a string. A number is mapped to the same
+	// port on the host. A string is passed to Docker unchanged and can be used to map ports
+	// differently, e.g. "8000:8010".
+	AppPort *DevContainerAppPort `json:"appPort"`
+	// Whether to overwrite the command specified in the image. The default is true.
+	//
+	// Whether to overwrite the command specified in the image. The default is false.
+	OverrideCommand *bool `json:"overrideCommand,omitempty"`
+	// The arguments required when starting in the container.
+	RunArgs []string `json:"runArgs,omitempty"`
+	// Action to take when the user disconnects from the container in their editor. The default
+	// is to stop the container.
+	//
+	// Action to take when the user disconnects from the primary container in their editor. The
+	// default is to stop all of the compose containers.
+	ShutdownAction *ShutdownAction `json:"shutdownAction,omitempty"`
+	// The path of the workspace folder inside the container.
+	//
+	// The path of the workspace folder inside the container. This is typically the target path
+	// of a volume mount in the docker-compose.yml.
+	WorkspaceFolder *string `json:"workspaceFolder,omitempty"`
+	// The --mount parameter for docker run. The default is to mount the project folder at
+	// /workspaces/$project.
+	WorkspaceMount *string `json:"workspaceMount,omitempty"`
+	// The name of the docker-compose file(s) used to start the services.
+	DockerComposeFile *CacheFrom `json:"dockerComposeFile"`
+	// An array of services that should be started and stopped.
+	RunServices []string `json:"runServices,omitempty"`
+	// The service you want to work on. This is considered the primary container for your dev
+	// environment which your editor will connect to.
+	Service *string `json:"service,omitempty"`
+	// The JSON schema of the `devcontainer.json` file.
+	Schema               *string                `json:"$schema,omitempty"`
+	AdditionalProperties map[string]interface{} `json:"additionalProperties,omitempty"`
+	// Passes docker capabilities to include when creating the dev container.
+	CapAdd []string `json:"capAdd,omitempty"`
+	// Container environment variables.
+	ContainerEnv map[string]string `json:"containerEnv,omitempty"`
+	// The user the container will be started with. The default is the user on the Docker image.
+	ContainerUser *string `json:"containerUser,omitempty"`
+	// Tool-specific configuration. Each tool should use a JSON object subproperty with a unique
+	// name to group its customizations.
+	Customizations map[string]interface{} `json:"customizations,omitempty"`
+	// Features to add to the dev container.
+	Features *Features `json:"features,omitempty"`
+	// Ports that are forwarded from the container to the local machine. Can be an integer port
+	// number, or a string of the format "host:port_number".
+	ForwardPorts []ForwardPort `json:"forwardPorts,omitempty"`
+	// Host hardware requirements.
+	HostRequirements *HostRequirements `json:"hostRequirements,omitempty"`
+	// Passes the --init flag when creating the dev container.
+	Init *bool `json:"init,omitempty"`
+	// A command to run locally (i.e Your host machine, cloud VM) before anything else. This
+	// command is run before "onCreateCommand". If this is a single string, it will be run in a
+	// shell. If this is an array of strings, it will be run as a single command without shell.
+	// If this is an object, each provided command will be run in parallel.
+	InitializeCommand *Command `json:"initializeCommand"`
+	// Mount points to set up when creating the container. See Docker's documentation for the
+	// --mount option for the supported syntax.
+	Mounts []MountElement `json:"mounts,omitempty"`
+	// A name for the dev container which can be displayed to the user.
+	Name *string `json:"name,omitempty"`
+	// A command to run when creating the container. This command is run after
+	// "initializeCommand" and before "updateContentCommand". If this is a single string, it
+	// will be run in a shell. If this is an array of strings, it will be run as a single
+	// command without shell. If this is an object, each provided command will be run in
+	// parallel.
+	OnCreateCommand      *Command              `json:"onCreateCommand"`
+	OtherPortsAttributes *OtherPortsAttributes `json:"otherPortsAttributes,omitempty"`
+	// Array consisting of the Feature id (without the semantic version) of Features in the
+	// order the user wants them to be installed.
+	OverrideFeatureInstallOrder []string         `json:"overrideFeatureInstallOrder,omitempty"`
+	PortsAttributes             *PortsAttributes `json:"portsAttributes,omitempty"`
+	// A command to run when attaching to the container. This command is run after
+	// "postStartCommand". If this is a single string, it will be run in a shell. If this is an
+	// array of strings, it will be run as a single command without shell. If this is an object,
+	// each provided command will be run in parallel.
+	PostAttachCommand *Command `json:"postAttachCommand"`
+	// A command to run after creating the container. This command is run after
+	// "updateContentCommand" and before "postStartCommand". If this is a single string, it will
+	// be run in a shell. If this is an array of strings, it will be run as a single command
+	// without shell. If this is an object, each provided command will be run in parallel.
+	PostCreateCommand *Command `json:"postCreateCommand"`
+	// A command to run after starting the container. This command is run after
+	// "postCreateCommand" and before "postAttachCommand". If this is a single string, it will
+	// be run in a shell. If this is an array of strings, it will be run as a single command
+	// without shell. If this is an object, each provided command will be run in parallel.
+	PostStartCommand *Command `json:"postStartCommand"`
+	// Passes the --privileged flag when creating the dev container.
+	Privileged *bool `json:"privileged,omitempty"`
+	// Remote environment variables to set for processes spawned in the container including
+	// lifecycle scripts and any remote editor/IDE server process.
+	RemoteEnv map[string]*string `json:"remoteEnv,omitempty"`
+	// The username to use for spawning processes in the container including lifecycle scripts
+	// and any remote editor/IDE server process. The default is the same user as the container.
+	RemoteUser *string `json:"remoteUser,omitempty"`
+	// Recommended secrets for this dev container. Recommendations are provided as environment
+	// variable keys with optional metadata.
+	Secrets *Secrets `json:"secrets,omitempty"`
+	// Passes docker security options to include when creating the dev container.
+	SecurityOpt []string `json:"securityOpt,omitempty"`
+	// A command to run when creating the container and rerun when the workspace content was
+	// updated while creating the container. This command is run after "onCreateCommand" and
+	// before "postCreateCommand". If this is a single string, it will be run in a shell. If
+	// this is an array of strings, it will be run as a single command without shell. If this is
+	// an object, each provided command will be run in parallel.
+	UpdateContentCommand *Command `json:"updateContentCommand"`
+	// Controls whether on Linux the container's user should be updated with the local user's
+	// UID and GID. On by default when opening from a local folder.
+	UpdateRemoteUserUID *bool `json:"updateRemoteUserUID,omitempty"`
+	// User environment probe to run. The default is "loginInteractiveShell".
+	UserEnvProbe *UserEnvProbe `json:"userEnvProbe,omitempty"`
+	// The user command to wait for before continuing execution in the background while the UI
+	// is starting up. The default is "updateContentCommand".
+	WaitFor *WaitFor `json:"waitFor,omitempty"`
+}
+
+// Docker build-related options.
+type BuildOptions struct {
+	// The location of the context folder for building the Docker image. The path is relative to
+	// the folder containing the `devcontainer.json` file.
+	Context *string `json:"context,omitempty"`
+	// The location of the Dockerfile that defines the contents of the container. The path is
+	// relative to the folder containing the `devcontainer.json` file.
+	Dockerfile *string `json:"dockerfile,omitempty"`
+	// Build arguments.
+	Args map[string]string `json:"args,omitempty"`
+	// The image to consider as a cache. Use an array to specify multiple images.
+	CacheFrom *CacheFrom `json:"cacheFrom"`
+	// Additional arguments passed to the build command.
+	Options []string `json:"options,omitempty"`
+	// Target stage in a multi-stage build.
+	Target *string `json:"target,omitempty"`
+}
+
+// Features to add to the dev container.
+type Features struct {
+	Fish       interface{} `json:"fish"`
+	Gradle     interface{} `json:"gradle"`
+	Homebrew   interface{} `json:"homebrew"`
+	Jupyterlab interface{} `json:"jupyterlab"`
+	Maven      interface{} `json:"maven"`
+}
+
+// Host hardware requirements.
+type HostRequirements struct {
+	// Number of required CPUs.
+	Cpus *int64    `json:"cpus,omitempty"`
+	GPU  *GPUUnion `json:"gpu"`
+	// Amount of required RAM in bytes. Supports units tb, gb, mb and kb.
+	Memory *string `json:"memory,omitempty"`
+	// Amount of required disk space in bytes. Supports units tb, gb, mb and kb.
+	Storage *string `json:"storage,omitempty"`
+}
+
+// Indicates whether a GPU is required. The string "optional" indicates that a GPU is
+// optional. An object value can be used to configure more detailed requirements.
+type GPUClass struct {
+	// Number of required cores.
+	Cores *int64 `json:"cores,omitempty"`
+	// Amount of required RAM in bytes. Supports units tb, gb, mb and kb.
+	Memory *string `json:"memory,omitempty"`
+}
+
+type Mount struct {
+	// Mount source.
+	Source *string `json:"source,omitempty"`
+	// Mount target.
+	Target string `json:"target"`
+	// Mount type.
+	Type Type `json:"type"`
+}
+
+type OtherPortsAttributes struct {
+	// Automatically prompt for elevation (if needed) when this port is forwarded. Elevate is
+	// required if the local port is a privileged port.
+	ElevateIfNeeded *bool `json:"elevateIfNeeded,omitempty"`
+	// Label that will be shown in the UI for this port.
+	Label *string `json:"label,omitempty"`
+	// Defines the action that occurs when the port is discovered for automatic forwarding
+	OnAutoForward *OnAutoForward `json:"onAutoForward,omitempty"`
+	// The protocol to use when forwarding this port.
+	Protocol         *Protocol `json:"protocol,omitempty"`
+	RequireLocalPort *bool     `json:"requireLocalPort,omitempty"`
+}
+
+type PortsAttributes struct{}
+
+// Recommended secrets for this dev container. Recommendations are provided as environment
+// variable keys with optional metadata.
+type Secrets struct{}
+
+type GPUEnum string
+
+const (
+	Optional GPUEnum = "optional"
+)
+
+// Mount type.
+type Type string
+
+const (
+	Bind   Type = "bind"
+	Volume Type = "volume"
+)
+
+// Defines the action that occurs when the port is discovered for automatic forwarding
+type OnAutoForward string
+
+const (
+	Ignore      OnAutoForward = "ignore"
+	Notify      OnAutoForward = "notify"
+	OpenBrowser OnAutoForward = "openBrowser"
+	OpenPreview OnAutoForward = "openPreview"
+	Silent      OnAutoForward = "silent"
+)
+
+// The protocol to use when forwarding this port.
+type Protocol string
+
+const (
+	HTTP  Protocol = "http"
+	HTTPS Protocol = "https"
+)
+
+// Action to take when the user disconnects from the container in their editor. The default
+// is to stop the container.
+//
+// Action to take when the user disconnects from the primary container in their editor. The
+// default is to stop all of the compose containers.
+type ShutdownAction string
+
+const (
+	ShutdownActionNone ShutdownAction = "none"
+	StopCompose        ShutdownAction = "stopCompose"
+	StopContainer      ShutdownAction = "stopContainer"
+)
+
+// User environment probe to run. The default is "loginInteractiveShell".
+type UserEnvProbe string
+
+const (
+	InteractiveShell      UserEnvProbe = "interactiveShell"
+	LoginInteractiveShell UserEnvProbe = "loginInteractiveShell"
+	LoginShell            UserEnvProbe = "loginShell"
+	UserEnvProbeNone      UserEnvProbe = "none"
+)
+
+// The user command to wait for before continuing execution in the background while the UI
+// is starting up. The default is "updateContentCommand".
+type WaitFor string
+
+const (
+	InitializeCommand    WaitFor = "initializeCommand"
+	OnCreateCommand      WaitFor = "onCreateCommand"
+	PostCreateCommand    WaitFor = "postCreateCommand"
+	PostStartCommand     WaitFor = "postStartCommand"
+	UpdateContentCommand WaitFor = "updateContentCommand"
+)
+
+// Application ports that are exposed by the container. This can be a single port or an
+// array of ports. Each port can be a number or a string. A number is mapped to the same
+// port on the host. A string is passed to Docker unchanged and can be used to map ports
+// differently, e.g. "8000:8010".
+type DevContainerAppPort struct {
+	Integer    *int64
+	String     *string
+	UnionArray []AppPortElement
+}
+
+func (x *DevContainerAppPort) UnmarshalJSON(data []byte) error {
+	x.UnionArray = nil
+	object, err := unmarshalUnion(data, &x.Integer, nil, nil, &x.String, true, &x.UnionArray, false, nil, false, nil, false, nil, false)
+	if err != nil {
+		return err
+	}
+	if object {
+	}
+	return nil
+}
+
+func (x *DevContainerAppPort) MarshalJSON() ([]byte, error) {
+	return marshalUnion(x.Integer, nil, nil, x.String, x.UnionArray != nil, x.UnionArray, false, nil, false, nil, false, nil, false)
+}
+
+// Application ports that are exposed by the container. This can be a single port or an
+// array of ports. Each port can be a number or a string. A number is mapped to the same
+// port on the host. A string is passed to Docker unchanged and can be used to map ports
+// differently, e.g. "8000:8010".
+type AppPortElement struct {
+	Integer *int64
+	String  *string
+}
+
+func (x *AppPortElement) UnmarshalJSON(data []byte) error {
+	object, err := unmarshalUnion(data, &x.Integer, nil, nil, &x.String, false, nil, false, nil, false, nil, false, nil, false)
+	if err != nil {
+		return err
+	}
+	if object {
+	}
+	return nil
+}
+
+func (x *AppPortElement) MarshalJSON() ([]byte, error) {
+	return marshalUnion(x.Integer, nil, nil, x.String, false, nil, false, nil, false, nil, false, nil, false)
+}
+
+// The image to consider as a cache. Use an array to specify multiple images.
+//
+// The name of the docker-compose file(s) used to start the services.
+type CacheFrom struct {
+	String      *string
+	StringArray []string
+}
+
+func (x *CacheFrom) UnmarshalJSON(data []byte) error {
+	x.StringArray = nil
+	object, err := unmarshalUnion(data, nil, nil, nil, &x.String, true, &x.StringArray, false, nil, false, nil, false, nil, false)
+	if err != nil {
+		return err
+	}
+	if object {
+	}
+	return nil
+}
+
+func (x *CacheFrom) MarshalJSON() ([]byte, error) {
+	return marshalUnion(nil, nil, nil, x.String, x.StringArray != nil, x.StringArray, false, nil, false, nil, false, nil, false)
+}
+
+type ForwardPort struct {
+	Integer *int64
+	String  *string
+}
+
+func (x *ForwardPort) UnmarshalJSON(data []byte) error {
+	object, err := unmarshalUnion(data, &x.Integer, nil, nil, &x.String, false, nil, false, nil, false, nil, false, nil, false)
+	if err != nil {
+		return err
+	}
+	if object {
+	}
+	return nil
+}
+
+func (x *ForwardPort) MarshalJSON() ([]byte, error) {
+	return marshalUnion(x.Integer, nil, nil, x.String, false, nil, false, nil, false, nil, false, nil, false)
+}
+
+type GPUUnion struct {
+	Bool     *bool
+	Enum     *GPUEnum
+	GPUClass *GPUClass
+}
+
+func (x *GPUUnion) UnmarshalJSON(data []byte) error {
+	x.GPUClass = nil
+	x.Enum = nil
+	var c GPUClass
+	object, err := unmarshalUnion(data, nil, nil, &x.Bool, nil, false, nil, true, &c, false, nil, true, &x.Enum, false)
+	if err != nil {
+		return err
+	}
+	if object {
+		x.GPUClass = &c
+	}
+	return nil
+}
+
+func (x *GPUUnion) MarshalJSON() ([]byte, error) {
+	return marshalUnion(nil, nil, x.Bool, nil, false, nil, x.GPUClass != nil, x.GPUClass, false, nil, x.Enum != nil, x.Enum, false)
+}
+
+// A command to run locally (i.e Your host machine, cloud VM) before anything else. This
+// command is run before "onCreateCommand". If this is a single string, it will be run in a
+// shell. If this is an array of strings, it will be run as a single command without shell.
+// If this is an object, each provided command will be run in parallel.
+//
+// A command to run when creating the container. This command is run after
+// "initializeCommand" and before "updateContentCommand". If this is a single string, it
+// will be run in a shell. If this is an array of strings, it will be run as a single
+// command without shell. If this is an object, each provided command will be run in
+// parallel.
+//
+// A command to run when attaching to the container. This command is run after
+// "postStartCommand". If this is a single string, it will be run in a shell. If this is an
+// array of strings, it will be run as a single command without shell. If this is an object,
+// each provided command will be run in parallel.
+//
+// A command to run after creating the container. This command is run after
+// "updateContentCommand" and before "postStartCommand". If this is a single string, it will
+// be run in a shell. If this is an array of strings, it will be run as a single command
+// without shell. If this is an object, each provided command will be run in parallel.
+//
+// A command to run after starting the container. This command is run after
+// "postCreateCommand" and before "postAttachCommand". If this is a single string, it will
+// be run in a shell. If this is an array of strings, it will be run as a single command
+// without shell. If this is an object, each provided command will be run in parallel.
+//
+// A command to run when creating the container and rerun when the workspace content was
+// updated while creating the container. This command is run after "onCreateCommand" and
+// before "postCreateCommand". If this is a single string, it will be run in a shell. If
+// this is an array of strings, it will be run as a single command without shell. If this is
+// an object, each provided command will be run in parallel.
+type Command struct {
+	String      *string
+	StringArray []string
+	UnionMap    map[string]*CacheFrom
+}
+
+func (x *Command) UnmarshalJSON(data []byte) error {
+	x.StringArray = nil
+	x.UnionMap = nil
+	object, err := unmarshalUnion(data, nil, nil, nil, &x.String, true, &x.StringArray, false, nil, true, &x.UnionMap, false, nil, false)
+	if err != nil {
+		return err
+	}
+	if object {
+	}
+	return nil
+}
+
+func (x *Command) MarshalJSON() ([]byte, error) {
+	return marshalUnion(nil, nil, nil, x.String, x.StringArray != nil, x.StringArray, false, nil, x.UnionMap != nil, x.UnionMap, false, nil, false)
+}
+
+type MountElement struct {
+	Mount  *Mount
+	String *string
+}
+
+func (x *MountElement) UnmarshalJSON(data []byte) error {
+	x.Mount = nil
+	var c Mount
+	object, err := unmarshalUnion(data, nil, nil, nil, &x.String, false, nil, true, &c, false, nil, false, nil, false)
+	if err != nil {
+		return err
+	}
+	if object {
+		x.Mount = &c
+	}
+	return nil
+}
+
+func (x *MountElement) MarshalJSON() ([]byte, error) {
+	return marshalUnion(nil, nil, nil, x.String, false, nil, x.Mount != nil, x.Mount, false, nil, false, nil, false)
+}
+
+func unmarshalUnion(data []byte, pi **int64, pf **float64, pb **bool, ps **string, haveArray bool, pa interface{}, haveObject bool, pc interface{}, haveMap bool, pm interface{}, haveEnum bool, pe interface{}, nullable bool) (bool, error) {
+	if pi != nil {
+		*pi = nil
+	}
+	if pf != nil {
+		*pf = nil
+	}
+	if pb != nil {
+		*pb = nil
+	}
+	if ps != nil {
+		*ps = nil
+	}
+
+	dec := json.NewDecoder(bytes.NewReader(data))
+	dec.UseNumber()
+	tok, err := dec.Token()
+	if err != nil {
+		return false, err
+	}
+
+	switch v := tok.(type) {
+	case json.Number:
+		if pi != nil {
+			i, err := v.Int64()
+			if err == nil {
+				*pi = &i
+				return false, nil
+			}
+		}
+		if pf != nil {
+			f, err := v.Float64()
+			if err == nil {
+				*pf = &f
+				return false, nil
+			}
+			return false, errors.New("Unparsable number")
+		}
+		return false, errors.New("Union does not contain number")
+	case float64:
+		return false, errors.New("Decoder should not return float64")
+	case bool:
+		if pb != nil {
+			*pb = &v
+			return false, nil
+		}
+		return false, errors.New("Union does not contain bool")
+	case string:
+		if haveEnum {
+			return false, json.Unmarshal(data, pe)
+		}
+		if ps != nil {
+			*ps = &v
+			return false, nil
+		}
+		return false, errors.New("Union does not contain string")
+	case nil:
+		if nullable {
+			return false, nil
+		}
+		return false, errors.New("Union does not contain null")
+	case json.Delim:
+		if v == '{' {
+			if haveObject {
+				return true, json.Unmarshal(data, pc)
+			}
+			if haveMap {
+				return false, json.Unmarshal(data, pm)
+			}
+			return false, errors.New("Union does not contain object")
+		}
+		if v == '[' {
+			if haveArray {
+				return false, json.Unmarshal(data, pa)
+			}
+			return false, errors.New("Union does not contain array")
+		}
+		return false, errors.New("Cannot handle delimiter")
+	}
+	return false, errors.New("Cannot unmarshal union")
+}
+
+func marshalUnion(pi *int64, pf *float64, pb *bool, ps *string, haveArray bool, pa interface{}, haveObject bool, pc interface{}, haveMap bool, pm interface{}, haveEnum bool, pe interface{}, nullable bool) ([]byte, error) {
+	if pi != nil {
+		return json.Marshal(*pi)
+	}
+	if pf != nil {
+		return json.Marshal(*pf)
+	}
+	if pb != nil {
+		return json.Marshal(*pb)
+	}
+	if ps != nil {
+		return json.Marshal(*ps)
+	}
+	if haveArray {
+		return json.Marshal(pa)
+	}
+	if haveObject {
+		return json.Marshal(pc)
+	}
+	if haveMap {
+		return json.Marshal(pm)
+	}
+	if haveEnum {
+		return json.Marshal(pe)
+	}
+	if nullable {
+		return json.Marshal(nil)
+	}
+	return nil, errors.New("Union must not be null")
+}

agent/agentcontainers/dcspec/dcspec_test.go

@@ -0,0 +1,148 @@
+package dcspec_test
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"slices"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/stretchr/testify/require"
+
+	"github.com/coder/coder/v2/agent/agentcontainers/dcspec"
+	"github.com/coder/coder/v2/coderd/util/ptr"
+)
+
+func TestUnmarshalDevContainer(t *testing.T) {
+	t.Parallel()
+
+	type testCase struct {
+		name    string
+		file    string
+		wantErr bool
+		want    dcspec.DevContainer
+	}
+	tests := []testCase{
+		{
+			name: "minimal",
+			file: filepath.Join("testdata", "minimal.json"),
+			want: dcspec.DevContainer{
+				Image: ptr.Ref("test-image"),
+			},
+		},
+		{
+			name: "arrays",
+			file: filepath.Join("testdata", "arrays.json"),
+			want: dcspec.DevContainer{
+				Image:   ptr.Ref("test-image"),
+				RunArgs: []string{"--network=host", "--privileged"},
+				ForwardPorts: []dcspec.ForwardPort{
+					{
+						Integer: ptr.Ref[int64](8080),
+					},
+					{
+						String: ptr.Ref("3000:3000"),
+					},
+				},
+			},
+		},
+		{
+			name:    "devcontainers/template-starter",
+			file:    filepath.Join("testdata", "devcontainers-template-starter.json"),
+			wantErr: false,
+			want: dcspec.DevContainer{
+				Image:    ptr.Ref("mcr.microsoft.com/devcontainers/javascript-node:1-18-bullseye"),
+				Features: &dcspec.Features{},
+				Customizations: map[string]interface{}{
+					"vscode": map[string]interface{}{
+						"extensions": []interface{}{
+							"mads-hartmann.bash-ide-vscode",
+							"dbaeumer.vscode-eslint",
+						},
+					},
+				},
+				PostCreateCommand: &dcspec.Command{
+					String: ptr.Ref("npm install -g @devcontainers/cli"),
+				},
+			},
+		},
+	}
+
+	var missingTests []string
+	files, err := filepath.Glob("testdata/*.json")
+	require.NoError(t, err, "glob test files failed")
+	for _, file := range files {
+		if !slices.ContainsFunc(tests, func(tt testCase) bool {
+			return tt.file == file
+		}) {
+			missingTests = append(missingTests, file)
+		}
+	}
+	require.Empty(t, missingTests, "missing tests case for files: %v", missingTests)
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			data, err := os.ReadFile(tt.file)
+			require.NoError(t, err, "read test file failed")
+
+			got, err := dcspec.UnmarshalDevContainer(data)
+			if tt.wantErr {
+				require.Error(t, err, "want error but got nil")
+				return
+			}
+			require.NoError(t, err, "unmarshal DevContainer failed")
+
+			// Compare the unmarshaled data with the expected data.
+			if diff := cmp.Diff(tt.want, got); diff != "" {
+				require.Empty(t, diff, "UnmarshalDevContainer() mismatch (-want +got):\n%s", diff)
+			}
+
+			// Test that marshaling works (without comparing to original).
+			marshaled, err := got.Marshal()
+			require.NoError(t, err, "marshal DevContainer back to JSON failed")
+			require.NotEmpty(t, marshaled, "marshaled JSON should not be empty")
+
+			// Verify the marshaled JSON can be unmarshaled back.
+			var unmarshaled interface{}
+			err = json.Unmarshal(marshaled, &unmarshaled)
+			require.NoError(t, err, "unmarshal marshaled JSON failed")
+		})
+	}
+}
+
+func TestUnmarshalDevContainer_EdgeCases(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name    string
+		json    string
+		wantErr bool
+	}{
+		{
+			name:    "empty JSON",
+			json:    "{}",
+			wantErr: false,
+		},
+		{
+			name:    "invalid JSON",
+			json:    "{not valid json",
+			wantErr: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			_, err := dcspec.UnmarshalDevContainer([]byte(tt.json))
+			if tt.wantErr {
+				require.Error(t, err, "want error but got nil")
+				return
+			}
+			require.NoError(t, err, "unmarshal DevContainer failed")
+		})
+	}
+}

agent/agentcontainers/dcspec/devContainer.base.schema.json

@@ -0,0 +1,771 @@
+{
+	"$schema": "https://json-schema.org/draft/2019-09/schema",
+	"description": "Defines a dev container",
+	"allowComments": true,
+	"allowTrailingCommas": false,
+	"definitions": {
+		"devContainerCommon": {
+			"type": "object",
+			"properties": {
+				"$schema": {
+					"type": "string",
+					"format": "uri",
+					"description": "The JSON schema of the `devcontainer.json` file."
+				},
+				"name": {
+					"type": "string",
+					"description": "A name for the dev container which can be displayed to the user."
+				},
+				"features": {
+					"type": "object",
+					"description": "Features to add to the dev container.",
+					"properties": {
+						"fish": {
+							"deprecated": true,
+							"deprecationMessage": "Legacy feature not supported. Please check https://containers.dev/features for replacements."
+						},
+						"maven": {
+							"deprecated": true,
+							"deprecationMessage": "Legacy feature will be removed in the future. Please check https://containers.dev/features for replacements. E.g., `ghcr.io/devcontainers/features/java` has an option to install Maven."
+						},
+						"gradle": {
+							"deprecated": true,
+							"deprecationMessage": "Legacy feature will be removed in the future. Please check https://containers.dev/features for replacements. E.g., `ghcr.io/devcontainers/features/java` has an option to install Gradle."
+						},
+						"homebrew": {
+							"deprecated": true,
+							"deprecationMessage": "Legacy feature not supported. Please check https://containers.dev/features for replacements."
+						},
+						"jupyterlab": {
+							"deprecated": true,
+							"deprecationMessage": "Legacy feature will be removed in the future. Please check https://containers.dev/features for replacements. E.g., `ghcr.io/devcontainers/features/python` has an option to install JupyterLab."
+						}
+					},
+					"additionalProperties": true
+				},
+				"overrideFeatureInstallOrder": {
+					"type": "array",
+					"description": "Array consisting of the Feature id (without the semantic version) of Features in the order the user wants them to be installed.",
+					"items": {
+						"type": "string"
+					}
+				},
+				"secrets": {
+					"type": "object",
+					"description": "Recommended secrets for this dev container. Recommendations are provided as environment variable keys with optional metadata.",
+					"patternProperties": {
+						"^[a-zA-Z_][a-zA-Z0-9_]*$": {
+							"type": "object",
+							"description": "Environment variable keys following unix-style naming conventions. eg: ^[a-zA-Z_][a-zA-Z0-9_]*$",
+							"properties": {
+								"description": {
+									"type": "string",
+									"description": "A description of the secret."
+								},
+								"documentationUrl": {
+									"type": "string",
+									"format": "uri",
+									"description": "A URL to documentation about the secret."
+								}
+							},
+							"additionalProperties": false
+						},
+						"additionalProperties": false
+					},
+					"additionalProperties": false
+				},
+				"forwardPorts": {
+					"type": "array",
+					"description": "Ports that are forwarded from the container to the local machine. Can be an integer port number, or a string of the format \"host:port_number\".",
+					"items": {
+						"oneOf": [
+							{
+								"type": "integer",
+								"maximum": 65535,
+								"minimum": 0
+							},
+							{
+								"type": "string",
+								"pattern": "^([a-z0-9-]+):(\\d{1,5})$"
+							}
+						]
+					}
+				},
+				"portsAttributes": {
+					"type": "object",
+					"patternProperties": {
+						"(^\\d+(-\\d+)?$)|(.+)": {
+							"type": "object",
+							"description": "A port, range of ports (ex. \"40000-55000\"), or regular expression (ex. \".+\\\\/server.js\").  For a port number or range, the attributes will apply to that port number or range of port numbers. Attributes which use a regular expression will apply to ports whose associated process command line matches the expression.",
+							"properties": {
+								"onAutoForward": {
+									"type": "string",
+									"enum": [
+										"notify",
+										"openBrowser",
+										"openBrowserOnce",
+										"openPreview",
+										"silent",
+										"ignore"
+									],
+									"enumDescriptions": [
+										"Shows a notification when a port is automatically forwarded.",
+										"Opens the browser when the port is automatically forwarded. Depending on your settings, this could open an embedded browser.",
+										"Opens the browser when the port is automatically forwarded, but only the first time the port is forward during a session. Depending on your settings, this could open an embedded browser.",
+										"Opens a preview in the same window when the port is automatically forwarded.",
+										"Shows no notification and takes no action when this port is automatically forwarded.",
+										"This port will not be automatically forwarded."
+									],
+									"description": "Defines the action that occurs when the port is discovered for automatic forwarding",
+									"default": "notify"
+								},
+								"elevateIfNeeded": {
+									"type": "boolean",
+									"description": "Automatically prompt for elevation (if needed) when this port is forwarded. Elevate is required if the local port is a privileged port.",
+									"default": false
+								},
+								"label": {
+									"type": "string",
+									"description": "Label that will be shown in the UI for this port.",
+									"default": "Application"
+								},
+								"requireLocalPort": {
+									"type": "boolean",
+									"markdownDescription": "When true, a modal dialog will show if the chosen local port isn't used for forwarding.",
+									"default": false
+								},
+								"protocol": {
+									"type": "string",
+									"enum": [
+										"http",
+										"https"
+									],
+									"description": "The protocol to use when forwarding this port."
+								}
+							},
+							"default": {
+								"label": "Application",
+								"onAutoForward": "notify"
+							}
+						}
+					},
+					"markdownDescription": "Set default properties that are applied when a specific port number is forwarded. For example:\n\n```\n\"3000\": {\n  \"label\": \"Application\"\n},\n\"40000-55000\": {\n  \"onAutoForward\": \"ignore\"\n},\n\".+\\\\/server.js\": {\n \"onAutoForward\": \"openPreview\"\n}\n```",
+					"defaultSnippets": [
+						{
+							"body": {
+								"${1:3000}": {
+									"label": "${2:Application}",
+									"onAutoForward": "notify"
+								}
+							}
+						}
+					],
+					"additionalProperties": false
+				},
+				"otherPortsAttributes": {
+					"type": "object",
+					"properties": {
+						"onAutoForward": {
+							"type": "string",
+							"enum": [
+								"notify",
+								"openBrowser",
+								"openPreview",
+								"silent",
+								"ignore"
+							],
+							"enumDescriptions": [
+								"Shows a notification when a port is automatically forwarded.",
+								"Opens the browser when the port is automatically forwarded. Depending on your settings, this could open an embedded browser.",
+								"Opens a preview in the same window when the port is automatically forwarded.",
+								"Shows no notification and takes no action when this port is automatically forwarded.",
+								"This port will not be automatically forwarded."
+							],
+							"description": "Defines the action that occurs when the port is discovered for automatic forwarding",
+							"default": "notify"
+						},
+						"elevateIfNeeded": {
+							"type": "boolean",
+							"description": "Automatically prompt for elevation (if needed) when this port is forwarded. Elevate is required if the local port is a privileged port.",
+							"default": false
+						},
+						"label": {
+							"type": "string",
+							"description": "Label that will be shown in the UI for this port.",
+							"default": "Application"
+						},
+						"requireLocalPort": {
+							"type": "boolean",
+							"markdownDescription": "When true, a modal dialog will show if the chosen local port isn't used for forwarding.",
+							"default": false
+						},
+						"protocol": {
+							"type": "string",
+							"enum": [
+								"http",
+								"https"
+							],
+							"description": "The protocol to use when forwarding this port."
+						}
+					},
+					"defaultSnippets": [
+						{
+							"body": {
+								"onAutoForward": "ignore"
+							}
+						}
+					],
+					"markdownDescription": "Set default properties that are applied to all ports that don't get properties from the setting `remote.portsAttributes`. For example:\n\n```\n{\n  \"onAutoForward\": \"ignore\"\n}\n```",
+					"additionalProperties": false
+				},
+				"updateRemoteUserUID": {
+					"type": "boolean",
+					"description": "Controls whether on Linux the container's user should be updated with the local user's UID and GID. On by default when opening from a local folder."
+				},
+				"containerEnv": {
+					"type": "object",
+					"additionalProperties": {
+						"type": "string"
+					},
+					"description": "Container environment variables."
+				},
+				"containerUser": {
+					"type": "string",
+					"description": "The user the container will be started with. The default is the user on the Docker image."
+				},
+				"mounts": {
+					"type": "array",
+					"description": "Mount points to set up when creating the container. See Docker's documentation for the --mount option for the supported syntax.",
+					"items": {
+						"anyOf": [
+							{
+								"$ref": "#/definitions/Mount"
+							},
+							{
+								"type": "string"
+							}
+						]
+					}
+				},
+				"init": {
+					"type": "boolean",
+					"description": "Passes the --init flag when creating the dev container."
+				},
+				"privileged": {
+					"type": "boolean",
+					"description": "Passes the --privileged flag when creating the dev container."
+				},
+				"capAdd": {
+					"type": "array",
+					"description": "Passes docker capabilities to include when creating the dev container.",
+					"examples": [
+						"SYS_PTRACE"
+					],
+					"items": {
+						"type": "string"
+					}
+				},
+				"securityOpt": {
+					"type": "array",
+					"description": "Passes docker security options to include when creating the dev container.",
+					"examples": [
+						"seccomp=unconfined"
+					],
+					"items": {
+						"type": "string"
+					}
+				},
+				"remoteEnv": {
+					"type": "object",
+					"additionalProperties": {
+						"type": [
+							"string",
+							"null"
+						]
+					},
+					"description": "Remote environment variables to set for processes spawned in the container including lifecycle scripts and any remote editor/IDE server process."
+				},
+				"remoteUser": {
+					"type": "string",
+					"description": "The username to use for spawning processes in the container including lifecycle scripts and any remote editor/IDE server process. The default is the same user as the container."
+				},
+				"initializeCommand": {
+					"type": [
+						"string",
+						"array",
+						"object"
+					],
+					"description": "A command to run locally (i.e Your host machine, cloud VM) before anything else. This command is run before \"onCreateCommand\". If this is a single string, it will be run in a shell. If this is an array of strings, it will be run as a single command without shell. If this is an object, each provided command will be run in parallel.",
+					"items": {
+						"type": "string"
+					},
+					"additionalProperties": {
+						"type": [
+							"string",
+							"array"
+						],
+						"items": {
+							"type": "string"
+						}
+					}
+				},
+				"onCreateCommand": {
+					"type": [
+						"string",
+						"array",
+						"object"
+					],
+					"description": "A command to run when creating the container. This command is run after \"initializeCommand\" and before \"updateContentCommand\". If this is a single string, it will be run in a shell. If this is an array of strings, it will be run as a single command without shell. If this is an object, each provided command will be run in parallel.",
+					"items": {
+						"type": "string"
+					},
+					"additionalProperties": {
+						"type": [
+							"string",
+							"array"
+						],
+						"items": {
+							"type": "string"
+						}
+					}
+				},
+				"updateContentCommand": {
+					"type": [
+						"string",
+						"array",
+						"object"
+					],
+					"description": "A command to run when creating the container and rerun when the workspace content was updated while creating the container. This command is run after \"onCreateCommand\" and before \"postCreateCommand\". If this is a single string, it will be run in a shell. If this is an array of strings, it will be run as a single command without shell. If this is an object, each provided command will be run in parallel.",
+					"items": {
+						"type": "string"
+					},
+					"additionalProperties": {
+						"type": [
+							"string",
+							"array"
+						],
+						"items": {
+							"type": "string"
+						}
+					}
+				},
+				"postCreateCommand": {
+					"type": [
+						"string",
+						"array",
+						"object"
+					],
+					"description": "A command to run after creating the container. This command is run after \"updateContentCommand\" and before \"postStartCommand\". If this is a single string, it will be run in a shell. If this is an array of strings, it will be run as a single command without shell. If this is an object, each provided command will be run in parallel.",
+					"items": {
+						"type": "string"
+					},
+					"additionalProperties": {
+						"type": [
+							"string",
+							"array"
+						],
+						"items": {
+							"type": "string"
+						}
+					}
+				},
+				"postStartCommand": {
+					"type": [
+						"string",
+						"array",
+						"object"
+					],
+					"description": "A command to run after starting the container. This command is run after \"postCreateCommand\" and before \"postAttachCommand\". If this is a single string, it will be run in a shell. If this is an array of strings, it will be run as a single command without shell. If this is an object, each provided command will be run in parallel.",
+					"items": {
+						"type": "string"
+					},
+					"additionalProperties": {
+						"type": [
+							"string",
+							"array"
+						],
+						"items": {
+							"type": "string"
+						}
+					}
+				},
+				"postAttachCommand": {
+					"type": [
+						"string",
+						"array",
+						"object"
+					],
+					"description": "A command to run when attaching to the container. This command is run after \"postStartCommand\". If this is a single string, it will be run in a shell. If this is an array of strings, it will be run as a single command without shell. If this is an object, each provided command will be run in parallel.",
+					"items": {
+						"type": "string"
+					},
+					"additionalProperties": {
+						"type": [
+							"string",
+							"array"
+						],
+						"items": {
+							"type": "string"
+						}
+					}
+				},
+				"waitFor": {
+					"type": "string",
+					"enum": [
+						"initializeCommand",
+						"onCreateCommand",
+						"updateContentCommand",
+						"postCreateCommand",
+						"postStartCommand"
+					],
+					"description": "The user command to wait for before continuing execution in the background while the UI is starting up. The default is \"updateContentCommand\"."
+				},
+				"userEnvProbe": {
+					"type": "string",
+					"enum": [
+						"none",
+						"loginShell",
+						"loginInteractiveShell",
+						"interactiveShell"
+					],
+					"description": "User environment probe to run. The default is \"loginInteractiveShell\"."
+				},
+				"hostRequirements": {
+					"type": "object",
+					"description": "Host hardware requirements.",
+					"properties": {
+						"cpus": {
+							"type": "integer",
+							"minimum": 1,
+							"description": "Number of required CPUs."
+						},
+						"memory": {
+							"type": "string",
+							"pattern": "^\\d+([tgmk]b)?$",
+							"description": "Amount of required RAM in bytes. Supports units tb, gb, mb and kb."
+						},
+						"storage": {
+							"type": "string",
+							"pattern": "^\\d+([tgmk]b)?$",
+							"description": "Amount of required disk space in bytes. Supports units tb, gb, mb and kb."
+						},
+						"gpu": {
+							"oneOf": [
+								{
+									"type": [
+										"boolean",
+										"string"
+									],
+									"enum": [
+										true,
+										false,
+										"optional"
+									],
+									"description": "Indicates whether a GPU is required. The string \"optional\" indicates that a GPU is optional. An object value can be used to configure more detailed requirements."
+								},
+								{
+									"type": "object",
+									"properties": {
+										"cores": {
+											"type": "integer",
+											"minimum": 1,
+											"description": "Number of required cores."
+										},
+										"memory": {
+											"type": "string",
+											"pattern": "^\\d+([tgmk]b)?$",
+											"description": "Amount of required RAM in bytes. Supports units tb, gb, mb and kb."
+										}
+									},
+									"description": "Indicates whether a GPU is required. The string \"optional\" indicates that a GPU is optional. An object value can be used to configure more detailed requirements.",
+									"additionalProperties": false
+								}
+							]
+						}
+					},
+					"unevaluatedProperties": false
+				},
+				"customizations": {
+					"type": "object",
+					"description": "Tool-specific configuration. Each tool should use a JSON object subproperty with a unique name to group its customizations."
+				},
+				"additionalProperties": {
+					"type": "object",
+					"additionalProperties": true
+				}
+			}
+		},
+		"nonComposeBase": {
+			"type": "object",
+			"properties": {
+				"appPort": {
+					"type": [
+						"integer",
+						"string",
+						"array"
+					],
+					"description": "Application ports that are exposed by the container. This can be a single port or an array of ports. Each port can be a number or a string. A number is mapped to the same port on the host. A string is passed to Docker unchanged and can be used to map ports differently, e.g. \"8000:8010\".",
+					"items": {
+						"type": [
+							"integer",
+							"string"
+						]
+					}
+				},
+				"runArgs": {
+					"type": "array",
+					"description": "The arguments required when starting in the container.",
+					"items": {
+						"type": "string"
+					}
+				},
+				"shutdownAction": {
+					"type": "string",
+					"enum": [
+						"none",
+						"stopContainer"
+					],
+					"description": "Action to take when the user disconnects from the container in their editor. The default is to stop the container."
+				},
+				"overrideCommand": {
+					"type": "boolean",
+					"description": "Whether to overwrite the command specified in the image. The default is true."
+				},
+				"workspaceFolder": {
+					"type": "string",
+					"description": "The path of the workspace folder inside the container."
+				},
+				"workspaceMount": {
+					"type": "string",
+					"description": "The --mount parameter for docker run. The default is to mount the project folder at /workspaces/$project."
+				}
+			}
+		},
+		"dockerfileContainer": {
+			"oneOf": [
+				{
+					"type": "object",
+					"properties": {
+						"build": {
+							"type": "object",
+							"description": "Docker build-related options.",
+							"allOf": [
+								{
+									"type": "object",
+									"properties": {
+										"dockerfile": {
+											"type": "string",
+											"description": "The location of the Dockerfile that defines the contents of the container. The path is relative to the folder containing the `devcontainer.json` file."
+										},
+										"context": {
+											"type": "string",
+											"description": "The location of the context folder for building the Docker image. The path is relative to the folder containing the `devcontainer.json` file."
+										}
+									},
+									"required": [
+										"dockerfile"
+									]
+								},
+								{
+									"$ref": "#/definitions/buildOptions"
+								}
+							],
+							"unevaluatedProperties": false
+						}
+					},
+					"required": [
+						"build"
+					]
+				},
+				{
+					"allOf": [
+						{
+							"type": "object",
+							"properties": {
+								"dockerFile": {
+									"type": "string",
+									"description": "The location of the Dockerfile that defines the contents of the container. The path is relative to the folder containing the `devcontainer.json` file."
+								},
+								"context": {
+									"type": "string",
+									"description": "The location of the context folder for building the Docker image. The path is relative to the folder containing the `devcontainer.json` file."
+								}
+							},
+							"required": [
+								"dockerFile"
+							]
+						},
+						{
+							"type": "object",
+							"properties": {
+								"build": {
+									"description": "Docker build-related options.",
+									"$ref": "#/definitions/buildOptions"
+								}
+							}
+						}
+					]
+				}
+			]
+		},
+		"buildOptions": {
+			"type": "object",
+			"properties": {
+				"target": {
+					"type": "string",
+					"description": "Target stage in a multi-stage build."
+				},
+				"args": {
+					"type": "object",
+					"additionalProperties": {
+						"type": [
+							"string"
+						]
+					},
+					"description": "Build arguments."
+				},
+				"cacheFrom": {
+					"type": [
+						"string",
+						"array"
+					],
+					"description": "The image to consider as a cache. Use an array to specify multiple images.",
+					"items": {
+						"type": "string"
+					}
+				},
+				"options": {
+					"type": "array",
+					"description": "Additional arguments passed to the build command.",
+					"items": {
+						"type": "string"
+					}
+				}
+			}
+		},
+		"imageContainer": {
+			"type": "object",
+			"properties": {
+				"image": {
+					"type": "string",
+					"description": "The docker image that will be used to create the container."
+				}
+			},
+			"required": [
+				"image"
+			]
+		},
+		"composeContainer": {
+			"type": "object",
+			"properties": {
+				"dockerComposeFile": {
+					"type": [
+						"string",
+						"array"
+					],
+					"description": "The name of the docker-compose file(s) used to start the services.",
+					"items": {
+						"type": "string"
+					}
+				},
+				"service": {
+					"type": "string",
+					"description": "The service you want to work on. This is considered the primary container for your dev environment which your editor will connect to."
+				},
+				"runServices": {
+					"type": "array",
+					"description": "An array of services that should be started and stopped.",
+					"items": {
+						"type": "string"
+					}
+				},
+				"workspaceFolder": {
+					"type": "string",
+					"description": "The path of the workspace folder inside the container. This is typically the target path of a volume mount in the docker-compose.yml."
+				},
+				"shutdownAction": {
+					"type": "string",
+					"enum": [
+						"none",
+						"stopCompose"
+					],
+					"description": "Action to take when the user disconnects from the primary container in their editor. The default is to stop all of the compose containers."
+				},
+				"overrideCommand": {
+					"type": "boolean",
+					"description": "Whether to overwrite the command specified in the image. The default is false."
+				}
+			},
+			"required": [
+				"dockerComposeFile",
+				"service",
+				"workspaceFolder"
+			]
+		},
+		"Mount": {
+			"type": "object",
+			"properties": {
+				"type": {
+					"type": "string",
+					"enum": [
+						"bind",
+						"volume"
+					],
+					"description": "Mount type."
+				},
+				"source": {
+					"type": "string",
+					"description": "Mount source."
+				},
+				"target": {
+					"type": "string",
+					"description": "Mount target."
+				}
+			},
+			"required": [
+				"type",
+				"target"
+			],
+			"additionalProperties": false
+		}
+	},
+	"oneOf": [
+		{
+			"allOf": [
+				{
+					"oneOf": [
+						{
+							"allOf": [
+								{
+									"oneOf": [
+										{
+											"$ref": "#/definitions/dockerfileContainer"
+										},
+										{
+											"$ref": "#/definitions/imageContainer"
+										}
+									]
+								},
+								{
+									"$ref": "#/definitions/nonComposeBase"
+								}
+							]
+						},
+						{
+							"$ref": "#/definitions/composeContainer"
+						}
+					]
+				},
+				{
+					"$ref": "#/definitions/devContainerCommon"
+				}
+			]
+		},
+		{
+			"type": "object",
+			"$ref": "#/definitions/devContainerCommon",
+			"additionalProperties": false
+		}
+	],
+	"unevaluatedProperties": false
+}

agent/agentcontainers/dcspec/doc.go

@@ -0,0 +1,5 @@
+// Package dcspec contains an automatically generated Devcontainer
+// specification.
+package dcspec
+
+//go:generate ./gen.sh

agent/agentcontainers/dcspec/gen.sh

@@ -0,0 +1,74 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# This script requires quicktype to be installed.
+# While you can install it using npm, we have it in our devDependencies
+# in ${PROJECT_ROOT}/package.json.
+PROJECT_ROOT="$(git rev-parse --show-toplevel)"
+if ! pnpm list | grep quicktype &>/dev/null; then
+	echo "quicktype is required to run this script!"
+	echo "Ensure that it is present in the devDependencies of ${PROJECT_ROOT}/package.json and then run pnpm install."
+	exit 1
+fi
+
+DEST_FILENAME="dcspec_gen.go"
+SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+DEST_PATH="${SCRIPT_DIR}/${DEST_FILENAME}"
+
+# Location of the JSON schema for the devcontainer specification.
+SCHEMA_SRC="https://raw.githubusercontent.com/devcontainers/spec/refs/heads/main/schemas/devContainer.base.schema.json"
+SCHEMA_DEST="${SCRIPT_DIR}/devContainer.base.schema.json"
+
+UPDATE_SCHEMA="${UPDATE_SCHEMA:-false}"
+if [[ "${UPDATE_SCHEMA}" = true || ! -f "${SCHEMA_DEST}" ]]; then
+	# Download the latest schema.
+	echo "Updating schema..."
+	curl --fail --silent --show-error --location --output "${SCHEMA_DEST}" "${SCHEMA_SRC}"
+else
+	echo "Using existing schema..."
+fi
+
+TMPDIR=$(mktemp -d)
+trap 'rm -rfv "$TMPDIR"' EXIT
+
+show_stderr=1
+exec 3>&2
+if [[ " $* " == *" --quiet "* ]] || [[ ${DCSPEC_QUIET:-false} == "true" ]]; then
+	# Redirect stderr to log because quicktype can't infer all types and
+	# we don't care right now.
+	show_stderr=0
+	exec 2>"${TMPDIR}/stderr.log"
+fi
+
+if ! pnpm exec quicktype \
+	--src-lang schema \
+	--lang go \
+	--top-level "DevContainer" \
+	--out "${TMPDIR}/${DEST_FILENAME}" \
+	--package "dcspec" \
+	"${SCHEMA_DEST}"; then
+	echo "quicktype failed to generate Go code." >&3
+	if [[ "${show_stderr}" -eq 1 ]]; then
+		cat "${TMPDIR}/stderr.log" >&3
+	fi
+	exit 1
+fi
+
+if [[ "${show_stderr}" -eq 0 ]]; then
+	# Restore stderr.
+	exec 2>&3
+fi
+exec 3>&-
+
+# Format the generated code.
+go run mvdan.cc/gofumpt@v0.4.0 -w -l "${TMPDIR}/${DEST_FILENAME}"
+
+# Add a header so that Go recognizes this as a generated file.
+if grep -q -- "\[-i extension\]" < <(sed -h 2>&1); then
+	# darwin sed
+	sed -i '' '1s/^/\/\/ Code generated by dcspec\/gen.sh. DO NOT EDIT.\n\/\/\n/' "${TMPDIR}/${DEST_FILENAME}"
+else
+	sed -i'' '1s/^/\/\/ Code generated by dcspec\/gen.sh. DO NOT EDIT.\n\/\/\n/' "${TMPDIR}/${DEST_FILENAME}"
+fi
+
+mv -v "${TMPDIR}/${DEST_FILENAME}" "${DEST_PATH}"

agent/agentcontainers/dcspec/testdata/arrays.json

@@ -0,0 +1,5 @@
+{
+	"image": "test-image",
+	"runArgs": ["--network=host", "--privileged"],
+	"forwardPorts": [8080, "3000:3000"]
+}

agent/agentcontainers/dcspec/testdata/devcontainers-template-starter.json

@@ -0,0 +1,12 @@
+{
+	"image": "mcr.microsoft.com/devcontainers/javascript-node:1-18-bullseye",
+	"features": {
+		"ghcr.io/devcontainers/features/docker-in-docker:2": {}
+	},
+	"customizations": {
+		"vscode": {
+			"extensions": ["mads-hartmann.bash-ide-vscode", "dbaeumer.vscode-eslint"]
+		}
+	},
+	"postCreateCommand": "npm install -g @devcontainers/cli"
+}

agent/agentcontainers/dcspec/testdata/minimal.json

@@ -0,0 +1 @@
+{ "image": "test-image" }

agent/agentcontainers/devcontainer.go

@@ -0,0 +1,119 @@
+package agentcontainers
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"cdr.dev/slog"
+	"github.com/coder/coder/v2/codersdk"
+)
+
+const (
+	// DevcontainerLocalFolderLabel is the label that contains the path to
+	// the local workspace folder for a devcontainer.
+	DevcontainerLocalFolderLabel = "devcontainer.local_folder"
+	// DevcontainerConfigFileLabel is the label that contains the path to
+	// the devcontainer.json configuration file.
+	DevcontainerConfigFileLabel = "devcontainer.config_file"
+)
+
+const devcontainerUpScriptTemplate = `
+if ! which devcontainer > /dev/null 2>&1; then
+  echo "ERROR: Unable to start devcontainer, @devcontainers/cli is not installed or not found in \$PATH." 1>&2
+  echo "Please install @devcontainers/cli by running \"npm install -g @devcontainers/cli\" or by using the \"devcontainers-cli\" Coder module." 1>&2
+  exit 1
+fi
+devcontainer up %s
+`
+
+// ExtractAndInitializeDevcontainerScripts extracts devcontainer scripts from
+// the given scripts and devcontainers. The devcontainer scripts are removed
+// from the returned scripts so that they can be run separately.
+//
+// Dev Containers have an inherent dependency on start scripts, since they
+// initialize the workspace (e.g. git clone, npm install, etc). This is
+// important if e.g. a Coder module to install @devcontainer/cli is used.
+func ExtractAndInitializeDevcontainerScripts(
+	devcontainers []codersdk.WorkspaceAgentDevcontainer,
+	scripts []codersdk.WorkspaceAgentScript,
+) (filteredScripts []codersdk.WorkspaceAgentScript, devcontainerScripts []codersdk.WorkspaceAgentScript) {
+ScriptLoop:
+	for _, script := range scripts {
+		for _, dc := range devcontainers {
+			// The devcontainer scripts match the devcontainer ID for
+			// identification.
+			if script.ID == dc.ID {
+				devcontainerScripts = append(devcontainerScripts, devcontainerStartupScript(dc, script))
+				continue ScriptLoop
+			}
+		}
+
+		filteredScripts = append(filteredScripts, script)
+	}
+
+	return filteredScripts, devcontainerScripts
+}
+
+func devcontainerStartupScript(dc codersdk.WorkspaceAgentDevcontainer, script codersdk.WorkspaceAgentScript) codersdk.WorkspaceAgentScript {
+	args := []string{
+		"--log-format json",
+		fmt.Sprintf("--workspace-folder %q", dc.WorkspaceFolder),
+	}
+	if dc.ConfigPath != "" {
+		args = append(args, fmt.Sprintf("--config %q", dc.ConfigPath))
+	}
+	cmd := fmt.Sprintf(devcontainerUpScriptTemplate, strings.Join(args, " "))
+	// Force the script to run in /bin/sh, since some shells (e.g. fish)
+	// don't support the script.
+	script.Script = fmt.Sprintf("/bin/sh -c '%s'", cmd)
+	// Disable RunOnStart, scripts have this set so that when devcontainers
+	// have not been enabled, a warning will be surfaced in the agent logs.
+	script.RunOnStart = false
+	return script
+}
+
+// ExpandAllDevcontainerPaths expands all devcontainer paths in the given
+// devcontainers. This is required by the devcontainer CLI, which requires
+// absolute paths for the workspace folder and config path.
+func ExpandAllDevcontainerPaths(logger slog.Logger, expandPath func(string) (string, error), devcontainers []codersdk.WorkspaceAgentDevcontainer) []codersdk.WorkspaceAgentDevcontainer {
+	expanded := make([]codersdk.WorkspaceAgentDevcontainer, 0, len(devcontainers))
+	for _, dc := range devcontainers {
+		expanded = append(expanded, expandDevcontainerPaths(logger, expandPath, dc))
+	}
+	return expanded
+}
+
+func expandDevcontainerPaths(logger slog.Logger, expandPath func(string) (string, error), dc codersdk.WorkspaceAgentDevcontainer) codersdk.WorkspaceAgentDevcontainer {
+	logger = logger.With(slog.F("devcontainer", dc.Name), slog.F("workspace_folder", dc.WorkspaceFolder), slog.F("config_path", dc.ConfigPath))
+
+	if wf, err := expandPath(dc.WorkspaceFolder); err != nil {
+		logger.Warn(context.Background(), "expand devcontainer workspace folder failed", slog.Error(err))
+	} else {
+		dc.WorkspaceFolder = wf
+	}
+	if dc.ConfigPath != "" {
+		// Let expandPath handle home directory, otherwise assume relative to
+		// workspace folder or absolute.
+		if dc.ConfigPath[0] == '~' {
+			if cp, err := expandPath(dc.ConfigPath); err != nil {
+				logger.Warn(context.Background(), "expand devcontainer config path failed", slog.Error(err))
+			} else {
+				dc.ConfigPath = cp
+			}
+		} else {
+			dc.ConfigPath = relativePathToAbs(dc.WorkspaceFolder, dc.ConfigPath)
+		}
+	}
+	return dc
+}
+
+func relativePathToAbs(workdir, path string) string {
+	path = os.ExpandEnv(path)
+	if !filepath.IsAbs(path) {
+		path = filepath.Join(workdir, path)
+	}
+	return path
+}

agent/agentcontainers/devcontainer_test.go

@@ -0,0 +1,274 @@
+package agentcontainers_test
+
+import (
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+	"github.com/google/uuid"
+	"github.com/stretchr/testify/require"
+
+	"cdr.dev/slog/sloggers/slogtest"
+	"github.com/coder/coder/v2/agent/agentcontainers"
+	"github.com/coder/coder/v2/codersdk"
+)
+
+func TestExtractAndInitializeDevcontainerScripts(t *testing.T) {
+	t.Parallel()
+
+	scriptIDs := []uuid.UUID{uuid.New(), uuid.New()}
+	devcontainerIDs := []uuid.UUID{uuid.New(), uuid.New()}
+
+	type args struct {
+		expandPath    func(string) (string, error)
+		devcontainers []codersdk.WorkspaceAgentDevcontainer
+		scripts       []codersdk.WorkspaceAgentScript
+	}
+	tests := []struct {
+		name                    string
+		args                    args
+		wantFilteredScripts     []codersdk.WorkspaceAgentScript
+		wantDevcontainerScripts []codersdk.WorkspaceAgentScript
+
+		skipOnWindowsDueToPathSeparator bool
+	}{
+		{
+			name: "no scripts",
+			args: args{
+				expandPath:    nil,
+				devcontainers: nil,
+				scripts:       nil,
+			},
+			wantFilteredScripts:     nil,
+			wantDevcontainerScripts: nil,
+		},
+		{
+			name: "no devcontainers",
+			args: args{
+				expandPath:    nil,
+				devcontainers: nil,
+				scripts: []codersdk.WorkspaceAgentScript{
+					{ID: scriptIDs[0]},
+					{ID: scriptIDs[1]},
+				},
+			},
+			wantFilteredScripts: []codersdk.WorkspaceAgentScript{
+				{ID: scriptIDs[0]},
+				{ID: scriptIDs[1]},
+			},
+			wantDevcontainerScripts: nil,
+		},
+		{
+			name: "no scripts match devcontainers",
+			args: args{
+				expandPath: nil,
+				devcontainers: []codersdk.WorkspaceAgentDevcontainer{
+					{ID: devcontainerIDs[0]},
+					{ID: devcontainerIDs[1]},
+				},
+				scripts: []codersdk.WorkspaceAgentScript{
+					{ID: scriptIDs[0]},
+					{ID: scriptIDs[1]},
+				},
+			},
+			wantFilteredScripts: []codersdk.WorkspaceAgentScript{
+				{ID: scriptIDs[0]},
+				{ID: scriptIDs[1]},
+			},
+			wantDevcontainerScripts: nil,
+		},
+		{
+			name: "scripts match devcontainers and sets RunOnStart=false",
+			args: args{
+				expandPath: nil,
+				devcontainers: []codersdk.WorkspaceAgentDevcontainer{
+					{ID: devcontainerIDs[0], WorkspaceFolder: "workspace1"},
+					{ID: devcontainerIDs[1], WorkspaceFolder: "workspace2"},
+				},
+				scripts: []codersdk.WorkspaceAgentScript{
+					{ID: scriptIDs[0], RunOnStart: true},
+					{ID: scriptIDs[1], RunOnStart: true},
+					{ID: devcontainerIDs[0], RunOnStart: true},
+					{ID: devcontainerIDs[1], RunOnStart: true},
+				},
+			},
+			wantFilteredScripts: []codersdk.WorkspaceAgentScript{
+				{ID: scriptIDs[0], RunOnStart: true},
+				{ID: scriptIDs[1], RunOnStart: true},
+			},
+			wantDevcontainerScripts: []codersdk.WorkspaceAgentScript{
+				{
+					ID:         devcontainerIDs[0],
+					Script:     "devcontainer up --log-format json --workspace-folder \"workspace1\"",
+					RunOnStart: false,
+				},
+				{
+					ID:         devcontainerIDs[1],
+					Script:     "devcontainer up --log-format json --workspace-folder \"workspace2\"",
+					RunOnStart: false,
+				},
+			},
+		},
+		{
+			name: "scripts match devcontainers with config path",
+			args: args{
+				expandPath: nil,
+				devcontainers: []codersdk.WorkspaceAgentDevcontainer{
+					{
+						ID:              devcontainerIDs[0],
+						WorkspaceFolder: "workspace1",
+						ConfigPath:      "config1",
+					},
+					{
+						ID:              devcontainerIDs[1],
+						WorkspaceFolder: "workspace2",
+						ConfigPath:      "config2",
+					},
+				},
+				scripts: []codersdk.WorkspaceAgentScript{
+					{ID: devcontainerIDs[0]},
+					{ID: devcontainerIDs[1]},
+				},
+			},
+			wantFilteredScripts: []codersdk.WorkspaceAgentScript{},
+			wantDevcontainerScripts: []codersdk.WorkspaceAgentScript{
+				{
+					ID:         devcontainerIDs[0],
+					Script:     "devcontainer up --log-format json --workspace-folder \"workspace1\" --config \"workspace1/config1\"",
+					RunOnStart: false,
+				},
+				{
+					ID:         devcontainerIDs[1],
+					Script:     "devcontainer up --log-format json --workspace-folder \"workspace2\" --config \"workspace2/config2\"",
+					RunOnStart: false,
+				},
+			},
+			skipOnWindowsDueToPathSeparator: true,
+		},
+		{
+			name: "scripts match devcontainers with expand path",
+			args: args{
+				expandPath: func(s string) (string, error) {
+					return "/home/" + s, nil
+				},
+				devcontainers: []codersdk.WorkspaceAgentDevcontainer{
+					{
+						ID:              devcontainerIDs[0],
+						WorkspaceFolder: "workspace1",
+						ConfigPath:      "config1",
+					},
+					{
+						ID:              devcontainerIDs[1],
+						WorkspaceFolder: "workspace2",
+						ConfigPath:      "config2",
+					},
+				},
+				scripts: []codersdk.WorkspaceAgentScript{
+					{ID: devcontainerIDs[0], RunOnStart: true},
+					{ID: devcontainerIDs[1], RunOnStart: true},
+				},
+			},
+			wantFilteredScripts: []codersdk.WorkspaceAgentScript{},
+			wantDevcontainerScripts: []codersdk.WorkspaceAgentScript{
+				{
+					ID:         devcontainerIDs[0],
+					Script:     "devcontainer up --log-format json --workspace-folder \"/home/workspace1\" --config \"/home/workspace1/config1\"",
+					RunOnStart: false,
+				},
+				{
+					ID:         devcontainerIDs[1],
+					Script:     "devcontainer up --log-format json --workspace-folder \"/home/workspace2\" --config \"/home/workspace2/config2\"",
+					RunOnStart: false,
+				},
+			},
+			skipOnWindowsDueToPathSeparator: true,
+		},
+		{
+			name: "expand config path when ~",
+			args: args{
+				expandPath: func(s string) (string, error) {
+					s = strings.Replace(s, "~/", "", 1)
+					if filepath.IsAbs(s) {
+						return s, nil
+					}
+					return "/home/" + s, nil
+				},
+				devcontainers: []codersdk.WorkspaceAgentDevcontainer{
+					{
+						ID:              devcontainerIDs[0],
+						WorkspaceFolder: "workspace1",
+						ConfigPath:      "~/config1",
+					},
+					{
+						ID:              devcontainerIDs[1],
+						WorkspaceFolder: "workspace2",
+						ConfigPath:      "/config2",
+					},
+				},
+				scripts: []codersdk.WorkspaceAgentScript{
+					{ID: devcontainerIDs[0], RunOnStart: true},
+					{ID: devcontainerIDs[1], RunOnStart: true},
+				},
+			},
+			wantFilteredScripts: []codersdk.WorkspaceAgentScript{},
+			wantDevcontainerScripts: []codersdk.WorkspaceAgentScript{
+				{
+					ID:         devcontainerIDs[0],
+					Script:     "devcontainer up --log-format json --workspace-folder \"/home/workspace1\" --config \"/home/config1\"",
+					RunOnStart: false,
+				},
+				{
+					ID:         devcontainerIDs[1],
+					Script:     "devcontainer up --log-format json --workspace-folder \"/home/workspace2\" --config \"/config2\"",
+					RunOnStart: false,
+				},
+			},
+			skipOnWindowsDueToPathSeparator: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+			if tt.skipOnWindowsDueToPathSeparator && filepath.Separator == '\\' {
+				t.Skip("Skipping test on Windows due to path separator difference.")
+			}
+
+			logger := slogtest.Make(t, nil)
+			if tt.args.expandPath == nil {
+				tt.args.expandPath = func(s string) (string, error) {
+					return s, nil
+				}
+			}
+			gotFilteredScripts, gotDevcontainerScripts := agentcontainers.ExtractAndInitializeDevcontainerScripts(
+				agentcontainers.ExpandAllDevcontainerPaths(logger, tt.args.expandPath, tt.args.devcontainers),
+				tt.args.scripts,
+			)
+
+			if diff := cmp.Diff(tt.wantFilteredScripts, gotFilteredScripts, cmpopts.EquateEmpty()); diff != "" {
+				t.Errorf("ExtractAndInitializeDevcontainerScripts() gotFilteredScripts mismatch (-want +got):\n%s", diff)
+			}
+
+			// Preprocess the devcontainer scripts to remove scripting part.
+			for i := range gotDevcontainerScripts {
+				gotDevcontainerScripts[i].Script = textGrep("devcontainer up", gotDevcontainerScripts[i].Script)
+				require.NotEmpty(t, gotDevcontainerScripts[i].Script, "devcontainer up script not found")
+			}
+			if diff := cmp.Diff(tt.wantDevcontainerScripts, gotDevcontainerScripts); diff != "" {
+				t.Errorf("ExtractAndInitializeDevcontainerScripts() gotDevcontainerScripts mismatch (-want +got):\n%s", diff)
+			}
+		})
+	}
+}
+
+// textGrep returns matching lines from multiline string.
+func textGrep(want, got string) (filtered string) {
+	var lines []string
+	for _, line := range strings.Split(got, "\n") {
+		if strings.Contains(line, want) {
+			lines = append(lines, line)
+		}
+	}
+	return strings.Join(lines, "\n")
+}

agent/agentcontainers/devcontainercli.go

@@ -0,0 +1,213 @@
+package agentcontainers
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"io"
+
+	"golang.org/x/xerrors"
+
+	"cdr.dev/slog"
+	"github.com/coder/coder/v2/agent/agentexec"
+)
+
+// DevcontainerCLI is an interface for the devcontainer CLI.
+type DevcontainerCLI interface {
+	Up(ctx context.Context, workspaceFolder, configPath string, opts ...DevcontainerCLIUpOptions) (id string, err error)
+}
+
+// DevcontainerCLIUpOptions are options for the devcontainer CLI up
+// command.
+type DevcontainerCLIUpOptions func(*devcontainerCLIUpConfig)
+
+// WithRemoveExistingContainer is an option to remove the existing
+// container.
+func WithRemoveExistingContainer() DevcontainerCLIUpOptions {
+	return func(o *devcontainerCLIUpConfig) {
+		o.removeExistingContainer = true
+	}
+}
+
+// WithOutput sets stdout and stderr writers for Up command logs.
+func WithOutput(stdout, stderr io.Writer) DevcontainerCLIUpOptions {
+	return func(o *devcontainerCLIUpConfig) {
+		o.stdout = stdout
+		o.stderr = stderr
+	}
+}
+
+type devcontainerCLIUpConfig struct {
+	removeExistingContainer bool
+	stdout                  io.Writer
+	stderr                  io.Writer
+}
+
+func applyDevcontainerCLIUpOptions(opts []DevcontainerCLIUpOptions) devcontainerCLIUpConfig {
+	conf := devcontainerCLIUpConfig{
+		removeExistingContainer: false,
+	}
+	for _, opt := range opts {
+		if opt != nil {
+			opt(&conf)
+		}
+	}
+	return conf
+}
+
+type devcontainerCLI struct {
+	logger slog.Logger
+	execer agentexec.Execer
+}
+
+var _ DevcontainerCLI = &devcontainerCLI{}
+
+func NewDevcontainerCLI(logger slog.Logger, execer agentexec.Execer) DevcontainerCLI {
+	return &devcontainerCLI{
+		execer: execer,
+		logger: logger,
+	}
+}
+
+func (d *devcontainerCLI) Up(ctx context.Context, workspaceFolder, configPath string, opts ...DevcontainerCLIUpOptions) (string, error) {
+	conf := applyDevcontainerCLIUpOptions(opts)
+	logger := d.logger.With(slog.F("workspace_folder", workspaceFolder), slog.F("config_path", configPath), slog.F("recreate", conf.removeExistingContainer))
+
+	args := []string{
+		"up",
+		"--log-format", "json",
+		"--workspace-folder", workspaceFolder,
+	}
+	if configPath != "" {
+		args = append(args, "--config", configPath)
+	}
+	if conf.removeExistingContainer {
+		args = append(args, "--remove-existing-container")
+	}
+	cmd := d.execer.CommandContext(ctx, "devcontainer", args...)
+
+	// Capture stdout for parsing and stream logs for both default and provided writers.
+	var stdoutBuf bytes.Buffer
+	stdoutWriters := []io.Writer{&stdoutBuf, &devcontainerCLILogWriter{ctx: ctx, logger: logger.With(slog.F("stdout", true))}}
+	if conf.stdout != nil {
+		stdoutWriters = append(stdoutWriters, conf.stdout)
+	}
+	cmd.Stdout = io.MultiWriter(stdoutWriters...)
+	// Stream stderr logs and provided writer if any.
+	stderrWriters := []io.Writer{&devcontainerCLILogWriter{ctx: ctx, logger: logger.With(slog.F("stderr", true))}}
+	if conf.stderr != nil {
+		stderrWriters = append(stderrWriters, conf.stderr)
+	}
+	cmd.Stderr = io.MultiWriter(stderrWriters...)
+
+	if err := cmd.Run(); err != nil {
+		if _, err2 := parseDevcontainerCLILastLine(ctx, logger, stdoutBuf.Bytes()); err2 != nil {
+			err = errors.Join(err, err2)
+		}
+		return "", err
+	}
+
+	result, err := parseDevcontainerCLILastLine(ctx, logger, stdoutBuf.Bytes())
+	if err != nil {
+		return "", err
+	}
+
+	return result.ContainerID, nil
+}
+
+// parseDevcontainerCLILastLine parses the last line of the devcontainer CLI output
+// which is a JSON object.
+func parseDevcontainerCLILastLine(ctx context.Context, logger slog.Logger, p []byte) (result devcontainerCLIResult, err error) {
+	s := bufio.NewScanner(bytes.NewReader(p))
+	var lastLine []byte
+	for s.Scan() {
+		b := s.Bytes()
+		if len(b) == 0 || b[0] != '{' {
+			continue
+		}
+		lastLine = b
+	}
+	if err = s.Err(); err != nil {
+		return result, err
+	}
+	if len(lastLine) == 0 || lastLine[0] != '{' {
+		logger.Error(ctx, "devcontainer result is not json", slog.F("result", string(lastLine)))
+		return result, xerrors.Errorf("devcontainer result is not json: %q", string(lastLine))
+	}
+	if err = json.Unmarshal(lastLine, &result); err != nil {
+		logger.Error(ctx, "parse devcontainer result failed", slog.Error(err), slog.F("result", string(lastLine)))
+		return result, err
+	}
+
+	return result, result.Err()
+}
+
+// devcontainerCLIResult is the result of the devcontainer CLI command.
+// It is parsed from the last line of the devcontainer CLI stdout which
+// is a JSON object.
+type devcontainerCLIResult struct {
+	Outcome string `json:"outcome"` // "error", "success".
+
+	// The following fields are set if outcome is success.
+	ContainerID           string `json:"containerId"`
+	RemoteUser            string `json:"remoteUser"`
+	RemoteWorkspaceFolder string `json:"remoteWorkspaceFolder"`
+
+	// The following fields are set if outcome is error.
+	Message     string `json:"message"`
+	Description string `json:"description"`
+}
+
+func (r devcontainerCLIResult) Err() error {
+	if r.Outcome == "success" {
+		return nil
+	}
+	return xerrors.Errorf("devcontainer up failed: %s (description: %s, message: %s)", r.Outcome, r.Description, r.Message)
+}
+
+// devcontainerCLIJSONLogLine is a log line from the devcontainer CLI.
+type devcontainerCLIJSONLogLine struct {
+	Type      string `json:"type"`      // "progress", "raw", "start", "stop", "text", etc.
+	Level     int    `json:"level"`     // 1, 2, 3.
+	Timestamp int    `json:"timestamp"` // Unix timestamp in milliseconds.
+	Text      string `json:"text"`
+
+	// More fields can be added here as needed.
+}
+
+// devcontainerCLILogWriter splits on newlines and logs each line
+// separately.
+type devcontainerCLILogWriter struct {
+	ctx    context.Context
+	logger slog.Logger
+}
+
+func (l *devcontainerCLILogWriter) Write(p []byte) (n int, err error) {
+	s := bufio.NewScanner(bytes.NewReader(p))
+	for s.Scan() {
+		line := s.Bytes()
+		if len(line) == 0 {
+			continue
+		}
+		if line[0] != '{' {
+			l.logger.Debug(l.ctx, "@devcontainer/cli", slog.F("line", string(line)))
+			continue
+		}
+		var logLine devcontainerCLIJSONLogLine
+		if err := json.Unmarshal(line, &logLine); err != nil {
+			l.logger.Error(l.ctx, "parse devcontainer json log line failed", slog.Error(err), slog.F("line", string(line)))
+			continue
+		}
+		if logLine.Level >= 3 {
+			l.logger.Info(l.ctx, "@devcontainer/cli", slog.F("line", string(line)))
+			continue
+		}
+		l.logger.Debug(l.ctx, "@devcontainer/cli", slog.F("line", string(line)))
+	}
+	if err := s.Err(); err != nil {
+		l.logger.Error(l.ctx, "devcontainer log line scan failed", slog.Error(err))
+	}
+	return len(p), nil
+}

agent/agentcontainers/devcontainercli_test.go

@@ -0,0 +1,393 @@
+package agentcontainers_test
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/ory/dockertest/v3"
+	"github.com/ory/dockertest/v3/docker"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"cdr.dev/slog"
+	"cdr.dev/slog/sloggers/slogtest"
+	"github.com/coder/coder/v2/agent/agentcontainers"
+	"github.com/coder/coder/v2/agent/agentexec"
+	"github.com/coder/coder/v2/pty"
+	"github.com/coder/coder/v2/testutil"
+)
+
+func TestDevcontainerCLI_ArgsAndParsing(t *testing.T) {
+	t.Parallel()
+
+	testExePath, err := os.Executable()
+	require.NoError(t, err, "get test executable path")
+
+	logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}).Leveled(slog.LevelDebug)
+
+	t.Run("Up", func(t *testing.T) {
+		t.Parallel()
+
+		tests := []struct {
+			name      string
+			logFile   string
+			workspace string
+			config    string
+			opts      []agentcontainers.DevcontainerCLIUpOptions
+			wantArgs  string
+			wantError bool
+		}{
+			{
+				name:      "success",
+				logFile:   "up.log",
+				workspace: "/test/workspace",
+				wantArgs:  "up --log-format json --workspace-folder /test/workspace",
+				wantError: false,
+			},
+			{
+				name:      "success with config",
+				logFile:   "up.log",
+				workspace: "/test/workspace",
+				config:    "/test/config.json",
+				wantArgs:  "up --log-format json --workspace-folder /test/workspace --config /test/config.json",
+				wantError: false,
+			},
+			{
+				name:      "already exists",
+				logFile:   "up-already-exists.log",
+				workspace: "/test/workspace",
+				wantArgs:  "up --log-format json --workspace-folder /test/workspace",
+				wantError: false,
+			},
+			{
+				name:      "docker error",
+				logFile:   "up-error-docker.log",
+				workspace: "/test/workspace",
+				wantArgs:  "up --log-format json --workspace-folder /test/workspace",
+				wantError: true,
+			},
+			{
+				name:      "bad outcome",
+				logFile:   "up-error-bad-outcome.log",
+				workspace: "/test/workspace",
+				wantArgs:  "up --log-format json --workspace-folder /test/workspace",
+				wantError: true,
+			},
+			{
+				name:      "does not exist",
+				logFile:   "up-error-does-not-exist.log",
+				workspace: "/test/workspace",
+				wantArgs:  "up --log-format json --workspace-folder /test/workspace",
+				wantError: true,
+			},
+			{
+				name:      "with remove existing container",
+				logFile:   "up.log",
+				workspace: "/test/workspace",
+				opts: []agentcontainers.DevcontainerCLIUpOptions{
+					agentcontainers.WithRemoveExistingContainer(),
+				},
+				wantArgs:  "up --log-format json --workspace-folder /test/workspace --remove-existing-container",
+				wantError: false,
+			},
+		}
+
+		for _, tt := range tests {
+			t.Run(tt.name, func(t *testing.T) {
+				t.Parallel()
+
+				ctx := testutil.Context(t, testutil.WaitMedium)
+
+				testExecer := &testDevcontainerExecer{
+					testExePath: testExePath,
+					wantArgs:    tt.wantArgs,
+					wantError:   tt.wantError,
+					logFile:     filepath.Join("testdata", "devcontainercli", "parse", tt.logFile),
+				}
+
+				dccli := agentcontainers.NewDevcontainerCLI(logger, testExecer)
+				containerID, err := dccli.Up(ctx, tt.workspace, tt.config, tt.opts...)
+				if tt.wantError {
+					assert.Error(t, err, "want error")
+					assert.Empty(t, containerID, "expected empty container ID")
+				} else {
+					assert.NoError(t, err, "want no error")
+					assert.NotEmpty(t, containerID, "expected non-empty container ID")
+				}
+			})
+		}
+	})
+}
+
+// TestDevcontainerCLI_WithOutput tests that WithOutput captures CLI
+// logs to provided writers.
+func TestDevcontainerCLI_WithOutput(t *testing.T) {
+	t.Parallel()
+
+	// Prepare test executable and logger.
+	testExePath, err := os.Executable()
+	require.NoError(t, err, "get test executable path")
+	logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}).Leveled(slog.LevelDebug)
+	ctx := testutil.Context(t, testutil.WaitMedium)
+
+	// Buffers to capture stdout and stderr.
+	outBuf := &bytes.Buffer{}
+	errBuf := &bytes.Buffer{}
+
+	// Simulate CLI execution with a standard up.log file.
+	wantArgs := "up --log-format json --workspace-folder /test/workspace"
+	testExecer := &testDevcontainerExecer{
+		testExePath: testExePath,
+		wantArgs:    wantArgs,
+		wantError:   false,
+		logFile:     filepath.Join("testdata", "devcontainercli", "parse", "up.log"),
+	}
+	dccli := agentcontainers.NewDevcontainerCLI(logger, testExecer)
+
+	// Call Up with WithOutput to capture CLI logs.
+	containerID, err := dccli.Up(ctx, "/test/workspace", "", agentcontainers.WithOutput(outBuf, errBuf))
+	require.NoError(t, err, "Up should succeed")
+	require.NotEmpty(t, containerID, "expected non-empty container ID")
+
+	// Read expected log content.
+	expLog, err := os.ReadFile(filepath.Join("testdata", "devcontainercli", "parse", "up.log"))
+	require.NoError(t, err, "reading expected log file")
+
+	// Verify stdout buffer contains the CLI logs and stderr is empty.
+	assert.Equal(t, string(expLog), outBuf.String(), "stdout buffer should match CLI logs")
+	assert.Empty(t, errBuf.String(), "stderr buffer should be empty on success")
+}
+
+// testDevcontainerExecer implements the agentexec.Execer interface for testing.
+type testDevcontainerExecer struct {
+	testExePath string
+	wantArgs    string
+	wantError   bool
+	logFile     string
+}
+
+// CommandContext returns a test binary command that simulates devcontainer responses.
+func (e *testDevcontainerExecer) CommandContext(ctx context.Context, name string, args ...string) *exec.Cmd {
+	// Only handle "devcontainer" commands.
+	if name != "devcontainer" {
+		// For non-devcontainer commands, use a standard execer.
+		return agentexec.DefaultExecer.CommandContext(ctx, name, args...)
+	}
+
+	// Create a command that runs the test binary with special flags
+	// that tell it to simulate a devcontainer command.
+	testArgs := []string{
+		"-test.run=TestDevcontainerHelperProcess",
+		"--",
+		name,
+	}
+	testArgs = append(testArgs, args...)
+
+	//nolint:gosec // This is a test binary, so we don't need to worry about command injection.
+	cmd := exec.CommandContext(ctx, e.testExePath, testArgs...)
+	// Set this environment variable so the child process knows it's the helper.
+	cmd.Env = append(os.Environ(),
+		"TEST_DEVCONTAINER_WANT_HELPER_PROCESS=1",
+		"TEST_DEVCONTAINER_WANT_ARGS="+e.wantArgs,
+		"TEST_DEVCONTAINER_WANT_ERROR="+fmt.Sprintf("%v", e.wantError),
+		"TEST_DEVCONTAINER_LOG_FILE="+e.logFile,
+	)
+
+	return cmd
+}
+
+// PTYCommandContext returns a PTY command.
+func (*testDevcontainerExecer) PTYCommandContext(_ context.Context, name string, args ...string) *pty.Cmd {
+	// This method shouldn't be called for our devcontainer tests.
+	panic("PTYCommandContext not expected in devcontainer tests")
+}
+
+// This is a special test helper that is executed as a subprocess.
+// It simulates the behavior of the devcontainer CLI.
+//
+//nolint:revive,paralleltest // This is a test helper function.
+func TestDevcontainerHelperProcess(t *testing.T) {
+	// If not called by the test as a helper process, do nothing.
+	if os.Getenv("TEST_DEVCONTAINER_WANT_HELPER_PROCESS") != "1" {
+		return
+	}
+
+	helperArgs := flag.Args()
+	if len(helperArgs) < 1 {
+		fmt.Fprintf(os.Stderr, "No command\n")
+		os.Exit(2)
+	}
+
+	if helperArgs[0] != "devcontainer" {
+		fmt.Fprintf(os.Stderr, "Unknown command: %s\n", helperArgs[0])
+		os.Exit(2)
+	}
+
+	// Verify arguments against expected arguments and skip
+	// "devcontainer", it's not included in the input args.
+	wantArgs := os.Getenv("TEST_DEVCONTAINER_WANT_ARGS")
+	gotArgs := strings.Join(helperArgs[1:], " ")
+	if gotArgs != wantArgs {
+		fmt.Fprintf(os.Stderr, "Arguments don't match.\nWant: %q\nGot:  %q\n",
+			wantArgs, gotArgs)
+		os.Exit(2)
+	}
+
+	logFilePath := os.Getenv("TEST_DEVCONTAINER_LOG_FILE")
+	output, err := os.ReadFile(logFilePath)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Reading log file %s failed: %v\n", logFilePath, err)
+		os.Exit(2)
+	}
+
+	_, _ = io.Copy(os.Stdout, bytes.NewReader(output))
+	if os.Getenv("TEST_DEVCONTAINER_WANT_ERROR") == "true" {
+		os.Exit(1)
+	}
+	os.Exit(0)
+}
+
+// TestDockerDevcontainerCLI tests the DevcontainerCLI component with real Docker containers.
+// This test verifies that containers can be created and recreated using the actual
+// devcontainer CLI and Docker. It is skipped by default and can be run with:
+//
+//	CODER_TEST_USE_DOCKER=1 go test ./agent/agentcontainers -run TestDockerDevcontainerCLI
+//
+// The test requires Docker to be installed and running.
+func TestDockerDevcontainerCLI(t *testing.T) {
+	t.Parallel()
+	if os.Getenv("CODER_TEST_USE_DOCKER") != "1" {
+		t.Skip("skipping Docker test; set CODER_TEST_USE_DOCKER=1 to run")
+	}
+	if _, err := exec.LookPath("devcontainer"); err != nil {
+		t.Fatal("this test requires the devcontainer CLI: npm install -g @devcontainers/cli")
+	}
+
+	// Connect to Docker.
+	pool, err := dockertest.NewPool("")
+	require.NoError(t, err, "connect to Docker")
+
+	t.Run("ContainerLifecycle", func(t *testing.T) {
+		t.Parallel()
+
+		// Set up workspace directory with a devcontainer configuration.
+		workspaceFolder := t.TempDir()
+		configPath := setupDevcontainerWorkspace(t, workspaceFolder)
+
+		// Use a long timeout because container operations are slow.
+		ctx := testutil.Context(t, testutil.WaitLong)
+		logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}).Leveled(slog.LevelDebug)
+
+		// Create the devcontainer CLI under test.
+		dccli := agentcontainers.NewDevcontainerCLI(logger, agentexec.DefaultExecer)
+
+		// Create a container.
+		firstID, err := dccli.Up(ctx, workspaceFolder, configPath)
+		require.NoError(t, err, "create container")
+		require.NotEmpty(t, firstID, "container ID should not be empty")
+		defer removeDevcontainerByID(t, pool, firstID)
+
+		// Verify container exists.
+		firstContainer, found := findDevcontainerByID(t, pool, firstID)
+		require.True(t, found, "container should exist")
+
+		// Remember the container creation time.
+		firstCreated := firstContainer.Created
+
+		// Recreate the container.
+		secondID, err := dccli.Up(ctx, workspaceFolder, configPath, agentcontainers.WithRemoveExistingContainer())
+		require.NoError(t, err, "recreate container")
+		require.NotEmpty(t, secondID, "recreated container ID should not be empty")
+		defer removeDevcontainerByID(t, pool, secondID)
+
+		// Verify the new container exists and is different.
+		secondContainer, found := findDevcontainerByID(t, pool, secondID)
+		require.True(t, found, "recreated container should exist")
+
+		// Verify it's a different container by checking creation time.
+		secondCreated := secondContainer.Created
+		assert.NotEqual(t, firstCreated, secondCreated, "recreated container should have different creation time")
+
+		// Verify the first container is removed by the recreation.
+		_, found = findDevcontainerByID(t, pool, firstID)
+		assert.False(t, found, "first container should be removed")
+	})
+}
+
+// setupDevcontainerWorkspace prepares a test environment with a minimal
+// devcontainer.json configuration and returns the path to the config file.
+func setupDevcontainerWorkspace(t *testing.T, workspaceFolder string) string {
+	t.Helper()
+
+	// Create the devcontainer directory structure.
+	devcontainerDir := filepath.Join(workspaceFolder, ".devcontainer")
+	err := os.MkdirAll(devcontainerDir, 0o755)
+	require.NoError(t, err, "create .devcontainer directory")
+
+	// Write a minimal configuration with test labels for identification.
+	configPath := filepath.Join(devcontainerDir, "devcontainer.json")
+	content := `{
+	"image": "alpine:latest",
+	"containerEnv": {
+		"TEST_CONTAINER": "true"
+	},
+	"runArgs": ["--label", "com.coder.test=devcontainercli"]
+}`
+	err = os.WriteFile(configPath, []byte(content), 0o600)
+	require.NoError(t, err, "create devcontainer.json file")
+
+	return configPath
+}
+
+// findDevcontainerByID locates a container by its ID and verifies it has our
+// test label. Returns the container and whether it was found.
+func findDevcontainerByID(t *testing.T, pool *dockertest.Pool, id string) (*docker.Container, bool) {
+	t.Helper()
+
+	container, err := pool.Client.InspectContainer(id)
+	if err != nil {
+		t.Logf("Inspect container failed: %v", err)
+		return nil, false
+	}
+	require.Equal(t, "devcontainercli", container.Config.Labels["com.coder.test"], "sanity check failed: container should have the test label")
+
+	return container, true
+}
+
+// removeDevcontainerByID safely cleans up a test container by ID, verifying
+// it has our test label before removal to prevent accidental deletion.
+func removeDevcontainerByID(t *testing.T, pool *dockertest.Pool, id string) {
+	t.Helper()
+
+	errNoSuchContainer := &docker.NoSuchContainer{}
+
+	// Check if the container has the expected label.
+	container, err := pool.Client.InspectContainer(id)
+	if err != nil {
+		if errors.As(err, &errNoSuchContainer) {
+			t.Logf("Container %s not found, skipping removal", id)
+			return
+		}
+		require.NoError(t, err, "inspect container")
+	}
+	require.Equal(t, "devcontainercli", container.Config.Labels["com.coder.test"], "sanity check failed: container should have the test label")
+
+	t.Logf("Removing container with ID: %s", id)
+	err = pool.Client.RemoveContainer(docker.RemoveContainerOptions{
+		ID:            container.ID,
+		Force:         true,
+		RemoveVolumes: true,
+	})
+	if err != nil && !errors.As(err, &errNoSuchContainer) {
+		assert.NoError(t, err, "remove container failed")
+	}
+}

agent/agentcontainers/testdata/container_binds/docker_inspect.json

@@ -0,0 +1,221 @@
+[
+    {
+        "Id": "fdc75ebefdc0243c0fce959e7685931691ac7aede278664a0e2c23af8a1e8d6a",
+        "Created": "2025-03-11T17:58:43.522505027Z",
+        "Path": "sleep",
+        "Args": [
+            "infinity"
+        ],
+        "State": {
+            "Status": "running",
+            "Running": true,
+            "Paused": false,
+            "Restarting": false,
+            "OOMKilled": false,
+            "Dead": false,
+            "Pid": 644296,
+            "ExitCode": 0,
+            "Error": "",
+            "StartedAt": "2025-03-11T17:58:43.569966691Z",
+            "FinishedAt": "0001-01-01T00:00:00Z"
+        },
+        "Image": "sha256:d4ccddb816ba27eaae22ef3d56175d53f47998e2acb99df1ae0e5b426b28a076",
+        "ResolvConfPath": "/var/lib/docker/containers/fdc75ebefdc0243c0fce959e7685931691ac7aede278664a0e2c23af8a1e8d6a/resolv.conf",
+        "HostnamePath": "/var/lib/docker/containers/fdc75ebefdc0243c0fce959e7685931691ac7aede278664a0e2c23af8a1e8d6a/hostname",
+        "HostsPath": "/var/lib/docker/containers/fdc75ebefdc0243c0fce959e7685931691ac7aede278664a0e2c23af8a1e8d6a/hosts",
+        "LogPath": "/var/lib/docker/containers/fdc75ebefdc0243c0fce959e7685931691ac7aede278664a0e2c23af8a1e8d6a/fdc75ebefdc0243c0fce959e7685931691ac7aede278664a0e2c23af8a1e8d6a-json.log",
+        "Name": "/silly_beaver",
+        "RestartCount": 0,
+        "Driver": "overlay2",
+        "Platform": "linux",
+        "MountLabel": "",
+        "ProcessLabel": "",
+        "AppArmorProfile": "",
+        "ExecIDs": null,
+        "HostConfig": {
+            "Binds": [
+                "/tmp/test/a:/var/coder/a:ro",
+                "/tmp/test/b:/var/coder/b"
+            ],
+            "ContainerIDFile": "",
+            "LogConfig": {
+                "Type": "json-file",
+                "Config": {}
+            },
+            "NetworkMode": "bridge",
+            "PortBindings": {},
+            "RestartPolicy": {
+                "Name": "no",
+                "MaximumRetryCount": 0
+            },
+            "AutoRemove": false,
+            "VolumeDriver": "",
+            "VolumesFrom": null,
+            "ConsoleSize": [
+                108,
+                176
+            ],
+            "CapAdd": null,
+            "CapDrop": null,
+            "CgroupnsMode": "private",
+            "Dns": [],
+            "DnsOptions": [],
+            "DnsSearch": [],
+            "ExtraHosts": null,
+            "GroupAdd": null,
+            "IpcMode": "private",
+            "Cgroup": "",
+            "Links": null,
+            "OomScoreAdj": 10,
+            "PidMode": "",
+            "Privileged": false,
+            "PublishAllPorts": false,
+            "ReadonlyRootfs": false,
+            "SecurityOpt": null,
+            "UTSMode": "",
+            "UsernsMode": "",
+            "ShmSize": 67108864,
+            "Runtime": "runc",
+            "Isolation": "",
+            "CpuShares": 0,
+            "Memory": 0,
+            "NanoCpus": 0,
+            "CgroupParent": "",
+            "BlkioWeight": 0,
+            "BlkioWeightDevice": [],
+            "BlkioDeviceReadBps": [],
+            "BlkioDeviceWriteBps": [],
+            "BlkioDeviceReadIOps": [],
+            "BlkioDeviceWriteIOps": [],
+            "CpuPeriod": 0,
+            "CpuQuota": 0,
+            "CpuRealtimePeriod": 0,
+            "CpuRealtimeRuntime": 0,
+            "CpusetCpus": "",
+            "CpusetMems": "",
+            "Devices": [],
+            "DeviceCgroupRules": null,
+            "DeviceRequests": null,
+            "MemoryReservation": 0,
+            "MemorySwap": 0,
+            "MemorySwappiness": null,
+            "OomKillDisable": null,
+            "PidsLimit": null,
+            "Ulimits": [],
+            "CpuCount": 0,
+            "CpuPercent": 0,
+            "IOMaximumIOps": 0,
+            "IOMaximumBandwidth": 0,
+            "MaskedPaths": [
+                "/proc/asound",
+                "/proc/acpi",
+                "/proc/kcore",
+                "/proc/keys",
+                "/proc/latency_stats",
+                "/proc/timer_list",
+                "/proc/timer_stats",
+                "/proc/sched_debug",
+                "/proc/scsi",
+                "/sys/firmware",
+                "/sys/devices/virtual/powercap"
+            ],
+            "ReadonlyPaths": [
+                "/proc/bus",
+                "/proc/fs",
+                "/proc/irq",
+                "/proc/sys",
+                "/proc/sysrq-trigger"
+            ]
+        },
+        "GraphDriver": {
+            "Data": {
+                "ID": "fdc75ebefdc0243c0fce959e7685931691ac7aede278664a0e2c23af8a1e8d6a",
+                "LowerDir": "/var/lib/docker/overlay2/c1519be93f8e138757310f6ed8c3946a524cdae2580ad8579913d19c3fe9ffd2-init/diff:/var/lib/docker/overlay2/4b4c37dfbdc0dc01b68d4fb1ddb86109398a2d73555439b874dbd23b87cd5c4b/diff",
+                "MergedDir": "/var/lib/docker/overlay2/c1519be93f8e138757310f6ed8c3946a524cdae2580ad8579913d19c3fe9ffd2/merged",
+                "UpperDir": "/var/lib/docker/overlay2/c1519be93f8e138757310f6ed8c3946a524cdae2580ad8579913d19c3fe9ffd2/diff",
+                "WorkDir": "/var/lib/docker/overlay2/c1519be93f8e138757310f6ed8c3946a524cdae2580ad8579913d19c3fe9ffd2/work"
+            },
+            "Name": "overlay2"
+        },
+        "Mounts": [
+            {
+                "Type": "bind",
+                "Source": "/tmp/test/a",
+                "Destination": "/var/coder/a",
+                "Mode": "ro",
+                "RW": false,
+                "Propagation": "rprivate"
+            },
+            {
+                "Type": "bind",
+                "Source": "/tmp/test/b",
+                "Destination": "/var/coder/b",
+                "Mode": "",
+                "RW": true,
+                "Propagation": "rprivate"
+            }
+        ],
+        "Config": {
+            "Hostname": "fdc75ebefdc0",
+            "Domainname": "",
+            "User": "",
+            "AttachStdin": false,
+            "AttachStdout": false,
+            "AttachStderr": false,
+            "Tty": false,
+            "OpenStdin": false,
+            "StdinOnce": false,
+            "Env": [
+                "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+            ],
+            "Cmd": [
+                "sleep",
+                "infinity"
+            ],
+            "Image": "debian:bookworm",
+            "Volumes": null,
+            "WorkingDir": "",
+            "Entrypoint": [],
+            "OnBuild": null,
+            "Labels": {}
+        },
+        "NetworkSettings": {
+            "Bridge": "",
+            "SandboxID": "46f98b32002740b63709e3ebf87c78efe652adfaa8753b85d79b814f26d88107",
+            "SandboxKey": "/var/run/docker/netns/46f98b320027",
+            "Ports": {},
+            "HairpinMode": false,
+            "LinkLocalIPv6Address": "",
+            "LinkLocalIPv6PrefixLen": 0,
+            "SecondaryIPAddresses": null,
+            "SecondaryIPv6Addresses": null,
+            "EndpointID": "356e429f15e354dd23250c7a3516aecf1a2afe9d58ea1dc2e97e33a75ac346a8",
+            "Gateway": "172.17.0.1",
+            "GlobalIPv6Address": "",
+            "GlobalIPv6PrefixLen": 0,
+            "IPAddress": "172.17.0.2",
+            "IPPrefixLen": 16,
+            "IPv6Gateway": "",
+            "MacAddress": "22:2c:26:d9:da:83",
+            "Networks": {
+                "bridge": {
+                    "IPAMConfig": null,
+                    "Links": null,
+                    "Aliases": null,
+                    "MacAddress": "22:2c:26:d9:da:83",
+                    "DriverOpts": null,
+                    "GwPriority": 0,
+                    "NetworkID": "c4dd768ab4945e41ad23fe3907c960dac811141592a861cc40038df7086a1ce1",
+                    "EndpointID": "356e429f15e354dd23250c7a3516aecf1a2afe9d58ea1dc2e97e33a75ac346a8",
+                    "Gateway": "172.17.0.1",
+                    "IPAddress": "172.17.0.2",
+                    "IPPrefixLen": 16,
+                    "IPv6Gateway": "",
+                    "GlobalIPv6Address": "",
+                    "GlobalIPv6PrefixLen": 0,
+                    "DNSNames": null
+                }
+            }
+        }
+    }
+]

agent/agentcontainers/testdata/container_differentport/docker_inspect.json

@@ -0,0 +1,222 @@
+[
+    {
+        "Id": "3090de8b72b1224758a94a11b827c82ba2b09c45524f1263dc4a2d83e19625ea",
+        "Created": "2025-03-11T17:57:08.862545133Z",
+        "Path": "sleep",
+        "Args": [
+            "infinity"
+        ],
+        "State": {
+            "Status": "running",
+            "Running": true,
+            "Paused": false,
+            "Restarting": false,
+            "OOMKilled": false,
+            "Dead": false,
+            "Pid": 640137,
+            "ExitCode": 0,
+            "Error": "",
+            "StartedAt": "2025-03-11T17:57:08.909898821Z",
+            "FinishedAt": "0001-01-01T00:00:00Z"
+        },
+        "Image": "sha256:d4ccddb816ba27eaae22ef3d56175d53f47998e2acb99df1ae0e5b426b28a076",
+        "ResolvConfPath": "/var/lib/docker/containers/3090de8b72b1224758a94a11b827c82ba2b09c45524f1263dc4a2d83e19625ea/resolv.conf",
+        "HostnamePath": "/var/lib/docker/containers/3090de8b72b1224758a94a11b827c82ba2b09c45524f1263dc4a2d83e19625ea/hostname",
+        "HostsPath": "/var/lib/docker/containers/3090de8b72b1224758a94a11b827c82ba2b09c45524f1263dc4a2d83e19625ea/hosts",
+        "LogPath": "/var/lib/docker/containers/3090de8b72b1224758a94a11b827c82ba2b09c45524f1263dc4a2d83e19625ea/3090de8b72b1224758a94a11b827c82ba2b09c45524f1263dc4a2d83e19625ea-json.log",
+        "Name": "/boring_ellis",
+        "RestartCount": 0,
+        "Driver": "overlay2",
+        "Platform": "linux",
+        "MountLabel": "",
+        "ProcessLabel": "",
+        "AppArmorProfile": "",
+        "ExecIDs": null,
+        "HostConfig": {
+            "Binds": null,
+            "ContainerIDFile": "",
+            "LogConfig": {
+                "Type": "json-file",
+                "Config": {}
+            },
+            "NetworkMode": "bridge",
+            "PortBindings": {
+                "23456/tcp": [
+                    {
+                        "HostIp": "",
+                        "HostPort": "12345"
+                    }
+                ]
+            },
+            "RestartPolicy": {
+                "Name": "no",
+                "MaximumRetryCount": 0
+            },
+            "AutoRemove": false,
+            "VolumeDriver": "",
+            "VolumesFrom": null,
+            "ConsoleSize": [
+                108,
+                176
+            ],
+            "CapAdd": null,
+            "CapDrop": null,
+            "CgroupnsMode": "private",
+            "Dns": [],
+            "DnsOptions": [],
+            "DnsSearch": [],
+            "ExtraHosts": null,
+            "GroupAdd": null,
+            "IpcMode": "private",
+            "Cgroup": "",
+            "Links": null,
+            "OomScoreAdj": 10,
+            "PidMode": "",
+            "Privileged": false,
+            "PublishAllPorts": false,
+            "ReadonlyRootfs": false,
+            "SecurityOpt": null,
+            "UTSMode": "",
+            "UsernsMode": "",
+            "ShmSize": 67108864,
+            "Runtime": "runc",
+            "Isolation": "",
+            "CpuShares": 0,
+            "Memory": 0,
+            "NanoCpus": 0,
+            "CgroupParent": "",
+            "BlkioWeight": 0,
+            "BlkioWeightDevice": [],
+            "BlkioDeviceReadBps": [],
+            "BlkioDeviceWriteBps": [],
+            "BlkioDeviceReadIOps": [],
+            "BlkioDeviceWriteIOps": [],
+            "CpuPeriod": 0,
+            "CpuQuota": 0,
+            "CpuRealtimePeriod": 0,
+            "CpuRealtimeRuntime": 0,
+            "CpusetCpus": "",
+            "CpusetMems": "",
+            "Devices": [],
+            "DeviceCgroupRules": null,
+            "DeviceRequests": null,
+            "MemoryReservation": 0,
+            "MemorySwap": 0,
+            "MemorySwappiness": null,
+            "OomKillDisable": null,
+            "PidsLimit": null,
+            "Ulimits": [],
+            "CpuCount": 0,
+            "CpuPercent": 0,
+            "IOMaximumIOps": 0,
+            "IOMaximumBandwidth": 0,
+            "MaskedPaths": [
+                "/proc/asound",
+                "/proc/acpi",
+                "/proc/kcore",
+                "/proc/keys",
+                "/proc/latency_stats",
+                "/proc/timer_list",
+                "/proc/timer_stats",
+                "/proc/sched_debug",
+                "/proc/scsi",
+                "/sys/firmware",
+                "/sys/devices/virtual/powercap"
+            ],
+            "ReadonlyPaths": [
+                "/proc/bus",
+                "/proc/fs",
+                "/proc/irq",
+                "/proc/sys",
+                "/proc/sysrq-trigger"
+            ]
+        },
+        "GraphDriver": {
+            "Data": {
+                "ID": "3090de8b72b1224758a94a11b827c82ba2b09c45524f1263dc4a2d83e19625ea",
+                "LowerDir": "/var/lib/docker/overlay2/e9f2dda207bde1f4b277f973457107d62cff259881901adcd9bcccfea9a92231-init/diff:/var/lib/docker/overlay2/4b4c37dfbdc0dc01b68d4fb1ddb86109398a2d73555439b874dbd23b87cd5c4b/diff",
+                "MergedDir": "/var/lib/docker/overlay2/e9f2dda207bde1f4b277f973457107d62cff259881901adcd9bcccfea9a92231/merged",
+                "UpperDir": "/var/lib/docker/overlay2/e9f2dda207bde1f4b277f973457107d62cff259881901adcd9bcccfea9a92231/diff",
+                "WorkDir": "/var/lib/docker/overlay2/e9f2dda207bde1f4b277f973457107d62cff259881901adcd9bcccfea9a92231/work"
+            },
+            "Name": "overlay2"
+        },
+        "Mounts": [],
+        "Config": {
+            "Hostname": "3090de8b72b1",
+            "Domainname": "",
+            "User": "",
+            "AttachStdin": false,
+            "AttachStdout": false,
+            "AttachStderr": false,
+            "ExposedPorts": {
+                "23456/tcp": {}
+            },
+            "Tty": false,
+            "OpenStdin": false,
+            "StdinOnce": false,
+            "Env": [
+                "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+            ],
+            "Cmd": [
+                "sleep",
+                "infinity"
+            ],
+            "Image": "debian:bookworm",
+            "Volumes": null,
+            "WorkingDir": "",
+            "Entrypoint": [],
+            "OnBuild": null,
+            "Labels": {}
+        },
+        "NetworkSettings": {
+            "Bridge": "",
+            "SandboxID": "ebcd8b749b4c719f90d80605c352b7aa508e4c61d9dcd2919654f18f17eb2840",
+            "SandboxKey": "/var/run/docker/netns/ebcd8b749b4c",
+            "Ports": {
+                "23456/tcp": [
+                    {
+                        "HostIp": "0.0.0.0",
+                        "HostPort": "12345"
+                    },
+                    {
+                        "HostIp": "::",
+                        "HostPort": "12345"
+                    }
+                ]
+            },
+            "HairpinMode": false,
+            "LinkLocalIPv6Address": "",
+            "LinkLocalIPv6PrefixLen": 0,
+            "SecondaryIPAddresses": null,
+            "SecondaryIPv6Addresses": null,
+            "EndpointID": "465824b3cc6bdd2b307e9c614815fd458b1baac113dee889c3620f0cac3183fa",
+            "Gateway": "172.17.0.1",
+            "GlobalIPv6Address": "",
+            "GlobalIPv6PrefixLen": 0,
+            "IPAddress": "172.17.0.2",
+            "IPPrefixLen": 16,
+            "IPv6Gateway": "",
+            "MacAddress": "52:b6:f6:7b:4b:5b",
+            "Networks": {
+                "bridge": {
+                    "IPAMConfig": null,
+                    "Links": null,
+                    "Aliases": null,
+                    "MacAddress": "52:b6:f6:7b:4b:5b",
+                    "DriverOpts": null,
+                    "GwPriority": 0,
+                    "NetworkID": "c4dd768ab4945e41ad23fe3907c960dac811141592a861cc40038df7086a1ce1",
+                    "EndpointID": "465824b3cc6bdd2b307e9c614815fd458b1baac113dee889c3620f0cac3183fa",
+                    "Gateway": "172.17.0.1",
+                    "IPAddress": "172.17.0.2",
+                    "IPPrefixLen": 16,
+                    "IPv6Gateway": "",
+                    "GlobalIPv6Address": "",
+                    "GlobalIPv6PrefixLen": 0,
+                    "DNSNames": null
+                }
+            }
+        }
+    }
+]

agent/agentcontainers/testdata/container_labels/docker_inspect.json

@@ -0,0 +1,204 @@
+[
+    {
+        "Id": "bd8818e670230fc6f36145b21cf8d6d35580355662aa4d9fe5ae1b188a4c905f",
+        "Created": "2025-03-11T20:03:28.071706536Z",
+        "Path": "sleep",
+        "Args": [
+            "infinity"
+        ],
+        "State": {
+            "Status": "running",
+            "Running": true,
+            "Paused": false,
+            "Restarting": false,
+            "OOMKilled": false,
+            "Dead": false,
+            "Pid": 913862,
+            "ExitCode": 0,
+            "Error": "",
+            "StartedAt": "2025-03-11T20:03:28.123599065Z",
+            "FinishedAt": "0001-01-01T00:00:00Z"
+        },
+        "Image": "sha256:d4ccddb816ba27eaae22ef3d56175d53f47998e2acb99df1ae0e5b426b28a076",
+        "ResolvConfPath": "/var/lib/docker/containers/bd8818e670230fc6f36145b21cf8d6d35580355662aa4d9fe5ae1b188a4c905f/resolv.conf",
+        "HostnamePath": "/var/lib/docker/containers/bd8818e670230fc6f36145b21cf8d6d35580355662aa4d9fe5ae1b188a4c905f/hostname",
+        "HostsPath": "/var/lib/docker/containers/bd8818e670230fc6f36145b21cf8d6d35580355662aa4d9fe5ae1b188a4c905f/hosts",
+        "LogPath": "/var/lib/docker/containers/bd8818e670230fc6f36145b21cf8d6d35580355662aa4d9fe5ae1b188a4c905f/bd8818e670230fc6f36145b21cf8d6d35580355662aa4d9fe5ae1b188a4c905f-json.log",
+        "Name": "/fervent_bardeen",
+        "RestartCount": 0,
+        "Driver": "overlay2",
+        "Platform": "linux",
+        "MountLabel": "",
+        "ProcessLabel": "",
+        "AppArmorProfile": "",
+        "ExecIDs": null,
+        "HostConfig": {
+            "Binds": null,
+            "ContainerIDFile": "",
+            "LogConfig": {
+                "Type": "json-file",
+                "Config": {}
+            },
+            "NetworkMode": "bridge",
+            "PortBindings": {},
+            "RestartPolicy": {
+                "Name": "no",
+                "MaximumRetryCount": 0
+            },
+            "AutoRemove": false,
+            "VolumeDriver": "",
+            "VolumesFrom": null,
+            "ConsoleSize": [
+                108,
+                176
+            ],
+            "CapAdd": null,
+            "CapDrop": null,
+            "CgroupnsMode": "private",
+            "Dns": [],
+            "DnsOptions": [],
+            "DnsSearch": [],
+            "ExtraHosts": null,
+            "GroupAdd": null,
+            "IpcMode": "private",
+            "Cgroup": "",
+            "Links": null,
+            "OomScoreAdj": 10,
+            "PidMode": "",
+            "Privileged": false,
+            "PublishAllPorts": false,
+            "ReadonlyRootfs": false,
+            "SecurityOpt": null,
+            "UTSMode": "",
+            "UsernsMode": "",
+            "ShmSize": 67108864,
+            "Runtime": "runc",
+            "Isolation": "",
+            "CpuShares": 0,
+            "Memory": 0,
+            "NanoCpus": 0,
+            "CgroupParent": "",
+            "BlkioWeight": 0,
+            "BlkioWeightDevice": [],
+            "BlkioDeviceReadBps": [],
+            "BlkioDeviceWriteBps": [],
+            "BlkioDeviceReadIOps": [],
+            "BlkioDeviceWriteIOps": [],
+            "CpuPeriod": 0,
+            "CpuQuota": 0,
+            "CpuRealtimePeriod": 0,
+            "CpuRealtimeRuntime": 0,
+            "CpusetCpus": "",
+            "CpusetMems": "",
+            "Devices": [],
+            "DeviceCgroupRules": null,
+            "DeviceRequests": null,
+            "MemoryReservation": 0,
+            "MemorySwap": 0,
+            "MemorySwappiness": null,
+            "OomKillDisable": null,
+            "PidsLimit": null,
+            "Ulimits": [],
+            "CpuCount": 0,
+            "CpuPercent": 0,
+            "IOMaximumIOps": 0,
+            "IOMaximumBandwidth": 0,
+            "MaskedPaths": [
+                "/proc/asound",
+                "/proc/acpi",
+                "/proc/kcore",
+                "/proc/keys",
+                "/proc/latency_stats",
+                "/proc/timer_list",
+                "/proc/timer_stats",
+                "/proc/sched_debug",
+                "/proc/scsi",
+                "/sys/firmware",
+                "/sys/devices/virtual/powercap"
+            ],
+            "ReadonlyPaths": [
+                "/proc/bus",
+                "/proc/fs",
+                "/proc/irq",
+                "/proc/sys",
+                "/proc/sysrq-trigger"
+            ]
+        },
+        "GraphDriver": {
+            "Data": {
+                "ID": "bd8818e670230fc6f36145b21cf8d6d35580355662aa4d9fe5ae1b188a4c905f",
+                "LowerDir": "/var/lib/docker/overlay2/55fc45976c381040c7d261c198333e6331889c51afe1500e2e7837c189a1b794-init/diff:/var/lib/docker/overlay2/4b4c37dfbdc0dc01b68d4fb1ddb86109398a2d73555439b874dbd23b87cd5c4b/diff",
+                "MergedDir": "/var/lib/docker/overlay2/55fc45976c381040c7d261c198333e6331889c51afe1500e2e7837c189a1b794/merged",
+                "UpperDir": "/var/lib/docker/overlay2/55fc45976c381040c7d261c198333e6331889c51afe1500e2e7837c189a1b794/diff",
+                "WorkDir": "/var/lib/docker/overlay2/55fc45976c381040c7d261c198333e6331889c51afe1500e2e7837c189a1b794/work"
+            },
+            "Name": "overlay2"
+        },
+        "Mounts": [],
+        "Config": {
+            "Hostname": "bd8818e67023",
+            "Domainname": "",
+            "User": "",
+            "AttachStdin": false,
+            "AttachStdout": false,
+            "AttachStderr": false,
+            "Tty": false,
+            "OpenStdin": false,
+            "StdinOnce": false,
+            "Env": [
+                "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+            ],
+            "Cmd": [
+                "sleep",
+                "infinity"
+            ],
+            "Image": "debian:bookworm",
+            "Volumes": null,
+            "WorkingDir": "",
+            "Entrypoint": [],
+            "OnBuild": null,
+            "Labels": {
+                "baz": "zap",
+                "foo": "bar"
+            }
+        },
+        "NetworkSettings": {
+            "Bridge": "",
+            "SandboxID": "24faa8b9aaa58c651deca0d85a3f7bcc6c3e5e1a24b6369211f736d6e82f8ab0",
+            "SandboxKey": "/var/run/docker/netns/24faa8b9aaa5",
+            "Ports": {},
+            "HairpinMode": false,
+            "LinkLocalIPv6Address": "",
+            "LinkLocalIPv6PrefixLen": 0,
+            "SecondaryIPAddresses": null,
+            "SecondaryIPv6Addresses": null,
+            "EndpointID": "c686f97d772d75c8ceed9285e06c1f671b71d4775d5513f93f26358c0f0b4671",
+            "Gateway": "172.17.0.1",
+            "GlobalIPv6Address": "",
+            "GlobalIPv6PrefixLen": 0,
+            "IPAddress": "172.17.0.2",
+            "IPPrefixLen": 16,
+            "IPv6Gateway": "",
+            "MacAddress": "96:88:4e:3b:11:44",
+            "Networks": {
+                "bridge": {
+                    "IPAMConfig": null,
+                    "Links": null,
+                    "Aliases": null,
+                    "MacAddress": "96:88:4e:3b:11:44",
+                    "DriverOpts": null,
+                    "GwPriority": 0,
+                    "NetworkID": "c4dd768ab4945e41ad23fe3907c960dac811141592a861cc40038df7086a1ce1",
+                    "EndpointID": "c686f97d772d75c8ceed9285e06c1f671b71d4775d5513f93f26358c0f0b4671",
+                    "Gateway": "172.17.0.1",
+                    "IPAddress": "172.17.0.2",
+                    "IPPrefixLen": 16,
+                    "IPv6Gateway": "",
+                    "GlobalIPv6Address": "",
+                    "GlobalIPv6PrefixLen": 0,
+                    "DNSNames": null
+                }
+            }
+        }
+    }
+]

agent/agentcontainers/testdata/container_sameport/docker_inspect.json

@@ -0,0 +1,222 @@
+[
+    {
+        "Id": "4eac5ce199d27b2329d0ff0ce1a6fc595612ced48eba3669aadb6c57ebef3fa2",
+        "Created": "2025-03-11T17:56:34.842164541Z",
+        "Path": "sleep",
+        "Args": [
+            "infinity"
+        ],
+        "State": {
+            "Status": "running",
+            "Running": true,
+            "Paused": false,
+            "Restarting": false,
+            "OOMKilled": false,
+            "Dead": false,
+            "Pid": 638449,
+            "ExitCode": 0,
+            "Error": "",
+            "StartedAt": "2025-03-11T17:56:34.894488648Z",
+            "FinishedAt": "0001-01-01T00:00:00Z"
+        },
+        "Image": "sha256:d4ccddb816ba27eaae22ef3d56175d53f47998e2acb99df1ae0e5b426b28a076",
+        "ResolvConfPath": "/var/lib/docker/containers/4eac5ce199d27b2329d0ff0ce1a6fc595612ced48eba3669aadb6c57ebef3fa2/resolv.conf",
+        "HostnamePath": "/var/lib/docker/containers/4eac5ce199d27b2329d0ff0ce1a6fc595612ced48eba3669aadb6c57ebef3fa2/hostname",
+        "HostsPath": "/var/lib/docker/containers/4eac5ce199d27b2329d0ff0ce1a6fc595612ced48eba3669aadb6c57ebef3fa2/hosts",
+        "LogPath": "/var/lib/docker/containers/4eac5ce199d27b2329d0ff0ce1a6fc595612ced48eba3669aadb6c57ebef3fa2/4eac5ce199d27b2329d0ff0ce1a6fc595612ced48eba3669aadb6c57ebef3fa2-json.log",
+        "Name": "/modest_varahamihira",
+        "RestartCount": 0,
+        "Driver": "overlay2",
+        "Platform": "linux",
+        "MountLabel": "",
+        "ProcessLabel": "",
+        "AppArmorProfile": "",
+        "ExecIDs": null,
+        "HostConfig": {
+            "Binds": null,
+            "ContainerIDFile": "",
+            "LogConfig": {
+                "Type": "json-file",
+                "Config": {}
+            },
+            "NetworkMode": "bridge",
+            "PortBindings": {
+                "12345/tcp": [
+                    {
+                        "HostIp": "",
+                        "HostPort": "12345"
+                    }
+                ]
+            },
+            "RestartPolicy": {
+                "Name": "no",
+                "MaximumRetryCount": 0
+            },
+            "AutoRemove": false,
+            "VolumeDriver": "",
+            "VolumesFrom": null,
+            "ConsoleSize": [
+                108,
+                176
+            ],
+            "CapAdd": null,
+            "CapDrop": null,
+            "CgroupnsMode": "private",
+            "Dns": [],
+            "DnsOptions": [],
+            "DnsSearch": [],
+            "ExtraHosts": null,
+            "GroupAdd": null,
+            "IpcMode": "private",
+            "Cgroup": "",
+            "Links": null,
+            "OomScoreAdj": 10,
+            "PidMode": "",
+            "Privileged": false,
+            "PublishAllPorts": false,
+            "ReadonlyRootfs": false,
+            "SecurityOpt": null,
+            "UTSMode": "",
+            "UsernsMode": "",
+            "ShmSize": 67108864,
+            "Runtime": "runc",
+            "Isolation": "",
+            "CpuShares": 0,
+            "Memory": 0,
+            "NanoCpus": 0,
+            "CgroupParent": "",
+            "BlkioWeight": 0,
+            "BlkioWeightDevice": [],
+            "BlkioDeviceReadBps": [],
+            "BlkioDeviceWriteBps": [],
+            "BlkioDeviceReadIOps": [],
+            "BlkioDeviceWriteIOps": [],
+            "CpuPeriod": 0,
+            "CpuQuota": 0,
+            "CpuRealtimePeriod": 0,
+            "CpuRealtimeRuntime": 0,
+            "CpusetCpus": "",
+            "CpusetMems": "",
+            "Devices": [],
+            "DeviceCgroupRules": null,
+            "DeviceRequests": null,
+            "MemoryReservation": 0,
+            "MemorySwap": 0,
+            "MemorySwappiness": null,
+            "OomKillDisable": null,
+            "PidsLimit": null,
+            "Ulimits": [],
+            "CpuCount": 0,
+            "CpuPercent": 0,
+            "IOMaximumIOps": 0,
+            "IOMaximumBandwidth": 0,
+            "MaskedPaths": [
+                "/proc/asound",
+                "/proc/acpi",
+                "/proc/kcore",
+                "/proc/keys",
+                "/proc/latency_stats",
+                "/proc/timer_list",
+                "/proc/timer_stats",
+                "/proc/sched_debug",
+                "/proc/scsi",
+                "/sys/firmware",
+                "/sys/devices/virtual/powercap"
+            ],
+            "ReadonlyPaths": [
+                "/proc/bus",
+                "/proc/fs",
+                "/proc/irq",
+                "/proc/sys",
+                "/proc/sysrq-trigger"
+            ]
+        },
+        "GraphDriver": {
+            "Data": {
+                "ID": "4eac5ce199d27b2329d0ff0ce1a6fc595612ced48eba3669aadb6c57ebef3fa2",
+                "LowerDir": "/var/lib/docker/overlay2/35deac62dd3f610275aaf145d091aaa487f73a3c99de5a90df8ab871c969bc0b-init/diff:/var/lib/docker/overlay2/4b4c37dfbdc0dc01b68d4fb1ddb86109398a2d73555439b874dbd23b87cd5c4b/diff",
+                "MergedDir": "/var/lib/docker/overlay2/35deac62dd3f610275aaf145d091aaa487f73a3c99de5a90df8ab871c969bc0b/merged",
+                "UpperDir": "/var/lib/docker/overlay2/35deac62dd3f610275aaf145d091aaa487f73a3c99de5a90df8ab871c969bc0b/diff",
+                "WorkDir": "/var/lib/docker/overlay2/35deac62dd3f610275aaf145d091aaa487f73a3c99de5a90df8ab871c969bc0b/work"
+            },
+            "Name": "overlay2"
+        },
+        "Mounts": [],
+        "Config": {
+            "Hostname": "4eac5ce199d2",
+            "Domainname": "",
+            "User": "",
+            "AttachStdin": false,
+            "AttachStdout": false,
+            "AttachStderr": false,
+            "ExposedPorts": {
+                "12345/tcp": {}
+            },
+            "Tty": false,
+            "OpenStdin": false,
+            "StdinOnce": false,
+            "Env": [
+                "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+            ],
+            "Cmd": [
+                "sleep",
+                "infinity"
+            ],
+            "Image": "debian:bookworm",
+            "Volumes": null,
+            "WorkingDir": "",
+            "Entrypoint": [],
+            "OnBuild": null,
+            "Labels": {}
+        },
+        "NetworkSettings": {
+            "Bridge": "",
+            "SandboxID": "5e966e97ba02013054e0ef15ef87f8629f359ad882fad4c57b33c768ad9b90dc",
+            "SandboxKey": "/var/run/docker/netns/5e966e97ba02",
+            "Ports": {
+                "12345/tcp": [
+                    {
+                        "HostIp": "0.0.0.0",
+                        "HostPort": "12345"
+                    },
+                    {
+                        "HostIp": "::",
+                        "HostPort": "12345"
+                    }
+                ]
+            },
+            "HairpinMode": false,
+            "LinkLocalIPv6Address": "",
+            "LinkLocalIPv6PrefixLen": 0,
+            "SecondaryIPAddresses": null,
+            "SecondaryIPv6Addresses": null,
+            "EndpointID": "f9e1896fc0ef48f3ea9aff3b4e98bc4291ba246412178331345f7b0745cccba9",
+            "Gateway": "172.17.0.1",
+            "GlobalIPv6Address": "",
+            "GlobalIPv6PrefixLen": 0,
+            "IPAddress": "172.17.0.2",
+            "IPPrefixLen": 16,
+            "IPv6Gateway": "",
+            "MacAddress": "be:a6:89:39:7e:b0",
+            "Networks": {
+                "bridge": {
+                    "IPAMConfig": null,
+                    "Links": null,
+                    "Aliases": null,
+                    "MacAddress": "be:a6:89:39:7e:b0",
+                    "DriverOpts": null,
+                    "GwPriority": 0,
+                    "NetworkID": "c4dd768ab4945e41ad23fe3907c960dac811141592a861cc40038df7086a1ce1",
+                    "EndpointID": "f9e1896fc0ef48f3ea9aff3b4e98bc4291ba246412178331345f7b0745cccba9",
+                    "Gateway": "172.17.0.1",
+                    "IPAddress": "172.17.0.2",
+                    "IPPrefixLen": 16,
+                    "IPv6Gateway": "",
+                    "GlobalIPv6Address": "",
+                    "GlobalIPv6PrefixLen": 0,
+                    "DNSNames": null
+                }
+            }
+        }
+    }
+]

agent/agentcontainers/testdata/container_sameportdiffip/docker_inspect.json

@@ -0,0 +1,51 @@
+[
+	{
+		"Id": "a",
+		"Created": "2025-03-11T17:56:34.842164541Z",
+		"State": {
+			"Running": true,
+			"ExitCode": 0,
+			"Error": ""
+		},
+		"Name": "/a",
+		"Mounts": [],
+		"Config": {
+			"Image": "debian:bookworm",
+			"Labels": {}
+		},
+		"NetworkSettings": {
+			"Ports": {
+				"8001/tcp": [
+					{
+						"HostIp": "0.0.0.0",
+						"HostPort": "8000"
+					}
+				]
+			}
+		}
+	},
+	{
+		"Id": "b",
+		"Created": "2025-03-11T17:56:34.842164541Z",
+		"State": {
+			"Running": true,
+			"ExitCode": 0,
+			"Error": ""
+		},
+		"Name": "/b",
+		"Config": {
+			"Image": "debian:bookworm",
+			"Labels": {}
+		},
+		"NetworkSettings": {
+			"Ports": {
+				"8001/tcp": [
+					{
+						"HostIp": "::",
+						"HostPort": "8000"
+					}
+				]
+			}
+		}
+	}
+]

agent/agentcontainers/testdata/container_simple/docker_inspect.json

@@ -0,0 +1,201 @@
+[
+    {
+        "Id": "6b539b8c60f5230b8b0fde2502cd2332d31c0d526a3e6eb6eef1cc39439b3286",
+        "Created": "2025-03-11T17:55:58.091280203Z",
+        "Path": "sleep",
+        "Args": [
+            "infinity"
+        ],
+        "State": {
+            "Status": "running",
+            "Running": true,
+            "Paused": false,
+            "Restarting": false,
+            "OOMKilled": false,
+            "Dead": false,
+            "Pid": 636855,
+            "ExitCode": 0,
+            "Error": "",
+            "StartedAt": "2025-03-11T17:55:58.142417459Z",
+            "FinishedAt": "0001-01-01T00:00:00Z"
+        },
+        "Image": "sha256:d4ccddb816ba27eaae22ef3d56175d53f47998e2acb99df1ae0e5b426b28a076",
+        "ResolvConfPath": "/var/lib/docker/containers/6b539b8c60f5230b8b0fde2502cd2332d31c0d526a3e6eb6eef1cc39439b3286/resolv.conf",
+        "HostnamePath": "/var/lib/docker/containers/6b539b8c60f5230b8b0fde2502cd2332d31c0d526a3e6eb6eef1cc39439b3286/hostname",
+        "HostsPath": "/var/lib/docker/containers/6b539b8c60f5230b8b0fde2502cd2332d31c0d526a3e6eb6eef1cc39439b3286/hosts",
+        "LogPath": "/var/lib/docker/containers/6b539b8c60f5230b8b0fde2502cd2332d31c0d526a3e6eb6eef1cc39439b3286/6b539b8c60f5230b8b0fde2502cd2332d31c0d526a3e6eb6eef1cc39439b3286-json.log",
+        "Name": "/eloquent_kowalevski",
+        "RestartCount": 0,
+        "Driver": "overlay2",
+        "Platform": "linux",
+        "MountLabel": "",
+        "ProcessLabel": "",
+        "AppArmorProfile": "",
+        "ExecIDs": null,
+        "HostConfig": {
+            "Binds": null,
+            "ContainerIDFile": "",
+            "LogConfig": {
+                "Type": "json-file",
+                "Config": {}
+            },
+            "NetworkMode": "bridge",
+            "PortBindings": {},
+            "RestartPolicy": {
+                "Name": "no",
+                "MaximumRetryCount": 0
+            },
+            "AutoRemove": false,
+            "VolumeDriver": "",
+            "VolumesFrom": null,
+            "ConsoleSize": [
+                108,
+                176
+            ],
+            "CapAdd": null,
+            "CapDrop": null,
+            "CgroupnsMode": "private",
+            "Dns": [],
+            "DnsOptions": [],
+            "DnsSearch": [],
+            "ExtraHosts": null,
+            "GroupAdd": null,
+            "IpcMode": "private",
+            "Cgroup": "",
+            "Links": null,
+            "OomScoreAdj": 10,
+            "PidMode": "",
+            "Privileged": false,
+            "PublishAllPorts": false,
+            "ReadonlyRootfs": false,
+            "SecurityOpt": null,
+            "UTSMode": "",
+            "UsernsMode": "",
+            "ShmSize": 67108864,
+            "Runtime": "runc",
+            "Isolation": "",
+            "CpuShares": 0,
+            "Memory": 0,
+            "NanoCpus": 0,
+            "CgroupParent": "",
+            "BlkioWeight": 0,
+            "BlkioWeightDevice": [],
+            "BlkioDeviceReadBps": [],
+            "BlkioDeviceWriteBps": [],
+            "BlkioDeviceReadIOps": [],
+            "BlkioDeviceWriteIOps": [],
+            "CpuPeriod": 0,
+            "CpuQuota": 0,
+            "CpuRealtimePeriod": 0,
+            "CpuRealtimeRuntime": 0,
+            "CpusetCpus": "",
+            "CpusetMems": "",
+            "Devices": [],
+            "DeviceCgroupRules": null,
+            "DeviceRequests": null,
+            "MemoryReservation": 0,
+            "MemorySwap": 0,
+            "MemorySwappiness": null,
+            "OomKillDisable": null,
+            "PidsLimit": null,
+            "Ulimits": [],
+            "CpuCount": 0,
+            "CpuPercent": 0,
+            "IOMaximumIOps": 0,
+            "IOMaximumBandwidth": 0,
+            "MaskedPaths": [
+                "/proc/asound",
+                "/proc/acpi",
+                "/proc/kcore",
+                "/proc/keys",
+                "/proc/latency_stats",
+                "/proc/timer_list",
+                "/proc/timer_stats",
+                "/proc/sched_debug",
+                "/proc/scsi",
+                "/sys/firmware",
+                "/sys/devices/virtual/powercap"
+            ],
+            "ReadonlyPaths": [
+                "/proc/bus",
+                "/proc/fs",
+                "/proc/irq",
+                "/proc/sys",
+                "/proc/sysrq-trigger"
+            ]
+        },
+        "GraphDriver": {
+            "Data": {
+                "ID": "6b539b8c60f5230b8b0fde2502cd2332d31c0d526a3e6eb6eef1cc39439b3286",
+                "LowerDir": "/var/lib/docker/overlay2/4093560d7757c088e24060e5ff6f32807d8e733008c42b8af7057fe4fe6f56ba-init/diff:/var/lib/docker/overlay2/4b4c37dfbdc0dc01b68d4fb1ddb86109398a2d73555439b874dbd23b87cd5c4b/diff",
+                "MergedDir": "/var/lib/docker/overlay2/4093560d7757c088e24060e5ff6f32807d8e733008c42b8af7057fe4fe6f56ba/merged",
+                "UpperDir": "/var/lib/docker/overlay2/4093560d7757c088e24060e5ff6f32807d8e733008c42b8af7057fe4fe6f56ba/diff",
+                "WorkDir": "/var/lib/docker/overlay2/4093560d7757c088e24060e5ff6f32807d8e733008c42b8af7057fe4fe6f56ba/work"
+            },
+            "Name": "overlay2"
+        },
+        "Mounts": [],
+        "Config": {
+            "Hostname": "6b539b8c60f5",
+            "Domainname": "",
+            "User": "",
+            "AttachStdin": false,
+            "AttachStdout": false,
+            "AttachStderr": false,
+            "Tty": false,
+            "OpenStdin": false,
+            "StdinOnce": false,
+            "Env": [
+                "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+            ],
+            "Cmd": [
+                "sleep",
+                "infinity"
+            ],
+            "Image": "debian:bookworm",
+            "Volumes": null,
+            "WorkingDir": "",
+            "Entrypoint": [],
+            "OnBuild": null,
+            "Labels": {}
+        },
+        "NetworkSettings": {
+            "Bridge": "",
+            "SandboxID": "08f2f3218a6d63ae149ab77672659d96b88bca350e85889240579ecb427e8011",
+            "SandboxKey": "/var/run/docker/netns/08f2f3218a6d",
+            "Ports": {},
+            "HairpinMode": false,
+            "LinkLocalIPv6Address": "",
+            "LinkLocalIPv6PrefixLen": 0,
+            "SecondaryIPAddresses": null,
+            "SecondaryIPv6Addresses": null,
+            "EndpointID": "f83bd20711df6d6ff7e2f44f4b5799636cd94596ae25ffe507a70f424073532c",
+            "Gateway": "172.17.0.1",
+            "GlobalIPv6Address": "",
+            "GlobalIPv6PrefixLen": 0,
+            "IPAddress": "172.17.0.2",
+            "IPPrefixLen": 16,
+            "IPv6Gateway": "",
+            "MacAddress": "f6:84:26:7a:10:5b",
+            "Networks": {
+                "bridge": {
+                    "IPAMConfig": null,
+                    "Links": null,
+                    "Aliases": null,
+                    "MacAddress": "f6:84:26:7a:10:5b",
+                    "DriverOpts": null,
+                    "GwPriority": 0,
+                    "NetworkID": "c4dd768ab4945e41ad23fe3907c960dac811141592a861cc40038df7086a1ce1",
+                    "EndpointID": "f83bd20711df6d6ff7e2f44f4b5799636cd94596ae25ffe507a70f424073532c",
+                    "Gateway": "172.17.0.1",
+                    "IPAddress": "172.17.0.2",
+                    "IPPrefixLen": 16,
+                    "IPv6Gateway": "",
+                    "GlobalIPv6Address": "",
+                    "GlobalIPv6PrefixLen": 0,
+                    "DNSNames": null
+                }
+            }
+        }
+    }
+]

agent/agentcontainers/testdata/container_volume/docker_inspect.json

@@ -0,0 +1,214 @@
+[
+    {
+        "Id": "b3688d98c007f53402a55e46d803f2f3ba9181d8e3f71a2eb19b392cf0377b4e",
+        "Created": "2025-03-11T17:59:42.039484134Z",
+        "Path": "sleep",
+        "Args": [
+            "infinity"
+        ],
+        "State": {
+            "Status": "running",
+            "Running": true,
+            "Paused": false,
+            "Restarting": false,
+            "OOMKilled": false,
+            "Dead": false,
+            "Pid": 646777,
+            "ExitCode": 0,
+            "Error": "",
+            "StartedAt": "2025-03-11T17:59:42.081315917Z",
+            "FinishedAt": "0001-01-01T00:00:00Z"
+        },
+        "Image": "sha256:d4ccddb816ba27eaae22ef3d56175d53f47998e2acb99df1ae0e5b426b28a076",
+        "ResolvConfPath": "/var/lib/docker/containers/b3688d98c007f53402a55e46d803f2f3ba9181d8e3f71a2eb19b392cf0377b4e/resolv.conf",
+        "HostnamePath": "/var/lib/docker/containers/b3688d98c007f53402a55e46d803f2f3ba9181d8e3f71a2eb19b392cf0377b4e/hostname",
+        "HostsPath": "/var/lib/docker/containers/b3688d98c007f53402a55e46d803f2f3ba9181d8e3f71a2eb19b392cf0377b4e/hosts",
+        "LogPath": "/var/lib/docker/containers/b3688d98c007f53402a55e46d803f2f3ba9181d8e3f71a2eb19b392cf0377b4e/b3688d98c007f53402a55e46d803f2f3ba9181d8e3f71a2eb19b392cf0377b4e-json.log",
+        "Name": "/upbeat_carver",
+        "RestartCount": 0,
+        "Driver": "overlay2",
+        "Platform": "linux",
+        "MountLabel": "",
+        "ProcessLabel": "",
+        "AppArmorProfile": "",
+        "ExecIDs": null,
+        "HostConfig": {
+            "Binds": [
+                "testvol:/testvol"
+            ],
+            "ContainerIDFile": "",
+            "LogConfig": {
+                "Type": "json-file",
+                "Config": {}
+            },
+            "NetworkMode": "bridge",
+            "PortBindings": {},
+            "RestartPolicy": {
+                "Name": "no",
+                "MaximumRetryCount": 0
+            },
+            "AutoRemove": false,
+            "VolumeDriver": "",
+            "VolumesFrom": null,
+            "ConsoleSize": [
+                108,
+                176
+            ],
+            "CapAdd": null,
+            "CapDrop": null,
+            "CgroupnsMode": "private",
+            "Dns": [],
+            "DnsOptions": [],
+            "DnsSearch": [],
+            "ExtraHosts": null,
+            "GroupAdd": null,
+            "IpcMode": "private",
+            "Cgroup": "",
+            "Links": null,
+            "OomScoreAdj": 10,
+            "PidMode": "",
+            "Privileged": false,
+            "PublishAllPorts": false,
+            "ReadonlyRootfs": false,
+            "SecurityOpt": null,
+            "UTSMode": "",
+            "UsernsMode": "",
+            "ShmSize": 67108864,
+            "Runtime": "runc",
+            "Isolation": "",
+            "CpuShares": 0,
+            "Memory": 0,
+            "NanoCpus": 0,
+            "CgroupParent": "",
+            "BlkioWeight": 0,
+            "BlkioWeightDevice": [],
+            "BlkioDeviceReadBps": [],
+            "BlkioDeviceWriteBps": [],
+            "BlkioDeviceReadIOps": [],
+            "BlkioDeviceWriteIOps": [],
+            "CpuPeriod": 0,
+            "CpuQuota": 0,
+            "CpuRealtimePeriod": 0,
+            "CpuRealtimeRuntime": 0,
+            "CpusetCpus": "",
+            "CpusetMems": "",
+            "Devices": [],
+            "DeviceCgroupRules": null,
+            "DeviceRequests": null,
+            "MemoryReservation": 0,
+            "MemorySwap": 0,
+            "MemorySwappiness": null,
+            "OomKillDisable": null,
+            "PidsLimit": null,
+            "Ulimits": [],
+            "CpuCount": 0,
+            "CpuPercent": 0,
+            "IOMaximumIOps": 0,
+            "IOMaximumBandwidth": 0,
+            "MaskedPaths": [
+                "/proc/asound",
+                "/proc/acpi",
+                "/proc/kcore",
+                "/proc/keys",
+                "/proc/latency_stats",
+                "/proc/timer_list",
+                "/proc/timer_stats",
+                "/proc/sched_debug",
+                "/proc/scsi",
+                "/sys/firmware",
+                "/sys/devices/virtual/powercap"
+            ],
+            "ReadonlyPaths": [
+                "/proc/bus",
+                "/proc/fs",
+                "/proc/irq",
+                "/proc/sys",
+                "/proc/sysrq-trigger"
+            ]
+        },
+        "GraphDriver": {
+            "Data": {
+                "ID": "b3688d98c007f53402a55e46d803f2f3ba9181d8e3f71a2eb19b392cf0377b4e",
+                "LowerDir": "/var/lib/docker/overlay2/d71790d2558bf17d7535451094e332780638a4e92697c021176f3447fc4c50f4-init/diff:/var/lib/docker/overlay2/4b4c37dfbdc0dc01b68d4fb1ddb86109398a2d73555439b874dbd23b87cd5c4b/diff",
+                "MergedDir": "/var/lib/docker/overlay2/d71790d2558bf17d7535451094e332780638a4e92697c021176f3447fc4c50f4/merged",
+                "UpperDir": "/var/lib/docker/overlay2/d71790d2558bf17d7535451094e332780638a4e92697c021176f3447fc4c50f4/diff",
+                "WorkDir": "/var/lib/docker/overlay2/d71790d2558bf17d7535451094e332780638a4e92697c021176f3447fc4c50f4/work"
+            },
+            "Name": "overlay2"
+        },
+        "Mounts": [
+            {
+                "Type": "volume",
+                "Name": "testvol",
+                "Source": "/var/lib/docker/volumes/testvol/_data",
+                "Destination": "/testvol",
+                "Driver": "local",
+                "Mode": "z",
+                "RW": true,
+                "Propagation": ""
+            }
+        ],
+        "Config": {
+            "Hostname": "b3688d98c007",
+            "Domainname": "",
+            "User": "",
+            "AttachStdin": false,
+            "AttachStdout": false,
+            "AttachStderr": false,
+            "Tty": false,
+            "OpenStdin": false,
+            "StdinOnce": false,
+            "Env": [
+                "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+            ],
+            "Cmd": [
+                "sleep",
+                "infinity"
+            ],
+            "Image": "debian:bookworm",
+            "Volumes": null,
+            "WorkingDir": "",
+            "Entrypoint": [],
+            "OnBuild": null,
+            "Labels": {}
+        },
+        "NetworkSettings": {
+            "Bridge": "",
+            "SandboxID": "e617ea865af5690d06c25df1c9a0154b98b4da6bbb9e0afae3b80ad29902538a",
+            "SandboxKey": "/var/run/docker/netns/e617ea865af5",
+            "Ports": {},
+            "HairpinMode": false,
+            "LinkLocalIPv6Address": "",
+            "LinkLocalIPv6PrefixLen": 0,
+            "SecondaryIPAddresses": null,
+            "SecondaryIPv6Addresses": null,
+            "EndpointID": "1a7bb5bbe4af0674476c95c5d1c913348bc82a5f01fd1c1b394afc44d1cf5a49",
+            "Gateway": "172.17.0.1",
+            "GlobalIPv6Address": "",
+            "GlobalIPv6PrefixLen": 0,
+            "IPAddress": "172.17.0.2",
+            "IPPrefixLen": 16,
+            "IPv6Gateway": "",
+            "MacAddress": "4a:d8:a5:47:1c:54",
+            "Networks": {
+                "bridge": {
+                    "IPAMConfig": null,
+                    "Links": null,
+                    "Aliases": null,
+                    "MacAddress": "4a:d8:a5:47:1c:54",
+                    "DriverOpts": null,
+                    "GwPriority": 0,
+                    "NetworkID": "c4dd768ab4945e41ad23fe3907c960dac811141592a861cc40038df7086a1ce1",
+                    "EndpointID": "1a7bb5bbe4af0674476c95c5d1c913348bc82a5f01fd1c1b394afc44d1cf5a49",
+                    "Gateway": "172.17.0.1",
+                    "IPAddress": "172.17.0.2",
+                    "IPPrefixLen": 16,
+                    "IPv6Gateway": "",
+                    "GlobalIPv6Address": "",
+                    "GlobalIPv6PrefixLen": 0,
+                    "DNSNames": null
+                }
+            }
+        }
+    }
+]

agent/agentcontainers/testdata/devcontainer_appport/docker_inspect.json

@@ -0,0 +1,230 @@
+[
+    {
+        "Id": "52d23691f4b954d083f117358ea763e20f69af584e1c08f479c5752629ee0be3",
+        "Created": "2025-03-11T17:02:42.613747761Z",
+        "Path": "/bin/sh",
+        "Args": [
+            "-c",
+            "echo Container started\ntrap \"exit 0\" 15\n\nexec \"$@\"\nwhile sleep 1 & wait $!; do :; done",
+            "-"
+        ],
+        "State": {
+            "Status": "running",
+            "Running": true,
+            "Paused": false,
+            "Restarting": false,
+            "OOMKilled": false,
+            "Dead": false,
+            "Pid": 526198,
+            "ExitCode": 0,
+            "Error": "",
+            "StartedAt": "2025-03-11T17:02:42.658905789Z",
+            "FinishedAt": "0001-01-01T00:00:00Z"
+        },
+        "Image": "sha256:d4ccddb816ba27eaae22ef3d56175d53f47998e2acb99df1ae0e5b426b28a076",
+        "ResolvConfPath": "/var/lib/docker/containers/52d23691f4b954d083f117358ea763e20f69af584e1c08f479c5752629ee0be3/resolv.conf",
+        "HostnamePath": "/var/lib/docker/containers/52d23691f4b954d083f117358ea763e20f69af584e1c08f479c5752629ee0be3/hostname",
+        "HostsPath": "/var/lib/docker/containers/52d23691f4b954d083f117358ea763e20f69af584e1c08f479c5752629ee0be3/hosts",
+        "LogPath": "/var/lib/docker/containers/52d23691f4b954d083f117358ea763e20f69af584e1c08f479c5752629ee0be3/52d23691f4b954d083f117358ea763e20f69af584e1c08f479c5752629ee0be3-json.log",
+        "Name": "/suspicious_margulis",
+        "RestartCount": 0,
+        "Driver": "overlay2",
+        "Platform": "linux",
+        "MountLabel": "",
+        "ProcessLabel": "",
+        "AppArmorProfile": "",
+        "ExecIDs": null,
+        "HostConfig": {
+            "Binds": null,
+            "ContainerIDFile": "",
+            "LogConfig": {
+                "Type": "json-file",
+                "Config": {}
+            },
+            "NetworkMode": "bridge",
+            "PortBindings": {
+                "8080/tcp": [
+                    {
+                        "HostIp": "",
+                        "HostPort": ""
+                    }
+                ]
+            },
+            "RestartPolicy": {
+                "Name": "no",
+                "MaximumRetryCount": 0
+            },
+            "AutoRemove": false,
+            "VolumeDriver": "",
+            "VolumesFrom": null,
+            "ConsoleSize": [
+                108,
+                176
+            ],
+            "CapAdd": null,
+            "CapDrop": null,
+            "CgroupnsMode": "private",
+            "Dns": [],
+            "DnsOptions": [],
+            "DnsSearch": [],
+            "ExtraHosts": null,
+            "GroupAdd": null,
+            "IpcMode": "private",
+            "Cgroup": "",
+            "Links": null,
+            "OomScoreAdj": 10,
+            "PidMode": "",
+            "Privileged": false,
+            "PublishAllPorts": false,
+            "ReadonlyRootfs": false,
+            "SecurityOpt": null,
+            "UTSMode": "",
+            "UsernsMode": "",
+            "ShmSize": 67108864,
+            "Runtime": "runc",
+            "Isolation": "",
+            "CpuShares": 0,
+            "Memory": 0,
+            "NanoCpus": 0,
+            "CgroupParent": "",
+            "BlkioWeight": 0,
+            "BlkioWeightDevice": [],
+            "BlkioDeviceReadBps": [],
+            "BlkioDeviceWriteBps": [],
+            "BlkioDeviceReadIOps": [],
+            "BlkioDeviceWriteIOps": [],
+            "CpuPeriod": 0,
+            "CpuQuota": 0,
+            "CpuRealtimePeriod": 0,
+            "CpuRealtimeRuntime": 0,
+            "CpusetCpus": "",
+            "CpusetMems": "",
+            "Devices": [],
+            "DeviceCgroupRules": null,
+            "DeviceRequests": null,
+            "MemoryReservation": 0,
+            "MemorySwap": 0,
+            "MemorySwappiness": null,
+            "OomKillDisable": null,
+            "PidsLimit": null,
+            "Ulimits": [],
+            "CpuCount": 0,
+            "CpuPercent": 0,
+            "IOMaximumIOps": 0,
+            "IOMaximumBandwidth": 0,
+            "MaskedPaths": [
+                "/proc/asound",
+                "/proc/acpi",
+                "/proc/kcore",
+                "/proc/keys",
+                "/proc/latency_stats",
+                "/proc/timer_list",
+                "/proc/timer_stats",
+                "/proc/sched_debug",
+                "/proc/scsi",
+                "/sys/firmware",
+                "/sys/devices/virtual/powercap"
+            ],
+            "ReadonlyPaths": [
+                "/proc/bus",
+                "/proc/fs",
+                "/proc/irq",
+                "/proc/sys",
+                "/proc/sysrq-trigger"
+            ]
+        },
+        "GraphDriver": {
+            "Data": {
+                "ID": "52d23691f4b954d083f117358ea763e20f69af584e1c08f479c5752629ee0be3",
+                "LowerDir": "/var/lib/docker/overlay2/e204eab11c98b3cacc18d5a0e7290c0c286a96d918c31e5c2fed4124132eec4f-init/diff:/var/lib/docker/overlay2/4b4c37dfbdc0dc01b68d4fb1ddb86109398a2d73555439b874dbd23b87cd5c4b/diff",
+                "MergedDir": "/var/lib/docker/overlay2/e204eab11c98b3cacc18d5a0e7290c0c286a96d918c31e5c2fed4124132eec4f/merged",
+                "UpperDir": "/var/lib/docker/overlay2/e204eab11c98b3cacc18d5a0e7290c0c286a96d918c31e5c2fed4124132eec4f/diff",
+                "WorkDir": "/var/lib/docker/overlay2/e204eab11c98b3cacc18d5a0e7290c0c286a96d918c31e5c2fed4124132eec4f/work"
+            },
+            "Name": "overlay2"
+        },
+        "Mounts": [],
+        "Config": {
+            "Hostname": "52d23691f4b9",
+            "Domainname": "",
+            "User": "",
+            "AttachStdin": false,
+            "AttachStdout": true,
+            "AttachStderr": true,
+            "ExposedPorts": {
+                "8080/tcp": {}
+            },
+            "Tty": false,
+            "OpenStdin": false,
+            "StdinOnce": false,
+            "Env": [
+                "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+            ],
+            "Cmd": [
+                "-c",
+                "echo Container started\ntrap \"exit 0\" 15\n\nexec \"$@\"\nwhile sleep 1 & wait $!; do :; done",
+                "-"
+            ],
+            "Image": "debian:bookworm",
+            "Volumes": null,
+            "WorkingDir": "",
+            "Entrypoint": [
+                "/bin/sh"
+            ],
+            "OnBuild": null,
+            "Labels": {
+                "devcontainer.config_file": "/home/coder/src/coder/coder/agent/agentcontainers/testdata/devcontainer_appport.json",
+                "devcontainer.metadata": "[]"
+            }
+        },
+        "NetworkSettings": {
+            "Bridge": "",
+            "SandboxID": "e4fa65f769e331c72e27f43af2d65073efca638fd413b7c57f763ee9ebf69020",
+            "SandboxKey": "/var/run/docker/netns/e4fa65f769e3",
+            "Ports": {
+                "8080/tcp": [
+                    {
+                        "HostIp": "0.0.0.0",
+                        "HostPort": "32768"
+                    },
+                    {
+                        "HostIp": "::",
+                        "HostPort": "32768"
+                    }
+                ]
+            },
+            "HairpinMode": false,
+            "LinkLocalIPv6Address": "",
+            "LinkLocalIPv6PrefixLen": 0,
+            "SecondaryIPAddresses": null,
+            "SecondaryIPv6Addresses": null,
+            "EndpointID": "14531bbbb26052456a4509e6d23753de45096ca8355ac11684c631d1656248ad",
+            "Gateway": "172.17.0.1",
+            "GlobalIPv6Address": "",
+            "GlobalIPv6PrefixLen": 0,
+            "IPAddress": "172.17.0.2",
+            "IPPrefixLen": 16,
+            "IPv6Gateway": "",
+            "MacAddress": "36:88:48:04:4e:b4",
+            "Networks": {
+                "bridge": {
+                    "IPAMConfig": null,
+                    "Links": null,
+                    "Aliases": null,
+                    "MacAddress": "36:88:48:04:4e:b4",
+                    "DriverOpts": null,
+                    "GwPriority": 0,
+                    "NetworkID": "c4dd768ab4945e41ad23fe3907c960dac811141592a861cc40038df7086a1ce1",
+                    "EndpointID": "14531bbbb26052456a4509e6d23753de45096ca8355ac11684c631d1656248ad",
+                    "Gateway": "172.17.0.1",
+                    "IPAddress": "172.17.0.2",
+                    "IPPrefixLen": 16,
+                    "IPv6Gateway": "",
+                    "GlobalIPv6Address": "",
+                    "GlobalIPv6PrefixLen": 0,
+                    "DNSNames": null
+                }
+            }
+        }
+    }
+]

agent/agentcontainers/testdata/devcontainer_forwardport/docker_inspect.json

@@ -0,0 +1,209 @@
+[
+    {
+        "Id": "4a16af2293fb75dc827a6949a3905dd57ea28cc008823218ce24fab1cb66c067",
+        "Created": "2025-03-11T17:03:55.022053072Z",
+        "Path": "/bin/sh",
+        "Args": [
+            "-c",
+            "echo Container started\ntrap \"exit 0\" 15\n\nexec \"$@\"\nwhile sleep 1 & wait $!; do :; done",
+            "-"
+        ],
+        "State": {
+            "Status": "running",
+            "Running": true,
+            "Paused": false,
+            "Restarting": false,
+            "OOMKilled": false,
+            "Dead": false,
+            "Pid": 529591,
+            "ExitCode": 0,
+            "Error": "",
+            "StartedAt": "2025-03-11T17:03:55.064323762Z",
+            "FinishedAt": "0001-01-01T00:00:00Z"
+        },
+        "Image": "sha256:d4ccddb816ba27eaae22ef3d56175d53f47998e2acb99df1ae0e5b426b28a076",
+        "ResolvConfPath": "/var/lib/docker/containers/4a16af2293fb75dc827a6949a3905dd57ea28cc008823218ce24fab1cb66c067/resolv.conf",
+        "HostnamePath": "/var/lib/docker/containers/4a16af2293fb75dc827a6949a3905dd57ea28cc008823218ce24fab1cb66c067/hostname",
+        "HostsPath": "/var/lib/docker/containers/4a16af2293fb75dc827a6949a3905dd57ea28cc008823218ce24fab1cb66c067/hosts",
+        "LogPath": "/var/lib/docker/containers/4a16af2293fb75dc827a6949a3905dd57ea28cc008823218ce24fab1cb66c067/4a16af2293fb75dc827a6949a3905dd57ea28cc008823218ce24fab1cb66c067-json.log",
+        "Name": "/serene_khayyam",
+        "RestartCount": 0,
+        "Driver": "overlay2",
+        "Platform": "linux",
+        "MountLabel": "",
+        "ProcessLabel": "",
+        "AppArmorProfile": "",
+        "ExecIDs": null,
+        "HostConfig": {
+            "Binds": null,
+            "ContainerIDFile": "",
+            "LogConfig": {
+                "Type": "json-file",
+                "Config": {}
+            },
+            "NetworkMode": "bridge",
+            "PortBindings": {},
+            "RestartPolicy": {
+                "Name": "no",
+                "MaximumRetryCount": 0
+            },
+            "AutoRemove": false,
+            "VolumeDriver": "",
+            "VolumesFrom": null,
+            "ConsoleSize": [
+                108,
+                176
+            ],
+            "CapAdd": null,
+            "CapDrop": null,
+            "CgroupnsMode": "private",
+            "Dns": [],
+            "DnsOptions": [],
+            "DnsSearch": [],
+            "ExtraHosts": null,
+            "GroupAdd": null,
+            "IpcMode": "private",
+            "Cgroup": "",
+            "Links": null,
+            "OomScoreAdj": 10,
+            "PidMode": "",
+            "Privileged": false,
+            "PublishAllPorts": false,
+            "ReadonlyRootfs": false,
+            "SecurityOpt": null,
+            "UTSMode": "",
+            "UsernsMode": "",
+            "ShmSize": 67108864,
+            "Runtime": "runc",
+            "Isolation": "",
+            "CpuShares": 0,
+            "Memory": 0,
+            "NanoCpus": 0,
+            "CgroupParent": "",
+            "BlkioWeight": 0,
+            "BlkioWeightDevice": [],
+            "BlkioDeviceReadBps": [],
+            "BlkioDeviceWriteBps": [],
+            "BlkioDeviceReadIOps": [],
+            "BlkioDeviceWriteIOps": [],
+            "CpuPeriod": 0,
+            "CpuQuota": 0,
+            "CpuRealtimePeriod": 0,
+            "CpuRealtimeRuntime": 0,
+            "CpusetCpus": "",
+            "CpusetMems": "",
+            "Devices": [],
+            "DeviceCgroupRules": null,
+            "DeviceRequests": null,
+            "MemoryReservation": 0,
+            "MemorySwap": 0,
+            "MemorySwappiness": null,
+            "OomKillDisable": null,
+            "PidsLimit": null,
+            "Ulimits": [],
+            "CpuCount": 0,
+            "CpuPercent": 0,
+            "IOMaximumIOps": 0,
+            "IOMaximumBandwidth": 0,
+            "MaskedPaths": [
+                "/proc/asound",
+                "/proc/acpi",
+                "/proc/kcore",
+                "/proc/keys",
+                "/proc/latency_stats",
+                "/proc/timer_list",
+                "/proc/timer_stats",
+                "/proc/sched_debug",
+                "/proc/scsi",
+                "/sys/firmware",
+                "/sys/devices/virtual/powercap"
+            ],
+            "ReadonlyPaths": [
+                "/proc/bus",
+                "/proc/fs",
+                "/proc/irq",
+                "/proc/sys",
+                "/proc/sysrq-trigger"
+            ]
+        },
+        "GraphDriver": {
+            "Data": {
+                "ID": "4a16af2293fb75dc827a6949a3905dd57ea28cc008823218ce24fab1cb66c067",
+                "LowerDir": "/var/lib/docker/overlay2/1974a49367024c771135c80dd6b62ba46cdebfa866e67a5408426c88a30bac3e-init/diff:/var/lib/docker/overlay2/4b4c37dfbdc0dc01b68d4fb1ddb86109398a2d73555439b874dbd23b87cd5c4b/diff",
+                "MergedDir": "/var/lib/docker/overlay2/1974a49367024c771135c80dd6b62ba46cdebfa866e67a5408426c88a30bac3e/merged",
+                "UpperDir": "/var/lib/docker/overlay2/1974a49367024c771135c80dd6b62ba46cdebfa866e67a5408426c88a30bac3e/diff",
+                "WorkDir": "/var/lib/docker/overlay2/1974a49367024c771135c80dd6b62ba46cdebfa866e67a5408426c88a30bac3e/work"
+            },
+            "Name": "overlay2"
+        },
+        "Mounts": [],
+        "Config": {
+            "Hostname": "4a16af2293fb",
+            "Domainname": "",
+            "User": "",
+            "AttachStdin": false,
+            "AttachStdout": true,
+            "AttachStderr": true,
+            "Tty": false,
+            "OpenStdin": false,
+            "StdinOnce": false,
+            "Env": [
+                "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+            ],
+            "Cmd": [
+                "-c",
+                "echo Container started\ntrap \"exit 0\" 15\n\nexec \"$@\"\nwhile sleep 1 & wait $!; do :; done",
+                "-"
+            ],
+            "Image": "debian:bookworm",
+            "Volumes": null,
+            "WorkingDir": "",
+            "Entrypoint": [
+                "/bin/sh"
+            ],
+            "OnBuild": null,
+            "Labels": {
+                "devcontainer.config_file": "/home/coder/src/coder/coder/agent/agentcontainers/testdata/devcontainer_forwardport.json",
+                "devcontainer.metadata": "[]"
+            }
+        },
+        "NetworkSettings": {
+            "Bridge": "",
+            "SandboxID": "e1c3bddb359d16c45d6d132561b83205af7809b01ed5cb985a8cb1b416b2ddd5",
+            "SandboxKey": "/var/run/docker/netns/e1c3bddb359d",
+            "Ports": {},
+            "HairpinMode": false,
+            "LinkLocalIPv6Address": "",
+            "LinkLocalIPv6PrefixLen": 0,
+            "SecondaryIPAddresses": null,
+            "SecondaryIPv6Addresses": null,
+            "EndpointID": "2899f34f5f8b928619952dc32566d82bc121b033453f72e5de4a743feabc423b",
+            "Gateway": "172.17.0.1",
+            "GlobalIPv6Address": "",
+            "GlobalIPv6PrefixLen": 0,
+            "IPAddress": "172.17.0.2",
+            "IPPrefixLen": 16,
+            "IPv6Gateway": "",
+            "MacAddress": "3e:94:61:83:1f:58",
+            "Networks": {
+                "bridge": {
+                    "IPAMConfig": null,
+                    "Links": null,
+                    "Aliases": null,
+                    "MacAddress": "3e:94:61:83:1f:58",
+                    "DriverOpts": null,
+                    "GwPriority": 0,
+                    "NetworkID": "c4dd768ab4945e41ad23fe3907c960dac811141592a861cc40038df7086a1ce1",
+                    "EndpointID": "2899f34f5f8b928619952dc32566d82bc121b033453f72e5de4a743feabc423b",
+                    "Gateway": "172.17.0.1",
+                    "IPAddress": "172.17.0.2",
+                    "IPPrefixLen": 16,
+                    "IPv6Gateway": "",
+                    "GlobalIPv6Address": "",
+                    "GlobalIPv6PrefixLen": 0,
+                    "DNSNames": null
+                }
+            }
+        }
+    }
+]

agent/agentcontainers/testdata/devcontainer_simple/docker_inspect.json

@@ -0,0 +1,209 @@
+[
+    {
+        "Id": "0b2a9fcf5727d9562943ce47d445019f4520e37a2aa7c6d9346d01af4f4f9aed",
+        "Created": "2025-03-11T17:01:05.751972661Z",
+        "Path": "/bin/sh",
+        "Args": [
+            "-c",
+            "echo Container started\ntrap \"exit 0\" 15\n\nexec \"$@\"\nwhile sleep 1 & wait $!; do :; done",
+            "-"
+        ],
+        "State": {
+            "Status": "running",
+            "Running": true,
+            "Paused": false,
+            "Restarting": false,
+            "OOMKilled": false,
+            "Dead": false,
+            "Pid": 521929,
+            "ExitCode": 0,
+            "Error": "",
+            "StartedAt": "2025-03-11T17:01:06.002539252Z",
+            "FinishedAt": "0001-01-01T00:00:00Z"
+        },
+        "Image": "sha256:d4ccddb816ba27eaae22ef3d56175d53f47998e2acb99df1ae0e5b426b28a076",
+        "ResolvConfPath": "/var/lib/docker/containers/0b2a9fcf5727d9562943ce47d445019f4520e37a2aa7c6d9346d01af4f4f9aed/resolv.conf",
+        "HostnamePath": "/var/lib/docker/containers/0b2a9fcf5727d9562943ce47d445019f4520e37a2aa7c6d9346d01af4f4f9aed/hostname",
+        "HostsPath": "/var/lib/docker/containers/0b2a9fcf5727d9562943ce47d445019f4520e37a2aa7c6d9346d01af4f4f9aed/hosts",
+        "LogPath": "/var/lib/docker/containers/0b2a9fcf5727d9562943ce47d445019f4520e37a2aa7c6d9346d01af4f4f9aed/0b2a9fcf5727d9562943ce47d445019f4520e37a2aa7c6d9346d01af4f4f9aed-json.log",
+        "Name": "/optimistic_hopper",
+        "RestartCount": 0,
+        "Driver": "overlay2",
+        "Platform": "linux",
+        "MountLabel": "",
+        "ProcessLabel": "",
+        "AppArmorProfile": "",
+        "ExecIDs": null,
+        "HostConfig": {
+            "Binds": null,
+            "ContainerIDFile": "",
+            "LogConfig": {
+                "Type": "json-file",
+                "Config": {}
+            },
+            "NetworkMode": "bridge",
+            "PortBindings": {},
+            "RestartPolicy": {
+                "Name": "no",
+                "MaximumRetryCount": 0
+            },
+            "AutoRemove": false,
+            "VolumeDriver": "",
+            "VolumesFrom": null,
+            "ConsoleSize": [
+                108,
+                176
+            ],
+            "CapAdd": null,
+            "CapDrop": null,
+            "CgroupnsMode": "private",
+            "Dns": [],
+            "DnsOptions": [],
+            "DnsSearch": [],
+            "ExtraHosts": null,
+            "GroupAdd": null,
+            "IpcMode": "private",
+            "Cgroup": "",
+            "Links": null,
+            "OomScoreAdj": 10,
+            "PidMode": "",
+            "Privileged": false,
+            "PublishAllPorts": false,
+            "ReadonlyRootfs": false,
+            "SecurityOpt": null,
+            "UTSMode": "",
+            "UsernsMode": "",
+            "ShmSize": 67108864,
+            "Runtime": "runc",
+            "Isolation": "",
+            "CpuShares": 0,
+            "Memory": 0,
+            "NanoCpus": 0,
+            "CgroupParent": "",
+            "BlkioWeight": 0,
+            "BlkioWeightDevice": [],
+            "BlkioDeviceReadBps": [],
+            "BlkioDeviceWriteBps": [],
+            "BlkioDeviceReadIOps": [],
+            "BlkioDeviceWriteIOps": [],
+            "CpuPeriod": 0,
+            "CpuQuota": 0,
+            "CpuRealtimePeriod": 0,
+            "CpuRealtimeRuntime": 0,
+            "CpusetCpus": "",
+            "CpusetMems": "",
+            "Devices": [],
+            "DeviceCgroupRules": null,
+            "DeviceRequests": null,
+            "MemoryReservation": 0,
+            "MemorySwap": 0,
+            "MemorySwappiness": null,
+            "OomKillDisable": null,
+            "PidsLimit": null,
+            "Ulimits": [],
+            "CpuCount": 0,
+            "CpuPercent": 0,
+            "IOMaximumIOps": 0,
+            "IOMaximumBandwidth": 0,
+            "MaskedPaths": [
+                "/proc/asound",
+                "/proc/acpi",
+                "/proc/kcore",
+                "/proc/keys",
+                "/proc/latency_stats",
+                "/proc/timer_list",
+                "/proc/timer_stats",
+                "/proc/sched_debug",
+                "/proc/scsi",
+                "/sys/firmware",
+                "/sys/devices/virtual/powercap"
+            ],
+            "ReadonlyPaths": [
+                "/proc/bus",
+                "/proc/fs",
+                "/proc/irq",
+                "/proc/sys",
+                "/proc/sysrq-trigger"
+            ]
+        },
+        "GraphDriver": {
+            "Data": {
+                "ID": "0b2a9fcf5727d9562943ce47d445019f4520e37a2aa7c6d9346d01af4f4f9aed",
+                "LowerDir": "/var/lib/docker/overlay2/b698fd9f03f25014d4936cdc64ed258342fe685f0dfd8813ed6928dd6de75219-init/diff:/var/lib/docker/overlay2/4b4c37dfbdc0dc01b68d4fb1ddb86109398a2d73555439b874dbd23b87cd5c4b/diff",
+                "MergedDir": "/var/lib/docker/overlay2/b698fd9f03f25014d4936cdc64ed258342fe685f0dfd8813ed6928dd6de75219/merged",
+                "UpperDir": "/var/lib/docker/overlay2/b698fd9f03f25014d4936cdc64ed258342fe685f0dfd8813ed6928dd6de75219/diff",
+                "WorkDir": "/var/lib/docker/overlay2/b698fd9f03f25014d4936cdc64ed258342fe685f0dfd8813ed6928dd6de75219/work"
+            },
+            "Name": "overlay2"
+        },
+        "Mounts": [],
+        "Config": {
+            "Hostname": "0b2a9fcf5727",
+            "Domainname": "",
+            "User": "",
+            "AttachStdin": false,
+            "AttachStdout": true,
+            "AttachStderr": true,
+            "Tty": false,
+            "OpenStdin": false,
+            "StdinOnce": false,
+            "Env": [
+                "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+            ],
+            "Cmd": [
+                "-c",
+                "echo Container started\ntrap \"exit 0\" 15\n\nexec \"$@\"\nwhile sleep 1 & wait $!; do :; done",
+                "-"
+            ],
+            "Image": "debian:bookworm",
+            "Volumes": null,
+            "WorkingDir": "",
+            "Entrypoint": [
+                "/bin/sh"
+            ],
+            "OnBuild": null,
+            "Labels": {
+                "devcontainer.config_file": "/home/coder/src/coder/coder/agent/agentcontainers/testdata/devcontainer_simple.json",
+                "devcontainer.metadata": "[]"
+            }
+        },
+        "NetworkSettings": {
+            "Bridge": "",
+            "SandboxID": "25a29a57c1330e0d0d2342af6e3291ffd3e812aca1a6e3f6a1630e74b73d0fc6",
+            "SandboxKey": "/var/run/docker/netns/25a29a57c133",
+            "Ports": {},
+            "HairpinMode": false,
+            "LinkLocalIPv6Address": "",
+            "LinkLocalIPv6PrefixLen": 0,
+            "SecondaryIPAddresses": null,
+            "SecondaryIPv6Addresses": null,
+            "EndpointID": "5c5ebda526d8fca90e841886ea81b77d7cc97fed56980c2aa89d275b84af7df2",
+            "Gateway": "172.17.0.1",
+            "GlobalIPv6Address": "",
+            "GlobalIPv6PrefixLen": 0,
+            "IPAddress": "172.17.0.2",
+            "IPPrefixLen": 16,
+            "IPv6Gateway": "",
+            "MacAddress": "32:b6:d9:ab:c3:61",
+            "Networks": {
+                "bridge": {
+                    "IPAMConfig": null,
+                    "Links": null,
+                    "Aliases": null,
+                    "MacAddress": "32:b6:d9:ab:c3:61",
+                    "DriverOpts": null,
+                    "GwPriority": 0,
+                    "NetworkID": "c4dd768ab4945e41ad23fe3907c960dac811141592a861cc40038df7086a1ce1",
+                    "EndpointID": "5c5ebda526d8fca90e841886ea81b77d7cc97fed56980c2aa89d275b84af7df2",
+                    "Gateway": "172.17.0.1",
+                    "IPAddress": "172.17.0.2",
+                    "IPPrefixLen": 16,
+                    "IPv6Gateway": "",
+                    "GlobalIPv6Address": "",
+                    "GlobalIPv6PrefixLen": 0,
+                    "DNSNames": null
+                }
+            }
+        }
+    }
+]

agent/agentcontainers/testdata/devcontainercli/parse/up-already-exists.log

@@ -0,0 +1,68 @@
+{"type":"text","level":3,"timestamp":1744102135254,"text":"@devcontainers/cli 0.75.0. Node.js v23.9.0. darwin 24.4.0 arm64."}
+{"type":"start","level":2,"timestamp":1744102135254,"text":"Run: docker buildx version"}
+{"type":"stop","level":2,"timestamp":1744102135300,"text":"Run: docker buildx version","startTimestamp":1744102135254}
+{"type":"text","level":2,"timestamp":1744102135300,"text":"github.com/docker/buildx v0.21.2 1360a9e8d25a2c3d03c2776d53ae62e6ff0a843d\r\n"}
+{"type":"text","level":2,"timestamp":1744102135300,"text":"\u001b[1m\u001b[31m\u001b[39m\u001b[22m\r\n"}
+{"type":"start","level":2,"timestamp":1744102135300,"text":"Run: docker -v"}
+{"type":"stop","level":2,"timestamp":1744102135309,"text":"Run: docker -v","startTimestamp":1744102135300}
+{"type":"start","level":2,"timestamp":1744102135309,"text":"Resolving Remote"}
+{"type":"start","level":2,"timestamp":1744102135311,"text":"Run: git rev-parse --show-cdup"}
+{"type":"stop","level":2,"timestamp":1744102135316,"text":"Run: git rev-parse --show-cdup","startTimestamp":1744102135311}
+{"type":"start","level":2,"timestamp":1744102135316,"text":"Run: docker ps -q -a --filter label=devcontainer.local_folder=/code/devcontainers-template-starter --filter label=devcontainer.config_file=/code/devcontainers-template-starter/.devcontainer/devcontainer.json"}
+{"type":"stop","level":2,"timestamp":1744102135333,"text":"Run: docker ps -q -a --filter label=devcontainer.local_folder=/code/devcontainers-template-starter --filter label=devcontainer.config_file=/code/devcontainers-template-starter/.devcontainer/devcontainer.json","startTimestamp":1744102135316}
+{"type":"start","level":2,"timestamp":1744102135333,"text":"Run: docker inspect --type container 4f22413fe134"}
+{"type":"stop","level":2,"timestamp":1744102135347,"text":"Run: docker inspect --type container 4f22413fe134","startTimestamp":1744102135333}
+{"type":"start","level":2,"timestamp":1744102135348,"text":"Run: docker ps -q -a --filter label=devcontainer.local_folder=/code/devcontainers-template-starter --filter label=devcontainer.config_file=/code/devcontainers-template-starter/.devcontainer/devcontainer.json"}
+{"type":"stop","level":2,"timestamp":1744102135364,"text":"Run: docker ps -q -a --filter label=devcontainer.local_folder=/code/devcontainers-template-starter --filter label=devcontainer.config_file=/code/devcontainers-template-starter/.devcontainer/devcontainer.json","startTimestamp":1744102135348}
+{"type":"start","level":2,"timestamp":1744102135364,"text":"Run: docker inspect --type container 4f22413fe134"}
+{"type":"stop","level":2,"timestamp":1744102135378,"text":"Run: docker inspect --type container 4f22413fe134","startTimestamp":1744102135364}
+{"type":"start","level":2,"timestamp":1744102135379,"text":"Inspecting container"}
+{"type":"start","level":2,"timestamp":1744102135379,"text":"Run: docker inspect --type container 4f22413fe13472200500a66ca057df5aafba6b45743afd499c3f26fc886eb236"}
+{"type":"stop","level":2,"timestamp":1744102135393,"text":"Run: docker inspect --type container 4f22413fe13472200500a66ca057df5aafba6b45743afd499c3f26fc886eb236","startTimestamp":1744102135379}
+{"type":"stop","level":2,"timestamp":1744102135393,"text":"Inspecting container","startTimestamp":1744102135379}
+{"type":"start","level":2,"timestamp":1744102135393,"text":"Run in container: /bin/sh"}
+{"type":"start","level":2,"timestamp":1744102135394,"text":"Run in container: uname -m"}
+{"type":"text","level":2,"timestamp":1744102135428,"text":"aarch64\n"}
+{"type":"text","level":2,"timestamp":1744102135428,"text":""}
+{"type":"stop","level":2,"timestamp":1744102135428,"text":"Run in container: uname -m","startTimestamp":1744102135394}
+{"type":"start","level":2,"timestamp":1744102135428,"text":"Run in container: (cat /etc/os-release || cat /usr/lib/os-release) 2>/dev/null"}
+{"type":"text","level":2,"timestamp":1744102135428,"text":"PRETTY_NAME=\"Debian GNU/Linux 11 (bullseye)\"\nNAME=\"Debian GNU/Linux\"\nVERSION_ID=\"11\"\nVERSION=\"11 (bullseye)\"\nVERSION_CODENAME=bullseye\nID=debian\nHOME_URL=\"https://www.debian.org/\"\nSUPPORT_URL=\"https://www.debian.org/support\"\nBUG_REPORT_URL=\"https://bugs.debian.org/\"\n"}
+{"type":"text","level":2,"timestamp":1744102135428,"text":""}
+{"type":"stop","level":2,"timestamp":1744102135428,"text":"Run in container: (cat /etc/os-release || cat /usr/lib/os-release) 2>/dev/null","startTimestamp":1744102135428}
+{"type":"start","level":2,"timestamp":1744102135429,"text":"Run in container:  (command -v getent >/dev/null 2>&1 && getent passwd 'node' || grep -E '^node|^[^:]*:[^:]*:node:' /etc/passwd || true)"}
+{"type":"stop","level":2,"timestamp":1744102135429,"text":"Run in container:  (command -v getent >/dev/null 2>&1 && getent passwd 'node' || grep -E '^node|^[^:]*:[^:]*:node:' /etc/passwd || true)","startTimestamp":1744102135429}
+{"type":"start","level":2,"timestamp":1744102135430,"text":"Run in container: test -f '/var/devcontainer/.patchEtcEnvironmentMarker'"}
+{"type":"text","level":2,"timestamp":1744102135430,"text":""}
+{"type":"text","level":2,"timestamp":1744102135430,"text":""}
+{"type":"stop","level":2,"timestamp":1744102135430,"text":"Run in container: test -f '/var/devcontainer/.patchEtcEnvironmentMarker'","startTimestamp":1744102135430}
+{"type":"start","level":2,"timestamp":1744102135430,"text":"Run in container: test -f '/var/devcontainer/.patchEtcProfileMarker'"}
+{"type":"text","level":2,"timestamp":1744102135430,"text":""}
+{"type":"text","level":2,"timestamp":1744102135430,"text":""}
+{"type":"stop","level":2,"timestamp":1744102135430,"text":"Run in container: test -f '/var/devcontainer/.patchEtcProfileMarker'","startTimestamp":1744102135430}
+{"type":"text","level":2,"timestamp":1744102135431,"text":"userEnvProbe: loginInteractiveShell (default)"}
+{"type":"text","level":1,"timestamp":1744102135431,"text":"LifecycleCommandExecutionMap: {\n    \"onCreateCommand\": [],\n    \"updateContentCommand\": [],\n    \"postCreateCommand\": [\n        {\n            \"origin\": \"devcontainer.json\",\n            \"command\": \"npm install -g @devcontainers/cli\"\n        }\n    ],\n    \"postStartCommand\": [],\n    \"postAttachCommand\": [],\n    \"initializeCommand\": []\n}"}
+{"type":"text","level":2,"timestamp":1744102135431,"text":"userEnvProbe: not found in cache"}
+{"type":"text","level":2,"timestamp":1744102135431,"text":"userEnvProbe shell: /bin/bash"}
+{"type":"start","level":2,"timestamp":1744102135431,"text":"Run in container: /bin/bash -lic echo -n 5805f204-cd2b-4911-8a88-96de28d5deb7; cat /proc/self/environ; echo -n 5805f204-cd2b-4911-8a88-96de28d5deb7"}
+{"type":"start","level":2,"timestamp":1744102135431,"text":"Run in container: mkdir -p '/home/node/.devcontainer' && CONTENT=\"$(cat '/home/node/.devcontainer/.onCreateCommandMarker' 2>/dev/null || echo ENOENT)\" && [ \"${CONTENT:-2025-04-07T09:21:41.201379807Z}\" != '2025-04-07T09:21:41.201379807Z' ] && echo '2025-04-07T09:21:41.201379807Z' > '/home/node/.devcontainer/.onCreateCommandMarker'"}
+{"type":"text","level":2,"timestamp":1744102135432,"text":""}
+{"type":"text","level":2,"timestamp":1744102135432,"text":""}
+{"type":"text","level":2,"timestamp":1744102135432,"text":"Exit code 1"}
+{"type":"stop","level":2,"timestamp":1744102135432,"text":"Run in container: mkdir -p '/home/node/.devcontainer' && CONTENT=\"$(cat '/home/node/.devcontainer/.onCreateCommandMarker' 2>/dev/null || echo ENOENT)\" && [ \"${CONTENT:-2025-04-07T09:21:41.201379807Z}\" != '2025-04-07T09:21:41.201379807Z' ] && echo '2025-04-07T09:21:41.201379807Z' > '/home/node/.devcontainer/.onCreateCommandMarker'","startTimestamp":1744102135431}
+{"type":"start","level":2,"timestamp":1744102135432,"text":"Run in container: mkdir -p '/home/node/.devcontainer' && CONTENT=\"$(cat '/home/node/.devcontainer/.updateContentCommandMarker' 2>/dev/null || echo ENOENT)\" && [ \"${CONTENT:-2025-04-07T09:21:41.201379807Z}\" != '2025-04-07T09:21:41.201379807Z' ] && echo '2025-04-07T09:21:41.201379807Z' > '/home/node/.devcontainer/.updateContentCommandMarker'"}
+{"type":"text","level":2,"timestamp":1744102135434,"text":""}
+{"type":"text","level":2,"timestamp":1744102135434,"text":""}
+{"type":"text","level":2,"timestamp":1744102135434,"text":"Exit code 1"}
+{"type":"stop","level":2,"timestamp":1744102135434,"text":"Run in container: mkdir -p '/home/node/.devcontainer' && CONTENT=\"$(cat '/home/node/.devcontainer/.updateContentCommandMarker' 2>/dev/null || echo ENOENT)\" && [ \"${CONTENT:-2025-04-07T09:21:41.201379807Z}\" != '2025-04-07T09:21:41.201379807Z' ] && echo '2025-04-07T09:21:41.201379807Z' > '/home/node/.devcontainer/.updateContentCommandMarker'","startTimestamp":1744102135432}
+{"type":"start","level":2,"timestamp":1744102135434,"text":"Run in container: mkdir -p '/home/node/.devcontainer' && CONTENT=\"$(cat '/home/node/.devcontainer/.postCreateCommandMarker' 2>/dev/null || echo ENOENT)\" && [ \"${CONTENT:-2025-04-07T09:21:41.201379807Z}\" != '2025-04-07T09:21:41.201379807Z' ] && echo '2025-04-07T09:21:41.201379807Z' > '/home/node/.devcontainer/.postCreateCommandMarker'"}
+{"type":"text","level":2,"timestamp":1744102135435,"text":""}
+{"type":"text","level":2,"timestamp":1744102135435,"text":""}
+{"type":"text","level":2,"timestamp":1744102135435,"text":"Exit code 1"}
+{"type":"stop","level":2,"timestamp":1744102135435,"text":"Run in container: mkdir -p '/home/node/.devcontainer' && CONTENT=\"$(cat '/home/node/.devcontainer/.postCreateCommandMarker' 2>/dev/null || echo ENOENT)\" && [ \"${CONTENT:-2025-04-07T09:21:41.201379807Z}\" != '2025-04-07T09:21:41.201379807Z' ] && echo '2025-04-07T09:21:41.201379807Z' > '/home/node/.devcontainer/.postCreateCommandMarker'","startTimestamp":1744102135434}
+{"type":"start","level":2,"timestamp":1744102135435,"text":"Run in container: mkdir -p '/home/node/.devcontainer' && CONTENT=\"$(cat '/home/node/.devcontainer/.postStartCommandMarker' 2>/dev/null || echo ENOENT)\" && [ \"${CONTENT:-2025-04-08T08:48:29.406495039Z}\" != '2025-04-08T08:48:29.406495039Z' ] && echo '2025-04-08T08:48:29.406495039Z' > '/home/node/.devcontainer/.postStartCommandMarker'"}
+{"type":"text","level":2,"timestamp":1744102135436,"text":""}
+{"type":"text","level":2,"timestamp":1744102135436,"text":""}
+{"type":"text","level":2,"timestamp":1744102135436,"text":"Exit code 1"}
+{"type":"stop","level":2,"timestamp":1744102135436,"text":"Run in container: mkdir -p '/home/node/.devcontainer' && CONTENT=\"$(cat '/home/node/.devcontainer/.postStartCommandMarker' 2>/dev/null || echo ENOENT)\" && [ \"${CONTENT:-2025-04-08T08:48:29.406495039Z}\" != '2025-04-08T08:48:29.406495039Z' ] && echo '2025-04-08T08:48:29.406495039Z' > '/home/node/.devcontainer/.postStartCommandMarker'","startTimestamp":1744102135435}
+{"type":"stop","level":2,"timestamp":1744102135436,"text":"Resolving Remote","startTimestamp":1744102135309}
+{"outcome":"success","containerId":"4f22413fe13472200500a66ca057df5aafba6b45743afd499c3f26fc886eb236","remoteUser":"node","remoteWorkspaceFolder":"/workspaces/devcontainers-template-starter"}

agent/agentcontainers/testdata/devcontainercli/parse/up-error-bad-outcome.log

@@ -0,0 +1 @@
+bad outcome

agent/agentcontainers/testdata/devcontainercli/parse/up-error-docker.log

@@ -0,0 +1,13 @@
+{"type":"text","level":3,"timestamp":1744102042893,"text":"@devcontainers/cli 0.75.0. Node.js v23.9.0. darwin 24.4.0 arm64."}
+{"type":"start","level":2,"timestamp":1744102042893,"text":"Run: docker buildx version"}
+{"type":"stop","level":2,"timestamp":1744102042941,"text":"Run: docker buildx version","startTimestamp":1744102042893}
+{"type":"text","level":2,"timestamp":1744102042941,"text":"github.com/docker/buildx v0.21.2 1360a9e8d25a2c3d03c2776d53ae62e6ff0a843d\r\n"}
+{"type":"text","level":2,"timestamp":1744102042941,"text":"\u001b[1m\u001b[31m\u001b[39m\u001b[22m\r\n"}
+{"type":"start","level":2,"timestamp":1744102042941,"text":"Run: docker -v"}
+{"type":"stop","level":2,"timestamp":1744102042950,"text":"Run: docker -v","startTimestamp":1744102042941}
+{"type":"start","level":2,"timestamp":1744102042950,"text":"Resolving Remote"}
+{"type":"start","level":2,"timestamp":1744102042952,"text":"Run: git rev-parse --show-cdup"}
+{"type":"stop","level":2,"timestamp":1744102042957,"text":"Run: git rev-parse --show-cdup","startTimestamp":1744102042952}
+{"type":"start","level":2,"timestamp":1744102042957,"text":"Run: docker ps -q -a --filter label=devcontainer.local_folder=/code/devcontainers-template-starter --filter label=devcontainer.config_file=/code/devcontainers-template-starter/.devcontainer/devcontainer.json"}
+{"type":"stop","level":2,"timestamp":1744102042967,"text":"Run: docker ps -q -a --filter label=devcontainer.local_folder=/code/devcontainers-template-starter --filter label=devcontainer.config_file=/code/devcontainers-template-starter/.devcontainer/devcontainer.json","startTimestamp":1744102042957}
+{"outcome":"error","message":"Command failed: docker ps -q -a --filter label=devcontainer.local_folder=/code/devcontainers-template-starter --filter label=devcontainer.config_file=/code/devcontainers-template-starter/.devcontainer/devcontainer.json","description":"An error occurred setting up the container."}

agent/agentcontainers/testdata/devcontainercli/parse/up-error-does-not-exist.log

@@ -0,0 +1,15 @@
+{"type":"text","level":3,"timestamp":1744102555495,"text":"@devcontainers/cli 0.75.0. Node.js v23.9.0. darwin 24.4.0 arm64."}
+{"type":"start","level":2,"timestamp":1744102555495,"text":"Run: docker buildx version"}
+{"type":"stop","level":2,"timestamp":1744102555539,"text":"Run: docker buildx version","startTimestamp":1744102555495}
+{"type":"text","level":2,"timestamp":1744102555539,"text":"github.com/docker/buildx v0.21.2 1360a9e8d25a2c3d03c2776d53ae62e6ff0a843d\r\n"}
+{"type":"text","level":2,"timestamp":1744102555539,"text":"\u001b[1m\u001b[31m\u001b[39m\u001b[22m\r\n"}
+{"type":"start","level":2,"timestamp":1744102555539,"text":"Run: docker -v"}
+{"type":"stop","level":2,"timestamp":1744102555548,"text":"Run: docker -v","startTimestamp":1744102555539}
+{"type":"start","level":2,"timestamp":1744102555548,"text":"Resolving Remote"}
+Error: Dev container config (/code/devcontainers-template-starter/foo/.devcontainer/devcontainer.json) not found.
+    at H6 (/opt/homebrew/Cellar/devcontainer/0.75.0/libexec/lib/node_modules/@devcontainers/cli/dist/spec-node/devContainersSpecCLI.js:484:3219)
+    at async BC (/opt/homebrew/Cellar/devcontainer/0.75.0/libexec/lib/node_modules/@devcontainers/cli/dist/spec-node/devContainersSpecCLI.js:484:4957)
+    at async d7 (/opt/homebrew/Cellar/devcontainer/0.75.0/libexec/lib/node_modules/@devcontainers/cli/dist/spec-node/devContainersSpecCLI.js:665:202)
+    at async f7 (/opt/homebrew/Cellar/devcontainer/0.75.0/libexec/lib/node_modules/@devcontainers/cli/dist/spec-node/devContainersSpecCLI.js:664:14804)
+    at async /opt/homebrew/Cellar/devcontainer/0.75.0/libexec/lib/node_modules/@devcontainers/cli/dist/spec-node/devContainersSpecCLI.js:484:1188
+{"outcome":"error","message":"Dev container config (/code/devcontainers-template-starter/foo/.devcontainer/devcontainer.json) not found.","description":"Dev container config (/code/devcontainers-template-starter/foo/.devcontainer/devcontainer.json) not found."}

agent/agentcontainers/watcher/noop.go

@@ -0,0 +1,48 @@
+package watcher
+
+import (
+	"context"
+	"sync"
+
+	"github.com/fsnotify/fsnotify"
+)
+
+// NewNoop creates a new watcher that does nothing.
+func NewNoop() Watcher {
+	return &noopWatcher{done: make(chan struct{})}
+}
+
+type noopWatcher struct {
+	mu     sync.Mutex
+	closed bool
+	done   chan struct{}
+}
+
+func (*noopWatcher) Add(string) error {
+	return nil
+}
+
+func (*noopWatcher) Remove(string) error {
+	return nil
+}
+
+// Next blocks until the context is canceled or the watcher is closed.
+func (n *noopWatcher) Next(ctx context.Context) (*fsnotify.Event, error) {
+	select {
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	case <-n.done:
+		return nil, ErrClosed
+	}
+}
+
+func (n *noopWatcher) Close() error {
+	n.mu.Lock()
+	defer n.mu.Unlock()
+	if n.closed {
+		return ErrClosed
+	}
+	n.closed = true
+	close(n.done)
+	return nil
+}

agent/agentcontainers/watcher/noop_test.go

@@ -0,0 +1,70 @@
+package watcher_test
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/coder/coder/v2/agent/agentcontainers/watcher"
+	"github.com/coder/coder/v2/testutil"
+)
+
+func TestNoopWatcher(t *testing.T) {
+	t.Parallel()
+
+	// Create the noop watcher under test.
+	wut := watcher.NewNoop()
+
+	// Test adding/removing files (should have no effect).
+	err := wut.Add("some-file.txt")
+	assert.NoError(t, err, "noop watcher should not return error on Add")
+
+	err = wut.Remove("some-file.txt")
+	assert.NoError(t, err, "noop watcher should not return error on Remove")
+
+	ctx, cancel := context.WithCancel(t.Context())
+	defer cancel()
+
+	// Start a goroutine to wait for Next to return.
+	errC := make(chan error, 1)
+	go func() {
+		_, err := wut.Next(ctx)
+		errC <- err
+	}()
+
+	select {
+	case <-errC:
+		require.Fail(t, "want Next to block")
+	default:
+	}
+
+	// Cancel the context and check that Next returns.
+	cancel()
+
+	select {
+	case err := <-errC:
+		assert.Error(t, err, "want Next error when context is canceled")
+	case <-time.After(testutil.WaitShort):
+		t.Fatal("want Next to return after context was canceled")
+	}
+
+	// Test Close.
+	err = wut.Close()
+	assert.NoError(t, err, "want no error on Close")
+}
+
+func TestNoopWatcher_CloseBeforeNext(t *testing.T) {
+	t.Parallel()
+
+	wut := watcher.NewNoop()
+
+	err := wut.Close()
+	require.NoError(t, err, "close watcher failed")
+
+	ctx := context.Background()
+	_, err = wut.Next(ctx)
+	assert.Error(t, err, "want Next to return error when watcher is closed")
+}

agent/agentcontainers/watcher/watcher.go

@@ -0,0 +1,195 @@
+// Package watcher provides file system watching capabilities for the
+// agent. It defines an interface for monitoring file changes and
+// implementations that can be used to detect when configuration files
+// are modified. This is primarily used to track changes to devcontainer
+// configuration files and notify users when containers need to be
+// recreated to apply the new configuration.
+package watcher
+
+import (
+	"context"
+	"path/filepath"
+	"sync"
+
+	"github.com/fsnotify/fsnotify"
+	"golang.org/x/xerrors"
+)
+
+var ErrClosed = xerrors.New("watcher closed")
+
+// Watcher defines an interface for monitoring file system changes.
+// Implementations track file modifications and provide an event stream
+// that clients can consume to react to changes.
+type Watcher interface {
+	// Add starts watching a file for changes.
+	Add(file string) error
+
+	// Remove stops watching a file for changes.
+	Remove(file string) error
+
+	// Next blocks until a file system event occurs or the context is canceled.
+	// It returns the next event or an error if the watcher encountered a problem.
+	Next(context.Context) (*fsnotify.Event, error)
+
+	// Close shuts down the watcher and releases any resources.
+	Close() error
+}
+
+type fsnotifyWatcher struct {
+	*fsnotify.Watcher
+
+	mu           sync.Mutex      // Protects following.
+	watchedFiles map[string]bool // Files being watched (absolute path -> bool).
+	watchedDirs  map[string]int  // Refcount of directories being watched (absolute path -> count).
+	closed       bool            // Protects closing of done.
+	done         chan struct{}
+}
+
+// NewFSNotify creates a new file system watcher that watches parent directories
+// instead of individual files for more reliable event detection.
+func NewFSNotify() (Watcher, error) {
+	w, err := fsnotify.NewWatcher()
+	if err != nil {
+		return nil, xerrors.Errorf("create fsnotify watcher: %w", err)
+	}
+	return &fsnotifyWatcher{
+		Watcher:      w,
+		done:         make(chan struct{}),
+		watchedFiles: make(map[string]bool),
+		watchedDirs:  make(map[string]int),
+	}, nil
+}
+
+func (f *fsnotifyWatcher) Add(file string) error {
+	absPath, err := filepath.Abs(file)
+	if err != nil {
+		return xerrors.Errorf("absolute path: %w", err)
+	}
+
+	dir := filepath.Dir(absPath)
+
+	f.mu.Lock()
+	defer f.mu.Unlock()
+
+	// Already watching this file.
+	if f.closed || f.watchedFiles[absPath] {
+		return nil
+	}
+
+	// Start watching the parent directory if not already watching.
+	if f.watchedDirs[dir] == 0 {
+		if err := f.Watcher.Add(dir); err != nil {
+			return xerrors.Errorf("add directory to watcher: %w", err)
+		}
+	}
+
+	// Increment the reference count for this directory.
+	f.watchedDirs[dir]++
+	// Mark this file as watched.
+	f.watchedFiles[absPath] = true
+
+	return nil
+}
+
+func (f *fsnotifyWatcher) Remove(file string) error {
+	absPath, err := filepath.Abs(file)
+	if err != nil {
+		return xerrors.Errorf("absolute path: %w", err)
+	}
+
+	dir := filepath.Dir(absPath)
+
+	f.mu.Lock()
+	defer f.mu.Unlock()
+
+	// Not watching this file.
+	if f.closed || !f.watchedFiles[absPath] {
+		return nil
+	}
+
+	// Remove the file from our watch list.
+	delete(f.watchedFiles, absPath)
+
+	// Decrement the reference count for this directory.
+	f.watchedDirs[dir]--
+
+	// If no more files in this directory are being watched, stop
+	// watching the directory.
+	if f.watchedDirs[dir] <= 0 {
+		f.watchedDirs[dir] = 0 // Ensure non-negative count.
+		if err := f.Watcher.Remove(dir); err != nil {
+			return xerrors.Errorf("remove directory from watcher: %w", err)
+		}
+		delete(f.watchedDirs, dir)
+	}
+
+	return nil
+}
+
+func (f *fsnotifyWatcher) Next(ctx context.Context) (event *fsnotify.Event, err error) {
+	defer func() {
+		if ctx.Err() != nil {
+			event = nil
+			err = ctx.Err()
+		}
+	}()
+
+	for {
+		select {
+		case <-ctx.Done():
+			return nil, ctx.Err()
+		case evt, ok := <-f.Events:
+			if !ok {
+				return nil, ErrClosed
+			}
+
+			// Get the absolute path to match against our watched files.
+			absPath, err := filepath.Abs(evt.Name)
+			if err != nil {
+				continue
+			}
+
+			f.mu.Lock()
+			if f.closed {
+				f.mu.Unlock()
+				return nil, ErrClosed
+			}
+			isWatched := f.watchedFiles[absPath]
+			f.mu.Unlock()
+			if !isWatched {
+				continue // Ignore events for files not being watched.
+			}
+
+			return &evt, nil
+
+		case err, ok := <-f.Errors:
+			if !ok {
+				return nil, ErrClosed
+			}
+			return nil, xerrors.Errorf("watcher error: %w", err)
+		case <-f.done:
+			return nil, ErrClosed
+		}
+	}
+}
+
+func (f *fsnotifyWatcher) Close() (err error) {
+	f.mu.Lock()
+	f.watchedFiles = nil
+	f.watchedDirs = nil
+	closed := f.closed
+	f.closed = true
+	f.mu.Unlock()
+
+	if closed {
+		return ErrClosed
+	}
+
+	close(f.done)
+
+	if err := f.Watcher.Close(); err != nil {
+		return xerrors.Errorf("close watcher: %w", err)
+	}
+
+	return nil
+}

agent/agentcontainers/watcher/watcher_test.go

@@ -0,0 +1,128 @@
+package watcher_test
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/fsnotify/fsnotify"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/coder/coder/v2/agent/agentcontainers/watcher"
+	"github.com/coder/coder/v2/testutil"
+)
+
+func TestFSNotifyWatcher(t *testing.T) {
+	t.Parallel()
+
+	// Create test files.
+	dir := t.TempDir()
+	testFile := filepath.Join(dir, "test.json")
+	err := os.WriteFile(testFile, []byte(`{"test": "initial"}`), 0o600)
+	require.NoError(t, err, "create test file failed")
+
+	// Create the watcher under test.
+	wut, err := watcher.NewFSNotify()
+	require.NoError(t, err, "create FSNotify watcher failed")
+	defer wut.Close()
+
+	// Add the test file to the watch list.
+	err = wut.Add(testFile)
+	require.NoError(t, err, "add file to watcher failed")
+
+	ctx := testutil.Context(t, testutil.WaitShort)
+
+	// Modify the test file to trigger an event.
+	err = os.WriteFile(testFile, []byte(`{"test": "modified"}`), 0o600)
+	require.NoError(t, err, "modify test file failed")
+
+	// Verify that we receive the event we want.
+	for {
+		event, err := wut.Next(ctx)
+		require.NoError(t, err, "next event failed")
+
+		require.NotNil(t, event, "want non-nil event")
+		if !event.Has(fsnotify.Write) {
+			t.Logf("Ignoring event: %s", event)
+			continue
+		}
+		require.Truef(t, event.Has(fsnotify.Write), "want write event: %s", event.String())
+		require.Equal(t, event.Name, testFile, "want event for test file")
+		break
+	}
+
+	// Rename the test file to trigger a rename event.
+	err = os.Rename(testFile, testFile+".bak")
+	require.NoError(t, err, "rename test file failed")
+
+	// Verify that we receive the event we want.
+	for {
+		event, err := wut.Next(ctx)
+		require.NoError(t, err, "next event failed")
+		require.NotNil(t, event, "want non-nil event")
+		if !event.Has(fsnotify.Rename) {
+			t.Logf("Ignoring event: %s", event)
+			continue
+		}
+		require.Truef(t, event.Has(fsnotify.Rename), "want rename event: %s", event.String())
+		require.Equal(t, event.Name, testFile, "want event for test file")
+		break
+	}
+
+	err = os.WriteFile(testFile, []byte(`{"test": "new"}`), 0o600)
+	require.NoError(t, err, "write new test file failed")
+
+	// Verify that we receive the event we want.
+	for {
+		event, err := wut.Next(ctx)
+		require.NoError(t, err, "next event failed")
+		require.NotNil(t, event, "want non-nil event")
+		if !event.Has(fsnotify.Create) {
+			t.Logf("Ignoring event: %s", event)
+			continue
+		}
+		require.Truef(t, event.Has(fsnotify.Create), "want create event: %s", event.String())
+		require.Equal(t, event.Name, testFile, "want event for test file")
+		break
+	}
+
+	err = os.WriteFile(testFile+".atomic", []byte(`{"test": "atomic"}`), 0o600)
+	require.NoError(t, err, "write new atomic test file failed")
+
+	err = os.Rename(testFile+".atomic", testFile)
+	require.NoError(t, err, "rename atomic test file failed")
+
+	// Verify that we receive the event we want.
+	for {
+		event, err := wut.Next(ctx)
+		require.NoError(t, err, "next event failed")
+		require.NotNil(t, event, "want non-nil event")
+		if !event.Has(fsnotify.Create) {
+			t.Logf("Ignoring event: %s", event)
+			continue
+		}
+		require.Truef(t, event.Has(fsnotify.Create), "want create event: %s", event.String())
+		require.Equal(t, event.Name, testFile, "want event for test file")
+		break
+	}
+
+	// Test removing the file from the watcher.
+	err = wut.Remove(testFile)
+	require.NoError(t, err, "remove file from watcher failed")
+}
+
+func TestFSNotifyWatcher_CloseBeforeNext(t *testing.T) {
+	t.Parallel()
+
+	wut, err := watcher.NewFSNotify()
+	require.NoError(t, err, "create FSNotify watcher failed")
+
+	err = wut.Close()
+	require.NoError(t, err, "close watcher failed")
+
+	ctx := context.Background()
+	_, err = wut.Next(ctx)
+	assert.Error(t, err, "want Next to return error when watcher is closed")
+}

agent/agentexec/cli_linux.go

@@ -17,6 +17,8 @@ import (
 	"golang.org/x/sys/unix"
 	"golang.org/x/xerrors"
 	"kernel.org/pub/linux/libs/security/libcap/cap"
+
+	"github.com/coder/coder/v2/agent/usershell"
 )
 
 // CLI runs the agent-exec command. It should only be called by the cli package.
@@ -114,7 +116,8 @@ func CLI() error {
 
 	// Remove environment variables specific to the agentexec command. This is
 	// especially important for environments that are attempting to develop Coder in Coder.
-	env := os.Environ()
+	ei := usershell.SystemEnvInfo{}
+	env := ei.Environ()
 	env = slices.DeleteFunc(env, func(e string) bool {
 		return strings.HasPrefix(e, EnvProcPrioMgmt) ||
 			strings.HasPrefix(e, EnvProcOOMScore) ||

agent/agentrsa/key.go

@@ -0,0 +1,87 @@
+package agentrsa
+
+import (
+	"crypto/rsa"
+	"math/big"
+	"math/rand"
+)
+
+// GenerateDeterministicKey generates an RSA private key deterministically based on the provided seed.
+// This function uses a deterministic random source to generate the primes p and q, ensuring that the
+// same seed will always produce the same private key. The generated key is 2048 bits in size.
+//
+// Reference: https://pkg.go.dev/crypto/rsa#GenerateKey
+func GenerateDeterministicKey(seed int64) *rsa.PrivateKey {
+	// Since the standard lib purposefully does not generate
+	// deterministic rsa keys, we need to do it ourselves.
+
+	// Create deterministic random source
+	// nolint: gosec
+	deterministicRand := rand.New(rand.NewSource(seed))
+
+	// Use fixed values for p and q based on the seed
+	p := big.NewInt(0)
+	q := big.NewInt(0)
+	e := big.NewInt(65537) // Standard RSA public exponent
+
+	for {
+		// Generate deterministic primes using the seeded random
+		// Each prime should be ~1024 bits to get a 2048-bit key
+		for {
+			p.SetBit(p, 1024, 1) // Ensure it's large enough
+			for i := range 1024 {
+				if deterministicRand.Int63()%2 == 1 {
+					p.SetBit(p, i, 1)
+				} else {
+					p.SetBit(p, i, 0)
+				}
+			}
+			p1 := new(big.Int).Sub(p, big.NewInt(1))
+			if p.ProbablyPrime(20) && new(big.Int).GCD(nil, nil, e, p1).Cmp(big.NewInt(1)) == 0 {
+				break
+			}
+		}
+
+		for {
+			q.SetBit(q, 1024, 1) // Ensure it's large enough
+			for i := range 1024 {
+				if deterministicRand.Int63()%2 == 1 {
+					q.SetBit(q, i, 1)
+				} else {
+					q.SetBit(q, i, 0)
+				}
+			}
+			q1 := new(big.Int).Sub(q, big.NewInt(1))
+			if q.ProbablyPrime(20) && p.Cmp(q) != 0 && new(big.Int).GCD(nil, nil, e, q1).Cmp(big.NewInt(1)) == 0 {
+				break
+			}
+		}
+
+		// Calculate phi = (p-1) * (q-1)
+		p1 := new(big.Int).Sub(p, big.NewInt(1))
+		q1 := new(big.Int).Sub(q, big.NewInt(1))
+		phi := new(big.Int).Mul(p1, q1)
+
+		// Calculate private exponent d
+		d := new(big.Int).ModInverse(e, phi)
+		if d != nil {
+			// Calculate n = p * q
+			n := new(big.Int).Mul(p, q)
+
+			// Create the private key
+			privateKey := &rsa.PrivateKey{
+				PublicKey: rsa.PublicKey{
+					N: n,
+					E: int(e.Int64()),
+				},
+				D:      d,
+				Primes: []*big.Int{p, q},
+			}
+
+			// Compute precomputed values
+			privateKey.Precompute()
+
+			return privateKey
+		}
+	}
+}

agent/agentrsa/key_test.go

@@ -0,0 +1,51 @@
+package agentrsa_test
+
+import (
+	"crypto/rsa"
+	"math/rand/v2"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/coder/coder/v2/agent/agentrsa"
+)
+
+func TestGenerateDeterministicKey(t *testing.T) {
+	t.Parallel()
+
+	key1 := agentrsa.GenerateDeterministicKey(1234)
+	key2 := agentrsa.GenerateDeterministicKey(1234)
+
+	assert.Equal(t, key1, key2)
+	assert.EqualExportedValues(t, key1, key2)
+}
+
+var result *rsa.PrivateKey
+
+func BenchmarkGenerateDeterministicKey(b *testing.B) {
+	var r *rsa.PrivateKey
+
+	for range b.N {
+		// always record the result of DeterministicPrivateKey to prevent
+		// the compiler eliminating the function call.
+		// #nosec G404 - Using math/rand is acceptable for benchmarking deterministic keys
+		r = agentrsa.GenerateDeterministicKey(rand.Int64())
+	}
+
+	// always store the result to a package level variable
+	// so the compiler cannot eliminate the Benchmark itself.
+	result = r
+}
+
+func FuzzGenerateDeterministicKey(f *testing.F) {
+	testcases := []int64{0, 1234, 1010101010}
+	for _, tc := range testcases {
+		f.Add(tc) // Use f.Add to provide a seed corpus
+	}
+	f.Fuzz(func(t *testing.T, seed int64) {
+		key1 := agentrsa.GenerateDeterministicKey(seed)
+		key2 := agentrsa.GenerateDeterministicKey(seed)
+		assert.Equal(t, key1, key2)
+		assert.EqualExportedValues(t, key1, key2)
+	})
+}

agent/agentscripts/agentscripts.go

@@ -10,7 +10,6 @@ import (
 	"os/user"
 	"path/filepath"
 	"sync"
-	"sync/atomic"
 	"time"
 
 	"github.com/google/uuid"
@@ -80,6 +79,21 @@ func New(opts Options) *Runner {
 
 type ScriptCompletedFunc func(context.Context, *proto.WorkspaceAgentScriptCompletedRequest) (*proto.WorkspaceAgentScriptCompletedResponse, error)
 
+type runnerScript struct {
+	runOnPostStart bool
+	codersdk.WorkspaceAgentScript
+}
+
+func toRunnerScript(scripts ...codersdk.WorkspaceAgentScript) []runnerScript {
+	var rs []runnerScript
+	for _, s := range scripts {
+		rs = append(rs, runnerScript{
+			WorkspaceAgentScript: s,
+		})
+	}
+	return rs
+}
+
 type Runner struct {
 	Options
 
@@ -89,8 +103,7 @@ type Runner struct {
 	closed          chan struct{}
 	closeMutex      sync.Mutex
 	cron            *cron.Cron
-	initialized     atomic.Bool
-	scripts         []codersdk.WorkspaceAgentScript
+	scripts         []runnerScript
 	dataDir         string
 	scriptCompleted ScriptCompletedFunc
 
@@ -98,6 +111,9 @@ type Runner struct {
 	// execute startup scripts, and scripts on a cron schedule. Both will increment
 	// this counter.
 	scriptsExecuted *prometheus.CounterVec
+
+	initMutex   sync.Mutex
+	initialized bool
 }
 
 // DataDir returns the directory where scripts data is stored.
@@ -119,16 +135,37 @@ func (r *Runner) RegisterMetrics(reg prometheus.Registerer) {
 	reg.MustRegister(r.scriptsExecuted)
 }
 
+// InitOption describes an option for the runner initialization.
+type InitOption func(*Runner)
+
+// WithPostStartScripts adds scripts that should be run after the workspace
+// start scripts but before the workspace is marked as started.
+func WithPostStartScripts(scripts ...codersdk.WorkspaceAgentScript) InitOption {
+	return func(r *Runner) {
+		for _, s := range scripts {
+			r.scripts = append(r.scripts, runnerScript{
+				runOnPostStart:       true,
+				WorkspaceAgentScript: s,
+			})
+		}
+	}
+}
+
 // Init initializes the runner with the provided scripts.
 // It also schedules any scripts that have a schedule.
 // This function must be called before Execute.
-func (r *Runner) Init(scripts []codersdk.WorkspaceAgentScript, scriptCompleted ScriptCompletedFunc) error {
-	if r.initialized.Load() {
+func (r *Runner) Init(scripts []codersdk.WorkspaceAgentScript, scriptCompleted ScriptCompletedFunc, opts ...InitOption) error {
+	r.initMutex.Lock()
+	defer r.initMutex.Unlock()
+	if r.initialized {
 		return xerrors.New("init: already initialized")
 	}
-	r.initialized.Store(true)
-	r.scripts = scripts
+	r.initialized = true
+	r.scripts = toRunnerScript(scripts...)
 	r.scriptCompleted = scriptCompleted
+	for _, opt := range opts {
+		opt(r)
+	}
 	r.Logger.Info(r.cronCtx, "initializing agent scripts", slog.F("script_count", len(scripts)), slog.F("log_dir", r.LogDir))
 
 	err := r.Filesystem.MkdirAll(r.ScriptBinDir(), 0o700)
@@ -136,13 +173,13 @@ func (r *Runner) Init(scripts []codersdk.WorkspaceAgentScript, scriptCompleted S
 		return xerrors.Errorf("create script bin dir: %w", err)
 	}
 
-	for _, script := range scripts {
+	for _, script := range r.scripts {
 		if script.Cron == "" {
 			continue
 		}
 		script := script
 		_, err := r.cron.AddFunc(script.Cron, func() {
-			err := r.trackRun(r.cronCtx, script, ExecuteCronScripts)
+			err := r.trackRun(r.cronCtx, script.WorkspaceAgentScript, ExecuteCronScripts)
 			if err != nil {
 				r.Logger.Warn(context.Background(), "run agent script on schedule", slog.Error(err))
 			}
@@ -186,16 +223,30 @@ type ExecuteOption int
 const (
 	ExecuteAllScripts ExecuteOption = iota
 	ExecuteStartScripts
+	ExecutePostStartScripts
 	ExecuteStopScripts
 	ExecuteCronScripts
 )
 
 // Execute runs a set of scripts according to a filter.
 func (r *Runner) Execute(ctx context.Context, option ExecuteOption) error {
+	initErr := func() error {
+		r.initMutex.Lock()
+		defer r.initMutex.Unlock()
+		if !r.initialized {
+			return xerrors.New("execute: not initialized")
+		}
+		return nil
+	}()
+	if initErr != nil {
+		return initErr
+	}
+
 	var eg errgroup.Group
 	for _, script := range r.scripts {
 		runScript := (option == ExecuteStartScripts && script.RunOnStart) ||
 			(option == ExecuteStopScripts && script.RunOnStop) ||
+			(option == ExecutePostStartScripts && script.runOnPostStart) ||
 			(option == ExecuteCronScripts && script.Cron != "") ||
 			option == ExecuteAllScripts
 
@@ -205,7 +256,7 @@ func (r *Runner) Execute(ctx context.Context, option ExecuteOption) error {
 
 		script := script
 		eg.Go(func() error {
-			err := r.trackRun(ctx, script, option)
+			err := r.trackRun(ctx, script.WorkspaceAgentScript, option)
 			if err != nil {
 				return xerrors.Errorf("run agent script %q: %w", script.LogSourceID, err)
 			}

agent/agentscripts/agentscripts_test.go

@@ -4,6 +4,8 @@ import (
 	"context"
 	"path/filepath"
 	"runtime"
+	"slices"
+	"sync"
 	"testing"
 	"time"
 
@@ -42,7 +44,7 @@ func TestExecuteBasic(t *testing.T) {
 	}}, aAPI.ScriptCompleted)
 	require.NoError(t, err)
 	require.NoError(t, runner.Execute(context.Background(), agentscripts.ExecuteAllScripts))
-	log := testutil.RequireRecvCtx(ctx, t, fLogger.logs)
+	log := testutil.TryReceive(ctx, t, fLogger.logs)
 	require.Equal(t, "hello", log.Output)
 }
 
@@ -100,13 +102,16 @@ func TestEnv(t *testing.T) {
 
 func TestTimeout(t *testing.T) {
 	t.Parallel()
+	if runtime.GOOS == "darwin" {
+		t.Skip("this test is flaky on macOS, see https://github.com/coder/internal/issues/329")
+	}
 	runner := setup(t, nil)
 	defer runner.Close()
 	aAPI := agenttest.NewFakeAgentAPI(t, testutil.Logger(t), nil, nil)
 	err := runner.Init([]codersdk.WorkspaceAgentScript{{
 		LogSourceID: uuid.New(),
 		Script:      "sleep infinity",
-		Timeout:     time.Millisecond,
+		Timeout:     100 * time.Millisecond,
 	}}, aAPI.ScriptCompleted)
 	require.NoError(t, err)
 	require.ErrorIs(t, runner.Execute(context.Background(), agentscripts.ExecuteAllScripts), agentscripts.ErrTimeout)
@@ -131,7 +136,7 @@ func TestScriptReportsTiming(t *testing.T) {
 	require.NoError(t, runner.Execute(ctx, agentscripts.ExecuteAllScripts))
 	runner.Close()
 
-	log := testutil.RequireRecvCtx(ctx, t, fLogger.logs)
+	log := testutil.TryReceive(ctx, t, fLogger.logs)
 	require.Equal(t, "hello", log.Output)
 
 	timings := aAPI.GetTimings()
@@ -151,11 +156,160 @@ func TestCronClose(t *testing.T) {
 	require.NoError(t, runner.Close(), "close runner")
 }
 
+func TestExecuteOptions(t *testing.T) {
+	t.Parallel()
+
+	startScript := codersdk.WorkspaceAgentScript{
+		ID:          uuid.New(),
+		LogSourceID: uuid.New(),
+		Script:      "echo start",
+		RunOnStart:  true,
+	}
+	stopScript := codersdk.WorkspaceAgentScript{
+		ID:          uuid.New(),
+		LogSourceID: uuid.New(),
+		Script:      "echo stop",
+		RunOnStop:   true,
+	}
+	postStartScript := codersdk.WorkspaceAgentScript{
+		ID:          uuid.New(),
+		LogSourceID: uuid.New(),
+		Script:      "echo poststart",
+	}
+	regularScript := codersdk.WorkspaceAgentScript{
+		ID:          uuid.New(),
+		LogSourceID: uuid.New(),
+		Script:      "echo regular",
+	}
+
+	scripts := []codersdk.WorkspaceAgentScript{
+		startScript,
+		stopScript,
+		regularScript,
+	}
+	allScripts := append(slices.Clone(scripts), postStartScript)
+
+	scriptByID := func(t *testing.T, id uuid.UUID) codersdk.WorkspaceAgentScript {
+		for _, script := range allScripts {
+			if script.ID == id {
+				return script
+			}
+		}
+		t.Fatal("script not found")
+		return codersdk.WorkspaceAgentScript{}
+	}
+
+	wantOutput := map[uuid.UUID]string{
+		startScript.ID:     "start",
+		stopScript.ID:      "stop",
+		postStartScript.ID: "poststart",
+		regularScript.ID:   "regular",
+	}
+
+	testCases := []struct {
+		name    string
+		option  agentscripts.ExecuteOption
+		wantRun []uuid.UUID
+	}{
+		{
+			name:    "ExecuteAllScripts",
+			option:  agentscripts.ExecuteAllScripts,
+			wantRun: []uuid.UUID{startScript.ID, stopScript.ID, regularScript.ID, postStartScript.ID},
+		},
+		{
+			name:    "ExecuteStartScripts",
+			option:  agentscripts.ExecuteStartScripts,
+			wantRun: []uuid.UUID{startScript.ID},
+		},
+		{
+			name:    "ExecutePostStartScripts",
+			option:  agentscripts.ExecutePostStartScripts,
+			wantRun: []uuid.UUID{postStartScript.ID},
+		},
+		{
+			name:    "ExecuteStopScripts",
+			option:  agentscripts.ExecuteStopScripts,
+			wantRun: []uuid.UUID{stopScript.ID},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+
+			ctx := testutil.Context(t, testutil.WaitMedium)
+			executedScripts := make(map[uuid.UUID]bool)
+			fLogger := &executeOptionTestLogger{
+				tb:              t,
+				executedScripts: executedScripts,
+				wantOutput:      wantOutput,
+			}
+
+			runner := setup(t, func(uuid.UUID) agentscripts.ScriptLogger {
+				return fLogger
+			})
+			defer runner.Close()
+
+			aAPI := agenttest.NewFakeAgentAPI(t, testutil.Logger(t), nil, nil)
+			err := runner.Init(
+				scripts,
+				aAPI.ScriptCompleted,
+				agentscripts.WithPostStartScripts(postStartScript),
+			)
+			require.NoError(t, err)
+
+			err = runner.Execute(ctx, tc.option)
+			require.NoError(t, err)
+
+			gotRun := map[uuid.UUID]bool{}
+			for _, id := range tc.wantRun {
+				gotRun[id] = true
+				require.True(t, executedScripts[id],
+					"script %s should have run when using filter %s", scriptByID(t, id).Script, tc.name)
+			}
+
+			for _, script := range allScripts {
+				if _, ok := gotRun[script.ID]; ok {
+					continue
+				}
+				require.False(t, executedScripts[script.ID],
+					"script %s should not have run when using filter %s", script.Script, tc.name)
+			}
+		})
+	}
+}
+
+type executeOptionTestLogger struct {
+	tb              testing.TB
+	executedScripts map[uuid.UUID]bool
+	wantOutput      map[uuid.UUID]string
+	mu              sync.Mutex
+}
+
+func (l *executeOptionTestLogger) Send(_ context.Context, logs ...agentsdk.Log) error {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	for _, log := range logs {
+		l.tb.Log(log.Output)
+		for id, output := range l.wantOutput {
+			if log.Output == output {
+				l.executedScripts[id] = true
+				break
+			}
+		}
+	}
+	return nil
+}
+
+func (*executeOptionTestLogger) Flush(context.Context) error {
+	return nil
+}
+
 func setup(t *testing.T, getScriptLogger func(logSourceID uuid.UUID) agentscripts.ScriptLogger) *agentscripts.Runner {
 	t.Helper()
 	if getScriptLogger == nil {
 		// noop
-		getScriptLogger = func(uuid uuid.UUID) agentscripts.ScriptLogger {
+		getScriptLogger = func(uuid.UUID) agentscripts.ScriptLogger {
 			return noopScriptLogger{}
 		}
 	}

agent/agentssh/agentssh.go

@@ -3,18 +3,16 @@ package agentssh
 import (
 	"bufio"
 	"context"
-	"crypto/rsa"
 	"errors"
 	"fmt"
 	"io"
-	"math/big"
-	"math/rand"
 	"net"
 	"os"
 	"os/exec"
 	"os/user"
 	"path/filepath"
 	"runtime"
+	"slices"
 	"strings"
 	"sync"
 	"time"
@@ -27,12 +25,13 @@ import (
 	"github.com/spf13/afero"
 	"go.uber.org/atomic"
 	gossh "golang.org/x/crypto/ssh"
-	"golang.org/x/exp/slices"
 	"golang.org/x/xerrors"
 
 	"cdr.dev/slog"
 
+	"github.com/coder/coder/v2/agent/agentcontainers"
 	"github.com/coder/coder/v2/agent/agentexec"
+	"github.com/coder/coder/v2/agent/agentrsa"
 	"github.com/coder/coder/v2/agent/usershell"
 	"github.com/coder/coder/v2/codersdk"
 	"github.com/coder/coder/v2/pty"
@@ -62,6 +61,14 @@ const (
 	// MagicSessionTypeEnvironmentVariable is used to track the purpose behind an SSH connection.
 	// This is stripped from any commands being executed, and is counted towards connection stats.
 	MagicSessionTypeEnvironmentVariable = "CODER_SSH_SESSION_TYPE"
+	// ContainerEnvironmentVariable is used to specify the target container for an SSH connection.
+	// This is stripped from any commands being executed.
+	// Only available if CODER_AGENT_DEVCONTAINERS_ENABLE=true.
+	ContainerEnvironmentVariable = "CODER_CONTAINER"
+	// ContainerUserEnvironmentVariable is used to specify the container user for
+	// an SSH connection.
+	// Only available if CODER_AGENT_DEVCONTAINERS_ENABLE=true.
+	ContainerUserEnvironmentVariable = "CODER_CONTAINER_USER"
 )
 
 // MagicSessionType enums.
@@ -80,6 +87,8 @@ const (
 // BlockedFileTransferCommands contains a list of restricted file transfer commands.
 var BlockedFileTransferCommands = []string{"nc", "rsync", "scp", "sftp"}
 
+type reportConnectionFunc func(id uuid.UUID, sessionType MagicSessionType, ip string) (disconnected func(code int, reason string))
+
 // Config sets configuration parameters for the agent SSH server.
 type Config struct {
 	// MaxTimeout sets the absolute connection timeout, none if empty. If set to
@@ -102,6 +111,11 @@ type Config struct {
 	X11DisplayOffset *int
 	// BlockFileTransfer restricts use of file transfer applications.
 	BlockFileTransfer bool
+	// ReportConnection.
+	ReportConnection reportConnectionFunc
+	// Experimental: allow connecting to running containers if
+	// CODER_AGENT_DEVCONTAINERS_ENABLE=true.
+	ExperimentalDevContainersEnabled bool
 }
 
 type Server struct {
@@ -110,6 +124,7 @@ type Server struct {
 	listeners map[net.Listener]struct{}
 	conns     map[net.Conn]struct{}
 	sessions  map[ssh.Session]struct{}
+	processes map[*os.Process]struct{}
 	closing   chan struct{}
 	// Wait for goroutines to exit, waited without
 	// a lock on mu but protected by closing.
@@ -154,6 +169,9 @@ func NewServer(ctx context.Context, logger slog.Logger, prometheusRegistry *prom
 			return home
 		}
 	}
+	if config.ReportConnection == nil {
+		config.ReportConnection = func(uuid.UUID, MagicSessionType, string) func(int, string) { return func(int, string) {} }
+	}
 
 	forwardHandler := &ssh.ForwardedTCPHandler{}
 	unixForwardHandler := newForwardedUnixHandler(logger)
@@ -165,6 +183,7 @@ func NewServer(ctx context.Context, logger slog.Logger, prometheusRegistry *prom
 		fs:        fs,
 		conns:     make(map[net.Conn]struct{}),
 		sessions:  make(map[ssh.Session]struct{}),
+		processes: make(map[*os.Process]struct{}),
 		logger:    logger,
 
 		config: config,
@@ -176,7 +195,7 @@ func NewServer(ctx context.Context, logger slog.Logger, prometheusRegistry *prom
 		ChannelHandlers: map[string]ssh.ChannelHandler{
 			"direct-tcpip": func(srv *ssh.Server, conn *gossh.ServerConn, newChan gossh.NewChannel, ctx ssh.Context) {
 				// Wrapper is designed to find and track JetBrains Gateway connections.
-				wrapped := NewJetbrainsChannelWatcher(ctx, s.logger, newChan, &s.connCountJetBrains)
+				wrapped := NewJetbrainsChannelWatcher(ctx, s.logger, s.config.ReportConnection, newChan, &s.connCountJetBrains)
 				ssh.DirectTCPIPHandler(srv, conn, wrapped, ctx)
 			},
 			"direct-streamlocal@openssh.com": directStreamLocalHandler,
@@ -206,7 +225,7 @@ func NewServer(ctx context.Context, logger slog.Logger, prometheusRegistry *prom
 				slog.F("destination_port", destinationPort))
 			return true
 		},
-		PtyCallback: func(ctx ssh.Context, pty ssh.Pty) bool {
+		PtyCallback: func(_ ssh.Context, _ ssh.Pty) bool {
 			return true
 		},
 		ReversePortForwardingCallback: func(ctx ssh.Context, bindHost string, bindPort uint32) bool {
@@ -223,7 +242,7 @@ func NewServer(ctx context.Context, logger slog.Logger, prometheusRegistry *prom
 			"cancel-streamlocal-forward@openssh.com": unixForwardHandler.HandleSSHRequest,
 		},
 		X11Callback: s.x11Callback,
-		ServerConfigCallback: func(ctx ssh.Context) *gossh.ServerConfig {
+		ServerConfigCallback: func(_ ssh.Context) *gossh.ServerConfig {
 			return &gossh.ServerConfig{
 				NoClientAuth: true,
 			}
@@ -290,6 +309,51 @@ func extractMagicSessionType(env []string) (magicType MagicSessionType, rawType
 	})
 }
 
+// sessionCloseTracker is a wrapper around Session that tracks the exit code.
+type sessionCloseTracker struct {
+	ssh.Session
+	exitOnce sync.Once
+	code     atomic.Int64
+}
+
+var _ ssh.Session = &sessionCloseTracker{}
+
+func (s *sessionCloseTracker) track(code int) {
+	s.exitOnce.Do(func() {
+		s.code.Store(int64(code))
+	})
+}
+
+func (s *sessionCloseTracker) exitCode() int {
+	return int(s.code.Load())
+}
+
+func (s *sessionCloseTracker) Exit(code int) error {
+	s.track(code)
+	return s.Session.Exit(code)
+}
+
+func (s *sessionCloseTracker) Close() error {
+	s.track(1)
+	return s.Session.Close()
+}
+
+func extractContainerInfo(env []string) (container, containerUser string, filteredEnv []string) {
+	for _, kv := range env {
+		if strings.HasPrefix(kv, ContainerEnvironmentVariable+"=") {
+			container = strings.TrimPrefix(kv, ContainerEnvironmentVariable+"=")
+		}
+
+		if strings.HasPrefix(kv, ContainerUserEnvironmentVariable+"=") {
+			containerUser = strings.TrimPrefix(kv, ContainerUserEnvironmentVariable+"=")
+		}
+	}
+
+	return container, containerUser, slices.DeleteFunc(env, func(kv string) bool {
+		return strings.HasPrefix(kv, ContainerEnvironmentVariable+"=") || strings.HasPrefix(kv, ContainerUserEnvironmentVariable+"=")
+	})
+}
+
 func (s *Server) sessionHandler(session ssh.Session) {
 	ctx := session.Context()
 	id := uuid.New()
@@ -302,16 +366,23 @@ func (s *Server) sessionHandler(session ssh.Session) {
 	)
 	logger.Info(ctx, "handling ssh session")
 
+	env := session.Environ()
+	magicType, magicTypeRaw, env := extractMagicSessionType(env)
+
 	if !s.trackSession(session, true) {
+		reason := "unable to accept new session, server is closing"
+		// Report connection attempt even if we couldn't accept it.
+		disconnected := s.config.ReportConnection(id, magicType, session.RemoteAddr().String())
+		defer disconnected(1, reason)
+
+		logger.Info(ctx, reason)
 		// See (*Server).Close() for why we call Close instead of Exit.
 		_ = session.Close()
-		logger.Info(ctx, "unable to accept new session, server is closing")
 		return
 	}
 	defer s.trackSession(session, false)
 
-	env := session.Environ()
-	magicType, magicTypeRaw, env := extractMagicSessionType(env)
+	reportSession := true
 
 	switch magicType {
 	case MagicSessionTypeVSCode:
@@ -320,6 +391,7 @@ func (s *Server) sessionHandler(session ssh.Session) {
 	case MagicSessionTypeJetBrains:
 		// Do nothing here because JetBrains launches hundreds of ssh sessions.
 		// We instead track JetBrains in the single persistent tcp forwarding channel.
+		reportSession = false
 	case MagicSessionTypeSSH:
 		s.connCountSSHSession.Add(1)
 		defer s.connCountSSHSession.Add(-1)
@@ -327,6 +399,20 @@ func (s *Server) sessionHandler(session ssh.Session) {
 		logger.Warn(ctx, "invalid magic ssh session type specified", slog.F("raw_type", magicTypeRaw))
 	}
 
+	closeCause := func(string) {}
+	if reportSession {
+		var reason string
+		closeCause = func(r string) { reason = r }
+
+		scr := &sessionCloseTracker{Session: session}
+		session = scr
+
+		disconnected := s.config.ReportConnection(id, magicType, session.RemoteAddr().String())
+		defer func() {
+			disconnected(scr.exitCode(), reason)
+		}()
+	}
+
 	if s.fileTransferBlocked(session) {
 		s.logger.Warn(ctx, "file transfer blocked", slog.F("session_subsystem", session.Subsystem()), slog.F("raw_command", session.RawCommand()))
 
@@ -335,17 +421,35 @@ func (s *Server) sessionHandler(session ssh.Session) {
 			errorMessage := fmt.Sprintf("\x02%s\n", BlockedFileTransferErrorMessage)
 			_, _ = session.Write([]byte(errorMessage))
 		}
+		closeCause("file transfer blocked")
 		_ = session.Exit(BlockedFileTransferErrorCode)
 		return
 	}
 
+	container, containerUser, env := extractContainerInfo(env)
+	if container != "" {
+		s.logger.Debug(ctx, "container info",
+			slog.F("container", container),
+			slog.F("container_user", containerUser),
+		)
+	}
+
 	switch ss := session.Subsystem(); ss {
 	case "":
 	case "sftp":
-		s.sftpHandler(logger, session)
+		if s.config.ExperimentalDevContainersEnabled && container != "" {
+			closeCause("sftp not yet supported with containers")
+			_ = session.Exit(1)
+			return
+		}
+		err := s.sftpHandler(logger, session)
+		if err != nil {
+			closeCause(err.Error())
+		}
 		return
 	default:
 		logger.Warn(ctx, "unsupported subsystem", slog.F("subsystem", ss))
+		closeCause(fmt.Sprintf("unsupported subsystem: %s", ss))
 		_ = session.Exit(1)
 		return
 	}
@@ -354,14 +458,15 @@ func (s *Server) sessionHandler(session ssh.Session) {
 	if hasX11 {
 		display, handled := s.x11Handler(session.Context(), x11)
 		if !handled {
-			_ = session.Exit(1)
 			logger.Error(ctx, "x11 handler failed")
+			closeCause("x11 handler failed")
+			_ = session.Exit(1)
 			return
 		}
 		env = append(env, fmt.Sprintf("DISPLAY=localhost:%d.%d", display, x11.ScreenNumber))
 	}
 
-	err := s.sessionStart(logger, session, env, magicType)
+	err := s.sessionStart(logger, session, env, magicType, container, containerUser)
 	var exitError *exec.ExitError
 	if xerrors.As(err, &exitError) {
 		code := exitError.ExitCode()
@@ -382,6 +487,8 @@ func (s *Server) sessionHandler(session ssh.Session) {
 			slog.F("exit_code", code),
 		)
 
+		closeCause(fmt.Sprintf("process exited with error status: %d", exitError.ExitCode()))
+
 		// TODO(mafredri): For signal exit, there's also an "exit-signal"
 		// request (session.Exit sends "exit-status"), however, since it's
 		// not implemented on the session interface and not used by
@@ -393,6 +500,7 @@ func (s *Server) sessionHandler(session ssh.Session) {
 		logger.Warn(ctx, "ssh session failed", slog.Error(err))
 		// This exit code is designed to be unlikely to be confused for a legit exit code
 		// from the process.
+		closeCause(err.Error())
 		_ = session.Exit(MagicSessionErrorCode)
 		return
 	}
@@ -431,18 +539,27 @@ func (s *Server) fileTransferBlocked(session ssh.Session) bool {
 	return false
 }
 
-func (s *Server) sessionStart(logger slog.Logger, session ssh.Session, env []string, magicType MagicSessionType) (retErr error) {
+func (s *Server) sessionStart(logger slog.Logger, session ssh.Session, env []string, magicType MagicSessionType, container, containerUser string) (retErr error) {
 	ctx := session.Context()
 
 	magicTypeLabel := magicTypeMetricLabel(magicType)
 	sshPty, windowSize, isPty := session.Pty()
+	ptyLabel := "no"
+	if isPty {
+		ptyLabel = "yes"
+	}
 
-	cmd, err := s.CreateCommand(ctx, session.RawCommand(), env, nil)
-	if err != nil {
-		ptyLabel := "no"
-		if isPty {
-			ptyLabel = "yes"
+	var ei usershell.EnvInfoer
+	var err error
+	if s.config.ExperimentalDevContainersEnabled && container != "" {
+		ei, err = agentcontainers.EnvInfo(ctx, s.Execer, container, containerUser)
+		if err != nil {
+			s.metrics.sessionErrors.WithLabelValues(magicTypeLabel, ptyLabel, "container_env_info").Add(1)
+			return err
 		}
+	}
+	cmd, err := s.CreateCommand(ctx, session.RawCommand(), env, ei)
+	if err != nil {
 		s.metrics.sessionErrors.WithLabelValues(magicTypeLabel, ptyLabel, "create_command").Add(1)
 		return err
 	}
@@ -450,11 +567,6 @@ func (s *Server) sessionStart(logger slog.Logger, session ssh.Session, env []str
 	if ssh.AgentRequested(session) {
 		l, err := ssh.NewAgentListener()
 		if err != nil {
-			ptyLabel := "no"
-			if isPty {
-				ptyLabel = "yes"
-			}
-
 			s.metrics.sessionErrors.WithLabelValues(magicTypeLabel, ptyLabel, "listener").Add(1)
 			return xerrors.Errorf("new agent listener: %w", err)
 		}
@@ -472,6 +584,15 @@ func (s *Server) sessionStart(logger slog.Logger, session ssh.Session, env []str
 func (s *Server) startNonPTYSession(logger slog.Logger, session ssh.Session, magicTypeLabel string, cmd *exec.Cmd) error {
 	s.metrics.sessionsTotal.WithLabelValues(magicTypeLabel, "no").Add(1)
 
+	// Create a process group and send SIGHUP to child processes,
+	// otherwise context cancellation will not propagate properly
+	// and SSH server close may be delayed.
+	cmd.SysProcAttr = cmdSysProcAttr()
+
+	// to match OpenSSH, we don't actually tear a non-TTY command down, even if the session ends.
+	// c.f. https://github.com/coder/coder/issues/18519#issuecomment-3019118271
+	cmd.Cancel = nil
+
 	cmd.Stdout = session
 	cmd.Stderr = session.Stderr()
 	// This blocks forever until stdin is received if we don't
@@ -493,6 +614,16 @@ func (s *Server) startNonPTYSession(logger slog.Logger, session ssh.Session, mag
 		s.metrics.sessionErrors.WithLabelValues(magicTypeLabel, "no", "start_command").Add(1)
 		return xerrors.Errorf("start: %w", err)
 	}
+
+	// Since we don't cancel the process when the session stops, we still need to tear it down if we are closing. So
+	// track it here.
+	if !s.trackProcess(cmd.Process, true) {
+		// must be closing
+		err = cmdCancel(logger, cmd.Process)
+		return xerrors.Errorf("failed to track process: %w", err)
+	}
+	defer s.trackProcess(cmd.Process, false)
+
 	sigs := make(chan ssh.Signal, 1)
 	session.Signals(sigs)
 	defer func() {
@@ -592,6 +723,7 @@ func (s *Server) startPTYSession(logger slog.Logger, session ptySession, magicTy
 					windowSize = nil
 					continue
 				}
+				// #nosec G115 - Safe conversions for terminal dimensions which are expected to be within uint16 range
 				resizeErr := ptty.Resize(uint16(win.Height), uint16(win.Width))
 				// If the pty is closed, then command has exited, no need to log.
 				if resizeErr != nil && !errors.Is(resizeErr, pty.ErrClosed) {
@@ -652,7 +784,7 @@ func handleSignal(logger slog.Logger, ssig ssh.Signal, signaler interface{ Signa
 	}
 }
 
-func (s *Server) sftpHandler(logger slog.Logger, session ssh.Session) {
+func (s *Server) sftpHandler(logger slog.Logger, session ssh.Session) error {
 	s.metrics.sftpConnectionsTotal.Add(1)
 
 	ctx := session.Context()
@@ -676,7 +808,7 @@ func (s *Server) sftpHandler(logger slog.Logger, session ssh.Session) {
 	server, err := sftp.NewServer(session, opts...)
 	if err != nil {
 		logger.Debug(ctx, "initialize sftp server", slog.Error(err))
-		return
+		return xerrors.Errorf("initialize sftp server: %w", err)
 	}
 	defer server.Close()
 
@@ -691,50 +823,12 @@ func (s *Server) sftpHandler(logger slog.Logger, session ssh.Session) {
 		// code but `scp` on macOS does (when using the default
 		// SFTP backend).
 		_ = session.Exit(0)
-		return
+		return nil
 	}
 	logger.Warn(ctx, "sftp server closed with error", slog.Error(err))
 	s.metrics.sftpServerErrors.Add(1)
 	_ = session.Exit(1)
-}
-
-// EnvInfoer encapsulates external information required by CreateCommand.
-type EnvInfoer interface {
-	// CurrentUser returns the current user.
-	CurrentUser() (*user.User, error)
-	// Environ returns the environment variables of the current process.
-	Environ() []string
-	// UserHomeDir returns the home directory of the current user.
-	UserHomeDir() (string, error)
-	// UserShell returns the shell of the given user.
-	UserShell(username string) (string, error)
-}
-
-type systemEnvInfoer struct{}
-
-var defaultEnvInfoer EnvInfoer = &systemEnvInfoer{}
-
-// DefaultEnvInfoer returns a default implementation of
-// EnvInfoer. This reads information using the default Go
-// implementations.
-func DefaultEnvInfoer() EnvInfoer {
-	return defaultEnvInfoer
-}
-
-func (systemEnvInfoer) CurrentUser() (*user.User, error) {
-	return user.Current()
-}
-
-func (systemEnvInfoer) Environ() []string {
-	return os.Environ()
-}
-
-func (systemEnvInfoer) UserHomeDir() (string, error) {
-	return userHomeDir()
-}
-
-func (systemEnvInfoer) UserShell(username string) (string, error) {
-	return usershell.Get(username)
+	return xerrors.Errorf("sftp server closed with error: %w", err)
 }
 
 // CreateCommand processes raw command input with OpenSSH-like behavior.
@@ -744,17 +838,17 @@ func (systemEnvInfoer) UserShell(username string) (string, error) {
 // alternative implementations for the dependencies of CreateCommand.
 // This is useful when creating a command to be run in a separate environment
 // (for example, a Docker container). Pass in nil to use the default.
-func (s *Server) CreateCommand(ctx context.Context, script string, env []string, deps EnvInfoer) (*pty.Cmd, error) {
-	if deps == nil {
-		deps = DefaultEnvInfoer()
+func (s *Server) CreateCommand(ctx context.Context, script string, env []string, ei usershell.EnvInfoer) (*pty.Cmd, error) {
+	if ei == nil {
+		ei = &usershell.SystemEnvInfo{}
 	}
-	currentUser, err := deps.CurrentUser()
+	currentUser, err := ei.User()
 	if err != nil {
 		return nil, xerrors.Errorf("get current user: %w", err)
 	}
 	username := currentUser.Username
 
-	shell, err := deps.UserShell(username)
+	shell, err := ei.Shell(username)
 	if err != nil {
 		return nil, xerrors.Errorf("get user shell: %w", err)
 	}
@@ -802,7 +896,18 @@ func (s *Server) CreateCommand(ctx context.Context, script string, env []string,
 		}
 	}
 
-	cmd := s.Execer.PTYCommandContext(ctx, name, args...)
+	// Modify command prior to execution. This will usually be a no-op, but not
+	// always. For example, to run a command in a Docker container, we need to
+	// modify the command to be `docker exec -it <container> <command>`.
+	modifiedName, modifiedArgs := ei.ModifyCommand(name, args...)
+	// Log if the command was modified.
+	if modifiedName != name && slices.Compare(modifiedArgs, args) != 0 {
+		s.logger.Debug(ctx, "modified command",
+			slog.F("before", append([]string{name}, args...)),
+			slog.F("after", append([]string{modifiedName}, modifiedArgs...)),
+		)
+	}
+	cmd := s.Execer.PTYCommandContext(ctx, modifiedName, modifiedArgs...)
 	cmd.Dir = s.config.WorkingDirectory()
 
 	// If the metadata directory doesn't exist, we run the command
@@ -810,14 +915,17 @@ func (s *Server) CreateCommand(ctx context.Context, script string, env []string,
 	_, err = os.Stat(cmd.Dir)
 	if cmd.Dir == "" || err != nil {
 		// Default to user home if a directory is not set.
-		homedir, err := deps.UserHomeDir()
+		homedir, err := ei.HomeDir()
 		if err != nil {
 			return nil, xerrors.Errorf("get home dir: %w", err)
 		}
 		cmd.Dir = homedir
 	}
-	cmd.Env = append(deps.Environ(), env...)
+	cmd.Env = append(ei.Environ(), env...)
+	// Set login variables (see `man login`).
 	cmd.Env = append(cmd.Env, fmt.Sprintf("USER=%s", username))
+	cmd.Env = append(cmd.Env, fmt.Sprintf("LOGNAME=%s", username))
+	cmd.Env = append(cmd.Env, fmt.Sprintf("SHELL=%s", shell))
 
 	// Set SSH connection environment variables (these are also set by OpenSSH
 	// and thus expected to be present by SSH clients). Since the agent does
@@ -839,7 +947,12 @@ func (s *Server) CreateCommand(ctx context.Context, script string, env []string,
 // Serve starts the server to handle incoming connections on the provided listener.
 // It returns an error if no host keys are set or if there is an issue accepting connections.
 func (s *Server) Serve(l net.Listener) (retErr error) {
-	if len(s.srv.HostSigners) == 0 {
+	// Ensure we're not mutating HostSigners as we're reading it.
+	s.mu.RLock()
+	noHostKeys := len(s.srv.HostSigners) == 0
+	s.mu.RUnlock()
+
+	if noHostKeys {
 		return xerrors.New("no host keys set")
 	}
 
@@ -954,6 +1067,27 @@ func (s *Server) trackSession(ss ssh.Session, add bool) (ok bool) {
 	return true
 }
 
+// trackCommand registers the process with the server. If the server is
+// closing, the process is not registered and should be closed.
+//
+//nolint:revive
+func (s *Server) trackProcess(p *os.Process, add bool) (ok bool) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if add {
+		if s.closing != nil {
+			// Server closed.
+			return false
+		}
+		s.wg.Add(1)
+		s.processes[p] = struct{}{}
+		return true
+	}
+	s.wg.Done()
+	delete(s.processes, p)
+	return true
+}
+
 // Close the server and all active connections. Server can be re-used
 // after Close is done.
 func (s *Server) Close() error {
@@ -962,32 +1096,47 @@ func (s *Server) Close() error {
 	// Guard against multiple calls to Close and
 	// accepting new connections during close.
 	if s.closing != nil {
+		closing := s.closing
 		s.mu.Unlock()
-		return xerrors.New("server is closing")
+		<-closing
+		return xerrors.New("server is closed")
 	}
 	s.closing = make(chan struct{})
 
+	ctx := context.Background()
+
+	s.logger.Debug(ctx, "closing server")
+
+	// Stop accepting new connections.
+	s.logger.Debug(ctx, "closing all active listeners", slog.F("count", len(s.listeners)))
+	for l := range s.listeners {
+		_ = l.Close()
+	}
+
 	// Close all active sessions to gracefully
 	// terminate client connections.
+	s.logger.Debug(ctx, "closing all active sessions", slog.F("count", len(s.sessions)))
 	for ss := range s.sessions {
 		// We call Close on the underlying channel here because we don't
 		// want to send an exit status to the client (via Exit()).
 		// Typically OpenSSH clients will return 255 as the exit status.
 		_ = ss.Close()
 	}
-
-	// Close all active listeners and connections.
-	for l := range s.listeners {
-		_ = l.Close()
-	}
+	s.logger.Debug(ctx, "closing all active connections", slog.F("count", len(s.conns)))
 	for c := range s.conns {
 		_ = c.Close()
 	}
 
-	// Close the underlying SSH server.
+	for p := range s.processes {
+		_ = cmdCancel(s.logger, p)
+	}
+
+	s.logger.Debug(ctx, "closing SSH server")
 	err := s.srv.Close()
 
 	s.mu.Unlock()
+
+	s.logger.Debug(ctx, "waiting for all goroutines to exit")
 	s.wg.Wait() // Wait for all goroutines to exit.
 
 	s.mu.Lock()
@@ -995,15 +1144,35 @@ func (s *Server) Close() error {
 	s.closing = nil
 	s.mu.Unlock()
 
+	s.logger.Debug(ctx, "closing server done")
+
 	return err
 }
 
-// Shutdown gracefully closes all active SSH connections and stops
-// accepting new connections.
-//
-// Shutdown is not implemented.
-func (*Server) Shutdown(_ context.Context) error {
-	// TODO(mafredri): Implement shutdown, SIGHUP running commands, etc.
+// Shutdown stops accepting new connections. The current implementation
+// calls Close() for simplicity instead of waiting for existing
+// connections to close. If the context times out, Shutdown will return
+// but Close() may not have completed.
+func (s *Server) Shutdown(ctx context.Context) error {
+	ch := make(chan error, 1)
+	go func() {
+		// TODO(mafredri): Implement shutdown, SIGHUP running commands, etc.
+		// For now we just close the server.
+		ch <- s.Close()
+	}()
+	var err error
+	select {
+	case <-ctx.Done():
+		err = ctx.Err()
+	case err = <-ch:
+	}
+	// Re-check for context cancellation precedence.
+	if ctx.Err() != nil {
+		err = ctx.Err()
+	}
+	if err != nil {
+		return xerrors.Errorf("close server: %w", err)
+	}
 	return nil
 }
 
@@ -1120,75 +1289,7 @@ func CoderSigner(seed int64) (gossh.Signer, error) {
 	// Clients should ignore the host key when connecting.
 	// The agent needs to authenticate with coderd to SSH,
 	// so SSH authentication doesn't improve security.
-
-	// Since the standard lib purposefully does not generate
-	// deterministic rsa keys, we need to do it ourselves.
-	coderHostKey := func() *rsa.PrivateKey {
-		// Create deterministic random source
-		// nolint: gosec
-		deterministicRand := rand.New(rand.NewSource(seed))
-
-		// Use fixed values for p and q based on the seed
-		p := big.NewInt(0)
-		q := big.NewInt(0)
-		e := big.NewInt(65537) // Standard RSA public exponent
-
-		// Generate deterministic primes using the seeded random
-		// Each prime should be ~1024 bits to get a 2048-bit key
-		for {
-			p.SetBit(p, 1024, 1) // Ensure it's large enough
-			for i := 0; i < 1024; i++ {
-				if deterministicRand.Int63()%2 == 1 {
-					p.SetBit(p, i, 1)
-				} else {
-					p.SetBit(p, i, 0)
-				}
-			}
-			if p.ProbablyPrime(20) {
-				break
-			}
-		}
-
-		for {
-			q.SetBit(q, 1024, 1) // Ensure it's large enough
-			for i := 0; i < 1024; i++ {
-				if deterministicRand.Int63()%2 == 1 {
-					q.SetBit(q, i, 1)
-				} else {
-					q.SetBit(q, i, 0)
-				}
-			}
-			if q.ProbablyPrime(20) && p.Cmp(q) != 0 {
-				break
-			}
-		}
-
-		// Calculate n = p * q
-		n := new(big.Int).Mul(p, q)
-
-		// Calculate phi = (p-1) * (q-1)
-		p1 := new(big.Int).Sub(p, big.NewInt(1))
-		q1 := new(big.Int).Sub(q, big.NewInt(1))
-		phi := new(big.Int).Mul(p1, q1)
-
-		// Calculate private exponent d
-		d := new(big.Int).ModInverse(e, phi)
-
-		// Create the private key
-		privateKey := &rsa.PrivateKey{
-			PublicKey: rsa.PublicKey{
-				N: n,
-				E: int(e.Int64()),
-			},
-			D:      d,
-			Primes: []*big.Int{p, q},
-		}
-
-		// Compute precomputed values
-		privateKey.Precompute()
-
-		return privateKey
-	}()
+	coderHostKey := agentrsa.GenerateDeterministicKey(seed)
 
 	coderSigner, err := gossh.NewSignerFromKey(coderHostKey)
 	return coderSigner, err

agent/agentssh/agentssh_test.go

@@ -13,6 +13,7 @@ import (
 	"strings"
 	"sync"
 	"testing"
+	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/spf13/afero"
@@ -21,6 +22,7 @@ import (
 	"go.uber.org/goleak"
 	"golang.org/x/crypto/ssh"
 
+	"cdr.dev/slog"
 	"cdr.dev/slog/sloggers/slogtest"
 
 	"github.com/coder/coder/v2/agent/agentexec"
@@ -124,7 +126,7 @@ type fakeEnvInfoer struct {
 	UserShellFn   func(string) (string, error)
 }
 
-func (f *fakeEnvInfoer) CurrentUser() (u *user.User, err error) {
+func (f *fakeEnvInfoer) User() (u *user.User, err error) {
 	return f.CurrentUserFn()
 }
 
@@ -132,62 +134,124 @@ func (f *fakeEnvInfoer) Environ() []string {
 	return f.EnvironFn()
 }
 
-func (f *fakeEnvInfoer) UserHomeDir() (string, error) {
+func (f *fakeEnvInfoer) HomeDir() (string, error) {
 	return f.UserHomeDirFn()
 }
 
-func (f *fakeEnvInfoer) UserShell(u string) (string, error) {
+func (f *fakeEnvInfoer) Shell(u string) (string, error) {
 	return f.UserShellFn(u)
 }
 
+func (*fakeEnvInfoer) ModifyCommand(cmd string, args ...string) (string, []string) {
+	return cmd, args
+}
+
 func TestNewServer_CloseActiveConnections(t *testing.T) {
 	t.Parallel()
 
-	ctx := context.Background()
-	logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true})
-	s, err := agentssh.NewServer(ctx, logger, prometheus.NewRegistry(), afero.NewMemMapFs(), agentexec.DefaultExecer, nil)
-	require.NoError(t, err)
-	defer s.Close()
-	err = s.UpdateHostSigner(42)
-	assert.NoError(t, err)
-
-	ln, err := net.Listen("tcp", "127.0.0.1:0")
-	require.NoError(t, err)
-
-	var wg sync.WaitGroup
-	wg.Add(2)
-	go func() {
-		defer wg.Done()
-		err := s.Serve(ln)
-		assert.Error(t, err) // Server is closed.
-	}()
-
-	pty := ptytest.New(t)
-
-	doClose := make(chan struct{})
-	go func() {
-		defer wg.Done()
-		c := sshClient(t, ln.Addr().String())
-		sess, err := c.NewSession()
-		assert.NoError(t, err)
-		sess.Stdin = pty.Input()
-		sess.Stdout = pty.Output()
-		sess.Stderr = pty.Output()
-
-		assert.NoError(t, err)
-		err = sess.Start("")
+	prepare := func(ctx context.Context, t *testing.T) (*agentssh.Server, func()) {
+		t.Helper()
+		logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}).Leveled(slog.LevelDebug)
+		s, err := agentssh.NewServer(ctx, logger, prometheus.NewRegistry(), afero.NewMemMapFs(), agentexec.DefaultExecer, nil)
+		require.NoError(t, err)
+		t.Cleanup(func() {
+			_ = s.Close()
+		})
+		err = s.UpdateHostSigner(42)
 		assert.NoError(t, err)
 
-		close(doClose)
-		err = sess.Wait()
-		assert.Error(t, err)
-	}()
+		ln, err := net.Listen("tcp", "127.0.0.1:0")
+		require.NoError(t, err)
 
-	<-doClose
-	err = s.Close()
-	require.NoError(t, err)
+		waitConns := make([]chan struct{}, 4)
 
-	wg.Wait()
+		var wg sync.WaitGroup
+		wg.Add(1 + len(waitConns))
+
+		go func() {
+			defer wg.Done()
+			err := s.Serve(ln)
+			assert.Error(t, err) // Server is closed.
+		}()
+
+		for i := 0; i < len(waitConns); i++ {
+			waitConns[i] = make(chan struct{})
+			go func(ch chan struct{}) {
+				defer wg.Done()
+				c := sshClient(t, ln.Addr().String())
+				sess, err := c.NewSession()
+				assert.NoError(t, err)
+				pty := ptytest.New(t)
+				sess.Stdin = pty.Input()
+				sess.Stdout = pty.Output()
+				sess.Stderr = pty.Output()
+
+				// Every other session will request a PTY.
+				if i%2 == 0 {
+					err = sess.RequestPty("xterm", 80, 80, nil)
+					assert.NoError(t, err)
+				}
+				// The 60 seconds here is intended to be longer than the
+				// test. The shutdown should propagate.
+				if runtime.GOOS == "windows" {
+					// Best effort to at least partially test this in Windows.
+					err = sess.Start("echo start\"ed\" && sleep 60")
+				} else {
+					err = sess.Start("/bin/bash -c 'trap \"sleep 60\" SIGTERM; echo start\"ed\"; sleep 60'")
+				}
+				assert.NoError(t, err)
+
+				// Allow the session to settle (i.e. reach echo).
+				pty.ExpectMatchContext(ctx, "started")
+				// Sleep a bit to ensure the sleep has started.
+				time.Sleep(testutil.IntervalMedium)
+
+				close(ch)
+
+				err = sess.Wait()
+				assert.Error(t, err)
+			}(waitConns[i])
+		}
+
+		for _, ch := range waitConns {
+			select {
+			case <-ctx.Done():
+				t.Fatal("timeout")
+			case <-ch:
+			}
+		}
+
+		return s, wg.Wait
+	}
+
+	t.Run("Close", func(t *testing.T) {
+		t.Parallel()
+		ctx := testutil.Context(t, testutil.WaitMedium)
+		s, wait := prepare(ctx, t)
+		err := s.Close()
+		require.NoError(t, err)
+		wait()
+	})
+
+	t.Run("Shutdown", func(t *testing.T) {
+		t.Parallel()
+		ctx := testutil.Context(t, testutil.WaitMedium)
+		s, wait := prepare(ctx, t)
+		err := s.Shutdown(ctx)
+		require.NoError(t, err)
+		wait()
+	})
+
+	t.Run("Shutdown Early", func(t *testing.T) {
+		t.Parallel()
+		ctx := testutil.Context(t, testutil.WaitMedium)
+		s, wait := prepare(ctx, t)
+		ctx, cancel := context.WithCancel(ctx)
+		cancel()
+		err := s.Shutdown(ctx)
+		require.ErrorIs(t, err, context.Canceled)
+		wait()
+	})
 }
 
 func TestNewServer_Signal(t *testing.T) {

agent/agentssh/exec_other.go

@@ -0,0 +1,22 @@
+//go:build !windows
+
+package agentssh
+
+import (
+	"context"
+	"os"
+	"syscall"
+
+	"cdr.dev/slog"
+)
+
+func cmdSysProcAttr() *syscall.SysProcAttr {
+	return &syscall.SysProcAttr{
+		Setsid: true,
+	}
+}
+
+func cmdCancel(logger slog.Logger, p *os.Process) error {
+	logger.Debug(context.Background(), "cmdCancel: sending SIGHUP to process and children", slog.F("pid", p.Pid))
+	return syscall.Kill(-p.Pid, syscall.SIGHUP)
+}

agent/agentssh/exec_windows.go

@@ -0,0 +1,23 @@
+package agentssh
+
+import (
+	"context"
+	"os"
+	"syscall"
+
+	"cdr.dev/slog"
+)
+
+func cmdSysProcAttr() *syscall.SysProcAttr {
+	return &syscall.SysProcAttr{}
+}
+
+func cmdCancel(logger slog.Logger, p *os.Process) error {
+	logger.Debug(context.Background(), "cmdCancel: killing process", slog.F("pid", p.Pid))
+	// Windows doesn't support sending signals to process groups, so we
+	// have to kill the process directly. In the future, we may want to
+	// implement a more sophisticated solution for process groups on
+	// Windows, but for now, this is a simple way to ensure that the
+	// process is terminated when the context is cancelled.
+	return p.Kill()
+}

agent/agentssh/jetbrainstrack.go

@@ -6,6 +6,7 @@ import (
 	"sync"
 
 	"github.com/gliderlabs/ssh"
+	"github.com/google/uuid"
 	"go.uber.org/atomic"
 	gossh "golang.org/x/crypto/ssh"
 
@@ -28,9 +29,11 @@ type JetbrainsChannelWatcher struct {
 	gossh.NewChannel
 	jetbrainsCounter *atomic.Int64
 	logger           slog.Logger
+	originAddr       string
+	reportConnection reportConnectionFunc
 }
 
-func NewJetbrainsChannelWatcher(ctx ssh.Context, logger slog.Logger, newChannel gossh.NewChannel, counter *atomic.Int64) gossh.NewChannel {
+func NewJetbrainsChannelWatcher(ctx ssh.Context, logger slog.Logger, reportConnection reportConnectionFunc, newChannel gossh.NewChannel, counter *atomic.Int64) gossh.NewChannel {
 	d := localForwardChannelData{}
 	if err := gossh.Unmarshal(newChannel.ExtraData(), &d); err != nil {
 		// If the data fails to unmarshal, do nothing.
@@ -61,12 +64,17 @@ func NewJetbrainsChannelWatcher(ctx ssh.Context, logger slog.Logger, newChannel
 		NewChannel:       newChannel,
 		jetbrainsCounter: counter,
 		logger:           logger.With(slog.F("destination_port", d.DestPort)),
+		originAddr:       d.OriginAddr,
+		reportConnection: reportConnection,
 	}
 }
 
 func (w *JetbrainsChannelWatcher) Accept() (gossh.Channel, <-chan *gossh.Request, error) {
+	disconnected := w.reportConnection(uuid.New(), MagicSessionTypeJetBrains, w.originAddr)
+
 	c, r, err := w.NewChannel.Accept()
 	if err != nil {
+		disconnected(1, err.Error())
 		return c, r, err
 	}
 	w.jetbrainsCounter.Add(1)
@@ -77,6 +85,7 @@ func (w *JetbrainsChannelWatcher) Accept() (gossh.Channel, <-chan *gossh.Request
 		Channel: c,
 		done: func() {
 			w.jetbrainsCounter.Add(-1)
+			disconnected(0, "")
 			// nolint: gocritic // JetBrains is a proper noun and should be capitalized
 			w.logger.Debug(context.Background(), "JetBrains watcher channel closed")
 		},

agent/agentssh/x11.go

@@ -116,7 +116,8 @@ func (s *Server) x11Handler(ctx ssh.Context, x11 ssh.X11) (displayNumber int, ha
 				OriginatorPort    uint32
 			}{
 				OriginatorAddress: tcpAddr.IP.String(),
-				OriginatorPort:    uint32(tcpAddr.Port),
+				// #nosec G115 - Safe conversion as TCP port numbers are within uint32 range (0-65535)
+				OriginatorPort: uint32(tcpAddr.Port),
 			}))
 			if err != nil {
 				s.logger.Warn(ctx, "failed to open X11 channel", slog.Error(err))
@@ -294,6 +295,7 @@ func addXauthEntry(ctx context.Context, fs afero.Fs, host string, display string
 		return xerrors.Errorf("failed to write family: %w", err)
 	}
 
+	// #nosec G115 - Safe conversion for host name length which is expected to be within uint16 range
 	err = binary.Write(file, binary.BigEndian, uint16(len(host)))
 	if err != nil {
 		return xerrors.Errorf("failed to write host length: %w", err)
@@ -303,6 +305,7 @@ func addXauthEntry(ctx context.Context, fs afero.Fs, host string, display string
 		return xerrors.Errorf("failed to write host: %w", err)
 	}
 
+	// #nosec G115 - Safe conversion for display name length which is expected to be within uint16 range
 	err = binary.Write(file, binary.BigEndian, uint16(len(display)))
 	if err != nil {
 		return xerrors.Errorf("failed to write display length: %w", err)
@@ -312,6 +315,7 @@ func addXauthEntry(ctx context.Context, fs afero.Fs, host string, display string
 		return xerrors.Errorf("failed to write display: %w", err)
 	}
 
+	// #nosec G115 - Safe conversion for auth protocol length which is expected to be within uint16 range
 	err = binary.Write(file, binary.BigEndian, uint16(len(authProtocol)))
 	if err != nil {
 		return xerrors.Errorf("failed to write auth protocol length: %w", err)
@@ -321,6 +325,7 @@ func addXauthEntry(ctx context.Context, fs afero.Fs, host string, display string
 		return xerrors.Errorf("failed to write auth protocol: %w", err)
 	}
 
+	// #nosec G115 - Safe conversion for auth cookie length which is expected to be within uint16 range
 	err = binary.Write(file, binary.BigEndian, uint16(len(authCookieBytes)))
 	if err != nil {
 		return xerrors.Errorf("failed to write auth cookie length: %w", err)

agent/agenttest/client.go

@@ -3,6 +3,7 @@ package agenttest
 import (
 	"context"
 	"io"
+	"slices"
 	"sync"
 	"sync/atomic"
 	"testing"
@@ -12,7 +13,6 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"golang.org/x/exp/maps"
-	"golang.org/x/exp/slices"
 	"golang.org/x/xerrors"
 	"google.golang.org/protobuf/types/known/durationpb"
 	"google.golang.org/protobuf/types/known/emptypb"
@@ -24,7 +24,7 @@ import (
 	agentproto "github.com/coder/coder/v2/agent/proto"
 	"github.com/coder/coder/v2/codersdk"
 	"github.com/coder/coder/v2/codersdk/agentsdk"
-	drpcsdk "github.com/coder/coder/v2/codersdk/drpc"
+	"github.com/coder/coder/v2/codersdk/drpcsdk"
 	"github.com/coder/coder/v2/tailnet"
 	"github.com/coder/coder/v2/tailnet/proto"
 	"github.com/coder/coder/v2/testutil"
@@ -60,6 +60,7 @@ func NewClient(t testing.TB,
 	err = agentproto.DRPCRegisterAgent(mux, fakeAAPI)
 	require.NoError(t, err)
 	server := drpcserver.NewWithOptions(mux, drpcserver.Options{
+		Manager: drpcsdk.DefaultDRPCOptions(nil),
 		Log: func(err error) {
 			if xerrors.Is(err, io.EOF) {
 				return
@@ -97,8 +98,8 @@ func (c *Client) Close() {
 	c.derpMapOnce.Do(func() { close(c.derpMapUpdates) })
 }
 
-func (c *Client) ConnectRPC24(ctx context.Context) (
-	agentproto.DRPCAgentClient24, proto.DRPCTailnetClient24, error,
+func (c *Client) ConnectRPC25(ctx context.Context) (
+	agentproto.DRPCAgentClient25, proto.DRPCTailnetClient25, error,
 ) {
 	conn, lis := drpcsdk.MemTransportPipe()
 	c.LastWorkspaceAgent = func() {
@@ -158,20 +159,24 @@ func (c *Client) SetLogsChannel(ch chan<- *agentproto.BatchCreateLogsRequest) {
 	c.fakeAgentAPI.SetLogsChannel(ch)
 }
 
+func (c *Client) GetConnectionReports() []*agentproto.ReportConnectionRequest {
+	return c.fakeAgentAPI.GetConnectionReports()
+}
+
 type FakeAgentAPI struct {
 	sync.Mutex
 	t      testing.TB
 	logger slog.Logger
 
-	manifest        *agentproto.Manifest
-	startupCh       chan *agentproto.Startup
-	statsCh         chan *agentproto.Stats
-	appHealthCh     chan *agentproto.BatchUpdateAppHealthRequest
-	logsCh          chan<- *agentproto.BatchCreateLogsRequest
-	lifecycleStates []codersdk.WorkspaceAgentLifecycle
-	metadata        map[string]agentsdk.Metadata
-	timings         []*agentproto.Timing
-	connections     []*agentproto.Connection
+	manifest          *agentproto.Manifest
+	startupCh         chan *agentproto.Startup
+	statsCh           chan *agentproto.Stats
+	appHealthCh       chan *agentproto.BatchUpdateAppHealthRequest
+	logsCh            chan<- *agentproto.BatchCreateLogsRequest
+	lifecycleStates   []codersdk.WorkspaceAgentLifecycle
+	metadata          map[string]agentsdk.Metadata
+	timings           []*agentproto.Timing
+	connectionReports []*agentproto.ReportConnectionRequest
 
 	getAnnouncementBannersFunc              func() ([]codersdk.BannerConfig, error)
 	getResourcesMonitoringConfigurationFunc func() (*agentproto.GetResourcesMonitoringConfigurationResponse, error)
@@ -348,12 +353,18 @@ func (f *FakeAgentAPI) ScriptCompleted(_ context.Context, req *agentproto.Worksp
 
 func (f *FakeAgentAPI) ReportConnection(_ context.Context, req *agentproto.ReportConnectionRequest) (*emptypb.Empty, error) {
 	f.Lock()
-	f.connections = append(f.connections, req.GetConnection())
+	f.connectionReports = append(f.connectionReports, req)
 	f.Unlock()
 
 	return &emptypb.Empty{}, nil
 }
 
+func (f *FakeAgentAPI) GetConnectionReports() []*agentproto.ReportConnectionRequest {
+	f.Lock()
+	defer f.Unlock()
+	return slices.Clone(f.connectionReports)
+}
+
 func NewFakeAgentAPI(t testing.TB, logger slog.Logger, manifest *agentproto.Manifest, statsCh chan *agentproto.Stats) *FakeAgentAPI {
 	return &FakeAgentAPI{
 		t:           t,

agent/api.go

@@ -7,12 +7,14 @@ import (
 
 	"github.com/go-chi/chi/v5"
 
+	"github.com/google/uuid"
+
 	"github.com/coder/coder/v2/agent/agentcontainers"
 	"github.com/coder/coder/v2/coderd/httpapi"
 	"github.com/coder/coder/v2/codersdk"
 )
 
-func (a *agent) apiHandler() http.Handler {
+func (a *agent) apiHandler() (http.Handler, func() error) {
 	r := chi.NewRouter()
 	r.Get("/", func(rw http.ResponseWriter, r *http.Request) {
 		httpapi.Write(r.Context(), rw, http.StatusOK, codersdk.Response{
@@ -36,18 +38,54 @@ func (a *agent) apiHandler() http.Handler {
 		ignorePorts:   cpy,
 		cacheDuration: cacheDuration,
 	}
-	ch := agentcontainers.New(agentcontainers.WithLister(a.lister))
+
+	if a.experimentalDevcontainersEnabled {
+		containerAPIOpts := []agentcontainers.Option{
+			agentcontainers.WithExecer(a.execer),
+			agentcontainers.WithScriptLogger(func(logSourceID uuid.UUID) agentcontainers.ScriptLogger {
+				return a.logSender.GetScriptLogger(logSourceID)
+			}),
+		}
+		manifest := a.manifest.Load()
+		if manifest != nil && len(manifest.Devcontainers) > 0 {
+			containerAPIOpts = append(
+				containerAPIOpts,
+				agentcontainers.WithDevcontainers(manifest.Devcontainers, manifest.Scripts),
+			)
+		}
+
+		// Append after to allow the agent options to override the default options.
+		containerAPIOpts = append(containerAPIOpts, a.containerAPIOptions...)
+
+		containerAPI := agentcontainers.NewAPI(a.logger.Named("containers"), containerAPIOpts...)
+		r.Mount("/api/v0/containers", containerAPI.Routes())
+		a.containerAPI.Store(containerAPI)
+	} else {
+		r.HandleFunc("/api/v0/containers", func(w http.ResponseWriter, r *http.Request) {
+			httpapi.Write(r.Context(), w, http.StatusForbidden, codersdk.Response{
+				Message: "The agent dev containers feature is experimental and not enabled by default.",
+				Detail:  "To enable this feature, set CODER_AGENT_DEVCONTAINERS_ENABLE=true in your template.",
+			})
+		})
+	}
+
 	promHandler := PrometheusMetricsHandler(a.prometheusRegistry, a.logger)
-	r.Get("/api/v0/containers", ch.ServeHTTP)
+
 	r.Get("/api/v0/listening-ports", lp.handler)
 	r.Get("/api/v0/netcheck", a.HandleNetcheck)
+	r.Post("/api/v0/list-directory", a.HandleLS)
 	r.Get("/debug/logs", a.HandleHTTPDebugLogs)
 	r.Get("/debug/magicsock", a.HandleHTTPDebugMagicsock)
 	r.Get("/debug/magicsock/debug-logging/{state}", a.HandleHTTPMagicsockDebugLoggingState)
 	r.Get("/debug/manifest", a.HandleHTTPDebugManifest)
 	r.Get("/debug/prometheus", promHandler.ServeHTTP)
 
-	return r
+	return r, func() error {
+		if containerAPI := a.containerAPI.Load(); containerAPI != nil {
+			return containerAPI.Close()
+		}
+		return nil
+	}
 }
 
 type listeningPortsHandler struct {

agent/apphealth.go

@@ -167,8 +167,8 @@ func shouldStartTicker(app codersdk.WorkspaceApp) bool {
 	return app.Healthcheck.URL != "" && app.Healthcheck.Interval > 0 && app.Healthcheck.Threshold > 0
 }
 
-func healthChanged(old map[uuid.UUID]codersdk.WorkspaceAppHealth, new map[uuid.UUID]codersdk.WorkspaceAppHealth) bool {
-	for name, newValue := range new {
+func healthChanged(old map[uuid.UUID]codersdk.WorkspaceAppHealth, updated map[uuid.UUID]codersdk.WorkspaceAppHealth) bool {
+	for name, newValue := range updated {
 		oldValue, found := old[name]
 		if !found {
 			return true

agent/apphealth_test.go

@@ -78,7 +78,7 @@ func TestAppHealth_Healthy(t *testing.T) {
 	healthchecksStarted := make([]string, 2)
 	for i := 0; i < 2; i++ {
 		c := healthcheckTrap.MustWait(ctx)
-		c.Release()
+		c.MustRelease(ctx)
 		healthchecksStarted[i] = c.Tags[1]
 	}
 	slices.Sort(healthchecksStarted)
@@ -87,12 +87,12 @@ func TestAppHealth_Healthy(t *testing.T) {
 	// advance the clock 1ms before the report ticker starts, so that it's not
 	// simultaneous with the checks.
 	mClock.Advance(time.Millisecond).MustWait(ctx)
-	reportTrap.MustWait(ctx).Release()
+	reportTrap.MustWait(ctx).MustRelease(ctx)
 
 	mClock.Advance(999 * time.Millisecond).MustWait(ctx) // app2 is now healthy
 
 	mClock.Advance(time.Millisecond).MustWait(ctx) // report gets triggered
-	update := testutil.RequireRecvCtx(ctx, t, fakeAPI.AppHealthCh())
+	update := testutil.TryReceive(ctx, t, fakeAPI.AppHealthCh())
 	require.Len(t, update.GetUpdates(), 2)
 	applyUpdate(t, apps, update)
 	require.Equal(t, codersdk.WorkspaceAppHealthHealthy, apps[1].Health)
@@ -101,7 +101,7 @@ func TestAppHealth_Healthy(t *testing.T) {
 	mClock.Advance(999 * time.Millisecond).MustWait(ctx) // app3 is now healthy
 
 	mClock.Advance(time.Millisecond).MustWait(ctx) // report gets triggered
-	update = testutil.RequireRecvCtx(ctx, t, fakeAPI.AppHealthCh())
+	update = testutil.TryReceive(ctx, t, fakeAPI.AppHealthCh())
 	require.Len(t, update.GetUpdates(), 2)
 	applyUpdate(t, apps, update)
 	require.Equal(t, codersdk.WorkspaceAppHealthHealthy, apps[1].Health)
@@ -143,11 +143,11 @@ func TestAppHealth_500(t *testing.T) {
 
 	fakeAPI, closeFn := setupAppReporter(ctx, t, slices.Clone(apps), handlers, mClock)
 	defer closeFn()
-	healthcheckTrap.MustWait(ctx).Release()
+	healthcheckTrap.MustWait(ctx).MustRelease(ctx)
 	// advance the clock 1ms before the report ticker starts, so that it's not
 	// simultaneous with the checks.
 	mClock.Advance(time.Millisecond).MustWait(ctx)
-	reportTrap.MustWait(ctx).Release()
+	reportTrap.MustWait(ctx).MustRelease(ctx)
 
 	mClock.Advance(999 * time.Millisecond).MustWait(ctx) // check gets triggered
 	mClock.Advance(time.Millisecond).MustWait(ctx)       // report gets triggered, but unsent since we are at the threshold
@@ -155,7 +155,7 @@ func TestAppHealth_500(t *testing.T) {
 	mClock.Advance(999 * time.Millisecond).MustWait(ctx) // 2nd check, crosses threshold
 	mClock.Advance(time.Millisecond).MustWait(ctx)       // 2nd report, sends update
 
-	update := testutil.RequireRecvCtx(ctx, t, fakeAPI.AppHealthCh())
+	update := testutil.TryReceive(ctx, t, fakeAPI.AppHealthCh())
 	require.Len(t, update.GetUpdates(), 1)
 	applyUpdate(t, apps, update)
 	require.Equal(t, codersdk.WorkspaceAppHealthUnhealthy, apps[0].Health)
@@ -202,28 +202,28 @@ func TestAppHealth_Timeout(t *testing.T) {
 
 	fakeAPI, closeFn := setupAppReporter(ctx, t, apps, handlers, mClock)
 	defer closeFn()
-	healthcheckTrap.MustWait(ctx).Release()
+	healthcheckTrap.MustWait(ctx).MustRelease(ctx)
 	// advance the clock 1ms before the report ticker starts, so that it's not
 	// simultaneous with the checks.
 	mClock.Set(ms(1)).MustWait(ctx)
-	reportTrap.MustWait(ctx).Release()
+	reportTrap.MustWait(ctx).MustRelease(ctx)
 
 	w := mClock.Set(ms(1000)) // 1st check starts
-	timeoutTrap.MustWait(ctx).Release()
+	timeoutTrap.MustWait(ctx).MustRelease(ctx)
 	mClock.Set(ms(1001)).MustWait(ctx) // report tick, no change
 	mClock.Set(ms(1999))               // timeout pops
 	w.MustWait(ctx)                    // 1st check finished
 	w = mClock.Set(ms(2000))           // 2nd check starts
-	timeoutTrap.MustWait(ctx).Release()
+	timeoutTrap.MustWait(ctx).MustRelease(ctx)
 	mClock.Set(ms(2001)).MustWait(ctx) // report tick, no change
 	mClock.Set(ms(2999))               // timeout pops
 	w.MustWait(ctx)                    // 2nd check finished
 	// app is now unhealthy after 2 timeouts
 	mClock.Set(ms(3000)) // 3rd check starts
-	timeoutTrap.MustWait(ctx).Release()
+	timeoutTrap.MustWait(ctx).MustRelease(ctx)
 	mClock.Set(ms(3001)).MustWait(ctx) // report tick, sends changes
 
-	update := testutil.RequireRecvCtx(ctx, t, fakeAPI.AppHealthCh())
+	update := testutil.TryReceive(ctx, t, fakeAPI.AppHealthCh())
 	require.Len(t, update.GetUpdates(), 1)
 	applyUpdate(t, apps, update)
 	require.Equal(t, codersdk.WorkspaceAppHealthUnhealthy, apps[0].Health)

agent/checkpoint_internal_test.go

@@ -44,6 +44,6 @@ func TestCheckpoint_WaitComplete(t *testing.T) {
 		errCh <- uut.wait(ctx)
 	}()
 	uut.complete(err)
-	got := testutil.RequireRecvCtx(ctx, t, errCh)
+	got := testutil.TryReceive(ctx, t, errCh)
 	require.Equal(t, err, got)
 }

agent/ls.go

@@ -0,0 +1,198 @@
+package agent
+
+import (
+	"errors"
+	"net/http"
+	"os"
+	"path/filepath"
+	"regexp"
+	"runtime"
+	"slices"
+	"strings"
+
+	"github.com/shirou/gopsutil/v4/disk"
+	"golang.org/x/xerrors"
+
+	"github.com/coder/coder/v2/coderd/httpapi"
+	"github.com/coder/coder/v2/codersdk"
+)
+
+var WindowsDriveRegex = regexp.MustCompile(`^[a-zA-Z]:\\$`)
+
+func (*agent) HandleLS(rw http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+
+	var query LSRequest
+	if !httpapi.Read(ctx, rw, r, &query) {
+		return
+	}
+
+	resp, err := listFiles(query)
+	if err != nil {
+		status := http.StatusInternalServerError
+		switch {
+		case errors.Is(err, os.ErrNotExist):
+			status = http.StatusNotFound
+		case errors.Is(err, os.ErrPermission):
+			status = http.StatusForbidden
+		default:
+		}
+		httpapi.Write(ctx, rw, status, codersdk.Response{
+			Message: err.Error(),
+		})
+		return
+	}
+
+	httpapi.Write(ctx, rw, http.StatusOK, resp)
+}
+
+func listFiles(query LSRequest) (LSResponse, error) {
+	var fullPath []string
+	switch query.Relativity {
+	case LSRelativityHome:
+		home, err := os.UserHomeDir()
+		if err != nil {
+			return LSResponse{}, xerrors.Errorf("failed to get user home directory: %w", err)
+		}
+		fullPath = []string{home}
+	case LSRelativityRoot:
+		if runtime.GOOS == "windows" {
+			if len(query.Path) == 0 {
+				return listDrives()
+			}
+			if !WindowsDriveRegex.MatchString(query.Path[0]) {
+				return LSResponse{}, xerrors.Errorf("invalid drive letter %q", query.Path[0])
+			}
+		} else {
+			fullPath = []string{"/"}
+		}
+	default:
+		return LSResponse{}, xerrors.Errorf("unsupported relativity type %q", query.Relativity)
+	}
+
+	fullPath = append(fullPath, query.Path...)
+	fullPathRelative := filepath.Join(fullPath...)
+	absolutePathString, err := filepath.Abs(fullPathRelative)
+	if err != nil {
+		return LSResponse{}, xerrors.Errorf("failed to get absolute path of %q: %w", fullPathRelative, err)
+	}
+
+	// codeql[go/path-injection] - The intent is to allow the user to navigate to any directory in their workspace.
+	f, err := os.Open(absolutePathString)
+	if err != nil {
+		return LSResponse{}, xerrors.Errorf("failed to open directory %q: %w", absolutePathString, err)
+	}
+	defer f.Close()
+
+	stat, err := f.Stat()
+	if err != nil {
+		return LSResponse{}, xerrors.Errorf("failed to stat directory %q: %w", absolutePathString, err)
+	}
+
+	if !stat.IsDir() {
+		return LSResponse{}, xerrors.Errorf("path %q is not a directory", absolutePathString)
+	}
+
+	// `contents` may be partially populated even if the operation fails midway.
+	contents, _ := f.ReadDir(-1)
+	respContents := make([]LSFile, 0, len(contents))
+	for _, file := range contents {
+		respContents = append(respContents, LSFile{
+			Name:               file.Name(),
+			AbsolutePathString: filepath.Join(absolutePathString, file.Name()),
+			IsDir:              file.IsDir(),
+		})
+	}
+
+	// Sort alphabetically: directories then files
+	slices.SortFunc(respContents, func(a, b LSFile) int {
+		if a.IsDir && !b.IsDir {
+			return -1
+		}
+		if !a.IsDir && b.IsDir {
+			return 1
+		}
+		return strings.Compare(a.Name, b.Name)
+	})
+
+	absolutePath := pathToArray(absolutePathString)
+
+	return LSResponse{
+		AbsolutePath:       absolutePath,
+		AbsolutePathString: absolutePathString,
+		Contents:           respContents,
+	}, nil
+}
+
+func listDrives() (LSResponse, error) {
+	// disk.Partitions() will return partitions even if there was a failure to
+	// get one. Any errored partitions will not be returned.
+	partitionStats, err := disk.Partitions(true)
+	if err != nil && len(partitionStats) == 0 {
+		// Only return the error if there were no partitions returned.
+		return LSResponse{}, xerrors.Errorf("failed to get partitions: %w", err)
+	}
+
+	contents := make([]LSFile, 0, len(partitionStats))
+	for _, a := range partitionStats {
+		// Drive letters on Windows have a trailing separator as part of their name.
+		// i.e. `os.Open("C:")` does not work, but `os.Open("C:\\")` does.
+		name := a.Mountpoint + string(os.PathSeparator)
+		contents = append(contents, LSFile{
+			Name:               name,
+			AbsolutePathString: name,
+			IsDir:              true,
+		})
+	}
+
+	return LSResponse{
+		AbsolutePath:       []string{},
+		AbsolutePathString: "",
+		Contents:           contents,
+	}, nil
+}
+
+func pathToArray(path string) []string {
+	out := strings.FieldsFunc(path, func(r rune) bool {
+		return r == os.PathSeparator
+	})
+	// Drive letters on Windows have a trailing separator as part of their name.
+	// i.e. `os.Open("C:")` does not work, but `os.Open("C:\\")` does.
+	if runtime.GOOS == "windows" && len(out) > 0 {
+		out[0] += string(os.PathSeparator)
+	}
+	return out
+}
+
+type LSRequest struct {
+	// e.g. [], ["repos", "coder"],
+	Path []string `json:"path"`
+	// Whether the supplied path is relative to the user's home directory,
+	// or the root directory.
+	Relativity LSRelativity `json:"relativity"`
+}
+
+type LSResponse struct {
+	AbsolutePath []string `json:"absolute_path"`
+	// Returned so clients can display the full path to the user, and
+	// copy it to configure file sync
+	// e.g. Windows: "C:\\Users\\coder"
+	//      Linux: "/home/coder"
+	AbsolutePathString string   `json:"absolute_path_string"`
+	Contents           []LSFile `json:"contents"`
+}
+
+type LSFile struct {
+	Name string `json:"name"`
+	// e.g. "C:\\Users\\coder\\hello.txt"
+	//      "/home/coder/hello.txt"
+	AbsolutePathString string `json:"absolute_path_string"`
+	IsDir              bool   `json:"is_dir"`
+}
+
+type LSRelativity string
+
+const (
+	LSRelativityRoot LSRelativity = "root"
+	LSRelativityHome LSRelativity = "home"
+)

agent/ls_internal_test.go

@@ -0,0 +1,208 @@
+package agent
+
+import (
+	"os"
+	"path/filepath"
+	"runtime"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestListFilesNonExistentDirectory(t *testing.T) {
+	t.Parallel()
+
+	query := LSRequest{
+		Path:       []string{"idontexist"},
+		Relativity: LSRelativityHome,
+	}
+	_, err := listFiles(query)
+	require.ErrorIs(t, err, os.ErrNotExist)
+}
+
+func TestListFilesPermissionDenied(t *testing.T) {
+	t.Parallel()
+
+	if runtime.GOOS == "windows" {
+		t.Skip("creating an unreadable-by-user directory is non-trivial on Windows")
+	}
+
+	home, err := os.UserHomeDir()
+	require.NoError(t, err)
+
+	tmpDir := t.TempDir()
+
+	reposDir := filepath.Join(tmpDir, "repos")
+	err = os.Mkdir(reposDir, 0o000)
+	require.NoError(t, err)
+
+	rel, err := filepath.Rel(home, reposDir)
+	require.NoError(t, err)
+
+	query := LSRequest{
+		Path:       pathToArray(rel),
+		Relativity: LSRelativityHome,
+	}
+	_, err = listFiles(query)
+	require.ErrorIs(t, err, os.ErrPermission)
+}
+
+func TestListFilesNotADirectory(t *testing.T) {
+	t.Parallel()
+
+	home, err := os.UserHomeDir()
+	require.NoError(t, err)
+
+	tmpDir := t.TempDir()
+
+	filePath := filepath.Join(tmpDir, "file.txt")
+	err = os.WriteFile(filePath, []byte("content"), 0o600)
+	require.NoError(t, err)
+
+	rel, err := filepath.Rel(home, filePath)
+	require.NoError(t, err)
+
+	query := LSRequest{
+		Path:       pathToArray(rel),
+		Relativity: LSRelativityHome,
+	}
+	_, err = listFiles(query)
+	require.ErrorContains(t, err, "is not a directory")
+}
+
+func TestListFilesSuccess(t *testing.T) {
+	t.Parallel()
+
+	tc := []struct {
+		name       string
+		baseFunc   func(t *testing.T) string
+		relativity LSRelativity
+	}{
+		{
+			name: "home",
+			baseFunc: func(t *testing.T) string {
+				home, err := os.UserHomeDir()
+				require.NoError(t, err)
+				return home
+			},
+			relativity: LSRelativityHome,
+		},
+		{
+			name: "root",
+			baseFunc: func(*testing.T) string {
+				if runtime.GOOS == "windows" {
+					return ""
+				}
+				return "/"
+			},
+			relativity: LSRelativityRoot,
+		},
+	}
+
+	// nolint:paralleltest // Not since Go v1.22.
+	for _, tc := range tc {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+
+			base := tc.baseFunc(t)
+			tmpDir := t.TempDir()
+
+			reposDir := filepath.Join(tmpDir, "repos")
+			err := os.Mkdir(reposDir, 0o755)
+			require.NoError(t, err)
+
+			downloadsDir := filepath.Join(tmpDir, "Downloads")
+			err = os.Mkdir(downloadsDir, 0o755)
+			require.NoError(t, err)
+
+			textFile := filepath.Join(tmpDir, "file.txt")
+			err = os.WriteFile(textFile, []byte("content"), 0o600)
+			require.NoError(t, err)
+
+			var queryComponents []string
+			// We can't get an absolute path relative to empty string on Windows.
+			if runtime.GOOS == "windows" && base == "" {
+				queryComponents = pathToArray(tmpDir)
+			} else {
+				rel, err := filepath.Rel(base, tmpDir)
+				require.NoError(t, err)
+				queryComponents = pathToArray(rel)
+			}
+
+			query := LSRequest{
+				Path:       queryComponents,
+				Relativity: tc.relativity,
+			}
+			resp, err := listFiles(query)
+			require.NoError(t, err)
+
+			require.Equal(t, tmpDir, resp.AbsolutePathString)
+			// Output is sorted
+			require.Equal(t, []LSFile{
+				{
+					Name:               "Downloads",
+					AbsolutePathString: downloadsDir,
+					IsDir:              true,
+				},
+				{
+					Name:               "repos",
+					AbsolutePathString: reposDir,
+					IsDir:              true,
+				},
+				{
+					Name:               "file.txt",
+					AbsolutePathString: textFile,
+					IsDir:              false,
+				},
+			}, resp.Contents)
+		})
+	}
+}
+
+func TestListFilesListDrives(t *testing.T) {
+	t.Parallel()
+
+	if runtime.GOOS != "windows" {
+		t.Skip("skipping test on non-Windows OS")
+	}
+
+	query := LSRequest{
+		Path:       []string{},
+		Relativity: LSRelativityRoot,
+	}
+	resp, err := listFiles(query)
+	require.NoError(t, err)
+	require.Contains(t, resp.Contents, LSFile{
+		Name:               "C:\\",
+		AbsolutePathString: "C:\\",
+		IsDir:              true,
+	})
+
+	query = LSRequest{
+		Path:       []string{"C:\\"},
+		Relativity: LSRelativityRoot,
+	}
+	resp, err = listFiles(query)
+	require.NoError(t, err)
+
+	query = LSRequest{
+		Path:       resp.AbsolutePath,
+		Relativity: LSRelativityRoot,
+	}
+	resp, err = listFiles(query)
+	require.NoError(t, err)
+	// System directory should always exist
+	require.Contains(t, resp.Contents, LSFile{
+		Name:               "Windows",
+		AbsolutePathString: "C:\\Windows",
+		IsDir:              true,
+	})
+
+	query = LSRequest{
+		// Network drives are not supported.
+		Path:       []string{"\\sshfs\\work"},
+		Relativity: LSRelativityRoot,
+	}
+	resp, err = listFiles(query)
+	require.ErrorContains(t, err, "drive")
+}

agent/metrics.go

@@ -89,21 +89,22 @@ func (a *agent) collectMetrics(ctx context.Context) []*proto.Stats_Metric {
 		for _, metric := range metricFamily.GetMetric() {
 			labels := toAgentMetricLabels(metric.Label)
 
-			if metric.Counter != nil {
+			switch {
+			case metric.Counter != nil:
 				collected = append(collected, &proto.Stats_Metric{
 					Name:   metricFamily.GetName(),
 					Type:   proto.Stats_Metric_COUNTER,
 					Value:  metric.Counter.GetValue(),
 					Labels: labels,
 				})
-			} else if metric.Gauge != nil {
+			case metric.Gauge != nil:
 				collected = append(collected, &proto.Stats_Metric{
 					Name:   metricFamily.GetName(),
 					Type:   proto.Stats_Metric_GAUGE,
 					Value:  metric.Gauge.GetValue(),
 					Labels: labels,
 				})
-			} else {
+			default:
 				a.logger.Error(ctx, "unsupported metric type", slog.F("type", metricFamily.Type.String()))
 			}
 		}