fix: fetch custom roles from workspace agent context (cherry-pick #16237) (#16246)

Co-authored-by: Steven Masley <Emyrk@users.noreply.github.com>

.devcontainer/devcontainer.json

@@ -1,13 +1,13 @@
 {
-  "name": "Development environments on your infrastructure",
-  "image": "codercom/oss-dogfood:latest",
+	"name": "Development environments on your infrastructure",
+	"image": "codercom/oss-dogfood:latest",
 
-  "features": {
-    // See all possible options here https://github.com/devcontainers/features/tree/main/src/docker-in-docker
-    "ghcr.io/devcontainers/features/docker-in-docker:2": {
-      "moby": "false"
-    }
-  },
-  // SYS_PTRACE to enable go debugging
-  "runArgs": ["--cap-add=SYS_PTRACE"]
+	"features": {
+		// See all possible options here https://github.com/devcontainers/features/tree/main/src/docker-in-docker
+		"ghcr.io/devcontainers/features/docker-in-docker:2": {
+			"moby": "false"
+		}
+	},
+	// SYS_PTRACE to enable go debugging
+	"runArgs": ["--cap-add=SYS_PTRACE"]
 }

.editorconfig

@@ -7,7 +7,7 @@ trim_trailing_whitespace = true
 insert_final_newline = true
 indent_style = tab
 
-[*.{md,json,yaml,yml,tf,tfvars,nix}]
+[*.{yaml,yml,tf,tfvars,nix}]
 indent_style = space
 indent_size = 2
 

.git-blame-ignore-revs

@@ -3,3 +3,5 @@
 
 # chore: format code with semicolons when using prettier (#9555)
 988c9af0153561397686c119da9d1336d2433fdd
+# chore: use tabs for prettier and biome (#14283)
+95a7c0c4f087744a22c2e88dd3c5d30024d5fb02

.gitattributes

@@ -1,7 +1,7 @@
 # Generated files
 coderd/apidoc/docs.go linguist-generated=true
-docs/api/*.md linguist-generated=true
-docs/cli/*.md linguist-generated=true
+docs/reference/api/*.md linguist-generated=true
+docs/reference/cli/*.md linguist-generated=true
 coderd/apidoc/swagger.json linguist-generated=true
 coderd/database/dump.sql linguist-generated=true
 peerbroker/proto/*.go linguist-generated=true

.github/.linkspector.yml

@@ -0,0 +1,22 @@
+dirs:
+  - docs
+excludedDirs:
+  # Downstream bug in linkspector means large markdown files fail to parse
+  # but these are autogenerated and shouldn't need checking
+  - docs/reference
+  # Older changelogs may contain broken links
+  - docs/changelogs
+ignorePatterns:
+  - pattern: "localhost"
+  - pattern: "example.com"
+  - pattern: "mailto:"
+  - pattern: "127.0.0.1"
+  - pattern: "0.0.0.0"
+  - pattern: "JFROG_URL"
+  - pattern: "coder.company.org"
+  # These real sites were blocking the linkspector action / GitHub runner IPs(?)
+  - pattern: "i.imgur.com"
+  - pattern: "code.visualstudio.com"
+  - pattern: "www.emacswiki.org"
+aliveStatusCodes:
+  - 200

.github/actions/setup-go/action.yaml

@@ -4,12 +4,12 @@ description: |
 inputs:
   version:
     description: "The Go version to use."
-    default: "1.22.3"
+    default: "1.22.8"
 runs:
   using: "composite"
   steps:
     - name: Setup Go
-      uses: buildjet/setup-go@v5
+      uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
       with:
         go-version: ${{ inputs.version }}
 

.github/actions/setup-node/action.yaml

@@ -11,16 +11,16 @@ runs:
   using: "composite"
   steps:
     - name: Install pnpm
-      uses: pnpm/action-setup@v3
-      with:
-        version: 8
+      uses: pnpm/action-setup@fe02b34f77f8bc703788d5817da081398fad5dd2 # v4.0.0
+
     - name: Setup Node
-      uses: buildjet/setup-node@v4.0.1
+      uses: actions/setup-node@0a44ba7841725637a19e28fa30b79a866c81b0a6 # v4.0.4
       with:
-        node-version: 18.19.0
+        node-version: 20.16.0
         # See https://github.com/actions/setup-node#caching-global-packages-data
         cache: "pnpm"
         cache-dependency-path: ${{ inputs.directory }}/pnpm-lock.yaml
+
     - name: Install root node_modules
       shell: bash
       run: ./scripts/pnpm_install.sh

.github/actions/setup-sqlc/action.yaml

@@ -5,6 +5,6 @@ runs:
   using: "composite"
   steps:
     - name: Setup sqlc
-      uses: sqlc-dev/setup-sqlc@v4
+      uses: sqlc-dev/setup-sqlc@c0209b9199cd1cce6a14fc27cabcec491b651761 # v4.0.0
       with:
         sqlc-version: "1.25.0"

.github/actions/setup-tf/action.yaml

@@ -5,7 +5,7 @@ runs:
   using: "composite"
   steps:
     - name: Install Terraform
-      uses: hashicorp/setup-terraform@v3
+      uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd # v3.1.2
       with:
-        terraform_version: 1.8.4
+        terraform_version: 1.9.8
         terraform_wrapper: false

.github/actions/upload-datadog/action.yaml

@@ -1,5 +1,6 @@
 name: Upload tests to datadog
-if: always()
+description: |
+  Uploads the test results to datadog.
 inputs:
   api-key:
     description: "Datadog API key"

.github/dependabot.yaml

@@ -39,6 +39,10 @@ updates:
       prefix: "chore"
     labels: []
     open-pull-requests-limit: 15
+    groups:
+      x:
+        patterns:
+          - "golang.org/x/*"
     ignore:
       # Ignore patch updates for all dependencies
       - dependency-name: "*"
@@ -47,7 +51,13 @@ updates:
 
   # Update our Dockerfile.
   - package-ecosystem: "docker"
-    directory: "/scripts/"
+    directories:
+      - "/dogfood/contents"
+      - "/scripts"
+      - "/examples/templates/docker/build"
+      - "/examples/parameters/build"
+      - "/scaletest/templates/scaletest-runner"
+      - "/scripts/ironbank"
     schedule:
       interval: "weekly"
       time: "06:00"
@@ -61,7 +71,12 @@ updates:
       - dependency-name: "terraform"
 
   - package-ecosystem: "npm"
-    directory: "/site/"
+    directories:
+      - "/site"
+      - "/offlinedocs"
+      - "/scripts"
+      - "/scripts/apidocgen"
+
     schedule:
       interval: "monthly"
       time: "06:00"
@@ -71,44 +86,35 @@ updates:
     commit-message:
       prefix: "chore"
     labels: []
+    groups:
+      xterm:
+        patterns:
+          - "@xterm*"
+      mui:
+        patterns:
+          - "@mui*"
+      react:
+        patterns:
+          - "react"
+          - "react-dom"
+          - "@types/react"
+          - "@types/react-dom"
+      emotion:
+        patterns:
+          - "@emotion*"
+        exclude-patterns:
+          - "jest-runner-eslint"
+      jest:
+        patterns:
+          - "jest"
+          - "@types/jest"
+      vite:
+        patterns:
+          - "vite*"
+          - "@vitejs/plugin-react"
     ignore:
-      # Ignore patch updates for all dependencies
+      # Ignore major version updates to avoid breaking changes
       - dependency-name: "*"
-        update-types:
-          - version-update:semver-patch
-      # Ignore major updates to Node.js types, because they need to
-      # correspond to the Node.js engine version
-      - dependency-name: "@types/node"
         update-types:
           - version-update:semver-major
     open-pull-requests-limit: 15
-    groups:
-      site:
-        patterns:
-          - "*"
-
-  - package-ecosystem: "npm"
-    directory: "/offlinedocs/"
-    schedule:
-      interval: "monthly"
-      time: "06:00"
-      timezone: "America/Chicago"
-    reviewers:
-      - "coder/ts"
-    commit-message:
-      prefix: "chore"
-    labels: []
-    ignore:
-      # Ignore patch updates for all dependencies
-      - dependency-name: "*"
-        update-types:
-          - version-update:semver-patch
-      # Ignore major updates to Node.js types, because they need to
-      # correspond to the Node.js engine version
-      - dependency-name: "@types/node"
-        update-types:
-          - version-update:semver-major
-    groups:
-      offlinedocs:
-        patterns:
-          - "*"

.github/workflows/ci.yaml

@@ -9,16 +9,7 @@ on:
   workflow_dispatch:
 
 permissions:
-  actions: none
-  checks: none
   contents: read
-  deployments: none
-  issues: none
-  packages: write
-  pull-requests: none
-  repository-projects: none
-  security-events: none
-  statuses: none
 
 # Cancel in-progress runs for pull requests when developers push
 # additional changes
@@ -42,13 +33,18 @@ jobs:
       offlinedocs: ${{ steps.filter.outputs.offlinedocs }}
       tailnet-integration: ${{ steps.filter.outputs.tailnet-integration }}
     steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
+        with:
+          egress-policy: audit
+
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
         with:
           fetch-depth: 1
       # For pull requests it's not necessary to checkout the code
       - name: check changed files
-        uses: dorny/paths-filter@v3
+        uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2
         id: filter
         with:
           filters: |
@@ -84,7 +80,8 @@ jobs:
               - "cmd/**"
               - "coderd/**"
               - "enterprise/**"
-              - "examples/*"
+              - "examples/**"
+              - "helm/**"
               - "provisioner/**"
               - "provisionerd/**"
               - "provisionersdk/**"
@@ -117,32 +114,53 @@ jobs:
         run: |
           echo "${{ toJSON(steps.filter )}}"
 
-  update-flake:
-    needs: changes
-    if: needs.changes.outputs.gomod == 'true'
-    runs-on: ${{ github.repository_owner == 'coder' && 'buildjet-8vcpu-ubuntu-2204' || 'ubuntu-latest' }}
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 1
+  # Disabled due to instability. See: https://github.com/coder/coder/issues/14553
+  # Re-enable once the flake hash calculation is stable.
+  # update-flake:
+  #   needs: changes
+  #   if: needs.changes.outputs.gomod == 'true'
+  #   runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-8' || 'ubuntu-latest' }}
+  #   steps:
+  #     - name: Checkout
+  #       uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
+  #       with:
+  #         fetch-depth: 1
+  #         # See: https://github.com/stefanzweifel/git-auto-commit-action?tab=readme-ov-file#commits-made-by-this-action-do-not-trigger-new-workflow-runs
+  #         token: ${{ secrets.CDRCI_GITHUB_TOKEN }}
 
-      - name: Setup Go
-        uses: ./.github/actions/setup-go
+  #     - name: Setup Go
+  #       uses: ./.github/actions/setup-go
 
-      - name: Update Nix Flake SRI Hash
-        run: ./scripts/update-flake.sh
+  #     - name: Update Nix Flake SRI Hash
+  #       run: ./scripts/update-flake.sh
 
-      - name: Ensure No Changes
-        run: git diff --exit-code
+  #     # auto update flake for dependabot
+  #     - uses: stefanzweifel/git-auto-commit-action@8621497c8c39c72f3e2a999a26b4ca1b5058a842 # v5.0.1
+  #       if: github.actor == 'dependabot[bot]'
+  #       with:
+  #         # Allows dependabot to still rebase!
+  #         commit_message: "[dependabot skip] Update Nix Flake SRI Hash"
+  #         commit_user_name: "dependabot[bot]"
+  #         commit_user_email: "49699333+dependabot[bot]@users.noreply.github.com>"
+  #         commit_author: "dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>"
+
+  #     # require everyone else to update it themselves
+  #     - name: Ensure No Changes
+  #       if: github.actor != 'dependabot[bot]'
+  #       run: git diff --exit-code
 
   lint:
     needs: changes
     if: needs.changes.outputs.offlinedocs-only == 'false' || needs.changes.outputs.ci == 'true' || github.ref == 'refs/heads/main'
-    runs-on: ${{ github.repository_owner == 'coder' && 'buildjet-8vcpu-ubuntu-2204' || 'ubuntu-latest' }}
+    runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-8' || 'ubuntu-latest' }}
     steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
+        with:
+          egress-policy: audit
+
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
         with:
           fetch-depth: 1
 
@@ -154,13 +172,13 @@ jobs:
 
       - name: Get golangci-lint cache dir
         run: |
-          linter_ver=$(egrep -o 'GOLANGCI_LINT_VERSION=\S+' dogfood/Dockerfile | cut -d '=' -f 2)
+          linter_ver=$(egrep -o 'GOLANGCI_LINT_VERSION=\S+' dogfood/contents/Dockerfile | cut -d '=' -f 2)
           go install github.com/golangci/golangci-lint/cmd/golangci-lint@v$linter_ver
           dir=$(golangci-lint cache status | awk '/Dir/ { print $2 }')
           echo "LINT_CACHE_DIR=$dir" >> $GITHUB_ENV
 
       - name: golangci-lint cache
-        uses: buildjet/cache@v4
+        uses: actions/cache@2cdf405574d6ef1f33a1d12acccd3ae82f47b3f2 # v4.1.0
         with:
           path: |
             ${{ env.LINT_CACHE_DIR }}
@@ -170,7 +188,7 @@ jobs:
 
       # Check for any typos
       - name: Check for typos
-        uses: crate-ci/typos@v1.21.0
+        uses: crate-ci/typos@b74202f74b4346efdbce7801d187ec57b266bac8 # v1.27.3
         with:
           config: .github/workflows/typos.toml
 
@@ -183,7 +201,7 @@ jobs:
 
       # Needed for helm chart linting
       - name: Install helm
-        uses: azure/setup-helm@v4
+        uses: azure/setup-helm@fe7b79cd5ee1e45176fcad797de68ecaf3ca4814 # v4.2.0
         with:
           version: v3.9.2
 
@@ -191,14 +209,30 @@ jobs:
         run: |
           make --output-sync=line -j lint
 
+      - name: Check workflow files
+        run: |
+          bash <(curl https://raw.githubusercontent.com/rhysd/actionlint/main/scripts/download-actionlint.bash) 1.7.4
+          ./actionlint -color -shellcheck= -ignore "set-output"
+        shell: bash
+
+      - name: Check for unstaged files
+        run: |
+          rm -f ./actionlint ./typos
+          ./scripts/check_unstaged.sh
+        shell: bash
+
   gen:
     timeout-minutes: 8
-    runs-on: ${{ github.repository_owner == 'coder' && 'buildjet-8vcpu-ubuntu-2204' || 'ubuntu-latest' }}
-    needs: changes
-    if: needs.changes.outputs.docs-only == 'false' || needs.changes.outputs.ci == 'true' || github.ref == 'refs/heads/main'
+    runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-8' || 'ubuntu-latest' }}
+    if: always()
     steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
+        with:
+          egress-policy: audit
+
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
         with:
           fetch-depth: 1
 
@@ -243,23 +277,25 @@ jobs:
   fmt:
     needs: changes
     if: needs.changes.outputs.offlinedocs-only == 'false' || needs.changes.outputs.ci == 'true' || github.ref == 'refs/heads/main'
-    runs-on: ${{ github.repository_owner == 'coder' && 'buildjet-8vcpu-ubuntu-2204' || 'ubuntu-latest' }}
+    runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-8' || 'ubuntu-latest' }}
     timeout-minutes: 7
     steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
+        with:
+          egress-policy: audit
+
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
         with:
           fetch-depth: 1
 
       - name: Setup Node
         uses: ./.github/actions/setup-node
 
+      # Use default Go version
       - name: Setup Go
-        uses: buildjet/setup-go@v5
-        with:
-          # This doesn't need caching. It's super fast anyways!
-          cache: false
-          go-version: 1.21.9
+        uses: ./.github/actions/setup-go
 
       - name: Install shfmt
         run: go install mvdan.cc/sh/v3/cmd/shfmt@v3.7.0
@@ -273,7 +309,7 @@ jobs:
         run: ./scripts/check_unstaged.sh
 
   test-go:
-    runs-on: ${{ matrix.os == 'ubuntu-latest' && github.repository_owner == 'coder' && 'buildjet-4vcpu-ubuntu-2204' || matrix.os == 'macos-latest' && github.repository_owner == 'coder' && 'macos-latest-xlarge' || matrix.os == 'windows-2022' && github.repository_owner == 'coder' && 'windows-latest-16-cores' || matrix.os }}
+    runs-on: ${{ matrix.os == 'ubuntu-latest' && github.repository_owner == 'coder' && 'depot-ubuntu-22.04-4' || matrix.os == 'macos-latest' && github.repository_owner == 'coder' && 'macos-latest-xlarge' || matrix.os == 'windows-2022' && github.repository_owner == 'coder' && 'windows-latest-16-cores' || matrix.os }}
     needs: changes
     if: needs.changes.outputs.go == 'true' || needs.changes.outputs.ci == 'true' || github.ref == 'refs/heads/main'
     timeout-minutes: 20
@@ -285,8 +321,13 @@ jobs:
           - macos-latest
           - windows-2022
     steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
+        with:
+          egress-policy: audit
+
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
         with:
           fetch-depth: 1
 
@@ -329,7 +370,7 @@ jobs:
           api-key: ${{ secrets.DATADOG_API_KEY }}
 
   test-go-pg:
-    runs-on: ${{ github.repository_owner == 'coder' && 'buildjet-8vcpu-ubuntu-2204' || 'ubuntu-latest' }}
+    runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-8' || 'ubuntu-latest' }}
     needs:
       - changes
     if: needs.changes.outputs.go == 'true' || needs.changes.outputs.ci == 'true' || github.ref == 'refs/heads/main'
@@ -339,8 +380,13 @@ jobs:
     # even if some of the preceding steps are slow.
     timeout-minutes: 25
     steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
+        with:
+          egress-policy: audit
+
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
         with:
           fetch-depth: 1
 
@@ -351,8 +397,55 @@ jobs:
         uses: ./.github/actions/setup-tf
 
       - name: Test with PostgreSQL Database
+        env:
+          POSTGRES_VERSION: "13"
+          TS_DEBUG_DISCO: "true"
+        run: |
+          make test-postgres
+
+      - name: Upload test stats to Datadog
+        timeout-minutes: 1
+        continue-on-error: true
+        uses: ./.github/actions/upload-datadog
+        if: success() || failure()
+        with:
+          api-key: ${{ secrets.DATADOG_API_KEY }}
+
+  # NOTE: this could instead be defined as a matrix strategy, but we want to
+  # only block merging if tests on postgres 13 fail. Using a matrix strategy
+  # here makes the check in the above `required` job rather complicated.
+  test-go-pg-16:
+    runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-8' || 'ubuntu-latest' }}
+    needs:
+      - changes
+    if: needs.changes.outputs.go == 'true' || needs.changes.outputs.ci == 'true' || github.ref == 'refs/heads/main'
+    # This timeout must be greater than the timeout set by `go test` in
+    # `make test-postgres` to ensure we receive a trace of running
+    # goroutines. Setting this to the timeout +5m should work quite well
+    # even if some of the preceding steps are slow.
+    timeout-minutes: 25
+    steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
+        with:
+          egress-policy: audit
+
+      - name: Checkout
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
+        with:
+          fetch-depth: 1
+
+      - name: Setup Go
+        uses: ./.github/actions/setup-go
+
+      - name: Setup Terraform
+        uses: ./.github/actions/setup-tf
+
+      - name: Test with PostgreSQL Database
+        env:
+          POSTGRES_VERSION: "16"
+          TS_DEBUG_DISCO: "true"
         run: |
-          export TS_DEBUG_DISCO=true
           make test-postgres
 
       - name: Upload test stats to Datadog
@@ -364,13 +457,18 @@ jobs:
           api-key: ${{ secrets.DATADOG_API_KEY }}
 
   test-go-race:
-    runs-on: ${{ github.repository_owner == 'coder' && 'buildjet-8vcpu-ubuntu-2204' || 'ubuntu-latest' }}
+    runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-16' || 'ubuntu-latest' }}
     needs: changes
     if: needs.changes.outputs.go == 'true' || needs.changes.outputs.ci == 'true' || github.ref == 'refs/heads/main'
     timeout-minutes: 25
     steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
+        with:
+          egress-policy: audit
+
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
         with:
           fetch-depth: 1
 
@@ -380,9 +478,13 @@ jobs:
       - name: Setup Terraform
         uses: ./.github/actions/setup-tf
 
+      # We run race tests with reduced parallelism because they use more CPU and we were finding
+      # instances where tests appear to hang for multiple seconds, resulting in flaky tests when
+      # short timeouts are used.
+      # c.f. discussion on https://github.com/coder/coder/pull/15106
       - name: Run Tests
         run: |
-          gotestsum --junitfile="gotests.xml" -- -race ./...
+          gotestsum --junitfile="gotests.xml" -- -race -parallel 4 -p 4 ./...
 
       - name: Upload test stats to Datadog
         timeout-minutes: 1
@@ -399,14 +501,19 @@ jobs:
   # These tests are skipped in the main go test jobs because they require root
   # and mess with networking.
   test-go-tailnet-integration:
-    runs-on: ${{ github.repository_owner == 'coder' && 'buildjet-8vcpu-ubuntu-2204' || 'ubuntu-latest' }}
+    runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-8' || 'ubuntu-latest' }}
     needs: changes
     # Unnecessary to run on main for now
     if: needs.changes.outputs.tailnet-integration == 'true' || needs.changes.outputs.ci == 'true'
     timeout-minutes: 20
     steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
+        with:
+          egress-policy: audit
+
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
         with:
           fetch-depth: 1
 
@@ -421,13 +528,18 @@ jobs:
         run: make test-tailnet-integration
 
   test-js:
-    runs-on: ${{ github.repository_owner == 'coder' && 'buildjet-8vcpu-ubuntu-2204' || 'ubuntu-latest' }}
+    runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-8' || 'ubuntu-latest' }}
     needs: changes
     if: needs.changes.outputs.ts == 'true' || needs.changes.outputs.ci == 'true' || github.ref == 'refs/heads/main'
     timeout-minutes: 20
     steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
+        with:
+          egress-policy: audit
+
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
         with:
           fetch-depth: 1
 
@@ -438,7 +550,8 @@ jobs:
         working-directory: site
 
   test-e2e:
-    runs-on: ${{ github.repository_owner == 'coder' && 'buildjet-16vcpu-ubuntu-2204' || 'ubuntu-latest' }}
+    # test-e2e fails on 2-core 8GB runners, so we use the 4-core 16GB runner
+    runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-4' || 'ubuntu-latest' }}
     needs: changes
     if: needs.changes.outputs.go == 'true' || needs.changes.outputs.ts == 'true' || needs.changes.outputs.ci == 'true' || github.ref == 'refs/heads/main'
     timeout-minutes: 20
@@ -446,14 +559,19 @@ jobs:
       fail-fast: false
       matrix:
         variant:
-          - enterprise: false
+          - premium: false
             name: test-e2e
-          - enterprise: true
-            name: test-e2e-enterprise
+          - premium: true
+            name: test-e2e-premium
     name: ${{ matrix.variant.name }}
     steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
+        with:
+          egress-policy: audit
+
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
         with:
           fetch-depth: 1
 
@@ -473,38 +591,35 @@ jobs:
       - run: pnpm playwright:install
         working-directory: site
 
-      # Run tests that don't require an enterprise license without an enterprise license
+      # Run tests that don't require a premium license without a premium license
       - run: pnpm playwright:test --forbid-only --workers 1
-        if: ${{ !matrix.variant.enterprise }}
+        if: ${{ !matrix.variant.premium }}
         env:
           DEBUG: pw:api
         working-directory: site
 
-      # Run all of the tests with an enterprise license
+      # Run all of the tests with a premium license
       - run: pnpm playwright:test --forbid-only --workers 1
-        if: ${{ matrix.variant.enterprise }}
+        if: ${{ matrix.variant.premium }}
         env:
           DEBUG: pw:api
-          CODER_E2E_ENTERPRISE_LICENSE: ${{ secrets.CODER_E2E_ENTERPRISE_LICENSE }}
-          CODER_E2E_REQUIRE_ENTERPRISE_TESTS: "1"
+          CODER_E2E_LICENSE: ${{ secrets.CODER_E2E_LICENSE }}
+          CODER_E2E_REQUIRE_PREMIUM_TESTS: "1"
         working-directory: site
-        # Temporarily allow these to fail so that I can gather data about which
-        # tests are failing.
-        continue-on-error: true
 
       - name: Upload Playwright Failed Tests
         if: always() && github.actor != 'dependabot[bot]' && runner.os == 'Linux' && !github.event.pull_request.head.repo.fork
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@604373da6381bf24206979c74d06a550515601b9 # v4.4.1
         with:
-          name: failed-test-videos${{ matrix.variant.enterprise && '-enterprise' || '-agpl' }}
+          name: failed-test-videos${{ matrix.variant.premium && '-premium' || '' }}
           path: ./site/test-results/**/*.webm
           retention-days: 7
 
       - name: Upload pprof dumps
         if: always() && github.actor != 'dependabot[bot]' && runner.os == 'Linux' && !github.event.pull_request.head.repo.fork
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@604373da6381bf24206979c74d06a550515601b9 # v4.4.1
         with:
-          name: debug-pprof-dumps${{ matrix.variant.enterprise && '-enterprise' || '-agpl'  }}
+          name: debug-pprof-dumps${{ matrix.variant.premium && '-premium' || ''  }}
           path: ./site/test-results/**/debug-pprof-*.txt
           retention-days: 7
 
@@ -514,8 +629,13 @@ jobs:
     needs: changes
     if: needs.changes.outputs.ts == 'true' || needs.changes.outputs.ci == 'true'
     steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
+        with:
+          egress-policy: audit
+
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
         with:
           # Required by Chromatic for build-over-build history, otherwise we
           # only get 1 commit on shallow checkout.
@@ -529,7 +649,7 @@ jobs:
       # the check to pass. This is desired in PRs, but not in mainline.
       - name: Publish to Chromatic (non-mainline)
         if: github.ref != 'refs/heads/main' && github.repository_owner == 'coder'
-        uses: chromaui/action@v10
+        uses: chromaui/action@30b6228aa809059d46219e0f556752e8672a7e26 # v11.11.0
         env:
           NODE_OPTIONS: "--max_old_space_size=4096"
           STORYBOOK: true
@@ -560,7 +680,7 @@ jobs:
       # infinitely "in progress" in mainline unless we re-review each build.
       - name: Publish to Chromatic (mainline)
         if: github.ref == 'refs/heads/main' && github.repository_owner == 'coder'
-        uses: chromaui/action@v10
+        uses: chromaui/action@30b6228aa809059d46219e0f556752e8672a7e26 # v11.11.0
         env:
           NODE_OPTIONS: "--max_old_space_size=4096"
           STORYBOOK: true
@@ -582,12 +702,17 @@ jobs:
   offlinedocs:
     name: offlinedocs
     needs: changes
-    runs-on: ${{ github.repository_owner == 'coder' && 'buildjet-8vcpu-ubuntu-2204' || 'ubuntu-latest' }}
+    runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-8' || 'ubuntu-latest' }}
     if: needs.changes.outputs.offlinedocs == 'true' || needs.changes.outputs.ci == 'true' || needs.changes.outputs.docs == 'true'
 
     steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
+        with:
+          egress-policy: audit
+
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
         with:
           # 0 is required here for version.sh to work.
           fetch-depth: 0
@@ -650,11 +775,15 @@ jobs:
       - test-e2e
       - offlinedocs
       - sqlc-vet
-      - dependency-license-review
     # Allow this job to run even if the needed jobs fail, are skipped or
     # cancelled.
     if: always()
     steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
+        with:
+          egress-policy: audit
+
       - name: Ensure required checks
         run: |
           echo "Checking required checks"
@@ -667,7 +796,6 @@ jobs:
           echo "- test-js: ${{ needs.test-js.result }}"
           echo "- test-e2e: ${{ needs.test-e2e.result }}"
           echo "- offlinedocs: ${{ needs.offlinedocs.result }}"
-          echo "- dependency-license-review: ${{ needs.dependency-license-review.result }}"
           echo
 
           # We allow skipped jobs to pass, but not failed or cancelled jobs.
@@ -678,25 +806,112 @@ jobs:
 
           echo "Required checks have passed"
 
+  # Builds the dylibs and upload it as an artifact so it can be embedded in the main build
+  build-dylib:
+    needs: changes
+    # We always build the dylibs on Go changes to verify we're not merging unbuildable code,
+    # but they need only be signed and uploaded on coder/coder main.
+    if: needs.changes.outputs.docs-only == 'false' || github.ref == 'refs/heads/main'
+    runs-on: ${{ github.repository_owner == 'coder' && 'depot-macos-latest' || 'macos-latest' }}
+    steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
+        with:
+          egress-policy: audit
+
+      - name: Checkout
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
+        with:
+          fetch-depth: 0
+
+      - name: Setup build tools
+        run: |
+          brew install bash gnu-getopt make
+          echo "$(brew --prefix bash)/bin" >> $GITHUB_PATH
+          echo "$(brew --prefix gnu-getopt)/bin" >> $GITHUB_PATH
+          echo "$(brew --prefix make)/libexec/gnubin" >> $GITHUB_PATH
+
+      - name: Setup Go
+        uses: ./.github/actions/setup-go
+
+      - name: Install rcodesign
+        if: ${{ github.repository_owner == 'coder' && github.ref == 'refs/heads/main' }}
+        run: |
+          set -euo pipefail
+          wget -O /tmp/rcodesign.tar.gz https://github.com/indygreg/apple-platform-rs/releases/download/apple-codesign%2F0.22.0/apple-codesign-0.22.0-macos-universal.tar.gz
+          sudo tar -xzf /tmp/rcodesign.tar.gz \
+            -C /usr/local/bin \
+            --strip-components=1 \
+            apple-codesign-0.22.0-macos-universal/rcodesign
+          rm /tmp/rcodesign.tar.gz
+
+      - name: Setup Apple Developer certificate and API key
+        if: ${{ github.repository_owner == 'coder' && github.ref == 'refs/heads/main' }}
+        run: |
+          set -euo pipefail
+          touch /tmp/{apple_cert.p12,apple_cert_password.txt,apple_apikey.p8}
+          chmod 600 /tmp/{apple_cert.p12,apple_cert_password.txt,apple_apikey.p8}
+          echo "$AC_CERTIFICATE_P12_BASE64" | base64 -d > /tmp/apple_cert.p12
+          echo "$AC_CERTIFICATE_PASSWORD" > /tmp/apple_cert_password.txt
+          echo "$AC_APIKEY_P8_BASE64" | base64 -d > /tmp/apple_apikey.p8
+        env:
+          AC_CERTIFICATE_P12_BASE64: ${{ secrets.AC_CERTIFICATE_P12_BASE64 }}
+          AC_CERTIFICATE_PASSWORD: ${{ secrets.AC_CERTIFICATE_PASSWORD }}
+          AC_APIKEY_P8_BASE64: ${{ secrets.AC_APIKEY_P8_BASE64 }}
+
+      - name: Build dylibs
+        run: |
+          set -euxo pipefail
+          go mod download
+
+          make gen/mark-fresh
+          make build/coder-dylib
+        env:
+          CODER_SIGN_DARWIN: ${{ github.ref == 'refs/heads/main' && '1' || '0' }}
+          AC_CERTIFICATE_FILE: /tmp/apple_cert.p12
+          AC_CERTIFICATE_PASSWORD_FILE: /tmp/apple_cert_password.txt
+
+      - name: Upload build artifacts
+        if: ${{ github.repository_owner == 'coder' && github.ref == 'refs/heads/main' }}
+        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+        with:
+          name: dylibs
+          path: |
+            ./build/*.h
+            ./build/*.dylib
+          retention-days: 7
+
+      - name: Delete Apple Developer certificate and API key
+        if: ${{ github.repository_owner == 'coder' && github.ref == 'refs/heads/main' }}
+        run: rm -f /tmp/{apple_cert.p12,apple_cert_password.txt,apple_apikey.p8}
+
   build:
     # This builds and publishes ghcr.io/coder/coder-preview:main for each commit
-    # to main branch. We are only building this for amd64 platform. (>95% pulls
-    # are for amd64)
-    needs: changes
+    # to main branch.
+    needs:
+      - changes
+      - build-dylib
     if: github.ref == 'refs/heads/main' && needs.changes.outputs.docs-only == 'false' && !github.event.pull_request.head.repo.fork
-    runs-on: ${{ github.repository_owner == 'coder' && 'buildjet-8vcpu-ubuntu-2204' || 'ubuntu-latest' }}
+    runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-8' || 'ubuntu-latest' }}
+    permissions:
+      packages: write # Needed to push images to ghcr.io
     env:
       DOCKER_CLI_EXPERIMENTAL: "enabled"
     outputs:
       IMAGE: ghcr.io/coder/coder-preview:${{ steps.build-docker.outputs.tag }}
     steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
+        with:
+          egress-policy: audit
+
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
         with:
           fetch-depth: 0
 
       - name: GHCR Login
-        uses: docker/login-action@v3
+        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
@@ -714,6 +929,18 @@ jobs:
       - name: Install zstd
         run: sudo apt-get install -y zstd
 
+      - name: Download dylibs
+        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
+        with:
+          name: dylibs
+          path: ./build
+
+      - name: Insert dylibs
+        run: |
+          mv ./build/*amd64.dylib ./site/out/bin/coder-vpn-darwin-amd64.dylib
+          mv ./build/*arm64.dylib ./site/out/bin/coder-vpn-darwin-arm64.dylib
+          mv ./build/*arm64.h     ./site/out/bin/coder-vpn-darwin-dylib.h
+
       - name: Build
         run: |
           set -euxo pipefail
@@ -744,13 +971,15 @@ jobs:
           echo "tag=$tag" >> $GITHUB_OUTPUT
 
           # build images for each architecture
-          make -j build/coder_"$version"_linux_{amd64,arm64,armv7}.tag
+          # note: omitting the -j argument to avoid race conditions when pushing
+          make build/coder_"$version"_linux_{amd64,arm64,armv7}.tag
 
           # only push if we are on main branch
           if [ "${{ github.ref }}" == "refs/heads/main" ]; then
             # build and push multi-arch manifest, this depends on the other images
             # being pushed so will automatically push them
-            make -j push/build/coder_"$version"_linux_{amd64,arm64,armv7}.tag
+            # note: omitting the -j argument to avoid race conditions when pushing
+            make push/build/coder_"$version"_linux_{amd64,arm64,armv7}.tag
 
             # Define specific tags
             tags=("$tag" "main" "latest")
@@ -769,7 +998,7 @@ jobs:
 
       - name: Prune old images
         if: github.ref == 'refs/heads/main'
-        uses: vlaurin/action-ghcr-prune@v0.6.0
+        uses: vlaurin/action-ghcr-prune@0cf7d39f88546edd31965acba78cdcb0be14d641 # v0.6.0
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           organization: coder
@@ -784,7 +1013,7 @@ jobs:
 
       - name: Upload build artifacts
         if: github.ref == 'refs/heads/main'
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@604373da6381bf24206979c74d06a550515601b9 # v4.4.1
         with:
           name: coder
           path: |
@@ -807,28 +1036,33 @@ jobs:
       contents: read
       id-token: write
     steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
+        with:
+          egress-policy: audit
+
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
         with:
           fetch-depth: 0
 
       - name: Authenticate to Google Cloud
-        uses: google-github-actions/auth@v2
+        uses: google-github-actions/auth@6fc4af4b145ae7821d527454aa9bd537d1f2dc5f # v2.1.7
         with:
           workload_identity_provider: projects/573722524737/locations/global/workloadIdentityPools/github/providers/github
           service_account: coder-ci@coder-dogfood.iam.gserviceaccount.com
 
       - name: Set up Google Cloud SDK
-        uses: google-github-actions/setup-gcloud@v2
+        uses: google-github-actions/setup-gcloud@6189d56e4096ee891640bb02ac264be376592d6a # v2.1.2
 
       - name: Set up Flux CLI
-        uses: fluxcd/flux2/action@main
+        uses: fluxcd/flux2/action@5350425cdcd5fa015337e09fa502153c0275bd4b # v2.4.0
         with:
-          # Keep this up to date with the version of flux installed in dogfood cluster
+          # Keep this and the github action up to date with the version of flux installed in dogfood cluster
           version: "2.2.1"
 
       - name: Get Cluster Credentials
-        uses: "google-github-actions/get-gke-credentials@v2"
+        uses: google-github-actions/get-gke-credentials@206d64b64b0eba0a6e2f25113d044c31776ca8d6 # v2.2.2
         with:
           cluster_name: dogfood-v2
           location: us-central1-a
@@ -864,13 +1098,18 @@ jobs:
     needs: build
     if: github.ref == 'refs/heads/main' && !github.event.pull_request.head.repo.fork
     steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
+        with:
+          egress-policy: audit
+
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
         with:
           fetch-depth: 0
 
       - name: Setup flyctl
-        uses: superfly/flyctl-actions/setup-flyctl@master
+        uses: superfly/flyctl-actions/setup-flyctl@fc53c09e1bc3be6f54706524e3b82c4f462f77be # v1.5
 
       - name: Deploy workspace proxies
         run: |
@@ -890,12 +1129,17 @@ jobs:
   # runs sqlc-vet to ensure all queries are valid. This catches any mistakes
   # in migrations or sqlc queries that makes a query unable to be prepared.
   sqlc-vet:
-    runs-on: ${{ github.repository_owner == 'coder' && 'buildjet-8vcpu-ubuntu-2204' || 'ubuntu-latest' }}
+    runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-8' || 'ubuntu-latest' }}
     needs: changes
     if: needs.changes.outputs.db == 'true' || needs.changes.outputs.ci == 'true' || github.ref == 'refs/heads/main'
     steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
+        with:
+          egress-policy: audit
+
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
         with:
           fetch-depth: 1
       # We need golang to run the migration main.go
@@ -908,43 +1152,3 @@ jobs:
       - name: Setup and run sqlc vet
         run: |
           make sqlc-vet
-
-  # dependency-license-review checks that no license-incompatible dependencies have been introduced.
-  # This action is not intended to do a vulnerability check since that is handled by a separate action.
-  dependency-license-review:
-    runs-on: ubuntu-latest
-    if: github.ref != 'refs/heads/main' && github.actor != 'dependabot[bot]'
-    steps:
-      - name: "Checkout Repository"
-        uses: actions/checkout@v4
-      - name: "Dependency Review"
-        id: review
-        uses: actions/dependency-review-action@v4.3.2
-        with:
-          allow-licenses: Apache-2.0, BSD-2-Clause, BSD-3-Clause, CC0-1.0, ISC, MIT, MIT-0, MPL-2.0
-          allow-dependencies-licenses: "pkg:golang/github.com/coder/wgtunnel@0.1.13-0.20240522110300-ade90dfb2da0"
-          license-check: true
-          vulnerability-check: false
-      - name: "Report"
-        # make sure this step runs even if the previous failed
-        if: always()
-        shell: bash
-        env:
-          VULNERABLE_CHANGES: ${{ steps.review.outputs.invalid-license-changes }}
-        run: |
-          fields=( "unlicensed" "unresolved" "forbidden" )
-
-          # This is unfortunate that we have to do this but the action does not support failing on
-          # an unknown license. The unknown dependency could easily have a GPL license which
-          # would be problematic for us.
-          # Track https://github.com/actions/dependency-review-action/issues/672 for when
-          # we can remove this brittle workaround.
-          for field in "${fields[@]}"; do
-            # Use jq to check if the array is not empty
-            if [[ $(echo "$VULNERABLE_CHANGES" | jq ".${field} | length") -ne 0 ]]; then
-              echo "Invalid or unknown licenses detected, contact @sreya to ensure your added dependency falls under one of our allowed licenses."
-              echo "$VULNERABLE_CHANGES" | jq
-              exit 1
-            fi
-          done
-          echo "No incompatible licenses detected"

.github/workflows/contrib.yaml

@@ -13,6 +13,11 @@ on:
       - opened
       - reopened
       - edited
+      # For jobs that don't run on draft PRs.
+      - ready_for_review
+
+permissions:
+  contents: read
 
 # Only run one instance per PR to ensure in-order execution.
 concurrency: pr-${{ github.ref }}
@@ -25,16 +30,26 @@ jobs:
     permissions:
       pull-requests: write
     steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
+        with:
+          egress-policy: audit
+
       - name: auto-approve dependabot
-        uses: hmarr/auto-approve-action@v4
+        uses: hmarr/auto-approve-action@f0939ea97e9205ef24d872e76833fa908a770363 # v4.0.0
         if: github.actor == 'dependabot[bot]'
 
   cla:
     runs-on: ubuntu-latest
     steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
+        with:
+          egress-policy: audit
+
       - name: cla
         if: (github.event.comment.body == 'recheck' || github.event.comment.body == 'I have read the CLA Document and I hereby sign the CLA') || github.event_name == 'pull_request_target'
-        uses: contributor-assistant/github-action@v2.4.0
+        uses: contributor-assistant/github-action@ca4a40a7d1004f18d9960b404b97e5f30a505a08 # v2.6.1
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           # the below token should have repo scope and must be manually added by you in the repository's secret
@@ -52,10 +67,15 @@ jobs:
   release-labels:
     runs-on: ubuntu-latest
     # Skip tagging for draft PRs.
-    if: ${{ github.event_name == 'pull_request_target' && success() && !github.event.pull_request.draft }}
+    if: ${{ github.event_name == 'pull_request_target' && !github.event.pull_request.draft }}
     steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
+        with:
+          egress-policy: audit
+
       - name: release-labels
-        uses: actions/github-script@v7
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
         with:
           # This script ensures PR title and labels are in sync:
           #

.github/workflows/docker-base.yaml

@@ -22,10 +22,6 @@ on:
 
 permissions:
   contents: read
-  # Necessary to push docker images to ghcr.io.
-  packages: write
-  # Necessary for depot.dev authentication.
-  id-token: write
 
 # Avoid running multiple jobs for the same commit.
 concurrency:
@@ -33,14 +29,24 @@ concurrency:
 
 jobs:
   build:
+    permissions:
+      # Necessary for depot.dev authentication.
+      id-token: write
+      # Necessary to push docker images to ghcr.io.
+      packages: write
     runs-on: ubuntu-latest
     if: github.repository_owner == 'coder'
     steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
+        with:
+          egress-policy: audit
+
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
 
       - name: Docker login
-        uses: docker/login-action@v3
+        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
@@ -50,16 +56,17 @@ jobs:
         run: mkdir base-build-context
 
       - name: Install depot.dev CLI
-        uses: depot/setup-action@v1
+        uses: depot/setup-action@b0b1ea4f69e92ebf5dea3f8713a1b0c37b2126a5 # v1.6.0
 
       # This uses OIDC authentication, so no auth variables are required.
       - name: Build base Docker image via depot.dev
-        uses: depot/build-push-action@v1
+        uses: depot/build-push-action@636daae76684e38c301daa0c5eca1c095b24e780 # v1.14.0
         with:
           project: wl5hnrrkns
           context: base-build-context
           file: scripts/Dockerfile.base
           platforms: linux/amd64,linux/arm64,linux/arm/v7
+          provenance: true
           pull: true
           no-cache: true
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/dogfood.yaml

@@ -17,16 +17,26 @@ on:
       - "flake.nix"
   workflow_dispatch:
 
+permissions:
+  # Necessary for GCP authentication (https://github.com/google-github-actions/setup-gcloud#usage)
+  id-token: write
+
 jobs:
   build_image:
+    if: github.actor != 'dependabot[bot]' # Skip Dependabot PRs
     runs-on: ubuntu-latest
     steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
+        with:
+          egress-policy: audit
+
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
 
       - name: Get branch name
         id: branch-name
-        uses: tj-actions/branch-names@v8
+        uses: tj-actions/branch-names@6871f53176ad61624f978536bbf089c574dc19a2 # v8.0.1
 
       - name: "Branch name to Docker tag name"
         id: docker-tag-name
@@ -37,38 +47,38 @@ jobs:
           echo "tag=${tag}" >> $GITHUB_OUTPUT
 
       - name: Set up Depot CLI
-        uses: depot/setup-action@v1
+        uses: depot/setup-action@b0b1ea4f69e92ebf5dea3f8713a1b0c37b2126a5 # v1.6.0
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@c47758b77c9736f4b2ef4073d4d51994fabfe349 # v3.7.1
 
       - name: Login to DockerHub
         if: github.ref == 'refs/heads/main'
-        uses: docker/login-action@v3
+        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_PASSWORD }}
 
       - name: Build and push Non-Nix image
-        uses: depot/build-push-action@v1
+        uses: depot/build-push-action@636daae76684e38c301daa0c5eca1c095b24e780 # v1.14.0
         with:
           project: b4q6ltmpzh
           token: ${{ secrets.DEPOT_TOKEN }}
           buildx-fallback: true
-          context: "{{defaultContext}}:dogfood"
+          context: "{{defaultContext}}:dogfood/contents"
           pull: true
           save: true
           push: ${{ github.ref == 'refs/heads/main' }}
           tags: "codercom/oss-dogfood:${{ steps.docker-tag-name.outputs.tag }},codercom/oss-dogfood:latest"
 
       - name: Build and push Nix image
-        uses: depot/build-push-action@v1
+        uses: depot/build-push-action@636daae76684e38c301daa0c5eca1c095b24e780 # v1.14.0
         with:
           project: b4q6ltmpzh
           token: ${{ secrets.DEPOT_TOKEN }}
           buildx-fallback: true
           context: "."
-          file: "dogfood/Dockerfile.nix"
+          file: "dogfood/contents/Dockerfile.nix"
           pull: true
           save: true
           push: ${{ github.ref == 'refs/heads/main' }}
@@ -78,17 +88,31 @@ jobs:
     needs: build_image
     runs-on: ubuntu-latest
     steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
+        with:
+          egress-policy: audit
+
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
 
       - name: Setup Terraform
         uses: ./.github/actions/setup-tf
 
+      - name: Authenticate to Google Cloud
+        uses: google-github-actions/auth@6fc4af4b145ae7821d527454aa9bd537d1f2dc5f # v2.1.7
+        with:
+          workload_identity_provider: projects/573722524737/locations/global/workloadIdentityPools/github/providers/github
+          service_account: coder-ci@coder-dogfood.iam.gserviceaccount.com
+
       - name: Terraform init and validate
         run: |
           cd dogfood
           terraform init -upgrade
           terraform validate
+          cd contents
+          terraform init -upgrade
+          terraform validate
 
       - name: Get short commit SHA
         if: github.ref == 'refs/heads/main'
@@ -100,22 +124,18 @@ jobs:
         id: message
         run: echo "pr_title=$(git log --format=%s -n 1 ${{ github.sha }})" >> $GITHUB_OUTPUT
 
-      - name: "Get latest Coder binary from the server"
-        if: github.ref == 'refs/heads/main'
-        run: |
-          curl -fsSL "https://dev.coder.com/bin/coder-linux-amd64" -o "./coder"
-          chmod +x "./coder"
-
       - name: "Push template"
         if: github.ref == 'refs/heads/main'
         run: |
-          ./coder templates push $CODER_TEMPLATE_NAME --directory $CODER_TEMPLATE_DIR --yes --name=$CODER_TEMPLATE_VERSION --message="$CODER_TEMPLATE_MESSAGE"
+          cd dogfood
+          terraform apply -auto-approve
         env:
-          # Consumed by Coder CLI
+          # Consumed by coderd provider
           CODER_URL: https://dev.coder.com
           CODER_SESSION_TOKEN: ${{ secrets.CODER_SESSION_TOKEN }}
           # Template source & details
-          CODER_TEMPLATE_NAME: ${{ secrets.CODER_TEMPLATE_NAME }}
-          CODER_TEMPLATE_VERSION: ${{ steps.vars.outputs.sha_short }}
-          CODER_TEMPLATE_DIR: ./dogfood
-          CODER_TEMPLATE_MESSAGE: ${{ steps.message.outputs.pr_title }}
+          TF_VAR_CODER_TEMPLATE_NAME: ${{ secrets.CODER_TEMPLATE_NAME }}
+          TF_VAR_CODER_TEMPLATE_VERSION: ${{ steps.vars.outputs.sha_short }}
+          TF_VAR_CODER_TEMPLATE_DIR: ./contents
+          TF_VAR_CODER_TEMPLATE_MESSAGE: ${{ steps.message.outputs.pr_title }}
+          TF_LOG: info

.github/workflows/mlc_config.json

@@ -1,26 +0,0 @@
-{
-  "ignorePatterns": [
-    {
-      "pattern": "://localhost"
-    },
-    {
-      "pattern": "://.*.?example\\.com"
-    },
-    {
-      "pattern": "developer.github.com"
-    },
-    {
-      "pattern": "docs.github.com"
-    },
-    {
-      "pattern": "support.google.com"
-    },
-    {
-      "pattern": "tailscale.com"
-    },
-    {
-      "pattern": "wireguard.com"
-    }
-  ],
-  "aliveStatusCodes": [200, 0]
-}

.github/workflows/nightly-gauntlet.yaml

@@ -6,18 +6,27 @@ on:
     # Every day at midnight
     - cron: "0 0 * * *"
   workflow_dispatch:
+
+permissions:
+  contents: read
+
 jobs:
   go-race:
     # While GitHub's toaster runners are likelier to flake, we want consistency
     # between this environment and the regular test environment for DataDog
     # statistics and to only show real workflow threats.
-    runs-on: "buildjet-8vcpu-ubuntu-2204"
+    runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-8' || 'ubuntu-latest' }}
     # This runner costs 0.016 USD per minute,
     # so 0.016 * 240 = 3.84 USD per run.
     timeout-minutes: 240
     steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
+        with:
+          egress-policy: audit
+
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
 
       - name: Setup Go
         uses: ./.github/actions/setup-go
@@ -40,11 +49,16 @@ jobs:
 
   go-timing:
     # We run these tests with p=1 so we don't need a lot of compute.
-    runs-on: "buildjet-2vcpu-ubuntu-2204"
+    runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04' || 'ubuntu-latest' }}
     timeout-minutes: 10
     steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
+        with:
+          egress-policy: audit
+
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
 
       - name: Setup Go
         uses: ./.github/actions/setup-go

.github/workflows/pr-auto-assign.yaml

@@ -13,5 +13,10 @@ jobs:
   assign-author:
     runs-on: ubuntu-latest
     steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
+        with:
+          egress-policy: audit
+
       - name: Assign author
-        uses: toshimaru/auto-author-assign@v2.1.0
+        uses: toshimaru/auto-author-assign@16f0022cf3d7970c106d8d1105f75a1165edb516 # v2.1.1

.github/workflows/pr-cleanup.yaml

@@ -9,12 +9,20 @@ on:
         required: true
 
 permissions:
-  packages: write
+  contents: read
 
 jobs:
   cleanup:
     runs-on: "ubuntu-latest"
+    permissions:
+      # Necessary to delete docker images from ghcr.io.
+      packages: write
     steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
+        with:
+          egress-policy: audit
+
       - name: Get PR number
         id: pr_number
         run: |
@@ -26,7 +34,7 @@ jobs:
 
       - name: Delete image
         continue-on-error: true
-        uses: bots-house/ghcr-delete-image-action@v1.1.0
+        uses: bots-house/ghcr-delete-image-action@3827559c68cb4dcdf54d813ea9853be6d468d3a4 # v1.1.0
         with:
           owner: coder
           name: coder-preview

.github/workflows/pr-deploy.yaml

@@ -30,8 +30,6 @@ env:
 
 permissions:
   contents: read
-  packages: write
-  pull-requests: write # needed for commenting on PRs
 
 jobs:
   check_pr:
@@ -39,8 +37,13 @@ jobs:
     outputs:
       PR_OPEN: ${{ steps.check_pr.outputs.pr_open }}
     steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
+        with:
+          egress-policy: audit
+
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
 
       - name: Check if PR is open
         id: check_pr
@@ -69,8 +72,13 @@ jobs:
 
     runs-on: "ubuntu-latest"
     steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
+        with:
+          egress-policy: audit
+
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
         with:
           fetch-depth: 0
 
@@ -101,8 +109,8 @@ jobs:
         run: |
           set -euo pipefail
           mkdir -p ~/.kube
-          echo "${{ secrets.PR_DEPLOYMENTS_KUBECONFIG }}" > ~/.kube/config
-          chmod 644 ~/.kube/config
+          echo "${{ secrets.PR_DEPLOYMENTS_KUBECONFIG_BASE64 }}" | base64 --decode > ~/.kube/config
+          chmod 600 ~/.kube/config
           export KUBECONFIG=~/.kube/config
 
       - name: Check if the helm deployment already exists
@@ -119,7 +127,7 @@ jobs:
           echo "NEW=$NEW" >> $GITHUB_OUTPUT
 
       - name: Check changed files
-        uses: dorny/paths-filter@v3
+        uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2
         id: filter
         with:
           base: ${{ github.ref }}
@@ -154,16 +162,23 @@ jobs:
           set -euo pipefail
           # build if the workflow is manually triggered and the deployment doesn't exist (first build or force rebuild)
           echo "first_or_force_build=${{ (github.event_name == 'workflow_dispatch' && steps.check_deployment.outputs.NEW == 'true') || github.event.inputs.build == 'true' }}" >> $GITHUB_OUTPUT
-          # build if the deployment alreday exist and there are changes in the files that we care about (automatic updates)
+          # build if the deployment already exist and there are changes in the files that we care about (automatic updates)
           echo "automatic_rebuild=${{ steps.check_deployment.outputs.NEW == 'false' && steps.filter.outputs.all_count > steps.filter.outputs.ignored_count }}" >> $GITHUB_OUTPUT
 
   comment-pr:
     needs: get_info
     if: needs.get_info.outputs.BUILD == 'true' || github.event.inputs.deploy == 'true'
     runs-on: "ubuntu-latest"
+    permissions:
+      pull-requests: write # needed for commenting on PRs
     steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
+        with:
+          egress-policy: audit
+
       - name: Find Comment
-        uses: peter-evans/find-comment@v3
+        uses: peter-evans/find-comment@3eae4d37986fb5a8592848f6a574fdf654e61f9e # v3.1.0
         id: fc
         with:
           issue-number: ${{ needs.get_info.outputs.PR_NUMBER }}
@@ -173,7 +188,7 @@ jobs:
 
       - name: Comment on PR
         id: comment_id
-        uses: peter-evans/create-or-update-comment@v4
+        uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4.0.0
         with:
           comment-id: ${{ steps.fc.outputs.comment-id }}
           issue-number: ${{ needs.get_info.outputs.PR_NUMBER }}
@@ -189,8 +204,11 @@ jobs:
     needs: get_info
     # Run build job only if there are changes in the files that we care about or if the workflow is manually triggered with --build flag
     if: needs.get_info.outputs.BUILD == 'true'
-    runs-on: ${{ github.repository_owner == 'coder' && 'buildjet-8vcpu-ubuntu-2204' || 'ubuntu-latest' }}
-    # This concurrency only cancels build jobs if a new build is triggred. It will avoid cancelling the current deployemtn in case of docs chnages.
+    runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-8' || 'ubuntu-latest' }}
+    permissions:
+      # Necessary to push docker images to ghcr.io.
+      packages: write
+    # This concurrency only cancels build jobs if a new build is triggred. It will avoid cancelling the current deployemtn in case of docs changes.
     concurrency:
       group: build-${{ github.workflow }}-${{ github.ref }}-${{ needs.get_info.outputs.BUILD }}
       cancel-in-progress: true
@@ -198,8 +216,13 @@ jobs:
       DOCKER_CLI_EXPERIMENTAL: "enabled"
       CODER_IMAGE_TAG: ${{ needs.get_info.outputs.CODER_IMAGE_TAG }}
     steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
+        with:
+          egress-policy: audit
+
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
         with:
           fetch-depth: 0
 
@@ -213,7 +236,7 @@ jobs:
         uses: ./.github/actions/setup-sqlc
 
       - name: GHCR Login
-        uses: docker/login-action@v3
+        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
@@ -242,6 +265,8 @@ jobs:
       always() && (needs.build.result == 'success' || needs.build.result == 'skipped') &&
       (needs.get_info.outputs.BUILD == 'true' || github.event.inputs.deploy == 'true')
     runs-on: "ubuntu-latest"
+    permissions:
+      pull-requests: write # needed for commenting on PRs
     env:
       CODER_IMAGE_TAG: ${{ needs.get_info.outputs.CODER_IMAGE_TAG }}
       PR_NUMBER: ${{ needs.get_info.outputs.PR_NUMBER }}
@@ -249,12 +274,17 @@ jobs:
       PR_URL: ${{ needs.get_info.outputs.PR_URL }}
       PR_HOSTNAME: "pr${{ needs.get_info.outputs.PR_NUMBER }}.${{ secrets.PR_DEPLOYMENTS_DOMAIN }}"
     steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
+        with:
+          egress-policy: audit
+
       - name: Set up kubeconfig
         run: |
           set -euo pipefail
           mkdir -p ~/.kube
-          echo "${{ secrets.PR_DEPLOYMENTS_KUBECONFIG }}" > ~/.kube/config
-          chmod 644 ~/.kube/config
+          echo "${{ secrets.PR_DEPLOYMENTS_KUBECONFIG_BASE64 }}" | base64 --decode > ~/.kube/config
+          chmod 600 ~/.kube/config
           export KUBECONFIG=~/.kube/config
 
       - name: Check if image exists
@@ -294,7 +324,7 @@ jobs:
           kubectl create namespace "pr${{ env.PR_NUMBER }}"
 
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
 
       - name: Check and Create Certificate
         if: needs.get_info.outputs.NEW == 'true' || github.event.inputs.deploy == 'true'
@@ -391,14 +421,14 @@ jobs:
           "${DEST}" version
           mv "${DEST}" /usr/local/bin/coder
 
-      - name: Create first user, template and workspace
+      - name: Create first user
         if: needs.get_info.outputs.NEW == 'true' || github.event.inputs.deploy == 'true'
         id: setup_deployment
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           set -euo pipefail
 
-          # Create first user
-
           # create a masked random password 12 characters long
           password=$(openssl rand -base64 16 | tr -d "=+/" | cut -c1-12)
 
@@ -407,20 +437,22 @@ jobs:
           echo "password=$password" >> $GITHUB_OUTPUT
 
           coder login \
-            --first-user-username coder \
+            --first-user-username pr${{ env.PR_NUMBER }}-admin \
             --first-user-email pr${{ env.PR_NUMBER }}@coder.com \
             --first-user-password $password \
-            --first-user-trial \
+            --first-user-trial=false \
             --use-token-as-session \
             https://${{ env.PR_HOSTNAME }}
 
-          # Create template
-          cd ./.github/pr-deployments/template
-          coder templates push -y --variable namespace=pr${{ env.PR_NUMBER }} kubernetes
+          # Create a user for the github.actor
+          # TODO: update once https://github.com/coder/coder/issues/15466 is resolved
+          # coder users create \
+          #   --username ${{ github.actor }} \
+          #   --login-type github
 
-          # Create workspace
-          coder create --template="kubernetes" kube --parameter cpu=2 --parameter memory=4 --parameter home_disk_size=2 -y
-          coder stop kube -y
+          # promote the user to admin role
+          # coder org members edit-role ${{ github.actor }} organization-admin
+          # TODO: update once https://github.com/coder/internal/issues/207 is resolved
 
       - name: Send Slack notification
         if: needs.get_info.outputs.NEW == 'true' || github.event.inputs.deploy == 'true'
@@ -432,7 +464,7 @@ jobs:
               "pr_url": "'"${{ env.PR_URL }}"'",
               "pr_title": "'"${{ env.PR_TITLE }}"'",
               "pr_access_url": "'"https://${{ env.PR_HOSTNAME }}"'",
-              "pr_username": "'"test"'",
+              "pr_username": "'"pr${{ env.PR_NUMBER }}-admin"'",
               "pr_email": "'"pr${{ env.PR_NUMBER }}@coder.com"'",
               "pr_password": "'"${{ steps.setup_deployment.outputs.password }}"'",
               "pr_actor": "'"${{ github.actor }}"'"
@@ -441,7 +473,7 @@ jobs:
           echo "Slack notification sent"
 
       - name: Find Comment
-        uses: peter-evans/find-comment@v3
+        uses: peter-evans/find-comment@3eae4d37986fb5a8592848f6a574fdf654e61f9e # v3.1.0
         id: fc
         with:
           issue-number: ${{ env.PR_NUMBER }}
@@ -450,7 +482,7 @@ jobs:
           direction: last
 
       - name: Comment on PR
-        uses: peter-evans/create-or-update-comment@v4
+        uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4.0.0
         env:
           STATUS: ${{ needs.get_info.outputs.NEW == 'true' && 'Created' || 'Updated' }}
         with:
@@ -465,3 +497,14 @@ jobs:
             cc: @${{ github.actor }}
           reactions: rocket
           reactions-edit-mode: replace
+
+      - name: Create template and workspace
+        if: needs.get_info.outputs.NEW == 'true' || github.event.inputs.deploy == 'true'
+        run: |
+          set -euo pipefail
+          cd .github/pr-deployments/template
+          coder templates push -y --variable namespace=pr${{ env.PR_NUMBER }} kubernetes
+
+          # Create workspace
+          coder create --template="kubernetes" kube --parameter cpu=2 --parameter memory=4 --parameter home_disk_size=2 -y
+          coder stop kube -y

.github/workflows/release-validation.yaml

@@ -0,0 +1,28 @@
+name: release-validation
+
+on:
+  push:
+    tags:
+      - "v*"
+
+permissions:
+  contents: read
+
+jobs:
+  network-performance:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
+        with:
+          egress-policy: audit
+
+      - name: Run Schmoder CI
+        uses: benc-uk/workflow-dispatch@e2e5e9a103e331dad343f381a29e654aea3cf8fc # v1.2.4
+        with:
+          workflow: ci.yaml
+          repo: coder/schmoder
+          inputs: '{ "num_releases": "3", "commit": "${{ github.sha }}" }'
+          token: ${{ secrets.CDRCI_SCHMODER_ACTIONS_TOKEN }}
+          ref: main

.github/workflows/release.yaml

@@ -18,12 +18,7 @@ on:
         default: false
 
 permissions:
-  # Required to publish a release
-  contents: write
-  # Necessary to push docker images to ghcr.io.
-  packages: write
-  # Necessary for GCP authentication (https://github.com/google-github-actions/setup-gcloud#usage)
-  id-token: write
+  contents: read
 
 concurrency: ${{ github.workflow }}-${{ github.ref }}
 
@@ -39,15 +34,27 @@ env:
 jobs:
   release:
     name: Build and publish
-    runs-on: ${{ github.repository_owner == 'coder' && 'buildjet-8vcpu-ubuntu-2204' || 'ubuntu-latest' }}
+    runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-8' || 'ubuntu-latest' }}
+    permissions:
+      # Required to publish a release
+      contents: write
+      # Necessary to push docker images to ghcr.io.
+      packages: write
+      # Necessary for GCP authentication (https://github.com/google-github-actions/setup-gcloud#usage)
+      id-token: write
     env:
       # Necessary for Docker manifest
       DOCKER_CLI_EXPERIMENTAL: "enabled"
     outputs:
       version: ${{ steps.version.outputs.version }}
     steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
+        with:
+          egress-policy: audit
+
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
         with:
           fetch-depth: 0
 
@@ -116,7 +123,7 @@ jobs:
           cat "$CODER_RELEASE_NOTES_FILE"
 
       - name: Docker Login
-        uses: docker/login-action@v3
+        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
@@ -130,7 +137,7 @@ jobs:
 
       # Necessary for signing Windows binaries.
       - name: Setup Java
-        uses: actions/setup-java@v4
+        uses: actions/setup-java@b36c23c0d998641eff861008f374ee103c25ac73 # v4.4.0
         with:
           distribution: "zulu"
           java-version: "11.0"
@@ -180,19 +187,19 @@ jobs:
 
       - name: Test migrations from current ref to main
         run: |
-          make test-migrations
+          POSTGRES_VERSION=13 make test-migrations
 
       # Setup GCloud for signing Windows binaries.
       - name: Authenticate to Google Cloud
         id: gcloud_auth
-        uses: google-github-actions/auth@v2
+        uses: google-github-actions/auth@6fc4af4b145ae7821d527454aa9bd537d1f2dc5f # v2.1.7
         with:
           workload_identity_provider: ${{ secrets.GCP_CODE_SIGNING_WORKLOAD_ID_PROVIDER }}
           service_account: ${{ secrets.GCP_CODE_SIGNING_SERVICE_ACCOUNT }}
           token_format: "access_token"
 
       - name: Setup GCloud SDK
-        uses: "google-github-actions/setup-gcloud@v2"
+        uses: google-github-actions/setup-gcloud@6189d56e4096ee891640bb02ac264be376592d6a # v2.1.2
 
       - name: Build binaries
         run: |
@@ -245,17 +252,18 @@ jobs:
 
       - name: Install depot.dev CLI
         if: steps.image-base-tag.outputs.tag != ''
-        uses: depot/setup-action@v1
+        uses: depot/setup-action@b0b1ea4f69e92ebf5dea3f8713a1b0c37b2126a5 # v1.6.0
 
       # This uses OIDC authentication, so no auth variables are required.
       - name: Build base Docker image via depot.dev
         if: steps.image-base-tag.outputs.tag != ''
-        uses: depot/build-push-action@v1
+        uses: depot/build-push-action@636daae76684e38c301daa0c5eca1c095b24e780 # v1.14.0
         with:
           project: wl5hnrrkns
           context: base-build-context
           file: scripts/Dockerfile.base
           platforms: linux/amd64,linux/arm64,linux/arm/v7
+          provenance: true
           pull: true
           no-cache: true
           push: true
@@ -297,7 +305,7 @@ jobs:
 
           # build Docker images for each architecture
           version="$(./scripts/version.sh)"
-          make -j build/coder_"$version"_linux_{amd64,arm64,armv7}.tag
+          make build/coder_"$version"_linux_{amd64,arm64,armv7}.tag
 
           # we can't build multi-arch if the images aren't pushed, so quit now
           # if dry-running
@@ -308,7 +316,7 @@ jobs:
 
           # build and push multi-arch manifest, this depends on the other images
           # being pushed so will automatically push them.
-          make -j push/build/coder_"$version"_linux.tag
+          make push/build/coder_"$version"_linux.tag
 
           # if the current version is equal to the highest (according to semver)
           # version in the repo, also create a multi-arch image as ":latest" and
@@ -358,13 +366,13 @@ jobs:
           CODER_GPG_RELEASE_KEY_BASE64: ${{ secrets.GPG_RELEASE_KEY_BASE64 }}
 
       - name: Authenticate to Google Cloud
-        uses: google-github-actions/auth@v2
+        uses: google-github-actions/auth@6fc4af4b145ae7821d527454aa9bd537d1f2dc5f # v2.1.7
         with:
           workload_identity_provider: ${{ secrets.GCP_WORKLOAD_ID_PROVIDER }}
           service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }}
 
       - name: Setup GCloud SDK
-        uses: "google-github-actions/setup-gcloud@v2"
+        uses: google-github-actions/setup-gcloud@6189d56e4096ee891640bb02ac264be376592d6a # 2.1.2
 
       - name: Publish Helm Chart
         if: ${{ !inputs.dry_run }}
@@ -383,7 +391,7 @@ jobs:
 
       - name: Upload artifacts to actions (if dry-run)
         if: ${{ inputs.dry_run }}
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@604373da6381bf24206979c74d06a550515601b9 # v4.4.1
         with:
           name: release-artifacts
           path: |
@@ -396,14 +404,14 @@ jobs:
             ./build/*.rpm
           retention-days: 7
 
-      - name: Start Packer builds
+      - name: Send repository-dispatch event
         if: ${{ !inputs.dry_run }}
-        uses: peter-evans/repository-dispatch@v3
+        uses: peter-evans/repository-dispatch@ff45666b9427631e3450c54a1bcbee4d9ff4d7c0 # v3.0.0
         with:
           token: ${{ secrets.CDRCI_GITHUB_TOKEN }}
           repository: coder/packages
           event-type: coder-release
-          client-payload: '{"coder_version": "${{ steps.version.outputs.version }}"}'
+          client-payload: '{"coder_version": "${{ steps.version.outputs.version }}", "release_channel": "${{ inputs.release_channel }}"}'
 
   publish-homebrew:
     name: Publish to Homebrew tap
@@ -414,6 +422,11 @@ jobs:
     steps:
       # TODO: skip this if it's not a new release (i.e. a backport). This is
       #       fine right now because it just makes a PR that we can close.
+      - name: Harden Runner
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
+        with:
+          egress-policy: audit
+
       - name: Update homebrew
         env:
           # Variables used by the `gh` command
@@ -485,13 +498,18 @@ jobs:
     if: ${{ !inputs.dry_run }}
 
     steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
+        with:
+          egress-policy: audit
+
       - name: Sync fork
         run: gh repo sync cdrci/winget-pkgs -b master
         env:
           GH_TOKEN: ${{ secrets.CDRCI_GITHUB_TOKEN }}
 
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
         with:
           fetch-depth: 0
 
@@ -570,8 +588,13 @@ jobs:
     needs: release
     if: ${{ !inputs.dry_run }}
     steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
+        with:
+          egress-policy: audit
+
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
         with:
           fetch-depth: 1
 

.github/workflows/scorecard.yml

@@ -0,0 +1,52 @@
+name: OpenSSF Scorecard
+on:
+  branch_protection_rule:
+  schedule:
+    - cron: "27 7 * * 3" # A random time to run weekly
+  push:
+    branches: ["main"]
+
+permissions: read-all
+
+jobs:
+  analysis:
+    name: Scorecard analysis
+    runs-on: ubuntu-latest
+    permissions:
+      # Needed to upload the results to code-scanning dashboard.
+      security-events: write
+      # Needed to publish results and get a badge (see publish_results below).
+      id-token: write
+
+    steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
+        with:
+          egress-policy: audit
+
+      - name: "Checkout code"
+        uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
+        with:
+          persist-credentials: false
+
+      - name: "Run analysis"
+        uses: ossf/scorecard-action@62b2cac7ed8198b15735ed49ab1e5cf35480ba46 # v2.4.0
+        with:
+          results_file: results.sarif
+          results_format: sarif
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_results: true
+
+      # Upload the results as artifacts.
+      - name: "Upload artifact"
+        uses: actions/upload-artifact@604373da6381bf24206979c74d06a550515601b9 # v4.4.1
+        with:
+          name: SARIF file
+          path: results.sarif
+          retention-days: 5
+
+      # Upload the results to GitHub's code scanning dashboard.
+      - name: "Upload to code-scanning"
+        uses: github/codeql-action/upload-sarif@f09c1c0a94de965c15400f5634aa42fac8fb8f88 # v3.27.5
+        with:
+          sarif_file: results.sarif

.github/workflows/security.yaml

@@ -3,7 +3,6 @@ name: "security"
 permissions:
   actions: read
   contents: read
-  security-events: write
 
 on:
   workflow_dispatch:
@@ -23,16 +22,23 @@ concurrency:
 
 jobs:
   codeql:
-    runs-on: ${{ github.repository_owner == 'coder' && 'buildjet-8vcpu-ubuntu-2204' || 'ubuntu-latest' }}
+    permissions:
+      security-events: write
+    runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-8' || 'ubuntu-latest' }}
     steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
+        with:
+          egress-policy: audit
+
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
 
       - name: Setup Go
         uses: ./.github/actions/setup-go
 
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@v3
+        uses: github/codeql-action/init@f09c1c0a94de965c15400f5634aa42fac8fb8f88 # v3.27.5
         with:
           languages: go, javascript
 
@@ -42,7 +48,7 @@ jobs:
           rm Makefile
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v3
+        uses: github/codeql-action/analyze@f09c1c0a94de965c15400f5634aa42fac8fb8f88 # v3.27.5
 
       - name: Send Slack notification on failure
         if: ${{ failure() }}
@@ -56,10 +62,17 @@ jobs:
             "${{ secrets.SLACK_SECURITY_FAILURE_WEBHOOK_URL }}"
 
   trivy:
-    runs-on: ${{ github.repository_owner == 'coder' && 'buildjet-8vcpu-ubuntu-2204' || 'ubuntu-latest' }}
+    permissions:
+      security-events: write
+    runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-8' || 'ubuntu-latest' }}
     steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
+        with:
+          egress-policy: audit
+
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
         with:
           fetch-depth: 0
 
@@ -85,13 +98,20 @@ jobs:
           # protoc must be in lockstep with our dogfood Dockerfile or the
           # version in the comments will differ. This is also defined in
           # ci.yaml.
-          set -x
-          cd dogfood
+          set -euxo pipefail
+          cd dogfood/contents
+          mkdir -p /usr/local/bin
+          mkdir -p /usr/local/include
+
           DOCKER_BUILDKIT=1 docker build . --target proto -t protoc
           protoc_path=/usr/local/bin/protoc
           docker run --rm --entrypoint cat protoc /tmp/bin/protoc > $protoc_path
           chmod +x $protoc_path
           protoc --version
+          # Copy the generated files to the include directory.
+          docker run --rm -v /usr/local/include:/target protoc cp -r /tmp/include/google /target/
+          ls -la /usr/local/include/google/protobuf/
+          stat /usr/local/include/google/protobuf/timestamp.proto
 
       - name: Build Coder linux amd64 Docker image
         id: build
@@ -110,11 +130,13 @@ jobs:
           # the registry.
           export CODER_IMAGE_BUILD_BASE_TAG="$(CODER_IMAGE_BASE=coder-base ./scripts/image_tag.sh --version "$version")"
 
-          make -j "$image_job"
+          # We would like to use make -j here, but it doesn't work with the some recent additions
+          # to our code generation.
+          make "$image_job"
           echo "image=$(cat "$image_job")" >> $GITHUB_OUTPUT
 
       - name: Run Trivy vulnerability scanner
-        uses: aquasecurity/trivy-action@fd25fed6972e341ff0007ddb61f77e88103953c2
+        uses: aquasecurity/trivy-action@18f2510ee396bbf400402947b394f2dd8c87dbb0
         with:
           image-ref: ${{ steps.build.outputs.image }}
           format: sarif
@@ -122,28 +144,18 @@ jobs:
           severity: "CRITICAL,HIGH"
 
       - name: Upload Trivy scan results to GitHub Security tab
-        uses: github/codeql-action/upload-sarif@v3
+        uses: github/codeql-action/upload-sarif@f09c1c0a94de965c15400f5634aa42fac8fb8f88 # v3.27.5
         with:
           sarif_file: trivy-results.sarif
           category: "Trivy"
 
       - name: Upload Trivy scan results as an artifact
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@604373da6381bf24206979c74d06a550515601b9 # v4.4.1
         with:
           name: trivy
           path: trivy-results.sarif
           retention-days: 7
 
-      # Prisma cloud scan runs last because it fails the entire job if it
-      # detects vulnerabilities. :|
-      - name: Run Prisma Cloud image scan
-        uses: PaloAltoNetworks/prisma-cloud-scan@v1
-        with:
-          pcc_console_url: ${{ secrets.PRISMA_CLOUD_URL }}
-          pcc_user: ${{ secrets.PRISMA_CLOUD_ACCESS_KEY }}
-          pcc_pass: ${{ secrets.PRISMA_CLOUD_SECRET_KEY }}
-          image_name: ${{ steps.build.outputs.image }}
-
       - name: Send Slack notification on failure
         if: ${{ failure() }}
         run: |

.github/workflows/stale.yaml

@@ -1,23 +1,36 @@
-name: Stale Issue, Banch and Old Workflows Cleanup
+name: Stale Issue, Branch and Old Workflows Cleanup
 on:
   schedule:
     # Every day at midnight
     - cron: "0 0 * * *"
   workflow_dispatch:
+
+permissions:
+  contents: read
+
 jobs:
   issues:
     runs-on: ubuntu-latest
     permissions:
+      # Needed to close issues.
       issues: write
+      # Needed to close PRs.
       pull-requests: write
-      actions: write
     steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
+        with:
+          egress-policy: audit
+
       - name: stale
-        uses: actions/stale@v9.0.0
+        uses: actions/stale@28ca1036281a5e5922ead5184a1bbf96e5fc984e # v9.0.0
         with:
           stale-issue-label: "stale"
           stale-pr-label: "stale"
-          days-before-stale: 180
+          # days-before-stale: 180
+          # essentially disabled for now while we work through polish issues
+          days-before-stale: 3650
+
           # Pull Requests become stale more quickly due to merge conflicts.
           # Also, we promote minimizing WIP.
           days-before-pr-stale: 7
@@ -31,7 +44,7 @@ jobs:
           # Start with the oldest issues, always.
           ascending: true
       - name: "Close old issues labeled likely-no"
-        uses: actions/github-script@v7
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           script: |
@@ -57,7 +70,7 @@ jobs:
               });
 
               const labelEvent = timeline.data.find(event => event.event === 'labeled' && event.label.name === 'likely-no');
-              
+
               if (labelEvent) {
                 console.log(`Issue #${issue.number} was labeled with 'likely-no' at ${labelEvent.created_at}`);
 
@@ -78,11 +91,19 @@ jobs:
 
   branches:
     runs-on: ubuntu-latest
+    permissions:
+      # Needed to delete branches.
+      contents: write
     steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
+        with:
+          egress-policy: audit
+
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
       - name: Run delete-old-branches-action
-        uses: beatlabs/delete-old-branches-action@v0.0.10
+        uses: beatlabs/delete-old-branches-action@6e94df089372a619c01ae2c2f666bf474f890911 # v0.0.10
         with:
           repo_token: ${{ github.token }}
           date: "6 months ago"
@@ -92,9 +113,17 @@ jobs:
           exclude_open_pr_branches: true
   del_runs:
     runs-on: ubuntu-latest
+    permissions:
+      # Needed to delete workflow runs.
+      actions: write
     steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
+        with:
+          egress-policy: audit
+
       - name: Delete PR Cleanup workflow runs
-        uses: Mattraks/delete-workflow-runs@v2
+        uses: Mattraks/delete-workflow-runs@39f0bbed25d76b34de5594dceab824811479e5de # v2.0.6
         with:
           token: ${{ github.token }}
           repository: ${{ github.repository }}
@@ -103,7 +132,7 @@ jobs:
           delete_workflow_pattern: pr-cleanup.yaml
 
       - name: Delete PR Deploy workflow skipped runs
-        uses: Mattraks/delete-workflow-runs@v2
+        uses: Mattraks/delete-workflow-runs@39f0bbed25d76b34de5594dceab824811479e5de # v2.0.6
         with:
           token: ${{ github.token }}
           repository: ${{ github.repository }}

.github/workflows/typos.toml

@@ -14,8 +14,16 @@ darcula = "darcula"
 Hashi = "Hashi"
 trialer = "trialer"
 encrypter = "encrypter"
-hel = "hel"             # as in helsinki
-pn = "pn"               # this is used as proto node
+# as in helsinki
+hel = "hel"
+# this is used as proto node
+pn = "pn"
+# typos doesn't like the EDE in TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA
+EDE = "EDE"
+# HELO is an SMTP command
+HELO = "HELO"
+LKE = "LKE"
+byt = "byt"
 
 [files]
 extend-exclude = [
@@ -34,4 +42,6 @@ extend-exclude = [
 	"tailnet/testdata/**",
 	"site/src/pages/SetupPage/countries.tsx",
 	"provisioner/terraform/testdata/**",
+	# notifications' golden files confuse the detector because of quoted-printable encoding
+	"coderd/notifications/testdata/**"
 ]

.github/workflows/weekly-docs.yaml

@@ -10,23 +10,32 @@ on:
     paths:
       - "docs/**"
 
+permissions:
+  contents: read
+
 jobs:
   check-docs:
     runs-on: ubuntu-latest
+    permissions:
+      pull-requests: write # required to post PR review comments by the action
     steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
+        with:
+          egress-policy: audit
+
       - name: Checkout
-        uses: actions/checkout@master
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
 
       - name: Check Markdown links
-        uses: gaurav-nelson/github-action-markdown-link-check@v1
+        uses: umbrelladocs/action-linkspector@fc382e19892aca958e189954912fe379a8df270c # v1.2.4
         id: markdown-link-check
         # checks all markdown files from /docs including all subfolders
         with:
-          use-quiet-mode: "yes"
-          use-verbose-mode: "yes"
-          config-file: ".github/workflows/mlc_config.json"
-          folder-path: "docs/"
-          file-path: "./README.md"
+          reporter: github-pr-review
+          config_file: ".github/.linkspector.yml"
+          fail_on_error: "true"
+          filter_mode: "nofilter"
 
       - name: Send Slack notification
         if: failure() && github.event_name == 'schedule'

.gitignore

@@ -17,6 +17,8 @@ yarn-error.log
 # Allow VSCode recommendations and default settings in project root.
 !/.vscode/extensions.json
 !/.vscode/settings.json
+# Allow code snippets
+!/.vscode/*.code-snippets
 
 # Front-end ignore patterns.
 .next/
@@ -68,3 +70,9 @@ result
 
 # Filebrowser.db
 **/filebrowser.db
+
+# pnpm
+.pnpm-store/
+
+# Zed
+.zed_server

.golangci.yaml

@@ -195,6 +195,11 @@ linters-settings:
       - name: var-naming
       - name: waitgroup-by-value
 
+  # irrelevant as of Go v1.22: https://go.dev/blog/loopvar-preview
+  govet:
+    disable:
+      - loopclosure
+
 issues:
   # Rules listed here: https://github.com/securego/gosec#available-rules
   exclude-rules:

.prettierignore

@@ -20,6 +20,8 @@ yarn-error.log
 # Allow VSCode recommendations and default settings in project root.
 !/.vscode/extensions.json
 !/.vscode/settings.json
+# Allow code snippets
+!/.vscode/*.code-snippets
 
 # Front-end ignore patterns.
 .next/
@@ -71,24 +73,24 @@ result
 
 # Filebrowser.db
 **/filebrowser.db
+
+# pnpm
+.pnpm-store/
+
+# Zed
+.zed_server
 # .prettierignore.include:
 # Helm templates contain variables that are invalid YAML and can't be formatted
 # by Prettier.
 helm/**/templates/*.yaml
 
-# Terraform state files used in tests, these are automatically generated.
-# Example: provisioner/terraform/testdata/instance-id/instance-id.tfstate.json
-**/testdata/**/*.tf*.json
-
 # Testdata shouldn't be formatted.
-scripts/apitypings/testdata/**/*.ts
-enterprise/tailnet/testdata/*.golden.html
-tailnet/testdata/*.golden.html
-
-# Generated files shouldn't be formatted.
-site/e2e/provisionerGenerated.ts
+testdata/
 
+# Ignore generated files
 **/pnpm-lock.yaml
-
-# Ignore generated JSON (e.g. examples/examples.gen.json).
 **/*.gen.json
+
+# Everything in site/ is formatted by Biome. For the rest of the repo though, we
+# need broader language support.
+site/

.prettierignore.include

@@ -2,19 +2,13 @@
 # by Prettier.
 helm/**/templates/*.yaml
 
-# Terraform state files used in tests, these are automatically generated.
-# Example: provisioner/terraform/testdata/instance-id/instance-id.tfstate.json
-**/testdata/**/*.tf*.json
-
 # Testdata shouldn't be formatted.
-scripts/apitypings/testdata/**/*.ts
-enterprise/tailnet/testdata/*.golden.html
-tailnet/testdata/*.golden.html
-
-# Generated files shouldn't be formatted.
-site/e2e/provisionerGenerated.ts
+testdata/
 
+# Ignore generated files
 **/pnpm-lock.yaml
-
-# Ignore generated JSON (e.g. examples/examples.gen.json).
 **/*.gen.json
+
+# Everything in site/ is formatted by Biome. For the rest of the repo though, we
+# need broader language support.
+site/

.prettierrc.yaml

@@ -4,13 +4,13 @@
 printWidth: 80
 proseWrap: always
 trailingComma: all
-useTabs: false
+useTabs: true
 tabWidth: 2
 overrides:
   - files:
       - README.md
-      - docs/api/**/*.md
-      - docs/cli/**/*.md
+      - docs/reference/api/**/*.md
+      - docs/reference/cli/**/*.md
       - docs/changelogs/*.md
       - .github/**/*.{yaml,yml,toml}
       - scripts/**/*.{yaml,yml,toml}

.vscode/extensions.json

@@ -1,15 +1,16 @@
 {
-  "recommendations": [
-    "github.vscode-codeql",
-    "golang.go",
-    "hashicorp.terraform",
-    "esbenp.prettier-vscode",
-    "foxundermoon.shell-format",
-    "emeraldwalk.runonsave",
-    "zxh404.vscode-proto3",
-    "redhat.vscode-yaml",
-    "streetsidesoftware.code-spell-checker",
-    "dbaeumer.vscode-eslint",
-    "EditorConfig.EditorConfig"
-  ]
+	"recommendations": [
+		"github.vscode-codeql",
+		"golang.go",
+		"hashicorp.terraform",
+		"esbenp.prettier-vscode",
+		"foxundermoon.shell-format",
+		"emeraldwalk.runonsave",
+		"zxh404.vscode-proto3",
+		"redhat.vscode-yaml",
+		"tekumara.typos-vscode",
+		"EditorConfig.EditorConfig",
+		"biomejs.biome",
+		"bradlc.vscode-tailwindcss"
+	]
 }

.vscode/markdown.code-snippets

@@ -0,0 +1,38 @@
+{
+	// For info about snippets, visit https://code.visualstudio.com/docs/editor/userdefinedsnippets
+
+    "admonition": {
+        "prefix": "#callout",
+        "body": [
+            "<blockquote class=\"admonition ${1|caution,important,note,tip,warning|}\">\n",
+            "${TM_SELECTED_TEXT:${2:add info here}}\n",
+            "</blockquote>\n"
+        ],
+        "description": "callout admonition caution info note tip warning"
+    },
+	"fenced code block": {
+		"prefix": "#codeblock",
+		"body": ["```${1|apache,bash,console,diff,Dockerfile,env,go,hcl,ini,json,lisp,md,powershell,shell,sql,text,tf,tsx,yaml|}", "${TM_SELECTED_TEXT}$0", "```"],
+		"description": "fenced code block"
+	},
+	"image": {
+		"prefix": "#image",
+		"body": "![${TM_SELECTED_TEXT:${1:alt}}](${2:url})$0",
+		"description": "image"
+	},
+	"tabs": {
+		"prefix": "#tabs",
+		"body": [
+			"<div class=\"tabs\">\n",
+			"${1:optional description}\n",
+			"## ${2:tab title}\n",
+			"${TM_SELECTED_TEXT:${3:first tab content}}\n",
+			"## ${4:tab title}\n",
+			"${5:second tab content}\n",
+			"## ${6:tab title}\n",
+			"${7:third tab content}\n",
+			"</div>\n"
+		],
+		"description": "tabs"
+	}
+}

.vscode/settings.json

@@ -1,227 +1,61 @@
 {
-  "cSpell.words": [
-    "afero",
-    "agentsdk",
-    "apps",
-    "ASKPASS",
-    "authcheck",
-    "autostop",
-    "awsidentity",
-    "bodyclose",
-    "buildinfo",
-    "buildname",
-    "circbuf",
-    "cliflag",
-    "cliui",
-    "codecov",
-    "coderd",
-    "coderdenttest",
-    "coderdtest",
-    "codersdk",
-    "contravariance",
-    "cronstrue",
-    "databasefake",
-    "dbgen",
-    "dbmem",
-    "dbtype",
-    "DERP",
-    "derphttp",
-    "derpmap",
-    "devel",
-    "devtunnel",
-    "dflags",
-    "drpc",
-    "drpcconn",
-    "drpcmux",
-    "drpcserver",
-    "Dsts",
-    "embeddedpostgres",
-    "enablements",
-    "enterprisemeta",
-    "errgroup",
-    "eventsourcemock",
-    "externalauth",
-    "Failf",
-    "fatih",
-    "Formik",
-    "gitauth",
-    "gitsshkey",
-    "goarch",
-    "gographviz",
-    "goleak",
-    "gonet",
-    "gossh",
-    "gsyslog",
-    "GTTY",
-    "hashicorp",
-    "hclsyntax",
-    "httpapi",
-    "httpmw",
-    "idtoken",
-    "Iflag",
-    "incpatch",
-    "initialisms",
-    "ipnstate",
-    "isatty",
-    "Jobf",
-    "Keygen",
-    "kirsle",
-    "Kubernetes",
-    "ldflags",
-    "magicsock",
-    "manifoldco",
-    "mapstructure",
-    "mattn",
-    "mitchellh",
-    "moby",
-    "namesgenerator",
-    "namespacing",
-    "netaddr",
-    "netip",
-    "netmap",
-    "netns",
-    "netstack",
-    "nettype",
-    "nfpms",
-    "nhooyr",
-    "nmcfg",
-    "nolint",
-    "nosec",
-    "ntqry",
-    "OIDC",
-    "oneof",
-    "opty",
-    "paralleltest",
-    "parameterscopeid",
-    "pqtype",
-    "prometheusmetrics",
-    "promhttp",
-    "protobuf",
-    "provisionerd",
-    "provisionerdserver",
-    "provisionersdk",
-    "ptty",
-    "ptys",
-    "ptytest",
-    "quickstart",
-    "reconfig",
-    "replicasync",
-    "retrier",
-    "rpty",
-    "SCIM",
-    "sdkproto",
-    "sdktrace",
-    "Signup",
-    "slogtest",
-    "sourcemapped",
-    "spinbutton",
-    "Srcs",
-    "stdbuf",
-    "stretchr",
-    "STTY",
-    "stuntest",
-    "tailbroker",
-    "tailcfg",
-    "tailexchange",
-    "tailnet",
-    "tailnettest",
-    "Tailscale",
-    "tanstack",
-    "tbody",
-    "TCGETS",
-    "tcpip",
-    "TCSETS",
-    "templateversions",
-    "testdata",
-    "testid",
-    "testutil",
-    "tfexec",
-    "tfjson",
-    "tfplan",
-    "tfstate",
-    "thead",
-    "tios",
-    "tmpdir",
-    "tokenconfig",
-    "Topbar",
-    "tparallel",
-    "trialer",
-    "trimprefix",
-    "tsdial",
-    "tslogger",
-    "tstun",
-    "turnconn",
-    "typegen",
-    "typesafe",
-    "unconvert",
-    "Untar",
-    "Userspace",
-    "VMID",
-    "walkthrough",
-    "weblinks",
-    "webrtc",
-    "wgcfg",
-    "wgconfig",
-    "wgengine",
-    "wgmonitor",
-    "wgnet",
-    "workspaceagent",
-    "workspaceagents",
-    "workspaceapp",
-    "workspaceapps",
-    "workspacebuilds",
-    "workspacename",
-    "wsjson",
-    "xerrors",
-    "xlarge",
-    "xsmall",
-    "yamux"
-  ],
-  "cSpell.ignorePaths": ["site/package.json", ".vscode/settings.json"],
-  "emeraldwalk.runonsave": {
-    "commands": [
-      {
-        "match": "database/queries/*.sql",
-        "cmd": "make gen"
-      },
-      {
-        "match": "provisionerd/proto/provisionerd.proto",
-        "cmd": "make provisionerd/proto/provisionerd.pb.go"
-      }
-    ]
-  },
-  "eslint.workingDirectories": ["./site"],
-  "search.exclude": {
-    "**.pb.go": true,
-    "**/*.gen.json": true,
-    "**/testdata/*": true,
-    "coderd/apidoc/**": true,
-    "docs/api/*.md": true,
-    "docs/templates/*.md": true,
-    "LICENSE": true,
-    "scripts/metricsdocgen/metrics": true,
-    "site/out/**": true,
-    "site/storybook-static/**": true,
-    "**.map": true,
-    "pnpm-lock.yaml": true
-  },
-  // Ensure files always have a newline.
-  "files.insertFinalNewline": true,
-  "go.lintTool": "golangci-lint",
-  "go.lintFlags": ["--fast"],
-  "go.coverageDecorator": {
-    "type": "gutter",
-    "coveredGutterStyle": "blockgreen",
-    "uncoveredGutterStyle": "blockred"
-  },
-  // The codersdk is used by coderd another other packages extensively.
-  // To reduce redundancy in tests, it's covered by other packages.
-  // Since package coverage pairing can't be defined, all packages cover
-  // all other packages.
-  "go.testFlags": ["-short", "-coverpkg=./..."],
-  // We often use a version of TypeScript that's ahead of the version shipped
-  // with VS Code.
-  "typescript.tsdk": "./site/node_modules/typescript/lib",
-  // Playwright tests in VSCode will open a browser to live "view" the test.
-  "playwright.reuseBrowser": true
+	"emeraldwalk.runonsave": {
+		"commands": [
+			{
+				"match": "database/queries/*.sql",
+				"cmd": "make gen"
+			},
+			{
+				"match": "provisionerd/proto/provisionerd.proto",
+				"cmd": "make provisionerd/proto/provisionerd.pb.go"
+			}
+		]
+	},
+	"search.exclude": {
+		"**.pb.go": true,
+		"**/*.gen.json": true,
+		"**/testdata/*": true,
+		"coderd/apidoc/**": true,
+		"docs/reference/api/*.md": true,
+		"docs/reference/cli/*.md": true,
+		"docs/templates/*.md": true,
+		"LICENSE": true,
+		"scripts/metricsdocgen/metrics": true,
+		"site/out/**": true,
+		"site/storybook-static/**": true,
+		"**.map": true,
+		"pnpm-lock.yaml": true
+	},
+	// Ensure files always have a newline.
+	"files.insertFinalNewline": true,
+	"go.lintTool": "golangci-lint",
+	"go.lintFlags": ["--fast"],
+	"go.coverageDecorator": {
+		"type": "gutter",
+		"coveredGutterStyle": "blockgreen",
+		"uncoveredGutterStyle": "blockred"
+	},
+	// The codersdk is used by coderd another other packages extensively.
+	// To reduce redundancy in tests, it's covered by other packages.
+	// Since package coverage pairing can't be defined, all packages cover
+	// all other packages.
+	"go.testFlags": ["-short", "-coverpkg=./..."],
+	// We often use a version of TypeScript that's ahead of the version shipped
+	// with VS Code.
+	"typescript.tsdk": "./site/node_modules/typescript/lib",
+	// Playwright tests in VSCode will open a browser to live "view" the test.
+	"playwright.reuseBrowser": true,
+
+	"[javascript][javascriptreact][json][jsonc][typescript][typescriptreact]": {
+		"editor.defaultFormatter": "biomejs.biome",
+		"editor.codeActionsOnSave": {
+			"quickfix.biome": "explicit"
+			//   "source.organizeImports.biome": "explicit"
+		}
+	},
+
+	"[css][html][markdown][yaml]": {
+		"editor.defaultFormatter": "esbenp.prettier-vscode"
+	},
+	"typos.config": ".github/workflows/typos.toml"
 }

CODEOWNERS

@@ -0,0 +1,6 @@
+# These APIs are versioned, so any changes need to be carefully reviewed for whether
+# to bump API major or minor versions.
+agent/proto/ @spikecurtis @johnstcn
+tailnet/proto/ @spikecurtis @johnstcn
+vpn/vpn.proto @spikecurtis @johnstcn
+vpn/version.go @spikecurtis @johnstcn

CODE_OF_CONDUCT.md

@@ -0,0 +1 @@
+[https://coder.com/docs/coder-oss/latest/contributing/CODE_OF_CONDUCT](https://coder.com/docs/contributing/CODE_OF_CONDUCT)

Makefile

@@ -36,6 +36,7 @@ GOOS         := $(shell go env GOOS)
 GOARCH       := $(shell go env GOARCH)
 GOOS_BIN_EXT := $(if $(filter windows, $(GOOS)),.exe,)
 VERSION      := $(shell ./scripts/version.sh)
+POSTGRES_VERSION ?= 16
 
 # Use the highest ZSTD compression level in CI.
 ifdef CI
@@ -78,8 +79,12 @@ PACKAGE_OS_ARCHES := linux_amd64 linux_armv7 linux_arm64
 # All architectures we build Docker images for (Linux only).
 DOCKER_ARCHES := amd64 arm64 armv7
 
+# All ${OS}_${ARCH} combos we build the desktop dylib for.
+DYLIB_ARCHES := darwin_amd64 darwin_arm64
+
 # Computed variables based on the above.
 CODER_SLIM_BINARIES      := $(addprefix build/coder-slim_$(VERSION)_,$(OS_ARCHES))
+CODER_DYLIBS             := $(foreach os_arch, $(DYLIB_ARCHES), build/coder-vpn_$(VERSION)_$(os_arch).dylib)
 CODER_FAT_BINARIES       := $(addprefix build/coder_$(VERSION)_,$(OS_ARCHES))
 CODER_ALL_BINARIES       := $(CODER_SLIM_BINARIES) $(CODER_FAT_BINARIES)
 CODER_TAR_GZ_ARCHIVES    := $(foreach os_arch, $(ARCHIVE_TAR_GZ), build/coder_$(VERSION)_$(os_arch).tar.gz)
@@ -237,6 +242,25 @@ $(CODER_ALL_BINARIES): go.mod go.sum \
 		cp "$@" "./site/out/bin/coder-$$os-$$arch$$dot_ext"
 	fi
 
+# This task builds Coder Desktop dylibs
+$(CODER_DYLIBS): go.mod go.sum $(GO_SRC_FILES)
+	@if [ "$(shell uname)" = "Darwin" ]; then
+		$(get-mode-os-arch-ext)
+		./scripts/build_go.sh \
+			--os "$$os" \
+			--arch "$$arch" \
+			--version "$(VERSION)" \
+			--output "$@" \
+			--dylib
+
+	else
+		echo "ERROR: Can't build dylib on non-Darwin OS" 1>&2
+		exit 1
+	fi
+
+# This task builds both dylibs
+build/coder-dylib: $(CODER_DYLIBS)
+
 # This task builds all archives. It parses the target name to get the metadata
 # for the build, so it must be specified in this format:
 #     build/coder_${version}_${os}_${arch}.${format}
@@ -390,25 +414,30 @@ BOLD := $(shell tput bold 2>/dev/null)
 GREEN := $(shell tput setaf 2 2>/dev/null)
 RESET := $(shell tput sgr0 2>/dev/null)
 
-fmt: fmt/eslint fmt/prettier fmt/terraform fmt/shfmt fmt/go
+fmt: fmt/ts fmt/go fmt/terraform fmt/shfmt fmt/prettier
 .PHONY: fmt
 
 fmt/go:
+	go mod tidy
 	echo "$(GREEN)==>$(RESET) $(BOLD)fmt/go$(RESET)"
 	# VS Code users should check out
 	# https://github.com/mvdan/gofumpt#visual-studio-code
 	go run mvdan.cc/gofumpt@v0.4.0 -w -l .
 .PHONY: fmt/go
 
-fmt/eslint:
-	echo "$(GREEN)==>$(RESET) $(BOLD)fmt/eslint$(RESET)"
+fmt/ts:
+	echo "$(GREEN)==>$(RESET) $(BOLD)fmt/ts$(RESET)"
 	cd site
-	pnpm run lint:fix
-.PHONY: fmt/eslint
+# Avoid writing files in CI to reduce file write activity
+ifdef CI
+	pnpm run check --linter-enabled=false
+else
+	pnpm run check:fix
+endif
+.PHONY: fmt/ts
 
-fmt/prettier:
+fmt/prettier: .prettierignore
 	echo "$(GREEN)==>$(RESET) $(BOLD)fmt/prettier$(RESET)"
-	cd site
 # Avoid writing files in CI to reduce file write activity
 ifdef CI
 	pnpm run format:check
@@ -441,14 +470,14 @@ lint/site-icons:
 
 lint/ts:
 	cd site
-	pnpm i && pnpm lint
+	pnpm lint
 .PHONY: lint/ts
 
 lint/go:
 	./scripts/check_enterprise_imports.sh
-	linter_ver=$(shell egrep -o 'GOLANGCI_LINT_VERSION=\S+' dogfood/Dockerfile | cut -d '=' -f 2)
-	go install github.com/golangci/golangci-lint/cmd/golangci-lint@v$$linter_ver
-	golangci-lint run
+	./scripts/check_codersdk_imports.sh
+	linter_ver=$(shell egrep -o 'GOLANGCI_LINT_VERSION=\S+' dogfood/contents/Dockerfile | cut -d '=' -f 2)
+	go run github.com/golangci/golangci-lint/cmd/golangci-lint@v$$linter_ver run
 .PHONY: lint/go
 
 lint/examples:
@@ -476,33 +505,39 @@ DB_GEN_FILES := \
 	coderd/database/dbauthz/dbauthz.go \
 	coderd/database/dbmock/dbmock.go
 
+TAILNETTEST_MOCKS := \
+	tailnet/tailnettest/coordinatormock.go \
+	tailnet/tailnettest/coordinateemock.go \
+	tailnet/tailnettest/workspaceupdatesprovidermock.go \
+	tailnet/tailnettest/subscriptionmock.go
+
+
 # all gen targets should be added here and to gen/mark-fresh
 gen: \
 	tailnet/proto/tailnet.pb.go \
 	agent/proto/agent.pb.go \
 	provisionersdk/proto/provisioner.pb.go \
 	provisionerd/proto/provisionerd.pb.go \
+	vpn/vpn.pb.go \
 	coderd/database/dump.sql \
 	$(DB_GEN_FILES) \
 	site/src/api/typesGenerated.ts \
 	coderd/rbac/object_gen.go \
 	codersdk/rbacresources_gen.go \
-	docs/admin/prometheus.md \
-	docs/cli.md \
-	docs/admin/audit-logs.md \
+	site/src/api/rbacresourcesGenerated.ts \
+	site/src/api/countriesGenerated.ts \
+	docs/admin/integrations/prometheus.md \
+	docs/reference/cli/index.md \
+	docs/admin/security/audit-logs.md \
 	coderd/apidoc/swagger.json \
 	.prettierignore.include \
 	.prettierignore \
 	provisioner/terraform/testdata/version \
-	site/.prettierrc.yaml \
-	site/.prettierignore \
-	site/.eslintignore \
 	site/e2e/provisionerGenerated.ts \
 	site/src/theme/icons.json \
 	examples/examples.gen.json \
-	tailnet/tailnettest/coordinatormock.go \
-	tailnet/tailnettest/coordinateemock.go \
-	tailnet/tailnettest/multiagentmock.go
+	$(TAILNETTEST_MOCKS) \
+	coderd/database/pubsub/psmock/psmock.go
 .PHONY: gen
 
 # Mark all generated files as fresh so make thinks they're up-to-date. This is
@@ -513,26 +548,27 @@ gen/mark-fresh:
 		agent/proto/agent.pb.go \
 		provisionersdk/proto/provisioner.pb.go \
 		provisionerd/proto/provisionerd.pb.go \
+		vpn/vpn.pb.go \
 		coderd/database/dump.sql \
 		$(DB_GEN_FILES) \
 		site/src/api/typesGenerated.ts \
 		coderd/rbac/object_gen.go \
-		docs/admin/prometheus.md \
-		docs/cli.md \
-		docs/admin/audit-logs.md \
+		codersdk/rbacresources_gen.go \
+		site/src/api/rbacresourcesGenerated.ts \
+		site/src/api/countriesGenerated.ts \
+		docs/admin/integrations/prometheus.md \
+		docs/reference/cli/index.md \
+		docs/admin/security/audit-logs.md \
 		coderd/apidoc/swagger.json \
 		.prettierignore.include \
 		.prettierignore \
-		site/.prettierrc.yaml \
-		site/.prettierignore \
-		site/.eslintignore \
 		site/e2e/provisionerGenerated.ts \
 		site/src/theme/icons.json \
 		examples/examples.gen.json \
-		tailnet/tailnettest/coordinatormock.go \
-		tailnet/tailnettest/coordinateemock.go \
-		tailnet/tailnettest/multiagentmock.go \
-	"
+		$(TAILNETTEST_MOCKS) \
+		coderd/database/pubsub/psmock/psmock.go \
+		"
+
 	for file in $$files; do
 		echo "$$file"
 		if [ ! -f "$$file" ]; then
@@ -562,7 +598,7 @@ coderd/database/dbmock/dbmock.go: coderd/database/db.go coderd/database/querier.
 coderd/database/pubsub/psmock/psmock.go: coderd/database/pubsub/pubsub.go
 	go generate ./coderd/database/pubsub/psmock
 
-tailnet/tailnettest/coordinatormock.go tailnet/tailnettest/multiagentmock.go tailnet/tailnettest/coordinateemock.go: tailnet/coordinator.go tailnet/multiagent.go
+$(TAILNETTEST_MOCKS): tailnet/coordinator.go tailnet/service.go
 	go generate ./tailnet/tailnettest/
 
 tailnet/proto/tailnet.pb.go: tailnet/proto/tailnet.proto
@@ -597,10 +633,15 @@ provisionerd/proto/provisionerd.pb.go: provisionerd/proto/provisionerd.proto
 		--go-drpc_opt=paths=source_relative \
 		./provisionerd/proto/provisionerd.proto
 
+vpn/vpn.pb.go: vpn/vpn.proto
+	protoc \
+		--go_out=. \
+		--go_opt=paths=source_relative \
+		./vpn/vpn.proto
+
 site/src/api/typesGenerated.ts: $(wildcard scripts/apitypings/*) $(shell find ./codersdk $(FIND_EXCLUSIONS) -type f -name '*.go')
 	go run ./scripts/apitypings/ > $@
 	./scripts/pnpm_install.sh
-	pnpm exec prettier --write "$@"
 
 site/e2e/provisionerGenerated.ts: provisionerd/proto/provisionerd.pb.go provisionersdk/proto/provisioner.pb.go
 	cd site
@@ -610,36 +651,45 @@ site/e2e/provisionerGenerated.ts: provisionerd/proto/provisionerd.pb.go provisio
 site/src/theme/icons.json: $(wildcard scripts/gensite/*) $(wildcard site/static/icon/*)
 	go run ./scripts/gensite/ -icons "$@"
 	./scripts/pnpm_install.sh
-	pnpm exec prettier --write "$@"
+	pnpm -C site/ exec biome format --write src/theme/icons.json
 
 examples/examples.gen.json: scripts/examplegen/main.go examples/examples.go $(shell find ./examples/templates)
 	go run ./scripts/examplegen/main.go > examples/examples.gen.json
 
-coderd/rbac/object_gen.go: scripts/rbacgen/main.go coderd/rbac/object.go
-	go run scripts/rbacgen/main.go rbac > coderd/rbac/object_gen.go
+coderd/rbac/object_gen.go: scripts/typegen/rbacobject.gotmpl scripts/typegen/main.go coderd/rbac/object.go coderd/rbac/policy/policy.go
+	go run scripts/typegen/main.go rbac object > coderd/rbac/object_gen.go
 
-codersdk/rbacresources_gen.go: scripts/rbacgen/main.go coderd/rbac/object.go
-	go run scripts/rbacgen/main.go codersdk > codersdk/rbacresources_gen.go
+codersdk/rbacresources_gen.go: scripts/typegen/codersdk.gotmpl scripts/typegen/main.go coderd/rbac/object.go coderd/rbac/policy/policy.go
+	# Do no overwrite codersdk/rbacresources_gen.go directly, as it would make the file empty, breaking
+ 	# the `codersdk` package and any parallel build targets.
+	go run scripts/typegen/main.go rbac codersdk > /tmp/rbacresources_gen.go
+	mv /tmp/rbacresources_gen.go codersdk/rbacresources_gen.go
 
-docs/admin/prometheus.md: scripts/metricsdocgen/main.go scripts/metricsdocgen/metrics
+site/src/api/rbacresourcesGenerated.ts: scripts/typegen/codersdk.gotmpl scripts/typegen/main.go coderd/rbac/object.go coderd/rbac/policy/policy.go
+	go run scripts/typegen/main.go rbac typescript > "$@"
+
+site/src/api/countriesGenerated.ts: scripts/typegen/countries.tstmpl scripts/typegen/main.go codersdk/countries.go
+	go run scripts/typegen/main.go countries > "$@"
+
+docs/admin/integrations/prometheus.md: scripts/metricsdocgen/main.go scripts/metricsdocgen/metrics
 	go run scripts/metricsdocgen/main.go
 	./scripts/pnpm_install.sh
-	pnpm exec prettier --write ./docs/admin/prometheus.md
+	pnpm exec prettier --write ./docs/admin/integrations/prometheus.md
 
-docs/cli.md: scripts/clidocgen/main.go examples/examples.gen.json $(GO_SRC_FILES)
+docs/reference/cli/index.md: scripts/clidocgen/main.go examples/examples.gen.json $(GO_SRC_FILES)
 	CI=true BASE_PATH="." go run ./scripts/clidocgen
 	./scripts/pnpm_install.sh
-	pnpm exec prettier --write ./docs/cli.md ./docs/cli/*.md ./docs/manifest.json
+	pnpm exec prettier --write ./docs/reference/cli/index.md ./docs/reference/cli/*.md ./docs/manifest.json
 
-docs/admin/audit-logs.md: coderd/database/querier.go scripts/auditdocgen/main.go enterprise/audit/table.go coderd/rbac/object_gen.go
+docs/admin/security/audit-logs.md: coderd/database/querier.go scripts/auditdocgen/main.go enterprise/audit/table.go coderd/rbac/object_gen.go
 	go run scripts/auditdocgen/main.go
 	./scripts/pnpm_install.sh
-	pnpm exec prettier --write ./docs/admin/audit-logs.md
+	pnpm exec prettier --write ./docs/admin/security/audit-logs.md
 
 coderd/apidoc/swagger.json: $(shell find ./scripts/apidocgen $(FIND_EXCLUSIONS) -type f) $(wildcard coderd/*.go) $(wildcard enterprise/coderd/*.go) $(wildcard codersdk/*.go) $(wildcard enterprise/wsproxy/wsproxysdk/*.go) $(DB_GEN_FILES) .swaggo docs/manifest.json coderd/rbac/object_gen.go
 	./scripts/apidocgen/generate.sh
 	./scripts/pnpm_install.sh
-	pnpm exec prettier --write ./docs/api ./docs/manifest.json ./coderd/apidoc/swagger.json
+	pnpm exec prettier --write ./docs/reference/api ./docs/manifest.json ./coderd/apidoc/swagger.json
 
 update-golden-files: \
 	cli/testdata/.gen-golden \
@@ -650,6 +700,7 @@ update-golden-files: \
 	enterprise/tailnet/testdata/.gen-golden \
 	tailnet/testdata/.gen-golden \
 	coderd/.gen-golden \
+	coderd/notifications/.gen-golden \
 	provisioner/terraform/testdata/.gen-golden
 .PHONY: update-golden-files
 
@@ -681,6 +732,10 @@ coderd/.gen-golden: $(wildcard coderd/testdata/*/*.golden) $(GO_SRC_FILES) $(wil
 	go test ./coderd -run="Test.*Golden$$" -update
 	touch "$@"
 
+coderd/notifications/.gen-golden: $(wildcard coderd/notifications/testdata/*/*.golden) $(GO_SRC_FILES) $(wildcard coderd/notifications/*_test.go)
+	go test ./coderd/notifications -run="Test.*Golden$$" -update
+	touch "$@"
+
 provisioner/terraform/testdata/.gen-golden: $(wildcard provisioner/terraform/testdata/*/*.golden) $(GO_SRC_FILES) $(wildcard provisioner/terraform/*_test.go)
 	go test ./provisioner/terraform -run="Test.*Golden$$" -update
 	touch "$@"
@@ -695,23 +750,6 @@ scripts/ci-report/testdata/.gen-golden: $(wildcard scripts/ci-report/testdata/*)
 	go test ./scripts/ci-report -run=TestOutputMatchesGoldenFile -update
 	touch "$@"
 
-# Generate a prettierrc for the site package that uses relative paths for
-# overrides. This allows us to share the same prettier config between the
-# site and the root of the repo.
-site/.prettierrc.yaml: .prettierrc.yaml
-	. ./scripts/lib.sh
-	dependencies yq
-
-	echo "# Code generated by Makefile (../$<). DO NOT EDIT." > "$@"
-	echo "" >> "$@"
-
-	# Replace all listed override files with relative paths inside site/.
-	# - ./ -> ../
-	# - ./site -> ./
-	yq \
-		'.overrides[].files |= map(. | sub("^./"; "") | sub("^"; "../") | sub("../site/"; "./") | sub("../!"; "!../"))' \
-		"$<" >> "$@"
-
 # Combine .gitignore with .prettierignore.include to generate .prettierignore.
 .prettierignore: .gitignore .prettierignore.include
 	echo "# Code generated by Makefile ($^). DO NOT EDIT." > "$@"
@@ -721,40 +759,6 @@ site/.prettierrc.yaml: .prettierrc.yaml
 		cat "$$f" >> "$@"
 	done
 
-# Generate ignore files based on gitignore into the site directory. We turn all
-# rules into relative paths for the `site/` directory (where applicable),
-# following the pattern format defined by git:
-# https://git-scm.com/docs/gitignore#_pattern_format
-#
-# This is done for compatibility reasons, see:
-# https://github.com/prettier/prettier/issues/8048
-# https://github.com/prettier/prettier/issues/8506
-# https://github.com/prettier/prettier/issues/8679
-site/.eslintignore site/.prettierignore: .prettierignore Makefile
-	rm -f "$@"
-	touch "$@"
-	# Skip generated by header, inherit `.prettierignore` header as-is.
-	while read -r rule; do
-		# Remove leading ! if present to simplify rule, added back at the end.
-		tmp="$${rule#!}"
-		ignore="$${rule%"$$tmp"}"
-		rule="$$tmp"
-		case "$$rule" in
-			# Comments or empty lines (include).
-			\#*|'') ;;
-			# Generic rules (include).
-			\*\**) ;;
-			# Site prefixed rules (include).
-			site/*) rule="$${rule#site/}";;
-			./site/*) rule="$${rule#./site/}";;
-			# Rules that are non-generic and don't start with site (rewrite).
-			/*) rule=.."$$rule";;
-			*/?*) rule=../"$$rule";;
-			*) ;;
-		esac
-		echo "$${ignore}$${rule}" >> "$@"
-	done < "$<"
-
 test:
 	$(GIT_FLAGS) gotestsum --format standard-quiet -- -v -short -count=1 ./...
 .PHONY: test
@@ -792,7 +796,7 @@ sqlc-vet: test-postgres-docker
 test-postgres: test-postgres-docker
 	# The postgres test is prone to failure, so we limit parallelism for
 	# more consistent execution.
-	$(GIT_FLAGS)  DB=ci DB_FROM=$(shell go run scripts/migrate-ci/main.go) gotestsum \
+	$(GIT_FLAGS)  DB=ci gotestsum \
 		--junitfile="gotests.xml" \
 		--jsonfile="gotests.json" \
 		--packages="./..." -- \
@@ -814,7 +818,7 @@ test-migrations: test-postgres-docker
 
 # NOTE: we set --memory to the same size as a GitHub runner.
 test-postgres-docker:
-	docker rm -f test-postgres-docker || true
+	docker rm -f test-postgres-docker-${POSTGRES_VERSION} || true
 	docker run \
 		--env POSTGRES_PASSWORD=postgres \
 		--env POSTGRES_USER=postgres \
@@ -822,11 +826,11 @@ test-postgres-docker:
 		--env PGDATA=/tmp \
 		--tmpfs /tmp \
 		--publish 5432:5432 \
-		--name test-postgres-docker \
+		--name test-postgres-docker-${POSTGRES_VERSION} \
 		--restart no \
 		--detach \
 		--memory 16GB \
-		gcr.io/coder-dev-1/postgres:13 \
+		gcr.io/coder-dev-1/postgres:${POSTGRES_VERSION} \
 		-c shared_buffers=1GB \
 		-c work_mem=1GB \
 		-c effective_cache_size=1GB \
@@ -844,7 +848,7 @@ test-postgres-docker:
 
 # Make sure to keep this in sync with test-go-race from .github/workflows/ci.yaml.
 test-race:
-	$(GIT_FLAGS) gotestsum --junitfile="gotests.xml" -- -race -count=1 ./...
+	$(GIT_FLAGS) gotestsum --junitfile="gotests.xml" -- -race -count=1 -parallel 4 -p 4 ./...
 .PHONY: test-race
 
 test-tailnet-integration:
@@ -865,3 +869,7 @@ test-tailnet-integration:
 test-clean:
 	go clean -testcache
 .PHONY: test-clean
+
+.PHONY: test-e2e
+test-e2e:
+	cd ./site && DEBUG=pw:api pnpm playwright:test --forbid-only --workers 1

README.md

@@ -20,17 +20,19 @@
   <br>
   <br>
 
-[Quickstart](#quickstart) | [Docs](https://coder.com/docs) | [Why Coder](https://coder.com/why) | [Enterprise](https://coder.com/docs/v2/latest/enterprise)
+[Quickstart](#quickstart) | [Docs](https://coder.com/docs) | [Why Coder](https://coder.com/why) | [Premium](https://coder.com/pricing#compare-plans)
 
 [![discord](https://img.shields.io/discord/747933592273027093?label=discord)](https://discord.gg/coder)
 [![release](https://img.shields.io/github/v/release/coder/coder)](https://github.com/coder/coder/releases/latest)
 [![godoc](https://pkg.go.dev/badge/github.com/coder/coder.svg)](https://pkg.go.dev/github.com/coder/coder)
-[![Go Report Card](https://goreportcard.com/badge/github.com/coder/coder)](https://goreportcard.com/report/github.com/coder/coder)
+[![Go Report Card](https://goreportcard.com/badge/github.com/coder/coder/v2)](https://goreportcard.com/report/github.com/coder/coder/v2)
+[![OpenSSF Best Practices](https://www.bestpractices.dev/projects/9511/badge)](https://www.bestpractices.dev/projects/9511)
+[![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/coder/coder/badge)](https://scorecard.dev/viewer/?uri=github.com%2Fcoder%2Fcoder)
 [![license](https://img.shields.io/github/license/coder/coder)](./LICENSE)
 
 </div>
 
-[Coder](https://coder.com) enables organizations to set up development environments in their public or private cloud infrastructure. Cloud development environments are defined with Terraform, connected through a secure high-speed Wireguard® tunnel, and are automatically shut down when not in use to save on costs. Coder gives engineering teams the flexibility to use the cloud for workloads that are most beneficial to them.
+[Coder](https://coder.com) enables organizations to set up development environments in their public or private cloud infrastructure. Cloud development environments are defined with Terraform, connected through a secure high-speed Wireguard® tunnel, and automatically shut down when not used to save on costs. Coder gives engineering teams the flexibility to use the cloud for workloads most beneficial to them.
 
 - Define cloud development environments in Terraform
   - EC2 VMs, Kubernetes Pods, Docker Containers, etc.
@@ -53,7 +55,7 @@ curl -L https://coder.com/install.sh | sh
 coder server
 
 # Navigate to http://localhost:3000 to create your initial user,
-# create a Docker template, and provision a workspace
+# create a Docker template and provision a workspace
 ```
 
 ## Install
@@ -69,7 +71,7 @@ curl -L https://coder.com/install.sh | sh
 
 You can run the install script with `--dry-run` to see the commands that will be used to install without executing them. Run the install script with `--help` for additional flags.
 
-> See [install](https://coder.com/docs/v2/latest/install) for additional methods.
+> See [install](https://coder.com/docs/install) for additional methods.
 
 Once installed, you can start a production deployment with a single command:
 
@@ -81,27 +83,27 @@ coder server
 coder server --postgres-url <url> --access-url <url>
 ```
 
-Use `coder --help` to get a list of flags and environment variables. Use our [install guides](https://coder.com/docs/v2/latest/install) for a full walkthrough.
+Use `coder --help` to get a list of flags and environment variables. Use our [install guides](https://coder.com/docs/install) for a complete walkthrough.
 
 ## Documentation
 
-Browse our docs [here](https://coder.com/docs/v2) or visit a specific section below:
+Browse our docs [here](https://coder.com/docs) or visit a specific section below:
 
-- [**Templates**](https://coder.com/docs/v2/latest/templates): Templates are written in Terraform and describe the infrastructure for workspaces
-- [**Workspaces**](https://coder.com/docs/v2/latest/workspaces): Workspaces contain the IDEs, dependencies, and configuration information needed for software development
-- [**IDEs**](https://coder.com/docs/v2/latest/ides): Connect your existing editor to a workspace
-- [**Administration**](https://coder.com/docs/v2/latest/admin): Learn how to operate Coder
-- [**Enterprise**](https://coder.com/docs/v2/latest/enterprise): Learn about our paid features built for large teams
+- [**Templates**](https://coder.com/docs/templates): Templates are written in Terraform and describe the infrastructure for workspaces
+- [**Workspaces**](https://coder.com/docs/workspaces): Workspaces contain the IDEs, dependencies, and configuration information needed for software development
+- [**IDEs**](https://coder.com/docs/ides): Connect your existing editor to a workspace
+- [**Administration**](https://coder.com/docs/admin): Learn how to operate Coder
+- [**Premium**](https://coder.com/pricing#compare-plans): Learn about our paid features built for large teams
 
 ## Support
 
 Feel free to [open an issue](https://github.com/coder/coder/issues/new) if you have questions, run into bugs, or have a feature request.
 
-[Join our Discord](https://discord.gg/coder) to provide feedback on in-progress features, and chat with the community using Coder!
+[Join our Discord](https://discord.gg/coder) to provide feedback on in-progress features and chat with the community using Coder!
 
 ## Integrations
 
-We are always working on new integrations. Feel free to open an issue to request an integration. Contributions are welcome in any official or community repositories.
+We are always working on new integrations. Please feel free to open an issue and ask for an integration. Contributions are welcome in any official or community repositories.
 
 ### Official
 
@@ -111,6 +113,7 @@ We are always working on new integrations. Feel free to open an issue to request
 - [**Module Registry**](https://registry.coder.com): Extend development environments with common use-cases
 - [**Kubernetes Log Stream**](https://github.com/coder/coder-logstream-kube): Stream Kubernetes Pod events to the Coder startup logs
 - [**Self-Hosted VS Code Extension Marketplace**](https://github.com/coder/code-marketplace): A private extension marketplace that works in restricted or airgapped networks integrating with [code-server](https://github.com/coder/code-server).
+- [**Setup Coder**](https://github.com/marketplace/actions/setup-coder): An action to setup coder CLI in GitHub workflows.
 
 ### Community
 
@@ -120,9 +123,9 @@ We are always working on new integrations. Feel free to open an issue to request
 ## Contributing
 
 We are always happy to see new contributors to Coder. If you are new to the Coder codebase, we have
-[a guide on how to get started](https://coder.com/docs/v2/latest/CONTRIBUTING). We'd love to see your
+[a guide on how to get started](https://coder.com/docs/CONTRIBUTING). We'd love to see your
 contributions!
 
 ## Hiring
 
-Apply [here](https://cdr.co/github-apply) if you're interested in joining our team.
+Apply [here](https://jobs.ashbyhq.com/coder?utm_source=github&utm_medium=readme&utm_campaign=unknown) if you're interested in joining our team.

agent/agent.go

@@ -3,12 +3,10 @@ package agent
 import (
 	"bytes"
 	"context"
-	"encoding/binary"
 	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
-	"net"
 	"net/http"
 	"net/netip"
 	"os"
@@ -31,7 +29,6 @@ import (
 	"golang.org/x/exp/slices"
 	"golang.org/x/sync/errgroup"
 	"golang.org/x/xerrors"
-	"storj.io/drpc"
 	"tailscale.com/net/speedtest"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/netlogtype"
@@ -82,7 +79,6 @@ type Options struct {
 	SSHMaxTimeout                time.Duration
 	TailnetListenPort            uint16
 	Subsystems                   []codersdk.AgentSubsystem
-	Addresses                    []netip.Prefix
 	PrometheusRegistry           *prometheus.Registry
 	ReportMetadataInterval       time.Duration
 	ServiceBannerRefreshInterval time.Duration
@@ -91,10 +87,13 @@ type Options struct {
 	ModifiedProcesses chan []*agentproc.Process
 	// ProcessManagementTick is used for testing process priority management.
 	ProcessManagementTick <-chan time.Time
+	BlockFileTransfer     bool
 }
 
 type Client interface {
-	ConnectRPC(ctx context.Context) (drpc.Conn, error)
+	ConnectRPC23(ctx context.Context) (
+		proto.DRPCAgentClient23, tailnetproto.DRPCTailnetClient23, error,
+	)
 	RewriteDERPMap(derpMap *tailcfg.DERPMap)
 }
 
@@ -179,11 +178,11 @@ func New(options Options) Agent {
 		announcementBannersRefreshInterval: options.ServiceBannerRefreshInterval,
 		sshMaxTimeout:                      options.SSHMaxTimeout,
 		subsystems:                         options.Subsystems,
-		addresses:                          options.Addresses,
 		syscaller:                          options.Syscaller,
 		modifiedProcs:                      options.ModifiedProcesses,
 		processManagementTick:              options.ProcessManagementTick,
 		logSender:                          agentsdk.NewLogSender(options.Logger),
+		blockFileTransfer:                  options.BlockFileTransfer,
 
 		prometheusRegistry: prometheusRegistry,
 		metrics:            newAgentMetrics(prometheusRegistry),
@@ -215,8 +214,8 @@ type agent struct {
 	portCacheDuration time.Duration
 	subsystems        []codersdk.AgentSubsystem
 
-	reconnectingPTYs       sync.Map
 	reconnectingPTYTimeout time.Duration
+	reconnectingPTYServer  *reconnectingpty.Server
 
 	// we track 2 contexts and associated cancel functions: "graceful" which is Done when it is time
 	// to start gracefully shutting down and "hard" which is Done when it is time to close
@@ -239,6 +238,7 @@ type agent struct {
 	sessionToken                       atomic.Pointer[string]
 	sshServer                          *agentssh.Server
 	sshMaxTimeout                      time.Duration
+	blockFileTransfer                  bool
 
 	lifecycleUpdate            chan struct{}
 	lifecycleReported          chan codersdk.WorkspaceAgentLifecycle
@@ -247,12 +247,9 @@ type agent struct {
 	lifecycleLastReportedIndex int // Keeps track of the last lifecycle state we successfully reported.
 
 	network       *tailnet.Conn
-	addresses     []netip.Prefix
 	statsReporter *statsReporter
 	logSender     *agentsdk.LogSender
 
-	connCountReconnectingPTY atomic.Int64
-
 	prometheusRegistry *prometheus.Registry
 	// metrics are prometheus registered metrics that will be collected and
 	// labeled in Coder with the agent + workspace.
@@ -277,6 +274,7 @@ func (a *agent) init() {
 		AnnouncementBanners: func() *[]codersdk.BannerConfig { return a.announcementBanners.Load() },
 		UpdateEnv:           a.updateCommandEnv,
 		WorkingDirectory:    func() string { return a.manifest.Load().Directory },
+		BlockFileTransfer:   a.blockFileTransfer,
 	})
 	if err != nil {
 		panic(err)
@@ -295,6 +293,13 @@ func (a *agent) init() {
 	// Register runner metrics. If the prom registry is nil, the metrics
 	// will not report anywhere.
 	a.scriptRunner.RegisterMetrics(a.prometheusRegistry)
+
+	a.reconnectingPTYServer = reconnectingpty.NewServer(
+		a.logger.Named("reconnecting-pty"),
+		a.sshServer,
+		a.metrics.connectionsTotal, a.metrics.reconnectingPTYErrors,
+		a.reconnectingPTYTimeout,
+	)
 	go a.runLoop()
 }
 
@@ -409,7 +414,7 @@ func (t *trySingleflight) Do(key string, fn func()) {
 	fn()
 }
 
-func (a *agent) reportMetadata(ctx context.Context, conn drpc.Conn) error {
+func (a *agent) reportMetadata(ctx context.Context, aAPI proto.DRPCAgentClient23) error {
 	tickerDone := make(chan struct{})
 	collectDone := make(chan struct{})
 	ctx, cancel := context.WithCancel(ctx)
@@ -571,7 +576,6 @@ func (a *agent) reportMetadata(ctx context.Context, conn drpc.Conn) error {
 		reportTimeout   = 30 * time.Second
 		reportError     = make(chan error, 1)
 		reportInFlight  = false
-		aAPI            = proto.NewDRPCAgentClient(conn)
 	)
 
 	for {
@@ -584,10 +588,12 @@ func (a *agent) reportMetadata(ctx context.Context, conn drpc.Conn) error {
 			updatedMetadata[mr.key] = mr.result
 			continue
 		case err := <-reportError:
-			a.logger.Debug(ctx, "batch update metadata complete", slog.Error(err))
+			logMsg := "batch update metadata complete"
 			if err != nil {
+				a.logger.Debug(ctx, logMsg, slog.Error(err))
 				return xerrors.Errorf("failed to report metadata: %w", err)
 			}
+			a.logger.Debug(ctx, logMsg)
 			reportInFlight = false
 		case <-report:
 			if len(updatedMetadata) == 0 {
@@ -624,8 +630,7 @@ func (a *agent) reportMetadata(ctx context.Context, conn drpc.Conn) error {
 
 // reportLifecycle reports the current lifecycle state once. All state
 // changes are reported in order.
-func (a *agent) reportLifecycle(ctx context.Context, conn drpc.Conn) error {
-	aAPI := proto.NewDRPCAgentClient(conn)
+func (a *agent) reportLifecycle(ctx context.Context, aAPI proto.DRPCAgentClient23) error {
 	for {
 		select {
 		case <-a.lifecycleUpdate:
@@ -707,8 +712,7 @@ func (a *agent) setLifecycle(state codersdk.WorkspaceAgentLifecycle) {
 // fetchServiceBannerLoop fetches the service banner on an interval.  It will
 // not be fetched immediately; the expectation is that it is primed elsewhere
 // (and must be done before the session actually starts).
-func (a *agent) fetchServiceBannerLoop(ctx context.Context, conn drpc.Conn) error {
-	aAPI := proto.NewDRPCAgentClient(conn)
+func (a *agent) fetchServiceBannerLoop(ctx context.Context, aAPI proto.DRPCAgentClient23) error {
 	ticker := time.NewTicker(a.announcementBannersRefreshInterval)
 	defer ticker.Stop()
 	for {
@@ -734,7 +738,7 @@ func (a *agent) fetchServiceBannerLoop(ctx context.Context, conn drpc.Conn) erro
 }
 
 func (a *agent) run() (retErr error) {
-	// This allows the agent to refresh it's token if necessary.
+	// This allows the agent to refresh its token if necessary.
 	// For instance identity this is required, since the instance
 	// may not have re-provisioned, but a new agent ID was created.
 	sessionToken, err := a.exchangeToken(a.hardCtx)
@@ -744,12 +748,12 @@ func (a *agent) run() (retErr error) {
 	a.sessionToken.Store(&sessionToken)
 
 	// ConnectRPC returns the dRPC connection we use for the Agent and Tailnet v2+ APIs
-	conn, err := a.client.ConnectRPC(a.hardCtx)
+	aAPI, tAPI, err := a.client.ConnectRPC23(a.hardCtx)
 	if err != nil {
 		return err
 	}
 	defer func() {
-		cErr := conn.Close()
+		cErr := aAPI.DRPCConn().Close()
 		if cErr != nil {
 			a.logger.Debug(a.hardCtx, "error closing drpc connection", slog.Error(err))
 		}
@@ -758,11 +762,10 @@ func (a *agent) run() (retErr error) {
 	// A lot of routines need the agent API / tailnet API connection.  We run them in their own
 	// goroutines in parallel, but errors in any routine will cause them all to exit so we can
 	// redial the coder server and retry.
-	connMan := newAPIConnRoutineManager(a.gracefulCtx, a.hardCtx, a.logger, conn)
+	connMan := newAPIConnRoutineManager(a.gracefulCtx, a.hardCtx, a.logger, aAPI, tAPI)
 
-	connMan.start("init notification banners", gracefulShutdownBehaviorStop,
-		func(ctx context.Context, conn drpc.Conn) error {
-			aAPI := proto.NewDRPCAgentClient(conn)
+	connMan.startAgentAPI("init notification banners", gracefulShutdownBehaviorStop,
+		func(ctx context.Context, aAPI proto.DRPCAgentClient23) error {
 			bannersProto, err := aAPI.GetAnnouncementBanners(ctx, &proto.GetAnnouncementBannersRequest{})
 			if err != nil {
 				return xerrors.Errorf("fetch service banner: %w", err)
@@ -778,9 +781,9 @@ func (a *agent) run() (retErr error) {
 
 	// sending logs gets gracefulShutdownBehaviorRemain because we want to send logs generated by
 	// shutdown scripts.
-	connMan.start("send logs", gracefulShutdownBehaviorRemain,
-		func(ctx context.Context, conn drpc.Conn) error {
-			err := a.logSender.SendLoop(ctx, proto.NewDRPCAgentClient(conn))
+	connMan.startAgentAPI("send logs", gracefulShutdownBehaviorRemain,
+		func(ctx context.Context, aAPI proto.DRPCAgentClient23) error {
+			err := a.logSender.SendLoop(ctx, aAPI)
 			if xerrors.Is(err, agentsdk.LogLimitExceededError) {
 				// we don't want this error to tear down the API connection and propagate to the
 				// other routines that use the API.  The LogSender has already dropped a warning
@@ -792,10 +795,10 @@ func (a *agent) run() (retErr error) {
 
 	// part of graceful shut down is reporting the final lifecycle states, e.g "ShuttingDown" so the
 	// lifecycle reporting has to be via gracefulShutdownBehaviorRemain
-	connMan.start("report lifecycle", gracefulShutdownBehaviorRemain, a.reportLifecycle)
+	connMan.startAgentAPI("report lifecycle", gracefulShutdownBehaviorRemain, a.reportLifecycle)
 
 	// metadata reporting can cease as soon as we start gracefully shutting down
-	connMan.start("report metadata", gracefulShutdownBehaviorStop, a.reportMetadata)
+	connMan.startAgentAPI("report metadata", gracefulShutdownBehaviorStop, a.reportMetadata)
 
 	// channels to sync goroutines below
 	//  handle manifest
@@ -816,55 +819,55 @@ func (a *agent) run() (retErr error) {
 	networkOK := newCheckpoint(a.logger)
 	manifestOK := newCheckpoint(a.logger)
 
-	connMan.start("handle manifest", gracefulShutdownBehaviorStop, a.handleManifest(manifestOK))
+	connMan.startAgentAPI("handle manifest", gracefulShutdownBehaviorStop, a.handleManifest(manifestOK))
 
-	connMan.start("app health reporter", gracefulShutdownBehaviorStop,
-		func(ctx context.Context, conn drpc.Conn) error {
+	connMan.startAgentAPI("app health reporter", gracefulShutdownBehaviorStop,
+		func(ctx context.Context, aAPI proto.DRPCAgentClient23) error {
 			if err := manifestOK.wait(ctx); err != nil {
 				return xerrors.Errorf("no manifest: %w", err)
 			}
 			manifest := a.manifest.Load()
 			NewWorkspaceAppHealthReporter(
-				a.logger, manifest.Apps, agentsdk.AppHealthPoster(proto.NewDRPCAgentClient(conn)),
+				a.logger, manifest.Apps, agentsdk.AppHealthPoster(aAPI),
 			)(ctx)
 			return nil
 		})
 
-	connMan.start("create or update network", gracefulShutdownBehaviorStop,
+	connMan.startAgentAPI("create or update network", gracefulShutdownBehaviorStop,
 		a.createOrUpdateNetwork(manifestOK, networkOK))
 
-	connMan.start("coordination", gracefulShutdownBehaviorStop,
-		func(ctx context.Context, conn drpc.Conn) error {
+	connMan.startTailnetAPI("coordination", gracefulShutdownBehaviorStop,
+		func(ctx context.Context, tAPI tailnetproto.DRPCTailnetClient23) error {
 			if err := networkOK.wait(ctx); err != nil {
 				return xerrors.Errorf("no network: %w", err)
 			}
-			return a.runCoordinator(ctx, conn, a.network)
+			return a.runCoordinator(ctx, tAPI, a.network)
 		},
 	)
 
-	connMan.start("derp map subscriber", gracefulShutdownBehaviorStop,
-		func(ctx context.Context, conn drpc.Conn) error {
+	connMan.startTailnetAPI("derp map subscriber", gracefulShutdownBehaviorStop,
+		func(ctx context.Context, tAPI tailnetproto.DRPCTailnetClient23) error {
 			if err := networkOK.wait(ctx); err != nil {
 				return xerrors.Errorf("no network: %w", err)
 			}
-			return a.runDERPMapSubscriber(ctx, conn, a.network)
+			return a.runDERPMapSubscriber(ctx, tAPI, a.network)
 		})
 
-	connMan.start("fetch service banner loop", gracefulShutdownBehaviorStop, a.fetchServiceBannerLoop)
+	connMan.startAgentAPI("fetch service banner loop", gracefulShutdownBehaviorStop, a.fetchServiceBannerLoop)
 
-	connMan.start("stats report loop", gracefulShutdownBehaviorStop, func(ctx context.Context, conn drpc.Conn) error {
+	connMan.startAgentAPI("stats report loop", gracefulShutdownBehaviorStop, func(ctx context.Context, aAPI proto.DRPCAgentClient23) error {
 		if err := networkOK.wait(ctx); err != nil {
 			return xerrors.Errorf("no network: %w", err)
 		}
-		return a.statsReporter.reportLoop(ctx, proto.NewDRPCAgentClient(conn))
+		return a.statsReporter.reportLoop(ctx, aAPI)
 	})
 
 	return connMan.wait()
 }
 
 // handleManifest returns a function that fetches and processes the manifest
-func (a *agent) handleManifest(manifestOK *checkpoint) func(ctx context.Context, conn drpc.Conn) error {
-	return func(ctx context.Context, conn drpc.Conn) error {
+func (a *agent) handleManifest(manifestOK *checkpoint) func(ctx context.Context, aAPI proto.DRPCAgentClient23) error {
+	return func(ctx context.Context, aAPI proto.DRPCAgentClient23) error {
 		var (
 			sentResult = false
 			err        error
@@ -874,7 +877,6 @@ func (a *agent) handleManifest(manifestOK *checkpoint) func(ctx context.Context,
 				manifestOK.complete(err)
 			}
 		}()
-		aAPI := proto.NewDRPCAgentClient(conn)
 		mp, err := aAPI.GetManifest(ctx, &proto.GetManifestRequest{})
 		if err != nil {
 			return xerrors.Errorf("fetch metadata: %w", err)
@@ -935,7 +937,7 @@ func (a *agent) handleManifest(manifestOK *checkpoint) func(ctx context.Context,
 				}
 			}
 
-			err = a.scriptRunner.Init(manifest.Scripts)
+			err = a.scriptRunner.Init(manifest.Scripts, aAPI.ScriptCompleted)
 			if err != nil {
 				return xerrors.Errorf("init script runner: %w", err)
 			}
@@ -943,9 +945,7 @@ func (a *agent) handleManifest(manifestOK *checkpoint) func(ctx context.Context,
 				start := time.Now()
 				// here we use the graceful context because the script runner is not directly tied
 				// to the agent API.
-				err := a.scriptRunner.Execute(a.gracefulCtx, func(script codersdk.WorkspaceAgentScript) bool {
-					return script.RunOnStart
-				})
+				err := a.scriptRunner.Execute(a.gracefulCtx, agentscripts.ExecuteStartScripts)
 				// Measure the time immediately after the script has finished
 				dur := time.Since(start).Seconds()
 				if err != nil {
@@ -976,8 +976,8 @@ func (a *agent) handleManifest(manifestOK *checkpoint) func(ctx context.Context,
 
 // createOrUpdateNetwork waits for the manifest to be set using manifestOK, then creates or updates
 // the tailnet using the information in the manifest
-func (a *agent) createOrUpdateNetwork(manifestOK, networkOK *checkpoint) func(context.Context, drpc.Conn) error {
-	return func(ctx context.Context, _ drpc.Conn) (retErr error) {
+func (a *agent) createOrUpdateNetwork(manifestOK, networkOK *checkpoint) func(context.Context, proto.DRPCAgentClient23) error {
+	return func(ctx context.Context, _ proto.DRPCAgentClient23) (retErr error) {
 		if err := manifestOK.wait(ctx); err != nil {
 			return xerrors.Errorf("no manifest: %w", err)
 		}
@@ -1108,18 +1108,14 @@ func (a *agent) updateCommandEnv(current []string) (updated []string, err error)
 	return updated, nil
 }
 
-func (a *agent) wireguardAddresses(agentID uuid.UUID) []netip.Prefix {
-	if len(a.addresses) == 0 {
-		return []netip.Prefix{
-			// This is the IP that should be used primarily.
-			netip.PrefixFrom(tailnet.IPFromUUID(agentID), 128),
-			// We also listen on the legacy codersdk.WorkspaceAgentIP. This
-			// allows for a transition away from wsconncache.
-			netip.PrefixFrom(workspacesdk.AgentIP, 128),
-		}
+func (*agent) wireguardAddresses(agentID uuid.UUID) []netip.Prefix {
+	return []netip.Prefix{
+		// This is the IP that should be used primarily.
+		tailnet.TailscaleServicePrefix.PrefixFromUUID(agentID),
+		// We'll need this address for CoderVPN, but aren't using it from clients until that feature
+		// is ready
+		tailnet.CoderServicePrefix.PrefixFromUUID(agentID),
 	}
-
-	return a.addresses
 }
 
 func (a *agent) trackGoroutine(fn func()) error {
@@ -1137,11 +1133,19 @@ func (a *agent) trackGoroutine(fn func()) error {
 }
 
 func (a *agent) createTailnet(ctx context.Context, agentID uuid.UUID, derpMap *tailcfg.DERPMap, derpForceWebSockets, disableDirectConnections bool) (_ *tailnet.Conn, err error) {
+	// Inject `CODER_AGENT_HEADER` into the DERP header.
+	var header http.Header
+	if client, ok := a.client.(*agentsdk.Client); ok {
+		if headerTransport, ok := client.SDK.HTTPClient.Transport.(*codersdk.HeaderTransport); ok {
+			header = headerTransport.Header
+		}
+	}
 	network, err := tailnet.NewConn(&tailnet.Options{
 		ID:                  agentID,
 		Addresses:           a.wireguardAddresses(agentID),
 		DERPMap:             derpMap,
 		DERPForceWebSockets: derpForceWebSockets,
+		DERPHeader:          &header,
 		Logger:              a.logger.Named("net.tailnet"),
 		ListenPort:          a.tailnetListenPort,
 		BlockEndpoints:      disableDirectConnections,
@@ -1180,55 +1184,12 @@ func (a *agent) createTailnet(ctx context.Context, agentID uuid.UUID, derpMap *t
 		}
 	}()
 	if err = a.trackGoroutine(func() {
-		logger := a.logger.Named("reconnecting-pty")
-		var wg sync.WaitGroup
-		for {
-			conn, err := reconnectingPTYListener.Accept()
-			if err != nil {
-				if !a.isClosed() {
-					logger.Debug(ctx, "accept pty failed", slog.Error(err))
-				}
-				break
-			}
-			clog := logger.With(
-				slog.F("remote", conn.RemoteAddr().String()),
-				slog.F("local", conn.LocalAddr().String()))
-			clog.Info(ctx, "accepted conn")
-			wg.Add(1)
-			closed := make(chan struct{})
-			go func() {
-				select {
-				case <-closed:
-				case <-a.hardCtx.Done():
-					_ = conn.Close()
-				}
-				wg.Done()
-			}()
-			go func() {
-				defer close(closed)
-				// This cannot use a JSON decoder, since that can
-				// buffer additional data that is required for the PTY.
-				rawLen := make([]byte, 2)
-				_, err = conn.Read(rawLen)
-				if err != nil {
-					return
-				}
-				length := binary.LittleEndian.Uint16(rawLen)
-				data := make([]byte, length)
-				_, err = conn.Read(data)
-				if err != nil {
-					return
-				}
-				var msg workspacesdk.AgentReconnectingPTYInit
-				err = json.Unmarshal(data, &msg)
-				if err != nil {
-					logger.Warn(ctx, "failed to unmarshal init", slog.F("raw", data))
-					return
-				}
-				_ = a.handleReconnectingPTY(ctx, clog, msg, conn)
-			}()
+		rPTYServeErr := a.reconnectingPTYServer.Serve(a.gracefulCtx, a.hardCtx, reconnectingPTYListener)
+		if rPTYServeErr != nil &&
+			a.gracefulCtx.Err() == nil &&
+			!strings.Contains(rPTYServeErr.Error(), "use of closed network connection") {
+			a.logger.Error(ctx, "error serving reconnecting PTY", slog.Error(err))
 		}
-		wg.Wait()
 	}); err != nil {
 		return nil, err
 	}
@@ -1307,9 +1268,9 @@ func (a *agent) createTailnet(ctx context.Context, agentID uuid.UUID, derpMap *t
 			_ = server.Close()
 		}()
 
-		err := server.Serve(apiListener)
-		if err != nil && !xerrors.Is(err, http.ErrServerClosed) && !strings.Contains(err.Error(), "use of closed network connection") {
-			a.logger.Critical(ctx, "serve HTTP API server", slog.Error(err))
+		apiServErr := server.Serve(apiListener)
+		if apiServErr != nil && !xerrors.Is(apiServErr, http.ErrServerClosed) && !strings.Contains(apiServErr.Error(), "use of closed network connection") {
+			a.logger.Critical(ctx, "serve HTTP API server", slog.Error(apiServErr))
 		}
 	}); err != nil {
 		return nil, err
@@ -1320,9 +1281,8 @@ func (a *agent) createTailnet(ctx context.Context, agentID uuid.UUID, derpMap *t
 
 // runCoordinator runs a coordinator and returns whether a reconnect
 // should occur.
-func (a *agent) runCoordinator(ctx context.Context, conn drpc.Conn, network *tailnet.Conn) error {
+func (a *agent) runCoordinator(ctx context.Context, tClient tailnetproto.DRPCTailnetClient23, network *tailnet.Conn) error {
 	defer a.logger.Debug(ctx, "disconnected from coordination RPC")
-	tClient := tailnetproto.NewDRPCTailnetClient(conn)
 	// we run the RPC on the hardCtx so that we have a chance to send the disconnect message if we
 	// gracefully shut down.
 	coordinate, err := tClient.Coordinate(a.hardCtx)
@@ -1347,19 +1307,20 @@ func (a *agent) runCoordinator(ctx context.Context, conn drpc.Conn, network *tai
 	defer close(disconnected)
 	a.closeMutex.Unlock()
 
-	coordination := tailnet.NewRemoteCoordination(a.logger, coordinate, network, uuid.Nil)
+	ctrl := tailnet.NewAgentCoordinationController(a.logger, network)
+	coordination := ctrl.New(coordinate)
 
 	errCh := make(chan error, 1)
 	go func() {
 		defer close(errCh)
 		select {
 		case <-ctx.Done():
-			err := coordination.Close()
+			err := coordination.Close(a.hardCtx)
 			if err != nil {
 				a.logger.Warn(ctx, "failed to close remote coordination", slog.Error(err))
 			}
 			return
-		case err := <-coordination.Error():
+		case err := <-coordination.Wait():
 			errCh <- err
 		}
 	}()
@@ -1367,11 +1328,10 @@ func (a *agent) runCoordinator(ctx context.Context, conn drpc.Conn, network *tai
 }
 
 // runDERPMapSubscriber runs a coordinator and returns if a reconnect should occur.
-func (a *agent) runDERPMapSubscriber(ctx context.Context, conn drpc.Conn, network *tailnet.Conn) error {
+func (a *agent) runDERPMapSubscriber(ctx context.Context, tClient tailnetproto.DRPCTailnetClient23, network *tailnet.Conn) error {
 	defer a.logger.Debug(ctx, "disconnected from derp map RPC")
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
-	tClient := tailnetproto.NewDRPCTailnetClient(conn)
 	stream, err := tClient.StreamDERPMaps(ctx, &tailnetproto.StreamDERPMapsRequest{})
 	if err != nil {
 		return xerrors.Errorf("stream DERP Maps: %w", err)
@@ -1394,87 +1354,6 @@ func (a *agent) runDERPMapSubscriber(ctx context.Context, conn drpc.Conn, networ
 	}
 }
 
-func (a *agent) handleReconnectingPTY(ctx context.Context, logger slog.Logger, msg workspacesdk.AgentReconnectingPTYInit, conn net.Conn) (retErr error) {
-	defer conn.Close()
-	a.metrics.connectionsTotal.Add(1)
-
-	a.connCountReconnectingPTY.Add(1)
-	defer a.connCountReconnectingPTY.Add(-1)
-
-	connectionID := uuid.NewString()
-	connLogger := logger.With(slog.F("message_id", msg.ID), slog.F("connection_id", connectionID))
-	connLogger.Debug(ctx, "starting handler")
-
-	defer func() {
-		if err := retErr; err != nil {
-			a.closeMutex.Lock()
-			closed := a.isClosed()
-			a.closeMutex.Unlock()
-
-			// If the agent is closed, we don't want to
-			// log this as an error since it's expected.
-			if closed {
-				connLogger.Info(ctx, "reconnecting pty failed with attach error (agent closed)", slog.Error(err))
-			} else {
-				connLogger.Error(ctx, "reconnecting pty failed with attach error", slog.Error(err))
-			}
-		}
-		connLogger.Info(ctx, "reconnecting pty connection closed")
-	}()
-
-	var rpty reconnectingpty.ReconnectingPTY
-	sendConnected := make(chan reconnectingpty.ReconnectingPTY, 1)
-	// On store, reserve this ID to prevent multiple concurrent new connections.
-	waitReady, ok := a.reconnectingPTYs.LoadOrStore(msg.ID, sendConnected)
-	if ok {
-		close(sendConnected) // Unused.
-		connLogger.Debug(ctx, "connecting to existing reconnecting pty")
-		c, ok := waitReady.(chan reconnectingpty.ReconnectingPTY)
-		if !ok {
-			return xerrors.Errorf("found invalid type in reconnecting pty map: %T", waitReady)
-		}
-		rpty, ok = <-c
-		if !ok || rpty == nil {
-			return xerrors.Errorf("reconnecting pty closed before connection")
-		}
-		c <- rpty // Put it back for the next reconnect.
-	} else {
-		connLogger.Debug(ctx, "creating new reconnecting pty")
-
-		connected := false
-		defer func() {
-			if !connected && retErr != nil {
-				a.reconnectingPTYs.Delete(msg.ID)
-				close(sendConnected)
-			}
-		}()
-
-		// Empty command will default to the users shell!
-		cmd, err := a.sshServer.CreateCommand(ctx, msg.Command, nil)
-		if err != nil {
-			a.metrics.reconnectingPTYErrors.WithLabelValues("create_command").Add(1)
-			return xerrors.Errorf("create command: %w", err)
-		}
-
-		rpty = reconnectingpty.New(ctx, cmd, &reconnectingpty.Options{
-			Timeout: a.reconnectingPTYTimeout,
-			Metrics: a.metrics.reconnectingPTYErrors,
-		}, logger.With(slog.F("message_id", msg.ID)))
-
-		if err = a.trackGoroutine(func() {
-			rpty.Wait()
-			a.reconnectingPTYs.Delete(msg.ID)
-		}); err != nil {
-			rpty.Close(err)
-			return xerrors.Errorf("start routine: %w", err)
-		}
-
-		connected = true
-		sendConnected <- rpty
-	}
-	return rpty.Attach(ctx, connectionID, conn, msg.Height, msg.Width, connLogger)
-}
-
 // Collect collects additional stats from the agent
 func (a *agent) Collect(ctx context.Context, networkStats map[netlogtype.Connection]netlogtype.Counts) *proto.Stats {
 	a.logger.Debug(context.Background(), "computing stats report")
@@ -1496,7 +1375,7 @@ func (a *agent) Collect(ctx context.Context, networkStats map[netlogtype.Connect
 	stats.SessionCountVscode = sshStats.VSCode
 	stats.SessionCountJetbrains = sshStats.JetBrains
 
-	stats.SessionCountReconnectingPty = a.connCountReconnectingPTY.Load()
+	stats.SessionCountReconnectingPty = a.reconnectingPTYServer.ConnCount()
 
 	// Compute the median connection latency!
 	a.logger.Debug(ctx, "starting peer latency measurement for stats")
@@ -1504,6 +1383,8 @@ func (a *agent) Collect(ctx context.Context, networkStats map[netlogtype.Connect
 	var mu sync.Mutex
 	status := a.network.Status()
 	durations := []float64{}
+	p2pConns := 0
+	derpConns := 0
 	pingCtx, cancelFunc := context.WithTimeout(ctx, 5*time.Second)
 	defer cancelFunc()
 	for nodeID, peer := range status.Peer {
@@ -1520,13 +1401,18 @@ func (a *agent) Collect(ctx context.Context, networkStats map[netlogtype.Connect
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
-			duration, _, _, err := a.network.Ping(pingCtx, addresses[0].Addr())
+			duration, p2p, _, err := a.network.Ping(pingCtx, addresses[0].Addr())
 			if err != nil {
 				return
 			}
 			mu.Lock()
 			defer mu.Unlock()
 			durations = append(durations, float64(duration.Microseconds()))
+			if p2p {
+				p2pConns++
+			} else {
+				derpConns++
+			}
 		}()
 	}
 	wg.Wait()
@@ -1546,6 +1432,9 @@ func (a *agent) Collect(ctx context.Context, networkStats map[netlogtype.Connect
 	// Agent metrics are changing all the time, so there is no need to perform
 	// reflect.DeepEqual to see if stats should be transferred.
 
+	// currentConnections behaves like a hypothetical `GaugeFuncVec` and is only set at collection time.
+	a.metrics.currentConnections.WithLabelValues("p2p").Set(float64(p2pConns))
+	a.metrics.currentConnections.WithLabelValues("derp").Set(float64(derpConns))
 	metricsCtx, cancelFunc := context.WithTimeout(ctx, 5*time.Second)
 	defer cancelFunc()
 	a.logger.Debug(ctx, "collecting agent metrics for stats")
@@ -1663,13 +1552,12 @@ func (a *agent) manageProcessPriority(ctx context.Context, debouncer *logDebounc
 		}
 
 		score, niceErr := proc.Niceness(a.syscaller)
-		if niceErr != nil && !xerrors.Is(niceErr, os.ErrPermission) {
+		if niceErr != nil && !isBenignProcessErr(niceErr) {
 			debouncer.Warn(ctx, "unable to get proc niceness",
 				slog.F("cmd", proc.Cmd()),
 				slog.F("pid", proc.PID),
 				slog.Error(niceErr),
 			)
-			continue
 		}
 
 		// We only want processes that don't have a nice value set
@@ -1683,7 +1571,7 @@ func (a *agent) manageProcessPriority(ctx context.Context, debouncer *logDebounc
 
 		if niceErr == nil {
 			err := proc.SetNiceness(a.syscaller, niceness)
-			if err != nil && !xerrors.Is(err, os.ErrPermission) {
+			if err != nil && !isBenignProcessErr(err) {
 				debouncer.Warn(ctx, "unable to set proc niceness",
 					slog.F("cmd", proc.Cmd()),
 					slog.F("pid", proc.PID),
@@ -1697,7 +1585,7 @@ func (a *agent) manageProcessPriority(ctx context.Context, debouncer *logDebounc
 		if oomScore != unsetOOMScore && oomScore != proc.OOMScoreAdj && !isCustomOOMScore(agentScore, proc) {
 			oomScoreStr := strconv.Itoa(oomScore)
 			err := afero.WriteFile(a.filesystem, fmt.Sprintf("/proc/%d/oom_score_adj", proc.PID), []byte(oomScoreStr), 0o644)
-			if err != nil && !xerrors.Is(err, os.ErrPermission) {
+			if err != nil && !isBenignProcessErr(err) {
 				debouncer.Warn(ctx, "unable to set oom_score_adj",
 					slog.F("cmd", proc.Cmd()),
 					slog.F("pid", proc.PID),
@@ -1781,7 +1669,7 @@ func (a *agent) HandleHTTPDebugLogs(w http.ResponseWriter, r *http.Request) {
 	}
 	defer f.Close()
 
-	// Limit to 10MB.
+	// Limit to 10MiB.
 	w.WriteHeader(http.StatusOK)
 	_, err = io.Copy(w, io.LimitReader(f, 10*1024*1024))
 	if err != nil && !errors.Is(err, io.EOF) {
@@ -1832,9 +1720,7 @@ func (a *agent) Close() error {
 	a.gracefulCancel()
 
 	lifecycleState := codersdk.WorkspaceAgentLifecycleOff
-	err = a.scriptRunner.Execute(a.hardCtx, func(script codersdk.WorkspaceAgentScript) bool {
-		return script.RunOnStop
-	})
+	err = a.scriptRunner.Execute(a.hardCtx, agentscripts.ExecuteStopScripts)
 	if err != nil {
 		a.logger.Warn(a.hardCtx, "shutdown script(s) failed", slog.Error(err))
 		if errors.Is(err, agentscripts.ErrTimeout) {
@@ -1968,13 +1854,17 @@ const (
 
 type apiConnRoutineManager struct {
 	logger    slog.Logger
-	conn      drpc.Conn
+	aAPI      proto.DRPCAgentClient23
+	tAPI      tailnetproto.DRPCTailnetClient23
 	eg        *errgroup.Group
 	stopCtx   context.Context
 	remainCtx context.Context
 }
 
-func newAPIConnRoutineManager(gracefulCtx, hardCtx context.Context, logger slog.Logger, conn drpc.Conn) *apiConnRoutineManager {
+func newAPIConnRoutineManager(
+	gracefulCtx, hardCtx context.Context, logger slog.Logger,
+	aAPI proto.DRPCAgentClient23, tAPI tailnetproto.DRPCTailnetClient23,
+) *apiConnRoutineManager {
 	// routines that remain in operation during graceful shutdown use the remainCtx.  They'll still
 	// exit if the errgroup hits an error, which usually means a problem with the conn.
 	eg, remainCtx := errgroup.WithContext(hardCtx)
@@ -1994,17 +1884,23 @@ func newAPIConnRoutineManager(gracefulCtx, hardCtx context.Context, logger slog.
 	stopCtx := eitherContext(remainCtx, gracefulCtx)
 	return &apiConnRoutineManager{
 		logger:    logger,
-		conn:      conn,
+		aAPI:      aAPI,
+		tAPI:      tAPI,
 		eg:        eg,
 		stopCtx:   stopCtx,
 		remainCtx: remainCtx,
 	}
 }
 
-func (a *apiConnRoutineManager) start(name string, b gracefulShutdownBehavior, f func(context.Context, drpc.Conn) error) {
+// startAgentAPI starts a routine that uses the Agent API. c.f. startTailnetAPI which is the same
+// but for Tailnet.
+func (a *apiConnRoutineManager) startAgentAPI(
+	name string, behavior gracefulShutdownBehavior,
+	f func(context.Context, proto.DRPCAgentClient23) error,
+) {
 	logger := a.logger.With(slog.F("name", name))
 	var ctx context.Context
-	switch b {
+	switch behavior {
 	case gracefulShutdownBehaviorStop:
 		ctx = a.stopCtx
 	case gracefulShutdownBehaviorRemain:
@@ -2013,8 +1909,45 @@ func (a *apiConnRoutineManager) start(name string, b gracefulShutdownBehavior, f
 		panic("unknown behavior")
 	}
 	a.eg.Go(func() error {
-		logger.Debug(ctx, "starting routine")
-		err := f(ctx, a.conn)
+		logger.Debug(ctx, "starting agent routine")
+		err := f(ctx, a.aAPI)
+		if xerrors.Is(err, context.Canceled) && ctx.Err() != nil {
+			logger.Debug(ctx, "swallowing context canceled")
+			// Don't propagate context canceled errors to the error group, because we don't want the
+			// graceful context being canceled to halt the work of routines with
+			// gracefulShutdownBehaviorRemain.  Note that we check both that the error is
+			// context.Canceled and that *our* context is currently canceled, because when Coderd
+			// unilaterally closes the API connection (for example if the build is outdated), it can
+			// sometimes show up as context.Canceled in our RPC calls.
+			return nil
+		}
+		logger.Debug(ctx, "routine exited", slog.Error(err))
+		if err != nil {
+			return xerrors.Errorf("error in routine %s: %w", name, err)
+		}
+		return nil
+	})
+}
+
+// startTailnetAPI starts a routine that uses the Tailnet API. c.f. startAgentAPI which is the same
+// but for the Agent API.
+func (a *apiConnRoutineManager) startTailnetAPI(
+	name string, behavior gracefulShutdownBehavior,
+	f func(context.Context, tailnetproto.DRPCTailnetClient23) error,
+) {
+	logger := a.logger.With(slog.F("name", name))
+	var ctx context.Context
+	switch behavior {
+	case gracefulShutdownBehaviorStop:
+		ctx = a.stopCtx
+	case gracefulShutdownBehaviorRemain:
+		ctx = a.remainCtx
+	default:
+		panic("unknown behavior")
+	}
+	a.eg.Go(func() error {
+		logger.Debug(ctx, "starting tailnet routine")
+		err := f(ctx, a.tAPI)
 		if xerrors.Is(err, context.Canceled) && ctx.Err() != nil {
 			logger.Debug(ctx, "swallowing context canceled")
 			// Don't propagate context canceled errors to the error group, because we don't want the
@@ -2133,3 +2066,14 @@ func (l *logDebouncer) log(ctx context.Context, level slog.Level, msg string, fi
 	}
 	l.messages[msg] = time.Now()
 }
+
+func isBenignProcessErr(err error) bool {
+	return err != nil &&
+		(xerrors.Is(err, os.ErrNotExist) ||
+			xerrors.Is(err, os.ErrPermission) ||
+			isNoSuchProcessErr(err))
+}
+
+func isNoSuchProcessErr(err error) bool {
+	return err != nil && strings.Contains(err.Error(), "no such process")
+}

agent/agent_test.go

@@ -19,6 +19,7 @@ import (
 	"path/filepath"
 	"regexp"
 	"runtime"
+	"strconv"
 	"strings"
 	"sync"
 	"sync/atomic"
@@ -970,6 +971,99 @@ func TestAgent_SCP(t *testing.T) {
 	require.NoError(t, err)
 }
 
+func TestAgent_FileTransferBlocked(t *testing.T) {
+	t.Parallel()
+
+	assertFileTransferBlocked := func(t *testing.T, errorMessage string) {
+		// NOTE: Checking content of the error message is flaky. Most likely there is a race condition, which results
+		// in stopping the client in different phases, and returning different errors:
+		// - client read the full error message: File transfer has been disabled.
+		// - client's stream was terminated before reading the error message: EOF
+		// - client just read the error code (Windows): Process exited with status 65
+		isErr := strings.Contains(errorMessage, agentssh.BlockedFileTransferErrorMessage) ||
+			strings.Contains(errorMessage, "EOF") ||
+			strings.Contains(errorMessage, "Process exited with status 65")
+		require.True(t, isErr, fmt.Sprintf("Message: "+errorMessage))
+	}
+
+	t.Run("SFTP", func(t *testing.T) {
+		t.Parallel()
+
+		ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
+		defer cancel()
+
+		//nolint:dogsled
+		conn, _, _, _, _ := setupAgent(t, agentsdk.Manifest{}, 0, func(_ *agenttest.Client, o *agent.Options) {
+			o.BlockFileTransfer = true
+		})
+		sshClient, err := conn.SSHClient(ctx)
+		require.NoError(t, err)
+		defer sshClient.Close()
+		_, err = sftp.NewClient(sshClient)
+		require.Error(t, err)
+		assertFileTransferBlocked(t, err.Error())
+	})
+
+	t.Run("SCP with go-scp package", func(t *testing.T) {
+		t.Parallel()
+
+		ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
+		defer cancel()
+
+		//nolint:dogsled
+		conn, _, _, _, _ := setupAgent(t, agentsdk.Manifest{}, 0, func(_ *agenttest.Client, o *agent.Options) {
+			o.BlockFileTransfer = true
+		})
+		sshClient, err := conn.SSHClient(ctx)
+		require.NoError(t, err)
+		defer sshClient.Close()
+		scpClient, err := scp.NewClientBySSH(sshClient)
+		require.NoError(t, err)
+		defer scpClient.Close()
+		tempFile := filepath.Join(t.TempDir(), "scp")
+		err = scpClient.CopyFile(context.Background(), strings.NewReader("hello world"), tempFile, "0755")
+		require.Error(t, err)
+		assertFileTransferBlocked(t, err.Error())
+	})
+
+	t.Run("Forbidden commands", func(t *testing.T) {
+		t.Parallel()
+
+		for _, c := range agentssh.BlockedFileTransferCommands {
+			t.Run(c, func(t *testing.T) {
+				t.Parallel()
+
+				ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
+				defer cancel()
+
+				//nolint:dogsled
+				conn, _, _, _, _ := setupAgent(t, agentsdk.Manifest{}, 0, func(_ *agenttest.Client, o *agent.Options) {
+					o.BlockFileTransfer = true
+				})
+				sshClient, err := conn.SSHClient(ctx)
+				require.NoError(t, err)
+				defer sshClient.Close()
+
+				session, err := sshClient.NewSession()
+				require.NoError(t, err)
+				defer session.Close()
+
+				stdout, err := session.StdoutPipe()
+				require.NoError(t, err)
+
+				//nolint:govet // we don't need `c := c` in Go 1.22
+				err = session.Start(c)
+				require.NoError(t, err)
+				defer session.Close()
+
+				msg, err := io.ReadAll(stdout)
+				require.NoError(t, err)
+				assertFileTransferBlocked(t, string(msg))
+			})
+		}
+	})
+}
+
 func TestAgent_EnvironmentVariables(t *testing.T) {
 	t.Parallel()
 	key := "EXAMPLE"
@@ -1414,7 +1508,7 @@ func TestAgent_Lifecycle(t *testing.T) {
 
 	t.Run("ShutdownScriptOnce", func(t *testing.T) {
 		t.Parallel()
-		logger := slogtest.Make(t, nil).Leveled(slog.LevelDebug)
+		logger := testutil.Logger(t)
 		expected := "this-is-shutdown"
 		derpMap, _ := tailnettest.RunDERPAndSTUN(t)
 
@@ -1424,10 +1518,12 @@ func TestAgent_Lifecycle(t *testing.T) {
 			agentsdk.Manifest{
 				DERPMap: derpMap,
 				Scripts: []codersdk.WorkspaceAgentScript{{
+					ID:         uuid.New(),
 					LogPath:    "coder-startup-script.log",
 					Script:     "echo 1",
 					RunOnStart: true,
 				}, {
+					ID:        uuid.New(),
 					LogPath:   "coder-shutdown-script.log",
 					Script:    "echo " + expected,
 					RunOnStop: true,
@@ -1719,20 +1815,45 @@ func TestAgent_Dial(t *testing.T) {
 
 			go func() {
 				defer close(done)
-				c, err := l.Accept()
-				if assert.NoError(t, err, "accept connection") {
-					defer c.Close()
-					testAccept(ctx, t, c)
+				for range 2 {
+					c, err := l.Accept()
+					if assert.NoError(t, err, "accept connection") {
+						testAccept(ctx, t, c)
+						_ = c.Close()
+					}
 				}
 			}()
 
+			agentID := uuid.UUID{0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8}
 			//nolint:dogsled
-			agentConn, _, _, _, _ := setupAgent(t, agentsdk.Manifest{}, 0)
+			agentConn, _, _, _, _ := setupAgent(t, agentsdk.Manifest{
+				AgentID: agentID,
+			}, 0)
 			require.True(t, agentConn.AwaitReachable(ctx))
 			conn, err := agentConn.DialContext(ctx, l.Addr().Network(), l.Addr().String())
 			require.NoError(t, err)
-			defer conn.Close()
 			testDial(ctx, t, conn)
+			err = conn.Close()
+			require.NoError(t, err)
+
+			// also connect via the CoderServicePrefix, to test that we can reach the agent on this
+			// IP. This will be required for CoderVPN.
+			_, rawPort, _ := net.SplitHostPort(l.Addr().String())
+			port, _ := strconv.ParseUint(rawPort, 10, 16)
+			ipp := netip.AddrPortFrom(tailnet.CoderServicePrefix.AddrFromUUID(agentID), uint16(port))
+
+			switch l.Addr().Network() {
+			case "tcp":
+				conn, err = agentConn.Conn.DialContextTCP(ctx, ipp)
+			case "udp":
+				conn, err = agentConn.Conn.DialContextUDP(ctx, ipp)
+			default:
+				t.Fatalf("unknown network: %s", l.Addr().Network())
+			}
+			require.NoError(t, err)
+			testDial(ctx, t, conn)
+			err = conn.Close()
+			require.NoError(t, err)
 		})
 	}
 }
@@ -1742,7 +1863,7 @@ func TestAgent_Dial(t *testing.T) {
 func TestAgent_UpdatedDERP(t *testing.T) {
 	t.Parallel()
 
-	logger := slogtest.Make(t, nil).Leveled(slog.LevelDebug)
+	logger := testutil.Logger(t)
 
 	originalDerpMap, _ := tailnettest.RunDERPAndSTUN(t)
 	require.NotNil(t, originalDerpMap)
@@ -1785,7 +1906,7 @@ func TestAgent_UpdatedDERP(t *testing.T) {
 	// Setup a client connection.
 	newClientConn := func(derpMap *tailcfg.DERPMap, name string) *workspacesdk.AgentConn {
 		conn, err := tailnet.NewConn(&tailnet.Options{
-			Addresses: []netip.Prefix{netip.PrefixFrom(tailnet.IP(), 128)},
+			Addresses: []netip.Prefix{tailnet.TailscaleServicePrefix.RandomPrefix()},
 			DERPMap:   derpMap,
 			Logger:    logger.Named(name),
 		})
@@ -1797,13 +1918,15 @@ func TestAgent_UpdatedDERP(t *testing.T) {
 		testCtx, testCtxCancel := context.WithCancel(context.Background())
 		t.Cleanup(testCtxCancel)
 		clientID := uuid.New()
-		coordination := tailnet.NewInMemoryCoordination(
-			testCtx, logger,
-			clientID, agentID,
-			coordinator, conn)
+		ctrl := tailnet.NewTunnelSrcCoordController(logger, conn)
+		ctrl.AddDestination(agentID)
+		auth := tailnet.ClientCoordinateeAuth{AgentID: agentID}
+		coordination := ctrl.New(tailnet.NewInMemoryCoordinatorClient(logger, clientID, auth, coordinator))
 		t.Cleanup(func() {
 			t.Logf("closing coordination %s", name)
-			err := coordination.Close()
+			cctx, ccancel := context.WithTimeout(testCtx, testutil.WaitShort)
+			defer ccancel()
+			err := coordination.Close(cctx)
 			if err != nil {
 				t.Logf("error closing in-memory coordination: %s", err.Error())
 			}
@@ -1896,7 +2019,7 @@ func TestAgent_Speedtest(t *testing.T) {
 
 func TestAgent_Reconnect(t *testing.T) {
 	t.Parallel()
-	logger := slogtest.Make(t, nil).Leveled(slog.LevelDebug)
+	logger := testutil.Logger(t)
 	// After the agent is disconnected from a coordinator, it's supposed
 	// to reconnect!
 	coordinator := tailnet.NewCoordinator(logger)
@@ -1937,7 +2060,7 @@ func TestAgent_Reconnect(t *testing.T) {
 
 func TestAgent_WriteVSCodeConfigs(t *testing.T) {
 	t.Parallel()
-	logger := slogtest.Make(t, nil).Leveled(slog.LevelDebug)
+	logger := testutil.Logger(t)
 	coordinator := tailnet.NewCoordinator(logger)
 	defer coordinator.Close()
 
@@ -2275,7 +2398,7 @@ func setupAgent(t *testing.T, metadata agentsdk.Manifest, ptyTimeout time.Durati
 		_ = agnt.Close()
 	})
 	conn, err := tailnet.NewConn(&tailnet.Options{
-		Addresses: []netip.Prefix{netip.PrefixFrom(tailnet.IP(), 128)},
+		Addresses: []netip.Prefix{netip.PrefixFrom(tailnet.TailscaleServicePrefix.RandomAddr(), 128)},
 		DERPMap:   metadata.DERPMap,
 		Logger:    logger.Named("client"),
 	})
@@ -2286,12 +2409,15 @@ func setupAgent(t *testing.T, metadata agentsdk.Manifest, ptyTimeout time.Durati
 	testCtx, testCtxCancel := context.WithCancel(context.Background())
 	t.Cleanup(testCtxCancel)
 	clientID := uuid.New()
-	coordination := tailnet.NewInMemoryCoordination(
-		testCtx, logger,
-		clientID, metadata.AgentID,
-		coordinator, conn)
+	ctrl := tailnet.NewTunnelSrcCoordController(logger, conn)
+	ctrl.AddDestination(metadata.AgentID)
+	auth := tailnet.ClientCoordinateeAuth{AgentID: metadata.AgentID}
+	coordination := ctrl.New(tailnet.NewInMemoryCoordinatorClient(
+		logger, clientID, auth, coordinator))
 	t.Cleanup(func() {
-		err := coordination.Close()
+		cctx, ccancel := context.WithTimeout(testCtx, testutil.WaitShort)
+		defer ccancel()
+		err := coordination.Close(cctx)
 		if err != nil {
 			t.Logf("error closing in-mem coordination: %s", err.Error())
 		}
@@ -2438,17 +2564,17 @@ func TestAgent_Metrics_SSH(t *testing.T) {
 	err = session.Shell()
 	require.NoError(t, err)
 
-	expected := []agentsdk.AgentMetric{
+	expected := []*proto.Stats_Metric{
 		{
 			Name:  "agent_reconnecting_pty_connections_total",
-			Type:  agentsdk.AgentMetricTypeCounter,
+			Type:  proto.Stats_Metric_COUNTER,
 			Value: 0,
 		},
 		{
 			Name:  "agent_sessions_total",
-			Type:  agentsdk.AgentMetricTypeCounter,
+			Type:  proto.Stats_Metric_COUNTER,
 			Value: 1,
-			Labels: []agentsdk.AgentMetricLabel{
+			Labels: []*proto.Stats_Metric_Label{
 				{
 					Name:  "magic_type",
 					Value: "ssh",
@@ -2461,30 +2587,46 @@ func TestAgent_Metrics_SSH(t *testing.T) {
 		},
 		{
 			Name:  "agent_ssh_server_failed_connections_total",
-			Type:  agentsdk.AgentMetricTypeCounter,
+			Type:  proto.Stats_Metric_COUNTER,
 			Value: 0,
 		},
 		{
 			Name:  "agent_ssh_server_sftp_connections_total",
-			Type:  agentsdk.AgentMetricTypeCounter,
+			Type:  proto.Stats_Metric_COUNTER,
 			Value: 0,
 		},
 		{
 			Name:  "agent_ssh_server_sftp_server_errors_total",
-			Type:  agentsdk.AgentMetricTypeCounter,
+			Type:  proto.Stats_Metric_COUNTER,
 			Value: 0,
 		},
 		{
-			Name:  "coderd_agentstats_startup_script_seconds",
-			Type:  agentsdk.AgentMetricTypeGauge,
+			Name:  "coderd_agentstats_currently_reachable_peers",
+			Type:  proto.Stats_Metric_GAUGE,
 			Value: 0,
-			Labels: []agentsdk.AgentMetricLabel{
+			Labels: []*proto.Stats_Metric_Label{
 				{
-					Name:  "success",
-					Value: "true",
+					Name:  "connection_type",
+					Value: "derp",
 				},
 			},
 		},
+		{
+			Name:  "coderd_agentstats_currently_reachable_peers",
+			Type:  proto.Stats_Metric_GAUGE,
+			Value: 1,
+			Labels: []*proto.Stats_Metric_Label{
+				{
+					Name:  "connection_type",
+					Value: "p2p",
+				},
+			},
+		},
+		{
+			Name:  "coderd_agentstats_startup_script_seconds",
+			Type:  proto.Stats_Metric_GAUGE,
+			Value: 1,
+		},
 	}
 
 	var actual []*promgo.MetricFamily
@@ -2493,17 +2635,33 @@ func TestAgent_Metrics_SSH(t *testing.T) {
 		if err != nil {
 			return false
 		}
-
-		if len(expected) != len(actual) {
-			return false
+		count := 0
+		for _, m := range actual {
+			count += len(m.GetMetric())
 		}
-
-		return verifyCollectedMetrics(t, expected, actual)
+		return count == len(expected)
 	}, testutil.WaitLong, testutil.IntervalFast)
 
-	require.Len(t, actual, len(expected))
-	collected := verifyCollectedMetrics(t, expected, actual)
-	require.True(t, collected, "expected metrics were not collected")
+	i := 0
+	for _, mf := range actual {
+		for _, m := range mf.GetMetric() {
+			assert.Equal(t, expected[i].Name, mf.GetName())
+			assert.Equal(t, expected[i].Type.String(), mf.GetType().String())
+			// Value is max expected
+			if expected[i].Type == proto.Stats_Metric_GAUGE {
+				assert.GreaterOrEqualf(t, expected[i].Value, m.GetGauge().GetValue(), "expected %s to be greater than or equal to %f, got %f", expected[i].Name, expected[i].Value, m.GetGauge().GetValue())
+			} else if expected[i].Type == proto.Stats_Metric_COUNTER {
+				assert.GreaterOrEqualf(t, expected[i].Value, m.GetCounter().GetValue(), "expected %s to be greater than or equal to %f, got %f", expected[i].Name, expected[i].Value, m.GetCounter().GetValue())
+			}
+			for j, lbl := range expected[i].Labels {
+				assert.Equal(t, m.GetLabel()[j], &promgo.LabelPair{
+					Name:  &lbl.Name,
+					Value: &lbl.Value,
+				})
+			}
+			i++
+		}
+	}
 
 	_ = stdin.Close()
 	err = session.Wait()
@@ -2735,28 +2893,6 @@ func TestAgent_ManageProcessPriority(t *testing.T) {
 	})
 }
 
-func verifyCollectedMetrics(t *testing.T, expected []agentsdk.AgentMetric, actual []*promgo.MetricFamily) bool {
-	t.Helper()
-
-	for i, e := range expected {
-		assert.Equal(t, e.Name, actual[i].GetName())
-		assert.Equal(t, string(e.Type), strings.ToLower(actual[i].GetType().String()))
-
-		for _, m := range actual[i].GetMetric() {
-			assert.Equal(t, e.Value, m.Counter.GetValue())
-
-			if len(m.GetLabel()) > 0 {
-				for j, lbl := range m.GetLabel() {
-					assert.Equal(t, e.Labels[j].Name, lbl.GetName())
-					assert.Equal(t, e.Labels[j].Value, lbl.GetValue())
-				}
-			}
-			m.GetLabel()
-		}
-	}
-	return true
-}
-
 type syncWriter struct {
 	mu sync.Mutex
 	w  io.Writer

agent/agentexec/cli_linux.go

@@ -0,0 +1,145 @@
+//go:build linux
+// +build linux
+
+package agentexec
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"os/exec"
+	"runtime"
+	"strconv"
+	"strings"
+	"syscall"
+
+	"golang.org/x/sys/unix"
+	"golang.org/x/xerrors"
+)
+
+// unset is set to an invalid value for nice and oom scores.
+const unset = -2000
+
+// CLI runs the agent-exec command. It should only be called by the cli package.
+func CLI() error {
+	// We lock the OS thread here to avoid a race condition where the nice priority
+	// we get is on a different thread from the one we set it on.
+	runtime.LockOSThread()
+	// Nop on success but we do it anyway in case of an error.
+	defer runtime.UnlockOSThread()
+
+	var (
+		fs   = flag.NewFlagSet("agent-exec", flag.ExitOnError)
+		nice = fs.Int("coder-nice", unset, "")
+		oom  = fs.Int("coder-oom", unset, "")
+	)
+
+	if len(os.Args) < 3 {
+		return xerrors.Errorf("malformed command %+v", os.Args)
+	}
+
+	// Parse everything after "coder agent-exec".
+	err := fs.Parse(os.Args[2:])
+	if err != nil {
+		return xerrors.Errorf("parse flags: %w", err)
+	}
+
+	// Get everything after "coder agent-exec --"
+	args := execArgs(os.Args)
+	if len(args) == 0 {
+		return xerrors.Errorf("no exec command provided %+v", os.Args)
+	}
+
+	if *nice == unset {
+		// If an explicit nice score isn't set, we use the default.
+		*nice, err = defaultNiceScore()
+		if err != nil {
+			return xerrors.Errorf("get default nice score: %w", err)
+		}
+	}
+
+	if *oom == unset {
+		// If an explicit oom score isn't set, we use the default.
+		*oom, err = defaultOOMScore()
+		if err != nil {
+			return xerrors.Errorf("get default oom score: %w", err)
+		}
+	}
+
+	err = unix.Setpriority(unix.PRIO_PROCESS, 0, *nice)
+	if err != nil {
+		return xerrors.Errorf("set nice score: %w", err)
+	}
+
+	err = writeOOMScoreAdj(*oom)
+	if err != nil {
+		return xerrors.Errorf("set oom score: %w", err)
+	}
+
+	path, err := exec.LookPath(args[0])
+	if err != nil {
+		return xerrors.Errorf("look path: %w", err)
+	}
+
+	return syscall.Exec(path, args, os.Environ())
+}
+
+func defaultNiceScore() (int, error) {
+	score, err := unix.Getpriority(unix.PRIO_PROCESS, 0)
+	if err != nil {
+		return 0, xerrors.Errorf("get nice score: %w", err)
+	}
+	// See https://linux.die.net/man/2/setpriority#Notes
+	score = 20 - score
+
+	score += 5
+	if score > 19 {
+		return 19, nil
+	}
+	return score, nil
+}
+
+func defaultOOMScore() (int, error) {
+	score, err := oomScoreAdj()
+	if err != nil {
+		return 0, xerrors.Errorf("get oom score: %w", err)
+	}
+
+	// If the agent has a negative oom_score_adj, we set the child to 0
+	// so it's treated like every other process.
+	if score < 0 {
+		return 0, nil
+	}
+
+	// If the agent is already almost at the maximum then set it to the max.
+	if score >= 998 {
+		return 1000, nil
+	}
+
+	// If the agent oom_score_adj is >=0, we set the child to slightly
+	// less than the maximum. If users want a different score they set it
+	// directly.
+	return 998, nil
+}
+
+func oomScoreAdj() (int, error) {
+	scoreStr, err := os.ReadFile("/proc/self/oom_score_adj")
+	if err != nil {
+		return 0, xerrors.Errorf("read oom_score_adj: %w", err)
+	}
+	return strconv.Atoi(strings.TrimSpace(string(scoreStr)))
+}
+
+func writeOOMScoreAdj(score int) error {
+	return os.WriteFile("/proc/self/oom_score_adj", []byte(fmt.Sprintf("%d", score)), 0o600)
+}
+
+// execArgs returns the arguments to pass to syscall.Exec after the "--" delimiter.
+func execArgs(args []string) []string {
+	for i, arg := range args {
+		if arg == "--" {
+			return args[i+1:]
+		}
+	}
+	return nil
+}

agent/agentexec/cli_linux_test.go

@@ -0,0 +1,178 @@
+//go:build linux
+// +build linux
+
+package agentexec_test
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"syscall"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+	"golang.org/x/sys/unix"
+
+	"github.com/coder/coder/v2/testutil"
+)
+
+func TestCLI(t *testing.T) {
+	t.Parallel()
+
+	t.Run("OK", func(t *testing.T) {
+		t.Parallel()
+
+		ctx := testutil.Context(t, testutil.WaitMedium)
+		cmd, path := cmd(ctx, t, 123, 12)
+		err := cmd.Start()
+		require.NoError(t, err)
+		go cmd.Wait()
+
+		waitForSentinel(ctx, t, cmd, path)
+		requireOOMScore(t, cmd.Process.Pid, 123)
+		requireNiceScore(t, cmd.Process.Pid, 12)
+	})
+
+	t.Run("Defaults", func(t *testing.T) {
+		t.Parallel()
+
+		ctx := testutil.Context(t, testutil.WaitMedium)
+		cmd, path := cmd(ctx, t, 0, 0)
+		err := cmd.Start()
+		require.NoError(t, err)
+		go cmd.Wait()
+
+		waitForSentinel(ctx, t, cmd, path)
+
+		expectedNice := expectedNiceScore(t)
+		expectedOOM := expectedOOMScore(t)
+		requireOOMScore(t, cmd.Process.Pid, expectedOOM)
+		requireNiceScore(t, cmd.Process.Pid, expectedNice)
+	})
+}
+
+func requireNiceScore(t *testing.T, pid int, score int) {
+	t.Helper()
+
+	nice, err := unix.Getpriority(unix.PRIO_PROCESS, pid)
+	require.NoError(t, err)
+	// See https://linux.die.net/man/2/setpriority#Notes
+	require.Equal(t, score, 20-nice)
+}
+
+func requireOOMScore(t *testing.T, pid int, expected int) {
+	t.Helper()
+
+	actual, err := os.ReadFile(fmt.Sprintf("/proc/%d/oom_score_adj", pid))
+	require.NoError(t, err)
+	score := strings.TrimSpace(string(actual))
+	require.Equal(t, strconv.Itoa(expected), score)
+}
+
+func waitForSentinel(ctx context.Context, t *testing.T, cmd *exec.Cmd, path string) {
+	t.Helper()
+
+	ticker := time.NewTicker(testutil.IntervalFast)
+	defer ticker.Stop()
+
+	// RequireEventually doesn't work well with require.NoError or similar require functions.
+	for {
+		err := cmd.Process.Signal(syscall.Signal(0))
+		require.NoError(t, err)
+
+		_, err = os.Stat(path)
+		if err == nil {
+			return
+		}
+
+		select {
+		case <-ticker.C:
+		case <-ctx.Done():
+			require.NoError(t, ctx.Err())
+		}
+	}
+}
+
+func cmd(ctx context.Context, t *testing.T, oom, nice int) (*exec.Cmd, string) {
+	var (
+		args = execArgs(oom, nice)
+		dir  = t.TempDir()
+		file = filepath.Join(dir, "sentinel")
+	)
+
+	args = append(args, "sh", "-c", fmt.Sprintf("touch %s && sleep 10m", file))
+	//nolint:gosec
+	cmd := exec.CommandContext(ctx, TestBin, args...)
+
+	// We set this so we can also easily kill the sleep process the shell spawns.
+	cmd.SysProcAttr = &syscall.SysProcAttr{
+		Setpgid: true,
+	}
+
+	cmd.Env = os.Environ()
+	var buf bytes.Buffer
+	cmd.Stdout = &buf
+	cmd.Stderr = &buf
+	t.Cleanup(func() {
+		// Print output of a command if the test fails.
+		if t.Failed() {
+			t.Logf("cmd %q output: %s", cmd.Args, buf.String())
+		}
+		if cmd.Process != nil {
+			// We use -cmd.Process.Pid to kill the whole process group.
+			_ = syscall.Kill(-cmd.Process.Pid, syscall.SIGINT)
+		}
+	})
+	return cmd, file
+}
+
+func expectedOOMScore(t *testing.T) int {
+	t.Helper()
+
+	score, err := os.ReadFile(fmt.Sprintf("/proc/%d/oom_score_adj", os.Getpid()))
+	require.NoError(t, err)
+
+	scoreInt, err := strconv.Atoi(strings.TrimSpace(string(score)))
+	require.NoError(t, err)
+
+	if scoreInt < 0 {
+		return 0
+	}
+	if scoreInt >= 998 {
+		return 1000
+	}
+	return 998
+}
+
+func expectedNiceScore(t *testing.T) int {
+	t.Helper()
+
+	score, err := unix.Getpriority(unix.PRIO_PROCESS, os.Getpid())
+	require.NoError(t, err)
+
+	// Priority is niceness + 20.
+	score = 20 - score
+	score += 5
+	if score > 19 {
+		return 19
+	}
+	return score
+}
+
+func execArgs(oom int, nice int) []string {
+	execArgs := []string{"agent-exec"}
+	if oom != 0 {
+		execArgs = append(execArgs, fmt.Sprintf("--coder-oom=%d", oom))
+	}
+	if nice != 0 {
+		execArgs = append(execArgs, fmt.Sprintf("--coder-nice=%d", nice))
+	}
+	execArgs = append(execArgs, "--")
+	return execArgs
+}

agent/agentexec/cli_other.go

@@ -0,0 +1,10 @@
+//go:build !linux
+// +build !linux
+
+package agentexec
+
+import "golang.org/x/xerrors"
+
+func CLI() error {
+	return xerrors.New("agent-exec is only supported on Linux")
+}

agent/agentexec/cmdtest/main_linux.go

@@ -0,0 +1,19 @@
+//go:build linux
+// +build linux
+
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/coder/coder/v2/agent/agentexec"
+)
+
+func main() {
+	err := agentexec.CLI()
+	if err != nil {
+		_, _ = fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+}

agent/agentexec/exec.go

@@ -0,0 +1,86 @@
+package agentexec
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strconv"
+
+	"golang.org/x/xerrors"
+)
+
+const (
+	// EnvProcPrioMgmt is the environment variable that determines whether
+	// we attempt to manage process CPU and OOM Killer priority.
+	EnvProcPrioMgmt  = "CODER_PROC_PRIO_MGMT"
+	EnvProcOOMScore  = "CODER_PROC_OOM_SCORE"
+	EnvProcNiceScore = "CODER_PROC_NICE_SCORE"
+)
+
+// CommandContext returns an exec.Cmd that calls "coder agent-exec" prior to exec'ing
+// the provided command if CODER_PROC_PRIO_MGMT is set, otherwise a normal exec.Cmd
+// is returned. All instances of exec.Cmd should flow through this function to ensure
+// proper resource constraints are applied to the child process.
+func CommandContext(ctx context.Context, cmd string, args ...string) (*exec.Cmd, error) {
+	_, enabled := os.LookupEnv(EnvProcPrioMgmt)
+	if runtime.GOOS != "linux" || !enabled {
+		return exec.CommandContext(ctx, cmd, args...), nil
+	}
+
+	executable, err := os.Executable()
+	if err != nil {
+		return nil, xerrors.Errorf("get executable: %w", err)
+	}
+
+	bin, err := filepath.EvalSymlinks(executable)
+	if err != nil {
+		return nil, xerrors.Errorf("eval symlinks: %w", err)
+	}
+
+	execArgs := []string{"agent-exec"}
+	if score, ok := envValInt(EnvProcOOMScore); ok {
+		execArgs = append(execArgs, oomScoreArg(score))
+	}
+
+	if score, ok := envValInt(EnvProcNiceScore); ok {
+		execArgs = append(execArgs, niceScoreArg(score))
+	}
+	execArgs = append(execArgs, "--", cmd)
+	execArgs = append(execArgs, args...)
+
+	return exec.CommandContext(ctx, bin, execArgs...), nil
+}
+
+// envValInt searches for a key in a list of environment variables and parses it to an int.
+// If the key is not found or cannot be parsed, returns 0 and false.
+func envValInt(key string) (int, bool) {
+	val, ok := os.LookupEnv(key)
+	if !ok {
+		return 0, false
+	}
+
+	i, err := strconv.Atoi(val)
+	if err != nil {
+		return 0, false
+	}
+	return i, true
+}
+
+// The following are flags used by the agent-exec command. We use flags instead of
+// environment variables to avoid having to deal with a caller overriding the
+// environment variables.
+const (
+	niceFlag = "coder-nice"
+	oomFlag  = "coder-oom"
+)
+
+func niceScoreArg(score int) string {
+	return fmt.Sprintf("--%s=%d", niceFlag, score)
+}
+
+func oomScoreArg(score int) string {
+	return fmt.Sprintf("--%s=%d", oomFlag, score)
+}

agent/agentexec/exec_test.go

@@ -0,0 +1,119 @@
+package agentexec_test
+
+import (
+	"context"
+	"os"
+	"os/exec"
+	"runtime"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/coder/coder/v2/agent/agentexec"
+)
+
+//nolint:paralleltest // we need to test environment variables
+func TestExec(t *testing.T) {
+	//nolint:paralleltest // we need to test environment variables
+	t.Run("NonLinux", func(t *testing.T) {
+		t.Setenv(agentexec.EnvProcPrioMgmt, "true")
+
+		if runtime.GOOS == "linux" {
+			t.Skip("skipping on linux")
+		}
+
+		cmd, err := agentexec.CommandContext(context.Background(), "sh", "-c", "sleep")
+		require.NoError(t, err)
+
+		path, err := exec.LookPath("sh")
+		require.NoError(t, err)
+		require.Equal(t, path, cmd.Path)
+		require.Equal(t, []string{"sh", "-c", "sleep"}, cmd.Args)
+	})
+
+	//nolint:paralleltest // we need to test environment variables
+	t.Run("Linux", func(t *testing.T) {
+		//nolint:paralleltest // we need to test environment variables
+		t.Run("Disabled", func(t *testing.T) {
+			if runtime.GOOS != "linux" {
+				t.Skip("skipping on linux")
+			}
+
+			cmd, err := agentexec.CommandContext(context.Background(), "sh", "-c", "sleep")
+			require.NoError(t, err)
+			path, err := exec.LookPath("sh")
+			require.NoError(t, err)
+			require.Equal(t, path, cmd.Path)
+			require.Equal(t, []string{"sh", "-c", "sleep"}, cmd.Args)
+		})
+
+		//nolint:paralleltest // we need to test environment variables
+		t.Run("Enabled", func(t *testing.T) {
+			t.Setenv(agentexec.EnvProcPrioMgmt, "hello")
+
+			if runtime.GOOS != "linux" {
+				t.Skip("skipping on linux")
+			}
+
+			executable, err := os.Executable()
+			require.NoError(t, err)
+
+			cmd, err := agentexec.CommandContext(context.Background(), "sh", "-c", "sleep")
+			require.NoError(t, err)
+			require.Equal(t, executable, cmd.Path)
+			require.Equal(t, []string{executable, "agent-exec", "--", "sh", "-c", "sleep"}, cmd.Args)
+		})
+
+		t.Run("Nice", func(t *testing.T) {
+			t.Setenv(agentexec.EnvProcPrioMgmt, "hello")
+			t.Setenv(agentexec.EnvProcNiceScore, "10")
+
+			if runtime.GOOS != "linux" {
+				t.Skip("skipping on linux")
+			}
+
+			executable, err := os.Executable()
+			require.NoError(t, err)
+
+			cmd, err := agentexec.CommandContext(context.Background(), "sh", "-c", "sleep")
+			require.NoError(t, err)
+			require.Equal(t, executable, cmd.Path)
+			require.Equal(t, []string{executable, "agent-exec", "--coder-nice=10", "--", "sh", "-c", "sleep"}, cmd.Args)
+		})
+
+		t.Run("OOM", func(t *testing.T) {
+			t.Setenv(agentexec.EnvProcPrioMgmt, "hello")
+			t.Setenv(agentexec.EnvProcOOMScore, "123")
+
+			if runtime.GOOS != "linux" {
+				t.Skip("skipping on linux")
+			}
+
+			executable, err := os.Executable()
+			require.NoError(t, err)
+
+			cmd, err := agentexec.CommandContext(context.Background(), "sh", "-c", "sleep")
+			require.NoError(t, err)
+			require.Equal(t, executable, cmd.Path)
+			require.Equal(t, []string{executable, "agent-exec", "--coder-oom=123", "--", "sh", "-c", "sleep"}, cmd.Args)
+		})
+
+		t.Run("Both", func(t *testing.T) {
+			t.Setenv(agentexec.EnvProcPrioMgmt, "hello")
+			t.Setenv(agentexec.EnvProcOOMScore, "432")
+			t.Setenv(agentexec.EnvProcNiceScore, "14")
+
+			if runtime.GOOS != "linux" {
+				t.Skip("skipping on linux")
+			}
+
+			executable, err := os.Executable()
+			require.NoError(t, err)
+
+			cmd, err := agentexec.CommandContext(context.Background(), "sh", "-c", "sleep")
+			require.NoError(t, err)
+			require.Equal(t, executable, cmd.Path)
+			require.Equal(t, []string{executable, "agent-exec", "--coder-oom=432", "--coder-nice=14", "--", "sh", "-c", "sleep"}, cmd.Args)
+		})
+	})
+}

agent/agentexec/main_linux_test.go

@@ -0,0 +1,46 @@
+//go:build linux
+// +build linux
+
+package agentexec_test
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"testing"
+)
+
+var TestBin string
+
+func TestMain(m *testing.M) {
+	code := func() int {
+		// We generate a unique directory per test invocation to avoid collisions between two
+		// processes attempting to create the same temp file.
+		dir := genDir()
+		defer os.RemoveAll(dir)
+		TestBin = buildBinary(dir)
+		return m.Run()
+	}()
+
+	os.Exit(code)
+}
+
+func buildBinary(dir string) string {
+	path := filepath.Join(dir, "agent-test")
+	out, err := exec.Command("go", "build", "-o", path, "./cmdtest").CombinedOutput()
+	mustf(err, "build binary: %s", out)
+	return path
+}
+
+func mustf(err error, msg string, args ...any) {
+	if err != nil {
+		panic(fmt.Sprintf(msg, args...))
+	}
+}
+
+func genDir() string {
+	dir, err := os.MkdirTemp(os.TempDir(), "agentexec")
+	mustf(err, "create temp dir: %v", err)
+	return dir
+}

agent/agentproc/proc_unix.go

@@ -45,8 +45,7 @@ func List(fs afero.Fs, syscaller Syscaller) ([]*Process, error) {
 
 		cmdline, err := afero.ReadFile(fs, filepath.Join(defaultProcDir, entry, "cmdline"))
 		if err != nil {
-			var errNo syscall.Errno
-			if xerrors.As(err, &errNo) && errNo == syscall.EPERM {
+			if isBenignError(err) {
 				continue
 			}
 			return nil, xerrors.Errorf("read cmdline: %w", err)
@@ -54,7 +53,7 @@ func List(fs afero.Fs, syscaller Syscaller) ([]*Process, error) {
 
 		oomScore, err := afero.ReadFile(fs, filepath.Join(defaultProcDir, entry, "oom_score_adj"))
 		if err != nil {
-			if xerrors.Is(err, os.ErrPermission) {
+			if isBenignError(err) {
 				continue
 			}
 
@@ -124,3 +123,12 @@ func (p *Process) Cmd() string {
 func (p *Process) cmdLine() []string {
 	return strings.Split(p.CmdLine, "\x00")
 }
+
+func isBenignError(err error) bool {
+	var errno syscall.Errno
+	if !xerrors.As(err, &errno) {
+		return false
+	}
+
+	return errno == syscall.ESRCH || errno == syscall.EPERM || xerrors.Is(err, os.ErrNotExist)
+}

agent/agentscripts/agentscripts.go

@@ -19,10 +19,13 @@ import (
 	"github.com/spf13/afero"
 	"golang.org/x/sync/errgroup"
 	"golang.org/x/xerrors"
+	"google.golang.org/protobuf/types/known/timestamppb"
 
 	"cdr.dev/slog"
 
 	"github.com/coder/coder/v2/agent/agentssh"
+	"github.com/coder/coder/v2/agent/proto"
+	"github.com/coder/coder/v2/coderd/database/dbtime"
 	"github.com/coder/coder/v2/codersdk"
 	"github.com/coder/coder/v2/codersdk/agentsdk"
 )
@@ -75,18 +78,21 @@ func New(opts Options) *Runner {
 	}
 }
 
+type ScriptCompletedFunc func(context.Context, *proto.WorkspaceAgentScriptCompletedRequest) (*proto.WorkspaceAgentScriptCompletedResponse, error)
+
 type Runner struct {
 	Options
 
-	cronCtx       context.Context
-	cronCtxCancel context.CancelFunc
-	cmdCloseWait  sync.WaitGroup
-	closed        chan struct{}
-	closeMutex    sync.Mutex
-	cron          *cron.Cron
-	initialized   atomic.Bool
-	scripts       []codersdk.WorkspaceAgentScript
-	dataDir       string
+	cronCtx         context.Context
+	cronCtxCancel   context.CancelFunc
+	cmdCloseWait    sync.WaitGroup
+	closed          chan struct{}
+	closeMutex      sync.Mutex
+	cron            *cron.Cron
+	initialized     atomic.Bool
+	scripts         []codersdk.WorkspaceAgentScript
+	dataDir         string
+	scriptCompleted ScriptCompletedFunc
 
 	// scriptsExecuted includes all scripts executed by the workspace agent. Agents
 	// execute startup scripts, and scripts on a cron schedule. Both will increment
@@ -116,12 +122,13 @@ func (r *Runner) RegisterMetrics(reg prometheus.Registerer) {
 // Init initializes the runner with the provided scripts.
 // It also schedules any scripts that have a schedule.
 // This function must be called before Execute.
-func (r *Runner) Init(scripts []codersdk.WorkspaceAgentScript) error {
+func (r *Runner) Init(scripts []codersdk.WorkspaceAgentScript, scriptCompleted ScriptCompletedFunc) error {
 	if r.initialized.Load() {
 		return xerrors.New("init: already initialized")
 	}
 	r.initialized.Store(true)
 	r.scripts = scripts
+	r.scriptCompleted = scriptCompleted
 	r.Logger.Info(r.cronCtx, "initializing agent scripts", slog.F("script_count", len(scripts)), slog.F("log_dir", r.LogDir))
 
 	err := r.Filesystem.MkdirAll(r.ScriptBinDir(), 0o700)
@@ -135,7 +142,7 @@ func (r *Runner) Init(scripts []codersdk.WorkspaceAgentScript) error {
 		}
 		script := script
 		_, err := r.cron.AddFunc(script.Cron, func() {
-			err := r.trackRun(r.cronCtx, script)
+			err := r.trackRun(r.cronCtx, script, ExecuteCronScripts)
 			if err != nil {
 				r.Logger.Warn(context.Background(), "run agent script on schedule", slog.Error(err))
 			}
@@ -172,22 +179,33 @@ func (r *Runner) StartCron() {
 	}
 }
 
+// ExecuteOption describes what scripts we want to execute.
+type ExecuteOption int
+
+// ExecuteOption enums.
+const (
+	ExecuteAllScripts ExecuteOption = iota
+	ExecuteStartScripts
+	ExecuteStopScripts
+	ExecuteCronScripts
+)
+
 // Execute runs a set of scripts according to a filter.
-func (r *Runner) Execute(ctx context.Context, filter func(script codersdk.WorkspaceAgentScript) bool) error {
-	if filter == nil {
-		// Execute em' all!
-		filter = func(script codersdk.WorkspaceAgentScript) bool {
-			return true
-		}
-	}
+func (r *Runner) Execute(ctx context.Context, option ExecuteOption) error {
 	var eg errgroup.Group
 	for _, script := range r.scripts {
-		if !filter(script) {
+		runScript := (option == ExecuteStartScripts && script.RunOnStart) ||
+			(option == ExecuteStopScripts && script.RunOnStop) ||
+			(option == ExecuteCronScripts && script.Cron != "") ||
+			option == ExecuteAllScripts
+
+		if !runScript {
 			continue
 		}
+
 		script := script
 		eg.Go(func() error {
-			err := r.trackRun(ctx, script)
+			err := r.trackRun(ctx, script, option)
 			if err != nil {
 				return xerrors.Errorf("run agent script %q: %w", script.LogSourceID, err)
 			}
@@ -198,8 +216,8 @@ func (r *Runner) Execute(ctx context.Context, filter func(script codersdk.Worksp
 }
 
 // trackRun wraps "run" with metrics.
-func (r *Runner) trackRun(ctx context.Context, script codersdk.WorkspaceAgentScript) error {
-	err := r.run(ctx, script)
+func (r *Runner) trackRun(ctx context.Context, script codersdk.WorkspaceAgentScript, option ExecuteOption) error {
+	err := r.run(ctx, script, option)
 	if err != nil {
 		r.scriptsExecuted.WithLabelValues("false").Add(1)
 	} else {
@@ -212,7 +230,7 @@ func (r *Runner) trackRun(ctx context.Context, script codersdk.WorkspaceAgentScr
 // If the timeout is exceeded, the process is sent an interrupt signal.
 // If the process does not exit after a few seconds, it is forcefully killed.
 // This function immediately returns after a timeout, and does not wait for the process to exit.
-func (r *Runner) run(ctx context.Context, script codersdk.WorkspaceAgentScript) error {
+func (r *Runner) run(ctx context.Context, script codersdk.WorkspaceAgentScript, option ExecuteOption) error {
 	logPath := script.LogPath
 	if logPath == "" {
 		logPath = fmt.Sprintf("coder-script-%s.log", script.LogSourceID)
@@ -299,9 +317,9 @@ func (r *Runner) run(ctx context.Context, script codersdk.WorkspaceAgentScript)
 	cmd.Stdout = io.MultiWriter(fileWriter, infoW)
 	cmd.Stderr = io.MultiWriter(fileWriter, errW)
 
-	start := time.Now()
+	start := dbtime.Now()
 	defer func() {
-		end := time.Now()
+		end := dbtime.Now()
 		execTime := end.Sub(start)
 		exitCode := 0
 		if err != nil {
@@ -314,6 +332,60 @@ func (r *Runner) run(ctx context.Context, script codersdk.WorkspaceAgentScript)
 		} else {
 			logger.Info(ctx, fmt.Sprintf("%s script completed", logPath), slog.F("execution_time", execTime), slog.F("exit_code", exitCode))
 		}
+
+		if r.scriptCompleted == nil {
+			logger.Debug(ctx, "r.scriptCompleted unexpectedly nil")
+			return
+		}
+
+		// We want to check this outside of the goroutine to avoid a race condition
+		timedOut := errors.Is(err, ErrTimeout)
+		pipesLeftOpen := errors.Is(err, ErrOutputPipesOpen)
+
+		err = r.trackCommandGoroutine(func() {
+			var stage proto.Timing_Stage
+			switch option {
+			case ExecuteStartScripts:
+				stage = proto.Timing_START
+			case ExecuteStopScripts:
+				stage = proto.Timing_STOP
+			case ExecuteCronScripts:
+				stage = proto.Timing_CRON
+			}
+
+			var status proto.Timing_Status
+			switch {
+			case timedOut:
+				status = proto.Timing_TIMED_OUT
+			case pipesLeftOpen:
+				status = proto.Timing_PIPES_LEFT_OPEN
+			case exitCode != 0:
+				status = proto.Timing_EXIT_FAILURE
+			default:
+				status = proto.Timing_OK
+			}
+
+			reportTimeout := 30 * time.Second
+			reportCtx, cancel := context.WithTimeout(context.Background(), reportTimeout)
+			defer cancel()
+
+			_, err := r.scriptCompleted(reportCtx, &proto.WorkspaceAgentScriptCompletedRequest{
+				Timing: &proto.Timing{
+					ScriptId: script.ID[:],
+					Start:    timestamppb.New(start),
+					End:      timestamppb.New(end),
+					ExitCode: int32(exitCode),
+					Stage:    stage,
+					Status:   status,
+				},
+			})
+			if err != nil {
+				logger.Error(ctx, fmt.Sprintf("reporting script completed: %s", err.Error()))
+			}
+		})
+		if err != nil {
+			logger.Error(ctx, fmt.Sprintf("reporting script completed: track command goroutine: %s", err.Error()))
+		}
 	}()
 
 	err = cmd.Start()
@@ -349,7 +421,7 @@ func (r *Runner) run(ctx context.Context, script codersdk.WorkspaceAgentScript)
 			"This usually means a child process was started with references to stdout or stderr. As a result, this " +
 				"process may now have been terminated. Consider redirecting the output or using a separate " +
 				"\"coder_script\" for the process, see " +
-				"https://coder.com/docs/v2/latest/templates/troubleshooting#startup-script-issues for more information.",
+				"https://coder.com/docs/templates/troubleshooting#startup-script-issues for more information.",
 		)
 		// Inform the user by propagating the message via log writers.
 		_, _ = fmt.Fprintf(cmd.Stderr, "WARNING: %s. %s\n", message, details)

agent/agentscripts/agentscripts_test.go

@@ -14,9 +14,9 @@ import (
 	"github.com/stretchr/testify/require"
 	"go.uber.org/goleak"
 
-	"cdr.dev/slog/sloggers/slogtest"
 	"github.com/coder/coder/v2/agent/agentscripts"
 	"github.com/coder/coder/v2/agent/agentssh"
+	"github.com/coder/coder/v2/agent/agenttest"
 	"github.com/coder/coder/v2/codersdk"
 	"github.com/coder/coder/v2/codersdk/agentsdk"
 	"github.com/coder/coder/v2/testutil"
@@ -34,14 +34,13 @@ func TestExecuteBasic(t *testing.T) {
 		return fLogger
 	})
 	defer runner.Close()
+	aAPI := agenttest.NewFakeAgentAPI(t, testutil.Logger(t), nil, nil)
 	err := runner.Init([]codersdk.WorkspaceAgentScript{{
 		LogSourceID: uuid.New(),
 		Script:      "echo hello",
-	}})
+	}}, aAPI.ScriptCompleted)
 	require.NoError(t, err)
-	require.NoError(t, runner.Execute(context.Background(), func(script codersdk.WorkspaceAgentScript) bool {
-		return true
-	}))
+	require.NoError(t, runner.Execute(context.Background(), agentscripts.ExecuteAllScripts))
 	log := testutil.RequireRecvCtx(ctx, t, fLogger.logs)
 	require.Equal(t, "hello", log.Output)
 }
@@ -61,18 +60,17 @@ func TestEnv(t *testing.T) {
 			cmd.exe /c echo %CODER_SCRIPT_BIN_DIR%
 		`
 	}
+	aAPI := agenttest.NewFakeAgentAPI(t, testutil.Logger(t), nil, nil)
 	err := runner.Init([]codersdk.WorkspaceAgentScript{{
 		LogSourceID: id,
 		Script:      script,
-	}})
+	}}, aAPI.ScriptCompleted)
 	require.NoError(t, err)
 
 	ctx := testutil.Context(t, testutil.WaitLong)
 
 	done := testutil.Go(t, func() {
-		err := runner.Execute(ctx, func(script codersdk.WorkspaceAgentScript) bool {
-			return true
-		})
+		err := runner.Execute(ctx, agentscripts.ExecuteAllScripts)
 		assert.NoError(t, err)
 	})
 	defer func() {
@@ -103,13 +101,44 @@ func TestTimeout(t *testing.T) {
 	t.Parallel()
 	runner := setup(t, nil)
 	defer runner.Close()
+	aAPI := agenttest.NewFakeAgentAPI(t, testutil.Logger(t), nil, nil)
 	err := runner.Init([]codersdk.WorkspaceAgentScript{{
 		LogSourceID: uuid.New(),
 		Script:      "sleep infinity",
 		Timeout:     time.Millisecond,
-	}})
+	}}, aAPI.ScriptCompleted)
 	require.NoError(t, err)
-	require.ErrorIs(t, runner.Execute(context.Background(), nil), agentscripts.ErrTimeout)
+	require.ErrorIs(t, runner.Execute(context.Background(), agentscripts.ExecuteAllScripts), agentscripts.ErrTimeout)
+}
+
+func TestScriptReportsTiming(t *testing.T) {
+	t.Parallel()
+
+	ctx := testutil.Context(t, testutil.WaitShort)
+	fLogger := newFakeScriptLogger()
+	runner := setup(t, func(uuid2 uuid.UUID) agentscripts.ScriptLogger {
+		return fLogger
+	})
+
+	aAPI := agenttest.NewFakeAgentAPI(t, testutil.Logger(t), nil, nil)
+	err := runner.Init([]codersdk.WorkspaceAgentScript{{
+		DisplayName: "say-hello",
+		LogSourceID: uuid.New(),
+		Script:      "echo hello",
+	}}, aAPI.ScriptCompleted)
+	require.NoError(t, err)
+	require.NoError(t, runner.Execute(ctx, agentscripts.ExecuteAllScripts))
+	runner.Close()
+
+	log := testutil.RequireRecvCtx(ctx, t, fLogger.logs)
+	require.Equal(t, "hello", log.Output)
+
+	timings := aAPI.GetTimings()
+	require.Equal(t, 1, len(timings))
+
+	timing := timings[0]
+	require.Equal(t, int32(0), timing.ExitCode)
+	require.GreaterOrEqual(t, timing.End.AsTime(), timing.Start.AsTime())
 }
 
 // TestCronClose exists because cron.Run() can happen after cron.Close().
@@ -130,7 +159,7 @@ func setup(t *testing.T, getScriptLogger func(logSourceID uuid.UUID) agentscript
 		}
 	}
 	fs := afero.NewMemMapFs()
-	logger := slogtest.Make(t, nil)
+	logger := testutil.Logger(t)
 	s, err := agentssh.NewServer(context.Background(), logger, prometheus.NewRegistry(), fs, nil)
 	require.NoError(t, err)
 	t.Cleanup(func() {

agent/agentssh/agentssh.go

@@ -52,8 +52,16 @@ const (
 	// MagicProcessCmdlineJetBrains is a string in a process's command line that
 	// uniquely identifies it as JetBrains software.
 	MagicProcessCmdlineJetBrains = "idea.vendor.name=JetBrains"
+
+	// BlockedFileTransferErrorCode indicates that SSH server restricted the raw command from performing
+	// the file transfer.
+	BlockedFileTransferErrorCode    = 65 // Error code: host not allowed to connect
+	BlockedFileTransferErrorMessage = "File transfer has been disabled."
 )
 
+// BlockedFileTransferCommands contains a list of restricted file transfer commands.
+var BlockedFileTransferCommands = []string{"nc", "rsync", "scp", "sftp"}
+
 // Config sets configuration parameters for the agent SSH server.
 type Config struct {
 	// MaxTimeout sets the absolute connection timeout, none if empty. If set to
@@ -71,9 +79,11 @@ type Config struct {
 	// where users will land when they connect via SSH. Default is the home
 	// directory of the user.
 	WorkingDirectory func() string
-	// X11SocketDir is the directory where X11 sockets are created. Default is
-	// /tmp/.X11-unix.
-	X11SocketDir string
+	// X11DisplayOffset is the offset to add to the X11 display number.
+	// Default is 10.
+	X11DisplayOffset *int
+	// BlockFileTransfer restricts use of file transfer applications.
+	BlockFileTransfer bool
 }
 
 type Server struct {
@@ -114,8 +124,9 @@ func NewServer(ctx context.Context, logger slog.Logger, prometheusRegistry *prom
 	if config == nil {
 		config = &Config{}
 	}
-	if config.X11SocketDir == "" {
-		config.X11SocketDir = filepath.Join(os.TempDir(), ".X11-unix")
+	if config.X11DisplayOffset == nil {
+		offset := X11DefaultDisplayOffset
+		config.X11DisplayOffset = &offset
 	}
 	if config.UpdateEnv == nil {
 		config.UpdateEnv = func(current []string) ([]string, error) { return current, nil }
@@ -263,13 +274,25 @@ func (s *Server) sessionHandler(session ssh.Session) {
 	extraEnv := make([]string, 0)
 	x11, hasX11 := session.X11()
 	if hasX11 {
-		handled := s.x11Handler(session.Context(), x11)
+		display, handled := s.x11Handler(session.Context(), x11)
 		if !handled {
 			_ = session.Exit(1)
 			logger.Error(ctx, "x11 handler failed")
 			return
 		}
-		extraEnv = append(extraEnv, fmt.Sprintf("DISPLAY=:%d.0", x11.ScreenNumber))
+		extraEnv = append(extraEnv, fmt.Sprintf("DISPLAY=localhost:%d.%d", display, x11.ScreenNumber))
+	}
+
+	if s.fileTransferBlocked(session) {
+		s.logger.Warn(ctx, "file transfer blocked", slog.F("session_subsystem", session.Subsystem()), slog.F("raw_command", session.RawCommand()))
+
+		if session.Subsystem() == "" { // sftp does not expect error, otherwise it fails with "package too long"
+			// Response format: <status_code><message body>\n
+			errorMessage := fmt.Sprintf("\x02%s\n", BlockedFileTransferErrorMessage)
+			_, _ = session.Write([]byte(errorMessage))
+		}
+		_ = session.Exit(BlockedFileTransferErrorCode)
+		return
 	}
 
 	switch ss := session.Subsystem(); ss {
@@ -322,6 +345,37 @@ func (s *Server) sessionHandler(session ssh.Session) {
 	_ = session.Exit(0)
 }
 
+// fileTransferBlocked method checks if the file transfer commands should be blocked.
+//
+// Warning: consider this mechanism as "Do not trespass" sign, as a violator can still ssh to the host,
+// smuggle the `scp` binary, or just manually send files outside with `curl` or `ftp`.
+// If a user needs a more sophisticated and battle-proof solution, consider full endpoint security.
+func (s *Server) fileTransferBlocked(session ssh.Session) bool {
+	if !s.config.BlockFileTransfer {
+		return false // file transfers are permitted
+	}
+	// File transfers are restricted.
+
+	if session.Subsystem() == "sftp" {
+		return true
+	}
+
+	cmd := session.Command()
+	if len(cmd) == 0 {
+		return false // no command?
+	}
+
+	c := cmd[0]
+	c = filepath.Base(c) // in case the binary is absolute path, /usr/sbin/scp
+
+	for _, cmd := range BlockedFileTransferCommands {
+		if cmd == c {
+			return true
+		}
+	}
+	return false
+}
+
 func (s *Server) sessionStart(logger slog.Logger, session ssh.Session, extraEnv []string) (retErr error) {
 	ctx := session.Context()
 	env := append(session.Environ(), extraEnv...)

agent/agentssh/agentssh_internal_test.go

@@ -17,8 +17,6 @@ import (
 
 	"github.com/coder/coder/v2/pty"
 	"github.com/coder/coder/v2/testutil"
-
-	"cdr.dev/slog/sloggers/slogtest"
 )
 
 const longScript = `
@@ -36,7 +34,7 @@ func Test_sessionStart_orphan(t *testing.T) {
 
 	ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitMedium)
 	defer cancel()
-	logger := slogtest.Make(t, nil)
+	logger := testutil.Logger(t)
 	s, err := NewServer(ctx, logger, prometheus.NewRegistry(), afero.NewMemMapFs(), nil)
 	require.NoError(t, err)
 	defer s.Close()

agent/agentssh/agentssh_test.go

@@ -35,7 +35,7 @@ func TestNewServer_ServeClient(t *testing.T) {
 	t.Parallel()
 
 	ctx := context.Background()
-	logger := slogtest.Make(t, nil)
+	logger := testutil.Logger(t)
 	s, err := agentssh.NewServer(ctx, logger, prometheus.NewRegistry(), afero.NewMemMapFs(), nil)
 	require.NoError(t, err)
 	defer s.Close()
@@ -76,7 +76,7 @@ func TestNewServer_ExecuteShebang(t *testing.T) {
 	}
 
 	ctx := context.Background()
-	logger := slogtest.Make(t, nil)
+	logger := testutil.Logger(t)
 	s, err := agentssh.NewServer(ctx, logger, prometheus.NewRegistry(), afero.NewMemMapFs(), nil)
 	require.NoError(t, err)
 	t.Cleanup(func() {
@@ -158,7 +158,7 @@ func TestNewServer_Signal(t *testing.T) {
 		t.Parallel()
 
 		ctx := context.Background()
-		logger := slogtest.Make(t, nil)
+		logger := testutil.Logger(t)
 		s, err := agentssh.NewServer(ctx, logger, prometheus.NewRegistry(), afero.NewMemMapFs(), nil)
 		require.NoError(t, err)
 		defer s.Close()
@@ -223,7 +223,7 @@ func TestNewServer_Signal(t *testing.T) {
 		t.Parallel()
 
 		ctx := context.Background()
-		logger := slogtest.Make(t, nil)
+		logger := testutil.Logger(t)
 		s, err := agentssh.NewServer(ctx, logger, prometheus.NewRegistry(), afero.NewMemMapFs(), nil)
 		require.NoError(t, err)
 		defer s.Close()

agent/agentssh/x11.go

@@ -7,6 +7,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"math"
 	"net"
 	"os"
 	"path/filepath"
@@ -22,61 +23,69 @@ import (
 	"cdr.dev/slog"
 )
 
+const (
+	// X11StartPort is the starting port for X11 forwarding, this is the
+	// port used for "DISPLAY=localhost:0".
+	X11StartPort = 6000
+	// X11DefaultDisplayOffset is the default offset for X11 forwarding.
+	X11DefaultDisplayOffset = 10
+)
+
 // x11Callback is called when the client requests X11 forwarding.
-// It adds an Xauthority entry to the Xauthority file.
-func (s *Server) x11Callback(ctx ssh.Context, x11 ssh.X11) bool {
-	hostname, err := os.Hostname()
-	if err != nil {
-		s.logger.Warn(ctx, "failed to get hostname", slog.Error(err))
-		s.metrics.x11HandlerErrors.WithLabelValues("hostname").Add(1)
-		return false
-	}
-
-	err = s.fs.MkdirAll(s.config.X11SocketDir, 0o700)
-	if err != nil {
-		s.logger.Warn(ctx, "failed to make the x11 socket dir", slog.F("dir", s.config.X11SocketDir), slog.Error(err))
-		s.metrics.x11HandlerErrors.WithLabelValues("socker_dir").Add(1)
-		return false
-	}
-
-	err = addXauthEntry(ctx, s.fs, hostname, strconv.Itoa(int(x11.ScreenNumber)), x11.AuthProtocol, x11.AuthCookie)
-	if err != nil {
-		s.logger.Warn(ctx, "failed to add Xauthority entry", slog.Error(err))
-		s.metrics.x11HandlerErrors.WithLabelValues("xauthority").Add(1)
-		return false
-	}
+func (*Server) x11Callback(_ ssh.Context, _ ssh.X11) bool {
+	// Always allow.
 	return true
 }
 
 // x11Handler is called when a session has requested X11 forwarding.
 // It listens for X11 connections and forwards them to the client.
-func (s *Server) x11Handler(ctx ssh.Context, x11 ssh.X11) bool {
+func (s *Server) x11Handler(ctx ssh.Context, x11 ssh.X11) (displayNumber int, handled bool) {
 	serverConn, valid := ctx.Value(ssh.ContextKeyConn).(*gossh.ServerConn)
 	if !valid {
 		s.logger.Warn(ctx, "failed to get server connection")
-		return false
+		return -1, false
 	}
-	// We want to overwrite the socket so that subsequent connections will succeed.
-	socketPath := filepath.Join(s.config.X11SocketDir, fmt.Sprintf("X%d", x11.ScreenNumber))
-	err := os.Remove(socketPath)
-	if err != nil && !errors.Is(err, os.ErrNotExist) {
-		s.logger.Warn(ctx, "failed to remove existing X11 socket", slog.Error(err))
-		return false
-	}
-	listener, err := net.Listen("unix", socketPath)
+
+	hostname, err := os.Hostname()
 	if err != nil {
-		s.logger.Warn(ctx, "failed to listen for X11", slog.Error(err))
-		return false
+		s.logger.Warn(ctx, "failed to get hostname", slog.Error(err))
+		s.metrics.x11HandlerErrors.WithLabelValues("hostname").Add(1)
+		return -1, false
+	}
+
+	ln, display, err := createX11Listener(ctx, *s.config.X11DisplayOffset)
+	if err != nil {
+		s.logger.Warn(ctx, "failed to create X11 listener", slog.Error(err))
+		s.metrics.x11HandlerErrors.WithLabelValues("listen").Add(1)
+		return -1, false
+	}
+	s.trackListener(ln, true)
+	defer func() {
+		if !handled {
+			s.trackListener(ln, false)
+			_ = ln.Close()
+		}
+	}()
+
+	err = addXauthEntry(ctx, s.fs, hostname, strconv.Itoa(display), x11.AuthProtocol, x11.AuthCookie)
+	if err != nil {
+		s.logger.Warn(ctx, "failed to add Xauthority entry", slog.Error(err))
+		s.metrics.x11HandlerErrors.WithLabelValues("xauthority").Add(1)
+		return -1, false
 	}
-	s.trackListener(listener, true)
 
 	go func() {
-		defer listener.Close()
-		defer s.trackListener(listener, false)
-		handledFirstConnection := false
+		// Don't leave the listener open after the session is gone.
+		<-ctx.Done()
+		_ = ln.Close()
+	}()
+
+	go func() {
+		defer ln.Close()
+		defer s.trackListener(ln, false)
 
 		for {
-			conn, err := listener.Accept()
+			conn, err := ln.Accept()
 			if err != nil {
 				if errors.Is(err, net.ErrClosed) {
 					return
@@ -84,40 +93,66 @@ func (s *Server) x11Handler(ctx ssh.Context, x11 ssh.X11) bool {
 				s.logger.Warn(ctx, "failed to accept X11 connection", slog.Error(err))
 				return
 			}
-			if x11.SingleConnection && handledFirstConnection {
-				s.logger.Warn(ctx, "X11 connection rejected because single connection is enabled")
+			if x11.SingleConnection {
+				s.logger.Debug(ctx, "single connection requested, closing X11 listener")
+				_ = ln.Close()
+			}
+
+			tcpConn, ok := conn.(*net.TCPConn)
+			if !ok {
+				s.logger.Warn(ctx, fmt.Sprintf("failed to cast connection to TCPConn. got: %T", conn))
 				_ = conn.Close()
 				continue
 			}
-			handledFirstConnection = true
-
-			unixConn, ok := conn.(*net.UnixConn)
+			tcpAddr, ok := tcpConn.LocalAddr().(*net.TCPAddr)
 			if !ok {
-				s.logger.Warn(ctx, fmt.Sprintf("failed to cast connection to UnixConn. got: %T", conn))
-				return
-			}
-			unixAddr, ok := unixConn.LocalAddr().(*net.UnixAddr)
-			if !ok {
-				s.logger.Warn(ctx, fmt.Sprintf("failed to cast local address to UnixAddr. got: %T", unixConn.LocalAddr()))
-				return
+				s.logger.Warn(ctx, fmt.Sprintf("failed to cast local address to TCPAddr. got: %T", tcpConn.LocalAddr()))
+				_ = conn.Close()
+				continue
 			}
 
 			channel, reqs, err := serverConn.OpenChannel("x11", gossh.Marshal(struct {
 				OriginatorAddress string
 				OriginatorPort    uint32
 			}{
-				OriginatorAddress: unixAddr.Name,
-				OriginatorPort:    0,
+				OriginatorAddress: tcpAddr.IP.String(),
+				OriginatorPort:    uint32(tcpAddr.Port),
 			}))
 			if err != nil {
 				s.logger.Warn(ctx, "failed to open X11 channel", slog.Error(err))
-				return
+				_ = conn.Close()
+				continue
 			}
 			go gossh.DiscardRequests(reqs)
-			go Bicopy(ctx, conn, channel)
+
+			if !s.trackConn(ln, conn, true) {
+				s.logger.Warn(ctx, "failed to track X11 connection")
+				_ = conn.Close()
+				continue
+			}
+			go func() {
+				defer s.trackConn(ln, conn, false)
+				Bicopy(ctx, conn, channel)
+			}()
 		}
 	}()
-	return true
+
+	return display, true
+}
+
+// createX11Listener creates a listener for X11 forwarding, it will use
+// the next available port starting from X11StartPort and displayOffset.
+func createX11Listener(ctx context.Context, displayOffset int) (ln net.Listener, display int, err error) {
+	var lc net.ListenConfig
+	// Look for an open port to listen on.
+	for port := X11StartPort + displayOffset; port < math.MaxUint16; port++ {
+		ln, err = lc.Listen(ctx, "tcp", fmt.Sprintf("localhost:%d", port))
+		if err == nil {
+			display = port - X11StartPort
+			return ln, display, nil
+		}
+	}
+	return nil, -1, xerrors.Errorf("failed to find open port for X11 listener: %w", err)
 }
 
 // addXauthEntry adds an Xauthority entry to the Xauthority file.

agent/agentssh/x11_test.go

@@ -1,12 +1,17 @@
 package agentssh_test
 
 import (
+	"bufio"
+	"bytes"
 	"context"
 	"encoding/hex"
+	"fmt"
 	"net"
 	"os"
 	"path/filepath"
 	"runtime"
+	"strconv"
+	"strings"
 	"testing"
 
 	"github.com/gliderlabs/ssh"
@@ -16,8 +21,6 @@ import (
 	"github.com/stretchr/testify/require"
 	gossh "golang.org/x/crypto/ssh"
 
-	"cdr.dev/slog"
-	"cdr.dev/slog/sloggers/slogtest"
 	"github.com/coder/coder/v2/agent/agentssh"
 	"github.com/coder/coder/v2/testutil"
 )
@@ -29,12 +32,9 @@ func TestServer_X11(t *testing.T) {
 	}
 
 	ctx := context.Background()
-	logger := slogtest.Make(t, nil).Leveled(slog.LevelDebug)
+	logger := testutil.Logger(t)
 	fs := afero.NewOsFs()
-	dir := t.TempDir()
-	s, err := agentssh.NewServer(ctx, logger, prometheus.NewRegistry(), fs, &agentssh.Config{
-		X11SocketDir: dir,
-	})
+	s, err := agentssh.NewServer(ctx, logger, prometheus.NewRegistry(), fs, &agentssh.Config{})
 	require.NoError(t, err)
 	defer s.Close()
 
@@ -53,21 +53,45 @@ func TestServer_X11(t *testing.T) {
 	sess, err := c.NewSession()
 	require.NoError(t, err)
 
+	wantScreenNumber := 1
 	reply, err := sess.SendRequest("x11-req", true, gossh.Marshal(ssh.X11{
 		AuthProtocol: "MIT-MAGIC-COOKIE-1",
 		AuthCookie:   hex.EncodeToString([]byte("cookie")),
-		ScreenNumber: 0,
+		ScreenNumber: uint32(wantScreenNumber),
 	}))
 	require.NoError(t, err)
 	assert.True(t, reply)
 
-	err = sess.Shell()
+	// Want: ~DISPLAY=localhost:10.1
+	out, err := sess.Output("echo DISPLAY=$DISPLAY")
 	require.NoError(t, err)
 
+	sc := bufio.NewScanner(bytes.NewReader(out))
+	displayNumber := -1
+	for sc.Scan() {
+		line := strings.TrimSpace(sc.Text())
+		t.Log(line)
+		if strings.HasPrefix(line, "DISPLAY=") {
+			parts := strings.SplitN(line, "=", 2)
+			display := parts[1]
+			parts = strings.SplitN(display, ":", 2)
+			parts = strings.SplitN(parts[1], ".", 2)
+			displayNumber, err = strconv.Atoi(parts[0])
+			require.NoError(t, err)
+			assert.GreaterOrEqual(t, displayNumber, 10, "display number should be >= 10")
+			gotScreenNumber, err := strconv.Atoi(parts[1])
+			require.NoError(t, err)
+			assert.Equal(t, wantScreenNumber, gotScreenNumber, "screen number should match")
+			break
+		}
+	}
+	require.NoError(t, sc.Err())
+	require.NotEqual(t, -1, displayNumber)
+
 	x11Chans := c.HandleChannelOpen("x11")
 	payload := "hello world"
 	require.Eventually(t, func() bool {
-		conn, err := net.Dial("unix", filepath.Join(dir, "X0"))
+		conn, err := net.Dial("tcp", fmt.Sprintf("localhost:%d", agentssh.X11StartPort+displayNumber))
 		if err == nil {
 			_, err = conn.Write([]byte(payload))
 			assert.NoError(t, err)

agent/agenttest/agent.go

@@ -7,10 +7,9 @@ import (
 
 	"github.com/stretchr/testify/assert"
 
-	"cdr.dev/slog"
-	"cdr.dev/slog/sloggers/slogtest"
 	"github.com/coder/coder/v2/agent"
 	"github.com/coder/coder/v2/codersdk/agentsdk"
+	"github.com/coder/coder/v2/testutil"
 )
 
 // New starts a new agent for use in tests.
@@ -24,7 +23,7 @@ func New(t testing.TB, coderURL *url.URL, agentToken string, opts ...func(*agent
 	t.Helper()
 
 	var o agent.Options
-	log := slogtest.Make(t, nil).Leveled(slog.LevelDebug).Named("agent")
+	log := testutil.Logger(t).Named("agent")
 	o.Logger = log
 
 	for _, opt := range opts {

agent/agenttest/client.go

@@ -15,7 +15,6 @@ import (
 	"golang.org/x/exp/slices"
 	"golang.org/x/xerrors"
 	"google.golang.org/protobuf/types/known/durationpb"
-	"storj.io/drpc"
 	"storj.io/drpc/drpcmux"
 	"storj.io/drpc/drpcserver"
 	"tailscale.com/tailcfg"
@@ -71,7 +70,6 @@ func NewClient(t testing.TB,
 		t:              t,
 		logger:         logger.Named("client"),
 		agentID:        agentID,
-		coordinator:    coordinator,
 		server:         server,
 		fakeAgentAPI:   fakeAAPI,
 		derpMapUpdates: derpMapUpdates,
@@ -82,7 +80,6 @@ type Client struct {
 	t                  testing.TB
 	logger             slog.Logger
 	agentID            uuid.UUID
-	coordinator        tailnet.Coordinator
 	server             *drpcserver.Server
 	fakeAgentAPI       *FakeAgentAPI
 	LastWorkspaceAgent func()
@@ -99,7 +96,9 @@ func (c *Client) Close() {
 	c.derpMapOnce.Do(func() { close(c.derpMapUpdates) })
 }
 
-func (c *Client) ConnectRPC(ctx context.Context) (drpc.Conn, error) {
+func (c *Client) ConnectRPC23(ctx context.Context) (
+	agentproto.DRPCAgentClient23, proto.DRPCTailnetClient23, error,
+) {
 	conn, lis := drpcsdk.MemTransportPipe()
 	c.LastWorkspaceAgent = func() {
 		_ = conn.Close()
@@ -117,7 +116,7 @@ func (c *Client) ConnectRPC(ctx context.Context) (drpc.Conn, error) {
 	go func() {
 		_ = c.server.Serve(serveCtx, lis)
 	}()
-	return conn, nil
+	return agentproto.NewDRPCAgentClient(conn), proto.NewDRPCTailnetClient(conn), nil
 }
 
 func (c *Client) GetLifecycleStates() []codersdk.WorkspaceAgentLifecycle {
@@ -170,6 +169,7 @@ type FakeAgentAPI struct {
 	logsCh          chan<- *agentproto.BatchCreateLogsRequest
 	lifecycleStates []codersdk.WorkspaceAgentLifecycle
 	metadata        map[string]agentsdk.Metadata
+	timings         []*agentproto.Timing
 
 	getAnnouncementBannersFunc func() ([]codersdk.BannerConfig, error)
 }
@@ -182,6 +182,12 @@ func (*FakeAgentAPI) GetServiceBanner(context.Context, *agentproto.GetServiceBan
 	return &agentproto.ServiceBanner{}, nil
 }
 
+func (f *FakeAgentAPI) GetTimings() []*agentproto.Timing {
+	f.Lock()
+	defer f.Unlock()
+	return slices.Clone(f.timings)
+}
+
 func (f *FakeAgentAPI) SetAnnouncementBannersFunc(fn func() ([]codersdk.BannerConfig, error)) {
 	f.Lock()
 	defer f.Unlock()
@@ -210,7 +216,12 @@ func (f *FakeAgentAPI) UpdateStats(ctx context.Context, req *agentproto.UpdateSt
 	f.logger.Debug(ctx, "update stats called", slog.F("req", req))
 	// empty request is sent to get the interval; but our tests don't want empty stats requests
 	if req.Stats != nil {
-		f.statsCh <- req.Stats
+		select {
+		case <-ctx.Done():
+			return nil, ctx.Err()
+		case f.statsCh <- req.Stats:
+			// OK!
+		}
 	}
 	return &agentproto.UpdateStatsResponse{ReportInterval: durationpb.New(statsInterval)}, nil
 }
@@ -233,17 +244,25 @@ func (f *FakeAgentAPI) UpdateLifecycle(_ context.Context, req *agentproto.Update
 
 func (f *FakeAgentAPI) BatchUpdateAppHealths(ctx context.Context, req *agentproto.BatchUpdateAppHealthRequest) (*agentproto.BatchUpdateAppHealthResponse, error) {
 	f.logger.Debug(ctx, "batch update app health", slog.F("req", req))
-	f.appHealthCh <- req
-	return &agentproto.BatchUpdateAppHealthResponse{}, nil
+	select {
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	case f.appHealthCh <- req:
+		return &agentproto.BatchUpdateAppHealthResponse{}, nil
+	}
 }
 
 func (f *FakeAgentAPI) AppHealthCh() <-chan *agentproto.BatchUpdateAppHealthRequest {
 	return f.appHealthCh
 }
 
-func (f *FakeAgentAPI) UpdateStartup(_ context.Context, req *agentproto.UpdateStartupRequest) (*agentproto.Startup, error) {
-	f.startupCh <- req.GetStartup()
-	return req.GetStartup(), nil
+func (f *FakeAgentAPI) UpdateStartup(ctx context.Context, req *agentproto.UpdateStartupRequest) (*agentproto.Startup, error) {
+	select {
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	case f.startupCh <- req.GetStartup():
+		return req.GetStartup(), nil
+	}
 }
 
 func (f *FakeAgentAPI) GetMetadata() map[string]agentsdk.Metadata {
@@ -288,6 +307,14 @@ func (f *FakeAgentAPI) BatchCreateLogs(ctx context.Context, req *agentproto.Batc
 	return &agentproto.BatchCreateLogsResponse{}, nil
 }
 
+func (f *FakeAgentAPI) ScriptCompleted(_ context.Context, req *agentproto.WorkspaceAgentScriptCompletedRequest) (*agentproto.WorkspaceAgentScriptCompletedResponse, error) {
+	f.Lock()
+	f.timings = append(f.timings, req.Timing)
+	f.Unlock()
+
+	return &agentproto.WorkspaceAgentScriptCompletedResponse{}, nil
+}
+
 func NewFakeAgentAPI(t testing.TB, logger slog.Logger, manifest *agentproto.Manifest, statsCh chan *agentproto.Stats) *FakeAgentAPI {
 	return &FakeAgentAPI{
 		t:           t,

agent/api.go

@@ -37,6 +37,7 @@ func (a *agent) apiHandler() http.Handler {
 	}
 	promHandler := PrometheusMetricsHandler(a.prometheusRegistry, a.logger)
 	r.Get("/api/v0/listening-ports", lp.handler)
+	r.Get("/api/v0/netcheck", a.HandleNetcheck)
 	r.Get("/debug/logs", a.HandleHTTPDebugLogs)
 	r.Get("/debug/magicsock", a.HandleHTTPDebugMagicsock)
 	r.Get("/debug/magicsock/debug-logging/{state}", a.HandleHTTPMagicsockDebugLoggingState)

agent/apphealth.go

@@ -12,12 +12,9 @@ import (
 	"cdr.dev/slog"
 	"github.com/coder/coder/v2/codersdk"
 	"github.com/coder/coder/v2/codersdk/agentsdk"
-	"github.com/coder/retry"
+	"github.com/coder/quartz"
 )
 
-// WorkspaceAgentApps fetches the workspace apps.
-type WorkspaceAgentApps func(context.Context) ([]codersdk.WorkspaceApp, error)
-
 // PostWorkspaceAgentAppHealth updates the workspace app health.
 type PostWorkspaceAgentAppHealth func(context.Context, agentsdk.PostAppHealthsRequest) error
 
@@ -26,15 +23,26 @@ type WorkspaceAppHealthReporter func(ctx context.Context)
 
 // NewWorkspaceAppHealthReporter creates a WorkspaceAppHealthReporter that reports app health to coderd.
 func NewWorkspaceAppHealthReporter(logger slog.Logger, apps []codersdk.WorkspaceApp, postWorkspaceAgentAppHealth PostWorkspaceAgentAppHealth) WorkspaceAppHealthReporter {
+	return NewAppHealthReporterWithClock(logger, apps, postWorkspaceAgentAppHealth, quartz.NewReal())
+}
+
+// NewAppHealthReporterWithClock is only called directly by test code.  Product code should call
+// NewAppHealthReporter.
+func NewAppHealthReporterWithClock(
+	logger slog.Logger,
+	apps []codersdk.WorkspaceApp,
+	postWorkspaceAgentAppHealth PostWorkspaceAgentAppHealth,
+	clk quartz.Clock,
+) WorkspaceAppHealthReporter {
 	logger = logger.Named("apphealth")
 
-	runHealthcheckLoop := func(ctx context.Context) error {
+	return func(ctx context.Context) {
 		ctx, cancel := context.WithCancel(ctx)
 		defer cancel()
 
 		// no need to run this loop if no apps for this workspace.
 		if len(apps) == 0 {
-			return nil
+			return
 		}
 
 		hasHealthchecksEnabled := false
@@ -49,7 +57,7 @@ func NewWorkspaceAppHealthReporter(logger slog.Logger, apps []codersdk.Workspace
 
 		// no need to run this loop if no health checks are configured.
 		if !hasHealthchecksEnabled {
-			return nil
+			return
 		}
 
 		// run a ticker for each app health check.
@@ -61,25 +69,29 @@ func NewWorkspaceAppHealthReporter(logger slog.Logger, apps []codersdk.Workspace
 			}
 			app := nextApp
 			go func() {
-				t := time.NewTicker(time.Duration(app.Healthcheck.Interval) * time.Second)
-				defer t.Stop()
+				_ = clk.TickerFunc(ctx, time.Duration(app.Healthcheck.Interval)*time.Second, func() error {
+					// We time out at the healthcheck interval to prevent getting too backed up, but
+					// set it 1ms early so that it's not simultaneous with the next tick in testing,
+					// which makes the test easier to understand.
+					//
+					// It would be idiomatic to use the http.Client.Timeout or a context.WithTimeout,
+					// but we are passing this off to the native http library, which is not aware
+					// of the clock library we are using. That means in testing, with a mock clock
+					// it will compare mocked times with real times, and we will get strange results.
+					// So, we just implement the timeout as a context we cancel with an AfterFunc
+					reqCtx, reqCancel := context.WithCancel(ctx)
+					timeout := clk.AfterFunc(
+						time.Duration(app.Healthcheck.Interval)*time.Second-time.Millisecond,
+						reqCancel,
+						"timeout", app.Slug)
+					defer timeout.Stop()
 
-				for {
-					select {
-					case <-ctx.Done():
-						return
-					case <-t.C:
-					}
-					// we set the http timeout to the healthcheck interval to prevent getting too backed up.
-					client := &http.Client{
-						Timeout: time.Duration(app.Healthcheck.Interval) * time.Second,
-					}
 					err := func() error {
-						req, err := http.NewRequestWithContext(ctx, http.MethodGet, app.Healthcheck.URL, nil)
+						req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, app.Healthcheck.URL, nil)
 						if err != nil {
 							return err
 						}
-						res, err := client.Do(req)
+						res, err := http.DefaultClient.Do(req)
 						if err != nil {
 							return err
 						}
@@ -118,54 +130,36 @@ func NewWorkspaceAppHealthReporter(logger slog.Logger, apps []codersdk.Workspace
 						mu.Unlock()
 						logger.Debug(ctx, "workspace app healthy", slog.F("id", app.ID.String()), slog.F("slug", app.Slug))
 					}
-
-					t.Reset(time.Duration(app.Healthcheck.Interval) * time.Second)
-				}
+					return nil
+				}, "healthcheck", app.Slug)
 			}()
 		}
 
 		mu.Lock()
 		lastHealth := copyHealth(health)
 		mu.Unlock()
-		reportTicker := time.NewTicker(time.Second)
-		defer reportTicker.Stop()
-		// every second we check if the health values of the apps have changed
-		// and if there is a change we will report the new values.
-		for {
-			select {
-			case <-ctx.Done():
+		reportTicker := clk.TickerFunc(ctx, time.Second, func() error {
+			mu.RLock()
+			changed := healthChanged(lastHealth, health)
+			mu.RUnlock()
+			if !changed {
 				return nil
-			case <-reportTicker.C:
-				mu.RLock()
-				changed := healthChanged(lastHealth, health)
-				mu.RUnlock()
-				if !changed {
-					continue
-				}
-
-				mu.Lock()
-				lastHealth = copyHealth(health)
-				mu.Unlock()
-				err := postWorkspaceAgentAppHealth(ctx, agentsdk.PostAppHealthsRequest{
-					Healths: lastHealth,
-				})
-				if err != nil {
-					logger.Error(ctx, "failed to report workspace app health", slog.Error(err))
-				} else {
-					logger.Debug(ctx, "sent workspace app health", slog.F("health", lastHealth))
-				}
 			}
-		}
-	}
 
-	return func(ctx context.Context) {
-		for r := retry.New(time.Second, 30*time.Second); r.Wait(ctx); {
-			err := runHealthcheckLoop(ctx)
-			if err == nil || xerrors.Is(err, context.Canceled) || xerrors.Is(err, context.DeadlineExceeded) {
-				return
+			mu.Lock()
+			lastHealth = copyHealth(health)
+			mu.Unlock()
+			err := postWorkspaceAgentAppHealth(ctx, agentsdk.PostAppHealthsRequest{
+				Healths: lastHealth,
+			})
+			if err != nil {
+				logger.Error(ctx, "failed to report workspace app health", slog.Error(err))
+			} else {
+				logger.Debug(ctx, "sent workspace app health", slog.F("health", lastHealth))
 			}
-			logger.Error(ctx, "failed running workspace app reporter", slog.Error(err))
-		}
+			return nil
+		}, "report")
+		_ = reportTicker.Wait() // only possible error is context done
 	}
 }
 

agent/apphealth_test.go

@@ -4,18 +4,14 @@ import (
 	"context"
 	"net/http"
 	"net/http/httptest"
+	"slices"
 	"strings"
-	"sync"
-	"sync/atomic"
 	"testing"
 	"time"
 
 	"github.com/google/uuid"
-	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
-	"cdr.dev/slog"
-	"cdr.dev/slog/sloggers/slogtest"
 	"github.com/coder/coder/v2/agent"
 	"github.com/coder/coder/v2/agent/agenttest"
 	"github.com/coder/coder/v2/agent/proto"
@@ -23,19 +19,22 @@ import (
 	"github.com/coder/coder/v2/codersdk"
 	"github.com/coder/coder/v2/codersdk/agentsdk"
 	"github.com/coder/coder/v2/testutil"
+	"github.com/coder/quartz"
 )
 
 func TestAppHealth_Healthy(t *testing.T) {
 	t.Parallel()
-	ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
+	ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitShort)
 	defer cancel()
 	apps := []codersdk.WorkspaceApp{
 		{
+			ID:          uuid.UUID{1},
 			Slug:        "app1",
 			Healthcheck: codersdk.Healthcheck{},
 			Health:      codersdk.WorkspaceAppHealthDisabled,
 		},
 		{
+			ID:   uuid.UUID{2},
 			Slug: "app2",
 			Healthcheck: codersdk.Healthcheck{
 				// URL: We don't set the URL for this test because the setup will
@@ -46,6 +45,7 @@ func TestAppHealth_Healthy(t *testing.T) {
 			Health: codersdk.WorkspaceAppHealthInitializing,
 		},
 		{
+			ID:   uuid.UUID{3},
 			Slug: "app3",
 			Healthcheck: codersdk.Healthcheck{
 				Interval:  2,
@@ -54,36 +54,71 @@ func TestAppHealth_Healthy(t *testing.T) {
 			Health: codersdk.WorkspaceAppHealthInitializing,
 		},
 	}
+	checks2 := 0
+	checks3 := 0
 	handlers := []http.Handler{
 		nil,
 		http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			checks2++
 			httpapi.Write(r.Context(), w, http.StatusOK, nil)
 		}),
 		http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			checks3++
 			httpapi.Write(r.Context(), w, http.StatusOK, nil)
 		}),
 	}
-	getApps, closeFn := setupAppReporter(ctx, t, apps, handlers)
-	defer closeFn()
-	apps, err := getApps(ctx)
-	require.NoError(t, err)
-	require.EqualValues(t, codersdk.WorkspaceAppHealthDisabled, apps[0].Health)
-	require.Eventually(t, func() bool {
-		apps, err := getApps(ctx)
-		if err != nil {
-			return false
-		}
+	mClock := quartz.NewMock(t)
+	healthcheckTrap := mClock.Trap().TickerFunc("healthcheck")
+	defer healthcheckTrap.Close()
+	reportTrap := mClock.Trap().TickerFunc("report")
+	defer reportTrap.Close()
 
-		return apps[1].Health == codersdk.WorkspaceAppHealthHealthy && apps[2].Health == codersdk.WorkspaceAppHealthHealthy
-	}, testutil.WaitLong, testutil.IntervalSlow)
+	fakeAPI, closeFn := setupAppReporter(ctx, t, slices.Clone(apps), handlers, mClock)
+	defer closeFn()
+	healthchecksStarted := make([]string, 2)
+	for i := 0; i < 2; i++ {
+		c := healthcheckTrap.MustWait(ctx)
+		c.Release()
+		healthchecksStarted[i] = c.Tags[1]
+	}
+	slices.Sort(healthchecksStarted)
+	require.Equal(t, []string{"app2", "app3"}, healthchecksStarted)
+
+	// advance the clock 1ms before the report ticker starts, so that it's not
+	// simultaneous with the checks.
+	mClock.Advance(time.Millisecond).MustWait(ctx)
+	reportTrap.MustWait(ctx).Release()
+
+	mClock.Advance(999 * time.Millisecond).MustWait(ctx) // app2 is now healthy
+
+	mClock.Advance(time.Millisecond).MustWait(ctx) // report gets triggered
+	update := testutil.RequireRecvCtx(ctx, t, fakeAPI.AppHealthCh())
+	require.Len(t, update.GetUpdates(), 2)
+	applyUpdate(t, apps, update)
+	require.Equal(t, codersdk.WorkspaceAppHealthHealthy, apps[1].Health)
+	require.Equal(t, codersdk.WorkspaceAppHealthInitializing, apps[2].Health)
+
+	mClock.Advance(999 * time.Millisecond).MustWait(ctx) // app3 is now healthy
+
+	mClock.Advance(time.Millisecond).MustWait(ctx) // report gets triggered
+	update = testutil.RequireRecvCtx(ctx, t, fakeAPI.AppHealthCh())
+	require.Len(t, update.GetUpdates(), 2)
+	applyUpdate(t, apps, update)
+	require.Equal(t, codersdk.WorkspaceAppHealthHealthy, apps[1].Health)
+	require.Equal(t, codersdk.WorkspaceAppHealthHealthy, apps[2].Health)
+
+	// ensure we aren't spamming
+	require.Equal(t, 2, checks2)
+	require.Equal(t, 1, checks3)
 }
 
 func TestAppHealth_500(t *testing.T) {
 	t.Parallel()
-	ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
+	ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitShort)
 	defer cancel()
 	apps := []codersdk.WorkspaceApp{
 		{
+			ID:   uuid.UUID{2},
 			Slug: "app2",
 			Healthcheck: codersdk.Healthcheck{
 				// URL: We don't set the URL for this test because the setup will
@@ -99,59 +134,40 @@ func TestAppHealth_500(t *testing.T) {
 			httpapi.Write(r.Context(), w, http.StatusInternalServerError, nil)
 		}),
 	}
-	getApps, closeFn := setupAppReporter(ctx, t, apps, handlers)
-	defer closeFn()
-	require.Eventually(t, func() bool {
-		apps, err := getApps(ctx)
-		if err != nil {
-			return false
-		}
 
-		return apps[0].Health == codersdk.WorkspaceAppHealthUnhealthy
-	}, testutil.WaitLong, testutil.IntervalSlow)
+	mClock := quartz.NewMock(t)
+	healthcheckTrap := mClock.Trap().TickerFunc("healthcheck")
+	defer healthcheckTrap.Close()
+	reportTrap := mClock.Trap().TickerFunc("report")
+	defer reportTrap.Close()
+
+	fakeAPI, closeFn := setupAppReporter(ctx, t, slices.Clone(apps), handlers, mClock)
+	defer closeFn()
+	healthcheckTrap.MustWait(ctx).Release()
+	// advance the clock 1ms before the report ticker starts, so that it's not
+	// simultaneous with the checks.
+	mClock.Advance(time.Millisecond).MustWait(ctx)
+	reportTrap.MustWait(ctx).Release()
+
+	mClock.Advance(999 * time.Millisecond).MustWait(ctx) // check gets triggered
+	mClock.Advance(time.Millisecond).MustWait(ctx)       // report gets triggered, but unsent since we are at the threshold
+
+	mClock.Advance(999 * time.Millisecond).MustWait(ctx) // 2nd check, crosses threshold
+	mClock.Advance(time.Millisecond).MustWait(ctx)       // 2nd report, sends update
+
+	update := testutil.RequireRecvCtx(ctx, t, fakeAPI.AppHealthCh())
+	require.Len(t, update.GetUpdates(), 1)
+	applyUpdate(t, apps, update)
+	require.Equal(t, codersdk.WorkspaceAppHealthUnhealthy, apps[0].Health)
 }
 
 func TestAppHealth_Timeout(t *testing.T) {
 	t.Parallel()
-	ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
-	defer cancel()
-	apps := []codersdk.WorkspaceApp{
-		{
-			Slug: "app2",
-			Healthcheck: codersdk.Healthcheck{
-				// URL: We don't set the URL for this test because the setup will
-				// create a httptest server for us and set it for us.
-				Interval:  1,
-				Threshold: 1,
-			},
-			Health: codersdk.WorkspaceAppHealthInitializing,
-		},
-	}
-	handlers := []http.Handler{
-		http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			// sleep longer than the interval to cause the health check to time out
-			time.Sleep(2 * time.Second)
-			httpapi.Write(r.Context(), w, http.StatusOK, nil)
-		}),
-	}
-	getApps, closeFn := setupAppReporter(ctx, t, apps, handlers)
-	defer closeFn()
-	require.Eventually(t, func() bool {
-		apps, err := getApps(ctx)
-		if err != nil {
-			return false
-		}
-
-		return apps[0].Health == codersdk.WorkspaceAppHealthUnhealthy
-	}, testutil.WaitLong, testutil.IntervalSlow)
-}
-
-func TestAppHealth_NotSpamming(t *testing.T) {
-	t.Parallel()
-	ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
+	ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitShort)
 	defer cancel()
 	apps := []codersdk.WorkspaceApp{
 		{
+			ID:   uuid.UUID{2},
 			Slug: "app2",
 			Healthcheck: codersdk.Healthcheck{
 				// URL: We don't set the URL for this test because the setup will
@@ -163,27 +179,65 @@ func TestAppHealth_NotSpamming(t *testing.T) {
 		},
 	}
 
-	counter := new(int32)
 	handlers := []http.Handler{
-		http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			atomic.AddInt32(counter, 1)
+		http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) {
+			// allow the request to time out
+			<-r.Context().Done()
 		}),
 	}
-	_, closeFn := setupAppReporter(ctx, t, apps, handlers)
+	mClock := quartz.NewMock(t)
+	start := mClock.Now()
+
+	// for this test, it's easier to think in the number of milliseconds elapsed
+	// since start.
+	ms := func(n int) time.Time {
+		return start.Add(time.Duration(n) * time.Millisecond)
+	}
+	healthcheckTrap := mClock.Trap().TickerFunc("healthcheck")
+	defer healthcheckTrap.Close()
+	reportTrap := mClock.Trap().TickerFunc("report")
+	defer reportTrap.Close()
+	timeoutTrap := mClock.Trap().AfterFunc("timeout")
+	defer timeoutTrap.Close()
+
+	fakeAPI, closeFn := setupAppReporter(ctx, t, apps, handlers, mClock)
 	defer closeFn()
-	// Ensure we haven't made more than 2 (expected 1 + 1 for buffer) requests in the last second.
-	// if there is a bug where we are spamming the healthcheck route this will catch it.
-	time.Sleep(time.Second)
-	require.LessOrEqual(t, atomic.LoadInt32(counter), int32(2))
+	healthcheckTrap.MustWait(ctx).Release()
+	// advance the clock 1ms before the report ticker starts, so that it's not
+	// simultaneous with the checks.
+	mClock.Set(ms(1)).MustWait(ctx)
+	reportTrap.MustWait(ctx).Release()
+
+	w := mClock.Set(ms(1000)) // 1st check starts
+	timeoutTrap.MustWait(ctx).Release()
+	mClock.Set(ms(1001)).MustWait(ctx) // report tick, no change
+	mClock.Set(ms(1999))               // timeout pops
+	w.MustWait(ctx)                    // 1st check finished
+	w = mClock.Set(ms(2000))           // 2nd check starts
+	timeoutTrap.MustWait(ctx).Release()
+	mClock.Set(ms(2001)).MustWait(ctx) // report tick, no change
+	mClock.Set(ms(2999))               // timeout pops
+	w.MustWait(ctx)                    // 2nd check finished
+	// app is now unhealthy after 2 timeouts
+	mClock.Set(ms(3000)) // 3rd check starts
+	timeoutTrap.MustWait(ctx).Release()
+	mClock.Set(ms(3001)).MustWait(ctx) // report tick, sends changes
+
+	update := testutil.RequireRecvCtx(ctx, t, fakeAPI.AppHealthCh())
+	require.Len(t, update.GetUpdates(), 1)
+	applyUpdate(t, apps, update)
+	require.Equal(t, codersdk.WorkspaceAppHealthUnhealthy, apps[0].Health)
 }
 
-func setupAppReporter(ctx context.Context, t *testing.T, apps []codersdk.WorkspaceApp, handlers []http.Handler) (agent.WorkspaceAgentApps, func()) {
+func setupAppReporter(
+	ctx context.Context, t *testing.T,
+	apps []codersdk.WorkspaceApp,
+	handlers []http.Handler,
+	clk quartz.Clock,
+) (*agenttest.FakeAgentAPI, func()) {
 	closers := []func(){}
-	for i, app := range apps {
-		if app.ID == uuid.Nil {
-			app.ID = uuid.New()
-			apps[i] = app
-		}
+	for _, app := range apps {
+		require.NotEqual(t, uuid.Nil, app.ID, "all apps must have ID set")
 	}
 	for i, handler := range handlers {
 		if handler == nil {
@@ -196,54 +250,39 @@ func setupAppReporter(ctx context.Context, t *testing.T, apps []codersdk.Workspa
 		closers = append(closers, ts.Close)
 	}
 
-	var mu sync.Mutex
-	workspaceAgentApps := func(context.Context) ([]codersdk.WorkspaceApp, error) {
-		mu.Lock()
-		defer mu.Unlock()
-		var newApps []codersdk.WorkspaceApp
-		return append(newApps, apps...), nil
-	}
-
 	// We don't care about manifest or stats in this test since it's not using
 	// a full agent and these RPCs won't get called.
 	//
 	// We use a proper fake agent API so we can test the conversion code and the
 	// request code as well. Before we were bypassing these by using a custom
 	// post function.
-	fakeAAPI := agenttest.NewFakeAgentAPI(t, slogtest.Make(t, nil), nil, nil)
+	fakeAAPI := agenttest.NewFakeAgentAPI(t, testutil.Logger(t), nil, nil)
 
-	// Process events from the channel and update the health of the apps.
-	go func() {
-		appHealthCh := fakeAAPI.AppHealthCh()
-		for {
-			select {
-			case <-ctx.Done():
-				return
-			case req := <-appHealthCh:
-				mu.Lock()
-				for _, update := range req.Updates {
-					updateID, err := uuid.FromBytes(update.Id)
-					assert.NoError(t, err)
-					updateHealth := codersdk.WorkspaceAppHealth(strings.ToLower(proto.AppHealth_name[int32(update.Health)]))
+	go agent.NewAppHealthReporterWithClock(
+		testutil.Logger(t),
+		apps, agentsdk.AppHealthPoster(fakeAAPI), clk,
+	)(ctx)
 
-					for i, app := range apps {
-						if app.ID != updateID {
-							continue
-						}
-						app.Health = updateHealth
-						apps[i] = app
-					}
-				}
-				mu.Unlock()
-			}
-		}
-	}()
-
-	go agent.NewWorkspaceAppHealthReporter(slogtest.Make(t, nil).Leveled(slog.LevelDebug), apps, agentsdk.AppHealthPoster(fakeAAPI))(ctx)
-
-	return workspaceAgentApps, func() {
+	return fakeAAPI, func() {
 		for _, closeFn := range closers {
 			closeFn()
 		}
 	}
 }
+
+func applyUpdate(t *testing.T, apps []codersdk.WorkspaceApp, req *proto.BatchUpdateAppHealthRequest) {
+	t.Helper()
+	for _, update := range req.Updates {
+		updateID, err := uuid.FromBytes(update.Id)
+		require.NoError(t, err)
+		updateHealth := codersdk.WorkspaceAppHealth(strings.ToLower(proto.AppHealth_name[int32(update.Health)]))
+
+		for i, app := range apps {
+			if app.ID != updateID {
+				continue
+			}
+			app.Health = updateHealth
+			apps[i] = app
+		}
+	}
+}

agent/checkpoint_internal_test.go

@@ -12,7 +12,7 @@ import (
 
 func TestCheckpoint_CompleteWait(t *testing.T) {
 	t.Parallel()
-	logger := slogtest.Make(t, nil)
+	logger := testutil.Logger(t)
 	ctx := testutil.Context(t, testutil.WaitShort)
 	uut := newCheckpoint(logger)
 	err := xerrors.New("test")
@@ -35,7 +35,7 @@ func TestCheckpoint_CompleteTwice(t *testing.T) {
 
 func TestCheckpoint_WaitComplete(t *testing.T) {
 	t.Parallel()
-	logger := slogtest.Make(t, nil)
+	logger := testutil.Logger(t)
 	ctx := testutil.Context(t, testutil.WaitShort)
 	uut := newCheckpoint(logger)
 	err := xerrors.New("test")

agent/health.go

@@ -0,0 +1,31 @@
+package agent
+
+import (
+	"net/http"
+
+	"github.com/coder/coder/v2/coderd/healthcheck/health"
+	"github.com/coder/coder/v2/coderd/httpapi"
+	"github.com/coder/coder/v2/codersdk"
+	"github.com/coder/coder/v2/codersdk/healthsdk"
+)
+
+func (a *agent) HandleNetcheck(rw http.ResponseWriter, r *http.Request) {
+	ni := a.TailnetConn().GetNetInfo()
+
+	ifReport, err := healthsdk.RunInterfacesReport()
+	if err != nil {
+		httpapi.Write(r.Context(), rw, http.StatusInternalServerError, codersdk.Response{
+			Message: "Failed to run interfaces report",
+			Detail:  err.Error(),
+		})
+		return
+	}
+
+	httpapi.Write(r.Context(), rw, http.StatusOK, healthsdk.AgentNetcheckReport{
+		BaseReport: healthsdk.BaseReport{
+			Severity: health.SeverityOK,
+		},
+		NetInfo:    ni,
+		Interfaces: ifReport,
+	})
+}

agent/metrics.go

@@ -19,6 +19,7 @@ type agentMetrics struct {
 	// startupScriptSeconds is the time in seconds that the start script(s)
 	// took to run. This is reported once per agent.
 	startupScriptSeconds *prometheus.GaugeVec
+	currentConnections   *prometheus.GaugeVec
 }
 
 func newAgentMetrics(registerer prometheus.Registerer) *agentMetrics {
@@ -45,10 +46,19 @@ func newAgentMetrics(registerer prometheus.Registerer) *agentMetrics {
 	}, []string{"success"})
 	registerer.MustRegister(startupScriptSeconds)
 
+	currentConnections := prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: "coderd",
+		Subsystem: "agentstats",
+		Name:      "currently_reachable_peers",
+		Help:      "The number of peers (e.g. clients) that are currently reachable over the encrypted network.",
+	}, []string{"connection_type"})
+	registerer.MustRegister(currentConnections)
+
 	return &agentMetrics{
 		connectionsTotal:      connectionsTotal,
 		reconnectingPTYErrors: reconnectingPTYErrors,
 		startupScriptSeconds:  startupScriptSeconds,
+		currentConnections:    currentConnections,
 	}
 }
 

agent/proto/agent.proto

@@ -41,6 +41,7 @@ message WorkspaceApp {
 		UNHEALTHY = 4;
 	}
 	Health health = 12;
+	bool hidden = 13;
 }
 
 message WorkspaceAgentScript {
@@ -52,6 +53,8 @@ message WorkspaceAgentScript {
 	bool run_on_stop = 6;
 	bool start_blocks_login = 7;
 	google.protobuf.Duration timeout = 8;
+	string display_name = 9;
+	bytes id = 10;
 }
 
 message WorkspaceAgentMetadata {
@@ -263,6 +266,35 @@ message BannerConfig {
 	string background_color = 3;
 }
 
+message WorkspaceAgentScriptCompletedRequest {
+    Timing timing = 1;
+}
+
+message WorkspaceAgentScriptCompletedResponse {
+}
+
+message Timing {
+    bytes script_id = 1;
+    google.protobuf.Timestamp start = 2;
+    google.protobuf.Timestamp end = 3;
+    int32 exit_code = 4;
+
+    enum Stage {
+        START = 0;
+        STOP = 1;
+        CRON = 2;
+	}
+    Stage stage = 5;
+
+    enum Status {
+        OK = 0;
+        EXIT_FAILURE = 1;
+        TIMED_OUT = 2;
+        PIPES_LEFT_OPEN = 3;
+    }
+    Status status = 6;
+}
+
 service Agent {
 	rpc GetManifest(GetManifestRequest) returns (Manifest);
 	rpc GetServiceBanner(GetServiceBannerRequest) returns (ServiceBanner);
@@ -273,4 +305,5 @@ service Agent {
 	rpc BatchUpdateMetadata(BatchUpdateMetadataRequest) returns (BatchUpdateMetadataResponse);
 	rpc BatchCreateLogs(BatchCreateLogsRequest) returns (BatchCreateLogsResponse);
 	rpc GetAnnouncementBanners(GetAnnouncementBannersRequest) returns (GetAnnouncementBannersResponse);
+	rpc ScriptCompleted(WorkspaceAgentScriptCompletedRequest) returns (WorkspaceAgentScriptCompletedResponse);
 }

agent/proto/agent_drpc.pb.go

@@ -47,6 +47,7 @@ type DRPCAgentClient interface {
 	BatchUpdateMetadata(ctx context.Context, in *BatchUpdateMetadataRequest) (*BatchUpdateMetadataResponse, error)
 	BatchCreateLogs(ctx context.Context, in *BatchCreateLogsRequest) (*BatchCreateLogsResponse, error)
 	GetAnnouncementBanners(ctx context.Context, in *GetAnnouncementBannersRequest) (*GetAnnouncementBannersResponse, error)
+	ScriptCompleted(ctx context.Context, in *WorkspaceAgentScriptCompletedRequest) (*WorkspaceAgentScriptCompletedResponse, error)
 }
 
 type drpcAgentClient struct {
@@ -140,6 +141,15 @@ func (c *drpcAgentClient) GetAnnouncementBanners(ctx context.Context, in *GetAnn
 	return out, nil
 }
 
+func (c *drpcAgentClient) ScriptCompleted(ctx context.Context, in *WorkspaceAgentScriptCompletedRequest) (*WorkspaceAgentScriptCompletedResponse, error) {
+	out := new(WorkspaceAgentScriptCompletedResponse)
+	err := c.cc.Invoke(ctx, "/coder.agent.v2.Agent/ScriptCompleted", drpcEncoding_File_agent_proto_agent_proto{}, in, out)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
 type DRPCAgentServer interface {
 	GetManifest(context.Context, *GetManifestRequest) (*Manifest, error)
 	GetServiceBanner(context.Context, *GetServiceBannerRequest) (*ServiceBanner, error)
@@ -150,6 +160,7 @@ type DRPCAgentServer interface {
 	BatchUpdateMetadata(context.Context, *BatchUpdateMetadataRequest) (*BatchUpdateMetadataResponse, error)
 	BatchCreateLogs(context.Context, *BatchCreateLogsRequest) (*BatchCreateLogsResponse, error)
 	GetAnnouncementBanners(context.Context, *GetAnnouncementBannersRequest) (*GetAnnouncementBannersResponse, error)
+	ScriptCompleted(context.Context, *WorkspaceAgentScriptCompletedRequest) (*WorkspaceAgentScriptCompletedResponse, error)
 }
 
 type DRPCAgentUnimplementedServer struct{}
@@ -190,9 +201,13 @@ func (s *DRPCAgentUnimplementedServer) GetAnnouncementBanners(context.Context, *
 	return nil, drpcerr.WithCode(errors.New("Unimplemented"), drpcerr.Unimplemented)
 }
 
+func (s *DRPCAgentUnimplementedServer) ScriptCompleted(context.Context, *WorkspaceAgentScriptCompletedRequest) (*WorkspaceAgentScriptCompletedResponse, error) {
+	return nil, drpcerr.WithCode(errors.New("Unimplemented"), drpcerr.Unimplemented)
+}
+
 type DRPCAgentDescription struct{}
 
-func (DRPCAgentDescription) NumMethods() int { return 9 }
+func (DRPCAgentDescription) NumMethods() int { return 10 }
 
 func (DRPCAgentDescription) Method(n int) (string, drpc.Encoding, drpc.Receiver, interface{}, bool) {
 	switch n {
@@ -277,6 +292,15 @@ func (DRPCAgentDescription) Method(n int) (string, drpc.Encoding, drpc.Receiver,
 						in1.(*GetAnnouncementBannersRequest),
 					)
 			}, DRPCAgentServer.GetAnnouncementBanners, true
+	case 9:
+		return "/coder.agent.v2.Agent/ScriptCompleted", drpcEncoding_File_agent_proto_agent_proto{},
+			func(srv interface{}, ctx context.Context, in1, in2 interface{}) (drpc.Message, error) {
+				return srv.(DRPCAgentServer).
+					ScriptCompleted(
+						ctx,
+						in1.(*WorkspaceAgentScriptCompletedRequest),
+					)
+			}, DRPCAgentServer.ScriptCompleted, true
 	default:
 		return "", nil, nil, nil, false
 	}
@@ -429,3 +453,19 @@ func (x *drpcAgent_GetAnnouncementBannersStream) SendAndClose(m *GetAnnouncement
 	}
 	return x.CloseSend()
 }
+
+type DRPCAgent_ScriptCompletedStream interface {
+	drpc.Stream
+	SendAndClose(*WorkspaceAgentScriptCompletedResponse) error
+}
+
+type drpcAgent_ScriptCompletedStream struct {
+	drpc.Stream
+}
+
+func (x *drpcAgent_ScriptCompletedStream) SendAndClose(m *WorkspaceAgentScriptCompletedResponse) error {
+	if err := x.MsgSend(m, drpcEncoding_File_agent_proto_agent_proto{}); err != nil {
+		return err
+	}
+	return x.CloseSend()
+}

agent/proto/agent_drpc_old.go

@@ -0,0 +1,42 @@
+package proto
+
+import (
+	"context"
+
+	"storj.io/drpc"
+)
+
+// DRPCAgentClient20 is the Agent API at v2.0.  Notably, it is missing GetAnnouncementBanners, but
+// is useful when you want to be maximally compatible with Coderd Release Versions from 2.9+
+type DRPCAgentClient20 interface {
+	DRPCConn() drpc.Conn
+
+	GetManifest(ctx context.Context, in *GetManifestRequest) (*Manifest, error)
+	GetServiceBanner(ctx context.Context, in *GetServiceBannerRequest) (*ServiceBanner, error)
+	UpdateStats(ctx context.Context, in *UpdateStatsRequest) (*UpdateStatsResponse, error)
+	UpdateLifecycle(ctx context.Context, in *UpdateLifecycleRequest) (*Lifecycle, error)
+	BatchUpdateAppHealths(ctx context.Context, in *BatchUpdateAppHealthRequest) (*BatchUpdateAppHealthResponse, error)
+	UpdateStartup(ctx context.Context, in *UpdateStartupRequest) (*Startup, error)
+	BatchUpdateMetadata(ctx context.Context, in *BatchUpdateMetadataRequest) (*BatchUpdateMetadataResponse, error)
+	BatchCreateLogs(ctx context.Context, in *BatchCreateLogsRequest) (*BatchCreateLogsResponse, error)
+}
+
+// DRPCAgentClient21 is the Agent API at v2.1. It is useful if you want to be maximally compatible
+// with Coderd Release Versions from 2.12+
+type DRPCAgentClient21 interface {
+	DRPCAgentClient20
+	GetAnnouncementBanners(ctx context.Context, in *GetAnnouncementBannersRequest) (*GetAnnouncementBannersResponse, error)
+}
+
+// DRPCAgentClient22 is the Agent API at v2.2. It is identical to 2.1, since the change was made on
+// the Tailnet API, which uses the same version number. Compatible with Coder v2.13+
+type DRPCAgentClient22 interface {
+	DRPCAgentClient21
+}
+
+// DRPCAgentClient23 is the Agent API at v2.3. It adds the ScriptCompleted RPC. Compatible with
+// Coder v2.18+
+type DRPCAgentClient23 interface {
+	DRPCAgentClient22
+	ScriptCompleted(ctx context.Context, in *WorkspaceAgentScriptCompletedRequest) (*WorkspaceAgentScriptCompletedResponse, error)
+}

agent/reconnectingpty/screen.go

@@ -67,8 +67,6 @@ func newScreen(ctx context.Context, cmd *pty.Cmd, options *Options, logger slog.
 		timeout: options.Timeout,
 	}
 
-	go rpty.lifecycle(ctx, logger)
-
 	// Socket paths are limited to around 100 characters on Linux and macOS which
 	// depending on the temporary directory can be a problem.  To give more leeway
 	// use a short ID.
@@ -124,6 +122,8 @@ func newScreen(ctx context.Context, cmd *pty.Cmd, options *Options, logger slog.
 		return rpty
 	}
 
+	go rpty.lifecycle(ctx, logger)
+
 	return rpty
 }
 

agent/reconnectingpty/server.go

@@ -0,0 +1,191 @@
+package reconnectingpty
+
+import (
+	"context"
+	"encoding/binary"
+	"encoding/json"
+	"net"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/prometheus/client_golang/prometheus"
+	"golang.org/x/xerrors"
+
+	"cdr.dev/slog"
+	"github.com/coder/coder/v2/agent/agentssh"
+	"github.com/coder/coder/v2/codersdk/workspacesdk"
+)
+
+type Server struct {
+	logger           slog.Logger
+	connectionsTotal prometheus.Counter
+	errorsTotal      *prometheus.CounterVec
+	commandCreator   *agentssh.Server
+	connCount        atomic.Int64
+	reconnectingPTYs sync.Map
+	timeout          time.Duration
+}
+
+// NewServer returns a new ReconnectingPTY server
+func NewServer(logger slog.Logger, commandCreator *agentssh.Server,
+	connectionsTotal prometheus.Counter, errorsTotal *prometheus.CounterVec,
+	timeout time.Duration,
+) *Server {
+	return &Server{
+		logger:           logger,
+		commandCreator:   commandCreator,
+		connectionsTotal: connectionsTotal,
+		errorsTotal:      errorsTotal,
+		timeout:          timeout,
+	}
+}
+
+func (s *Server) Serve(ctx, hardCtx context.Context, l net.Listener) (retErr error) {
+	var wg sync.WaitGroup
+	for {
+		if ctx.Err() != nil {
+			break
+		}
+		conn, err := l.Accept()
+		if err != nil {
+			s.logger.Debug(ctx, "accept pty failed", slog.Error(err))
+			retErr = err
+			break
+		}
+		clog := s.logger.With(
+			slog.F("remote", conn.RemoteAddr().String()),
+			slog.F("local", conn.LocalAddr().String()))
+		clog.Info(ctx, "accepted conn")
+		wg.Add(1)
+		closed := make(chan struct{})
+		go func() {
+			select {
+			case <-closed:
+			case <-hardCtx.Done():
+				_ = conn.Close()
+			}
+			wg.Done()
+		}()
+		wg.Add(1)
+		go func() {
+			defer close(closed)
+			defer wg.Done()
+			_ = s.handleConn(ctx, clog, conn)
+		}()
+	}
+	wg.Wait()
+	return retErr
+}
+
+func (s *Server) ConnCount() int64 {
+	return s.connCount.Load()
+}
+
+func (s *Server) handleConn(ctx context.Context, logger slog.Logger, conn net.Conn) (retErr error) {
+	defer conn.Close()
+	s.connectionsTotal.Add(1)
+	s.connCount.Add(1)
+	defer s.connCount.Add(-1)
+
+	// This cannot use a JSON decoder, since that can
+	// buffer additional data that is required for the PTY.
+	rawLen := make([]byte, 2)
+	_, err := conn.Read(rawLen)
+	if err != nil {
+		// logging at info since a single incident isn't too worrying (the client could just have
+		// hung up), but if we get a lot of these we'd want to investigate.
+		logger.Info(ctx, "failed to read AgentReconnectingPTYInit length", slog.Error(err))
+		return nil
+	}
+	length := binary.LittleEndian.Uint16(rawLen)
+	data := make([]byte, length)
+	_, err = conn.Read(data)
+	if err != nil {
+		// logging at info since a single incident isn't too worrying (the client could just have
+		// hung up), but if we get a lot of these we'd want to investigate.
+		logger.Info(ctx, "failed to read AgentReconnectingPTYInit", slog.Error(err))
+		return nil
+	}
+	var msg workspacesdk.AgentReconnectingPTYInit
+	err = json.Unmarshal(data, &msg)
+	if err != nil {
+		logger.Warn(ctx, "failed to unmarshal init", slog.F("raw", data))
+		return nil
+	}
+
+	connectionID := uuid.NewString()
+	connLogger := logger.With(slog.F("message_id", msg.ID), slog.F("connection_id", connectionID))
+	connLogger.Debug(ctx, "starting handler")
+
+	defer func() {
+		if err := retErr; err != nil {
+			// If the context is done, we don't want to log this as an error since it's expected.
+			if ctx.Err() != nil {
+				connLogger.Info(ctx, "reconnecting pty failed with attach error (agent closed)", slog.Error(err))
+			} else {
+				connLogger.Error(ctx, "reconnecting pty failed with attach error", slog.Error(err))
+			}
+		}
+		connLogger.Info(ctx, "reconnecting pty connection closed")
+	}()
+
+	var rpty ReconnectingPTY
+	sendConnected := make(chan ReconnectingPTY, 1)
+	// On store, reserve this ID to prevent multiple concurrent new connections.
+	waitReady, ok := s.reconnectingPTYs.LoadOrStore(msg.ID, sendConnected)
+	if ok {
+		close(sendConnected) // Unused.
+		connLogger.Debug(ctx, "connecting to existing reconnecting pty")
+		c, ok := waitReady.(chan ReconnectingPTY)
+		if !ok {
+			return xerrors.Errorf("found invalid type in reconnecting pty map: %T", waitReady)
+		}
+		rpty, ok = <-c
+		if !ok || rpty == nil {
+			return xerrors.Errorf("reconnecting pty closed before connection")
+		}
+		c <- rpty // Put it back for the next reconnect.
+	} else {
+		connLogger.Debug(ctx, "creating new reconnecting pty")
+
+		connected := false
+		defer func() {
+			if !connected && retErr != nil {
+				s.reconnectingPTYs.Delete(msg.ID)
+				close(sendConnected)
+			}
+		}()
+
+		// Empty command will default to the users shell!
+		cmd, err := s.commandCreator.CreateCommand(ctx, msg.Command, nil)
+		if err != nil {
+			s.errorsTotal.WithLabelValues("create_command").Add(1)
+			return xerrors.Errorf("create command: %w", err)
+		}
+
+		rpty = New(ctx, cmd, &Options{
+			Timeout: s.timeout,
+			Metrics: s.errorsTotal,
+		}, logger.With(slog.F("message_id", msg.ID)))
+
+		done := make(chan struct{})
+		go func() {
+			select {
+			case <-done:
+			case <-ctx.Done():
+				rpty.Close(ctx.Err())
+			}
+		}()
+
+		go func() {
+			rpty.Wait()
+			s.reconnectingPTYs.Delete(msg.ID)
+		}()
+
+		connected = true
+		sendConnected <- rpty
+	}
+	return rpty.Attach(ctx, connectionID, conn, msg.Height, msg.Width, connLogger)
+}

agent/stats_internal_test.go

@@ -18,7 +18,6 @@ import (
 
 	"cdr.dev/slog"
 	"cdr.dev/slog/sloggers/slogjson"
-	"cdr.dev/slog/sloggers/slogtest"
 	"github.com/coder/coder/v2/agent/proto"
 	"github.com/coder/coder/v2/testutil"
 )
@@ -26,7 +25,7 @@ import (
 func TestStatsReporter(t *testing.T) {
 	t.Parallel()
 	ctx := testutil.Context(t, testutil.WaitShort)
-	logger := slogtest.Make(t, nil).Leveled(slog.LevelDebug)
+	logger := testutil.Logger(t)
 	fSource := newFakeNetworkStatsSource(ctx, t)
 	fCollector := newFakeCollector(t)
 	fDest := newFakeStatsDest()

apiversion/apiversion.go

@@ -26,7 +26,7 @@ type APIVersion struct {
 }
 
 func (v *APIVersion) WithBackwardCompat(majs ...int) *APIVersion {
-	v.additionalMajors = append(v.additionalMajors, majs[:]...)
+	v.additionalMajors = append(v.additionalMajors, majs...)
 	return v
 }
 

archive/archive.go

@@ -1,4 +1,4 @@
-package coderd
+package archive
 
 import (
 	"archive/tar"
@@ -10,21 +10,22 @@ import (
 	"strings"
 )
 
-func CreateTarFromZip(zipReader *zip.Reader) ([]byte, error) {
+// CreateTarFromZip converts the given zipReader to a tar archive.
+func CreateTarFromZip(zipReader *zip.Reader, maxSize int64) ([]byte, error) {
 	var tarBuffer bytes.Buffer
-	err := writeTarArchive(&tarBuffer, zipReader)
+	err := writeTarArchive(&tarBuffer, zipReader, maxSize)
 	if err != nil {
 		return nil, err
 	}
 	return tarBuffer.Bytes(), nil
 }
 
-func writeTarArchive(w io.Writer, zipReader *zip.Reader) error {
+func writeTarArchive(w io.Writer, zipReader *zip.Reader, maxSize int64) error {
 	tarWriter := tar.NewWriter(w)
 	defer tarWriter.Close()
 
 	for _, file := range zipReader.File {
-		err := processFileInZipArchive(file, tarWriter)
+		err := processFileInZipArchive(file, tarWriter, maxSize)
 		if err != nil {
 			return err
 		}
@@ -32,7 +33,7 @@ func writeTarArchive(w io.Writer, zipReader *zip.Reader) error {
 	return nil
 }
 
-func processFileInZipArchive(file *zip.File, tarWriter *tar.Writer) error {
+func processFileInZipArchive(file *zip.File, tarWriter *tar.Writer, maxSize int64) error {
 	fileReader, err := file.Open()
 	if err != nil {
 		return err
@@ -52,7 +53,7 @@ func processFileInZipArchive(file *zip.File, tarWriter *tar.Writer) error {
 		return err
 	}
 
-	n, err := io.CopyN(tarWriter, fileReader, httpFileMaxBytes)
+	n, err := io.CopyN(tarWriter, fileReader, maxSize)
 	log.Println(file.Name, n, err)
 	if errors.Is(err, io.EOF) {
 		err = nil
@@ -60,16 +61,18 @@ func processFileInZipArchive(file *zip.File, tarWriter *tar.Writer) error {
 	return err
 }
 
-func CreateZipFromTar(tarReader *tar.Reader) ([]byte, error) {
+// CreateZipFromTar converts the given tarReader to a zip archive.
+func CreateZipFromTar(tarReader *tar.Reader, maxSize int64) ([]byte, error) {
 	var zipBuffer bytes.Buffer
-	err := WriteZipArchive(&zipBuffer, tarReader)
+	err := WriteZip(&zipBuffer, tarReader, maxSize)
 	if err != nil {
 		return nil, err
 	}
 	return zipBuffer.Bytes(), nil
 }
 
-func WriteZipArchive(w io.Writer, tarReader *tar.Reader) error {
+// WriteZip writes the given tarReader to w.
+func WriteZip(w io.Writer, tarReader *tar.Reader, maxSize int64) error {
 	zipWriter := zip.NewWriter(w)
 	defer zipWriter.Close()
 
@@ -100,7 +103,7 @@ func WriteZipArchive(w io.Writer, tarReader *tar.Reader) error {
 			return err
 		}
 
-		_, err = io.CopyN(zipEntry, tarReader, httpFileMaxBytes)
+		_, err = io.CopyN(zipEntry, tarReader, maxSize)
 		if errors.Is(err, io.EOF) {
 			err = nil
 		}

archive/archive_test.go

@@ -1,10 +1,9 @@
-package coderd_test
+package archive_test
 
 import (
 	"archive/tar"
 	"archive/zip"
 	"bytes"
-	"io"
 	"io/fs"
 	"os"
 	"os/exec"
@@ -12,13 +11,12 @@ import (
 	"runtime"
 	"strings"
 	"testing"
-	"time"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-	"golang.org/x/xerrors"
 
-	"github.com/coder/coder/v2/coderd"
+	"github.com/coder/coder/v2/archive"
+	"github.com/coder/coder/v2/archive/archivetest"
 	"github.com/coder/coder/v2/testutil"
 )
 
@@ -30,18 +28,17 @@ func TestCreateTarFromZip(t *testing.T) {
 
 	// Read a zip file we prepared earlier
 	ctx := testutil.Context(t, testutil.WaitShort)
-	zipBytes, err := os.ReadFile(filepath.Join("testdata", "test.zip"))
-	require.NoError(t, err, "failed to read sample zip file")
+	zipBytes := archivetest.TestZipFileBytes()
 	// Assert invariant
-	assertSampleZipFile(t, zipBytes)
+	archivetest.AssertSampleZipFile(t, zipBytes)
 
 	zr, err := zip.NewReader(bytes.NewReader(zipBytes), int64(len(zipBytes)))
 	require.NoError(t, err, "failed to parse sample zip file")
 
-	tarBytes, err := coderd.CreateTarFromZip(zr)
+	tarBytes, err := archive.CreateTarFromZip(zr, int64(len(zipBytes)))
 	require.NoError(t, err, "failed to convert zip to tar")
 
-	assertSampleTarFile(t, tarBytes)
+	archivetest.AssertSampleTarFile(t, tarBytes)
 
 	tempDir := t.TempDir()
 	tempFilePath := filepath.Join(tempDir, "test.tar")
@@ -60,14 +57,13 @@ func TestCreateZipFromTar(t *testing.T) {
 	}
 	t.Run("OK", func(t *testing.T) {
 		t.Parallel()
-		tarBytes, err := os.ReadFile(filepath.Join(".", "testdata", "test.tar"))
-		require.NoError(t, err, "failed to read sample tar file")
+		tarBytes := archivetest.TestTarFileBytes()
 
 		tr := tar.NewReader(bytes.NewReader(tarBytes))
-		zipBytes, err := coderd.CreateZipFromTar(tr)
+		zipBytes, err := archive.CreateZipFromTar(tr, int64(len(tarBytes)))
 		require.NoError(t, err)
 
-		assertSampleZipFile(t, zipBytes)
+		archivetest.AssertSampleZipFile(t, zipBytes)
 
 		tempDir := t.TempDir()
 		tempFilePath := filepath.Join(tempDir, "test.zip")
@@ -99,7 +95,7 @@ func TestCreateZipFromTar(t *testing.T) {
 
 		// When: we convert this to a zip
 		tr := tar.NewReader(&tarBytes)
-		zipBytes, err := coderd.CreateZipFromTar(tr)
+		zipBytes, err := archive.CreateZipFromTar(tr, int64(tarBytes.Len()))
 		require.NoError(t, err)
 
 		// Then: the resulting zip should contain a corresponding directory
@@ -133,7 +129,7 @@ func assertExtractedFiles(t *testing.T, dir string, checkModePerm bool) {
 			if checkModePerm {
 				assert.Equal(t, fs.ModePerm&0o755, stat.Mode().Perm(), "expected mode 0755 on directory")
 			}
-			assert.Equal(t, archiveRefTime(t).UTC(), stat.ModTime().UTC(), "unexpected modtime of %q", path)
+			assert.Equal(t, archivetest.ArchiveRefTime(t).UTC(), stat.ModTime().UTC(), "unexpected modtime of %q", path)
 		case "/test/hello.txt":
 			stat, err := os.Stat(path)
 			assert.NoError(t, err, "failed to stat path %q", path)
@@ -168,84 +164,3 @@ func assertExtractedFiles(t *testing.T, dir string, checkModePerm bool) {
 		return nil
 	})
 }
-
-func assertSampleTarFile(t *testing.T, tarBytes []byte) {
-	t.Helper()
-
-	tr := tar.NewReader(bytes.NewReader(tarBytes))
-	for {
-		hdr, err := tr.Next()
-		if err != nil {
-			if err == io.EOF {
-				return
-			}
-			require.NoError(t, err)
-		}
-
-		// Note: ignoring timezones here.
-		require.Equal(t, archiveRefTime(t).UTC(), hdr.ModTime.UTC())
-
-		switch hdr.Name {
-		case "test/":
-			require.Equal(t, hdr.Typeflag, byte(tar.TypeDir))
-		case "test/hello.txt":
-			require.Equal(t, hdr.Typeflag, byte(tar.TypeReg))
-			bs, err := io.ReadAll(tr)
-			if err != nil && !xerrors.Is(err, io.EOF) {
-				require.NoError(t, err)
-			}
-			require.Equal(t, "hello", string(bs))
-		case "test/dir/":
-			require.Equal(t, hdr.Typeflag, byte(tar.TypeDir))
-		case "test/dir/world.txt":
-			require.Equal(t, hdr.Typeflag, byte(tar.TypeReg))
-			bs, err := io.ReadAll(tr)
-			if err != nil && !xerrors.Is(err, io.EOF) {
-				require.NoError(t, err)
-			}
-			require.Equal(t, "world", string(bs))
-		default:
-			require.Failf(t, "unexpected file in tar", hdr.Name)
-		}
-	}
-}
-
-func assertSampleZipFile(t *testing.T, zipBytes []byte) {
-	t.Helper()
-
-	zr, err := zip.NewReader(bytes.NewReader(zipBytes), int64(len(zipBytes)))
-	require.NoError(t, err)
-
-	for _, f := range zr.File {
-		// Note: ignoring timezones here.
-		require.Equal(t, archiveRefTime(t).UTC(), f.Modified.UTC())
-		switch f.Name {
-		case "test/", "test/dir/":
-			// directory
-		case "test/hello.txt":
-			rc, err := f.Open()
-			require.NoError(t, err)
-			bs, err := io.ReadAll(rc)
-			_ = rc.Close()
-			require.NoError(t, err)
-			require.Equal(t, "hello", string(bs))
-		case "test/dir/world.txt":
-			rc, err := f.Open()
-			require.NoError(t, err)
-			bs, err := io.ReadAll(rc)
-			_ = rc.Close()
-			require.NoError(t, err)
-			require.Equal(t, "world", string(bs))
-		default:
-			require.Failf(t, "unexpected file in zip", f.Name)
-		}
-	}
-}
-
-// archiveRefTime is the Go reference time. The contents of the sample tar and zip files
-// in testdata/ all have their modtimes set to the below in some timezone.
-func archiveRefTime(t *testing.T) time.Time {
-	locMST, err := time.LoadLocation("MST")
-	require.NoError(t, err, "failed to load MST timezone")
-	return time.Date(2006, 1, 2, 3, 4, 5, 0, locMST)
-}

archive/archivetest/archivetest.go

@@ -0,0 +1,113 @@
+package archivetest
+
+import (
+	"archive/tar"
+	"archive/zip"
+	"bytes"
+	_ "embed"
+	"io"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+	"golang.org/x/xerrors"
+)
+
+//go:embed testdata/test.tar
+var testTarFileBytes []byte
+
+//go:embed testdata/test.zip
+var testZipFileBytes []byte
+
+// TestTarFileBytes returns the content of testdata/test.tar
+func TestTarFileBytes() []byte {
+	return append([]byte{}, testTarFileBytes...)
+}
+
+// TestZipFileBytes returns the content of testdata/test.zip
+func TestZipFileBytes() []byte {
+	return append([]byte{}, testZipFileBytes...)
+}
+
+// AssertSampleTarfile compares the content of tarBytes against testdata/test.tar.
+func AssertSampleTarFile(t *testing.T, tarBytes []byte) {
+	t.Helper()
+
+	tr := tar.NewReader(bytes.NewReader(tarBytes))
+	for {
+		hdr, err := tr.Next()
+		if err != nil {
+			if err == io.EOF {
+				return
+			}
+			require.NoError(t, err)
+		}
+
+		// Note: ignoring timezones here.
+		require.Equal(t, ArchiveRefTime(t).UTC(), hdr.ModTime.UTC())
+
+		switch hdr.Name {
+		case "test/":
+			require.Equal(t, hdr.Typeflag, byte(tar.TypeDir))
+		case "test/hello.txt":
+			require.Equal(t, hdr.Typeflag, byte(tar.TypeReg))
+			bs, err := io.ReadAll(tr)
+			if err != nil && !xerrors.Is(err, io.EOF) {
+				require.NoError(t, err)
+			}
+			require.Equal(t, "hello", string(bs))
+		case "test/dir/":
+			require.Equal(t, hdr.Typeflag, byte(tar.TypeDir))
+		case "test/dir/world.txt":
+			require.Equal(t, hdr.Typeflag, byte(tar.TypeReg))
+			bs, err := io.ReadAll(tr)
+			if err != nil && !xerrors.Is(err, io.EOF) {
+				require.NoError(t, err)
+			}
+			require.Equal(t, "world", string(bs))
+		default:
+			require.Failf(t, "unexpected file in tar", hdr.Name)
+		}
+	}
+}
+
+// AssertSampleZipFile compares the content of zipBytes against testdata/test.zip.
+func AssertSampleZipFile(t *testing.T, zipBytes []byte) {
+	t.Helper()
+
+	zr, err := zip.NewReader(bytes.NewReader(zipBytes), int64(len(zipBytes)))
+	require.NoError(t, err)
+
+	for _, f := range zr.File {
+		// Note: ignoring timezones here.
+		require.Equal(t, ArchiveRefTime(t).UTC(), f.Modified.UTC())
+		switch f.Name {
+		case "test/", "test/dir/":
+			// directory
+		case "test/hello.txt":
+			rc, err := f.Open()
+			require.NoError(t, err)
+			bs, err := io.ReadAll(rc)
+			_ = rc.Close()
+			require.NoError(t, err)
+			require.Equal(t, "hello", string(bs))
+		case "test/dir/world.txt":
+			rc, err := f.Open()
+			require.NoError(t, err)
+			bs, err := io.ReadAll(rc)
+			_ = rc.Close()
+			require.NoError(t, err)
+			require.Equal(t, "world", string(bs))
+		default:
+			require.Failf(t, "unexpected file in zip", f.Name)
+		}
+	}
+}
+
+// archiveRefTime is the Go reference time. The contents of the sample tar and zip files
+// in testdata/ all have their modtimes set to the below in some timezone.
+func ArchiveRefTime(t *testing.T) time.Time {
+	locMST, err := time.LoadLocation("MST")
+	require.NoError(t, err, "failed to load MST timezone")
+	return time.Date(2006, 1, 2, 3, 4, 5, 0, locMST)
+}

buildinfo/buildinfo.go

@@ -24,6 +24,9 @@ var (
 	// Updated by buildinfo_slim.go on start.
 	slim bool
 
+	// Updated by buildinfo_site.go on start.
+	site bool
+
 	// Injected with ldflags at build, see scripts/build_go.sh
 	tag  string
 	agpl string // either "true" or "false", ldflags does not support bools
@@ -95,6 +98,11 @@ func IsSlim() bool {
 	return slim
 }
 
+// HasSite returns true if the frontend is embedded in the build.
+func HasSite() bool {
+	return site
+}
+
 // IsAGPL returns true if this is an AGPL build.
 func IsAGPL() bool {
 	return strings.Contains(agpl, "t")

buildinfo/buildinfo_site.go

@@ -0,0 +1,7 @@
+//go:build embed
+
+package buildinfo
+
+func init() {
+	site = true
+}

cli/agent.go

@@ -12,7 +12,6 @@ import (
 	"runtime"
 	"strconv"
 	"strings"
-	"sync"
 	"time"
 
 	"cloud.google.com/go/compute/metadata"
@@ -27,8 +26,10 @@ import (
 	"cdr.dev/slog/sloggers/slogstackdriver"
 	"github.com/coder/coder/v2/agent"
 	"github.com/coder/coder/v2/agent/agentproc"
+	"github.com/coder/coder/v2/agent/agentssh"
 	"github.com/coder/coder/v2/agent/reaper"
 	"github.com/coder/coder/v2/buildinfo"
+	"github.com/coder/coder/v2/cli/clilog"
 	"github.com/coder/coder/v2/codersdk"
 	"github.com/coder/coder/v2/codersdk/agentsdk"
 	"github.com/coder/serpent"
@@ -48,6 +49,9 @@ func (r *RootCmd) workspaceAgent() *serpent.Command {
 		slogHumanPath       string
 		slogJSONPath        string
 		slogStackdriverPath string
+		blockFileTransfer   bool
+		agentHeaderCommand  string
+		agentHeader         []string
 	)
 	cmd := &serpent.Command{
 		Use:   "agent",
@@ -106,7 +110,7 @@ func (r *RootCmd) workspaceAgent() *serpent.Command {
 			// Spawn a reaper so that we don't accumulate a ton
 			// of zombie processes.
 			if reaper.IsInitProcess() && !noReap && isLinux {
-				logWriter := &lumberjackWriteCloseFixer{w: &lumberjack.Logger{
+				logWriter := &clilog.LumberjackWriteCloseFixer{Writer: &lumberjack.Logger{
 					Filename: filepath.Join(logDir, "coder-agent-init.log"),
 					MaxSize:  5, // MB
 					// Without this, rotated logs will never be deleted.
@@ -149,7 +153,7 @@ func (r *RootCmd) workspaceAgent() *serpent.Command {
 			// reaper.
 			go DumpHandler(ctx, "agent")
 
-			logWriter := &lumberjackWriteCloseFixer{w: &lumberjack.Logger{
+			logWriter := &clilog.LumberjackWriteCloseFixer{Writer: &lumberjack.Logger{
 				Filename: filepath.Join(logDir, "coder-agent.log"),
 				MaxSize:  5, // MB
 				// Per customer incident on November 17th, 2023, its helpful
@@ -174,6 +178,14 @@ func (r *RootCmd) workspaceAgent() *serpent.Command {
 			// with large payloads can take a bit. e.g. startup scripts
 			// may take a while to insert.
 			client.SDK.HTTPClient.Timeout = 30 * time.Second
+			// Attach header transport so we process --agent-header and
+			// --agent-header-command flags
+			headerTransport, err := headerTransport(ctx, r.agentURL, agentHeader, agentHeaderCommand)
+			if err != nil {
+				return xerrors.Errorf("configure header transport: %w", err)
+			}
+			headerTransport.Transport = client.SDK.HTTPClient.Transport
+			client.SDK.HTTPClient.Transport = headerTransport
 
 			// Enable pprof handler
 			// This prevents the pprof import from being accidentally deleted.
@@ -314,6 +326,8 @@ func (r *RootCmd) workspaceAgent() *serpent.Command {
 				// Intentionally set this to nil. It's mainly used
 				// for testing.
 				ModifiedProcesses: nil,
+
+				BlockFileTransfer: blockFileTransfer,
 			})
 
 			promHandler := agent.PrometheusMetricsHandler(prometheusRegistry, logger)
@@ -357,6 +371,18 @@ func (r *RootCmd) workspaceAgent() *serpent.Command {
 			Value:       serpent.StringOf(&pprofAddress),
 			Description: "The address to serve pprof.",
 		},
+		{
+			Flag:        "agent-header-command",
+			Env:         "CODER_AGENT_HEADER_COMMAND",
+			Value:       serpent.StringOf(&agentHeaderCommand),
+			Description: "An external command that outputs additional HTTP headers added to all requests. The command must output each header as `key=value` on its own line.",
+		},
+		{
+			Flag:        "agent-header",
+			Env:         "CODER_AGENT_HEADER",
+			Value:       serpent.StringArrayOf(&agentHeader),
+			Description: "Additional HTTP headers added to all requests. Provide as " + `key=value` + ". Can be specified multiple times.",
+		},
 		{
 			Flag: "no-reap",
 
@@ -417,6 +443,13 @@ func (r *RootCmd) workspaceAgent() *serpent.Command {
 			Default:     "",
 			Value:       serpent.StringOf(&slogStackdriverPath),
 		},
+		{
+			Flag:        "block-file-transfer",
+			Default:     "false",
+			Env:         "CODER_AGENT_BLOCK_FILE_TRANSFER",
+			Description: fmt.Sprintf("Block file transfer using known applications: %s.", strings.Join(agentssh.BlockedFileTransferCommands, ",")),
+			Value:       serpent.BoolOf(&blockFileTransfer),
+		},
 	}
 
 	return cmd
@@ -445,33 +478,6 @@ func ServeHandler(ctx context.Context, logger slog.Logger, handler http.Handler,
 	}
 }
 
-// lumberjackWriteCloseFixer is a wrapper around an io.WriteCloser that
-// prevents writes after Close. This is necessary because lumberjack
-// re-opens the file on Write.
-type lumberjackWriteCloseFixer struct {
-	w      io.WriteCloser
-	mu     sync.Mutex // Protects following.
-	closed bool
-}
-
-func (c *lumberjackWriteCloseFixer) Close() error {
-	c.mu.Lock()
-	defer c.mu.Unlock()
-
-	c.closed = true
-	return c.w.Close()
-}
-
-func (c *lumberjackWriteCloseFixer) Write(p []byte) (int, error) {
-	c.mu.Lock()
-	defer c.mu.Unlock()
-
-	if c.closed {
-		return 0, io.ErrClosedPipe
-	}
-	return c.w.Write(p)
-}
-
 // extractPort handles different url strings.
 // - localhost:6060
 // - http://localhost:6060

cli/agent_test.go

@@ -3,10 +3,12 @@ package cli_test
 import (
 	"context"
 	"fmt"
+	"net/http"
 	"os"
 	"path/filepath"
 	"runtime"
 	"strings"
+	"sync/atomic"
 	"testing"
 
 	"github.com/google/uuid"
@@ -15,6 +17,7 @@ import (
 
 	"github.com/coder/coder/v2/agent"
 	"github.com/coder/coder/v2/cli/clitest"
+	"github.com/coder/coder/v2/coderd"
 	"github.com/coder/coder/v2/coderd/coderdtest"
 	"github.com/coder/coder/v2/coderd/database"
 	"github.com/coder/coder/v2/coderd/database/dbfake"
@@ -32,7 +35,7 @@ func TestWorkspaceAgent(t *testing.T) {
 
 		client, db := coderdtest.NewWithDatabase(t, nil)
 		user := coderdtest.CreateFirstUser(t, client)
-		r := dbfake.WorkspaceBuild(t, db, database.Workspace{
+		r := dbfake.WorkspaceBuild(t, db, database.WorkspaceTable{
 			OrganizationID: user.OrganizationID,
 			OwnerID:        user.UserID,
 		}).
@@ -68,7 +71,7 @@ func TestWorkspaceAgent(t *testing.T) {
 			AzureCertificates: certificates,
 		})
 		user := coderdtest.CreateFirstUser(t, client)
-		r := dbfake.WorkspaceBuild(t, db, database.Workspace{
+		r := dbfake.WorkspaceBuild(t, db, database.WorkspaceTable{
 			OrganizationID: user.OrganizationID,
 			OwnerID:        user.UserID,
 		}).WithAgent(func(agents []*proto.Agent) []*proto.Agent {
@@ -107,7 +110,7 @@ func TestWorkspaceAgent(t *testing.T) {
 			AWSCertificates: certificates,
 		})
 		user := coderdtest.CreateFirstUser(t, client)
-		r := dbfake.WorkspaceBuild(t, db, database.Workspace{
+		r := dbfake.WorkspaceBuild(t, db, database.WorkspaceTable{
 			OrganizationID: user.OrganizationID,
 			OwnerID:        user.UserID,
 		}).WithAgent(func(agents []*proto.Agent) []*proto.Agent {
@@ -148,7 +151,7 @@ func TestWorkspaceAgent(t *testing.T) {
 		})
 		owner := coderdtest.CreateFirstUser(t, client)
 		member, memberUser := coderdtest.CreateAnotherUser(t, client, owner.OrganizationID)
-		r := dbfake.WorkspaceBuild(t, db, database.Workspace{
+		r := dbfake.WorkspaceBuild(t, db, database.WorkspaceTable{
 			OrganizationID: owner.OrganizationID,
 			OwnerID:        memberUser.ID,
 		}).WithAgent(func(agents []*proto.Agent) []*proto.Agent {
@@ -202,7 +205,7 @@ func TestWorkspaceAgent(t *testing.T) {
 
 		client, db := coderdtest.NewWithDatabase(t, nil)
 		user := coderdtest.CreateFirstUser(t, client)
-		r := dbfake.WorkspaceBuild(t, db, database.Workspace{
+		r := dbfake.WorkspaceBuild(t, db, database.WorkspaceTable{
 			OrganizationID: user.OrganizationID,
 			OwnerID:        user.UserID,
 		}).WithAgent().Do()
@@ -229,6 +232,93 @@ func TestWorkspaceAgent(t *testing.T) {
 		require.Equal(t, codersdk.AgentSubsystemEnvbox, resources[0].Agents[0].Subsystems[0])
 		require.Equal(t, codersdk.AgentSubsystemExectrace, resources[0].Agents[0].Subsystems[1])
 	})
+	t.Run("Headers&DERPHeaders", func(t *testing.T) {
+		t.Parallel()
+
+		// Create a coderd API instance the hard way since we need to change the
+		// handler to inject our custom /derp handler.
+		dv := coderdtest.DeploymentValues(t)
+		dv.DERP.Config.BlockDirect = true
+		setHandler, cancelFunc, serverURL, newOptions := coderdtest.NewOptions(t, &coderdtest.Options{
+			DeploymentValues: dv,
+		})
+
+		// We set the handler after server creation for the access URL.
+		coderAPI := coderd.New(newOptions)
+		setHandler(coderAPI.RootHandler)
+		provisionerCloser := coderdtest.NewProvisionerDaemon(t, coderAPI)
+		t.Cleanup(func() {
+			_ = provisionerCloser.Close()
+		})
+		client := codersdk.New(serverURL)
+		t.Cleanup(func() {
+			cancelFunc()
+			_ = provisionerCloser.Close()
+			_ = coderAPI.Close()
+			client.HTTPClient.CloseIdleConnections()
+		})
+
+		var (
+			admin              = coderdtest.CreateFirstUser(t, client)
+			member, memberUser = coderdtest.CreateAnotherUser(t, client, admin.OrganizationID)
+			called             int64
+			derpCalled         int64
+		)
+
+		setHandler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			// Ignore client requests
+			if r.Header.Get("X-Testing") == "agent" {
+				assert.Equal(t, "Ethan was Here!", r.Header.Get("Cool-Header"))
+				assert.Equal(t, "very-wow-"+client.URL.String(), r.Header.Get("X-Process-Testing"))
+				assert.Equal(t, "more-wow", r.Header.Get("X-Process-Testing2"))
+				if strings.HasPrefix(r.URL.Path, "/derp") {
+					atomic.AddInt64(&derpCalled, 1)
+				} else {
+					atomic.AddInt64(&called, 1)
+				}
+			}
+			coderAPI.RootHandler.ServeHTTP(w, r)
+		}))
+		r := dbfake.WorkspaceBuild(t, coderAPI.Database, database.WorkspaceTable{
+			OrganizationID: memberUser.OrganizationIDs[0],
+			OwnerID:        memberUser.ID,
+		}).WithAgent().Do()
+
+		coderURLEnv := "$CODER_URL"
+		if runtime.GOOS == "windows" {
+			coderURLEnv = "%CODER_URL%"
+		}
+
+		logDir := t.TempDir()
+		agentInv, _ := clitest.New(t,
+			"agent",
+			"--auth", "token",
+			"--agent-token", r.AgentToken,
+			"--agent-url", client.URL.String(),
+			"--log-dir", logDir,
+			"--agent-header", "X-Testing=agent",
+			"--agent-header", "Cool-Header=Ethan was Here!",
+			"--agent-header-command", "printf X-Process-Testing=very-wow-"+coderURLEnv+"'\\r\\n'X-Process-Testing2=more-wow",
+		)
+		clitest.Start(t, agentInv)
+		coderdtest.NewWorkspaceAgentWaiter(t, client, r.Workspace.ID).
+			MatchResources(matchAgentWithVersion).Wait()
+
+		ctx := testutil.Context(t, testutil.WaitLong)
+		clientInv, root := clitest.New(t,
+			"-v",
+			"--no-feature-warning",
+			"--no-version-warning",
+			"ping", r.Workspace.Name,
+			"-n", "1",
+		)
+		clitest.SetupConfig(t, member, root)
+		err := clientInv.WithContext(ctx).Run()
+		require.NoError(t, err)
+
+		require.Greater(t, atomic.LoadInt64(&called), int64(0), "expected coderd to be reached with custom headers")
+		require.Greater(t, atomic.LoadInt64(&derpCalled), int64(0), "expected /derp to be called with custom headers")
+	})
 }
 
 func matchAgentWithVersion(rs []codersdk.WorkspaceResource) bool {

cli/autoupdate_test.go

@@ -24,7 +24,7 @@ func TestAutoUpdate(t *testing.T) {
 		version := coderdtest.CreateTemplateVersion(t, client, owner.OrganizationID, nil)
 		coderdtest.AwaitTemplateVersionJobCompleted(t, client, version.ID)
 		template := coderdtest.CreateTemplate(t, client, owner.OrganizationID, version.ID)
-		workspace := coderdtest.CreateWorkspace(t, member, owner.OrganizationID, template.ID)
+		workspace := coderdtest.CreateWorkspace(t, member, template.ID)
 		coderdtest.AwaitWorkspaceBuildJobCompleted(t, client, workspace.LatestBuild.ID)
 		require.Equal(t, codersdk.AutomaticUpdatesNever, workspace.AutomaticUpdates)
 

cli/clilog/clilog.go

@@ -4,11 +4,12 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"os"
 	"regexp"
 	"strings"
+	"sync"
 
 	"golang.org/x/xerrors"
+	"gopkg.in/natefinch/lumberjack.v2"
 
 	"cdr.dev/slog"
 	"cdr.dev/slog/sloggers/sloghuman"
@@ -104,7 +105,6 @@ func (b *Builder) Build(inv *serpent.Invocation) (log slog.Logger, closeLog func
 	addSinkIfProvided := func(sinkFn func(io.Writer) slog.Sink, loc string) error {
 		switch loc {
 		case "":
-
 		case "/dev/stdout":
 			sinks = append(sinks, sinkFn(inv.Stdout))
 
@@ -112,12 +112,14 @@ func (b *Builder) Build(inv *serpent.Invocation) (log slog.Logger, closeLog func
 			sinks = append(sinks, sinkFn(inv.Stderr))
 
 		default:
-			fi, err := os.OpenFile(loc, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0o644)
-			if err != nil {
-				return xerrors.Errorf("open log file %q: %w", loc, err)
-			}
-			closers = append(closers, fi.Close)
-			sinks = append(sinks, sinkFn(fi))
+			logWriter := &LumberjackWriteCloseFixer{Writer: &lumberjack.Logger{
+				Filename: loc,
+				MaxSize:  5, // MB
+				// Without this, rotated logs will never be deleted.
+				MaxBackups: 1,
+			}}
+			closers = append(closers, logWriter.Close)
+			sinks = append(sinks, sinkFn(logWriter))
 		}
 		return nil
 	}
@@ -209,3 +211,30 @@ func (f *debugFilterSink) Sync() {
 		sink.Sync()
 	}
 }
+
+// LumberjackWriteCloseFixer is a wrapper around an io.WriteCloser that
+// prevents writes after Close. This is necessary because lumberjack
+// re-opens the file on Write.
+type LumberjackWriteCloseFixer struct {
+	Writer io.WriteCloser
+	mu     sync.Mutex // Protects following.
+	closed bool
+}
+
+func (c *LumberjackWriteCloseFixer) Close() error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	c.closed = true
+	return c.Writer.Close()
+}
+
+func (c *LumberjackWriteCloseFixer) Write(p []byte) (int, error) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	if c.closed {
+		return 0, io.ErrClosedPipe
+	}
+	return c.Writer.Write(p)
+}

cli/clilog/clilog_test.go

@@ -2,7 +2,6 @@ package clilog_test
 
 import (
 	"encoding/json"
-	"io/fs"
 	"os"
 	"path/filepath"
 	"strings"
@@ -145,30 +144,6 @@ func TestBuilder(t *testing.T) {
 			assertLogsJSON(t, tempJSON, info, infoLog, warn, warnLog)
 		})
 	})
-
-	t.Run("NotFound", func(t *testing.T) {
-		t.Parallel()
-
-		tempFile := filepath.Join(t.TempDir(), "doesnotexist", "test.log")
-		cmd := &serpent.Command{
-			Use: "test",
-			Handler: func(inv *serpent.Invocation) error {
-				logger, closeLog, err := clilog.New(
-					clilog.WithFilter("foo", "baz"),
-					clilog.WithHuman(tempFile),
-					clilog.WithVerbose(),
-				).Build(inv)
-				if err != nil {
-					return err
-				}
-				defer closeLog()
-				logger.Error(inv.Context(), "you will never see this")
-				return nil
-			},
-		}
-		err := cmd.Invoke().Run()
-		require.ErrorIs(t, err, fs.ErrNotExist)
-	})
 }
 
 var (

cli/clistat/container.go

@@ -12,6 +12,7 @@ import (
 const (
 	procMounts                           = "/proc/mounts"
 	procOneCgroup                        = "/proc/1/cgroup"
+	sysCgroupType                        = "/sys/fs/cgroup/cgroup.type"
 	kubernetesDefaultServiceAccountToken = "/var/run/secrets/kubernetes.io/serviceaccount/token" //nolint:gosec
 )
 
@@ -65,6 +66,17 @@ func IsContainerized(fs afero.Fs) (ok bool, err error) {
 		}
 	}
 
+	// Adapted from https://github.com/systemd/systemd/blob/88bbf187a9b2ebe0732caa1e886616ae5f8186da/src/basic/virt.c#L603-L605
+	// The file `/sys/fs/cgroup/cgroup.type` does not exist on the root cgroup.
+	// If this file exists we can be sure we're in a container.
+	cgTypeExists, err := afero.Exists(fs, sysCgroupType)
+	if err != nil {
+		return false, xerrors.Errorf("check file exists %s: %w", sysCgroupType, err)
+	}
+	if cgTypeExists {
+		return true, nil
+	}
+
 	// If we get here, we are _probably_ not running in a container.
 	return false, nil
 }

cli/clistat/stat_internal_test.go

@@ -309,6 +309,12 @@ func TestIsContainerized(t *testing.T) {
 			Expected: true,
 			Error:    "",
 		},
+		{
+			Name:     "Docker (Cgroupns=private)",
+			FS:       fsContainerCgroupV2PrivateCgroupns,
+			Expected: true,
+			Error:    "",
+		},
 	} {
 		tt := tt
 		t.Run(tt.Name, func(t *testing.T) {
@@ -374,6 +380,12 @@ proc /proc/sys proc ro,nosuid,nodev,noexec,relatime 0 0`,
 		cgroupV2MemoryUsageBytes: "536870912",
 		cgroupV2MemoryStat:       "inactive_file 268435456",
 	}
+	fsContainerCgroupV2PrivateCgroupns = map[string]string{
+		procOneCgroup: "0::/",
+		procMounts: `overlay / overlay rw,relatime,lowerdir=/some/path:/some/path,upperdir=/some/path:/some/path,workdir=/some/path:/some/path 0 0
+proc /proc/sys proc ro,nosuid,nodev,noexec,relatime 0 0`,
+		sysCgroupType: "domain",
+	}
 	fsContainerCgroupV1 = map[string]string{
 		procOneCgroup: "0::/docker/aa86ac98959eeedeae0ecb6e0c9ddd8ae8b97a9d0fdccccf7ea7a474f4e0bb1f",
 		procMounts: `overlay / overlay rw,relatime,lowerdir=/some/path:/some/path,upperdir=/some/path:/some/path,workdir=/some/path:/some/path 0 0

cli/clitest/golden.go

@@ -11,6 +11,7 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/google/uuid"
 	"github.com/stretchr/testify/require"
 
 	"github.com/coder/coder/v2/cli/config"
@@ -183,11 +184,11 @@ func prepareTestData(t *testing.T) (*codersdk.Client, map[string]string) {
 		IncludeProvisionerDaemon: true,
 	})
 	firstUser := coderdtest.CreateFirstUser(t, rootClient)
-	secondUser, err := rootClient.CreateUser(ctx, codersdk.CreateUserRequest{
-		Email:          "testuser2@coder.com",
-		Username:       "testuser2",
-		Password:       coderdtest.FirstUserParams.Password,
-		OrganizationID: firstUser.OrganizationID,
+	secondUser, err := rootClient.CreateUserWithOrgs(ctx, codersdk.CreateUserRequestWithOrgs{
+		Email:           "testuser2@coder.com",
+		Username:        "testuser2",
+		Password:        coderdtest.FirstUserParams.Password,
+		OrganizationIDs: []uuid.UUID{firstUser.OrganizationID},
 	})
 	require.NoError(t, err)
 	version := coderdtest.CreateTemplateVersion(t, rootClient, firstUser.OrganizationID, nil)
@@ -195,7 +196,7 @@ func prepareTestData(t *testing.T) (*codersdk.Client, map[string]string) {
 	template := coderdtest.CreateTemplate(t, rootClient, firstUser.OrganizationID, version.ID, func(req *codersdk.CreateTemplateRequest) {
 		req.Name = "test-template"
 	})
-	workspace := coderdtest.CreateWorkspace(t, rootClient, firstUser.OrganizationID, template.ID, func(req *codersdk.CreateWorkspaceRequest) {
+	workspace := coderdtest.CreateWorkspace(t, rootClient, template.ID, func(req *codersdk.CreateWorkspaceRequest) {
 		req.Name = "test-workspace"
 	})
 	workspaceBuild := coderdtest.AwaitWorkspaceBuildJobCompleted(t, rootClient, workspace.LatestBuild.ID)

cli/cliui/agent.go

@@ -10,8 +10,11 @@ import (
 
 	"github.com/google/uuid"
 	"golang.org/x/xerrors"
+	"tailscale.com/tailcfg"
 
 	"github.com/coder/coder/v2/codersdk"
+	"github.com/coder/coder/v2/codersdk/healthsdk"
+	"github.com/coder/coder/v2/codersdk/workspacesdk"
 	"github.com/coder/coder/v2/tailnet"
 )
 
@@ -22,6 +25,7 @@ type AgentOptions struct {
 	Fetch         func(ctx context.Context, agentID uuid.UUID) (codersdk.WorkspaceAgent, error)
 	FetchLogs     func(ctx context.Context, agentID uuid.UUID, after int64, follow bool) (<-chan []codersdk.WorkspaceAgentLog, io.Closer, error)
 	Wait          bool // If true, wait for the agent to be ready (startup script).
+	DocsURL       string
 }
 
 // Agent displays a spinning indicator that waits for a workspace agent to connect.
@@ -116,7 +120,7 @@ func Agent(ctx context.Context, writer io.Writer, agentID uuid.UUID, opts AgentO
 			if agent.Status == codersdk.WorkspaceAgentTimeout {
 				now := time.Now()
 				sw.Log(now, codersdk.LogLevelInfo, "The workspace agent is having trouble connecting, wait for it to connect or restart your workspace.")
-				sw.Log(now, codersdk.LogLevelInfo, troubleshootingMessage(agent, "https://coder.com/docs/v2/latest/templates#agent-connection-issues"))
+				sw.Log(now, codersdk.LogLevelInfo, troubleshootingMessage(agent, fmt.Sprintf("%s/templates#agent-connection-issues", opts.DocsURL)))
 				for agent.Status == codersdk.WorkspaceAgentTimeout {
 					if agent, err = fetch(); err != nil {
 						return xerrors.Errorf("fetch: %w", err)
@@ -132,11 +136,14 @@ func Agent(ctx context.Context, writer io.Writer, agentID uuid.UUID, opts AgentO
 			}
 
 			stage := "Running workspace agent startup scripts"
-			follow := opts.Wait
+			follow := opts.Wait && agent.LifecycleState.Starting()
 			if !follow {
 				stage += " (non-blocking)"
 			}
 			sw.Start(stage)
+			if follow {
+				sw.Log(time.Time{}, codersdk.LogLevelInfo, "==> ℹ︎ To connect immediately, reconnect with --wait=no or CODER_SSH_WAIT=no, see --help for more information.")
+			}
 
 			err = func() error { // Use func because of defer in for loop.
 				logStream, logsCloser, err := opts.FetchLogs(ctx, agent.ID, 0, follow)
@@ -206,19 +213,25 @@ func Agent(ctx context.Context, writer io.Writer, agentID uuid.UUID, opts AgentO
 			case codersdk.WorkspaceAgentLifecycleReady:
 				sw.Complete(stage, safeDuration(sw, agent.ReadyAt, agent.StartedAt))
 			case codersdk.WorkspaceAgentLifecycleStartTimeout:
-				sw.Fail(stage, 0)
+				// Backwards compatibility: Avoid printing warning if
+				// coderd is old and doesn't set ReadyAt for timeouts.
+				if agent.ReadyAt == nil {
+					sw.Fail(stage, 0)
+				} else {
+					sw.Fail(stage, safeDuration(sw, agent.ReadyAt, agent.StartedAt))
+				}
 				sw.Log(time.Time{}, codersdk.LogLevelWarn, "Warning: A startup script timed out and your workspace may be incomplete.")
 			case codersdk.WorkspaceAgentLifecycleStartError:
 				sw.Fail(stage, safeDuration(sw, agent.ReadyAt, agent.StartedAt))
 				// Use zero time (omitted) to separate these from the startup logs.
 				sw.Log(time.Time{}, codersdk.LogLevelWarn, "Warning: A startup script exited with an error and your workspace may be incomplete.")
-				sw.Log(time.Time{}, codersdk.LogLevelWarn, troubleshootingMessage(agent, "https://coder.com/docs/v2/latest/templates/troubleshooting#startup-script-exited-with-an-error"))
+				sw.Log(time.Time{}, codersdk.LogLevelWarn, troubleshootingMessage(agent, fmt.Sprintf("%s/templates#startup-script-exited-with-an-error", opts.DocsURL)))
 			default:
 				switch {
 				case agent.LifecycleState.Starting():
 					// Use zero time (omitted) to separate these from the startup logs.
 					sw.Log(time.Time{}, codersdk.LogLevelWarn, "Notice: The startup scripts are still running and your workspace may be incomplete.")
-					sw.Log(time.Time{}, codersdk.LogLevelWarn, troubleshootingMessage(agent, "https://coder.com/docs/v2/latest/templates/troubleshooting#your-workspace-may-be-incomplete"))
+					sw.Log(time.Time{}, codersdk.LogLevelWarn, troubleshootingMessage(agent, fmt.Sprintf("%s/templates#your-workspace-may-be-incomplete", opts.DocsURL)))
 					// Note: We don't complete or fail the stage here, it's
 					// intentionally left open to indicate this stage didn't
 					// complete.
@@ -240,7 +253,7 @@ func Agent(ctx context.Context, writer io.Writer, agentID uuid.UUID, opts AgentO
 			stage := "The workspace agent lost connection"
 			sw.Start(stage)
 			sw.Log(time.Now(), codersdk.LogLevelWarn, "Wait for it to reconnect or restart your workspace.")
-			sw.Log(time.Now(), codersdk.LogLevelWarn, troubleshootingMessage(agent, "https://coder.com/docs/v2/latest/templates/troubleshooting#agent-connection-issues"))
+			sw.Log(time.Now(), codersdk.LogLevelWarn, troubleshootingMessage(agent, fmt.Sprintf("%s/templates#agent-connection-issues", opts.DocsURL)))
 
 			disconnectedAt := agent.DisconnectedAt
 			for agent.Status == codersdk.WorkspaceAgentDisconnected {
@@ -297,7 +310,7 @@ func PeerDiagnostics(w io.Writer, d tailnet.PeerDiagnostics) {
 		_, _ = fmt.Fprint(w, "✘ not connected to DERP\n")
 	}
 	if d.SentNode {
-		_, _ = fmt.Fprint(w, "✔ sent local data to Coder networking coodinator\n")
+		_, _ = fmt.Fprint(w, "✔ sent local data to Coder networking coordinator\n")
 	} else {
 		_, _ = fmt.Fprint(w, "✘ have not sent local data to Coder networking coordinator\n")
 	}
@@ -337,3 +350,115 @@ func PeerDiagnostics(w io.Writer, d tailnet.PeerDiagnostics) {
 		_, _ = fmt.Fprint(w, "✘ Wireguard is not connected\n")
 	}
 }
+
+type ConnDiags struct {
+	ConnInfo           workspacesdk.AgentConnectionInfo
+	PingP2P            bool
+	DisableDirect      bool
+	LocalNetInfo       *tailcfg.NetInfo
+	LocalInterfaces    *healthsdk.InterfacesReport
+	AgentNetcheck      *healthsdk.AgentNetcheckReport
+	ClientIPIsAWS      bool
+	AgentIPIsAWS       bool
+	Verbose            bool
+	TroubleshootingURL string
+}
+
+func (d ConnDiags) Write(w io.Writer) {
+	_, _ = fmt.Fprintln(w, "")
+	general, client, agent := d.splitDiagnostics()
+	for _, msg := range general {
+		_, _ = fmt.Fprintln(w, msg)
+	}
+	if len(general) > 0 {
+		_, _ = fmt.Fprintln(w, "")
+	}
+	if len(client) > 0 {
+		_, _ = fmt.Fprint(w, "Possible client-side issues with direct connection:\n\n")
+		for _, msg := range client {
+			_, _ = fmt.Fprintf(w, " - %s\n\n", msg)
+		}
+	}
+	if len(agent) > 0 {
+		_, _ = fmt.Fprint(w, "Possible agent-side issues with direct connections:\n\n")
+		for _, msg := range agent {
+			_, _ = fmt.Fprintf(w, " - %s\n\n", msg)
+		}
+	}
+}
+
+func (d ConnDiags) splitDiagnostics() (general, client, agent []string) {
+	if d.AgentNetcheck != nil {
+		for _, msg := range d.AgentNetcheck.Interfaces.Warnings {
+			agent = append(agent, msg.Message)
+		}
+		if len(d.AgentNetcheck.Interfaces.Warnings) > 0 {
+			agent[len(agent)-1] += fmt.Sprintf("\n%s#low-mtu", d.TroubleshootingURL)
+		}
+	}
+
+	if d.LocalInterfaces != nil {
+		for _, msg := range d.LocalInterfaces.Warnings {
+			client = append(client, msg.Message)
+		}
+		if len(d.LocalInterfaces.Warnings) > 0 {
+			client[len(client)-1] += fmt.Sprintf("\n%s#low-mtu", d.TroubleshootingURL)
+		}
+	}
+
+	if d.PingP2P && !d.Verbose {
+		return general, client, agent
+	}
+
+	if d.DisableDirect {
+		general = append(general, "❗ Direct connections are disabled locally, by `--disable-direct-connections` or `CODER_DISABLE_DIRECT_CONNECTIONS`.\n"+
+			"   They may still be established over a private network.")
+		if !d.Verbose {
+			return general, client, agent
+		}
+	}
+
+	if d.ConnInfo.DisableDirectConnections {
+		general = append(general,
+			fmt.Sprintf("❗ Your Coder administrator has blocked direct connections\n   %s#disabled-deployment-wide", d.TroubleshootingURL))
+		if !d.Verbose {
+			return general, client, agent
+		}
+	}
+
+	if !d.ConnInfo.DERPMap.HasSTUN() {
+		general = append(general,
+			fmt.Sprintf("❗ The DERP map is not configured to use STUN\n   %s#no-stun-servers", d.TroubleshootingURL))
+	} else if d.LocalNetInfo != nil && !d.LocalNetInfo.UDP {
+		client = append(client,
+			fmt.Sprintf("Client could not connect to STUN over UDP\n   %s#udp-blocked", d.TroubleshootingURL))
+	}
+
+	if d.LocalNetInfo != nil && d.LocalNetInfo.MappingVariesByDestIP.EqualBool(true) {
+		client = append(client,
+			fmt.Sprintf("Client is potentially behind a hard NAT, as multiple endpoints were retrieved from different STUN servers\n  %s#endpoint-dependent-nat-hard-nat", d.TroubleshootingURL))
+	}
+
+	if d.AgentNetcheck != nil && d.AgentNetcheck.NetInfo != nil {
+		if d.AgentNetcheck.NetInfo.MappingVariesByDestIP.EqualBool(true) {
+			agent = append(agent,
+				fmt.Sprintf("Agent is potentially behind a hard NAT, as multiple endpoints were retrieved from different STUN servers\n   %s#endpoint-dependent-nat-hard-nat", d.TroubleshootingURL))
+		}
+		if !d.AgentNetcheck.NetInfo.UDP {
+			agent = append(agent,
+				fmt.Sprintf("Agent could not connect to STUN over UDP\n   %s#udp-blocked", d.TroubleshootingURL))
+		}
+	}
+
+	if d.ClientIPIsAWS {
+		client = append(client,
+			fmt.Sprintf("Client IP address is within an AWS range (AWS uses hard NAT)\n   %s#endpoint-dependent-nat-hard-nat", d.TroubleshootingURL))
+	}
+
+	if d.AgentIPIsAWS {
+		agent = append(agent,
+			fmt.Sprintf("Agent IP address is within an AWS range (AWS uses hard NAT)\n   %s#endpoint-dependent-nat-hard-nat", d.TroubleshootingURL))
+	}
+
+	return general, client, agent
+}

cli/cliui/agent_test.go

@@ -20,8 +20,11 @@ import (
 
 	"github.com/coder/coder/v2/cli/clitest"
 	"github.com/coder/coder/v2/cli/cliui"
+	"github.com/coder/coder/v2/coderd/healthcheck/health"
 	"github.com/coder/coder/v2/coderd/util/ptr"
 	"github.com/coder/coder/v2/codersdk"
+	"github.com/coder/coder/v2/codersdk/healthsdk"
+	"github.com/coder/coder/v2/codersdk/workspacesdk"
 	"github.com/coder/coder/v2/tailnet"
 	"github.com/coder/coder/v2/testutil"
 	"github.com/coder/serpent"
@@ -95,6 +98,8 @@ func TestAgent(t *testing.T) {
 			iter: []func(context.Context, *testing.T, *codersdk.WorkspaceAgent, <-chan string, chan []codersdk.WorkspaceAgentLog) error{
 				func(_ context.Context, _ *testing.T, agent *codersdk.WorkspaceAgent, _ <-chan string, _ chan []codersdk.WorkspaceAgentLog) error {
 					agent.Status = codersdk.WorkspaceAgentConnecting
+					agent.LifecycleState = codersdk.WorkspaceAgentLifecycleStarting
+					agent.StartedAt = ptr.Ref(time.Now())
 					return nil
 				},
 				func(_ context.Context, t *testing.T, agent *codersdk.WorkspaceAgent, output <-chan string, _ chan []codersdk.WorkspaceAgentLog) error {
@@ -104,6 +109,7 @@ func TestAgent(t *testing.T) {
 					agent.Status = codersdk.WorkspaceAgentConnected
 					agent.LifecycleState = codersdk.WorkspaceAgentLifecycleStartTimeout
 					agent.FirstConnectedAt = ptr.Ref(time.Now())
+					agent.ReadyAt = ptr.Ref(time.Now())
 					return nil
 				},
 			},
@@ -226,6 +232,7 @@ func TestAgent(t *testing.T) {
 			},
 			want: []string{
 				"⧗ Running workspace agent startup scripts",
+				"ℹ︎ To connect immediately, reconnect with --wait=no or CODER_SSH_WAIT=no, see --help for more information.",
 				"testing: Hello world",
 				"Bye now",
 				"✔ Running workspace agent startup scripts",
@@ -254,9 +261,9 @@ func TestAgent(t *testing.T) {
 				},
 			},
 			want: []string{
-				"⧗ Running workspace agent startup scripts",
+				"⧗ Running workspace agent startup scripts (non-blocking)",
 				"Hello world",
-				"✘ Running workspace agent startup scripts",
+				"✘ Running workspace agent startup scripts (non-blocking)",
 				"Warning: A startup script exited with an error and your workspace may be incomplete.",
 				"For more information and troubleshooting, see",
 			},
@@ -306,6 +313,7 @@ func TestAgent(t *testing.T) {
 			},
 			want: []string{
 				"⧗ Running workspace agent startup scripts",
+				"ℹ︎ To connect immediately, reconnect with --wait=no or CODER_SSH_WAIT=no, see --help for more information.",
 				"Hello world",
 				"✔ Running workspace agent startup scripts",
 			},
@@ -525,7 +533,7 @@ func TestPeerDiagnostics(t *testing.T) {
 				LastWireguardHandshake: time.Time{},
 			},
 			want: []*regexp.Regexp{
-				regexp.MustCompile(`^✔ sent local data to Coder networking coodinator$`),
+				regexp.MustCompile(`^✔ sent local data to Coder networking coordinator$`),
 			},
 		},
 		{
@@ -667,3 +675,197 @@ func TestPeerDiagnostics(t *testing.T) {
 		})
 	}
 }
+
+func TestConnDiagnostics(t *testing.T) {
+	t.Parallel()
+	testCases := []struct {
+		name  string
+		diags cliui.ConnDiags
+		want  []string
+	}{
+		{
+			name: "DirectBlocked",
+			diags: cliui.ConnDiags{
+				ConnInfo: workspacesdk.AgentConnectionInfo{
+					DERPMap:                  &tailcfg.DERPMap{},
+					DisableDirectConnections: true,
+				},
+			},
+			want: []string{
+				`❗ Your Coder administrator has blocked direct connections`,
+			},
+		},
+		{
+			name: "NoStun",
+			diags: cliui.ConnDiags{
+				ConnInfo: workspacesdk.AgentConnectionInfo{
+					DERPMap: &tailcfg.DERPMap{},
+				},
+				LocalNetInfo: &tailcfg.NetInfo{},
+			},
+			want: []string{
+				`The DERP map is not configured to use STUN`,
+			},
+		},
+		{
+			name: "ClientHasStunNoUDP",
+			diags: cliui.ConnDiags{
+				ConnInfo: workspacesdk.AgentConnectionInfo{
+					DERPMap: &tailcfg.DERPMap{
+						Regions: map[int]*tailcfg.DERPRegion{
+							999: {
+								Nodes: []*tailcfg.DERPNode{
+									{
+										STUNPort: 1337,
+									},
+								},
+							},
+						},
+					},
+				},
+				LocalNetInfo: &tailcfg.NetInfo{
+					UDP: false,
+				},
+			},
+			want: []string{
+				`Client could not connect to STUN over UDP`,
+			},
+		},
+		{
+			name: "AgentHasStunNoUDP",
+			diags: cliui.ConnDiags{
+				ConnInfo: workspacesdk.AgentConnectionInfo{
+					DERPMap: &tailcfg.DERPMap{
+						Regions: map[int]*tailcfg.DERPRegion{
+							999: {
+								Nodes: []*tailcfg.DERPNode{
+									{
+										STUNPort: 1337,
+									},
+								},
+							},
+						},
+					},
+				},
+				AgentNetcheck: &healthsdk.AgentNetcheckReport{
+					NetInfo: &tailcfg.NetInfo{
+						UDP: false,
+					},
+				},
+			},
+			want: []string{
+				`Agent could not connect to STUN over UDP`,
+			},
+		},
+		{
+			name: "ClientHardNat",
+			diags: cliui.ConnDiags{
+				ConnInfo: workspacesdk.AgentConnectionInfo{
+					DERPMap: &tailcfg.DERPMap{},
+				},
+				LocalNetInfo: &tailcfg.NetInfo{
+					MappingVariesByDestIP: "true",
+				},
+			},
+			want: []string{
+				`Client is potentially behind a hard NAT, as multiple endpoints were retrieved from different STUN servers`,
+			},
+		},
+		{
+			name: "AgentHardNat",
+			diags: cliui.ConnDiags{
+				ConnInfo: workspacesdk.AgentConnectionInfo{
+					DERPMap: &tailcfg.DERPMap{},
+				},
+				LocalNetInfo: &tailcfg.NetInfo{},
+				AgentNetcheck: &healthsdk.AgentNetcheckReport{
+					NetInfo: &tailcfg.NetInfo{MappingVariesByDestIP: "true"},
+				},
+			},
+			want: []string{
+				`Agent is potentially behind a hard NAT, as multiple endpoints were retrieved from different STUN servers`,
+			},
+		},
+		{
+			name: "AgentInterfaceWarnings",
+			diags: cliui.ConnDiags{
+				ConnInfo: workspacesdk.AgentConnectionInfo{
+					DERPMap: &tailcfg.DERPMap{},
+				},
+				AgentNetcheck: &healthsdk.AgentNetcheckReport{
+					Interfaces: healthsdk.InterfacesReport{
+						BaseReport: healthsdk.BaseReport{
+							Warnings: []health.Message{
+								health.Messagef(health.CodeInterfaceSmallMTU, "Network interface eth0 has MTU 1280, (less than 1378), which may degrade the quality of direct connections"),
+							},
+						},
+					},
+				},
+			},
+			want: []string{
+				`Network interface eth0 has MTU 1280, (less than 1378), which may degrade the quality of direct connections`,
+			},
+		},
+		{
+			name: "LocalInterfaceWarnings",
+			diags: cliui.ConnDiags{
+				ConnInfo: workspacesdk.AgentConnectionInfo{
+					DERPMap: &tailcfg.DERPMap{},
+				},
+				LocalInterfaces: &healthsdk.InterfacesReport{
+					BaseReport: healthsdk.BaseReport{
+						Warnings: []health.Message{
+							health.Messagef(health.CodeInterfaceSmallMTU, "Network interface eth1 has MTU 1310, (less than 1378), which may degrade the quality of direct connections"),
+						},
+					},
+				},
+			},
+			want: []string{
+				`Network interface eth1 has MTU 1310, (less than 1378), which may degrade the quality of direct connections`,
+			},
+		},
+		{
+			name: "ClientAWSIP",
+			diags: cliui.ConnDiags{
+				ConnInfo: workspacesdk.AgentConnectionInfo{
+					DERPMap: &tailcfg.DERPMap{},
+				},
+				ClientIPIsAWS: true,
+				AgentIPIsAWS:  false,
+			},
+			want: []string{
+				`Client IP address is within an AWS range (AWS uses hard NAT)`,
+			},
+		},
+		{
+			name: "AgentAWSIP",
+			diags: cliui.ConnDiags{
+				ConnInfo: workspacesdk.AgentConnectionInfo{
+					DERPMap: &tailcfg.DERPMap{},
+				},
+				ClientIPIsAWS: false,
+				AgentIPIsAWS:  true,
+			},
+			want: []string{
+				`Agent IP address is within an AWS range (AWS uses hard NAT)`,
+			},
+		},
+	}
+	for _, tc := range testCases {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			r, w := io.Pipe()
+			go func() {
+				defer w.Close()
+				tc.diags.Write(w)
+			}()
+			bytes, err := io.ReadAll(r)
+			require.NoError(t, err)
+			output := string(bytes)
+			for _, want := range tc.want {
+				require.Contains(t, output, want)
+			}
+		})
+	}
+}

cli/cliui/cliui.go

@@ -22,6 +22,7 @@ type Styles struct {
 	DateTimeStamp,
 	Error,
 	Field,
+	Hyperlink,
 	Keyword,
 	Placeholder,
 	Prompt,
@@ -37,17 +38,21 @@ var (
 )
 
 var (
-	Green   = Color("#04B575")
-	Red     = Color("#ED567A")
-	Fuchsia = Color("#EE6FF8")
-	Yellow  = Color("#ECFD65")
-	Blue    = Color("#5000ff")
+	// ANSI color codes
+	red           = Color("1")
+	green         = Color("2")
+	yellow        = Color("3")
+	magenta       = Color("5")
+	white         = Color("7")
+	brightBlue    = Color("12")
+	brightMagenta = Color("13")
 )
 
 // Color returns a color for the given string.
 func Color(s string) termenv.Color {
 	colorOnce.Do(func() {
-		color = termenv.NewOutput(os.Stdout).ColorProfile()
+		color = termenv.NewOutput(os.Stdout).EnvColorProfile()
+
 		if flag.Lookup("test.v") != nil {
 			// Use a consistent colorless profile in tests so that results
 			// are deterministic.
@@ -123,42 +128,49 @@ func init() {
 	DefaultStyles = Styles{
 		Code: pretty.Style{
 			ifTerm(pretty.XPad(1, 1)),
-			pretty.FgColor(Red),
-			pretty.BgColor(color.Color("#2c2c2c")),
+			pretty.FgColor(Color("#ED567A")),
+			pretty.BgColor(Color("#2C2C2C")),
 		},
 		DateTimeStamp: pretty.Style{
-			pretty.FgColor(color.Color("#7571F9")),
+			pretty.FgColor(brightBlue),
 		},
 		Error: pretty.Style{
-			pretty.FgColor(Red),
+			pretty.FgColor(red),
 		},
 		Field: pretty.Style{
 			pretty.XPad(1, 1),
-			pretty.FgColor(color.Color("#FFFFFF")),
-			pretty.BgColor(color.Color("#2b2a2a")),
+			pretty.FgColor(Color("#FFFFFF")),
+			pretty.BgColor(Color("#2B2A2A")),
+		},
+		Fuchsia: pretty.Style{
+			pretty.FgColor(brightMagenta),
+		},
+		FocusedPrompt: pretty.Style{
+			pretty.FgColor(white),
+			pretty.Wrap("> ", ""),
+			pretty.FgColor(brightBlue),
+		},
+		Hyperlink: pretty.Style{
+			pretty.FgColor(magenta),
+			pretty.Underline(),
 		},
 		Keyword: pretty.Style{
-			pretty.FgColor(Green),
+			pretty.FgColor(green),
 		},
 		Placeholder: pretty.Style{
-			pretty.FgColor(color.Color("#4d46b3")),
+			pretty.FgColor(magenta),
 		},
 		Prompt: pretty.Style{
-			pretty.FgColor(color.Color("#5C5C5C")),
-			pretty.Wrap("> ", ""),
+			pretty.FgColor(white),
+			pretty.Wrap("  ", ""),
 		},
 		Warn: pretty.Style{
-			pretty.FgColor(Yellow),
+			pretty.FgColor(yellow),
 		},
 		Wrap: pretty.Style{
 			pretty.LineWrap(80),
 		},
 	}
-
-	DefaultStyles.FocusedPrompt = append(
-		DefaultStyles.Prompt,
-		pretty.FgColor(Blue),
-	)
 }
 
 // ValidateNotEmpty is a helper function to disallow empty inputs!

cli/cliui/output.go

@@ -7,6 +7,7 @@ import (
 	"reflect"
 	"strings"
 
+	"github.com/jedib0t/go-pretty/v6/table"
 	"golang.org/x/xerrors"
 
 	"github.com/coder/serpent"
@@ -64,8 +65,8 @@ func (f *OutputFormatter) AttachOptions(opts *serpent.OptionSet) {
 			Flag:          "output",
 			FlagShorthand: "o",
 			Default:       f.formats[0].ID(),
-			Value:         serpent.StringOf(&f.formatID),
-			Description:   "Output format. Available formats: " + strings.Join(formatNames, ", ") + ".",
+			Value:         serpent.EnumOf(&f.formatID, formatNames...),
+			Description:   "Output format.",
 		},
 	)
 }
@@ -135,15 +136,19 @@ func (f *tableFormat) AttachOptions(opts *serpent.OptionSet) {
 			Flag:          "column",
 			FlagShorthand: "c",
 			Default:       strings.Join(f.defaultColumns, ","),
-			Value:         serpent.StringArrayOf(&f.columns),
-			Description:   "Columns to display in table output. Available columns: " + strings.Join(f.allColumns, ", ") + ".",
+			Value:         serpent.EnumArrayOf(&f.columns, f.allColumns...),
+			Description:   "Columns to display in table output.",
 		},
 	)
 }
 
 // Format implements OutputFormat.
 func (f *tableFormat) Format(_ context.Context, data any) (string, error) {
-	return DisplayTable(data, f.sort, f.columns)
+	headers := make(table.Row, len(f.allColumns))
+	for i, header := range f.allColumns {
+		headers[i] = header
+	}
+	return renderTable(data, f.sort, headers, f.columns)
 }
 
 type jsonFormat struct{}

cli/cliui/output_test.go

@@ -106,11 +106,11 @@ func Test_OutputFormatter(t *testing.T) {
 
 		fs := cmd.Options.FlagSet()
 
-		selected, err := fs.GetString("output")
-		require.NoError(t, err)
-		require.Equal(t, "json", selected)
+		selected := cmd.Options.ByFlag("output")
+		require.NotNil(t, selected)
+		require.Equal(t, "json", selected.Value.String())
 		usage := fs.FlagUsages()
-		require.Contains(t, usage, "Available formats: json, foo")
+		require.Contains(t, usage, "Output format.")
 		require.Contains(t, usage, "foo flag 1234")
 
 		ctx := context.Background()
@@ -129,11 +129,10 @@ func Test_OutputFormatter(t *testing.T) {
 		require.Equal(t, "foo", out)
 		require.EqualValues(t, 1, atomic.LoadInt64(&called))
 
-		require.NoError(t, fs.Set("output", "bar"))
+		require.Error(t, fs.Set("output", "bar"))
 		out, err = f.Format(ctx, data)
-		require.Error(t, err)
-		require.ErrorContains(t, err, "bar")
-		require.Equal(t, "", out)
-		require.EqualValues(t, 1, atomic.LoadInt64(&called))
+		require.NoError(t, err)
+		require.Equal(t, "foo", out)
+		require.EqualValues(t, 2, atomic.LoadInt64(&called))
 	})
 }

cli/cliui/parameter.go

@@ -43,7 +43,10 @@ func RichParameter(inv *serpent.Invocation, templateVersionParameter codersdk.Te
 			return "", err
 		}
 
-		values, err := MultiSelect(inv, options)
+		values, err := MultiSelect(inv, MultiSelectOptions{
+			Options:  options,
+			Defaults: options,
+		})
 		if err == nil {
 			v, err := json.Marshal(&values)
 			if err != nil {

cli/cliui/prompt.go

@@ -1,10 +1,10 @@
 package cliui
 
 import (
-	"bufio"
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"io"
 	"os"
 	"os/signal"
 	"strings"
@@ -96,14 +96,13 @@ func Prompt(inv *serpent.Invocation, opts PromptOptions) (string, error) {
 			signal.Notify(interrupt, os.Interrupt)
 			defer signal.Stop(interrupt)
 
-			reader := bufio.NewReader(inv.Stdin)
-			line, err = reader.ReadString('\n')
+			line, err = readUntil(inv.Stdin, '\n')
 
 			// Check if the first line beings with JSON object or array chars.
 			// This enables multiline JSON to be pasted into an input, and have
 			// it parse properly.
 			if err == nil && (strings.HasPrefix(line, "{") || strings.HasPrefix(line, "[")) {
-				line, err = promptJSON(reader, line)
+				line, err = promptJSON(inv.Stdin, line)
 			}
 		}
 		if err != nil {
@@ -144,7 +143,7 @@ func Prompt(inv *serpent.Invocation, opts PromptOptions) (string, error) {
 	}
 }
 
-func promptJSON(reader *bufio.Reader, line string) (string, error) {
+func promptJSON(reader io.Reader, line string) (string, error) {
 	var data bytes.Buffer
 	for {
 		_, _ = data.WriteString(line)
@@ -162,7 +161,7 @@ func promptJSON(reader *bufio.Reader, line string) (string, error) {
 			// Read line-by-line. We can't use a JSON decoder
 			// here because it doesn't work by newline, so
 			// reads will block.
-			line, err = reader.ReadString('\n')
+			line, err = readUntil(reader, '\n')
 			if err != nil {
 				break
 			}
@@ -179,3 +178,29 @@ func promptJSON(reader *bufio.Reader, line string) (string, error) {
 	}
 	return line, nil
 }
+
+// readUntil the first occurrence of delim in the input, returning a string containing the data up
+// to and including the delimiter. Unlike `bufio`, it only reads until the delimiter and no further
+// bytes. If readUntil encounters an error before finding a delimiter, it returns the data read
+// before the error and the error itself (often io.EOF). readUntil returns err != nil if and only if
+// the returned data does not end in delim.
+func readUntil(r io.Reader, delim byte) (string, error) {
+	var (
+		have []byte
+		b    = make([]byte, 1)
+	)
+	for {
+		n, err := r.Read(b)
+		if n > 0 {
+			have = append(have, b[0])
+			if b[0] == delim {
+				// match `bufio` in that we only return non-nil if we didn't find the delimiter,
+				// regardless of whether we also erred.
+				return string(have), nil
+			}
+		}
+		if err != nil {
+			return string(have), err
+		}
+	}
+}

cli/cliui/prompt_test.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"golang.org/x/xerrors"
 
 	"github.com/coder/coder/v2/cli/cliui"
 	"github.com/coder/coder/v2/pty"
@@ -22,10 +23,11 @@ func TestPrompt(t *testing.T) {
 	t.Parallel()
 	t.Run("Success", func(t *testing.T) {
 		t.Parallel()
+		ctx := testutil.Context(t, testutil.WaitShort)
 		ptty := ptytest.New(t)
 		msgChan := make(chan string)
 		go func() {
-			resp, err := newPrompt(ptty, cliui.PromptOptions{
+			resp, err := newPrompt(ctx, ptty, cliui.PromptOptions{
 				Text: "Example",
 			}, nil)
 			assert.NoError(t, err)
@@ -33,15 +35,17 @@ func TestPrompt(t *testing.T) {
 		}()
 		ptty.ExpectMatch("Example")
 		ptty.WriteLine("hello")
-		require.Equal(t, "hello", <-msgChan)
+		resp := testutil.RequireRecvCtx(ctx, t, msgChan)
+		require.Equal(t, "hello", resp)
 	})
 
 	t.Run("Confirm", func(t *testing.T) {
 		t.Parallel()
+		ctx := testutil.Context(t, testutil.WaitShort)
 		ptty := ptytest.New(t)
 		doneChan := make(chan string)
 		go func() {
-			resp, err := newPrompt(ptty, cliui.PromptOptions{
+			resp, err := newPrompt(ctx, ptty, cliui.PromptOptions{
 				Text:      "Example",
 				IsConfirm: true,
 			}, nil)
@@ -50,18 +54,20 @@ func TestPrompt(t *testing.T) {
 		}()
 		ptty.ExpectMatch("Example")
 		ptty.WriteLine("yes")
-		require.Equal(t, "yes", <-doneChan)
+		resp := testutil.RequireRecvCtx(ctx, t, doneChan)
+		require.Equal(t, "yes", resp)
 	})
 
 	t.Run("Skip", func(t *testing.T) {
 		t.Parallel()
+		ctx := testutil.Context(t, testutil.WaitShort)
 		ptty := ptytest.New(t)
 		var buf bytes.Buffer
 
 		// Copy all data written out to a buffer. When we close the ptty, we can
 		// no longer read from the ptty.Output(), but we can read what was
 		// written to the buffer.
-		dataRead, doneReading := context.WithTimeout(context.Background(), testutil.WaitShort)
+		dataRead, doneReading := context.WithCancel(ctx)
 		go func() {
 			// This will throw an error sometimes. The underlying ptty
 			// has its own cleanup routines in t.Cleanup. Instead of
@@ -74,7 +80,7 @@ func TestPrompt(t *testing.T) {
 
 		doneChan := make(chan string)
 		go func() {
-			resp, err := newPrompt(ptty, cliui.PromptOptions{
+			resp, err := newPrompt(ctx, ptty, cliui.PromptOptions{
 				Text:      "ShouldNotSeeThis",
 				IsConfirm: true,
 			}, func(inv *serpent.Invocation) {
@@ -85,7 +91,8 @@ func TestPrompt(t *testing.T) {
 			doneChan <- resp
 		}()
 
-		require.Equal(t, "yes", <-doneChan)
+		resp := testutil.RequireRecvCtx(ctx, t, doneChan)
+		require.Equal(t, "yes", resp)
 		// Close the reader to end the io.Copy
 		require.NoError(t, ptty.Close(), "close eof reader")
 		// Wait for the IO copy to finish
@@ -96,10 +103,11 @@ func TestPrompt(t *testing.T) {
 	})
 	t.Run("JSON", func(t *testing.T) {
 		t.Parallel()
+		ctx := testutil.Context(t, testutil.WaitShort)
 		ptty := ptytest.New(t)
 		doneChan := make(chan string)
 		go func() {
-			resp, err := newPrompt(ptty, cliui.PromptOptions{
+			resp, err := newPrompt(ctx, ptty, cliui.PromptOptions{
 				Text: "Example",
 			}, nil)
 			assert.NoError(t, err)
@@ -107,15 +115,17 @@ func TestPrompt(t *testing.T) {
 		}()
 		ptty.ExpectMatch("Example")
 		ptty.WriteLine("{}")
-		require.Equal(t, "{}", <-doneChan)
+		resp := testutil.RequireRecvCtx(ctx, t, doneChan)
+		require.Equal(t, "{}", resp)
 	})
 
 	t.Run("BadJSON", func(t *testing.T) {
 		t.Parallel()
+		ctx := testutil.Context(t, testutil.WaitShort)
 		ptty := ptytest.New(t)
 		doneChan := make(chan string)
 		go func() {
-			resp, err := newPrompt(ptty, cliui.PromptOptions{
+			resp, err := newPrompt(ctx, ptty, cliui.PromptOptions{
 				Text: "Example",
 			}, nil)
 			assert.NoError(t, err)
@@ -123,15 +133,17 @@ func TestPrompt(t *testing.T) {
 		}()
 		ptty.ExpectMatch("Example")
 		ptty.WriteLine("{a")
-		require.Equal(t, "{a", <-doneChan)
+		resp := testutil.RequireRecvCtx(ctx, t, doneChan)
+		require.Equal(t, "{a", resp)
 	})
 
 	t.Run("MultilineJSON", func(t *testing.T) {
 		t.Parallel()
+		ctx := testutil.Context(t, testutil.WaitShort)
 		ptty := ptytest.New(t)
 		doneChan := make(chan string)
 		go func() {
-			resp, err := newPrompt(ptty, cliui.PromptOptions{
+			resp, err := newPrompt(ctx, ptty, cliui.PromptOptions{
 				Text: "Example",
 			}, nil)
 			assert.NoError(t, err)
@@ -141,11 +153,37 @@ func TestPrompt(t *testing.T) {
 		ptty.WriteLine(`{
 "test": "wow"
 }`)
-		require.Equal(t, `{"test":"wow"}`, <-doneChan)
+		resp := testutil.RequireRecvCtx(ctx, t, doneChan)
+		require.Equal(t, `{"test":"wow"}`, resp)
+	})
+
+	t.Run("InvalidValid", func(t *testing.T) {
+		t.Parallel()
+		ctx := testutil.Context(t, testutil.WaitShort)
+		ptty := ptytest.New(t)
+		doneChan := make(chan string)
+		go func() {
+			resp, err := newPrompt(ctx, ptty, cliui.PromptOptions{
+				Text: "Example",
+				Validate: func(s string) error {
+					t.Logf("validate: %q", s)
+					if s != "valid" {
+						return xerrors.New("invalid")
+					}
+					return nil
+				},
+			}, nil)
+			assert.NoError(t, err)
+			doneChan <- resp
+		}()
+		ptty.ExpectMatch("Example")
+		ptty.WriteLine("foo\nbar\nbaz\n\n\nvalid\n")
+		resp := testutil.RequireRecvCtx(ctx, t, doneChan)
+		require.Equal(t, "valid", resp)
 	})
 }
 
-func newPrompt(ptty *ptytest.PTY, opts cliui.PromptOptions, invOpt func(inv *serpent.Invocation)) (string, error) {
+func newPrompt(ctx context.Context, ptty *ptytest.PTY, opts cliui.PromptOptions, invOpt func(inv *serpent.Invocation)) (string, error) {
 	value := ""
 	cmd := &serpent.Command{
 		Handler: func(inv *serpent.Invocation) error {
@@ -163,7 +201,7 @@ func newPrompt(ptty *ptytest.PTY, opts cliui.PromptOptions, invOpt func(inv *ser
 	inv.Stdout = ptty.Output()
 	inv.Stderr = ptty.Output()
 	inv.Stdin = ptty.Input()
-	return value, inv.WithContext(context.Background()).Run()
+	return value, inv.WithContext(ctx).Run()
 }
 
 func TestPasswordTerminalState(t *testing.T) {

cli/cliui/select.go

@@ -1,61 +1,59 @@
 package cliui
 
 import (
-	"errors"
 	"flag"
-	"io"
+	"fmt"
 	"os"
+	"os/signal"
+	"strings"
+	"syscall"
 
-	"github.com/AlecAivazis/survey/v2"
-	"github.com/AlecAivazis/survey/v2/terminal"
+	"github.com/charmbracelet/bubbles/textinput"
+	tea "github.com/charmbracelet/bubbletea"
 	"golang.org/x/xerrors"
 
 	"github.com/coder/coder/v2/codersdk"
+	"github.com/coder/pretty"
 	"github.com/coder/serpent"
 )
 
-func init() {
-	survey.SelectQuestionTemplate = `
-{{- define "option"}}
-    {{- "  " }}{{- if eq .SelectedIndex .CurrentIndex }}{{color "green" }}{{ .Config.Icons.SelectFocus.Text }} {{else}}{{color "default"}}  {{end}}
-    {{- .CurrentOpt.Value}}
-    {{- color "reset"}}
-{{end}}
+const defaultSelectModelHeight = 7
 
-{{- if not .ShowAnswer }}
-{{- if .Config.Icons.Help.Text }}
-{{- if .FilterMessage }}{{ "Search:" }}{{ .FilterMessage }}
-{{- else }}
-{{- color "black+h"}}{{- "Type to search" }}{{color "reset"}}
-{{- end }}
-{{- "\n" }}
-{{- end }}
-{{- "\n" }}
-{{- range $ix, $option := .PageEntries}}
-  {{- template "option" $.IterateOption $ix $option}}
-{{- end}}
-{{- end }}`
+type terminateMsg struct{}
 
-	survey.MultiSelectQuestionTemplate = `
-{{- define "option"}}
-    {{- if eq .SelectedIndex .CurrentIndex }}{{color .Config.Icons.SelectFocus.Format }}{{ .Config.Icons.SelectFocus.Text }}{{color "reset"}}{{else}} {{end}}
-    {{- if index .Checked .CurrentOpt.Index }}{{color .Config.Icons.MarkedOption.Format }} {{ .Config.Icons.MarkedOption.Text }} {{else}}{{color .Config.Icons.UnmarkedOption.Format }} {{ .Config.Icons.UnmarkedOption.Text }} {{end}}
-    {{- color "reset"}}
-    {{- " "}}{{- .CurrentOpt.Value}}
-{{end}}
-{{- if .ShowHelp }}{{- color .Config.Icons.Help.Format }}{{ .Config.Icons.Help.Text }} {{ .Help }}{{color "reset"}}{{"\n"}}{{end}}
-{{- if not .ShowAnswer }}
-  {{- "\n"}}
-  {{- range $ix, $option := .PageEntries}}
-    {{- template "option" $.IterateOption $ix $option}}
-  {{- end}}
-{{- end}}`
+func installSignalHandler(p *tea.Program) func() {
+	ch := make(chan struct{})
+
+	go func() {
+		sig := make(chan os.Signal, 1)
+		signal.Notify(sig, os.Interrupt, syscall.SIGTERM)
+
+		defer func() {
+			signal.Stop(sig)
+			close(ch)
+		}()
+
+		for {
+			select {
+			case <-ch:
+				return
+
+			case <-sig:
+				p.Send(terminateMsg{})
+			}
+		}
+	}()
+
+	return func() {
+		ch <- struct{}{}
+	}
 }
 
 type SelectOptions struct {
 	Options []string
 	// Default will be highlighted first if it's a valid option.
 	Default    string
+	Message    string
 	Size       int
 	HideSearch bool
 }
@@ -112,66 +110,531 @@ func Select(inv *serpent.Invocation, opts SelectOptions) (string, error) {
 		return opts.Options[0], nil
 	}
 
-	var defaultOption interface{}
-	if opts.Default != "" {
-		defaultOption = opts.Default
+	initialModel := selectModel{
+		search:     textinput.New(),
+		hideSearch: opts.HideSearch,
+		options:    opts.Options,
+		height:     opts.Size,
+		message:    opts.Message,
 	}
 
-	var value string
-	err := survey.AskOne(&survey.Select{
-		Options:  opts.Options,
-		Default:  defaultOption,
-		PageSize: opts.Size,
-	}, &value, survey.WithIcons(func(is *survey.IconSet) {
-		is.Help.Text = "Type to search"
-		if opts.HideSearch {
-			is.Help.Text = ""
-		}
-	}), survey.WithStdio(fileReadWriter{
-		Reader: inv.Stdin,
-	}, fileReadWriter{
-		Writer: inv.Stdout,
-	}, inv.Stdout))
-	if errors.Is(err, terminal.InterruptErr) {
-		return value, Canceled
+	if initialModel.height == 0 {
+		initialModel.height = defaultSelectModelHeight
 	}
-	return value, err
+
+	initialModel.search.Prompt = ""
+	initialModel.search.Focus()
+
+	p := tea.NewProgram(
+		initialModel,
+		tea.WithoutSignalHandler(),
+		tea.WithContext(inv.Context()),
+		tea.WithInput(inv.Stdin),
+		tea.WithOutput(inv.Stdout),
+	)
+
+	closeSignalHandler := installSignalHandler(p)
+	defer closeSignalHandler()
+
+	m, err := p.Run()
+	if err != nil {
+		return "", err
+	}
+
+	model, ok := m.(selectModel)
+	if !ok {
+		return "", xerrors.New(fmt.Sprintf("unknown model found %T (%+v)", m, m))
+	}
+
+	if model.canceled {
+		return "", Canceled
+	}
+
+	return model.selected, nil
 }
 
-func MultiSelect(inv *serpent.Invocation, items []string) ([]string, error) {
+type selectModel struct {
+	search     textinput.Model
+	options    []string
+	cursor     int
+	height     int
+	message    string
+	selected   string
+	canceled   bool
+	hideSearch bool
+}
+
+func (selectModel) Init() tea.Cmd {
+	return nil
+}
+
+//nolint:revive // The linter complains about modifying 'm' but this is typical practice for bubbletea
+func (m selectModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
+	var cmd tea.Cmd
+
+	switch msg := msg.(type) {
+	case terminateMsg:
+		m.canceled = true
+		return m, tea.Quit
+
+	case tea.KeyMsg:
+		switch msg.Type {
+		case tea.KeyCtrlC:
+			m.canceled = true
+			return m, tea.Quit
+
+		case tea.KeyEnter:
+			options := m.filteredOptions()
+			if len(options) != 0 {
+				m.selected = options[m.cursor]
+				return m, tea.Quit
+			}
+
+		case tea.KeyUp:
+			options := m.filteredOptions()
+			if m.cursor > 0 {
+				m.cursor--
+			} else {
+				m.cursor = len(options) - 1
+			}
+
+		case tea.KeyDown:
+			options := m.filteredOptions()
+			if m.cursor < len(options)-1 {
+				m.cursor++
+			} else {
+				m.cursor = 0
+			}
+		}
+	}
+
+	if !m.hideSearch {
+		oldSearch := m.search.Value()
+		m.search, cmd = m.search.Update(msg)
+
+		// If the search query has changed then we need to ensure
+		// the cursor is still pointing at a valid option.
+		if m.search.Value() != oldSearch {
+			options := m.filteredOptions()
+
+			if m.cursor > len(options)-1 {
+				m.cursor = max(0, len(options)-1)
+			}
+		}
+	}
+
+	return m, cmd
+}
+
+func (m selectModel) View() string {
+	var s strings.Builder
+
+	msg := pretty.Sprintf(pretty.Bold(), "? %s", m.message)
+
+	if m.selected != "" {
+		selected := pretty.Sprint(DefaultStyles.Keyword, m.selected)
+		_, _ = s.WriteString(fmt.Sprintf("%s %s\n", msg, selected))
+
+		return s.String()
+	}
+
+	if m.hideSearch {
+		_, _ = s.WriteString(fmt.Sprintf("%s [Use arrows to move]\n", msg))
+	} else {
+		_, _ = s.WriteString(fmt.Sprintf(
+			"%s %s[Use arrows to move, type to filter]\n",
+			msg,
+			m.search.View(),
+		))
+	}
+
+	options, start := m.viewableOptions()
+
+	for i, option := range options {
+		// Is this the currently selected option?
+		style := pretty.Wrap("  ", "")
+		if m.cursor == start+i {
+			style = pretty.Style{
+				pretty.Wrap("> ", ""),
+				DefaultStyles.Keyword,
+			}
+		}
+
+		_, _ = s.WriteString(pretty.Sprint(style, option))
+		_, _ = s.WriteString("\n")
+	}
+
+	return s.String()
+}
+
+func (m selectModel) viewableOptions() ([]string, int) {
+	options := m.filteredOptions()
+	halfHeight := m.height / 2
+	bottom := 0
+	top := len(options)
+
+	switch {
+	case m.cursor <= halfHeight:
+		top = min(top, m.height)
+	case m.cursor < top-halfHeight:
+		bottom = max(0, m.cursor-halfHeight)
+		top = min(top, m.cursor+halfHeight+1)
+	default:
+		bottom = max(0, top-m.height)
+	}
+
+	return options[bottom:top], bottom
+}
+
+func (m selectModel) filteredOptions() []string {
+	options := []string{}
+	for _, o := range m.options {
+		filter := strings.ToLower(m.search.Value())
+		option := strings.ToLower(o)
+
+		if strings.Contains(option, filter) {
+			options = append(options, o)
+		}
+	}
+	return options
+}
+
+type MultiSelectOptions struct {
+	Message           string
+	Options           []string
+	Defaults          []string
+	EnableCustomInput bool
+}
+
+func MultiSelect(inv *serpent.Invocation, opts MultiSelectOptions) ([]string, error) {
 	// Similar hack is applied to Select()
 	if flag.Lookup("test.v") != nil {
-		return items, nil
+		return opts.Defaults, nil
 	}
 
-	prompt := &survey.MultiSelect{
-		Options: items,
-		Default: items,
+	options := make([]*multiSelectOption, len(opts.Options))
+	for i, option := range opts.Options {
+		chosen := false
+		for _, d := range opts.Defaults {
+			if option == d {
+				chosen = true
+				break
+			}
+		}
+
+		options[i] = &multiSelectOption{
+			option: option,
+			chosen: chosen,
+		}
 	}
 
-	var values []string
-	err := survey.AskOne(prompt, &values, survey.WithStdio(fileReadWriter{
-		Reader: inv.Stdin,
-	}, fileReadWriter{
-		Writer: inv.Stdout,
-	}, inv.Stdout))
-	if errors.Is(err, terminal.InterruptErr) {
+	initialModel := multiSelectModel{
+		search:            textinput.New(),
+		options:           options,
+		message:           opts.Message,
+		enableCustomInput: opts.EnableCustomInput,
+	}
+
+	initialModel.search.Prompt = ""
+	initialModel.search.Focus()
+
+	p := tea.NewProgram(
+		initialModel,
+		tea.WithoutSignalHandler(),
+		tea.WithContext(inv.Context()),
+		tea.WithInput(inv.Stdin),
+		tea.WithOutput(inv.Stdout),
+	)
+
+	closeSignalHandler := installSignalHandler(p)
+	defer closeSignalHandler()
+
+	m, err := p.Run()
+	if err != nil {
+		return nil, err
+	}
+
+	model, ok := m.(multiSelectModel)
+	if !ok {
+		return nil, xerrors.New(fmt.Sprintf("unknown model found %T (%+v)", m, m))
+	}
+
+	if model.canceled {
 		return nil, Canceled
 	}
-	return values, err
+
+	return model.selectedOptions(), nil
 }
 
-type fileReadWriter struct {
-	io.Reader
-	io.Writer
+type multiSelectOption struct {
+	option string
+	chosen bool
 }
 
-func (f fileReadWriter) Fd() uintptr {
-	if file, ok := f.Reader.(*os.File); ok {
-		return file.Fd()
-	}
-	if file, ok := f.Writer.(*os.File); ok {
-		return file.Fd()
-	}
-	return 0
+type multiSelectModel struct {
+	search            textinput.Model
+	options           []*multiSelectOption
+	cursor            int
+	message           string
+	canceled          bool
+	selected          bool
+	isCustomInputMode bool   // track if we're adding a custom option
+	customInput       string // store custom input
+	enableCustomInput bool   // control whether custom input is allowed
+}
+
+func (multiSelectModel) Init() tea.Cmd {
+	return nil
+}
+
+//nolint:revive // For same reason as previous Update definition
+func (m multiSelectModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
+	var cmd tea.Cmd
+
+	if m.isCustomInputMode {
+		return m.handleCustomInputMode(msg)
+	}
+
+	switch msg := msg.(type) {
+	case terminateMsg:
+		m.canceled = true
+		return m, tea.Quit
+
+	case tea.KeyMsg:
+		switch msg.Type {
+		case tea.KeyCtrlC:
+			m.canceled = true
+			return m, tea.Quit
+
+		case tea.KeyEnter:
+			// Switch to custom input mode if we're on the "+ Add custom value:" option
+			if m.enableCustomInput && m.cursor == len(m.filteredOptions()) {
+				m.isCustomInputMode = true
+				return m, nil
+			}
+			if len(m.options) != 0 {
+				m.selected = true
+				return m, tea.Quit
+			}
+
+		case tea.KeySpace:
+			options := m.filteredOptions()
+			if len(options) != 0 {
+				options[m.cursor].chosen = !options[m.cursor].chosen
+			}
+			// We back out early here otherwise a space will be inserted
+			// into the search field.
+			return m, nil
+
+		case tea.KeyUp:
+			maxIndex := m.getMaxIndex()
+			if m.cursor > 0 {
+				m.cursor--
+			} else {
+				m.cursor = maxIndex
+			}
+
+		case tea.KeyDown:
+			maxIndex := m.getMaxIndex()
+			if m.cursor < maxIndex {
+				m.cursor++
+			} else {
+				m.cursor = 0
+			}
+
+		case tea.KeyRight:
+			options := m.filteredOptions()
+			for _, option := range options {
+				option.chosen = true
+			}
+
+		case tea.KeyLeft:
+			options := m.filteredOptions()
+			for _, option := range options {
+				option.chosen = false
+			}
+		}
+	}
+
+	oldSearch := m.search.Value()
+	m.search, cmd = m.search.Update(msg)
+
+	// If the search query has changed then we need to ensure
+	// the cursor is still pointing at a valid option.
+	if m.search.Value() != oldSearch {
+		options := m.filteredOptions()
+		if m.cursor > len(options)-1 {
+			m.cursor = max(0, len(options)-1)
+		}
+	}
+
+	return m, cmd
+}
+
+func (m multiSelectModel) getMaxIndex() int {
+	options := m.filteredOptions()
+	if m.enableCustomInput {
+		// Include the "+ Add custom value" entry
+		return len(options)
+	}
+	// Includes only the actual options
+	return len(options) - 1
+}
+
+// handleCustomInputMode manages keyboard interactions when in custom input mode
+func (m *multiSelectModel) handleCustomInputMode(msg tea.Msg) (tea.Model, tea.Cmd) {
+	keyMsg, ok := msg.(tea.KeyMsg)
+	if !ok {
+		return m, nil
+	}
+
+	switch keyMsg.Type {
+	case tea.KeyEnter:
+		return m.handleCustomInputSubmission()
+
+	case tea.KeyCtrlC:
+		m.canceled = true
+		return m, tea.Quit
+
+	case tea.KeyBackspace:
+		return m.handleCustomInputBackspace()
+
+	default:
+		m.customInput += keyMsg.String()
+		return m, nil
+	}
+}
+
+// handleCustomInputSubmission processes the submission of custom input
+func (m *multiSelectModel) handleCustomInputSubmission() (tea.Model, tea.Cmd) {
+	if m.customInput == "" {
+		m.isCustomInputMode = false
+		return m, nil
+	}
+
+	// Clear search to ensure option is visible and cursor points to the new option
+	m.search.SetValue("")
+
+	// Check for duplicates
+	for i, opt := range m.options {
+		if opt.option == m.customInput {
+			// If the option exists but isn't chosen, select it
+			if !opt.chosen {
+				opt.chosen = true
+			}
+
+			// Point cursor to the new option
+			m.cursor = i
+
+			// Reset custom input mode to disabled
+			m.isCustomInputMode = false
+			m.customInput = ""
+			return m, nil
+		}
+	}
+
+	// Add new unique option
+	m.options = append(m.options, &multiSelectOption{
+		option: m.customInput,
+		chosen: true,
+	})
+
+	// Point cursor to the newly added option
+	m.cursor = len(m.options) - 1
+
+	// Reset custom input mode to disabled
+	m.customInput = ""
+	m.isCustomInputMode = false
+	return m, nil
+}
+
+// handleCustomInputBackspace handles backspace in custom input mode
+func (m *multiSelectModel) handleCustomInputBackspace() (tea.Model, tea.Cmd) {
+	if len(m.customInput) > 0 {
+		m.customInput = m.customInput[:len(m.customInput)-1]
+	}
+	return m, nil
+}
+
+func (m multiSelectModel) View() string {
+	var s strings.Builder
+
+	msg := pretty.Sprintf(pretty.Bold(), "? %s", m.message)
+
+	if m.selected {
+		selected := pretty.Sprint(DefaultStyles.Keyword, strings.Join(m.selectedOptions(), ", "))
+		_, _ = s.WriteString(fmt.Sprintf("%s %s\n", msg, selected))
+
+		return s.String()
+	}
+
+	if m.isCustomInputMode {
+		_, _ = s.WriteString(fmt.Sprintf("%s\nEnter custom value: %s\n", msg, m.customInput))
+		return s.String()
+	}
+
+	_, _ = s.WriteString(fmt.Sprintf(
+		"%s %s[Use arrows to move, space to select, <right> to all, <left> to none, type to filter]\n",
+		msg,
+		m.search.View(),
+	))
+
+	options := m.filteredOptions()
+	for i, option := range options {
+		cursor := "  "
+		chosen := "[ ]"
+		o := option.option
+
+		if m.cursor == i {
+			cursor = pretty.Sprint(DefaultStyles.Keyword, "> ")
+			chosen = pretty.Sprint(DefaultStyles.Keyword, "[ ]")
+			o = pretty.Sprint(DefaultStyles.Keyword, o)
+		}
+
+		if option.chosen {
+			chosen = pretty.Sprint(DefaultStyles.Keyword, "[x]")
+		}
+
+		_, _ = s.WriteString(fmt.Sprintf(
+			"%s%s %s\n",
+			cursor,
+			chosen,
+			o,
+		))
+	}
+
+	if m.enableCustomInput {
+		// Add the "+ Add custom value" option at the bottom
+		cursor := "  "
+		text := " + Add custom value"
+		if m.cursor == len(options) {
+			cursor = pretty.Sprint(DefaultStyles.Keyword, "> ")
+			text = pretty.Sprint(DefaultStyles.Keyword, text)
+		}
+		_, _ = s.WriteString(fmt.Sprintf("%s%s\n", cursor, text))
+	}
+	return s.String()
+}
+
+func (m multiSelectModel) filteredOptions() []*multiSelectOption {
+	options := []*multiSelectOption{}
+	for _, o := range m.options {
+		filter := strings.ToLower(m.search.Value())
+		option := strings.ToLower(o.option)
+
+		if strings.Contains(option, filter) {
+			options = append(options, o)
+		}
+	}
+	return options
+}
+
+func (m multiSelectModel) selectedOptions() []string {
+	selected := []string{}
+	for _, o := range m.options {
+		if o.chosen {
+			selected = append(selected, o.option)
+		}
+	}
+	return selected
 }

cli/cliui/select_test.go

@@ -101,13 +101,49 @@ func TestMultiSelect(t *testing.T) {
 		}()
 		require.Equal(t, items, <-msgChan)
 	})
+
+	t.Run("MultiSelectWithCustomInput", func(t *testing.T) {
+		t.Parallel()
+		items := []string{"Code", "Chairs", "Whale", "Diamond", "Carrot"}
+		ptty := ptytest.New(t)
+		msgChan := make(chan []string)
+		go func() {
+			resp, err := newMultiSelectWithCustomInput(ptty, items)
+			assert.NoError(t, err)
+			msgChan <- resp
+		}()
+		require.Equal(t, items, <-msgChan)
+	})
 }
 
-func newMultiSelect(ptty *ptytest.PTY, items []string) ([]string, error) {
+func newMultiSelectWithCustomInput(ptty *ptytest.PTY, items []string) ([]string, error) {
 	var values []string
 	cmd := &serpent.Command{
 		Handler: func(inv *serpent.Invocation) error {
-			selectedItems, err := cliui.MultiSelect(inv, items)
+			selectedItems, err := cliui.MultiSelect(inv, cliui.MultiSelectOptions{
+				Options:           items,
+				Defaults:          items,
+				EnableCustomInput: true,
+			})
+			if err == nil {
+				values = selectedItems
+			}
+			return err
+		},
+	}
+	inv := cmd.Invoke()
+	ptty.Attach(inv)
+	return values, inv.Run()
+}
+
+func newMultiSelect(ptty *ptytest.PTY, items []string) ([]string, error) {
+	var values []string
+	cmd := &serpent.Command{
+		Handler: func(inv *serpent.Invocation) error {
+			selectedItems, err := cliui.MultiSelect(inv, cliui.MultiSelectOptions{
+				Options:  items,
+				Defaults: items,
+			})
 			if err == nil {
 				values = selectedItems
 			}

cli/cliui/table.go

@@ -22,6 +22,13 @@ func Table() table.Writer {
 	return tableWriter
 }
 
+// This type can be supplied as part of a slice to DisplayTable
+// or to a `TableFormat` `Format` call to render a separator.
+// Leading separators are not supported and trailing separators
+// are ignored by the table formatter.
+// e.g. `[]any{someRow, TableSeparator, someRow}`
+type TableSeparator struct{}
+
 // filterTableColumns returns configurations to hide columns
 // that are not provided in the array. If the array is empty,
 // no filtering will occur!
@@ -47,8 +54,12 @@ func filterTableColumns(header table.Row, columns []string) []table.ColumnConfig
 	return columnConfigs
 }
 
-// DisplayTable renders a table as a string. The input argument must be a slice
-// of structs. At least one field in the struct must have a `table:""` tag
+// DisplayTable renders a table as a string. The input argument can be:
+//   - a struct slice.
+//   - an interface slice, where the first element is a struct,
+//     and all other elements are of the same type, or a TableSeparator.
+//
+// At least one field in the struct must have a `table:""` tag
 // containing the name of the column in the outputted table.
 //
 // If `sort` is not specified, the field with the `table:"$NAME,default_sort"`
@@ -66,11 +77,20 @@ func DisplayTable(out any, sort string, filterColumns []string) (string, error)
 	v := reflect.Indirect(reflect.ValueOf(out))
 
 	if v.Kind() != reflect.Slice {
-		return "", xerrors.Errorf("DisplayTable called with a non-slice type")
+		return "", xerrors.New("DisplayTable called with a non-slice type")
+	}
+	var tableType reflect.Type
+	if v.Type().Elem().Kind() == reflect.Interface {
+		if v.Len() == 0 {
+			return "", xerrors.New("DisplayTable called with empty interface slice")
+		}
+		tableType = reflect.Indirect(reflect.ValueOf(v.Index(0).Interface())).Type()
+	} else {
+		tableType = v.Type().Elem()
 	}
 
 	// Get the list of table column headers.
-	headersRaw, defaultSort, err := typeToTableHeaders(v.Type().Elem(), true)
+	headersRaw, defaultSort, err := typeToTableHeaders(tableType, true)
 	if err != nil {
 		return "", xerrors.Errorf("get table headers recursively for type %q: %w", v.Type().Elem().String(), err)
 	}
@@ -82,9 +102,8 @@ func DisplayTable(out any, sort string, filterColumns []string) (string, error)
 	}
 	headers := make(table.Row, len(headersRaw))
 	for i, header := range headersRaw {
-		headers[i] = header
+		headers[i] = strings.ReplaceAll(header, "_", " ")
 	}
-
 	// Verify that the given sort column and filter columns are valid.
 	if sort != "" || len(filterColumns) != 0 {
 		headersMap := make(map[string]string, len(headersRaw))
@@ -130,6 +149,11 @@ func DisplayTable(out any, sort string, filterColumns []string) (string, error)
 			return "", xerrors.Errorf("specified sort column %q not found in table headers, available columns are %q", sort, strings.Join(headersRaw, `", "`))
 		}
 	}
+	return renderTable(out, sort, headers, filterColumns)
+}
+
+func renderTable(out any, sort string, headers table.Row, filterColumns []string) (string, error) {
+	v := reflect.Indirect(reflect.ValueOf(out))
 
 	// Setup the table formatter.
 	tw := Table()
@@ -143,15 +167,22 @@ func DisplayTable(out any, sort string, filterColumns []string) (string, error)
 
 	// Write each struct to the table.
 	for i := 0; i < v.Len(); i++ {
+		cur := v.Index(i).Interface()
+		_, ok := cur.(TableSeparator)
+		if ok {
+			tw.AppendSeparator()
+			continue
+		}
 		// Format the row as a slice.
-		rowMap, err := valueToTableMap(v.Index(i))
+		// ValueToTableMap does what `reflect.Indirect` does
+		rowMap, err := valueToTableMap(reflect.ValueOf(cur))
 		if err != nil {
 			return "", xerrors.Errorf("get table row map %v: %w", i, err)
 		}
 
 		rowSlice := make([]any, len(headers))
-		for i, h := range headersRaw {
-			v, ok := rowMap[h]
+		for i, h := range headers {
+			v, ok := rowMap[h.(string)]
 			if !ok {
 				v = nil
 			}
@@ -168,12 +199,34 @@ func DisplayTable(out any, sort string, filterColumns []string) (string, error)
 				if val != nil {
 					v = *val
 				}
+			case *time.Duration:
+				if val != nil {
+					v = val.String()
+				}
 			case fmt.Stringer:
 				if val != nil {
 					v = val.String()
 				}
 			}
 
+			// Guard against nil dereferences
+			if v != nil {
+				rt := reflect.TypeOf(v)
+				switch rt.Kind() {
+				case reflect.Slice:
+					// By default, the behavior is '%v', which just returns a string like
+					// '[a b c]'. This will add commas in between each value.
+					strs := make([]string, 0)
+					vt := reflect.ValueOf(v)
+					for i := 0; i < vt.Len(); i++ {
+						strs = append(strs, fmt.Sprintf("%v", vt.Index(i).Interface()))
+					}
+					v = "[" + strings.Join(strs, ", ") + "]"
+				default:
+					// Leave it as it is
+				}
+			}
+
 			rowSlice[i] = v
 		}
 
@@ -188,25 +241,28 @@ func DisplayTable(out any, sort string, filterColumns []string) (string, error)
 // returned. If the table tag is malformed, an error is returned.
 //
 // The returned name is transformed from "snake_case" to "normal text".
-func parseTableStructTag(field reflect.StructField) (name string, defaultSort, recursive bool, skipParentName bool, err error) {
+func parseTableStructTag(field reflect.StructField) (name string, defaultSort, noSortOpt, recursive, skipParentName bool, err error) {
 	tags, err := structtag.Parse(string(field.Tag))
 	if err != nil {
-		return "", false, false, false, xerrors.Errorf("parse struct field tag %q: %w", string(field.Tag), err)
+		return "", false, false, false, false, xerrors.Errorf("parse struct field tag %q: %w", string(field.Tag), err)
 	}
 
 	tag, err := tags.Get("table")
 	if err != nil || tag.Name == "-" {
 		// tags.Get only returns an error if the tag is not found.
-		return "", false, false, false, nil
+		return "", false, false, false, false, nil
 	}
 
 	defaultSortOpt := false
+	noSortOpt = false
 	recursiveOpt := false
 	skipParentNameOpt := false
 	for _, opt := range tag.Options {
 		switch opt {
 		case "default_sort":
 			defaultSortOpt = true
+		case "nosort":
+			noSortOpt = true
 		case "recursive":
 			recursiveOpt = true
 		case "recursive_inline":
@@ -216,11 +272,11 @@ func parseTableStructTag(field reflect.StructField) (name string, defaultSort, r
 			recursiveOpt = true
 			skipParentNameOpt = true
 		default:
-			return "", false, false, false, xerrors.Errorf("unknown option %q in struct field tag", opt)
+			return "", false, false, false, false, xerrors.Errorf("unknown option %q in struct field tag", opt)
 		}
 	}
 
-	return strings.ReplaceAll(tag.Name, "_", " "), defaultSortOpt, recursiveOpt, skipParentNameOpt, nil
+	return strings.ReplaceAll(tag.Name, "_", " "), defaultSortOpt, noSortOpt, recursiveOpt, skipParentNameOpt, nil
 }
 
 func isStructOrStructPointer(t reflect.Type) bool {
@@ -244,12 +300,16 @@ func typeToTableHeaders(t reflect.Type, requireDefault bool) ([]string, string,
 
 	headers := []string{}
 	defaultSortName := ""
+	noSortOpt := false
 	for i := 0; i < t.NumField(); i++ {
 		field := t.Field(i)
-		name, defaultSort, recursive, skip, err := parseTableStructTag(field)
+		name, defaultSort, noSort, recursive, skip, err := parseTableStructTag(field)
 		if err != nil {
 			return nil, "", xerrors.Errorf("parse struct tags for field %q in type %q: %w", field.Name, t.String(), err)
 		}
+		if requireDefault && noSort {
+			noSortOpt = true
+		}
 
 		if name == "" && (recursive && skip) {
 			return nil, "", xerrors.Errorf("a name is required for the field %q. "+
@@ -292,8 +352,8 @@ func typeToTableHeaders(t reflect.Type, requireDefault bool) ([]string, string,
 		headers = append(headers, name)
 	}
 
-	if defaultSortName == "" && requireDefault {
-		return nil, "", xerrors.Errorf("no field marked as default_sort in type %q", t.String())
+	if defaultSortName == "" && requireDefault && !noSortOpt {
+		return nil, "", xerrors.Errorf("no field marked as default_sort or nosort in type %q", t.String())
 	}
 
 	return headers, defaultSortName, nil
@@ -320,7 +380,7 @@ func valueToTableMap(val reflect.Value) (map[string]any, error) {
 	for i := 0; i < val.NumField(); i++ {
 		field := val.Type().Field(i)
 		fieldVal := val.Field(i)
-		name, _, recursive, skip, err := parseTableStructTag(field)
+		name, _, _, recursive, skip, err := parseTableStructTag(field)
 		if err != nil {
 			return nil, xerrors.Errorf("parse struct tags for field %q in type %T: %w", field.Name, val, err)
 		}

cli/cliui/table_test.go

@@ -138,10 +138,10 @@ func Test_DisplayTable(t *testing.T) {
 		t.Parallel()
 
 		expected := `
-NAME  AGE  ROLES    SUB 1 NAME  SUB 1 AGE  SUB 2 NAME  SUB 2 AGE  SUB 3 INNER NAME  SUB 3 INNER AGE  SUB 4       TIME                  TIME PTR
-bar    20  [a]      bar1               21  <nil>       <nil>      bar3                           23  {bar4 24 }  2022-08-02T15:49:10Z  <nil>
-baz    30  []       baz1               31  <nil>       <nil>      baz3                           33  {baz4 34 }  2022-08-02T15:49:10Z  <nil>
-foo    10  [a b c]  foo1               11  foo2        12         foo3                           13  {foo4 14 }  2022-08-02T15:49:10Z  2022-08-02T15:49:10Z
+NAME  AGE  ROLES      SUB 1 NAME  SUB 1 AGE  SUB 2 NAME  SUB 2 AGE  SUB 3 INNER NAME  SUB 3 INNER AGE  SUB 4       TIME                  TIME PTR
+bar    20  [a]        bar1               21  <nil>       <nil>      bar3                           23  {bar4 24 }  2022-08-02T15:49:10Z  <nil>
+baz    30  []         baz1               31  <nil>       <nil>      baz3                           33  {baz4 34 }  2022-08-02T15:49:10Z  <nil>
+foo    10  [a, b, c]  foo1               11  foo2        12         foo3                           13  {foo4 14 }  2022-08-02T15:49:10Z  2022-08-02T15:49:10Z
 		`
 
 		// Test with non-pointer values.
@@ -165,10 +165,10 @@ foo    10  [a b c]  foo1               11  foo2        12         foo3
 		t.Parallel()
 
 		expected := `
-NAME  AGE  ROLES    SUB 1 NAME  SUB 1 AGE  SUB 2 NAME  SUB 2 AGE  SUB 3 INNER NAME  SUB 3 INNER AGE  SUB 4       TIME                  TIME PTR
-foo    10  [a b c]  foo1               11  foo2        12         foo3                           13  {foo4 14 }  2022-08-02T15:49:10Z  2022-08-02T15:49:10Z
-bar    20  [a]      bar1               21  <nil>       <nil>      bar3                           23  {bar4 24 }  2022-08-02T15:49:10Z  <nil>
-baz    30  []       baz1               31  <nil>       <nil>      baz3                           33  {baz4 34 }  2022-08-02T15:49:10Z  <nil>
+NAME  AGE  ROLES      SUB 1 NAME  SUB 1 AGE  SUB 2 NAME  SUB 2 AGE  SUB 3 INNER NAME  SUB 3 INNER AGE  SUB 4       TIME                  TIME PTR
+foo    10  [a, b, c]  foo1               11  foo2        12         foo3                           13  {foo4 14 }  2022-08-02T15:49:10Z  2022-08-02T15:49:10Z
+bar    20  [a]        bar1               21  <nil>       <nil>      bar3                           23  {bar4 24 }  2022-08-02T15:49:10Z  <nil>
+baz    30  []         baz1               31  <nil>       <nil>      baz3                           33  {baz4 34 }  2022-08-02T15:49:10Z  <nil>
 		`
 
 		out, err := cliui.DisplayTable(in, "age", nil)
@@ -218,6 +218,42 @@ Alice   25
 		compareTables(t, expected, out)
 	})
 
+	// This test ensures we can display dynamically typed slices
+	t.Run("Interfaces", func(t *testing.T) {
+		t.Parallel()
+
+		in := []any{tableTest1{}}
+		out, err := cliui.DisplayTable(in, "", nil)
+		t.Log("rendered table:\n" + out)
+		require.NoError(t, err)
+		other := []tableTest1{{}}
+		expected, err := cliui.DisplayTable(other, "", nil)
+		require.NoError(t, err)
+		compareTables(t, expected, out)
+	})
+
+	t.Run("WithSeparator", func(t *testing.T) {
+		t.Parallel()
+		expected := `
+NAME  AGE  ROLES      SUB 1 NAME  SUB 1 AGE  SUB 2 NAME  SUB 2 AGE  SUB 3 INNER NAME  SUB 3 INNER AGE  SUB 4       TIME                  TIME PTR
+bar    20  [a]        bar1               21  <nil>       <nil>      bar3                           23  {bar4 24 }  2022-08-02T15:49:10Z  <nil>
+---------------------------------------------------------------------------------------------------------------------------------------------------------------
+baz    30  []         baz1               31  <nil>       <nil>      baz3                           33  {baz4 34 }  2022-08-02T15:49:10Z  <nil>
+---------------------------------------------------------------------------------------------------------------------------------------------------------------
+foo    10  [a, b, c]  foo1               11  foo2        12         foo3                           13  {foo4 14 }  2022-08-02T15:49:10Z  2022-08-02T15:49:10Z
+		`
+
+		var inlineIn []any
+		for _, v := range in {
+			inlineIn = append(inlineIn, v)
+			inlineIn = append(inlineIn, cliui.TableSeparator{})
+		}
+		out, err := cliui.DisplayTable(inlineIn, "", nil)
+		t.Log("rendered table:\n" + out)
+		require.NoError(t, err)
+		compareTables(t, expected, out)
+	})
+
 	// This test ensures that safeties against invalid use of `table` tags
 	// causes errors (even without data).
 	t.Run("Errors", func(t *testing.T) {
@@ -255,14 +291,6 @@ Alice   25
 				_, err := cliui.DisplayTable(in, "", nil)
 				require.Error(t, err)
 			})
-
-			t.Run("WithData", func(t *testing.T) {
-				t.Parallel()
-
-				in := []any{tableTest1{}}
-				_, err := cliui.DisplayTable(in, "", nil)
-				require.Error(t, err)
-			})
 		})
 
 		t.Run("NotStruct", func(t *testing.T) {

cli/cliutil/awscheck.go

@@ -0,0 +1,114 @@
+package cliutil
+
+import (
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+	"net/netip"
+	"time"
+
+	"golang.org/x/xerrors"
+)
+
+const AWSIPRangesURL = "https://ip-ranges.amazonaws.com/ip-ranges.json"
+
+type awsIPv4Prefix struct {
+	Prefix             string `json:"ip_prefix"`
+	Region             string `json:"region"`
+	Service            string `json:"service"`
+	NetworkBorderGroup string `json:"network_border_group"`
+}
+
+type awsIPv6Prefix struct {
+	Prefix             string `json:"ipv6_prefix"`
+	Region             string `json:"region"`
+	Service            string `json:"service"`
+	NetworkBorderGroup string `json:"network_border_group"`
+}
+
+type AWSIPRanges struct {
+	V4 []netip.Prefix
+	V6 []netip.Prefix
+}
+
+type awsIPRangesResponse struct {
+	SyncToken    string          `json:"syncToken"`
+	CreateDate   string          `json:"createDate"`
+	IPV4Prefixes []awsIPv4Prefix `json:"prefixes"`
+	IPV6Prefixes []awsIPv6Prefix `json:"ipv6_prefixes"`
+}
+
+func FetchAWSIPRanges(ctx context.Context, url string) (*AWSIPRanges, error) {
+	client := &http.Client{}
+	reqCtx, reqCancel := context.WithTimeout(ctx, 5*time.Second)
+	defer reqCancel()
+	req, _ := http.NewRequestWithContext(reqCtx, http.MethodGet, url, nil)
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		b, _ := io.ReadAll(resp.Body)
+		return nil, xerrors.Errorf("unexpected status code %d: %s", resp.StatusCode, b)
+	}
+
+	var body awsIPRangesResponse
+	err = json.NewDecoder(resp.Body).Decode(&body)
+	if err != nil {
+		return nil, xerrors.Errorf("json decode: %w", err)
+	}
+
+	out := &AWSIPRanges{
+		V4: make([]netip.Prefix, 0, len(body.IPV4Prefixes)),
+		V6: make([]netip.Prefix, 0, len(body.IPV6Prefixes)),
+	}
+
+	for _, p := range body.IPV4Prefixes {
+		prefix, err := netip.ParsePrefix(p.Prefix)
+		if err != nil {
+			return nil, xerrors.Errorf("parse ip prefix: %w", err)
+		}
+		if prefix.Addr().Is6() {
+			return nil, xerrors.Errorf("ipv4 prefix contains ipv6 address: %s", p.Prefix)
+		}
+		out.V4 = append(out.V4, prefix)
+	}
+
+	for _, p := range body.IPV6Prefixes {
+		prefix, err := netip.ParsePrefix(p.Prefix)
+		if err != nil {
+			return nil, xerrors.Errorf("parse ip prefix: %w", err)
+		}
+		if prefix.Addr().Is4() {
+			return nil, xerrors.Errorf("ipv6 prefix contains ipv4 address: %s", p.Prefix)
+		}
+		out.V6 = append(out.V6, prefix)
+	}
+
+	return out, nil
+}
+
+// CheckIP checks if the given IP address is an AWS IP.
+func (r *AWSIPRanges) CheckIP(ip netip.Addr) bool {
+	if ip.IsLoopback() || ip.IsLinkLocalMulticast() || ip.IsLinkLocalUnicast() || ip.IsPrivate() {
+		return false
+	}
+
+	if ip.Is4() {
+		for _, p := range r.V4 {
+			if p.Contains(ip) {
+				return true
+			}
+		}
+	} else {
+		for _, p := range r.V6 {
+			if p.Contains(ip) {
+				return true
+			}
+		}
+	}
+	return false
+}