fix(scripts/release): handle cherry-pick bot titles in check commit metadata (cherry-pick #17535) (#17537)

Co-authored-by: Mathias Fredriksson <mafredri@gmail.com>

.devcontainer/devcontainer.json

@@ -1,13 +1,13 @@
 {
-  "name": "Development environments on your infrastructure",
-  "image": "codercom/oss-dogfood:latest",
+	"name": "Development environments on your infrastructure",
+	"image": "codercom/oss-dogfood:latest",
 
-  "features": {
-    // See all possible options here https://github.com/devcontainers/features/tree/main/src/docker-in-docker
-    "ghcr.io/devcontainers/features/docker-in-docker:2": {
-      "moby": "false"
-    }
-  },
-  // SYS_PTRACE to enable go debugging
-  "runArgs": ["--cap-add=SYS_PTRACE"]
+	"features": {
+		// See all possible options here https://github.com/devcontainers/features/tree/main/src/docker-in-docker
+		"ghcr.io/devcontainers/features/docker-in-docker:2": {
+			"moby": "false"
+		}
+	},
+	// SYS_PTRACE to enable go debugging
+	"runArgs": ["--cap-add=SYS_PTRACE"]
 }

.dockerignore

@@ -1,6 +0,0 @@
-# Ignore all files and folders
-**
-
-# Include flake.nix and flake.lock
-!flake.nix
-!flake.lock

.editorconfig

@@ -7,7 +7,7 @@ trim_trailing_whitespace = true
 insert_final_newline = true
 indent_style = tab
 
-[*.{md,json,yaml,yml,tf,tfvars,nix}]
+[*.{yaml,yml,tf,tfvars,nix}]
 indent_style = space
 indent_size = 2
 

.git-blame-ignore-revs

@@ -3,3 +3,5 @@
 
 # chore: format code with semicolons when using prettier (#9555)
 988c9af0153561397686c119da9d1336d2433fdd
+# chore: use tabs for prettier and biome (#14283)
+95a7c0c4f087744a22c2e88dd3c5d30024d5fb02

.gitattributes

@@ -1,7 +1,7 @@
 # Generated files
 coderd/apidoc/docs.go linguist-generated=true
-docs/api/*.md linguist-generated=true
-docs/cli/*.md linguist-generated=true
+docs/reference/api/*.md linguist-generated=true
+docs/reference/cli/*.md linguist-generated=true
 coderd/apidoc/swagger.json linguist-generated=true
 coderd/database/dump.sql linguist-generated=true
 peerbroker/proto/*.go linguist-generated=true

.github/.linkspector.yml

@@ -0,0 +1,24 @@
+dirs:
+  - docs
+excludedDirs:
+  # Downstream bug in linkspector means large markdown files fail to parse
+  # but these are autogenerated and shouldn't need checking
+  - docs/reference
+  # Older changelogs may contain broken links
+  - docs/changelogs
+ignorePatterns:
+  - pattern: "localhost"
+  - pattern: "example.com"
+  - pattern: "mailto:"
+  - pattern: "127.0.0.1"
+  - pattern: "0.0.0.0"
+  - pattern: "JFROG_URL"
+  - pattern: "coder.company.org"
+  # These real sites were blocking the linkspector action / GitHub runner IPs(?)
+  - pattern: "i.imgur.com"
+  - pattern: "code.visualstudio.com"
+  - pattern: "www.emacswiki.org"
+  - pattern: "linux.die.net/man"
+  - pattern: "www.gnu.org"
+aliveStatusCodes:
+  - 200

.github/actions/setup-go/action.yaml

@@ -4,12 +4,12 @@ description: |
 inputs:
   version:
     description: "The Go version to use."
-    default: "1.22.4"
+    default: "1.22.8"
 runs:
   using: "composite"
   steps:
     - name: Setup Go
-      uses: actions/setup-go@v5
+      uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
       with:
         go-version: ${{ inputs.version }}
 

.github/actions/setup-imdisk/action.yaml

@@ -0,0 +1,27 @@
+name: "Setup ImDisk"
+if: runner.os == 'Windows'
+description: |
+  Sets up the ImDisk toolkit for Windows and creates a RAM disk on drive R:.
+runs:
+  using: "composite"
+  steps:
+    - name: Download ImDisk
+      if: runner.os == 'Windows'
+      shell: bash
+      run: |
+        mkdir imdisk
+        cd imdisk
+        curl -L -o files.cab https://github.com/coder/imdisk-artifacts/raw/92a17839ebc0ee3e69be019f66b3e9b5d2de4482/files.cab
+        curl -L -o install.bat https://github.com/coder/imdisk-artifacts/raw/92a17839ebc0ee3e69be019f66b3e9b5d2de4482/install.bat
+        cd ..
+
+    - name: Install ImDisk
+      shell: cmd
+      run: |
+        cd imdisk
+        install.bat /silent
+
+    - name: Create RAM Disk
+      shell: cmd
+      run: |
+        imdisk -a -s 4096M -m R: -p "/fs:ntfs /q /y"

.github/actions/setup-node/action.yaml

@@ -11,16 +11,16 @@ runs:
   using: "composite"
   steps:
     - name: Install pnpm
-      uses: pnpm/action-setup@v3
-      with:
-        version: 8
+      uses: pnpm/action-setup@fe02b34f77f8bc703788d5817da081398fad5dd2 # v4.0.0
+
     - name: Setup Node
-      uses: actions/setup-node@v4.0.1
+      uses: actions/setup-node@0a44ba7841725637a19e28fa30b79a866c81b0a6 # v4.0.4
       with:
-        node-version: 18.19.0
+        node-version: 20.16.0
         # See https://github.com/actions/setup-node#caching-global-packages-data
         cache: "pnpm"
         cache-dependency-path: ${{ inputs.directory }}/pnpm-lock.yaml
+
     - name: Install root node_modules
       shell: bash
       run: ./scripts/pnpm_install.sh

.github/actions/setup-sqlc/action.yaml

@@ -5,6 +5,6 @@ runs:
   using: "composite"
   steps:
     - name: Setup sqlc
-      uses: sqlc-dev/setup-sqlc@v4
+      uses: sqlc-dev/setup-sqlc@c0209b9199cd1cce6a14fc27cabcec491b651761 # v4.0.0
       with:
-        sqlc-version: "1.25.0"
+        sqlc-version: "1.27.0"

.github/actions/setup-tf/action.yaml

@@ -5,7 +5,7 @@ runs:
   using: "composite"
   steps:
     - name: Install Terraform
-      uses: hashicorp/setup-terraform@v3
+      uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd # v3.1.2
       with:
-        terraform_version: 1.8.4
+        terraform_version: 1.10.5
         terraform_wrapper: false

.github/actions/upload-datadog/action.yaml

@@ -1,5 +1,6 @@
 name: Upload tests to datadog
-if: always()
+description: |
+  Uploads the test results to datadog.
 inputs:
   api-key:
     description: "Datadog API key"

.github/cherry-pick-bot.yml

@@ -0,0 +1,2 @@
+enabled: true
+preservePullRequestTitle: true

.github/dependabot.yaml

@@ -39,6 +39,10 @@ updates:
       prefix: "chore"
     labels: []
     open-pull-requests-limit: 15
+    groups:
+      x:
+        patterns:
+          - "golang.org/x/*"
     ignore:
       # Ignore patch updates for all dependencies
       - dependency-name: "*"
@@ -47,7 +51,13 @@ updates:
 
   # Update our Dockerfile.
   - package-ecosystem: "docker"
-    directory: "/scripts/"
+    directories:
+      - "/dogfood/contents"
+      - "/scripts"
+      - "/examples/templates/docker/build"
+      - "/examples/parameters/build"
+      - "/scaletest/templates/scaletest-runner"
+      - "/scripts/ironbank"
     schedule:
       interval: "weekly"
       time: "06:00"
@@ -64,6 +74,9 @@ updates:
     directories:
       - "/site"
       - "/offlinedocs"
+      - "/scripts"
+      - "/scripts/apidocgen"
+
     schedule:
       interval: "monthly"
       time: "06:00"
@@ -73,14 +86,35 @@ updates:
     commit-message:
       prefix: "chore"
     labels: []
+    groups:
+      xterm:
+        patterns:
+          - "@xterm*"
+      mui:
+        patterns:
+          - "@mui*"
+      react:
+        patterns:
+          - "react"
+          - "react-dom"
+          - "@types/react"
+          - "@types/react-dom"
+      emotion:
+        patterns:
+          - "@emotion*"
+        exclude-patterns:
+          - "jest-runner-eslint"
+      jest:
+        patterns:
+          - "jest"
+          - "@types/jest"
+      vite:
+        patterns:
+          - "vite*"
+          - "@vitejs/plugin-react"
     ignore:
-      # Ignore patch updates for all dependencies
+      # Ignore major version updates to avoid breaking changes
       - dependency-name: "*"
-        update-types:
-          - version-update:semver-patch
-      # Ignore major updates to Node.js types, because they need to
-      # correspond to the Node.js engine version
-      - dependency-name: "@types/node"
         update-types:
           - version-update:semver-major
     open-pull-requests-limit: 15

.github/workflows/contrib.yaml

@@ -3,7 +3,7 @@ name: contrib
 on:
   issue_comment:
     types: [created]
-  pull_request_target:
+  pull_request:
     types:
       - opened
       - closed
@@ -13,32 +13,103 @@ on:
       - opened
       - reopened
       - edited
+      # For jobs that don't run on draft PRs.
+      - ready_for_review
+
+permissions:
+  contents: read
 
 # Only run one instance per PR to ensure in-order execution.
 concurrency: pr-${{ github.ref }}
 
 jobs:
   # Dependabot is annoying, but this makes it a bit less so.
-  auto-approve-dependabot:
+  dependabot-automerge:
     runs-on: ubuntu-latest
-    if: github.event_name == 'pull_request_target'
+    if: github.event_name == 'pull_request' && github.event.pull_request.user.login == 'dependabot[bot]' && github.repository == 'coder/coder'
     permissions:
       pull-requests: write
+      contents: write
     steps:
-      - name: auto-approve dependabot
-        uses: hmarr/auto-approve-action@v4
-        if: github.actor == 'dependabot[bot]'
+      - name: Dependabot metadata
+        id: metadata
+        uses: dependabot/fetch-metadata@d7267f607e9d3fb96fc2fbe83e0af444713e90b7 # v2.3.0
+        with:
+          github-token: "${{ secrets.GITHUB_TOKEN }}"
+
+      - name: Approve the PR
+        run: gh pr review --approve "$PR_URL"
+        env:
+          PR_URL: ${{github.event.pull_request.html_url}}
+          GH_TOKEN: ${{secrets.GITHUB_TOKEN}}
+
+      - name: Enable auto-merge for Dependabot PRs
+        run: gh pr merge --auto --squash "$PR_URL"
+        env:
+          PR_URL: ${{github.event.pull_request.html_url}}
+          GH_TOKEN: ${{secrets.GITHUB_TOKEN}}
+
+  dependabot-automerge-notify:
+    # Send a slack notification when a dependabot PR is merged.
+    runs-on: ubuntu-latest
+    if: github.event_name == 'pull_request' && github.event.pull_request.user.login == 'dependabot[bot]' && github.repository == 'coder/coder' && github.event.pull_request.merged
+    steps:
+      - name: Send Slack notification
+        env:
+          PR_URL: ${{github.event.pull_request.html_url}}
+          PR_TITLE: ${{github.event.pull_request.title}}
+          PR_NUMBER: ${{github.event.pull_request.number}}
+        run: |
+          curl -X POST -H 'Content-type: application/json' \
+          --data '{
+            "username": "dependabot",
+            "icon_url": "https://avatars.githubusercontent.com/u/27347476",
+            "blocks": [
+              {
+                "type": "header",
+                "text": {
+                  "type": "plain_text",
+                  "text": ":pr-merged: Auto merged Dependabot PR #${{ env.PR_NUMBER }}",
+                  "emoji": true
+                }
+              },
+              {
+                "type": "section",
+                "fields": [
+                  {
+                    "type": "mrkdwn",
+                    "text": "${{ env.PR_TITLE }}"
+                  }
+                ]
+              },
+              {
+                "type": "actions",
+                "elements": [
+                  {
+                    "type": "button",
+                    "text": {
+                      "type": "plain_text",
+                      "text": "View PR"
+                    },
+                    "url": "${{ env.PR_URL }}"
+                  }
+                ]
+              }
+            ]
+          }' ${{ secrets.DEPENDABOT_PRS_SLACK_WEBHOOK }}
 
   cla:
     runs-on: ubuntu-latest
+    permissions:
+      pull-requests: write
     steps:
       - name: cla
-        if: (github.event.comment.body == 'recheck' || github.event.comment.body == 'I have read the CLA Document and I hereby sign the CLA') || github.event_name == 'pull_request_target'
-        uses: contributor-assistant/github-action@v2.4.0
+        if: (github.event.comment.body == 'recheck' || github.event.comment.body == 'I have read the CLA Document and I hereby sign the CLA') || github.event_name == 'pull_request'
+        uses: contributor-assistant/github-action@ca4a40a7d1004f18d9960b404b97e5f30a505a08 # v2.6.1
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           # the below token should have repo scope and must be manually added by you in the repository's secret
-          PERSONAL_ACCESS_TOKEN: ${{ secrets.CDRCOMMUNITY_GITHUB_TOKEN }}
+          PERSONAL_ACCESS_TOKEN: ${{ secrets.CDRCI2_GITHUB_TOKEN }}
         with:
           remote-organization-name: "coder"
           remote-repository-name: "cla"
@@ -52,10 +123,10 @@ jobs:
   release-labels:
     runs-on: ubuntu-latest
     # Skip tagging for draft PRs.
-    if: ${{ github.event_name == 'pull_request_target' && success() && !github.event.pull_request.draft }}
+    if: ${{ github.event_name == 'pull_request' && !github.event.pull_request.draft }}
     steps:
       - name: release-labels
-        uses: actions/github-script@v7
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
         with:
           # This script ensures PR title and labels are in sync:
           #

.github/workflows/docker-base.yaml

@@ -22,10 +22,6 @@ on:
 
 permissions:
   contents: read
-  # Necessary to push docker images to ghcr.io.
-  packages: write
-  # Necessary for depot.dev authentication.
-  id-token: write
 
 # Avoid running multiple jobs for the same commit.
 concurrency:
@@ -33,14 +29,24 @@ concurrency:
 
 jobs:
   build:
+    permissions:
+      # Necessary for depot.dev authentication.
+      id-token: write
+      # Necessary to push docker images to ghcr.io.
+      packages: write
     runs-on: ubuntu-latest
     if: github.repository_owner == 'coder'
     steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@cb605e52c26070c328afc4562f0b4ada7618a84e # v2.10.4
+        with:
+          egress-policy: audit
+
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
 
       - name: Docker login
-        uses: docker/login-action@v3
+        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
@@ -50,16 +56,17 @@ jobs:
         run: mkdir base-build-context
 
       - name: Install depot.dev CLI
-        uses: depot/setup-action@v1
+        uses: depot/setup-action@b0b1ea4f69e92ebf5dea3f8713a1b0c37b2126a5 # v1.6.0
 
       # This uses OIDC authentication, so no auth variables are required.
       - name: Build base Docker image via depot.dev
-        uses: depot/build-push-action@v1
+        uses: depot/build-push-action@636daae76684e38c301daa0c5eca1c095b24e780 # v1.14.0
         with:
           project: wl5hnrrkns
           context: base-build-context
           file: scripts/Dockerfile.base
           platforms: linux/amd64,linux/arm64,linux/arm/v7
+          provenance: true
           pull: true
           no-cache: true
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/docs-ci.yaml

@@ -0,0 +1,45 @@
+name: Docs CI
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - "docs/**"
+      - "**.md"
+      - ".github/workflows/docs-ci.yaml"
+
+  pull_request:
+    paths:
+      - "docs/**"
+      - "**.md"
+      - ".github/workflows/docs-ci.yaml"
+
+jobs:
+  docs:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
+
+      - name: Setup Node
+        uses: ./.github/actions/setup-node
+
+      - uses: tj-actions/changed-files@d6e91a2266cdb9d62096cebf1e8546899c6aa18f # v45.0.6
+        id: changed-files
+        with:
+          files: |
+            docs/**
+            **.md
+          separator: ","
+
+      - name: lint
+        if: steps.changed-files.outputs.any_changed == 'true'
+        run: |
+          pnpm exec markdownlint-cli2 ${{ steps.changed-files.outputs.all_changed_files }}
+
+      - name: fmt
+        if: steps.changed-files.outputs.any_changed == 'true'
+        run: |
+          # markdown-table-formatter requires a space separated list of files
+          echo ${{ steps.changed-files.outputs.all_changed_files }} | tr ',' '\n' | pnpm exec markdown-table-formatter --check

.github/workflows/dogfood.yaml

@@ -17,16 +17,32 @@ on:
       - "flake.nix"
   workflow_dispatch:
 
+permissions:
+  # Necessary for GCP authentication (https://github.com/google-github-actions/setup-gcloud#usage)
+  id-token: write
+
 jobs:
   build_image:
-    runs-on: ubuntu-latest
+    if: github.actor != 'dependabot[bot]' # Skip Dependabot PRs
+    runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-4' || 'ubuntu-latest' }}
     steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@cb605e52c26070c328afc4562f0b4ada7618a84e # v2.10.4
+        with:
+          egress-policy: audit
+
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
+
+      - name: Setup Nix
+        uses: DeterminateSystems/nix-installer-action@e50d5f73bfe71c2dd0aa4218de8f4afa59f8f81d # v16
+
+      - name: Setup GHA Nix cache
+        uses: DeterminateSystems/magic-nix-cache-action@6221693898146dc97e38ad0e013488a16477a4c4 # v9
 
       - name: Get branch name
         id: branch-name
-        uses: tj-actions/branch-names@v8
+        uses: tj-actions/branch-names@6871f53176ad61624f978536bbf089c574dc19a2 # v8.0.1
 
       - name: "Branch name to Docker tag name"
         id: docker-tag-name
@@ -37,58 +53,75 @@ jobs:
           echo "tag=${tag}" >> $GITHUB_OUTPUT
 
       - name: Set up Depot CLI
-        uses: depot/setup-action@v1
+        uses: depot/setup-action@b0b1ea4f69e92ebf5dea3f8713a1b0c37b2126a5 # v1.6.0
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@6524bf65af31da8d45b59e8c27de4bd072b392f5 # v3.8.0
 
       - name: Login to DockerHub
         if: github.ref == 'refs/heads/main'
-        uses: docker/login-action@v3
+        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_PASSWORD }}
 
       - name: Build and push Non-Nix image
-        uses: depot/build-push-action@v1
+        uses: depot/build-push-action@636daae76684e38c301daa0c5eca1c095b24e780 # v1.14.0
         with:
           project: b4q6ltmpzh
           token: ${{ secrets.DEPOT_TOKEN }}
           buildx-fallback: true
-          context: "{{defaultContext}}:dogfood"
+          context: "{{defaultContext}}:dogfood/contents"
           pull: true
           save: true
           push: ${{ github.ref == 'refs/heads/main' }}
           tags: "codercom/oss-dogfood:${{ steps.docker-tag-name.outputs.tag }},codercom/oss-dogfood:latest"
 
-      - name: Build and push Nix image
-        uses: depot/build-push-action@v1
-        with:
-          project: b4q6ltmpzh
-          token: ${{ secrets.DEPOT_TOKEN }}
-          buildx-fallback: true
-          context: "."
-          file: "dogfood/Dockerfile.nix"
-          pull: true
-          save: true
-          push: ${{ github.ref == 'refs/heads/main' }}
-          tags: "codercom/oss-dogfood-nix:${{ steps.docker-tag-name.outputs.tag }},codercom/oss-dogfood-nix:latest"
+      - name: Build Nix image
+        run: nix build .#dev_image
+
+      - name: Push Nix image
+        if: github.ref == 'refs/heads/main'
+        run: |
+          docker load -i result
+
+          CURRENT_SYSTEM=$(nix eval --impure --raw --expr 'builtins.currentSystem')
+
+          docker image tag codercom/oss-dogfood-nix:latest-$CURRENT_SYSTEM codercom/oss-dogfood-nix:${{ steps.docker-tag-name.outputs.tag }}
+          docker image push codercom/oss-dogfood-nix:${{ steps.docker-tag-name.outputs.tag }}
+
+          docker image tag codercom/oss-dogfood-nix:latest-$CURRENT_SYSTEM codercom/oss-dogfood-nix:latest
+          docker image push codercom/oss-dogfood-nix:latest
 
   deploy_template:
     needs: build_image
     runs-on: ubuntu-latest
     steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@cb605e52c26070c328afc4562f0b4ada7618a84e # v2.10.4
+        with:
+          egress-policy: audit
+
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
 
       - name: Setup Terraform
         uses: ./.github/actions/setup-tf
 
+      - name: Authenticate to Google Cloud
+        uses: google-github-actions/auth@6fc4af4b145ae7821d527454aa9bd537d1f2dc5f # v2.1.7
+        with:
+          workload_identity_provider: projects/573722524737/locations/global/workloadIdentityPools/github/providers/github
+          service_account: coder-ci@coder-dogfood.iam.gserviceaccount.com
+
       - name: Terraform init and validate
         run: |
           cd dogfood
           terraform init -upgrade
           terraform validate
+          cd contents
+          terraform init -upgrade
+          terraform validate
 
       - name: Get short commit SHA
         if: github.ref == 'refs/heads/main'
@@ -100,22 +133,18 @@ jobs:
         id: message
         run: echo "pr_title=$(git log --format=%s -n 1 ${{ github.sha }})" >> $GITHUB_OUTPUT
 
-      - name: "Get latest Coder binary from the server"
-        if: github.ref == 'refs/heads/main'
-        run: |
-          curl -fsSL "https://dev.coder.com/bin/coder-linux-amd64" -o "./coder"
-          chmod +x "./coder"
-
       - name: "Push template"
         if: github.ref == 'refs/heads/main'
         run: |
-          ./coder templates push $CODER_TEMPLATE_NAME --directory $CODER_TEMPLATE_DIR --yes --name=$CODER_TEMPLATE_VERSION --message="$CODER_TEMPLATE_MESSAGE"
+          cd dogfood
+          terraform apply -auto-approve
         env:
-          # Consumed by Coder CLI
+          # Consumed by coderd provider
           CODER_URL: https://dev.coder.com
           CODER_SESSION_TOKEN: ${{ secrets.CODER_SESSION_TOKEN }}
           # Template source & details
-          CODER_TEMPLATE_NAME: ${{ secrets.CODER_TEMPLATE_NAME }}
-          CODER_TEMPLATE_VERSION: ${{ steps.vars.outputs.sha_short }}
-          CODER_TEMPLATE_DIR: ./dogfood
-          CODER_TEMPLATE_MESSAGE: ${{ steps.message.outputs.pr_title }}
+          TF_VAR_CODER_TEMPLATE_NAME: ${{ secrets.CODER_TEMPLATE_NAME }}
+          TF_VAR_CODER_TEMPLATE_VERSION: ${{ steps.vars.outputs.sha_short }}
+          TF_VAR_CODER_TEMPLATE_DIR: ./contents
+          TF_VAR_CODER_TEMPLATE_MESSAGE: ${{ steps.message.outputs.pr_title }}
+          TF_LOG: info

.github/workflows/mlc_config.json

@@ -1,26 +0,0 @@
-{
-  "ignorePatterns": [
-    {
-      "pattern": "://localhost"
-    },
-    {
-      "pattern": "://.*.?example\\.com"
-    },
-    {
-      "pattern": "developer.github.com"
-    },
-    {
-      "pattern": "docs.github.com"
-    },
-    {
-      "pattern": "support.google.com"
-    },
-    {
-      "pattern": "tailscale.com"
-    },
-    {
-      "pattern": "wireguard.com"
-    }
-  ],
-  "aliveStatusCodes": [200, 0]
-}

.github/workflows/nightly-gauntlet.yaml

@@ -3,21 +3,37 @@
 name: nightly-gauntlet
 on:
   schedule:
-    # Every day at midnight
-    - cron: "0 0 * * *"
+    # Every day at 4AM
+    - cron: "0 4 * * 1-5"
   workflow_dispatch:
+
+permissions:
+  contents: read
+
 jobs:
-  go-race:
-    # While GitHub's toaster runners are likelier to flake, we want consistency
-    # between this environment and the regular test environment for DataDog
-    # statistics and to only show real workflow threats.
-    runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-8' || 'ubuntu-latest' }}
-    # This runner costs 0.016 USD per minute,
-    # so 0.016 * 240 = 3.84 USD per run.
-    timeout-minutes: 240
+  test-go-pg:
+    runs-on: ${{ matrix.os == 'macos-latest' && github.repository_owner == 'coder' && 'depot-macos-latest' || matrix.os == 'windows-2022' && github.repository_owner == 'coder' && 'windows-latest-16-cores' || matrix.os }}
+    if: github.ref == 'refs/heads/main'
+    # This timeout must be greater than the timeout set by `go test` in
+    # `make test-postgres` to ensure we receive a trace of running
+    # goroutines. Setting this to the timeout +5m should work quite well
+    # even if some of the preceding steps are slow.
+    timeout-minutes: 25
+    strategy:
+      matrix:
+        os:
+          - macos-latest
+          - windows-2022
     steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@cb605e52c26070c328afc4562f0b4ada7618a84e # v2.10.4
+        with:
+          egress-policy: audit
+
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
+        with:
+          fetch-depth: 1
 
       - name: Setup Go
         uses: ./.github/actions/setup-go
@@ -25,36 +41,101 @@ jobs:
       - name: Setup Terraform
         uses: ./.github/actions/setup-tf
 
-      - name: Run Tests
-        run: |
-          # -race is likeliest to catch flaky tests
-          # due to correctness detection and its performance
-          # impact.
-          gotestsum --junitfile="gotests.xml" -- -timeout=240m -count=10 -race ./...
+      # Sets up the ImDisk toolkit for Windows and creates a RAM disk on drive R:.
+      - name: Setup ImDisk
+        if: runner.os == 'Windows'
+        uses: ./.github/actions/setup-imdisk
 
-      - name: Upload test results to DataDog
+      - name: Test with PostgreSQL Database
+        env:
+          POSTGRES_VERSION: "13"
+          TS_DEBUG_DISCO: "true"
+          LC_CTYPE: "en_US.UTF-8"
+          LC_ALL: "en_US.UTF-8"
+        shell: bash
+        run: |
+          # if macOS, install google-chrome for scaletests
+          # As another concern, should we really have this kind of external dependency
+          # requirement on standard CI?
+          if [ "${{ matrix.os }}" == "macos-latest" ]; then
+            brew install google-chrome
+          fi
+
+          # By default Go will use the number of logical CPUs, which
+          # is a fine default.
+          PARALLEL_FLAG=""
+
+          # macOS will output "The default interactive shell is now zsh"
+          # intermittently in CI...
+          if [ "${{ matrix.os }}" == "macos-latest" ]; then
+            touch ~/.bash_profile && echo "export BASH_SILENCE_DEPRECATION_WARNING=1" >> ~/.bash_profile
+          fi
+
+          if [ "${{ runner.os }}" == "Windows" ]; then
+            # Create a temp dir on the R: ramdisk drive for Windows. The default
+            # C: drive is extremely slow: https://github.com/actions/runner-images/issues/8755
+            mkdir -p "R:/temp/embedded-pg"
+            go run scripts/embedded-pg/main.go -path "R:/temp/embedded-pg"
+          else
+            go run scripts/embedded-pg/main.go
+          fi
+
+          # Reduce test parallelism, mirroring what we do for race tests.
+          # We'd been encountering issues with timing related flakes, and
+          # this seems to help.
+          DB=ci gotestsum --format standard-quiet -- -v -short -count=1 -parallel 4 -p 4 ./...
+
+      - name: Upload test stats to Datadog
+        timeout-minutes: 1
+        continue-on-error: true
         uses: ./.github/actions/upload-datadog
-        if: always()
+        if: success() || failure()
         with:
           api-key: ${{ secrets.DATADOG_API_KEY }}
 
-  go-timing:
-    # We run these tests with p=1 so we don't need a lot of compute.
-    runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04' || 'ubuntu-latest' }}
-    timeout-minutes: 10
+  notify-slack-on-failure:
+    needs:
+      - test-go-pg
+    runs-on: ubuntu-latest
+    if: failure() && github.ref == 'refs/heads/main'
+
     steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Setup Go
-        uses: ./.github/actions/setup-go
-
-      - name: Run Tests
+      - name: Send Slack notification
         run: |
-          gotestsum --junitfile="gotests.xml" -- --tags="timing" -p=1 -run='_Timing/' ./...
-
-      - name: Upload test results to DataDog
-        uses: ./.github/actions/upload-datadog
-        if: always()
-        with:
-          api-key: ${{ secrets.DATADOG_API_KEY }}
+          curl -X POST -H 'Content-type: application/json' \
+          --data '{
+            "blocks": [
+              {
+                "type": "header",
+                "text": {
+                  "type": "plain_text",
+                  "text": "❌ Nightly gauntlet failed",
+                  "emoji": true
+                }
+              },
+              {
+                "type": "section",
+                "fields": [
+                  {
+                    "type": "mrkdwn",
+                    "text": "*Workflow:*\n${{ github.workflow }}"
+                  },
+                  {
+                    "type": "mrkdwn",
+                    "text": "*Committer:*\n${{ github.actor }}"
+                  },
+                  {
+                    "type": "mrkdwn",
+                    "text": "*Commit:*\n${{ github.sha }}"
+                  }
+                ]
+              },
+              {
+                "type": "section",
+                "text": {
+                  "type": "mrkdwn",
+                  "text": "*View failure:* <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|Click here>"
+                }
+              }
+            ]
+          }' ${{ secrets.CI_FAILURE_SLACK_WEBHOOK }}

.github/workflows/pr-auto-assign.yaml

@@ -13,5 +13,10 @@ jobs:
   assign-author:
     runs-on: ubuntu-latest
     steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@cb605e52c26070c328afc4562f0b4ada7618a84e # v2.10.4
+        with:
+          egress-policy: audit
+
       - name: Assign author
-        uses: toshimaru/auto-author-assign@v2.1.1
+        uses: toshimaru/auto-author-assign@16f0022cf3d7970c106d8d1105f75a1165edb516 # v2.1.1

.github/workflows/pr-cleanup.yaml

@@ -9,12 +9,20 @@ on:
         required: true
 
 permissions:
-  packages: write
+  contents: read
 
 jobs:
   cleanup:
     runs-on: "ubuntu-latest"
+    permissions:
+      # Necessary to delete docker images from ghcr.io.
+      packages: write
     steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@cb605e52c26070c328afc4562f0b4ada7618a84e # v2.10.4
+        with:
+          egress-policy: audit
+
       - name: Get PR number
         id: pr_number
         run: |
@@ -26,7 +34,7 @@ jobs:
 
       - name: Delete image
         continue-on-error: true
-        uses: bots-house/ghcr-delete-image-action@v1.1.0
+        uses: bots-house/ghcr-delete-image-action@3827559c68cb4dcdf54d813ea9853be6d468d3a4 # v1.1.0
         with:
           owner: coder
           name: coder-preview

.github/workflows/pr-deploy.yaml

@@ -7,6 +7,7 @@ on:
   push:
     branches-ignore:
       - main
+      - "temp-cherry-pick-*"
   workflow_dispatch:
     inputs:
       experiments:
@@ -30,8 +31,6 @@ env:
 
 permissions:
   contents: read
-  packages: write
-  pull-requests: write # needed for commenting on PRs
 
 jobs:
   check_pr:
@@ -39,8 +38,13 @@ jobs:
     outputs:
       PR_OPEN: ${{ steps.check_pr.outputs.pr_open }}
     steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@cb605e52c26070c328afc4562f0b4ada7618a84e # v2.10.4
+        with:
+          egress-policy: audit
+
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
 
       - name: Check if PR is open
         id: check_pr
@@ -69,8 +73,13 @@ jobs:
 
     runs-on: "ubuntu-latest"
     steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@cb605e52c26070c328afc4562f0b4ada7618a84e # v2.10.4
+        with:
+          egress-policy: audit
+
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
         with:
           fetch-depth: 0
 
@@ -101,8 +110,8 @@ jobs:
         run: |
           set -euo pipefail
           mkdir -p ~/.kube
-          echo "${{ secrets.PR_DEPLOYMENTS_KUBECONFIG }}" > ~/.kube/config
-          chmod 644 ~/.kube/config
+          echo "${{ secrets.PR_DEPLOYMENTS_KUBECONFIG_BASE64 }}" | base64 --decode > ~/.kube/config
+          chmod 600 ~/.kube/config
           export KUBECONFIG=~/.kube/config
 
       - name: Check if the helm deployment already exists
@@ -119,7 +128,7 @@ jobs:
           echo "NEW=$NEW" >> $GITHUB_OUTPUT
 
       - name: Check changed files
-        uses: dorny/paths-filter@v3
+        uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2
         id: filter
         with:
           base: ${{ github.ref }}
@@ -154,16 +163,23 @@ jobs:
           set -euo pipefail
           # build if the workflow is manually triggered and the deployment doesn't exist (first build or force rebuild)
           echo "first_or_force_build=${{ (github.event_name == 'workflow_dispatch' && steps.check_deployment.outputs.NEW == 'true') || github.event.inputs.build == 'true' }}" >> $GITHUB_OUTPUT
-          # build if the deployment alreday exist and there are changes in the files that we care about (automatic updates)
+          # build if the deployment already exist and there are changes in the files that we care about (automatic updates)
           echo "automatic_rebuild=${{ steps.check_deployment.outputs.NEW == 'false' && steps.filter.outputs.all_count > steps.filter.outputs.ignored_count }}" >> $GITHUB_OUTPUT
 
   comment-pr:
     needs: get_info
     if: needs.get_info.outputs.BUILD == 'true' || github.event.inputs.deploy == 'true'
     runs-on: "ubuntu-latest"
+    permissions:
+      pull-requests: write # needed for commenting on PRs
     steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@cb605e52c26070c328afc4562f0b4ada7618a84e # v2.10.4
+        with:
+          egress-policy: audit
+
       - name: Find Comment
-        uses: peter-evans/find-comment@v3
+        uses: peter-evans/find-comment@3eae4d37986fb5a8592848f6a574fdf654e61f9e # v3.1.0
         id: fc
         with:
           issue-number: ${{ needs.get_info.outputs.PR_NUMBER }}
@@ -173,7 +189,7 @@ jobs:
 
       - name: Comment on PR
         id: comment_id
-        uses: peter-evans/create-or-update-comment@v4
+        uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4.0.0
         with:
           comment-id: ${{ steps.fc.outputs.comment-id }}
           issue-number: ${{ needs.get_info.outputs.PR_NUMBER }}
@@ -190,7 +206,10 @@ jobs:
     # Run build job only if there are changes in the files that we care about or if the workflow is manually triggered with --build flag
     if: needs.get_info.outputs.BUILD == 'true'
     runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-8' || 'ubuntu-latest' }}
-    # This concurrency only cancels build jobs if a new build is triggred. It will avoid cancelling the current deployemtn in case of docs chnages.
+    permissions:
+      # Necessary to push docker images to ghcr.io.
+      packages: write
+    # This concurrency only cancels build jobs if a new build is triggred. It will avoid cancelling the current deployemtn in case of docs changes.
     concurrency:
       group: build-${{ github.workflow }}-${{ github.ref }}-${{ needs.get_info.outputs.BUILD }}
       cancel-in-progress: true
@@ -198,8 +217,13 @@ jobs:
       DOCKER_CLI_EXPERIMENTAL: "enabled"
       CODER_IMAGE_TAG: ${{ needs.get_info.outputs.CODER_IMAGE_TAG }}
     steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@cb605e52c26070c328afc4562f0b4ada7618a84e # v2.10.4
+        with:
+          egress-policy: audit
+
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
         with:
           fetch-depth: 0
 
@@ -213,7 +237,7 @@ jobs:
         uses: ./.github/actions/setup-sqlc
 
       - name: GHCR Login
-        uses: docker/login-action@v3
+        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
@@ -242,6 +266,8 @@ jobs:
       always() && (needs.build.result == 'success' || needs.build.result == 'skipped') &&
       (needs.get_info.outputs.BUILD == 'true' || github.event.inputs.deploy == 'true')
     runs-on: "ubuntu-latest"
+    permissions:
+      pull-requests: write # needed for commenting on PRs
     env:
       CODER_IMAGE_TAG: ${{ needs.get_info.outputs.CODER_IMAGE_TAG }}
       PR_NUMBER: ${{ needs.get_info.outputs.PR_NUMBER }}
@@ -249,12 +275,17 @@ jobs:
       PR_URL: ${{ needs.get_info.outputs.PR_URL }}
       PR_HOSTNAME: "pr${{ needs.get_info.outputs.PR_NUMBER }}.${{ secrets.PR_DEPLOYMENTS_DOMAIN }}"
     steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@cb605e52c26070c328afc4562f0b4ada7618a84e # v2.10.4
+        with:
+          egress-policy: audit
+
       - name: Set up kubeconfig
         run: |
           set -euo pipefail
           mkdir -p ~/.kube
-          echo "${{ secrets.PR_DEPLOYMENTS_KUBECONFIG }}" > ~/.kube/config
-          chmod 644 ~/.kube/config
+          echo "${{ secrets.PR_DEPLOYMENTS_KUBECONFIG_BASE64 }}" | base64 --decode > ~/.kube/config
+          chmod 600 ~/.kube/config
           export KUBECONFIG=~/.kube/config
 
       - name: Check if image exists
@@ -294,7 +325,7 @@ jobs:
           kubectl create namespace "pr${{ env.PR_NUMBER }}"
 
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
 
       - name: Check and Create Certificate
         if: needs.get_info.outputs.NEW == 'true' || github.event.inputs.deploy == 'true'
@@ -391,14 +422,14 @@ jobs:
           "${DEST}" version
           mv "${DEST}" /usr/local/bin/coder
 
-      - name: Create first user, template and workspace
+      - name: Create first user
         if: needs.get_info.outputs.NEW == 'true' || github.event.inputs.deploy == 'true'
         id: setup_deployment
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           set -euo pipefail
 
-          # Create first user
-
           # create a masked random password 12 characters long
           password=$(openssl rand -base64 16 | tr -d "=+/" | cut -c1-12)
 
@@ -407,20 +438,22 @@ jobs:
           echo "password=$password" >> $GITHUB_OUTPUT
 
           coder login \
-            --first-user-username coder \
+            --first-user-username pr${{ env.PR_NUMBER }}-admin \
             --first-user-email pr${{ env.PR_NUMBER }}@coder.com \
             --first-user-password $password \
-            --first-user-trial \
+            --first-user-trial=false \
             --use-token-as-session \
             https://${{ env.PR_HOSTNAME }}
 
-          # Create template
-          cd ./.github/pr-deployments/template
-          coder templates push -y --variable namespace=pr${{ env.PR_NUMBER }} kubernetes
+          # Create a user for the github.actor
+          # TODO: update once https://github.com/coder/coder/issues/15466 is resolved
+          # coder users create \
+          #   --username ${{ github.actor }} \
+          #   --login-type github
 
-          # Create workspace
-          coder create --template="kubernetes" kube --parameter cpu=2 --parameter memory=4 --parameter home_disk_size=2 -y
-          coder stop kube -y
+          # promote the user to admin role
+          # coder org members edit-role ${{ github.actor }} organization-admin
+          # TODO: update once https://github.com/coder/internal/issues/207 is resolved
 
       - name: Send Slack notification
         if: needs.get_info.outputs.NEW == 'true' || github.event.inputs.deploy == 'true'
@@ -432,7 +465,7 @@ jobs:
               "pr_url": "'"${{ env.PR_URL }}"'",
               "pr_title": "'"${{ env.PR_TITLE }}"'",
               "pr_access_url": "'"https://${{ env.PR_HOSTNAME }}"'",
-              "pr_username": "'"test"'",
+              "pr_username": "'"pr${{ env.PR_NUMBER }}-admin"'",
               "pr_email": "'"pr${{ env.PR_NUMBER }}@coder.com"'",
               "pr_password": "'"${{ steps.setup_deployment.outputs.password }}"'",
               "pr_actor": "'"${{ github.actor }}"'"
@@ -441,7 +474,7 @@ jobs:
           echo "Slack notification sent"
 
       - name: Find Comment
-        uses: peter-evans/find-comment@v3
+        uses: peter-evans/find-comment@3eae4d37986fb5a8592848f6a574fdf654e61f9e # v3.1.0
         id: fc
         with:
           issue-number: ${{ env.PR_NUMBER }}
@@ -450,7 +483,7 @@ jobs:
           direction: last
 
       - name: Comment on PR
-        uses: peter-evans/create-or-update-comment@v4
+        uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4.0.0
         env:
           STATUS: ${{ needs.get_info.outputs.NEW == 'true' && 'Created' || 'Updated' }}
         with:
@@ -465,3 +498,14 @@ jobs:
             cc: @${{ github.actor }}
           reactions: rocket
           reactions-edit-mode: replace
+
+      - name: Create template and workspace
+        if: needs.get_info.outputs.NEW == 'true' || github.event.inputs.deploy == 'true'
+        run: |
+          set -euo pipefail
+          cd .github/pr-deployments/template
+          coder templates push -y --variable namespace=pr${{ env.PR_NUMBER }} kubernetes
+
+          # Create workspace
+          coder create --template="kubernetes" kube --parameter cpu=2 --parameter memory=4 --parameter home_disk_size=2 -y
+          coder stop kube -y

.github/workflows/release-validation.yaml

@@ -0,0 +1,28 @@
+name: release-validation
+
+on:
+  push:
+    tags:
+      - "v*"
+
+permissions:
+  contents: read
+
+jobs:
+  network-performance:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@cb605e52c26070c328afc4562f0b4ada7618a84e # v2.10.4
+        with:
+          egress-policy: audit
+
+      - name: Run Schmoder CI
+        uses: benc-uk/workflow-dispatch@e2e5e9a103e331dad343f381a29e654aea3cf8fc # v1.2.4
+        with:
+          workflow: ci.yaml
+          repo: coder/schmoder
+          inputs: '{ "num_releases": "3", "commit": "${{ github.sha }}" }'
+          token: ${{ secrets.CDRCI_SCHMODER_ACTIONS_TOKEN }}
+          ref: main

.github/workflows/release.yaml

@@ -18,12 +18,7 @@ on:
         default: false
 
 permissions:
-  # Required to publish a release
-  contents: write
-  # Necessary to push docker images to ghcr.io.
-  packages: write
-  # Necessary for GCP authentication (https://github.com/google-github-actions/setup-gcloud#usage)
-  id-token: write
+  contents: read
 
 concurrency: ${{ github.workflow }}-${{ github.ref }}
 
@@ -37,17 +32,109 @@ env:
   CODER_RELEASE_NOTES: ${{ inputs.release_notes }}
 
 jobs:
+  # build-dylib is a separate job to build the dylib on macOS.
+  build-dylib:
+    runs-on: ${{ github.repository_owner == 'coder' && 'depot-macos-latest' || 'macos-latest' }}
+    steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@cb605e52c26070c328afc4562f0b4ada7618a84e # v2.10.4
+        with:
+          egress-policy: audit
+
+      - name: Checkout
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
+        with:
+          fetch-depth: 0
+
+      # If the event that triggered the build was an annotated tag (which our
+      # tags are supposed to be), actions/checkout has a bug where the tag in
+      # question is only a lightweight tag and not a full annotated tag. This
+      # command seems to fix it.
+      # https://github.com/actions/checkout/issues/290
+      - name: Fetch git tags
+        run: git fetch --tags --force
+
+      - name: Setup build tools
+        run: |
+          brew install bash gnu-getopt make
+          echo "$(brew --prefix bash)/bin" >> $GITHUB_PATH
+          echo "$(brew --prefix gnu-getopt)/bin" >> $GITHUB_PATH
+          echo "$(brew --prefix make)/libexec/gnubin" >> $GITHUB_PATH
+
+      - name: Setup Go
+        uses: ./.github/actions/setup-go
+
+      - name: Install rcodesign
+        run: |
+          set -euo pipefail
+          wget -O /tmp/rcodesign.tar.gz https://github.com/indygreg/apple-platform-rs/releases/download/apple-codesign%2F0.22.0/apple-codesign-0.22.0-macos-universal.tar.gz
+          sudo tar -xzf /tmp/rcodesign.tar.gz \
+            -C /usr/local/bin \
+            --strip-components=1 \
+            apple-codesign-0.22.0-macos-universal/rcodesign
+          rm /tmp/rcodesign.tar.gz
+
+      - name: Setup Apple Developer certificate and API key
+        run: |
+          set -euo pipefail
+          touch /tmp/{apple_cert.p12,apple_cert_password.txt,apple_apikey.p8}
+          chmod 600 /tmp/{apple_cert.p12,apple_cert_password.txt,apple_apikey.p8}
+          echo "$AC_CERTIFICATE_P12_BASE64" | base64 -d > /tmp/apple_cert.p12
+          echo "$AC_CERTIFICATE_PASSWORD" > /tmp/apple_cert_password.txt
+          echo "$AC_APIKEY_P8_BASE64" | base64 -d > /tmp/apple_apikey.p8
+        env:
+          AC_CERTIFICATE_P12_BASE64: ${{ secrets.AC_CERTIFICATE_P12_BASE64 }}
+          AC_CERTIFICATE_PASSWORD: ${{ secrets.AC_CERTIFICATE_PASSWORD }}
+          AC_APIKEY_P8_BASE64: ${{ secrets.AC_APIKEY_P8_BASE64 }}
+
+      - name: Build dylibs
+        run: |
+          set -euxo pipefail
+          go mod download
+
+          make gen/mark-fresh
+          make build/coder-dylib
+        env:
+          CODER_SIGN_DARWIN: 1
+          AC_CERTIFICATE_FILE: /tmp/apple_cert.p12
+          AC_CERTIFICATE_PASSWORD_FILE: /tmp/apple_cert_password.txt
+
+      - name: Upload build artifacts
+        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+        with:
+          name: dylibs
+          path: |
+            ./build/*.h
+            ./build/*.dylib
+          retention-days: 7
+
+      - name: Delete Apple Developer certificate and API key
+        run: rm -f /tmp/{apple_cert.p12,apple_cert_password.txt,apple_apikey.p8}
+
   release:
     name: Build and publish
+    needs: build-dylib
     runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-8' || 'ubuntu-latest' }}
+    permissions:
+      # Required to publish a release
+      contents: write
+      # Necessary to push docker images to ghcr.io.
+      packages: write
+      # Necessary for GCP authentication (https://github.com/google-github-actions/setup-gcloud#usage)
+      id-token: write
     env:
       # Necessary for Docker manifest
       DOCKER_CLI_EXPERIMENTAL: "enabled"
     outputs:
       version: ${{ steps.version.outputs.version }}
     steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@cb605e52c26070c328afc4562f0b4ada7618a84e # v2.10.4
+        with:
+          egress-policy: audit
+
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
         with:
           fetch-depth: 0
 
@@ -116,7 +203,7 @@ jobs:
           cat "$CODER_RELEASE_NOTES_FILE"
 
       - name: Docker Login
-        uses: docker/login-action@v3
+        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
@@ -130,7 +217,7 @@ jobs:
 
       # Necessary for signing Windows binaries.
       - name: Setup Java
-        uses: actions/setup-java@v4
+        uses: actions/setup-java@b36c23c0d998641eff861008f374ee103c25ac73 # v4.4.0
         with:
           distribution: "zulu"
           java-version: "11.0"
@@ -138,6 +225,18 @@ jobs:
       - name: Install nsis and zstd
         run: sudo apt-get install -y nsis zstd
 
+      - name: Download dylibs
+        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
+        with:
+          name: dylibs
+          path: ./build
+
+      - name: Insert dylibs
+        run: |
+          mv ./build/*amd64.dylib ./site/out/bin/coder-vpn-darwin-amd64.dylib
+          mv ./build/*arm64.dylib ./site/out/bin/coder-vpn-darwin-arm64.dylib
+          mv ./build/*arm64.h     ./site/out/bin/coder-vpn-darwin-dylib.h
+
       - name: Install nfpm
         run: |
           set -euo pipefail
@@ -180,19 +279,19 @@ jobs:
 
       - name: Test migrations from current ref to main
         run: |
-          make test-migrations
+          POSTGRES_VERSION=13 make test-migrations
 
       # Setup GCloud for signing Windows binaries.
       - name: Authenticate to Google Cloud
         id: gcloud_auth
-        uses: google-github-actions/auth@v2
+        uses: google-github-actions/auth@6fc4af4b145ae7821d527454aa9bd537d1f2dc5f # v2.1.7
         with:
           workload_identity_provider: ${{ secrets.GCP_CODE_SIGNING_WORKLOAD_ID_PROVIDER }}
           service_account: ${{ secrets.GCP_CODE_SIGNING_SERVICE_ACCOUNT }}
           token_format: "access_token"
 
       - name: Setup GCloud SDK
-        uses: "google-github-actions/setup-gcloud@v2"
+        uses: google-github-actions/setup-gcloud@6189d56e4096ee891640bb02ac264be376592d6a # v2.1.2
 
       - name: Build binaries
         run: |
@@ -245,17 +344,18 @@ jobs:
 
       - name: Install depot.dev CLI
         if: steps.image-base-tag.outputs.tag != ''
-        uses: depot/setup-action@v1
+        uses: depot/setup-action@b0b1ea4f69e92ebf5dea3f8713a1b0c37b2126a5 # v1.6.0
 
       # This uses OIDC authentication, so no auth variables are required.
       - name: Build base Docker image via depot.dev
         if: steps.image-base-tag.outputs.tag != ''
-        uses: depot/build-push-action@v1
+        uses: depot/build-push-action@636daae76684e38c301daa0c5eca1c095b24e780 # v1.14.0
         with:
           project: wl5hnrrkns
           context: base-build-context
           file: scripts/Dockerfile.base
           platforms: linux/amd64,linux/arm64,linux/arm/v7
+          provenance: true
           pull: true
           no-cache: true
           push: true
@@ -263,6 +363,7 @@ jobs:
             ${{ steps.image-base-tag.outputs.tag }}
 
       - name: Verify that images are pushed properly
+        if: steps.image-base-tag.outputs.tag != ''
         run: |
           # retry 10 times with a 5 second delay as the images may not be
           # available immediately
@@ -295,10 +396,6 @@ jobs:
         run: |
           set -euxo pipefail
 
-          # build Docker images for each architecture
-          version="$(./scripts/version.sh)"
-          make -j build/coder_"$version"_linux_{amd64,arm64,armv7}.tag
-
           # we can't build multi-arch if the images aren't pushed, so quit now
           # if dry-running
           if [[ "$CODER_RELEASE" != *t* ]]; then
@@ -306,9 +403,13 @@ jobs:
             exit 0
           fi
 
+          # build Docker images for each architecture
+          version="$(./scripts/version.sh)"
+          make build/coder_"$version"_linux_{amd64,arm64,armv7}.tag
+
           # build and push multi-arch manifest, this depends on the other images
           # being pushed so will automatically push them.
-          make -j push/build/coder_"$version"_linux.tag
+          make push/build/coder_"$version"_linux.tag
 
           # if the current version is equal to the highest (according to semver)
           # version in the repo, also create a multi-arch image as ":latest" and
@@ -358,13 +459,13 @@ jobs:
           CODER_GPG_RELEASE_KEY_BASE64: ${{ secrets.GPG_RELEASE_KEY_BASE64 }}
 
       - name: Authenticate to Google Cloud
-        uses: google-github-actions/auth@v2
+        uses: google-github-actions/auth@6fc4af4b145ae7821d527454aa9bd537d1f2dc5f # v2.1.7
         with:
           workload_identity_provider: ${{ secrets.GCP_WORKLOAD_ID_PROVIDER }}
           service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }}
 
       - name: Setup GCloud SDK
-        uses: "google-github-actions/setup-gcloud@v2"
+        uses: google-github-actions/setup-gcloud@6189d56e4096ee891640bb02ac264be376592d6a # 2.1.2
 
       - name: Publish Helm Chart
         if: ${{ !inputs.dry_run }}
@@ -383,7 +484,7 @@ jobs:
 
       - name: Upload artifacts to actions (if dry-run)
         if: ${{ inputs.dry_run }}
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@604373da6381bf24206979c74d06a550515601b9 # v4.4.1
         with:
           name: release-artifacts
           path: |
@@ -396,14 +497,14 @@ jobs:
             ./build/*.rpm
           retention-days: 7
 
-      - name: Start Packer builds
+      - name: Send repository-dispatch event
         if: ${{ !inputs.dry_run }}
-        uses: peter-evans/repository-dispatch@v3
+        uses: peter-evans/repository-dispatch@ff45666b9427631e3450c54a1bcbee4d9ff4d7c0 # v3.0.0
         with:
           token: ${{ secrets.CDRCI_GITHUB_TOKEN }}
           repository: coder/packages
           event-type: coder-release
-          client-payload: '{"coder_version": "${{ steps.version.outputs.version }}"}'
+          client-payload: '{"coder_version": "${{ steps.version.outputs.version }}", "release_channel": "${{ inputs.release_channel }}"}'
 
   publish-homebrew:
     name: Publish to Homebrew tap
@@ -414,6 +515,11 @@ jobs:
     steps:
       # TODO: skip this if it's not a new release (i.e. a backport). This is
       #       fine right now because it just makes a PR that we can close.
+      - name: Harden Runner
+        uses: step-security/harden-runner@cb605e52c26070c328afc4562f0b4ada7618a84e # v2.10.4
+        with:
+          egress-policy: audit
+
       - name: Update homebrew
         env:
           # Variables used by the `gh` command
@@ -485,13 +591,18 @@ jobs:
     if: ${{ !inputs.dry_run }}
 
     steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@cb605e52c26070c328afc4562f0b4ada7618a84e # v2.10.4
+        with:
+          egress-policy: audit
+
       - name: Sync fork
         run: gh repo sync cdrci/winget-pkgs -b master
         env:
           GH_TOKEN: ${{ secrets.CDRCI_GITHUB_TOKEN }}
 
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
         with:
           fetch-depth: 0
 
@@ -570,8 +681,13 @@ jobs:
     needs: release
     if: ${{ !inputs.dry_run }}
     steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@cb605e52c26070c328afc4562f0b4ada7618a84e # v2.10.4
+        with:
+          egress-policy: audit
+
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
         with:
           fetch-depth: 1
 

.github/workflows/scorecard.yml

@@ -0,0 +1,52 @@
+name: OpenSSF Scorecard
+on:
+  branch_protection_rule:
+  schedule:
+    - cron: "27 7 * * 3" # A random time to run weekly
+  push:
+    branches: ["main"]
+
+permissions: read-all
+
+jobs:
+  analysis:
+    name: Scorecard analysis
+    runs-on: ubuntu-latest
+    permissions:
+      # Needed to upload the results to code-scanning dashboard.
+      security-events: write
+      # Needed to publish results and get a badge (see publish_results below).
+      id-token: write
+
+    steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@cb605e52c26070c328afc4562f0b4ada7618a84e # v2.10.4
+        with:
+          egress-policy: audit
+
+      - name: "Checkout code"
+        uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
+        with:
+          persist-credentials: false
+
+      - name: "Run analysis"
+        uses: ossf/scorecard-action@62b2cac7ed8198b15735ed49ab1e5cf35480ba46 # v2.4.0
+        with:
+          results_file: results.sarif
+          results_format: sarif
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_results: true
+
+      # Upload the results as artifacts.
+      - name: "Upload artifact"
+        uses: actions/upload-artifact@604373da6381bf24206979c74d06a550515601b9 # v4.4.1
+        with:
+          name: SARIF file
+          path: results.sarif
+          retention-days: 5
+
+      # Upload the results to GitHub's code scanning dashboard.
+      - name: "Upload to code-scanning"
+        uses: github/codeql-action/upload-sarif@f6091c0113d1dcf9b98e269ee48e8a7e51b7bdd4 # v3.28.5
+        with:
+          sarif_file: results.sarif

.github/workflows/security.yaml

@@ -3,7 +3,6 @@ name: "security"
 permissions:
   actions: read
   contents: read
-  security-events: write
 
 on:
   workflow_dispatch:
@@ -23,16 +22,23 @@ concurrency:
 
 jobs:
   codeql:
+    permissions:
+      security-events: write
     runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-8' || 'ubuntu-latest' }}
     steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@cb605e52c26070c328afc4562f0b4ada7618a84e # v2.10.4
+        with:
+          egress-policy: audit
+
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
 
       - name: Setup Go
         uses: ./.github/actions/setup-go
 
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@v3
+        uses: github/codeql-action/init@f6091c0113d1dcf9b98e269ee48e8a7e51b7bdd4 # v3.28.5
         with:
           languages: go, javascript
 
@@ -42,7 +48,7 @@ jobs:
           rm Makefile
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v3
+        uses: github/codeql-action/analyze@f6091c0113d1dcf9b98e269ee48e8a7e51b7bdd4 # v3.28.5
 
       - name: Send Slack notification on failure
         if: ${{ failure() }}
@@ -56,10 +62,17 @@ jobs:
             "${{ secrets.SLACK_SECURITY_FAILURE_WEBHOOK_URL }}"
 
   trivy:
+    permissions:
+      security-events: write
     runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-8' || 'ubuntu-latest' }}
     steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@cb605e52c26070c328afc4562f0b4ada7618a84e # v2.10.4
+        with:
+          egress-policy: audit
+
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
         with:
           fetch-depth: 0
 
@@ -73,25 +86,32 @@ jobs:
         uses: ./.github/actions/setup-sqlc
 
       - name: Install yq
-        run: go run github.com/mikefarah/yq/v4@v4.30.6
+        run: go run github.com/mikefarah/yq/v4@v4.44.3
       - name: Install mockgen
-        run: go install go.uber.org/mock/mockgen@v0.4.0
+        run: go install go.uber.org/mock/mockgen@v0.5.0
       - name: Install protoc-gen-go
         run: go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.30
       - name: Install protoc-gen-go-drpc
-        run: go install storj.io/drpc/cmd/protoc-gen-go-drpc@v0.0.33
+        run: go install storj.io/drpc/cmd/protoc-gen-go-drpc@v0.0.34
       - name: Install Protoc
         run: |
           # protoc must be in lockstep with our dogfood Dockerfile or the
           # version in the comments will differ. This is also defined in
           # ci.yaml.
-          set -x
-          cd dogfood
+          set -euxo pipefail
+          cd dogfood/contents
+          mkdir -p /usr/local/bin
+          mkdir -p /usr/local/include
+
           DOCKER_BUILDKIT=1 docker build . --target proto -t protoc
           protoc_path=/usr/local/bin/protoc
           docker run --rm --entrypoint cat protoc /tmp/bin/protoc > $protoc_path
           chmod +x $protoc_path
           protoc --version
+          # Copy the generated files to the include directory.
+          docker run --rm -v /usr/local/include:/target protoc cp -r /tmp/include/google /target/
+          ls -la /usr/local/include/google/protobuf/
+          stat /usr/local/include/google/protobuf/timestamp.proto
 
       - name: Build Coder linux amd64 Docker image
         id: build
@@ -110,11 +130,13 @@ jobs:
           # the registry.
           export CODER_IMAGE_BUILD_BASE_TAG="$(CODER_IMAGE_BASE=coder-base ./scripts/image_tag.sh --version "$version")"
 
-          make -j "$image_job"
+          # We would like to use make -j here, but it doesn't work with the some recent additions
+          # to our code generation.
+          make "$image_job"
           echo "image=$(cat "$image_job")" >> $GITHUB_OUTPUT
 
       - name: Run Trivy vulnerability scanner
-        uses: aquasecurity/trivy-action@7c2007bcb556501da015201bcba5aa14069b74e2
+        uses: aquasecurity/trivy-action@18f2510ee396bbf400402947b394f2dd8c87dbb0
         with:
           image-ref: ${{ steps.build.outputs.image }}
           format: sarif
@@ -122,28 +144,18 @@ jobs:
           severity: "CRITICAL,HIGH"
 
       - name: Upload Trivy scan results to GitHub Security tab
-        uses: github/codeql-action/upload-sarif@v3
+        uses: github/codeql-action/upload-sarif@f6091c0113d1dcf9b98e269ee48e8a7e51b7bdd4 # v3.28.5
         with:
           sarif_file: trivy-results.sarif
           category: "Trivy"
 
       - name: Upload Trivy scan results as an artifact
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@604373da6381bf24206979c74d06a550515601b9 # v4.4.1
         with:
           name: trivy
           path: trivy-results.sarif
           retention-days: 7
 
-      # Prisma cloud scan runs last because it fails the entire job if it
-      # detects vulnerabilities. :|
-      - name: Run Prisma Cloud image scan
-        uses: PaloAltoNetworks/prisma-cloud-scan@v1
-        with:
-          pcc_console_url: ${{ secrets.PRISMA_CLOUD_URL }}
-          pcc_user: ${{ secrets.PRISMA_CLOUD_ACCESS_KEY }}
-          pcc_pass: ${{ secrets.PRISMA_CLOUD_SECRET_KEY }}
-          image_name: ${{ steps.build.outputs.image }}
-
       - name: Send Slack notification on failure
         if: ${{ failure() }}
         run: |

.github/workflows/stale.yaml

@@ -1,23 +1,36 @@
-name: Stale Issue, Banch and Old Workflows Cleanup
+name: Stale Issue, Branch and Old Workflows Cleanup
 on:
   schedule:
     # Every day at midnight
     - cron: "0 0 * * *"
   workflow_dispatch:
+
+permissions:
+  contents: read
+
 jobs:
   issues:
     runs-on: ubuntu-latest
     permissions:
+      # Needed to close issues.
       issues: write
+      # Needed to close PRs.
       pull-requests: write
-      actions: write
     steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@cb605e52c26070c328afc4562f0b4ada7618a84e # v2.10.4
+        with:
+          egress-policy: audit
+
       - name: stale
-        uses: actions/stale@v9.0.0
+        uses: actions/stale@28ca1036281a5e5922ead5184a1bbf96e5fc984e # v9.0.0
         with:
           stale-issue-label: "stale"
           stale-pr-label: "stale"
-          days-before-stale: 180
+          # days-before-stale: 180
+          # essentially disabled for now while we work through polish issues
+          days-before-stale: 3650
+
           # Pull Requests become stale more quickly due to merge conflicts.
           # Also, we promote minimizing WIP.
           days-before-pr-stale: 7
@@ -31,7 +44,7 @@ jobs:
           # Start with the oldest issues, always.
           ascending: true
       - name: "Close old issues labeled likely-no"
-        uses: actions/github-script@v7
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           script: |
@@ -57,7 +70,7 @@ jobs:
               });
 
               const labelEvent = timeline.data.find(event => event.event === 'labeled' && event.label.name === 'likely-no');
-              
+
               if (labelEvent) {
                 console.log(`Issue #${issue.number} was labeled with 'likely-no' at ${labelEvent.created_at}`);
 
@@ -78,11 +91,19 @@ jobs:
 
   branches:
     runs-on: ubuntu-latest
+    permissions:
+      # Needed to delete branches.
+      contents: write
     steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@cb605e52c26070c328afc4562f0b4ada7618a84e # v2.10.4
+        with:
+          egress-policy: audit
+
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
       - name: Run delete-old-branches-action
-        uses: beatlabs/delete-old-branches-action@v0.0.10
+        uses: beatlabs/delete-old-branches-action@6e94df089372a619c01ae2c2f666bf474f890911 # v0.0.10
         with:
           repo_token: ${{ github.token }}
           date: "6 months ago"
@@ -92,9 +113,17 @@ jobs:
           exclude_open_pr_branches: true
   del_runs:
     runs-on: ubuntu-latest
+    permissions:
+      # Needed to delete workflow runs.
+      actions: write
     steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@cb605e52c26070c328afc4562f0b4ada7618a84e # v2.10.4
+        with:
+          egress-policy: audit
+
       - name: Delete PR Cleanup workflow runs
-        uses: Mattraks/delete-workflow-runs@v2
+        uses: Mattraks/delete-workflow-runs@39f0bbed25d76b34de5594dceab824811479e5de # v2.0.6
         with:
           token: ${{ github.token }}
           repository: ${{ github.repository }}
@@ -103,7 +132,7 @@ jobs:
           delete_workflow_pattern: pr-cleanup.yaml
 
       - name: Delete PR Deploy workflow skipped runs
-        uses: Mattraks/delete-workflow-runs@v2
+        uses: Mattraks/delete-workflow-runs@39f0bbed25d76b34de5594dceab824811479e5de # v2.0.6
         with:
           token: ${{ github.token }}
           repository: ${{ github.repository }}

.github/workflows/typos.toml

@@ -14,8 +14,16 @@ darcula = "darcula"
 Hashi = "Hashi"
 trialer = "trialer"
 encrypter = "encrypter"
-hel = "hel"             # as in helsinki
-pn = "pn"               # this is used as proto node
+# as in helsinki
+hel = "hel"
+# this is used as proto node
+pn = "pn"
+# typos doesn't like the EDE in TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA
+EDE = "EDE"
+# HELO is an SMTP command
+HELO = "HELO"
+LKE = "LKE"
+byt = "byt"
 
 [files]
 extend-exclude = [
@@ -27,11 +35,12 @@ extend-exclude = [
 	# These files contain base64 strings that confuse the detector
 	"**XService**.ts",
 	"**identity.go",
-	"scripts/ci-report/testdata/**",
 	"**/*_test.go",
 	"**/*.test.tsx",
 	"**/pnpm-lock.yaml",
 	"tailnet/testdata/**",
 	"site/src/pages/SetupPage/countries.tsx",
 	"provisioner/terraform/testdata/**",
+	# notifications' golden files confuse the detector because of quoted-printable encoding
+	"coderd/notifications/testdata/**"
 ]

.github/workflows/weekly-docs.yaml

@@ -10,23 +10,33 @@ on:
     paths:
       - "docs/**"
 
+permissions:
+  contents: read
+
 jobs:
   check-docs:
-    runs-on: ubuntu-latest
+    # later versions of Ubuntu have disabled unprivileged user namespaces, which are required by the action
+    runs-on: ubuntu-22.04
+    permissions:
+      pull-requests: write # required to post PR review comments by the action
     steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@cb605e52c26070c328afc4562f0b4ada7618a84e # v2.10.4
+        with:
+          egress-policy: audit
+
       - name: Checkout
-        uses: actions/checkout@master
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
 
       - name: Check Markdown links
-        uses: gaurav-nelson/github-action-markdown-link-check@v1
+        uses: umbrelladocs/action-linkspector@de84085e0f51452a470558693d7d308fbb2fa261 # v1.2.5
         id: markdown-link-check
         # checks all markdown files from /docs including all subfolders
         with:
-          use-quiet-mode: "yes"
-          use-verbose-mode: "yes"
-          config-file: ".github/workflows/mlc_config.json"
-          folder-path: "docs/"
-          file-path: "./README.md"
+          reporter: github-pr-review
+          config_file: ".github/.linkspector.yml"
+          fail_on_error: "true"
+          filter_mode: "nofilter"
 
       - name: Send Slack notification
         if: failure() && github.event_name == 'schedule'

.gitignore

@@ -17,6 +17,8 @@ yarn-error.log
 # Allow VSCode recommendations and default settings in project root.
 !/.vscode/extensions.json
 !/.vscode/settings.json
+# Allow code snippets
+!/.vscode/*.code-snippets
 
 # Front-end ignore patterns.
 .next/
@@ -34,6 +36,7 @@ site/.swc
 .gen-golden
 
 # Build
+bin/
 build/
 dist/
 out/
@@ -52,6 +55,7 @@ site/stats/
 
 # direnv
 .envrc
+.direnv
 *.test
 
 # Loadtesting
@@ -71,3 +75,9 @@ result
 
 # pnpm
 .pnpm-store/
+
+# Zed
+.zed_server
+
+# dlv debug binaries for go tests
+__debug_bin*

.golangci.yaml

@@ -175,8 +175,6 @@ linters-settings:
       - name: modifies-value-receiver
       - name: package-comments
       - name: range
-      - name: range-val-address
-      - name: range-val-in-closure
       - name: receiver-naming
       - name: redefines-builtin-id
       - name: string-of-int
@@ -195,6 +193,15 @@ linters-settings:
       - name: var-naming
       - name: waitgroup-by-value
 
+  # irrelevant as of Go v1.22: https://go.dev/blog/loopvar-preview
+  govet:
+    disable:
+      - loopclosure
+  gosec:
+    excludes:
+      # Implicit memory aliasing of items from a range statement (irrelevant as of Go v1.22)
+      - G601
+
 issues:
   # Rules listed here: https://github.com/securego/gosec#available-rules
   exclude-rules:
@@ -233,7 +240,6 @@ linters:
     - errname
     - errorlint
     - exhaustruct
-    - exportloopref
     - forcetypeassert
     - gocritic
     # gocyclo is may be useful in the future when we start caring

.markdownlint.jsonc

@@ -0,0 +1,31 @@
+// Example markdownlint configuration with all properties set to their default value
+{
+	"MD010": { "spaces_per_tab": 4}, // No hard tabs: we use 4 spaces per tab
+
+	"MD013": false, // Line length: we are not following a strict line lnegth in markdown files
+
+	"MD024": { "siblings_only": true }, // Multiple headings with the same content:
+
+	"MD033": false, // Inline HTML: we use it in some places
+
+	"MD034": false, // Bare URL: we use it in some places in generated docs e.g.
+	// codersdk/deployment.go L597, L1177, L2287, L2495, L2533
+	// codersdk/workspaceproxy.go L196, L200-L201
+	// coderd/tracing/exporter.go L26
+	// cli/exp_scaletest.go L-9
+
+	"MD041": false, // First line in file should be a top level heading: All of our changelogs do not start with a top level heading
+	// TODO: We need to update /home/coder/repos/coder/coder/scripts/release/generate_release_notes.sh to generate changelogs that follow this rule
+
+	"MD052": false, // Image reference: Not a valid reference in generated docs
+	// docs/reference/cli/server.md L628
+
+	"MD055": false, // Table pipe style: Some of the generated tables do not have ending pipes
+	// docs/reference/api/schema.md
+	// docs/reference/api/templates.md
+	// docs/reference/cli/server.md
+
+	"MD056": false // Table column count: Some of the auto-generated tables have issues. TODO: This is probably because of splitting cell content to multiple lines.
+	// docs/reference/api/schema.md
+	// docs/reference/api/templates.md
+}

.prettierignore

@@ -1,97 +0,0 @@
-# Code generated by Makefile (.gitignore .prettierignore.include). DO NOT EDIT.
-
-# .gitignore:
-# Common ignore patterns, these rules applies in both root and subdirectories.
-.DS_Store
-.eslintcache
-.gitpod.yml
-.idea
-**/*.swp
-gotests.coverage
-gotests.xml
-gotests_stats.json
-gotests.json
-node_modules/
-vendor/
-yarn-error.log
-
-# VSCode settings.
-**/.vscode/*
-# Allow VSCode recommendations and default settings in project root.
-!/.vscode/extensions.json
-!/.vscode/settings.json
-
-# Front-end ignore patterns.
-.next/
-site/build-storybook.log
-site/coverage/
-site/storybook-static/
-site/test-results/*
-site/e2e/test-results/*
-site/e2e/states/*.json
-site/e2e/.auth.json
-site/playwright-report/*
-site/.swc
-
-# Make target for updating golden files (any dir).
-.gen-golden
-
-# Build
-build/
-dist/
-out/
-
-# Bundle analysis
-site/stats/
-
-*.tfstate
-*.tfstate.backup
-*.tfplan
-*.lock.hcl
-.terraform/
-
-**/.coderv2/*
-**/__debug_bin
-
-# direnv
-.envrc
-*.test
-
-# Loadtesting
-./scaletest/terraform/.terraform
-./scaletest/terraform/.terraform.lock.hcl
-scaletest/terraform/secrets.tfvars
-.terraform.tfstate.*
-
-# Nix
-result
-
-# Data dumps from unit tests
-**/*.test.sql
-
-# Filebrowser.db
-**/filebrowser.db
-
-# pnpm
-.pnpm-store/
-# .prettierignore.include:
-# Helm templates contain variables that are invalid YAML and can't be formatted
-# by Prettier.
-helm/**/templates/*.yaml
-
-# Terraform state files used in tests, these are automatically generated.
-# Example: provisioner/terraform/testdata/instance-id/instance-id.tfstate.json
-**/testdata/**/*.tf*.json
-
-# Testdata shouldn't be formatted.
-scripts/apitypings/testdata/**/*.ts
-enterprise/tailnet/testdata/*.golden.html
-tailnet/testdata/*.golden.html
-
-# Generated files shouldn't be formatted.
-site/e2e/provisionerGenerated.ts
-
-**/pnpm-lock.yaml
-
-# Ignore generated JSON (e.g. examples/examples.gen.json).
-**/*.gen.json

.prettierignore.include

@@ -1,20 +0,0 @@
-# Helm templates contain variables that are invalid YAML and can't be formatted
-# by Prettier.
-helm/**/templates/*.yaml
-
-# Terraform state files used in tests, these are automatically generated.
-# Example: provisioner/terraform/testdata/instance-id/instance-id.tfstate.json
-**/testdata/**/*.tf*.json
-
-# Testdata shouldn't be formatted.
-scripts/apitypings/testdata/**/*.ts
-enterprise/tailnet/testdata/*.golden.html
-tailnet/testdata/*.golden.html
-
-# Generated files shouldn't be formatted.
-site/e2e/provisionerGenerated.ts
-
-**/pnpm-lock.yaml
-
-# Ignore generated JSON (e.g. examples/examples.gen.json).
-**/*.gen.json

.prettierrc.yaml

@@ -4,13 +4,13 @@
 printWidth: 80
 proseWrap: always
 trailingComma: all
-useTabs: false
+useTabs: true
 tabWidth: 2
 overrides:
   - files:
       - README.md
-      - docs/api/**/*.md
-      - docs/cli/**/*.md
+      - docs/reference/api/**/*.md
+      - docs/reference/cli/**/*.md
       - docs/changelogs/*.md
       - .github/**/*.{yaml,yml,toml}
       - scripts/**/*.{yaml,yml,toml}

.vscode/extensions.json

@@ -1,15 +1,16 @@
 {
-  "recommendations": [
-    "github.vscode-codeql",
-    "golang.go",
-    "hashicorp.terraform",
-    "esbenp.prettier-vscode",
-    "foxundermoon.shell-format",
-    "emeraldwalk.runonsave",
-    "zxh404.vscode-proto3",
-    "redhat.vscode-yaml",
-    "streetsidesoftware.code-spell-checker",
-    "dbaeumer.vscode-eslint",
-    "EditorConfig.EditorConfig"
-  ]
+	"recommendations": [
+		"biomejs.biome",
+		"bradlc.vscode-tailwindcss",
+		"DavidAnson.vscode-markdownlint",
+		"EditorConfig.EditorConfig",
+		"emeraldwalk.runonsave",
+		"foxundermoon.shell-format",
+		"github.vscode-codeql",
+		"golang.go",
+		"hashicorp.terraform",
+		"redhat.vscode-yaml",
+		"tekumara.typos-vscode",
+		"zxh404.vscode-proto3"
+	]
 }

.vscode/markdown.code-snippets

@@ -0,0 +1,46 @@
+{
+	// For info about snippets, visit https://code.visualstudio.com/docs/editor/userdefinedsnippets
+
+    "admonition": {
+        "prefix": "#callout",
+        "body": [
+            "<blockquote class=\"admonition ${1|caution,important,note,tip,warning|}\">\n",
+            "${TM_SELECTED_TEXT:${2:add info here}}\n",
+            "</blockquote>\n"
+        ],
+        "description": "callout admonition caution info note tip warning"
+    },
+	"fenced code block": {
+		"prefix": "#codeblock",
+		"body": ["```${1|apache,bash,console,diff,Dockerfile,env,go,hcl,ini,json,lisp,md,powershell,shell,sql,text,tf,tsx,yaml|}", "${TM_SELECTED_TEXT}$0", "```"],
+		"description": "fenced code block"
+	},
+	"image": {
+		"prefix": "#image",
+		"body": "![${TM_SELECTED_TEXT:${1:alt}}](${2:url})$0",
+		"description": "image"
+	},
+	"premium-feature": {
+		"prefix": "#premium-feature",
+		"body": [
+			"<blockquote class=\"info\">\n",
+			"${1:feature} ${2|is,are|} an Enterprise and Premium feature. [Learn more](https://coder.com/pricing#compare-plans).\n",
+			"</blockquote>"
+		]
+	},
+	"tabs": {
+		"prefix": "#tabs",
+		"body": [
+			"<div class=\"tabs\">\n",
+			"${1:optional description}\n",
+			"## ${2:tab title}\n",
+			"${TM_SELECTED_TEXT:${3:first tab content}}\n",
+			"## ${4:tab title}\n",
+			"${5:second tab content}\n",
+			"## ${6:tab title}\n",
+			"${7:third tab content}\n",
+			"</div>\n"
+		],
+		"description": "tabs"
+	}
+}

.vscode/settings.json

@@ -1,227 +1,61 @@
 {
-  "cSpell.words": [
-    "afero",
-    "agentsdk",
-    "apps",
-    "ASKPASS",
-    "authcheck",
-    "autostop",
-    "awsidentity",
-    "bodyclose",
-    "buildinfo",
-    "buildname",
-    "circbuf",
-    "cliflag",
-    "cliui",
-    "codecov",
-    "coderd",
-    "coderdenttest",
-    "coderdtest",
-    "codersdk",
-    "contravariance",
-    "cronstrue",
-    "databasefake",
-    "dbgen",
-    "dbmem",
-    "dbtype",
-    "DERP",
-    "derphttp",
-    "derpmap",
-    "devel",
-    "devtunnel",
-    "dflags",
-    "drpc",
-    "drpcconn",
-    "drpcmux",
-    "drpcserver",
-    "Dsts",
-    "embeddedpostgres",
-    "enablements",
-    "enterprisemeta",
-    "errgroup",
-    "eventsourcemock",
-    "externalauth",
-    "Failf",
-    "fatih",
-    "Formik",
-    "gitauth",
-    "gitsshkey",
-    "goarch",
-    "gographviz",
-    "goleak",
-    "gonet",
-    "gossh",
-    "gsyslog",
-    "GTTY",
-    "hashicorp",
-    "hclsyntax",
-    "httpapi",
-    "httpmw",
-    "idtoken",
-    "Iflag",
-    "incpatch",
-    "initialisms",
-    "ipnstate",
-    "isatty",
-    "Jobf",
-    "Keygen",
-    "kirsle",
-    "Kubernetes",
-    "ldflags",
-    "magicsock",
-    "manifoldco",
-    "mapstructure",
-    "mattn",
-    "mitchellh",
-    "moby",
-    "namesgenerator",
-    "namespacing",
-    "netaddr",
-    "netip",
-    "netmap",
-    "netns",
-    "netstack",
-    "nettype",
-    "nfpms",
-    "nhooyr",
-    "nmcfg",
-    "nolint",
-    "nosec",
-    "ntqry",
-    "OIDC",
-    "oneof",
-    "opty",
-    "paralleltest",
-    "parameterscopeid",
-    "pqtype",
-    "prometheusmetrics",
-    "promhttp",
-    "protobuf",
-    "provisionerd",
-    "provisionerdserver",
-    "provisionersdk",
-    "ptty",
-    "ptys",
-    "ptytest",
-    "quickstart",
-    "reconfig",
-    "replicasync",
-    "retrier",
-    "rpty",
-    "SCIM",
-    "sdkproto",
-    "sdktrace",
-    "Signup",
-    "slogtest",
-    "sourcemapped",
-    "spinbutton",
-    "Srcs",
-    "stdbuf",
-    "stretchr",
-    "STTY",
-    "stuntest",
-    "tailbroker",
-    "tailcfg",
-    "tailexchange",
-    "tailnet",
-    "tailnettest",
-    "Tailscale",
-    "tanstack",
-    "tbody",
-    "TCGETS",
-    "tcpip",
-    "TCSETS",
-    "templateversions",
-    "testdata",
-    "testid",
-    "testutil",
-    "tfexec",
-    "tfjson",
-    "tfplan",
-    "tfstate",
-    "thead",
-    "tios",
-    "tmpdir",
-    "tokenconfig",
-    "Topbar",
-    "tparallel",
-    "trialer",
-    "trimprefix",
-    "tsdial",
-    "tslogger",
-    "tstun",
-    "turnconn",
-    "typegen",
-    "typesafe",
-    "unconvert",
-    "Untar",
-    "Userspace",
-    "VMID",
-    "walkthrough",
-    "weblinks",
-    "webrtc",
-    "wgcfg",
-    "wgconfig",
-    "wgengine",
-    "wgmonitor",
-    "wgnet",
-    "workspaceagent",
-    "workspaceagents",
-    "workspaceapp",
-    "workspaceapps",
-    "workspacebuilds",
-    "workspacename",
-    "wsjson",
-    "xerrors",
-    "xlarge",
-    "xsmall",
-    "yamux"
-  ],
-  "cSpell.ignorePaths": ["site/package.json", ".vscode/settings.json"],
-  "emeraldwalk.runonsave": {
-    "commands": [
-      {
-        "match": "database/queries/*.sql",
-        "cmd": "make gen"
-      },
-      {
-        "match": "provisionerd/proto/provisionerd.proto",
-        "cmd": "make provisionerd/proto/provisionerd.pb.go"
-      }
-    ]
-  },
-  "eslint.workingDirectories": ["./site"],
-  "search.exclude": {
-    "**.pb.go": true,
-    "**/*.gen.json": true,
-    "**/testdata/*": true,
-    "coderd/apidoc/**": true,
-    "docs/api/*.md": true,
-    "docs/templates/*.md": true,
-    "LICENSE": true,
-    "scripts/metricsdocgen/metrics": true,
-    "site/out/**": true,
-    "site/storybook-static/**": true,
-    "**.map": true,
-    "pnpm-lock.yaml": true
-  },
-  // Ensure files always have a newline.
-  "files.insertFinalNewline": true,
-  "go.lintTool": "golangci-lint",
-  "go.lintFlags": ["--fast"],
-  "go.coverageDecorator": {
-    "type": "gutter",
-    "coveredGutterStyle": "blockgreen",
-    "uncoveredGutterStyle": "blockred"
-  },
-  // The codersdk is used by coderd another other packages extensively.
-  // To reduce redundancy in tests, it's covered by other packages.
-  // Since package coverage pairing can't be defined, all packages cover
-  // all other packages.
-  "go.testFlags": ["-short", "-coverpkg=./..."],
-  // We often use a version of TypeScript that's ahead of the version shipped
-  // with VS Code.
-  "typescript.tsdk": "./site/node_modules/typescript/lib",
-  // Playwright tests in VSCode will open a browser to live "view" the test.
-  "playwright.reuseBrowser": true
+	"emeraldwalk.runonsave": {
+		"commands": [
+			{
+				"match": "database/queries/*.sql",
+				"cmd": "make gen"
+			},
+			{
+				"match": "provisionerd/proto/provisionerd.proto",
+				"cmd": "make provisionerd/proto/provisionerd.pb.go"
+			}
+		]
+	},
+	"search.exclude": {
+		"**.pb.go": true,
+		"**/*.gen.json": true,
+		"**/testdata/*": true,
+		"coderd/apidoc/**": true,
+		"docs/reference/api/*.md": true,
+		"docs/reference/cli/*.md": true,
+		"docs/templates/*.md": true,
+		"LICENSE": true,
+		"scripts/metricsdocgen/metrics": true,
+		"site/out/**": true,
+		"site/storybook-static/**": true,
+		"**.map": true,
+		"pnpm-lock.yaml": true
+	},
+	// Ensure files always have a newline.
+	"files.insertFinalNewline": true,
+	"go.lintTool": "golangci-lint",
+	"go.lintFlags": ["--fast"],
+	"go.coverageDecorator": {
+		"type": "gutter",
+		"coveredGutterStyle": "blockgreen",
+		"uncoveredGutterStyle": "blockred"
+	},
+	// The codersdk is used by coderd another other packages extensively.
+	// To reduce redundancy in tests, it's covered by other packages.
+	// Since package coverage pairing can't be defined, all packages cover
+	// all other packages.
+	"go.testFlags": ["-short", "-coverpkg=./..."],
+	// We often use a version of TypeScript that's ahead of the version shipped
+	// with VS Code.
+	"typescript.tsdk": "./site/node_modules/typescript/lib",
+	// Playwright tests in VSCode will open a browser to live "view" the test.
+	"playwright.reuseBrowser": true,
+
+	"[javascript][javascriptreact][json][jsonc][typescript][typescriptreact]": {
+		"editor.defaultFormatter": "biomejs.biome",
+		"editor.codeActionsOnSave": {
+			"quickfix.biome": "explicit"
+			//   "source.organizeImports.biome": "explicit"
+		}
+	},
+
+	"[css][html][markdown][yaml]": {
+		"editor.defaultFormatter": "esbenp.prettier-vscode"
+	},
+	"typos.config": ".github/workflows/typos.toml"
 }

CODEOWNERS

@@ -0,0 +1,6 @@
+# These APIs are versioned, so any changes need to be carefully reviewed for whether
+# to bump API major or minor versions.
+agent/proto/ @spikecurtis @johnstcn
+tailnet/proto/ @spikecurtis @johnstcn
+vpn/vpn.proto @spikecurtis @johnstcn
+vpn/version.go @spikecurtis @johnstcn

CODE_OF_CONDUCT.md

@@ -0,0 +1,2 @@
+<!-- markdownlint-disable MD041 -->
+[https://coder.com/docs/contributing/CODE_OF_CONDUCT](https://coder.com/docs/contributing/CODE_OF_CONDUCT)

CONTRIBUTING.md

@@ -0,0 +1,2 @@
+<!-- markdownlint-disable MD041 -->
+[https://coder.com/docs/CONTRIBUTING](https://coder.com/docs/CONTRIBUTING)

Makefile

@@ -79,8 +79,12 @@ PACKAGE_OS_ARCHES := linux_amd64 linux_armv7 linux_arm64
 # All architectures we build Docker images for (Linux only).
 DOCKER_ARCHES := amd64 arm64 armv7
 
+# All ${OS}_${ARCH} combos we build the desktop dylib for.
+DYLIB_ARCHES := darwin_amd64 darwin_arm64
+
 # Computed variables based on the above.
 CODER_SLIM_BINARIES      := $(addprefix build/coder-slim_$(VERSION)_,$(OS_ARCHES))
+CODER_DYLIBS             := $(foreach os_arch, $(DYLIB_ARCHES), build/coder-vpn_$(VERSION)_$(os_arch).dylib)
 CODER_FAT_BINARIES       := $(addprefix build/coder_$(VERSION)_,$(OS_ARCHES))
 CODER_ALL_BINARIES       := $(CODER_SLIM_BINARIES) $(CODER_FAT_BINARIES)
 CODER_TAR_GZ_ARCHIVES    := $(foreach os_arch, $(ARCHIVE_TAR_GZ), build/coder_$(VERSION)_$(os_arch).tar.gz)
@@ -238,6 +242,26 @@ $(CODER_ALL_BINARIES): go.mod go.sum \
 		cp "$@" "./site/out/bin/coder-$$os-$$arch$$dot_ext"
 	fi
 
+# This task builds Coder Desktop dylibs
+$(CODER_DYLIBS): go.mod go.sum $(GO_SRC_FILES)
+	@if [ "$(shell uname)" = "Darwin" ]; then
+		$(get-mode-os-arch-ext)
+		./scripts/build_go.sh \
+			--os "$$os" \
+			--arch "$$arch" \
+			--version "$(VERSION)" \
+			--output "$@" \
+			--dylib
+
+	else
+		echo "ERROR: Can't build dylib on non-Darwin OS" 1>&2
+		exit 1
+	fi
+
+# This task builds both dylibs
+build/coder-dylib: $(CODER_DYLIBS)
+.PHONY: build/coder-dylib
+
 # This task builds all archives. It parses the target name to get the metadata
 # for the build, so it must be specified in this format:
 #     build/coder_${version}_${os}_${arch}.${format}
@@ -364,15 +388,35 @@ $(foreach chart,$(charts),build/$(chart)_helm_$(VERSION).tgz): build/%_helm_$(VE
 		--chart $* \
 		--output "$@"
 
-site/out/index.html: site/package.json $(shell find ./site $(FIND_EXCLUSIONS) -type f \( -name '*.ts' -o -name '*.tsx' \))
-	cd site
+node_modules/.installed: package.json
+	./scripts/pnpm_install.sh
+
+offlinedocs/node_modules/.installed: offlinedocs/package.json
+	cd offlinedocs/
+	../scripts/pnpm_install.sh
+
+site/node_modules/.installed: site/package.json
+	cd site/
+	../scripts/pnpm_install.sh
+
+SITE_GEN_FILES := \
+	site/src/api/typesGenerated.ts \
+	site/src/api/rbacresourcesGenerated.ts \
+	site/src/api/countriesGenerated.ts \
+	site/src/theme/icons.json
+
+site/out/index.html: \
+	site/node_modules/.installed \
+	site/static/install.sh \
+	$(SITE_GEN_FILES) \
+	$(shell find ./site $(FIND_EXCLUSIONS) -type f \( -name '*.ts' -o -name '*.tsx' \))
+	cd site/
 	# prevents this directory from getting to big, and causing "too much data" errors
 	rm -rf out/assets/
-	../scripts/pnpm_install.sh
 	pnpm build
 
-offlinedocs/out/index.html: $(shell find ./offlinedocs $(FIND_EXCLUSIONS) -type f) $(shell find ./docs $(FIND_EXCLUSIONS) -type f | sed 's: :\\ :g')
-	cd offlinedocs
+offlinedocs/out/index.html: offlinedocs/node_modules/.installed $(shell find ./offlinedocs $(FIND_EXCLUSIONS) -type f) $(shell find ./docs $(FIND_EXCLUSIONS) -type f | sed 's: :\\ :g')
+	cd offlinedocs/
 	../scripts/pnpm_install.sh
 	pnpm export
 
@@ -391,32 +435,40 @@ BOLD := $(shell tput bold 2>/dev/null)
 GREEN := $(shell tput setaf 2 2>/dev/null)
 RESET := $(shell tput sgr0 2>/dev/null)
 
-fmt: fmt/eslint fmt/prettier fmt/terraform fmt/shfmt fmt/go
+fmt: fmt/ts fmt/go fmt/terraform fmt/shfmt fmt/biome fmt/markdown
 .PHONY: fmt
 
 fmt/go:
+	go mod tidy
 	echo "$(GREEN)==>$(RESET) $(BOLD)fmt/go$(RESET)"
 	# VS Code users should check out
 	# https://github.com/mvdan/gofumpt#visual-studio-code
-	go run mvdan.cc/gofumpt@v0.4.0 -w -l .
+	find . $(FIND_EXCLUSIONS) -type f -name '*.go' -print0 | \
+		xargs -0 grep --null -L "DO NOT EDIT" | \
+		xargs -0 go run mvdan.cc/gofumpt@v0.4.0 -w -l
 .PHONY: fmt/go
 
-fmt/eslint:
-	echo "$(GREEN)==>$(RESET) $(BOLD)fmt/eslint$(RESET)"
+fmt/ts: site/node_modules/.installed
+	echo "$(GREEN)==>$(RESET) $(BOLD)fmt/ts$(RESET)"
 	cd site
-	pnpm run lint:fix
-.PHONY: fmt/eslint
+# Avoid writing files in CI to reduce file write activity
+ifdef CI
+	pnpm run check --linter-enabled=false
+else
+	pnpm run check:fix
+endif
+.PHONY: fmt/ts
 
-fmt/prettier:
-	echo "$(GREEN)==>$(RESET) $(BOLD)fmt/prettier$(RESET)"
-	cd site
+fmt/biome: site/node_modules/.installed
+	echo "$(GREEN)==>$(RESET) $(BOLD)fmt/biome$(RESET)"
+	cd site/
 # Avoid writing files in CI to reduce file write activity
 ifdef CI
 	pnpm run format:check
 else
 	pnpm run format
 endif
-.PHONY: fmt/prettier
+.PHONY: fmt/biome
 
 fmt/terraform: $(wildcard *.tf)
 	echo "$(GREEN)==>$(RESET) $(BOLD)fmt/terraform$(RESET)"
@@ -433,23 +485,28 @@ else
 endif
 .PHONY: fmt/shfmt
 
-lint: lint/shellcheck lint/go lint/ts lint/examples lint/helm lint/site-icons
+fmt/markdown: node_modules/.installed
+	echo "$(GREEN)==>$(RESET) $(BOLD)fmt/markdown$(RESET)"
+	pnpm format-docs
+.PHONY: fmt/markdown
+
+lint: lint/shellcheck lint/go lint/ts lint/examples lint/helm lint/site-icons lint/markdown
 .PHONY: lint
 
 lint/site-icons:
 	./scripts/check_site_icons.sh
 .PHONY: lint/site-icons
 
-lint/ts:
-	cd site
-	pnpm i && pnpm lint
+lint/ts: site/node_modules/.installed
+	cd site/
+	pnpm lint
 .PHONY: lint/ts
 
 lint/go:
 	./scripts/check_enterprise_imports.sh
-	linter_ver=$(shell egrep -o 'GOLANGCI_LINT_VERSION=\S+' dogfood/Dockerfile | cut -d '=' -f 2)
-	go install github.com/golangci/golangci-lint/cmd/golangci-lint@v$$linter_ver
-	golangci-lint run
+	./scripts/check_codersdk_imports.sh
+	linter_ver=$(shell egrep -o 'GOLANGCI_LINT_VERSION=\S+' dogfood/contents/Dockerfile | cut -d '=' -f 2)
+	go run github.com/golangci/golangci-lint/cmd/golangci-lint@v$$linter_ver run
 .PHONY: lint/go
 
 lint/examples:
@@ -463,13 +520,18 @@ lint/shellcheck: $(SHELL_SRC_FILES)
 .PHONY: lint/shellcheck
 
 lint/helm:
-	cd helm
+	cd helm/
 	make lint
 .PHONY: lint/helm
 
+lint/markdown: node_modules/.installed
+	pnpm lint-docs
+.PHONY: lint/markdown
+
 # All files generated by the database should be added here, and this can be used
 # as a target for jobs that need to run after the database is generated.
 DB_GEN_FILES := \
+	coderd/database/dump.sql \
 	coderd/database/querier.go \
 	coderd/database/unique_constraint.go \
 	coderd/database/dbmem/dbmem.go \
@@ -477,35 +539,40 @@ DB_GEN_FILES := \
 	coderd/database/dbauthz/dbauthz.go \
 	coderd/database/dbmock/dbmock.go
 
-# all gen targets should be added here and to gen/mark-fresh
-gen: \
+TAILNETTEST_MOCKS := \
+	tailnet/tailnettest/coordinatormock.go \
+	tailnet/tailnettest/coordinateemock.go \
+	tailnet/tailnettest/workspaceupdatesprovidermock.go \
+	tailnet/tailnettest/subscriptionmock.go
+
+GEN_FILES := \
 	tailnet/proto/tailnet.pb.go \
 	agent/proto/agent.pb.go \
 	provisionersdk/proto/provisioner.pb.go \
 	provisionerd/proto/provisionerd.pb.go \
-	coderd/database/dump.sql \
+	vpn/vpn.pb.go \
 	$(DB_GEN_FILES) \
-	site/src/api/typesGenerated.ts \
+	$(SITE_GEN_FILES) \
 	coderd/rbac/object_gen.go \
 	codersdk/rbacresources_gen.go \
-	docs/admin/prometheus.md \
-	docs/cli.md \
-	docs/admin/audit-logs.md \
+	docs/admin/integrations/prometheus.md \
+	docs/reference/cli/index.md \
+	docs/admin/security/audit-logs.md \
 	coderd/apidoc/swagger.json \
-	.prettierignore.include \
-	.prettierignore \
 	provisioner/terraform/testdata/version \
-	site/.prettierrc.yaml \
-	site/.prettierignore \
-	site/.eslintignore \
 	site/e2e/provisionerGenerated.ts \
-	site/src/theme/icons.json \
 	examples/examples.gen.json \
-	tailnet/tailnettest/coordinatormock.go \
-	tailnet/tailnettest/coordinateemock.go \
-	tailnet/tailnettest/multiagentmock.go
+	$(TAILNETTEST_MOCKS) \
+	coderd/database/pubsub/psmock/psmock.go \
+	coderd/httpmw/loggermw/loggermock/loggermock.go
+
+# all gen targets should be added here and to gen/mark-fresh
+gen: gen/db $(GEN_FILES)
 .PHONY: gen
 
+gen/db: $(DB_GEN_FILES)
+.PHONY: gen/db
+
 # Mark all generated files as fresh so make thinks they're up-to-date. This is
 # used during releases so we don't run generation scripts.
 gen/mark-fresh:
@@ -514,26 +581,26 @@ gen/mark-fresh:
 		agent/proto/agent.pb.go \
 		provisionersdk/proto/provisioner.pb.go \
 		provisionerd/proto/provisionerd.pb.go \
+		vpn/vpn.pb.go \
 		coderd/database/dump.sql \
 		$(DB_GEN_FILES) \
 		site/src/api/typesGenerated.ts \
 		coderd/rbac/object_gen.go \
-		docs/admin/prometheus.md \
-		docs/cli.md \
-		docs/admin/audit-logs.md \
+		codersdk/rbacresources_gen.go \
+		site/src/api/rbacresourcesGenerated.ts \
+		site/src/api/countriesGenerated.ts \
+		docs/admin/integrations/prometheus.md \
+		docs/reference/cli/index.md \
+		docs/admin/security/audit-logs.md \
 		coderd/apidoc/swagger.json \
-		.prettierignore.include \
-		.prettierignore \
-		site/.prettierrc.yaml \
-		site/.prettierignore \
-		site/.eslintignore \
 		site/e2e/provisionerGenerated.ts \
 		site/src/theme/icons.json \
 		examples/examples.gen.json \
-		tailnet/tailnettest/coordinatormock.go \
-		tailnet/tailnettest/coordinateemock.go \
-		tailnet/tailnettest/multiagentmock.go \
-	"
+		$(TAILNETTEST_MOCKS) \
+		coderd/database/pubsub/psmock/psmock.go \
+		coderd/httpmw/loggermw/loggermock/loggermock.go \
+		"
+
 	for file in $$files; do
 		echo "$$file"
 		if [ ! -f "$$file" ]; then
@@ -542,7 +609,7 @@ gen/mark-fresh:
 		fi
 
 		# touch sets the mtime of the file to the current time
-		touch $$file
+		touch "$$file"
 	done
 .PHONY: gen/mark-fresh
 
@@ -563,7 +630,10 @@ coderd/database/dbmock/dbmock.go: coderd/database/db.go coderd/database/querier.
 coderd/database/pubsub/psmock/psmock.go: coderd/database/pubsub/pubsub.go
 	go generate ./coderd/database/pubsub/psmock
 
-tailnet/tailnettest/coordinatormock.go tailnet/tailnettest/multiagentmock.go tailnet/tailnettest/coordinateemock.go: tailnet/coordinator.go tailnet/multiagent.go
+coderd/httpmw/loggermw/loggermock/loggermock.go: coderd/httpmw/loggermw/logger.go
+	go generate ./coderd/httpmw/loggermw/loggermock/
+
+$(TAILNETTEST_MOCKS): tailnet/coordinator.go tailnet/service.go
 	go generate ./tailnet/tailnettest/
 
 tailnet/proto/tailnet.pb.go: tailnet/proto/tailnet.proto
@@ -598,64 +668,105 @@ provisionerd/proto/provisionerd.pb.go: provisionerd/proto/provisionerd.proto
 		--go-drpc_opt=paths=source_relative \
 		./provisionerd/proto/provisionerd.proto
 
-site/src/api/typesGenerated.ts: $(wildcard scripts/apitypings/*) $(shell find ./codersdk $(FIND_EXCLUSIONS) -type f -name '*.go')
-	go run ./scripts/apitypings/ > $@
-	./scripts/pnpm_install.sh
-	pnpm exec prettier --write "$@"
+vpn/vpn.pb.go: vpn/vpn.proto
+	protoc \
+		--go_out=. \
+		--go_opt=paths=source_relative \
+		./vpn/vpn.proto
 
-site/e2e/provisionerGenerated.ts: provisionerd/proto/provisionerd.pb.go provisionersdk/proto/provisioner.pb.go
-	cd site
-	../scripts/pnpm_install.sh
+site/src/api/typesGenerated.ts: site/node_modules/.installed $(wildcard scripts/apitypings/*) $(shell find ./codersdk $(FIND_EXCLUSIONS) -type f -name '*.go')
+	# -C sets the directory for the go run command
+	go run -C ./scripts/apitypings main.go > $@
+	cd site/
+	pnpm exec biome format --write src/api/typesGenerated.ts
+
+site/e2e/provisionerGenerated.ts: site/node_modules/.installed provisionerd/proto/provisionerd.pb.go provisionersdk/proto/provisioner.pb.go
+	cd site/
 	pnpm run gen:provisioner
 
-site/src/theme/icons.json: $(wildcard scripts/gensite/*) $(wildcard site/static/icon/*)
+site/src/theme/icons.json: site/node_modules/.installed $(wildcard scripts/gensite/*) $(wildcard site/static/icon/*)
 	go run ./scripts/gensite/ -icons "$@"
-	./scripts/pnpm_install.sh
-	pnpm exec prettier --write "$@"
+	cd site/
+	pnpm exec biome format --write src/theme/icons.json
 
 examples/examples.gen.json: scripts/examplegen/main.go examples/examples.go $(shell find ./examples/templates)
 	go run ./scripts/examplegen/main.go > examples/examples.gen.json
 
-coderd/rbac/object_gen.go: scripts/rbacgen/rbacobject.gotmpl scripts/rbacgen/main.go coderd/rbac/object.go
-	go run scripts/rbacgen/main.go rbac > coderd/rbac/object_gen.go
+coderd/rbac/object_gen.go: scripts/typegen/rbacobject.gotmpl scripts/typegen/main.go coderd/rbac/object.go coderd/rbac/policy/policy.go
+	tempdir=$(shell mktemp -d /tmp/typegen_rbac_object.XXXXXX)
+	go run ./scripts/typegen/main.go rbac object > "$$tempdir/object_gen.go"
+	mv -v "$$tempdir/object_gen.go" coderd/rbac/object_gen.go
+	rmdir -v "$$tempdir"
 
-codersdk/rbacresources_gen.go: scripts/rbacgen/codersdk.gotmpl scripts/rbacgen/main.go coderd/rbac/object.go
-	go run scripts/rbacgen/main.go codersdk > codersdk/rbacresources_gen.go
+codersdk/rbacresources_gen.go: scripts/typegen/codersdk.gotmpl scripts/typegen/main.go coderd/rbac/object.go coderd/rbac/policy/policy.go
+	# Do no overwrite codersdk/rbacresources_gen.go directly, as it would make the file empty, breaking
+ 	# the `codersdk` package and any parallel build targets.
+	go run scripts/typegen/main.go rbac codersdk > /tmp/rbacresources_gen.go
+	mv /tmp/rbacresources_gen.go codersdk/rbacresources_gen.go
 
-docs/admin/prometheus.md: scripts/metricsdocgen/main.go scripts/metricsdocgen/metrics
+site/src/api/rbacresourcesGenerated.ts: site/node_modules/.installed scripts/typegen/codersdk.gotmpl scripts/typegen/main.go coderd/rbac/object.go coderd/rbac/policy/policy.go
+	go run scripts/typegen/main.go rbac typescript > "$@"
+	cd site/
+	pnpm exec biome format --write src/api/rbacresourcesGenerated.ts
+
+site/src/api/countriesGenerated.ts: site/node_modules/.installed scripts/typegen/countries.tstmpl scripts/typegen/main.go codersdk/countries.go
+	go run scripts/typegen/main.go countries > "$@"
+	cd site/
+	pnpm exec biome format --write src/api/countriesGenerated.ts
+
+docs/admin/integrations/prometheus.md: node_modules/.installed scripts/metricsdocgen/main.go scripts/metricsdocgen/metrics
 	go run scripts/metricsdocgen/main.go
-	./scripts/pnpm_install.sh
-	pnpm exec prettier --write ./docs/admin/prometheus.md
+	pnpm exec markdownlint-cli2 --fix ./docs/admin/integrations/prometheus.md
+	pnpm exec markdown-table-formatter ./docs/admin/integrations/prometheus.md
 
-docs/cli.md: scripts/clidocgen/main.go examples/examples.gen.json $(GO_SRC_FILES)
+docs/reference/cli/index.md: node_modules/.installed site/node_modules/.installed scripts/clidocgen/main.go examples/examples.gen.json $(GO_SRC_FILES)
 	CI=true BASE_PATH="." go run ./scripts/clidocgen
-	./scripts/pnpm_install.sh
-	pnpm exec prettier --write ./docs/cli.md ./docs/cli/*.md ./docs/manifest.json
+	pnpm exec markdownlint-cli2 --fix ./docs/reference/cli/*.md
+	pnpm exec markdown-table-formatter ./docs/reference/cli/*.md
+	cd site/
+	pnpm exec biome format --write ../docs/manifest.json
 
-docs/admin/audit-logs.md: coderd/database/querier.go scripts/auditdocgen/main.go enterprise/audit/table.go coderd/rbac/object_gen.go
+docs/admin/security/audit-logs.md: node_modules/.installed coderd/database/querier.go scripts/auditdocgen/main.go enterprise/audit/table.go coderd/rbac/object_gen.go
 	go run scripts/auditdocgen/main.go
-	./scripts/pnpm_install.sh
-	pnpm exec prettier --write ./docs/admin/audit-logs.md
+	pnpm exec markdownlint-cli2 --fix ./docs/admin/security/audit-logs.md
+	pnpm exec markdown-table-formatter ./docs/admin/security/audit-logs.md
 
-coderd/apidoc/swagger.json: $(shell find ./scripts/apidocgen $(FIND_EXCLUSIONS) -type f) $(wildcard coderd/*.go) $(wildcard enterprise/coderd/*.go) $(wildcard codersdk/*.go) $(wildcard enterprise/wsproxy/wsproxysdk/*.go) $(DB_GEN_FILES) .swaggo docs/manifest.json coderd/rbac/object_gen.go
+coderd/apidoc/swagger.json: node_modules/.installed site/node_modules/.installed $(shell find ./scripts/apidocgen $(FIND_EXCLUSIONS) -type f) $(wildcard coderd/*.go) $(wildcard enterprise/coderd/*.go) $(wildcard codersdk/*.go) $(wildcard enterprise/wsproxy/wsproxysdk/*.go) $(DB_GEN_FILES) .swaggo docs/manifest.json coderd/rbac/object_gen.go
 	./scripts/apidocgen/generate.sh
-	./scripts/pnpm_install.sh
-	pnpm exec prettier --write ./docs/api ./docs/manifest.json ./coderd/apidoc/swagger.json
+	pnpm exec markdownlint-cli2 --fix ./docs/reference/api/*.md
+	pnpm exec markdown-table-formatter ./docs/reference/api/*.md
+	cd site/
+	pnpm exec biome format --write ../docs/manifest.json ../coderd/apidoc/swagger.json
 
 update-golden-files: \
 	cli/testdata/.gen-golden \
-	helm/coder/tests/testdata/.gen-golden \
-	helm/provisioner/tests/testdata/.gen-golden \
-	scripts/ci-report/testdata/.gen-golden \
+	coderd/.gen-golden \
+	coderd/notifications/.gen-golden \
 	enterprise/cli/testdata/.gen-golden \
 	enterprise/tailnet/testdata/.gen-golden \
-	tailnet/testdata/.gen-golden \
-	coderd/.gen-golden \
-	provisioner/terraform/testdata/.gen-golden
+	helm/coder/tests/testdata/.gen-golden \
+	helm/provisioner/tests/testdata/.gen-golden \
+	provisioner/terraform/testdata/.gen-golden \
+	tailnet/testdata/.gen-golden
 .PHONY: update-golden-files
 
+clean/golden-files:
+	find . -type f -name '.gen-golden' -delete
+	find \
+		cli/testdata \
+		coderd/notifications/testdata \
+		coderd/testdata \
+		enterprise/cli/testdata \
+		enterprise/tailnet/testdata \
+		helm/coder/tests/testdata \
+		helm/provisioner/tests/testdata \
+		provisioner/terraform/testdata \
+		tailnet/testdata \
+		-type f -name '*.golden' -delete
+.PHONY: clean/golden-files
+
 cli/testdata/.gen-golden: $(wildcard cli/testdata/*.golden) $(wildcard cli/*.tpl) $(GO_SRC_FILES) $(wildcard cli/*_test.go)
-	go test ./cli -run="Test(CommandHelp|ServerYAML|ErrorExamples)" -update
+	go test ./cli -run="Test(CommandHelp|ServerYAML|ErrorExamples|.*Golden)" -update
 	touch "$@"
 
 enterprise/cli/testdata/.gen-golden: $(wildcard enterprise/cli/testdata/*.golden) $(wildcard cli/*.tpl) $(GO_SRC_FILES) $(wildcard enterprise/cli/*_test.go)
@@ -682,6 +793,10 @@ coderd/.gen-golden: $(wildcard coderd/testdata/*/*.golden) $(GO_SRC_FILES) $(wil
 	go test ./coderd -run="Test.*Golden$$" -update
 	touch "$@"
 
+coderd/notifications/.gen-golden: $(wildcard coderd/notifications/testdata/*/*.golden) $(GO_SRC_FILES) $(wildcard coderd/notifications/*_test.go)
+	go test ./coderd/notifications -run="Test.*Golden$$" -update
+	touch "$@"
+
 provisioner/terraform/testdata/.gen-golden: $(wildcard provisioner/terraform/testdata/*/*.golden) $(GO_SRC_FILES) $(wildcard provisioner/terraform/*_test.go)
 	go test ./provisioner/terraform -run="Test.*Golden$$" -update
 	touch "$@"
@@ -692,74 +807,14 @@ provisioner/terraform/testdata/version:
 	fi
 .PHONY: provisioner/terraform/testdata/version
 
-scripts/ci-report/testdata/.gen-golden: $(wildcard scripts/ci-report/testdata/*) $(wildcard scripts/ci-report/*.go)
-	go test ./scripts/ci-report -run=TestOutputMatchesGoldenFile -update
-	touch "$@"
-
-# Generate a prettierrc for the site package that uses relative paths for
-# overrides. This allows us to share the same prettier config between the
-# site and the root of the repo.
-site/.prettierrc.yaml: .prettierrc.yaml
-	. ./scripts/lib.sh
-	dependencies yq
-
-	echo "# Code generated by Makefile (../$<). DO NOT EDIT." > "$@"
-	echo "" >> "$@"
-
-	# Replace all listed override files with relative paths inside site/.
-	# - ./ -> ../
-	# - ./site -> ./
-	yq \
-		'.overrides[].files |= map(. | sub("^./"; "") | sub("^"; "../") | sub("../site/"; "./") | sub("../!"; "!../"))' \
-		"$<" >> "$@"
-
-# Combine .gitignore with .prettierignore.include to generate .prettierignore.
-.prettierignore: .gitignore .prettierignore.include
-	echo "# Code generated by Makefile ($^). DO NOT EDIT." > "$@"
-	echo "" >> "$@"
-	for f in $^; do
-		echo "# $${f}:" >> "$@"
-		cat "$$f" >> "$@"
-	done
-
-# Generate ignore files based on gitignore into the site directory. We turn all
-# rules into relative paths for the `site/` directory (where applicable),
-# following the pattern format defined by git:
-# https://git-scm.com/docs/gitignore#_pattern_format
-#
-# This is done for compatibility reasons, see:
-# https://github.com/prettier/prettier/issues/8048
-# https://github.com/prettier/prettier/issues/8506
-# https://github.com/prettier/prettier/issues/8679
-site/.eslintignore site/.prettierignore: .prettierignore Makefile
-	rm -f "$@"
-	touch "$@"
-	# Skip generated by header, inherit `.prettierignore` header as-is.
-	while read -r rule; do
-		# Remove leading ! if present to simplify rule, added back at the end.
-		tmp="$${rule#!}"
-		ignore="$${rule%"$$tmp"}"
-		rule="$$tmp"
-		case "$$rule" in
-			# Comments or empty lines (include).
-			\#*|'') ;;
-			# Generic rules (include).
-			\*\**) ;;
-			# Site prefixed rules (include).
-			site/*) rule="$${rule#site/}";;
-			./site/*) rule="$${rule#./site/}";;
-			# Rules that are non-generic and don't start with site (rewrite).
-			/*) rule=.."$$rule";;
-			*/?*) rule=../"$$rule";;
-			*) ;;
-		esac
-		echo "$${ignore}$${rule}" >> "$@"
-	done < "$<"
-
 test:
 	$(GIT_FLAGS) gotestsum --format standard-quiet -- -v -short -count=1 ./...
 .PHONY: test
 
+test-cli:
+	$(GIT_FLAGS) gotestsum --format standard-quiet -- -v -short -count=1 ./cli/...
+.PHONY: test-cli
+
 # sqlc-cloud-is-setup will fail if no SQLc auth token is set. Use this as a
 # dependency for any sqlc-cloud related targets.
 sqlc-cloud-is-setup:
@@ -793,7 +848,7 @@ sqlc-vet: test-postgres-docker
 test-postgres: test-postgres-docker
 	# The postgres test is prone to failure, so we limit parallelism for
 	# more consistent execution.
-	$(GIT_FLAGS)  DB=ci DB_FROM=$(shell go run scripts/migrate-ci/main.go) gotestsum \
+	$(GIT_FLAGS)  DB=ci gotestsum \
 		--junitfile="gotests.xml" \
 		--jsonfile="gotests.json" \
 		--packages="./..." -- \
@@ -812,10 +867,35 @@ test-migrations: test-postgres-docker
 	if [[ "$${COMMIT_FROM}" == "$${COMMIT_TO}" ]]; then echo "Nothing to do!"; exit 0; fi
 	echo "DROP DATABASE IF EXISTS migrate_test_$${COMMIT_FROM}; CREATE DATABASE migrate_test_$${COMMIT_FROM};" | psql 'postgresql://postgres:postgres@localhost:5432/postgres?sslmode=disable'
 	go run ./scripts/migrate-test/main.go --from="$$COMMIT_FROM" --to="$$COMMIT_TO" --postgres-url="postgresql://postgres:postgres@localhost:5432/migrate_test_$${COMMIT_FROM}?sslmode=disable"
+.PHONY: test-migrations
 
 # NOTE: we set --memory to the same size as a GitHub runner.
 test-postgres-docker:
 	docker rm -f test-postgres-docker-${POSTGRES_VERSION} || true
+
+	# Try pulling up to three times to avoid CI flakes.
+	docker pull gcr.io/coder-dev-1/postgres:${POSTGRES_VERSION} || {
+		retries=2
+		for try in $(seq 1 ${retries}); do
+			echo "Failed to pull image, retrying (${try}/${retries})..."
+			sleep 1
+			if docker pull gcr.io/coder-dev-1/postgres:${POSTGRES_VERSION}; then
+				break
+			fi
+		done
+	}
+
+	# Make sure to not overallocate work_mem and max_connections as each
+	# connection will be allowed to use this much memory. Try adjusting
+	# shared_buffers instead, if needed.
+	#
+	# - work_mem=8MB * max_connections=1000 = 8GB
+	# - shared_buffers=2GB + effective_cache_size=1GB = 3GB
+	#
+	# This leaves 5GB for the rest of the system _and_ storing the
+	# database in memory (--tmpfs).
+	#
+	# https://www.postgresql.org/docs/current/runtime-config-resource.html#GUC-WORK-MEM
 	docker run \
 		--env POSTGRES_PASSWORD=postgres \
 		--env POSTGRES_USER=postgres \
@@ -828,9 +908,9 @@ test-postgres-docker:
 		--detach \
 		--memory 16GB \
 		gcr.io/coder-dev-1/postgres:${POSTGRES_VERSION} \
-		-c shared_buffers=1GB \
-		-c work_mem=1GB \
+		-c shared_buffers=2GB \
 		-c effective_cache_size=1GB \
+		-c work_mem=8MB \
 		-c max_connections=1000 \
 		-c fsync=off \
 		-c synchronous_commit=off \
@@ -845,7 +925,7 @@ test-postgres-docker:
 
 # Make sure to keep this in sync with test-go-race from .github/workflows/ci.yaml.
 test-race:
-	$(GIT_FLAGS) gotestsum --junitfile="gotests.xml" -- -race -count=1 ./...
+	$(GIT_FLAGS) gotestsum --junitfile="gotests.xml" -- -race -count=1 -parallel 4 -p 4 ./...
 .PHONY: test-race
 
 test-tailnet-integration:
@@ -859,6 +939,7 @@ test-tailnet-integration:
 			-timeout=5m \
 			-count=1 \
 			./tailnet/test/integration
+.PHONY: test-tailnet-integration
 
 # Note: we used to add this to the test target, but it's not necessary and we can
 # achieve the desired result by specifying -count=1 in the go test invocation
@@ -867,6 +948,19 @@ test-clean:
 	go clean -testcache
 .PHONY: test-clean
 
+site/e2e/bin/coder: go.mod go.sum $(GO_SRC_FILES)
+	go build -o $@ \
+		-tags ts_omit_aws,ts_omit_bird,ts_omit_tap,ts_omit_kube \
+		./enterprise/cmd/coder
+
+test-e2e: site/e2e/bin/coder site/node_modules/.installed site/out/index.html
+	cd site/
+ifdef CI
+	DEBUG=pw:api pnpm playwright:test --forbid-only --workers 1
+else
+	pnpm playwright:test
+endif
 .PHONY: test-e2e
-test-e2e:
-	cd ./site && DEBUG=pw:api pnpm playwright:test --forbid-only --workers 1
+
+dogfood/contents/nix.hash: flake.nix flake.lock
+	sha256sum flake.nix flake.lock >./dogfood/contents/nix.hash

README.md

@@ -1,9 +1,10 @@
+<!-- markdownlint-disable MD041 -->
 <div align="center">
   <a href="https://coder.com#gh-light-mode-only">
-    <img src="./docs/images/logo-black.png" style="width: 128px">
+    <img src="./docs/images/logo-black.png" alt="Coder Logo Light" style="width: 128px">
   </a>
   <a href="https://coder.com#gh-dark-mode-only">
-    <img src="./docs/images/logo-white.png" style="width: 128px">
+    <img src="./docs/images/logo-white.png" alt="Coder Logo Dark" style="width: 128px">
   </a>
 
   <h1>
@@ -11,21 +12,23 @@
   </h1>
 
   <a href="https://coder.com#gh-light-mode-only">
-    <img src="./docs/images/banner-black.png" style="width: 650px">
+    <img src="./docs/images/banner-black.png" alt="Coder Banner Light" style="width: 650px">
   </a>
   <a href="https://coder.com#gh-dark-mode-only">
-    <img src="./docs/images/banner-white.png" style="width: 650px">
+    <img src="./docs/images/banner-white.png" alt="Coder Banner Dark" style="width: 650px">
   </a>
 
   <br>
   <br>
 
-[Quickstart](#quickstart) | [Docs](https://coder.com/docs) | [Why Coder](https://coder.com/why) | [Enterprise](https://coder.com/docs/enterprise)
+[Quickstart](#quickstart) | [Docs](https://coder.com/docs) | [Why Coder](https://coder.com/why) | [Premium](https://coder.com/pricing#compare-plans)
 
 [![discord](https://img.shields.io/discord/747933592273027093?label=discord)](https://discord.gg/coder)
 [![release](https://img.shields.io/github/v/release/coder/coder)](https://github.com/coder/coder/releases/latest)
 [![godoc](https://pkg.go.dev/badge/github.com/coder/coder.svg)](https://pkg.go.dev/github.com/coder/coder)
 [![Go Report Card](https://goreportcard.com/badge/github.com/coder/coder/v2)](https://goreportcard.com/report/github.com/coder/coder/v2)
+[![OpenSSF Best Practices](https://www.bestpractices.dev/projects/9511/badge)](https://www.bestpractices.dev/projects/9511)
+[![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/coder/coder/badge)](https://scorecard.dev/viewer/?uri=github.com%2Fcoder%2Fcoder)
 [![license](https://img.shields.io/github/license/coder/coder)](./LICENSE)
 
 </div>
@@ -38,14 +41,14 @@
 - Onboard developers in seconds instead of days
 
 <p align="center">
-  <img src="./docs/images/hero-image.png">
+  <img src="./docs/images/hero-image.png" alt="Coder Hero Image">
 </p>
 
 ## Quickstart
 
 The most convenient way to try Coder is to install it on your local machine and experiment with provisioning cloud development environments using Docker (works on Linux, macOS, and Windows).
 
-```
+```shell
 # First, install Coder
 curl -L https://coder.com/install.sh | sh
 
@@ -63,7 +66,7 @@ The easiest way to install Coder is to use our
 and macOS. For Windows, use the latest `..._installer.exe` file from GitHub
 Releases.
 
-```bash
+```shell
 curl -L https://coder.com/install.sh | sh
 ```
 
@@ -91,7 +94,7 @@ Browse our docs [here](https://coder.com/docs) or visit a specific section below
 - [**Workspaces**](https://coder.com/docs/workspaces): Workspaces contain the IDEs, dependencies, and configuration information needed for software development
 - [**IDEs**](https://coder.com/docs/ides): Connect your existing editor to a workspace
 - [**Administration**](https://coder.com/docs/admin): Learn how to operate Coder
-- [**Enterprise**](https://coder.com/docs/enterprise): Learn about our paid features built for large teams
+- [**Premium**](https://coder.com/pricing#compare-plans): Learn about our paid features built for large teams
 
 ## Support
 
@@ -111,6 +114,7 @@ We are always working on new integrations. Please feel free to open an issue and
 - [**Module Registry**](https://registry.coder.com): Extend development environments with common use-cases
 - [**Kubernetes Log Stream**](https://github.com/coder/coder-logstream-kube): Stream Kubernetes Pod events to the Coder startup logs
 - [**Self-Hosted VS Code Extension Marketplace**](https://github.com/coder/code-marketplace): A private extension marketplace that works in restricted or airgapped networks integrating with [code-server](https://github.com/coder/code-server).
+- [**Setup Coder**](https://github.com/marketplace/actions/setup-coder): An action to setup coder CLI in GitHub workflows.
 
 ### Community
 

SECURITY.md

@@ -8,7 +8,7 @@ to us, what we expect, what you can expect from us.
 
 You can see the pretty version [here](https://coder.com/security/policy)
 
-# Why Coder's security matters
+## Why Coder's security matters
 
 If an attacker could fully compromise a Coder installation, they could spin up
 expensive workstations, steal valuable credentials, or steal proprietary source
@@ -16,13 +16,13 @@ code. We take this risk very seriously and employ routine pen testing,
 vulnerability scanning, and code reviews. We also welcome the contributions from
 the community that helped make this product possible.
 
-# Where should I report security issues?
+## Where should I report security issues?
 
-Please report security issues to security@coder.com, providing all relevant
+Please report security issues to <security@coder.com>, providing all relevant
 information. The more details you provide, the easier it will be for us to
 triage and fix the issue.
 
-# Out of Scope
+## Out of Scope
 
 Our primary concern is around an abuse of the Coder application that allows an
 attacker to gain access to another users workspace, or spin up unwanted
@@ -40,7 +40,7 @@ workspaces.
   out-of-scope systems should be reported to the appropriate vendor or
   applicable authority.
 
-# Our Commitments
+## Our Commitments
 
 When working with us, according to this policy, you can expect us to:
 
@@ -53,7 +53,7 @@ When working with us, according to this policy, you can expect us to:
 - Extend Safe Harbor for your vulnerability research that is related to this
   policy.
 
-# Our Expectations
+## Our Expectations
 
 In participating in our vulnerability disclosure program in good faith, we ask
 that you:

agent/agent_test.go

@@ -19,10 +19,9 @@ import (
 	"path/filepath"
 	"regexp"
 	"runtime"
+	"strconv"
 	"strings"
-	"sync"
 	"sync/atomic"
-	"syscall"
 	"testing"
 	"time"
 
@@ -36,7 +35,6 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"go.uber.org/goleak"
-	"go.uber.org/mock/gomock"
 	"golang.org/x/crypto/ssh"
 	"golang.org/x/exp/slices"
 	"golang.org/x/xerrors"
@@ -44,11 +42,8 @@ import (
 	"tailscale.com/tailcfg"
 
 	"cdr.dev/slog"
-	"cdr.dev/slog/sloggers/sloghuman"
 	"cdr.dev/slog/sloggers/slogtest"
 	"github.com/coder/coder/v2/agent"
-	"github.com/coder/coder/v2/agent/agentproc"
-	"github.com/coder/coder/v2/agent/agentproc/agentproctest"
 	"github.com/coder/coder/v2/agent/agentssh"
 	"github.com/coder/coder/v2/agent/agenttest"
 	"github.com/coder/coder/v2/agent/proto"
@@ -63,7 +58,7 @@ import (
 )
 
 func TestMain(m *testing.M) {
-	goleak.VerifyTestMain(m)
+	goleak.VerifyTestMain(m, testutil.GoleakOptions...)
 }
 
 // NOTE: These tests only work when your default shell is bash for some reason.
@@ -1507,7 +1502,7 @@ func TestAgent_Lifecycle(t *testing.T) {
 
 	t.Run("ShutdownScriptOnce", func(t *testing.T) {
 		t.Parallel()
-		logger := slogtest.Make(t, nil).Leveled(slog.LevelDebug)
+		logger := testutil.Logger(t)
 		expected := "this-is-shutdown"
 		derpMap, _ := tailnettest.RunDERPAndSTUN(t)
 
@@ -1517,10 +1512,12 @@ func TestAgent_Lifecycle(t *testing.T) {
 			agentsdk.Manifest{
 				DERPMap: derpMap,
 				Scripts: []codersdk.WorkspaceAgentScript{{
+					ID:         uuid.New(),
 					LogPath:    "coder-startup-script.log",
 					Script:     "echo 1",
 					RunOnStart: true,
 				}, {
+					ID:        uuid.New(),
 					LogPath:   "coder-shutdown-script.log",
 					Script:    "echo " + expected,
 					RunOnStop: true,
@@ -1812,20 +1809,45 @@ func TestAgent_Dial(t *testing.T) {
 
 			go func() {
 				defer close(done)
-				c, err := l.Accept()
-				if assert.NoError(t, err, "accept connection") {
-					defer c.Close()
-					testAccept(ctx, t, c)
+				for range 2 {
+					c, err := l.Accept()
+					if assert.NoError(t, err, "accept connection") {
+						testAccept(ctx, t, c)
+						_ = c.Close()
+					}
 				}
 			}()
 
+			agentID := uuid.UUID{0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8}
 			//nolint:dogsled
-			agentConn, _, _, _, _ := setupAgent(t, agentsdk.Manifest{}, 0)
+			agentConn, _, _, _, _ := setupAgent(t, agentsdk.Manifest{
+				AgentID: agentID,
+			}, 0)
 			require.True(t, agentConn.AwaitReachable(ctx))
 			conn, err := agentConn.DialContext(ctx, l.Addr().Network(), l.Addr().String())
 			require.NoError(t, err)
-			defer conn.Close()
 			testDial(ctx, t, conn)
+			err = conn.Close()
+			require.NoError(t, err)
+
+			// also connect via the CoderServicePrefix, to test that we can reach the agent on this
+			// IP. This will be required for CoderVPN.
+			_, rawPort, _ := net.SplitHostPort(l.Addr().String())
+			port, _ := strconv.ParseUint(rawPort, 10, 16)
+			ipp := netip.AddrPortFrom(tailnet.CoderServicePrefix.AddrFromUUID(agentID), uint16(port))
+
+			switch l.Addr().Network() {
+			case "tcp":
+				conn, err = agentConn.Conn.DialContextTCP(ctx, ipp)
+			case "udp":
+				conn, err = agentConn.Conn.DialContextUDP(ctx, ipp)
+			default:
+				t.Fatalf("unknown network: %s", l.Addr().Network())
+			}
+			require.NoError(t, err)
+			testDial(ctx, t, conn)
+			err = conn.Close()
+			require.NoError(t, err)
 		})
 	}
 }
@@ -1835,7 +1857,7 @@ func TestAgent_Dial(t *testing.T) {
 func TestAgent_UpdatedDERP(t *testing.T) {
 	t.Parallel()
 
-	logger := slogtest.Make(t, nil).Leveled(slog.LevelDebug)
+	logger := testutil.Logger(t)
 
 	originalDerpMap, _ := tailnettest.RunDERPAndSTUN(t)
 	require.NotNil(t, originalDerpMap)
@@ -1878,7 +1900,7 @@ func TestAgent_UpdatedDERP(t *testing.T) {
 	// Setup a client connection.
 	newClientConn := func(derpMap *tailcfg.DERPMap, name string) *workspacesdk.AgentConn {
 		conn, err := tailnet.NewConn(&tailnet.Options{
-			Addresses: []netip.Prefix{netip.PrefixFrom(tailnet.IP(), 128)},
+			Addresses: []netip.Prefix{tailnet.TailscaleServicePrefix.RandomPrefix()},
 			DERPMap:   derpMap,
 			Logger:    logger.Named(name),
 		})
@@ -1890,13 +1912,15 @@ func TestAgent_UpdatedDERP(t *testing.T) {
 		testCtx, testCtxCancel := context.WithCancel(context.Background())
 		t.Cleanup(testCtxCancel)
 		clientID := uuid.New()
-		coordination := tailnet.NewInMemoryCoordination(
-			testCtx, logger,
-			clientID, agentID,
-			coordinator, conn)
+		ctrl := tailnet.NewTunnelSrcCoordController(logger, conn)
+		ctrl.AddDestination(agentID)
+		auth := tailnet.ClientCoordinateeAuth{AgentID: agentID}
+		coordination := ctrl.New(tailnet.NewInMemoryCoordinatorClient(logger, clientID, auth, coordinator))
 		t.Cleanup(func() {
 			t.Logf("closing coordination %s", name)
-			err := coordination.Close()
+			cctx, ccancel := context.WithTimeout(testCtx, testutil.WaitShort)
+			defer ccancel()
+			err := coordination.Close(cctx)
 			if err != nil {
 				t.Logf("error closing in-memory coordination: %s", err.Error())
 			}
@@ -1989,7 +2013,7 @@ func TestAgent_Speedtest(t *testing.T) {
 
 func TestAgent_Reconnect(t *testing.T) {
 	t.Parallel()
-	logger := slogtest.Make(t, nil).Leveled(slog.LevelDebug)
+	logger := testutil.Logger(t)
 	// After the agent is disconnected from a coordinator, it's supposed
 	// to reconnect!
 	coordinator := tailnet.NewCoordinator(logger)
@@ -2030,7 +2054,7 @@ func TestAgent_Reconnect(t *testing.T) {
 
 func TestAgent_WriteVSCodeConfigs(t *testing.T) {
 	t.Parallel()
-	logger := slogtest.Make(t, nil).Leveled(slog.LevelDebug)
+	logger := testutil.Logger(t)
 	coordinator := tailnet.NewCoordinator(logger)
 	defer coordinator.Close()
 
@@ -2368,7 +2392,7 @@ func setupAgent(t *testing.T, metadata agentsdk.Manifest, ptyTimeout time.Durati
 		_ = agnt.Close()
 	})
 	conn, err := tailnet.NewConn(&tailnet.Options{
-		Addresses: []netip.Prefix{netip.PrefixFrom(tailnet.IP(), 128)},
+		Addresses: []netip.Prefix{netip.PrefixFrom(tailnet.TailscaleServicePrefix.RandomAddr(), 128)},
 		DERPMap:   metadata.DERPMap,
 		Logger:    logger.Named("client"),
 	})
@@ -2379,12 +2403,15 @@ func setupAgent(t *testing.T, metadata agentsdk.Manifest, ptyTimeout time.Durati
 	testCtx, testCtxCancel := context.WithCancel(context.Background())
 	t.Cleanup(testCtxCancel)
 	clientID := uuid.New()
-	coordination := tailnet.NewInMemoryCoordination(
-		testCtx, logger,
-		clientID, metadata.AgentID,
-		coordinator, conn)
+	ctrl := tailnet.NewTunnelSrcCoordController(logger, conn)
+	ctrl.AddDestination(metadata.AgentID)
+	auth := tailnet.ClientCoordinateeAuth{AgentID: metadata.AgentID}
+	coordination := ctrl.New(tailnet.NewInMemoryCoordinatorClient(
+		logger, clientID, auth, coordinator))
 	t.Cleanup(func() {
-		err := coordination.Close()
+		cctx, ccancel := context.WithTimeout(testCtx, testutil.WaitShort)
+		defer ccancel()
+		err := coordination.Close(cctx)
 		if err != nil {
 			t.Logf("error closing in-mem coordination: %s", err.Error())
 		}
@@ -2531,17 +2558,17 @@ func TestAgent_Metrics_SSH(t *testing.T) {
 	err = session.Shell()
 	require.NoError(t, err)
 
-	expected := []agentsdk.AgentMetric{
+	expected := []*proto.Stats_Metric{
 		{
 			Name:  "agent_reconnecting_pty_connections_total",
-			Type:  agentsdk.AgentMetricTypeCounter,
+			Type:  proto.Stats_Metric_COUNTER,
 			Value: 0,
 		},
 		{
 			Name:  "agent_sessions_total",
-			Type:  agentsdk.AgentMetricTypeCounter,
+			Type:  proto.Stats_Metric_COUNTER,
 			Value: 1,
-			Labels: []agentsdk.AgentMetricLabel{
+			Labels: []*proto.Stats_Metric_Label{
 				{
 					Name:  "magic_type",
 					Value: "ssh",
@@ -2554,30 +2581,46 @@ func TestAgent_Metrics_SSH(t *testing.T) {
 		},
 		{
 			Name:  "agent_ssh_server_failed_connections_total",
-			Type:  agentsdk.AgentMetricTypeCounter,
+			Type:  proto.Stats_Metric_COUNTER,
 			Value: 0,
 		},
 		{
 			Name:  "agent_ssh_server_sftp_connections_total",
-			Type:  agentsdk.AgentMetricTypeCounter,
+			Type:  proto.Stats_Metric_COUNTER,
 			Value: 0,
 		},
 		{
 			Name:  "agent_ssh_server_sftp_server_errors_total",
-			Type:  agentsdk.AgentMetricTypeCounter,
+			Type:  proto.Stats_Metric_COUNTER,
 			Value: 0,
 		},
 		{
-			Name:  "coderd_agentstats_startup_script_seconds",
-			Type:  agentsdk.AgentMetricTypeGauge,
+			Name:  "coderd_agentstats_currently_reachable_peers",
+			Type:  proto.Stats_Metric_GAUGE,
 			Value: 0,
-			Labels: []agentsdk.AgentMetricLabel{
+			Labels: []*proto.Stats_Metric_Label{
 				{
-					Name:  "success",
-					Value: "true",
+					Name:  "connection_type",
+					Value: "derp",
 				},
 			},
 		},
+		{
+			Name:  "coderd_agentstats_currently_reachable_peers",
+			Type:  proto.Stats_Metric_GAUGE,
+			Value: 1,
+			Labels: []*proto.Stats_Metric_Label{
+				{
+					Name:  "connection_type",
+					Value: "p2p",
+				},
+			},
+		},
+		{
+			Name:  "coderd_agentstats_startup_script_seconds",
+			Type:  proto.Stats_Metric_GAUGE,
+			Value: 1,
+		},
 	}
 
 	var actual []*promgo.MetricFamily
@@ -2586,281 +2629,39 @@ func TestAgent_Metrics_SSH(t *testing.T) {
 		if err != nil {
 			return false
 		}
-
-		if len(expected) != len(actual) {
-			return false
+		count := 0
+		for _, m := range actual {
+			count += len(m.GetMetric())
 		}
-
-		return verifyCollectedMetrics(t, expected, actual)
+		return count == len(expected)
 	}, testutil.WaitLong, testutil.IntervalFast)
 
-	require.Len(t, actual, len(expected))
-	collected := verifyCollectedMetrics(t, expected, actual)
-	require.True(t, collected, "expected metrics were not collected")
+	i := 0
+	for _, mf := range actual {
+		for _, m := range mf.GetMetric() {
+			assert.Equal(t, expected[i].Name, mf.GetName())
+			assert.Equal(t, expected[i].Type.String(), mf.GetType().String())
+			// Value is max expected
+			if expected[i].Type == proto.Stats_Metric_GAUGE {
+				assert.GreaterOrEqualf(t, expected[i].Value, m.GetGauge().GetValue(), "expected %s to be greater than or equal to %f, got %f", expected[i].Name, expected[i].Value, m.GetGauge().GetValue())
+			} else if expected[i].Type == proto.Stats_Metric_COUNTER {
+				assert.GreaterOrEqualf(t, expected[i].Value, m.GetCounter().GetValue(), "expected %s to be greater than or equal to %f, got %f", expected[i].Name, expected[i].Value, m.GetCounter().GetValue())
+			}
+			for j, lbl := range expected[i].Labels {
+				assert.Equal(t, m.GetLabel()[j], &promgo.LabelPair{
+					Name:  &lbl.Name,
+					Value: &lbl.Value,
+				})
+			}
+			i++
+		}
+	}
 
 	_ = stdin.Close()
 	err = session.Wait()
 	require.NoError(t, err)
 }
 
-func TestAgent_ManageProcessPriority(t *testing.T) {
-	t.Parallel()
-
-	t.Run("OK", func(t *testing.T) {
-		t.Parallel()
-
-		if runtime.GOOS != "linux" {
-			t.Skip("Skipping non-linux environment")
-		}
-
-		var (
-			expectedProcs = map[int32]agentproc.Process{}
-			fs            = afero.NewMemMapFs()
-			syscaller     = agentproctest.NewMockSyscaller(gomock.NewController(t))
-			ticker        = make(chan time.Time)
-			modProcs      = make(chan []*agentproc.Process)
-			logger        = slog.Make(sloghuman.Sink(io.Discard))
-		)
-
-		requireFileWrite(t, fs, "/proc/self/oom_score_adj", "-500")
-
-		// Create some processes.
-		for i := 0; i < 4; i++ {
-			// Create a prioritized process.
-			var proc agentproc.Process
-			if i == 0 {
-				proc = agentproctest.GenerateProcess(t, fs,
-					func(p *agentproc.Process) {
-						p.CmdLine = "./coder\x00agent\x00--no-reap"
-						p.PID = int32(i)
-					},
-				)
-			} else {
-				proc = agentproctest.GenerateProcess(t, fs,
-					func(p *agentproc.Process) {
-						// Make the cmd something similar to a prioritized
-						// process but differentiate the arguments.
-						p.CmdLine = "./coder\x00stat"
-					},
-				)
-
-				syscaller.EXPECT().GetPriority(proc.PID).Return(20, nil)
-				syscaller.EXPECT().SetPriority(proc.PID, 10).Return(nil)
-			}
-			syscaller.EXPECT().
-				Kill(proc.PID, syscall.Signal(0)).
-				Return(nil)
-
-			expectedProcs[proc.PID] = proc
-		}
-
-		_, _, _, _, _ = setupAgent(t, agentsdk.Manifest{}, 0, func(c *agenttest.Client, o *agent.Options) {
-			o.Syscaller = syscaller
-			o.ModifiedProcesses = modProcs
-			o.EnvironmentVariables = map[string]string{agent.EnvProcPrioMgmt: "1"}
-			o.Filesystem = fs
-			o.Logger = logger
-			o.ProcessManagementTick = ticker
-		})
-		actualProcs := <-modProcs
-		require.Len(t, actualProcs, len(expectedProcs)-1)
-		for _, proc := range actualProcs {
-			requireFileEquals(t, fs, fmt.Sprintf("/proc/%d/oom_score_adj", proc.PID), "0")
-		}
-	})
-
-	t.Run("IgnoreCustomNice", func(t *testing.T) {
-		t.Parallel()
-
-		if runtime.GOOS != "linux" {
-			t.Skip("Skipping non-linux environment")
-		}
-
-		var (
-			expectedProcs = map[int32]agentproc.Process{}
-			fs            = afero.NewMemMapFs()
-			ticker        = make(chan time.Time)
-			syscaller     = agentproctest.NewMockSyscaller(gomock.NewController(t))
-			modProcs      = make(chan []*agentproc.Process)
-			logger        = slog.Make(sloghuman.Sink(io.Discard))
-		)
-
-		err := afero.WriteFile(fs, "/proc/self/oom_score_adj", []byte("0"), 0o644)
-		require.NoError(t, err)
-
-		// Create some processes.
-		for i := 0; i < 3; i++ {
-			proc := agentproctest.GenerateProcess(t, fs)
-			syscaller.EXPECT().
-				Kill(proc.PID, syscall.Signal(0)).
-				Return(nil)
-
-			if i == 0 {
-				// Set a random nice score. This one should not be adjusted by
-				// our management loop.
-				syscaller.EXPECT().GetPriority(proc.PID).Return(25, nil)
-			} else {
-				syscaller.EXPECT().GetPriority(proc.PID).Return(20, nil)
-				syscaller.EXPECT().SetPriority(proc.PID, 10).Return(nil)
-			}
-
-			expectedProcs[proc.PID] = proc
-		}
-
-		_, _, _, _, _ = setupAgent(t, agentsdk.Manifest{}, 0, func(c *agenttest.Client, o *agent.Options) {
-			o.Syscaller = syscaller
-			o.ModifiedProcesses = modProcs
-			o.EnvironmentVariables = map[string]string{agent.EnvProcPrioMgmt: "1"}
-			o.Filesystem = fs
-			o.Logger = logger
-			o.ProcessManagementTick = ticker
-		})
-		actualProcs := <-modProcs
-		// We should ignore the process with a custom nice score.
-		require.Len(t, actualProcs, 2)
-		for _, proc := range actualProcs {
-			_, ok := expectedProcs[proc.PID]
-			require.True(t, ok)
-			requireFileEquals(t, fs, fmt.Sprintf("/proc/%d/oom_score_adj", proc.PID), "998")
-		}
-	})
-
-	t.Run("CustomOOMScore", func(t *testing.T) {
-		t.Parallel()
-
-		if runtime.GOOS != "linux" {
-			t.Skip("Skipping non-linux environment")
-		}
-
-		var (
-			fs        = afero.NewMemMapFs()
-			ticker    = make(chan time.Time)
-			syscaller = agentproctest.NewMockSyscaller(gomock.NewController(t))
-			modProcs  = make(chan []*agentproc.Process)
-			logger    = slog.Make(sloghuman.Sink(io.Discard))
-		)
-
-		err := afero.WriteFile(fs, "/proc/self/oom_score_adj", []byte("0"), 0o644)
-		require.NoError(t, err)
-
-		// Create some processes.
-		for i := 0; i < 3; i++ {
-			proc := agentproctest.GenerateProcess(t, fs)
-			syscaller.EXPECT().
-				Kill(proc.PID, syscall.Signal(0)).
-				Return(nil)
-			syscaller.EXPECT().GetPriority(proc.PID).Return(20, nil)
-			syscaller.EXPECT().SetPriority(proc.PID, 10).Return(nil)
-		}
-
-		_, _, _, _, _ = setupAgent(t, agentsdk.Manifest{}, 0, func(c *agenttest.Client, o *agent.Options) {
-			o.Syscaller = syscaller
-			o.ModifiedProcesses = modProcs
-			o.EnvironmentVariables = map[string]string{
-				agent.EnvProcPrioMgmt: "1",
-				agent.EnvProcOOMScore: "-567",
-			}
-			o.Filesystem = fs
-			o.Logger = logger
-			o.ProcessManagementTick = ticker
-		})
-		actualProcs := <-modProcs
-		// We should ignore the process with a custom nice score.
-		require.Len(t, actualProcs, 3)
-		for _, proc := range actualProcs {
-			requireFileEquals(t, fs, fmt.Sprintf("/proc/%d/oom_score_adj", proc.PID), "-567")
-		}
-	})
-
-	t.Run("DisabledByDefault", func(t *testing.T) {
-		t.Parallel()
-
-		if runtime.GOOS != "linux" {
-			t.Skip("Skipping non-linux environment")
-		}
-
-		var (
-			buf bytes.Buffer
-			wr  = &syncWriter{
-				w: &buf,
-			}
-		)
-		log := slog.Make(sloghuman.Sink(wr)).Leveled(slog.LevelDebug)
-
-		_, _, _, _, _ = setupAgent(t, agentsdk.Manifest{}, 0, func(c *agenttest.Client, o *agent.Options) {
-			o.Logger = log
-		})
-
-		require.Eventually(t, func() bool {
-			wr.mu.Lock()
-			defer wr.mu.Unlock()
-			return strings.Contains(buf.String(), "process priority not enabled")
-		}, testutil.WaitLong, testutil.IntervalFast)
-	})
-
-	t.Run("DisabledForNonLinux", func(t *testing.T) {
-		t.Parallel()
-
-		if runtime.GOOS == "linux" {
-			t.Skip("Skipping linux environment")
-		}
-
-		var (
-			buf bytes.Buffer
-			wr  = &syncWriter{
-				w: &buf,
-			}
-		)
-		log := slog.Make(sloghuman.Sink(wr)).Leveled(slog.LevelDebug)
-
-		_, _, _, _, _ = setupAgent(t, agentsdk.Manifest{}, 0, func(c *agenttest.Client, o *agent.Options) {
-			o.Logger = log
-			// Try to enable it so that we can assert that non-linux
-			// environments are truly disabled.
-			o.EnvironmentVariables = map[string]string{agent.EnvProcPrioMgmt: "1"}
-		})
-		require.Eventually(t, func() bool {
-			wr.mu.Lock()
-			defer wr.mu.Unlock()
-
-			return strings.Contains(buf.String(), "process priority not enabled")
-		}, testutil.WaitLong, testutil.IntervalFast)
-	})
-}
-
-func verifyCollectedMetrics(t *testing.T, expected []agentsdk.AgentMetric, actual []*promgo.MetricFamily) bool {
-	t.Helper()
-
-	for i, e := range expected {
-		assert.Equal(t, e.Name, actual[i].GetName())
-		assert.Equal(t, string(e.Type), strings.ToLower(actual[i].GetType().String()))
-
-		for _, m := range actual[i].GetMetric() {
-			assert.Equal(t, e.Value, m.Counter.GetValue())
-
-			if len(m.GetLabel()) > 0 {
-				for j, lbl := range m.GetLabel() {
-					assert.Equal(t, e.Labels[j].Name, lbl.GetName())
-					assert.Equal(t, e.Labels[j].Value, lbl.GetValue())
-				}
-			}
-			m.GetLabel()
-		}
-	}
-	return true
-}
-
-type syncWriter struct {
-	mu sync.Mutex
-	w  io.Writer
-}
-
-func (s *syncWriter) Write(p []byte) (int, error) {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-	return s.w.Write(p)
-}
-
 // echoOnce accepts a single connection, reads 4 bytes and echos them back
 func echoOnce(t *testing.T, ll net.Listener) {
 	t.Helper()
@@ -2890,17 +2691,3 @@ func requireEcho(t *testing.T, conn net.Conn) {
 	require.NoError(t, err)
 	require.Equal(t, "test", string(b))
 }
-
-func requireFileWrite(t testing.TB, fs afero.Fs, fp, data string) {
-	t.Helper()
-	err := afero.WriteFile(fs, fp, []byte(data), 0o600)
-	require.NoError(t, err)
-}
-
-func requireFileEquals(t testing.TB, fs afero.Fs, fp, expect string) {
-	t.Helper()
-	actual, err := afero.ReadFile(fs, fp)
-	require.NoError(t, err)
-
-	require.Equal(t, expect, string(actual))
-}

agent/agentexec/cli_linux.go

@@ -0,0 +1,202 @@
+//go:build linux
+// +build linux
+
+package agentexec
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"os/exec"
+	"runtime"
+	"slices"
+	"strconv"
+	"strings"
+	"syscall"
+
+	"golang.org/x/sys/unix"
+	"golang.org/x/xerrors"
+	"kernel.org/pub/linux/libs/security/libcap/cap"
+)
+
+// CLI runs the agent-exec command. It should only be called by the cli package.
+func CLI() error {
+	// We lock the OS thread here to avoid a race condition where the nice priority
+	// we set gets applied to a different thread than the one we exec the provided
+	// command on.
+	runtime.LockOSThread()
+	// Nop on success but we do it anyway in case of an error.
+	defer runtime.UnlockOSThread()
+
+	var (
+		fs   = flag.NewFlagSet("agent-exec", flag.ExitOnError)
+		nice = fs.Int("coder-nice", unset, "")
+		oom  = fs.Int("coder-oom", unset, "")
+	)
+
+	if len(os.Args) < 3 {
+		return xerrors.Errorf("malformed command %+v", os.Args)
+	}
+
+	// Parse everything after "coder agent-exec".
+	err := fs.Parse(os.Args[2:])
+	if err != nil {
+		return xerrors.Errorf("parse flags: %w", err)
+	}
+
+	// Get everything after "coder agent-exec --"
+	args := execArgs(os.Args)
+	if len(args) == 0 {
+		return xerrors.Errorf("no exec command provided %+v", os.Args)
+	}
+
+	if *oom == unset {
+		// If an explicit oom score isn't set, we use the default.
+		*oom, err = defaultOOMScore()
+		if err != nil {
+			return xerrors.Errorf("get default oom score: %w", err)
+		}
+	}
+
+	if *nice == unset {
+		// If an explicit nice score isn't set, we use the default.
+		*nice, err = defaultNiceScore()
+		if err != nil {
+			return xerrors.Errorf("get default nice score: %w", err)
+		}
+	}
+
+	// We drop effective caps prior to setting dumpable so that we limit the
+	// impact of someone attempting to hijack the process (i.e. with a debugger)
+	// to take advantage of the capabilities of the agent process. We encourage
+	// users to set cap_net_admin on the agent binary for improved networking
+	// performance and doing so results in the process having its SET_DUMPABLE
+	// attribute disabled (meaning we cannot adjust the oom score).
+	err = dropEffectiveCaps()
+	if err != nil {
+		printfStdErr("failed to drop effective caps: %v", err)
+	}
+
+	// Set dumpable to 1 so that we can adjust the oom score. If the process
+	// doesn't have capabilities or has an suid/sgid bit set, this is already
+	// set.
+	err = unix.Prctl(unix.PR_SET_DUMPABLE, 1, 0, 0, 0)
+	if err != nil {
+		printfStdErr("failed to set dumpable: %v", err)
+	}
+
+	err = writeOOMScoreAdj(*oom)
+	if err != nil {
+		// We alert the user instead of failing the command since it can be difficult to debug
+		// for a template admin otherwise. It's quite possible (and easy) to set an
+		// inappriopriate value for oom_score_adj.
+		printfStdErr("failed to adjust oom score to %d for cmd %+v: %v", *oom, execArgs(os.Args), err)
+	}
+
+	// Set dumpable back to 0 just to be safe. It's not inherited for execve anyways.
+	err = unix.Prctl(unix.PR_SET_DUMPABLE, 0, 0, 0, 0)
+	if err != nil {
+		printfStdErr("failed to unset dumpable: %v", err)
+	}
+
+	err = unix.Setpriority(unix.PRIO_PROCESS, 0, *nice)
+	if err != nil {
+		// We alert the user instead of failing the command since it can be difficult to debug
+		// for a template admin otherwise. It's quite possible (and easy) to set an
+		// inappriopriate value for niceness.
+		printfStdErr("failed to adjust niceness to %d for cmd %+v: %v", *nice, args, err)
+	}
+
+	path, err := exec.LookPath(args[0])
+	if err != nil {
+		return xerrors.Errorf("look path: %w", err)
+	}
+
+	// Remove environment variables specific to the agentexec command. This is
+	// especially important for environments that are attempting to develop Coder in Coder.
+	env := os.Environ()
+	env = slices.DeleteFunc(env, func(e string) bool {
+		return strings.HasPrefix(e, EnvProcPrioMgmt) ||
+			strings.HasPrefix(e, EnvProcOOMScore) ||
+			strings.HasPrefix(e, EnvProcNiceScore)
+	})
+
+	return syscall.Exec(path, args, env)
+}
+
+func defaultNiceScore() (int, error) {
+	score, err := unix.Getpriority(unix.PRIO_PROCESS, 0)
+	if err != nil {
+		return 0, xerrors.Errorf("get nice score: %w", err)
+	}
+	// See https://linux.die.net/man/2/setpriority#Notes
+	score = 20 - score
+
+	score += 5
+	if score > 19 {
+		return 19, nil
+	}
+	return score, nil
+}
+
+func defaultOOMScore() (int, error) {
+	score, err := oomScoreAdj()
+	if err != nil {
+		return 0, xerrors.Errorf("get oom score: %w", err)
+	}
+
+	// If the agent has a negative oom_score_adj, we set the child to 0
+	// so it's treated like every other process.
+	if score < 0 {
+		return 0, nil
+	}
+
+	// If the agent is already almost at the maximum then set it to the max.
+	if score >= 998 {
+		return 1000, nil
+	}
+
+	// If the agent oom_score_adj is >=0, we set the child to slightly
+	// less than the maximum. If users want a different score they set it
+	// directly.
+	return 998, nil
+}
+
+func oomScoreAdj() (int, error) {
+	scoreStr, err := os.ReadFile("/proc/self/oom_score_adj")
+	if err != nil {
+		return 0, xerrors.Errorf("read oom_score_adj: %w", err)
+	}
+	return strconv.Atoi(strings.TrimSpace(string(scoreStr)))
+}
+
+func writeOOMScoreAdj(score int) error {
+	return os.WriteFile(fmt.Sprintf("/proc/%d/oom_score_adj", os.Getpid()), []byte(fmt.Sprintf("%d", score)), 0o600)
+}
+
+// execArgs returns the arguments to pass to syscall.Exec after the "--" delimiter.
+func execArgs(args []string) []string {
+	for i, arg := range args {
+		if arg == "--" {
+			return args[i+1:]
+		}
+	}
+	return nil
+}
+
+func printfStdErr(format string, a ...any) {
+	_, _ = fmt.Fprintf(os.Stderr, "coder-agent: %s\n", fmt.Sprintf(format, a...))
+}
+
+func dropEffectiveCaps() error {
+	proc := cap.GetProc()
+	err := proc.ClearFlag(cap.Effective)
+	if err != nil {
+		return xerrors.Errorf("clear effective caps: %w", err)
+	}
+	err = proc.SetProc()
+	if err != nil {
+		return xerrors.Errorf("set proc: %w", err)
+	}
+	return nil
+}

agent/agentexec/cli_linux_test.go

@@ -0,0 +1,252 @@
+//go:build linux
+// +build linux
+
+package agentexec_test
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"slices"
+	"strconv"
+	"strings"
+	"syscall"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+	"golang.org/x/sys/unix"
+	"golang.org/x/xerrors"
+
+	"github.com/coder/coder/v2/agent/agentexec"
+	"github.com/coder/coder/v2/testutil"
+)
+
+//nolint:paralleltest // This test is sensitive to environment variables
+func TestCLI(t *testing.T) {
+	t.Run("OK", func(t *testing.T) {
+		ctx := testutil.Context(t, testutil.WaitMedium)
+		cmd, path := cmd(ctx, t, 123, 12)
+		err := cmd.Start()
+		require.NoError(t, err)
+		go cmd.Wait()
+
+		waitForSentinel(ctx, t, cmd, path)
+		requireOOMScore(t, cmd.Process.Pid, 123)
+		requireNiceScore(t, cmd.Process.Pid, 12)
+	})
+
+	t.Run("FiltersEnv", func(t *testing.T) {
+		ctx := testutil.Context(t, testutil.WaitMedium)
+		cmd, path := cmd(ctx, t, 123, 12)
+		cmd.Env = append(cmd.Env, fmt.Sprintf("%s=true", agentexec.EnvProcPrioMgmt))
+		cmd.Env = append(cmd.Env, fmt.Sprintf("%s=123", agentexec.EnvProcOOMScore))
+		cmd.Env = append(cmd.Env, fmt.Sprintf("%s=12", agentexec.EnvProcNiceScore))
+		// Ensure unrelated environment variables are preserved.
+		cmd.Env = append(cmd.Env, "CODER_TEST_ME_AGENTEXEC=true")
+		err := cmd.Start()
+		require.NoError(t, err)
+		go cmd.Wait()
+		waitForSentinel(ctx, t, cmd, path)
+
+		env := procEnv(t, cmd.Process.Pid)
+		hasExecEnvs := slices.ContainsFunc(
+			env,
+			func(e string) bool {
+				return strings.HasPrefix(e, agentexec.EnvProcPrioMgmt) ||
+					strings.HasPrefix(e, agentexec.EnvProcOOMScore) ||
+					strings.HasPrefix(e, agentexec.EnvProcNiceScore)
+			})
+		require.False(t, hasExecEnvs, "expected environment variables to be filtered")
+		userEnv := slices.Contains(env, "CODER_TEST_ME_AGENTEXEC=true")
+		require.True(t, userEnv, "expected user environment variables to be preserved")
+	})
+
+	t.Run("Defaults", func(t *testing.T) {
+		ctx := testutil.Context(t, testutil.WaitMedium)
+		cmd, path := cmd(ctx, t, 0, 0)
+		err := cmd.Start()
+		require.NoError(t, err)
+		go cmd.Wait()
+
+		waitForSentinel(ctx, t, cmd, path)
+
+		expectedNice := expectedNiceScore(t)
+		expectedOOM := expectedOOMScore(t)
+		requireOOMScore(t, cmd.Process.Pid, expectedOOM)
+		requireNiceScore(t, cmd.Process.Pid, expectedNice)
+	})
+
+	t.Run("Capabilities", func(t *testing.T) {
+		testdir := filepath.Dir(TestBin)
+		capDir := filepath.Join(testdir, "caps")
+		err := os.Mkdir(capDir, 0o755)
+		require.NoError(t, err)
+		bin := buildBinary(capDir)
+		// Try to set capabilities on the binary. This should work fine in CI but
+		// it's possible some developers may be working in an environment where they don't have the necessary permissions.
+		err = setCaps(t, bin, "cap_net_admin")
+		if os.Getenv("CI") != "" {
+			require.NoError(t, err)
+		} else if err != nil {
+			t.Skipf("unable to set capabilities for test: %v", err)
+		}
+		ctx := testutil.Context(t, testutil.WaitMedium)
+		cmd, path := binCmd(ctx, t, bin, 123, 12)
+		err = cmd.Start()
+		require.NoError(t, err)
+		go cmd.Wait()
+
+		waitForSentinel(ctx, t, cmd, path)
+		// This is what we're really testing, a binary with added capabilities requires setting dumpable.
+		requireOOMScore(t, cmd.Process.Pid, 123)
+		requireNiceScore(t, cmd.Process.Pid, 12)
+	})
+}
+
+func requireNiceScore(t *testing.T, pid int, score int) {
+	t.Helper()
+
+	nice, err := unix.Getpriority(unix.PRIO_PROCESS, pid)
+	require.NoError(t, err)
+	// See https://linux.die.net/man/2/setpriority#Notes
+	require.Equal(t, score, 20-nice)
+}
+
+func requireOOMScore(t *testing.T, pid int, expected int) {
+	t.Helper()
+
+	actual, err := os.ReadFile(fmt.Sprintf("/proc/%d/oom_score_adj", pid))
+	require.NoError(t, err)
+	score := strings.TrimSpace(string(actual))
+	require.Equal(t, strconv.Itoa(expected), score)
+}
+
+func waitForSentinel(ctx context.Context, t *testing.T, cmd *exec.Cmd, path string) {
+	t.Helper()
+
+	ticker := time.NewTicker(testutil.IntervalFast)
+	defer ticker.Stop()
+
+	// RequireEventually doesn't work well with require.NoError or similar require functions.
+	for {
+		err := cmd.Process.Signal(syscall.Signal(0))
+		require.NoError(t, err)
+
+		_, err = os.Stat(path)
+		if err == nil {
+			return
+		}
+
+		select {
+		case <-ticker.C:
+		case <-ctx.Done():
+			require.NoError(t, ctx.Err())
+		}
+	}
+}
+
+func binCmd(ctx context.Context, t *testing.T, bin string, oom, nice int) (*exec.Cmd, string) {
+	var (
+		args = execArgs(oom, nice)
+		dir  = t.TempDir()
+		file = filepath.Join(dir, "sentinel")
+	)
+
+	args = append(args, "sh", "-c", fmt.Sprintf("touch %s && sleep 10m", file))
+	//nolint:gosec
+	cmd := exec.CommandContext(ctx, bin, args...)
+
+	// We set this so we can also easily kill the sleep process the shell spawns.
+	cmd.SysProcAttr = &syscall.SysProcAttr{
+		Setpgid: true,
+	}
+
+	cmd.Env = os.Environ()
+	var buf bytes.Buffer
+	cmd.Stdout = &buf
+	cmd.Stderr = &buf
+	t.Cleanup(func() {
+		// Print output of a command if the test fails.
+		if t.Failed() {
+			t.Logf("cmd %q output: %s", cmd.Args, buf.String())
+		}
+		if cmd.Process != nil {
+			// We use -cmd.Process.Pid to kill the whole process group.
+			_ = syscall.Kill(-cmd.Process.Pid, syscall.SIGINT)
+		}
+	})
+	return cmd, file
+}
+
+func cmd(ctx context.Context, t *testing.T, oom, nice int) (*exec.Cmd, string) {
+	return binCmd(ctx, t, TestBin, oom, nice)
+}
+
+func expectedOOMScore(t *testing.T) int {
+	t.Helper()
+
+	score, err := os.ReadFile(fmt.Sprintf("/proc/%d/oom_score_adj", os.Getpid()))
+	require.NoError(t, err)
+
+	scoreInt, err := strconv.Atoi(strings.TrimSpace(string(score)))
+	require.NoError(t, err)
+
+	if scoreInt < 0 {
+		return 0
+	}
+	if scoreInt >= 998 {
+		return 1000
+	}
+	return 998
+}
+
+// procEnv returns the environment variables for a given process.
+func procEnv(t *testing.T, pid int) []string {
+	t.Helper()
+
+	env, err := os.ReadFile(fmt.Sprintf("/proc/%d/environ", pid))
+	require.NoError(t, err)
+	return strings.Split(string(env), "\x00")
+}
+
+func expectedNiceScore(t *testing.T) int {
+	t.Helper()
+
+	score, err := unix.Getpriority(unix.PRIO_PROCESS, os.Getpid())
+	require.NoError(t, err)
+
+	// Priority is niceness + 20.
+	score = 20 - score
+	score += 5
+	if score > 19 {
+		return 19
+	}
+	return score
+}
+
+func execArgs(oom int, nice int) []string {
+	execArgs := []string{"agent-exec"}
+	if oom != 0 {
+		execArgs = append(execArgs, fmt.Sprintf("--coder-oom=%d", oom))
+	}
+	if nice != 0 {
+		execArgs = append(execArgs, fmt.Sprintf("--coder-nice=%d", nice))
+	}
+	execArgs = append(execArgs, "--")
+	return execArgs
+}
+
+func setCaps(t *testing.T, bin string, caps ...string) error {
+	t.Helper()
+
+	setcap := fmt.Sprintf("sudo -n setcap %s=ep %s", strings.Join(caps, ", "), bin)
+	out, err := exec.CommandContext(context.Background(), "sh", "-c", setcap).CombinedOutput()
+	if err != nil {
+		return xerrors.Errorf("setcap %q (%s): %w", setcap, out, err)
+	}
+	return nil
+}

agent/agentexec/cli_other.go

@@ -0,0 +1,10 @@
+//go:build !linux
+// +build !linux
+
+package agentexec
+
+import "golang.org/x/xerrors"
+
+func CLI() error {
+	return xerrors.New("agent-exec is only supported on Linux")
+}

agent/agentexec/cmdtest/main_linux.go

@@ -0,0 +1,19 @@
+//go:build linux
+// +build linux
+
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/coder/coder/v2/agent/agentexec"
+)
+
+func main() {
+	err := agentexec.CLI()
+	if err != nil {
+		_, _ = fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+}

agent/agentexec/exec.go

@@ -0,0 +1,149 @@
+package agentexec
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strconv"
+
+	"golang.org/x/xerrors"
+
+	"github.com/coder/coder/v2/pty"
+)
+
+const (
+	// EnvProcPrioMgmt is the environment variable that determines whether
+	// we attempt to manage process CPU and OOM Killer priority.
+	EnvProcPrioMgmt  = "CODER_PROC_PRIO_MGMT"
+	EnvProcOOMScore  = "CODER_PROC_OOM_SCORE"
+	EnvProcNiceScore = "CODER_PROC_NICE_SCORE"
+
+	// unset is set to an invalid value for nice and oom scores.
+	unset = -2000
+)
+
+var DefaultExecer Execer = execer{}
+
+// Execer defines an abstraction for creating exec.Cmd variants. It's unfortunately
+// necessary because we need to be able to wrap child processes with "coder agent-exec"
+// for templates that expect the agent to manage process priority.
+type Execer interface {
+	// CommandContext returns an exec.Cmd that calls "coder agent-exec" prior to exec'ing
+	// the provided command if CODER_PROC_PRIO_MGMT is set, otherwise a normal exec.Cmd
+	// is returned. All instances of exec.Cmd should flow through this function to ensure
+	// proper resource constraints are applied to the child process.
+	CommandContext(ctx context.Context, cmd string, args ...string) *exec.Cmd
+	// PTYCommandContext returns an pty.Cmd that calls "coder agent-exec" prior to exec'ing
+	// the provided command if CODER_PROC_PRIO_MGMT is set, otherwise a normal pty.Cmd
+	// is returned. All instances of pty.Cmd should flow through this function to ensure
+	// proper resource constraints are applied to the child process.
+	PTYCommandContext(ctx context.Context, cmd string, args ...string) *pty.Cmd
+}
+
+func NewExecer() (Execer, error) {
+	_, enabled := os.LookupEnv(EnvProcPrioMgmt)
+	if runtime.GOOS != "linux" || !enabled {
+		return DefaultExecer, nil
+	}
+
+	executable, err := os.Executable()
+	if err != nil {
+		return nil, xerrors.Errorf("get executable: %w", err)
+	}
+
+	bin, err := filepath.EvalSymlinks(executable)
+	if err != nil {
+		return nil, xerrors.Errorf("eval symlinks: %w", err)
+	}
+
+	oomScore, ok := envValInt(EnvProcOOMScore)
+	if !ok {
+		oomScore = unset
+	}
+
+	niceScore, ok := envValInt(EnvProcNiceScore)
+	if !ok {
+		niceScore = unset
+	}
+
+	return priorityExecer{
+		binPath:   bin,
+		oomScore:  oomScore,
+		niceScore: niceScore,
+	}, nil
+}
+
+type execer struct{}
+
+func (execer) CommandContext(ctx context.Context, cmd string, args ...string) *exec.Cmd {
+	return exec.CommandContext(ctx, cmd, args...)
+}
+
+func (execer) PTYCommandContext(ctx context.Context, cmd string, args ...string) *pty.Cmd {
+	return pty.CommandContext(ctx, cmd, args...)
+}
+
+type priorityExecer struct {
+	binPath   string
+	oomScore  int
+	niceScore int
+}
+
+func (e priorityExecer) CommandContext(ctx context.Context, cmd string, args ...string) *exec.Cmd {
+	cmd, args = e.agentExecCmd(cmd, args...)
+	return exec.CommandContext(ctx, cmd, args...)
+}
+
+func (e priorityExecer) PTYCommandContext(ctx context.Context, cmd string, args ...string) *pty.Cmd {
+	cmd, args = e.agentExecCmd(cmd, args...)
+	return pty.CommandContext(ctx, cmd, args...)
+}
+
+func (e priorityExecer) agentExecCmd(cmd string, args ...string) (string, []string) {
+	execArgs := []string{"agent-exec"}
+	if e.oomScore != unset {
+		execArgs = append(execArgs, oomScoreArg(e.oomScore))
+	}
+
+	if e.niceScore != unset {
+		execArgs = append(execArgs, niceScoreArg(e.niceScore))
+	}
+	execArgs = append(execArgs, "--", cmd)
+	execArgs = append(execArgs, args...)
+
+	return e.binPath, execArgs
+}
+
+// envValInt searches for a key in a list of environment variables and parses it to an int.
+// If the key is not found or cannot be parsed, returns 0 and false.
+func envValInt(key string) (int, bool) {
+	val, ok := os.LookupEnv(key)
+	if !ok {
+		return 0, false
+	}
+
+	i, err := strconv.Atoi(val)
+	if err != nil {
+		return 0, false
+	}
+	return i, true
+}
+
+// The following are flags used by the agent-exec command. We use flags instead of
+// environment variables to avoid having to deal with a caller overriding the
+// environment variables.
+const (
+	niceFlag = "coder-nice"
+	oomFlag  = "coder-oom"
+)
+
+func niceScoreArg(score int) string {
+	return fmt.Sprintf("--%s=%d", niceFlag, score)
+}
+
+func oomScoreArg(score int) string {
+	return fmt.Sprintf("--%s=%d", oomFlag, score)
+}

agent/agentexec/exec_internal_test.go

@@ -0,0 +1,84 @@
+package agentexec
+
+import (
+	"context"
+	"os/exec"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestExecer(t *testing.T) {
+	t.Parallel()
+
+	t.Run("Default", func(t *testing.T) {
+		t.Parallel()
+
+		cmd := DefaultExecer.CommandContext(context.Background(), "sh", "-c", "sleep")
+
+		path, err := exec.LookPath("sh")
+		require.NoError(t, err)
+		require.Equal(t, path, cmd.Path)
+		require.Equal(t, []string{"sh", "-c", "sleep"}, cmd.Args)
+	})
+
+	t.Run("Priority", func(t *testing.T) {
+		t.Parallel()
+
+		t.Run("OK", func(t *testing.T) {
+			t.Parallel()
+
+			e := priorityExecer{
+				binPath:   "/foo/bar/baz",
+				oomScore:  unset,
+				niceScore: unset,
+			}
+
+			cmd := e.CommandContext(context.Background(), "sh", "-c", "sleep")
+			require.Equal(t, e.binPath, cmd.Path)
+			require.Equal(t, []string{e.binPath, "agent-exec", "--", "sh", "-c", "sleep"}, cmd.Args)
+		})
+
+		t.Run("Nice", func(t *testing.T) {
+			t.Parallel()
+
+			e := priorityExecer{
+				binPath:   "/foo/bar/baz",
+				oomScore:  unset,
+				niceScore: 10,
+			}
+
+			cmd := e.CommandContext(context.Background(), "sh", "-c", "sleep")
+			require.Equal(t, e.binPath, cmd.Path)
+			require.Equal(t, []string{e.binPath, "agent-exec", "--coder-nice=10", "--", "sh", "-c", "sleep"}, cmd.Args)
+		})
+
+		t.Run("OOM", func(t *testing.T) {
+			t.Parallel()
+
+			e := priorityExecer{
+				binPath:   "/foo/bar/baz",
+				oomScore:  123,
+				niceScore: unset,
+			}
+
+			cmd := e.CommandContext(context.Background(), "sh", "-c", "sleep")
+			require.Equal(t, e.binPath, cmd.Path)
+			require.Equal(t, []string{e.binPath, "agent-exec", "--coder-oom=123", "--", "sh", "-c", "sleep"}, cmd.Args)
+		})
+
+		t.Run("Both", func(t *testing.T) {
+			t.Parallel()
+
+			e := priorityExecer{
+				binPath:   "/foo/bar/baz",
+				oomScore:  432,
+				niceScore: 14,
+			}
+
+			cmd := e.CommandContext(context.Background(), "sh", "-c", "sleep")
+			require.Equal(t, e.binPath, cmd.Path)
+			require.Equal(t, []string{e.binPath, "agent-exec", "--coder-oom=432", "--coder-nice=14", "--", "sh", "-c", "sleep"}, cmd.Args)
+		})
+	})
+}

agent/agentexec/main_linux_test.go

@@ -0,0 +1,46 @@
+//go:build linux
+// +build linux
+
+package agentexec_test
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"testing"
+)
+
+var TestBin string
+
+func TestMain(m *testing.M) {
+	code := func() int {
+		// We generate a unique directory per test invocation to avoid collisions between two
+		// processes attempting to create the same temp file.
+		dir := genDir()
+		defer os.RemoveAll(dir)
+		TestBin = buildBinary(dir)
+		return m.Run()
+	}()
+
+	os.Exit(code)
+}
+
+func buildBinary(dir string) string {
+	path := filepath.Join(dir, "agent-test")
+	out, err := exec.Command("go", "build", "-o", path, "./cmdtest").CombinedOutput()
+	mustf(err, "build binary: %s", out)
+	return path
+}
+
+func mustf(err error, msg string, args ...any) {
+	if err != nil {
+		panic(fmt.Sprintf(msg, args...))
+	}
+}
+
+func genDir() string {
+	dir, err := os.MkdirTemp(os.TempDir(), "agentexec")
+	mustf(err, "create temp dir: %v", err)
+	return dir
+}

agent/agentproc/agentproctest/doc.go

@@ -1,5 +0,0 @@
-// Package agentproctest contains utility functions
-// for testing process management in the agent.
-package agentproctest
-
-//go:generate mockgen -destination ./syscallermock.go -package agentproctest github.com/coder/coder/v2/agent/agentproc Syscaller

agent/agentproc/agentproctest/proc.go

@@ -1,55 +0,0 @@
-package agentproctest
-
-import (
-	"fmt"
-	"strconv"
-	"testing"
-
-	"github.com/spf13/afero"
-	"github.com/stretchr/testify/require"
-
-	"github.com/coder/coder/v2/agent/agentproc"
-	"github.com/coder/coder/v2/cryptorand"
-)
-
-func GenerateProcess(t *testing.T, fs afero.Fs, muts ...func(*agentproc.Process)) agentproc.Process {
-	t.Helper()
-
-	pid, err := cryptorand.Intn(1<<31 - 1)
-	require.NoError(t, err)
-
-	arg1, err := cryptorand.String(5)
-	require.NoError(t, err)
-
-	arg2, err := cryptorand.String(5)
-	require.NoError(t, err)
-
-	arg3, err := cryptorand.String(5)
-	require.NoError(t, err)
-
-	cmdline := fmt.Sprintf("%s\x00%s\x00%s", arg1, arg2, arg3)
-
-	process := agentproc.Process{
-		CmdLine:     cmdline,
-		PID:         int32(pid),
-		OOMScoreAdj: 0,
-	}
-
-	for _, mut := range muts {
-		mut(&process)
-	}
-
-	process.Dir = fmt.Sprintf("%s/%d", "/proc", process.PID)
-
-	err = fs.MkdirAll(process.Dir, 0o555)
-	require.NoError(t, err)
-
-	err = afero.WriteFile(fs, fmt.Sprintf("%s/cmdline", process.Dir), []byte(process.CmdLine), 0o444)
-	require.NoError(t, err)
-
-	score := strconv.Itoa(process.OOMScoreAdj)
-	err = afero.WriteFile(fs, fmt.Sprintf("%s/oom_score_adj", process.Dir), []byte(score), 0o444)
-	require.NoError(t, err)
-
-	return process
-}

agent/agentproc/agentproctest/syscallermock.go

@@ -1,83 +0,0 @@
-// Code generated by MockGen. DO NOT EDIT.
-// Source: github.com/coder/coder/v2/agent/agentproc (interfaces: Syscaller)
-//
-// Generated by this command:
-//
-//	mockgen -destination ./syscallermock.go -package agentproctest github.com/coder/coder/v2/agent/agentproc Syscaller
-//
-
-// Package agentproctest is a generated GoMock package.
-package agentproctest
-
-import (
-	reflect "reflect"
-	syscall "syscall"
-
-	gomock "go.uber.org/mock/gomock"
-)
-
-// MockSyscaller is a mock of Syscaller interface.
-type MockSyscaller struct {
-	ctrl     *gomock.Controller
-	recorder *MockSyscallerMockRecorder
-}
-
-// MockSyscallerMockRecorder is the mock recorder for MockSyscaller.
-type MockSyscallerMockRecorder struct {
-	mock *MockSyscaller
-}
-
-// NewMockSyscaller creates a new mock instance.
-func NewMockSyscaller(ctrl *gomock.Controller) *MockSyscaller {
-	mock := &MockSyscaller{ctrl: ctrl}
-	mock.recorder = &MockSyscallerMockRecorder{mock}
-	return mock
-}
-
-// EXPECT returns an object that allows the caller to indicate expected use.
-func (m *MockSyscaller) EXPECT() *MockSyscallerMockRecorder {
-	return m.recorder
-}
-
-// GetPriority mocks base method.
-func (m *MockSyscaller) GetPriority(arg0 int32) (int, error) {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "GetPriority", arg0)
-	ret0, _ := ret[0].(int)
-	ret1, _ := ret[1].(error)
-	return ret0, ret1
-}
-
-// GetPriority indicates an expected call of GetPriority.
-func (mr *MockSyscallerMockRecorder) GetPriority(arg0 any) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPriority", reflect.TypeOf((*MockSyscaller)(nil).GetPriority), arg0)
-}
-
-// Kill mocks base method.
-func (m *MockSyscaller) Kill(arg0 int32, arg1 syscall.Signal) error {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "Kill", arg0, arg1)
-	ret0, _ := ret[0].(error)
-	return ret0
-}
-
-// Kill indicates an expected call of Kill.
-func (mr *MockSyscallerMockRecorder) Kill(arg0, arg1 any) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Kill", reflect.TypeOf((*MockSyscaller)(nil).Kill), arg0, arg1)
-}
-
-// SetPriority mocks base method.
-func (m *MockSyscaller) SetPriority(arg0 int32, arg1 int) error {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "SetPriority", arg0, arg1)
-	ret0, _ := ret[0].(error)
-	return ret0
-}
-
-// SetPriority indicates an expected call of SetPriority.
-func (mr *MockSyscallerMockRecorder) SetPriority(arg0, arg1 any) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetPriority", reflect.TypeOf((*MockSyscaller)(nil).SetPriority), arg0, arg1)
-}

agent/agentproc/doc.go

@@ -1,3 +0,0 @@
-// Package agentproc contains logic for interfacing with local
-// processes running in the same context as the agent.
-package agentproc

agent/agentproc/proc_other.go

@@ -1,24 +0,0 @@
-//go:build !linux
-// +build !linux
-
-package agentproc
-
-import (
-	"github.com/spf13/afero"
-)
-
-func (*Process) Niceness(Syscaller) (int, error) {
-	return 0, errUnimplemented
-}
-
-func (*Process) SetNiceness(Syscaller, int) error {
-	return errUnimplemented
-}
-
-func (*Process) Cmd() string {
-	return ""
-}
-
-func List(afero.Fs, Syscaller) ([]*Process, error) {
-	return nil, errUnimplemented
-}

agent/agentproc/proc_test.go

@@ -1,166 +0,0 @@
-package agentproc_test
-
-import (
-	"runtime"
-	"syscall"
-	"testing"
-
-	"github.com/spf13/afero"
-	"github.com/stretchr/testify/require"
-	"go.uber.org/mock/gomock"
-	"golang.org/x/xerrors"
-
-	"github.com/coder/coder/v2/agent/agentproc"
-	"github.com/coder/coder/v2/agent/agentproc/agentproctest"
-)
-
-func TestList(t *testing.T) {
-	t.Parallel()
-
-	if runtime.GOOS != "linux" {
-		t.Skipf("skipping non-linux environment")
-	}
-
-	t.Run("OK", func(t *testing.T) {
-		t.Parallel()
-
-		var (
-			fs            = afero.NewMemMapFs()
-			sc            = agentproctest.NewMockSyscaller(gomock.NewController(t))
-			expectedProcs = make(map[int32]agentproc.Process)
-		)
-
-		for i := 0; i < 4; i++ {
-			proc := agentproctest.GenerateProcess(t, fs)
-			expectedProcs[proc.PID] = proc
-
-			sc.EXPECT().
-				Kill(proc.PID, syscall.Signal(0)).
-				Return(nil)
-		}
-
-		actualProcs, err := agentproc.List(fs, sc)
-		require.NoError(t, err)
-		require.Len(t, actualProcs, len(expectedProcs))
-		for _, proc := range actualProcs {
-			expected, ok := expectedProcs[proc.PID]
-			require.True(t, ok)
-			require.Equal(t, expected.PID, proc.PID)
-			require.Equal(t, expected.CmdLine, proc.CmdLine)
-			require.Equal(t, expected.Dir, proc.Dir)
-		}
-	})
-
-	t.Run("FinishedProcess", func(t *testing.T) {
-		t.Parallel()
-
-		var (
-			fs            = afero.NewMemMapFs()
-			sc            = agentproctest.NewMockSyscaller(gomock.NewController(t))
-			expectedProcs = make(map[int32]agentproc.Process)
-		)
-
-		for i := 0; i < 3; i++ {
-			proc := agentproctest.GenerateProcess(t, fs)
-			expectedProcs[proc.PID] = proc
-
-			sc.EXPECT().
-				Kill(proc.PID, syscall.Signal(0)).
-				Return(nil)
-		}
-
-		// Create a process that's already finished. We're not adding
-		// it to the map because it should be skipped over.
-		proc := agentproctest.GenerateProcess(t, fs)
-		sc.EXPECT().
-			Kill(proc.PID, syscall.Signal(0)).
-			Return(xerrors.New("os: process already finished"))
-
-		actualProcs, err := agentproc.List(fs, sc)
-		require.NoError(t, err)
-		require.Len(t, actualProcs, len(expectedProcs))
-		for _, proc := range actualProcs {
-			expected, ok := expectedProcs[proc.PID]
-			require.True(t, ok)
-			require.Equal(t, expected.PID, proc.PID)
-			require.Equal(t, expected.CmdLine, proc.CmdLine)
-			require.Equal(t, expected.Dir, proc.Dir)
-		}
-	})
-
-	t.Run("NoSuchProcess", func(t *testing.T) {
-		t.Parallel()
-
-		var (
-			fs            = afero.NewMemMapFs()
-			sc            = agentproctest.NewMockSyscaller(gomock.NewController(t))
-			expectedProcs = make(map[int32]agentproc.Process)
-		)
-
-		for i := 0; i < 3; i++ {
-			proc := agentproctest.GenerateProcess(t, fs)
-			expectedProcs[proc.PID] = proc
-
-			sc.EXPECT().
-				Kill(proc.PID, syscall.Signal(0)).
-				Return(nil)
-		}
-
-		// Create a process that doesn't exist. We're not adding
-		// it to the map because it should be skipped over.
-		proc := agentproctest.GenerateProcess(t, fs)
-		sc.EXPECT().
-			Kill(proc.PID, syscall.Signal(0)).
-			Return(syscall.ESRCH)
-
-		actualProcs, err := agentproc.List(fs, sc)
-		require.NoError(t, err)
-		require.Len(t, actualProcs, len(expectedProcs))
-		for _, proc := range actualProcs {
-			expected, ok := expectedProcs[proc.PID]
-			require.True(t, ok)
-			require.Equal(t, expected.PID, proc.PID)
-			require.Equal(t, expected.CmdLine, proc.CmdLine)
-			require.Equal(t, expected.Dir, proc.Dir)
-		}
-	})
-}
-
-// These tests are not very interesting but they provide some modicum of
-// confidence.
-func TestProcess(t *testing.T) {
-	t.Parallel()
-
-	if runtime.GOOS != "linux" {
-		t.Skipf("skipping non-linux environment")
-	}
-
-	t.Run("SetNiceness", func(t *testing.T) {
-		t.Parallel()
-
-		var (
-			sc   = agentproctest.NewMockSyscaller(gomock.NewController(t))
-			proc = &agentproc.Process{
-				PID: 32,
-			}
-			score = 20
-		)
-
-		sc.EXPECT().SetPriority(proc.PID, score).Return(nil)
-		err := proc.SetNiceness(sc, score)
-		require.NoError(t, err)
-	})
-
-	t.Run("Cmd", func(t *testing.T) {
-		t.Parallel()
-
-		var (
-			proc = &agentproc.Process{
-				CmdLine: "helloworld\x00--arg1\x00--arg2",
-			}
-			expectedName = "helloworld --arg1 --arg2"
-		)
-
-		require.Equal(t, expectedName, proc.Cmd())
-	})
-}

agent/agentproc/proc_unix.go

@@ -1,126 +0,0 @@
-//go:build linux
-// +build linux
-
-package agentproc
-
-import (
-	"errors"
-	"os"
-	"path/filepath"
-	"strconv"
-	"strings"
-	"syscall"
-
-	"github.com/spf13/afero"
-	"golang.org/x/xerrors"
-)
-
-func List(fs afero.Fs, syscaller Syscaller) ([]*Process, error) {
-	d, err := fs.Open(defaultProcDir)
-	if err != nil {
-		return nil, xerrors.Errorf("open dir %q: %w", defaultProcDir, err)
-	}
-	defer d.Close()
-
-	entries, err := d.Readdirnames(0)
-	if err != nil {
-		return nil, xerrors.Errorf("readdirnames: %w", err)
-	}
-
-	processes := make([]*Process, 0, len(entries))
-	for _, entry := range entries {
-		pid, err := strconv.ParseInt(entry, 10, 32)
-		if err != nil {
-			continue
-		}
-
-		// Check that the process still exists.
-		exists, err := isProcessExist(syscaller, int32(pid))
-		if err != nil {
-			return nil, xerrors.Errorf("check process exists: %w", err)
-		}
-		if !exists {
-			continue
-		}
-
-		cmdline, err := afero.ReadFile(fs, filepath.Join(defaultProcDir, entry, "cmdline"))
-		if err != nil {
-			var errNo syscall.Errno
-			if xerrors.As(err, &errNo) && errNo == syscall.EPERM {
-				continue
-			}
-			return nil, xerrors.Errorf("read cmdline: %w", err)
-		}
-
-		oomScore, err := afero.ReadFile(fs, filepath.Join(defaultProcDir, entry, "oom_score_adj"))
-		if err != nil {
-			if xerrors.Is(err, os.ErrPermission) {
-				continue
-			}
-
-			return nil, xerrors.Errorf("read oom_score_adj: %w", err)
-		}
-
-		oom, err := strconv.Atoi(strings.TrimSpace(string(oomScore)))
-		if err != nil {
-			return nil, xerrors.Errorf("convert oom score: %w", err)
-		}
-
-		processes = append(processes, &Process{
-			PID:         int32(pid),
-			CmdLine:     string(cmdline),
-			Dir:         filepath.Join(defaultProcDir, entry),
-			OOMScoreAdj: oom,
-		})
-	}
-
-	return processes, nil
-}
-
-func isProcessExist(syscaller Syscaller, pid int32) (bool, error) {
-	err := syscaller.Kill(pid, syscall.Signal(0))
-	if err == nil {
-		return true, nil
-	}
-	if err.Error() == "os: process already finished" {
-		return false, nil
-	}
-
-	var errno syscall.Errno
-	if !errors.As(err, &errno) {
-		return false, err
-	}
-
-	switch errno {
-	case syscall.ESRCH:
-		return false, nil
-	case syscall.EPERM:
-		return true, nil
-	}
-
-	return false, xerrors.Errorf("kill: %w", err)
-}
-
-func (p *Process) Niceness(sc Syscaller) (int, error) {
-	nice, err := sc.GetPriority(p.PID)
-	if err != nil {
-		return 0, xerrors.Errorf("get priority for %q: %w", p.CmdLine, err)
-	}
-	return nice, nil
-}
-
-func (p *Process) SetNiceness(sc Syscaller, score int) error {
-	err := sc.SetPriority(p.PID, score)
-	if err != nil {
-		return xerrors.Errorf("set priority for %q: %w", p.CmdLine, err)
-	}
-	return nil
-}
-
-func (p *Process) Cmd() string {
-	return strings.Join(p.cmdLine(), " ")
-}
-
-func (p *Process) cmdLine() []string {
-	return strings.Split(p.CmdLine, "\x00")
-}

agent/agentproc/syscaller.go

@@ -1,21 +0,0 @@
-package agentproc
-
-import (
-	"syscall"
-)
-
-type Syscaller interface {
-	SetPriority(pid int32, priority int) error
-	GetPriority(pid int32) (int, error)
-	Kill(pid int32, sig syscall.Signal) error
-}
-
-// nolint: unused // used on some but no all platforms
-const defaultProcDir = "/proc"
-
-type Process struct {
-	Dir         string
-	CmdLine     string
-	PID         int32
-	OOMScoreAdj int
-}

agent/agentproc/syscaller_other.go

@@ -1,30 +0,0 @@
-//go:build !linux
-// +build !linux
-
-package agentproc
-
-import (
-	"syscall"
-
-	"golang.org/x/xerrors"
-)
-
-func NewSyscaller() Syscaller {
-	return nopSyscaller{}
-}
-
-var errUnimplemented = xerrors.New("unimplemented")
-
-type nopSyscaller struct{}
-
-func (nopSyscaller) SetPriority(int32, int) error {
-	return errUnimplemented
-}
-
-func (nopSyscaller) GetPriority(int32) (int, error) {
-	return 0, errUnimplemented
-}
-
-func (nopSyscaller) Kill(int32, syscall.Signal) error {
-	return errUnimplemented
-}

agent/agentproc/syscaller_unix.go

@@ -1,42 +0,0 @@
-//go:build linux
-// +build linux
-
-package agentproc
-
-import (
-	"syscall"
-
-	"golang.org/x/sys/unix"
-	"golang.org/x/xerrors"
-)
-
-func NewSyscaller() Syscaller {
-	return UnixSyscaller{}
-}
-
-type UnixSyscaller struct{}
-
-func (UnixSyscaller) SetPriority(pid int32, nice int) error {
-	err := unix.Setpriority(unix.PRIO_PROCESS, int(pid), nice)
-	if err != nil {
-		return xerrors.Errorf("set priority: %w", err)
-	}
-	return nil
-}
-
-func (UnixSyscaller) GetPriority(pid int32) (int, error) {
-	nice, err := unix.Getpriority(0, int(pid))
-	if err != nil {
-		return 0, xerrors.Errorf("get priority: %w", err)
-	}
-	return nice, nil
-}
-
-func (UnixSyscaller) Kill(pid int32, sig syscall.Signal) error {
-	err := syscall.Kill(int(pid), sig)
-	if err != nil {
-		return xerrors.Errorf("kill: %w", err)
-	}
-
-	return nil
-}

agent/agentscripts/agentscripts.go

@@ -19,10 +19,13 @@ import (
 	"github.com/spf13/afero"
 	"golang.org/x/sync/errgroup"
 	"golang.org/x/xerrors"
+	"google.golang.org/protobuf/types/known/timestamppb"
 
 	"cdr.dev/slog"
 
 	"github.com/coder/coder/v2/agent/agentssh"
+	"github.com/coder/coder/v2/agent/proto"
+	"github.com/coder/coder/v2/coderd/database/dbtime"
 	"github.com/coder/coder/v2/codersdk"
 	"github.com/coder/coder/v2/codersdk/agentsdk"
 )
@@ -75,18 +78,21 @@ func New(opts Options) *Runner {
 	}
 }
 
+type ScriptCompletedFunc func(context.Context, *proto.WorkspaceAgentScriptCompletedRequest) (*proto.WorkspaceAgentScriptCompletedResponse, error)
+
 type Runner struct {
 	Options
 
-	cronCtx       context.Context
-	cronCtxCancel context.CancelFunc
-	cmdCloseWait  sync.WaitGroup
-	closed        chan struct{}
-	closeMutex    sync.Mutex
-	cron          *cron.Cron
-	initialized   atomic.Bool
-	scripts       []codersdk.WorkspaceAgentScript
-	dataDir       string
+	cronCtx         context.Context
+	cronCtxCancel   context.CancelFunc
+	cmdCloseWait    sync.WaitGroup
+	closed          chan struct{}
+	closeMutex      sync.Mutex
+	cron            *cron.Cron
+	initialized     atomic.Bool
+	scripts         []codersdk.WorkspaceAgentScript
+	dataDir         string
+	scriptCompleted ScriptCompletedFunc
 
 	// scriptsExecuted includes all scripts executed by the workspace agent. Agents
 	// execute startup scripts, and scripts on a cron schedule. Both will increment
@@ -116,12 +122,13 @@ func (r *Runner) RegisterMetrics(reg prometheus.Registerer) {
 // Init initializes the runner with the provided scripts.
 // It also schedules any scripts that have a schedule.
 // This function must be called before Execute.
-func (r *Runner) Init(scripts []codersdk.WorkspaceAgentScript) error {
+func (r *Runner) Init(scripts []codersdk.WorkspaceAgentScript, scriptCompleted ScriptCompletedFunc) error {
 	if r.initialized.Load() {
 		return xerrors.New("init: already initialized")
 	}
 	r.initialized.Store(true)
 	r.scripts = scripts
+	r.scriptCompleted = scriptCompleted
 	r.Logger.Info(r.cronCtx, "initializing agent scripts", slog.F("script_count", len(scripts)), slog.F("log_dir", r.LogDir))
 
 	err := r.Filesystem.MkdirAll(r.ScriptBinDir(), 0o700)
@@ -135,7 +142,7 @@ func (r *Runner) Init(scripts []codersdk.WorkspaceAgentScript) error {
 		}
 		script := script
 		_, err := r.cron.AddFunc(script.Cron, func() {
-			err := r.trackRun(r.cronCtx, script)
+			err := r.trackRun(r.cronCtx, script, ExecuteCronScripts)
 			if err != nil {
 				r.Logger.Warn(context.Background(), "run agent script on schedule", slog.Error(err))
 			}
@@ -172,22 +179,33 @@ func (r *Runner) StartCron() {
 	}
 }
 
+// ExecuteOption describes what scripts we want to execute.
+type ExecuteOption int
+
+// ExecuteOption enums.
+const (
+	ExecuteAllScripts ExecuteOption = iota
+	ExecuteStartScripts
+	ExecuteStopScripts
+	ExecuteCronScripts
+)
+
 // Execute runs a set of scripts according to a filter.
-func (r *Runner) Execute(ctx context.Context, filter func(script codersdk.WorkspaceAgentScript) bool) error {
-	if filter == nil {
-		// Execute em' all!
-		filter = func(script codersdk.WorkspaceAgentScript) bool {
-			return true
-		}
-	}
+func (r *Runner) Execute(ctx context.Context, option ExecuteOption) error {
 	var eg errgroup.Group
 	for _, script := range r.scripts {
-		if !filter(script) {
+		runScript := (option == ExecuteStartScripts && script.RunOnStart) ||
+			(option == ExecuteStopScripts && script.RunOnStop) ||
+			(option == ExecuteCronScripts && script.Cron != "") ||
+			option == ExecuteAllScripts
+
+		if !runScript {
 			continue
 		}
+
 		script := script
 		eg.Go(func() error {
-			err := r.trackRun(ctx, script)
+			err := r.trackRun(ctx, script, option)
 			if err != nil {
 				return xerrors.Errorf("run agent script %q: %w", script.LogSourceID, err)
 			}
@@ -198,8 +216,8 @@ func (r *Runner) Execute(ctx context.Context, filter func(script codersdk.Worksp
 }
 
 // trackRun wraps "run" with metrics.
-func (r *Runner) trackRun(ctx context.Context, script codersdk.WorkspaceAgentScript) error {
-	err := r.run(ctx, script)
+func (r *Runner) trackRun(ctx context.Context, script codersdk.WorkspaceAgentScript, option ExecuteOption) error {
+	err := r.run(ctx, script, option)
 	if err != nil {
 		r.scriptsExecuted.WithLabelValues("false").Add(1)
 	} else {
@@ -212,7 +230,7 @@ func (r *Runner) trackRun(ctx context.Context, script codersdk.WorkspaceAgentScr
 // If the timeout is exceeded, the process is sent an interrupt signal.
 // If the process does not exit after a few seconds, it is forcefully killed.
 // This function immediately returns after a timeout, and does not wait for the process to exit.
-func (r *Runner) run(ctx context.Context, script codersdk.WorkspaceAgentScript) error {
+func (r *Runner) run(ctx context.Context, script codersdk.WorkspaceAgentScript, option ExecuteOption) error {
 	logPath := script.LogPath
 	if logPath == "" {
 		logPath = fmt.Sprintf("coder-script-%s.log", script.LogSourceID)
@@ -299,9 +317,9 @@ func (r *Runner) run(ctx context.Context, script codersdk.WorkspaceAgentScript)
 	cmd.Stdout = io.MultiWriter(fileWriter, infoW)
 	cmd.Stderr = io.MultiWriter(fileWriter, errW)
 
-	start := time.Now()
+	start := dbtime.Now()
 	defer func() {
-		end := time.Now()
+		end := dbtime.Now()
 		execTime := end.Sub(start)
 		exitCode := 0
 		if err != nil {
@@ -314,6 +332,60 @@ func (r *Runner) run(ctx context.Context, script codersdk.WorkspaceAgentScript)
 		} else {
 			logger.Info(ctx, fmt.Sprintf("%s script completed", logPath), slog.F("execution_time", execTime), slog.F("exit_code", exitCode))
 		}
+
+		if r.scriptCompleted == nil {
+			logger.Debug(ctx, "r.scriptCompleted unexpectedly nil")
+			return
+		}
+
+		// We want to check this outside of the goroutine to avoid a race condition
+		timedOut := errors.Is(err, ErrTimeout)
+		pipesLeftOpen := errors.Is(err, ErrOutputPipesOpen)
+
+		err = r.trackCommandGoroutine(func() {
+			var stage proto.Timing_Stage
+			switch option {
+			case ExecuteStartScripts:
+				stage = proto.Timing_START
+			case ExecuteStopScripts:
+				stage = proto.Timing_STOP
+			case ExecuteCronScripts:
+				stage = proto.Timing_CRON
+			}
+
+			var status proto.Timing_Status
+			switch {
+			case timedOut:
+				status = proto.Timing_TIMED_OUT
+			case pipesLeftOpen:
+				status = proto.Timing_PIPES_LEFT_OPEN
+			case exitCode != 0:
+				status = proto.Timing_EXIT_FAILURE
+			default:
+				status = proto.Timing_OK
+			}
+
+			reportTimeout := 30 * time.Second
+			reportCtx, cancel := context.WithTimeout(context.Background(), reportTimeout)
+			defer cancel()
+
+			_, err := r.scriptCompleted(reportCtx, &proto.WorkspaceAgentScriptCompletedRequest{
+				Timing: &proto.Timing{
+					ScriptId: script.ID[:],
+					Start:    timestamppb.New(start),
+					End:      timestamppb.New(end),
+					ExitCode: int32(exitCode),
+					Stage:    stage,
+					Status:   status,
+				},
+			})
+			if err != nil {
+				logger.Error(ctx, fmt.Sprintf("reporting script completed: %s", err.Error()))
+			}
+		})
+		if err != nil {
+			logger.Error(ctx, fmt.Sprintf("reporting script completed: track command goroutine: %s", err.Error()))
+		}
 	}()
 
 	err = cmd.Start()
@@ -349,7 +421,7 @@ func (r *Runner) run(ctx context.Context, script codersdk.WorkspaceAgentScript)
 			"This usually means a child process was started with references to stdout or stderr. As a result, this " +
 				"process may now have been terminated. Consider redirecting the output or using a separate " +
 				"\"coder_script\" for the process, see " +
-				"https://coder.com/docs/v2/latest/templates/troubleshooting#startup-script-issues for more information.",
+				"https://coder.com/docs/templates/troubleshooting#startup-script-issues for more information.",
 		)
 		// Inform the user by propagating the message via log writers.
 		_, _ = fmt.Fprintf(cmd.Stderr, "WARNING: %s. %s\n", message, details)

agent/agentscripts/agentscripts_test.go

@@ -14,16 +14,17 @@ import (
 	"github.com/stretchr/testify/require"
 	"go.uber.org/goleak"
 
-	"cdr.dev/slog/sloggers/slogtest"
+	"github.com/coder/coder/v2/agent/agentexec"
 	"github.com/coder/coder/v2/agent/agentscripts"
 	"github.com/coder/coder/v2/agent/agentssh"
+	"github.com/coder/coder/v2/agent/agenttest"
 	"github.com/coder/coder/v2/codersdk"
 	"github.com/coder/coder/v2/codersdk/agentsdk"
 	"github.com/coder/coder/v2/testutil"
 )
 
 func TestMain(m *testing.M) {
-	goleak.VerifyTestMain(m)
+	goleak.VerifyTestMain(m, testutil.GoleakOptions...)
 }
 
 func TestExecuteBasic(t *testing.T) {
@@ -34,14 +35,13 @@ func TestExecuteBasic(t *testing.T) {
 		return fLogger
 	})
 	defer runner.Close()
+	aAPI := agenttest.NewFakeAgentAPI(t, testutil.Logger(t), nil, nil)
 	err := runner.Init([]codersdk.WorkspaceAgentScript{{
 		LogSourceID: uuid.New(),
 		Script:      "echo hello",
-	}})
+	}}, aAPI.ScriptCompleted)
 	require.NoError(t, err)
-	require.NoError(t, runner.Execute(context.Background(), func(script codersdk.WorkspaceAgentScript) bool {
-		return true
-	}))
+	require.NoError(t, runner.Execute(context.Background(), agentscripts.ExecuteAllScripts))
 	log := testutil.RequireRecvCtx(ctx, t, fLogger.logs)
 	require.Equal(t, "hello", log.Output)
 }
@@ -61,18 +61,17 @@ func TestEnv(t *testing.T) {
 			cmd.exe /c echo %CODER_SCRIPT_BIN_DIR%
 		`
 	}
+	aAPI := agenttest.NewFakeAgentAPI(t, testutil.Logger(t), nil, nil)
 	err := runner.Init([]codersdk.WorkspaceAgentScript{{
 		LogSourceID: id,
 		Script:      script,
-	}})
+	}}, aAPI.ScriptCompleted)
 	require.NoError(t, err)
 
 	ctx := testutil.Context(t, testutil.WaitLong)
 
 	done := testutil.Go(t, func() {
-		err := runner.Execute(ctx, func(script codersdk.WorkspaceAgentScript) bool {
-			return true
-		})
+		err := runner.Execute(ctx, agentscripts.ExecuteAllScripts)
 		assert.NoError(t, err)
 	})
 	defer func() {
@@ -103,13 +102,44 @@ func TestTimeout(t *testing.T) {
 	t.Parallel()
 	runner := setup(t, nil)
 	defer runner.Close()
+	aAPI := agenttest.NewFakeAgentAPI(t, testutil.Logger(t), nil, nil)
 	err := runner.Init([]codersdk.WorkspaceAgentScript{{
 		LogSourceID: uuid.New(),
 		Script:      "sleep infinity",
 		Timeout:     time.Millisecond,
-	}})
+	}}, aAPI.ScriptCompleted)
 	require.NoError(t, err)
-	require.ErrorIs(t, runner.Execute(context.Background(), nil), agentscripts.ErrTimeout)
+	require.ErrorIs(t, runner.Execute(context.Background(), agentscripts.ExecuteAllScripts), agentscripts.ErrTimeout)
+}
+
+func TestScriptReportsTiming(t *testing.T) {
+	t.Parallel()
+
+	ctx := testutil.Context(t, testutil.WaitShort)
+	fLogger := newFakeScriptLogger()
+	runner := setup(t, func(uuid2 uuid.UUID) agentscripts.ScriptLogger {
+		return fLogger
+	})
+
+	aAPI := agenttest.NewFakeAgentAPI(t, testutil.Logger(t), nil, nil)
+	err := runner.Init([]codersdk.WorkspaceAgentScript{{
+		DisplayName: "say-hello",
+		LogSourceID: uuid.New(),
+		Script:      "echo hello",
+	}}, aAPI.ScriptCompleted)
+	require.NoError(t, err)
+	require.NoError(t, runner.Execute(ctx, agentscripts.ExecuteAllScripts))
+	runner.Close()
+
+	log := testutil.RequireRecvCtx(ctx, t, fLogger.logs)
+	require.Equal(t, "hello", log.Output)
+
+	timings := aAPI.GetTimings()
+	require.Equal(t, 1, len(timings))
+
+	timing := timings[0]
+	require.Equal(t, int32(0), timing.ExitCode)
+	require.GreaterOrEqual(t, timing.End.AsTime(), timing.Start.AsTime())
 }
 
 // TestCronClose exists because cron.Run() can happen after cron.Close().
@@ -130,8 +160,8 @@ func setup(t *testing.T, getScriptLogger func(logSourceID uuid.UUID) agentscript
 		}
 	}
 	fs := afero.NewMemMapFs()
-	logger := slogtest.Make(t, nil)
-	s, err := agentssh.NewServer(context.Background(), logger, prometheus.NewRegistry(), fs, nil)
+	logger := testutil.Logger(t)
+	s, err := agentssh.NewServer(context.Background(), logger, prometheus.NewRegistry(), fs, agentexec.DefaultExecer, nil)
 	require.NoError(t, err)
 	t.Cleanup(func() {
 		_ = s.Close()

agent/agentssh/agentssh.go

@@ -30,6 +30,7 @@ import (
 
 	"cdr.dev/slog"
 
+	"github.com/coder/coder/v2/agent/agentexec"
 	"github.com/coder/coder/v2/agent/usershell"
 	"github.com/coder/coder/v2/codersdk"
 	"github.com/coder/coder/v2/pty"
@@ -79,9 +80,9 @@ type Config struct {
 	// where users will land when they connect via SSH. Default is the home
 	// directory of the user.
 	WorkingDirectory func() string
-	// X11SocketDir is the directory where X11 sockets are created. Default is
-	// /tmp/.X11-unix.
-	X11SocketDir string
+	// X11DisplayOffset is the offset to add to the X11 display number.
+	// Default is 10.
+	X11DisplayOffset *int
 	// BlockFileTransfer restricts use of file transfer applications.
 	BlockFileTransfer bool
 }
@@ -97,6 +98,7 @@ type Server struct {
 	// a lock on mu but protected by closing.
 	wg sync.WaitGroup
 
+	Execer agentexec.Execer
 	logger slog.Logger
 	srv    *ssh.Server
 
@@ -109,7 +111,7 @@ type Server struct {
 	metrics *sshServerMetrics
 }
 
-func NewServer(ctx context.Context, logger slog.Logger, prometheusRegistry *prometheus.Registry, fs afero.Fs, config *Config) (*Server, error) {
+func NewServer(ctx context.Context, logger slog.Logger, prometheusRegistry *prometheus.Registry, fs afero.Fs, execer agentexec.Execer, config *Config) (*Server, error) {
 	// Clients' should ignore the host key when connecting.
 	// The agent needs to authenticate with coderd to SSH,
 	// so SSH authentication doesn't improve security.
@@ -124,8 +126,9 @@ func NewServer(ctx context.Context, logger slog.Logger, prometheusRegistry *prom
 	if config == nil {
 		config = &Config{}
 	}
-	if config.X11SocketDir == "" {
-		config.X11SocketDir = filepath.Join(os.TempDir(), ".X11-unix")
+	if config.X11DisplayOffset == nil {
+		offset := X11DefaultDisplayOffset
+		config.X11DisplayOffset = &offset
 	}
 	if config.UpdateEnv == nil {
 		config.UpdateEnv = func(current []string) ([]string, error) { return current, nil }
@@ -151,6 +154,7 @@ func NewServer(ctx context.Context, logger slog.Logger, prometheusRegistry *prom
 
 	metrics := newSSHServerMetrics(prometheusRegistry)
 	s := &Server{
+		Execer:    execer,
 		listeners: make(map[net.Listener]struct{}),
 		fs:        fs,
 		conns:     make(map[net.Conn]struct{}),
@@ -273,13 +277,13 @@ func (s *Server) sessionHandler(session ssh.Session) {
 	extraEnv := make([]string, 0)
 	x11, hasX11 := session.X11()
 	if hasX11 {
-		handled := s.x11Handler(session.Context(), x11)
+		display, handled := s.x11Handler(session.Context(), x11)
 		if !handled {
 			_ = session.Exit(1)
 			logger.Error(ctx, "x11 handler failed")
 			return
 		}
-		extraEnv = append(extraEnv, fmt.Sprintf("DISPLAY=:%d.0", x11.ScreenNumber))
+		extraEnv = append(extraEnv, fmt.Sprintf("DISPLAY=localhost:%d.%d", display, x11.ScreenNumber))
 	}
 
 	if s.fileTransferBlocked(session) {
@@ -724,7 +728,7 @@ func (s *Server) CreateCommand(ctx context.Context, script string, env []string)
 		}
 	}
 
-	cmd := pty.CommandContext(ctx, name, args...)
+	cmd := s.Execer.PTYCommandContext(ctx, name, args...)
 	cmd.Dir = s.config.WorkingDirectory()
 
 	// If the metadata directory doesn't exist, we run the command

agent/agentssh/agentssh_internal_test.go

@@ -15,10 +15,9 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
+	"github.com/coder/coder/v2/agent/agentexec"
 	"github.com/coder/coder/v2/pty"
 	"github.com/coder/coder/v2/testutil"
-
-	"cdr.dev/slog/sloggers/slogtest"
 )
 
 const longScript = `
@@ -36,8 +35,8 @@ func Test_sessionStart_orphan(t *testing.T) {
 
 	ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitMedium)
 	defer cancel()
-	logger := slogtest.Make(t, nil)
-	s, err := NewServer(ctx, logger, prometheus.NewRegistry(), afero.NewMemMapFs(), nil)
+	logger := testutil.Logger(t)
+	s, err := NewServer(ctx, logger, prometheus.NewRegistry(), afero.NewMemMapFs(), agentexec.DefaultExecer, nil)
 	require.NoError(t, err)
 	defer s.Close()
 

agent/agentssh/agentssh_test.go

@@ -22,21 +22,22 @@ import (
 
 	"cdr.dev/slog/sloggers/slogtest"
 
+	"github.com/coder/coder/v2/agent/agentexec"
 	"github.com/coder/coder/v2/agent/agentssh"
 	"github.com/coder/coder/v2/pty/ptytest"
 	"github.com/coder/coder/v2/testutil"
 )
 
 func TestMain(m *testing.M) {
-	goleak.VerifyTestMain(m)
+	goleak.VerifyTestMain(m, testutil.GoleakOptions...)
 }
 
 func TestNewServer_ServeClient(t *testing.T) {
 	t.Parallel()
 
 	ctx := context.Background()
-	logger := slogtest.Make(t, nil)
-	s, err := agentssh.NewServer(ctx, logger, prometheus.NewRegistry(), afero.NewMemMapFs(), nil)
+	logger := testutil.Logger(t)
+	s, err := agentssh.NewServer(ctx, logger, prometheus.NewRegistry(), afero.NewMemMapFs(), agentexec.DefaultExecer, nil)
 	require.NoError(t, err)
 	defer s.Close()
 
@@ -76,8 +77,8 @@ func TestNewServer_ExecuteShebang(t *testing.T) {
 	}
 
 	ctx := context.Background()
-	logger := slogtest.Make(t, nil)
-	s, err := agentssh.NewServer(ctx, logger, prometheus.NewRegistry(), afero.NewMemMapFs(), nil)
+	logger := testutil.Logger(t)
+	s, err := agentssh.NewServer(ctx, logger, prometheus.NewRegistry(), afero.NewMemMapFs(), agentexec.DefaultExecer, nil)
 	require.NoError(t, err)
 	t.Cleanup(func() {
 		_ = s.Close()
@@ -108,7 +109,7 @@ func TestNewServer_CloseActiveConnections(t *testing.T) {
 
 	ctx := context.Background()
 	logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true})
-	s, err := agentssh.NewServer(ctx, logger, prometheus.NewRegistry(), afero.NewMemMapFs(), nil)
+	s, err := agentssh.NewServer(ctx, logger, prometheus.NewRegistry(), afero.NewMemMapFs(), agentexec.DefaultExecer, nil)
 	require.NoError(t, err)
 	defer s.Close()
 
@@ -158,8 +159,8 @@ func TestNewServer_Signal(t *testing.T) {
 		t.Parallel()
 
 		ctx := context.Background()
-		logger := slogtest.Make(t, nil)
-		s, err := agentssh.NewServer(ctx, logger, prometheus.NewRegistry(), afero.NewMemMapFs(), nil)
+		logger := testutil.Logger(t)
+		s, err := agentssh.NewServer(ctx, logger, prometheus.NewRegistry(), afero.NewMemMapFs(), agentexec.DefaultExecer, nil)
 		require.NoError(t, err)
 		defer s.Close()
 
@@ -223,8 +224,8 @@ func TestNewServer_Signal(t *testing.T) {
 		t.Parallel()
 
 		ctx := context.Background()
-		logger := slogtest.Make(t, nil)
-		s, err := agentssh.NewServer(ctx, logger, prometheus.NewRegistry(), afero.NewMemMapFs(), nil)
+		logger := testutil.Logger(t)
+		s, err := agentssh.NewServer(ctx, logger, prometheus.NewRegistry(), afero.NewMemMapFs(), agentexec.DefaultExecer, nil)
 		require.NoError(t, err)
 		defer s.Close()
 

agent/agentssh/x11.go

@@ -7,6 +7,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"math"
 	"net"
 	"os"
 	"path/filepath"
@@ -22,61 +23,69 @@ import (
 	"cdr.dev/slog"
 )
 
+const (
+	// X11StartPort is the starting port for X11 forwarding, this is the
+	// port used for "DISPLAY=localhost:0".
+	X11StartPort = 6000
+	// X11DefaultDisplayOffset is the default offset for X11 forwarding.
+	X11DefaultDisplayOffset = 10
+)
+
 // x11Callback is called when the client requests X11 forwarding.
-// It adds an Xauthority entry to the Xauthority file.
-func (s *Server) x11Callback(ctx ssh.Context, x11 ssh.X11) bool {
-	hostname, err := os.Hostname()
-	if err != nil {
-		s.logger.Warn(ctx, "failed to get hostname", slog.Error(err))
-		s.metrics.x11HandlerErrors.WithLabelValues("hostname").Add(1)
-		return false
-	}
-
-	err = s.fs.MkdirAll(s.config.X11SocketDir, 0o700)
-	if err != nil {
-		s.logger.Warn(ctx, "failed to make the x11 socket dir", slog.F("dir", s.config.X11SocketDir), slog.Error(err))
-		s.metrics.x11HandlerErrors.WithLabelValues("socker_dir").Add(1)
-		return false
-	}
-
-	err = addXauthEntry(ctx, s.fs, hostname, strconv.Itoa(int(x11.ScreenNumber)), x11.AuthProtocol, x11.AuthCookie)
-	if err != nil {
-		s.logger.Warn(ctx, "failed to add Xauthority entry", slog.Error(err))
-		s.metrics.x11HandlerErrors.WithLabelValues("xauthority").Add(1)
-		return false
-	}
+func (*Server) x11Callback(_ ssh.Context, _ ssh.X11) bool {
+	// Always allow.
 	return true
 }
 
 // x11Handler is called when a session has requested X11 forwarding.
 // It listens for X11 connections and forwards them to the client.
-func (s *Server) x11Handler(ctx ssh.Context, x11 ssh.X11) bool {
+func (s *Server) x11Handler(ctx ssh.Context, x11 ssh.X11) (displayNumber int, handled bool) {
 	serverConn, valid := ctx.Value(ssh.ContextKeyConn).(*gossh.ServerConn)
 	if !valid {
 		s.logger.Warn(ctx, "failed to get server connection")
-		return false
+		return -1, false
 	}
-	// We want to overwrite the socket so that subsequent connections will succeed.
-	socketPath := filepath.Join(s.config.X11SocketDir, fmt.Sprintf("X%d", x11.ScreenNumber))
-	err := os.Remove(socketPath)
-	if err != nil && !errors.Is(err, os.ErrNotExist) {
-		s.logger.Warn(ctx, "failed to remove existing X11 socket", slog.Error(err))
-		return false
-	}
-	listener, err := net.Listen("unix", socketPath)
+
+	hostname, err := os.Hostname()
 	if err != nil {
-		s.logger.Warn(ctx, "failed to listen for X11", slog.Error(err))
-		return false
+		s.logger.Warn(ctx, "failed to get hostname", slog.Error(err))
+		s.metrics.x11HandlerErrors.WithLabelValues("hostname").Add(1)
+		return -1, false
+	}
+
+	ln, display, err := createX11Listener(ctx, *s.config.X11DisplayOffset)
+	if err != nil {
+		s.logger.Warn(ctx, "failed to create X11 listener", slog.Error(err))
+		s.metrics.x11HandlerErrors.WithLabelValues("listen").Add(1)
+		return -1, false
+	}
+	s.trackListener(ln, true)
+	defer func() {
+		if !handled {
+			s.trackListener(ln, false)
+			_ = ln.Close()
+		}
+	}()
+
+	err = addXauthEntry(ctx, s.fs, hostname, strconv.Itoa(display), x11.AuthProtocol, x11.AuthCookie)
+	if err != nil {
+		s.logger.Warn(ctx, "failed to add Xauthority entry", slog.Error(err))
+		s.metrics.x11HandlerErrors.WithLabelValues("xauthority").Add(1)
+		return -1, false
 	}
-	s.trackListener(listener, true)
 
 	go func() {
-		defer listener.Close()
-		defer s.trackListener(listener, false)
-		handledFirstConnection := false
+		// Don't leave the listener open after the session is gone.
+		<-ctx.Done()
+		_ = ln.Close()
+	}()
+
+	go func() {
+		defer ln.Close()
+		defer s.trackListener(ln, false)
 
 		for {
-			conn, err := listener.Accept()
+			conn, err := ln.Accept()
 			if err != nil {
 				if errors.Is(err, net.ErrClosed) {
 					return
@@ -84,40 +93,66 @@ func (s *Server) x11Handler(ctx ssh.Context, x11 ssh.X11) bool {
 				s.logger.Warn(ctx, "failed to accept X11 connection", slog.Error(err))
 				return
 			}
-			if x11.SingleConnection && handledFirstConnection {
-				s.logger.Warn(ctx, "X11 connection rejected because single connection is enabled")
+			if x11.SingleConnection {
+				s.logger.Debug(ctx, "single connection requested, closing X11 listener")
+				_ = ln.Close()
+			}
+
+			tcpConn, ok := conn.(*net.TCPConn)
+			if !ok {
+				s.logger.Warn(ctx, fmt.Sprintf("failed to cast connection to TCPConn. got: %T", conn))
 				_ = conn.Close()
 				continue
 			}
-			handledFirstConnection = true
-
-			unixConn, ok := conn.(*net.UnixConn)
+			tcpAddr, ok := tcpConn.LocalAddr().(*net.TCPAddr)
 			if !ok {
-				s.logger.Warn(ctx, fmt.Sprintf("failed to cast connection to UnixConn. got: %T", conn))
-				return
-			}
-			unixAddr, ok := unixConn.LocalAddr().(*net.UnixAddr)
-			if !ok {
-				s.logger.Warn(ctx, fmt.Sprintf("failed to cast local address to UnixAddr. got: %T", unixConn.LocalAddr()))
-				return
+				s.logger.Warn(ctx, fmt.Sprintf("failed to cast local address to TCPAddr. got: %T", tcpConn.LocalAddr()))
+				_ = conn.Close()
+				continue
 			}
 
 			channel, reqs, err := serverConn.OpenChannel("x11", gossh.Marshal(struct {
 				OriginatorAddress string
 				OriginatorPort    uint32
 			}{
-				OriginatorAddress: unixAddr.Name,
-				OriginatorPort:    0,
+				OriginatorAddress: tcpAddr.IP.String(),
+				OriginatorPort:    uint32(tcpAddr.Port),
 			}))
 			if err != nil {
 				s.logger.Warn(ctx, "failed to open X11 channel", slog.Error(err))
-				return
+				_ = conn.Close()
+				continue
 			}
 			go gossh.DiscardRequests(reqs)
-			go Bicopy(ctx, conn, channel)
+
+			if !s.trackConn(ln, conn, true) {
+				s.logger.Warn(ctx, "failed to track X11 connection")
+				_ = conn.Close()
+				continue
+			}
+			go func() {
+				defer s.trackConn(ln, conn, false)
+				Bicopy(ctx, conn, channel)
+			}()
 		}
 	}()
-	return true
+
+	return display, true
+}
+
+// createX11Listener creates a listener for X11 forwarding, it will use
+// the next available port starting from X11StartPort and displayOffset.
+func createX11Listener(ctx context.Context, displayOffset int) (ln net.Listener, display int, err error) {
+	var lc net.ListenConfig
+	// Look for an open port to listen on.
+	for port := X11StartPort + displayOffset; port < math.MaxUint16; port++ {
+		ln, err = lc.Listen(ctx, "tcp", fmt.Sprintf("localhost:%d", port))
+		if err == nil {
+			display = port - X11StartPort
+			return ln, display, nil
+		}
+	}
+	return nil, -1, xerrors.Errorf("failed to find open port for X11 listener: %w", err)
 }
 
 // addXauthEntry adds an Xauthority entry to the Xauthority file.

agent/agentssh/x11_test.go

@@ -1,12 +1,17 @@
 package agentssh_test
 
 import (
+	"bufio"
+	"bytes"
 	"context"
 	"encoding/hex"
+	"fmt"
 	"net"
 	"os"
 	"path/filepath"
 	"runtime"
+	"strconv"
+	"strings"
 	"testing"
 
 	"github.com/gliderlabs/ssh"
@@ -16,8 +21,7 @@ import (
 	"github.com/stretchr/testify/require"
 	gossh "golang.org/x/crypto/ssh"
 
-	"cdr.dev/slog"
-	"cdr.dev/slog/sloggers/slogtest"
+	"github.com/coder/coder/v2/agent/agentexec"
 	"github.com/coder/coder/v2/agent/agentssh"
 	"github.com/coder/coder/v2/testutil"
 )
@@ -29,12 +33,9 @@ func TestServer_X11(t *testing.T) {
 	}
 
 	ctx := context.Background()
-	logger := slogtest.Make(t, nil).Leveled(slog.LevelDebug)
+	logger := testutil.Logger(t)
 	fs := afero.NewOsFs()
-	dir := t.TempDir()
-	s, err := agentssh.NewServer(ctx, logger, prometheus.NewRegistry(), fs, &agentssh.Config{
-		X11SocketDir: dir,
-	})
+	s, err := agentssh.NewServer(ctx, logger, prometheus.NewRegistry(), fs, agentexec.DefaultExecer, &agentssh.Config{})
 	require.NoError(t, err)
 	defer s.Close()
 
@@ -53,21 +54,45 @@ func TestServer_X11(t *testing.T) {
 	sess, err := c.NewSession()
 	require.NoError(t, err)
 
+	wantScreenNumber := 1
 	reply, err := sess.SendRequest("x11-req", true, gossh.Marshal(ssh.X11{
 		AuthProtocol: "MIT-MAGIC-COOKIE-1",
 		AuthCookie:   hex.EncodeToString([]byte("cookie")),
-		ScreenNumber: 0,
+		ScreenNumber: uint32(wantScreenNumber),
 	}))
 	require.NoError(t, err)
 	assert.True(t, reply)
 
-	err = sess.Shell()
+	// Want: ~DISPLAY=localhost:10.1
+	out, err := sess.Output("echo DISPLAY=$DISPLAY")
 	require.NoError(t, err)
 
+	sc := bufio.NewScanner(bytes.NewReader(out))
+	displayNumber := -1
+	for sc.Scan() {
+		line := strings.TrimSpace(sc.Text())
+		t.Log(line)
+		if strings.HasPrefix(line, "DISPLAY=") {
+			parts := strings.SplitN(line, "=", 2)
+			display := parts[1]
+			parts = strings.SplitN(display, ":", 2)
+			parts = strings.SplitN(parts[1], ".", 2)
+			displayNumber, err = strconv.Atoi(parts[0])
+			require.NoError(t, err)
+			assert.GreaterOrEqual(t, displayNumber, 10, "display number should be >= 10")
+			gotScreenNumber, err := strconv.Atoi(parts[1])
+			require.NoError(t, err)
+			assert.Equal(t, wantScreenNumber, gotScreenNumber, "screen number should match")
+			break
+		}
+	}
+	require.NoError(t, sc.Err())
+	require.NotEqual(t, -1, displayNumber)
+
 	x11Chans := c.HandleChannelOpen("x11")
 	payload := "hello world"
 	require.Eventually(t, func() bool {
-		conn, err := net.Dial("unix", filepath.Join(dir, "X0"))
+		conn, err := net.Dial("tcp", fmt.Sprintf("localhost:%d", agentssh.X11StartPort+displayNumber))
 		if err == nil {
 			_, err = conn.Write([]byte(payload))
 			assert.NoError(t, err)

agent/agenttest/agent.go

@@ -7,10 +7,9 @@ import (
 
 	"github.com/stretchr/testify/assert"
 
-	"cdr.dev/slog"
-	"cdr.dev/slog/sloggers/slogtest"
 	"github.com/coder/coder/v2/agent"
 	"github.com/coder/coder/v2/codersdk/agentsdk"
+	"github.com/coder/coder/v2/testutil"
 )
 
 // New starts a new agent for use in tests.
@@ -24,7 +23,7 @@ func New(t testing.TB, coderURL *url.URL, agentToken string, opts ...func(*agent
 	t.Helper()
 
 	var o agent.Options
-	log := slogtest.Make(t, nil).Leveled(slog.LevelDebug).Named("agent")
+	log := testutil.Logger(t).Named("agent")
 	o.Logger = log
 
 	for _, opt := range opts {

agent/agenttest/client.go

@@ -15,7 +15,6 @@ import (
 	"golang.org/x/exp/slices"
 	"golang.org/x/xerrors"
 	"google.golang.org/protobuf/types/known/durationpb"
-	"storj.io/drpc"
 	"storj.io/drpc/drpcmux"
 	"storj.io/drpc/drpcserver"
 	"tailscale.com/tailcfg"
@@ -71,7 +70,6 @@ func NewClient(t testing.TB,
 		t:              t,
 		logger:         logger.Named("client"),
 		agentID:        agentID,
-		coordinator:    coordinator,
 		server:         server,
 		fakeAgentAPI:   fakeAAPI,
 		derpMapUpdates: derpMapUpdates,
@@ -82,7 +80,6 @@ type Client struct {
 	t                  testing.TB
 	logger             slog.Logger
 	agentID            uuid.UUID
-	coordinator        tailnet.Coordinator
 	server             *drpcserver.Server
 	fakeAgentAPI       *FakeAgentAPI
 	LastWorkspaceAgent func()
@@ -99,7 +96,9 @@ func (c *Client) Close() {
 	c.derpMapOnce.Do(func() { close(c.derpMapUpdates) })
 }
 
-func (c *Client) ConnectRPC(ctx context.Context) (drpc.Conn, error) {
+func (c *Client) ConnectRPC23(ctx context.Context) (
+	agentproto.DRPCAgentClient23, proto.DRPCTailnetClient23, error,
+) {
 	conn, lis := drpcsdk.MemTransportPipe()
 	c.LastWorkspaceAgent = func() {
 		_ = conn.Close()
@@ -117,7 +116,7 @@ func (c *Client) ConnectRPC(ctx context.Context) (drpc.Conn, error) {
 	go func() {
 		_ = c.server.Serve(serveCtx, lis)
 	}()
-	return conn, nil
+	return agentproto.NewDRPCAgentClient(conn), proto.NewDRPCTailnetClient(conn), nil
 }
 
 func (c *Client) GetLifecycleStates() []codersdk.WorkspaceAgentLifecycle {
@@ -170,6 +169,7 @@ type FakeAgentAPI struct {
 	logsCh          chan<- *agentproto.BatchCreateLogsRequest
 	lifecycleStates []codersdk.WorkspaceAgentLifecycle
 	metadata        map[string]agentsdk.Metadata
+	timings         []*agentproto.Timing
 
 	getAnnouncementBannersFunc func() ([]codersdk.BannerConfig, error)
 }
@@ -182,6 +182,12 @@ func (*FakeAgentAPI) GetServiceBanner(context.Context, *agentproto.GetServiceBan
 	return &agentproto.ServiceBanner{}, nil
 }
 
+func (f *FakeAgentAPI) GetTimings() []*agentproto.Timing {
+	f.Lock()
+	defer f.Unlock()
+	return slices.Clone(f.timings)
+}
+
 func (f *FakeAgentAPI) SetAnnouncementBannersFunc(fn func() ([]codersdk.BannerConfig, error)) {
 	f.Lock()
 	defer f.Unlock()
@@ -210,7 +216,12 @@ func (f *FakeAgentAPI) UpdateStats(ctx context.Context, req *agentproto.UpdateSt
 	f.logger.Debug(ctx, "update stats called", slog.F("req", req))
 	// empty request is sent to get the interval; but our tests don't want empty stats requests
 	if req.Stats != nil {
-		f.statsCh <- req.Stats
+		select {
+		case <-ctx.Done():
+			return nil, ctx.Err()
+		case f.statsCh <- req.Stats:
+			// OK!
+		}
 	}
 	return &agentproto.UpdateStatsResponse{ReportInterval: durationpb.New(statsInterval)}, nil
 }
@@ -233,17 +244,25 @@ func (f *FakeAgentAPI) UpdateLifecycle(_ context.Context, req *agentproto.Update
 
 func (f *FakeAgentAPI) BatchUpdateAppHealths(ctx context.Context, req *agentproto.BatchUpdateAppHealthRequest) (*agentproto.BatchUpdateAppHealthResponse, error) {
 	f.logger.Debug(ctx, "batch update app health", slog.F("req", req))
-	f.appHealthCh <- req
-	return &agentproto.BatchUpdateAppHealthResponse{}, nil
+	select {
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	case f.appHealthCh <- req:
+		return &agentproto.BatchUpdateAppHealthResponse{}, nil
+	}
 }
 
 func (f *FakeAgentAPI) AppHealthCh() <-chan *agentproto.BatchUpdateAppHealthRequest {
 	return f.appHealthCh
 }
 
-func (f *FakeAgentAPI) UpdateStartup(_ context.Context, req *agentproto.UpdateStartupRequest) (*agentproto.Startup, error) {
-	f.startupCh <- req.GetStartup()
-	return req.GetStartup(), nil
+func (f *FakeAgentAPI) UpdateStartup(ctx context.Context, req *agentproto.UpdateStartupRequest) (*agentproto.Startup, error) {
+	select {
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	case f.startupCh <- req.GetStartup():
+		return req.GetStartup(), nil
+	}
 }
 
 func (f *FakeAgentAPI) GetMetadata() map[string]agentsdk.Metadata {
@@ -288,6 +307,14 @@ func (f *FakeAgentAPI) BatchCreateLogs(ctx context.Context, req *agentproto.Batc
 	return &agentproto.BatchCreateLogsResponse{}, nil
 }
 
+func (f *FakeAgentAPI) ScriptCompleted(_ context.Context, req *agentproto.WorkspaceAgentScriptCompletedRequest) (*agentproto.WorkspaceAgentScriptCompletedResponse, error) {
+	f.Lock()
+	f.timings = append(f.timings, req.Timing)
+	f.Unlock()
+
+	return &agentproto.WorkspaceAgentScriptCompletedResponse{}, nil
+}
+
 func NewFakeAgentAPI(t testing.TB, logger slog.Logger, manifest *agentproto.Manifest, statsCh chan *agentproto.Stats) *FakeAgentAPI {
 	return &FakeAgentAPI{
 		t:           t,

agent/api.go

@@ -37,6 +37,7 @@ func (a *agent) apiHandler() http.Handler {
 	}
 	promHandler := PrometheusMetricsHandler(a.prometheusRegistry, a.logger)
 	r.Get("/api/v0/listening-ports", lp.handler)
+	r.Get("/api/v0/netcheck", a.HandleNetcheck)
 	r.Get("/debug/logs", a.HandleHTTPDebugLogs)
 	r.Get("/debug/magicsock", a.HandleHTTPDebugMagicsock)
 	r.Get("/debug/magicsock/debug-logging/{state}", a.HandleHTTPMagicsockDebugLoggingState)

agent/apphealth.go

@@ -10,9 +10,9 @@ import (
 	"golang.org/x/xerrors"
 
 	"cdr.dev/slog"
-	"github.com/coder/coder/v2/clock"
 	"github.com/coder/coder/v2/codersdk"
 	"github.com/coder/coder/v2/codersdk/agentsdk"
+	"github.com/coder/quartz"
 )
 
 // PostWorkspaceAgentAppHealth updates the workspace app health.
@@ -23,7 +23,7 @@ type WorkspaceAppHealthReporter func(ctx context.Context)
 
 // NewWorkspaceAppHealthReporter creates a WorkspaceAppHealthReporter that reports app health to coderd.
 func NewWorkspaceAppHealthReporter(logger slog.Logger, apps []codersdk.WorkspaceApp, postWorkspaceAgentAppHealth PostWorkspaceAgentAppHealth) WorkspaceAppHealthReporter {
-	return NewAppHealthReporterWithClock(logger, apps, postWorkspaceAgentAppHealth, clock.NewReal())
+	return NewAppHealthReporterWithClock(logger, apps, postWorkspaceAgentAppHealth, quartz.NewReal())
 }
 
 // NewAppHealthReporterWithClock is only called directly by test code.  Product code should call
@@ -32,7 +32,7 @@ func NewAppHealthReporterWithClock(
 	logger slog.Logger,
 	apps []codersdk.WorkspaceApp,
 	postWorkspaceAgentAppHealth PostWorkspaceAgentAppHealth,
-	clk clock.Clock,
+	clk quartz.Clock,
 ) WorkspaceAppHealthReporter {
 	logger = logger.Named("apphealth")
 

agent/apphealth_test.go

@@ -12,16 +12,14 @@ import (
 	"github.com/google/uuid"
 	"github.com/stretchr/testify/require"
 
-	"cdr.dev/slog"
-	"cdr.dev/slog/sloggers/slogtest"
 	"github.com/coder/coder/v2/agent"
 	"github.com/coder/coder/v2/agent/agenttest"
 	"github.com/coder/coder/v2/agent/proto"
-	"github.com/coder/coder/v2/clock"
 	"github.com/coder/coder/v2/coderd/httpapi"
 	"github.com/coder/coder/v2/codersdk"
 	"github.com/coder/coder/v2/codersdk/agentsdk"
 	"github.com/coder/coder/v2/testutil"
+	"github.com/coder/quartz"
 )
 
 func TestAppHealth_Healthy(t *testing.T) {
@@ -69,7 +67,7 @@ func TestAppHealth_Healthy(t *testing.T) {
 			httpapi.Write(r.Context(), w, http.StatusOK, nil)
 		}),
 	}
-	mClock := clock.NewMock(t)
+	mClock := quartz.NewMock(t)
 	healthcheckTrap := mClock.Trap().TickerFunc("healthcheck")
 	defer healthcheckTrap.Close()
 	reportTrap := mClock.Trap().TickerFunc("report")
@@ -137,7 +135,7 @@ func TestAppHealth_500(t *testing.T) {
 		}),
 	}
 
-	mClock := clock.NewMock(t)
+	mClock := quartz.NewMock(t)
 	healthcheckTrap := mClock.Trap().TickerFunc("healthcheck")
 	defer healthcheckTrap.Close()
 	reportTrap := mClock.Trap().TickerFunc("report")
@@ -187,7 +185,7 @@ func TestAppHealth_Timeout(t *testing.T) {
 			<-r.Context().Done()
 		}),
 	}
-	mClock := clock.NewMock(t)
+	mClock := quartz.NewMock(t)
 	start := mClock.Now()
 
 	// for this test, it's easier to think in the number of milliseconds elapsed
@@ -235,7 +233,7 @@ func setupAppReporter(
 	ctx context.Context, t *testing.T,
 	apps []codersdk.WorkspaceApp,
 	handlers []http.Handler,
-	clk clock.Clock,
+	clk quartz.Clock,
 ) (*agenttest.FakeAgentAPI, func()) {
 	closers := []func(){}
 	for _, app := range apps {
@@ -258,10 +256,10 @@ func setupAppReporter(
 	// We use a proper fake agent API so we can test the conversion code and the
 	// request code as well. Before we were bypassing these by using a custom
 	// post function.
-	fakeAAPI := agenttest.NewFakeAgentAPI(t, slogtest.Make(t, nil), nil, nil)
+	fakeAAPI := agenttest.NewFakeAgentAPI(t, testutil.Logger(t), nil, nil)
 
 	go agent.NewAppHealthReporterWithClock(
-		slogtest.Make(t, nil).Leveled(slog.LevelDebug),
+		testutil.Logger(t),
 		apps, agentsdk.AppHealthPoster(fakeAAPI), clk,
 	)(ctx)
 

agent/checkpoint_internal_test.go

@@ -12,7 +12,7 @@ import (
 
 func TestCheckpoint_CompleteWait(t *testing.T) {
 	t.Parallel()
-	logger := slogtest.Make(t, nil)
+	logger := testutil.Logger(t)
 	ctx := testutil.Context(t, testutil.WaitShort)
 	uut := newCheckpoint(logger)
 	err := xerrors.New("test")
@@ -35,7 +35,7 @@ func TestCheckpoint_CompleteTwice(t *testing.T) {
 
 func TestCheckpoint_WaitComplete(t *testing.T) {
 	t.Parallel()
-	logger := slogtest.Make(t, nil)
+	logger := testutil.Logger(t)
 	ctx := testutil.Context(t, testutil.WaitShort)
 	uut := newCheckpoint(logger)
 	err := xerrors.New("test")

agent/health.go

@@ -0,0 +1,31 @@
+package agent
+
+import (
+	"net/http"
+
+	"github.com/coder/coder/v2/coderd/healthcheck/health"
+	"github.com/coder/coder/v2/coderd/httpapi"
+	"github.com/coder/coder/v2/codersdk"
+	"github.com/coder/coder/v2/codersdk/healthsdk"
+)
+
+func (a *agent) HandleNetcheck(rw http.ResponseWriter, r *http.Request) {
+	ni := a.TailnetConn().GetNetInfo()
+
+	ifReport, err := healthsdk.RunInterfacesReport()
+	if err != nil {
+		httpapi.Write(r.Context(), rw, http.StatusInternalServerError, codersdk.Response{
+			Message: "Failed to run interfaces report",
+			Detail:  err.Error(),
+		})
+		return
+	}
+
+	httpapi.Write(r.Context(), rw, http.StatusOK, healthsdk.AgentNetcheckReport{
+		BaseReport: healthsdk.BaseReport{
+			Severity: health.SeverityOK,
+		},
+		NetInfo:    ni,
+		Interfaces: ifReport,
+	})
+}

agent/metrics.go

@@ -19,6 +19,7 @@ type agentMetrics struct {
 	// startupScriptSeconds is the time in seconds that the start script(s)
 	// took to run. This is reported once per agent.
 	startupScriptSeconds *prometheus.GaugeVec
+	currentConnections   *prometheus.GaugeVec
 }
 
 func newAgentMetrics(registerer prometheus.Registerer) *agentMetrics {
@@ -45,10 +46,19 @@ func newAgentMetrics(registerer prometheus.Registerer) *agentMetrics {
 	}, []string{"success"})
 	registerer.MustRegister(startupScriptSeconds)
 
+	currentConnections := prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: "coderd",
+		Subsystem: "agentstats",
+		Name:      "currently_reachable_peers",
+		Help:      "The number of peers (e.g. clients) that are currently reachable over the encrypted network.",
+	}, []string{"connection_type"})
+	registerer.MustRegister(currentConnections)
+
 	return &agentMetrics{
 		connectionsTotal:      connectionsTotal,
 		reconnectingPTYErrors: reconnectingPTYErrors,
 		startupScriptSeconds:  startupScriptSeconds,
+		currentConnections:    currentConnections,
 	}
 }
 

agent/proto/agent.proto

@@ -41,6 +41,7 @@ message WorkspaceApp {
 		UNHEALTHY = 4;
 	}
 	Health health = 12;
+	bool hidden = 13;
 }
 
 message WorkspaceAgentScript {
@@ -52,6 +53,8 @@ message WorkspaceAgentScript {
 	bool run_on_stop = 6;
 	bool start_blocks_login = 7;
 	google.protobuf.Duration timeout = 8;
+	string display_name = 9;
+	bytes id = 10;
 }
 
 message WorkspaceAgentMetadata {
@@ -263,6 +266,35 @@ message BannerConfig {
 	string background_color = 3;
 }
 
+message WorkspaceAgentScriptCompletedRequest {
+    Timing timing = 1;
+}
+
+message WorkspaceAgentScriptCompletedResponse {
+}
+
+message Timing {
+    bytes script_id = 1;
+    google.protobuf.Timestamp start = 2;
+    google.protobuf.Timestamp end = 3;
+    int32 exit_code = 4;
+
+    enum Stage {
+        START = 0;
+        STOP = 1;
+        CRON = 2;
+	}
+    Stage stage = 5;
+
+    enum Status {
+        OK = 0;
+        EXIT_FAILURE = 1;
+        TIMED_OUT = 2;
+        PIPES_LEFT_OPEN = 3;
+    }
+    Status status = 6;
+}
+
 service Agent {
 	rpc GetManifest(GetManifestRequest) returns (Manifest);
 	rpc GetServiceBanner(GetServiceBannerRequest) returns (ServiceBanner);
@@ -273,4 +305,5 @@ service Agent {
 	rpc BatchUpdateMetadata(BatchUpdateMetadataRequest) returns (BatchUpdateMetadataResponse);
 	rpc BatchCreateLogs(BatchCreateLogsRequest) returns (BatchCreateLogsResponse);
 	rpc GetAnnouncementBanners(GetAnnouncementBannersRequest) returns (GetAnnouncementBannersResponse);
+	rpc ScriptCompleted(WorkspaceAgentScriptCompletedRequest) returns (WorkspaceAgentScriptCompletedResponse);
 }

agent/proto/agent_drpc.pb.go

@@ -1,5 +1,5 @@
 // Code generated by protoc-gen-go-drpc. DO NOT EDIT.
-// protoc-gen-go-drpc version: v0.0.33
+// protoc-gen-go-drpc version: v0.0.34
 // source: agent/proto/agent.proto
 
 package proto
@@ -47,6 +47,7 @@ type DRPCAgentClient interface {
 	BatchUpdateMetadata(ctx context.Context, in *BatchUpdateMetadataRequest) (*BatchUpdateMetadataResponse, error)
 	BatchCreateLogs(ctx context.Context, in *BatchCreateLogsRequest) (*BatchCreateLogsResponse, error)
 	GetAnnouncementBanners(ctx context.Context, in *GetAnnouncementBannersRequest) (*GetAnnouncementBannersResponse, error)
+	ScriptCompleted(ctx context.Context, in *WorkspaceAgentScriptCompletedRequest) (*WorkspaceAgentScriptCompletedResponse, error)
 }
 
 type drpcAgentClient struct {
@@ -140,6 +141,15 @@ func (c *drpcAgentClient) GetAnnouncementBanners(ctx context.Context, in *GetAnn
 	return out, nil
 }
 
+func (c *drpcAgentClient) ScriptCompleted(ctx context.Context, in *WorkspaceAgentScriptCompletedRequest) (*WorkspaceAgentScriptCompletedResponse, error) {
+	out := new(WorkspaceAgentScriptCompletedResponse)
+	err := c.cc.Invoke(ctx, "/coder.agent.v2.Agent/ScriptCompleted", drpcEncoding_File_agent_proto_agent_proto{}, in, out)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
 type DRPCAgentServer interface {
 	GetManifest(context.Context, *GetManifestRequest) (*Manifest, error)
 	GetServiceBanner(context.Context, *GetServiceBannerRequest) (*ServiceBanner, error)
@@ -150,6 +160,7 @@ type DRPCAgentServer interface {
 	BatchUpdateMetadata(context.Context, *BatchUpdateMetadataRequest) (*BatchUpdateMetadataResponse, error)
 	BatchCreateLogs(context.Context, *BatchCreateLogsRequest) (*BatchCreateLogsResponse, error)
 	GetAnnouncementBanners(context.Context, *GetAnnouncementBannersRequest) (*GetAnnouncementBannersResponse, error)
+	ScriptCompleted(context.Context, *WorkspaceAgentScriptCompletedRequest) (*WorkspaceAgentScriptCompletedResponse, error)
 }
 
 type DRPCAgentUnimplementedServer struct{}
@@ -190,9 +201,13 @@ func (s *DRPCAgentUnimplementedServer) GetAnnouncementBanners(context.Context, *
 	return nil, drpcerr.WithCode(errors.New("Unimplemented"), drpcerr.Unimplemented)
 }
 
+func (s *DRPCAgentUnimplementedServer) ScriptCompleted(context.Context, *WorkspaceAgentScriptCompletedRequest) (*WorkspaceAgentScriptCompletedResponse, error) {
+	return nil, drpcerr.WithCode(errors.New("Unimplemented"), drpcerr.Unimplemented)
+}
+
 type DRPCAgentDescription struct{}
 
-func (DRPCAgentDescription) NumMethods() int { return 9 }
+func (DRPCAgentDescription) NumMethods() int { return 10 }
 
 func (DRPCAgentDescription) Method(n int) (string, drpc.Encoding, drpc.Receiver, interface{}, bool) {
 	switch n {
@@ -277,6 +292,15 @@ func (DRPCAgentDescription) Method(n int) (string, drpc.Encoding, drpc.Receiver,
 						in1.(*GetAnnouncementBannersRequest),
 					)
 			}, DRPCAgentServer.GetAnnouncementBanners, true
+	case 9:
+		return "/coder.agent.v2.Agent/ScriptCompleted", drpcEncoding_File_agent_proto_agent_proto{},
+			func(srv interface{}, ctx context.Context, in1, in2 interface{}) (drpc.Message, error) {
+				return srv.(DRPCAgentServer).
+					ScriptCompleted(
+						ctx,
+						in1.(*WorkspaceAgentScriptCompletedRequest),
+					)
+			}, DRPCAgentServer.ScriptCompleted, true
 	default:
 		return "", nil, nil, nil, false
 	}
@@ -429,3 +453,19 @@ func (x *drpcAgent_GetAnnouncementBannersStream) SendAndClose(m *GetAnnouncement
 	}
 	return x.CloseSend()
 }
+
+type DRPCAgent_ScriptCompletedStream interface {
+	drpc.Stream
+	SendAndClose(*WorkspaceAgentScriptCompletedResponse) error
+}
+
+type drpcAgent_ScriptCompletedStream struct {
+	drpc.Stream
+}
+
+func (x *drpcAgent_ScriptCompletedStream) SendAndClose(m *WorkspaceAgentScriptCompletedResponse) error {
+	if err := x.MsgSend(m, drpcEncoding_File_agent_proto_agent_proto{}); err != nil {
+		return err
+	}
+	return x.CloseSend()
+}

agent/proto/agent_drpc_old.go

@@ -0,0 +1,42 @@
+package proto
+
+import (
+	"context"
+
+	"storj.io/drpc"
+)
+
+// DRPCAgentClient20 is the Agent API at v2.0.  Notably, it is missing GetAnnouncementBanners, but
+// is useful when you want to be maximally compatible with Coderd Release Versions from 2.9+
+type DRPCAgentClient20 interface {
+	DRPCConn() drpc.Conn
+
+	GetManifest(ctx context.Context, in *GetManifestRequest) (*Manifest, error)
+	GetServiceBanner(ctx context.Context, in *GetServiceBannerRequest) (*ServiceBanner, error)
+	UpdateStats(ctx context.Context, in *UpdateStatsRequest) (*UpdateStatsResponse, error)
+	UpdateLifecycle(ctx context.Context, in *UpdateLifecycleRequest) (*Lifecycle, error)
+	BatchUpdateAppHealths(ctx context.Context, in *BatchUpdateAppHealthRequest) (*BatchUpdateAppHealthResponse, error)
+	UpdateStartup(ctx context.Context, in *UpdateStartupRequest) (*Startup, error)
+	BatchUpdateMetadata(ctx context.Context, in *BatchUpdateMetadataRequest) (*BatchUpdateMetadataResponse, error)
+	BatchCreateLogs(ctx context.Context, in *BatchCreateLogsRequest) (*BatchCreateLogsResponse, error)
+}
+
+// DRPCAgentClient21 is the Agent API at v2.1. It is useful if you want to be maximally compatible
+// with Coderd Release Versions from 2.12+
+type DRPCAgentClient21 interface {
+	DRPCAgentClient20
+	GetAnnouncementBanners(ctx context.Context, in *GetAnnouncementBannersRequest) (*GetAnnouncementBannersResponse, error)
+}
+
+// DRPCAgentClient22 is the Agent API at v2.2. It is identical to 2.1, since the change was made on
+// the Tailnet API, which uses the same version number. Compatible with Coder v2.13+
+type DRPCAgentClient22 interface {
+	DRPCAgentClient21
+}
+
+// DRPCAgentClient23 is the Agent API at v2.3. It adds the ScriptCompleted RPC. Compatible with
+// Coder v2.18+
+type DRPCAgentClient23 interface {
+	DRPCAgentClient22
+	ScriptCompleted(ctx context.Context, in *WorkspaceAgentScriptCompletedRequest) (*WorkspaceAgentScriptCompletedResponse, error)
+}

agent/reconnectingpty/buffered.go

@@ -14,6 +14,7 @@ import (
 
 	"cdr.dev/slog"
 
+	"github.com/coder/coder/v2/agent/agentexec"
 	"github.com/coder/coder/v2/pty"
 )
 
@@ -39,7 +40,7 @@ type bufferedReconnectingPTY struct {
 
 // newBuffered starts the buffered pty.  If the context ends the process will be
 // killed.
-func newBuffered(ctx context.Context, cmd *pty.Cmd, options *Options, logger slog.Logger) *bufferedReconnectingPTY {
+func newBuffered(ctx context.Context, logger slog.Logger, execer agentexec.Execer, cmd *pty.Cmd, options *Options) *bufferedReconnectingPTY {
 	rpty := &bufferedReconnectingPTY{
 		activeConns: map[string]net.Conn{},
 		command:     cmd,
@@ -58,7 +59,7 @@ func newBuffered(ctx context.Context, cmd *pty.Cmd, options *Options, logger slo
 
 	// Add TERM then start the command with a pty.  pty.Cmd duplicates Path as the
 	// first argument so remove it.
-	cmdWithEnv := pty.CommandContext(ctx, cmd.Path, cmd.Args[1:]...)
+	cmdWithEnv := execer.PTYCommandContext(ctx, cmd.Path, cmd.Args[1:]...)
 	cmdWithEnv.Env = append(rpty.command.Env, "TERM=xterm-256color")
 	cmdWithEnv.Dir = rpty.command.Dir
 	ptty, process, err := pty.Start(cmdWithEnv)

agent/reconnectingpty/reconnectingpty.go

@@ -14,6 +14,7 @@ import (
 	"golang.org/x/xerrors"
 
 	"cdr.dev/slog"
+	"github.com/coder/coder/v2/agent/agentexec"
 	"github.com/coder/coder/v2/codersdk/workspacesdk"
 	"github.com/coder/coder/v2/pty"
 )
@@ -55,7 +56,7 @@ type ReconnectingPTY interface {
 // close itself (and all connections to it) if nothing is attached for the
 // duration of the timeout, if the context ends, or the process exits (buffered
 // backend only).
-func New(ctx context.Context, cmd *pty.Cmd, options *Options, logger slog.Logger) ReconnectingPTY {
+func New(ctx context.Context, logger slog.Logger, execer agentexec.Execer, cmd *pty.Cmd, options *Options) ReconnectingPTY {
 	if options.Timeout == 0 {
 		options.Timeout = 5 * time.Minute
 	}
@@ -75,9 +76,9 @@ func New(ctx context.Context, cmd *pty.Cmd, options *Options, logger slog.Logger
 
 	switch backendType {
 	case "screen":
-		return newScreen(ctx, cmd, options, logger)
+		return newScreen(ctx, logger, execer, cmd, options)
 	default:
-		return newBuffered(ctx, cmd, options, logger)
+		return newBuffered(ctx, logger, execer, cmd, options)
 	}
 }
 

agent/reconnectingpty/screen.go

@@ -9,7 +9,6 @@ import (
 	"io"
 	"net"
 	"os"
-	"os/exec"
 	"path/filepath"
 	"strings"
 	"sync"
@@ -20,11 +19,13 @@ import (
 	"golang.org/x/xerrors"
 
 	"cdr.dev/slog"
+	"github.com/coder/coder/v2/agent/agentexec"
 	"github.com/coder/coder/v2/pty"
 )
 
 // screenReconnectingPTY provides a reconnectable PTY via `screen`.
 type screenReconnectingPTY struct {
+	execer  agentexec.Execer
 	command *pty.Cmd
 
 	// id holds the id of the session for both creating and attaching.  This will
@@ -59,16 +60,15 @@ type screenReconnectingPTY struct {
 // spawns the daemon with a hardcoded 24x80 size it is not a very good user
 // experience.  Instead we will let the attach command spawn the daemon on its
 // own which causes it to spawn with the specified size.
-func newScreen(ctx context.Context, cmd *pty.Cmd, options *Options, logger slog.Logger) *screenReconnectingPTY {
+func newScreen(ctx context.Context, logger slog.Logger, execer agentexec.Execer, cmd *pty.Cmd, options *Options) *screenReconnectingPTY {
 	rpty := &screenReconnectingPTY{
+		execer:  execer,
 		command: cmd,
 		metrics: options.Metrics,
 		state:   newState(),
 		timeout: options.Timeout,
 	}
 
-	go rpty.lifecycle(ctx, logger)
-
 	// Socket paths are limited to around 100 characters on Linux and macOS which
 	// depending on the temporary directory can be a problem.  To give more leeway
 	// use a short ID.
@@ -124,6 +124,8 @@ func newScreen(ctx context.Context, cmd *pty.Cmd, options *Options, logger slog.
 		return rpty
 	}
 
+	go rpty.lifecycle(ctx, logger)
+
 	return rpty
 }
 
@@ -210,7 +212,7 @@ func (rpty *screenReconnectingPTY) doAttach(ctx context.Context, conn net.Conn,
 	logger.Debug(ctx, "spawning screen client", slog.F("screen_id", rpty.id))
 
 	// Wrap the command with screen and tie it to the connection's context.
-	cmd := pty.CommandContext(ctx, "screen", append([]string{
+	cmd := rpty.execer.PTYCommandContext(ctx, "screen", append([]string{
 		// -S is for setting the session's name.
 		"-S", rpty.id,
 		// -U tells screen to use UTF-8 encoding.
@@ -327,10 +329,10 @@ func (rpty *screenReconnectingPTY) sendCommand(ctx context.Context, command stri
 	defer cancel()
 
 	var lastErr error
-	run := func() bool {
+	run := func() (bool, error) {
 		var stdout bytes.Buffer
 		//nolint:gosec
-		cmd := exec.CommandContext(ctx, "screen",
+		cmd := rpty.execer.CommandContext(ctx, "screen",
 			// -x targets an attached session.
 			"-x", rpty.id,
 			// -c is the flag for the config file.
@@ -343,13 +345,13 @@ func (rpty *screenReconnectingPTY) sendCommand(ctx context.Context, command stri
 		cmd.Stdout = &stdout
 		err := cmd.Run()
 		if err == nil {
-			return true
+			return true, nil
 		}
 
 		stdoutStr := stdout.String()
 		for _, se := range successErrors {
 			if strings.Contains(stdoutStr, se) {
-				return true
+				return true, nil
 			}
 		}
 
@@ -359,11 +361,15 @@ func (rpty *screenReconnectingPTY) sendCommand(ctx context.Context, command stri
 			lastErr = xerrors.Errorf("`screen -x %s -X %s`: %w: %s", rpty.id, command, err, stdoutStr)
 		}
 
-		return false
+		return false, nil
 	}
 
 	// Run immediately.
-	if done := run(); done {
+	done, err := run()
+	if err != nil {
+		return err
+	}
+	if done {
 		return nil
 	}
 
@@ -379,7 +385,11 @@ func (rpty *screenReconnectingPTY) sendCommand(ctx context.Context, command stri
 			}
 			return errors.Join(ctx.Err(), lastErr)
 		case <-ticker.C:
-			if done := run(); done {
+			done, err := run()
+			if err != nil {
+				return err
+			}
+			if done {
 				return nil
 			}
 		}

agent/reconnectingpty/server.go

@@ -0,0 +1,196 @@
+package reconnectingpty
+
+import (
+	"context"
+	"encoding/binary"
+	"encoding/json"
+	"net"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/prometheus/client_golang/prometheus"
+	"golang.org/x/xerrors"
+
+	"cdr.dev/slog"
+	"github.com/coder/coder/v2/agent/agentssh"
+	"github.com/coder/coder/v2/codersdk/workspacesdk"
+)
+
+type Server struct {
+	logger           slog.Logger
+	connectionsTotal prometheus.Counter
+	errorsTotal      *prometheus.CounterVec
+	commandCreator   *agentssh.Server
+	connCount        atomic.Int64
+	reconnectingPTYs sync.Map
+	timeout          time.Duration
+}
+
+// NewServer returns a new ReconnectingPTY server
+func NewServer(logger slog.Logger, commandCreator *agentssh.Server,
+	connectionsTotal prometheus.Counter, errorsTotal *prometheus.CounterVec,
+	timeout time.Duration,
+) *Server {
+	return &Server{
+		logger:           logger,
+		commandCreator:   commandCreator,
+		connectionsTotal: connectionsTotal,
+		errorsTotal:      errorsTotal,
+		timeout:          timeout,
+	}
+}
+
+func (s *Server) Serve(ctx, hardCtx context.Context, l net.Listener) (retErr error) {
+	var wg sync.WaitGroup
+	for {
+		if ctx.Err() != nil {
+			break
+		}
+		conn, err := l.Accept()
+		if err != nil {
+			s.logger.Debug(ctx, "accept pty failed", slog.Error(err))
+			retErr = err
+			break
+		}
+		clog := s.logger.With(
+			slog.F("remote", conn.RemoteAddr().String()),
+			slog.F("local", conn.LocalAddr().String()))
+		clog.Info(ctx, "accepted conn")
+		wg.Add(1)
+		closed := make(chan struct{})
+		go func() {
+			select {
+			case <-closed:
+			case <-hardCtx.Done():
+				_ = conn.Close()
+			}
+			wg.Done()
+		}()
+		wg.Add(1)
+		go func() {
+			defer close(closed)
+			defer wg.Done()
+			_ = s.handleConn(ctx, clog, conn)
+		}()
+	}
+	wg.Wait()
+	return retErr
+}
+
+func (s *Server) ConnCount() int64 {
+	return s.connCount.Load()
+}
+
+func (s *Server) handleConn(ctx context.Context, logger slog.Logger, conn net.Conn) (retErr error) {
+	defer conn.Close()
+	s.connectionsTotal.Add(1)
+	s.connCount.Add(1)
+	defer s.connCount.Add(-1)
+
+	// This cannot use a JSON decoder, since that can
+	// buffer additional data that is required for the PTY.
+	rawLen := make([]byte, 2)
+	_, err := conn.Read(rawLen)
+	if err != nil {
+		// logging at info since a single incident isn't too worrying (the client could just have
+		// hung up), but if we get a lot of these we'd want to investigate.
+		logger.Info(ctx, "failed to read AgentReconnectingPTYInit length", slog.Error(err))
+		return nil
+	}
+	length := binary.LittleEndian.Uint16(rawLen)
+	data := make([]byte, length)
+	_, err = conn.Read(data)
+	if err != nil {
+		// logging at info since a single incident isn't too worrying (the client could just have
+		// hung up), but if we get a lot of these we'd want to investigate.
+		logger.Info(ctx, "failed to read AgentReconnectingPTYInit", slog.Error(err))
+		return nil
+	}
+	var msg workspacesdk.AgentReconnectingPTYInit
+	err = json.Unmarshal(data, &msg)
+	if err != nil {
+		logger.Warn(ctx, "failed to unmarshal init", slog.F("raw", data))
+		return nil
+	}
+
+	connectionID := uuid.NewString()
+	connLogger := logger.With(slog.F("message_id", msg.ID), slog.F("connection_id", connectionID))
+	connLogger.Debug(ctx, "starting handler")
+
+	defer func() {
+		if err := retErr; err != nil {
+			// If the context is done, we don't want to log this as an error since it's expected.
+			if ctx.Err() != nil {
+				connLogger.Info(ctx, "reconnecting pty failed with attach error (agent closed)", slog.Error(err))
+			} else {
+				connLogger.Error(ctx, "reconnecting pty failed with attach error", slog.Error(err))
+			}
+		}
+		connLogger.Info(ctx, "reconnecting pty connection closed")
+	}()
+
+	var rpty ReconnectingPTY
+	sendConnected := make(chan ReconnectingPTY, 1)
+	// On store, reserve this ID to prevent multiple concurrent new connections.
+	waitReady, ok := s.reconnectingPTYs.LoadOrStore(msg.ID, sendConnected)
+	if ok {
+		close(sendConnected) // Unused.
+		connLogger.Debug(ctx, "connecting to existing reconnecting pty")
+		c, ok := waitReady.(chan ReconnectingPTY)
+		if !ok {
+			return xerrors.Errorf("found invalid type in reconnecting pty map: %T", waitReady)
+		}
+		rpty, ok = <-c
+		if !ok || rpty == nil {
+			return xerrors.Errorf("reconnecting pty closed before connection")
+		}
+		c <- rpty // Put it back for the next reconnect.
+	} else {
+		connLogger.Debug(ctx, "creating new reconnecting pty")
+
+		connected := false
+		defer func() {
+			if !connected && retErr != nil {
+				s.reconnectingPTYs.Delete(msg.ID)
+				close(sendConnected)
+			}
+		}()
+
+		// Empty command will default to the users shell!
+		cmd, err := s.commandCreator.CreateCommand(ctx, msg.Command, nil)
+		if err != nil {
+			s.errorsTotal.WithLabelValues("create_command").Add(1)
+			return xerrors.Errorf("create command: %w", err)
+		}
+
+		rpty = New(ctx,
+			logger.With(slog.F("message_id", msg.ID)),
+			s.commandCreator.Execer,
+			cmd,
+			&Options{
+				Timeout: s.timeout,
+				Metrics: s.errorsTotal,
+			},
+		)
+
+		done := make(chan struct{})
+		go func() {
+			select {
+			case <-done:
+			case <-ctx.Done():
+				rpty.Close(ctx.Err())
+			}
+		}()
+
+		go func() {
+			rpty.Wait()
+			s.reconnectingPTYs.Delete(msg.ID)
+		}()
+
+		connected = true
+		sendConnected <- rpty
+	}
+	return rpty.Attach(ctx, connectionID, conn, msg.Height, msg.Width, connLogger)
+}

agent/stats.go

@@ -2,6 +2,7 @@ package agent
 
 import (
 	"context"
+	"maps"
 	"sync"
 	"time"
 
@@ -32,7 +33,7 @@ type statsDest interface {
 // statsDest (agent API in prod)
 type statsReporter struct {
 	*sync.Cond
-	networkStats *map[netlogtype.Connection]netlogtype.Counts
+	networkStats map[netlogtype.Connection]netlogtype.Counts
 	unreported   bool
 	lastInterval time.Duration
 
@@ -54,8 +55,15 @@ func (s *statsReporter) callback(_, _ time.Time, virtual, _ map[netlogtype.Conne
 	s.L.Lock()
 	defer s.L.Unlock()
 	s.logger.Debug(context.Background(), "got stats callback")
-	s.networkStats = &virtual
-	s.unreported = true
+	// Accumulate stats until they've been reported.
+	if s.unreported && len(s.networkStats) > 0 {
+		for k, v := range virtual {
+			s.networkStats[k] = s.networkStats[k].Add(v)
+		}
+	} else {
+		s.networkStats = maps.Clone(virtual)
+		s.unreported = true
+	}
 	s.Broadcast()
 }
 
@@ -96,9 +104,8 @@ func (s *statsReporter) reportLoop(ctx context.Context, dest statsDest) error {
 		if ctxDone {
 			return nil
 		}
-		networkStats := *s.networkStats
 		s.unreported = false
-		if err = s.reportLocked(ctx, dest, networkStats); err != nil {
+		if err = s.reportLocked(ctx, dest, s.networkStats); err != nil {
 			return xerrors.Errorf("report stats: %w", err)
 		}
 	}

agent/stats_internal_test.go

@@ -1,10 +1,7 @@
 package agent
 
 import (
-	"bytes"
 	"context"
-	"encoding/json"
-	"io"
 	"net/netip"
 	"sync"
 	"testing"
@@ -16,9 +13,6 @@ import (
 
 	"tailscale.com/types/netlogtype"
 
-	"cdr.dev/slog"
-	"cdr.dev/slog/sloggers/slogjson"
-	"cdr.dev/slog/sloggers/slogtest"
 	"github.com/coder/coder/v2/agent/proto"
 	"github.com/coder/coder/v2/testutil"
 )
@@ -26,7 +20,7 @@ import (
 func TestStatsReporter(t *testing.T) {
 	t.Parallel()
 	ctx := testutil.Context(t, testutil.WaitShort)
-	logger := slogtest.Make(t, nil).Leveled(slog.LevelDebug)
+	logger := testutil.Logger(t)
 	fSource := newFakeNetworkStatsSource(ctx, t)
 	fCollector := newFakeCollector(t)
 	fDest := newFakeStatsDest()
@@ -70,7 +64,7 @@ func TestStatsReporter(t *testing.T) {
 	require.Equal(t, netStats, gotNetStats)
 
 	// while we are collecting the stats, send in two new netStats to simulate
-	// what happens if we don't keep up.  Only the latest should be kept.
+	// what happens if we don't keep up.  The stats should be accumulated.
 	netStats0 := map[netlogtype.Connection]netlogtype.Counts{
 		{
 			Proto: ipproto.TCP,
@@ -108,9 +102,21 @@ func TestStatsReporter(t *testing.T) {
 	require.Equal(t, stats, update.Stats)
 	testutil.RequireSendCtx(ctx, t, fDest.resps, &proto.UpdateStatsResponse{ReportInterval: durationpb.New(interval)})
 
-	// second update -- only netStats1 is reported
+	// second update -- netStat0 and netStats1 are accumulated and reported
+	wantNetStats := map[netlogtype.Connection]netlogtype.Counts{
+		{
+			Proto: ipproto.TCP,
+			Src:   netip.MustParseAddrPort("192.168.1.33:4887"),
+			Dst:   netip.MustParseAddrPort("192.168.2.99:9999"),
+		}: {
+			TxPackets: 21,
+			TxBytes:   21,
+			RxPackets: 21,
+			RxBytes:   21,
+		},
+	}
 	gotNetStats = testutil.RequireRecvCtx(ctx, t, fCollector.calls)
-	require.Equal(t, netStats1, gotNetStats)
+	require.Equal(t, wantNetStats, gotNetStats)
 	stats = &proto.Stats{SessionCountJetbrains: 66}
 	testutil.RequireSendCtx(ctx, t, fCollector.stats, stats)
 	update = testutil.RequireRecvCtx(ctx, t, fDest.reqs)
@@ -214,58 +220,3 @@ func newFakeStatsDest() *fakeStatsDest {
 		resps: make(chan *proto.UpdateStatsResponse),
 	}
 }
-
-func Test_logDebouncer(t *testing.T) {
-	t.Parallel()
-
-	var (
-		buf    bytes.Buffer
-		logger = slog.Make(slogjson.Sink(&buf))
-		ctx    = context.Background()
-	)
-
-	debouncer := &logDebouncer{
-		logger:   logger,
-		messages: map[string]time.Time{},
-		interval: time.Minute,
-	}
-
-	fields := map[string]interface{}{
-		"field_1": float64(1),
-		"field_2": "2",
-	}
-
-	debouncer.Error(ctx, "my message", "field_1", 1, "field_2", "2")
-	debouncer.Warn(ctx, "another message", "field_1", 1, "field_2", "2")
-	// Shouldn't log this.
-	debouncer.Warn(ctx, "another message", "field_1", 1, "field_2", "2")
-
-	require.Len(t, debouncer.messages, 2)
-
-	type entry struct {
-		Msg    string                 `json:"msg"`
-		Level  string                 `json:"level"`
-		Fields map[string]interface{} `json:"fields"`
-	}
-
-	assertLog := func(msg string, level string, fields map[string]interface{}) {
-		line, err := buf.ReadString('\n')
-		require.NoError(t, err)
-
-		var e entry
-		err = json.Unmarshal([]byte(line), &e)
-		require.NoError(t, err)
-		require.Equal(t, msg, e.Msg)
-		require.Equal(t, level, e.Level)
-		require.Equal(t, fields, e.Fields)
-	}
-	assertLog("my message", "ERROR", fields)
-	assertLog("another message", "WARN", fields)
-
-	debouncer.messages["another message"] = time.Now().Add(-2 * time.Minute)
-	debouncer.Warn(ctx, "another message", "field_1", 1, "field_2", "2")
-	assertLog("another message", "WARN", fields)
-	// Assert nothing else was written.
-	_, err := buf.ReadString('\n')
-	require.ErrorIs(t, err, io.EOF)
-}

apiversion/apiversion.go

@@ -26,7 +26,7 @@ type APIVersion struct {
 }
 
 func (v *APIVersion) WithBackwardCompat(majs ...int) *APIVersion {
-	v.additionalMajors = append(v.additionalMajors, majs[:]...)
+	v.additionalMajors = append(v.additionalMajors, majs...)
 	return v
 }
 

apiversion/doc.go

@@ -0,0 +1,26 @@
+// Package apiversion provides an API version type that can be used to validate
+// compatibility between two API versions.
+//
+// NOTE: API VERSIONS ARE NOT SEMANTIC VERSIONS.
+//
+// API versions are represented as major.minor where major and minor are both
+// positive integers.
+//
+// API versions are not directly tied to a specific release of the software.
+// Instead, they are used to represent the capabilities of the server. For
+// example, a server that supports API version 1.2 should be able to handle
+// requests from clients that support API version 1.0, 1.1, or 1.2.
+// However, a server that supports API version 2.0 is not required to handle
+// requests from clients that support API version 1.x.
+// Clients may need to negotiate with the server to determine the highest
+// supported API version.
+//
+// When making a change to the API, use the following rules to determine the
+// next API version:
+//  1. If the change is backward-compatible, increment the minor version.
+//     Examples of backward-compatible changes include adding new fields to
+//     a response or adding new endpoints.
+//  2. If the change is not backward-compatible, increment the major version.
+//     Examples of non-backward-compatible changes include removing or renaming
+//     fields.
+package apiversion

archive/archive.go

@@ -1,4 +1,4 @@
-package coderd
+package archive
 
 import (
 	"archive/tar"
@@ -10,21 +10,22 @@ import (
 	"strings"
 )
 
-func CreateTarFromZip(zipReader *zip.Reader) ([]byte, error) {
+// CreateTarFromZip converts the given zipReader to a tar archive.
+func CreateTarFromZip(zipReader *zip.Reader, maxSize int64) ([]byte, error) {
 	var tarBuffer bytes.Buffer
-	err := writeTarArchive(&tarBuffer, zipReader)
+	err := writeTarArchive(&tarBuffer, zipReader, maxSize)
 	if err != nil {
 		return nil, err
 	}
 	return tarBuffer.Bytes(), nil
 }
 
-func writeTarArchive(w io.Writer, zipReader *zip.Reader) error {
+func writeTarArchive(w io.Writer, zipReader *zip.Reader, maxSize int64) error {
 	tarWriter := tar.NewWriter(w)
 	defer tarWriter.Close()
 
 	for _, file := range zipReader.File {
-		err := processFileInZipArchive(file, tarWriter)
+		err := processFileInZipArchive(file, tarWriter, maxSize)
 		if err != nil {
 			return err
 		}
@@ -32,7 +33,7 @@ func writeTarArchive(w io.Writer, zipReader *zip.Reader) error {
 	return nil
 }
 
-func processFileInZipArchive(file *zip.File, tarWriter *tar.Writer) error {
+func processFileInZipArchive(file *zip.File, tarWriter *tar.Writer, maxSize int64) error {
 	fileReader, err := file.Open()
 	if err != nil {
 		return err
@@ -52,7 +53,7 @@ func processFileInZipArchive(file *zip.File, tarWriter *tar.Writer) error {
 		return err
 	}
 
-	n, err := io.CopyN(tarWriter, fileReader, httpFileMaxBytes)
+	n, err := io.CopyN(tarWriter, fileReader, maxSize)
 	log.Println(file.Name, n, err)
 	if errors.Is(err, io.EOF) {
 		err = nil
@@ -60,16 +61,18 @@ func processFileInZipArchive(file *zip.File, tarWriter *tar.Writer) error {
 	return err
 }
 
-func CreateZipFromTar(tarReader *tar.Reader) ([]byte, error) {
+// CreateZipFromTar converts the given tarReader to a zip archive.
+func CreateZipFromTar(tarReader *tar.Reader, maxSize int64) ([]byte, error) {
 	var zipBuffer bytes.Buffer
-	err := WriteZipArchive(&zipBuffer, tarReader)
+	err := WriteZip(&zipBuffer, tarReader, maxSize)
 	if err != nil {
 		return nil, err
 	}
 	return zipBuffer.Bytes(), nil
 }
 
-func WriteZipArchive(w io.Writer, tarReader *tar.Reader) error {
+// WriteZip writes the given tarReader to w.
+func WriteZip(w io.Writer, tarReader *tar.Reader, maxSize int64) error {
 	zipWriter := zip.NewWriter(w)
 	defer zipWriter.Close()
 
@@ -100,7 +103,7 @@ func WriteZipArchive(w io.Writer, tarReader *tar.Reader) error {
 			return err
 		}
 
-		_, err = io.CopyN(zipEntry, tarReader, httpFileMaxBytes)
+		_, err = io.CopyN(zipEntry, tarReader, maxSize)
 		if errors.Is(err, io.EOF) {
 			err = nil
 		}

archive/archive_test.go

@@ -1,10 +1,9 @@
-package coderd_test
+package archive_test
 
 import (
 	"archive/tar"
 	"archive/zip"
 	"bytes"
-	"io"
 	"io/fs"
 	"os"
 	"os/exec"
@@ -12,13 +11,12 @@ import (
 	"runtime"
 	"strings"
 	"testing"
-	"time"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-	"golang.org/x/xerrors"
 
-	"github.com/coder/coder/v2/coderd"
+	"github.com/coder/coder/v2/archive"
+	"github.com/coder/coder/v2/archive/archivetest"
 	"github.com/coder/coder/v2/testutil"
 )
 
@@ -30,18 +28,17 @@ func TestCreateTarFromZip(t *testing.T) {
 
 	// Read a zip file we prepared earlier
 	ctx := testutil.Context(t, testutil.WaitShort)
-	zipBytes, err := os.ReadFile(filepath.Join("testdata", "test.zip"))
-	require.NoError(t, err, "failed to read sample zip file")
+	zipBytes := archivetest.TestZipFileBytes()
 	// Assert invariant
-	assertSampleZipFile(t, zipBytes)
+	archivetest.AssertSampleZipFile(t, zipBytes)
 
 	zr, err := zip.NewReader(bytes.NewReader(zipBytes), int64(len(zipBytes)))
 	require.NoError(t, err, "failed to parse sample zip file")
 
-	tarBytes, err := coderd.CreateTarFromZip(zr)
+	tarBytes, err := archive.CreateTarFromZip(zr, int64(len(zipBytes)))
 	require.NoError(t, err, "failed to convert zip to tar")
 
-	assertSampleTarFile(t, tarBytes)
+	archivetest.AssertSampleTarFile(t, tarBytes)
 
 	tempDir := t.TempDir()
 	tempFilePath := filepath.Join(tempDir, "test.tar")
@@ -60,14 +57,13 @@ func TestCreateZipFromTar(t *testing.T) {
 	}
 	t.Run("OK", func(t *testing.T) {
 		t.Parallel()
-		tarBytes, err := os.ReadFile(filepath.Join(".", "testdata", "test.tar"))
-		require.NoError(t, err, "failed to read sample tar file")
+		tarBytes := archivetest.TestTarFileBytes()
 
 		tr := tar.NewReader(bytes.NewReader(tarBytes))
-		zipBytes, err := coderd.CreateZipFromTar(tr)
+		zipBytes, err := archive.CreateZipFromTar(tr, int64(len(tarBytes)))
 		require.NoError(t, err)
 
-		assertSampleZipFile(t, zipBytes)
+		archivetest.AssertSampleZipFile(t, zipBytes)
 
 		tempDir := t.TempDir()
 		tempFilePath := filepath.Join(tempDir, "test.zip")
@@ -99,7 +95,7 @@ func TestCreateZipFromTar(t *testing.T) {
 
 		// When: we convert this to a zip
 		tr := tar.NewReader(&tarBytes)
-		zipBytes, err := coderd.CreateZipFromTar(tr)
+		zipBytes, err := archive.CreateZipFromTar(tr, int64(tarBytes.Len()))
 		require.NoError(t, err)
 
 		// Then: the resulting zip should contain a corresponding directory
@@ -133,7 +129,7 @@ func assertExtractedFiles(t *testing.T, dir string, checkModePerm bool) {
 			if checkModePerm {
 				assert.Equal(t, fs.ModePerm&0o755, stat.Mode().Perm(), "expected mode 0755 on directory")
 			}
-			assert.Equal(t, archiveRefTime(t).UTC(), stat.ModTime().UTC(), "unexpected modtime of %q", path)
+			assert.Equal(t, archivetest.ArchiveRefTime(t).UTC(), stat.ModTime().UTC(), "unexpected modtime of %q", path)
 		case "/test/hello.txt":
 			stat, err := os.Stat(path)
 			assert.NoError(t, err, "failed to stat path %q", path)
@@ -168,84 +164,3 @@ func assertExtractedFiles(t *testing.T, dir string, checkModePerm bool) {
 		return nil
 	})
 }
-
-func assertSampleTarFile(t *testing.T, tarBytes []byte) {
-	t.Helper()
-
-	tr := tar.NewReader(bytes.NewReader(tarBytes))
-	for {
-		hdr, err := tr.Next()
-		if err != nil {
-			if err == io.EOF {
-				return
-			}
-			require.NoError(t, err)
-		}
-
-		// Note: ignoring timezones here.
-		require.Equal(t, archiveRefTime(t).UTC(), hdr.ModTime.UTC())
-
-		switch hdr.Name {
-		case "test/":
-			require.Equal(t, hdr.Typeflag, byte(tar.TypeDir))
-		case "test/hello.txt":
-			require.Equal(t, hdr.Typeflag, byte(tar.TypeReg))
-			bs, err := io.ReadAll(tr)
-			if err != nil && !xerrors.Is(err, io.EOF) {
-				require.NoError(t, err)
-			}
-			require.Equal(t, "hello", string(bs))
-		case "test/dir/":
-			require.Equal(t, hdr.Typeflag, byte(tar.TypeDir))
-		case "test/dir/world.txt":
-			require.Equal(t, hdr.Typeflag, byte(tar.TypeReg))
-			bs, err := io.ReadAll(tr)
-			if err != nil && !xerrors.Is(err, io.EOF) {
-				require.NoError(t, err)
-			}
-			require.Equal(t, "world", string(bs))
-		default:
-			require.Failf(t, "unexpected file in tar", hdr.Name)
-		}
-	}
-}
-
-func assertSampleZipFile(t *testing.T, zipBytes []byte) {
-	t.Helper()
-
-	zr, err := zip.NewReader(bytes.NewReader(zipBytes), int64(len(zipBytes)))
-	require.NoError(t, err)
-
-	for _, f := range zr.File {
-		// Note: ignoring timezones here.
-		require.Equal(t, archiveRefTime(t).UTC(), f.Modified.UTC())
-		switch f.Name {
-		case "test/", "test/dir/":
-			// directory
-		case "test/hello.txt":
-			rc, err := f.Open()
-			require.NoError(t, err)
-			bs, err := io.ReadAll(rc)
-			_ = rc.Close()
-			require.NoError(t, err)
-			require.Equal(t, "hello", string(bs))
-		case "test/dir/world.txt":
-			rc, err := f.Open()
-			require.NoError(t, err)
-			bs, err := io.ReadAll(rc)
-			_ = rc.Close()
-			require.NoError(t, err)
-			require.Equal(t, "world", string(bs))
-		default:
-			require.Failf(t, "unexpected file in zip", f.Name)
-		}
-	}
-}
-
-// archiveRefTime is the Go reference time. The contents of the sample tar and zip files
-// in testdata/ all have their modtimes set to the below in some timezone.
-func archiveRefTime(t *testing.T) time.Time {
-	locMST, err := time.LoadLocation("MST")
-	require.NoError(t, err, "failed to load MST timezone")
-	return time.Date(2006, 1, 2, 3, 4, 5, 0, locMST)
-}

archive/archivetest/archivetest.go

@@ -0,0 +1,113 @@
+package archivetest
+
+import (
+	"archive/tar"
+	"archive/zip"
+	"bytes"
+	_ "embed"
+	"io"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+	"golang.org/x/xerrors"
+)
+
+//go:embed testdata/test.tar
+var testTarFileBytes []byte
+
+//go:embed testdata/test.zip
+var testZipFileBytes []byte
+
+// TestTarFileBytes returns the content of testdata/test.tar
+func TestTarFileBytes() []byte {
+	return append([]byte{}, testTarFileBytes...)
+}
+
+// TestZipFileBytes returns the content of testdata/test.zip
+func TestZipFileBytes() []byte {
+	return append([]byte{}, testZipFileBytes...)
+}
+
+// AssertSampleTarfile compares the content of tarBytes against testdata/test.tar.
+func AssertSampleTarFile(t *testing.T, tarBytes []byte) {
+	t.Helper()
+
+	tr := tar.NewReader(bytes.NewReader(tarBytes))
+	for {
+		hdr, err := tr.Next()
+		if err != nil {
+			if err == io.EOF {
+				return
+			}
+			require.NoError(t, err)
+		}
+
+		// Note: ignoring timezones here.
+		require.Equal(t, ArchiveRefTime(t).UTC(), hdr.ModTime.UTC())
+
+		switch hdr.Name {
+		case "test/":
+			require.Equal(t, hdr.Typeflag, byte(tar.TypeDir))
+		case "test/hello.txt":
+			require.Equal(t, hdr.Typeflag, byte(tar.TypeReg))
+			bs, err := io.ReadAll(tr)
+			if err != nil && !xerrors.Is(err, io.EOF) {
+				require.NoError(t, err)
+			}
+			require.Equal(t, "hello", string(bs))
+		case "test/dir/":
+			require.Equal(t, hdr.Typeflag, byte(tar.TypeDir))
+		case "test/dir/world.txt":
+			require.Equal(t, hdr.Typeflag, byte(tar.TypeReg))
+			bs, err := io.ReadAll(tr)
+			if err != nil && !xerrors.Is(err, io.EOF) {
+				require.NoError(t, err)
+			}
+			require.Equal(t, "world", string(bs))
+		default:
+			require.Failf(t, "unexpected file in tar", hdr.Name)
+		}
+	}
+}
+
+// AssertSampleZipFile compares the content of zipBytes against testdata/test.zip.
+func AssertSampleZipFile(t *testing.T, zipBytes []byte) {
+	t.Helper()
+
+	zr, err := zip.NewReader(bytes.NewReader(zipBytes), int64(len(zipBytes)))
+	require.NoError(t, err)
+
+	for _, f := range zr.File {
+		// Note: ignoring timezones here.
+		require.Equal(t, ArchiveRefTime(t).UTC(), f.Modified.UTC())
+		switch f.Name {
+		case "test/", "test/dir/":
+			// directory
+		case "test/hello.txt":
+			rc, err := f.Open()
+			require.NoError(t, err)
+			bs, err := io.ReadAll(rc)
+			_ = rc.Close()
+			require.NoError(t, err)
+			require.Equal(t, "hello", string(bs))
+		case "test/dir/world.txt":
+			rc, err := f.Open()
+			require.NoError(t, err)
+			bs, err := io.ReadAll(rc)
+			_ = rc.Close()
+			require.NoError(t, err)
+			require.Equal(t, "world", string(bs))
+		default:
+			require.Failf(t, "unexpected file in zip", f.Name)
+		}
+	}
+}
+
+// archiveRefTime is the Go reference time. The contents of the sample tar and zip files
+// in testdata/ all have their modtimes set to the below in some timezone.
+func ArchiveRefTime(t *testing.T) time.Time {
+	locMST, err := time.LoadLocation("MST")
+	require.NoError(t, err, "failed to load MST timezone")
+	return time.Date(2006, 1, 2, 3, 4, 5, 0, locMST)
+}

buildinfo/buildinfo.go

@@ -24,6 +24,9 @@ var (
 	// Updated by buildinfo_slim.go on start.
 	slim bool
 
+	// Updated by buildinfo_site.go on start.
+	site bool
+
 	// Injected with ldflags at build, see scripts/build_go.sh
 	tag  string
 	agpl string // either "true" or "false", ldflags does not support bools
@@ -95,6 +98,11 @@ func IsSlim() bool {
 	return slim
 }
 
+// HasSite returns true if the frontend is embedded in the build.
+func HasSite() bool {
+	return site
+}
+
 // IsAGPL returns true if this is an AGPL build.
 func IsAGPL() bool {
 	return strings.Contains(agpl, "t")

buildinfo/buildinfo_site.go

@@ -0,0 +1,7 @@
+//go:build embed
+
+package buildinfo
+
+func init() {
+	site = true
+}

cli/agent.go

@@ -12,7 +12,6 @@ import (
 	"runtime"
 	"strconv"
 	"strings"
-	"sync"
 	"time"
 
 	"cloud.google.com/go/compute/metadata"
@@ -26,10 +25,11 @@ import (
 	"cdr.dev/slog/sloggers/slogjson"
 	"cdr.dev/slog/sloggers/slogstackdriver"
 	"github.com/coder/coder/v2/agent"
-	"github.com/coder/coder/v2/agent/agentproc"
+	"github.com/coder/coder/v2/agent/agentexec"
 	"github.com/coder/coder/v2/agent/agentssh"
 	"github.com/coder/coder/v2/agent/reaper"
 	"github.com/coder/coder/v2/buildinfo"
+	"github.com/coder/coder/v2/cli/clilog"
 	"github.com/coder/coder/v2/codersdk"
 	"github.com/coder/coder/v2/codersdk/agentsdk"
 	"github.com/coder/serpent"
@@ -50,6 +50,8 @@ func (r *RootCmd) workspaceAgent() *serpent.Command {
 		slogJSONPath        string
 		slogStackdriverPath string
 		blockFileTransfer   bool
+		agentHeaderCommand  string
+		agentHeader         []string
 	)
 	cmd := &serpent.Command{
 		Use:   "agent",
@@ -108,7 +110,7 @@ func (r *RootCmd) workspaceAgent() *serpent.Command {
 			// Spawn a reaper so that we don't accumulate a ton
 			// of zombie processes.
 			if reaper.IsInitProcess() && !noReap && isLinux {
-				logWriter := &lumberjackWriteCloseFixer{w: &lumberjack.Logger{
+				logWriter := &clilog.LumberjackWriteCloseFixer{Writer: &lumberjack.Logger{
 					Filename: filepath.Join(logDir, "coder-agent-init.log"),
 					MaxSize:  5, // MB
 					// Without this, rotated logs will never be deleted.
@@ -151,7 +153,7 @@ func (r *RootCmd) workspaceAgent() *serpent.Command {
 			// reaper.
 			go DumpHandler(ctx, "agent")
 
-			logWriter := &lumberjackWriteCloseFixer{w: &lumberjack.Logger{
+			logWriter := &clilog.LumberjackWriteCloseFixer{Writer: &lumberjack.Logger{
 				Filename: filepath.Join(logDir, "coder-agent.log"),
 				MaxSize:  5, // MB
 				// Per customer incident on November 17th, 2023, its helpful
@@ -169,6 +171,7 @@ func (r *RootCmd) workspaceAgent() *serpent.Command {
 				slog.F("auth", auth),
 				slog.F("version", version),
 			)
+
 			client := agentsdk.New(r.agentURL)
 			client.SDK.SetLogger(logger)
 			// Set a reasonable timeout so requests can't hang forever!
@@ -176,6 +179,14 @@ func (r *RootCmd) workspaceAgent() *serpent.Command {
 			// with large payloads can take a bit. e.g. startup scripts
 			// may take a while to insert.
 			client.SDK.HTTPClient.Timeout = 30 * time.Second
+			// Attach header transport so we process --agent-header and
+			// --agent-header-command flags
+			headerTransport, err := headerTransport(ctx, r.agentURL, agentHeader, agentHeaderCommand)
+			if err != nil {
+				return xerrors.Errorf("configure header transport: %w", err)
+			}
+			headerTransport.Transport = client.SDK.HTTPClient.Transport
+			client.SDK.HTTPClient.Transport = headerTransport
 
 			// Enable pprof handler
 			// This prevents the pprof import from being accidentally deleted.
@@ -282,11 +293,25 @@ func (r *RootCmd) workspaceAgent() *serpent.Command {
 			environmentVariables := map[string]string{
 				"GIT_ASKPASS": executablePath,
 			}
-			if v, ok := os.LookupEnv(agent.EnvProcPrioMgmt); ok {
-				environmentVariables[agent.EnvProcPrioMgmt] = v
+
+			enabled := os.Getenv(agentexec.EnvProcPrioMgmt)
+			if enabled != "" && runtime.GOOS == "linux" {
+				logger.Info(ctx, "process priority management enabled",
+					slog.F("env_var", agentexec.EnvProcPrioMgmt),
+					slog.F("enabled", enabled),
+					slog.F("os", runtime.GOOS),
+				)
+			} else {
+				logger.Info(ctx, "process priority management not enabled (linux-only) ",
+					slog.F("env_var", agentexec.EnvProcPrioMgmt),
+					slog.F("enabled", enabled),
+					slog.F("os", runtime.GOOS),
+				)
 			}
-			if v, ok := os.LookupEnv(agent.EnvProcOOMScore); ok {
-				environmentVariables[agent.EnvProcOOMScore] = v
+
+			execer, err := agentexec.NewExecer()
+			if err != nil {
+				return xerrors.Errorf("create agent execer: %w", err)
 			}
 
 			agnt := agent.New(agent.Options{
@@ -312,12 +337,8 @@ func (r *RootCmd) workspaceAgent() *serpent.Command {
 				Subsystems:           subsystems,
 
 				PrometheusRegistry: prometheusRegistry,
-				Syscaller:          agentproc.NewSyscaller(),
-				// Intentionally set this to nil. It's mainly used
-				// for testing.
-				ModifiedProcesses: nil,
-
-				BlockFileTransfer: blockFileTransfer,
+				BlockFileTransfer:  blockFileTransfer,
+				Execer:             execer,
 			})
 
 			promHandler := agent.PrometheusMetricsHandler(prometheusRegistry, logger)
@@ -361,6 +382,18 @@ func (r *RootCmd) workspaceAgent() *serpent.Command {
 			Value:       serpent.StringOf(&pprofAddress),
 			Description: "The address to serve pprof.",
 		},
+		{
+			Flag:        "agent-header-command",
+			Env:         "CODER_AGENT_HEADER_COMMAND",
+			Value:       serpent.StringOf(&agentHeaderCommand),
+			Description: "An external command that outputs additional HTTP headers added to all requests. The command must output each header as `key=value` on its own line.",
+		},
+		{
+			Flag:        "agent-header",
+			Env:         "CODER_AGENT_HEADER",
+			Value:       serpent.StringArrayOf(&agentHeader),
+			Description: "Additional HTTP headers added to all requests. Provide as " + `key=value` + ". Can be specified multiple times.",
+		},
 		{
 			Flag: "no-reap",
 
@@ -456,33 +489,6 @@ func ServeHandler(ctx context.Context, logger slog.Logger, handler http.Handler,
 	}
 }
 
-// lumberjackWriteCloseFixer is a wrapper around an io.WriteCloser that
-// prevents writes after Close. This is necessary because lumberjack
-// re-opens the file on Write.
-type lumberjackWriteCloseFixer struct {
-	w      io.WriteCloser
-	mu     sync.Mutex // Protects following.
-	closed bool
-}
-
-func (c *lumberjackWriteCloseFixer) Close() error {
-	c.mu.Lock()
-	defer c.mu.Unlock()
-
-	c.closed = true
-	return c.w.Close()
-}
-
-func (c *lumberjackWriteCloseFixer) Write(p []byte) (int, error) {
-	c.mu.Lock()
-	defer c.mu.Unlock()
-
-	if c.closed {
-		return 0, io.ErrClosedPipe
-	}
-	return c.w.Write(p)
-}
-
 // extractPort handles different url strings.
 // - localhost:6060
 // - http://localhost:6060