chore: add cherry-picks for patch 2.18.2 (#16061)

Co-authored-by: Cian Johnston <cian@coder.com>
Co-authored-by: Joobi S B <joobisb@gmail.com>

.devcontainer/Dockerfile

@@ -1,83 +0,0 @@
-FROM ubuntu
-SHELL ["/bin/bash", "-o", "pipefail", "-c"]
-
-ENV EDITOR=vim
-
-RUN apt-get update && apt-get upgrade --yes
-
-RUN apt-get install --yes \
-	ca-certificates \
-	bash-completion \
-	build-essential \
-	curl \
-	cmake \
-	direnv \
-	emacs-nox \
-	gnupg \
-	htop \
-	jq \
-	less \
-	lsb-release \
-	lsof \
-	man-db \
-	nano \
-	neovim \
-	ssl-cert \
-	sudo \
-	unzip \
-	xz-utils \
-	zip
-
-# configure locales to UTF8
-RUN apt-get install locales && locale-gen en_US.UTF-8
-ENV LANG='en_US.UTF-8' LANGUAGE='en_US:en' LC_ALL='en_US.UTF-8'
-
-# configure direnv
-RUN direnv hook bash >> $HOME/.bashrc
-
-# install nix
-RUN sh <(curl -L https://nixos.org/nix/install) --daemon
-
-RUN mkdir -p $HOME/.config/nix $HOME/.config/nixpkgs \
-	&& echo 'sandbox = false' >> $HOME/.config/nix/nix.conf \
-	&& echo '{ allowUnfree = true; }' >> $HOME/.config/nixpkgs/config.nix \
-	&& echo '. $HOME/.nix-profile/etc/profile.d/nix.sh' >> $HOME/.bashrc
-
-
-# install docker and configure daemon to use vfs as GitHub codespaces requires vfs
-# https://github.com/moby/moby/issues/13742#issuecomment-725197223
-RUN mkdir -p /etc/apt/keyrings \
-	&& curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg \
-	&& echo \
-	"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \
-	$(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null \
-	&& apt-get update \
-	&& apt-get install --yes docker-ce docker-ce-cli containerd.io docker-compose-plugin \
-	&& mkdir -p /etc/docker \
-	&& echo '{"cgroup-parent":"/actions_job","storage-driver":"vfs"}' >> /etc/docker/daemon.json
-
-# install golang and language tooling
-ENV GO_VERSION=1.20
-ENV GOPATH=$HOME/go-packages
-ENV GOROOT=$HOME/go
-ENV PATH=$GOROOT/bin:$GOPATH/bin:$PATH
-RUN curl -fsSL https://dl.google.com/go/go$GO_VERSION.linux-amd64.tar.gz | tar xzs
-RUN echo 'export PATH=$GOPATH/bin:$PATH' >> $HOME/.bashrc
-
-RUN bash -c ". $HOME/.bashrc \
-	go install -v golang.org/x/tools/gopls@latest \
-	&& go install -v mvdan.cc/sh/v3/cmd/shfmt@latest \
-	&& go install -v github.com/mikefarah/yq/v4@v4.30.6 \
-	"
-
-# install nodejs
-RUN bash -c "$(curl -fsSL https://deb.nodesource.com/setup_14.x)" \
-	&& apt-get install -y nodejs
-
-# install zstd
-RUN bash -c "$(curl -fsSL https://raw.githubusercontent.com/horta/zstd.install/main/install)"
-
-# install nfpm
-RUN echo 'deb [trusted=yes] https://repo.goreleaser.com/apt/ /' | sudo tee /etc/apt/sources.list.d/goreleaser.list \
-	&& apt update \
-	&& apt install nfpm

.devcontainer/devcontainer.json

@@ -1,24 +1,13 @@
-// For format details, see https://aka.ms/devcontainer.json
 {
-  "name": "Development environments on your infrastructure",
+	"name": "Development environments on your infrastructure",
+	"image": "codercom/oss-dogfood:latest",
 
-  // Sets the run context to one level up instead of the .devcontainer folder.
-  "context": ".",
-
-  // Update the 'dockerFile' property if you aren't using the standard 'Dockerfile' filename.
-  "dockerFile": "Dockerfile",
-
-  // Use 'forwardPorts' to make a list of ports inside the container available locally.
-  // "forwardPorts": [],
-
-  "postStartCommand": "dockerd",
-
-  // privileged is required by GitHub codespaces - https://github.com/microsoft/vscode-dev-containers/issues/727
-  "runArgs": [
-    "--cap-add=SYS_PTRACE",
-    "--security-opt",
-    "seccomp=unconfined",
-    "--privileged",
-    "--init"
-  ]
+	"features": {
+		// See all possible options here https://github.com/devcontainers/features/tree/main/src/docker-in-docker
+		"ghcr.io/devcontainers/features/docker-in-docker:2": {
+			"moby": "false"
+		}
+	},
+	// SYS_PTRACE to enable go debugging
+	"runArgs": ["--cap-add=SYS_PTRACE"]
 }

.dockerignore

@@ -0,0 +1,6 @@
+# Ignore all files and folders
+**
+
+# Include flake.nix and flake.lock
+!flake.nix
+!flake.lock

.editorconfig

@@ -7,7 +7,7 @@ trim_trailing_whitespace = true
 insert_final_newline = true
 indent_style = tab
 
-[*.{md,json,yaml,yml,tf,tfvars,nix}]
+[*.{yaml,yml,tf,tfvars,nix}]
 indent_style = space
 indent_size = 2
 

.git-blame-ignore-revs

@@ -0,0 +1,7 @@
+# If you would like `git blame` to ignore commits from this file, run...
+#   git config blame.ignoreRevsFile .git-blame-ignore-revs
+
+# chore: format code with semicolons when using prettier (#9555)
+988c9af0153561397686c119da9d1336d2433fdd
+# chore: use tabs for prettier and biome (#14283)
+95a7c0c4f087744a22c2e88dd3c5d30024d5fb02

.gitattributes

@@ -1,11 +1,17 @@
 # Generated files
 coderd/apidoc/docs.go linguist-generated=true
+docs/reference/api/*.md linguist-generated=true
+docs/reference/cli/*.md linguist-generated=true
 coderd/apidoc/swagger.json linguist-generated=true
 coderd/database/dump.sql linguist-generated=true
 peerbroker/proto/*.go linguist-generated=true
 provisionerd/proto/*.go linguist-generated=true
+provisionerd/proto/version.go linguist-generated=false
 provisionersdk/proto/*.go linguist-generated=true
 *.tfplan.json linguist-generated=true
 *.tfstate.json linguist-generated=true
 *.tfstate.dot linguist-generated=true
 *.tfplan.dot linguist-generated=true
+site/e2e/provisionerGenerated.ts linguist-generated=true
+site/src/api/typesGenerated.ts linguist-generated=true
+site/src/pages/SetupPage/countries.tsx linguist-generated=true

.github/.linkspector.yml

@@ -0,0 +1,22 @@
+dirs:
+  - docs
+excludedDirs:
+  # Downstream bug in linkspector means large markdown files fail to parse
+  # but these are autogenerated and shouldn't need checking
+  - docs/reference
+  # Older changelogs may contain broken links
+  - docs/changelogs
+ignorePatterns:
+  - pattern: "localhost"
+  - pattern: "example.com"
+  - pattern: "mailto:"
+  - pattern: "127.0.0.1"
+  - pattern: "0.0.0.0"
+  - pattern: "JFROG_URL"
+  - pattern: "coder.company.org"
+  # These real sites were blocking the linkspector action / GitHub runner IPs(?)
+  - pattern: "i.imgur.com"
+  - pattern: "code.visualstudio.com"
+  - pattern: "www.emacswiki.org"
+aliveStatusCodes:
+  - 200

.github/CODEOWNERS

@@ -1,3 +0,0 @@
-docs/ @coder/docs
-README.md @coder/docs
-ADOPTERS.md @coder/docs

.github/actions/setup-go/action.yaml

@@ -0,0 +1,24 @@
+name: "Setup Go"
+description: |
+  Sets up the Go environment for tests, builds, etc.
+inputs:
+  version:
+    description: "The Go version to use."
+    default: "1.22.8"
+runs:
+  using: "composite"
+  steps:
+    - name: Setup Go
+      uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
+      with:
+        go-version: ${{ inputs.version }}
+
+    - name: Install gotestsum
+      shell: bash
+      run: go install gotest.tools/gotestsum@latest
+
+    # It isn't necessary that we ever do this, but it helps
+    # separate the "setup" from the "run" times.
+    - name: go mod download
+      shell: bash
+      run: go mod download -x

.github/actions/setup-node/action.yaml

@@ -0,0 +1,31 @@
+name: "Setup Node"
+description: |
+  Sets up the node environment for tests, builds, etc.
+inputs:
+  directory:
+    description: |
+      The directory to run the setup in.
+    required: false
+    default: "site"
+runs:
+  using: "composite"
+  steps:
+    - name: Install pnpm
+      uses: pnpm/action-setup@fe02b34f77f8bc703788d5817da081398fad5dd2 # v4.0.0
+
+    - name: Setup Node
+      uses: actions/setup-node@0a44ba7841725637a19e28fa30b79a866c81b0a6 # v4.0.4
+      with:
+        node-version: 20.16.0
+        # See https://github.com/actions/setup-node#caching-global-packages-data
+        cache: "pnpm"
+        cache-dependency-path: ${{ inputs.directory }}/pnpm-lock.yaml
+
+    - name: Install root node_modules
+      shell: bash
+      run: ./scripts/pnpm_install.sh
+
+    - name: Install node_modules
+      shell: bash
+      run: ../scripts/pnpm_install.sh
+      working-directory: ${{ inputs.directory }}

.github/actions/setup-sqlc/action.yaml

@@ -0,0 +1,10 @@
+name: Setup sqlc
+description: |
+  Sets up the sqlc environment for tests, builds, etc.
+runs:
+  using: "composite"
+  steps:
+    - name: Setup sqlc
+      uses: sqlc-dev/setup-sqlc@c0209b9199cd1cce6a14fc27cabcec491b651761 # v4.0.0
+      with:
+        sqlc-version: "1.25.0"

.github/actions/setup-tf/action.yaml

@@ -0,0 +1,11 @@
+name: "Setup Terraform"
+description: |
+  Sets up Terraform for tests, builds, etc.
+runs:
+  using: "composite"
+  steps:
+    - name: Install Terraform
+      uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd # v3.1.2
+      with:
+        terraform_version: 1.9.8
+        terraform_wrapper: false

.github/actions/upload-datadog/action.yaml

@@ -0,0 +1,28 @@
+name: Upload tests to datadog
+description: |
+  Uploads the test results to datadog.
+inputs:
+  api-key:
+    description: "Datadog API key"
+    required: true
+runs:
+  using: "composite"
+  steps:
+    - shell: bash
+      run: |
+        owner=${{ github.repository_owner	 }}
+        echo "owner: $owner"
+        if [[  $owner != "coder" ]]; then
+          echo "Not a pull request from the main repo, skipping..."
+          exit 0
+        fi
+        if [[ -z "${{ inputs.api-key }}" ]]; then
+          # This can happen for dependabot.
+          echo "No API key provided, skipping..."
+          exit 0
+        fi
+        npm install -g @datadog/datadog-ci@2.21.0
+        datadog-ci junit upload --service coder ./gotests.xml \
+          --tags os:${{runner.os}} --tags runner_name:${{runner.name}}
+      env:
+        DATADOG_API_KEY: ${{ inputs.api-key }}

.github/codecov.yml

@@ -1,43 +0,0 @@
-codecov:
-  require_ci_to_pass: false
-  notify:
-    after_n_builds: 5
-
-comment: false
-
-github_checks:
-  annotations: false
-
-coverage:
-  range: 50..75
-  round: down
-  precision: 2
-  status:
-    patch:
-      default:
-        informational: yes
-    project:
-      default:
-        target: 65%
-        informational: true
-
-ignore:
-  # This is generated code.
-  - coderd/database/models.go
-  - coderd/database/queries.sql.go
-  - coderd/database/databasefake
-  # These are generated or don't require tests.
-  - cmd
-  - coderd/tunnel
-  - coderd/database/dump
-  - coderd/database/postgres
-  - peerbroker/proto
-  - provisionerd/proto
-  - provisionersdk/proto
-  - scripts
-  - site/.storybook
-  - rules.go
-  # Packages used for writing tests.
-  - cli/clitest
-  - coderd/coderdtest
-  - pty/ptytest

.github/dependabot.yaml

@@ -3,12 +3,12 @@ updates:
   - package-ecosystem: "github-actions"
     directory: "/"
     schedule:
-      interval: "monthly"
+      interval: "weekly"
       time: "06:00"
       timezone: "America/Chicago"
     labels: []
     commit-message:
-      prefix: "chore"
+      prefix: "ci"
     ignore:
       # These actions deliver the latest versions by updating the major
       # release tag, so ignore minor and patch versions
@@ -24,51 +24,97 @@ updates:
         update-types:
           - version-update:semver-minor
           - version-update:semver-patch
+    groups:
+      github-actions:
+        patterns:
+          - "*"
 
   - package-ecosystem: "gomod"
     directory: "/"
     schedule:
-      interval: "monthly"
+      interval: "weekly"
+      time: "06:00"
+      timezone: "America/Chicago"
+    commit-message:
+      prefix: "chore"
+    labels: []
+    open-pull-requests-limit: 15
+    groups:
+      x:
+        patterns:
+          - "golang.org/x/*"
+    ignore:
+      # Ignore patch updates for all dependencies
+      - dependency-name: "*"
+        update-types:
+          - version-update:semver-patch
+
+  # Update our Dockerfile.
+  - package-ecosystem: "docker"
+    directories:
+      - "/dogfood/contents"
+      - "/scripts"
+      - "/examples/templates/docker/build"
+      - "/examples/parameters/build"
+      - "/scaletest/templates/scaletest-runner"
+      - "/scripts/ironbank"
+    schedule:
+      interval: "weekly"
       time: "06:00"
       timezone: "America/Chicago"
     commit-message:
       prefix: "chore"
     labels: []
     ignore:
-      # Ignore patch updates for all dependencies
-      - dependency-name: "*"
-        update-types:
-          - version-update:semver-patch
+      # We need to coordinate terraform updates with the version hardcoded in
+      # our Go code.
+      - dependency-name: "terraform"
 
   - package-ecosystem: "npm"
-    directory: "/site/"
+    directories:
+      - "/site"
+      - "/offlinedocs"
+      - "/scripts"
+      - "/scripts/apidocgen"
+
     schedule:
       interval: "monthly"
       time: "06:00"
       timezone: "America/Chicago"
+    reviewers:
+      - "coder/ts"
     commit-message:
       prefix: "chore"
     labels: []
+    groups:
+      xterm:
+        patterns:
+          - "@xterm*"
+      mui:
+        patterns:
+          - "@mui*"
+      react:
+        patterns:
+          - "react"
+          - "react-dom"
+          - "@types/react"
+          - "@types/react-dom"
+      emotion:
+        patterns:
+          - "@emotion*"
+        exclude-patterns:
+          - "jest-runner-eslint"
+      jest:
+        patterns:
+          - "jest"
+          - "@types/jest"
+      vite:
+        patterns:
+          - "vite*"
+          - "@vitejs/plugin-react"
     ignore:
-      # Ignore patch updates for all dependencies
+      # Ignore major version updates to avoid breaking changes
       - dependency-name: "*"
-        update-types:
-          - version-update:semver-patch
-      # Ignore major updates to Node.js types, because they need to
-      # correspond to the Node.js engine version
-      - dependency-name: "@types/node"
         update-types:
           - version-update:semver-major
-
-  - package-ecosystem: "terraform"
-    directory: "/examples/templates"
-    schedule:
-      interval: "monthly"
-      time: "06:00"
-      timezone: "America/Chicago"
-    commit-message:
-      prefix: "chore"
-    labels: []
-    ignore:
-      # We likely want to update this ourselves.
-      - dependency-name: "coder/coder"
+    open-pull-requests-limit: 15

.github/fly-wsproxies/jnb-coder.toml

@@ -0,0 +1,34 @@
+app = "jnb-coder"
+primary_region = "jnb"
+
+[experimental]
+  entrypoint = ["/bin/sh", "-c", "CODER_DERP_SERVER_RELAY_URL=\"http://[${FLY_PRIVATE_IP}]:3000\" /opt/coder wsproxy server"]
+  auto_rollback = true
+
+[build]
+  image = "ghcr.io/coder/coder-preview:main"
+
+[env]
+  CODER_ACCESS_URL = "https://jnb.fly.dev.coder.com"
+  CODER_HTTP_ADDRESS = "0.0.0.0:3000"
+  CODER_PRIMARY_ACCESS_URL = "https://dev.coder.com"
+  CODER_WILDCARD_ACCESS_URL = "*--apps.jnb.fly.dev.coder.com"
+  CODER_VERBOSE = "true"
+
+[http_service]
+  internal_port = 3000
+  force_https = true
+  auto_stop_machines = true
+  auto_start_machines = true
+  min_machines_running = 0
+
+# Ref: https://fly.io/docs/reference/configuration/#http_service-concurrency
+[http_service.concurrency]
+  type = "requests"
+  soft_limit = 50
+  hard_limit = 100
+
+[[vm]]
+  cpu_kind = "shared"
+  cpus = 2
+  memory_mb = 512

.github/fly-wsproxies/paris-coder.toml

@@ -0,0 +1,34 @@
+app = "paris-coder"
+primary_region = "cdg"
+
+[experimental]
+  entrypoint = ["/bin/sh", "-c", "CODER_DERP_SERVER_RELAY_URL=\"http://[${FLY_PRIVATE_IP}]:3000\" /opt/coder wsproxy server"]
+  auto_rollback = true
+
+[build]
+  image = "ghcr.io/coder/coder-preview:main"
+
+[env]
+  CODER_ACCESS_URL = "https://paris.fly.dev.coder.com"
+  CODER_HTTP_ADDRESS = "0.0.0.0:3000"
+  CODER_PRIMARY_ACCESS_URL = "https://dev.coder.com"
+  CODER_WILDCARD_ACCESS_URL = "*--apps.paris.fly.dev.coder.com"
+  CODER_VERBOSE = "true"
+
+[http_service]
+  internal_port = 3000
+  force_https = true
+  auto_stop_machines = true
+  auto_start_machines = true
+  min_machines_running = 0
+
+# Ref: https://fly.io/docs/reference/configuration/#http_service-concurrency
+[http_service.concurrency]
+  type = "requests"
+  soft_limit = 50
+  hard_limit = 100
+
+[[vm]]
+  cpu_kind = "shared"
+  cpus = 2
+  memory_mb = 512

.github/fly-wsproxies/sao-paulo-coder.toml

@@ -0,0 +1,34 @@
+app = "sao-paulo-coder"
+primary_region = "gru"
+
+[experimental]
+  entrypoint = ["/bin/sh", "-c", "CODER_DERP_SERVER_RELAY_URL=\"http://[${FLY_PRIVATE_IP}]:3000\" /opt/coder wsproxy server"]
+  auto_rollback = true
+
+[build]
+  image = "ghcr.io/coder/coder-preview:main"
+
+[env]
+  CODER_ACCESS_URL = "https://sao-paulo.fly.dev.coder.com"
+  CODER_HTTP_ADDRESS = "0.0.0.0:3000"
+  CODER_PRIMARY_ACCESS_URL = "https://dev.coder.com"
+  CODER_WILDCARD_ACCESS_URL = "*--apps.sao-paulo.fly.dev.coder.com"
+  CODER_VERBOSE = "true"
+
+[http_service]
+  internal_port = 3000
+  force_https = true
+  auto_stop_machines = true
+  auto_start_machines = true
+  min_machines_running = 0
+
+# Ref: https://fly.io/docs/reference/configuration/#http_service-concurrency
+[http_service.concurrency]
+  type = "requests"
+  soft_limit = 50
+  hard_limit = 100
+
+[[vm]]
+  cpu_kind = "shared"
+  cpus = 2
+  memory_mb = 512

.github/fly-wsproxies/sydney-coder.toml

@@ -0,0 +1,34 @@
+app = "sydney-coder"
+primary_region = "syd"
+
+[experimental]
+  entrypoint = ["/bin/sh", "-c", "CODER_DERP_SERVER_RELAY_URL=\"http://[${FLY_PRIVATE_IP}]:3000\" /opt/coder wsproxy server"]
+  auto_rollback = true
+
+[build]
+  image = "ghcr.io/coder/coder-preview:main"
+
+[env]
+  CODER_ACCESS_URL = "https://sydney.fly.dev.coder.com"
+  CODER_HTTP_ADDRESS = "0.0.0.0:3000"
+  CODER_PRIMARY_ACCESS_URL = "https://dev.coder.com"
+  CODER_WILDCARD_ACCESS_URL = "*--apps.sydney.fly.dev.coder.com"
+  CODER_VERBOSE = "true"
+
+[http_service]
+  internal_port = 3000
+  force_https = true
+  auto_stop_machines = true
+  auto_start_machines = true
+  min_machines_running = 0
+
+# Ref: https://fly.io/docs/reference/configuration/#http_service-concurrency
+[http_service.concurrency]
+  type = "requests"
+  soft_limit = 50
+  hard_limit = 100
+
+[[vm]]
+  cpu_kind = "shared"
+  cpus = 2
+  memory_mb = 512

.github/pr-deployments/certificate.yaml

@@ -0,0 +1,13 @@
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: pr${PR_NUMBER}-tls
+  namespace: pr-deployment-certs
+spec:
+  secretName: pr${PR_NUMBER}-tls
+  issuerRef:
+    name: letsencrypt
+    kind: ClusterIssuer
+  dnsNames:
+    - "${PR_HOSTNAME}"
+    - "*.${PR_HOSTNAME}"

.github/pr-deployments/rbac.yaml

@@ -0,0 +1,31 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: coder-workspace-pr${PR_NUMBER}
+  namespace: pr${PR_NUMBER}
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: coder-workspace-pr${PR_NUMBER}
+  namespace: pr${PR_NUMBER}
+rules:
+  - apiGroups: ["*"]
+    resources: ["*"]
+    verbs: ["*"]
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: coder-workspace-pr${PR_NUMBER}
+  namespace: pr${PR_NUMBER}
+subjects:
+  - kind: ServiceAccount
+    name: coder-workspace-pr${PR_NUMBER}
+    namespace: pr${PR_NUMBER}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: coder-workspace-pr${PR_NUMBER}

.github/pr-deployments/template/main.tf

@@ -0,0 +1,314 @@
+terraform {
+  required_providers {
+    coder = {
+      source = "coder/coder"
+    }
+    kubernetes = {
+      source = "hashicorp/kubernetes"
+    }
+  }
+}
+
+provider "coder" {
+}
+
+variable "namespace" {
+  type        = string
+  description = "The Kubernetes namespace to create workspaces in (must exist prior to creating workspaces)"
+}
+
+data "coder_parameter" "cpu" {
+  name         = "cpu"
+  display_name = "CPU"
+  description  = "The number of CPU cores"
+  default      = "2"
+  icon         = "/icon/memory.svg"
+  mutable      = true
+  option {
+    name  = "2 Cores"
+    value = "2"
+  }
+  option {
+    name  = "4 Cores"
+    value = "4"
+  }
+  option {
+    name  = "6 Cores"
+    value = "6"
+  }
+  option {
+    name  = "8 Cores"
+    value = "8"
+  }
+}
+
+data "coder_parameter" "memory" {
+  name         = "memory"
+  display_name = "Memory"
+  description  = "The amount of memory in GB"
+  default      = "2"
+  icon         = "/icon/memory.svg"
+  mutable      = true
+  option {
+    name  = "2 GB"
+    value = "2"
+  }
+  option {
+    name  = "4 GB"
+    value = "4"
+  }
+  option {
+    name  = "6 GB"
+    value = "6"
+  }
+  option {
+    name  = "8 GB"
+    value = "8"
+  }
+}
+
+data "coder_parameter" "home_disk_size" {
+  name         = "home_disk_size"
+  display_name = "Home disk size"
+  description  = "The size of the home disk in GB"
+  default      = "10"
+  type         = "number"
+  icon         = "/emojis/1f4be.png"
+  mutable      = false
+  validation {
+    min = 1
+    max = 99999
+  }
+}
+
+provider "kubernetes" {
+  config_path = null
+}
+
+data "coder_workspace" "me" {}
+data "coder_workspace_owner" "me" {}
+
+resource "coder_agent" "main" {
+  os             = "linux"
+  arch           = "amd64"
+  startup_script = <<-EOT
+    set -e
+
+    # install and start code-server
+    curl -fsSL https://code-server.dev/install.sh | sh -s -- --method=standalone --prefix=/tmp/code-server
+    /tmp/code-server/bin/code-server --auth none --port 13337 >/tmp/code-server.log 2>&1 &
+
+  EOT
+
+  # The following metadata blocks are optional. They are used to display
+  # information about your workspace in the dashboard. You can remove them
+  # if you don't want to display any information.
+  # For basic resources, you can use the `coder stat` command.
+  # If you need more control, you can write your own script.
+  metadata {
+    display_name = "CPU Usage"
+    key          = "0_cpu_usage"
+    script       = "coder stat cpu"
+    interval     = 10
+    timeout      = 1
+  }
+
+  metadata {
+    display_name = "RAM Usage"
+    key          = "1_ram_usage"
+    script       = "coder stat mem"
+    interval     = 10
+    timeout      = 1
+  }
+
+  metadata {
+    display_name = "Home Disk"
+    key          = "3_home_disk"
+    script       = "coder stat disk --path $${HOME}"
+    interval     = 60
+    timeout      = 1
+  }
+
+  metadata {
+    display_name = "CPU Usage (Host)"
+    key          = "4_cpu_usage_host"
+    script       = "coder stat cpu --host"
+    interval     = 10
+    timeout      = 1
+  }
+
+  metadata {
+    display_name = "Memory Usage (Host)"
+    key          = "5_mem_usage_host"
+    script       = "coder stat mem --host"
+    interval     = 10
+    timeout      = 1
+  }
+
+  metadata {
+    display_name = "Load Average (Host)"
+    key          = "6_load_host"
+    # get load avg scaled by number of cores
+    script   = <<EOT
+      echo "`cat /proc/loadavg | awk '{ print $1 }'` `nproc`" | awk '{ printf "%0.2f", $1/$2 }'
+    EOT
+    interval = 60
+    timeout  = 1
+  }
+}
+
+# code-server
+resource "coder_app" "code-server" {
+  agent_id     = coder_agent.main.id
+  slug         = "code-server"
+  display_name = "code-server"
+  icon         = "/icon/code.svg"
+  url          = "http://localhost:13337?folder=/home/coder"
+  subdomain    = false
+  share        = "owner"
+
+  healthcheck {
+    url       = "http://localhost:13337/healthz"
+    interval  = 3
+    threshold = 10
+  }
+}
+
+resource "kubernetes_persistent_volume_claim" "home" {
+  metadata {
+    name      = "coder-${lower(data.coder_workspace_owner.me.name)}-${lower(data.coder_workspace.me.name)}-home"
+    namespace = var.namespace
+    labels = {
+      "app.kubernetes.io/name"     = "coder-pvc"
+      "app.kubernetes.io/instance" = "coder-pvc-${lower(data.coder_workspace_owner.me.name)}-${lower(data.coder_workspace.me.name)}"
+      "app.kubernetes.io/part-of"  = "coder"
+      //Coder-specific labels.
+      "com.coder.resource"       = "true"
+      "com.coder.workspace.id"   = data.coder_workspace.me.id
+      "com.coder.workspace.name" = data.coder_workspace.me.name
+      "com.coder.user.id"        = data.coder_workspace_owner.me.id
+      "com.coder.user.username"  = data.coder_workspace_owner.me.name
+    }
+    annotations = {
+      "com.coder.user.email" = data.coder_workspace_owner.me.email
+    }
+  }
+  wait_until_bound = false
+  spec {
+    access_modes = ["ReadWriteOnce"]
+    resources {
+      requests = {
+        storage = "${data.coder_parameter.home_disk_size.value}Gi"
+      }
+    }
+  }
+}
+
+resource "kubernetes_deployment" "main" {
+  count = data.coder_workspace.me.start_count
+  depends_on = [
+    kubernetes_persistent_volume_claim.home
+  ]
+  wait_for_rollout = false
+  metadata {
+    name      = "coder-${lower(data.coder_workspace_owner.me.name)}-${lower(data.coder_workspace.me.name)}"
+    namespace = var.namespace
+    labels = {
+      "app.kubernetes.io/name"     = "coder-workspace"
+      "app.kubernetes.io/instance" = "coder-workspace-${lower(data.coder_workspace_owner.me.name)}-${lower(data.coder_workspace.me.name)}"
+      "app.kubernetes.io/part-of"  = "coder"
+      "com.coder.resource"         = "true"
+      "com.coder.workspace.id"     = data.coder_workspace.me.id
+      "com.coder.workspace.name"   = data.coder_workspace.me.name
+      "com.coder.user.id"          = data.coder_workspace_owner.me.id
+      "com.coder.user.username"    = data.coder_workspace_owner.me.name
+    }
+    annotations = {
+      "com.coder.user.email" = data.coder_workspace_owner.me.email
+    }
+  }
+
+  spec {
+    replicas = 1
+    selector {
+      match_labels = {
+        "app.kubernetes.io/name" = "coder-workspace"
+      }
+    }
+    strategy {
+      type = "Recreate"
+    }
+
+    template {
+      metadata {
+        labels = {
+          "app.kubernetes.io/name" = "coder-workspace"
+        }
+      }
+      spec {
+        security_context {
+          run_as_user = 1000
+          fs_group    = 1000
+        }
+
+        service_account_name = "coder-workspace-${var.namespace}"
+        container {
+          name              = "dev"
+          image             = "bencdr/devops-tools"
+          image_pull_policy = "Always"
+          command           = ["sh", "-c", coder_agent.main.init_script]
+          security_context {
+            run_as_user = "1000"
+          }
+          env {
+            name  = "CODER_AGENT_TOKEN"
+            value = coder_agent.main.token
+          }
+          resources {
+            requests = {
+              "cpu"    = "250m"
+              "memory" = "512Mi"
+            }
+            limits = {
+              "cpu"    = "${data.coder_parameter.cpu.value}"
+              "memory" = "${data.coder_parameter.memory.value}Gi"
+            }
+          }
+          volume_mount {
+            mount_path = "/home/coder"
+            name       = "home"
+            read_only  = false
+          }
+        }
+
+        volume {
+          name = "home"
+          persistent_volume_claim {
+            claim_name = kubernetes_persistent_volume_claim.home.metadata.0.name
+            read_only  = false
+          }
+        }
+
+        affinity {
+          // This affinity attempts to spread out all workspace pods evenly across
+          // nodes.
+          pod_anti_affinity {
+            preferred_during_scheduling_ignored_during_execution {
+              weight = 1
+              pod_affinity_term {
+                topology_key = "kubernetes.io/hostname"
+                label_selector {
+                  match_expressions {
+                    key      = "app.kubernetes.io/name"
+                    operator = "In"
+                    values   = ["coder-workspace"]
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}

.github/pr-deployments/values.yaml

@@ -0,0 +1,38 @@
+coder:
+  image:
+    repo: "${REPO}"
+    tag: "pr${PR_NUMBER}"
+    pullPolicy: Always
+  service:
+    type: ClusterIP
+  ingress:
+    enable: true
+    className: traefik
+    host: "${PR_HOSTNAME}"
+    wildcardHost: "*.${PR_HOSTNAME}"
+    tls:
+      enable: true
+      secretName: "pr${PR_NUMBER}-tls"
+      wildcardSecretName: "pr${PR_NUMBER}-tls"
+  env:
+    - name: "CODER_ACCESS_URL"
+      value: "https://${PR_HOSTNAME}"
+    - name: "CODER_WILDCARD_ACCESS_URL"
+      value: "*.${PR_HOSTNAME}"
+    - name: "CODER_EXPERIMENTS"
+      value: "${EXPERIMENTS}"
+    - name: CODER_PG_CONNECTION_URL
+      valueFrom:
+        secretKeyRef:
+          name: coder-db-url
+          key: url
+    - name: "CODER_OAUTH2_GITHUB_ALLOW_SIGNUPS"
+      value: "true"
+    - name: "CODER_OAUTH2_GITHUB_CLIENT_ID"
+      value: "${PR_DEPLOYMENTS_GITHUB_OAUTH_CLIENT_ID}"
+    - name: "CODER_OAUTH2_GITHUB_CLIENT_SECRET"
+      value: "${PR_DEPLOYMENTS_GITHUB_OAUTH_CLIENT_SECRET}"
+    - name: "CODER_OAUTH2_GITHUB_ALLOWED_ORGS"
+      value: "coder"
+    - name: "CODER_DERP_CONFIG_URL"
+      value: "https://controlplane.tailscale.com/derpmap/default"

.github/workflows/contrib.yaml

@@ -13,6 +13,11 @@ on:
       - opened
       - reopened
       - edited
+      # For jobs that don't run on draft PRs.
+      - ready_for_review
+
+permissions:
+  contents: read
 
 # Only run one instance per PR to ensure in-order execution.
 concurrency: pr-${{ github.ref }}
@@ -25,15 +30,26 @@ jobs:
     permissions:
       pull-requests: write
     steps:
-      - uses: hmarr/auto-approve-action@v3
+      - name: Harden Runner
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
+        with:
+          egress-policy: audit
+
+      - name: auto-approve dependabot
+        uses: hmarr/auto-approve-action@f0939ea97e9205ef24d872e76833fa908a770363 # v4.0.0
         if: github.actor == 'dependabot[bot]'
 
   cla:
     runs-on: ubuntu-latest
     steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
+        with:
+          egress-policy: audit
+
       - name: cla
         if: (github.event.comment.body == 'recheck' || github.event.comment.body == 'I have read the CLA Document and I hereby sign the CLA') || github.event_name == 'pull_request_target'
-        uses: contributor-assistant/github-action@v2.3.0
+        uses: contributor-assistant/github-action@ca4a40a7d1004f18d9960b404b97e5f30a505a08 # v2.6.1
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           # the below token should have repo scope and must be manually added by you in the repository's secret
@@ -45,27 +61,21 @@ jobs:
           path-to-document: "https://github.com/coder/cla/blob/main/README.md"
           # branch should not be protected
           branch: "main"
-          allowlist: dependabot*
-
-  title:
-    runs-on: ubuntu-latest
-    if: github.event_name == 'pull_request_target'
-    steps:
-      - name: Validate PR title
-        uses: amannn/action-semantic-pull-request@v5
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          requireScope: false
+          # Some users have signed a corporate CLA with Coder so are exempt from signing our community one.
+          allowlist: "coryb,aaronlehmann,dependabot*"
 
   release-labels:
     runs-on: ubuntu-latest
-    # Depend on lint so that title is Conventional Commits-compatible.
-    needs: [title]
     # Skip tagging for draft PRs.
-    if: ${{ github.event_name == 'pull_request_target' && success() && !github.event.pull_request.draft }}
+    if: ${{ github.event_name == 'pull_request_target' && !github.event.pull_request.draft }}
     steps:
-      - uses: actions/github-script@v6
+      - name: Harden Runner
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
+        with:
+          egress-policy: audit
+
+      - name: release-labels
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
         with:
           # This script ensures PR title and labels are in sync:
           #

.github/workflows/cron-weekly.yaml

@@ -1,32 +0,0 @@
-name: Weekly Cron
-# runs every monday at 9 am
-on:
-  schedule:
-    - cron: "0 9 * * 1"
-  workflow_dispatch: # allows to run manually for testing
-
-jobs:
-  check-docs:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@master
-
-      - name: Check Markdown links
-        uses: gaurav-nelson/github-action-markdown-link-check@v1
-        id: markdown-link-check
-        # checks all markdown files from /docs including all subfolders
-        with:
-          use-quiet-mode: "yes"
-          use-verbose-mode: "yes"
-          config-file: ".github/workflows/mlc_config.json"
-          folder-path: "docs/"
-          file-path: "./README.md"
-
-      - name: Send Slack notification
-        if: failure()
-        run: |
-          curl -X POST -H 'Content-type: application/json' -d '{"msg":"Broken links found in the documentation. Please check the logs at ${{ env.LOGS_URL }}"}' ${{ secrets.DOCS_LINK_SLACK_WEBHOOK }}
-          echo "Sent Slack notification"
-        env:
-          LOGS_URL: https://github.com/coder/coder/actions/runs/${{ github.run_id }}

.github/workflows/docker-base.yaml

@@ -8,6 +8,11 @@ on:
       - scripts/Dockerfile.base
       - scripts/Dockerfile
 
+  pull_request:
+    paths:
+      - scripts/Dockerfile.base
+      - .github/workflows/docker-base.yaml
+
   schedule:
     # Run every week at 09:43 on Monday, Wednesday and Friday. We build this
     # frequently to ensure that packages are up-to-date.
@@ -17,10 +22,6 @@ on:
 
 permissions:
   contents: read
-  # Necessary to push docker images to ghcr.io.
-  packages: write
-  # Necessary for depot.dev authentication.
-  id-token: write
 
 # Avoid running multiple jobs for the same commit.
 concurrency:
@@ -28,13 +29,24 @@ concurrency:
 
 jobs:
   build:
+    permissions:
+      # Necessary for depot.dev authentication.
+      id-token: write
+      # Necessary to push docker images to ghcr.io.
+      packages: write
     runs-on: ubuntu-latest
     if: github.repository_owner == 'coder'
     steps:
-      - uses: actions/checkout@v3
+      - name: Harden Runner
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
+        with:
+          egress-policy: audit
+
+      - name: Checkout
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
 
       - name: Docker login
-        uses: docker/login-action@v2
+        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
@@ -44,23 +56,25 @@ jobs:
         run: mkdir base-build-context
 
       - name: Install depot.dev CLI
-        uses: depot/setup-action@v1
+        uses: depot/setup-action@b0b1ea4f69e92ebf5dea3f8713a1b0c37b2126a5 # v1.6.0
 
       # This uses OIDC authentication, so no auth variables are required.
       - name: Build base Docker image via depot.dev
-        uses: depot/build-push-action@v1
+        uses: depot/build-push-action@636daae76684e38c301daa0c5eca1c095b24e780 # v1.14.0
         with:
           project: wl5hnrrkns
           context: base-build-context
           file: scripts/Dockerfile.base
           platforms: linux/amd64,linux/arm64,linux/arm/v7
+          provenance: true
           pull: true
           no-cache: true
-          push: true
+          push: ${{ github.event_name != 'pull_request' }}
           tags: |
             ghcr.io/coder/coder-base:latest
 
       - name: Verify that images are pushed properly
+        if: github.event_name != 'pull_request'
         run: |
           # retry 10 times with a 5 second delay as the images may not be
           # available immediately

.github/workflows/dogfood.yaml

@@ -6,18 +6,37 @@ on:
       - main
     paths:
       - "dogfood/**"
+      - ".github/workflows/dogfood.yaml"
+      - "flake.lock"
+      - "flake.nix"
   pull_request:
     paths:
       - "dogfood/**"
+      - ".github/workflows/dogfood.yaml"
+      - "flake.lock"
+      - "flake.nix"
   workflow_dispatch:
 
+permissions:
+  # Necessary for GCP authentication (https://github.com/google-github-actions/setup-gcloud#usage)
+  id-token: write
+
 jobs:
-  deploy_image:
+  build_image:
+    if: github.actor != 'dependabot[bot]' # Skip Dependabot PRs
     runs-on: ubuntu-latest
     steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
+        with:
+          egress-policy: audit
+
+      - name: Checkout
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
+
       - name: Get branch name
         id: branch-name
-        uses: tj-actions/branch-names@v6.5
+        uses: tj-actions/branch-names@6871f53176ad61624f978536bbf089c574dc19a2 # v8.0.1
 
       - name: "Branch name to Docker tag name"
         id: docker-tag-name
@@ -27,47 +46,96 @@ jobs:
           tag=${tag//\//--}
           echo "tag=${tag}" >> $GITHUB_OUTPUT
 
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
+      - name: Set up Depot CLI
+        uses: depot/setup-action@b0b1ea4f69e92ebf5dea3f8713a1b0c37b2126a5 # v1.6.0
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@c47758b77c9736f4b2ef4073d4d51994fabfe349 # v3.7.1
 
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        if: github.ref == 'refs/heads/main'
+        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_PASSWORD }}
 
-      - name: Build and push
-        uses: docker/build-push-action@v4
+      - name: Build and push Non-Nix image
+        uses: depot/build-push-action@636daae76684e38c301daa0c5eca1c095b24e780 # v1.14.0
         with:
-          context: "{{defaultContext}}:dogfood"
-          push: true
+          project: b4q6ltmpzh
+          token: ${{ secrets.DEPOT_TOKEN }}
+          buildx-fallback: true
+          context: "{{defaultContext}}:dogfood/contents"
+          pull: true
+          save: true
+          push: ${{ github.ref == 'refs/heads/main' }}
           tags: "codercom/oss-dogfood:${{ steps.docker-tag-name.outputs.tag }},codercom/oss-dogfood:latest"
-          cache-from: type=registry,ref=codercom/oss-dogfood:latest
-          cache-to: type=inline
+
+      - name: Build and push Nix image
+        uses: depot/build-push-action@636daae76684e38c301daa0c5eca1c095b24e780 # v1.14.0
+        with:
+          project: b4q6ltmpzh
+          token: ${{ secrets.DEPOT_TOKEN }}
+          buildx-fallback: true
+          context: "."
+          file: "dogfood/contents/Dockerfile.nix"
+          pull: true
+          save: true
+          push: ${{ github.ref == 'refs/heads/main' }}
+          tags: "codercom/oss-dogfood-nix:${{ steps.docker-tag-name.outputs.tag }},codercom/oss-dogfood-nix:latest"
+
   deploy_template:
+    needs: build_image
     runs-on: ubuntu-latest
     steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
+        with:
+          egress-policy: audit
+
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
+
+      - name: Setup Terraform
+        uses: ./.github/actions/setup-tf
+
+      - name: Authenticate to Google Cloud
+        uses: google-github-actions/auth@6fc4af4b145ae7821d527454aa9bd537d1f2dc5f # v2.1.7
+        with:
+          workload_identity_provider: projects/573722524737/locations/global/workloadIdentityPools/github/providers/github
+          service_account: coder-ci@coder-dogfood.iam.gserviceaccount.com
+
+      - name: Terraform init and validate
+        run: |
+          cd dogfood
+          terraform init -upgrade
+          terraform validate
+          cd contents
+          terraform init -upgrade
+          terraform validate
+
       - name: Get short commit SHA
+        if: github.ref == 'refs/heads/main'
         id: vars
         run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
-      - name: "Install latest Coder"
-        run: |
-          curl -L https://coder.com/install.sh | sh
-        # env:
-        #   VERSION: 0.x
+
+      - name: Get latest commit title
+        if: github.ref == 'refs/heads/main'
+        id: message
+        run: echo "pr_title=$(git log --format=%s -n 1 ${{ github.sha }})" >> $GITHUB_OUTPUT
+
       - name: "Push template"
+        if: github.ref == 'refs/heads/main'
         run: |
-          coder templates push $CODER_TEMPLATE_NAME --directory $CODER_TEMPLATE_DIR --yes --name=$CODER_TEMPLATE_VERSION
+          cd dogfood
+          terraform apply -auto-approve
         env:
-          # Consumed by Coder CLI
+          # Consumed by coderd provider
           CODER_URL: https://dev.coder.com
           CODER_SESSION_TOKEN: ${{ secrets.CODER_SESSION_TOKEN }}
           # Template source & details
-          CODER_TEMPLATE_NAME: ${{ secrets.CODER_TEMPLATE_NAME }}
-          CODER_TEMPLATE_VERSION: ${{ steps.vars.outputs.sha_short }}
-          CODER_TEMPLATE_DIR: ./dogfood
+          TF_VAR_CODER_TEMPLATE_NAME: ${{ secrets.CODER_TEMPLATE_NAME }}
+          TF_VAR_CODER_TEMPLATE_VERSION: ${{ steps.vars.outputs.sha_short }}
+          TF_VAR_CODER_TEMPLATE_DIR: ./contents
+          TF_VAR_CODER_TEMPLATE_MESSAGE: ${{ steps.message.outputs.pr_title }}
+          TF_LOG: info

.github/workflows/mlc_config.json

@@ -1,23 +0,0 @@
-{
-  "ignorePatterns": [
-    {
-      "pattern": "://localhost"
-    },
-    {
-      "pattern": "://.*.?example\\.com"
-    },
-    {
-      "pattern": "developer.github.com"
-    },
-    {
-      "pattern": "docs.github.com"
-    },
-    {
-      "pattern": "support.google.com"
-    },
-    {
-      "pattern": "tailscale.com"
-    }
-  ],
-  "aliveStatusCodes": [200, 0]
-}

.github/workflows/nightly-gauntlet.yaml

@@ -0,0 +1,74 @@
+# The nightly-gauntlet runs tests that are either too flaky or too slow to block
+# every PR.
+name: nightly-gauntlet
+on:
+  schedule:
+    # Every day at midnight
+    - cron: "0 0 * * *"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  go-race:
+    # While GitHub's toaster runners are likelier to flake, we want consistency
+    # between this environment and the regular test environment for DataDog
+    # statistics and to only show real workflow threats.
+    runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-8' || 'ubuntu-latest' }}
+    # This runner costs 0.016 USD per minute,
+    # so 0.016 * 240 = 3.84 USD per run.
+    timeout-minutes: 240
+    steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
+        with:
+          egress-policy: audit
+
+      - name: Checkout
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
+
+      - name: Setup Go
+        uses: ./.github/actions/setup-go
+
+      - name: Setup Terraform
+        uses: ./.github/actions/setup-tf
+
+      - name: Run Tests
+        run: |
+          # -race is likeliest to catch flaky tests
+          # due to correctness detection and its performance
+          # impact.
+          gotestsum --junitfile="gotests.xml" -- -timeout=240m -count=10 -race ./...
+
+      - name: Upload test results to DataDog
+        uses: ./.github/actions/upload-datadog
+        if: always()
+        with:
+          api-key: ${{ secrets.DATADOG_API_KEY }}
+
+  go-timing:
+    # We run these tests with p=1 so we don't need a lot of compute.
+    runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04' || 'ubuntu-latest' }}
+    timeout-minutes: 10
+    steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
+        with:
+          egress-policy: audit
+
+      - name: Checkout
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
+
+      - name: Setup Go
+        uses: ./.github/actions/setup-go
+
+      - name: Run Tests
+        run: |
+          gotestsum --junitfile="gotests.xml" -- --tags="timing" -p=1 -run='_Timing/' ./...
+
+      - name: Upload test results to DataDog
+        uses: ./.github/actions/upload-datadog
+        if: always()
+        with:
+          api-key: ${{ secrets.DATADOG_API_KEY }}

.github/workflows/pr-auto-assign.yaml

@@ -13,4 +13,10 @@ jobs:
   assign-author:
     runs-on: ubuntu-latest
     steps:
-      - uses: toshimaru/auto-author-assign@v1.6.2
+      - name: Harden Runner
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
+        with:
+          egress-policy: audit
+
+      - name: Assign author
+        uses: toshimaru/auto-author-assign@16f0022cf3d7970c106d8d1105f75a1165edb516 # v2.1.1

.github/workflows/pr-cleanup.yaml

@@ -0,0 +1,81 @@
+name: pr-cleanup
+on:
+  pull_request:
+    types: closed
+  workflow_dispatch:
+    inputs:
+      pr_number:
+        description: "PR number"
+        required: true
+
+permissions:
+  contents: read
+
+jobs:
+  cleanup:
+    runs-on: "ubuntu-latest"
+    permissions:
+      # Necessary to delete docker images from ghcr.io.
+      packages: write
+    steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
+        with:
+          egress-policy: audit
+
+      - name: Get PR number
+        id: pr_number
+        run: |
+          if [ -n "${{ github.event.pull_request.number }}" ]; then
+            echo "PR_NUMBER=${{ github.event.pull_request.number }}" >> $GITHUB_OUTPUT
+          else
+            echo "PR_NUMBER=${{ github.event.inputs.pr_number }}" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Delete image
+        continue-on-error: true
+        uses: bots-house/ghcr-delete-image-action@3827559c68cb4dcdf54d813ea9853be6d468d3a4 # v1.1.0
+        with:
+          owner: coder
+          name: coder-preview
+          token: ${{ secrets.GITHUB_TOKEN }}
+          tag: pr${{ steps.pr_number.outputs.PR_NUMBER }}
+
+      - name: Set up kubeconfig
+        run: |
+          set -euo pipefail
+          mkdir -p ~/.kube
+          echo "${{ secrets.PR_DEPLOYMENTS_KUBECONFIG }}" > ~/.kube/config
+          export KUBECONFIG=~/.kube/config
+
+      - name: Delete helm release
+        run: |
+          set -euo pipefail
+          helm delete --namespace "pr${{ steps.pr_number.outputs.PR_NUMBER }}" "pr${{ steps.pr_number.outputs.PR_NUMBER }}" || echo "helm release not found"
+
+      - name: "Remove PR namespace"
+        run: |
+          kubectl delete namespace "pr${{ steps.pr_number.outputs.PR_NUMBER }}" || echo "namespace not found"
+
+      - name: "Remove DNS records"
+        run: |
+          set -euo pipefail
+          # Get identifier for the record
+          record_id=$(curl -X GET "https://api.cloudflare.com/client/v4/zones/${{ secrets.PR_DEPLOYMENTS_ZONE_ID }}/dns_records?name=%2A.pr${{ steps.pr_number.outputs.PR_NUMBER }}.${{ secrets.PR_DEPLOYMENTS_DOMAIN }}" \
+          -H "Authorization: Bearer ${{ secrets.PR_DEPLOYMENTS_CLOUDFLARE_API_TOKEN }}" \
+          -H "Content-Type:application/json" | jq -r '.result[0].id') || echo "DNS record not found"
+
+          echo "::add-mask::$record_id"
+
+          # Delete the record
+          (
+            curl -X DELETE "https://api.cloudflare.com/client/v4/zones/${{ secrets.PR_DEPLOYMENTS_ZONE_ID }}/dns_records/$record_id" \
+            -H "Authorization: Bearer ${{ secrets.PR_DEPLOYMENTS_CLOUDFLARE_API_TOKEN }}" \
+            -H "Content-Type:application/json" | jq -r '.success'
+          ) || echo "DNS record not found"
+
+      - name: "Delete certificate"
+        if: ${{ github.event.pull_request.merged == true }}
+        run: |
+          set -euxo pipefail
+          kubectl delete certificate "pr${{ steps.pr_number.outputs.PR_NUMBER }}-tls" -n pr-deployment-certs || echo "certificate not found"

.github/workflows/pr-deploy.yaml

@@ -0,0 +1,510 @@
+# This action will trigger when
+# 1. when the workflow is manually triggered
+# 2. ./scripts/deploy_pr.sh is run locally
+# 3. when a PR is updated
+name: Deploy PR
+on:
+  push:
+    branches-ignore:
+      - main
+  workflow_dispatch:
+    inputs:
+      experiments:
+        description: "Experiments to enable"
+        required: false
+        type: string
+        default: "*"
+      build:
+        description: "Force new build"
+        required: false
+        type: boolean
+        default: false
+      deploy:
+        description: "Force new deployment"
+        required: false
+        type: boolean
+        default: false
+
+env:
+  REPO: ghcr.io/coder/coder-preview
+
+permissions:
+  contents: read
+
+jobs:
+  check_pr:
+    runs-on: ubuntu-latest
+    outputs:
+      PR_OPEN: ${{ steps.check_pr.outputs.pr_open }}
+    steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
+        with:
+          egress-policy: audit
+
+      - name: Checkout
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
+
+      - name: Check if PR is open
+        id: check_pr
+        run: |
+          set -euo pipefail
+          pr_open=true
+          if [[ "$(gh pr view --json state | jq -r '.state')" != "OPEN" ]]; then
+            echo "PR doesn't exist or is closed."
+            pr_open=false
+          fi
+          echo "pr_open=$pr_open" >> $GITHUB_OUTPUT
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+  get_info:
+    needs: check_pr
+    if: ${{ needs.check_pr.outputs.PR_OPEN == 'true' }}
+    outputs:
+      PR_NUMBER: ${{ steps.pr_info.outputs.PR_NUMBER }}
+      PR_TITLE: ${{ steps.pr_info.outputs.PR_TITLE }}
+      PR_URL: ${{ steps.pr_info.outputs.PR_URL }}
+      CODER_BASE_IMAGE_TAG: ${{ steps.set_tags.outputs.CODER_BASE_IMAGE_TAG }}
+      CODER_IMAGE_TAG: ${{ steps.set_tags.outputs.CODER_IMAGE_TAG }}
+      NEW: ${{ steps.check_deployment.outputs.NEW }}
+      BUILD: ${{ steps.build_conditionals.outputs.first_or_force_build == 'true' || steps.build_conditionals.outputs.automatic_rebuild == 'true' }}
+
+    runs-on: "ubuntu-latest"
+    steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
+        with:
+          egress-policy: audit
+
+      - name: Checkout
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
+        with:
+          fetch-depth: 0
+
+      - name: Get PR number, title, and branch name
+        id: pr_info
+        run: |
+          set -euo pipefail
+          PR_NUMBER=$(gh pr view --json number | jq -r '.number')
+          PR_TITLE=$(gh pr view --json title | jq -r '.title')
+          PR_URL=$(gh pr view --json url | jq -r '.url')
+          echo "PR_URL=$PR_URL" >> $GITHUB_OUTPUT
+          echo "PR_NUMBER=$PR_NUMBER" >> $GITHUB_OUTPUT
+          echo "PR_TITLE=$PR_TITLE" >> $GITHUB_OUTPUT
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set required tags
+        id: set_tags
+        run: |
+          set -euo pipefail
+          echo "CODER_BASE_IMAGE_TAG=$CODER_BASE_IMAGE_TAG" >> $GITHUB_OUTPUT
+          echo "CODER_IMAGE_TAG=$CODER_IMAGE_TAG" >> $GITHUB_OUTPUT
+        env:
+          CODER_BASE_IMAGE_TAG: ghcr.io/coder/coder-preview-base:pr${{ steps.pr_info.outputs.PR_NUMBER }}
+          CODER_IMAGE_TAG: ghcr.io/coder/coder-preview:pr${{ steps.pr_info.outputs.PR_NUMBER }}
+
+      - name: Set up kubeconfig
+        run: |
+          set -euo pipefail
+          mkdir -p ~/.kube
+          echo "${{ secrets.PR_DEPLOYMENTS_KUBECONFIG_BASE64 }}" | base64 --decode > ~/.kube/config
+          chmod 600 ~/.kube/config
+          export KUBECONFIG=~/.kube/config
+
+      - name: Check if the helm deployment already exists
+        id: check_deployment
+        run: |
+          set -euo pipefail
+          if helm status "pr${{ steps.pr_info.outputs.PR_NUMBER }}" --namespace "pr${{ steps.pr_info.outputs.PR_NUMBER }}" > /dev/null 2>&1; then
+            echo "Deployment already exists. Skipping deployment."
+            NEW=false
+          else
+            echo "Deployment doesn't exist."
+            NEW=true
+          fi
+          echo "NEW=$NEW" >> $GITHUB_OUTPUT
+
+      - name: Check changed files
+        uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2
+        id: filter
+        with:
+          base: ${{ github.ref }}
+          filters: |
+            all:
+              - "**"
+            ignored:
+              - "docs/**"
+              - "README.md"
+              - "examples/web-server/**"
+              - "examples/monitoring/**"
+              - "examples/lima/**"
+              - ".github/**"
+              - "offlinedocs/**"
+              - ".devcontainer/**"
+              - "helm/**"
+              - "*[^g][^o][^.][^s][^u][^m]*"
+              - "*[^g][^o][^.][^m][^o][^d]*"
+              - "*[^M][^a][^k][^e][^f][^i][^l][^e]*"
+              - "scripts/**/*[^D][^o][^c][^k][^e][^r][^f][^i][^l][^e]*"
+              - "scripts/**/*[^D][^o][^c][^k][^e][^r][^f][^i][^l][^e][.][b][^a][^s][^e]*"
+
+      - name: Print number of changed files
+        run: |
+          set -euo pipefail
+          echo "Total number of changed files: ${{ steps.filter.outputs.all_count }}"
+          echo "Number of ignored files: ${{ steps.filter.outputs.ignored_count }}"
+
+      - name: Build conditionals
+        id: build_conditionals
+        run: |
+          set -euo pipefail
+          # build if the workflow is manually triggered and the deployment doesn't exist (first build or force rebuild)
+          echo "first_or_force_build=${{ (github.event_name == 'workflow_dispatch' && steps.check_deployment.outputs.NEW == 'true') || github.event.inputs.build == 'true' }}" >> $GITHUB_OUTPUT
+          # build if the deployment already exist and there are changes in the files that we care about (automatic updates)
+          echo "automatic_rebuild=${{ steps.check_deployment.outputs.NEW == 'false' && steps.filter.outputs.all_count > steps.filter.outputs.ignored_count }}" >> $GITHUB_OUTPUT
+
+  comment-pr:
+    needs: get_info
+    if: needs.get_info.outputs.BUILD == 'true' || github.event.inputs.deploy == 'true'
+    runs-on: "ubuntu-latest"
+    permissions:
+      pull-requests: write # needed for commenting on PRs
+    steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
+        with:
+          egress-policy: audit
+
+      - name: Find Comment
+        uses: peter-evans/find-comment@3eae4d37986fb5a8592848f6a574fdf654e61f9e # v3.1.0
+        id: fc
+        with:
+          issue-number: ${{ needs.get_info.outputs.PR_NUMBER }}
+          comment-author: "github-actions[bot]"
+          body-includes: ":rocket:"
+          direction: last
+
+      - name: Comment on PR
+        id: comment_id
+        uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4.0.0
+        with:
+          comment-id: ${{ steps.fc.outputs.comment-id }}
+          issue-number: ${{ needs.get_info.outputs.PR_NUMBER }}
+          edit-mode: replace
+          body: |
+            ---
+            :rocket: Deploying PR ${{ needs.get_info.outputs.PR_NUMBER }} ...
+            ---
+          reactions: eyes
+          reactions-edit-mode: replace
+
+  build:
+    needs: get_info
+    # Run build job only if there are changes in the files that we care about or if the workflow is manually triggered with --build flag
+    if: needs.get_info.outputs.BUILD == 'true'
+    runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-8' || 'ubuntu-latest' }}
+    permissions:
+      # Necessary to push docker images to ghcr.io.
+      packages: write
+    # This concurrency only cancels build jobs if a new build is triggred. It will avoid cancelling the current deployemtn in case of docs changes.
+    concurrency:
+      group: build-${{ github.workflow }}-${{ github.ref }}-${{ needs.get_info.outputs.BUILD }}
+      cancel-in-progress: true
+    env:
+      DOCKER_CLI_EXPERIMENTAL: "enabled"
+      CODER_IMAGE_TAG: ${{ needs.get_info.outputs.CODER_IMAGE_TAG }}
+    steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
+        with:
+          egress-policy: audit
+
+      - name: Checkout
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
+        with:
+          fetch-depth: 0
+
+      - name: Setup Node
+        uses: ./.github/actions/setup-node
+
+      - name: Setup Go
+        uses: ./.github/actions/setup-go
+
+      - name: Setup sqlc
+        uses: ./.github/actions/setup-sqlc
+
+      - name: GHCR Login
+        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push Linux amd64 Docker image
+        run: |
+          set -euo pipefail
+          go mod download
+          make gen/mark-fresh
+          export DOCKER_IMAGE_NO_PREREQUISITES=true
+          version="$(./scripts/version.sh)"
+          export CODER_IMAGE_BUILD_BASE_TAG="$(CODER_IMAGE_BASE=coder-base ./scripts/image_tag.sh --version "$version")"
+          make -j build/coder_linux_amd64
+          ./scripts/build_docker.sh \
+            --arch amd64 \
+            --target ${{ env.CODER_IMAGE_TAG }} \
+            --version $version \
+            --push \
+            build/coder_linux_amd64
+
+  deploy:
+    needs: [build, get_info]
+    # Run deploy job only if build job was successful or skipped
+    if: |
+      always() && (needs.build.result == 'success' || needs.build.result == 'skipped') &&
+      (needs.get_info.outputs.BUILD == 'true' || github.event.inputs.deploy == 'true')
+    runs-on: "ubuntu-latest"
+    permissions:
+      pull-requests: write # needed for commenting on PRs
+    env:
+      CODER_IMAGE_TAG: ${{ needs.get_info.outputs.CODER_IMAGE_TAG }}
+      PR_NUMBER: ${{ needs.get_info.outputs.PR_NUMBER }}
+      PR_TITLE: ${{ needs.get_info.outputs.PR_TITLE }}
+      PR_URL: ${{ needs.get_info.outputs.PR_URL }}
+      PR_HOSTNAME: "pr${{ needs.get_info.outputs.PR_NUMBER }}.${{ secrets.PR_DEPLOYMENTS_DOMAIN }}"
+    steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
+        with:
+          egress-policy: audit
+
+      - name: Set up kubeconfig
+        run: |
+          set -euo pipefail
+          mkdir -p ~/.kube
+          echo "${{ secrets.PR_DEPLOYMENTS_KUBECONFIG_BASE64 }}" | base64 --decode > ~/.kube/config
+          chmod 600 ~/.kube/config
+          export KUBECONFIG=~/.kube/config
+
+      - name: Check if image exists
+        run: |
+          set -euo pipefail
+          foundTag=$(
+            gh api /orgs/coder/packages/container/coder-preview/versions |
+            jq -r --arg tag "pr${{ env.PR_NUMBER }}" '.[] |
+            select(.metadata.container.tags == [$tag]) |
+            .metadata.container.tags[0]'
+          )
+          if [ -z "$foundTag" ]; then
+            echo "Image not found"
+            echo "${{ env.CODER_IMAGE_TAG }} not found in ghcr.io/coder/coder-preview"
+            exit 1
+          else
+            echo "Image found"
+            echo "$foundTag tag found in ghcr.io/coder/coder-preview"
+          fi
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Add DNS record to Cloudflare
+        if: needs.get_info.outputs.NEW == 'true'
+        run: |
+          curl -X POST "https://api.cloudflare.com/client/v4/zones/${{ secrets.PR_DEPLOYMENTS_ZONE_ID }}/dns_records" \
+            -H "Authorization: Bearer ${{ secrets.PR_DEPLOYMENTS_CLOUDFLARE_API_TOKEN }}" \
+            -H "Content-Type:application/json" \
+            --data '{"type":"CNAME","name":"*.${{ env.PR_HOSTNAME }}","content":"${{ env.PR_HOSTNAME }}","ttl":1,"proxied":false}'
+
+      - name: Create PR namespace
+        if: needs.get_info.outputs.NEW == 'true' || github.event.inputs.deploy == 'true'
+        run: |
+          set -euo pipefail
+          # try to delete the namespace, but don't fail if it doesn't exist
+          kubectl delete namespace "pr${{ env.PR_NUMBER }}" || true
+          kubectl create namespace "pr${{ env.PR_NUMBER }}"
+
+      - name: Checkout
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
+
+      - name: Check and Create Certificate
+        if: needs.get_info.outputs.NEW == 'true' || github.event.inputs.deploy == 'true'
+        run: |
+          # Using kubectl to check if a Certificate resource already exists
+          # we are doing this to avoid letsenrypt rate limits
+          if ! kubectl get certificate pr${{ env.PR_NUMBER }}-tls -n pr-deployment-certs > /dev/null 2>&1; then
+            echo "Certificate doesn't exist. Creating a new one."
+            envsubst < ./.github/pr-deployments/certificate.yaml | kubectl apply -f -
+          else
+            echo "Certificate exists. Skipping certificate creation."
+          fi
+          echo "Copy certificate from pr-deployment-certs to pr${{ env.PR_NUMBER }} namespace"
+          until kubectl get secret pr${{ env.PR_NUMBER }}-tls -n pr-deployment-certs &> /dev/null
+          do
+            echo "Waiting for secret pr${{ env.PR_NUMBER }}-tls to be created..."
+            sleep 5
+          done
+          (
+            kubectl get secret pr${{ env.PR_NUMBER }}-tls -n pr-deployment-certs -o json |
+            jq 'del(.metadata.namespace,.metadata.creationTimestamp,.metadata.resourceVersion,.metadata.selfLink,.metadata.uid,.metadata.managedFields)' |
+            kubectl -n pr${{ env.PR_NUMBER }} apply -f -
+          )
+
+      - name: Set up PostgreSQL database
+        if: needs.get_info.outputs.NEW == 'true' || github.event.inputs.deploy == 'true'
+        run: |
+          helm repo add bitnami https://charts.bitnami.com/bitnami
+          helm install coder-db bitnami/postgresql \
+            --namespace pr${{ env.PR_NUMBER }} \
+            --set auth.username=coder \
+            --set auth.password=coder \
+            --set auth.database=coder \
+            --set persistence.size=10Gi
+          kubectl create secret generic coder-db-url -n pr${{ env.PR_NUMBER }} \
+            --from-literal=url="postgres://coder:coder@coder-db-postgresql.pr${{ env.PR_NUMBER }}.svc.cluster.local:5432/coder?sslmode=disable"
+
+      - name: Create a service account, role, and rolebinding for the PR namespace
+        if: needs.get_info.outputs.NEW == 'true' || github.event.inputs.deploy == 'true'
+        run: |
+          set -euo pipefail
+          # Create service account, role, rolebinding
+          envsubst < ./.github/pr-deployments/rbac.yaml | kubectl apply -f -
+
+      - name: Create values.yaml
+        env:
+          EXPERIMENTS: ${{ github.event.inputs.experiments }}
+          PR_DEPLOYMENTS_GITHUB_OAUTH_CLIENT_ID: ${{ secrets.PR_DEPLOYMENTS_GITHUB_OAUTH_CLIENT_ID }}
+          PR_DEPLOYMENTS_GITHUB_OAUTH_CLIENT_SECRET: ${{ secrets.PR_DEPLOYMENTS_GITHUB_OAUTH_CLIENT_SECRET }}
+        run: |
+          set -euo pipefail
+          envsubst < ./.github/pr-deployments/values.yaml > ./pr-deploy-values.yaml
+
+      - name: Install/Upgrade Helm chart
+        run: |
+          set -euo pipefail
+          helm dependency update --skip-refresh ./helm/coder
+          helm upgrade --install "pr${{ env.PR_NUMBER }}" ./helm/coder \
+          --namespace "pr${{ env.PR_NUMBER }}" \
+          --values ./pr-deploy-values.yaml \
+          --force
+
+      - name: Install coder-logstream-kube
+        if: needs.get_info.outputs.NEW == 'true' || github.event.inputs.deploy == 'true'
+        run: |
+          helm repo add coder-logstream-kube https://helm.coder.com/logstream-kube
+          helm upgrade --install coder-logstream-kube coder-logstream-kube/coder-logstream-kube \
+            --namespace "pr${{ env.PR_NUMBER }}" \
+            --set url="https://${{ env.PR_HOSTNAME }}"
+
+      - name: Get Coder binary
+        if: needs.get_info.outputs.NEW == 'true' || github.event.inputs.deploy == 'true'
+        run: |
+          set -euo pipefail
+
+          DEST="${HOME}/coder"
+          URL="https://${{ env.PR_HOSTNAME }}/bin/coder-linux-amd64"
+
+          mkdir -p "$(dirname ${DEST})"
+
+          COUNT=0
+          until $(curl --output /dev/null --silent --head --fail "$URL"); do
+              printf '.'
+              sleep 5
+              COUNT=$((COUNT+1))
+              if [ $COUNT -ge 60 ]; then
+                echo "Timed out waiting for URL to be available"
+                exit 1
+              fi
+          done
+
+          curl -fsSL "$URL" -o "${DEST}"
+          chmod +x "${DEST}"
+          "${DEST}" version
+          mv "${DEST}" /usr/local/bin/coder
+
+      - name: Create first user
+        if: needs.get_info.outputs.NEW == 'true' || github.event.inputs.deploy == 'true'
+        id: setup_deployment
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          set -euo pipefail
+
+          # create a masked random password 12 characters long
+          password=$(openssl rand -base64 16 | tr -d "=+/" | cut -c1-12)
+
+          # add mask so that the password is not printed to the logs
+          echo "::add-mask::$password"
+          echo "password=$password" >> $GITHUB_OUTPUT
+
+          coder login \
+            --first-user-username pr${{ env.PR_NUMBER }}-admin \
+            --first-user-email pr${{ env.PR_NUMBER }}@coder.com \
+            --first-user-password $password \
+            --first-user-trial=false \
+            --use-token-as-session \
+            https://${{ env.PR_HOSTNAME }}
+
+          # Create a user for the github.actor
+          # TODO: update once https://github.com/coder/coder/issues/15466 is resolved
+          # coder users create \
+          #   --username ${{ github.actor }} \
+          #   --login-type github
+
+          # promote the user to admin role
+          # coder org members edit-role ${{ github.actor }} organization-admin
+          # TODO: update once https://github.com/coder/internal/issues/207 is resolved
+
+      - name: Send Slack notification
+        if: needs.get_info.outputs.NEW == 'true' || github.event.inputs.deploy == 'true'
+        run: |
+          curl -s -o /dev/null -X POST -H 'Content-type: application/json' \
+            -d \
+            '{
+              "pr_number": "'"${{ env.PR_NUMBER }}"'",
+              "pr_url": "'"${{ env.PR_URL }}"'",
+              "pr_title": "'"${{ env.PR_TITLE }}"'",
+              "pr_access_url": "'"https://${{ env.PR_HOSTNAME }}"'",
+              "pr_username": "'"pr${{ env.PR_NUMBER }}-admin"'",
+              "pr_email": "'"pr${{ env.PR_NUMBER }}@coder.com"'",
+              "pr_password": "'"${{ steps.setup_deployment.outputs.password }}"'",
+              "pr_actor": "'"${{ github.actor }}"'"
+            }' \
+            ${{ secrets.PR_DEPLOYMENTS_SLACK_WEBHOOK }}
+          echo "Slack notification sent"
+
+      - name: Find Comment
+        uses: peter-evans/find-comment@3eae4d37986fb5a8592848f6a574fdf654e61f9e # v3.1.0
+        id: fc
+        with:
+          issue-number: ${{ env.PR_NUMBER }}
+          comment-author: "github-actions[bot]"
+          body-includes: ":rocket:"
+          direction: last
+
+      - name: Comment on PR
+        uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4.0.0
+        env:
+          STATUS: ${{ needs.get_info.outputs.NEW == 'true' && 'Created' || 'Updated' }}
+        with:
+          issue-number: ${{ env.PR_NUMBER }}
+          edit-mode: replace
+          comment-id: ${{ steps.fc.outputs.comment-id }}
+          body: |
+            ---
+            :heavy_check_mark: PR ${{ env.PR_NUMBER }} ${{ env.STATUS }} successfully.
+            :rocket: Access the credentials [here](${{ secrets.PR_DEPLOYMENTS_SLACK_CHANNEL_URL }}).
+            ---
+            cc: @${{ github.actor }}
+          reactions: rocket
+          reactions-edit-mode: replace
+
+      - name: Create template and workspace
+        if: needs.get_info.outputs.NEW == 'true' || github.event.inputs.deploy == 'true'
+        run: |
+          set -euo pipefail
+          cd .github/pr-deployments/template
+          coder templates push -y --variable namespace=pr${{ env.PR_NUMBER }} kubernetes
+
+          # Create workspace
+          coder create --template="kubernetes" kube --parameter cpu=2 --parameter memory=4 --parameter home_disk_size=2 -y
+          coder stop kube -y

.github/workflows/release-validation.yaml

@@ -0,0 +1,28 @@
+name: release-validation
+
+on:
+  push:
+    tags:
+      - "v*"
+
+permissions:
+  contents: read
+
+jobs:
+  network-performance:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
+        with:
+          egress-policy: audit
+
+      - name: Run Schmoder CI
+        uses: benc-uk/workflow-dispatch@e2e5e9a103e331dad343f381a29e654aea3cf8fc # v1.2.4
+        with:
+          workflow: ci.yaml
+          repo: coder/schmoder
+          inputs: '{ "num_releases": "3", "commit": "${{ github.sha }}" }'
+          token: ${{ secrets.CDRCI_SCHMODER_ACTIONS_TOKEN }}
+          ref: main

.github/workflows/release.yaml

@@ -1,11 +1,16 @@
 # GitHub release workflow.
 name: Release
 on:
-  push:
-    tags:
-      - "v*"
   workflow_dispatch:
     inputs:
+      release_channel:
+        type: choice
+        description: Release channel
+        options:
+          - mainline
+          - stable
+      release_notes:
+        description: Release notes for the publishing the release. This is required to create a release.
       dry_run:
         description: Perform a dry-run release (devel). Note that ref must be an annotated tag when run without dry-run.
         type: boolean
@@ -13,12 +18,7 @@ on:
         default: false
 
 permissions:
-  # Required to publish a release
-  contents: write
-  # Necessary to push docker images to ghcr.io.
-  packages: write
-  # Necessary for GCP authentication (https://github.com/google-github-actions/setup-gcloud#usage)
-  id-token: write
+  contents: read
 
 concurrency: ${{ github.workflow }}-${{ github.ref }}
 
@@ -28,18 +28,33 @@ env:
   # https://github.blog/changelog/2022-06-10-github-actions-inputs-unified-across-manual-and-reusable-workflows/
   CODER_RELEASE: ${{ !inputs.dry_run }}
   CODER_DRY_RUN: ${{ inputs.dry_run }}
+  CODER_RELEASE_CHANNEL: ${{ inputs.release_channel }}
+  CODER_RELEASE_NOTES: ${{ inputs.release_notes }}
 
 jobs:
   release:
     name: Build and publish
-    runs-on: ${{ github.repository_owner == 'coder' && 'ubuntu-latest-8-cores' || 'ubuntu-latest' }}
+    runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-8' || 'ubuntu-latest' }}
+    permissions:
+      # Required to publish a release
+      contents: write
+      # Necessary to push docker images to ghcr.io.
+      packages: write
+      # Necessary for GCP authentication (https://github.com/google-github-actions/setup-gcloud#usage)
+      id-token: write
     env:
       # Necessary for Docker manifest
       DOCKER_CLI_EXPERIMENTAL: "enabled"
     outputs:
       version: ${{ steps.version.outputs.version }}
     steps:
-      - uses: actions/checkout@v3
+      - name: Harden Runner
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
+        with:
+          egress-policy: audit
+
+      - name: Checkout
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
         with:
           fetch-depth: 0
 
@@ -61,21 +76,45 @@ jobs:
           echo "CODER_FORCE_VERSION=$version" >> $GITHUB_ENV
           echo "$version"
 
-      - name: Create release notes
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          # We always have to set this since there might be commits on
-          # main that didn't have a PR.
-          CODER_IGNORE_MISSING_COMMIT_METADATA: "1"
+      # Verify that all expectations for a release are met.
+      - name: Verify release input
+        if: ${{ !inputs.dry_run }}
+        run: |
+          set -euo pipefail
+
+          if [[ "${GITHUB_REF}" != "refs/tags/v"* ]]; then
+            echo "Ref must be a semver tag when creating a release, did you use scripts/release.sh?"
+            exit 1
+          fi
+
+          # 2.10.2 -> release/2.10
+          version="$(./scripts/version.sh)"
+          release_branch=release/${version%.*}
+          branch_contains_tag=$(git branch --remotes --contains "${GITHUB_REF}" --list "*/${release_branch}" --format='%(refname)')
+          if [[ -z "${branch_contains_tag}" ]]; then
+            echo "Ref tag must exist in a branch named ${release_branch} when creating a release, did you use scripts/release.sh?"
+            exit 1
+          fi
+
+          if [[ -z "${CODER_RELEASE_NOTES}" ]]; then
+            echo "Release notes are required to create a release, did you use scripts/release.sh?"
+            exit 1
+          fi
+
+          echo "Release inputs verified:"
+          echo
+          echo "- Ref: ${GITHUB_REF}"
+          echo "- Version: ${version}"
+          echo "- Release channel: ${CODER_RELEASE_CHANNEL}"
+          echo "- Release branch: ${release_branch}"
+          echo "- Release notes: true"
+
+      - name: Create release notes file
         run: |
           set -euo pipefail
-          ref=HEAD
-          old_version="$(git describe --abbrev=0 "$ref^1")"
-          version="$(./scripts/version.sh)"
 
-          # Generate notes.
           release_notes_file="$(mktemp -t release_notes.XXXXXX)"
-          ./scripts/release/generate_release_notes.sh --old-version "$old_version" --new-version "$version" --ref "$ref" >> "$release_notes_file"
+          echo "$CODER_RELEASE_NOTES" > "$release_notes_file"
           echo CODER_RELEASE_NOTES_FILE="$release_notes_file" >> $GITHUB_ENV
 
       - name: Show release notes
@@ -84,26 +123,24 @@ jobs:
           cat "$CODER_RELEASE_NOTES_FILE"
 
       - name: Docker Login
-        uses: docker/login-action@v2
+        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - uses: actions/setup-go@v4
-        with:
-          go-version: "~1.20"
+      - name: Setup Go
+        uses: ./.github/actions/setup-go
 
-      - name: Cache Node
-        id: cache-node
-        uses: actions/cache@v3
+      - name: Setup Node
+        uses: ./.github/actions/setup-node
+
+      # Necessary for signing Windows binaries.
+      - name: Setup Java
+        uses: actions/setup-java@b36c23c0d998641eff861008f374ee103c25ac73 # v4.4.0
         with:
-          path: |
-            **/node_modules
-            .eslintcache
-          key: js-${{ runner.os }}-test-${{ hashFiles('**/yarn.lock') }}
-          restore-keys: |
-            js-${{ runner.os }}-
+          distribution: "zulu"
+          java-version: "11.0"
 
       - name: Install nsis and zstd
         run: sudo apt-get install -y nsis zstd
@@ -111,7 +148,7 @@ jobs:
       - name: Install nfpm
         run: |
           set -euo pipefail
-          wget -O /tmp/nfpm.deb https://github.com/goreleaser/nfpm/releases/download/v2.18.1/nfpm_amd64.deb
+          wget -O /tmp/nfpm.deb https://github.com/goreleaser/nfpm/releases/download/v2.35.1/nfpm_2.35.1_amd64.deb
           sudo dpkg -i /tmp/nfpm.deb
           rm /tmp/nfpm.deb
 
@@ -138,6 +175,32 @@ jobs:
           AC_CERTIFICATE_PASSWORD: ${{ secrets.AC_CERTIFICATE_PASSWORD }}
           AC_APIKEY_P8_BASE64: ${{ secrets.AC_APIKEY_P8_BASE64 }}
 
+      - name: Setup Windows EV Signing Certificate
+        run: |
+          set -euo pipefail
+          touch /tmp/ev_cert.pem
+          chmod 600 /tmp/ev_cert.pem
+          echo "$EV_SIGNING_CERT" > /tmp/ev_cert.pem
+          wget https://github.com/ebourg/jsign/releases/download/6.0/jsign-6.0.jar -O /tmp/jsign-6.0.jar
+        env:
+          EV_SIGNING_CERT: ${{ secrets.EV_SIGNING_CERT }}
+
+      - name: Test migrations from current ref to main
+        run: |
+          POSTGRES_VERSION=13 make test-migrations
+
+      # Setup GCloud for signing Windows binaries.
+      - name: Authenticate to Google Cloud
+        id: gcloud_auth
+        uses: google-github-actions/auth@6fc4af4b145ae7821d527454aa9bd537d1f2dc5f # v2.1.7
+        with:
+          workload_identity_provider: ${{ secrets.GCP_CODE_SIGNING_WORKLOAD_ID_PROVIDER }}
+          service_account: ${{ secrets.GCP_CODE_SIGNING_SERVICE_ACCOUNT }}
+          token_format: "access_token"
+
+      - name: Setup GCloud SDK
+        uses: google-github-actions/setup-gcloud@6189d56e4096ee891640bb02ac264be376592d6a # v2.1.2
+
       - name: Build binaries
         run: |
           set -euo pipefail
@@ -149,18 +212,29 @@ jobs:
             build/coder_"$version"_linux_{amd64,armv7,arm64}.{tar.gz,apk,deb,rpm} \
             build/coder_"$version"_{darwin,windows}_{amd64,arm64}.zip \
             build/coder_"$version"_windows_amd64_installer.exe \
-            build/coder_helm_"$version".tgz
+            build/coder_helm_"$version".tgz \
+            build/provisioner_helm_"$version".tgz
         env:
+          CODER_SIGN_WINDOWS: "1"
           CODER_SIGN_DARWIN: "1"
           AC_CERTIFICATE_FILE: /tmp/apple_cert.p12
           AC_CERTIFICATE_PASSWORD_FILE: /tmp/apple_cert_password.txt
           AC_APIKEY_ISSUER_ID: ${{ secrets.AC_APIKEY_ISSUER_ID }}
           AC_APIKEY_ID: ${{ secrets.AC_APIKEY_ID }}
           AC_APIKEY_FILE: /tmp/apple_apikey.p8
+          EV_KEY: ${{ secrets.EV_KEY }}
+          EV_KEYSTORE: ${{ secrets.EV_KEYSTORE }}
+          EV_TSA_URL: ${{ secrets.EV_TSA_URL }}
+          EV_CERTIFICATE_PATH: /tmp/ev_cert.pem
+          GCLOUD_ACCESS_TOKEN: ${{ steps.gcloud_auth.outputs.access_token }}
+          JSIGN_PATH: /tmp/jsign-6.0.jar
 
       - name: Delete Apple Developer certificate and API key
         run: rm -f /tmp/{apple_cert.p12,apple_cert_password.txt,apple_apikey.p8}
 
+      - name: Delete Windows EV Signing Cert
+        run: rm /tmp/ev_cert.pem
+
       - name: Determine base image tag
         id: image-base-tag
         run: |
@@ -178,17 +252,18 @@ jobs:
 
       - name: Install depot.dev CLI
         if: steps.image-base-tag.outputs.tag != ''
-        uses: depot/setup-action@v1
+        uses: depot/setup-action@b0b1ea4f69e92ebf5dea3f8713a1b0c37b2126a5 # v1.6.0
 
       # This uses OIDC authentication, so no auth variables are required.
       - name: Build base Docker image via depot.dev
         if: steps.image-base-tag.outputs.tag != ''
-        uses: depot/build-push-action@v1
+        uses: depot/build-push-action@636daae76684e38c301daa0c5eca1c095b24e780 # v1.14.0
         with:
           project: wl5hnrrkns
           context: base-build-context
           file: scripts/Dockerfile.base
           platforms: linux/amd64,linux/arm64,linux/arm/v7
+          provenance: true
           pull: true
           no-cache: true
           push: true
@@ -230,7 +305,7 @@ jobs:
 
           # build Docker images for each architecture
           version="$(./scripts/version.sh)"
-          make -j build/coder_"$version"_linux_{amd64,arm64,armv7}.tag
+          make build/coder_"$version"_linux_{amd64,arm64,armv7}.tag
 
           # we can't build multi-arch if the images aren't pushed, so quit now
           # if dry-running
@@ -241,7 +316,7 @@ jobs:
 
           # build and push multi-arch manifest, this depends on the other images
           # being pushed so will automatically push them.
-          make -j push/build/coder_"$version"_linux.tag
+          make push/build/coder_"$version"_linux.tag
 
           # if the current version is equal to the highest (according to semver)
           # version in the repo, also create a multi-arch image as ":latest" and
@@ -255,6 +330,11 @@ jobs:
         env:
           CODER_BASE_IMAGE_TAG: ${{ steps.image-base-tag.outputs.tag }}
 
+      - name: Generate offline docs
+        run: |
+          version="$(./scripts/version.sh)"
+          make -j build/coder_docs_"$version".tgz
+
       - name: ls build
         run: ls -lh build
 
@@ -263,6 +343,9 @@ jobs:
           set -euo pipefail
 
           publish_args=()
+          if [[ $CODER_RELEASE_CHANNEL == "stable" ]]; then
+            publish_args+=(--stable)
+          fi
           if [[ $CODER_DRY_RUN == *t* ]]; then
             publish_args+=(--dry-run)
           fi
@@ -283,13 +366,13 @@ jobs:
           CODER_GPG_RELEASE_KEY_BASE64: ${{ secrets.GPG_RELEASE_KEY_BASE64 }}
 
       - name: Authenticate to Google Cloud
-        uses: google-github-actions/auth@v1
+        uses: google-github-actions/auth@6fc4af4b145ae7821d527454aa9bd537d1f2dc5f # v2.1.7
         with:
           workload_identity_provider: ${{ secrets.GCP_WORKLOAD_ID_PROVIDER }}
           service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }}
 
       - name: Setup GCloud SDK
-        uses: "google-github-actions/setup-gcloud@v1"
+        uses: google-github-actions/setup-gcloud@6189d56e4096ee891640bb02ac264be376592d6a # 2.1.2
 
       - name: Publish Helm Chart
         if: ${{ !inputs.dry_run }}
@@ -298,15 +381,17 @@ jobs:
           version="$(./scripts/version.sh)"
           mkdir -p build/helm
           cp "build/coder_helm_${version}.tgz" build/helm
+          cp "build/provisioner_helm_${version}.tgz" build/helm
           gsutil cp gs://helm.coder.com/v2/index.yaml build/helm/index.yaml
           helm repo index build/helm --url https://helm.coder.com/v2 --merge build/helm/index.yaml
           gsutil -h "Cache-Control:no-cache,max-age=0" cp build/helm/coder_helm_${version}.tgz gs://helm.coder.com/v2
+          gsutil -h "Cache-Control:no-cache,max-age=0" cp build/helm/provisioner_helm_${version}.tgz gs://helm.coder.com/v2
           gsutil -h "Cache-Control:no-cache,max-age=0" cp build/helm/index.yaml gs://helm.coder.com/v2
           gsutil -h "Cache-Control:no-cache,max-age=0" cp helm/artifacthub-repo.yml gs://helm.coder.com/v2
 
       - name: Upload artifacts to actions (if dry-run)
         if: ${{ inputs.dry_run }}
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@604373da6381bf24206979c74d06a550515601b9 # v4.4.1
         with:
           name: release-artifacts
           path: |
@@ -319,21 +404,112 @@ jobs:
             ./build/*.rpm
           retention-days: 7
 
-      - name: Start Packer builds
+      - name: Send repository-dispatch event
         if: ${{ !inputs.dry_run }}
-        uses: peter-evans/repository-dispatch@v2
+        uses: peter-evans/repository-dispatch@ff45666b9427631e3450c54a1bcbee4d9ff4d7c0 # v3.0.0
         with:
           token: ${{ secrets.CDRCI_GITHUB_TOKEN }}
           repository: coder/packages
           event-type: coder-release
-          client-payload: '{"coder_version": "${{ steps.version.outputs.version }}"}'
+          client-payload: '{"coder_version": "${{ steps.version.outputs.version }}", "release_channel": "${{ inputs.release_channel }}"}'
+
+  publish-homebrew:
+    name: Publish to Homebrew tap
+    runs-on: ubuntu-latest
+    needs: release
+    if: ${{ !inputs.dry_run }}
+
+    steps:
+      # TODO: skip this if it's not a new release (i.e. a backport). This is
+      #       fine right now because it just makes a PR that we can close.
+      - name: Harden Runner
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
+        with:
+          egress-policy: audit
+
+      - name: Update homebrew
+        env:
+          # Variables used by the `gh` command
+          GH_REPO: coder/homebrew-coder
+          GH_TOKEN: ${{ secrets.CDRCI_GITHUB_TOKEN }}
+        run: |
+          # Keep version number around for reference, removing any potential leading v
+          coder_version="$(echo "${{ needs.release.outputs.version }}" | tr -d v)"
+
+          set -euxo pipefail
+
+          # Setup Git
+          git config --global user.email "ci@coder.com"
+          git config --global user.name "Coder CI"
+          git config --global credential.helper "store"
+
+          temp_dir="$(mktemp -d)"
+          cd "$temp_dir"
+
+          # Download checksums
+          checksums_url="$(gh release view --repo coder/coder "v$coder_version" --json assets \
+            | jq -r ".assets | map(.url) | .[]" \
+            | grep -e ".checksums.txt\$")"
+          wget "$checksums_url" -O checksums.txt
+
+          # Get the SHAs
+          darwin_arm_sha="$(cat checksums.txt | grep "darwin_arm64.zip" | awk '{ print $1 }')"
+          darwin_intel_sha="$(cat checksums.txt | grep "darwin_amd64.zip" | awk '{ print $1 }')"
+          linux_sha="$(cat checksums.txt | grep "linux_amd64.tar.gz" | awk '{ print $1 }')"
+
+          echo "macOS arm64: $darwin_arm_sha"
+          echo "macOS amd64: $darwin_intel_sha"
+          echo "Linux amd64: $linux_sha"
+
+          # Check out the homebrew repo
+          git clone "https://github.com/$GH_REPO" homebrew-coder
+          brew_branch="auto-release/$coder_version"
+          cd homebrew-coder
+
+          # Check if a PR already exists.
+          pr_count="$(gh pr list --search "head:$brew_branch" --json id,closed | jq -r ".[] | select(.closed == false) | .id" | wc -l)"
+          if [[ "$pr_count" > 0 ]]; then
+            echo "Bailing out as PR already exists" 2>&1
+            exit 0
+          fi
+
+          # Set up cdrci credentials for pushing to homebrew-coder
+          echo "https://x-access-token:$GH_TOKEN@github.com" >> ~/.git-credentials
+          # Update the formulae and push
+          git checkout -b "$brew_branch"
+          ./scripts/update-v2.sh "$coder_version" "$darwin_arm_sha" "$darwin_intel_sha" "$linux_sha"
+          git add .
+          git commit -m "coder $coder_version"
+          git push -u origin -f "$brew_branch"
+
+          # Create PR
+          gh pr create \
+            -B master -H "$brew_branch" \
+            -t "coder $coder_version" \
+            -b "" \
+            -r "${{ github.actor }}" \
+            -a "${{ github.actor }}" \
+            -b "This automatic PR was triggered by the release of Coder v$coder_version"
 
   publish-winget:
     name: Publish to winget-pkgs
     runs-on: windows-latest
     needs: release
+    if: ${{ !inputs.dry_run }}
+
     steps:
-      - uses: actions/checkout@v3
+      - name: Harden Runner
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
+        with:
+          egress-policy: audit
+
+      - name: Sync fork
+        run: gh repo sync cdrci/winget-pkgs -b master
+        env:
+          GH_TOKEN: ${{ secrets.CDRCI_GITHUB_TOKEN }}
+
+      - name: Checkout
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
         with:
           fetch-depth: 0
 
@@ -358,33 +534,26 @@ jobs:
 
           $release_assets = gh release view --repo coder/coder "v${version}" --json assets | `
             ConvertFrom-Json
-          # Get the installer URL from the release assets.
-          $installer_url = $release_assets.assets | `
+          # Get the installer URLs from the release assets.
+          $amd64_installer_url = $release_assets.assets | `
             Where-Object name -Match ".*_windows_amd64_installer.exe$" | `
             Select -ExpandProperty url
+          $amd64_zip_url = $release_assets.assets | `
+            Where-Object name -Match ".*_windows_amd64.zip$" | `
+            Select -ExpandProperty url
+          $arm64_zip_url = $release_assets.assets | `
+            Where-Object name -Match ".*_windows_arm64.zip$" | `
+            Select -ExpandProperty url
 
-          echo "Installer URL: ${installer_url}"
+          echo "amd64 Installer URL: ${amd64_installer_url}"
+          echo "amd64 zip URL: ${amd64_zip_url}"
+          echo "arm64 zip URL: ${arm64_zip_url}"
           echo "Package version: ${version}"
 
-          # Bail if dry-run.
-          if ($env:CODER_DRY_RUN -match "t") {
-            echo "Skipping submission due to dry-run."
-            exit 0
-          }
-
-          # The URL "|X64" suffix forces the architecture as it cannot be
-          # sniffed properly from the URL. wingetcreate checks both the URL and
-          # binary magic bytes for the architecture and they need to both match,
-          # but they only check for `x64`, `win64` and `_64` in the URL. Our URL
-          # contains `amd64` which doesn't match sadly.
-          #
-          # wingetcreate will still do the binary magic bytes check, so if we
-          # accidentally change the architecture of the installer, it will fail
-          # submission.
           .\wingetcreate.exe update Coder.Coder `
             --submit `
             --version "${version}" `
-            --urls "${installer_url}|X64" `
+            --urls "${amd64_installer_url}" "${amd64_zip_url}" "${arm64_zip_url}" `
             --token "$env:WINGET_GH_TOKEN"
 
         env:
@@ -395,8 +564,9 @@ jobs:
           WINGET_GH_TOKEN: ${{ secrets.CDRCI_GITHUB_TOKEN }}
 
       - name: Comment on PR
-        if: ${{ !inputs.dry_run }}
         run: |
+          # wait 30 seconds
+          Start-Sleep -Seconds 30.0
           # Find the PR that wingetcreate just made.
           $version = "${{ needs.release.outputs.version }}".Trim('v')
           $pr_list = gh pr list --repo microsoft/winget-pkgs --search "author:cdrci Coder.Coder version ${version}" --limit 1 --json number | `
@@ -409,3 +579,34 @@ jobs:
           # For gh CLI. We need a real token since we're commenting on a PR in a
           # different repo.
           GH_TOKEN: ${{ secrets.CDRCI_GITHUB_TOKEN }}
+
+  # publish-sqlc pushes the latest schema to sqlc cloud.
+  # At present these pushes cannot be tagged, so the last push is always the latest.
+  publish-sqlc:
+    name: "Publish to schema sqlc cloud"
+    runs-on: "ubuntu-latest"
+    needs: release
+    if: ${{ !inputs.dry_run }}
+    steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
+        with:
+          egress-policy: audit
+
+      - name: Checkout
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
+        with:
+          fetch-depth: 1
+
+      # We need golang to run the migration main.go
+      - name: Setup Go
+        uses: ./.github/actions/setup-go
+
+      - name: Setup sqlc
+        uses: ./.github/actions/setup-sqlc
+
+      - name: Push schema to sqlc cloud
+        # Don't block a release on this
+        continue-on-error: true
+        run: |
+          make sqlc-push

.github/workflows/scorecard.yml

@@ -0,0 +1,52 @@
+name: OpenSSF Scorecard
+on:
+  branch_protection_rule:
+  schedule:
+    - cron: "27 7 * * 3" # A random time to run weekly
+  push:
+    branches: ["main"]
+
+permissions: read-all
+
+jobs:
+  analysis:
+    name: Scorecard analysis
+    runs-on: ubuntu-latest
+    permissions:
+      # Needed to upload the results to code-scanning dashboard.
+      security-events: write
+      # Needed to publish results and get a badge (see publish_results below).
+      id-token: write
+
+    steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
+        with:
+          egress-policy: audit
+
+      - name: "Checkout code"
+        uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
+        with:
+          persist-credentials: false
+
+      - name: "Run analysis"
+        uses: ossf/scorecard-action@62b2cac7ed8198b15735ed49ab1e5cf35480ba46 # v2.4.0
+        with:
+          results_file: results.sarif
+          results_format: sarif
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_results: true
+
+      # Upload the results as artifacts.
+      - name: "Upload artifact"
+        uses: actions/upload-artifact@604373da6381bf24206979c74d06a550515601b9 # v4.4.1
+        with:
+          name: SARIF file
+          path: results.sarif
+          retention-days: 5
+
+      # Upload the results to GitHub's code scanning dashboard.
+      - name: "Upload to code-scanning"
+        uses: github/codeql-action/upload-sarif@f09c1c0a94de965c15400f5634aa42fac8fb8f88 # v3.27.5
+        with:
+          sarif_file: results.sarif

.github/workflows/security.yaml

@@ -3,14 +3,16 @@ name: "security"
 permissions:
   actions: read
   contents: read
-  security-events: write
 
 on:
   workflow_dispatch:
 
+  # Uncomment when testing.
+  # pull_request:
+
   schedule:
     # Run every 6 hours Monday-Friday!
-    - cron: "0 0,6,12,18 * * 1-5"
+    - cron: "0 0/6 * * 1-5"
 
 # Cancel in-progress runs for pull requests when developers push
 # additional changes
@@ -20,30 +22,25 @@ concurrency:
 
 jobs:
   codeql:
-    runs-on: ${{ github.repository_owner == 'coder' && 'ubuntu-latest-8-cores' || 'ubuntu-latest' }}
+    permissions:
+      security-events: write
+    runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-8' || 'ubuntu-latest' }}
     steps:
-      - uses: actions/checkout@v3
-
-      - name: Initialize CodeQL
-        uses: github/codeql-action/init@v2
+      - name: Harden Runner
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
         with:
-          languages: go, javascript
+          egress-policy: audit
+
+      - name: Checkout
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
 
       - name: Setup Go
-        uses: actions/setup-go@v4
-        with:
-          go-version: "~1.20"
+        uses: ./.github/actions/setup-go
 
-      - name: Go Cache Paths
-        id: go-cache-paths
-        run: |
-          echo "GOMODCACHE=$(go env GOMODCACHE)" >> $GITHUB_OUTPUT
-
-      - name: Go Mod Cache
-        uses: actions/cache@v3
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@f09c1c0a94de965c15400f5634aa42fac8fb8f88 # v3.27.5
         with:
-          path: ${{ steps.go-cache-paths.outputs.GOMODCACHE }}
-          key: ${{ runner.os }}-release-go-mod-${{ hashFiles('**/go.sum') }}
+          languages: go, javascript
 
       # Workaround to prevent CodeQL from building the dashboard.
       - name: Remove Makefile
@@ -51,7 +48,7 @@ jobs:
           rm Makefile
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v2
+        uses: github/codeql-action/analyze@f09c1c0a94de965c15400f5634aa42fac8fb8f88 # v3.27.5
 
       - name: Send Slack notification on failure
         if: ${{ failure() }}
@@ -65,59 +62,56 @@ jobs:
             "${{ secrets.SLACK_SECURITY_FAILURE_WEBHOOK_URL }}"
 
   trivy:
-    runs-on: ${{ github.repository_owner == 'coder' && 'ubuntu-latest-8-cores' || 'ubuntu-latest' }}
+    permissions:
+      security-events: write
+    runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-8' || 'ubuntu-latest' }}
     steps:
-      - uses: actions/checkout@v3
+      - name: Harden Runner
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
+        with:
+          egress-policy: audit
+
+      - name: Checkout
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
         with:
           fetch-depth: 0
 
-      - uses: actions/setup-go@v4
-        with:
-          go-version: "~1.20"
+      - name: Setup Go
+        uses: ./.github/actions/setup-go
 
-      - name: Go Cache Paths
-        id: go-cache-paths
-        run: |
-          echo "GOMODCACHE=$(go env GOMODCACHE)" >> $GITHUB_OUTPUT
+      - name: Setup Node
+        uses: ./.github/actions/setup-node
 
-      - name: Go Mod Cache
-        uses: actions/cache@v3
-        with:
-          path: ${{ steps.go-cache-paths.outputs.GOMODCACHE }}
-          key: ${{ runner.os }}-release-go-mod-${{ hashFiles('**/go.sum') }}
+      - name: Setup sqlc
+        uses: ./.github/actions/setup-sqlc
 
-      - name: Cache Node
-        id: cache-node
-        uses: actions/cache@v3
-        with:
-          path: |
-            **/node_modules
-            .eslintcache
-          key: js-${{ runner.os }}-test-${{ hashFiles('**/yarn.lock') }}
-          restore-keys: |
-            js-${{ runner.os }}-
-
-      - name: Install sqlc
-        run: |
-          curl -sSL https://github.com/kyleconroy/sqlc/releases/download/v1.17.2/sqlc_1.17.2_linux_amd64.tar.gz | sudo tar -C /usr/bin -xz sqlc
       - name: Install yq
         run: go run github.com/mikefarah/yq/v4@v4.30.6
+      - name: Install mockgen
+        run: go install go.uber.org/mock/mockgen@v0.4.0
       - name: Install protoc-gen-go
-        run: go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.26
+        run: go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.30
       - name: Install protoc-gen-go-drpc
-        run: go install storj.io/drpc/cmd/protoc-gen-go-drpc@v0.0.26
+        run: go install storj.io/drpc/cmd/protoc-gen-go-drpc@v0.0.33
       - name: Install Protoc
         run: |
           # protoc must be in lockstep with our dogfood Dockerfile or the
           # version in the comments will differ. This is also defined in
           # ci.yaml.
-          set -x
-          cd dogfood
+          set -euxo pipefail
+          cd dogfood/contents
+          mkdir -p /usr/local/bin
+          mkdir -p /usr/local/include
+
           DOCKER_BUILDKIT=1 docker build . --target proto -t protoc
           protoc_path=/usr/local/bin/protoc
           docker run --rm --entrypoint cat protoc /tmp/bin/protoc > $protoc_path
           chmod +x $protoc_path
           protoc --version
+          # Copy the generated files to the include directory.
+          docker run --rm -v /usr/local/include:/target protoc cp -r /tmp/include/google /target/
+          ls -la /usr/local/include/google/protobuf/
+          stat /usr/local/include/google/protobuf/timestamp.proto
 
       - name: Build Coder linux amd64 Docker image
         id: build
@@ -136,11 +130,13 @@ jobs:
           # the registry.
           export CODER_IMAGE_BUILD_BASE_TAG="$(CODER_IMAGE_BASE=coder-base ./scripts/image_tag.sh --version "$version")"
 
-          make -j "$image_job"
+          # We would like to use make -j here, but it doesn't work with the some recent additions
+          # to our code generation.
+          make "$image_job"
           echo "image=$(cat "$image_job")" >> $GITHUB_OUTPUT
 
       - name: Run Trivy vulnerability scanner
-        uses: aquasecurity/trivy-action@e5f43133f6e8736992c9f3c1b3296e24b37e17f2
+        uses: aquasecurity/trivy-action@18f2510ee396bbf400402947b394f2dd8c87dbb0
         with:
           image-ref: ${{ steps.build.outputs.image }}
           format: sarif
@@ -148,13 +144,13 @@ jobs:
           severity: "CRITICAL,HIGH"
 
       - name: Upload Trivy scan results to GitHub Security tab
-        uses: github/codeql-action/upload-sarif@v2
+        uses: github/codeql-action/upload-sarif@f09c1c0a94de965c15400f5634aa42fac8fb8f88 # v3.27.5
         with:
           sarif_file: trivy-results.sarif
           category: "Trivy"
 
       - name: Upload Trivy scan results as an artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@604373da6381bf24206979c74d06a550515601b9 # v4.4.1
         with:
           name: trivy
           path: trivy-results.sarif
@@ -163,7 +159,7 @@ jobs:
       - name: Send Slack notification on failure
         if: ${{ failure() }}
         run: |
-          msg="❌ CodeQL Failed\n\nhttps://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+          msg="❌ Trivy Failed\n\nhttps://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
           curl \
             -qfsSL \
             -X POST \

.github/workflows/stale.yaml

@@ -1,44 +1,109 @@
-name: Stale Issue and Branch Cleanup
+name: Stale Issue, Branch and Old Workflows Cleanup
 on:
   schedule:
     # Every day at midnight
     - cron: "0 0 * * *"
   workflow_dispatch:
+
+permissions:
+  contents: read
+
 jobs:
   issues:
     runs-on: ubuntu-latest
     permissions:
+      # Needed to close issues.
       issues: write
+      # Needed to close PRs.
       pull-requests: write
     steps:
-      - uses: actions/stale@v8.0.0
+      - name: Harden Runner
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
+        with:
+          egress-policy: audit
+
+      - name: stale
+        uses: actions/stale@28ca1036281a5e5922ead5184a1bbf96e5fc984e # v9.0.0
         with:
           stale-issue-label: "stale"
           stale-pr-label: "stale"
-          days-before-stale: 90
+          # days-before-stale: 180
+          # essentially disabled for now while we work through polish issues
+          days-before-stale: 3650
+
           # Pull Requests become stale more quickly due to merge conflicts.
           # Also, we promote minimizing WIP.
           days-before-pr-stale: 7
           days-before-pr-close: 3
-          stale-pr-message: >
-            This Pull Request is becoming stale. In order to minimize WIP,
-            prevent merge conflicts and keep the tracker readable, I'm going
-            close to this PR in 3 days if there isn't more activity.
-          stale-issue-message: >
-            This issue is becoming stale. In order to keep the tracker readable
-            and actionable, I'm going close to this issue in 7 days if there
-            isn't more activity.
+          # We rarely take action in response to the message, so avoid
+          # cluttering the issue and just close the oldies.
+          stale-pr-message: ""
+          stale-issue-message: ""
           # Upped from 30 since we have a big tracker and was hitting the limit.
           operations-per-run: 60
           # Start with the oldest issues, always.
           ascending: true
+      - name: "Close old issues labeled likely-no"
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const thirtyDaysAgo = new Date(new Date().setDate(new Date().getDate() - 30));
+            console.log(`Looking for issues labeled with 'likely-no' more than 30 days ago, which is after ${thirtyDaysAgo.toISOString()}`);
+
+            const issues = await github.rest.issues.listForRepo({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              labels: 'likely-no',
+              state: 'open',
+            });
+
+            console.log(`Found ${issues.data.length} open issues labeled with 'likely-no'`);
+
+            for (const issue of issues.data) {
+              console.log(`Checking issue #${issue.number} created at ${issue.created_at}`);
+
+              const timeline = await github.rest.issues.listEventsForTimeline({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: issue.number,
+              });
+
+              const labelEvent = timeline.data.find(event => event.event === 'labeled' && event.label.name === 'likely-no');
+
+              if (labelEvent) {
+                console.log(`Issue #${issue.number} was labeled with 'likely-no' at ${labelEvent.created_at}`);
+
+                if (new Date(labelEvent.created_at) < thirtyDaysAgo) {
+                  console.log(`Issue #${issue.number} is older than 30 days with 'likely-no' label, closing issue.`);
+                  await github.rest.issues.update({
+                    owner: context.repo.owner,
+                    repo: context.repo.repo,
+                    issue_number: issue.number,
+                    state: 'closed',
+                    state_reason: 'not_planned'
+                  });
+                }
+              } else {
+                console.log(`Issue #${issue.number} does not have a 'likely-no' label event in its timeline.`);
+              }
+            }
+
   branches:
     runs-on: ubuntu-latest
+    permissions:
+      # Needed to delete branches.
+      contents: write
     steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
+        with:
+          egress-policy: audit
+
       - name: Checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
       - name: Run delete-old-branches-action
-        uses: beatlabs/delete-old-branches-action@v0.0.9
+        uses: beatlabs/delete-old-branches-action@6e94df089372a619c01ae2c2f666bf474f890911 # v0.0.10
         with:
           repo_token: ${{ github.token }}
           date: "6 months ago"
@@ -46,3 +111,31 @@ jobs:
           delete_tags: false
           # extra_protected_branch_regex: ^(foo|bar)$
           exclude_open_pr_branches: true
+  del_runs:
+    runs-on: ubuntu-latest
+    permissions:
+      # Needed to delete workflow runs.
+      actions: write
+    steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
+        with:
+          egress-policy: audit
+
+      - name: Delete PR Cleanup workflow runs
+        uses: Mattraks/delete-workflow-runs@39f0bbed25d76b34de5594dceab824811479e5de # v2.0.6
+        with:
+          token: ${{ github.token }}
+          repository: ${{ github.repository }}
+          retain_days: 30
+          keep_minimum_runs: 30
+          delete_workflow_pattern: pr-cleanup.yaml
+
+      - name: Delete PR Deploy workflow skipped runs
+        uses: Mattraks/delete-workflow-runs@39f0bbed25d76b34de5594dceab824811479e5de # v2.0.6
+        with:
+          token: ${{ github.token }}
+          repository: ${{ github.repository }}
+          retain_days: 30
+          keep_minimum_runs: 30
+          delete_workflow_pattern: pr-deploy.yaml

.github/workflows/typos.toml

@@ -4,6 +4,7 @@ Jetbrains = "JetBrains"
 IST = "IST"
 MacOS = "macOS"
 AKS = "AKS"
+O_WRONLY = "O_WRONLY"
 
 [default.extend-words]
 AKS = "AKS"
@@ -13,16 +14,34 @@ darcula = "darcula"
 Hashi = "Hashi"
 trialer = "trialer"
 encrypter = "encrypter"
+# as in helsinki
+hel = "hel"
+# this is used as proto node
+pn = "pn"
+# typos doesn't like the EDE in TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA
+EDE = "EDE"
+# HELO is an SMTP command
+HELO = "HELO"
+LKE = "LKE"
+byt = "byt"
 
 [files]
 extend-exclude = [
-    "**.svg",
-    "**.png",
-    "**.lock",
-    "go.sum",
-    "go.mod",
-    # These files contain base64 strings that confuse the detector
-    "**XService**.ts",
-    "**identity.go",
-    "scripts/ci-report/testdata/**",
+	"**.svg",
+	"**.png",
+	"**.lock",
+	"go.sum",
+	"go.mod",
+	# These files contain base64 strings that confuse the detector
+	"**XService**.ts",
+	"**identity.go",
+	"scripts/ci-report/testdata/**",
+	"**/*_test.go",
+	"**/*.test.tsx",
+	"**/pnpm-lock.yaml",
+	"tailnet/testdata/**",
+	"site/src/pages/SetupPage/countries.tsx",
+	"provisioner/terraform/testdata/**",
+	# notifications' golden files confuse the detector because of quoted-printable encoding
+	"coderd/notifications/testdata/**"
 ]

.github/workflows/weekly-docs.yaml

@@ -0,0 +1,46 @@
+name: weekly-docs
+# runs every monday at 9 am
+on:
+  schedule:
+    - cron: "0 9 * * 1"
+  workflow_dispatch: # allows to run manually for testing
+  pull_request:
+    branches:
+      - main
+    paths:
+      - "docs/**"
+
+permissions:
+  contents: read
+
+jobs:
+  check-docs:
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: write # required to post PR review comments by the action
+    steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
+        with:
+          egress-policy: audit
+
+      - name: Checkout
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
+
+      - name: Check Markdown links
+        uses: umbrelladocs/action-linkspector@fc382e19892aca958e189954912fe379a8df270c # v1.2.4
+        id: markdown-link-check
+        # checks all markdown files from /docs including all subfolders
+        with:
+          reporter: github-pr-review
+          config_file: ".github/.linkspector.yml"
+          fail_on_error: "true"
+          filter_mode: "nofilter"
+
+      - name: Send Slack notification
+        if: failure() && github.event_name == 'schedule'
+        run: |
+          curl -X POST -H 'Content-type: application/json' -d '{"msg":"Broken links found in the documentation. Please check the logs at ${{ env.LOGS_URL }}"}' ${{ secrets.DOCS_LINK_SLACK_WEBHOOK }}
+          echo "Sent Slack notification"
+        env:
+          LOGS_URL: https://github.com/coder/coder/actions/runs/${{ github.run_id }}

.gitignore

@@ -17,27 +17,28 @@ yarn-error.log
 # Allow VSCode recommendations and default settings in project root.
 !/.vscode/extensions.json
 !/.vscode/settings.json
+# Allow code snippets
+!/.vscode/*.code-snippets
 
 # Front-end ignore patterns.
 .next/
-site/**/*.typegen.ts
 site/build-storybook.log
 site/coverage/
 site/storybook-static/
 site/test-results/*
 site/e2e/test-results/*
 site/e2e/states/*.json
+site/e2e/.auth.json
 site/playwright-report/*
 site/.swc
-site/dist/
 
 # Make target for updating golden files (any dir).
 .gen-golden
 
 # Build
-/build/
-/dist/
-site/out/
+build/
+dist/
+out/
 
 # Bundle analysis
 site/stats/
@@ -48,9 +49,30 @@ site/stats/
 *.lock.hcl
 .terraform/
 
-/.coderv2/*
+**/.coderv2/*
 **/__debug_bin
 
 # direnv
 .envrc
 *.test
+
+# Loadtesting
+./scaletest/terraform/.terraform
+./scaletest/terraform/.terraform.lock.hcl
+scaletest/terraform/secrets.tfvars
+.terraform.tfstate.*
+
+# Nix
+result
+
+# Data dumps from unit tests
+**/*.test.sql
+
+# Filebrowser.db
+**/filebrowser.db
+
+# pnpm
+.pnpm-store/
+
+# Zed
+.zed_server

.golangci.yaml

@@ -2,8 +2,19 @@
 # Over time we should try tightening some of these.
 
 linters-settings:
+  dupl:
+    # goal: 100
+    threshold: 412
+
+  exhaustruct:
+    include:
+      # Gradually extend to cover more of the codebase.
+      - 'httpmw\.\w+'
+      # We want to enforce all values are specified when inserting or updating
+      # a database row. Ref: #9936
+      - 'github.com/coder/coder/v2/coderd/database\.[^G][^e][^t]\w+Params'
   gocognit:
-    min-complexity: 46 # Min code complexity (def 30).
+    min-complexity: 300
 
   goconst:
     min-len: 4 # Min length of string consts (def 3).
@@ -54,7 +65,6 @@ linters-settings:
       # - importShadow
       - indexAlloc
       - initClause
-      - ioutilDeprecated
       - mapKey
       - methodExprCall
       # - nestingReduce
@@ -115,9 +125,6 @@ linters-settings:
   goimports:
     local-prefixes: coder.com,cdr.dev,go.coder.com,github.com/cdr,github.com/coder
 
-  gocyclo:
-    min-complexity: 50
-
   importas:
     no-unaliased: true
 
@@ -127,7 +134,8 @@ linters-settings:
       - trialer
 
   nestif:
-    min-complexity: 4 # Min complexity of if statements (def 5, goal 4)
+    # goal: 10
+    min-complexity: 20
 
   revive:
     # see https://github.com/mgechev/revive#available-rules for details.
@@ -187,6 +195,11 @@ linters-settings:
       - name: var-naming
       - name: waitgroup-by-value
 
+  # irrelevant as of Go v1.22: https://go.dev/blog/loopvar-preview
+  govet:
+    disable:
+      - loopclosure
+
 issues:
   # Rules listed here: https://github.com/securego/gosec#available-rules
   exclude-rules:
@@ -195,18 +208,22 @@ issues:
         # We use assertions rather than explicitly checking errors in tests
         - errcheck
         - forcetypeassert
+        - exhaustruct # This is unhelpful in tests.
+    - path: scripts/*
+      linters:
+        - exhaustruct
 
   fix: true
   max-issues-per-linter: 0
   max-same-issues: 0
 
 run:
-  concurrency: 4
   skip-dirs:
     - node_modules
+    - .git
   skip-files:
     - scripts/rules.go
-  timeout: 5m
+  timeout: 10m
 
 # Over time, add more and more linters from
 # https://golangci-lint.run/usage/linters/ as the code improves.
@@ -220,10 +237,16 @@ linters:
     - errcheck
     - errname
     - errorlint
+    - exhaustruct
     - exportloopref
     - forcetypeassert
     - gocritic
-    - gocyclo
+    # gocyclo is may be useful in the future when we start caring
+    # about testing complexity, but for the time being we should
+    # create a good culture around cognitive complexity.
+    # - gocyclo
+    - gocognit
+    - nestif
     - goimports
     - gomodguard
     - gosec
@@ -259,3 +282,4 @@ linters:
     - typecheck
     - unconvert
     - unused
+    - dupl

.prettierignore

@@ -20,27 +20,28 @@ yarn-error.log
 # Allow VSCode recommendations and default settings in project root.
 !/.vscode/extensions.json
 !/.vscode/settings.json
+# Allow code snippets
+!/.vscode/*.code-snippets
 
 # Front-end ignore patterns.
 .next/
-site/**/*.typegen.ts
 site/build-storybook.log
 site/coverage/
 site/storybook-static/
 site/test-results/*
 site/e2e/test-results/*
 site/e2e/states/*.json
+site/e2e/.auth.json
 site/playwright-report/*
 site/.swc
-site/dist/
 
 # Make target for updating golden files (any dir).
 .gen-golden
 
 # Build
-/build/
-/dist/
-site/out/
+build/
+dist/
+out/
 
 # Bundle analysis
 site/stats/
@@ -51,20 +52,45 @@ site/stats/
 *.lock.hcl
 .terraform/
 
-/.coderv2/*
+**/.coderv2/*
 **/__debug_bin
 
 # direnv
 .envrc
 *.test
+
+# Loadtesting
+./scaletest/terraform/.terraform
+./scaletest/terraform/.terraform.lock.hcl
+scaletest/terraform/secrets.tfvars
+.terraform.tfstate.*
+
+# Nix
+result
+
+# Data dumps from unit tests
+**/*.test.sql
+
+# Filebrowser.db
+**/filebrowser.db
+
+# pnpm
+.pnpm-store/
+
+# Zed
+.zed_server
 # .prettierignore.include:
 # Helm templates contain variables that are invalid YAML and can't be formatted
 # by Prettier.
-helm/templates/*.yaml
-
-# Terraform state files used in tests, these are automatically generated.
-# Example: provisioner/terraform/testdata/instance-id/instance-id.tfstate.json
-**/testdata/**/*.tf*.json
+helm/**/templates/*.yaml
 
 # Testdata shouldn't be formatted.
-scripts/apitypings/testdata/**/*.ts
+testdata/
+
+# Ignore generated files
+**/pnpm-lock.yaml
+**/*.gen.json
+
+# Everything in site/ is formatted by Biome. For the rest of the repo though, we
+# need broader language support.
+site/

.prettierignore.include

@@ -1,10 +1,14 @@
 # Helm templates contain variables that are invalid YAML and can't be formatted
 # by Prettier.
-helm/templates/*.yaml
-
-# Terraform state files used in tests, these are automatically generated.
-# Example: provisioner/terraform/testdata/instance-id/instance-id.tfstate.json
-**/testdata/**/*.tf*.json
+helm/**/templates/*.yaml
 
 # Testdata shouldn't be formatted.
-scripts/apitypings/testdata/**/*.ts
+testdata/
+
+# Ignore generated files
+**/pnpm-lock.yaml
+**/*.gen.json
+
+# Everything in site/ is formatted by Biome. For the rest of the repo though, we
+# need broader language support.
+site/

.prettierrc.yaml

@@ -1,16 +1,18 @@
 # This config file is used in conjunction with `.editorconfig` to specify
 # formatting for prettier-supported files. See `.editorconfig` and
-# `site/.editorconfig`for whitespace formatting options.
+# `site/.editorconfig` for whitespace formatting options.
 printWidth: 80
-semi: false
+proseWrap: always
 trailingComma: all
+useTabs: true
+tabWidth: 2
 overrides:
   - files:
       - README.md
+      - docs/reference/api/**/*.md
+      - docs/reference/cli/**/*.md
+      - docs/changelogs/*.md
+      - .github/**/*.{yaml,yml,toml}
+      - scripts/**/*.{yaml,yml,toml}
     options:
       proseWrap: preserve
-  - files:
-      - "site/**/*.yaml"
-      - "site/**/*.yml"
-    options:
-      proseWrap: always

.swaggo

@@ -1,8 +1,8 @@
 // Replace all NullTime with string
-replace github.com/coder/coder/codersdk.NullTime string
+replace github.com/coder/coder/v2/codersdk.NullTime string
 // Prevent swaggo from rendering enums for time.Duration
 replace time.Duration int64
 // Do not expose "echo" provider
-replace github.com/coder/coder/codersdk.ProvisionerType string
+replace github.com/coder/coder/v2/codersdk.ProvisionerType string
 // Do not render netip.Addr
 replace netip.Addr string

.vscode/extensions.json

@@ -1,15 +1,16 @@
 {
-  "recommendations": [
-    "github.vscode-codeql",
-    "golang.go",
-    "hashicorp.terraform",
-    "esbenp.prettier-vscode",
-    "foxundermoon.shell-format",
-    "emeraldwalk.runonsave",
-    "zxh404.vscode-proto3",
-    "redhat.vscode-yaml",
-    "streetsidesoftware.code-spell-checker",
-    "dbaeumer.vscode-eslint",
-    "EditorConfig.EditorConfig"
-  ]
+	"recommendations": [
+		"github.vscode-codeql",
+		"golang.go",
+		"hashicorp.terraform",
+		"esbenp.prettier-vscode",
+		"foxundermoon.shell-format",
+		"emeraldwalk.runonsave",
+		"zxh404.vscode-proto3",
+		"redhat.vscode-yaml",
+		"tekumara.typos-vscode",
+		"EditorConfig.EditorConfig",
+		"biomejs.biome",
+		"bradlc.vscode-tailwindcss"
+	]
 }

.vscode/markdown.code-snippets

@@ -0,0 +1,38 @@
+{
+	// For info about snippets, visit https://code.visualstudio.com/docs/editor/userdefinedsnippets
+
+    "admonition": {
+        "prefix": "#callout",
+        "body": [
+            "<blockquote class=\"admonition ${1|caution,important,note,tip,warning|}\">\n",
+            "${TM_SELECTED_TEXT:${2:add info here}}\n",
+            "</blockquote>\n"
+        ],
+        "description": "callout admonition caution info note tip warning"
+    },
+	"fenced code block": {
+		"prefix": "#codeblock",
+		"body": ["```${1|apache,bash,console,diff,Dockerfile,env,go,hcl,ini,json,lisp,md,powershell,shell,sql,text,tf,tsx,yaml|}", "${TM_SELECTED_TEXT}$0", "```"],
+		"description": "fenced code block"
+	},
+	"image": {
+		"prefix": "#image",
+		"body": "![${TM_SELECTED_TEXT:${1:alt}}](${2:url})$0",
+		"description": "image"
+	},
+	"tabs": {
+		"prefix": "#tabs",
+		"body": [
+			"<div class=\"tabs\">\n",
+			"${1:optional description}\n",
+			"## ${2:tab title}\n",
+			"${TM_SELECTED_TEXT:${3:first tab content}}\n",
+			"## ${4:tab title}\n",
+			"${5:second tab content}\n",
+			"## ${6:tab title}\n",
+			"${7:third tab content}\n",
+			"</div>\n"
+		],
+		"description": "tabs"
+	}
+}

.vscode/settings.json

@@ -1,220 +1,61 @@
 {
-  "cSpell.words": [
-    "afero",
-    "agentsdk",
-    "apps",
-    "ASKPASS",
-    "authcheck",
-    "autostop",
-    "awsidentity",
-    "bodyclose",
-    "buildinfo",
-    "buildname",
-    "circbuf",
-    "cliflag",
-    "cliui",
-    "codecov",
-    "coderd",
-    "coderdenttest",
-    "coderdtest",
-    "codersdk",
-    "cronstrue",
-    "databasefake",
-    "dbtype",
-    "DERP",
-    "derphttp",
-    "derpmap",
-    "devel",
-    "devtunnel",
-    "dflags",
-    "drpc",
-    "drpcconn",
-    "drpcmux",
-    "drpcserver",
-    "Dsts",
-    "embeddedpostgres",
-    "enablements",
-    "enterprisemeta",
-    "errgroup",
-    "eventsourcemock",
-    "Failf",
-    "fatih",
-    "Formik",
-    "gitauth",
-    "gitsshkey",
-    "goarch",
-    "gographviz",
-    "goleak",
-    "gonet",
-    "gossh",
-    "gsyslog",
-    "GTTY",
-    "hashicorp",
-    "hclsyntax",
-    "httpapi",
-    "httpmw",
-    "idtoken",
-    "Iflag",
-    "incpatch",
-    "ipnstate",
-    "isatty",
-    "Jobf",
-    "Keygen",
-    "kirsle",
-    "Kubernetes",
-    "ldflags",
-    "magicsock",
-    "manifoldco",
-    "mapstructure",
-    "mattn",
-    "mitchellh",
-    "moby",
-    "namesgenerator",
-    "namespacing",
-    "netaddr",
-    "netip",
-    "netmap",
-    "netns",
-    "netstack",
-    "nettype",
-    "nfpms",
-    "nhooyr",
-    "nmcfg",
-    "nolint",
-    "nosec",
-    "ntqry",
-    "OIDC",
-    "oneof",
-    "opty",
-    "paralleltest",
-    "parameterscopeid",
-    "pqtype",
-    "prometheusmetrics",
-    "promhttp",
-    "protobuf",
-    "provisionerd",
-    "provisionerdserver",
-    "provisionersdk",
-    "ptty",
-    "ptys",
-    "ptytest",
-    "quickstart",
-    "reconfig",
-    "replicasync",
-    "retrier",
-    "rpty",
-    "SCIM",
-    "sdkproto",
-    "sdktrace",
-    "Signup",
-    "slogtest",
-    "sourcemapped",
-    "Srcs",
-    "stdbuf",
-    "stretchr",
-    "STTY",
-    "stuntest",
-    "tanstack",
-    "tailbroker",
-    "tailcfg",
-    "tailexchange",
-    "tailnet",
-    "tailnettest",
-    "Tailscale",
-    "tbody",
-    "TCGETS",
-    "tcpip",
-    "TCSETS",
-    "templateversions",
-    "testdata",
-    "testid",
-    "testutil",
-    "tfexec",
-    "tfjson",
-    "tfplan",
-    "tfstate",
-    "thead",
-    "tios",
-    "tmpdir",
-    "tokenconfig",
-    "tparallel",
-    "trialer",
-    "trimprefix",
-    "tsdial",
-    "tslogger",
-    "tstun",
-    "turnconn",
-    "typegen",
-    "typesafe",
-    "unconvert",
-    "Untar",
-    "Userspace",
-    "VMID",
-    "walkthrough",
-    "weblinks",
-    "webrtc",
-    "wgcfg",
-    "wgconfig",
-    "wgengine",
-    "wgmonitor",
-    "wgnet",
-    "workspaceagent",
-    "workspaceagents",
-    "workspaceapp",
-    "workspaceapps",
-    "workspacebuilds",
-    "workspacename",
-    "wsconncache",
-    "wsjson",
-    "xerrors",
-    "xstate",
-    "yamux"
-  ],
-  "cSpell.ignorePaths": ["site/package.json", ".vscode/settings.json"],
-  "emeraldwalk.runonsave": {
-    "commands": [
-      {
-        "match": "database/queries/*.sql",
-        "cmd": "make gen"
-      },
-      {
-        "match": "provisionerd/proto/provisionerd.proto",
-        "cmd": "make provisionerd/proto/provisionerd.pb.go"
-      }
-    ]
-  },
-  "eslint.workingDirectories": ["./site"],
-  "files.exclude": {
-    "**/node_modules": true
-  },
-  "search.exclude": {
-    "scripts/metricsdocgen/metrics": true,
-    "docs/api/*.md": true
-  },
-  // Ensure files always have a newline.
-  "files.insertFinalNewline": true,
-  "go.lintTool": "golangci-lint",
-  "go.lintFlags": ["--fast"],
-  "go.lintOnSave": "package",
-  "go.coverOnSave": true,
-  "go.coverageDecorator": {
-    "type": "gutter",
-    "coveredGutterStyle": "blockgreen",
-    "uncoveredGutterStyle": "blockred"
-  },
-  // The codersdk is used by coderd another other packages extensively.
-  // To reduce redundancy in tests, it's covered by other packages.
-  // Since package coverage pairing can't be defined, all packages cover
-  // all other packages.
-  "go.testFlags": ["-short", "-coverpkg=./..."],
-  // We often use a version of TypeScript that's ahead of the version shipped
-  // with VS Code.
-  "typescript.tsdk": "./site/node_modules/typescript/lib",
-  "grammarly.selectors": [
-    {
-      "language": "markdown",
-      "scheme": "file",
-      "pattern": "docs/contributing/frontend.md"
-    }
-  ]
+	"emeraldwalk.runonsave": {
+		"commands": [
+			{
+				"match": "database/queries/*.sql",
+				"cmd": "make gen"
+			},
+			{
+				"match": "provisionerd/proto/provisionerd.proto",
+				"cmd": "make provisionerd/proto/provisionerd.pb.go"
+			}
+		]
+	},
+	"search.exclude": {
+		"**.pb.go": true,
+		"**/*.gen.json": true,
+		"**/testdata/*": true,
+		"coderd/apidoc/**": true,
+		"docs/reference/api/*.md": true,
+		"docs/reference/cli/*.md": true,
+		"docs/templates/*.md": true,
+		"LICENSE": true,
+		"scripts/metricsdocgen/metrics": true,
+		"site/out/**": true,
+		"site/storybook-static/**": true,
+		"**.map": true,
+		"pnpm-lock.yaml": true
+	},
+	// Ensure files always have a newline.
+	"files.insertFinalNewline": true,
+	"go.lintTool": "golangci-lint",
+	"go.lintFlags": ["--fast"],
+	"go.coverageDecorator": {
+		"type": "gutter",
+		"coveredGutterStyle": "blockgreen",
+		"uncoveredGutterStyle": "blockred"
+	},
+	// The codersdk is used by coderd another other packages extensively.
+	// To reduce redundancy in tests, it's covered by other packages.
+	// Since package coverage pairing can't be defined, all packages cover
+	// all other packages.
+	"go.testFlags": ["-short", "-coverpkg=./..."],
+	// We often use a version of TypeScript that's ahead of the version shipped
+	// with VS Code.
+	"typescript.tsdk": "./site/node_modules/typescript/lib",
+	// Playwright tests in VSCode will open a browser to live "view" the test.
+	"playwright.reuseBrowser": true,
+
+	"[javascript][javascriptreact][json][jsonc][typescript][typescriptreact]": {
+		"editor.defaultFormatter": "biomejs.biome",
+		"editor.codeActionsOnSave": {
+			"quickfix.biome": "explicit"
+			//   "source.organizeImports.biome": "explicit"
+		}
+	},
+
+	"[css][html][markdown][yaml]": {
+		"editor.defaultFormatter": "esbenp.prettier-vscode"
+	},
+	"typos.config": ".github/workflows/typos.toml"
 }

CODEOWNERS

@@ -0,0 +1,6 @@
+# These APIs are versioned, so any changes need to be carefully reviewed for whether
+# to bump API major or minor versions.
+agent/proto/ @spikecurtis @johnstcn
+tailnet/proto/ @spikecurtis @johnstcn
+vpn/vpn.proto @spikecurtis @johnstcn
+vpn/version.go @spikecurtis @johnstcn

CODE_OF_CONDUCT.md

@@ -0,0 +1 @@
+[https://coder.com/docs/coder-oss/latest/contributing/CODE_OF_CONDUCT](https://coder.com/docs/contributing/CODE_OF_CONDUCT)

Makefile

@@ -36,6 +36,7 @@ GOOS         := $(shell go env GOOS)
 GOARCH       := $(shell go env GOARCH)
 GOOS_BIN_EXT := $(if $(filter windows, $(GOOS)),.exe,)
 VERSION      := $(shell ./scripts/version.sh)
+POSTGRES_VERSION ?= 16
 
 # Use the highest ZSTD compression level in CI.
 ifdef CI
@@ -50,11 +51,14 @@ endif
 # Note, all find statements should be written with `.` or `./path` as
 # the search path so that these exclusions match.
 FIND_EXCLUSIONS= \
-	-not \( \( -path '*/.git/*' -o -path './build/*' -o -path './vendor/*' -o -path './.coderv2/*' -o -path '*/node_modules/*' -o -path './site/out/*' \) -prune \)
+	-not \( \( -path '*/.git/*' -o -path './build/*' -o -path './vendor/*' -o -path './.coderv2/*' -o -path '*/node_modules/*' -o -path '*/out/*' -o -path './coderd/apidoc/*' -o -path '*/.next/*' -o -path '*/.terraform/*' \) -prune \)
 # Source files used for make targets, evaluated on use.
-GO_SRC_FILES = $(shell find . $(FIND_EXCLUSIONS) -type f -name '*.go')
+GO_SRC_FILES := $(shell find . $(FIND_EXCLUSIONS) -type f -name '*.go' -not -name '*_test.go')
 # All the shell files in the repo, excluding ignored files.
-SHELL_SRC_FILES = $(shell find . $(FIND_EXCLUSIONS) -type f -name '*.sh')
+SHELL_SRC_FILES := $(shell find . $(FIND_EXCLUSIONS) -type f -name '*.sh')
+
+# Ensure we don't use the user's git configs which might cause side-effects
+GIT_FLAGS = GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_SYSTEM=/dev/null
 
 # All ${OS}_${ARCH} combos we build for. Windows binaries have the .exe suffix.
 OS_ARCHES := \
@@ -75,8 +79,12 @@ PACKAGE_OS_ARCHES := linux_amd64 linux_armv7 linux_arm64
 # All architectures we build Docker images for (Linux only).
 DOCKER_ARCHES := amd64 arm64 armv7
 
+# All ${OS}_${ARCH} combos we build the desktop dylib for.
+DYLIB_ARCHES := darwin_amd64 darwin_arm64
+
 # Computed variables based on the above.
 CODER_SLIM_BINARIES      := $(addprefix build/coder-slim_$(VERSION)_,$(OS_ARCHES))
+CODER_DYLIBS             := $(foreach os_arch, $(DYLIB_ARCHES), build/coder-vpn_$(VERSION)_$(os_arch).dylib)
 CODER_FAT_BINARIES       := $(addprefix build/coder_$(VERSION)_,$(OS_ARCHES))
 CODER_ALL_BINARIES       := $(CODER_SLIM_BINARIES) $(CODER_FAT_BINARIES)
 CODER_TAR_GZ_ARCHIVES    := $(foreach os_arch, $(ARCHIVE_TAR_GZ), build/coder_$(VERSION)_$(os_arch).tar.gz)
@@ -107,9 +115,9 @@ endif
 
 
 clean:
-	rm -rf build site/out
-	mkdir -p build site/out/bin
-	git restore site/out
+	rm -rf build/ site/build/ site/out/
+	mkdir -p build/ site/out/bin/
+	git restore site/out/
 .PHONY: clean
 
 build-slim: $(CODER_SLIM_BINARIES)
@@ -200,7 +208,8 @@ endef
 # calling this manually.
 $(CODER_ALL_BINARIES): go.mod go.sum \
 	$(GO_SRC_FILES) \
-	$(shell find ./examples/templates)
+	$(shell find ./examples/templates) \
+	site/static/error.html
 
 	$(get-mode-os-arch-ext)
 	if [[ "$$os" != "windows" ]] && [[ "$$ext" != "" ]]; then
@@ -233,6 +242,25 @@ $(CODER_ALL_BINARIES): go.mod go.sum \
 		cp "$@" "./site/out/bin/coder-$$os-$$arch$$dot_ext"
 	fi
 
+# This task builds Coder Desktop dylibs
+$(CODER_DYLIBS): go.mod go.sum $(GO_SRC_FILES)
+	@if [ "$(shell uname)" = "Darwin" ]; then
+		$(get-mode-os-arch-ext)
+		./scripts/build_go.sh \
+			--os "$$os" \
+			--arch "$$arch" \
+			--version "$(VERSION)" \
+			--output "$@" \
+			--dylib
+
+	else
+		echo "ERROR: Can't build dylib on non-Darwin OS" 1>&2
+		exit 1
+	fi
+
+# This task builds both dylibs
+build/coder-dylib: $(CODER_DYLIBS)
+
 # This task builds all archives. It parses the target name to get the metadata
 # for the build, so it must be specified in this format:
 #     build/coder_${version}_${os}_${arch}.${format}
@@ -344,21 +372,35 @@ push/$(CODER_MAIN_IMAGE): $(CODER_MAIN_IMAGE)
 	docker manifest push "$$image_tag"
 .PHONY: push/$(CODER_MAIN_IMAGE)
 
+# Helm charts that are available
+charts = coder provisioner
+
 # Shortcut for Helm chart package.
-build/coder_helm.tgz: build/coder_helm_$(VERSION).tgz
+$(foreach chart,$(charts),build/$(chart)_helm.tgz): build/%_helm.tgz: build/%_helm_$(VERSION).tgz
 	rm -f "$@"
 	ln "$<" "$@"
 
 # Helm chart package.
-build/coder_helm_$(VERSION).tgz:
+$(foreach chart,$(charts),build/$(chart)_helm_$(VERSION).tgz): build/%_helm_$(VERSION).tgz:
 	./scripts/helm.sh \
 		--version "$(VERSION)" \
+		--chart $* \
 		--output "$@"
 
 site/out/index.html: site/package.json $(shell find ./site $(FIND_EXCLUSIONS) -type f \( -name '*.ts' -o -name '*.tsx' \))
-	./scripts/yarn_install.sh
 	cd site
-	yarn build
+	# prevents this directory from getting to big, and causing "too much data" errors
+	rm -rf out/assets/
+	../scripts/pnpm_install.sh
+	pnpm build
+
+offlinedocs/out/index.html: $(shell find ./offlinedocs $(FIND_EXCLUSIONS) -type f) $(shell find ./docs $(FIND_EXCLUSIONS) -type f | sed 's: :\\ :g')
+	cd offlinedocs
+	../scripts/pnpm_install.sh
+	pnpm export
+
+build/coder_docs_$(VERSION).tgz: offlinedocs/out/index.html
+	tar -czf "$@" -C offlinedocs/out .
 
 install: build/coder_$(VERSION)_$(GOOS)_$(GOARCH)$(GOOS_BIN_EXT)
 	install_dir="$$(go env GOPATH)/bin"
@@ -368,32 +410,49 @@ install: build/coder_$(VERSION)_$(GOOS)_$(GOARCH)$(GOOS_BIN_EXT)
 	cp "$<" "$$output_file"
 .PHONY: install
 
-fmt: fmt/prettier fmt/terraform fmt/shfmt fmt/go
+BOLD := $(shell tput bold 2>/dev/null)
+GREEN := $(shell tput setaf 2 2>/dev/null)
+RESET := $(shell tput sgr0 2>/dev/null)
+
+fmt: fmt/ts fmt/go fmt/terraform fmt/shfmt fmt/prettier
 .PHONY: fmt
 
 fmt/go:
+	go mod tidy
+	echo "$(GREEN)==>$(RESET) $(BOLD)fmt/go$(RESET)"
 	# VS Code users should check out
 	# https://github.com/mvdan/gofumpt#visual-studio-code
 	go run mvdan.cc/gofumpt@v0.4.0 -w -l .
 .PHONY: fmt/go
 
-fmt/prettier:
-	echo "--- prettier"
+fmt/ts:
+	echo "$(GREEN)==>$(RESET) $(BOLD)fmt/ts$(RESET)"
 	cd site
 # Avoid writing files in CI to reduce file write activity
 ifdef CI
-	yarn run format:check
+	pnpm run check --linter-enabled=false
 else
-	yarn run format:write
+	pnpm run check:fix
+endif
+.PHONY: fmt/ts
+
+fmt/prettier: .prettierignore
+	echo "$(GREEN)==>$(RESET) $(BOLD)fmt/prettier$(RESET)"
+# Avoid writing files in CI to reduce file write activity
+ifdef CI
+	pnpm run format:check
+else
+	pnpm run format
 endif
 .PHONY: fmt/prettier
 
 fmt/terraform: $(wildcard *.tf)
+	echo "$(GREEN)==>$(RESET) $(BOLD)fmt/terraform$(RESET)"
 	terraform fmt -recursive
 .PHONY: fmt/terraform
 
 fmt/shfmt: $(SHELL_SRC_FILES)
-	echo "--- shfmt"
+	echo "$(GREEN)==>$(RESET) $(BOLD)fmt/shfmt$(RESET)"
 # Only do diff check in CI, errors on diff.
 ifdef CI
 	shfmt -d $(SHELL_SRC_FILES)
@@ -402,59 +461,114 @@ else
 endif
 .PHONY: fmt/shfmt
 
-lint: lint/shellcheck lint/go
+lint: lint/shellcheck lint/go lint/ts lint/examples lint/helm lint/site-icons
 .PHONY: lint
 
+lint/site-icons:
+	./scripts/check_site_icons.sh
+.PHONY: lint/site-icons
+
+lint/ts:
+	cd site
+	pnpm lint
+.PHONY: lint/ts
+
 lint/go:
 	./scripts/check_enterprise_imports.sh
-	golangci-lint run
+	./scripts/check_codersdk_imports.sh
+	linter_ver=$(shell egrep -o 'GOLANGCI_LINT_VERSION=\S+' dogfood/contents/Dockerfile | cut -d '=' -f 2)
+	go run github.com/golangci/golangci-lint/cmd/golangci-lint@v$$linter_ver run
 .PHONY: lint/go
 
+lint/examples:
+	go run ./scripts/examplegen/main.go -lint
+.PHONY: lint/examples
+
 # Use shfmt to determine the shell files, takes editorconfig into consideration.
 lint/shellcheck: $(SHELL_SRC_FILES)
 	echo "--- shellcheck"
 	shellcheck --external-sources $(SHELL_SRC_FILES)
 .PHONY: lint/shellcheck
 
+lint/helm:
+	cd helm
+	make lint
+.PHONY: lint/helm
+
+# All files generated by the database should be added here, and this can be used
+# as a target for jobs that need to run after the database is generated.
+DB_GEN_FILES := \
+	coderd/database/querier.go \
+	coderd/database/unique_constraint.go \
+	coderd/database/dbmem/dbmem.go \
+	coderd/database/dbmetrics/dbmetrics.go \
+	coderd/database/dbauthz/dbauthz.go \
+	coderd/database/dbmock/dbmock.go
+
+TAILNETTEST_MOCKS := \
+	tailnet/tailnettest/coordinatormock.go \
+	tailnet/tailnettest/coordinateemock.go \
+	tailnet/tailnettest/workspaceupdatesprovidermock.go \
+	tailnet/tailnettest/subscriptionmock.go
+
+
 # all gen targets should be added here and to gen/mark-fresh
 gen: \
-	coderd/database/dump.sql \
-	coderd/database/querier.go \
+	tailnet/proto/tailnet.pb.go \
+	agent/proto/agent.pb.go \
 	provisionersdk/proto/provisioner.pb.go \
 	provisionerd/proto/provisionerd.pb.go \
+	vpn/vpn.pb.go \
+	coderd/database/dump.sql \
+	$(DB_GEN_FILES) \
 	site/src/api/typesGenerated.ts \
 	coderd/rbac/object_gen.go \
-	docs/admin/prometheus.md \
-	docs/cli.md \
-	docs/admin/audit-logs.md \
+	codersdk/rbacresources_gen.go \
+	site/src/api/rbacresourcesGenerated.ts \
+	site/src/api/countriesGenerated.ts \
+	docs/admin/integrations/prometheus.md \
+	docs/reference/cli/index.md \
+	docs/admin/security/audit-logs.md \
 	coderd/apidoc/swagger.json \
 	.prettierignore.include \
 	.prettierignore \
-	site/.prettierrc.yaml \
-	site/.prettierignore \
-	site/.eslintignore
+	provisioner/terraform/testdata/version \
+	site/e2e/provisionerGenerated.ts \
+	site/src/theme/icons.json \
+	examples/examples.gen.json \
+	$(TAILNETTEST_MOCKS) \
+	coderd/database/pubsub/psmock/psmock.go
 .PHONY: gen
 
 # Mark all generated files as fresh so make thinks they're up-to-date. This is
 # used during releases so we don't run generation scripts.
 gen/mark-fresh:
 	files="\
-		coderd/database/dump.sql \
-		coderd/database/querier.go \
+		tailnet/proto/tailnet.pb.go \
+		agent/proto/agent.pb.go \
 		provisionersdk/proto/provisioner.pb.go \
 		provisionerd/proto/provisionerd.pb.go \
+		vpn/vpn.pb.go \
+		coderd/database/dump.sql \
+		$(DB_GEN_FILES) \
 		site/src/api/typesGenerated.ts \
 		coderd/rbac/object_gen.go \
-		docs/admin/prometheus.md \
-		docs/cli.md \
-		docs/admin/audit-logs.md \
+		codersdk/rbacresources_gen.go \
+		site/src/api/rbacresourcesGenerated.ts \
+		site/src/api/countriesGenerated.ts \
+		docs/admin/integrations/prometheus.md \
+		docs/reference/cli/index.md \
+		docs/admin/security/audit-logs.md \
 		coderd/apidoc/swagger.json \
 		.prettierignore.include \
 		.prettierignore \
-		site/.prettierrc.yaml \
-		site/.prettierignore \
-		site/.eslintignore \
-	"
+		site/e2e/provisionerGenerated.ts \
+		site/src/theme/icons.json \
+		examples/examples.gen.json \
+		$(TAILNETTEST_MOCKS) \
+		coderd/database/pubsub/psmock/psmock.go \
+		"
+
 	for file in $$files; do
 		echo "$$file"
 		if [ ! -f "$$file" ]; then
@@ -473,9 +587,36 @@ coderd/database/dump.sql: coderd/database/gen/dump/main.go $(wildcard coderd/dat
 	go run ./coderd/database/gen/dump/main.go
 
 # Generates Go code for querying the database.
-coderd/database/querier.go: coderd/database/sqlc.yaml coderd/database/dump.sql $(wildcard coderd/database/queries/*.sql) coderd/database/gen/enum/main.go
+# coderd/database/queries.sql.go
+# coderd/database/models.go
+coderd/database/querier.go: coderd/database/sqlc.yaml coderd/database/dump.sql $(wildcard coderd/database/queries/*.sql)
 	./coderd/database/generate.sh
 
+coderd/database/dbmock/dbmock.go: coderd/database/db.go coderd/database/querier.go
+	go generate ./coderd/database/dbmock/
+
+coderd/database/pubsub/psmock/psmock.go: coderd/database/pubsub/pubsub.go
+	go generate ./coderd/database/pubsub/psmock
+
+$(TAILNETTEST_MOCKS): tailnet/coordinator.go tailnet/service.go
+	go generate ./tailnet/tailnettest/
+
+tailnet/proto/tailnet.pb.go: tailnet/proto/tailnet.proto
+	protoc \
+		--go_out=. \
+		--go_opt=paths=source_relative \
+		--go-drpc_out=. \
+		--go-drpc_opt=paths=source_relative \
+		./tailnet/proto/tailnet.proto
+
+agent/proto/agent.pb.go: agent/proto/agent.proto
+	protoc \
+		--go_out=. \
+		--go_opt=paths=source_relative \
+		--go-drpc_out=. \
+		--go-drpc_opt=paths=source_relative \
+		./agent/proto/agent.proto
+
 provisionersdk/proto/provisioner.pb.go: provisionersdk/proto/provisioner.proto
 	protoc \
 		--go_out=. \
@@ -492,65 +633,123 @@ provisionerd/proto/provisionerd.pb.go: provisionerd/proto/provisionerd.proto
 		--go-drpc_opt=paths=source_relative \
 		./provisionerd/proto/provisionerd.proto
 
-site/src/api/typesGenerated.ts: scripts/apitypings/main.go $(shell find ./codersdk $(FIND_EXCLUSIONS) -type f -name '*.go')
-	go run scripts/apitypings/main.go > site/src/api/typesGenerated.ts
+vpn/vpn.pb.go: vpn/vpn.proto
+	protoc \
+		--go_out=. \
+		--go_opt=paths=source_relative \
+		./vpn/vpn.proto
+
+site/src/api/typesGenerated.ts: $(wildcard scripts/apitypings/*) $(shell find ./codersdk $(FIND_EXCLUSIONS) -type f -name '*.go')
+	go run ./scripts/apitypings/ > $@
+	./scripts/pnpm_install.sh
+
+site/e2e/provisionerGenerated.ts: provisionerd/proto/provisionerd.pb.go provisionersdk/proto/provisioner.pb.go
 	cd site
-	yarn run format:types
+	../scripts/pnpm_install.sh
+	pnpm run gen:provisioner
 
-coderd/rbac/object_gen.go: scripts/rbacgen/main.go coderd/rbac/object.go
-	go run scripts/rbacgen/main.go ./coderd/rbac > coderd/rbac/object_gen.go
+site/src/theme/icons.json: $(wildcard scripts/gensite/*) $(wildcard site/static/icon/*)
+	go run ./scripts/gensite/ -icons "$@"
+	./scripts/pnpm_install.sh
+	pnpm -C site/ exec biome format --write src/theme/icons.json
 
-docs/admin/prometheus.md: scripts/metricsdocgen/main.go scripts/metricsdocgen/metrics
+examples/examples.gen.json: scripts/examplegen/main.go examples/examples.go $(shell find ./examples/templates)
+	go run ./scripts/examplegen/main.go > examples/examples.gen.json
+
+coderd/rbac/object_gen.go: scripts/typegen/rbacobject.gotmpl scripts/typegen/main.go coderd/rbac/object.go coderd/rbac/policy/policy.go
+	go run scripts/typegen/main.go rbac object > coderd/rbac/object_gen.go
+
+codersdk/rbacresources_gen.go: scripts/typegen/codersdk.gotmpl scripts/typegen/main.go coderd/rbac/object.go coderd/rbac/policy/policy.go
+	# Do no overwrite codersdk/rbacresources_gen.go directly, as it would make the file empty, breaking
+ 	# the `codersdk` package and any parallel build targets.
+	go run scripts/typegen/main.go rbac codersdk > /tmp/rbacresources_gen.go
+	mv /tmp/rbacresources_gen.go codersdk/rbacresources_gen.go
+
+site/src/api/rbacresourcesGenerated.ts: scripts/typegen/codersdk.gotmpl scripts/typegen/main.go coderd/rbac/object.go coderd/rbac/policy/policy.go
+	go run scripts/typegen/main.go rbac typescript > "$@"
+
+site/src/api/countriesGenerated.ts: scripts/typegen/countries.tstmpl scripts/typegen/main.go codersdk/countries.go
+	go run scripts/typegen/main.go countries > "$@"
+
+docs/admin/integrations/prometheus.md: scripts/metricsdocgen/main.go scripts/metricsdocgen/metrics
 	go run scripts/metricsdocgen/main.go
-	cd site
-	yarn run format:write:only ../docs/admin/prometheus.md
+	./scripts/pnpm_install.sh
+	pnpm exec prettier --write ./docs/admin/integrations/prometheus.md
 
-docs/cli.md: scripts/clidocgen/main.go $(GO_SRC_FILES) docs/manifest.json
-	BASE_PATH="." go run ./scripts/clidocgen
-	cd site
-	yarn run format:write:only ../docs/cli.md ../docs/cli/*.md ../docs/manifest.json
+docs/reference/cli/index.md: scripts/clidocgen/main.go examples/examples.gen.json $(GO_SRC_FILES)
+	CI=true BASE_PATH="." go run ./scripts/clidocgen
+	./scripts/pnpm_install.sh
+	pnpm exec prettier --write ./docs/reference/cli/index.md ./docs/reference/cli/*.md ./docs/manifest.json
 
-docs/admin/audit-logs.md: scripts/auditdocgen/main.go enterprise/audit/table.go coderd/rbac/object_gen.go
+docs/admin/security/audit-logs.md: coderd/database/querier.go scripts/auditdocgen/main.go enterprise/audit/table.go coderd/rbac/object_gen.go
 	go run scripts/auditdocgen/main.go
-	cd site
-	yarn run format:write:only ../docs/admin/audit-logs.md
+	./scripts/pnpm_install.sh
+	pnpm exec prettier --write ./docs/admin/security/audit-logs.md
 
-coderd/apidoc/swagger.json: $(shell find ./scripts/apidocgen $(FIND_EXCLUSIONS) -type f) $(wildcard coderd/*.go) $(wildcard enterprise/coderd/*.go) $(wildcard codersdk/*.go) $(wildcard enterprise/wsproxy/wsproxysdk/*.go) coderd/database/querier.go .swaggo docs/manifest.json coderd/rbac/object_gen.go
+coderd/apidoc/swagger.json: $(shell find ./scripts/apidocgen $(FIND_EXCLUSIONS) -type f) $(wildcard coderd/*.go) $(wildcard enterprise/coderd/*.go) $(wildcard codersdk/*.go) $(wildcard enterprise/wsproxy/wsproxysdk/*.go) $(DB_GEN_FILES) .swaggo docs/manifest.json coderd/rbac/object_gen.go
 	./scripts/apidocgen/generate.sh
-	yarn run --cwd=site format:write:only ../docs/api ../docs/manifest.json ../coderd/apidoc/swagger.json
+	./scripts/pnpm_install.sh
+	pnpm exec prettier --write ./docs/reference/api ./docs/manifest.json ./coderd/apidoc/swagger.json
 
-update-golden-files: cli/testdata/.gen-golden helm/tests/testdata/.gen-golden scripts/ci-report/testdata/.gen-golden
+update-golden-files: \
+	cli/testdata/.gen-golden \
+	helm/coder/tests/testdata/.gen-golden \
+	helm/provisioner/tests/testdata/.gen-golden \
+	scripts/ci-report/testdata/.gen-golden \
+	enterprise/cli/testdata/.gen-golden \
+	enterprise/tailnet/testdata/.gen-golden \
+	tailnet/testdata/.gen-golden \
+	coderd/.gen-golden \
+	coderd/notifications/.gen-golden \
+	provisioner/terraform/testdata/.gen-golden
 .PHONY: update-golden-files
 
-cli/testdata/.gen-golden: $(wildcard cli/testdata/*.golden) $(wildcard cli/*.tpl) $(GO_SRC_FILES)
-	go test ./cli -run="Test(CommandHelp|ServerYAML)" -update
+cli/testdata/.gen-golden: $(wildcard cli/testdata/*.golden) $(wildcard cli/*.tpl) $(GO_SRC_FILES) $(wildcard cli/*_test.go)
+	go test ./cli -run="Test(CommandHelp|ServerYAML|ErrorExamples)" -update
 	touch "$@"
 
-helm/tests/testdata/.gen-golden: $(wildcard helm/tests/testdata/*.yaml) $(wildcard helm/tests/testdata/*.golden) $(GO_SRC_FILES)
-	go test ./helm/tests -run=TestUpdateGoldenFiles -update
+enterprise/cli/testdata/.gen-golden: $(wildcard enterprise/cli/testdata/*.golden) $(wildcard cli/*.tpl) $(GO_SRC_FILES) $(wildcard enterprise/cli/*_test.go)
+	go test ./enterprise/cli -run="TestEnterpriseCommandHelp" -update
 	touch "$@"
 
+tailnet/testdata/.gen-golden: $(wildcard tailnet/testdata/*.golden.html) $(GO_SRC_FILES) $(wildcard tailnet/*_test.go)
+	go test ./tailnet -run="TestDebugTemplate" -update
+	touch "$@"
+
+enterprise/tailnet/testdata/.gen-golden: $(wildcard enterprise/tailnet/testdata/*.golden.html) $(GO_SRC_FILES) $(wildcard enterprise/tailnet/*_test.go)
+	go test ./enterprise/tailnet -run="TestDebugTemplate" -update
+	touch "$@"
+
+helm/coder/tests/testdata/.gen-golden: $(wildcard helm/coder/tests/testdata/*.yaml) $(wildcard helm/coder/tests/testdata/*.golden) $(GO_SRC_FILES) $(wildcard helm/coder/tests/*_test.go)
+	go test ./helm/coder/tests -run=TestUpdateGoldenFiles -update
+	touch "$@"
+
+helm/provisioner/tests/testdata/.gen-golden: $(wildcard helm/provisioner/tests/testdata/*.yaml) $(wildcard helm/provisioner/tests/testdata/*.golden) $(GO_SRC_FILES) $(wildcard helm/provisioner/tests/*_test.go)
+	go test ./helm/provisioner/tests -run=TestUpdateGoldenFiles -update
+	touch "$@"
+
+coderd/.gen-golden: $(wildcard coderd/testdata/*/*.golden) $(GO_SRC_FILES) $(wildcard coderd/*_test.go)
+	go test ./coderd -run="Test.*Golden$$" -update
+	touch "$@"
+
+coderd/notifications/.gen-golden: $(wildcard coderd/notifications/testdata/*/*.golden) $(GO_SRC_FILES) $(wildcard coderd/notifications/*_test.go)
+	go test ./coderd/notifications -run="Test.*Golden$$" -update
+	touch "$@"
+
+provisioner/terraform/testdata/.gen-golden: $(wildcard provisioner/terraform/testdata/*/*.golden) $(GO_SRC_FILES) $(wildcard provisioner/terraform/*_test.go)
+	go test ./provisioner/terraform -run="Test.*Golden$$" -update
+	touch "$@"
+
+provisioner/terraform/testdata/version:
+	if [[ "$(shell cat provisioner/terraform/testdata/version.txt)" != "$(shell terraform version -json | jq -r '.terraform_version')" ]]; then
+		./provisioner/terraform/testdata/generate.sh
+	fi
+.PHONY: provisioner/terraform/testdata/version
+
 scripts/ci-report/testdata/.gen-golden: $(wildcard scripts/ci-report/testdata/*) $(wildcard scripts/ci-report/*.go)
 	go test ./scripts/ci-report -run=TestOutputMatchesGoldenFile -update
 	touch "$@"
 
-# Generate a prettierrc for the site package that uses relative paths for
-# overrides. This allows us to share the same prettier config between the
-# site and the root of the repo.
-site/.prettierrc.yaml: .prettierrc.yaml
-	. ./scripts/lib.sh
-	dependencies yq
-
-	echo "# Code generated by Makefile (../$<). DO NOT EDIT." > "$@"
-	echo "" >> "$@"
-
-	# Replace all listed override files with relative paths inside site/.
-	# - ./ -> ../
-	# - ./site -> ./
-	yq \
-		'.overrides[].files |= map(. | sub("^./"; "") | sub("^"; "../") | sub("../site/"; "./"))' \
-		"$<" >> "$@"
-
 # Combine .gitignore with .prettierignore.include to generate .prettierignore.
 .prettierignore: .gitignore .prettierignore.include
 	echo "# Code generated by Makefile ($^). DO NOT EDIT." > "$@"
@@ -560,61 +759,66 @@ site/.prettierrc.yaml: .prettierrc.yaml
 		cat "$$f" >> "$@"
 	done
 
-# Generate ignore files based on gitignore into the site directory. We turn all
-# rules into relative paths for the `site/` directory (where applicable),
-# following the pattern format defined by git:
-# https://git-scm.com/docs/gitignore#_pattern_format
-#
-# This is done for compatibility reasons, see:
-# https://github.com/prettier/prettier/issues/8048
-# https://github.com/prettier/prettier/issues/8506
-# https://github.com/prettier/prettier/issues/8679
-site/.eslintignore site/.prettierignore: .prettierignore Makefile
-	rm -f "$@"
-	touch "$@"
-	# Skip generated by header, inherit `.prettierignore` header as-is.
-	while read -r rule; do
-		# Remove leading ! if present to simplify rule, added back at the end.
-		tmp="$${rule#!}"
-		ignore="$${rule%"$$tmp"}"
-		rule="$$tmp"
-		case "$$rule" in
-			# Comments or empty lines (include).
-			\#*|'') ;;
-			# Generic rules (include).
-			\*\**) ;;
-			# Site prefixed rules (include).
-			site/*) rule="$${rule#site/}";;
-			./site/*) rule="$${rule#./site/}";;
-			# Rules that are non-generic and don't start with site (rewrite).
-			/*) rule=.."$$rule";;
-			*/?*) rule=../"$$rule";;
-			*) ;;
-		esac
-		echo "$${ignore}$${rule}" >> "$@"
-	done < "$<"
-
-test: test-clean
-	gotestsum -- -v -short ./...
+test:
+	$(GIT_FLAGS) gotestsum --format standard-quiet -- -v -short -count=1 ./...
 .PHONY: test
 
+# sqlc-cloud-is-setup will fail if no SQLc auth token is set. Use this as a
+# dependency for any sqlc-cloud related targets.
+sqlc-cloud-is-setup:
+	if [[ "$(SQLC_AUTH_TOKEN)" == "" ]]; then
+		echo "ERROR: 'SQLC_AUTH_TOKEN' must be set to auth with sqlc cloud before running verify." 1>&2
+		exit 1
+	fi
+.PHONY: sqlc-cloud-is-setup
+
+sqlc-push: sqlc-cloud-is-setup test-postgres-docker
+	echo "--- sqlc push"
+	SQLC_DATABASE_URL="postgresql://postgres:postgres@localhost:5432/$(shell go run scripts/migrate-ci/main.go)" \
+	sqlc push -f coderd/database/sqlc.yaml && echo "Passed sqlc push"
+.PHONY: sqlc-push
+
+sqlc-verify: sqlc-cloud-is-setup test-postgres-docker
+	echo "--- sqlc verify"
+	SQLC_DATABASE_URL="postgresql://postgres:postgres@localhost:5432/$(shell go run scripts/migrate-ci/main.go)" \
+	sqlc verify -f coderd/database/sqlc.yaml && echo "Passed sqlc verify"
+.PHONY: sqlc-verify
+
+sqlc-vet: test-postgres-docker
+	echo "--- sqlc vet"
+	SQLC_DATABASE_URL="postgresql://postgres:postgres@localhost:5432/$(shell go run scripts/migrate-ci/main.go)" \
+	sqlc vet -f coderd/database/sqlc.yaml && echo "Passed sqlc vet"
+.PHONY: sqlc-vet
+
 # When updating -timeout for this test, keep in sync with
 # test-go-postgres (.github/workflows/coder.yaml).
-test-postgres: test-clean test-postgres-docker
+# Do add coverage flags so that test caching works.
+test-postgres: test-postgres-docker
 	# The postgres test is prone to failure, so we limit parallelism for
 	# more consistent execution.
-	DB=ci DB_FROM=$(shell go run scripts/migrate-ci/main.go) gotestsum \
+	$(GIT_FLAGS)  DB=ci gotestsum \
 		--junitfile="gotests.xml" \
 		--jsonfile="gotests.json" \
 		--packages="./..." -- \
-		-covermode=atomic -coverprofile="gotests.coverage" -timeout=20m \
-		-parallel=4 \
-		-coverpkg=./... \
-		-count=1 -race -failfast
+		-timeout=20m \
+		-failfast \
+		-count=1
 .PHONY: test-postgres
 
+test-migrations: test-postgres-docker
+	echo "--- test migrations"
+	set -euo pipefail
+	COMMIT_FROM=$(shell git log -1 --format='%h' HEAD)
+	echo "COMMIT_FROM=$${COMMIT_FROM}"
+	COMMIT_TO=$(shell git log -1 --format='%h' origin/main)
+	echo "COMMIT_TO=$${COMMIT_TO}"
+	if [[ "$${COMMIT_FROM}" == "$${COMMIT_TO}" ]]; then echo "Nothing to do!"; exit 0; fi
+	echo "DROP DATABASE IF EXISTS migrate_test_$${COMMIT_FROM}; CREATE DATABASE migrate_test_$${COMMIT_FROM};" | psql 'postgresql://postgres:postgres@localhost:5432/postgres?sslmode=disable'
+	go run ./scripts/migrate-test/main.go --from="$$COMMIT_FROM" --to="$$COMMIT_TO" --postgres-url="postgresql://postgres:postgres@localhost:5432/migrate_test_$${COMMIT_FROM}?sslmode=disable"
+
+# NOTE: we set --memory to the same size as a GitHub runner.
 test-postgres-docker:
-	docker rm -f test-postgres-docker || true
+	docker rm -f test-postgres-docker-${POSTGRES_VERSION} || true
 	docker run \
 		--env POSTGRES_PASSWORD=postgres \
 		--env POSTGRES_USER=postgres \
@@ -622,11 +826,14 @@ test-postgres-docker:
 		--env PGDATA=/tmp \
 		--tmpfs /tmp \
 		--publish 5432:5432 \
-		--name test-postgres-docker \
+		--name test-postgres-docker-${POSTGRES_VERSION} \
 		--restart no \
 		--detach \
-		postgres:13 \
+		--memory 16GB \
+		gcr.io/coder-dev-1/postgres:${POSTGRES_VERSION} \
 		-c shared_buffers=1GB \
+		-c work_mem=1GB \
+		-c effective_cache_size=1GB \
 		-c max_connections=1000 \
 		-c fsync=off \
 		-c synchronous_commit=off \
@@ -639,6 +846,30 @@ test-postgres-docker:
 	done
 .PHONY: test-postgres-docker
 
+# Make sure to keep this in sync with test-go-race from .github/workflows/ci.yaml.
+test-race:
+	$(GIT_FLAGS) gotestsum --junitfile="gotests.xml" -- -race -count=1 -parallel 4 -p 4 ./...
+.PHONY: test-race
+
+test-tailnet-integration:
+	env \
+		CODER_TAILNET_TESTS=true \
+		CODER_MAGICSOCK_DEBUG_LOGGING=true \
+		TS_DEBUG_NETCHECK=true \
+		GOTRACEBACK=single \
+		go test \
+			-exec "sudo -E" \
+			-timeout=5m \
+			-count=1 \
+			./tailnet/test/integration
+
+# Note: we used to add this to the test target, but it's not necessary and we can
+# achieve the desired result by specifying -count=1 in the go test invocation
+# instead. Keeping it here for convenience.
 test-clean:
 	go clean -testcache
 .PHONY: test-clean
+
+.PHONY: test-e2e
+test-e2e:
+	cd ./site && DEBUG=pw:api pnpm playwright:test --forbid-only --workers 1

README.md

@@ -7,7 +7,7 @@
   </a>
 
   <h1>
-  Self-Hosted Remote Development Environments
+  Self-Hosted Cloud Development Environments
   </h1>
 
   <a href="https://coder.com#gh-light-mode-only">
@@ -20,20 +20,21 @@
   <br>
   <br>
 
-[Quickstart](#quickstart) | [Docs](https://coder.com/docs) | [Why Coder](https://coder.com/why) | [Enterprise](https://coder.com/docs/v2/latest/enterprise)
+[Quickstart](#quickstart) | [Docs](https://coder.com/docs) | [Why Coder](https://coder.com/why) | [Premium](https://coder.com/pricing#compare-plans)
 
 [![discord](https://img.shields.io/discord/747933592273027093?label=discord)](https://discord.gg/coder)
-[![codecov](https://codecov.io/gh/coder/coder/branch/main/graph/badge.svg?token=TNLW3OAP6G)](https://codecov.io/gh/coder/coder)
 [![release](https://img.shields.io/github/v/release/coder/coder)](https://github.com/coder/coder/releases/latest)
 [![godoc](https://pkg.go.dev/badge/github.com/coder/coder.svg)](https://pkg.go.dev/github.com/coder/coder)
-[![Go Report Card](https://goreportcard.com/badge/github.com/coder/coder)](https://goreportcard.com/report/github.com/coder/coder)
+[![Go Report Card](https://goreportcard.com/badge/github.com/coder/coder/v2)](https://goreportcard.com/report/github.com/coder/coder/v2)
+[![OpenSSF Best Practices](https://www.bestpractices.dev/projects/9511/badge)](https://www.bestpractices.dev/projects/9511)
+[![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/coder/coder/badge)](https://scorecard.dev/viewer/?uri=github.com%2Fcoder%2Fcoder)
 [![license](https://img.shields.io/github/license/coder/coder)](./LICENSE)
 
 </div>
 
-[Coder](https://coder.com) enables organizations to set up development environments in the cloud. Environments are defined with Terraform, connected through a secure high-speed Wireguard® tunnel, and are automatically shut down when not in use to save on costs. Coder gives engineering teams the flexibility to use the cloud for workloads that are most beneficial to them.
+[Coder](https://coder.com) enables organizations to set up development environments in their public or private cloud infrastructure. Cloud development environments are defined with Terraform, connected through a secure high-speed Wireguard® tunnel, and automatically shut down when not used to save on costs. Coder gives engineering teams the flexibility to use the cloud for workloads most beneficial to them.
 
-- Define development environments in Terraform
+- Define cloud development environments in Terraform
   - EC2 VMs, Kubernetes Pods, Docker Containers, etc.
 - Automatically shutdown idle resources to save on costs
 - Onboard developers in seconds instead of days
@@ -44,7 +45,7 @@
 
 ## Quickstart
 
-The most convenient way to try Coder is to install it on your local machine and experiment with provisioning development environments using Docker (works on Linux, macOS, and Windows).
+The most convenient way to try Coder is to install it on your local machine and experiment with provisioning cloud development environments using Docker (works on Linux, macOS, and Windows).
 
 ```
 # First, install Coder
@@ -53,8 +54,8 @@ curl -L https://coder.com/install.sh | sh
 # Start the Coder server (caches data in ~/.cache/coder)
 coder server
 
-# Navigate to http://localhost:3000 to create your initial user
-# Create a Docker template, and provision a workspace
+# Navigate to http://localhost:3000 to create your initial user,
+# create a Docker template and provision a workspace
 ```
 
 ## Install
@@ -68,13 +69,13 @@ Releases.
 curl -L https://coder.com/install.sh | sh
 ```
 
-You can run the install script with `--dry-run` to see the commands that will be used to install without executing them. You can modify the installation process by including flags. Run the install script with `--help` for reference.
+You can run the install script with `--dry-run` to see the commands that will be used to install without executing them. Run the install script with `--help` for additional flags.
 
-> See [install](docs/install) for additional methods.
+> See [install](https://coder.com/docs/install) for additional methods.
 
-Once installed, you can start a production deployment<sup>1</sup> with a single command:
+Once installed, you can start a production deployment with a single command:
 
-```console
+```shell
 # Automatically sets up an external access URL on *.try.coder.app
 coder server
 
@@ -82,44 +83,49 @@ coder server
 coder server --postgres-url <url> --access-url <url>
 ```
 
-> <sup>1</sup> For production deployments, set up an external PostgreSQL instance for reliability.
-
-Use `coder --help` to get a list of flags and environment variables. Use our [install guides](https://coder.com/docs/v2/latest/install) for a full walkthrough.
+Use `coder --help` to get a list of flags and environment variables. Use our [install guides](https://coder.com/docs/install) for a complete walkthrough.
 
 ## Documentation
 
-Browse our docs [here](https://coder.com/docs/v2) or visit a specific section below:
+Browse our docs [here](https://coder.com/docs) or visit a specific section below:
 
-- [**Templates**](https://coder.com/docs/v2/latest/templates): Templates are written in Terraform and describe the infrastructure for workspaces
-- [**Workspaces**](https://coder.com/docs/v2/latest/workspaces): Workspaces contain the IDEs, dependencies, and configuration information needed for software development
-- [**IDEs**](https://coder.com/docs/v2/latest/ides): Connect your existing editor to a workspace
-- [**Administration**](https://coder.com/docs/v2/latest/admin): Learn how to operate Coder
-- [**Enterprise**](https://coder.com/docs/v2/latest/enterprise): Learn about our paid features built for large teams
+- [**Templates**](https://coder.com/docs/templates): Templates are written in Terraform and describe the infrastructure for workspaces
+- [**Workspaces**](https://coder.com/docs/workspaces): Workspaces contain the IDEs, dependencies, and configuration information needed for software development
+- [**IDEs**](https://coder.com/docs/ides): Connect your existing editor to a workspace
+- [**Administration**](https://coder.com/docs/admin): Learn how to operate Coder
+- [**Premium**](https://coder.com/pricing#compare-plans): Learn about our paid features built for large teams
 
-## Community and Support
+## Support
 
 Feel free to [open an issue](https://github.com/coder/coder/issues/new) if you have questions, run into bugs, or have a feature request.
 
-[Join our Discord](https://discord.gg/coder) to provide feedback on in-progress features, and chat with the community using Coder!
+[Join our Discord](https://discord.gg/coder) to provide feedback on in-progress features and chat with the community using Coder!
 
-## Contributing
+## Integrations
 
-Contributions are welcome! Read the [contributing docs](https://coder.com/docs/v2/latest/CONTRIBUTING) to get started.
-
-Find our list of contributors [here](https://github.com/coder/coder/graphs/contributors).
-
-## Related
-
-We are always working on new integrations. Feel free to open an issue to request an integration. Contributions are welcome in any official or community repositories.
+We are always working on new integrations. Please feel free to open an issue and ask for an integration. Contributions are welcome in any official or community repositories.
 
 ### Official
 
 - [**VS Code Extension**](https://marketplace.visualstudio.com/items?itemName=coder.coder-remote): Open any Coder workspace in VS Code with a single click
 - [**JetBrains Gateway Extension**](https://plugins.jetbrains.com/plugin/19620-coder): Open any Coder workspace in JetBrains Gateway with a single click
+- [**Dev Container Builder**](https://github.com/coder/envbuilder): Build development environments using `devcontainer.json` on Docker, Kubernetes, and OpenShift
+- [**Module Registry**](https://registry.coder.com): Extend development environments with common use-cases
+- [**Kubernetes Log Stream**](https://github.com/coder/coder-logstream-kube): Stream Kubernetes Pod events to the Coder startup logs
 - [**Self-Hosted VS Code Extension Marketplace**](https://github.com/coder/code-marketplace): A private extension marketplace that works in restricted or airgapped networks integrating with [code-server](https://github.com/coder/code-server).
+- [**Setup Coder**](https://github.com/marketplace/actions/setup-coder): An action to setup coder CLI in GitHub workflows.
 
 ### Community
 
 - [**Provision Coder with Terraform**](https://github.com/ElliotG/coder-oss-tf): Provision Coder on Google GKE, Azure AKS, AWS EKS, DigitalOcean DOKS, IBMCloud K8s, OVHCloud K8s, and Scaleway K8s Kapsule with Terraform
-- [**Coder GitHub Action**](https://github.com/marketplace/actions/update-coder-template): A GitHub Action that updates Coder templates
-- [**Various Templates**](./examples/templates/community-templates.md): Hetzner Cloud, Docker in Docker, and other templates the community has built.
+- [**Coder Template GitHub Action**](https://github.com/marketplace/actions/update-coder-template): A GitHub Action that updates Coder templates
+
+## Contributing
+
+We are always happy to see new contributors to Coder. If you are new to the Coder codebase, we have
+[a guide on how to get started](https://coder.com/docs/CONTRIBUTING). We'd love to see your
+contributions!
+
+## Hiring
+
+Apply [here](https://jobs.ashbyhq.com/coder?utm_source=github&utm_medium=readme&utm_campaign=unknown) if you're interested in joining our team.

SECURITY.md

@@ -1,7 +1,7 @@
 # Coder Security
 
-Coder welcomes feedback from security researchers and the general public
-to help improve our security. If you believe you have discovered a vulnerability,
+Coder welcomes feedback from security researchers and the general public to help
+improve our security. If you believe you have discovered a vulnerability,
 privacy issue, exposed data, or other security issues in any of our assets, we
 want to hear from you. This policy outlines steps for reporting vulnerabilities
 to us, what we expect, what you can expect from us.
@@ -10,64 +10,72 @@ You can see the pretty version [here](https://coder.com/security/policy)
 
 # Why Coder's security matters
 
-If an attacker could fully compromise a Coder installation, they could spin
-up expensive workstations, steal valuable credentials, or steal proprietary
-source code. We take this risk very seriously and employ routine pen testing,
-vulnerability scanning, and code reviews. We also welcome the contributions
-from the community that helped make this product possible.
+If an attacker could fully compromise a Coder installation, they could spin up
+expensive workstations, steal valuable credentials, or steal proprietary source
+code. We take this risk very seriously and employ routine pen testing,
+vulnerability scanning, and code reviews. We also welcome the contributions from
+the community that helped make this product possible.
 
 # Where should I report security issues?
 
-Please report security issues to security@coder.com, providing
-all relevant information. The more details you provide, the easier it will be
-for us to triage and fix the issue.
+Please report security issues to security@coder.com, providing all relevant
+information. The more details you provide, the easier it will be for us to
+triage and fix the issue.
 
 # Out of Scope
 
-Our primary concern is around an abuse of the Coder application that allows
-an attacker to gain access to another users workspace, or spin up unwanted
+Our primary concern is around an abuse of the Coder application that allows an
+attacker to gain access to another users workspace, or spin up unwanted
 workspaces.
 
 - DOS/DDOS attacks affecting availability --> While we do support rate limiting
-  of requests, we primarily leave this to the owner of the Coder installation. Our
-  rationale is that a DOS attack only affecting availability is not a valuable
-  target for attackers.
+  of requests, we primarily leave this to the owner of the Coder installation.
+  Our rationale is that a DOS attack only affecting availability is not a
+  valuable target for attackers.
 - Abuse of a compromised user credential --> If a user credential is compromised
-  outside of the Coder ecosystem, then we consider it beyond the scope of our application.
-  However, if an unprivileged user could escalate their permissions or gain access
-  to another workspace, that is a cause for concern.
+  outside of the Coder ecosystem, then we consider it beyond the scope of our
+  application. However, if an unprivileged user could escalate their permissions
+  or gain access to another workspace, that is a cause for concern.
 - Vulnerabilities in third party systems --> Vulnerabilities discovered in
-  out-of-scope systems should be reported to the appropriate vendor or applicable authority.
+  out-of-scope systems should be reported to the appropriate vendor or
+  applicable authority.
 
 # Our Commitments
 
 When working with us, according to this policy, you can expect us to:
 
-- Respond to your report promptly, and work with you to understand and validate your report;
-- Strive to keep you informed about the progress of a vulnerability as it is processed;
-- Work to remediate discovered vulnerabilities in a timely manner, within our operational constraints; and
-- Extend Safe Harbor for your vulnerability research that is related to this policy.
+- Respond to your report promptly, and work with you to understand and validate
+  your report;
+- Strive to keep you informed about the progress of a vulnerability as it is
+  processed;
+- Work to remediate discovered vulnerabilities in a timely manner, within our
+  operational constraints; and
+- Extend Safe Harbor for your vulnerability research that is related to this
+  policy.
 
 # Our Expectations
 
-In participating in our vulnerability disclosure program in good faith, we ask that you:
+In participating in our vulnerability disclosure program in good faith, we ask
+that you:
 
-- Play by the rules, including following this policy and any other relevant agreements.
-  If there is any inconsistency between this policy and any other applicable terms, the
-  terms of this policy will prevail;
+- Play by the rules, including following this policy and any other relevant
+  agreements. If there is any inconsistency between this policy and any other
+  applicable terms, the terms of this policy will prevail;
 - Report any vulnerability you’ve discovered promptly;
-- Avoid violating the privacy of others, disrupting our systems, destroying data, and/or
-  harming user experience;
+- Avoid violating the privacy of others, disrupting our systems, destroying
+  data, and/or harming user experience;
 - Use only the Official Channels to discuss vulnerability information with us;
-- Provide us a reasonable amount of time (at least 90 days from the initial report) to
-  resolve the issue before you disclose it publicly;
-- Perform testing only on in-scope systems, and respect systems and activities which
-  are out-of-scope;
-- If a vulnerability provides unintended access to data: Limit the amount of data you
-  access to the minimum required for effectively demonstrating a Proof of Concept; and
-  cease testing and submit a report immediately if you encounter any user data during testing,
-  such as Personally Identifiable Information (PII), Personal Healthcare Information (PHI),
-  credit card data, or proprietary information;
-- You should only interact with test accounts you own or with explicit permission from
+- Provide us a reasonable amount of time (at least 90 days from the initial
+  report) to resolve the issue before you disclose it publicly;
+- Perform testing only on in-scope systems, and respect systems and activities
+  which are out-of-scope;
+- If a vulnerability provides unintended access to data: Limit the amount of
+  data you access to the minimum required for effectively demonstrating a Proof
+  of Concept; and cease testing and submit a report immediately if you encounter
+  any user data during testing, such as Personally Identifiable Information
+  (PII), Personal Healthcare Information (PHI), credit card data, or proprietary
+  information;
+- You should only interact with test accounts you own or with explicit
+  permission from
 - the account holder; and
 - Do not engage in extortion.

agent/agentexec/cli_linux.go

@@ -0,0 +1,145 @@
+//go:build linux
+// +build linux
+
+package agentexec
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"os/exec"
+	"runtime"
+	"strconv"
+	"strings"
+	"syscall"
+
+	"golang.org/x/sys/unix"
+	"golang.org/x/xerrors"
+)
+
+// unset is set to an invalid value for nice and oom scores.
+const unset = -2000
+
+// CLI runs the agent-exec command. It should only be called by the cli package.
+func CLI() error {
+	// We lock the OS thread here to avoid a race condition where the nice priority
+	// we get is on a different thread from the one we set it on.
+	runtime.LockOSThread()
+	// Nop on success but we do it anyway in case of an error.
+	defer runtime.UnlockOSThread()
+
+	var (
+		fs   = flag.NewFlagSet("agent-exec", flag.ExitOnError)
+		nice = fs.Int("coder-nice", unset, "")
+		oom  = fs.Int("coder-oom", unset, "")
+	)
+
+	if len(os.Args) < 3 {
+		return xerrors.Errorf("malformed command %+v", os.Args)
+	}
+
+	// Parse everything after "coder agent-exec".
+	err := fs.Parse(os.Args[2:])
+	if err != nil {
+		return xerrors.Errorf("parse flags: %w", err)
+	}
+
+	// Get everything after "coder agent-exec --"
+	args := execArgs(os.Args)
+	if len(args) == 0 {
+		return xerrors.Errorf("no exec command provided %+v", os.Args)
+	}
+
+	if *nice == unset {
+		// If an explicit nice score isn't set, we use the default.
+		*nice, err = defaultNiceScore()
+		if err != nil {
+			return xerrors.Errorf("get default nice score: %w", err)
+		}
+	}
+
+	if *oom == unset {
+		// If an explicit oom score isn't set, we use the default.
+		*oom, err = defaultOOMScore()
+		if err != nil {
+			return xerrors.Errorf("get default oom score: %w", err)
+		}
+	}
+
+	err = unix.Setpriority(unix.PRIO_PROCESS, 0, *nice)
+	if err != nil {
+		return xerrors.Errorf("set nice score: %w", err)
+	}
+
+	err = writeOOMScoreAdj(*oom)
+	if err != nil {
+		return xerrors.Errorf("set oom score: %w", err)
+	}
+
+	path, err := exec.LookPath(args[0])
+	if err != nil {
+		return xerrors.Errorf("look path: %w", err)
+	}
+
+	return syscall.Exec(path, args, os.Environ())
+}
+
+func defaultNiceScore() (int, error) {
+	score, err := unix.Getpriority(unix.PRIO_PROCESS, 0)
+	if err != nil {
+		return 0, xerrors.Errorf("get nice score: %w", err)
+	}
+	// See https://linux.die.net/man/2/setpriority#Notes
+	score = 20 - score
+
+	score += 5
+	if score > 19 {
+		return 19, nil
+	}
+	return score, nil
+}
+
+func defaultOOMScore() (int, error) {
+	score, err := oomScoreAdj()
+	if err != nil {
+		return 0, xerrors.Errorf("get oom score: %w", err)
+	}
+
+	// If the agent has a negative oom_score_adj, we set the child to 0
+	// so it's treated like every other process.
+	if score < 0 {
+		return 0, nil
+	}
+
+	// If the agent is already almost at the maximum then set it to the max.
+	if score >= 998 {
+		return 1000, nil
+	}
+
+	// If the agent oom_score_adj is >=0, we set the child to slightly
+	// less than the maximum. If users want a different score they set it
+	// directly.
+	return 998, nil
+}
+
+func oomScoreAdj() (int, error) {
+	scoreStr, err := os.ReadFile("/proc/self/oom_score_adj")
+	if err != nil {
+		return 0, xerrors.Errorf("read oom_score_adj: %w", err)
+	}
+	return strconv.Atoi(strings.TrimSpace(string(scoreStr)))
+}
+
+func writeOOMScoreAdj(score int) error {
+	return os.WriteFile("/proc/self/oom_score_adj", []byte(fmt.Sprintf("%d", score)), 0o600)
+}
+
+// execArgs returns the arguments to pass to syscall.Exec after the "--" delimiter.
+func execArgs(args []string) []string {
+	for i, arg := range args {
+		if arg == "--" {
+			return args[i+1:]
+		}
+	}
+	return nil
+}

agent/agentexec/cli_linux_test.go

@@ -0,0 +1,178 @@
+//go:build linux
+// +build linux
+
+package agentexec_test
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"syscall"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+	"golang.org/x/sys/unix"
+
+	"github.com/coder/coder/v2/testutil"
+)
+
+func TestCLI(t *testing.T) {
+	t.Parallel()
+
+	t.Run("OK", func(t *testing.T) {
+		t.Parallel()
+
+		ctx := testutil.Context(t, testutil.WaitMedium)
+		cmd, path := cmd(ctx, t, 123, 12)
+		err := cmd.Start()
+		require.NoError(t, err)
+		go cmd.Wait()
+
+		waitForSentinel(ctx, t, cmd, path)
+		requireOOMScore(t, cmd.Process.Pid, 123)
+		requireNiceScore(t, cmd.Process.Pid, 12)
+	})
+
+	t.Run("Defaults", func(t *testing.T) {
+		t.Parallel()
+
+		ctx := testutil.Context(t, testutil.WaitMedium)
+		cmd, path := cmd(ctx, t, 0, 0)
+		err := cmd.Start()
+		require.NoError(t, err)
+		go cmd.Wait()
+
+		waitForSentinel(ctx, t, cmd, path)
+
+		expectedNice := expectedNiceScore(t)
+		expectedOOM := expectedOOMScore(t)
+		requireOOMScore(t, cmd.Process.Pid, expectedOOM)
+		requireNiceScore(t, cmd.Process.Pid, expectedNice)
+	})
+}
+
+func requireNiceScore(t *testing.T, pid int, score int) {
+	t.Helper()
+
+	nice, err := unix.Getpriority(unix.PRIO_PROCESS, pid)
+	require.NoError(t, err)
+	// See https://linux.die.net/man/2/setpriority#Notes
+	require.Equal(t, score, 20-nice)
+}
+
+func requireOOMScore(t *testing.T, pid int, expected int) {
+	t.Helper()
+
+	actual, err := os.ReadFile(fmt.Sprintf("/proc/%d/oom_score_adj", pid))
+	require.NoError(t, err)
+	score := strings.TrimSpace(string(actual))
+	require.Equal(t, strconv.Itoa(expected), score)
+}
+
+func waitForSentinel(ctx context.Context, t *testing.T, cmd *exec.Cmd, path string) {
+	t.Helper()
+
+	ticker := time.NewTicker(testutil.IntervalFast)
+	defer ticker.Stop()
+
+	// RequireEventually doesn't work well with require.NoError or similar require functions.
+	for {
+		err := cmd.Process.Signal(syscall.Signal(0))
+		require.NoError(t, err)
+
+		_, err = os.Stat(path)
+		if err == nil {
+			return
+		}
+
+		select {
+		case <-ticker.C:
+		case <-ctx.Done():
+			require.NoError(t, ctx.Err())
+		}
+	}
+}
+
+func cmd(ctx context.Context, t *testing.T, oom, nice int) (*exec.Cmd, string) {
+	var (
+		args = execArgs(oom, nice)
+		dir  = t.TempDir()
+		file = filepath.Join(dir, "sentinel")
+	)
+
+	args = append(args, "sh", "-c", fmt.Sprintf("touch %s && sleep 10m", file))
+	//nolint:gosec
+	cmd := exec.CommandContext(ctx, TestBin, args...)
+
+	// We set this so we can also easily kill the sleep process the shell spawns.
+	cmd.SysProcAttr = &syscall.SysProcAttr{
+		Setpgid: true,
+	}
+
+	cmd.Env = os.Environ()
+	var buf bytes.Buffer
+	cmd.Stdout = &buf
+	cmd.Stderr = &buf
+	t.Cleanup(func() {
+		// Print output of a command if the test fails.
+		if t.Failed() {
+			t.Logf("cmd %q output: %s", cmd.Args, buf.String())
+		}
+		if cmd.Process != nil {
+			// We use -cmd.Process.Pid to kill the whole process group.
+			_ = syscall.Kill(-cmd.Process.Pid, syscall.SIGINT)
+		}
+	})
+	return cmd, file
+}
+
+func expectedOOMScore(t *testing.T) int {
+	t.Helper()
+
+	score, err := os.ReadFile(fmt.Sprintf("/proc/%d/oom_score_adj", os.Getpid()))
+	require.NoError(t, err)
+
+	scoreInt, err := strconv.Atoi(strings.TrimSpace(string(score)))
+	require.NoError(t, err)
+
+	if scoreInt < 0 {
+		return 0
+	}
+	if scoreInt >= 998 {
+		return 1000
+	}
+	return 998
+}
+
+func expectedNiceScore(t *testing.T) int {
+	t.Helper()
+
+	score, err := unix.Getpriority(unix.PRIO_PROCESS, os.Getpid())
+	require.NoError(t, err)
+
+	// Priority is niceness + 20.
+	score = 20 - score
+	score += 5
+	if score > 19 {
+		return 19
+	}
+	return score
+}
+
+func execArgs(oom int, nice int) []string {
+	execArgs := []string{"agent-exec"}
+	if oom != 0 {
+		execArgs = append(execArgs, fmt.Sprintf("--coder-oom=%d", oom))
+	}
+	if nice != 0 {
+		execArgs = append(execArgs, fmt.Sprintf("--coder-nice=%d", nice))
+	}
+	execArgs = append(execArgs, "--")
+	return execArgs
+}

agent/agentexec/cli_other.go

@@ -0,0 +1,10 @@
+//go:build !linux
+// +build !linux
+
+package agentexec
+
+import "golang.org/x/xerrors"
+
+func CLI() error {
+	return xerrors.New("agent-exec is only supported on Linux")
+}

agent/agentexec/cmdtest/main_linux.go

@@ -0,0 +1,19 @@
+//go:build linux
+// +build linux
+
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/coder/coder/v2/agent/agentexec"
+)
+
+func main() {
+	err := agentexec.CLI()
+	if err != nil {
+		_, _ = fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+}

agent/agentexec/exec.go

@@ -0,0 +1,86 @@
+package agentexec
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strconv"
+
+	"golang.org/x/xerrors"
+)
+
+const (
+	// EnvProcPrioMgmt is the environment variable that determines whether
+	// we attempt to manage process CPU and OOM Killer priority.
+	EnvProcPrioMgmt  = "CODER_PROC_PRIO_MGMT"
+	EnvProcOOMScore  = "CODER_PROC_OOM_SCORE"
+	EnvProcNiceScore = "CODER_PROC_NICE_SCORE"
+)
+
+// CommandContext returns an exec.Cmd that calls "coder agent-exec" prior to exec'ing
+// the provided command if CODER_PROC_PRIO_MGMT is set, otherwise a normal exec.Cmd
+// is returned. All instances of exec.Cmd should flow through this function to ensure
+// proper resource constraints are applied to the child process.
+func CommandContext(ctx context.Context, cmd string, args ...string) (*exec.Cmd, error) {
+	_, enabled := os.LookupEnv(EnvProcPrioMgmt)
+	if runtime.GOOS != "linux" || !enabled {
+		return exec.CommandContext(ctx, cmd, args...), nil
+	}
+
+	executable, err := os.Executable()
+	if err != nil {
+		return nil, xerrors.Errorf("get executable: %w", err)
+	}
+
+	bin, err := filepath.EvalSymlinks(executable)
+	if err != nil {
+		return nil, xerrors.Errorf("eval symlinks: %w", err)
+	}
+
+	execArgs := []string{"agent-exec"}
+	if score, ok := envValInt(EnvProcOOMScore); ok {
+		execArgs = append(execArgs, oomScoreArg(score))
+	}
+
+	if score, ok := envValInt(EnvProcNiceScore); ok {
+		execArgs = append(execArgs, niceScoreArg(score))
+	}
+	execArgs = append(execArgs, "--", cmd)
+	execArgs = append(execArgs, args...)
+
+	return exec.CommandContext(ctx, bin, execArgs...), nil
+}
+
+// envValInt searches for a key in a list of environment variables and parses it to an int.
+// If the key is not found or cannot be parsed, returns 0 and false.
+func envValInt(key string) (int, bool) {
+	val, ok := os.LookupEnv(key)
+	if !ok {
+		return 0, false
+	}
+
+	i, err := strconv.Atoi(val)
+	if err != nil {
+		return 0, false
+	}
+	return i, true
+}
+
+// The following are flags used by the agent-exec command. We use flags instead of
+// environment variables to avoid having to deal with a caller overriding the
+// environment variables.
+const (
+	niceFlag = "coder-nice"
+	oomFlag  = "coder-oom"
+)
+
+func niceScoreArg(score int) string {
+	return fmt.Sprintf("--%s=%d", niceFlag, score)
+}
+
+func oomScoreArg(score int) string {
+	return fmt.Sprintf("--%s=%d", oomFlag, score)
+}

agent/agentexec/exec_test.go

@@ -0,0 +1,119 @@
+package agentexec_test
+
+import (
+	"context"
+	"os"
+	"os/exec"
+	"runtime"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/coder/coder/v2/agent/agentexec"
+)
+
+//nolint:paralleltest // we need to test environment variables
+func TestExec(t *testing.T) {
+	//nolint:paralleltest // we need to test environment variables
+	t.Run("NonLinux", func(t *testing.T) {
+		t.Setenv(agentexec.EnvProcPrioMgmt, "true")
+
+		if runtime.GOOS == "linux" {
+			t.Skip("skipping on linux")
+		}
+
+		cmd, err := agentexec.CommandContext(context.Background(), "sh", "-c", "sleep")
+		require.NoError(t, err)
+
+		path, err := exec.LookPath("sh")
+		require.NoError(t, err)
+		require.Equal(t, path, cmd.Path)
+		require.Equal(t, []string{"sh", "-c", "sleep"}, cmd.Args)
+	})
+
+	//nolint:paralleltest // we need to test environment variables
+	t.Run("Linux", func(t *testing.T) {
+		//nolint:paralleltest // we need to test environment variables
+		t.Run("Disabled", func(t *testing.T) {
+			if runtime.GOOS != "linux" {
+				t.Skip("skipping on linux")
+			}
+
+			cmd, err := agentexec.CommandContext(context.Background(), "sh", "-c", "sleep")
+			require.NoError(t, err)
+			path, err := exec.LookPath("sh")
+			require.NoError(t, err)
+			require.Equal(t, path, cmd.Path)
+			require.Equal(t, []string{"sh", "-c", "sleep"}, cmd.Args)
+		})
+
+		//nolint:paralleltest // we need to test environment variables
+		t.Run("Enabled", func(t *testing.T) {
+			t.Setenv(agentexec.EnvProcPrioMgmt, "hello")
+
+			if runtime.GOOS != "linux" {
+				t.Skip("skipping on linux")
+			}
+
+			executable, err := os.Executable()
+			require.NoError(t, err)
+
+			cmd, err := agentexec.CommandContext(context.Background(), "sh", "-c", "sleep")
+			require.NoError(t, err)
+			require.Equal(t, executable, cmd.Path)
+			require.Equal(t, []string{executable, "agent-exec", "--", "sh", "-c", "sleep"}, cmd.Args)
+		})
+
+		t.Run("Nice", func(t *testing.T) {
+			t.Setenv(agentexec.EnvProcPrioMgmt, "hello")
+			t.Setenv(agentexec.EnvProcNiceScore, "10")
+
+			if runtime.GOOS != "linux" {
+				t.Skip("skipping on linux")
+			}
+
+			executable, err := os.Executable()
+			require.NoError(t, err)
+
+			cmd, err := agentexec.CommandContext(context.Background(), "sh", "-c", "sleep")
+			require.NoError(t, err)
+			require.Equal(t, executable, cmd.Path)
+			require.Equal(t, []string{executable, "agent-exec", "--coder-nice=10", "--", "sh", "-c", "sleep"}, cmd.Args)
+		})
+
+		t.Run("OOM", func(t *testing.T) {
+			t.Setenv(agentexec.EnvProcPrioMgmt, "hello")
+			t.Setenv(agentexec.EnvProcOOMScore, "123")
+
+			if runtime.GOOS != "linux" {
+				t.Skip("skipping on linux")
+			}
+
+			executable, err := os.Executable()
+			require.NoError(t, err)
+
+			cmd, err := agentexec.CommandContext(context.Background(), "sh", "-c", "sleep")
+			require.NoError(t, err)
+			require.Equal(t, executable, cmd.Path)
+			require.Equal(t, []string{executable, "agent-exec", "--coder-oom=123", "--", "sh", "-c", "sleep"}, cmd.Args)
+		})
+
+		t.Run("Both", func(t *testing.T) {
+			t.Setenv(agentexec.EnvProcPrioMgmt, "hello")
+			t.Setenv(agentexec.EnvProcOOMScore, "432")
+			t.Setenv(agentexec.EnvProcNiceScore, "14")
+
+			if runtime.GOOS != "linux" {
+				t.Skip("skipping on linux")
+			}
+
+			executable, err := os.Executable()
+			require.NoError(t, err)
+
+			cmd, err := agentexec.CommandContext(context.Background(), "sh", "-c", "sleep")
+			require.NoError(t, err)
+			require.Equal(t, executable, cmd.Path)
+			require.Equal(t, []string{executable, "agent-exec", "--coder-oom=432", "--coder-nice=14", "--", "sh", "-c", "sleep"}, cmd.Args)
+		})
+	})
+}

agent/agentexec/main_linux_test.go

@@ -0,0 +1,46 @@
+//go:build linux
+// +build linux
+
+package agentexec_test
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"testing"
+)
+
+var TestBin string
+
+func TestMain(m *testing.M) {
+	code := func() int {
+		// We generate a unique directory per test invocation to avoid collisions between two
+		// processes attempting to create the same temp file.
+		dir := genDir()
+		defer os.RemoveAll(dir)
+		TestBin = buildBinary(dir)
+		return m.Run()
+	}()
+
+	os.Exit(code)
+}
+
+func buildBinary(dir string) string {
+	path := filepath.Join(dir, "agent-test")
+	out, err := exec.Command("go", "build", "-o", path, "./cmdtest").CombinedOutput()
+	mustf(err, "build binary: %s", out)
+	return path
+}
+
+func mustf(err error, msg string, args ...any) {
+	if err != nil {
+		panic(fmt.Sprintf(msg, args...))
+	}
+}
+
+func genDir() string {
+	dir, err := os.MkdirTemp(os.TempDir(), "agentexec")
+	mustf(err, "create temp dir: %v", err)
+	return dir
+}

agent/agentproc/agentproctest/doc.go

@@ -0,0 +1,5 @@
+// Package agentproctest contains utility functions
+// for testing process management in the agent.
+package agentproctest
+
+//go:generate mockgen -destination ./syscallermock.go -package agentproctest github.com/coder/coder/v2/agent/agentproc Syscaller

agent/agentproc/agentproctest/proc.go

@@ -0,0 +1,55 @@
+package agentproctest
+
+import (
+	"fmt"
+	"strconv"
+	"testing"
+
+	"github.com/spf13/afero"
+	"github.com/stretchr/testify/require"
+
+	"github.com/coder/coder/v2/agent/agentproc"
+	"github.com/coder/coder/v2/cryptorand"
+)
+
+func GenerateProcess(t *testing.T, fs afero.Fs, muts ...func(*agentproc.Process)) agentproc.Process {
+	t.Helper()
+
+	pid, err := cryptorand.Intn(1<<31 - 1)
+	require.NoError(t, err)
+
+	arg1, err := cryptorand.String(5)
+	require.NoError(t, err)
+
+	arg2, err := cryptorand.String(5)
+	require.NoError(t, err)
+
+	arg3, err := cryptorand.String(5)
+	require.NoError(t, err)
+
+	cmdline := fmt.Sprintf("%s\x00%s\x00%s", arg1, arg2, arg3)
+
+	process := agentproc.Process{
+		CmdLine:     cmdline,
+		PID:         int32(pid),
+		OOMScoreAdj: 0,
+	}
+
+	for _, mut := range muts {
+		mut(&process)
+	}
+
+	process.Dir = fmt.Sprintf("%s/%d", "/proc", process.PID)
+
+	err = fs.MkdirAll(process.Dir, 0o555)
+	require.NoError(t, err)
+
+	err = afero.WriteFile(fs, fmt.Sprintf("%s/cmdline", process.Dir), []byte(process.CmdLine), 0o444)
+	require.NoError(t, err)
+
+	score := strconv.Itoa(process.OOMScoreAdj)
+	err = afero.WriteFile(fs, fmt.Sprintf("%s/oom_score_adj", process.Dir), []byte(score), 0o444)
+	require.NoError(t, err)
+
+	return process
+}

agent/agentproc/agentproctest/syscallermock.go

@@ -0,0 +1,83 @@
+// Code generated by MockGen. DO NOT EDIT.
+// Source: github.com/coder/coder/v2/agent/agentproc (interfaces: Syscaller)
+//
+// Generated by this command:
+//
+//	mockgen -destination ./syscallermock.go -package agentproctest github.com/coder/coder/v2/agent/agentproc Syscaller
+//
+
+// Package agentproctest is a generated GoMock package.
+package agentproctest
+
+import (
+	reflect "reflect"
+	syscall "syscall"
+
+	gomock "go.uber.org/mock/gomock"
+)
+
+// MockSyscaller is a mock of Syscaller interface.
+type MockSyscaller struct {
+	ctrl     *gomock.Controller
+	recorder *MockSyscallerMockRecorder
+}
+
+// MockSyscallerMockRecorder is the mock recorder for MockSyscaller.
+type MockSyscallerMockRecorder struct {
+	mock *MockSyscaller
+}
+
+// NewMockSyscaller creates a new mock instance.
+func NewMockSyscaller(ctrl *gomock.Controller) *MockSyscaller {
+	mock := &MockSyscaller{ctrl: ctrl}
+	mock.recorder = &MockSyscallerMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use.
+func (m *MockSyscaller) EXPECT() *MockSyscallerMockRecorder {
+	return m.recorder
+}
+
+// GetPriority mocks base method.
+func (m *MockSyscaller) GetPriority(arg0 int32) (int, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetPriority", arg0)
+	ret0, _ := ret[0].(int)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// GetPriority indicates an expected call of GetPriority.
+func (mr *MockSyscallerMockRecorder) GetPriority(arg0 any) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPriority", reflect.TypeOf((*MockSyscaller)(nil).GetPriority), arg0)
+}
+
+// Kill mocks base method.
+func (m *MockSyscaller) Kill(arg0 int32, arg1 syscall.Signal) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Kill", arg0, arg1)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// Kill indicates an expected call of Kill.
+func (mr *MockSyscallerMockRecorder) Kill(arg0, arg1 any) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Kill", reflect.TypeOf((*MockSyscaller)(nil).Kill), arg0, arg1)
+}
+
+// SetPriority mocks base method.
+func (m *MockSyscaller) SetPriority(arg0 int32, arg1 int) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "SetPriority", arg0, arg1)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// SetPriority indicates an expected call of SetPriority.
+func (mr *MockSyscallerMockRecorder) SetPriority(arg0, arg1 any) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetPriority", reflect.TypeOf((*MockSyscaller)(nil).SetPriority), arg0, arg1)
+}

agent/agentproc/doc.go

@@ -0,0 +1,3 @@
+// Package agentproc contains logic for interfacing with local
+// processes running in the same context as the agent.
+package agentproc

agent/agentproc/proc_other.go

@@ -0,0 +1,24 @@
+//go:build !linux
+// +build !linux
+
+package agentproc
+
+import (
+	"github.com/spf13/afero"
+)
+
+func (*Process) Niceness(Syscaller) (int, error) {
+	return 0, errUnimplemented
+}
+
+func (*Process) SetNiceness(Syscaller, int) error {
+	return errUnimplemented
+}
+
+func (*Process) Cmd() string {
+	return ""
+}
+
+func List(afero.Fs, Syscaller) ([]*Process, error) {
+	return nil, errUnimplemented
+}

agent/agentproc/proc_test.go

@@ -0,0 +1,166 @@
+package agentproc_test
+
+import (
+	"runtime"
+	"syscall"
+	"testing"
+
+	"github.com/spf13/afero"
+	"github.com/stretchr/testify/require"
+	"go.uber.org/mock/gomock"
+	"golang.org/x/xerrors"
+
+	"github.com/coder/coder/v2/agent/agentproc"
+	"github.com/coder/coder/v2/agent/agentproc/agentproctest"
+)
+
+func TestList(t *testing.T) {
+	t.Parallel()
+
+	if runtime.GOOS != "linux" {
+		t.Skipf("skipping non-linux environment")
+	}
+
+	t.Run("OK", func(t *testing.T) {
+		t.Parallel()
+
+		var (
+			fs            = afero.NewMemMapFs()
+			sc            = agentproctest.NewMockSyscaller(gomock.NewController(t))
+			expectedProcs = make(map[int32]agentproc.Process)
+		)
+
+		for i := 0; i < 4; i++ {
+			proc := agentproctest.GenerateProcess(t, fs)
+			expectedProcs[proc.PID] = proc
+
+			sc.EXPECT().
+				Kill(proc.PID, syscall.Signal(0)).
+				Return(nil)
+		}
+
+		actualProcs, err := agentproc.List(fs, sc)
+		require.NoError(t, err)
+		require.Len(t, actualProcs, len(expectedProcs))
+		for _, proc := range actualProcs {
+			expected, ok := expectedProcs[proc.PID]
+			require.True(t, ok)
+			require.Equal(t, expected.PID, proc.PID)
+			require.Equal(t, expected.CmdLine, proc.CmdLine)
+			require.Equal(t, expected.Dir, proc.Dir)
+		}
+	})
+
+	t.Run("FinishedProcess", func(t *testing.T) {
+		t.Parallel()
+
+		var (
+			fs            = afero.NewMemMapFs()
+			sc            = agentproctest.NewMockSyscaller(gomock.NewController(t))
+			expectedProcs = make(map[int32]agentproc.Process)
+		)
+
+		for i := 0; i < 3; i++ {
+			proc := agentproctest.GenerateProcess(t, fs)
+			expectedProcs[proc.PID] = proc
+
+			sc.EXPECT().
+				Kill(proc.PID, syscall.Signal(0)).
+				Return(nil)
+		}
+
+		// Create a process that's already finished. We're not adding
+		// it to the map because it should be skipped over.
+		proc := agentproctest.GenerateProcess(t, fs)
+		sc.EXPECT().
+			Kill(proc.PID, syscall.Signal(0)).
+			Return(xerrors.New("os: process already finished"))
+
+		actualProcs, err := agentproc.List(fs, sc)
+		require.NoError(t, err)
+		require.Len(t, actualProcs, len(expectedProcs))
+		for _, proc := range actualProcs {
+			expected, ok := expectedProcs[proc.PID]
+			require.True(t, ok)
+			require.Equal(t, expected.PID, proc.PID)
+			require.Equal(t, expected.CmdLine, proc.CmdLine)
+			require.Equal(t, expected.Dir, proc.Dir)
+		}
+	})
+
+	t.Run("NoSuchProcess", func(t *testing.T) {
+		t.Parallel()
+
+		var (
+			fs            = afero.NewMemMapFs()
+			sc            = agentproctest.NewMockSyscaller(gomock.NewController(t))
+			expectedProcs = make(map[int32]agentproc.Process)
+		)
+
+		for i := 0; i < 3; i++ {
+			proc := agentproctest.GenerateProcess(t, fs)
+			expectedProcs[proc.PID] = proc
+
+			sc.EXPECT().
+				Kill(proc.PID, syscall.Signal(0)).
+				Return(nil)
+		}
+
+		// Create a process that doesn't exist. We're not adding
+		// it to the map because it should be skipped over.
+		proc := agentproctest.GenerateProcess(t, fs)
+		sc.EXPECT().
+			Kill(proc.PID, syscall.Signal(0)).
+			Return(syscall.ESRCH)
+
+		actualProcs, err := agentproc.List(fs, sc)
+		require.NoError(t, err)
+		require.Len(t, actualProcs, len(expectedProcs))
+		for _, proc := range actualProcs {
+			expected, ok := expectedProcs[proc.PID]
+			require.True(t, ok)
+			require.Equal(t, expected.PID, proc.PID)
+			require.Equal(t, expected.CmdLine, proc.CmdLine)
+			require.Equal(t, expected.Dir, proc.Dir)
+		}
+	})
+}
+
+// These tests are not very interesting but they provide some modicum of
+// confidence.
+func TestProcess(t *testing.T) {
+	t.Parallel()
+
+	if runtime.GOOS != "linux" {
+		t.Skipf("skipping non-linux environment")
+	}
+
+	t.Run("SetNiceness", func(t *testing.T) {
+		t.Parallel()
+
+		var (
+			sc   = agentproctest.NewMockSyscaller(gomock.NewController(t))
+			proc = &agentproc.Process{
+				PID: 32,
+			}
+			score = 20
+		)
+
+		sc.EXPECT().SetPriority(proc.PID, score).Return(nil)
+		err := proc.SetNiceness(sc, score)
+		require.NoError(t, err)
+	})
+
+	t.Run("Cmd", func(t *testing.T) {
+		t.Parallel()
+
+		var (
+			proc = &agentproc.Process{
+				CmdLine: "helloworld\x00--arg1\x00--arg2",
+			}
+			expectedName = "helloworld --arg1 --arg2"
+		)
+
+		require.Equal(t, expectedName, proc.Cmd())
+	})
+}

agent/agentproc/proc_unix.go

@@ -0,0 +1,134 @@
+//go:build linux
+// +build linux
+
+package agentproc
+
+import (
+	"errors"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"syscall"
+
+	"github.com/spf13/afero"
+	"golang.org/x/xerrors"
+)
+
+func List(fs afero.Fs, syscaller Syscaller) ([]*Process, error) {
+	d, err := fs.Open(defaultProcDir)
+	if err != nil {
+		return nil, xerrors.Errorf("open dir %q: %w", defaultProcDir, err)
+	}
+	defer d.Close()
+
+	entries, err := d.Readdirnames(0)
+	if err != nil {
+		return nil, xerrors.Errorf("readdirnames: %w", err)
+	}
+
+	processes := make([]*Process, 0, len(entries))
+	for _, entry := range entries {
+		pid, err := strconv.ParseInt(entry, 10, 32)
+		if err != nil {
+			continue
+		}
+
+		// Check that the process still exists.
+		exists, err := isProcessExist(syscaller, int32(pid))
+		if err != nil {
+			return nil, xerrors.Errorf("check process exists: %w", err)
+		}
+		if !exists {
+			continue
+		}
+
+		cmdline, err := afero.ReadFile(fs, filepath.Join(defaultProcDir, entry, "cmdline"))
+		if err != nil {
+			if isBenignError(err) {
+				continue
+			}
+			return nil, xerrors.Errorf("read cmdline: %w", err)
+		}
+
+		oomScore, err := afero.ReadFile(fs, filepath.Join(defaultProcDir, entry, "oom_score_adj"))
+		if err != nil {
+			if isBenignError(err) {
+				continue
+			}
+
+			return nil, xerrors.Errorf("read oom_score_adj: %w", err)
+		}
+
+		oom, err := strconv.Atoi(strings.TrimSpace(string(oomScore)))
+		if err != nil {
+			return nil, xerrors.Errorf("convert oom score: %w", err)
+		}
+
+		processes = append(processes, &Process{
+			PID:         int32(pid),
+			CmdLine:     string(cmdline),
+			Dir:         filepath.Join(defaultProcDir, entry),
+			OOMScoreAdj: oom,
+		})
+	}
+
+	return processes, nil
+}
+
+func isProcessExist(syscaller Syscaller, pid int32) (bool, error) {
+	err := syscaller.Kill(pid, syscall.Signal(0))
+	if err == nil {
+		return true, nil
+	}
+	if err.Error() == "os: process already finished" {
+		return false, nil
+	}
+
+	var errno syscall.Errno
+	if !errors.As(err, &errno) {
+		return false, err
+	}
+
+	switch errno {
+	case syscall.ESRCH:
+		return false, nil
+	case syscall.EPERM:
+		return true, nil
+	}
+
+	return false, xerrors.Errorf("kill: %w", err)
+}
+
+func (p *Process) Niceness(sc Syscaller) (int, error) {
+	nice, err := sc.GetPriority(p.PID)
+	if err != nil {
+		return 0, xerrors.Errorf("get priority for %q: %w", p.CmdLine, err)
+	}
+	return nice, nil
+}
+
+func (p *Process) SetNiceness(sc Syscaller, score int) error {
+	err := sc.SetPriority(p.PID, score)
+	if err != nil {
+		return xerrors.Errorf("set priority for %q: %w", p.CmdLine, err)
+	}
+	return nil
+}
+
+func (p *Process) Cmd() string {
+	return strings.Join(p.cmdLine(), " ")
+}
+
+func (p *Process) cmdLine() []string {
+	return strings.Split(p.CmdLine, "\x00")
+}
+
+func isBenignError(err error) bool {
+	var errno syscall.Errno
+	if !xerrors.As(err, &errno) {
+		return false
+	}
+
+	return errno == syscall.ESRCH || errno == syscall.EPERM || xerrors.Is(err, os.ErrNotExist)
+}

agent/agentproc/syscaller.go

@@ -0,0 +1,21 @@
+package agentproc
+
+import (
+	"syscall"
+)
+
+type Syscaller interface {
+	SetPriority(pid int32, priority int) error
+	GetPriority(pid int32) (int, error)
+	Kill(pid int32, sig syscall.Signal) error
+}
+
+// nolint: unused // used on some but no all platforms
+const defaultProcDir = "/proc"
+
+type Process struct {
+	Dir         string
+	CmdLine     string
+	PID         int32
+	OOMScoreAdj int
+}

agent/agentproc/syscaller_other.go

@@ -0,0 +1,30 @@
+//go:build !linux
+// +build !linux
+
+package agentproc
+
+import (
+	"syscall"
+
+	"golang.org/x/xerrors"
+)
+
+func NewSyscaller() Syscaller {
+	return nopSyscaller{}
+}
+
+var errUnimplemented = xerrors.New("unimplemented")
+
+type nopSyscaller struct{}
+
+func (nopSyscaller) SetPriority(int32, int) error {
+	return errUnimplemented
+}
+
+func (nopSyscaller) GetPriority(int32) (int, error) {
+	return 0, errUnimplemented
+}
+
+func (nopSyscaller) Kill(int32, syscall.Signal) error {
+	return errUnimplemented
+}

agent/agentproc/syscaller_unix.go

@@ -0,0 +1,42 @@
+//go:build linux
+// +build linux
+
+package agentproc
+
+import (
+	"syscall"
+
+	"golang.org/x/sys/unix"
+	"golang.org/x/xerrors"
+)
+
+func NewSyscaller() Syscaller {
+	return UnixSyscaller{}
+}
+
+type UnixSyscaller struct{}
+
+func (UnixSyscaller) SetPriority(pid int32, nice int) error {
+	err := unix.Setpriority(unix.PRIO_PROCESS, int(pid), nice)
+	if err != nil {
+		return xerrors.Errorf("set priority: %w", err)
+	}
+	return nil
+}
+
+func (UnixSyscaller) GetPriority(pid int32) (int, error) {
+	nice, err := unix.Getpriority(0, int(pid))
+	if err != nil {
+		return 0, xerrors.Errorf("get priority: %w", err)
+	}
+	return nice, nil
+}
+
+func (UnixSyscaller) Kill(pid int32, sig syscall.Signal) error {
+	err := syscall.Kill(int(pid), sig)
+	if err != nil {
+		return xerrors.Errorf("kill: %w", err)
+	}
+
+	return nil
+}

agent/agentscripts/agentscripts.go

@@ -0,0 +1,472 @@
+package agentscripts
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"os/user"
+	"path/filepath"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/robfig/cron/v3"
+	"github.com/spf13/afero"
+	"golang.org/x/sync/errgroup"
+	"golang.org/x/xerrors"
+	"google.golang.org/protobuf/types/known/timestamppb"
+
+	"cdr.dev/slog"
+
+	"github.com/coder/coder/v2/agent/agentssh"
+	"github.com/coder/coder/v2/agent/proto"
+	"github.com/coder/coder/v2/coderd/database/dbtime"
+	"github.com/coder/coder/v2/codersdk"
+	"github.com/coder/coder/v2/codersdk/agentsdk"
+)
+
+var (
+	// ErrTimeout is returned when a script times out.
+	ErrTimeout = xerrors.New("script timed out")
+	// ErrOutputPipesOpen is returned when a script exits leaving the output
+	// pipe(s) (stdout, stderr) open. This happens because we set WaitDelay on
+	// the command, which gives us two things:
+	//
+	// 1. The ability to ensure that a script exits (this is important for e.g.
+	//    blocking login, and avoiding doing so indefinitely)
+	// 2. Improved command cancellation on timeout
+	ErrOutputPipesOpen = xerrors.New("script exited without closing output pipes")
+
+	parser = cron.NewParser(cron.Second | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.DowOptional)
+)
+
+type ScriptLogger interface {
+	Send(ctx context.Context, log ...agentsdk.Log) error
+	Flush(context.Context) error
+}
+
+// Options are a set of options for the runner.
+type Options struct {
+	DataDirBase     string
+	LogDir          string
+	Logger          slog.Logger
+	SSHServer       *agentssh.Server
+	Filesystem      afero.Fs
+	GetScriptLogger func(logSourceID uuid.UUID) ScriptLogger
+}
+
+// New creates a runner for the provided scripts.
+func New(opts Options) *Runner {
+	cronCtx, cronCtxCancel := context.WithCancel(context.Background())
+	return &Runner{
+		Options:       opts,
+		cronCtx:       cronCtx,
+		cronCtxCancel: cronCtxCancel,
+		cron:          cron.New(cron.WithParser(parser)),
+		closed:        make(chan struct{}),
+		dataDir:       filepath.Join(opts.DataDirBase, "coder-script-data"),
+		scriptsExecuted: prometheus.NewCounterVec(prometheus.CounterOpts{
+			Namespace: "agent",
+			Subsystem: "scripts",
+			Name:      "executed_total",
+		}, []string{"success"}),
+	}
+}
+
+type ScriptCompletedFunc func(context.Context, *proto.WorkspaceAgentScriptCompletedRequest) (*proto.WorkspaceAgentScriptCompletedResponse, error)
+
+type Runner struct {
+	Options
+
+	cronCtx         context.Context
+	cronCtxCancel   context.CancelFunc
+	cmdCloseWait    sync.WaitGroup
+	closed          chan struct{}
+	closeMutex      sync.Mutex
+	cron            *cron.Cron
+	initialized     atomic.Bool
+	scripts         []codersdk.WorkspaceAgentScript
+	dataDir         string
+	scriptCompleted ScriptCompletedFunc
+
+	// scriptsExecuted includes all scripts executed by the workspace agent. Agents
+	// execute startup scripts, and scripts on a cron schedule. Both will increment
+	// this counter.
+	scriptsExecuted *prometheus.CounterVec
+}
+
+// DataDir returns the directory where scripts data is stored.
+func (r *Runner) DataDir() string {
+	return r.dataDir
+}
+
+// ScriptBinDir returns the directory where scripts can store executable
+// binaries.
+func (r *Runner) ScriptBinDir() string {
+	return filepath.Join(r.dataDir, "bin")
+}
+
+func (r *Runner) RegisterMetrics(reg prometheus.Registerer) {
+	if reg == nil {
+		// If no registry, do nothing.
+		return
+	}
+	reg.MustRegister(r.scriptsExecuted)
+}
+
+// Init initializes the runner with the provided scripts.
+// It also schedules any scripts that have a schedule.
+// This function must be called before Execute.
+func (r *Runner) Init(scripts []codersdk.WorkspaceAgentScript, scriptCompleted ScriptCompletedFunc) error {
+	if r.initialized.Load() {
+		return xerrors.New("init: already initialized")
+	}
+	r.initialized.Store(true)
+	r.scripts = scripts
+	r.scriptCompleted = scriptCompleted
+	r.Logger.Info(r.cronCtx, "initializing agent scripts", slog.F("script_count", len(scripts)), slog.F("log_dir", r.LogDir))
+
+	err := r.Filesystem.MkdirAll(r.ScriptBinDir(), 0o700)
+	if err != nil {
+		return xerrors.Errorf("create script bin dir: %w", err)
+	}
+
+	for _, script := range scripts {
+		if script.Cron == "" {
+			continue
+		}
+		script := script
+		_, err := r.cron.AddFunc(script.Cron, func() {
+			err := r.trackRun(r.cronCtx, script, ExecuteCronScripts)
+			if err != nil {
+				r.Logger.Warn(context.Background(), "run agent script on schedule", slog.Error(err))
+			}
+		})
+		if err != nil {
+			return xerrors.Errorf("add schedule: %w", err)
+		}
+	}
+	return nil
+}
+
+// StartCron starts the cron scheduler.
+// This is done async to allow for the caller to execute scripts prior.
+func (r *Runner) StartCron() {
+	// cron.Start() and cron.Stop() does not guarantee that the cron goroutine
+	// has exited by the time the `cron.Stop()` context returns, so we need to
+	// track it manually.
+	err := r.trackCommandGoroutine(func() {
+		// Since this is run async, in quick unit tests, it is possible the
+		// Close() function gets called before we even start the cron.
+		// In these cases, the Run() will never end.
+		// So if we are closed, we just return, and skip the Run() entirely.
+		select {
+		case <-r.cronCtx.Done():
+			// The cronCtx is canceled before cron.Close() happens. So if the ctx is
+			// canceled, then Close() will be called, or it is about to be called.
+			// So do nothing!
+		default:
+			r.cron.Run()
+		}
+	})
+	if err != nil {
+		r.Logger.Warn(context.Background(), "start cron failed", slog.Error(err))
+	}
+}
+
+// ExecuteOption describes what scripts we want to execute.
+type ExecuteOption int
+
+// ExecuteOption enums.
+const (
+	ExecuteAllScripts ExecuteOption = iota
+	ExecuteStartScripts
+	ExecuteStopScripts
+	ExecuteCronScripts
+)
+
+// Execute runs a set of scripts according to a filter.
+func (r *Runner) Execute(ctx context.Context, option ExecuteOption) error {
+	var eg errgroup.Group
+	for _, script := range r.scripts {
+		runScript := (option == ExecuteStartScripts && script.RunOnStart) ||
+			(option == ExecuteStopScripts && script.RunOnStop) ||
+			(option == ExecuteCronScripts && script.Cron != "") ||
+			option == ExecuteAllScripts
+
+		if !runScript {
+			continue
+		}
+
+		script := script
+		eg.Go(func() error {
+			err := r.trackRun(ctx, script, option)
+			if err != nil {
+				return xerrors.Errorf("run agent script %q: %w", script.LogSourceID, err)
+			}
+			return nil
+		})
+	}
+	return eg.Wait()
+}
+
+// trackRun wraps "run" with metrics.
+func (r *Runner) trackRun(ctx context.Context, script codersdk.WorkspaceAgentScript, option ExecuteOption) error {
+	err := r.run(ctx, script, option)
+	if err != nil {
+		r.scriptsExecuted.WithLabelValues("false").Add(1)
+	} else {
+		r.scriptsExecuted.WithLabelValues("true").Add(1)
+	}
+	return err
+}
+
+// run executes the provided script with the timeout.
+// If the timeout is exceeded, the process is sent an interrupt signal.
+// If the process does not exit after a few seconds, it is forcefully killed.
+// This function immediately returns after a timeout, and does not wait for the process to exit.
+func (r *Runner) run(ctx context.Context, script codersdk.WorkspaceAgentScript, option ExecuteOption) error {
+	logPath := script.LogPath
+	if logPath == "" {
+		logPath = fmt.Sprintf("coder-script-%s.log", script.LogSourceID)
+	}
+	if logPath[0] == '~' {
+		// First we check the environment.
+		homeDir, err := os.UserHomeDir()
+		if err != nil {
+			u, err := user.Current()
+			if err != nil {
+				return xerrors.Errorf("current user: %w", err)
+			}
+			homeDir = u.HomeDir
+		}
+		logPath = filepath.Join(homeDir, logPath[1:])
+	}
+	logPath = os.ExpandEnv(logPath)
+	if !filepath.IsAbs(logPath) {
+		logPath = filepath.Join(r.LogDir, logPath)
+	}
+
+	scriptDataDir := filepath.Join(r.DataDir(), script.LogSourceID.String())
+	err := r.Filesystem.MkdirAll(scriptDataDir, 0o700)
+	if err != nil {
+		return xerrors.Errorf("%s script: create script temp dir: %w", scriptDataDir, err)
+	}
+
+	logger := r.Logger.With(
+		slog.F("log_source_id", script.LogSourceID),
+		slog.F("log_path", logPath),
+		slog.F("script_data_dir", scriptDataDir),
+	)
+	logger.Info(ctx, "running agent script", slog.F("script", script.Script))
+
+	fileWriter, err := r.Filesystem.OpenFile(logPath, os.O_CREATE|os.O_RDWR, 0o600)
+	if err != nil {
+		return xerrors.Errorf("open %s script log file: %w", logPath, err)
+	}
+	defer func() {
+		err := fileWriter.Close()
+		if err != nil {
+			logger.Warn(ctx, fmt.Sprintf("close %s script log file", logPath), slog.Error(err))
+		}
+	}()
+
+	var cmd *exec.Cmd
+	cmdCtx := ctx
+	if script.Timeout > 0 {
+		var ctxCancel context.CancelFunc
+		cmdCtx, ctxCancel = context.WithTimeout(ctx, script.Timeout)
+		defer ctxCancel()
+	}
+	cmdPty, err := r.SSHServer.CreateCommand(cmdCtx, script.Script, nil)
+	if err != nil {
+		return xerrors.Errorf("%s script: create command: %w", logPath, err)
+	}
+	cmd = cmdPty.AsExec()
+	cmd.SysProcAttr = cmdSysProcAttr()
+	cmd.WaitDelay = 10 * time.Second
+	cmd.Cancel = cmdCancel(cmd)
+
+	// Expose env vars that can be used in the script for storing data
+	// and binaries. In the future, we may want to expose more env vars
+	// for the script to use, like CODER_SCRIPT_DATA_DIR for persistent
+	// storage.
+	cmd.Env = append(cmd.Env, "CODER_SCRIPT_DATA_DIR="+scriptDataDir)
+	cmd.Env = append(cmd.Env, "CODER_SCRIPT_BIN_DIR="+r.ScriptBinDir())
+
+	scriptLogger := r.GetScriptLogger(script.LogSourceID)
+	// If ctx is canceled here (or in a writer below), we may be
+	// discarding logs, but that's okay because we're shutting down
+	// anyway. We could consider creating a new context here if we
+	// want better control over flush during shutdown.
+	defer func() {
+		if err := scriptLogger.Flush(ctx); err != nil {
+			logger.Warn(ctx, "flush startup logs failed", slog.Error(err))
+		}
+	}()
+
+	infoW := agentsdk.LogsWriter(ctx, scriptLogger.Send, script.LogSourceID, codersdk.LogLevelInfo)
+	defer infoW.Close()
+	errW := agentsdk.LogsWriter(ctx, scriptLogger.Send, script.LogSourceID, codersdk.LogLevelError)
+	defer errW.Close()
+	cmd.Stdout = io.MultiWriter(fileWriter, infoW)
+	cmd.Stderr = io.MultiWriter(fileWriter, errW)
+
+	start := dbtime.Now()
+	defer func() {
+		end := dbtime.Now()
+		execTime := end.Sub(start)
+		exitCode := 0
+		if err != nil {
+			exitCode = 255 // Unknown status.
+			var exitError *exec.ExitError
+			if xerrors.As(err, &exitError) {
+				exitCode = exitError.ExitCode()
+			}
+			logger.Warn(ctx, fmt.Sprintf("%s script failed", logPath), slog.F("execution_time", execTime), slog.F("exit_code", exitCode), slog.Error(err))
+		} else {
+			logger.Info(ctx, fmt.Sprintf("%s script completed", logPath), slog.F("execution_time", execTime), slog.F("exit_code", exitCode))
+		}
+
+		if r.scriptCompleted == nil {
+			logger.Debug(ctx, "r.scriptCompleted unexpectedly nil")
+			return
+		}
+
+		// We want to check this outside of the goroutine to avoid a race condition
+		timedOut := errors.Is(err, ErrTimeout)
+		pipesLeftOpen := errors.Is(err, ErrOutputPipesOpen)
+
+		err = r.trackCommandGoroutine(func() {
+			var stage proto.Timing_Stage
+			switch option {
+			case ExecuteStartScripts:
+				stage = proto.Timing_START
+			case ExecuteStopScripts:
+				stage = proto.Timing_STOP
+			case ExecuteCronScripts:
+				stage = proto.Timing_CRON
+			}
+
+			var status proto.Timing_Status
+			switch {
+			case timedOut:
+				status = proto.Timing_TIMED_OUT
+			case pipesLeftOpen:
+				status = proto.Timing_PIPES_LEFT_OPEN
+			case exitCode != 0:
+				status = proto.Timing_EXIT_FAILURE
+			default:
+				status = proto.Timing_OK
+			}
+
+			reportTimeout := 30 * time.Second
+			reportCtx, cancel := context.WithTimeout(context.Background(), reportTimeout)
+			defer cancel()
+
+			_, err := r.scriptCompleted(reportCtx, &proto.WorkspaceAgentScriptCompletedRequest{
+				Timing: &proto.Timing{
+					ScriptId: script.ID[:],
+					Start:    timestamppb.New(start),
+					End:      timestamppb.New(end),
+					ExitCode: int32(exitCode),
+					Stage:    stage,
+					Status:   status,
+				},
+			})
+			if err != nil {
+				logger.Error(ctx, fmt.Sprintf("reporting script completed: %s", err.Error()))
+			}
+		})
+		if err != nil {
+			logger.Error(ctx, fmt.Sprintf("reporting script completed: track command goroutine: %s", err.Error()))
+		}
+	}()
+
+	err = cmd.Start()
+	if err != nil {
+		if errors.Is(err, context.DeadlineExceeded) {
+			return ErrTimeout
+		}
+		return xerrors.Errorf("%s script: start command: %w", logPath, err)
+	}
+
+	cmdDone := make(chan error, 1)
+	err = r.trackCommandGoroutine(func() {
+		cmdDone <- cmd.Wait()
+	})
+	if err != nil {
+		return xerrors.Errorf("%s script: track command goroutine: %w", logPath, err)
+	}
+	select {
+	case <-cmdCtx.Done():
+		// Wait for the command to drain!
+		select {
+		case <-cmdDone:
+		case <-time.After(10 * time.Second):
+		}
+		err = cmdCtx.Err()
+	case err = <-cmdDone:
+	}
+	switch {
+	case errors.Is(err, exec.ErrWaitDelay):
+		err = ErrOutputPipesOpen
+		message := fmt.Sprintf("script exited successfully, but output pipes were not closed after %s", cmd.WaitDelay)
+		details := fmt.Sprint(
+			"This usually means a child process was started with references to stdout or stderr. As a result, this " +
+				"process may now have been terminated. Consider redirecting the output or using a separate " +
+				"\"coder_script\" for the process, see " +
+				"https://coder.com/docs/templates/troubleshooting#startup-script-issues for more information.",
+		)
+		// Inform the user by propagating the message via log writers.
+		_, _ = fmt.Fprintf(cmd.Stderr, "WARNING: %s. %s\n", message, details)
+		// Also log to agent logs for ease of debugging.
+		r.Logger.Warn(ctx, message, slog.F("details", details), slog.Error(err))
+
+	case errors.Is(err, context.DeadlineExceeded):
+		err = ErrTimeout
+	}
+	return err
+}
+
+func (r *Runner) Close() error {
+	r.closeMutex.Lock()
+	defer r.closeMutex.Unlock()
+	if r.isClosed() {
+		return nil
+	}
+	close(r.closed)
+	// Must cancel the cron ctx BEFORE stopping the cron.
+	r.cronCtxCancel()
+	<-r.cron.Stop().Done()
+	r.cmdCloseWait.Wait()
+	return nil
+}
+
+func (r *Runner) trackCommandGoroutine(fn func()) error {
+	r.closeMutex.Lock()
+	defer r.closeMutex.Unlock()
+	if r.isClosed() {
+		return xerrors.New("track command goroutine: closed")
+	}
+	r.cmdCloseWait.Add(1)
+	go func() {
+		defer r.cmdCloseWait.Done()
+		fn()
+	}()
+	return nil
+}
+
+func (r *Runner) isClosed() bool {
+	select {
+	case <-r.closed:
+		return true
+	default:
+		return false
+	}
+}

agent/agentscripts/agentscripts_other.go

@@ -0,0 +1,20 @@
+//go:build !windows
+
+package agentscripts
+
+import (
+	"os/exec"
+	"syscall"
+)
+
+func cmdSysProcAttr() *syscall.SysProcAttr {
+	return &syscall.SysProcAttr{
+		Setsid: true,
+	}
+}
+
+func cmdCancel(cmd *exec.Cmd) func() error {
+	return func() error {
+		return syscall.Kill(-cmd.Process.Pid, syscall.SIGHUP)
+	}
+}

agent/agentscripts/agentscripts_test.go

@@ -0,0 +1,210 @@
+package agentscripts_test
+
+import (
+	"context"
+	"path/filepath"
+	"runtime"
+	"testing"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/spf13/afero"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.uber.org/goleak"
+
+	"github.com/coder/coder/v2/agent/agentscripts"
+	"github.com/coder/coder/v2/agent/agentssh"
+	"github.com/coder/coder/v2/agent/agenttest"
+	"github.com/coder/coder/v2/codersdk"
+	"github.com/coder/coder/v2/codersdk/agentsdk"
+	"github.com/coder/coder/v2/testutil"
+)
+
+func TestMain(m *testing.M) {
+	goleak.VerifyTestMain(m)
+}
+
+func TestExecuteBasic(t *testing.T) {
+	t.Parallel()
+	ctx := testutil.Context(t, testutil.WaitShort)
+	fLogger := newFakeScriptLogger()
+	runner := setup(t, func(uuid2 uuid.UUID) agentscripts.ScriptLogger {
+		return fLogger
+	})
+	defer runner.Close()
+	aAPI := agenttest.NewFakeAgentAPI(t, testutil.Logger(t), nil, nil)
+	err := runner.Init([]codersdk.WorkspaceAgentScript{{
+		LogSourceID: uuid.New(),
+		Script:      "echo hello",
+	}}, aAPI.ScriptCompleted)
+	require.NoError(t, err)
+	require.NoError(t, runner.Execute(context.Background(), agentscripts.ExecuteAllScripts))
+	log := testutil.RequireRecvCtx(ctx, t, fLogger.logs)
+	require.Equal(t, "hello", log.Output)
+}
+
+func TestEnv(t *testing.T) {
+	t.Parallel()
+	fLogger := newFakeScriptLogger()
+	runner := setup(t, func(uuid2 uuid.UUID) agentscripts.ScriptLogger {
+		return fLogger
+	})
+	defer runner.Close()
+	id := uuid.New()
+	script := "echo $CODER_SCRIPT_DATA_DIR\necho $CODER_SCRIPT_BIN_DIR\n"
+	if runtime.GOOS == "windows" {
+		script = `
+			cmd.exe /c echo %CODER_SCRIPT_DATA_DIR%
+			cmd.exe /c echo %CODER_SCRIPT_BIN_DIR%
+		`
+	}
+	aAPI := agenttest.NewFakeAgentAPI(t, testutil.Logger(t), nil, nil)
+	err := runner.Init([]codersdk.WorkspaceAgentScript{{
+		LogSourceID: id,
+		Script:      script,
+	}}, aAPI.ScriptCompleted)
+	require.NoError(t, err)
+
+	ctx := testutil.Context(t, testutil.WaitLong)
+
+	done := testutil.Go(t, func() {
+		err := runner.Execute(ctx, agentscripts.ExecuteAllScripts)
+		assert.NoError(t, err)
+	})
+	defer func() {
+		select {
+		case <-ctx.Done():
+		case <-done:
+		}
+	}()
+
+	var log []agentsdk.Log
+	for {
+		select {
+		case <-ctx.Done():
+			require.Fail(t, "timed out waiting for logs")
+		case l := <-fLogger.logs:
+			t.Logf("log: %s", l.Output)
+			log = append(log, l)
+		}
+		if len(log) >= 2 {
+			break
+		}
+	}
+	require.Contains(t, log[0].Output, filepath.Join(runner.DataDir(), id.String()))
+	require.Contains(t, log[1].Output, runner.ScriptBinDir())
+}
+
+func TestTimeout(t *testing.T) {
+	t.Parallel()
+	runner := setup(t, nil)
+	defer runner.Close()
+	aAPI := agenttest.NewFakeAgentAPI(t, testutil.Logger(t), nil, nil)
+	err := runner.Init([]codersdk.WorkspaceAgentScript{{
+		LogSourceID: uuid.New(),
+		Script:      "sleep infinity",
+		Timeout:     time.Millisecond,
+	}}, aAPI.ScriptCompleted)
+	require.NoError(t, err)
+	require.ErrorIs(t, runner.Execute(context.Background(), agentscripts.ExecuteAllScripts), agentscripts.ErrTimeout)
+}
+
+func TestScriptReportsTiming(t *testing.T) {
+	t.Parallel()
+
+	ctx := testutil.Context(t, testutil.WaitShort)
+	fLogger := newFakeScriptLogger()
+	runner := setup(t, func(uuid2 uuid.UUID) agentscripts.ScriptLogger {
+		return fLogger
+	})
+
+	aAPI := agenttest.NewFakeAgentAPI(t, testutil.Logger(t), nil, nil)
+	err := runner.Init([]codersdk.WorkspaceAgentScript{{
+		DisplayName: "say-hello",
+		LogSourceID: uuid.New(),
+		Script:      "echo hello",
+	}}, aAPI.ScriptCompleted)
+	require.NoError(t, err)
+	require.NoError(t, runner.Execute(ctx, agentscripts.ExecuteAllScripts))
+	runner.Close()
+
+	log := testutil.RequireRecvCtx(ctx, t, fLogger.logs)
+	require.Equal(t, "hello", log.Output)
+
+	timings := aAPI.GetTimings()
+	require.Equal(t, 1, len(timings))
+
+	timing := timings[0]
+	require.Equal(t, int32(0), timing.ExitCode)
+	require.GreaterOrEqual(t, timing.End.AsTime(), timing.Start.AsTime())
+}
+
+// TestCronClose exists because cron.Run() can happen after cron.Close().
+// If this happens, there used to be a deadlock.
+func TestCronClose(t *testing.T) {
+	t.Parallel()
+	runner := agentscripts.New(agentscripts.Options{})
+	runner.StartCron()
+	require.NoError(t, runner.Close(), "close runner")
+}
+
+func setup(t *testing.T, getScriptLogger func(logSourceID uuid.UUID) agentscripts.ScriptLogger) *agentscripts.Runner {
+	t.Helper()
+	if getScriptLogger == nil {
+		// noop
+		getScriptLogger = func(uuid uuid.UUID) agentscripts.ScriptLogger {
+			return noopScriptLogger{}
+		}
+	}
+	fs := afero.NewMemMapFs()
+	logger := testutil.Logger(t)
+	s, err := agentssh.NewServer(context.Background(), logger, prometheus.NewRegistry(), fs, nil)
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		_ = s.Close()
+	})
+	return agentscripts.New(agentscripts.Options{
+		LogDir:          t.TempDir(),
+		DataDirBase:     t.TempDir(),
+		Logger:          logger,
+		SSHServer:       s,
+		Filesystem:      fs,
+		GetScriptLogger: getScriptLogger,
+	})
+}
+
+type noopScriptLogger struct{}
+
+func (noopScriptLogger) Send(context.Context, ...agentsdk.Log) error {
+	return nil
+}
+
+func (noopScriptLogger) Flush(context.Context) error {
+	return nil
+}
+
+type fakeScriptLogger struct {
+	logs chan agentsdk.Log
+}
+
+func (f *fakeScriptLogger) Send(ctx context.Context, logs ...agentsdk.Log) error {
+	for _, log := range logs {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case f.logs <- log:
+			// OK!
+		}
+	}
+	return nil
+}
+
+func (*fakeScriptLogger) Flush(context.Context) error {
+	return nil
+}
+
+func newFakeScriptLogger() *fakeScriptLogger {
+	return &fakeScriptLogger{make(chan agentsdk.Log, 100)}
+}

agent/agentscripts/agentscripts_windows.go

@@ -0,0 +1,17 @@
+package agentscripts
+
+import (
+	"os"
+	"os/exec"
+	"syscall"
+)
+
+func cmdSysProcAttr() *syscall.SysProcAttr {
+	return &syscall.SysProcAttr{}
+}
+
+func cmdCancel(cmd *exec.Cmd) func() error {
+	return func() error {
+		return cmd.Process.Signal(os.Interrupt)
+	}
+}

agent/agentssh/agentssh.go

@@ -19,7 +19,10 @@ import (
 	"time"
 
 	"github.com/gliderlabs/ssh"
+	"github.com/google/uuid"
+	"github.com/kballard/go-shellquote"
 	"github.com/pkg/sftp"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/spf13/afero"
 	"go.uber.org/atomic"
 	gossh "golang.org/x/crypto/ssh"
@@ -27,9 +30,9 @@ import (
 
 	"cdr.dev/slog"
 
-	"github.com/coder/coder/agent/usershell"
-	"github.com/coder/coder/codersdk/agentsdk"
-	"github.com/coder/coder/pty"
+	"github.com/coder/coder/v2/agent/usershell"
+	"github.com/coder/coder/v2/codersdk"
+	"github.com/coder/coder/v2/pty"
 )
 
 const (
@@ -43,10 +46,46 @@ const (
 	MagicSessionTypeEnvironmentVariable = "CODER_SSH_SESSION_TYPE"
 	// MagicSessionTypeVSCode is set in the SSH config by the VS Code extension to identify itself.
 	MagicSessionTypeVSCode = "vscode"
-	// MagicSessionTypeJetBrains is set in the SSH config by the JetBrains extension to identify itself.
+	// MagicSessionTypeJetBrains is set in the SSH config by the JetBrains
+	// extension to identify itself.
 	MagicSessionTypeJetBrains = "jetbrains"
+	// MagicProcessCmdlineJetBrains is a string in a process's command line that
+	// uniquely identifies it as JetBrains software.
+	MagicProcessCmdlineJetBrains = "idea.vendor.name=JetBrains"
+
+	// BlockedFileTransferErrorCode indicates that SSH server restricted the raw command from performing
+	// the file transfer.
+	BlockedFileTransferErrorCode    = 65 // Error code: host not allowed to connect
+	BlockedFileTransferErrorMessage = "File transfer has been disabled."
 )
 
+// BlockedFileTransferCommands contains a list of restricted file transfer commands.
+var BlockedFileTransferCommands = []string{"nc", "rsync", "scp", "sftp"}
+
+// Config sets configuration parameters for the agent SSH server.
+type Config struct {
+	// MaxTimeout sets the absolute connection timeout, none if empty. If set to
+	// 3 seconds or more, keep alive will be used instead.
+	MaxTimeout time.Duration
+	// MOTDFile returns the path to the message of the day file. If set, the
+	// file will be displayed to the user upon login.
+	MOTDFile func() string
+	// ServiceBanner returns the configuration for the Coder service banner.
+	AnnouncementBanners func() *[]codersdk.BannerConfig
+	// UpdateEnv updates the environment variables for the command to be
+	// executed. It can be used to add, modify or replace environment variables.
+	UpdateEnv func(current []string) (updated []string, err error)
+	// WorkingDirectory sets the working directory for commands and defines
+	// where users will land when they connect via SSH. Default is the home
+	// directory of the user.
+	WorkingDirectory func() string
+	// X11DisplayOffset is the offset to add to the X11 display number.
+	// Default is 10.
+	X11DisplayOffset *int
+	// BlockFileTransfer restricts use of file transfer applications.
+	BlockFileTransfer bool
+}
+
 type Server struct {
 	mu        sync.RWMutex // Protects following.
 	fs        afero.Fs
@@ -58,20 +97,19 @@ type Server struct {
 	// a lock on mu but protected by closing.
 	wg sync.WaitGroup
 
-	logger       slog.Logger
-	srv          *ssh.Server
-	x11SocketDir string
+	logger slog.Logger
+	srv    *ssh.Server
 
-	Env        map[string]string
-	AgentToken func() string
-	Manifest   *atomic.Pointer[agentsdk.Manifest]
+	config *Config
 
 	connCountVSCode     atomic.Int64
 	connCountJetBrains  atomic.Int64
 	connCountSSHSession atomic.Int64
+
+	metrics *sshServerMetrics
 }
 
-func NewServer(ctx context.Context, logger slog.Logger, fs afero.Fs, maxTimeout time.Duration, x11SocketDir string) (*Server, error) {
+func NewServer(ctx context.Context, logger slog.Logger, prometheusRegistry *prometheus.Registry, fs afero.Fs, config *Config) (*Server, error) {
 	// Clients' should ignore the host key when connecting.
 	// The agent needs to authenticate with coderd to SSH,
 	// so SSH authentication doesn't improve security.
@@ -83,38 +121,78 @@ func NewServer(ctx context.Context, logger slog.Logger, fs afero.Fs, maxTimeout
 	if err != nil {
 		return nil, err
 	}
-	if x11SocketDir == "" {
-		x11SocketDir = filepath.Join(os.TempDir(), ".X11-unix")
+	if config == nil {
+		config = &Config{}
+	}
+	if config.X11DisplayOffset == nil {
+		offset := X11DefaultDisplayOffset
+		config.X11DisplayOffset = &offset
+	}
+	if config.UpdateEnv == nil {
+		config.UpdateEnv = func(current []string) ([]string, error) { return current, nil }
+	}
+	if config.MOTDFile == nil {
+		config.MOTDFile = func() string { return "" }
+	}
+	if config.AnnouncementBanners == nil {
+		config.AnnouncementBanners = func() *[]codersdk.BannerConfig { return &[]codersdk.BannerConfig{} }
+	}
+	if config.WorkingDirectory == nil {
+		config.WorkingDirectory = func() string {
+			home, err := userHomeDir()
+			if err != nil {
+				return ""
+			}
+			return home
+		}
 	}
 
 	forwardHandler := &ssh.ForwardedTCPHandler{}
-	unixForwardHandler := &forwardedUnixHandler{log: logger}
+	unixForwardHandler := newForwardedUnixHandler(logger)
 
+	metrics := newSSHServerMetrics(prometheusRegistry)
 	s := &Server{
-		listeners:    make(map[net.Listener]struct{}),
-		fs:           fs,
-		conns:        make(map[net.Conn]struct{}),
-		sessions:     make(map[ssh.Session]struct{}),
-		logger:       logger,
-		x11SocketDir: x11SocketDir,
+		listeners: make(map[net.Listener]struct{}),
+		fs:        fs,
+		conns:     make(map[net.Conn]struct{}),
+		sessions:  make(map[ssh.Session]struct{}),
+		logger:    logger,
+
+		config: config,
+
+		metrics: metrics,
 	}
 
-	s.srv = &ssh.Server{
+	srv := &ssh.Server{
 		ChannelHandlers: map[string]ssh.ChannelHandler{
-			"direct-tcpip":                   ssh.DirectTCPIPHandler,
+			"direct-tcpip": func(srv *ssh.Server, conn *gossh.ServerConn, newChan gossh.NewChannel, ctx ssh.Context) {
+				// Wrapper is designed to find and track JetBrains Gateway connections.
+				wrapped := NewJetbrainsChannelWatcher(ctx, s.logger, newChan, &s.connCountJetBrains)
+				ssh.DirectTCPIPHandler(srv, conn, wrapped, ctx)
+			},
 			"direct-streamlocal@openssh.com": directStreamLocalHandler,
 			"session":                        ssh.DefaultSessionHandler,
 		},
-		ConnectionFailedCallback: func(_ net.Conn, err error) {
-			s.logger.Info(ctx, "ssh connection ended", slog.Error(err))
+		ConnectionFailedCallback: func(conn net.Conn, err error) {
+			s.logger.Warn(ctx, "ssh connection failed",
+				slog.F("remote_addr", conn.RemoteAddr()),
+				slog.F("local_addr", conn.LocalAddr()),
+				slog.Error(err))
+			metrics.failedConnectionsTotal.Add(1)
+		},
+		ConnectionCompleteCallback: func(conn *gossh.ServerConn, err error) {
+			s.logger.Info(ctx, "ssh connection complete",
+				slog.F("remote_addr", conn.RemoteAddr()),
+				slog.F("local_addr", conn.LocalAddr()),
+				slog.Error(err))
 		},
 		Handler:     s.sessionHandler,
 		HostSigners: []ssh.Signer{randomSigner},
 		LocalPortForwardingCallback: func(ctx ssh.Context, destinationHost string, destinationPort uint32) bool {
 			// Allow local port forwarding all!
 			s.logger.Debug(ctx, "local port forward",
-				slog.F("destination-host", destinationHost),
-				slog.F("destination-port", destinationPort))
+				slog.F("destination_host", destinationHost),
+				slog.F("destination_port", destinationPort))
 			return true
 		},
 		PtyCallback: func(ctx ssh.Context, pty ssh.Pty) bool {
@@ -122,9 +200,9 @@ func NewServer(ctx context.Context, logger slog.Logger, fs afero.Fs, maxTimeout
 		},
 		ReversePortForwardingCallback: func(ctx ssh.Context, bindHost string, bindPort uint32) bool {
 			// Allow reverse port forwarding all!
-			s.logger.Debug(ctx, "local port forward",
-				slog.F("bind-host", bindHost),
-				slog.F("bind-port", bindPort))
+			s.logger.Debug(ctx, "reverse port forward",
+				slog.F("bind_host", bindHost),
+				slog.F("bind_port", bindPort))
 			return true
 		},
 		RequestHandlers: map[string]ssh.RequestHandler{
@@ -142,9 +220,21 @@ func NewServer(ctx context.Context, logger slog.Logger, fs afero.Fs, maxTimeout
 		SubsystemHandlers: map[string]ssh.SubsystemHandler{
 			"sftp": s.sessionHandler,
 		},
-		MaxTimeout: maxTimeout,
 	}
 
+	// The MaxTimeout functionality has been substituted with the introduction
+	// of the KeepAlive feature. In cases where very short timeouts are set, the
+	// SSH server will automatically switch to the connection timeout for both
+	// read and write operations.
+	if config.MaxTimeout >= 3*time.Second {
+		srv.ClientAliveCountMax = 3
+		srv.ClientAliveInterval = config.MaxTimeout / time.Duration(srv.ClientAliveCountMax)
+		srv.MaxTimeout = 0
+	} else {
+		srv.MaxTimeout = config.MaxTimeout
+	}
+
+	s.srv = srv
 	return s, nil
 }
 
@@ -163,55 +253,130 @@ func (s *Server) ConnStats() ConnStats {
 }
 
 func (s *Server) sessionHandler(session ssh.Session) {
+	ctx := session.Context()
+	logger := s.logger.With(
+		slog.F("remote_addr", session.RemoteAddr()),
+		slog.F("local_addr", session.LocalAddr()),
+		// Assigning a random uuid for each session is useful for tracking
+		// logs for the same ssh session.
+		slog.F("id", uuid.NewString()),
+	)
+	logger.Info(ctx, "handling ssh session")
+
 	if !s.trackSession(session, true) {
 		// See (*Server).Close() for why we call Close instead of Exit.
 		_ = session.Close()
+		logger.Info(ctx, "unable to accept new session, server is closing")
 		return
 	}
 	defer s.trackSession(session, false)
 
-	ctx := session.Context()
-
 	extraEnv := make([]string, 0)
 	x11, hasX11 := session.X11()
 	if hasX11 {
-		handled := s.x11Handler(session.Context(), x11)
+		display, handled := s.x11Handler(session.Context(), x11)
 		if !handled {
 			_ = session.Exit(1)
+			logger.Error(ctx, "x11 handler failed")
 			return
 		}
-		extraEnv = append(extraEnv, fmt.Sprintf("DISPLAY=:%d.0", x11.ScreenNumber))
+		extraEnv = append(extraEnv, fmt.Sprintf("DISPLAY=localhost:%d.%d", display, x11.ScreenNumber))
+	}
+
+	if s.fileTransferBlocked(session) {
+		s.logger.Warn(ctx, "file transfer blocked", slog.F("session_subsystem", session.Subsystem()), slog.F("raw_command", session.RawCommand()))
+
+		if session.Subsystem() == "" { // sftp does not expect error, otherwise it fails with "package too long"
+			// Response format: <status_code><message body>\n
+			errorMessage := fmt.Sprintf("\x02%s\n", BlockedFileTransferErrorMessage)
+			_, _ = session.Write([]byte(errorMessage))
+		}
+		_ = session.Exit(BlockedFileTransferErrorCode)
+		return
 	}
 
 	switch ss := session.Subsystem(); ss {
 	case "":
 	case "sftp":
-		s.sftpHandler(session)
+		s.sftpHandler(logger, session)
 		return
 	default:
-		s.logger.Debug(ctx, "unsupported subsystem", slog.F("subsystem", ss))
+		logger.Warn(ctx, "unsupported subsystem", slog.F("subsystem", ss))
 		_ = session.Exit(1)
 		return
 	}
 
-	err := s.sessionStart(session, extraEnv)
+	err := s.sessionStart(logger, session, extraEnv)
 	var exitError *exec.ExitError
 	if xerrors.As(err, &exitError) {
-		s.logger.Debug(ctx, "ssh session returned", slog.Error(exitError))
-		_ = session.Exit(exitError.ExitCode())
+		code := exitError.ExitCode()
+		if code == -1 {
+			// If we return -1 here, it will be transmitted as an
+			// uint32(4294967295). This exit code is nonsense, so
+			// instead we return 255 (same as OpenSSH). This is
+			// also the same exit code that the shell returns for
+			// -1.
+			//
+			// For signals, we could consider sending 128+signal
+			// instead (however, OpenSSH doesn't seem to do this).
+			code = 255
+		}
+		logger.Info(ctx, "ssh session returned",
+			slog.Error(exitError),
+			slog.F("process_exit_code", exitError.ExitCode()),
+			slog.F("exit_code", code),
+		)
+
+		// TODO(mafredri): For signal exit, there's also an "exit-signal"
+		// request (session.Exit sends "exit-status"), however, since it's
+		// not implemented on the session interface and not used by
+		// OpenSSH, we'll leave it for now.
+		_ = session.Exit(code)
 		return
 	}
 	if err != nil {
-		s.logger.Warn(ctx, "ssh session failed", slog.Error(err))
+		logger.Warn(ctx, "ssh session failed", slog.Error(err))
 		// This exit code is designed to be unlikely to be confused for a legit exit code
 		// from the process.
 		_ = session.Exit(MagicSessionErrorCode)
 		return
 	}
+	logger.Info(ctx, "normal ssh session exit")
 	_ = session.Exit(0)
 }
 
-func (s *Server) sessionStart(session ssh.Session, extraEnv []string) (retErr error) {
+// fileTransferBlocked method checks if the file transfer commands should be blocked.
+//
+// Warning: consider this mechanism as "Do not trespass" sign, as a violator can still ssh to the host,
+// smuggle the `scp` binary, or just manually send files outside with `curl` or `ftp`.
+// If a user needs a more sophisticated and battle-proof solution, consider full endpoint security.
+func (s *Server) fileTransferBlocked(session ssh.Session) bool {
+	if !s.config.BlockFileTransfer {
+		return false // file transfers are permitted
+	}
+	// File transfers are restricted.
+
+	if session.Subsystem() == "sftp" {
+		return true
+	}
+
+	cmd := session.Command()
+	if len(cmd) == 0 {
+		return false // no command?
+	}
+
+	c := cmd[0]
+	c = filepath.Base(c) // in case the binary is absolute path, /usr/sbin/scp
+
+	for _, cmd := range BlockedFileTransferCommands {
+		if cmd == c {
+			return true
+		}
+	}
+	return false
+}
+
+func (s *Server) sessionStart(logger slog.Logger, session ssh.Session, extraEnv []string) (retErr error) {
 	ctx := session.Context()
 	env := append(session.Environ(), extraEnv...)
 	var magicType string
@@ -219,31 +384,47 @@ func (s *Server) sessionStart(session ssh.Session, extraEnv []string) (retErr er
 		if !strings.HasPrefix(kv, MagicSessionTypeEnvironmentVariable) {
 			continue
 		}
-		magicType = strings.TrimPrefix(kv, MagicSessionTypeEnvironmentVariable+"=")
+		magicType = strings.ToLower(strings.TrimPrefix(kv, MagicSessionTypeEnvironmentVariable+"="))
 		env = append(env[:index], env[index+1:]...)
 	}
+
+	// Always force lowercase checking to be case-insensitive.
 	switch magicType {
 	case MagicSessionTypeVSCode:
 		s.connCountVSCode.Add(1)
 		defer s.connCountVSCode.Add(-1)
 	case MagicSessionTypeJetBrains:
-		s.connCountJetBrains.Add(1)
-		defer s.connCountJetBrains.Add(-1)
+		// Do nothing here because JetBrains launches hundreds of ssh sessions.
+		// We instead track JetBrains in the single persistent tcp forwarding channel.
 	case "":
 		s.connCountSSHSession.Add(1)
 		defer s.connCountSSHSession.Add(-1)
 	default:
-		s.logger.Warn(ctx, "invalid magic ssh session type specified", slog.F("type", magicType))
+		logger.Warn(ctx, "invalid magic ssh session type specified", slog.F("type", magicType))
 	}
 
+	magicTypeLabel := magicTypeMetricLabel(magicType)
+	sshPty, windowSize, isPty := session.Pty()
+
 	cmd, err := s.CreateCommand(ctx, session.RawCommand(), env)
 	if err != nil {
+		ptyLabel := "no"
+		if isPty {
+			ptyLabel = "yes"
+		}
+		s.metrics.sessionErrors.WithLabelValues(magicTypeLabel, ptyLabel, "create_command").Add(1)
 		return err
 	}
 
 	if ssh.AgentRequested(session) {
 		l, err := ssh.NewAgentListener()
 		if err != nil {
+			ptyLabel := "no"
+			if isPty {
+				ptyLabel = "yes"
+			}
+
+			s.metrics.sessionErrors.WithLabelValues(magicTypeLabel, ptyLabel, "listener").Add(1)
 			return xerrors.Errorf("new agent listener: %w", err)
 		}
 		defer l.Close()
@@ -251,30 +432,47 @@ func (s *Server) sessionStart(session ssh.Session, extraEnv []string) (retErr er
 		cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", "SSH_AUTH_SOCK", l.Addr().String()))
 	}
 
-	sshPty, windowSize, isPty := session.Pty()
 	if isPty {
-		return s.startPTYSession(session, cmd, sshPty, windowSize)
+		return s.startPTYSession(logger, session, magicTypeLabel, cmd, sshPty, windowSize)
 	}
-	return startNonPTYSession(session, cmd.AsExec())
+	return s.startNonPTYSession(logger, session, magicTypeLabel, cmd.AsExec())
 }
 
-func startNonPTYSession(session ssh.Session, cmd *exec.Cmd) error {
+func (s *Server) startNonPTYSession(logger slog.Logger, session ssh.Session, magicTypeLabel string, cmd *exec.Cmd) error {
+	s.metrics.sessionsTotal.WithLabelValues(magicTypeLabel, "no").Add(1)
+
 	cmd.Stdout = session
 	cmd.Stderr = session.Stderr()
 	// This blocks forever until stdin is received if we don't
 	// use StdinPipe. It's unknown what causes this.
 	stdinPipe, err := cmd.StdinPipe()
 	if err != nil {
+		s.metrics.sessionErrors.WithLabelValues(magicTypeLabel, "no", "stdin_pipe").Add(1)
 		return xerrors.Errorf("create stdin pipe: %w", err)
 	}
 	go func() {
-		_, _ = io.Copy(stdinPipe, session)
+		_, err := io.Copy(stdinPipe, session)
+		if err != nil {
+			s.metrics.sessionErrors.WithLabelValues(magicTypeLabel, "no", "stdin_io_copy").Add(1)
+		}
 		_ = stdinPipe.Close()
 	}()
 	err = cmd.Start()
 	if err != nil {
+		s.metrics.sessionErrors.WithLabelValues(magicTypeLabel, "no", "start_command").Add(1)
 		return xerrors.Errorf("start: %w", err)
 	}
+	sigs := make(chan ssh.Signal, 1)
+	session.Signals(sigs)
+	defer func() {
+		session.Signals(nil)
+		close(sigs)
+	}()
+	go func() {
+		for sig := range sigs {
+			s.handleSignal(logger, sig, cmd.Process, magicTypeLabel)
+		}
+	}()
 	return cmd.Wait()
 }
 
@@ -285,23 +483,36 @@ type ptySession interface {
 	Context() ssh.Context
 	DisablePTYEmulation()
 	RawCommand() string
+	Signals(chan<- ssh.Signal)
 }
 
-func (s *Server) startPTYSession(session ptySession, cmd *pty.Cmd, sshPty ssh.Pty, windowSize <-chan ssh.Window) (retErr error) {
+func (s *Server) startPTYSession(logger slog.Logger, session ptySession, magicTypeLabel string, cmd *pty.Cmd, sshPty ssh.Pty, windowSize <-chan ssh.Window) (retErr error) {
+	s.metrics.sessionsTotal.WithLabelValues(magicTypeLabel, "yes").Add(1)
+
 	ctx := session.Context()
 	// Disable minimal PTY emulation set by gliderlabs/ssh (NL-to-CRNL).
 	// See https://github.com/coder/coder/issues/3371.
 	session.DisablePTYEmulation()
 
-	if !isQuietLogin(session.RawCommand()) {
-		manifest := s.Manifest.Load()
-		if manifest != nil {
-			err := showMOTD(session, manifest.MOTDFile)
-			if err != nil {
-				s.logger.Error(ctx, "show MOTD", slog.Error(err))
+	if isLoginShell(session.RawCommand()) {
+		banners := s.config.AnnouncementBanners()
+		if banners != nil {
+			for _, banner := range *banners {
+				err := showAnnouncementBanner(session, banner)
+				if err != nil {
+					logger.Error(ctx, "agent failed to show announcement banner", slog.Error(err))
+					s.metrics.sessionErrors.WithLabelValues(magicTypeLabel, "yes", "announcement_banner").Add(1)
+					break
+				}
 			}
-		} else {
-			s.logger.Warn(ctx, "metadata lookup failed, unable to show MOTD")
+		}
+	}
+
+	if !isQuietLogin(s.fs, session.RawCommand()) {
+		err := showMOTD(s.fs, session, s.config.MOTDFile())
+		if err != nil {
+			logger.Error(ctx, "agent failed to show MOTD", slog.Error(err))
+			s.metrics.sessionErrors.WithLabelValues(magicTypeLabel, "yes", "motd").Add(1)
 		}
 	}
 
@@ -310,32 +521,61 @@ func (s *Server) startPTYSession(session ptySession, cmd *pty.Cmd, sshPty ssh.Pt
 	// The pty package sets `SSH_TTY` on supported platforms.
 	ptty, process, err := pty.Start(cmd, pty.WithPTYOption(
 		pty.WithSSHRequest(sshPty),
-		pty.WithLogger(slog.Stdlib(ctx, s.logger, slog.LevelInfo)),
+		pty.WithLogger(slog.Stdlib(ctx, logger, slog.LevelInfo)),
 	))
 	if err != nil {
+		s.metrics.sessionErrors.WithLabelValues(magicTypeLabel, "yes", "start_command").Add(1)
 		return xerrors.Errorf("start command: %w", err)
 	}
 	defer func() {
 		closeErr := ptty.Close()
 		if closeErr != nil {
-			s.logger.Warn(ctx, "failed to close tty", slog.Error(closeErr))
+			logger.Warn(ctx, "failed to close tty", slog.Error(closeErr))
+			s.metrics.sessionErrors.WithLabelValues(magicTypeLabel, "yes", "close").Add(1)
 			if retErr == nil {
 				retErr = closeErr
 			}
 		}
 	}()
+	sigs := make(chan ssh.Signal, 1)
+	session.Signals(sigs)
+	defer func() {
+		session.Signals(nil)
+		close(sigs)
+	}()
 	go func() {
-		for win := range windowSize {
-			resizeErr := ptty.Resize(uint16(win.Height), uint16(win.Width))
-			// If the pty is closed, then command has exited, no need to log.
-			if resizeErr != nil && !errors.Is(resizeErr, pty.ErrClosed) {
-				s.logger.Warn(ctx, "failed to resize tty", slog.Error(resizeErr))
+		for {
+			if sigs == nil && windowSize == nil {
+				return
+			}
+
+			select {
+			case sig, ok := <-sigs:
+				if !ok {
+					sigs = nil
+					continue
+				}
+				s.handleSignal(logger, sig, process, magicTypeLabel)
+			case win, ok := <-windowSize:
+				if !ok {
+					windowSize = nil
+					continue
+				}
+				resizeErr := ptty.Resize(uint16(win.Height), uint16(win.Width))
+				// If the pty is closed, then command has exited, no need to log.
+				if resizeErr != nil && !errors.Is(resizeErr, pty.ErrClosed) {
+					logger.Warn(ctx, "failed to resize tty", slog.Error(resizeErr))
+					s.metrics.sessionErrors.WithLabelValues(magicTypeLabel, "yes", "resize").Add(1)
+				}
 			}
 		}
 	}()
 
 	go func() {
-		_, _ = io.Copy(ptty.InputWriter(), session)
+		_, err := io.Copy(ptty.InputWriter(), session)
+		if err != nil {
+			s.metrics.sessionErrors.WithLabelValues(magicTypeLabel, "yes", "input_io_copy").Add(1)
+		}
 	}()
 
 	// We need to wait for the command output to finish copying.  It's safe to
@@ -347,8 +587,9 @@ func (s *Server) startPTYSession(session ptySession, cmd *pty.Cmd, sshPty ssh.Pt
 	// 2. The client hangs up, which cancels the command's Context, and go will
 	//    kill the command's process.  This then has the same effect as (1).
 	n, err := io.Copy(session, ptty.OutputReader())
-	s.logger.Debug(ctx, "copy output done", slog.F("bytes", n), slog.Error(err))
+	logger.Debug(ctx, "copy output done", slog.F("bytes", n), slog.Error(err))
 	if err != nil {
+		s.metrics.sessionErrors.WithLabelValues(magicTypeLabel, "yes", "output_io_copy").Add(1)
 		return xerrors.Errorf("copy error: %w", err)
 	}
 	// We've gotten all the output, but we need to wait for the process to
@@ -359,7 +600,8 @@ func (s *Server) startPTYSession(session ptySession, cmd *pty.Cmd, sshPty ssh.Pt
 	// ExitErrors just mean the command we run returned a non-zero exit code, which is normal
 	// and not something to be concerned about.  But, if it's something else, we should log it.
 	if err != nil && !xerrors.As(err, &exitErr) {
-		s.logger.Warn(ctx, "wait error", slog.Error(err))
+		logger.Warn(ctx, "process wait exited with error", slog.Error(err))
+		s.metrics.sessionErrors.WithLabelValues(magicTypeLabel, "yes", "wait").Add(1)
 	}
 	if err != nil {
 		return xerrors.Errorf("process wait: %w", err)
@@ -367,7 +609,21 @@ func (s *Server) startPTYSession(session ptySession, cmd *pty.Cmd, sshPty ssh.Pt
 	return nil
 }
 
-func (s *Server) sftpHandler(session ssh.Session) {
+func (s *Server) handleSignal(logger slog.Logger, ssig ssh.Signal, signaler interface{ Signal(os.Signal) error }, magicTypeLabel string) {
+	ctx := context.Background()
+	sig := osSignalFrom(ssig)
+	logger = logger.With(slog.F("ssh_signal", ssig), slog.F("signal", sig.String()))
+	logger.Info(ctx, "received signal from client")
+	err := signaler.Signal(sig)
+	if err != nil {
+		logger.Warn(ctx, "signaling the process failed", slog.Error(err))
+		s.metrics.sessionErrors.WithLabelValues(magicTypeLabel, "yes", "signal").Add(1)
+	}
+}
+
+func (s *Server) sftpHandler(logger slog.Logger, session ssh.Session) {
+	s.metrics.sftpConnectionsTotal.Add(1)
+
 	ctx := session.Context()
 
 	// Typically sftp sessions don't request a TTY, but if they do,
@@ -381,20 +637,20 @@ func (s *Server) sftpHandler(session ssh.Session) {
 	// directory so that SFTP connections land there.
 	homedir, err := userHomeDir()
 	if err != nil {
-		s.logger.Warn(ctx, "get sftp working directory failed, unable to get home dir", slog.Error(err))
+		logger.Warn(ctx, "get sftp working directory failed, unable to get home dir", slog.Error(err))
 	} else {
 		opts = append(opts, sftp.WithServerWorkingDirectory(homedir))
 	}
 
 	server, err := sftp.NewServer(session, opts...)
 	if err != nil {
-		s.logger.Debug(ctx, "initialize sftp server", slog.Error(err))
+		logger.Debug(ctx, "initialize sftp server", slog.Error(err))
 		return
 	}
 	defer server.Close()
 
 	err = server.Serve()
-	if errors.Is(err, io.EOF) {
+	if err == nil || errors.Is(err, io.EOF) {
 		// Unless we call `session.Exit(0)` here, the client won't
 		// receive `exit-status` because `(*sftp.Server).Close()`
 		// calls `Close()` on the underlying connection (session),
@@ -406,7 +662,8 @@ func (s *Server) sftpHandler(session ssh.Session) {
 		_ = session.Exit(0)
 		return
 	}
-	s.logger.Warn(ctx, "sftp server closed with error", slog.Error(err))
+	logger.Warn(ctx, "sftp server closed with error", slog.Error(err))
+	s.metrics.sftpServerErrors.Add(1)
 	_ = session.Exit(1)
 }
 
@@ -425,19 +682,38 @@ func (s *Server) CreateCommand(ctx context.Context, script string, env []string)
 		return nil, xerrors.Errorf("get user shell: %w", err)
 	}
 
-	manifest := s.Manifest.Load()
-	if manifest == nil {
-		return nil, xerrors.Errorf("no metadata was provided")
-	}
-
 	// OpenSSH executes all commands with the users current shell.
 	// We replicate that behavior for IDE support.
 	caller := "-c"
 	if runtime.GOOS == "windows" {
 		caller = "/c"
 	}
+	name := shell
 	args := []string{caller, script}
 
+	// A preceding space is generally not idiomatic for a shebang,
+	// but in Terraform it's quite standard to use <<EOF for a multi-line
+	// string which would indent with spaces, so we accept it for user-ease.
+	if strings.HasPrefix(strings.TrimSpace(script), "#!") {
+		// If the script starts with a shebang, we should
+		// execute it directly. This is useful for running
+		// scripts that aren't executable.
+		shebang := strings.SplitN(strings.TrimSpace(script), "\n", 2)[0]
+		shebang = strings.TrimSpace(shebang)
+		shebang = strings.TrimPrefix(shebang, "#!")
+		words, err := shellquote.Split(shebang)
+		if err != nil {
+			return nil, xerrors.Errorf("split shebang: %w", err)
+		}
+		name = words[0]
+		if len(words) > 1 {
+			args = words[1:]
+		} else {
+			args = []string{}
+		}
+		args = append(args, caller, script)
+	}
+
 	// gliderlabs/ssh returns a command slice of zero
 	// when a shell is requested.
 	if len(script) == 0 {
@@ -449,8 +725,8 @@ func (s *Server) CreateCommand(ctx context.Context, script string, env []string)
 		}
 	}
 
-	cmd := pty.CommandContext(ctx, shell, args...)
-	cmd.Dir = manifest.Directory
+	cmd := pty.CommandContext(ctx, name, args...)
+	cmd.Dir = s.config.WorkingDirectory()
 
 	// If the metadata directory doesn't exist, we run the command
 	// in the users home directory.
@@ -464,21 +740,7 @@ func (s *Server) CreateCommand(ctx context.Context, script string, env []string)
 		cmd.Dir = homedir
 	}
 	cmd.Env = append(os.Environ(), env...)
-	executablePath, err := os.Executable()
-	if err != nil {
-		return nil, xerrors.Errorf("getting os executable: %w", err)
-	}
-	// Set environment variables reliable detection of being inside a
-	// Coder workspace.
-	cmd.Env = append(cmd.Env, "CODER=true")
 	cmd.Env = append(cmd.Env, fmt.Sprintf("USER=%s", username))
-	// Git on Windows resolves with UNIX-style paths.
-	// If using backslashes, it's unable to find the executable.
-	unixExecutablePath := strings.ReplaceAll(executablePath, "\\", "/")
-	cmd.Env = append(cmd.Env, fmt.Sprintf(`GIT_SSH_COMMAND=%s gitssh --`, unixExecutablePath))
-
-	// Specific Coder subcommands require the agent token exposed!
-	cmd.Env = append(cmd.Env, fmt.Sprintf("CODER_AGENT_TOKEN=%s", s.AgentToken()))
 
 	// Set SSH connection environment variables (these are also set by OpenSSH
 	// and thus expected to be present by SSH clients). Since the agent does
@@ -489,32 +751,20 @@ func (s *Server) CreateCommand(ctx context.Context, script string, env []string)
 	cmd.Env = append(cmd.Env, fmt.Sprintf("SSH_CLIENT=%s %s %s", srcAddr, srcPort, dstPort))
 	cmd.Env = append(cmd.Env, fmt.Sprintf("SSH_CONNECTION=%s %s %s %s", srcAddr, srcPort, dstAddr, dstPort))
 
-	// This adds the ports dialog to code-server that enables
-	// proxying a port dynamically.
-	cmd.Env = append(cmd.Env, fmt.Sprintf("VSCODE_PROXY_URI=%s", manifest.VSCodePortProxyURI))
-
-	// Hide Coder message on code-server's "Getting Started" page
-	cmd.Env = append(cmd.Env, "CS_DISABLE_GETTING_STARTED_OVERRIDE=true")
-
-	// Load environment variables passed via the agent.
-	// These should override all variables we manually specify.
-	for envKey, value := range manifest.EnvironmentVariables {
-		// Expanding environment variables allows for customization
-		// of the $PATH, among other variables. Customers can prepend
-		// or append to the $PATH, so allowing expand is required!
-		cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", envKey, os.ExpandEnv(value)))
-	}
-
-	// Agent-level environment variables should take over all!
-	// This is used for setting agent-specific variables like "CODER_AGENT_TOKEN".
-	for envKey, value := range s.Env {
-		cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", envKey, value))
+	cmd.Env, err = s.config.UpdateEnv(cmd.Env)
+	if err != nil {
+		return nil, xerrors.Errorf("apply env: %w", err)
 	}
 
 	return cmd, nil
 }
 
-func (s *Server) Serve(l net.Listener) error {
+func (s *Server) Serve(l net.Listener) (retErr error) {
+	s.logger.Info(context.Background(), "started serving listener", slog.F("listen_addr", l.Addr()))
+	defer func() {
+		s.logger.Info(context.Background(), "stopped serving listener",
+			slog.F("listen_addr", l.Addr()), slog.Error(retErr))
+	}()
 	defer l.Close()
 
 	s.trackListener(l, true)
@@ -529,16 +779,21 @@ func (s *Server) Serve(l net.Listener) error {
 }
 
 func (s *Server) handleConn(l net.Listener, c net.Conn) {
+	logger := s.logger.With(
+		slog.F("remote_addr", c.RemoteAddr()),
+		slog.F("local_addr", c.LocalAddr()),
+		slog.F("listen_addr", l.Addr()))
 	defer c.Close()
 
 	if !s.trackConn(l, c, true) {
 		// Server is closed or we no longer want
 		// connections from this listener.
-		s.logger.Debug(context.Background(), "received connection after server closed")
+		logger.Info(context.Background(), "received connection after server closed")
 		return
 	}
 	defer s.trackConn(l, c, false)
-
+	logger.Info(context.Background(), "started serving connection")
+	// note: srv.ConnectionCompleteCallback logs completion of the connection
 	s.srv.HandleConn(c)
 }
 
@@ -607,9 +862,11 @@ func (s *Server) trackSession(ss ssh.Session, add bool) (ok bool) {
 			// Server closed.
 			return false
 		}
+		s.wg.Add(1)
 		s.sessions[ss] = struct{}{}
 		return true
 	}
+	s.wg.Done()
 	delete(s.sessions, ss)
 	return true
 }
@@ -667,12 +924,16 @@ func (*Server) Shutdown(_ context.Context) error {
 	return nil
 }
 
+func isLoginShell(rawCommand string) bool {
+	return len(rawCommand) == 0
+}
+
 // isQuietLogin checks if the SSH server should perform a quiet login or not.
 //
 // https://github.com/openssh/openssh-portable/blob/25bd659cc72268f2858c5415740c442ee950049f/session.c#L816
-func isQuietLogin(rawCommand string) bool {
+func isQuietLogin(fs afero.Fs, rawCommand string) bool {
 	// We are always quiet unless this is a login shell.
-	if len(rawCommand) != 0 {
+	if !isLoginShell(rawCommand) {
 		return true
 	}
 
@@ -683,20 +944,32 @@ func isQuietLogin(rawCommand string) bool {
 		return false
 	}
 
-	_, err = os.Stat(filepath.Join(homedir, ".hushlogin"))
+	_, err = fs.Stat(filepath.Join(homedir, ".hushlogin"))
 	return err == nil
 }
 
+// showAnnouncementBanner will write the service banner if enabled and not blank
+// along with a blank line for spacing.
+func showAnnouncementBanner(session io.Writer, banner codersdk.BannerConfig) error {
+	if banner.Enabled && banner.Message != "" {
+		// The banner supports Markdown so we might want to parse it but Markdown is
+		// still fairly readable in its raw form.
+		message := strings.TrimSpace(banner.Message) + "\n\n"
+		return writeWithCarriageReturn(strings.NewReader(message), session)
+	}
+	return nil
+}
+
 // showMOTD will output the message of the day from
 // the given filename to dest, if the file exists.
 //
 // https://github.com/openssh/openssh-portable/blob/25bd659cc72268f2858c5415740c442ee950049f/session.c#L784
-func showMOTD(dest io.Writer, filename string) error {
+func showMOTD(fs afero.Fs, dest io.Writer, filename string) error {
 	if filename == "" {
 		return nil
 	}
 
-	f, err := os.Open(filename)
+	f, err := fs.Open(filename)
 	if err != nil {
 		if xerrors.Is(err, os.ErrNotExist) {
 			// This is not an error, there simply isn't a MOTD to show.
@@ -706,19 +979,22 @@ func showMOTD(dest io.Writer, filename string) error {
 	}
 	defer f.Close()
 
-	s := bufio.NewScanner(f)
+	return writeWithCarriageReturn(f, dest)
+}
+
+// writeWithCarriageReturn writes each line with a carriage return to ensure
+// that each line starts at the beginning of the terminal.
+func writeWithCarriageReturn(src io.Reader, dest io.Writer) error {
+	s := bufio.NewScanner(src)
 	for s.Scan() {
-		// Carriage return ensures each line starts
-		// at the beginning of the terminal.
-		_, err = fmt.Fprint(dest, s.Text()+"\r\n")
+		_, err := fmt.Fprint(dest, s.Text()+"\r\n")
 		if err != nil {
-			return xerrors.Errorf("write MOTD: %w", err)
+			return xerrors.Errorf("write line: %w", err)
 		}
 	}
 	if err := s.Err(); err != nil {
-		return xerrors.Errorf("read MOTD: %w", err)
+		return xerrors.Errorf("read line: %w", err)
 	}
-
 	return nil
 }
 

agent/agentssh/agentssh_internal_test.go

@@ -10,14 +10,13 @@ import (
 	"testing"
 
 	gliderssh "github.com/gliderlabs/ssh"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/spf13/afero"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
-	"github.com/coder/coder/pty"
-	"github.com/coder/coder/testutil"
-
-	"cdr.dev/slog/sloggers/slogtest"
+	"github.com/coder/coder/v2/pty"
+	"github.com/coder/coder/v2/testutil"
 )
 
 const longScript = `
@@ -35,9 +34,10 @@ func Test_sessionStart_orphan(t *testing.T) {
 
 	ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitMedium)
 	defer cancel()
-	logger := slogtest.Make(t, nil)
-	s, err := NewServer(ctx, logger, afero.NewMemMapFs(), 0, "")
+	logger := testutil.Logger(t)
+	s, err := NewServer(ctx, logger, prometheus.NewRegistry(), afero.NewMemMapFs(), nil)
 	require.NoError(t, err)
+	defer s.Close()
 
 	// Here we're going to call the handler directly with a faked SSH session
 	// that just uses io.Pipes instead of a network socket.  There is a large
@@ -57,10 +57,11 @@ func Test_sessionStart_orphan(t *testing.T) {
 	done := make(chan struct{})
 	go func() {
 		defer close(done)
+
 		// we don't really care what the error is here.  In the larger scenario,
 		// the client has disconnected, so we can't return any error information
 		// to them.
-		_ = s.startPTYSession(sess, cmd, ptyInfo, windowSize)
+		_ = s.startPTYSession(logger, sess, "ssh", cmd, ptyInfo, windowSize)
 	}()
 
 	readDone := make(chan struct{})
@@ -111,6 +112,11 @@ type testSSHContext struct {
 	context.Context
 }
 
+var (
+	_ gliderssh.Context = testSSHContext{}
+	_ ptySession        = &testSession{}
+)
+
 func newTestSession(ctx context.Context) (toClient *io.PipeReader, fromClient *io.PipeWriter, s ptySession) {
 	toClient, fromPty := io.Pipe()
 	toPty, fromClient := io.Pipe()
@@ -141,6 +147,10 @@ func (s *testSession) Write(p []byte) (n int, err error) {
 	return s.fromPty.Write(p)
 }
 
+func (*testSession) Signals(_ chan<- gliderssh.Signal) {
+	// Not implemented, but will be called.
+}
+
 func (testSSHContext) Lock() {
 	panic("not implemented")
 }
@@ -188,3 +198,7 @@ func (testSSHContext) Permissions() *gliderssh.Permissions {
 func (testSSHContext) SetValue(_, _ interface{}) {
 	panic("not implemented")
 }
+
+func (testSSHContext) KeepAlive() *gliderssh.SessionKeepAlive {
+	panic("not implemented")
+}

agent/agentssh/agentssh_test.go

@@ -3,25 +3,28 @@
 package agentssh_test
 
 import (
+	"bufio"
 	"bytes"
 	"context"
+	"fmt"
 	"net"
+	"runtime"
 	"strings"
 	"sync"
 	"testing"
 
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/spf13/afero"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-	"go.uber.org/atomic"
 	"go.uber.org/goleak"
 	"golang.org/x/crypto/ssh"
 
 	"cdr.dev/slog/sloggers/slogtest"
 
-	"github.com/coder/coder/agent/agentssh"
-	"github.com/coder/coder/codersdk/agentsdk"
-	"github.com/coder/coder/pty/ptytest"
+	"github.com/coder/coder/v2/agent/agentssh"
+	"github.com/coder/coder/v2/pty/ptytest"
+	"github.com/coder/coder/v2/testutil"
 )
 
 func TestMain(m *testing.M) {
@@ -32,13 +35,10 @@ func TestNewServer_ServeClient(t *testing.T) {
 	t.Parallel()
 
 	ctx := context.Background()
-	logger := slogtest.Make(t, nil)
-	s, err := agentssh.NewServer(ctx, logger, afero.NewMemMapFs(), 0, "")
+	logger := testutil.Logger(t)
+	s, err := agentssh.NewServer(ctx, logger, prometheus.NewRegistry(), afero.NewMemMapFs(), nil)
 	require.NoError(t, err)
-
-	// The assumption is that these are set before serving SSH connections.
-	s.AgentToken = func() string { return "" }
-	s.Manifest = atomic.NewPointer(&agentsdk.Manifest{})
+	defer s.Close()
 
 	ln, err := net.Listen("tcp", "127.0.0.1:0")
 	require.NoError(t, err)
@@ -54,8 +54,8 @@ func TestNewServer_ServeClient(t *testing.T) {
 
 	var b bytes.Buffer
 	sess, err := c.NewSession()
-	sess.Stdout = &b
 	require.NoError(t, err)
+	sess.Stdout = &b
 	err = sess.Start("echo hello")
 	require.NoError(t, err)
 
@@ -69,17 +69,48 @@ func TestNewServer_ServeClient(t *testing.T) {
 	<-done
 }
 
+func TestNewServer_ExecuteShebang(t *testing.T) {
+	t.Parallel()
+	if runtime.GOOS == "windows" {
+		t.Skip("bash doesn't exist on Windows")
+	}
+
+	ctx := context.Background()
+	logger := testutil.Logger(t)
+	s, err := agentssh.NewServer(ctx, logger, prometheus.NewRegistry(), afero.NewMemMapFs(), nil)
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		_ = s.Close()
+	})
+
+	t.Run("Basic", func(t *testing.T) {
+		t.Parallel()
+		cmd, err := s.CreateCommand(ctx, `#!/bin/bash
+		echo test`, nil)
+		require.NoError(t, err)
+		output, err := cmd.AsExec().CombinedOutput()
+		require.NoError(t, err)
+		require.Equal(t, "test\n", string(output))
+	})
+	t.Run("Args", func(t *testing.T) {
+		t.Parallel()
+		cmd, err := s.CreateCommand(ctx, `#!/usr/bin/env bash
+		echo test`, nil)
+		require.NoError(t, err)
+		output, err := cmd.AsExec().CombinedOutput()
+		require.NoError(t, err)
+		require.Equal(t, "test\n", string(output))
+	})
+}
+
 func TestNewServer_CloseActiveConnections(t *testing.T) {
 	t.Parallel()
 
 	ctx := context.Background()
 	logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true})
-	s, err := agentssh.NewServer(ctx, logger, afero.NewMemMapFs(), 0, "")
+	s, err := agentssh.NewServer(ctx, logger, prometheus.NewRegistry(), afero.NewMemMapFs(), nil)
 	require.NoError(t, err)
-
-	// The assumption is that these are set before serving SSH connections.
-	s.AgentToken = func() string { return "" }
-	s.Manifest = atomic.NewPointer(&agentsdk.Manifest{})
+	defer s.Close()
 
 	ln, err := net.Listen("tcp", "127.0.0.1:0")
 	require.NoError(t, err)
@@ -99,6 +130,7 @@ func TestNewServer_CloseActiveConnections(t *testing.T) {
 		defer wg.Done()
 		c := sshClient(t, ln.Addr().String())
 		sess, err := c.NewSession()
+		assert.NoError(t, err)
 		sess.Stdin = pty.Input()
 		sess.Stdout = pty.Output()
 		sess.Stderr = pty.Output()
@@ -119,6 +151,151 @@ func TestNewServer_CloseActiveConnections(t *testing.T) {
 	wg.Wait()
 }
 
+func TestNewServer_Signal(t *testing.T) {
+	t.Parallel()
+
+	t.Run("Stdout", func(t *testing.T) {
+		t.Parallel()
+
+		ctx := context.Background()
+		logger := testutil.Logger(t)
+		s, err := agentssh.NewServer(ctx, logger, prometheus.NewRegistry(), afero.NewMemMapFs(), nil)
+		require.NoError(t, err)
+		defer s.Close()
+
+		ln, err := net.Listen("tcp", "127.0.0.1:0")
+		require.NoError(t, err)
+
+		done := make(chan struct{})
+		go func() {
+			defer close(done)
+			err := s.Serve(ln)
+			assert.Error(t, err) // Server is closed.
+		}()
+		defer func() {
+			err := s.Close()
+			require.NoError(t, err)
+			<-done
+		}()
+
+		c := sshClient(t, ln.Addr().String())
+
+		sess, err := c.NewSession()
+		require.NoError(t, err)
+		r, err := sess.StdoutPipe()
+		require.NoError(t, err)
+
+		// Perform multiple sleeps since the interrupt signal doesn't propagate to
+		// the process group, this lets us exit early.
+		sleeps := strings.Repeat("sleep 1 && ", int(testutil.WaitMedium.Seconds()))
+		err = sess.Start(fmt.Sprintf("echo hello && %s echo bye", sleeps))
+		require.NoError(t, err)
+
+		sc := bufio.NewScanner(r)
+		for sc.Scan() {
+			t.Log(sc.Text())
+			if strings.Contains(sc.Text(), "hello") {
+				break
+			}
+		}
+		require.NoError(t, sc.Err())
+
+		err = sess.Signal(ssh.SIGKILL)
+		require.NoError(t, err)
+
+		// Assumption, signal propagates and the command exists, closing stdout.
+		for sc.Scan() {
+			t.Log(sc.Text())
+			require.NotContains(t, sc.Text(), "bye")
+		}
+		require.NoError(t, sc.Err())
+
+		err = sess.Wait()
+		exitErr := &ssh.ExitError{}
+		require.ErrorAs(t, err, &exitErr)
+		wantCode := 255
+		if runtime.GOOS == "windows" {
+			wantCode = 1
+		}
+		require.Equal(t, wantCode, exitErr.ExitStatus())
+	})
+	t.Run("PTY", func(t *testing.T) {
+		t.Parallel()
+
+		ctx := context.Background()
+		logger := testutil.Logger(t)
+		s, err := agentssh.NewServer(ctx, logger, prometheus.NewRegistry(), afero.NewMemMapFs(), nil)
+		require.NoError(t, err)
+		defer s.Close()
+
+		ln, err := net.Listen("tcp", "127.0.0.1:0")
+		require.NoError(t, err)
+
+		done := make(chan struct{})
+		go func() {
+			defer close(done)
+			err := s.Serve(ln)
+			assert.Error(t, err) // Server is closed.
+		}()
+		defer func() {
+			err := s.Close()
+			require.NoError(t, err)
+			<-done
+		}()
+
+		c := sshClient(t, ln.Addr().String())
+
+		pty := ptytest.New(t)
+
+		sess, err := c.NewSession()
+		require.NoError(t, err)
+		r, err := sess.StdoutPipe()
+		require.NoError(t, err)
+
+		// Note, we request pty but don't use ptytest here because we can't
+		// easily test for no text before EOF.
+		sess.Stdin = pty.Input()
+		sess.Stderr = pty.Output()
+
+		err = sess.RequestPty("xterm", 80, 80, nil)
+		require.NoError(t, err)
+
+		// Perform multiple sleeps since the interrupt signal doesn't propagate to
+		// the process group, this lets us exit early.
+		sleeps := strings.Repeat("sleep 1 && ", int(testutil.WaitMedium.Seconds()))
+		err = sess.Start(fmt.Sprintf("echo hello && %s echo bye", sleeps))
+		require.NoError(t, err)
+
+		sc := bufio.NewScanner(r)
+		for sc.Scan() {
+			t.Log(sc.Text())
+			if strings.Contains(sc.Text(), "hello") {
+				break
+			}
+		}
+		require.NoError(t, sc.Err())
+
+		err = sess.Signal(ssh.SIGKILL)
+		require.NoError(t, err)
+
+		// Assumption, signal propagates and the command exists, closing stdout.
+		for sc.Scan() {
+			t.Log(sc.Text())
+			require.NotContains(t, sc.Text(), "bye")
+		}
+		require.NoError(t, sc.Err())
+
+		err = sess.Wait()
+		exitErr := &ssh.ExitError{}
+		require.ErrorAs(t, err, &exitErr)
+		wantCode := 255
+		if runtime.GOOS == "windows" {
+			wantCode = 1
+		}
+		require.Equal(t, wantCode, exitErr.ExitStatus())
+	})
+}
+
 func sshClient(t *testing.T, addr string) *ssh.Client {
 	conn, err := net.Dial("tcp", addr)
 	require.NoError(t, err)

agent/agentssh/forward.go

@@ -2,11 +2,14 @@ package agentssh
 
 import (
 	"context"
+	"errors"
 	"fmt"
+	"io/fs"
 	"net"
 	"os"
 	"path/filepath"
 	"sync"
+	"syscall"
 
 	"github.com/gliderlabs/ssh"
 	gossh "golang.org/x/crypto/ssh"
@@ -33,70 +36,98 @@ type forwardedStreamLocalPayload struct {
 type forwardedUnixHandler struct {
 	sync.Mutex
 	log      slog.Logger
-	forwards map[string]net.Listener
+	forwards map[forwardKey]net.Listener
+}
+
+type forwardKey struct {
+	sessionID string
+	addr      string
+}
+
+func newForwardedUnixHandler(log slog.Logger) *forwardedUnixHandler {
+	return &forwardedUnixHandler{
+		log:      log,
+		forwards: make(map[forwardKey]net.Listener),
+	}
 }
 
 func (h *forwardedUnixHandler) HandleSSHRequest(ctx ssh.Context, _ *ssh.Server, req *gossh.Request) (bool, []byte) {
-	h.Lock()
-	if h.forwards == nil {
-		h.forwards = make(map[string]net.Listener)
-	}
-	h.Unlock()
+	h.log.Debug(ctx, "handling SSH unix forward")
 	conn, ok := ctx.Value(ssh.ContextKeyConn).(*gossh.ServerConn)
 	if !ok {
 		h.log.Warn(ctx, "SSH unix forward request from client with no gossh connection")
 		return false, nil
 	}
+	log := h.log.With(slog.F("session_id", ctx.SessionID()), slog.F("remote_addr", conn.RemoteAddr()))
 
 	switch req.Type {
 	case "streamlocal-forward@openssh.com":
 		var reqPayload streamLocalForwardPayload
 		err := gossh.Unmarshal(req.Payload, &reqPayload)
 		if err != nil {
-			h.log.Warn(ctx, "parse streamlocal-forward@openssh.com request payload from client", slog.Error(err))
+			h.log.Warn(ctx, "parse streamlocal-forward@openssh.com request (SSH unix forward) payload from client", slog.Error(err))
 			return false, nil
 		}
 
 		addr := reqPayload.SocketPath
+		log = log.With(slog.F("socket_path", addr))
+		log.Debug(ctx, "request begin SSH unix forward")
+
+		key := forwardKey{
+			sessionID: ctx.SessionID(),
+			addr:      addr,
+		}
+
 		h.Lock()
-		_, ok := h.forwards[addr]
+		_, ok := h.forwards[key]
 		h.Unlock()
 		if ok {
-			h.log.Warn(ctx, "SSH unix forward request for socket path that is already being forwarded (maybe to another client?)",
-				slog.F("socket_path", addr),
-			)
-			return false, nil
+			// In cases where `ExitOnForwardFailure=yes` is set, returning false
+			// here will cause the connection to be closed. To avoid this, and
+			// to match OpenSSH behavior, we silently ignore the second forward
+			// request.
+			log.Warn(ctx, "SSH unix forward request for socket path that is already being forwarded on this session, ignoring")
+			return true, nil
 		}
 
 		// Create socket parent dir if not exists.
 		parentDir := filepath.Dir(addr)
 		err = os.MkdirAll(parentDir, 0o700)
 		if err != nil {
-			h.log.Warn(ctx, "create parent dir for SSH unix forward request",
+			log.Warn(ctx, "create parent dir for SSH unix forward request",
 				slog.F("parent_dir", parentDir),
-				slog.F("socket_path", addr),
 				slog.Error(err),
 			)
 			return false, nil
 		}
 
-		ln, err := net.Listen("unix", addr)
-		if err != nil {
-			h.log.Warn(ctx, "listen on Unix socket for SSH unix forward request",
-				slog.F("socket_path", addr),
-				slog.Error(err),
-			)
+		// Remove existing socket if it exists. We do not use os.Remove() here
+		// so that directories are kept. Note that it's possible that we will
+		// overwrite a regular file here. Both of these behaviors match OpenSSH,
+		// however, which is why we unlink.
+		err = unlink(addr)
+		if err != nil && !errors.Is(err, fs.ErrNotExist) {
+			log.Warn(ctx, "remove existing socket for SSH unix forward request", slog.Error(err))
 			return false, nil
 		}
 
+		lc := &net.ListenConfig{}
+		ln, err := lc.Listen(ctx, "unix", addr)
+		if err != nil {
+			log.Warn(ctx, "listen on Unix socket for SSH unix forward request", slog.Error(err))
+			return false, nil
+		}
+		log.Debug(ctx, "SSH unix forward listening on socket")
+
 		// The listener needs to successfully start before it can be added to
 		// the map, so we don't have to worry about checking for an existing
 		// listener.
 		//
 		// This is also what the upstream TCP version of this code does.
 		h.Lock()
-		h.forwards[addr] = ln
+		h.forwards[key] = ln
 		h.Unlock()
+		log.Debug(ctx, "SSH unix forward added to cache")
 
 		ctx, cancel := context.WithCancel(ctx)
 		go func() {
@@ -110,14 +141,13 @@ func (h *forwardedUnixHandler) HandleSSHRequest(ctx ssh.Context, _ *ssh.Server,
 				c, err := ln.Accept()
 				if err != nil {
 					if !xerrors.Is(err, net.ErrClosed) {
-						h.log.Warn(ctx, "accept on local Unix socket for SSH unix forward request",
-							slog.F("socket_path", addr),
-							slog.Error(err),
-						)
+						log.Warn(ctx, "accept on local Unix socket for SSH unix forward request", slog.Error(err))
 					}
 					// closed below
+					log.Debug(ctx, "SSH unix forward listener closed")
 					break
 				}
+				log.Debug(ctx, "accepted SSH unix forward connection")
 				payload := gossh.Marshal(&forwardedStreamLocalPayload{
 					SocketPath: addr,
 				})
@@ -125,10 +155,7 @@ func (h *forwardedUnixHandler) HandleSSHRequest(ctx ssh.Context, _ *ssh.Server,
 				go func() {
 					ch, reqs, err := conn.OpenChannel("forwarded-streamlocal@openssh.com", payload)
 					if err != nil {
-						h.log.Warn(ctx, "open SSH channel to forward Unix connection to client",
-							slog.F("socket_path", addr),
-							slog.Error(err),
-						)
+						h.log.Warn(ctx, "open SSH unix forward channel to client", slog.Error(err))
 						_ = c.Close()
 						return
 					}
@@ -138,11 +165,11 @@ func (h *forwardedUnixHandler) HandleSSHRequest(ctx ssh.Context, _ *ssh.Server,
 			}
 
 			h.Lock()
-			ln2, ok := h.forwards[addr]
-			if ok && ln2 == ln {
-				delete(h.forwards, addr)
+			if ln2, ok := h.forwards[key]; ok && ln2 == ln {
+				delete(h.forwards, key)
 			}
 			h.Unlock()
+			log.Debug(ctx, "SSH unix forward listener removed from cache")
 			_ = ln.Close()
 		}()
 
@@ -152,15 +179,25 @@ func (h *forwardedUnixHandler) HandleSSHRequest(ctx ssh.Context, _ *ssh.Server,
 		var reqPayload streamLocalForwardPayload
 		err := gossh.Unmarshal(req.Payload, &reqPayload)
 		if err != nil {
-			h.log.Warn(ctx, "parse cancel-streamlocal-forward@openssh.com request payload from client", slog.Error(err))
+			h.log.Warn(ctx, "parse cancel-streamlocal-forward@openssh.com (SSH unix forward) request payload from client", slog.Error(err))
 			return false, nil
 		}
-		h.Lock()
-		ln, ok := h.forwards[reqPayload.SocketPath]
-		h.Unlock()
-		if ok {
-			_ = ln.Close()
+		log.Debug(ctx, "request to cancel SSH unix forward", slog.F("socket_path", reqPayload.SocketPath))
+
+		key := forwardKey{
+			sessionID: ctx.SessionID(),
+			addr:      reqPayload.SocketPath,
 		}
+
+		h.Lock()
+		ln, ok := h.forwards[key]
+		delete(h.forwards, key)
+		h.Unlock()
+		if !ok {
+			log.Warn(ctx, "SSH unix forward not found in cache")
+			return true, nil
+		}
+		_ = ln.Close()
 		return true, nil
 
 	default:
@@ -201,3 +238,15 @@ func directStreamLocalHandler(_ *ssh.Server, _ *gossh.ServerConn, newChan gossh.
 
 	Bicopy(ctx, ch, dconn)
 }
+
+// unlink removes files and unlike os.Remove, directories are kept.
+func unlink(path string) error {
+	// Ignore EINTR like os.Remove, see ignoringEINTR in os/file_posix.go
+	// for more details.
+	for {
+		err := syscall.Unlink(path)
+		if !errors.Is(err, syscall.EINTR) {
+			return err
+		}
+	}
+}

agent/agentssh/jetbrainstrack.go

@@ -0,0 +1,97 @@
+package agentssh
+
+import (
+	"context"
+	"strings"
+	"sync"
+
+	"github.com/gliderlabs/ssh"
+	"go.uber.org/atomic"
+	gossh "golang.org/x/crypto/ssh"
+
+	"cdr.dev/slog"
+)
+
+// localForwardChannelData is copied from the ssh package.
+type localForwardChannelData struct {
+	DestAddr string
+	DestPort uint32
+
+	OriginAddr string
+	OriginPort uint32
+}
+
+// JetbrainsChannelWatcher is used to track JetBrains port forwarded (Gateway)
+// channels. If the port forward is something other than JetBrains, this struct
+// is a noop.
+type JetbrainsChannelWatcher struct {
+	gossh.NewChannel
+	jetbrainsCounter *atomic.Int64
+	logger           slog.Logger
+}
+
+func NewJetbrainsChannelWatcher(ctx ssh.Context, logger slog.Logger, newChannel gossh.NewChannel, counter *atomic.Int64) gossh.NewChannel {
+	d := localForwardChannelData{}
+	if err := gossh.Unmarshal(newChannel.ExtraData(), &d); err != nil {
+		// If the data fails to unmarshal, do nothing.
+		logger.Warn(ctx, "failed to unmarshal port forward data", slog.Error(err))
+		return newChannel
+	}
+
+	// If we do get a port, we should be able to get the matching PID and from
+	// there look up the invocation.
+	cmdline, err := getListeningPortProcessCmdline(d.DestPort)
+	if err != nil {
+		logger.Warn(ctx, "failed to inspect port",
+			slog.F("destination_port", d.DestPort),
+			slog.Error(err))
+		return newChannel
+	}
+
+	// If this is not JetBrains, then we do not need to do anything special.  We
+	// attempt to match on something that appears unique to JetBrains software.
+	if !strings.Contains(strings.ToLower(cmdline), strings.ToLower(MagicProcessCmdlineJetBrains)) {
+		return newChannel
+	}
+
+	logger.Debug(ctx, "discovered forwarded JetBrains process",
+		slog.F("destination_port", d.DestPort))
+
+	return &JetbrainsChannelWatcher{
+		NewChannel:       newChannel,
+		jetbrainsCounter: counter,
+		logger:           logger.With(slog.F("destination_port", d.DestPort)),
+	}
+}
+
+func (w *JetbrainsChannelWatcher) Accept() (gossh.Channel, <-chan *gossh.Request, error) {
+	c, r, err := w.NewChannel.Accept()
+	if err != nil {
+		return c, r, err
+	}
+	w.jetbrainsCounter.Add(1)
+	// nolint: gocritic // JetBrains is a proper noun and should be capitalized
+	w.logger.Debug(context.Background(), "JetBrains watcher accepted channel")
+
+	return &ChannelOnClose{
+		Channel: c,
+		done: func() {
+			w.jetbrainsCounter.Add(-1)
+			// nolint: gocritic // JetBrains is a proper noun and should be capitalized
+			w.logger.Debug(context.Background(), "JetBrains watcher channel closed")
+		},
+	}, r, err
+}
+
+type ChannelOnClose struct {
+	gossh.Channel
+	// once ensures close only decrements the counter once.
+	// Because close can be called multiple times.
+	once sync.Once
+	done func()
+}
+
+func (c *ChannelOnClose) Close() error {
+	c.once.Do(c.done)
+	return c.Channel.Close()
+}

agent/agentssh/metrics.go

@@ -0,0 +1,85 @@
+package agentssh
+
+import (
+	"strings"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+type sshServerMetrics struct {
+	failedConnectionsTotal prometheus.Counter
+	sftpConnectionsTotal   prometheus.Counter
+	sftpServerErrors       prometheus.Counter
+	x11HandlerErrors       *prometheus.CounterVec
+	sessionsTotal          *prometheus.CounterVec
+	sessionErrors          *prometheus.CounterVec
+}
+
+func newSSHServerMetrics(registerer prometheus.Registerer) *sshServerMetrics {
+	failedConnectionsTotal := prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace: "agent", Subsystem: "ssh_server", Name: "failed_connections_total",
+	})
+	registerer.MustRegister(failedConnectionsTotal)
+
+	sftpConnectionsTotal := prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace: "agent", Subsystem: "ssh_server", Name: "sftp_connections_total",
+	})
+	registerer.MustRegister(sftpConnectionsTotal)
+
+	sftpServerErrors := prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace: "agent", Subsystem: "ssh_server", Name: "sftp_server_errors_total",
+	})
+	registerer.MustRegister(sftpServerErrors)
+
+	x11HandlerErrors := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: "agent",
+			Subsystem: "x11_handler",
+			Name:      "errors_total",
+		},
+		[]string{"error_type"},
+	)
+	registerer.MustRegister(x11HandlerErrors)
+
+	sessionsTotal := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: "agent",
+			Subsystem: "sessions",
+			Name:      "total",
+		},
+		[]string{"magic_type", "pty"},
+	)
+	registerer.MustRegister(sessionsTotal)
+
+	sessionErrors := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: "agent",
+			Subsystem: "sessions",
+			Name:      "errors_total",
+		},
+		[]string{"magic_type", "pty", "error_type"},
+	)
+	registerer.MustRegister(sessionErrors)
+
+	return &sshServerMetrics{
+		failedConnectionsTotal: failedConnectionsTotal,
+		sftpConnectionsTotal:   sftpConnectionsTotal,
+		sftpServerErrors:       sftpServerErrors,
+		x11HandlerErrors:       x11HandlerErrors,
+		sessionsTotal:          sessionsTotal,
+		sessionErrors:          sessionErrors,
+	}
+}
+
+func magicTypeMetricLabel(magicType string) string {
+	switch magicType {
+	case MagicSessionTypeVSCode:
+	case MagicSessionTypeJetBrains:
+	case "":
+		magicType = "ssh"
+	default:
+		magicType = "unknown"
+	}
+	// Always be case insensitive
+	return strings.ToLower(magicType)
+}

agent/agentssh/portinspection_supported.go

@@ -0,0 +1,51 @@
+//go:build linux
+
+package agentssh
+
+import (
+	"errors"
+	"fmt"
+	"os"
+
+	"github.com/cakturk/go-netstat/netstat"
+	"golang.org/x/xerrors"
+)
+
+func getListeningPortProcessCmdline(port uint32) (string, error) {
+	acceptFn := func(s *netstat.SockTabEntry) bool {
+		return s.LocalAddr != nil && uint32(s.LocalAddr.Port) == port
+	}
+	tabs4, err4 := netstat.TCPSocks(acceptFn)
+	tabs6, err6 := netstat.TCP6Socks(acceptFn)
+
+	// In the common case, we want to check ipv4 listening addresses.  If this
+	// fails, we should return an error.  We also need to check ipv6.  The
+	// assumption is, if we have an err4, and 0 ipv6 addresses listed, then we are
+	// interested in the err4 (and vice versa).  So return both errors (at least 1
+	// is non-nil) if the other list is empty.
+	if (err4 != nil && len(tabs6) == 0) || (err6 != nil && len(tabs4) == 0) {
+		return "", xerrors.Errorf("inspect port %d: %w", port, errors.Join(err4, err6))
+	}
+
+	var proc *netstat.Process
+	if len(tabs4) > 0 {
+		proc = tabs4[0].Process
+	} else if len(tabs6) > 0 {
+		proc = tabs6[0].Process
+	}
+	if proc == nil {
+		// Either nothing is listening on this port or we were unable to read the
+		// process details (permission issues reading /proc/$pid/* potentially).
+		// Or, perhaps /proc/net/tcp{,6} is not listing the port for some reason.
+		return "", nil
+	}
+
+	// The process name provided by go-netstat does not include the full command
+	// line so grab that instead.
+	pid := proc.Pid
+	data, err := os.ReadFile(fmt.Sprintf("/proc/%d/cmdline", pid))
+	if err != nil {
+		return "", xerrors.Errorf("read /proc/%d/cmdline: %w", pid, err)
+	}
+	return string(data), nil
+}

agent/agentssh/portinspection_unsupported.go

@@ -0,0 +1,9 @@
+//go:build !linux
+
+package agentssh
+
+func getListeningPortProcessCmdline(uint32) (string, error) {
+	// We are not worrying about other platforms at the moment because Gateway
+	// only supports Linux anyway.
+	return "", nil
+}

agent/agentssh/signal_other.go

@@ -0,0 +1,45 @@
+//go:build !windows
+
+package agentssh
+
+import (
+	"os"
+
+	"github.com/gliderlabs/ssh"
+	"golang.org/x/sys/unix"
+)
+
+func osSignalFrom(sig ssh.Signal) os.Signal {
+	switch sig {
+	case ssh.SIGABRT:
+		return unix.SIGABRT
+	case ssh.SIGALRM:
+		return unix.SIGALRM
+	case ssh.SIGFPE:
+		return unix.SIGFPE
+	case ssh.SIGHUP:
+		return unix.SIGHUP
+	case ssh.SIGILL:
+		return unix.SIGILL
+	case ssh.SIGINT:
+		return unix.SIGINT
+	case ssh.SIGKILL:
+		return unix.SIGKILL
+	case ssh.SIGPIPE:
+		return unix.SIGPIPE
+	case ssh.SIGQUIT:
+		return unix.SIGQUIT
+	case ssh.SIGSEGV:
+		return unix.SIGSEGV
+	case ssh.SIGTERM:
+		return unix.SIGTERM
+	case ssh.SIGUSR1:
+		return unix.SIGUSR1
+	case ssh.SIGUSR2:
+		return unix.SIGUSR2
+
+	// Unhandled, use sane fallback.
+	default:
+		return unix.SIGKILL
+	}
+}

agent/agentssh/signal_windows.go

@@ -0,0 +1,15 @@
+package agentssh
+
+import (
+	"os"
+
+	"github.com/gliderlabs/ssh"
+)
+
+func osSignalFrom(sig ssh.Signal) os.Signal {
+	switch sig {
+	// Signals are not supported on Windows.
+	default:
+		return os.Kill
+	}
+}

agent/agentssh/x11.go

@@ -6,6 +6,8 @@ import (
 	"encoding/hex"
 	"errors"
 	"fmt"
+	"io"
+	"math"
 	"net"
 	"os"
 	"path/filepath"
@@ -21,58 +23,69 @@ import (
 	"cdr.dev/slog"
 )
 
+const (
+	// X11StartPort is the starting port for X11 forwarding, this is the
+	// port used for "DISPLAY=localhost:0".
+	X11StartPort = 6000
+	// X11DefaultDisplayOffset is the default offset for X11 forwarding.
+	X11DefaultDisplayOffset = 10
+)
+
 // x11Callback is called when the client requests X11 forwarding.
-// It adds an Xauthority entry to the Xauthority file.
-func (s *Server) x11Callback(ctx ssh.Context, x11 ssh.X11) bool {
-	hostname, err := os.Hostname()
-	if err != nil {
-		s.logger.Warn(ctx, "failed to get hostname", slog.Error(err))
-		return false
-	}
-
-	err = s.fs.MkdirAll(s.x11SocketDir, 0o700)
-	if err != nil {
-		s.logger.Warn(ctx, "failed to make the x11 socket dir", slog.F("dir", s.x11SocketDir), slog.Error(err))
-		return false
-	}
-
-	err = addXauthEntry(ctx, s.fs, hostname, strconv.Itoa(int(x11.ScreenNumber)), x11.AuthProtocol, x11.AuthCookie)
-	if err != nil {
-		s.logger.Warn(ctx, "failed to add Xauthority entry", slog.Error(err))
-		return false
-	}
+func (*Server) x11Callback(_ ssh.Context, _ ssh.X11) bool {
+	// Always allow.
 	return true
 }
 
 // x11Handler is called when a session has requested X11 forwarding.
 // It listens for X11 connections and forwards them to the client.
-func (s *Server) x11Handler(ctx ssh.Context, x11 ssh.X11) bool {
+func (s *Server) x11Handler(ctx ssh.Context, x11 ssh.X11) (displayNumber int, handled bool) {
 	serverConn, valid := ctx.Value(ssh.ContextKeyConn).(*gossh.ServerConn)
 	if !valid {
 		s.logger.Warn(ctx, "failed to get server connection")
-		return false
+		return -1, false
 	}
-	// We want to overwrite the socket so that subsequent connections will succeed.
-	socketPath := filepath.Join(s.x11SocketDir, fmt.Sprintf("X%d", x11.ScreenNumber))
-	err := os.Remove(socketPath)
-	if err != nil && !errors.Is(err, os.ErrNotExist) {
-		s.logger.Warn(ctx, "failed to remove existing X11 socket", slog.Error(err))
-		return false
-	}
-	listener, err := net.Listen("unix", socketPath)
+
+	hostname, err := os.Hostname()
 	if err != nil {
-		s.logger.Warn(ctx, "failed to listen for X11", slog.Error(err))
-		return false
+		s.logger.Warn(ctx, "failed to get hostname", slog.Error(err))
+		s.metrics.x11HandlerErrors.WithLabelValues("hostname").Add(1)
+		return -1, false
+	}
+
+	ln, display, err := createX11Listener(ctx, *s.config.X11DisplayOffset)
+	if err != nil {
+		s.logger.Warn(ctx, "failed to create X11 listener", slog.Error(err))
+		s.metrics.x11HandlerErrors.WithLabelValues("listen").Add(1)
+		return -1, false
+	}
+	s.trackListener(ln, true)
+	defer func() {
+		if !handled {
+			s.trackListener(ln, false)
+			_ = ln.Close()
+		}
+	}()
+
+	err = addXauthEntry(ctx, s.fs, hostname, strconv.Itoa(display), x11.AuthProtocol, x11.AuthCookie)
+	if err != nil {
+		s.logger.Warn(ctx, "failed to add Xauthority entry", slog.Error(err))
+		s.metrics.x11HandlerErrors.WithLabelValues("xauthority").Add(1)
+		return -1, false
 	}
-	s.trackListener(listener, true)
 
 	go func() {
-		defer listener.Close()
-		defer s.trackListener(listener, false)
-		handledFirstConnection := false
+		// Don't leave the listener open after the session is gone.
+		<-ctx.Done()
+		_ = ln.Close()
+	}()
+
+	go func() {
+		defer ln.Close()
+		defer s.trackListener(ln, false)
 
 		for {
-			conn, err := listener.Accept()
+			conn, err := ln.Accept()
 			if err != nil {
 				if errors.Is(err, net.ErrClosed) {
 					return
@@ -80,40 +93,66 @@ func (s *Server) x11Handler(ctx ssh.Context, x11 ssh.X11) bool {
 				s.logger.Warn(ctx, "failed to accept X11 connection", slog.Error(err))
 				return
 			}
-			if x11.SingleConnection && handledFirstConnection {
-				s.logger.Warn(ctx, "X11 connection rejected because single connection is enabled")
+			if x11.SingleConnection {
+				s.logger.Debug(ctx, "single connection requested, closing X11 listener")
+				_ = ln.Close()
+			}
+
+			tcpConn, ok := conn.(*net.TCPConn)
+			if !ok {
+				s.logger.Warn(ctx, fmt.Sprintf("failed to cast connection to TCPConn. got: %T", conn))
 				_ = conn.Close()
 				continue
 			}
-			handledFirstConnection = true
-
-			unixConn, ok := conn.(*net.UnixConn)
+			tcpAddr, ok := tcpConn.LocalAddr().(*net.TCPAddr)
 			if !ok {
-				s.logger.Warn(ctx, fmt.Sprintf("failed to cast connection to UnixConn. got: %T", conn))
-				return
-			}
-			unixAddr, ok := unixConn.LocalAddr().(*net.UnixAddr)
-			if !ok {
-				s.logger.Warn(ctx, fmt.Sprintf("failed to cast local address to UnixAddr. got: %T", unixConn.LocalAddr()))
-				return
+				s.logger.Warn(ctx, fmt.Sprintf("failed to cast local address to TCPAddr. got: %T", tcpConn.LocalAddr()))
+				_ = conn.Close()
+				continue
 			}
 
 			channel, reqs, err := serverConn.OpenChannel("x11", gossh.Marshal(struct {
 				OriginatorAddress string
 				OriginatorPort    uint32
 			}{
-				OriginatorAddress: unixAddr.Name,
-				OriginatorPort:    0,
+				OriginatorAddress: tcpAddr.IP.String(),
+				OriginatorPort:    uint32(tcpAddr.Port),
 			}))
 			if err != nil {
 				s.logger.Warn(ctx, "failed to open X11 channel", slog.Error(err))
-				return
+				_ = conn.Close()
+				continue
 			}
 			go gossh.DiscardRequests(reqs)
-			go Bicopy(ctx, conn, channel)
+
+			if !s.trackConn(ln, conn, true) {
+				s.logger.Warn(ctx, "failed to track X11 connection")
+				_ = conn.Close()
+				continue
+			}
+			go func() {
+				defer s.trackConn(ln, conn, false)
+				Bicopy(ctx, conn, channel)
+			}()
 		}
 	}()
-	return true
+
+	return display, true
+}
+
+// createX11Listener creates a listener for X11 forwarding, it will use
+// the next available port starting from X11StartPort and displayOffset.
+func createX11Listener(ctx context.Context, displayOffset int) (ln net.Listener, display int, err error) {
+	var lc net.ListenConfig
+	// Look for an open port to listen on.
+	for port := X11StartPort + displayOffset; port < math.MaxUint16; port++ {
+		ln, err = lc.Listen(ctx, "tcp", fmt.Sprintf("localhost:%d", port))
+		if err == nil {
+			display = port - X11StartPort
+			return ln, display, nil
+		}
+	}
+	return nil, -1, xerrors.Errorf("failed to find open port for X11 listener: %w", err)
 }
 
 // addXauthEntry adds an Xauthority entry to the Xauthority file.
@@ -138,7 +177,7 @@ func addXauthEntry(ctx context.Context, fs afero.Fs, host string, display string
 	}
 
 	// Open or create the Xauthority file
-	file, err := fs.OpenFile(xauthPath, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0o600)
+	file, err := fs.OpenFile(xauthPath, os.O_RDWR|os.O_CREATE, 0o600)
 	if err != nil {
 		return xerrors.Errorf("failed to open Xauthority file: %w", err)
 	}
@@ -150,7 +189,105 @@ func addXauthEntry(ctx context.Context, fs afero.Fs, host string, display string
 		return xerrors.Errorf("failed to decode auth cookie: %w", err)
 	}
 
-	// Write Xauthority entry
+	// Read the Xauthority file and look for an existing entry for the host,
+	// display, and auth protocol. If an entry is found, overwrite the auth
+	// cookie (if it fits). Otherwise, mark the entry for deletion.
+	type deleteEntry struct {
+		start, end int
+	}
+	var deleteEntries []deleteEntry
+	pos := 0
+	updated := false
+	for {
+		entry, err := readXauthEntry(file)
+		if err != nil {
+			if errors.Is(err, io.EOF) {
+				break
+			}
+			return xerrors.Errorf("failed to read Xauthority entry: %w", err)
+		}
+
+		nextPos := pos + entry.Len()
+		cookieStartPos := nextPos - len(entry.authCookie)
+
+		if entry.family == 0x0100 && entry.address == host && entry.display == display && entry.authProtocol == authProtocol {
+			if !updated && len(entry.authCookie) == len(authCookieBytes) {
+				// Overwrite the auth cookie
+				_, err := file.WriteAt(authCookieBytes, int64(cookieStartPos))
+				if err != nil {
+					return xerrors.Errorf("failed to write auth cookie: %w", err)
+				}
+				updated = true
+			} else {
+				// Mark entry for deletion.
+				if len(deleteEntries) > 0 && deleteEntries[len(deleteEntries)-1].end == pos {
+					deleteEntries[len(deleteEntries)-1].end = nextPos
+				} else {
+					deleteEntries = append(deleteEntries, deleteEntry{
+						start: pos,
+						end:   nextPos,
+					})
+				}
+			}
+		}
+
+		pos = nextPos
+	}
+
+	// In case the magic cookie changed, or we've previously bloated the
+	// Xauthority file, we may have to delete entries.
+	if len(deleteEntries) > 0 {
+		// Read the entire file into memory. This is not ideal, but it's the
+		// simplest way to delete entries from the middle of the file. The
+		// Xauthority file is small, so this should be fine.
+		_, err = file.Seek(0, io.SeekStart)
+		if err != nil {
+			return xerrors.Errorf("failed to seek Xauthority file: %w", err)
+		}
+		data, err := io.ReadAll(file)
+		if err != nil {
+			return xerrors.Errorf("failed to read Xauthority file: %w", err)
+		}
+
+		// Delete the entries in reverse order.
+		for i := len(deleteEntries) - 1; i >= 0; i-- {
+			entry := deleteEntries[i]
+			// Safety check: ensure the entry is still there.
+			if entry.start > len(data) || entry.end > len(data) {
+				continue
+			}
+			data = append(data[:entry.start], data[entry.end:]...)
+		}
+
+		// Write the data back to the file.
+		_, err = file.Seek(0, io.SeekStart)
+		if err != nil {
+			return xerrors.Errorf("failed to seek Xauthority file: %w", err)
+		}
+		_, err = file.Write(data)
+		if err != nil {
+			return xerrors.Errorf("failed to write Xauthority file: %w", err)
+		}
+
+		// Truncate the file.
+		err = file.Truncate(int64(len(data)))
+		if err != nil {
+			return xerrors.Errorf("failed to truncate Xauthority file: %w", err)
+		}
+	}
+
+	// Return if we've already updated the entry.
+	if updated {
+		return nil
+	}
+
+	// Ensure we're at the end (append).
+	_, err = file.Seek(0, io.SeekEnd)
+	if err != nil {
+		return xerrors.Errorf("failed to seek Xauthority file: %w", err)
+	}
+
+	// Append Xauthority entry.
 	family := uint16(0x0100) // FamilyLocal
 	err = binary.Write(file, binary.BigEndian, family)
 	if err != nil {
@@ -195,3 +332,96 @@ func addXauthEntry(ctx context.Context, fs afero.Fs, host string, display string
 
 	return nil
 }
+
+// xauthEntry is an representation of an Xauthority entry.
+//
+// The Xauthority file format is as follows:
+//
+// - 16-bit family
+// - 16-bit address length
+// - address
+// - 16-bit display length
+// - display
+// - 16-bit auth protocol length
+// - auth protocol
+// - 16-bit auth cookie length
+// - auth cookie
+type xauthEntry struct {
+	family       uint16
+	address      string
+	display      string
+	authProtocol string
+	authCookie   []byte
+}
+
+func (e xauthEntry) Len() int {
+	// 5 * uint16 = 10 bytes for the family/length fields.
+	return 2*5 + len(e.address) + len(e.display) + len(e.authProtocol) + len(e.authCookie)
+}
+
+func readXauthEntry(r io.Reader) (xauthEntry, error) {
+	var entry xauthEntry
+
+	// Read family
+	err := binary.Read(r, binary.BigEndian, &entry.family)
+	if err != nil {
+		return xauthEntry{}, xerrors.Errorf("failed to read family: %w", err)
+	}
+
+	// Read address
+	var addressLength uint16
+	err = binary.Read(r, binary.BigEndian, &addressLength)
+	if err != nil {
+		return xauthEntry{}, xerrors.Errorf("failed to read address length: %w", err)
+	}
+
+	addressBytes := make([]byte, addressLength)
+	_, err = r.Read(addressBytes)
+	if err != nil {
+		return xauthEntry{}, xerrors.Errorf("failed to read address: %w", err)
+	}
+	entry.address = string(addressBytes)
+
+	// Read display
+	var displayLength uint16
+	err = binary.Read(r, binary.BigEndian, &displayLength)
+	if err != nil {
+		return xauthEntry{}, xerrors.Errorf("failed to read display length: %w", err)
+	}
+
+	displayBytes := make([]byte, displayLength)
+	_, err = r.Read(displayBytes)
+	if err != nil {
+		return xauthEntry{}, xerrors.Errorf("failed to read display: %w", err)
+	}
+	entry.display = string(displayBytes)
+
+	// Read auth protocol
+	var authProtocolLength uint16
+	err = binary.Read(r, binary.BigEndian, &authProtocolLength)
+	if err != nil {
+		return xauthEntry{}, xerrors.Errorf("failed to read auth protocol length: %w", err)
+	}
+
+	authProtocolBytes := make([]byte, authProtocolLength)
+	_, err = r.Read(authProtocolBytes)
+	if err != nil {
+		return xauthEntry{}, xerrors.Errorf("failed to read auth protocol: %w", err)
+	}
+	entry.authProtocol = string(authProtocolBytes)
+
+	// Read auth cookie
+	var authCookieLength uint16
+	err = binary.Read(r, binary.BigEndian, &authCookieLength)
+	if err != nil {
+		return xauthEntry{}, xerrors.Errorf("failed to read auth cookie length: %w", err)
+	}
+
+	entry.authCookie = make([]byte, authCookieLength)
+	_, err = r.Read(entry.authCookie)
+	if err != nil {
+		return xauthEntry{}, xerrors.Errorf("failed to read auth cookie: %w", err)
+	}
+
+	return entry, nil
+}

agent/agentssh/x11_internal_test.go

@@ -0,0 +1,254 @@
+package agentssh
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/spf13/afero"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func Test_addXauthEntry(t *testing.T) {
+	t.Parallel()
+
+	type testEntry struct {
+		address      string
+		display      string
+		authProtocol string
+		authCookie   string
+	}
+	tests := []struct {
+		name         string
+		authFile     []byte
+		wantAuthFile []byte
+		entries      []testEntry
+	}{
+		{
+			name:     "add entry",
+			authFile: nil,
+			wantAuthFile: []byte{
+				// w/unix:0  MIT-MAGIC-COOKIE-1  00
+				//
+				// 00000000: 0100 0001 7700 0130 0012 4d49 542d 4d41  ....w..0..MIT-MA
+				// 00000010: 4749 432d 434f 4f4b 4945 2d31 0001 00    GIC-COOKIE-1...
+				0x01, 0x00, 0x00, 0x01, 0x77, 0x00, 0x01, 0x30,
+				0x00, 0x12, 0x4d, 0x49, 0x54, 0x2d, 0x4d, 0x41,
+				0x47, 0x49, 0x43, 0x2d, 0x43, 0x4f, 0x4f, 0x4b,
+				0x49, 0x45, 0x2d, 0x31, 0x00, 0x01, 0x00,
+			},
+			entries: []testEntry{
+				{
+					address:      "w",
+					display:      "0",
+					authProtocol: "MIT-MAGIC-COOKIE-1",
+					authCookie:   "00",
+				},
+			},
+		},
+		{
+			name:     "add two entries",
+			authFile: []byte{},
+			wantAuthFile: []byte{
+				// w/unix:0  MIT-MAGIC-COOKIE-1  00
+				// w/unix:1  MIT-MAGIC-COOKIE-1  11
+				//
+				// 00000000: 0100 0001 7700 0130 0012 4d49 542d 4d41  ....w..0..MIT-MA
+				// 00000010: 4749 432d 434f 4f4b 4945 2d31 0001 0001  GIC-COOKIE-1....
+				// 00000020: 0000 0177 0001 3100 124d 4954 2d4d 4147  ...w..1..MIT-MAG
+				// 00000030: 4943 2d43 4f4f 4b49 452d 3100 0111       IC-COOKIE-1...
+				0x01, 0x00, 0x00, 0x01, 0x77, 0x00, 0x01, 0x30,
+				0x00, 0x12, 0x4d, 0x49, 0x54, 0x2d, 0x4d, 0x41,
+				0x47, 0x49, 0x43, 0x2d, 0x43, 0x4f, 0x4f, 0x4b,
+				0x49, 0x45, 0x2d, 0x31, 0x00, 0x01, 0x00,
+				0x01, 0x00, 0x00, 0x01, 0x77, 0x00, 0x01, 0x31,
+				0x00, 0x12, 0x4d, 0x49, 0x54, 0x2d, 0x4d, 0x41,
+				0x47, 0x49, 0x43, 0x2d, 0x43, 0x4f, 0x4f, 0x4b,
+				0x49, 0x45, 0x2d, 0x31, 0x00, 0x01, 0x11,
+			},
+			entries: []testEntry{
+				{
+					address:      "w",
+					display:      "0",
+					authProtocol: "MIT-MAGIC-COOKIE-1",
+					authCookie:   "00",
+				},
+				{
+					address:      "w",
+					display:      "1",
+					authProtocol: "MIT-MAGIC-COOKIE-1",
+					authCookie:   "11",
+				},
+			},
+		},
+		{
+			name: "update entry with new auth cookie length",
+			authFile: []byte{
+				// w/unix:0  MIT-MAGIC-COOKIE-1  00
+				// w/unix:1  MIT-MAGIC-COOKIE-1  11
+				//
+				// 00000000: 0100 0001 7700 0130 0012 4d49 542d 4d41  ....w..0..MIT-MA
+				// 00000010: 4749 432d 434f 4f4b 4945 2d31 0001 0001  GIC-COOKIE-1....
+				// 00000020: 0000 0177 0001 3100 124d 4954 2d4d 4147  ...w..1..MIT-MAG
+				// 00000030: 4943 2d43 4f4f 4b49 452d 3100 0111       IC-COOKIE-1...
+				0x01, 0x00, 0x00, 0x01, 0x77, 0x00, 0x01, 0x30,
+				0x00, 0x12, 0x4d, 0x49, 0x54, 0x2d, 0x4d, 0x41,
+				0x47, 0x49, 0x43, 0x2d, 0x43, 0x4f, 0x4f, 0x4b,
+				0x49, 0x45, 0x2d, 0x31, 0x00, 0x01, 0x00,
+				0x01, 0x00, 0x00, 0x01, 0x77, 0x00, 0x01, 0x31,
+				0x00, 0x12, 0x4d, 0x49, 0x54, 0x2d, 0x4d, 0x41,
+				0x47, 0x49, 0x43, 0x2d, 0x43, 0x4f, 0x4f, 0x4b,
+				0x49, 0x45, 0x2d, 0x31, 0x00, 0x01, 0x11,
+			},
+			wantAuthFile: []byte{
+				// The order changed, due to new length of auth cookie resulting
+				// in remove + append, we verify that the implementation is
+				// behaving as expected (changing the order is not a requirement,
+				// simply an implementation detail).
+				0x01, 0x00, 0x00, 0x01, 0x77, 0x00, 0x01, 0x31,
+				0x00, 0x12, 0x4d, 0x49, 0x54, 0x2d, 0x4d, 0x41,
+				0x47, 0x49, 0x43, 0x2d, 0x43, 0x4f, 0x4f, 0x4b,
+				0x49, 0x45, 0x2d, 0x31, 0x00, 0x01, 0x11,
+				0x01, 0x00, 0x00, 0x01, 0x77, 0x00, 0x01, 0x30,
+				0x00, 0x12, 0x4d, 0x49, 0x54, 0x2d, 0x4d, 0x41,
+				0x47, 0x49, 0x43, 0x2d, 0x43, 0x4f, 0x4f, 0x4b,
+				0x49, 0x45, 0x2d, 0x31, 0x00, 0x02, 0xff, 0xff,
+			},
+			entries: []testEntry{
+				{
+					address:      "w",
+					display:      "0",
+					authProtocol: "MIT-MAGIC-COOKIE-1",
+					authCookie:   "ffff",
+				},
+			},
+		},
+		{
+			name: "update entry",
+			authFile: []byte{
+				// 00000000: 0100 0001 7700 0130 0012 4d49 542d 4d41  ....w..0..MIT-MA
+				// 00000010: 4749 432d 434f 4f4b 4945 2d31 0001 0001  GIC-COOKIE-1....
+				// 00000020: 0000 0177 0001 3100 124d 4954 2d4d 4147  ...w..1..MIT-MAG
+				// 00000030: 4943 2d43 4f4f 4b49 452d 3100 0111       IC-COOKIE-1...
+				0x01, 0x00, 0x00, 0x01, 0x77, 0x00, 0x01, 0x30,
+				0x00, 0x12, 0x4d, 0x49, 0x54, 0x2d, 0x4d, 0x41,
+				0x47, 0x49, 0x43, 0x2d, 0x43, 0x4f, 0x4f, 0x4b,
+				0x49, 0x45, 0x2d, 0x31, 0x00, 0x01, 0x00,
+				0x01, 0x00, 0x00, 0x01, 0x77, 0x00, 0x01, 0x31,
+				0x00, 0x12, 0x4d, 0x49, 0x54, 0x2d, 0x4d, 0x41,
+				0x47, 0x49, 0x43, 0x2d, 0x43, 0x4f, 0x4f, 0x4b,
+				0x49, 0x45, 0x2d, 0x31, 0x00, 0x01, 0x11,
+			},
+			wantAuthFile: []byte{
+				// 00000000: 0100 0001 7700 0130 0012 4d49 542d 4d41  ....w..0..MIT-MA
+				// 00000010: 4749 432d 434f 4f4b 4945 2d31 0001 0001  GIC-COOKIE-1....
+				// 00000020: 0000 0177 0001 3100 124d 4954 2d4d 4147  ...w..1..MIT-MAG
+				// 00000030: 4943 2d43 4f4f 4b49 452d 3100 0111       IC-COOKIE-1...
+				0x01, 0x00, 0x00, 0x01, 0x77, 0x00, 0x01, 0x30,
+				0x00, 0x12, 0x4d, 0x49, 0x54, 0x2d, 0x4d, 0x41,
+				0x47, 0x49, 0x43, 0x2d, 0x43, 0x4f, 0x4f, 0x4b,
+				0x49, 0x45, 0x2d, 0x31, 0x00, 0x01, 0xff,
+				0x01, 0x00, 0x00, 0x01, 0x77, 0x00, 0x01, 0x31,
+				0x00, 0x12, 0x4d, 0x49, 0x54, 0x2d, 0x4d, 0x41,
+				0x47, 0x49, 0x43, 0x2d, 0x43, 0x4f, 0x4f, 0x4b,
+				0x49, 0x45, 0x2d, 0x31, 0x00, 0x01, 0x11,
+			},
+			entries: []testEntry{
+				{
+					address:      "w",
+					display:      "0",
+					authProtocol: "MIT-MAGIC-COOKIE-1",
+					authCookie:   "ff",
+				},
+			},
+		},
+		{
+			name: "clean up old entries",
+			authFile: []byte{
+				// w/unix:0  MIT-MAGIC-COOKIE-1  80507df050756cdefa504b65adb3bcfb
+				// w/unix:0  MIT-MAGIC-COOKIE-1  267b37f6cbc11b97beb826bb1aab8570
+				// w/unix:0  MIT-MAGIC-COOKIE-1  516e22e2b11d1bd0115dff09c028ca5c
+				//
+				// 00000000: 0100 0001 7700 0130 0012 4d49 542d 4d41  ....w..0..MIT-MA
+				// 00000010: 4749 432d 434f 4f4b 4945 2d31 0010 8050  GIC-COOKIE-1...P
+				// 00000020: 7df0 5075 6cde fa50 4b65 adb3 bcfb 0100  }.Pul..PKe......
+				// 00000030: 0001 7700 0130 0012 4d49 542d 4d41 4749  ..w..0..MIT-MAGI
+				// 00000040: 432d 434f 4f4b 4945 2d31 0010 267b 37f6  C-COOKIE-1..&{7.
+				// 00000050: cbc1 1b97 beb8 26bb 1aab 8570 0100 0001  ......&....p....
+				// 00000060: 7700 0130 0012 4d49 542d 4d41 4749 432d  w..0..MIT-MAGIC-
+				// 00000070: 434f 4f4b 4945 2d31 0010 516e 22e2 b11d  COOKIE-1..Qn"...
+				// 00000080: 1bd0 115d ff09 c028 ca5c                 ...]...(.\
+				0x01, 0x00, 0x00, 0x01, 0x77, 0x00, 0x01, 0x30,
+				0x00, 0x12, 0x4d, 0x49, 0x54, 0x2d, 0x4d, 0x41,
+				0x47, 0x49, 0x43, 0x2d, 0x43, 0x4f, 0x4f, 0x4b,
+				0x49, 0x45, 0x2d, 0x31, 0x00, 0x10, 0x80, 0x50,
+				0x7d, 0xf0, 0x50, 0x75, 0x6c, 0xde, 0xfa, 0x50,
+				0x4b, 0x65, 0xad, 0xb3, 0xbc, 0xfb, 0x01, 0x00,
+				0x00, 0x01, 0x77, 0x00, 0x01, 0x30, 0x00, 0x12,
+				0x4d, 0x49, 0x54, 0x2d, 0x4d, 0x41, 0x47, 0x49,
+				0x43, 0x2d, 0x43, 0x4f, 0x4f, 0x4b, 0x49, 0x45,
+				0x2d, 0x31, 0x00, 0x10, 0x26, 0x7b, 0x37, 0xf6,
+				0xcb, 0xc1, 0x1b, 0x97, 0xbe, 0xb8, 0x26, 0xbb,
+				0x1a, 0xab, 0x85, 0x70, 0x01, 0x00, 0x00, 0x01,
+				0x77, 0x00, 0x01, 0x30, 0x00, 0x12, 0x4d, 0x49,
+				0x54, 0x2d, 0x4d, 0x41, 0x47, 0x49, 0x43, 0x2d,
+				0x43, 0x4f, 0x4f, 0x4b, 0x49, 0x45, 0x2d, 0x31,
+				0x00, 0x10, 0x51, 0x6e, 0x22, 0xe2, 0xb1, 0x1d,
+				0x1b, 0xd0, 0x11, 0x5d, 0xff, 0x09, 0xc0, 0x28,
+				0xca, 0x5c,
+			},
+			wantAuthFile: []byte{
+				// w/unix:0  MIT-MAGIC-COOKIE-1  516e5bc892b7162b844abd1fc1a7c16e
+				//
+				// 00000000: 0100 0001 7700 0130 0012 4d49 542d 4d41  ....w..0..MIT-MA
+				// 00000010: 4749 432d 434f 4f4b 4945 2d31 0010 516e  GIC-COOKIE-1..Qn
+				// 00000020: 5bc8 92b7 162b 844a bd1f c1a7 c16e       [....+.J.....n
+				0x01, 0x00, 0x00, 0x01, 0x77, 0x00, 0x01, 0x30,
+				0x00, 0x12, 0x4d, 0x49, 0x54, 0x2d, 0x4d, 0x41,
+				0x47, 0x49, 0x43, 0x2d, 0x43, 0x4f, 0x4f, 0x4b,
+				0x49, 0x45, 0x2d, 0x31, 0x00, 0x10, 0x51, 0x6e,
+				0x5b, 0xc8, 0x92, 0xb7, 0x16, 0x2b, 0x84, 0x4a,
+				0xbd, 0x1f, 0xc1, 0xa7, 0xc1, 0x6e,
+			},
+			entries: []testEntry{
+				{
+					address:      "w",
+					display:      "0",
+					authProtocol: "MIT-MAGIC-COOKIE-1",
+					authCookie:   "516e5bc892b7162b844abd1fc1a7c16e",
+				},
+			},
+		},
+	}
+
+	homedir, err := os.UserHomeDir()
+	require.NoError(t, err)
+
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			fs := afero.NewMemMapFs()
+			if tt.authFile != nil {
+				err := afero.WriteFile(fs, filepath.Join(homedir, ".Xauthority"), tt.authFile, 0o600)
+				require.NoError(t, err)
+			}
+
+			for _, entry := range tt.entries {
+				err := addXauthEntry(context.Background(), fs, entry.address, entry.display, entry.authProtocol, entry.authCookie)
+				require.NoError(t, err)
+			}
+
+			gotAuthFile, err := afero.ReadFile(fs, filepath.Join(homedir, ".Xauthority"))
+			require.NoError(t, err)
+
+			if diff := cmp.Diff(tt.wantAuthFile, gotAuthFile); diff != "" {
+				assert.Failf(t, "addXauthEntry() mismatch", "(-want +got):\n%s", diff)
+			}
+		})
+	}
+}

agent/agentssh/x11_test.go

@@ -1,26 +1,28 @@
 package agentssh_test
 
 import (
+	"bufio"
+	"bytes"
 	"context"
 	"encoding/hex"
+	"fmt"
 	"net"
 	"os"
 	"path/filepath"
 	"runtime"
+	"strconv"
+	"strings"
 	"testing"
 
 	"github.com/gliderlabs/ssh"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/spf13/afero"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-	"go.uber.org/atomic"
 	gossh "golang.org/x/crypto/ssh"
 
-	"cdr.dev/slog"
-	"cdr.dev/slog/sloggers/slogtest"
-	"github.com/coder/coder/agent/agentssh"
-	"github.com/coder/coder/codersdk/agentsdk"
-	"github.com/coder/coder/testutil"
+	"github.com/coder/coder/v2/agent/agentssh"
+	"github.com/coder/coder/v2/testutil"
 )
 
 func TestServer_X11(t *testing.T) {
@@ -30,17 +32,12 @@ func TestServer_X11(t *testing.T) {
 	}
 
 	ctx := context.Background()
-	logger := slogtest.Make(t, nil).Leveled(slog.LevelDebug)
+	logger := testutil.Logger(t)
 	fs := afero.NewOsFs()
-	dir := t.TempDir()
-	s, err := agentssh.NewServer(ctx, logger, fs, 0, dir)
+	s, err := agentssh.NewServer(ctx, logger, prometheus.NewRegistry(), fs, &agentssh.Config{})
 	require.NoError(t, err)
 	defer s.Close()
 
-	// The assumption is that these are set before serving SSH connections.
-	s.AgentToken = func() string { return "" }
-	s.Manifest = atomic.NewPointer(&agentsdk.Manifest{})
-
 	ln, err := net.Listen("tcp", "127.0.0.1:0")
 	require.NoError(t, err)
 
@@ -56,21 +53,45 @@ func TestServer_X11(t *testing.T) {
 	sess, err := c.NewSession()
 	require.NoError(t, err)
 
+	wantScreenNumber := 1
 	reply, err := sess.SendRequest("x11-req", true, gossh.Marshal(ssh.X11{
 		AuthProtocol: "MIT-MAGIC-COOKIE-1",
 		AuthCookie:   hex.EncodeToString([]byte("cookie")),
-		ScreenNumber: 0,
+		ScreenNumber: uint32(wantScreenNumber),
 	}))
 	require.NoError(t, err)
 	assert.True(t, reply)
 
-	err = sess.Shell()
+	// Want: ~DISPLAY=localhost:10.1
+	out, err := sess.Output("echo DISPLAY=$DISPLAY")
 	require.NoError(t, err)
 
+	sc := bufio.NewScanner(bytes.NewReader(out))
+	displayNumber := -1
+	for sc.Scan() {
+		line := strings.TrimSpace(sc.Text())
+		t.Log(line)
+		if strings.HasPrefix(line, "DISPLAY=") {
+			parts := strings.SplitN(line, "=", 2)
+			display := parts[1]
+			parts = strings.SplitN(display, ":", 2)
+			parts = strings.SplitN(parts[1], ".", 2)
+			displayNumber, err = strconv.Atoi(parts[0])
+			require.NoError(t, err)
+			assert.GreaterOrEqual(t, displayNumber, 10, "display number should be >= 10")
+			gotScreenNumber, err := strconv.Atoi(parts[1])
+			require.NoError(t, err)
+			assert.Equal(t, wantScreenNumber, gotScreenNumber, "screen number should match")
+			break
+		}
+	}
+	require.NoError(t, sc.Err())
+	require.NotEqual(t, -1, displayNumber)
+
 	x11Chans := c.HandleChannelOpen("x11")
 	payload := "hello world"
 	require.Eventually(t, func() bool {
-		conn, err := net.Dial("unix", filepath.Join(dir, "X0"))
+		conn, err := net.Dial("tcp", fmt.Sprintf("localhost:%d", agentssh.X11StartPort+displayNumber))
 		if err == nil {
 			_, err = conn.Write([]byte(payload))
 			assert.NoError(t, err)

agent/agenttest/agent.go

@@ -0,0 +1,56 @@
+package agenttest
+
+import (
+	"context"
+	"net/url"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/coder/coder/v2/agent"
+	"github.com/coder/coder/v2/codersdk/agentsdk"
+	"github.com/coder/coder/v2/testutil"
+)
+
+// New starts a new agent for use in tests.
+// The agent will use the provided coder URL and session token.
+// The options passed to agent.New() can be modified by passing an optional
+// variadic func(*agent.Options).
+// Returns the agent. Closing the agent is handled by the test cleanup.
+// It is the responsibility of the caller to call coderdtest.AwaitWorkspaceAgents
+// to ensure agent is connected.
+func New(t testing.TB, coderURL *url.URL, agentToken string, opts ...func(*agent.Options)) agent.Agent {
+	t.Helper()
+
+	var o agent.Options
+	log := testutil.Logger(t).Named("agent")
+	o.Logger = log
+
+	for _, opt := range opts {
+		opt(&o)
+	}
+
+	if o.Client == nil {
+		agentClient := agentsdk.New(coderURL)
+		agentClient.SetSessionToken(agentToken)
+		agentClient.SDK.SetLogger(log)
+		o.Client = agentClient
+	}
+
+	if o.ExchangeToken == nil {
+		o.ExchangeToken = func(_ context.Context) (string, error) {
+			return agentToken, nil
+		}
+	}
+
+	if o.LogDir == "" {
+		o.LogDir = t.TempDir()
+	}
+
+	agt := agent.New(o)
+	t.Cleanup(func() {
+		assert.NoError(t, agt.Close(), "failed to close agent during cleanup")
+	})
+
+	return agt
+}

agent/agenttest/client.go

@@ -0,0 +1,327 @@
+package agenttest
+
+import (
+	"context"
+	"io"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/exp/maps"
+	"golang.org/x/exp/slices"
+	"golang.org/x/xerrors"
+	"google.golang.org/protobuf/types/known/durationpb"
+	"storj.io/drpc/drpcmux"
+	"storj.io/drpc/drpcserver"
+	"tailscale.com/tailcfg"
+
+	"cdr.dev/slog"
+	agentproto "github.com/coder/coder/v2/agent/proto"
+	"github.com/coder/coder/v2/codersdk"
+	"github.com/coder/coder/v2/codersdk/agentsdk"
+	drpcsdk "github.com/coder/coder/v2/codersdk/drpc"
+	"github.com/coder/coder/v2/tailnet"
+	"github.com/coder/coder/v2/tailnet/proto"
+	"github.com/coder/coder/v2/testutil"
+)
+
+const statsInterval = 500 * time.Millisecond
+
+func NewClient(t testing.TB,
+	logger slog.Logger,
+	agentID uuid.UUID,
+	manifest agentsdk.Manifest,
+	statsChan chan *agentproto.Stats,
+	coordinator tailnet.Coordinator,
+) *Client {
+	if manifest.AgentID == uuid.Nil {
+		manifest.AgentID = agentID
+	}
+	coordPtr := atomic.Pointer[tailnet.Coordinator]{}
+	coordPtr.Store(&coordinator)
+	mux := drpcmux.New()
+	derpMapUpdates := make(chan *tailcfg.DERPMap)
+	drpcService := &tailnet.DRPCService{
+		CoordPtr:               &coordPtr,
+		Logger:                 logger.Named("tailnetsvc"),
+		DerpMapUpdateFrequency: time.Microsecond,
+		DerpMapFn:              func() *tailcfg.DERPMap { return <-derpMapUpdates },
+	}
+	err := proto.DRPCRegisterTailnet(mux, drpcService)
+	require.NoError(t, err)
+	mp, err := agentsdk.ProtoFromManifest(manifest)
+	require.NoError(t, err)
+	fakeAAPI := NewFakeAgentAPI(t, logger, mp, statsChan)
+	err = agentproto.DRPCRegisterAgent(mux, fakeAAPI)
+	require.NoError(t, err)
+	server := drpcserver.NewWithOptions(mux, drpcserver.Options{
+		Log: func(err error) {
+			if xerrors.Is(err, io.EOF) {
+				return
+			}
+			logger.Debug(context.Background(), "drpc server error", slog.Error(err))
+		},
+	})
+	return &Client{
+		t:              t,
+		logger:         logger.Named("client"),
+		agentID:        agentID,
+		server:         server,
+		fakeAgentAPI:   fakeAAPI,
+		derpMapUpdates: derpMapUpdates,
+	}
+}
+
+type Client struct {
+	t                  testing.TB
+	logger             slog.Logger
+	agentID            uuid.UUID
+	server             *drpcserver.Server
+	fakeAgentAPI       *FakeAgentAPI
+	LastWorkspaceAgent func()
+
+	mu             sync.Mutex // Protects following.
+	logs           []agentsdk.Log
+	derpMapUpdates chan *tailcfg.DERPMap
+	derpMapOnce    sync.Once
+}
+
+func (*Client) RewriteDERPMap(*tailcfg.DERPMap) {}
+
+func (c *Client) Close() {
+	c.derpMapOnce.Do(func() { close(c.derpMapUpdates) })
+}
+
+func (c *Client) ConnectRPC23(ctx context.Context) (
+	agentproto.DRPCAgentClient23, proto.DRPCTailnetClient23, error,
+) {
+	conn, lis := drpcsdk.MemTransportPipe()
+	c.LastWorkspaceAgent = func() {
+		_ = conn.Close()
+		_ = lis.Close()
+	}
+	c.t.Cleanup(c.LastWorkspaceAgent)
+	serveCtx, cancel := context.WithCancel(ctx)
+	c.t.Cleanup(cancel)
+	streamID := tailnet.StreamID{
+		Name: "agenttest",
+		ID:   c.agentID,
+		Auth: tailnet.AgentCoordinateeAuth{ID: c.agentID},
+	}
+	serveCtx = tailnet.WithStreamID(serveCtx, streamID)
+	go func() {
+		_ = c.server.Serve(serveCtx, lis)
+	}()
+	return agentproto.NewDRPCAgentClient(conn), proto.NewDRPCTailnetClient(conn), nil
+}
+
+func (c *Client) GetLifecycleStates() []codersdk.WorkspaceAgentLifecycle {
+	return c.fakeAgentAPI.GetLifecycleStates()
+}
+
+func (c *Client) GetStartup() <-chan *agentproto.Startup {
+	return c.fakeAgentAPI.startupCh
+}
+
+func (c *Client) GetMetadata() map[string]agentsdk.Metadata {
+	return c.fakeAgentAPI.GetMetadata()
+}
+
+func (c *Client) GetStartupLogs() []agentsdk.Log {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return c.logs
+}
+
+func (c *Client) SetAnnouncementBannersFunc(f func() ([]codersdk.BannerConfig, error)) {
+	c.fakeAgentAPI.SetAnnouncementBannersFunc(f)
+}
+
+func (c *Client) PushDERPMapUpdate(update *tailcfg.DERPMap) error {
+	timer := time.NewTimer(testutil.WaitShort)
+	defer timer.Stop()
+	select {
+	case c.derpMapUpdates <- update:
+	case <-timer.C:
+		return xerrors.New("timeout waiting to push derp map update")
+	}
+
+	return nil
+}
+
+func (c *Client) SetLogsChannel(ch chan<- *agentproto.BatchCreateLogsRequest) {
+	c.fakeAgentAPI.SetLogsChannel(ch)
+}
+
+type FakeAgentAPI struct {
+	sync.Mutex
+	t      testing.TB
+	logger slog.Logger
+
+	manifest        *agentproto.Manifest
+	startupCh       chan *agentproto.Startup
+	statsCh         chan *agentproto.Stats
+	appHealthCh     chan *agentproto.BatchUpdateAppHealthRequest
+	logsCh          chan<- *agentproto.BatchCreateLogsRequest
+	lifecycleStates []codersdk.WorkspaceAgentLifecycle
+	metadata        map[string]agentsdk.Metadata
+	timings         []*agentproto.Timing
+
+	getAnnouncementBannersFunc func() ([]codersdk.BannerConfig, error)
+}
+
+func (f *FakeAgentAPI) GetManifest(context.Context, *agentproto.GetManifestRequest) (*agentproto.Manifest, error) {
+	return f.manifest, nil
+}
+
+func (*FakeAgentAPI) GetServiceBanner(context.Context, *agentproto.GetServiceBannerRequest) (*agentproto.ServiceBanner, error) {
+	return &agentproto.ServiceBanner{}, nil
+}
+
+func (f *FakeAgentAPI) GetTimings() []*agentproto.Timing {
+	f.Lock()
+	defer f.Unlock()
+	return slices.Clone(f.timings)
+}
+
+func (f *FakeAgentAPI) SetAnnouncementBannersFunc(fn func() ([]codersdk.BannerConfig, error)) {
+	f.Lock()
+	defer f.Unlock()
+	f.getAnnouncementBannersFunc = fn
+	f.logger.Info(context.Background(), "updated notification banners")
+}
+
+func (f *FakeAgentAPI) GetAnnouncementBanners(context.Context, *agentproto.GetAnnouncementBannersRequest) (*agentproto.GetAnnouncementBannersResponse, error) {
+	f.Lock()
+	defer f.Unlock()
+	if f.getAnnouncementBannersFunc == nil {
+		return &agentproto.GetAnnouncementBannersResponse{AnnouncementBanners: []*agentproto.BannerConfig{}}, nil
+	}
+	banners, err := f.getAnnouncementBannersFunc()
+	if err != nil {
+		return nil, err
+	}
+	bannersProto := make([]*agentproto.BannerConfig, 0, len(banners))
+	for _, banner := range banners {
+		bannersProto = append(bannersProto, agentsdk.ProtoFromBannerConfig(banner))
+	}
+	return &agentproto.GetAnnouncementBannersResponse{AnnouncementBanners: bannersProto}, nil
+}
+
+func (f *FakeAgentAPI) UpdateStats(ctx context.Context, req *agentproto.UpdateStatsRequest) (*agentproto.UpdateStatsResponse, error) {
+	f.logger.Debug(ctx, "update stats called", slog.F("req", req))
+	// empty request is sent to get the interval; but our tests don't want empty stats requests
+	if req.Stats != nil {
+		select {
+		case <-ctx.Done():
+			return nil, ctx.Err()
+		case f.statsCh <- req.Stats:
+			// OK!
+		}
+	}
+	return &agentproto.UpdateStatsResponse{ReportInterval: durationpb.New(statsInterval)}, nil
+}
+
+func (f *FakeAgentAPI) GetLifecycleStates() []codersdk.WorkspaceAgentLifecycle {
+	f.Lock()
+	defer f.Unlock()
+	return slices.Clone(f.lifecycleStates)
+}
+
+func (f *FakeAgentAPI) UpdateLifecycle(_ context.Context, req *agentproto.UpdateLifecycleRequest) (*agentproto.Lifecycle, error) {
+	f.Lock()
+	defer f.Unlock()
+	s, err := agentsdk.LifecycleStateFromProto(req.GetLifecycle().GetState())
+	if assert.NoError(f.t, err) {
+		f.lifecycleStates = append(f.lifecycleStates, s)
+	}
+	return req.GetLifecycle(), nil
+}
+
+func (f *FakeAgentAPI) BatchUpdateAppHealths(ctx context.Context, req *agentproto.BatchUpdateAppHealthRequest) (*agentproto.BatchUpdateAppHealthResponse, error) {
+	f.logger.Debug(ctx, "batch update app health", slog.F("req", req))
+	select {
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	case f.appHealthCh <- req:
+		return &agentproto.BatchUpdateAppHealthResponse{}, nil
+	}
+}
+
+func (f *FakeAgentAPI) AppHealthCh() <-chan *agentproto.BatchUpdateAppHealthRequest {
+	return f.appHealthCh
+}
+
+func (f *FakeAgentAPI) UpdateStartup(ctx context.Context, req *agentproto.UpdateStartupRequest) (*agentproto.Startup, error) {
+	select {
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	case f.startupCh <- req.GetStartup():
+		return req.GetStartup(), nil
+	}
+}
+
+func (f *FakeAgentAPI) GetMetadata() map[string]agentsdk.Metadata {
+	f.Lock()
+	defer f.Unlock()
+	return maps.Clone(f.metadata)
+}
+
+func (f *FakeAgentAPI) BatchUpdateMetadata(ctx context.Context, req *agentproto.BatchUpdateMetadataRequest) (*agentproto.BatchUpdateMetadataResponse, error) {
+	f.Lock()
+	defer f.Unlock()
+	if f.metadata == nil {
+		f.metadata = make(map[string]agentsdk.Metadata)
+	}
+	for _, md := range req.Metadata {
+		smd := agentsdk.MetadataFromProto(md)
+		f.metadata[md.Key] = smd
+		f.logger.Debug(ctx, "post metadata", slog.F("key", md.Key), slog.F("md", md))
+	}
+	return &agentproto.BatchUpdateMetadataResponse{}, nil
+}
+
+func (f *FakeAgentAPI) SetLogsChannel(ch chan<- *agentproto.BatchCreateLogsRequest) {
+	f.Lock()
+	defer f.Unlock()
+	f.logsCh = ch
+}
+
+func (f *FakeAgentAPI) BatchCreateLogs(ctx context.Context, req *agentproto.BatchCreateLogsRequest) (*agentproto.BatchCreateLogsResponse, error) {
+	f.logger.Info(ctx, "batch create logs called", slog.F("req", req))
+	f.Lock()
+	ch := f.logsCh
+	f.Unlock()
+	if ch != nil {
+		select {
+		case <-ctx.Done():
+			return nil, ctx.Err()
+		case ch <- req:
+			// ok
+		}
+	}
+	return &agentproto.BatchCreateLogsResponse{}, nil
+}
+
+func (f *FakeAgentAPI) ScriptCompleted(_ context.Context, req *agentproto.WorkspaceAgentScriptCompletedRequest) (*agentproto.WorkspaceAgentScriptCompletedResponse, error) {
+	f.Lock()
+	f.timings = append(f.timings, req.Timing)
+	f.Unlock()
+
+	return &agentproto.WorkspaceAgentScriptCompletedResponse{}, nil
+}
+
+func NewFakeAgentAPI(t testing.TB, logger slog.Logger, manifest *agentproto.Manifest, statsCh chan *agentproto.Stats) *FakeAgentAPI {
+	return &FakeAgentAPI{
+		t:           t,
+		logger:      logger.Named("FakeAgentAPI"),
+		manifest:    manifest,
+		statsCh:     statsCh,
+		startupCh:   make(chan *agentproto.Startup, 100),
+		appHealthCh: make(chan *agentproto.BatchUpdateAppHealthRequest, 100),
+	}
+}

agent/api.go

@@ -5,10 +5,10 @@ import (
 	"sync"
 	"time"
 
-	"github.com/go-chi/chi"
+	"github.com/go-chi/chi/v5"
 
-	"github.com/coder/coder/coderd/httpapi"
-	"github.com/coder/coder/codersdk"
+	"github.com/coder/coder/v2/coderd/httpapi"
+	"github.com/coder/coder/v2/codersdk"
 )
 
 func (a *agent) apiHandler() http.Handler {
@@ -26,17 +26,37 @@ func (a *agent) apiHandler() http.Handler {
 		cpy[k] = b
 	}
 
-	lp := &listeningPortsHandler{ignorePorts: cpy}
+	cacheDuration := 1 * time.Second
+	if a.portCacheDuration > 0 {
+		cacheDuration = a.portCacheDuration
+	}
+
+	lp := &listeningPortsHandler{
+		ignorePorts:   cpy,
+		cacheDuration: cacheDuration,
+	}
+	promHandler := PrometheusMetricsHandler(a.prometheusRegistry, a.logger)
 	r.Get("/api/v0/listening-ports", lp.handler)
+	r.Get("/api/v0/netcheck", a.HandleNetcheck)
+	r.Get("/debug/logs", a.HandleHTTPDebugLogs)
+	r.Get("/debug/magicsock", a.HandleHTTPDebugMagicsock)
+	r.Get("/debug/magicsock/debug-logging/{state}", a.HandleHTTPMagicsockDebugLoggingState)
+	r.Get("/debug/manifest", a.HandleHTTPDebugManifest)
+	r.Get("/debug/prometheus", promHandler.ServeHTTP)
 
 	return r
 }
 
 type listeningPortsHandler struct {
-	mut         sync.Mutex
-	ports       []codersdk.WorkspaceAgentListeningPort
-	mtime       time.Time
-	ignorePorts map[int]string
+	ignorePorts   map[int]string
+	cacheDuration time.Duration
+
+	//nolint: unused  // used on some but not all platforms
+	mut sync.Mutex
+	//nolint: unused  // used on some but not all platforms
+	ports []codersdk.WorkspaceAgentListeningPort
+	//nolint: unused  // used on some but not all platforms
+	mtime time.Time
 }
 
 // handler returns a list of listening ports. This is tested by coderd's

agent/apphealth.go

@@ -10,14 +10,11 @@ import (
 	"golang.org/x/xerrors"
 
 	"cdr.dev/slog"
-	"github.com/coder/coder/codersdk"
-	"github.com/coder/coder/codersdk/agentsdk"
-	"github.com/coder/retry"
+	"github.com/coder/coder/v2/codersdk"
+	"github.com/coder/coder/v2/codersdk/agentsdk"
+	"github.com/coder/quartz"
 )
 
-// WorkspaceAgentApps fetches the workspace apps.
-type WorkspaceAgentApps func(context.Context) ([]codersdk.WorkspaceApp, error)
-
 // PostWorkspaceAgentAppHealth updates the workspace app health.
 type PostWorkspaceAgentAppHealth func(context.Context, agentsdk.PostAppHealthsRequest) error
 
@@ -26,10 +23,26 @@ type WorkspaceAppHealthReporter func(ctx context.Context)
 
 // NewWorkspaceAppHealthReporter creates a WorkspaceAppHealthReporter that reports app health to coderd.
 func NewWorkspaceAppHealthReporter(logger slog.Logger, apps []codersdk.WorkspaceApp, postWorkspaceAgentAppHealth PostWorkspaceAgentAppHealth) WorkspaceAppHealthReporter {
-	runHealthcheckLoop := func(ctx context.Context) error {
+	return NewAppHealthReporterWithClock(logger, apps, postWorkspaceAgentAppHealth, quartz.NewReal())
+}
+
+// NewAppHealthReporterWithClock is only called directly by test code.  Product code should call
+// NewAppHealthReporter.
+func NewAppHealthReporterWithClock(
+	logger slog.Logger,
+	apps []codersdk.WorkspaceApp,
+	postWorkspaceAgentAppHealth PostWorkspaceAgentAppHealth,
+	clk quartz.Clock,
+) WorkspaceAppHealthReporter {
+	logger = logger.Named("apphealth")
+
+	return func(ctx context.Context) {
+		ctx, cancel := context.WithCancel(ctx)
+		defer cancel()
+
 		// no need to run this loop if no apps for this workspace.
 		if len(apps) == 0 {
-			return nil
+			return
 		}
 
 		hasHealthchecksEnabled := false
@@ -44,7 +57,7 @@ func NewWorkspaceAppHealthReporter(logger slog.Logger, apps []codersdk.Workspace
 
 		// no need to run this loop if no health checks are configured.
 		if !hasHealthchecksEnabled {
-			return nil
+			return
 		}
 
 		// run a ticker for each app health check.
@@ -56,25 +69,29 @@ func NewWorkspaceAppHealthReporter(logger slog.Logger, apps []codersdk.Workspace
 			}
 			app := nextApp
 			go func() {
-				t := time.NewTicker(time.Duration(app.Healthcheck.Interval) * time.Second)
-				defer t.Stop()
+				_ = clk.TickerFunc(ctx, time.Duration(app.Healthcheck.Interval)*time.Second, func() error {
+					// We time out at the healthcheck interval to prevent getting too backed up, but
+					// set it 1ms early so that it's not simultaneous with the next tick in testing,
+					// which makes the test easier to understand.
+					//
+					// It would be idiomatic to use the http.Client.Timeout or a context.WithTimeout,
+					// but we are passing this off to the native http library, which is not aware
+					// of the clock library we are using. That means in testing, with a mock clock
+					// it will compare mocked times with real times, and we will get strange results.
+					// So, we just implement the timeout as a context we cancel with an AfterFunc
+					reqCtx, reqCancel := context.WithCancel(ctx)
+					timeout := clk.AfterFunc(
+						time.Duration(app.Healthcheck.Interval)*time.Second-time.Millisecond,
+						reqCancel,
+						"timeout", app.Slug)
+					defer timeout.Stop()
 
-				for {
-					select {
-					case <-ctx.Done():
-						return
-					case <-t.C:
-					}
-					// we set the http timeout to the healthcheck interval to prevent getting too backed up.
-					client := &http.Client{
-						Timeout: time.Duration(app.Healthcheck.Interval) * time.Second,
-					}
 					err := func() error {
-						req, err := http.NewRequestWithContext(ctx, http.MethodGet, app.Healthcheck.URL, nil)
+						req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, app.Healthcheck.URL, nil)
 						if err != nil {
 							return err
 						}
-						res, err := client.Do(req)
+						res, err := http.DefaultClient.Do(req)
 						if err != nil {
 							return err
 						}
@@ -87,6 +104,7 @@ func NewWorkspaceAppHealthReporter(logger slog.Logger, apps []codersdk.Workspace
 						return nil
 					}()
 					if err != nil {
+						nowUnhealthy := false
 						mu.Lock()
 						if failures[app.ID] < int(app.Healthcheck.Threshold) {
 							// increment the failure count and keep status the same.
@@ -96,61 +114,52 @@ func NewWorkspaceAppHealthReporter(logger slog.Logger, apps []codersdk.Workspace
 							// set to unhealthy if we hit the failure threshold.
 							// we stop incrementing at the threshold to prevent the failure value from increasing forever.
 							health[app.ID] = codersdk.WorkspaceAppHealthUnhealthy
+							nowUnhealthy = true
 						}
 						mu.Unlock()
+						logger.Debug(ctx, "error checking app health",
+							slog.F("id", app.ID.String()),
+							slog.F("slug", app.Slug),
+							slog.F("now_unhealthy", nowUnhealthy), slog.Error(err),
+						)
 					} else {
 						mu.Lock()
 						// we only need one successful health check to be considered healthy.
 						health[app.ID] = codersdk.WorkspaceAppHealthHealthy
 						failures[app.ID] = 0
 						mu.Unlock()
+						logger.Debug(ctx, "workspace app healthy", slog.F("id", app.ID.String()), slog.F("slug", app.Slug))
 					}
-
-					t.Reset(time.Duration(app.Healthcheck.Interval) * time.Second)
-				}
+					return nil
+				}, "healthcheck", app.Slug)
 			}()
 		}
 
 		mu.Lock()
 		lastHealth := copyHealth(health)
 		mu.Unlock()
-		reportTicker := time.NewTicker(time.Second)
-		defer reportTicker.Stop()
-		// every second we check if the health values of the apps have changed
-		// and if there is a change we will report the new values.
-		for {
-			select {
-			case <-ctx.Done():
+		reportTicker := clk.TickerFunc(ctx, time.Second, func() error {
+			mu.RLock()
+			changed := healthChanged(lastHealth, health)
+			mu.RUnlock()
+			if !changed {
 				return nil
-			case <-reportTicker.C:
-				mu.RLock()
-				changed := healthChanged(lastHealth, health)
-				mu.RUnlock()
-				if !changed {
-					continue
-				}
-
-				mu.Lock()
-				lastHealth = copyHealth(health)
-				mu.Unlock()
-				err := postWorkspaceAgentAppHealth(ctx, agentsdk.PostAppHealthsRequest{
-					Healths: lastHealth,
-				})
-				if err != nil {
-					logger.Error(ctx, "failed to report workspace app stat", slog.Error(err))
-				}
 			}
-		}
-	}
 
-	return func(ctx context.Context) {
-		for r := retry.New(time.Second, 30*time.Second); r.Wait(ctx); {
-			err := runHealthcheckLoop(ctx)
-			if err == nil || xerrors.Is(err, context.Canceled) || xerrors.Is(err, context.DeadlineExceeded) {
-				return
+			mu.Lock()
+			lastHealth = copyHealth(health)
+			mu.Unlock()
+			err := postWorkspaceAgentAppHealth(ctx, agentsdk.PostAppHealthsRequest{
+				Healths: lastHealth,
+			})
+			if err != nil {
+				logger.Error(ctx, "failed to report workspace app health", slog.Error(err))
+			} else {
+				logger.Debug(ctx, "sent workspace app health", slog.F("health", lastHealth))
 			}
-			logger.Error(ctx, "failed running workspace app reporter", slog.Error(err))
-		}
+			return nil
+		}, "report")
+		_ = reportTicker.Wait() // only possible error is context done
 	}
 }
 

agent/apphealth_test.go

@@ -4,33 +4,37 @@ import (
 	"context"
 	"net/http"
 	"net/http/httptest"
-	"sync"
-	"sync/atomic"
+	"slices"
+	"strings"
 	"testing"
 	"time"
 
+	"github.com/google/uuid"
 	"github.com/stretchr/testify/require"
 
-	"cdr.dev/slog"
-	"cdr.dev/slog/sloggers/slogtest"
-	"github.com/coder/coder/agent"
-	"github.com/coder/coder/coderd/httpapi"
-	"github.com/coder/coder/codersdk"
-	"github.com/coder/coder/codersdk/agentsdk"
-	"github.com/coder/coder/testutil"
+	"github.com/coder/coder/v2/agent"
+	"github.com/coder/coder/v2/agent/agenttest"
+	"github.com/coder/coder/v2/agent/proto"
+	"github.com/coder/coder/v2/coderd/httpapi"
+	"github.com/coder/coder/v2/codersdk"
+	"github.com/coder/coder/v2/codersdk/agentsdk"
+	"github.com/coder/coder/v2/testutil"
+	"github.com/coder/quartz"
 )
 
 func TestAppHealth_Healthy(t *testing.T) {
 	t.Parallel()
-	ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
+	ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitShort)
 	defer cancel()
 	apps := []codersdk.WorkspaceApp{
 		{
+			ID:          uuid.UUID{1},
 			Slug:        "app1",
 			Healthcheck: codersdk.Healthcheck{},
 			Health:      codersdk.WorkspaceAppHealthDisabled,
 		},
 		{
+			ID:   uuid.UUID{2},
 			Slug: "app2",
 			Healthcheck: codersdk.Healthcheck{
 				// URL: We don't set the URL for this test because the setup will
@@ -40,34 +44,81 @@ func TestAppHealth_Healthy(t *testing.T) {
 			},
 			Health: codersdk.WorkspaceAppHealthInitializing,
 		},
+		{
+			ID:   uuid.UUID{3},
+			Slug: "app3",
+			Healthcheck: codersdk.Healthcheck{
+				Interval:  2,
+				Threshold: 1,
+			},
+			Health: codersdk.WorkspaceAppHealthInitializing,
+		},
 	}
+	checks2 := 0
+	checks3 := 0
 	handlers := []http.Handler{
 		nil,
 		http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			checks2++
+			httpapi.Write(r.Context(), w, http.StatusOK, nil)
+		}),
+		http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			checks3++
 			httpapi.Write(r.Context(), w, http.StatusOK, nil)
 		}),
 	}
-	getApps, closeFn := setupAppReporter(ctx, t, apps, handlers)
-	defer closeFn()
-	apps, err := getApps(ctx)
-	require.NoError(t, err)
-	require.EqualValues(t, codersdk.WorkspaceAppHealthDisabled, apps[0].Health)
-	require.Eventually(t, func() bool {
-		apps, err := getApps(ctx)
-		if err != nil {
-			return false
-		}
+	mClock := quartz.NewMock(t)
+	healthcheckTrap := mClock.Trap().TickerFunc("healthcheck")
+	defer healthcheckTrap.Close()
+	reportTrap := mClock.Trap().TickerFunc("report")
+	defer reportTrap.Close()
 
-		return apps[1].Health == codersdk.WorkspaceAppHealthHealthy
-	}, testutil.WaitLong, testutil.IntervalSlow)
+	fakeAPI, closeFn := setupAppReporter(ctx, t, slices.Clone(apps), handlers, mClock)
+	defer closeFn()
+	healthchecksStarted := make([]string, 2)
+	for i := 0; i < 2; i++ {
+		c := healthcheckTrap.MustWait(ctx)
+		c.Release()
+		healthchecksStarted[i] = c.Tags[1]
+	}
+	slices.Sort(healthchecksStarted)
+	require.Equal(t, []string{"app2", "app3"}, healthchecksStarted)
+
+	// advance the clock 1ms before the report ticker starts, so that it's not
+	// simultaneous with the checks.
+	mClock.Advance(time.Millisecond).MustWait(ctx)
+	reportTrap.MustWait(ctx).Release()
+
+	mClock.Advance(999 * time.Millisecond).MustWait(ctx) // app2 is now healthy
+
+	mClock.Advance(time.Millisecond).MustWait(ctx) // report gets triggered
+	update := testutil.RequireRecvCtx(ctx, t, fakeAPI.AppHealthCh())
+	require.Len(t, update.GetUpdates(), 2)
+	applyUpdate(t, apps, update)
+	require.Equal(t, codersdk.WorkspaceAppHealthHealthy, apps[1].Health)
+	require.Equal(t, codersdk.WorkspaceAppHealthInitializing, apps[2].Health)
+
+	mClock.Advance(999 * time.Millisecond).MustWait(ctx) // app3 is now healthy
+
+	mClock.Advance(time.Millisecond).MustWait(ctx) // report gets triggered
+	update = testutil.RequireRecvCtx(ctx, t, fakeAPI.AppHealthCh())
+	require.Len(t, update.GetUpdates(), 2)
+	applyUpdate(t, apps, update)
+	require.Equal(t, codersdk.WorkspaceAppHealthHealthy, apps[1].Health)
+	require.Equal(t, codersdk.WorkspaceAppHealthHealthy, apps[2].Health)
+
+	// ensure we aren't spamming
+	require.Equal(t, 2, checks2)
+	require.Equal(t, 1, checks3)
 }
 
 func TestAppHealth_500(t *testing.T) {
 	t.Parallel()
-	ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
+	ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitShort)
 	defer cancel()
 	apps := []codersdk.WorkspaceApp{
 		{
+			ID:   uuid.UUID{2},
 			Slug: "app2",
 			Healthcheck: codersdk.Healthcheck{
 				// URL: We don't set the URL for this test because the setup will
@@ -83,59 +134,40 @@ func TestAppHealth_500(t *testing.T) {
 			httpapi.Write(r.Context(), w, http.StatusInternalServerError, nil)
 		}),
 	}
-	getApps, closeFn := setupAppReporter(ctx, t, apps, handlers)
-	defer closeFn()
-	require.Eventually(t, func() bool {
-		apps, err := getApps(ctx)
-		if err != nil {
-			return false
-		}
 
-		return apps[0].Health == codersdk.WorkspaceAppHealthUnhealthy
-	}, testutil.WaitLong, testutil.IntervalSlow)
+	mClock := quartz.NewMock(t)
+	healthcheckTrap := mClock.Trap().TickerFunc("healthcheck")
+	defer healthcheckTrap.Close()
+	reportTrap := mClock.Trap().TickerFunc("report")
+	defer reportTrap.Close()
+
+	fakeAPI, closeFn := setupAppReporter(ctx, t, slices.Clone(apps), handlers, mClock)
+	defer closeFn()
+	healthcheckTrap.MustWait(ctx).Release()
+	// advance the clock 1ms before the report ticker starts, so that it's not
+	// simultaneous with the checks.
+	mClock.Advance(time.Millisecond).MustWait(ctx)
+	reportTrap.MustWait(ctx).Release()
+
+	mClock.Advance(999 * time.Millisecond).MustWait(ctx) // check gets triggered
+	mClock.Advance(time.Millisecond).MustWait(ctx)       // report gets triggered, but unsent since we are at the threshold
+
+	mClock.Advance(999 * time.Millisecond).MustWait(ctx) // 2nd check, crosses threshold
+	mClock.Advance(time.Millisecond).MustWait(ctx)       // 2nd report, sends update
+
+	update := testutil.RequireRecvCtx(ctx, t, fakeAPI.AppHealthCh())
+	require.Len(t, update.GetUpdates(), 1)
+	applyUpdate(t, apps, update)
+	require.Equal(t, codersdk.WorkspaceAppHealthUnhealthy, apps[0].Health)
 }
 
 func TestAppHealth_Timeout(t *testing.T) {
 	t.Parallel()
-	ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
-	defer cancel()
-	apps := []codersdk.WorkspaceApp{
-		{
-			Slug: "app2",
-			Healthcheck: codersdk.Healthcheck{
-				// URL: We don't set the URL for this test because the setup will
-				// create a httptest server for us and set it for us.
-				Interval:  1,
-				Threshold: 1,
-			},
-			Health: codersdk.WorkspaceAppHealthInitializing,
-		},
-	}
-	handlers := []http.Handler{
-		http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			// sleep longer than the interval to cause the health check to time out
-			time.Sleep(2 * time.Second)
-			httpapi.Write(r.Context(), w, http.StatusOK, nil)
-		}),
-	}
-	getApps, closeFn := setupAppReporter(ctx, t, apps, handlers)
-	defer closeFn()
-	require.Eventually(t, func() bool {
-		apps, err := getApps(ctx)
-		if err != nil {
-			return false
-		}
-
-		return apps[0].Health == codersdk.WorkspaceAppHealthUnhealthy
-	}, testutil.WaitLong, testutil.IntervalSlow)
-}
-
-func TestAppHealth_NotSpamming(t *testing.T) {
-	t.Parallel()
-	ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
+	ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitShort)
 	defer cancel()
 	apps := []codersdk.WorkspaceApp{
 		{
+			ID:   uuid.UUID{2},
 			Slug: "app2",
 			Healthcheck: codersdk.Healthcheck{
 				// URL: We don't set the URL for this test because the setup will
@@ -147,22 +179,66 @@ func TestAppHealth_NotSpamming(t *testing.T) {
 		},
 	}
 
-	counter := new(int32)
 	handlers := []http.Handler{
-		http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			atomic.AddInt32(counter, 1)
+		http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) {
+			// allow the request to time out
+			<-r.Context().Done()
 		}),
 	}
-	_, closeFn := setupAppReporter(ctx, t, apps, handlers)
+	mClock := quartz.NewMock(t)
+	start := mClock.Now()
+
+	// for this test, it's easier to think in the number of milliseconds elapsed
+	// since start.
+	ms := func(n int) time.Time {
+		return start.Add(time.Duration(n) * time.Millisecond)
+	}
+	healthcheckTrap := mClock.Trap().TickerFunc("healthcheck")
+	defer healthcheckTrap.Close()
+	reportTrap := mClock.Trap().TickerFunc("report")
+	defer reportTrap.Close()
+	timeoutTrap := mClock.Trap().AfterFunc("timeout")
+	defer timeoutTrap.Close()
+
+	fakeAPI, closeFn := setupAppReporter(ctx, t, apps, handlers, mClock)
 	defer closeFn()
-	// Ensure we haven't made more than 2 (expected 1 + 1 for buffer) requests in the last second.
-	// if there is a bug where we are spamming the healthcheck route this will catch it.
-	time.Sleep(time.Second)
-	require.LessOrEqual(t, atomic.LoadInt32(counter), int32(2))
+	healthcheckTrap.MustWait(ctx).Release()
+	// advance the clock 1ms before the report ticker starts, so that it's not
+	// simultaneous with the checks.
+	mClock.Set(ms(1)).MustWait(ctx)
+	reportTrap.MustWait(ctx).Release()
+
+	w := mClock.Set(ms(1000)) // 1st check starts
+	timeoutTrap.MustWait(ctx).Release()
+	mClock.Set(ms(1001)).MustWait(ctx) // report tick, no change
+	mClock.Set(ms(1999))               // timeout pops
+	w.MustWait(ctx)                    // 1st check finished
+	w = mClock.Set(ms(2000))           // 2nd check starts
+	timeoutTrap.MustWait(ctx).Release()
+	mClock.Set(ms(2001)).MustWait(ctx) // report tick, no change
+	mClock.Set(ms(2999))               // timeout pops
+	w.MustWait(ctx)                    // 2nd check finished
+	// app is now unhealthy after 2 timeouts
+	mClock.Set(ms(3000)) // 3rd check starts
+	timeoutTrap.MustWait(ctx).Release()
+	mClock.Set(ms(3001)).MustWait(ctx) // report tick, sends changes
+
+	update := testutil.RequireRecvCtx(ctx, t, fakeAPI.AppHealthCh())
+	require.Len(t, update.GetUpdates(), 1)
+	applyUpdate(t, apps, update)
+	require.Equal(t, codersdk.WorkspaceAppHealthUnhealthy, apps[0].Health)
 }
 
-func setupAppReporter(ctx context.Context, t *testing.T, apps []codersdk.WorkspaceApp, handlers []http.Handler) (agent.WorkspaceAgentApps, func()) {
+func setupAppReporter(
+	ctx context.Context, t *testing.T,
+	apps []codersdk.WorkspaceApp,
+	handlers []http.Handler,
+	clk quartz.Clock,
+) (*agenttest.FakeAgentAPI, func()) {
 	closers := []func(){}
+	for _, app := range apps {
+		require.NotEqual(t, uuid.Nil, app.ID, "all apps must have ID set")
+	}
 	for i, handler := range handlers {
 		if handler == nil {
 			continue
@@ -174,34 +250,39 @@ func setupAppReporter(ctx context.Context, t *testing.T, apps []codersdk.Workspa
 		closers = append(closers, ts.Close)
 	}
 
-	var mu sync.Mutex
-	workspaceAgentApps := func(context.Context) ([]codersdk.WorkspaceApp, error) {
-		mu.Lock()
-		defer mu.Unlock()
-		var newApps []codersdk.WorkspaceApp
-		return append(newApps, apps...), nil
-	}
-	postWorkspaceAgentAppHealth := func(_ context.Context, req agentsdk.PostAppHealthsRequest) error {
-		mu.Lock()
-		for id, health := range req.Healths {
-			for i, app := range apps {
-				if app.ID != id {
-					continue
-				}
-				app.Health = health
-				apps[i] = app
-			}
-		}
-		mu.Unlock()
+	// We don't care about manifest or stats in this test since it's not using
+	// a full agent and these RPCs won't get called.
+	//
+	// We use a proper fake agent API so we can test the conversion code and the
+	// request code as well. Before we were bypassing these by using a custom
+	// post function.
+	fakeAAPI := agenttest.NewFakeAgentAPI(t, testutil.Logger(t), nil, nil)
 
-		return nil
-	}
+	go agent.NewAppHealthReporterWithClock(
+		testutil.Logger(t),
+		apps, agentsdk.AppHealthPoster(fakeAAPI), clk,
+	)(ctx)
 
-	go agent.NewWorkspaceAppHealthReporter(slogtest.Make(t, nil).Leveled(slog.LevelDebug), apps, postWorkspaceAgentAppHealth)(ctx)
-
-	return workspaceAgentApps, func() {
+	return fakeAAPI, func() {
 		for _, closeFn := range closers {
 			closeFn()
 		}
 	}
 }
+
+func applyUpdate(t *testing.T, apps []codersdk.WorkspaceApp, req *proto.BatchUpdateAppHealthRequest) {
+	t.Helper()
+	for _, update := range req.Updates {
+		updateID, err := uuid.FromBytes(update.Id)
+		require.NoError(t, err)
+		updateHealth := codersdk.WorkspaceAppHealth(strings.ToLower(proto.AppHealth_name[int32(update.Health)]))
+
+		for i, app := range apps {
+			if app.ID != updateID {
+				continue
+			}
+			app.Health = updateHealth
+			apps[i] = app
+		}
+	}
+}

agent/checkpoint.go

@@ -0,0 +1,51 @@
+package agent
+
+import (
+	"context"
+	"runtime"
+	"sync"
+
+	"cdr.dev/slog"
+)
+
+// checkpoint allows a goroutine to communicate when it is OK to proceed beyond some async condition
+// to other dependent goroutines.
+type checkpoint struct {
+	logger slog.Logger
+	mu     sync.Mutex
+	called bool
+	done   chan struct{}
+	err    error
+}
+
+// complete the checkpoint.  Pass nil to indicate the checkpoint was ok.  It is an error to call this
+// more than once.
+func (c *checkpoint) complete(err error) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if c.called {
+		b := make([]byte, 2048)
+		n := runtime.Stack(b, false)
+		c.logger.Critical(context.Background(), "checkpoint complete called more than once", slog.F("stacktrace", b[:n]))
+		return
+	}
+	c.called = true
+	c.err = err
+	close(c.done)
+}
+
+func (c *checkpoint) wait(ctx context.Context) error {
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	case <-c.done:
+		return c.err
+	}
+}
+
+func newCheckpoint(logger slog.Logger) *checkpoint {
+	return &checkpoint{
+		logger: logger,
+		done:   make(chan struct{}),
+	}
+}

agent/checkpoint_internal_test.go

@@ -0,0 +1,49 @@
+package agent
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"golang.org/x/xerrors"
+
+	"cdr.dev/slog/sloggers/slogtest"
+	"github.com/coder/coder/v2/testutil"
+)
+
+func TestCheckpoint_CompleteWait(t *testing.T) {
+	t.Parallel()
+	logger := testutil.Logger(t)
+	ctx := testutil.Context(t, testutil.WaitShort)
+	uut := newCheckpoint(logger)
+	err := xerrors.New("test")
+	uut.complete(err)
+	got := uut.wait(ctx)
+	require.Equal(t, err, got)
+}
+
+func TestCheckpoint_CompleteTwice(t *testing.T) {
+	t.Parallel()
+	logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true})
+	ctx := testutil.Context(t, testutil.WaitShort)
+	uut := newCheckpoint(logger)
+	err := xerrors.New("test")
+	uut.complete(err)
+	uut.complete(nil) // drops CRITICAL log
+	got := uut.wait(ctx)
+	require.Equal(t, err, got)
+}
+
+func TestCheckpoint_WaitComplete(t *testing.T) {
+	t.Parallel()
+	logger := testutil.Logger(t)
+	ctx := testutil.Context(t, testutil.WaitShort)
+	uut := newCheckpoint(logger)
+	err := xerrors.New("test")
+	errCh := make(chan error, 1)
+	go func() {
+		errCh <- uut.wait(ctx)
+	}()
+	uut.complete(err)
+	got := testutil.RequireRecvCtx(ctx, t, errCh)
+	require.Equal(t, err, got)
+}

agent/health.go

@@ -0,0 +1,31 @@
+package agent
+
+import (
+	"net/http"
+
+	"github.com/coder/coder/v2/coderd/healthcheck/health"
+	"github.com/coder/coder/v2/coderd/httpapi"
+	"github.com/coder/coder/v2/codersdk"
+	"github.com/coder/coder/v2/codersdk/healthsdk"
+)
+
+func (a *agent) HandleNetcheck(rw http.ResponseWriter, r *http.Request) {
+	ni := a.TailnetConn().GetNetInfo()
+
+	ifReport, err := healthsdk.RunInterfacesReport()
+	if err != nil {
+		httpapi.Write(r.Context(), rw, http.StatusInternalServerError, codersdk.Response{
+			Message: "Failed to run interfaces report",
+			Detail:  err.Error(),
+		})
+		return
+	}
+
+	httpapi.Write(r.Context(), rw, http.StatusOK, healthsdk.AgentNetcheckReport{
+		BaseReport: healthsdk.BaseReport{
+			Severity: health.SeverityOK,
+		},
+		NetInfo:    ni,
+		Interfaces: ifReport,
+	})
+}

agent/metrics.go

@@ -1,32 +1,131 @@
 package agent
 
 import (
+	"context"
 	"fmt"
 	"strings"
 
+	"github.com/prometheus/client_golang/prometheus"
+	prompb "github.com/prometheus/client_model/go"
 	"tailscale.com/util/clientmetric"
 
-	"github.com/coder/coder/codersdk/agentsdk"
+	"cdr.dev/slog"
+	"github.com/coder/coder/v2/agent/proto"
 )
 
-func collectMetrics() []agentsdk.AgentMetric {
-	// Tailscale metrics
+type agentMetrics struct {
+	connectionsTotal      prometheus.Counter
+	reconnectingPTYErrors *prometheus.CounterVec
+	// startupScriptSeconds is the time in seconds that the start script(s)
+	// took to run. This is reported once per agent.
+	startupScriptSeconds *prometheus.GaugeVec
+	currentConnections   *prometheus.GaugeVec
+}
+
+func newAgentMetrics(registerer prometheus.Registerer) *agentMetrics {
+	connectionsTotal := prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace: "agent", Subsystem: "reconnecting_pty", Name: "connections_total",
+	})
+	registerer.MustRegister(connectionsTotal)
+
+	reconnectingPTYErrors := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: "agent",
+			Subsystem: "reconnecting_pty",
+			Name:      "errors_total",
+		},
+		[]string{"error_type"},
+	)
+	registerer.MustRegister(reconnectingPTYErrors)
+
+	startupScriptSeconds := prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: "coderd",
+		Subsystem: "agentstats",
+		Name:      "startup_script_seconds",
+		Help:      "Amount of time taken to run the startup script in seconds.",
+	}, []string{"success"})
+	registerer.MustRegister(startupScriptSeconds)
+
+	currentConnections := prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: "coderd",
+		Subsystem: "agentstats",
+		Name:      "currently_reachable_peers",
+		Help:      "The number of peers (e.g. clients) that are currently reachable over the encrypted network.",
+	}, []string{"connection_type"})
+	registerer.MustRegister(currentConnections)
+
+	return &agentMetrics{
+		connectionsTotal:      connectionsTotal,
+		reconnectingPTYErrors: reconnectingPTYErrors,
+		startupScriptSeconds:  startupScriptSeconds,
+		currentConnections:    currentConnections,
+	}
+}
+
+func (a *agent) collectMetrics(ctx context.Context) []*proto.Stats_Metric {
+	var collected []*proto.Stats_Metric
+
+	// Tailscale internal metrics
 	metrics := clientmetric.Metrics()
-	collected := make([]agentsdk.AgentMetric, 0, len(metrics))
 	for _, m := range metrics {
 		if isIgnoredMetric(m.Name()) {
 			continue
 		}
 
-		collected = append(collected, agentsdk.AgentMetric{
+		collected = append(collected, &proto.Stats_Metric{
 			Name:  m.Name(),
 			Type:  asMetricType(m.Type()),
 			Value: float64(m.Value()),
 		})
 	}
+
+	metricFamilies, err := a.prometheusRegistry.Gather()
+	if err != nil {
+		a.logger.Error(ctx, "can't gather agent metrics", slog.Error(err))
+		return collected
+	}
+
+	for _, metricFamily := range metricFamilies {
+		for _, metric := range metricFamily.GetMetric() {
+			labels := toAgentMetricLabels(metric.Label)
+
+			if metric.Counter != nil {
+				collected = append(collected, &proto.Stats_Metric{
+					Name:   metricFamily.GetName(),
+					Type:   proto.Stats_Metric_COUNTER,
+					Value:  metric.Counter.GetValue(),
+					Labels: labels,
+				})
+			} else if metric.Gauge != nil {
+				collected = append(collected, &proto.Stats_Metric{
+					Name:   metricFamily.GetName(),
+					Type:   proto.Stats_Metric_GAUGE,
+					Value:  metric.Gauge.GetValue(),
+					Labels: labels,
+				})
+			} else {
+				a.logger.Error(ctx, "unsupported metric type", slog.F("type", metricFamily.Type.String()))
+			}
+		}
+	}
 	return collected
 }
 
+func toAgentMetricLabels(metricLabels []*prompb.LabelPair) []*proto.Stats_Metric_Label {
+	if len(metricLabels) == 0 {
+		return nil
+	}
+
+	labels := make([]*proto.Stats_Metric_Label, 0, len(metricLabels))
+	for _, metricLabel := range metricLabels {
+		labels = append(labels, &proto.Stats_Metric_Label{
+			Name:  metricLabel.GetName(),
+			Value: metricLabel.GetValue(),
+		})
+	}
+	return labels
+}
+
 // isIgnoredMetric checks if the metric should be ignored, as Coder agent doesn't use related features.
 // Expected metric families: magicsock_*, derp_*, tstun_*, netcheck_*, portmap_*, etc.
 func isIgnoredMetric(metricName string) bool {
@@ -40,12 +139,12 @@ func isIgnoredMetric(metricName string) bool {
 	return false
 }
 
-func asMetricType(typ clientmetric.Type) agentsdk.AgentMetricType {
+func asMetricType(typ clientmetric.Type) proto.Stats_Metric_Type {
 	switch typ {
 	case clientmetric.TypeGauge:
-		return agentsdk.AgentMetricTypeGauge
+		return proto.Stats_Metric_GAUGE
 	case clientmetric.TypeCounter:
-		return agentsdk.AgentMetricTypeCounter
+		return proto.Stats_Metric_COUNTER
 	default:
 		panic(fmt.Sprintf("unknown metric type: %d", typ))
 	}

agent/ports_supported.go

@@ -8,14 +8,15 @@ import (
 	"github.com/cakturk/go-netstat/netstat"
 	"golang.org/x/xerrors"
 
-	"github.com/coder/coder/codersdk"
+	"github.com/coder/coder/v2/codersdk"
+	"github.com/coder/coder/v2/codersdk/workspacesdk"
 )
 
 func (lp *listeningPortsHandler) getListeningPorts() ([]codersdk.WorkspaceAgentListeningPort, error) {
 	lp.mut.Lock()
 	defer lp.mut.Unlock()
 
-	if time.Since(lp.mtime) < time.Second {
+	if time.Since(lp.mtime) < lp.cacheDuration {
 		// copy
 		ports := make([]codersdk.WorkspaceAgentListeningPort, len(lp.ports))
 		copy(ports, lp.ports)
@@ -32,7 +33,7 @@ func (lp *listeningPortsHandler) getListeningPorts() ([]codersdk.WorkspaceAgentL
 	seen := make(map[uint16]struct{}, len(tabs))
 	ports := []codersdk.WorkspaceAgentListeningPort{}
 	for _, tab := range tabs {
-		if tab.LocalAddr == nil || tab.LocalAddr.Port < codersdk.WorkspaceAgentMinimumListeningPort {
+		if tab.LocalAddr == nil || tab.LocalAddr.Port < workspacesdk.AgentMinimumListeningPort {
 			continue
 		}