chore: revert "chore: remove workspace_actions experiment (#10030)" (#10363)

* Remove terminalXService

This is a prelude to the change I actually want to make, which is to
send the size of the terminal on the web socket URL after we do a fit.
I have found xstate so confusing that it was easier to just rewrite it.

* Fix hanging tests

I am not really sure what ws.connected is doing but it seems to somehow
block updates.  Something to do with `act()` maybe?

Basically, the useEffect creating the terminal never updates once the
config query finishes, so the web socket is never created, and the test
hangs forever.

It might have been working before only because the web socket was
created using xstate rather than useEffect and once it connected it
would unblock and React could update again but this is just a guess.

* Ignore other config changes

The terminal only cares about the renderer specifically, no need to
recreate the terminal if something else changes.

* Break out port forward URL open to util

Felt like this could be broken out to reduce the component size.  Also
trying to figure out why it is causing the terminal to create multiple
times.

* Prevent handleWebLink change from recreating terminal

Depending on the timing, handleWebLink was causing the terminal to get
recreated.  We only need to create the terminal once unless the render
type changes.

Recreating the terminal was also recreating the web socket pointlessly.

.devcontainer/Dockerfile

@@ -1,83 +0,0 @@
-FROM ubuntu
-SHELL ["/bin/bash", "-o", "pipefail", "-c"]
-
-ENV EDITOR=vim
-
-RUN apt-get update && apt-get upgrade --yes
-
-RUN apt-get install --yes \
-	ca-certificates \
-	bash-completion \
-	build-essential \
-	curl \
-	cmake \
-	direnv \
-	emacs-nox \
-	gnupg \
-	htop \
-	jq \
-	less \
-	lsb-release \
-	lsof \
-	man-db \
-	nano \
-	neovim \
-	ssl-cert \
-	sudo \
-	unzip \
-	xz-utils \
-	zip
-
-# configure locales to UTF8
-RUN apt-get install locales && locale-gen en_US.UTF-8
-ENV LANG='en_US.UTF-8' LANGUAGE='en_US:en' LC_ALL='en_US.UTF-8'
-
-# configure direnv
-RUN direnv hook bash >> $HOME/.bashrc
-
-# install nix
-RUN sh <(curl -L https://nixos.org/nix/install) --daemon
-
-RUN mkdir -p $HOME/.config/nix $HOME/.config/nixpkgs \
-	&& echo 'sandbox = false' >> $HOME/.config/nix/nix.conf \
-	&& echo '{ allowUnfree = true; }' >> $HOME/.config/nixpkgs/config.nix \
-	&& echo '. $HOME/.nix-profile/etc/profile.d/nix.sh' >> $HOME/.bashrc
-
-
-# install docker and configure daemon to use vfs as GitHub codespaces requires vfs
-# https://github.com/moby/moby/issues/13742#issuecomment-725197223
-RUN mkdir -p /etc/apt/keyrings \
-	&& curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg \
-	&& echo \
-	"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \
-	$(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null \
-	&& apt-get update \
-	&& apt-get install --yes docker-ce docker-ce-cli containerd.io docker-compose-plugin \
-	&& mkdir -p /etc/docker \
-	&& echo '{"cgroup-parent":"/actions_job","storage-driver":"vfs"}' >> /etc/docker/daemon.json
-
-# install golang and language tooling
-ENV GO_VERSION=1.20
-ENV GOPATH=$HOME/go-packages
-ENV GOROOT=$HOME/go
-ENV PATH=$GOROOT/bin:$GOPATH/bin:$PATH
-RUN curl -fsSL https://dl.google.com/go/go$GO_VERSION.linux-amd64.tar.gz | tar xzs
-RUN echo 'export PATH=$GOPATH/bin:$PATH' >> $HOME/.bashrc
-
-RUN bash -c ". $HOME/.bashrc \
-	go install -v golang.org/x/tools/gopls@latest \
-	&& go install -v mvdan.cc/sh/v3/cmd/shfmt@latest \
-	&& go install -v github.com/mikefarah/yq/v4@v4.30.6 \
-	"
-
-# install nodejs
-RUN bash -c "$(curl -fsSL https://deb.nodesource.com/setup_14.x)" \
-	&& apt-get install -y nodejs
-
-# install zstd
-RUN bash -c "$(curl -fsSL https://raw.githubusercontent.com/horta/zstd.install/main/install)"
-
-# install nfpm
-RUN echo 'deb [trusted=yes] https://repo.goreleaser.com/apt/ /' | sudo tee /etc/apt/sources.list.d/goreleaser.list \
-	&& apt update \
-	&& apt install nfpm

.devcontainer/devcontainer.json

@@ -1,24 +1,12 @@
-// For format details, see https://aka.ms/devcontainer.json
 {
   "name": "Development environments on your infrastructure",
+  "image": "codercom/oss-dogfood:latest",
 
-  // Sets the run context to one level up instead of the .devcontainer folder.
-  "context": ".",
-
-  // Update the 'dockerFile' property if you aren't using the standard 'Dockerfile' filename.
-  "dockerFile": "Dockerfile",
-
-  // Use 'forwardPorts' to make a list of ports inside the container available locally.
-  // "forwardPorts": [],
-
-  "postStartCommand": "dockerd",
-
-  // privileged is required by GitHub codespaces - https://github.com/microsoft/vscode-dev-containers/issues/727
-  "runArgs": [
-    "--cap-add=SYS_PTRACE",
-    "--security-opt",
-    "seccomp=unconfined",
-    "--privileged",
-    "--init"
-  ]
+  "features": {
+    // See all possible options here https://github.com/devcontainers/features/tree/main/src/docker-in-docker
+    "ghcr.io/devcontainers/features/docker-in-docker:2": {}
+  },
+  // SYS_PTRACE to enable go debugging
+  // without --priviliged the Github Codespace build fails (not required otherwise)
+  "runArgs": ["--cap-add=SYS_PTRACE", "--privileged"]
 }

.git-blame-ignore-revs

@@ -0,0 +1,5 @@
+# If you would like `git blame` to ignore commits from this file, run...
+#   git config blame.ignoreRevsFile .git-blame-ignore-revs
+
+# chore: format code with semicolons when using prettier (#9555)
+988c9af0153561397686c119da9d1336d2433fdd

.gitattributes

@@ -1,5 +1,7 @@
 # Generated files
 coderd/apidoc/docs.go linguist-generated=true
+docs/api/*.md linguist-generated=true
+docs/cli/*.md linguist-generated=true
 coderd/apidoc/swagger.json linguist-generated=true
 coderd/database/dump.sql linguist-generated=true
 peerbroker/proto/*.go linguist-generated=true
@@ -9,3 +11,4 @@ provisionersdk/proto/*.go linguist-generated=true
 *.tfstate.json linguist-generated=true
 *.tfstate.dot linguist-generated=true
 *.tfplan.dot linguist-generated=true
+site/src/api/typesGenerated.ts linguist-generated=true

.github/CODEOWNERS

@@ -1,3 +0,0 @@
-docs/ @coder/docs
-README.md @coder/docs
-ADOPTERS.md @coder/docs

.github/actions/setup-go/action.yaml

@@ -0,0 +1,70 @@
+name: "Setup Go"
+description: |
+  Sets up the Go environment for tests, builds, etc.
+inputs:
+  version:
+    description: "The Go version to use."
+    default: "1.20.10"
+runs:
+  using: "composite"
+  steps:
+    - name: Cache go toolchain
+      uses: buildjet/cache@v3
+      with:
+        path: |
+          ${{ runner.tool_cache }}/go/${{ inputs.version }}
+        key: gotoolchain-${{ runner.os }}-${{ inputs.version }}
+        restore-keys: |
+          gotoolchain-${{ runner.os }}-
+
+    - name: Setup Go
+      uses: buildjet/setup-go@v4
+      with:
+        # We do our own caching for implementation clarity.
+        cache: false
+        go-version: ${{ inputs.version }}
+
+    - name: Get cache dirs
+      shell: bash
+      run: |
+        set -x
+        echo "GOMODCACHE=$(go env GOMODCACHE)" >> $GITHUB_ENV
+        echo "GOCACHE=$(go env GOCACHE)" >> $GITHUB_ENV
+
+    # We split up GOMODCACHE from GOCACHE because the latter must be invalidated
+    # on code change, but the former can be kept.
+    - name: Cache $GOMODCACHE
+      uses: buildjet/cache@v3
+      with:
+        path: |
+          ${{ env.GOMODCACHE }}
+        key: gomodcache-${{ runner.os }}-${{ hashFiles('**/go.sum') }}-${{ github.job }}
+        # restore-keys aren't used because it causes the cache to grow
+        # infinitely. go.sum changes very infrequently, so rebuilding from
+        # scratch every now and then isn't terrible.
+
+    - name: Cache $GOCACHE
+      uses: buildjet/cache@v3
+      with:
+        path: |
+          ${{ env.GOCACHE }}
+        # Job name must be included in the key for effective test cache reuse.
+        # The key format is intentionally different than GOMODCACHE, because any
+        # time a Go file changes we invalidate this cache, whereas GOMODCACHE is
+        # only invalidated when go.sum changes.
+        # The number in the key is incremented when the cache gets too large,
+        # since this technically grows without bound.
+        key: gocache2-${{ runner.os }}-${{ github.job }}-${{ hashFiles('**/*.go', 'go.**') }}
+        restore-keys: |
+          gocache2-${{ runner.os }}-${{ github.job }}-
+          gocache2-${{ runner.os }}-
+
+    - name: Install gotestsum
+      shell: bash
+      run: go install gotest.tools/gotestsum@latest
+
+    # It isn't necessary that we ever do this, but it helps
+    # separate the "setup" from the "run" times.
+    - name: go mod download
+      shell: bash
+      run: go mod download -x

.github/actions/setup-node/action.yaml

@@ -0,0 +1,31 @@
+name: "Setup Node"
+description: |
+  Sets up the node environment for tests, builds, etc.
+inputs:
+  directory:
+    description: |
+      The directory to run the setup in.
+    required: false
+    default: "site"
+runs:
+  using: "composite"
+  steps:
+    - name: Install pnpm
+      uses: pnpm/action-setup@v2
+      with:
+        version: 8
+    - name: Setup Node
+      uses: buildjet/setup-node@v3
+      with:
+        node-version: 18.17.0
+        # See https://github.com/actions/setup-node#caching-global-packages-data
+        cache: "pnpm"
+        cache-dependency-path: ${{ inputs.directory }}/pnpm-lock.yaml
+    - name: Install root node_modules
+      shell: bash
+      run: ./scripts/pnpm_install.sh
+
+    - name: Install node_modules
+      shell: bash
+      run: ../scripts/pnpm_install.sh
+      working-directory: ${{ inputs.directory }}

.github/actions/setup-sqlc/action.yaml

@@ -0,0 +1,10 @@
+name: Setup sqlc
+description: |
+  Sets up the sqlc environment for tests, builds, etc.
+runs:
+  using: "composite"
+  steps:
+    - name: Setup sqlc
+      uses: sqlc-dev/setup-sqlc@v4
+      with:
+        sqlc-version: "1.20.0"

.github/actions/setup-tf/action.yaml

@@ -0,0 +1,11 @@
+name: "Setup Terraform"
+description: |
+  Sets up Terraform for tests, builds, etc.
+runs:
+  using: "composite"
+  steps:
+    - name: Install Terraform
+      uses: hashicorp/setup-terraform@v2
+      with:
+        terraform_version: 1.5.5
+        terraform_wrapper: false

.github/actions/upload-datadog/action.yaml

@@ -0,0 +1,27 @@
+name: Upload tests to datadog
+if: always()
+inputs:
+  api-key:
+    description: "Datadog API key"
+    required: true
+runs:
+  using: "composite"
+  steps:
+    - shell: bash
+      run: |
+        owner=${{ github.repository_owner	 }}
+        echo "owner: $owner"
+        if [[  $owner != "coder" ]]; then
+          echo "Not a pull request from the main repo, skipping..."
+          exit 0
+        fi
+        if [[ -z "${{ inputs.api-key }}" ]]; then
+          # This can happen for dependabot.
+          echo "No API key provided, skipping..."
+          exit 0
+        fi
+        npm install -g @datadog/datadog-ci@2.21.0
+        datadog-ci junit upload --service coder ./gotests.xml \
+          --tags os:${{runner.os}} --tags runner_name:${{runner.name}}
+      env:
+        DATADOG_API_KEY: ${{ inputs.api-key }}

.github/dependabot.yaml

@@ -3,12 +3,12 @@ updates:
   - package-ecosystem: "github-actions"
     directory: "/"
     schedule:
-      interval: "monthly"
+      interval: "weekly"
       time: "06:00"
       timezone: "America/Chicago"
     labels: []
     commit-message:
-      prefix: "chore"
+      prefix: "ci"
     ignore:
       # These actions deliver the latest versions by updating the major
       # release tag, so ignore minor and patch versions
@@ -24,11 +24,15 @@ updates:
         update-types:
           - version-update:semver-minor
           - version-update:semver-patch
+    groups:
+      github-actions:
+        patterns:
+          - "*"
 
   - package-ecosystem: "gomod"
     directory: "/"
     schedule:
-      interval: "monthly"
+      interval: "weekly"
       time: "06:00"
       timezone: "America/Chicago"
     commit-message:
@@ -39,6 +43,33 @@ updates:
       - dependency-name: "*"
         update-types:
           - version-update:semver-patch
+    groups:
+      otel:
+        patterns:
+          - "go.nhat.io/otelsql"
+          - "go.opentelemetry.io/otel*"
+      golang-x:
+        patterns:
+          - "golang.org/x/*"
+
+  # Update our Dockerfile.
+  - package-ecosystem: "docker"
+    directory: "/scripts/"
+    schedule:
+      interval: "weekly"
+      time: "06:00"
+      timezone: "America/Chicago"
+    commit-message:
+      prefix: "chore"
+    labels: []
+    ignore:
+      # We need to coordinate terraform updates with the version hardcoded in
+      # our Go code.
+      - dependency-name: "terraform"
+    groups:
+      scripts-docker:
+        patterns:
+          - "*"
 
   - package-ecosystem: "npm"
     directory: "/site/"
@@ -46,6 +77,56 @@ updates:
       interval: "monthly"
       time: "06:00"
       timezone: "America/Chicago"
+    reviewers:
+      - "coder/ts"
+    commit-message:
+      prefix: "chore"
+    labels: []
+    ignore:
+      # Ignore patch updates for all dependencies
+      - dependency-name: "*"
+        update-types:
+          - version-update:semver-patch
+      # Ignore major updates to Node.js types, because they need to
+      # correspond to the Node.js engine version
+      - dependency-name: "@types/node"
+        update-types:
+          - version-update:semver-major
+    open-pull-requests-limit: 15
+    groups:
+      react:
+        patterns:
+          - "react*"
+          - "@types/react*"
+      xterm:
+        patterns:
+          - "xterm*"
+      xstate:
+        patterns:
+          - "xstate"
+          - "@xstate*"
+      mui:
+        patterns:
+          - "@mui*"
+      storybook:
+        patterns:
+          - "@storybook*"
+          - "storybook*"
+      eslint:
+        patterns:
+          - "eslint*"
+          - "@eslint*"
+          - "@typescript-eslint/eslint-plugin"
+          - "@typescript-eslint/parser"
+
+  - package-ecosystem: "npm"
+    directory: "/offlinedocs/"
+    schedule:
+      interval: "monthly"
+      time: "06:00"
+      timezone: "America/Chicago"
+    reviewers:
+      - "coder/ts"
     commit-message:
       prefix: "chore"
     labels: []
@@ -60,10 +141,11 @@ updates:
         update-types:
           - version-update:semver-major
 
+  # Update dogfood.
   - package-ecosystem: "terraform"
-    directory: "/examples/templates"
+    directory: "/dogfood/"
     schedule:
-      interval: "monthly"
+      interval: "weekly"
       time: "06:00"
       timezone: "America/Chicago"
     commit-message:

.github/pr-deployments/certificate.yaml

@@ -0,0 +1,13 @@
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: pr${PR_NUMBER}-tls
+  namespace: pr-deployment-certs
+spec:
+  secretName: pr${PR_NUMBER}-tls
+  issuerRef:
+    name: letsencrypt
+    kind: ClusterIssuer
+  dnsNames:
+    - "${PR_HOSTNAME}"
+    - "*.${PR_HOSTNAME}"

.github/pr-deployments/rbac.yaml

@@ -0,0 +1,31 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: coder-workspace-pr${PR_NUMBER}
+  namespace: pr${PR_NUMBER}
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: coder-workspace-pr${PR_NUMBER}
+  namespace: pr${PR_NUMBER}
+rules:
+  - apiGroups: ["*"]
+    resources: ["*"]
+    verbs: ["*"]
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: coder-workspace-pr${PR_NUMBER}
+  namespace: pr${PR_NUMBER}
+subjects:
+  - kind: ServiceAccount
+    name: coder-workspace-pr${PR_NUMBER}
+    namespace: pr${PR_NUMBER}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: coder-workspace-pr${PR_NUMBER}

.github/pr-deployments/template/main.tf

@@ -0,0 +1,314 @@
+terraform {
+  required_providers {
+    coder = {
+      source = "coder/coder"
+    }
+    kubernetes = {
+      source = "hashicorp/kubernetes"
+    }
+  }
+}
+
+provider "coder" {
+}
+
+variable "namespace" {
+  type        = string
+  description = "The Kubernetes namespace to create workspaces in (must exist prior to creating workspaces)"
+}
+
+data "coder_parameter" "cpu" {
+  name         = "cpu"
+  display_name = "CPU"
+  description  = "The number of CPU cores"
+  default      = "2"
+  icon         = "/icon/memory.svg"
+  mutable      = true
+  option {
+    name  = "2 Cores"
+    value = "2"
+  }
+  option {
+    name  = "4 Cores"
+    value = "4"
+  }
+  option {
+    name  = "6 Cores"
+    value = "6"
+  }
+  option {
+    name  = "8 Cores"
+    value = "8"
+  }
+}
+
+data "coder_parameter" "memory" {
+  name         = "memory"
+  display_name = "Memory"
+  description  = "The amount of memory in GB"
+  default      = "2"
+  icon         = "/icon/memory.svg"
+  mutable      = true
+  option {
+    name  = "2 GB"
+    value = "2"
+  }
+  option {
+    name  = "4 GB"
+    value = "4"
+  }
+  option {
+    name  = "6 GB"
+    value = "6"
+  }
+  option {
+    name  = "8 GB"
+    value = "8"
+  }
+}
+
+data "coder_parameter" "home_disk_size" {
+  name         = "home_disk_size"
+  display_name = "Home disk size"
+  description  = "The size of the home disk in GB"
+  default      = "10"
+  type         = "number"
+  icon         = "/emojis/1f4be.png"
+  mutable      = false
+  validation {
+    min = 1
+    max = 99999
+  }
+}
+
+provider "kubernetes" {
+  config_path = null
+}
+
+data "coder_workspace" "me" {}
+
+resource "coder_agent" "main" {
+  os                     = "linux"
+  arch                   = "amd64"
+  startup_script_timeout = 180
+  startup_script         = <<-EOT
+    set -e
+
+    # install and start code-server
+    curl -fsSL https://code-server.dev/install.sh | sh -s -- --method=standalone --prefix=/tmp/code-server
+    /tmp/code-server/bin/code-server --auth none --port 13337 >/tmp/code-server.log 2>&1 &
+
+  EOT
+
+  # The following metadata blocks are optional. They are used to display
+  # information about your workspace in the dashboard. You can remove them
+  # if you don't want to display any information.
+  # For basic resources, you can use the `coder stat` command.
+  # If you need more control, you can write your own script.
+  metadata {
+    display_name = "CPU Usage"
+    key          = "0_cpu_usage"
+    script       = "coder stat cpu"
+    interval     = 10
+    timeout      = 1
+  }
+
+  metadata {
+    display_name = "RAM Usage"
+    key          = "1_ram_usage"
+    script       = "coder stat mem"
+    interval     = 10
+    timeout      = 1
+  }
+
+  metadata {
+    display_name = "Home Disk"
+    key          = "3_home_disk"
+    script       = "coder stat disk --path $${HOME}"
+    interval     = 60
+    timeout      = 1
+  }
+
+  metadata {
+    display_name = "CPU Usage (Host)"
+    key          = "4_cpu_usage_host"
+    script       = "coder stat cpu --host"
+    interval     = 10
+    timeout      = 1
+  }
+
+  metadata {
+    display_name = "Memory Usage (Host)"
+    key          = "5_mem_usage_host"
+    script       = "coder stat mem --host"
+    interval     = 10
+    timeout      = 1
+  }
+
+  metadata {
+    display_name = "Load Average (Host)"
+    key          = "6_load_host"
+    # get load avg scaled by number of cores
+    script   = <<EOT
+      echo "`cat /proc/loadavg | awk '{ print $1 }'` `nproc`" | awk '{ printf "%0.2f", $1/$2 }'
+    EOT
+    interval = 60
+    timeout  = 1
+  }
+}
+
+# code-server
+resource "coder_app" "code-server" {
+  agent_id     = coder_agent.main.id
+  slug         = "code-server"
+  display_name = "code-server"
+  icon         = "/icon/code.svg"
+  url          = "http://localhost:13337?folder=/home/coder"
+  subdomain    = false
+  share        = "owner"
+
+  healthcheck {
+    url       = "http://localhost:13337/healthz"
+    interval  = 3
+    threshold = 10
+  }
+}
+
+resource "kubernetes_persistent_volume_claim" "home" {
+  metadata {
+    name      = "coder-${lower(data.coder_workspace.me.owner)}-${lower(data.coder_workspace.me.name)}-home"
+    namespace = var.namespace
+    labels = {
+      "app.kubernetes.io/name"     = "coder-pvc"
+      "app.kubernetes.io/instance" = "coder-pvc-${lower(data.coder_workspace.me.owner)}-${lower(data.coder_workspace.me.name)}"
+      "app.kubernetes.io/part-of"  = "coder"
+      //Coder-specific labels.
+      "com.coder.resource"       = "true"
+      "com.coder.workspace.id"   = data.coder_workspace.me.id
+      "com.coder.workspace.name" = data.coder_workspace.me.name
+      "com.coder.user.id"        = data.coder_workspace.me.owner_id
+      "com.coder.user.username"  = data.coder_workspace.me.owner
+    }
+    annotations = {
+      "com.coder.user.email" = data.coder_workspace.me.owner_email
+    }
+  }
+  wait_until_bound = false
+  spec {
+    access_modes = ["ReadWriteOnce"]
+    resources {
+      requests = {
+        storage = "${data.coder_parameter.home_disk_size.value}Gi"
+      }
+    }
+  }
+}
+
+resource "kubernetes_deployment" "main" {
+  count = data.coder_workspace.me.start_count
+  depends_on = [
+    kubernetes_persistent_volume_claim.home
+  ]
+  wait_for_rollout = false
+  metadata {
+    name      = "coder-${lower(data.coder_workspace.me.owner)}-${lower(data.coder_workspace.me.name)}"
+    namespace = var.namespace
+    labels = {
+      "app.kubernetes.io/name"     = "coder-workspace"
+      "app.kubernetes.io/instance" = "coder-workspace-${lower(data.coder_workspace.me.owner)}-${lower(data.coder_workspace.me.name)}"
+      "app.kubernetes.io/part-of"  = "coder"
+      "com.coder.resource"         = "true"
+      "com.coder.workspace.id"     = data.coder_workspace.me.id
+      "com.coder.workspace.name"   = data.coder_workspace.me.name
+      "com.coder.user.id"          = data.coder_workspace.me.owner_id
+      "com.coder.user.username"    = data.coder_workspace.me.owner
+    }
+    annotations = {
+      "com.coder.user.email" = data.coder_workspace.me.owner_email
+    }
+  }
+
+  spec {
+    replicas = 1
+    selector {
+      match_labels = {
+        "app.kubernetes.io/name" = "coder-workspace"
+      }
+    }
+    strategy {
+      type = "Recreate"
+    }
+
+    template {
+      metadata {
+        labels = {
+          "app.kubernetes.io/name" = "coder-workspace"
+        }
+      }
+      spec {
+        security_context {
+          run_as_user = 1000
+          fs_group    = 1000
+        }
+
+        service_account_name = "coder-workspace-${var.namespace}"
+        container {
+          name              = "dev"
+          image             = "bencdr/devops-tools"
+          image_pull_policy = "Always"
+          command           = ["sh", "-c", coder_agent.main.init_script]
+          security_context {
+            run_as_user = "1000"
+          }
+          env {
+            name  = "CODER_AGENT_TOKEN"
+            value = coder_agent.main.token
+          }
+          resources {
+            requests = {
+              "cpu"    = "250m"
+              "memory" = "512Mi"
+            }
+            limits = {
+              "cpu"    = "${data.coder_parameter.cpu.value}"
+              "memory" = "${data.coder_parameter.memory.value}Gi"
+            }
+          }
+          volume_mount {
+            mount_path = "/home/coder"
+            name       = "home"
+            read_only  = false
+          }
+        }
+
+        volume {
+          name = "home"
+          persistent_volume_claim {
+            claim_name = kubernetes_persistent_volume_claim.home.metadata.0.name
+            read_only  = false
+          }
+        }
+
+        affinity {
+          // This affinity attempts to spread out all workspace pods evenly across
+          // nodes.
+          pod_anti_affinity {
+            preferred_during_scheduling_ignored_during_execution {
+              weight = 1
+              pod_affinity_term {
+                topology_key = "kubernetes.io/hostname"
+                label_selector {
+                  match_expressions {
+                    key      = "app.kubernetes.io/name"
+                    operator = "In"
+                    values   = ["coder-workspace"]
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}

.github/pr-deployments/values.yaml

@@ -0,0 +1,38 @@
+coder:
+  image:
+    repo: "${REPO}"
+    tag: "pr${PR_NUMBER}"
+    pullPolicy: Always
+  service:
+    type: ClusterIP
+  ingress:
+    enable: true
+    className: traefik
+    host: "${PR_HOSTNAME}"
+    wildcardHost: "*.${PR_HOSTNAME}"
+    tls:
+      enable: true
+      secretName: "pr${PR_NUMBER}-tls"
+      wildcardSecretName: "pr${PR_NUMBER}-tls"
+  env:
+    - name: "CODER_ACCESS_URL"
+      value: "https://${PR_HOSTNAME}"
+    - name: "CODER_WILDCARD_ACCESS_URL"
+      value: "*.${PR_HOSTNAME}"
+    - name: "CODER_EXPERIMENTS"
+      value: "${EXPERIMENTS}"
+    - name: CODER_PG_CONNECTION_URL
+      valueFrom:
+        secretKeyRef:
+          name: coder-db-url
+          key: url
+    - name: "CODER_OAUTH2_GITHUB_ALLOW_SIGNUPS"
+      value: "true"
+    - name: "CODER_OAUTH2_GITHUB_CLIENT_ID"
+      value: "${PR_DEPLOYMENTS_GITHUB_OAUTH_CLIENT_ID}"
+    - name: "CODER_OAUTH2_GITHUB_CLIENT_SECRET"
+      value: "${PR_DEPLOYMENTS_GITHUB_OAUTH_CLIENT_SECRET}"
+    - name: "CODER_OAUTH2_GITHUB_ALLOWED_ORGS"
+      value: "coder"
+    - name: "CODER_DERP_CONFIG_URL"
+      value: "https://controlplane.tailscale.com/derpmap/default"

.github/pull_request_template.md

@@ -1,3 +0,0 @@
-<!--
-Check if your change requires documentation edits before merging: https://coder.com/docs/coder. Make edits in `docs/`.
--->

.github/workflows/contrib.yaml

@@ -19,13 +19,14 @@ concurrency: pr-${{ github.ref }}
 
 jobs:
   # Dependabot is annoying, but this makes it a bit less so.
-  auto-approve:
+  auto-approve-dependabot:
     runs-on: ubuntu-latest
     if: github.event_name == 'pull_request_target'
     permissions:
       pull-requests: write
     steps:
-      - uses: hmarr/auto-approve-action@v3
+      - name: auto-approve dependabot
+        uses: hmarr/auto-approve-action@v3
         if: github.actor == 'dependabot[bot]'
 
   cla:
@@ -33,7 +34,7 @@ jobs:
     steps:
       - name: cla
         if: (github.event.comment.body == 'recheck' || github.event.comment.body == 'I have read the CLA Document and I hereby sign the CLA') || github.event_name == 'pull_request_target'
-        uses: contributor-assistant/github-action@v2.2.1
+        uses: contributor-assistant/github-action@v2.3.1
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           # the below token should have repo scope and must be manually added by you in the repository's secret
@@ -45,27 +46,16 @@ jobs:
           path-to-document: "https://github.com/coder/cla/blob/main/README.md"
           # branch should not be protected
           branch: "main"
-          allowlist: dependabot*
-
-  title:
-    runs-on: ubuntu-latest
-    if: github.event_name == 'pull_request_target'
-    steps:
-      - name: Validate PR title
-        uses: amannn/action-semantic-pull-request@v5
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          requireScope: false
+          # Some users have signed a corporate CLA with Coder so are exempt from signing our community one.
+          allowlist: "coryb,aaronlehmann,dependabot*"
 
   release-labels:
     runs-on: ubuntu-latest
-    # Depend on lint so that title is Conventional Commits-compatible.
-    needs: [title]
     # Skip tagging for draft PRs.
     if: ${{ github.event_name == 'pull_request_target' && success() && !github.event.pull_request.draft }}
     steps:
-      - uses: actions/github-script@v6
+      - name: release-labels
+        uses: actions/github-script@v6
         with:
           # This script ensures PR title and labels are in sync:
           #

.github/workflows/docker-base.yaml

@@ -31,10 +31,11 @@ jobs:
     runs-on: ubuntu-latest
     if: github.repository_owner == 'coder'
     steps:
-      - uses: actions/checkout@v3
+      - name: Checkout
+        uses: actions/checkout@v4
 
       - name: Docker login
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
@@ -53,8 +54,38 @@ jobs:
           project: wl5hnrrkns
           context: base-build-context
           file: scripts/Dockerfile.base
+          platforms: linux/amd64,linux/arm64,linux/arm/v7
           pull: true
           no-cache: true
           push: true
           tags: |
             ghcr.io/coder/coder-base:latest
+
+      - name: Verify that images are pushed properly
+        run: |
+          # retry 10 times with a 5 second delay as the images may not be
+          # available immediately
+          for i in {1..10}; do
+            rc=0
+            raw_manifests=$(docker buildx imagetools inspect --raw ghcr.io/coder/coder-base:latest) || rc=$?
+            if [[ "$rc" -eq 0 ]]; then
+              break
+            fi
+            if [[ "$i" -eq 10 ]]; then
+              echo "Failed to pull manifests after 10 retries"
+              exit 1
+            fi
+            echo "Failed to pull manifests, retrying in 5 seconds"
+            sleep 5
+          done
+
+          manifests=$(
+            echo "$raw_manifests" | \
+              jq -r '.manifests[].platform | .os + "/" + .architecture + (if .variant then "/" + .variant else "" end)'
+          )
+
+          # Verify all 3 platforms are present.
+          set -euxo pipefail
+          echo "$manifests" | grep -q linux/amd64
+          echo "$manifests" | grep -q linux/arm64
+          echo "$manifests" | grep -q linux/arm/v7

.github/workflows/dogfood.yaml

@@ -5,19 +5,29 @@ on:
     branches:
       - main
     paths:
+      - "flake.nix"
+      - "flake.lock"
       - "dogfood/**"
-  pull_request:
-    paths:
-      - "dogfood/**"
+      - ".github/workflows/dogfood.yaml"
+  # Uncomment these lines when testing with CI.
+  # pull_request:
+  #   paths:
+  #     - "flake.nix"
+  #     - "flake.lock"
+  #     - "dogfood/**"
+  #     - ".github/workflows/dogfood.yaml"
   workflow_dispatch:
 
 jobs:
   deploy_image:
-    runs-on: ubuntu-latest
+    runs-on: buildjet-4vcpu-ubuntu-2204
     steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
       - name: Get branch name
         id: branch-name
-        uses: tj-actions/branch-names@v6.4
+        uses: tj-actions/branch-names@v6.5
 
       - name: "Branch name to Docker tag name"
         id: docker-tag-name
@@ -27,42 +37,48 @@ jobs:
           tag=${tag//\//--}
           echo "tag=${tag}" >> $GITHUB_OUTPUT
 
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
+      - name: Install Nix
+        uses: DeterminateSystems/nix-installer-action@v6
 
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+      - name: Run the Magic Nix Cache
+        uses: DeterminateSystems/magic-nix-cache-action@v2
+
+      - run: nix build .#devEnvImage && ./result | docker load
 
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_PASSWORD }}
 
-      - name: Build and push
-        uses: docker/build-push-action@v4
-        with:
-          context: "{{defaultContext}}:dogfood"
-          push: true
-          tags: "codercom/oss-dogfood:${{ steps.docker-tag-name.outputs.tag }},codercom/oss-dogfood:latest"
-          cache-from: type=registry,ref=codercom/oss-dogfood:latest
-          cache-to: type=inline
+      - name: Tag and Push
+        run: |
+          docker tag codercom/oss-dogfood:latest codercom/oss-dogfood:${{ steps.docker-tag-name.outputs.tag }}
+          docker push codercom/oss-dogfood -a
+
   deploy_template:
+    needs: deploy_image
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
+
       - name: Get short commit SHA
         id: vars
         run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
-      - name: "Install latest Coder"
+
+      - name: Get latest commit title
+        id: message
+        run: echo "pr_title=$(git log --format=%s -n 1 ${{ github.sha }})" >> $GITHUB_OUTPUT
+
+      - name: "Get latest Coder binary from the server"
         run: |
-          curl -L https://coder.com/install.sh | sh
-        # env:
-        #   VERSION: 0.x
+          curl -fsSL "https://dev.coder.com/bin/coder-linux-amd64" -o "./coder"
+          chmod +x "./coder"
+
       - name: "Push template"
         run: |
-          coder templates push $CODER_TEMPLATE_NAME --directory $CODER_TEMPLATE_DIR --yes --name=$CODER_TEMPLATE_VERSION
+          ./coder templates push $CODER_TEMPLATE_NAME --directory $CODER_TEMPLATE_DIR --yes --name=$CODER_TEMPLATE_VERSION --message="$CODER_TEMPLATE_MESSAGE"
         env:
           # Consumed by Coder CLI
           CODER_URL: https://dev.coder.com
@@ -71,3 +87,4 @@ jobs:
           CODER_TEMPLATE_NAME: ${{ secrets.CODER_TEMPLATE_NAME }}
           CODER_TEMPLATE_VERSION: ${{ steps.vars.outputs.sha_short }}
           CODER_TEMPLATE_DIR: ./dogfood
+          CODER_TEMPLATE_MESSAGE: ${{ steps.message.outputs.pr_title }}

.github/workflows/mlc_config.json

@@ -18,5 +18,6 @@
     {
       "pattern": "tailscale.com"
     }
-  ]
+  ],
+  "aliveStatusCodes": [200, 0]
 }

.github/workflows/nightly-gauntlet.yaml

@@ -0,0 +1,60 @@
+# The nightly-gauntlet runs tests that are either too flaky or too slow to block
+# every PR.
+name: nightly-gauntlet
+on:
+  schedule:
+    # Every day at midnight
+    - cron: "0 0 * * *"
+  workflow_dispatch:
+jobs:
+  go-race:
+    # While GitHub's toaster runners are likelier to flake, we want consistency
+    # between this environment and the regular test environment for DataDog
+    # statistics and to only show real workflow threats.
+    runs-on: "buildjet-8vcpu-ubuntu-2204"
+    # This runner costs 0.016 USD per minute,
+    # so 0.016 * 240 = 3.84 USD per run.
+    timeout-minutes: 240
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Go
+        uses: ./.github/actions/setup-go
+
+      - name: Setup Terraform
+        uses: ./.github/actions/setup-tf
+
+      - name: Run Tests
+        run: |
+          # -race is likeliest to catch flaky tests
+          # due to correctness detection and its performance
+          # impact.
+          gotestsum --junitfile="gotests.xml" -- -timeout=240m -count=10 -race ./...
+
+      - name: Upload test results to DataDog
+        uses: ./.github/actions/upload-datadog
+        if: always()
+        with:
+          api-key: ${{ secrets.DATADOG_API_KEY }}
+
+  go-timing:
+    # We run these tests with p=1 so we don't need a lot of compute.
+    runs-on: "buildjet-2vcpu-ubuntu-2204"
+    timeout-minutes: 10
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Go
+        uses: ./.github/actions/setup-go
+
+      - name: Run Tests
+        run: |
+          gotestsum --junitfile="gotests.xml" -- --tags="timing" -p=1 -run='_Timing/' ./...
+
+      - name: Upload test results to DataDog
+        uses: ./.github/actions/upload-datadog
+        if: always()
+        with:
+          api-key: ${{ secrets.DATADOG_API_KEY }}

.github/workflows/pr-auto-assign.yaml

@@ -0,0 +1,17 @@
+# Filtering pull requests is much easier when we can reliably guarantee
+# that the "Assignee" field is populated.
+name: PR Auto Assign
+
+on:
+  pull_request_target:
+    types: [opened]
+
+permissions:
+  pull-requests: write
+
+jobs:
+  assign-author:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Assign author
+        uses: toshimaru/auto-author-assign@v2.0.1

.github/workflows/pr-cleanup.yaml

@@ -0,0 +1,73 @@
+name: pr-cleanup
+on:
+  pull_request:
+    types: closed
+  workflow_dispatch:
+    inputs:
+      pr_number:
+        description: "PR number"
+        required: true
+
+permissions:
+  packages: write
+
+jobs:
+  cleanup:
+    runs-on: "ubuntu-latest"
+    steps:
+      - name: Get PR number
+        id: pr_number
+        run: |
+          if [ -n "${{ github.event.pull_request.number }}" ]; then
+            echo "PR_NUMBER=${{ github.event.pull_request.number }}" >> $GITHUB_OUTPUT
+          else
+            echo "PR_NUMBER=${{ github.event.inputs.pr_number }}" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Delete image
+        continue-on-error: true
+        uses: bots-house/ghcr-delete-image-action@v1.1.0
+        with:
+          owner: coder
+          name: coder-preview
+          token: ${{ secrets.GITHUB_TOKEN }}
+          tag: pr${{ steps.pr_number.outputs.PR_NUMBER }}
+
+      - name: Set up kubeconfig
+        run: |
+          set -euo pipefail
+          mkdir -p ~/.kube
+          echo "${{ secrets.PR_DEPLOYMENTS_KUBECONFIG }}" > ~/.kube/config
+          export KUBECONFIG=~/.kube/config
+
+      - name: Delete helm release
+        run: |
+          set -euo pipefail
+          helm delete --namespace "pr${{ steps.pr_number.outputs.PR_NUMBER }}" "pr${{ steps.pr_number.outputs.PR_NUMBER }}" || echo "helm release not found"
+
+      - name: "Remove PR namespace"
+        run: |
+          kubectl delete namespace "pr${{ steps.pr_number.outputs.PR_NUMBER }}" || echo "namespace not found"
+
+      - name: "Remove DNS records"
+        run: |
+          set -euo pipefail
+          # Get identifier for the record
+          record_id=$(curl -X GET "https://api.cloudflare.com/client/v4/zones/${{ secrets.PR_DEPLOYMENTS_ZONE_ID }}/dns_records?name=%2A.pr${{ steps.pr_number.outputs.PR_NUMBER }}.${{ secrets.PR_DEPLOYMENTS_DOMAIN }}" \
+          -H "Authorization: Bearer ${{ secrets.PR_DEPLOYMENTS_CLOUDFLARE_API_TOKEN }}" \
+          -H "Content-Type:application/json" | jq -r '.result[0].id') || echo "DNS record not found"
+
+          echo "::add-mask::$record_id"
+
+          # Delete the record
+          (
+            curl -X DELETE "https://api.cloudflare.com/client/v4/zones/${{ secrets.PR_DEPLOYMENTS_ZONE_ID }}/dns_records/$record_id" \
+            -H "Authorization: Bearer ${{ secrets.PR_DEPLOYMENTS_CLOUDFLARE_API_TOKEN }}" \
+            -H "Content-Type:application/json" | jq -r '.success'
+          ) || echo "DNS record not found"
+
+      - name: "Delete certificate"
+        if: ${{ github.event.pull_request.merged == true }}
+        run: |
+          set -euxo pipefail
+          kubectl delete certificate "pr${{ steps.pr_number.outputs.PR_NUMBER }}-tls" -n pr-deployment-certs || echo "certificate not found"

.github/workflows/pr-deploy.yaml

@@ -0,0 +1,470 @@
+# This action will trigger when
+# 1. when the workflow is manually triggered
+# 2. ./scripts/deploy_pr.sh is run locally
+# 3. when a PR is updated
+name: Deploy PR
+on:
+  push:
+    branches-ignore:
+      - main
+  workflow_dispatch:
+    inputs:
+      pr_number:
+        description: "PR number"
+        type: number
+        required: true
+      experiments:
+        description: "Experiments to enable"
+        required: false
+        type: string
+        default: "*"
+      build:
+        description: "Force new build"
+        required: false
+        type: boolean
+        default: false
+      deploy:
+        description: "Force new deployment"
+        required: false
+        type: boolean
+        default: false
+
+env:
+  REPO: ghcr.io/coder/coder-preview
+
+permissions:
+  contents: read
+  packages: write
+  pull-requests: write # needed for commenting on PRs
+
+jobs:
+  check_pr:
+    runs-on: ubuntu-latest
+    outputs:
+      PR_OPEN: ${{ steps.check_pr.outputs.pr_open }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Check if PR is open
+        id: check_pr
+        run: |
+          set -euo pipefail
+          pr_open=true
+          if [[ "$(gh pr view --json state | jq -r '.state')" != "OPEN" ]]; then
+            echo "PR doesn't exist or is closed."
+            pr_open=false
+          fi
+          echo "pr_open=$pr_open" >> $GITHUB_OUTPUT
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+  get_info:
+    needs: check_pr
+    if: ${{ needs.check_pr.outputs.PR_OPEN == 'true' }}
+    outputs:
+      PR_NUMBER: ${{ steps.pr_info.outputs.PR_NUMBER }}
+      PR_TITLE: ${{ steps.pr_info.outputs.PR_TITLE }}
+      PR_URL: ${{ steps.pr_info.outputs.PR_URL }}
+      CODER_BASE_IMAGE_TAG: ${{ steps.set_tags.outputs.CODER_BASE_IMAGE_TAG }}
+      CODER_IMAGE_TAG: ${{ steps.set_tags.outputs.CODER_IMAGE_TAG }}
+      NEW: ${{ steps.check_deployment.outputs.NEW }}
+      BUILD: ${{ steps.build_conditionals.outputs.first_or_force_build == 'true' || steps.build_conditionals.outputs.automatic_rebuild == 'true' }}
+
+    runs-on: "ubuntu-latest"
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Get PR number, title, and branch name
+        id: pr_info
+        run: |
+          set -euo pipefail
+          PR_NUMBER=$(gh pr view --json number | jq -r '.number')
+          PR_TITLE=$(gh pr view --json title | jq -r '.title')
+          PR_URL=$(gh pr view --json url | jq -r '.url')
+          echo "PR_URL=$PR_URL" >> $GITHUB_OUTPUT
+          echo "PR_NUMBER=$PR_NUMBER" >> $GITHUB_OUTPUT
+          echo "PR_TITLE=$PR_TITLE" >> $GITHUB_OUTPUT
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set required tags
+        id: set_tags
+        run: |
+          set -euo pipefail
+          echo "CODER_BASE_IMAGE_TAG=$CODER_BASE_IMAGE_TAG" >> $GITHUB_OUTPUT
+          echo "CODER_IMAGE_TAG=$CODER_IMAGE_TAG" >> $GITHUB_OUTPUT
+        env:
+          CODER_BASE_IMAGE_TAG: ghcr.io/coder/coder-preview-base:pr${{ steps.pr_info.outputs.PR_NUMBER }}
+          CODER_IMAGE_TAG: ghcr.io/coder/coder-preview:pr${{ steps.pr_info.outputs.PR_NUMBER }}
+
+      - name: Set up kubeconfig
+        run: |
+          set -euo pipefail
+          mkdir -p ~/.kube
+          echo "${{ secrets.PR_DEPLOYMENTS_KUBECONFIG }}" > ~/.kube/config
+          chmod 644 ~/.kube/config
+          export KUBECONFIG=~/.kube/config
+
+      - name: Check if the helm deployment already exists
+        id: check_deployment
+        run: |
+          set -euo pipefail
+          if helm status "pr${{ steps.pr_info.outputs.PR_NUMBER }}" --namespace "pr${{ steps.pr_info.outputs.PR_NUMBER }}" > /dev/null 2>&1; then
+            echo "Deployment already exists. Skipping deployment."
+            NEW=false
+          else
+            echo "Deployment doesn't exist."
+            NEW=true
+          fi
+          echo "NEW=$NEW" >> $GITHUB_OUTPUT
+
+      - name: Check changed files
+        uses: dorny/paths-filter@v2
+        id: filter
+        with:
+          base: ${{ github.ref }}
+          filters: |
+            all:
+              - "**"
+            ignored:
+              - "docs/**"
+              - "README.md"
+              - "examples/web-server/**"
+              - "examples/monitoring/**"
+              - "examples/lima/**"
+              - ".github/**"
+              - "offlinedocs/**"
+              - ".devcontainer/**"
+              - "helm/**"
+              - "*[^g][^o][^.][^s][^u][^m]*"
+              - "*[^g][^o][^.][^m][^o][^d]*"
+              - "*[^M][^a][^k][^e][^f][^i][^l][^e]*"
+              - "scripts/**/*[^D][^o][^c][^k][^e][^r][^f][^i][^l][^e]*"
+              - "scripts/**/*[^D][^o][^c][^k][^e][^r][^f][^i][^l][^e][.][b][^a][^s][^e]*"
+
+      - name: Print number of changed files
+        run: |
+          set -euo pipefail
+          echo "Total number of changed files: ${{ steps.filter.outputs.all_count }}"
+          echo "Number of ignored files: ${{ steps.filter.outputs.ignored_count }}"
+
+      - name: Build conditionals
+        id: build_conditionals
+        run: |
+          set -euo pipefail
+          # build if the workflow is manually triggered and the deployment doesn't exist (first build or force rebuild)
+          echo "first_or_force_build=${{ (github.event_name == 'workflow_dispatch' && steps.check_deployment.outputs.NEW == 'true') || github.event.inputs.build == 'true' }}" >> $GITHUB_OUTPUT
+          # build if the deployment alreday exist and there are changes in the files that we care about (automatic updates)
+          echo "automatic_rebuild=${{ steps.check_deployment.outputs.NEW == 'false' && steps.filter.outputs.all_count > steps.filter.outputs.ignored_count }}" >> $GITHUB_OUTPUT
+
+  comment-pr:
+    needs: get_info
+    if: needs.get_info.outputs.BUILD == 'true' || github.event.inputs.deploy == 'true'
+    runs-on: "ubuntu-latest"
+    steps:
+      - name: Find Comment
+        uses: peter-evans/find-comment@v2
+        id: fc
+        with:
+          issue-number: ${{ needs.get_info.outputs.PR_NUMBER }}
+          comment-author: "github-actions[bot]"
+          body-includes: ":rocket:"
+          direction: last
+
+      - name: Comment on PR
+        id: comment_id
+        uses: peter-evans/create-or-update-comment@v3
+        with:
+          comment-id: ${{ steps.fc.outputs.comment-id }}
+          issue-number: ${{ needs.get_info.outputs.PR_NUMBER }}
+          edit-mode: replace
+          body: |
+            ---
+            :rocket: Deploying PR ${{ needs.get_info.outputs.PR_NUMBER }} ...
+            ---
+          reactions: eyes
+          reactions-edit-mode: replace
+
+  build:
+    needs: get_info
+    # Run build job only if there are changes in the files that we care about or if the workflow is manually triggered with --build flag
+    if: needs.get_info.outputs.BUILD == 'true'
+    runs-on: ${{ github.repository_owner == 'coder' && 'buildjet-8vcpu-ubuntu-2204' || 'ubuntu-latest' }}
+    # This concurrency only cancels build jobs if a new build is triggred. It will avoid cancelling the current deployemtn in case of docs chnages.
+    concurrency:
+      group: build-${{ github.workflow }}-${{ github.ref }}-${{ needs.get_info.outputs.BUILD }}
+      cancel-in-progress: true
+    env:
+      DOCKER_CLI_EXPERIMENTAL: "enabled"
+      CODER_IMAGE_TAG: ${{ needs.get_info.outputs.CODER_IMAGE_TAG }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Setup Node
+        uses: ./.github/actions/setup-node
+
+      - name: Setup Go
+        uses: ./.github/actions/setup-go
+
+      - name: Setup sqlc
+        uses: ./.github/actions/setup-sqlc
+
+      - name: GHCR Login
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push Linux amd64 Docker image
+        run: |
+          set -euo pipefail
+          go mod download
+          make gen/mark-fresh
+          export DOCKER_IMAGE_NO_PREREQUISITES=true
+          version="$(./scripts/version.sh)"
+          export CODER_IMAGE_BUILD_BASE_TAG="$(CODER_IMAGE_BASE=coder-base ./scripts/image_tag.sh --version "$version")"
+          make -j build/coder_linux_amd64
+          ./scripts/build_docker.sh \
+            --arch amd64 \
+            --target ${{ env.CODER_IMAGE_TAG }} \
+            --version $version \
+            --push \
+            build/coder_linux_amd64
+
+  deploy:
+    needs: [build, get_info]
+    # Run deploy job only if build job was successful or skipped
+    if: |
+      always() && (needs.build.result == 'success' || needs.build.result == 'skipped') &&
+      (needs.get_info.outputs.BUILD == 'true' || github.event.inputs.deploy == 'true')
+    runs-on: "ubuntu-latest"
+    env:
+      CODER_IMAGE_TAG: ${{ needs.get_info.outputs.CODER_IMAGE_TAG }}
+      PR_NUMBER: ${{ needs.get_info.outputs.PR_NUMBER }}
+      PR_TITLE: ${{ needs.get_info.outputs.PR_TITLE }}
+      PR_URL: ${{ needs.get_info.outputs.PR_URL }}
+      PR_HOSTNAME: "pr${{ needs.get_info.outputs.PR_NUMBER }}.${{ secrets.PR_DEPLOYMENTS_DOMAIN }}"
+    steps:
+      - name: Set up kubeconfig
+        run: |
+          set -euo pipefail
+          mkdir -p ~/.kube
+          echo "${{ secrets.PR_DEPLOYMENTS_KUBECONFIG }}" > ~/.kube/config
+          chmod 644 ~/.kube/config
+          export KUBECONFIG=~/.kube/config
+
+      - name: Check if image exists
+        run: |
+          set -euo pipefail
+          foundTag=$(
+            gh api /orgs/coder/packages/container/coder-preview/versions |
+            jq -r --arg tag "pr${{ env.PR_NUMBER }}" '.[] |
+            select(.metadata.container.tags == [$tag]) |
+            .metadata.container.tags[0]'
+          )
+          if [ -z "$foundTag" ]; then
+            echo "Image not found"
+            echo "${{ env.CODER_IMAGE_TAG }} not found in ghcr.io/coder/coder-preview"
+            exit 1
+          else
+            echo "Image found"
+            echo "$foundTag tag found in ghcr.io/coder/coder-preview"
+          fi
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Add DNS record to Cloudflare
+        if: needs.get_info.outputs.NEW == 'true'
+        run: |
+          curl -X POST "https://api.cloudflare.com/client/v4/zones/${{ secrets.PR_DEPLOYMENTS_ZONE_ID }}/dns_records" \
+            -H "Authorization: Bearer ${{ secrets.PR_DEPLOYMENTS_CLOUDFLARE_API_TOKEN }}" \
+            -H "Content-Type:application/json" \
+            --data '{"type":"CNAME","name":"*.${{ env.PR_HOSTNAME }}","content":"${{ env.PR_HOSTNAME }}","ttl":1,"proxied":false}'
+
+      - name: Create PR namespace
+        if: needs.get_info.outputs.NEW == 'true' || github.event.inputs.deploy == 'true'
+        run: |
+          set -euo pipefail
+          # try to delete the namespace, but don't fail if it doesn't exist
+          kubectl delete namespace "pr${{ env.PR_NUMBER }}" || true
+          kubectl create namespace "pr${{ env.PR_NUMBER }}"
+
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Check and Create Certificate
+        if: needs.get_info.outputs.NEW == 'true' || github.event.inputs.deploy == 'true'
+        run: |
+          # Using kubectl to check if a Certificate resource already exists
+          # we are doing this to avoid letsenrypt rate limits
+          if ! kubectl get certificate pr${{ env.PR_NUMBER }}-tls -n pr-deployment-certs > /dev/null 2>&1; then
+            echo "Certificate doesn't exist. Creating a new one."
+            envsubst < ./.github/pr-deployments/certificate.yaml | kubectl apply -f -
+          else
+            echo "Certificate exists. Skipping certificate creation."
+          fi
+          echo "Copy certificate from pr-deployment-certs to pr${{ env.PR_NUMBER }} namespace"
+          until kubectl get secret pr${{ env.PR_NUMBER }}-tls -n pr-deployment-certs &> /dev/null
+          do
+            echo "Waiting for secret pr${{ env.PR_NUMBER }}-tls to be created..."
+            sleep 5
+          done
+          (
+            kubectl get secret pr${{ env.PR_NUMBER }}-tls -n pr-deployment-certs -o json |
+            jq 'del(.metadata.namespace,.metadata.creationTimestamp,.metadata.resourceVersion,.metadata.selfLink,.metadata.uid,.metadata.managedFields)' |
+            kubectl -n pr${{ env.PR_NUMBER }} apply -f -
+          )
+
+      - name: Set up PostgreSQL database
+        if: needs.get_info.outputs.NEW == 'true' || github.event.inputs.deploy == 'true'
+        run: |
+          helm repo add bitnami https://charts.bitnami.com/bitnami
+          helm install coder-db bitnami/postgresql \
+            --namespace pr${{ env.PR_NUMBER }} \
+            --set auth.username=coder \
+            --set auth.password=coder \
+            --set auth.database=coder \
+            --set persistence.size=10Gi
+          kubectl create secret generic coder-db-url -n pr${{ env.PR_NUMBER }} \
+            --from-literal=url="postgres://coder:coder@coder-db-postgresql.pr${{ env.PR_NUMBER }}.svc.cluster.local:5432/coder?sslmode=disable"
+
+      - name: Create a service account, role, and rolebinding for the PR namespace
+        if: needs.get_info.outputs.NEW == 'true' || github.event.inputs.deploy == 'true'
+        run: |
+          set -euo pipefail
+          # Create service account, role, rolebinding
+          envsubst < ./.github/pr-deployments/rbac.yaml | kubectl apply -f -
+
+      - name: Create values.yaml
+        env:
+          EXPERIMENTS: ${{ github.event.inputs.experiments }}
+          PR_DEPLOYMENTS_GITHUB_OAUTH_CLIENT_ID: ${{ secrets.PR_DEPLOYMENTS_GITHUB_OAUTH_CLIENT_ID }}
+          PR_DEPLOYMENTS_GITHUB_OAUTH_CLIENT_SECRET: ${{ secrets.PR_DEPLOYMENTS_GITHUB_OAUTH_CLIENT_SECRET }}
+        run: |
+          set -euo pipefail
+          envsubst < ./.github/pr-deployments/values.yaml > ./pr-deploy-values.yaml
+
+      - name: Install/Upgrade Helm chart
+        run: |
+          set -euo pipefail
+          helm upgrade --install "pr${{ env.PR_NUMBER }}" ./helm/coder \
+          --namespace "pr${{ env.PR_NUMBER }}" \
+          --values ./pr-deploy-values.yaml \
+          --force
+
+      - name: Install coder-logstream-kube
+        if: needs.get_info.outputs.NEW == 'true' || github.event.inputs.deploy == 'true'
+        run: |
+          helm repo add coder-logstream-kube https://helm.coder.com/logstream-kube
+          helm upgrade --install coder-logstream-kube coder-logstream-kube/coder-logstream-kube \
+            --namespace "pr${{ env.PR_NUMBER }}" \
+            --set url="https://${{ env.PR_HOSTNAME }}"
+
+      - name: Get Coder binary
+        if: needs.get_info.outputs.NEW == 'true' || github.event.inputs.deploy == 'true'
+        run: |
+          set -euo pipefail
+
+          DEST="${HOME}/coder"
+          URL="https://${{ env.PR_HOSTNAME }}/bin/coder-linux-amd64"
+
+          mkdir -p "$(dirname ${DEST})"
+
+          COUNT=0
+          until $(curl --output /dev/null --silent --head --fail "$URL"); do
+              printf '.'
+              sleep 5
+              COUNT=$((COUNT+1))
+              if [ $COUNT -ge 60 ]; then
+                echo "Timed out waiting for URL to be available"
+                exit 1
+              fi
+          done
+
+          curl -fsSL "$URL" -o "${DEST}"
+          chmod +x "${DEST}"
+          "${DEST}" version
+          mv "${DEST}" /usr/local/bin/coder
+
+      - name: Create first user, template and workspace
+        if: needs.get_info.outputs.NEW == 'true' || github.event.inputs.deploy == 'true'
+        id: setup_deployment
+        run: |
+          set -euo pipefail
+
+          # Create first user
+
+          # create a masked random password 12 characters long
+          password=$(openssl rand -base64 16 | tr -d "=+/" | cut -c1-12)
+
+          # add mask so that the password is not printed to the logs
+          echo "::add-mask::$password"
+          echo "password=$password" >> $GITHUB_OUTPUT
+
+          coder login \
+            --first-user-username coder \
+            --first-user-email pr${{ env.PR_NUMBER }}@coder.com \
+            --first-user-password $password \
+            --first-user-trial \
+            --use-token-as-session \
+            https://${{ env.PR_HOSTNAME }}
+
+          # Create template
+          cd ./.github/pr-deployments/template
+          coder templates create -y --variable namespace=pr${{ env.PR_NUMBER }} kubernetes
+
+          # Create workspace
+          coder create --template="kubernetes" kube --parameter cpu=2 --parameter memory=4 --parameter home_disk_size=2 -y
+          coder stop kube -y
+
+      - name: Send Slack notification
+        if: needs.get_info.outputs.NEW == 'true' || github.event.inputs.deploy == 'true'
+        run: |
+          curl -s -o /dev/null -X POST -H 'Content-type: application/json' \
+            -d \
+            '{
+              "pr_number": "'"${{ env.PR_NUMBER }}"'",
+              "pr_url": "'"${{ env.PR_URL }}"'",
+              "pr_title": "'"${{ env.PR_TITLE }}"'",
+              "pr_access_url": "'"https://${{ env.PR_HOSTNAME }}"'",
+              "pr_username": "'"test"'",
+              "pr_email": "'"pr${{ env.PR_NUMBER }}@coder.com"'",
+              "pr_password": "'"${{ steps.setup_deployment.outputs.password }}"'",
+              "pr_actor": "'"${{ github.actor }}"'"
+            }' \
+            ${{ secrets.PR_DEPLOYMENTS_SLACK_WEBHOOK }}
+          echo "Slack notification sent"
+
+      - name: Find Comment
+        uses: peter-evans/find-comment@v2
+        id: fc
+        with:
+          issue-number: ${{ env.PR_NUMBER }}
+          comment-author: "github-actions[bot]"
+          body-includes: ":rocket:"
+          direction: last
+
+      - name: Comment on PR
+        uses: peter-evans/create-or-update-comment@v3
+        env:
+          STATUS: ${{ needs.get_info.outputs.NEW == 'true' && 'Created' || 'Updated' }}
+        with:
+          issue-number: ${{ env.PR_NUMBER }}
+          edit-mode: replace
+          comment-id: ${{ steps.fc.outputs.comment-id }}
+          body: |
+            ---
+            :heavy_check_mark: PR ${{ env.PR_NUMBER }} ${{ env.STATUS }} successfully.
+            :rocket: Access the credentials [here](${{ secrets.PR_DEPLOYMENTS_SLACK_CHANNEL_URL }}).
+            ---
+            cc: @${{ github.actor }}
+          reactions: rocket
+          reactions-edit-mode: replace

.github/workflows/release.yaml

@@ -32,14 +32,15 @@ env:
 jobs:
   release:
     name: Build and publish
-    runs-on: ${{ github.repository_owner == 'coder' && 'ubuntu-latest-8-cores' || 'ubuntu-latest' }}
+    runs-on: ${{ github.repository_owner == 'coder' && 'buildjet-8vcpu-ubuntu-2204' || 'ubuntu-latest' }}
     env:
       # Necessary for Docker manifest
       DOCKER_CLI_EXPERIMENTAL: "enabled"
     outputs:
       version: ${{ steps.version.outputs.version }}
     steps:
-      - uses: actions/checkout@v3
+      - name: Checkout
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
@@ -71,11 +72,11 @@ jobs:
           set -euo pipefail
           ref=HEAD
           old_version="$(git describe --abbrev=0 "$ref^1")"
-          version="$(./scripts/version.sh)"
+          version="v$(./scripts/version.sh)"
 
           # Generate notes.
           release_notes_file="$(mktemp -t release_notes.XXXXXX)"
-          ./scripts/release/generate_release_notes.sh --old-version "$old_version" --new-version "$version" --ref "$ref" >> "$release_notes_file"
+          ./scripts/release/generate_release_notes.sh --check-for-changelog --old-version "$old_version" --new-version "$version" --ref "$ref" >> "$release_notes_file"
           echo CODER_RELEASE_NOTES_FILE="$release_notes_file" >> $GITHUB_ENV
 
       - name: Show release notes
@@ -84,26 +85,17 @@ jobs:
           cat "$CODER_RELEASE_NOTES_FILE"
 
       - name: Docker Login
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - uses: actions/setup-go@v3
-        with:
-          go-version: "~1.20"
+      - name: Setup Go
+        uses: ./.github/actions/setup-go
 
-      - name: Cache Node
-        id: cache-node
-        uses: actions/cache@v3
-        with:
-          path: |
-            **/node_modules
-            .eslintcache
-          key: js-${{ runner.os }}-test-${{ hashFiles('**/yarn.lock') }}
-          restore-keys: |
-            js-${{ runner.os }}-
+      - name: Setup Node
+        uses: ./.github/actions/setup-node
 
       - name: Install nsis and zstd
         run: sudo apt-get install -y nsis zstd
@@ -149,7 +141,8 @@ jobs:
             build/coder_"$version"_linux_{amd64,armv7,arm64}.{tar.gz,apk,deb,rpm} \
             build/coder_"$version"_{darwin,windows}_{amd64,arm64}.zip \
             build/coder_"$version"_windows_amd64_installer.exe \
-            build/coder_helm_"$version".tgz
+            build/coder_helm_"$version".tgz \
+            build/provisioner_helm_"$version".tgz
         env:
           CODER_SIGN_DARWIN: "1"
           AC_CERTIFICATE_FILE: /tmp/apple_cert.p12
@@ -188,12 +181,42 @@ jobs:
           project: wl5hnrrkns
           context: base-build-context
           file: scripts/Dockerfile.base
+          platforms: linux/amd64,linux/arm64,linux/arm/v7
           pull: true
           no-cache: true
           push: true
           tags: |
             ${{ steps.image-base-tag.outputs.tag }}
 
+      - name: Verify that images are pushed properly
+        run: |
+          # retry 10 times with a 5 second delay as the images may not be
+          # available immediately
+          for i in {1..10}; do
+            rc=0
+            raw_manifests=$(docker buildx imagetools inspect --raw "${{ steps.image-base-tag.outputs.tag }}") || rc=$?
+            if [[ "$rc" -eq 0 ]]; then
+              break
+            fi
+            if [[ "$i" -eq 10 ]]; then
+              echo "Failed to pull manifests after 10 retries"
+              exit 1
+            fi
+            echo "Failed to pull manifests, retrying in 5 seconds"
+            sleep 5
+          done
+
+          manifests=$(
+            echo "$raw_manifests" | \
+              jq -r '.manifests[].platform | .os + "/" + .architecture + (if .variant then "/" + .variant else "" end)'
+          )
+
+          # Verify all 3 platforms are present.
+          set -euxo pipefail
+          echo "$manifests" | grep -q linux/amd64
+          echo "$manifests" | grep -q linux/arm64
+          echo "$manifests" | grep -q linux/arm/v7
+
       - name: Build Linux Docker images
         run: |
           set -euxo pipefail
@@ -225,6 +248,11 @@ jobs:
         env:
           CODER_BASE_IMAGE_TAG: ${{ steps.image-base-tag.outputs.tag }}
 
+      - name: Generate offline docs
+        run: |
+          version="$(./scripts/version.sh)"
+          make -j build/coder_docs_"$version".tgz
+
       - name: ls build
         run: ls -lh build
 
@@ -268,14 +296,17 @@ jobs:
           version="$(./scripts/version.sh)"
           mkdir -p build/helm
           cp "build/coder_helm_${version}.tgz" build/helm
+          cp "build/provisioner_helm_${version}.tgz" build/helm
           gsutil cp gs://helm.coder.com/v2/index.yaml build/helm/index.yaml
           helm repo index build/helm --url https://helm.coder.com/v2 --merge build/helm/index.yaml
           gsutil -h "Cache-Control:no-cache,max-age=0" cp build/helm/coder_helm_${version}.tgz gs://helm.coder.com/v2
+          gsutil -h "Cache-Control:no-cache,max-age=0" cp build/helm/provisioner_helm_${version}.tgz gs://helm.coder.com/v2
           gsutil -h "Cache-Control:no-cache,max-age=0" cp build/helm/index.yaml gs://helm.coder.com/v2
+          gsutil -h "Cache-Control:no-cache,max-age=0" cp helm/artifacthub-repo.yml gs://helm.coder.com/v2
 
       - name: Upload artifacts to actions (if dry-run)
         if: ${{ inputs.dry_run }}
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: release-artifacts
           path: |
@@ -297,12 +328,88 @@ jobs:
           event-type: coder-release
           client-payload: '{"coder_version": "${{ steps.version.outputs.version }}"}'
 
+  publish-homebrew:
+    name: Publish to Homebrew tap
+    runs-on: ubuntu-latest
+    needs: release
+    if: ${{ !inputs.dry_run }}
+
+    steps:
+      # TODO: skip this if it's not a new release (i.e. a backport). This is
+      #       fine right now because it just makes a PR that we can close.
+      - name: Update homebrew
+        env:
+          # Variables used by the `gh` command
+          GH_REPO: coder/homebrew-coder
+          GH_TOKEN: ${{ secrets.CDRCI_GITHUB_TOKEN }}
+        run: |
+          # Keep version number around for reference, removing any potential leading v
+          coder_version="$(echo "${{ needs.release.outputs.version }}" | tr -d v)"
+
+          set -euxo pipefail
+
+          # Setup Git
+          git config --global user.email "ci@coder.com"
+          git config --global user.name "Coder CI"
+          git config --global credential.helper "store"
+
+          temp_dir="$(mktemp -d)"
+          cd "$temp_dir"
+
+          # Download checksums
+          checksums_url="$(gh release view --repo coder/coder "v$coder_version" --json assets \
+            | jq -r ".assets | map(.url) | .[]" \
+            | grep -e ".checksums.txt\$")"
+          wget "$checksums_url" -O checksums.txt
+
+          # Get the SHAs
+          darwin_arm_sha="$(cat checksums.txt | grep "darwin_arm64.zip" | awk '{ print $1 }')"
+          darwin_intel_sha="$(cat checksums.txt | grep "darwin_amd64.zip" | awk '{ print $1 }')"
+          linux_sha="$(cat checksums.txt | grep "linux_amd64.tar.gz" | awk '{ print $1 }')"
+
+          echo "macOS arm64: $darwin_arm_sha"
+          echo "macOS amd64: $darwin_intel_sha"
+          echo "Linux amd64: $linux_sha"
+
+          # Check out the homebrew repo
+          git clone "https://github.com/$GH_REPO" homebrew-coder
+          brew_branch="auto-release/$coder_version"
+          cd homebrew-coder
+
+          # Check if a PR already exists.
+          pr_count="$(gh pr list --search "head:$brew_branch" --json id,closed | jq -r ".[] | select(.closed == false) | .id" | wc -l)"
+          if [[ "$pr_count" > 0 ]]; then
+            echo "Bailing out as PR already exists" 2>&1
+            exit 0
+          fi
+
+          # Set up cdrci credentials for pushing to homebrew-coder
+          echo "https://x-access-token:$GH_TOKEN@github.com" >> ~/.git-credentials
+          # Update the formulae and push
+          git checkout -b "$brew_branch"
+          ./scripts/update-v2.sh "$coder_version" "$darwin_arm_sha" "$darwin_intel_sha" "$linux_sha"
+          git add .
+          git commit -m "coder $coder_version"
+          git push -u origin -f "$brew_branch"
+
+          # Create PR
+          gh pr create \
+            -B master -H "$brew_branch" \
+            -t "coder $coder_version" \
+            -b "" \
+            -r "${{ github.actor }}" \
+            -a "${{ github.actor }}" \
+            -b "This automatic PR was triggered by the release of Coder v$coder_version"
+
   publish-winget:
     name: Publish to winget-pkgs
     runs-on: windows-latest
     needs: release
+    if: ${{ !inputs.dry_run }}
+
     steps:
-      - uses: actions/checkout@v3
+      - name: Checkout
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
@@ -335,12 +442,6 @@ jobs:
           echo "Installer URL: ${installer_url}"
           echo "Package version: ${version}"
 
-          # Bail if dry-run.
-          if ($env:CODER_DRY_RUN -match "t") {
-            echo "Skipping submission due to dry-run."
-            exit 0
-          }
-
           # The URL "|X64" suffix forces the architecture as it cannot be
           # sniffed properly from the URL. wingetcreate checks both the URL and
           # binary magic bytes for the architecture and they need to both match,
@@ -364,8 +465,9 @@ jobs:
           WINGET_GH_TOKEN: ${{ secrets.CDRCI_GITHUB_TOKEN }}
 
       - name: Comment on PR
-        if: ${{ !inputs.dry_run }}
         run: |
+          # wait 30 seconds
+          Start-Sleep -Seconds 30.0
           # Find the PR that wingetcreate just made.
           $version = "${{ needs.release.outputs.version }}".Trim('v')
           $pr_list = gh pr list --repo microsoft/winget-pkgs --search "author:cdrci Coder.Coder version ${version}" --limit 1 --json number | `
@@ -378,3 +480,66 @@ jobs:
           # For gh CLI. We need a real token since we're commenting on a PR in a
           # different repo.
           GH_TOKEN: ${{ secrets.CDRCI_GITHUB_TOKEN }}
+
+  publish-chocolatey:
+    name: Publish to Chocolatey
+    runs-on: windows-latest
+    needs: release
+    if: ${{ !inputs.dry_run }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      # Same reason as for release.
+      - name: Fetch git tags
+        run: git fetch --tags --force
+
+      # From https://chocolatey.org
+      - name: Install Chocolatey
+        run: |
+          Set-ExecutionPolicy Bypass -Scope Process -Force
+          [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072
+
+          iex ((New-Object System.Net.WebClient).DownloadString('https://community.chocolatey.org/install.ps1'))
+
+      - name: Build chocolatey package
+        run: |
+          cd scripts/chocolatey
+
+          # The package version is the same as the tag minus the leading "v".
+          # The version in this output already has the leading "v" removed but
+          # we do it again to be safe.
+          $version = "${{ needs.release.outputs.version }}".Trim('v')
+
+          $release_assets = gh release view --repo coder/coder "v${version}" --json assets | `
+            ConvertFrom-Json
+
+          # Get the URL for the Windows ZIP from the release assets.
+          $zip_url = $release_assets.assets | `
+            Where-Object name -Match ".*_windows_amd64.zip$" | `
+            Select -ExpandProperty url
+
+          echo "ZIP URL: ${zip_url}"
+          echo "Package version: ${version}"
+
+          echo "Downloading ZIP..."
+          Invoke-WebRequest $zip_url -OutFile assets.zip
+
+          echo "Extracting ZIP..."
+          Expand-Archive assets.zip -DestinationPath assets/
+
+          # No need to specify nuspec if there's only one in the directory.
+          choco pack --version=$version binary_path=assets/coder.exe
+
+          choco apikey --api-key $env:CHOCO_API_KEY --source https://push.chocolatey.org/
+
+          # No need to specify nupkg if there's only one in the directory.
+          choco push --source https://push.chocolatey.org/
+
+        env:
+          CHOCO_API_KEY: ${{ secrets.CHOCO_API_KEY }}
+          # We need a GitHub token for the gh CLI to function under GitHub Actions
+          GH_TOKEN: ${{ secrets.CDRCI_GITHUB_TOKEN }}

.github/workflows/security.yaml

@@ -6,17 +6,14 @@ permissions:
   security-events: write
 
 on:
-  push:
-    branches: ["main"]
-
-  pull_request:
-    branches: ["main"]
-
   workflow_dispatch:
 
+  # Uncomment when testing.
+  # pull_request:
+
   schedule:
-    # Run every week at 10:24 on Thursday.
-    - cron: "24 10 * * 4"
+    # Run every 6 hours Monday-Friday!
+    - cron: "0 0/6 * * 1-5"
 
 # Cancel in-progress runs for pull requests when developers push
 # additional changes
@@ -26,9 +23,10 @@ concurrency:
 
 jobs:
   codeql:
-    runs-on: ${{ github.repository_owner == 'coder' && 'ubuntu-latest-8-cores' || 'ubuntu-latest' }}
+    runs-on: ${{ github.repository_owner == 'coder' && 'buildjet-8vcpu-ubuntu-2204' || 'ubuntu-latest' }}
     steps:
-      - uses: actions/checkout@v3
+      - name: Checkout
+        uses: actions/checkout@v4
 
       - name: Initialize CodeQL
         uses: github/codeql-action/init@v2
@@ -36,20 +34,7 @@ jobs:
           languages: go, javascript
 
       - name: Setup Go
-        uses: actions/setup-go@v3
-        with:
-          go-version: "~1.20"
-
-      - name: Go Cache Paths
-        id: go-cache-paths
-        run: |
-          echo "GOMODCACHE=$(go env GOMODCACHE)" >> $GITHUB_OUTPUT
-
-      - name: Go Mod Cache
-        uses: actions/cache@v3
-        with:
-          path: ${{ steps.go-cache-paths.outputs.GOMODCACHE }}
-          key: ${{ runner.os }}-release-go-mod-${{ hashFiles('**/go.sum') }}
+        uses: ./.github/actions/setup-go
 
       # Workaround to prevent CodeQL from building the dashboard.
       - name: Remove Makefile
@@ -59,41 +44,54 @@ jobs:
       - name: Perform CodeQL Analysis
         uses: github/codeql-action/analyze@v2
 
+      - name: Send Slack notification on failure
+        if: ${{ failure() }}
+        run: |
+          msg="❌ CodeQL Failed\n\nhttps://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+          curl \
+            -qfsSL \
+            -X POST \
+            -H "Content-Type: application/json" \
+            --data "{\"content\": \"$msg\"}" \
+            "${{ secrets.SLACK_SECURITY_FAILURE_WEBHOOK_URL }}"
+
   trivy:
-    runs-on: ${{ github.repository_owner == 'coder' && 'ubuntu-latest-8-cores' || 'ubuntu-latest' }}
+    runs-on: ${{ github.repository_owner == 'coder' && 'buildjet-8vcpu-ubuntu-2204' || 'ubuntu-latest' }}
     steps:
-      - uses: actions/checkout@v3
+      - name: Checkout
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
-      - uses: actions/setup-go@v3
-        with:
-          go-version: "~1.20"
+      - name: Setup Go
+        uses: ./.github/actions/setup-go
 
-      - name: Go Cache Paths
-        id: go-cache-paths
-        run: |
-          echo "GOMODCACHE=$(go env GOMODCACHE)" >> $GITHUB_OUTPUT
+      - name: Setup Node
+        uses: ./.github/actions/setup-node
 
-      - name: Go Mod Cache
-        uses: actions/cache@v3
-        with:
-          path: ${{ steps.go-cache-paths.outputs.GOMODCACHE }}
-          key: ${{ runner.os }}-release-go-mod-${{ hashFiles('**/go.sum') }}
-
-      - name: Cache Node
-        id: cache-node
-        uses: actions/cache@v3
-        with:
-          path: |
-            **/node_modules
-            .eslintcache
-          key: js-${{ runner.os }}-test-${{ hashFiles('**/yarn.lock') }}
-          restore-keys: |
-            js-${{ runner.os }}-
+      - name: Setup sqlc
+        uses: ./.github/actions/setup-sqlc
 
       - name: Install yq
         run: go run github.com/mikefarah/yq/v4@v4.30.6
+      - name: Install mockgen
+        run: go install github.com/golang/mock/mockgen@v1.6.0
+      - name: Install protoc-gen-go
+        run: go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.30
+      - name: Install protoc-gen-go-drpc
+        run: go install storj.io/drpc/cmd/protoc-gen-go-drpc@v0.0.33
+      - name: Install Protoc
+        run: |
+          # protoc must be in lockstep with our dogfood Dockerfile or the
+          # version in the comments will differ. This is also defined in
+          # ci.yaml.
+          set -x
+          cd dogfood
+          DOCKER_BUILDKIT=1 docker build . --target proto -t protoc
+          protoc_path=/usr/local/bin/protoc
+          docker run --rm --entrypoint cat protoc /tmp/bin/protoc > $protoc_path
+          chmod +x $protoc_path
+          protoc --version
 
       - name: Build Coder linux amd64 Docker image
         id: build
@@ -115,8 +113,16 @@ jobs:
           make -j "$image_job"
           echo "image=$(cat "$image_job")" >> $GITHUB_OUTPUT
 
+      - name: Run Prisma Cloud image scan
+        uses: PaloAltoNetworks/prisma-cloud-scan@v1
+        with:
+          pcc_console_url: ${{ secrets.PRISMA_CLOUD_URL }}
+          pcc_user: ${{ secrets.PRISMA_CLOUD_ACCESS_KEY }}
+          pcc_pass: ${{ secrets.PRISMA_CLOUD_SECRET_KEY }}
+          image_name: ${{ steps.build.outputs.image }}
+
       - name: Run Trivy vulnerability scanner
-        uses: aquasecurity/trivy-action@9ab158e8597f3b310480b9a69402b419bc03dbd5
+        uses: aquasecurity/trivy-action@fbd16365eb88e12433951383f5e99bd901fc618f
         with:
           image-ref: ${{ steps.build.outputs.image }}
           format: sarif
@@ -130,8 +136,19 @@ jobs:
           category: "Trivy"
 
       - name: Upload Trivy scan results as an artifact
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: trivy
           path: trivy-results.sarif
           retention-days: 7
+
+      - name: Send Slack notification on failure
+        if: ${{ failure() }}
+        run: |
+          msg="❌ Trivy Failed\n\nhttps://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+          curl \
+            -qfsSL \
+            -X POST \
+            -H "Content-Type: application/json" \
+            --data "{\"content\": \"$msg\"}" \
+            "${{ secrets.SLACK_SECURITY_FAILURE_WEBHOOK_URL }}"

.github/workflows/stale.yaml

@@ -1,4 +1,4 @@
-name: Stale Issue and Branch Cleanup
+name: Stale Issue, Banch and Old Workflows Cleanup
 on:
   schedule:
     # Every day at midnight
@@ -10,24 +10,22 @@ jobs:
     permissions:
       issues: write
       pull-requests: write
+      actions: write
     steps:
-      - uses: actions/stale@v7.0.0
+      - name: stale
+        uses: actions/stale@v8.0.0
         with:
           stale-issue-label: "stale"
           stale-pr-label: "stale"
-          days-before-stale: 90
+          days-before-stale: 180
           # Pull Requests become stale more quickly due to merge conflicts.
           # Also, we promote minimizing WIP.
           days-before-pr-stale: 7
           days-before-pr-close: 3
-          stale-pr-message: >
-            This Pull Request is becoming stale. In order to minimize WIP,
-            prevent merge conflicts and keep the tracker readable, I'm going
-            close to this PR in 3 days if there isn't more activity.
-          stale-issue-message: >
-            This issue is becoming stale. In order to keep the tracker readable
-            and actionable, I'm going close to this issue in 7 days if there
-            isn't more activity.
+          # We rarely take action in response to the message, so avoid
+          # cluttering the issue and just close the oldies.
+          stale-pr-message: ""
+          stale-issue-message: ""
           # Upped from 30 since we have a big tracker and was hitting the limit.
           operations-per-run: 60
           # Start with the oldest issues, always.
@@ -36,9 +34,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - name: Run delete-old-branches-action
-        uses: beatlabs/delete-old-branches-action@v0.0.9
+        uses: beatlabs/delete-old-branches-action@v0.0.10
         with:
           repo_token: ${{ github.token }}
           date: "6 months ago"
@@ -46,3 +44,24 @@ jobs:
           delete_tags: false
           # extra_protected_branch_regex: ^(foo|bar)$
           exclude_open_pr_branches: true
+  del_runs:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Delete PR Cleanup workflow runs
+        uses: Mattraks/delete-workflow-runs@v2
+        with:
+          token: ${{ github.token }}
+          repository: ${{ github.repository }}
+          retain_days: 1
+          keep_minimum_runs: 1
+          delete_workflow_pattern: pr-cleanup.yaml
+
+      - name: Delete PR Deploy workflow skipped runs
+        uses: Mattraks/delete-workflow-runs@v2
+        with:
+          token: ${{ github.token }}
+          repository: ${{ github.repository }}
+          retain_days: 0
+          keep_minimum_runs: 0
+          delete_run_by_conclusion_pattern: skipped
+          delete_workflow_pattern: pr-deploy.yaml

.github/workflows/typos.toml

@@ -4,6 +4,7 @@ Jetbrains = "JetBrains"
 IST = "IST"
 MacOS = "macOS"
 AKS = "AKS"
+O_WRONLY = "O_WRONLY"
 
 [default.extend-words]
 AKS = "AKS"
@@ -16,12 +17,16 @@ encrypter = "encrypter"
 
 [files]
 extend-exclude = [
-    "**.svg",
-    "**.png",
-    "**.lock",
-    "go.sum",
-    "go.mod",
-    # These files contain base64 strings that confuse the detector
-    "**XService**.ts",
-    "**identity.go",
+	"**.svg",
+	"**.png",
+	"**.lock",
+	"go.sum",
+	"go.mod",
+	# These files contain base64 strings that confuse the detector
+	"**XService**.ts",
+	"**identity.go",
+	"scripts/ci-report/testdata/**",
+	"**/*_test.go",
+	"**/*.test.tsx",
+	"**/pnpm-lock.yaml",
 ]

.github/workflows/weekly-docs.yaml

@@ -0,0 +1,32 @@
+name: weekly-docs
+# runs every monday at 9 am
+on:
+  schedule:
+    - cron: "0 9 * * 1"
+  workflow_dispatch: # allows to run manually for testing
+
+jobs:
+  check-docs:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@master
+
+      - name: Check Markdown links
+        uses: gaurav-nelson/github-action-markdown-link-check@v1
+        id: markdown-link-check
+        # checks all markdown files from /docs including all subfolders
+        with:
+          use-quiet-mode: "yes"
+          use-verbose-mode: "yes"
+          config-file: ".github/workflows/mlc_config.json"
+          folder-path: "docs/"
+          file-path: "./README.md"
+
+      - name: Send Slack notification
+        if: failure()
+        run: |
+          curl -X POST -H 'Content-type: application/json' -d '{"msg":"Broken links found in the documentation. Please check the logs at ${{ env.LOGS_URL }}"}' ${{ secrets.DOCS_LINK_SLACK_WEBHOOK }}
+          echo "Sent Slack notification"
+        env:
+          LOGS_URL: https://github.com/coder/coder/actions/runs/${{ github.run_id }}

.gitignore

@@ -6,7 +6,8 @@
 **/*.swp
 gotests.coverage
 gotests.xml
-gotestsum.json
+gotests_stats.json
+gotests.json
 node_modules/
 vendor/
 yarn-error.log
@@ -26,15 +27,17 @@ site/storybook-static/
 site/test-results/*
 site/e2e/test-results/*
 site/e2e/states/*.json
+site/e2e/.auth.json
 site/playwright-report/*
+site/.swc
 
-# Make target for updating golden files.
-cli/testdata/.gen-golden
+# Make target for updating golden files (any dir).
+.gen-golden
 
 # Build
-/build/
-/dist/
-site/out/
+build/
+dist/
+out/
 
 # Bundle analysis
 site/stats/
@@ -45,8 +48,24 @@ site/stats/
 *.lock.hcl
 .terraform/
 
-/.coderv2/*
+**/.coderv2/*
 **/__debug_bin
 
 # direnv
 .envrc
+*.test
+
+# Loadtesting
+./scaletest/terraform/.terraform
+./scaletest/terraform/.terraform.lock.hcl
+scaletest/terraform/secrets.tfvars
+.terraform.tfstate.*
+
+# Nix
+result
+
+# Data dumps from unit tests
+**/*.test.sql
+
+# Filebrowser.db
+**/filebrowser.db

.golangci.yaml

@@ -2,8 +2,19 @@
 # Over time we should try tightening some of these.
 
 linters-settings:
+  dupl:
+    # goal: 100
+    threshold: 412
+
+  exhaustruct:
+    include:
+      # Gradually extend to cover more of the codebase.
+      - 'httpmw\.\w+'
+      # We want to enforce all values are specified when inserting or updating
+      # a database row. Ref: #9936
+      - 'github.com/coder/coder/v2/coderd/database\.[^G][^e][^t]\w+Params'
   gocognit:
-    min-complexity: 46 # Min code complexity (def 30).
+    min-complexity: 300
 
   goconst:
     min-len: 4 # Min length of string consts (def 3).
@@ -54,7 +65,6 @@ linters-settings:
       # - importShadow
       - indexAlloc
       - initClause
-      - ioutilDeprecated
       - mapKey
       - methodExprCall
       # - nestingReduce
@@ -115,9 +125,6 @@ linters-settings:
   goimports:
     local-prefixes: coder.com,cdr.dev,go.coder.com,github.com/cdr,github.com/coder
 
-  gocyclo:
-    min-complexity: 50
-
   importas:
     no-unaliased: true
 
@@ -127,7 +134,8 @@ linters-settings:
       - trialer
 
   nestif:
-    min-complexity: 4 # Min complexity of if statements (def 5, goal 4)
+    # goal: 10
+    min-complexity: 20
 
   revive:
     # see https://github.com/mgechev/revive#available-rules for details.
@@ -194,18 +202,23 @@ issues:
       linters:
         # We use assertions rather than explicitly checking errors in tests
         - errcheck
+        - forcetypeassert
+        - exhaustruct # This is unhelpful in tests.
+    - path: scripts/*
+      linters:
+        - exhaustruct
 
   fix: true
   max-issues-per-linter: 0
   max-same-issues: 0
 
 run:
-  concurrency: 4
   skip-dirs:
     - node_modules
+    - .git
   skip-files:
     - scripts/rules.go
-  timeout: 5m
+  timeout: 10m
 
 # Over time, add more and more linters from
 # https://golangci-lint.run/usage/linters/ as the code improves.
@@ -215,15 +228,20 @@ linters:
     - asciicheck
     - bidichk
     - bodyclose
-    - deadcode
     - dogsled
     - errcheck
     - errname
     - errorlint
+    - exhaustruct
     - exportloopref
     - forcetypeassert
     - gocritic
-    - gocyclo
+    # gocyclo is may be useful in the future when we start caring
+    # about testing complexity, but for the time being we should
+    # create a good culture around cognitive complexity.
+    # - gocyclo
+    - gocognit
+    - nestif
     - goimports
     - gomodguard
     - gosec
@@ -259,4 +277,4 @@ linters:
     - typecheck
     - unconvert
     - unused
-    - varcheck
+    - dupl

.prettierignore

@@ -9,7 +9,8 @@
 **/*.swp
 gotests.coverage
 gotests.xml
-gotestsum.json
+gotests_stats.json
+gotests.json
 node_modules/
 vendor/
 yarn-error.log
@@ -29,15 +30,17 @@ site/storybook-static/
 site/test-results/*
 site/e2e/test-results/*
 site/e2e/states/*.json
+site/e2e/.auth.json
 site/playwright-report/*
+site/.swc
 
-# Make target for updating golden files.
-cli/testdata/.gen-golden
+# Make target for updating golden files (any dir).
+.gen-golden
 
 # Build
-/build/
-/dist/
-site/out/
+build/
+dist/
+out/
 
 # Bundle analysis
 site/stats/
@@ -48,15 +51,31 @@ site/stats/
 *.lock.hcl
 .terraform/
 
-/.coderv2/*
+**/.coderv2/*
 **/__debug_bin
 
 # direnv
 .envrc
+*.test
+
+# Loadtesting
+./scaletest/terraform/.terraform
+./scaletest/terraform/.terraform.lock.hcl
+scaletest/terraform/secrets.tfvars
+.terraform.tfstate.*
+
+# Nix
+result
+
+# Data dumps from unit tests
+**/*.test.sql
+
+# Filebrowser.db
+**/filebrowser.db
 # .prettierignore.include:
 # Helm templates contain variables that are invalid YAML and can't be formatted
 # by Prettier.
-helm/templates/*.yaml
+helm/**/templates/*.yaml
 
 # Terraform state files used in tests, these are automatically generated.
 # Example: provisioner/terraform/testdata/instance-id/instance-id.tfstate.json
@@ -64,3 +83,11 @@ helm/templates/*.yaml
 
 # Testdata shouldn't be formatted.
 scripts/apitypings/testdata/**/*.ts
+
+# Generated files shouldn't be formatted.
+site/e2e/provisionerGenerated.ts
+
+**/pnpm-lock.yaml
+
+# Ignore generated JSON (e.g. examples/examples.gen.json).
+**/*.gen.json

.prettierignore.include

@@ -1,6 +1,6 @@
 # Helm templates contain variables that are invalid YAML and can't be formatted
 # by Prettier.
-helm/templates/*.yaml
+helm/**/templates/*.yaml
 
 # Terraform state files used in tests, these are automatically generated.
 # Example: provisioner/terraform/testdata/instance-id/instance-id.tfstate.json
@@ -8,3 +8,11 @@ helm/templates/*.yaml
 
 # Testdata shouldn't be formatted.
 scripts/apitypings/testdata/**/*.ts
+
+# Generated files shouldn't be formatted.
+site/e2e/provisionerGenerated.ts
+
+**/pnpm-lock.yaml
+
+# Ignore generated JSON (e.g. examples/examples.gen.json).
+**/*.gen.json

.prettierrc.yaml

@@ -1,16 +1,18 @@
 # This config file is used in conjunction with `.editorconfig` to specify
 # formatting for prettier-supported files. See `.editorconfig` and
-# `site/.editorconfig`for whitespace formatting options.
+# `site/.editorconfig` for whitespace formatting options.
 printWidth: 80
-semi: false
+proseWrap: always
 trailingComma: all
+useTabs: false
+tabWidth: 2
 overrides:
   - files:
       - README.md
+      - docs/api/**/*.md
+      - docs/cli/**/*.md
+      - docs/changelogs/*.md
+      - .github/**/*.{yaml,yml,toml}
+      - scripts/**/*.{yaml,yml,toml}
     options:
       proseWrap: preserve
-  - files:
-      - "site/**/*.yaml"
-      - "site/**/*.yml"
-    options:
-      proseWrap: always

.swaggo

@@ -1,8 +1,8 @@
 // Replace all NullTime with string
-replace github.com/coder/coder/codersdk.NullTime string
+replace github.com/coder/coder/v2/codersdk.NullTime string
 // Prevent swaggo from rendering enums for time.Duration
 replace time.Duration int64
 // Do not expose "echo" provider
-replace github.com/coder/coder/codersdk.ProvisionerType string
+replace github.com/coder/coder/v2/codersdk.ProvisionerType string
 // Do not render netip.Addr
 replace netip.Addr string

.vscode/settings.json

@@ -4,6 +4,7 @@
     "agentsdk",
     "apps",
     "ASKPASS",
+    "authcheck",
     "autostop",
     "awsidentity",
     "bodyclose",
@@ -19,6 +20,8 @@
     "codersdk",
     "cronstrue",
     "databasefake",
+    "dbfake",
+    "dbgen",
     "dbtype",
     "DERP",
     "derphttp",
@@ -33,8 +36,10 @@
     "Dsts",
     "embeddedpostgres",
     "enablements",
+    "enterprisemeta",
     "errgroup",
     "eventsourcemock",
+    "externalauth",
     "Failf",
     "fatih",
     "Formik",
@@ -89,7 +94,6 @@
     "pqtype",
     "prometheusmetrics",
     "promhttp",
-    "promptui",
     "protobuf",
     "provisionerd",
     "provisionerdserver",
@@ -113,6 +117,7 @@
     "stretchr",
     "STTY",
     "stuntest",
+    "tanstack",
     "tailbroker",
     "tailcfg",
     "tailexchange",
@@ -134,6 +139,7 @@
     "thead",
     "tios",
     "tmpdir",
+    "tokenconfig",
     "tparallel",
     "trialer",
     "trimprefix",
@@ -181,12 +187,20 @@
     ]
   },
   "eslint.workingDirectories": ["./site"],
-  "files.exclude": {
-    "**/node_modules": true
-  },
   "search.exclude": {
+    "**.pb.go": true,
+    "**/*.gen.json": true,
+    "**/testdata/*": true,
+    "**Generated.ts": true,
+    "coderd/apidoc/**": true,
+    "docs/api/*.md": true,
+    "docs/templates/*.md": true,
+    "LICENSE": true,
     "scripts/metricsdocgen/metrics": true,
-    "docs/api/*.md": true
+    "site/out/**": true,
+    "site/storybook-static/**": true,
+    "**.map": true,
+    "pnpm-lock.yaml": true
   },
   // Ensure files always have a newline.
   "files.insertFinalNewline": true,
@@ -206,12 +220,5 @@
   "go.testFlags": ["-short", "-coverpkg=./..."],
   // We often use a version of TypeScript that's ahead of the version shipped
   // with VS Code.
-  "typescript.tsdk": "./site/node_modules/typescript/lib",
-  "grammarly.selectors": [
-    {
-      "language": "markdown",
-      "scheme": "file",
-      "pattern": "docs/contributing/frontend.md"
-    }
-  ]
+  "typescript.tsdk": "./site/node_modules/typescript/lib"
 }

Makefile

@@ -50,11 +50,11 @@ endif
 # Note, all find statements should be written with `.` or `./path` as
 # the search path so that these exclusions match.
 FIND_EXCLUSIONS= \
-	-not \( \( -path '*/.git/*' -o -path './build/*' -o -path './vendor/*' -o -path './.coderv2/*' -o -path '*/node_modules/*' -o -path './site/out/*' \) -prune \)
+	-not \( \( -path '*/.git/*' -o -path './build/*' -o -path './vendor/*' -o -path './.coderv2/*' -o -path '*/node_modules/*' -o -path '*/out/*' -o -path './coderd/apidoc/*' -o -path '*/.next/*' \) -prune \)
 # Source files used for make targets, evaluated on use.
-GO_SRC_FILES = $(shell find . $(FIND_EXCLUSIONS) -type f -name '*.go')
+GO_SRC_FILES := $(shell find . $(FIND_EXCLUSIONS) -type f -name '*.go' -not -name '*_test.go')
 # All the shell files in the repo, excluding ignored files.
-SHELL_SRC_FILES = $(shell find . $(FIND_EXCLUSIONS) -type f -name '*.sh')
+SHELL_SRC_FILES := $(shell find . $(FIND_EXCLUSIONS) -type f -name '*.sh')
 
 # All ${OS}_${ARCH} combos we build for. Windows binaries have the .exe suffix.
 OS_ARCHES := \
@@ -107,9 +107,9 @@ endif
 
 
 clean:
-	rm -rf build site/out
-	mkdir -p build site/out/bin
-	git restore site/out
+	rm -rf build/ site/build/ site/out/
+	mkdir -p build/ site/out/bin/
+	git restore site/out/
 .PHONY: clean
 
 build-slim: $(CODER_SLIM_BINARIES)
@@ -344,21 +344,33 @@ push/$(CODER_MAIN_IMAGE): $(CODER_MAIN_IMAGE)
 	docker manifest push "$$image_tag"
 .PHONY: push/$(CODER_MAIN_IMAGE)
 
+# Helm charts that are available
+charts = coder provisioner
+
 # Shortcut for Helm chart package.
-build/coder_helm.tgz: build/coder_helm_$(VERSION).tgz
+$(foreach chart,$(charts),build/$(chart)_helm.tgz): build/%_helm.tgz: build/%_helm_$(VERSION).tgz
 	rm -f "$@"
 	ln "$<" "$@"
 
 # Helm chart package.
-build/coder_helm_$(VERSION).tgz:
+$(foreach chart,$(charts),build/$(chart)_helm_$(VERSION).tgz): build/%_helm_$(VERSION).tgz:
 	./scripts/helm.sh \
 		--version "$(VERSION)" \
+		--chart $* \
 		--output "$@"
 
 site/out/index.html: site/package.json $(shell find ./site $(FIND_EXCLUSIONS) -type f \( -name '*.ts' -o -name '*.tsx' \))
-	./scripts/yarn_install.sh
 	cd site
-	yarn build
+	../scripts/pnpm_install.sh
+	pnpm build
+
+offlinedocs/out/index.html: $(shell find ./offlinedocs $(FIND_EXCLUSIONS) -type f) $(shell find ./docs $(FIND_EXCLUSIONS) -type f | sed 's: :\\ :g')
+	cd offlinedocs
+	../scripts/pnpm_install.sh
+	pnpm export
+
+build/coder_docs_$(VERSION).tgz: offlinedocs/out/index.html
+	tar -czf "$@" -C offlinedocs/out .
 
 install: build/coder_$(VERSION)_$(GOOS)_$(GOARCH)$(GOOS_BIN_EXT)
 	install_dir="$$(go env GOPATH)/bin"
@@ -382,9 +394,9 @@ fmt/prettier:
 	cd site
 # Avoid writing files in CI to reduce file write activity
 ifdef CI
-	yarn run format:check
+	pnpm run format:check
 else
-	yarn run format:write
+	pnpm run format:write
 endif
 .PHONY: fmt/prettier
 
@@ -402,11 +414,21 @@ else
 endif
 .PHONY: fmt/shfmt
 
-lint: lint/shellcheck lint/go
+lint: lint/shellcheck lint/go lint/ts lint/helm lint/site-icons
 .PHONY: lint
 
+lint/site-icons:
+	./scripts/check_site_icons.sh
+.PHONY: lint/site-icons
+
+lint/ts:
+	cd site
+	pnpm i && pnpm lint
+.PHONY: lint/ts
+
 lint/go:
 	./scripts/check_enterprise_imports.sh
+	go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.53.2
 	golangci-lint run
 .PHONY: lint/go
 
@@ -416,35 +438,55 @@ lint/shellcheck: $(SHELL_SRC_FILES)
 	shellcheck --external-sources $(SHELL_SRC_FILES)
 .PHONY: lint/shellcheck
 
+lint/helm:
+	cd helm
+	make lint
+.PHONY: lint/helm
+
+# All files generated by the database should be added here, and this can be used
+# as a target for jobs that need to run after the database is generated.
+DB_GEN_FILES := \
+	coderd/database/querier.go \
+	coderd/database/unique_constraint.go \
+	coderd/database/dbfake/dbfake.go \
+	coderd/database/dbmetrics/dbmetrics.go \
+	coderd/database/dbauthz/dbauthz.go \
+	coderd/database/dbmock/dbmock.go
+
 # all gen targets should be added here and to gen/mark-fresh
 gen: \
-	coderd/database/dump.sql \
-	coderd/database/querier.go \
 	provisionersdk/proto/provisioner.pb.go \
 	provisionerd/proto/provisionerd.pb.go \
+	coderd/database/dump.sql \
+	$(DB_GEN_FILES) \
 	site/src/api/typesGenerated.ts \
+	coderd/rbac/object_gen.go \
 	docs/admin/prometheus.md \
-	docs/cli/coder.md \
+	docs/cli.md \
 	docs/admin/audit-logs.md \
 	coderd/apidoc/swagger.json \
 	.prettierignore.include \
 	.prettierignore \
 	site/.prettierrc.yaml \
 	site/.prettierignore \
-	site/.eslintignore
+	site/.eslintignore \
+	site/e2e/provisionerGenerated.ts \
+	site/src/theme/icons.json \
+	examples/examples.gen.json
 .PHONY: gen
 
 # Mark all generated files as fresh so make thinks they're up-to-date. This is
 # used during releases so we don't run generation scripts.
 gen/mark-fresh:
 	files="\
-		coderd/database/dump.sql \
-		coderd/database/querier.go \
 		provisionersdk/proto/provisioner.pb.go \
 		provisionerd/proto/provisionerd.pb.go \
+		coderd/database/dump.sql \
+		$(DB_GEN_FILES) \
 		site/src/api/typesGenerated.ts \
+		coderd/rbac/object_gen.go \
 		docs/admin/prometheus.md \
-		docs/cli/coder.md \
+		docs/cli.md \
 		docs/admin/audit-logs.md \
 		coderd/apidoc/swagger.json \
 		.prettierignore.include \
@@ -452,6 +494,9 @@ gen/mark-fresh:
 		site/.prettierrc.yaml \
 		site/.prettierignore \
 		site/.eslintignore \
+		site/e2e/provisionerGenerated.ts \
+		site/src/theme/icons.json \
+		examples/examples.gen.json \
 	"
 	for file in $$files; do
 		echo "$$file"
@@ -471,9 +516,14 @@ coderd/database/dump.sql: coderd/database/gen/dump/main.go $(wildcard coderd/dat
 	go run ./coderd/database/gen/dump/main.go
 
 # Generates Go code for querying the database.
-coderd/database/querier.go: coderd/database/sqlc.yaml coderd/database/dump.sql $(wildcard coderd/database/queries/*.sql) coderd/database/gen/enum/main.go
+# coderd/database/queries.sql.go
+# coderd/database/models.go
+coderd/database/querier.go: coderd/database/sqlc.yaml coderd/database/dump.sql $(wildcard coderd/database/queries/*.sql)
 	./coderd/database/generate.sh
 
+coderd/database/dbmock/dbmock.go: coderd/database/db.go coderd/database/querier.go
+	go generate ./coderd/database/dbmock/
+
 provisionersdk/proto/provisioner.pb.go: provisionersdk/proto/provisioner.proto
 	protoc \
 		--go_out=. \
@@ -490,35 +540,70 @@ provisionerd/proto/provisionerd.pb.go: provisionerd/proto/provisionerd.proto
 		--go-drpc_opt=paths=source_relative \
 		./provisionerd/proto/provisionerd.proto
 
-site/src/api/typesGenerated.ts: scripts/apitypings/main.go $(shell find ./codersdk $(FIND_EXCLUSIONS) -type f -name '*.go')
-	go run scripts/apitypings/main.go > site/src/api/typesGenerated.ts
+site/src/api/typesGenerated.ts: $(wildcard scripts/apitypings/*) $(shell find ./codersdk $(FIND_EXCLUSIONS) -type f -name '*.go')
+	go run ./scripts/apitypings/ > $@
+	pnpm run format:write:only "$@"
+
+site/e2e/provisionerGenerated.ts: provisionerd/proto/provisionerd.pb.go provisionersdk/proto/provisioner.pb.go
 	cd site
-	yarn run format:types
+	../scripts/pnpm_install.sh
+	pnpm run gen:provisioner
+
+site/src/theme/icons.json: $(wildcard scripts/gensite/*) $(wildcard site/static/icon/*)
+	go run ./scripts/gensite/ -icons "$@"
+	pnpm run format:write:only "$@"
+
+examples/examples.gen.json: scripts/examplegen/main.go examples/examples.go $(shell find ./examples/templates)
+	go run ./scripts/examplegen/main.go > examples/examples.gen.json
+
+coderd/rbac/object_gen.go: scripts/rbacgen/main.go coderd/rbac/object.go
+	go run scripts/rbacgen/main.go ./coderd/rbac > coderd/rbac/object_gen.go
 
 docs/admin/prometheus.md: scripts/metricsdocgen/main.go scripts/metricsdocgen/metrics
 	go run scripts/metricsdocgen/main.go
-	cd site
-	yarn run format:write:only ../docs/admin/prometheus.md
+	pnpm run format:write:only ./docs/admin/prometheus.md
 
-docs/cli/coder.md: scripts/clidocgen/main.go $(GO_SRC_FILES) docs/manifest.json
-	BASE_PATH="." go run scripts/clidocgen/main.go
-	cd site
-	yarn run format:write:only ../docs/cli/*.md ../docs/manifest.json
+docs/cli.md: scripts/clidocgen/main.go examples/examples.gen.json $(GO_SRC_FILES)
+	CI=true BASE_PATH="." go run ./scripts/clidocgen
+	pnpm run format:write:only ./docs/cli.md ./docs/cli/*.md ./docs/manifest.json
 
-docs/admin/audit-logs.md: scripts/auditdocgen/main.go enterprise/audit/table.go
+docs/admin/audit-logs.md: scripts/auditdocgen/main.go enterprise/audit/table.go coderd/rbac/object_gen.go
 	go run scripts/auditdocgen/main.go
-	cd site
-	yarn run format:write:only ../docs/admin/audit-logs.md
+	pnpm run format:write:only ./docs/admin/audit-logs.md
 
-coderd/apidoc/swagger.json: $(shell find ./scripts/apidocgen $(FIND_EXCLUSIONS) -type f) $(wildcard coderd/*.go) $(wildcard enterprise/coderd/*.go) $(wildcard codersdk/*.go) .swaggo docs/manifest.json
+coderd/apidoc/swagger.json: $(shell find ./scripts/apidocgen $(FIND_EXCLUSIONS) -type f) $(wildcard coderd/*.go) $(wildcard enterprise/coderd/*.go) $(wildcard codersdk/*.go) $(wildcard enterprise/wsproxy/wsproxysdk/*.go) $(DB_GEN_FILES) .swaggo docs/manifest.json coderd/rbac/object_gen.go
 	./scripts/apidocgen/generate.sh
-	yarn run --cwd=site format:write:only ../docs/api ../docs/manifest.json ../coderd/apidoc/swagger.json
+	pnpm run format:write:only ./docs/api ./docs/manifest.json ./coderd/apidoc/swagger.json
 
-update-golden-files: cli/testdata/.gen-golden
+update-golden-files: cli/testdata/.gen-golden helm/coder/tests/testdata/.gen-golden helm/provisioner/tests/testdata/.gen-golden scripts/ci-report/testdata/.gen-golden enterprise/cli/testdata/.gen-golden coderd/.gen-golden provisioner/terraform/testdata/.gen-golden
 .PHONY: update-golden-files
 
-cli/testdata/.gen-golden: $(wildcard cli/testdata/*.golden) $(GO_SRC_FILES)
-	go test ./cli -run=TestCommandHelp -update
+cli/testdata/.gen-golden: $(wildcard cli/testdata/*.golden) $(wildcard cli/*.tpl) $(GO_SRC_FILES) $(wildcard cli/*_test.go)
+	go test ./cli -run="Test(CommandHelp|ServerYAML)" -update
+	touch "$@"
+
+enterprise/cli/testdata/.gen-golden: $(wildcard enterprise/cli/testdata/*.golden) $(wildcard cli/*.tpl) $(GO_SRC_FILES) $(wildcard enterprise/cli/*_test.go)
+	go test ./enterprise/cli -run="TestEnterpriseCommandHelp" -update
+	touch "$@"
+
+helm/coder/tests/testdata/.gen-golden: $(wildcard helm/coder/tests/testdata/*.yaml) $(wildcard helm/coder/tests/testdata/*.golden) $(GO_SRC_FILES) $(wildcard helm/coder/tests/*_test.go)
+	go test ./helm/coder/tests -run=TestUpdateGoldenFiles -update
+	touch "$@"
+
+helm/provisioner/tests/testdata/.gen-golden: $(wildcard helm/provisioner/tests/testdata/*.yaml) $(wildcard helm/provisioner/tests/testdata/*.golden) $(GO_SRC_FILES) $(wildcard helm/provisioner/tests/*_test.go)
+	go test ./helm/provisioner/tests -run=TestUpdateGoldenFiles -update
+	touch "$@"
+
+coderd/.gen-golden: $(wildcard coderd/testdata/*/*.golden) $(GO_SRC_FILES) $(wildcard coderd/*_test.go)
+	go test ./coderd -run="Test.*Golden$$" -update
+	touch "$@"
+
+provisioner/terraform/testdata/.gen-golden: $(wildcard provisioner/terraform/testdata/*/*.golden) $(GO_SRC_FILES) $(wildcard provisioner/terraform/*_test.go)
+	go test ./provisioner/terraform -run="Test.*Golden$$" -update
+	touch "$@"
+
+scripts/ci-report/testdata/.gen-golden: $(wildcard scripts/ci-report/testdata/*) $(wildcard scripts/ci-report/*.go)
+	go test ./scripts/ci-report -run=TestOutputMatchesGoldenFile -update
 	touch "$@"
 
 # Generate a prettierrc for the site package that uses relative paths for
@@ -535,7 +620,7 @@ site/.prettierrc.yaml: .prettierrc.yaml
 	# - ./ -> ../
 	# - ./site -> ./
 	yq \
-		'.overrides[].files |= map(. | sub("^./"; "") | sub("^"; "../") | sub("../site/"; "./"))' \
+		'.overrides[].files |= map(. | sub("^./"; "") | sub("^"; "../") | sub("../site/"; "./") | sub("../!"; "!../"))' \
 		"$<" >> "$@"
 
 # Combine .gitignore with .prettierignore.include to generate .prettierignore.
@@ -581,22 +666,23 @@ site/.eslintignore site/.prettierignore: .prettierignore Makefile
 		echo "$${ignore}$${rule}" >> "$@"
 	done < "$<"
 
-test: test-clean
-	gotestsum -- -v -short ./...
+test:
+	gotestsum --format standard-quiet -- -v -short -count=1 ./...
 .PHONY: test
 
 # When updating -timeout for this test, keep in sync with
 # test-go-postgres (.github/workflows/coder.yaml).
-test-postgres: test-clean test-postgres-docker
+# Do add coverage flags so that test caching works.
+test-postgres: test-postgres-docker
 	# The postgres test is prone to failure, so we limit parallelism for
 	# more consistent execution.
 	DB=ci DB_FROM=$(shell go run scripts/migrate-ci/main.go) gotestsum \
 		--junitfile="gotests.xml" \
+		--jsonfile="gotests.json" \
 		--packages="./..." -- \
-		-covermode=atomic -coverprofile="gotests.coverage" -timeout=20m \
-		-parallel=4 \
-		-coverpkg=./... \
-		-count=1 -race -failfast
+		-timeout=20m \
+		-failfast \
+		-count=1
 .PHONY: test-postgres
 
 test-postgres-docker:
@@ -611,8 +697,10 @@ test-postgres-docker:
 		--name test-postgres-docker \
 		--restart no \
 		--detach \
-		postgres:13 \
+		gcr.io/coder-dev-1/postgres:13 \
 		-c shared_buffers=1GB \
+		-c work_mem=1GB \
+		-c effective_cache_size=1GB \
 		-c max_connections=1000 \
 		-c fsync=off \
 		-c synchronous_commit=off \
@@ -625,6 +713,14 @@ test-postgres-docker:
 	done
 .PHONY: test-postgres-docker
 
+# Make sure to keep this in sync with test-go-race from .github/workflows/ci.yaml.
+test-race:
+	gotestsum --junitfile="gotests.xml" -- -race -count=1 ./...
+.PHONY: test-race
+
+# Note: we used to add this to the test target, but it's not necessary and we can
+# achieve the desired result by specifying -count=1 in the go test invocation
+# instead. Keeping it here for convenience.
 test-clean:
 	go clean -testcache
 .PHONY: test-clean

README.md

@@ -74,7 +74,7 @@ You can run the install script with `--dry-run` to see the commands that will be
 
 Once installed, you can start a production deployment<sup>1</sup> with a single command:
 
-```console
+```shell
 # Automatically sets up an external access URL on *.try.coder.app
 coder server
 
@@ -84,7 +84,7 @@ coder server --postgres-url <url> --access-url <url>
 
 > <sup>1</sup> For production deployments, set up an external PostgreSQL instance for reliability.
 
-Use `coder --help` to get a list of flags and environment variables. Use our [quickstart guide](https://coder.com/docs/v2/latest/quickstart) for a full walkthrough.
+Use `coder --help` to get a list of flags and environment variables. Use our [install guides](https://coder.com/docs/v2/latest/install) for a full walkthrough.
 
 ## Documentation
 

SECURITY.md

@@ -1,7 +1,7 @@
 # Coder Security
 
-Coder welcomes feedback from security researchers and the general public
-to help improve our security. If you believe you have discovered a vulnerability,
+Coder welcomes feedback from security researchers and the general public to help
+improve our security. If you believe you have discovered a vulnerability,
 privacy issue, exposed data, or other security issues in any of our assets, we
 want to hear from you. This policy outlines steps for reporting vulnerabilities
 to us, what we expect, what you can expect from us.
@@ -10,64 +10,72 @@ You can see the pretty version [here](https://coder.com/security/policy)
 
 # Why Coder's security matters
 
-If an attacker could fully compromise a Coder installation, they could spin
-up expensive workstations, steal valuable credentials, or steal proprietary
-source code. We take this risk very seriously and employ routine pen testing,
-vulnerability scanning, and code reviews. We also welcome the contributions
-from the community that helped make this product possible.
+If an attacker could fully compromise a Coder installation, they could spin up
+expensive workstations, steal valuable credentials, or steal proprietary source
+code. We take this risk very seriously and employ routine pen testing,
+vulnerability scanning, and code reviews. We also welcome the contributions from
+the community that helped make this product possible.
 
 # Where should I report security issues?
 
-Please report security issues to security@coder.com, providing
-all relevant information. The more details you provide, the easier it will be
-for us to triage and fix the issue.
+Please report security issues to security@coder.com, providing all relevant
+information. The more details you provide, the easier it will be for us to
+triage and fix the issue.
 
 # Out of Scope
 
-Our primary concern is around an abuse of the Coder application that allows
-an attacker to gain access to another users workspace, or spin up unwanted
+Our primary concern is around an abuse of the Coder application that allows an
+attacker to gain access to another users workspace, or spin up unwanted
 workspaces.
 
 - DOS/DDOS attacks affecting availability --> While we do support rate limiting
-  of requests, we primarily leave this to the owner of the Coder installation. Our
-  rationale is that a DOS attack only affecting availability is not a valuable
-  target for attackers.
+  of requests, we primarily leave this to the owner of the Coder installation.
+  Our rationale is that a DOS attack only affecting availability is not a
+  valuable target for attackers.
 - Abuse of a compromised user credential --> If a user credential is compromised
-  outside of the Coder ecosystem, then we consider it beyond the scope of our application.
-  However, if an unprivileged user could escalate their permissions or gain access
-  to another workspace, that is a cause for concern.
+  outside of the Coder ecosystem, then we consider it beyond the scope of our
+  application. However, if an unprivileged user could escalate their permissions
+  or gain access to another workspace, that is a cause for concern.
 - Vulnerabilities in third party systems --> Vulnerabilities discovered in
-  out-of-scope systems should be reported to the appropriate vendor or applicable authority.
+  out-of-scope systems should be reported to the appropriate vendor or
+  applicable authority.
 
 # Our Commitments
 
 When working with us, according to this policy, you can expect us to:
 
-- Respond to your report promptly, and work with you to understand and validate your report;
-- Strive to keep you informed about the progress of a vulnerability as it is processed;
-- Work to remediate discovered vulnerabilities in a timely manner, within our operational constraints; and
-- Extend Safe Harbor for your vulnerability research that is related to this policy.
+- Respond to your report promptly, and work with you to understand and validate
+  your report;
+- Strive to keep you informed about the progress of a vulnerability as it is
+  processed;
+- Work to remediate discovered vulnerabilities in a timely manner, within our
+  operational constraints; and
+- Extend Safe Harbor for your vulnerability research that is related to this
+  policy.
 
 # Our Expectations
 
-In participating in our vulnerability disclosure program in good faith, we ask that you:
+In participating in our vulnerability disclosure program in good faith, we ask
+that you:
 
-- Play by the rules, including following this policy and any other relevant agreements.
-  If there is any inconsistency between this policy and any other applicable terms, the
-  terms of this policy will prevail;
+- Play by the rules, including following this policy and any other relevant
+  agreements. If there is any inconsistency between this policy and any other
+  applicable terms, the terms of this policy will prevail;
 - Report any vulnerability you’ve discovered promptly;
-- Avoid violating the privacy of others, disrupting our systems, destroying data, and/or
-  harming user experience;
+- Avoid violating the privacy of others, disrupting our systems, destroying
+  data, and/or harming user experience;
 - Use only the Official Channels to discuss vulnerability information with us;
-- Provide us a reasonable amount of time (at least 90 days from the initial report) to
-  resolve the issue before you disclose it publicly;
-- Perform testing only on in-scope systems, and respect systems and activities which
-  are out-of-scope;
-- If a vulnerability provides unintended access to data: Limit the amount of data you
-  access to the minimum required for effectively demonstrating a Proof of Concept; and
-  cease testing and submit a report immediately if you encounter any user data during testing,
-  such as Personally Identifiable Information (PII), Personal Healthcare Information (PHI),
-  credit card data, or proprietary information;
-- You should only interact with test accounts you own or with explicit permission from
+- Provide us a reasonable amount of time (at least 90 days from the initial
+  report) to resolve the issue before you disclose it publicly;
+- Perform testing only on in-scope systems, and respect systems and activities
+  which are out-of-scope;
+- If a vulnerability provides unintended access to data: Limit the amount of
+  data you access to the minimum required for effectively demonstrating a Proof
+  of Concept; and cease testing and submit a report immediately if you encounter
+  any user data during testing, such as Personally Identifiable Information
+  (PII), Personal Healthcare Information (PHI), credit card data, or proprietary
+  information;
+- You should only interact with test accounts you own or with explicit
+  permission from
 - the account holder; and
 - Do not engage in extortion.

agent/agentproc/agentproctest/doc.go

@@ -0,0 +1,5 @@
+// Package agentproctest contains utility functions
+// for testing process management in the agent.
+package agentproctest
+
+//go:generate mockgen -destination ./syscallermock.go -package agentproctest github.com/coder/coder/v2/agent/agentproc Syscaller

agent/agentproc/agentproctest/proc.go

@@ -0,0 +1,49 @@
+package agentproctest
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/spf13/afero"
+	"github.com/stretchr/testify/require"
+
+	"github.com/coder/coder/v2/agent/agentproc"
+	"github.com/coder/coder/v2/cryptorand"
+)
+
+func GenerateProcess(t *testing.T, fs afero.Fs, muts ...func(*agentproc.Process)) agentproc.Process {
+	t.Helper()
+
+	pid, err := cryptorand.Intn(1<<31 - 1)
+	require.NoError(t, err)
+
+	arg1, err := cryptorand.String(5)
+	require.NoError(t, err)
+
+	arg2, err := cryptorand.String(5)
+	require.NoError(t, err)
+
+	arg3, err := cryptorand.String(5)
+	require.NoError(t, err)
+
+	cmdline := fmt.Sprintf("%s\x00%s\x00%s", arg1, arg2, arg3)
+
+	process := agentproc.Process{
+		CmdLine: cmdline,
+		PID:     int32(pid),
+	}
+
+	for _, mut := range muts {
+		mut(&process)
+	}
+
+	process.Dir = fmt.Sprintf("%s/%d", "/proc", process.PID)
+
+	err = fs.MkdirAll(process.Dir, 0o555)
+	require.NoError(t, err)
+
+	err = afero.WriteFile(fs, fmt.Sprintf("%s/cmdline", process.Dir), []byte(process.CmdLine), 0o444)
+	require.NoError(t, err)
+
+	return process
+}

agent/agentproc/agentproctest/syscallermock.go

@@ -0,0 +1,78 @@
+// Code generated by MockGen. DO NOT EDIT.
+// Source: github.com/coder/coder/v2/agent/agentproc (interfaces: Syscaller)
+
+// Package agentproctest is a generated GoMock package.
+package agentproctest
+
+import (
+	reflect "reflect"
+	syscall "syscall"
+
+	gomock "github.com/golang/mock/gomock"
+)
+
+// MockSyscaller is a mock of Syscaller interface.
+type MockSyscaller struct {
+	ctrl     *gomock.Controller
+	recorder *MockSyscallerMockRecorder
+}
+
+// MockSyscallerMockRecorder is the mock recorder for MockSyscaller.
+type MockSyscallerMockRecorder struct {
+	mock *MockSyscaller
+}
+
+// NewMockSyscaller creates a new mock instance.
+func NewMockSyscaller(ctrl *gomock.Controller) *MockSyscaller {
+	mock := &MockSyscaller{ctrl: ctrl}
+	mock.recorder = &MockSyscallerMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use.
+func (m *MockSyscaller) EXPECT() *MockSyscallerMockRecorder {
+	return m.recorder
+}
+
+// GetPriority mocks base method.
+func (m *MockSyscaller) GetPriority(arg0 int32) (int, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetPriority", arg0)
+	ret0, _ := ret[0].(int)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// GetPriority indicates an expected call of GetPriority.
+func (mr *MockSyscallerMockRecorder) GetPriority(arg0 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPriority", reflect.TypeOf((*MockSyscaller)(nil).GetPriority), arg0)
+}
+
+// Kill mocks base method.
+func (m *MockSyscaller) Kill(arg0 int32, arg1 syscall.Signal) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Kill", arg0, arg1)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// Kill indicates an expected call of Kill.
+func (mr *MockSyscallerMockRecorder) Kill(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Kill", reflect.TypeOf((*MockSyscaller)(nil).Kill), arg0, arg1)
+}
+
+// SetPriority mocks base method.
+func (m *MockSyscaller) SetPriority(arg0 int32, arg1 int) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "SetPriority", arg0, arg1)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// SetPriority indicates an expected call of SetPriority.
+func (mr *MockSyscallerMockRecorder) SetPriority(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetPriority", reflect.TypeOf((*MockSyscaller)(nil).SetPriority), arg0, arg1)
+}

agent/agentproc/doc.go

@@ -0,0 +1,3 @@
+// Package agentproc contains logic for interfacing with local
+// processes running in the same context as the agent.
+package agentproc

agent/agentproc/proc_other.go

@@ -0,0 +1,24 @@
+//go:build !linux
+// +build !linux
+
+package agentproc
+
+import (
+	"github.com/spf13/afero"
+)
+
+func (p *Process) Niceness(sc Syscaller) (int, error) {
+	return 0, errUnimplemented
+}
+
+func (p *Process) SetNiceness(sc Syscaller, score int) error {
+	return errUnimplemented
+}
+
+func (p *Process) Cmd() string {
+	return ""
+}
+
+func List(fs afero.Fs, syscaller Syscaller) ([]*Process, error) {
+	return nil, errUnimplemented
+}

agent/agentproc/proc_test.go

@@ -0,0 +1,166 @@
+package agentproc_test
+
+import (
+	"runtime"
+	"syscall"
+	"testing"
+
+	"github.com/golang/mock/gomock"
+	"github.com/spf13/afero"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/xerrors"
+
+	"github.com/coder/coder/v2/agent/agentproc"
+	"github.com/coder/coder/v2/agent/agentproc/agentproctest"
+)
+
+func TestList(t *testing.T) {
+	t.Parallel()
+
+	if runtime.GOOS != "linux" {
+		t.Skipf("skipping non-linux environment")
+	}
+
+	t.Run("OK", func(t *testing.T) {
+		t.Parallel()
+
+		var (
+			fs            = afero.NewMemMapFs()
+			sc            = agentproctest.NewMockSyscaller(gomock.NewController(t))
+			expectedProcs = make(map[int32]agentproc.Process)
+		)
+
+		for i := 0; i < 4; i++ {
+			proc := agentproctest.GenerateProcess(t, fs)
+			expectedProcs[proc.PID] = proc
+
+			sc.EXPECT().
+				Kill(proc.PID, syscall.Signal(0)).
+				Return(nil)
+		}
+
+		actualProcs, err := agentproc.List(fs, sc)
+		require.NoError(t, err)
+		require.Len(t, actualProcs, len(expectedProcs))
+		for _, proc := range actualProcs {
+			expected, ok := expectedProcs[proc.PID]
+			require.True(t, ok)
+			require.Equal(t, expected.PID, proc.PID)
+			require.Equal(t, expected.CmdLine, proc.CmdLine)
+			require.Equal(t, expected.Dir, proc.Dir)
+		}
+	})
+
+	t.Run("FinishedProcess", func(t *testing.T) {
+		t.Parallel()
+
+		var (
+			fs            = afero.NewMemMapFs()
+			sc            = agentproctest.NewMockSyscaller(gomock.NewController(t))
+			expectedProcs = make(map[int32]agentproc.Process)
+		)
+
+		for i := 0; i < 3; i++ {
+			proc := agentproctest.GenerateProcess(t, fs)
+			expectedProcs[proc.PID] = proc
+
+			sc.EXPECT().
+				Kill(proc.PID, syscall.Signal(0)).
+				Return(nil)
+		}
+
+		// Create a process that's already finished. We're not adding
+		// it to the map because it should be skipped over.
+		proc := agentproctest.GenerateProcess(t, fs)
+		sc.EXPECT().
+			Kill(proc.PID, syscall.Signal(0)).
+			Return(xerrors.New("os: process already finished"))
+
+		actualProcs, err := agentproc.List(fs, sc)
+		require.NoError(t, err)
+		require.Len(t, actualProcs, len(expectedProcs))
+		for _, proc := range actualProcs {
+			expected, ok := expectedProcs[proc.PID]
+			require.True(t, ok)
+			require.Equal(t, expected.PID, proc.PID)
+			require.Equal(t, expected.CmdLine, proc.CmdLine)
+			require.Equal(t, expected.Dir, proc.Dir)
+		}
+	})
+
+	t.Run("NoSuchProcess", func(t *testing.T) {
+		t.Parallel()
+
+		var (
+			fs            = afero.NewMemMapFs()
+			sc            = agentproctest.NewMockSyscaller(gomock.NewController(t))
+			expectedProcs = make(map[int32]agentproc.Process)
+		)
+
+		for i := 0; i < 3; i++ {
+			proc := agentproctest.GenerateProcess(t, fs)
+			expectedProcs[proc.PID] = proc
+
+			sc.EXPECT().
+				Kill(proc.PID, syscall.Signal(0)).
+				Return(nil)
+		}
+
+		// Create a process that doesn't exist. We're not adding
+		// it to the map because it should be skipped over.
+		proc := agentproctest.GenerateProcess(t, fs)
+		sc.EXPECT().
+			Kill(proc.PID, syscall.Signal(0)).
+			Return(syscall.ESRCH)
+
+		actualProcs, err := agentproc.List(fs, sc)
+		require.NoError(t, err)
+		require.Len(t, actualProcs, len(expectedProcs))
+		for _, proc := range actualProcs {
+			expected, ok := expectedProcs[proc.PID]
+			require.True(t, ok)
+			require.Equal(t, expected.PID, proc.PID)
+			require.Equal(t, expected.CmdLine, proc.CmdLine)
+			require.Equal(t, expected.Dir, proc.Dir)
+		}
+	})
+}
+
+// These tests are not very interesting but they provide some modicum of
+// confidence.
+func TestProcess(t *testing.T) {
+	t.Parallel()
+
+	if runtime.GOOS != "linux" {
+		t.Skipf("skipping non-linux environment")
+	}
+
+	t.Run("SetNiceness", func(t *testing.T) {
+		t.Parallel()
+
+		var (
+			sc   = agentproctest.NewMockSyscaller(gomock.NewController(t))
+			proc = &agentproc.Process{
+				PID: 32,
+			}
+			score = 20
+		)
+
+		sc.EXPECT().SetPriority(proc.PID, score).Return(nil)
+		err := proc.SetNiceness(sc, score)
+		require.NoError(t, err)
+	})
+
+	t.Run("Cmd", func(t *testing.T) {
+		t.Parallel()
+
+		var (
+			proc = &agentproc.Process{
+				CmdLine: "helloworld\x00--arg1\x00--arg2",
+			}
+			expectedName = "helloworld --arg1 --arg2"
+		)
+
+		require.Equal(t, expectedName, proc.Cmd())
+	})
+}

agent/agentproc/proc_unix.go

@@ -0,0 +1,109 @@
+//go:build linux
+// +build linux
+
+package agentproc
+
+import (
+	"errors"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"syscall"
+
+	"github.com/spf13/afero"
+	"golang.org/x/xerrors"
+)
+
+func List(fs afero.Fs, syscaller Syscaller) ([]*Process, error) {
+	d, err := fs.Open(defaultProcDir)
+	if err != nil {
+		return nil, xerrors.Errorf("open dir %q: %w", defaultProcDir, err)
+	}
+	defer d.Close()
+
+	entries, err := d.Readdirnames(0)
+	if err != nil {
+		return nil, xerrors.Errorf("readdirnames: %w", err)
+	}
+
+	processes := make([]*Process, 0, len(entries))
+	for _, entry := range entries {
+		pid, err := strconv.ParseInt(entry, 10, 32)
+		if err != nil {
+			continue
+		}
+
+		// Check that the process still exists.
+		exists, err := isProcessExist(syscaller, int32(pid))
+		if err != nil {
+			return nil, xerrors.Errorf("check process exists: %w", err)
+		}
+		if !exists {
+			continue
+		}
+
+		cmdline, err := afero.ReadFile(fs, filepath.Join(defaultProcDir, entry, "cmdline"))
+		if err != nil {
+			var errNo syscall.Errno
+			if xerrors.As(err, &errNo) && errNo == syscall.EPERM {
+				continue
+			}
+			return nil, xerrors.Errorf("read cmdline: %w", err)
+		}
+		processes = append(processes, &Process{
+			PID:     int32(pid),
+			CmdLine: string(cmdline),
+			Dir:     filepath.Join(defaultProcDir, entry),
+		})
+	}
+
+	return processes, nil
+}
+
+func isProcessExist(syscaller Syscaller, pid int32) (bool, error) {
+	err := syscaller.Kill(pid, syscall.Signal(0))
+	if err == nil {
+		return true, nil
+	}
+	if err.Error() == "os: process already finished" {
+		return false, nil
+	}
+
+	var errno syscall.Errno
+	if !errors.As(err, &errno) {
+		return false, err
+	}
+
+	switch errno {
+	case syscall.ESRCH:
+		return false, nil
+	case syscall.EPERM:
+		return true, nil
+	}
+
+	return false, xerrors.Errorf("kill: %w", err)
+}
+
+func (p *Process) Niceness(sc Syscaller) (int, error) {
+	nice, err := sc.GetPriority(p.PID)
+	if err != nil {
+		return 0, xerrors.Errorf("get priority for %q: %w", p.CmdLine, err)
+	}
+	return nice, nil
+}
+
+func (p *Process) SetNiceness(sc Syscaller, score int) error {
+	err := sc.SetPriority(p.PID, score)
+	if err != nil {
+		return xerrors.Errorf("set priority for %q: %w", p.CmdLine, err)
+	}
+	return nil
+}
+
+func (p *Process) Cmd() string {
+	return strings.Join(p.cmdLine(), " ")
+}
+
+func (p *Process) cmdLine() []string {
+	return strings.Split(p.CmdLine, "\x00")
+}

agent/agentproc/syscaller.go

@@ -0,0 +1,19 @@
+package agentproc
+
+import (
+	"syscall"
+)
+
+type Syscaller interface {
+	SetPriority(pid int32, priority int) error
+	GetPriority(pid int32) (int, error)
+	Kill(pid int32, sig syscall.Signal) error
+}
+
+const defaultProcDir = "/proc"
+
+type Process struct {
+	Dir     string
+	CmdLine string
+	PID     int32
+}

agent/agentproc/syscaller_other.go

@@ -0,0 +1,30 @@
+//go:build !linux
+// +build !linux
+
+package agentproc
+
+import (
+	"syscall"
+
+	"golang.org/x/xerrors"
+)
+
+func NewSyscaller() Syscaller {
+	return nopSyscaller{}
+}
+
+var errUnimplemented = xerrors.New("unimplemented")
+
+type nopSyscaller struct{}
+
+func (nopSyscaller) SetPriority(pid int32, priority int) error {
+	return errUnimplemented
+}
+
+func (nopSyscaller) GetPriority(pid int32) (int, error) {
+	return 0, errUnimplemented
+}
+
+func (nopSyscaller) Kill(pid int32, sig syscall.Signal) error {
+	return errUnimplemented
+}

agent/agentproc/syscaller_unix.go

@@ -0,0 +1,42 @@
+//go:build linux
+// +build linux
+
+package agentproc
+
+import (
+	"syscall"
+
+	"golang.org/x/sys/unix"
+	"golang.org/x/xerrors"
+)
+
+func NewSyscaller() Syscaller {
+	return UnixSyscaller{}
+}
+
+type UnixSyscaller struct{}
+
+func (UnixSyscaller) SetPriority(pid int32, nice int) error {
+	err := unix.Setpriority(unix.PRIO_PROCESS, int(pid), nice)
+	if err != nil {
+		return xerrors.Errorf("set priority: %w", err)
+	}
+	return nil
+}
+
+func (UnixSyscaller) GetPriority(pid int32) (int, error) {
+	nice, err := unix.Getpriority(0, int(pid))
+	if err != nil {
+		return 0, xerrors.Errorf("get priority: %w", err)
+	}
+	return nice, nil
+}
+
+func (UnixSyscaller) Kill(pid int32, sig syscall.Signal) error {
+	err := syscall.Kill(int(pid), sig)
+	if err != nil {
+		return xerrors.Errorf("kill: %w", err)
+	}
+
+	return nil
+}

agent/agentscripts/agentscripts.go

@@ -0,0 +1,283 @@
+package agentscripts
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"os/user"
+	"path/filepath"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/robfig/cron/v3"
+	"github.com/spf13/afero"
+	"golang.org/x/sync/errgroup"
+	"golang.org/x/xerrors"
+
+	"cdr.dev/slog"
+	"github.com/coder/coder/v2/agent/agentssh"
+	"github.com/coder/coder/v2/codersdk"
+	"github.com/coder/coder/v2/codersdk/agentsdk"
+)
+
+var (
+	// ErrTimeout is returned when a script times out.
+	ErrTimeout = xerrors.New("script timed out")
+
+	parser = cron.NewParser(cron.Second | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.DowOptional)
+)
+
+// Options are a set of options for the runner.
+type Options struct {
+	LogDir     string
+	Logger     slog.Logger
+	SSHServer  *agentssh.Server
+	Filesystem afero.Fs
+	PatchLogs  func(ctx context.Context, req agentsdk.PatchLogs) error
+}
+
+// New creates a runner for the provided scripts.
+func New(opts Options) *Runner {
+	cronCtx, cronCtxCancel := context.WithCancel(context.Background())
+	return &Runner{
+		Options:       opts,
+		cronCtx:       cronCtx,
+		cronCtxCancel: cronCtxCancel,
+		cron:          cron.New(cron.WithParser(parser)),
+		closed:        make(chan struct{}),
+	}
+}
+
+type Runner struct {
+	Options
+
+	cronCtx       context.Context
+	cronCtxCancel context.CancelFunc
+	cmdCloseWait  sync.WaitGroup
+	closed        chan struct{}
+	closeMutex    sync.Mutex
+	cron          *cron.Cron
+	initialized   atomic.Bool
+	scripts       []codersdk.WorkspaceAgentScript
+}
+
+// Init initializes the runner with the provided scripts.
+// It also schedules any scripts that have a schedule.
+// This function must be called before Execute.
+func (r *Runner) Init(scripts []codersdk.WorkspaceAgentScript) error {
+	if r.initialized.Load() {
+		return xerrors.New("init: already initialized")
+	}
+	r.initialized.Store(true)
+	r.scripts = scripts
+	r.Logger.Info(r.cronCtx, "initializing agent scripts", slog.F("script_count", len(scripts)), slog.F("log_dir", r.LogDir))
+
+	for _, script := range scripts {
+		if script.Cron == "" {
+			continue
+		}
+		script := script
+		_, err := r.cron.AddFunc(script.Cron, func() {
+			err := r.run(r.cronCtx, script)
+			if err != nil {
+				r.Logger.Warn(context.Background(), "run agent script on schedule", slog.Error(err))
+			}
+		})
+		if err != nil {
+			return xerrors.Errorf("add schedule: %w", err)
+		}
+	}
+	return nil
+}
+
+// StartCron starts the cron scheduler.
+// This is done async to allow for the caller to execute scripts prior.
+func (r *Runner) StartCron() {
+	r.cron.Start()
+}
+
+// Execute runs a set of scripts according to a filter.
+func (r *Runner) Execute(ctx context.Context, filter func(script codersdk.WorkspaceAgentScript) bool) error {
+	if filter == nil {
+		// Execute em' all!
+		filter = func(script codersdk.WorkspaceAgentScript) bool {
+			return true
+		}
+	}
+	var eg errgroup.Group
+	for _, script := range r.scripts {
+		if !filter(script) {
+			continue
+		}
+		script := script
+		eg.Go(func() error {
+			err := r.run(ctx, script)
+			if err != nil {
+				return xerrors.Errorf("run agent script %q: %w", script.LogSourceID, err)
+			}
+			return nil
+		})
+	}
+	return eg.Wait()
+}
+
+// run executes the provided script with the timeout.
+// If the timeout is exceeded, the process is sent an interrupt signal.
+// If the process does not exit after a few seconds, it is forcefully killed.
+// This function immediately returns after a timeout, and does not wait for the process to exit.
+func (r *Runner) run(ctx context.Context, script codersdk.WorkspaceAgentScript) error {
+	logPath := script.LogPath
+	if logPath == "" {
+		logPath = fmt.Sprintf("coder-script-%s.log", script.LogSourceID)
+	}
+	if logPath[0] == '~' {
+		// First we check the environment.
+		homeDir, err := os.UserHomeDir()
+		if err != nil {
+			u, err := user.Current()
+			if err != nil {
+				return xerrors.Errorf("current user: %w", err)
+			}
+			homeDir = u.HomeDir
+		}
+		logPath = filepath.Join(homeDir, logPath[1:])
+	}
+	logPath = os.ExpandEnv(logPath)
+	if !filepath.IsAbs(logPath) {
+		logPath = filepath.Join(r.LogDir, logPath)
+	}
+	logger := r.Logger.With(slog.F("log_path", logPath))
+	logger.Info(ctx, "running agent script", slog.F("script", script.Script))
+
+	fileWriter, err := r.Filesystem.OpenFile(logPath, os.O_CREATE|os.O_RDWR, 0o600)
+	if err != nil {
+		return xerrors.Errorf("open %s script log file: %w", logPath, err)
+	}
+	defer func() {
+		err := fileWriter.Close()
+		if err != nil {
+			logger.Warn(ctx, fmt.Sprintf("close %s script log file", logPath), slog.Error(err))
+		}
+	}()
+
+	var cmd *exec.Cmd
+	cmdCtx := ctx
+	if script.Timeout > 0 {
+		var ctxCancel context.CancelFunc
+		cmdCtx, ctxCancel = context.WithTimeout(ctx, script.Timeout)
+		defer ctxCancel()
+	}
+	cmdPty, err := r.SSHServer.CreateCommand(cmdCtx, script.Script, nil)
+	if err != nil {
+		return xerrors.Errorf("%s script: create command: %w", logPath, err)
+	}
+	cmd = cmdPty.AsExec()
+	cmd.SysProcAttr = cmdSysProcAttr()
+	cmd.WaitDelay = 10 * time.Second
+	cmd.Cancel = cmdCancel(cmd)
+
+	send, flushAndClose := agentsdk.LogsSender(script.LogSourceID, r.PatchLogs, logger)
+	// If ctx is canceled here (or in a writer below), we may be
+	// discarding logs, but that's okay because we're shutting down
+	// anyway. We could consider creating a new context here if we
+	// want better control over flush during shutdown.
+	defer func() {
+		if err := flushAndClose(ctx); err != nil {
+			logger.Warn(ctx, "flush startup logs failed", slog.Error(err))
+		}
+	}()
+
+	infoW := agentsdk.LogsWriter(ctx, send, script.LogSourceID, codersdk.LogLevelInfo)
+	defer infoW.Close()
+	errW := agentsdk.LogsWriter(ctx, send, script.LogSourceID, codersdk.LogLevelError)
+	defer errW.Close()
+	cmd.Stdout = io.MultiWriter(fileWriter, infoW)
+	cmd.Stderr = io.MultiWriter(fileWriter, errW)
+
+	start := time.Now()
+	defer func() {
+		end := time.Now()
+		execTime := end.Sub(start)
+		exitCode := 0
+		if err != nil {
+			exitCode = 255 // Unknown status.
+			var exitError *exec.ExitError
+			if xerrors.As(err, &exitError) {
+				exitCode = exitError.ExitCode()
+			}
+			logger.Warn(ctx, fmt.Sprintf("%s script failed", logPath), slog.F("execution_time", execTime), slog.F("exit_code", exitCode), slog.Error(err))
+		} else {
+			logger.Info(ctx, fmt.Sprintf("%s script completed", logPath), slog.F("execution_time", execTime), slog.F("exit_code", exitCode))
+		}
+	}()
+
+	err = cmd.Start()
+	if err != nil {
+		if errors.Is(err, context.DeadlineExceeded) {
+			return ErrTimeout
+		}
+		return xerrors.Errorf("%s script: start command: %w", logPath, err)
+	}
+
+	cmdDone := make(chan error, 1)
+	err = r.trackCommandGoroutine(func() {
+		cmdDone <- cmd.Wait()
+	})
+	if err != nil {
+		return xerrors.Errorf("%s script: track command goroutine: %w", logPath, err)
+	}
+	select {
+	case <-cmdCtx.Done():
+		// Wait for the command to drain!
+		select {
+		case <-cmdDone:
+		case <-time.After(10 * time.Second):
+		}
+		err = cmdCtx.Err()
+	case err = <-cmdDone:
+	}
+	if errors.Is(err, context.DeadlineExceeded) {
+		err = ErrTimeout
+	}
+	return err
+}
+
+func (r *Runner) Close() error {
+	r.closeMutex.Lock()
+	defer r.closeMutex.Unlock()
+	if r.isClosed() {
+		return nil
+	}
+	close(r.closed)
+	r.cronCtxCancel()
+	r.cron.Stop()
+	r.cmdCloseWait.Wait()
+	return nil
+}
+
+func (r *Runner) trackCommandGoroutine(fn func()) error {
+	r.closeMutex.Lock()
+	defer r.closeMutex.Unlock()
+	if r.isClosed() {
+		return xerrors.New("track command goroutine: closed")
+	}
+	r.cmdCloseWait.Add(1)
+	go func() {
+		defer r.cmdCloseWait.Done()
+		fn()
+	}()
+	return nil
+}
+
+func (r *Runner) isClosed() bool {
+	select {
+	case <-r.closed:
+		return true
+	default:
+		return false
+	}
+}

agent/agentscripts/agentscripts_other.go

@@ -0,0 +1,20 @@
+//go:build !windows
+
+package agentscripts
+
+import (
+	"os/exec"
+	"syscall"
+)
+
+func cmdSysProcAttr() *syscall.SysProcAttr {
+	return &syscall.SysProcAttr{
+		Setsid: true,
+	}
+}
+
+func cmdCancel(cmd *exec.Cmd) func() error {
+	return func() error {
+		return syscall.Kill(-cmd.Process.Pid, syscall.SIGHUP)
+	}
+}

agent/agentscripts/agentscripts_test.go

@@ -0,0 +1,80 @@
+package agentscripts_test
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/spf13/afero"
+	"github.com/stretchr/testify/require"
+	"go.uber.org/atomic"
+	"go.uber.org/goleak"
+
+	"cdr.dev/slog/sloggers/slogtest"
+	"github.com/coder/coder/v2/agent/agentscripts"
+	"github.com/coder/coder/v2/agent/agentssh"
+	"github.com/coder/coder/v2/codersdk"
+	"github.com/coder/coder/v2/codersdk/agentsdk"
+)
+
+func TestMain(m *testing.M) {
+	goleak.VerifyTestMain(m)
+}
+
+func TestExecuteBasic(t *testing.T) {
+	t.Parallel()
+	logs := make(chan agentsdk.PatchLogs, 1)
+	runner := setup(t, func(ctx context.Context, req agentsdk.PatchLogs) error {
+		logs <- req
+		return nil
+	})
+	defer runner.Close()
+	err := runner.Init([]codersdk.WorkspaceAgentScript{{
+		Script: "echo hello",
+	}})
+	require.NoError(t, err)
+	require.NoError(t, runner.Execute(context.Background(), func(script codersdk.WorkspaceAgentScript) bool {
+		return true
+	}))
+	log := <-logs
+	require.Equal(t, "hello", log.Logs[0].Output)
+}
+
+func TestTimeout(t *testing.T) {
+	t.Parallel()
+	runner := setup(t, nil)
+	defer runner.Close()
+	err := runner.Init([]codersdk.WorkspaceAgentScript{{
+		Script:  "sleep infinity",
+		Timeout: time.Millisecond,
+	}})
+	require.NoError(t, err)
+	require.ErrorIs(t, runner.Execute(context.Background(), nil), agentscripts.ErrTimeout)
+}
+
+func setup(t *testing.T, patchLogs func(ctx context.Context, req agentsdk.PatchLogs) error) *agentscripts.Runner {
+	t.Helper()
+	if patchLogs == nil {
+		// noop
+		patchLogs = func(ctx context.Context, req agentsdk.PatchLogs) error {
+			return nil
+		}
+	}
+	fs := afero.NewMemMapFs()
+	logger := slogtest.Make(t, nil)
+	s, err := agentssh.NewServer(context.Background(), logger, prometheus.NewRegistry(), fs, 0, "")
+	require.NoError(t, err)
+	s.AgentToken = func() string { return "" }
+	s.Manifest = atomic.NewPointer(&agentsdk.Manifest{})
+	t.Cleanup(func() {
+		_ = s.Close()
+	})
+	return agentscripts.New(agentscripts.Options{
+		LogDir:     t.TempDir(),
+		Logger:     logger,
+		SSHServer:  s,
+		Filesystem: fs,
+		PatchLogs:  patchLogs,
+	})
+}

agent/agentscripts/agentscripts_windows.go

@@ -0,0 +1,17 @@
+package agentscripts
+
+import (
+	"os"
+	"os/exec"
+	"syscall"
+)
+
+func cmdSysProcAttr() *syscall.SysProcAttr {
+	return &syscall.SysProcAttr{}
+}
+
+func cmdCancel(cmd *exec.Cmd) func() error {
+	return func() error {
+		return cmd.Process.Signal(os.Interrupt)
+	}
+}

agent/agentssh/agentssh.go

@@ -0,0 +1,875 @@
+package agentssh
+
+import (
+	"bufio"
+	"context"
+	"crypto/rand"
+	"crypto/rsa"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"os"
+	"os/exec"
+	"os/user"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/gliderlabs/ssh"
+	"github.com/kballard/go-shellquote"
+	"github.com/pkg/sftp"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/spf13/afero"
+	"go.uber.org/atomic"
+	gossh "golang.org/x/crypto/ssh"
+	"golang.org/x/xerrors"
+
+	"cdr.dev/slog"
+
+	"github.com/coder/coder/v2/agent/usershell"
+	"github.com/coder/coder/v2/codersdk"
+	"github.com/coder/coder/v2/codersdk/agentsdk"
+	"github.com/coder/coder/v2/pty"
+)
+
+const (
+	// MagicSessionErrorCode indicates that something went wrong with the session, rather than the
+	// command just returning a nonzero exit code, and is chosen as an arbitrary, high number
+	// unlikely to shadow other exit codes, which are typically 1, 2, 3, etc.
+	MagicSessionErrorCode = 229
+
+	// MagicSessionTypeEnvironmentVariable is used to track the purpose behind an SSH connection.
+	// This is stripped from any commands being executed, and is counted towards connection stats.
+	MagicSessionTypeEnvironmentVariable = "CODER_SSH_SESSION_TYPE"
+	// MagicSessionTypeVSCode is set in the SSH config by the VS Code extension to identify itself.
+	MagicSessionTypeVSCode = "vscode"
+	// MagicSessionTypeJetBrains is set in the SSH config by the JetBrains extension to identify itself.
+	MagicSessionTypeJetBrains = "jetbrains"
+)
+
+type Server struct {
+	mu        sync.RWMutex // Protects following.
+	fs        afero.Fs
+	listeners map[net.Listener]struct{}
+	conns     map[net.Conn]struct{}
+	sessions  map[ssh.Session]struct{}
+	closing   chan struct{}
+	// Wait for goroutines to exit, waited without
+	// a lock on mu but protected by closing.
+	wg sync.WaitGroup
+
+	logger       slog.Logger
+	srv          *ssh.Server
+	x11SocketDir string
+
+	Env           map[string]string
+	AgentToken    func() string
+	Manifest      *atomic.Pointer[agentsdk.Manifest]
+	ServiceBanner *atomic.Pointer[codersdk.ServiceBannerConfig]
+
+	connCountVSCode     atomic.Int64
+	connCountJetBrains  atomic.Int64
+	connCountSSHSession atomic.Int64
+
+	metrics *sshServerMetrics
+}
+
+func NewServer(ctx context.Context, logger slog.Logger, prometheusRegistry *prometheus.Registry, fs afero.Fs, maxTimeout time.Duration, x11SocketDir string) (*Server, error) {
+	// Clients' should ignore the host key when connecting.
+	// The agent needs to authenticate with coderd to SSH,
+	// so SSH authentication doesn't improve security.
+	randomHostKey, err := rsa.GenerateKey(rand.Reader, 2048)
+	if err != nil {
+		return nil, err
+	}
+	randomSigner, err := gossh.NewSignerFromKey(randomHostKey)
+	if err != nil {
+		return nil, err
+	}
+	if x11SocketDir == "" {
+		x11SocketDir = filepath.Join(os.TempDir(), ".X11-unix")
+	}
+
+	forwardHandler := &ssh.ForwardedTCPHandler{}
+	unixForwardHandler := &forwardedUnixHandler{log: logger}
+
+	metrics := newSSHServerMetrics(prometheusRegistry)
+	s := &Server{
+		listeners:    make(map[net.Listener]struct{}),
+		fs:           fs,
+		conns:        make(map[net.Conn]struct{}),
+		sessions:     make(map[ssh.Session]struct{}),
+		logger:       logger,
+		x11SocketDir: x11SocketDir,
+
+		metrics: metrics,
+	}
+
+	srv := &ssh.Server{
+		ChannelHandlers: map[string]ssh.ChannelHandler{
+			"direct-tcpip":                   ssh.DirectTCPIPHandler,
+			"direct-streamlocal@openssh.com": directStreamLocalHandler,
+			"session":                        ssh.DefaultSessionHandler,
+		},
+		ConnectionFailedCallback: func(conn net.Conn, err error) {
+			s.logger.Warn(ctx, "ssh connection failed",
+				slog.F("remote_addr", conn.RemoteAddr()),
+				slog.F("local_addr", conn.LocalAddr()),
+				slog.Error(err))
+			metrics.failedConnectionsTotal.Add(1)
+		},
+		ConnectionCompleteCallback: func(conn *gossh.ServerConn, err error) {
+			s.logger.Info(ctx, "ssh connection complete",
+				slog.F("remote_addr", conn.RemoteAddr()),
+				slog.F("local_addr", conn.LocalAddr()),
+				slog.Error(err))
+		},
+		Handler:     s.sessionHandler,
+		HostSigners: []ssh.Signer{randomSigner},
+		LocalPortForwardingCallback: func(ctx ssh.Context, destinationHost string, destinationPort uint32) bool {
+			// Allow local port forwarding all!
+			s.logger.Debug(ctx, "local port forward",
+				slog.F("destination_host", destinationHost),
+				slog.F("destination_port", destinationPort))
+			return true
+		},
+		PtyCallback: func(ctx ssh.Context, pty ssh.Pty) bool {
+			return true
+		},
+		ReversePortForwardingCallback: func(ctx ssh.Context, bindHost string, bindPort uint32) bool {
+			// Allow reverse port forwarding all!
+			s.logger.Debug(ctx, "local port forward",
+				slog.F("bind_host", bindHost),
+				slog.F("bind_port", bindPort))
+			return true
+		},
+		RequestHandlers: map[string]ssh.RequestHandler{
+			"tcpip-forward":                          forwardHandler.HandleSSHRequest,
+			"cancel-tcpip-forward":                   forwardHandler.HandleSSHRequest,
+			"streamlocal-forward@openssh.com":        unixForwardHandler.HandleSSHRequest,
+			"cancel-streamlocal-forward@openssh.com": unixForwardHandler.HandleSSHRequest,
+		},
+		X11Callback: s.x11Callback,
+		ServerConfigCallback: func(ctx ssh.Context) *gossh.ServerConfig {
+			return &gossh.ServerConfig{
+				NoClientAuth: true,
+			}
+		},
+		SubsystemHandlers: map[string]ssh.SubsystemHandler{
+			"sftp": s.sessionHandler,
+		},
+	}
+
+	// The MaxTimeout functionality has been substituted with the introduction of the KeepAlive feature.
+	// In cases where very short timeouts are set, the SSH server will automatically switch to the connection timeout for both read and write operations.
+	if maxTimeout >= 3*time.Second {
+		srv.ClientAliveCountMax = 3
+		srv.ClientAliveInterval = maxTimeout / time.Duration(srv.ClientAliveCountMax)
+		srv.MaxTimeout = 0
+	} else {
+		srv.MaxTimeout = maxTimeout
+	}
+
+	s.srv = srv
+	return s, nil
+}
+
+type ConnStats struct {
+	Sessions  int64
+	VSCode    int64
+	JetBrains int64
+}
+
+func (s *Server) ConnStats() ConnStats {
+	return ConnStats{
+		Sessions:  s.connCountSSHSession.Load(),
+		VSCode:    s.connCountVSCode.Load(),
+		JetBrains: s.connCountJetBrains.Load(),
+	}
+}
+
+func (s *Server) sessionHandler(session ssh.Session) {
+	logger := s.logger.With(slog.F("remote_addr", session.RemoteAddr()), slog.F("local_addr", session.LocalAddr()))
+	logger.Info(session.Context(), "handling ssh session")
+	ctx := session.Context()
+	if !s.trackSession(session, true) {
+		// See (*Server).Close() for why we call Close instead of Exit.
+		_ = session.Close()
+		logger.Info(ctx, "unable to accept new session, server is closing")
+		return
+	}
+	defer s.trackSession(session, false)
+
+	extraEnv := make([]string, 0)
+	x11, hasX11 := session.X11()
+	if hasX11 {
+		handled := s.x11Handler(session.Context(), x11)
+		if !handled {
+			_ = session.Exit(1)
+			logger.Error(ctx, "x11 handler failed")
+			return
+		}
+		extraEnv = append(extraEnv, fmt.Sprintf("DISPLAY=:%d.0", x11.ScreenNumber))
+	}
+
+	switch ss := session.Subsystem(); ss {
+	case "":
+	case "sftp":
+		s.sftpHandler(session)
+		return
+	default:
+		logger.Warn(ctx, "unsupported subsystem", slog.F("subsystem", ss))
+		_ = session.Exit(1)
+		return
+	}
+
+	err := s.sessionStart(session, extraEnv)
+	var exitError *exec.ExitError
+	if xerrors.As(err, &exitError) {
+		logger.Info(ctx, "ssh session returned", slog.Error(exitError))
+		_ = session.Exit(exitError.ExitCode())
+		return
+	}
+	if err != nil {
+		logger.Warn(ctx, "ssh session failed", slog.Error(err))
+		// This exit code is designed to be unlikely to be confused for a legit exit code
+		// from the process.
+		_ = session.Exit(MagicSessionErrorCode)
+		return
+	}
+	logger.Info(ctx, "normal ssh session exit")
+	_ = session.Exit(0)
+}
+
+func (s *Server) sessionStart(session ssh.Session, extraEnv []string) (retErr error) {
+	ctx := session.Context()
+	env := append(session.Environ(), extraEnv...)
+	var magicType string
+	for index, kv := range env {
+		if !strings.HasPrefix(kv, MagicSessionTypeEnvironmentVariable) {
+			continue
+		}
+		magicType = strings.TrimPrefix(kv, MagicSessionTypeEnvironmentVariable+"=")
+		env = append(env[:index], env[index+1:]...)
+	}
+
+	// Always force lowercase checking to be case-insensitive.
+	switch strings.ToLower(magicType) {
+	case strings.ToLower(MagicSessionTypeVSCode):
+		s.connCountVSCode.Add(1)
+		defer s.connCountVSCode.Add(-1)
+	case strings.ToLower(MagicSessionTypeJetBrains):
+		s.connCountJetBrains.Add(1)
+		defer s.connCountJetBrains.Add(-1)
+	case "":
+		s.connCountSSHSession.Add(1)
+		defer s.connCountSSHSession.Add(-1)
+	default:
+		s.logger.Warn(ctx, "invalid magic ssh session type specified", slog.F("type", magicType))
+	}
+
+	magicTypeLabel := magicTypeMetricLabel(magicType)
+	sshPty, windowSize, isPty := session.Pty()
+
+	cmd, err := s.CreateCommand(ctx, session.RawCommand(), env)
+	if err != nil {
+		ptyLabel := "no"
+		if isPty {
+			ptyLabel = "yes"
+		}
+		s.metrics.sessionErrors.WithLabelValues(magicTypeLabel, ptyLabel, "create_command").Add(1)
+		return err
+	}
+
+	if ssh.AgentRequested(session) {
+		l, err := ssh.NewAgentListener()
+		if err != nil {
+			ptyLabel := "no"
+			if isPty {
+				ptyLabel = "yes"
+			}
+
+			s.metrics.sessionErrors.WithLabelValues(magicTypeLabel, ptyLabel, "listener").Add(1)
+			return xerrors.Errorf("new agent listener: %w", err)
+		}
+		defer l.Close()
+		go ssh.ForwardAgentConnections(l, session)
+		cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", "SSH_AUTH_SOCK", l.Addr().String()))
+	}
+
+	if isPty {
+		return s.startPTYSession(session, magicTypeLabel, cmd, sshPty, windowSize)
+	}
+	return s.startNonPTYSession(session, magicTypeLabel, cmd.AsExec())
+}
+
+func (s *Server) startNonPTYSession(session ssh.Session, magicTypeLabel string, cmd *exec.Cmd) error {
+	s.metrics.sessionsTotal.WithLabelValues(magicTypeLabel, "no").Add(1)
+
+	cmd.Stdout = session
+	cmd.Stderr = session.Stderr()
+	// This blocks forever until stdin is received if we don't
+	// use StdinPipe. It's unknown what causes this.
+	stdinPipe, err := cmd.StdinPipe()
+	if err != nil {
+		s.metrics.sessionErrors.WithLabelValues(magicTypeLabel, "no", "stdin_pipe").Add(1)
+		return xerrors.Errorf("create stdin pipe: %w", err)
+	}
+	go func() {
+		_, err := io.Copy(stdinPipe, session)
+		if err != nil {
+			s.metrics.sessionErrors.WithLabelValues(magicTypeLabel, "no", "stdin_io_copy").Add(1)
+		}
+		_ = stdinPipe.Close()
+	}()
+	err = cmd.Start()
+	if err != nil {
+		s.metrics.sessionErrors.WithLabelValues(magicTypeLabel, "no", "start_command").Add(1)
+		return xerrors.Errorf("start: %w", err)
+	}
+	return cmd.Wait()
+}
+
+// ptySession is the interface to the ssh.Session that startPTYSession uses
+// we use an interface here so that we can fake it in tests.
+type ptySession interface {
+	io.ReadWriter
+	Context() ssh.Context
+	DisablePTYEmulation()
+	RawCommand() string
+}
+
+func (s *Server) startPTYSession(session ptySession, magicTypeLabel string, cmd *pty.Cmd, sshPty ssh.Pty, windowSize <-chan ssh.Window) (retErr error) {
+	s.metrics.sessionsTotal.WithLabelValues(magicTypeLabel, "yes").Add(1)
+
+	ctx := session.Context()
+	// Disable minimal PTY emulation set by gliderlabs/ssh (NL-to-CRNL).
+	// See https://github.com/coder/coder/issues/3371.
+	session.DisablePTYEmulation()
+
+	if isLoginShell(session.RawCommand()) {
+		serviceBanner := s.ServiceBanner.Load()
+		if serviceBanner != nil {
+			err := showServiceBanner(session, serviceBanner)
+			if err != nil {
+				s.logger.Error(ctx, "agent failed to show service banner", slog.Error(err))
+				s.metrics.sessionErrors.WithLabelValues(magicTypeLabel, "yes", "service_banner").Add(1)
+			}
+		}
+	}
+
+	if !isQuietLogin(s.fs, session.RawCommand()) {
+		manifest := s.Manifest.Load()
+		if manifest != nil {
+			err := showMOTD(s.fs, session, manifest.MOTDFile)
+			if err != nil {
+				s.logger.Error(ctx, "agent failed to show MOTD", slog.Error(err))
+				s.metrics.sessionErrors.WithLabelValues(magicTypeLabel, "yes", "motd").Add(1)
+			}
+		} else {
+			s.logger.Warn(ctx, "metadata lookup failed, unable to show MOTD")
+		}
+	}
+
+	cmd.Env = append(cmd.Env, fmt.Sprintf("TERM=%s", sshPty.Term))
+
+	// The pty package sets `SSH_TTY` on supported platforms.
+	ptty, process, err := pty.Start(cmd, pty.WithPTYOption(
+		pty.WithSSHRequest(sshPty),
+		pty.WithLogger(slog.Stdlib(ctx, s.logger, slog.LevelInfo)),
+	))
+	if err != nil {
+		s.metrics.sessionErrors.WithLabelValues(magicTypeLabel, "yes", "start_command").Add(1)
+		return xerrors.Errorf("start command: %w", err)
+	}
+	defer func() {
+		closeErr := ptty.Close()
+		if closeErr != nil {
+			s.logger.Warn(ctx, "failed to close tty", slog.Error(closeErr))
+			s.metrics.sessionErrors.WithLabelValues(magicTypeLabel, "yes", "close").Add(1)
+			if retErr == nil {
+				retErr = closeErr
+			}
+		}
+	}()
+	go func() {
+		for win := range windowSize {
+			resizeErr := ptty.Resize(uint16(win.Height), uint16(win.Width))
+			// If the pty is closed, then command has exited, no need to log.
+			if resizeErr != nil && !errors.Is(resizeErr, pty.ErrClosed) {
+				s.logger.Warn(ctx, "failed to resize tty", slog.Error(resizeErr))
+				s.metrics.sessionErrors.WithLabelValues(magicTypeLabel, "yes", "resize").Add(1)
+			}
+		}
+	}()
+
+	go func() {
+		_, err := io.Copy(ptty.InputWriter(), session)
+		if err != nil {
+			s.metrics.sessionErrors.WithLabelValues(magicTypeLabel, "yes", "input_io_copy").Add(1)
+		}
+	}()
+
+	// We need to wait for the command output to finish copying.  It's safe to
+	// just do this copy on the main handler goroutine because one of two things
+	// will happen:
+	//
+	// 1. The command completes & closes the TTY, which then triggers an error
+	//    after we've Read() all the buffered data from the PTY.
+	// 2. The client hangs up, which cancels the command's Context, and go will
+	//    kill the command's process.  This then has the same effect as (1).
+	n, err := io.Copy(session, ptty.OutputReader())
+	s.logger.Debug(ctx, "copy output done", slog.F("bytes", n), slog.Error(err))
+	if err != nil {
+		s.metrics.sessionErrors.WithLabelValues(magicTypeLabel, "yes", "output_io_copy").Add(1)
+		return xerrors.Errorf("copy error: %w", err)
+	}
+	// We've gotten all the output, but we need to wait for the process to
+	// complete so that we can get the exit code.  This returns
+	// immediately if the TTY was closed as part of the command exiting.
+	err = process.Wait()
+	var exitErr *exec.ExitError
+	// ExitErrors just mean the command we run returned a non-zero exit code, which is normal
+	// and not something to be concerned about.  But, if it's something else, we should log it.
+	if err != nil && !xerrors.As(err, &exitErr) {
+		s.logger.Warn(ctx, "process wait exited with error", slog.Error(err))
+		s.metrics.sessionErrors.WithLabelValues(magicTypeLabel, "yes", "wait").Add(1)
+	}
+	if err != nil {
+		return xerrors.Errorf("process wait: %w", err)
+	}
+	return nil
+}
+
+func (s *Server) sftpHandler(session ssh.Session) {
+	s.metrics.sftpConnectionsTotal.Add(1)
+
+	ctx := session.Context()
+
+	// Typically sftp sessions don't request a TTY, but if they do,
+	// we must ensure the gliderlabs/ssh CRLF emulation is disabled.
+	// Otherwise sftp will be broken. This can happen if a user sets
+	// `RequestTTY force` in their SSH config.
+	session.DisablePTYEmulation()
+
+	var opts []sftp.ServerOption
+	// Change current working directory to the users home
+	// directory so that SFTP connections land there.
+	homedir, err := userHomeDir()
+	if err != nil {
+		s.logger.Warn(ctx, "get sftp working directory failed, unable to get home dir", slog.Error(err))
+	} else {
+		opts = append(opts, sftp.WithServerWorkingDirectory(homedir))
+	}
+
+	server, err := sftp.NewServer(session, opts...)
+	if err != nil {
+		s.logger.Debug(ctx, "initialize sftp server", slog.Error(err))
+		return
+	}
+	defer server.Close()
+
+	err = server.Serve()
+	if errors.Is(err, io.EOF) {
+		// Unless we call `session.Exit(0)` here, the client won't
+		// receive `exit-status` because `(*sftp.Server).Close()`
+		// calls `Close()` on the underlying connection (session),
+		// which actually calls `channel.Close()` because it isn't
+		// wrapped. This causes sftp clients to receive a non-zero
+		// exit code. Typically sftp clients don't echo this exit
+		// code but `scp` on macOS does (when using the default
+		// SFTP backend).
+		_ = session.Exit(0)
+		return
+	}
+	s.logger.Warn(ctx, "sftp server closed with error", slog.Error(err))
+	s.metrics.sftpServerErrors.Add(1)
+	_ = session.Exit(1)
+}
+
+// CreateCommand processes raw command input with OpenSSH-like behavior.
+// If the script provided is empty, it will default to the users shell.
+// This injects environment variables specified by the user at launch too.
+func (s *Server) CreateCommand(ctx context.Context, script string, env []string) (*pty.Cmd, error) {
+	currentUser, err := user.Current()
+	if err != nil {
+		return nil, xerrors.Errorf("get current user: %w", err)
+	}
+	username := currentUser.Username
+
+	shell, err := usershell.Get(username)
+	if err != nil {
+		return nil, xerrors.Errorf("get user shell: %w", err)
+	}
+
+	manifest := s.Manifest.Load()
+	if manifest == nil {
+		return nil, xerrors.Errorf("no metadata was provided")
+	}
+
+	// OpenSSH executes all commands with the users current shell.
+	// We replicate that behavior for IDE support.
+	caller := "-c"
+	if runtime.GOOS == "windows" {
+		caller = "/c"
+	}
+	name := shell
+	args := []string{caller, script}
+
+	// A preceding space is generally not idiomatic for a shebang,
+	// but in Terraform it's quite standard to use <<EOF for a multi-line
+	// string which would indent with spaces, so we accept it for user-ease.
+	if strings.HasPrefix(strings.TrimSpace(script), "#!") {
+		// If the script starts with a shebang, we should
+		// execute it directly. This is useful for running
+		// scripts that aren't executable.
+		shebang := strings.SplitN(strings.TrimSpace(script), "\n", 2)[0]
+		shebang = strings.TrimSpace(shebang)
+		shebang = strings.TrimPrefix(shebang, "#!")
+		words, err := shellquote.Split(shebang)
+		if err != nil {
+			return nil, xerrors.Errorf("split shebang: %w", err)
+		}
+		name = words[0]
+		if len(words) > 1 {
+			args = words[1:]
+		} else {
+			args = []string{}
+		}
+		args = append(args, caller, script)
+	}
+
+	// gliderlabs/ssh returns a command slice of zero
+	// when a shell is requested.
+	if len(script) == 0 {
+		args = []string{}
+		if runtime.GOOS != "windows" {
+			// On Linux and macOS, we should start a login
+			// shell to consume juicy environment variables!
+			args = append(args, "-l")
+		}
+	}
+
+	cmd := pty.CommandContext(ctx, name, args...)
+	cmd.Dir = manifest.Directory
+
+	// If the metadata directory doesn't exist, we run the command
+	// in the users home directory.
+	_, err = os.Stat(cmd.Dir)
+	if cmd.Dir == "" || err != nil {
+		// Default to user home if a directory is not set.
+		homedir, err := userHomeDir()
+		if err != nil {
+			return nil, xerrors.Errorf("get home dir: %w", err)
+		}
+		cmd.Dir = homedir
+	}
+	cmd.Env = append(os.Environ(), env...)
+	executablePath, err := os.Executable()
+	if err != nil {
+		return nil, xerrors.Errorf("getting os executable: %w", err)
+	}
+	// Set environment variables reliable detection of being inside a
+	// Coder workspace.
+	cmd.Env = append(cmd.Env, "CODER=true")
+	cmd.Env = append(cmd.Env, fmt.Sprintf("USER=%s", username))
+	// Git on Windows resolves with UNIX-style paths.
+	// If using backslashes, it's unable to find the executable.
+	unixExecutablePath := strings.ReplaceAll(executablePath, "\\", "/")
+	cmd.Env = append(cmd.Env, fmt.Sprintf(`GIT_SSH_COMMAND=%s gitssh --`, unixExecutablePath))
+
+	// Specific Coder subcommands require the agent token exposed!
+	cmd.Env = append(cmd.Env, fmt.Sprintf("CODER_AGENT_TOKEN=%s", s.AgentToken()))
+
+	// Set SSH connection environment variables (these are also set by OpenSSH
+	// and thus expected to be present by SSH clients). Since the agent does
+	// networking in-memory, trying to provide accurate values here would be
+	// nonsensical. For now, we hard code these values so that they're present.
+	srcAddr, srcPort := "0.0.0.0", "0"
+	dstAddr, dstPort := "0.0.0.0", "0"
+	cmd.Env = append(cmd.Env, fmt.Sprintf("SSH_CLIENT=%s %s %s", srcAddr, srcPort, dstPort))
+	cmd.Env = append(cmd.Env, fmt.Sprintf("SSH_CONNECTION=%s %s %s %s", srcAddr, srcPort, dstAddr, dstPort))
+
+	// This adds the ports dialog to code-server that enables
+	// proxying a port dynamically.
+	cmd.Env = append(cmd.Env, fmt.Sprintf("VSCODE_PROXY_URI=%s", manifest.VSCodePortProxyURI))
+
+	// Hide Coder message on code-server's "Getting Started" page
+	cmd.Env = append(cmd.Env, "CS_DISABLE_GETTING_STARTED_OVERRIDE=true")
+
+	// Load environment variables passed via the agent.
+	// These should override all variables we manually specify.
+	for envKey, value := range manifest.EnvironmentVariables {
+		// Expanding environment variables allows for customization
+		// of the $PATH, among other variables. Customers can prepend
+		// or append to the $PATH, so allowing expand is required!
+		cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", envKey, os.ExpandEnv(value)))
+	}
+
+	// Agent-level environment variables should take over all!
+	// This is used for setting agent-specific variables like "CODER_AGENT_TOKEN".
+	for envKey, value := range s.Env {
+		cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", envKey, value))
+	}
+
+	return cmd, nil
+}
+
+func (s *Server) Serve(l net.Listener) (retErr error) {
+	s.logger.Info(context.Background(), "started serving listener", slog.F("listen_addr", l.Addr()))
+	defer func() {
+		s.logger.Info(context.Background(), "stopped serving listener",
+			slog.F("listen_addr", l.Addr()), slog.Error(retErr))
+	}()
+	defer l.Close()
+
+	s.trackListener(l, true)
+	defer s.trackListener(l, false)
+	for {
+		conn, err := l.Accept()
+		if err != nil {
+			return err
+		}
+		go s.handleConn(l, conn)
+	}
+}
+
+func (s *Server) handleConn(l net.Listener, c net.Conn) {
+	logger := s.logger.With(
+		slog.F("remote_addr", c.RemoteAddr()),
+		slog.F("local_addr", c.LocalAddr()),
+		slog.F("listen_addr", l.Addr()))
+	defer c.Close()
+
+	if !s.trackConn(l, c, true) {
+		// Server is closed or we no longer want
+		// connections from this listener.
+		logger.Info(context.Background(), "received connection after server closed")
+		return
+	}
+	defer s.trackConn(l, c, false)
+	logger.Info(context.Background(), "started serving connection")
+	// note: srv.ConnectionCompleteCallback logs completion of the connection
+	s.srv.HandleConn(c)
+}
+
+// trackListener registers the listener with the server. If the server is
+// closing, the function will block until the server is closed.
+//
+//nolint:revive
+func (s *Server) trackListener(l net.Listener, add bool) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if add {
+		for s.closing != nil {
+			closing := s.closing
+			// Wait until close is complete before
+			// serving a new listener.
+			s.mu.Unlock()
+			<-closing
+			s.mu.Lock()
+		}
+		s.wg.Add(1)
+		s.listeners[l] = struct{}{}
+		return
+	}
+	s.wg.Done()
+	delete(s.listeners, l)
+}
+
+// trackConn registers the connection with the server. If the server is
+// closed or the listener is closed, the connection is not registered
+// and should be closed.
+//
+//nolint:revive
+func (s *Server) trackConn(l net.Listener, c net.Conn, add bool) (ok bool) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if add {
+		found := false
+		for ll := range s.listeners {
+			if l == ll {
+				found = true
+				break
+			}
+		}
+		if s.closing != nil || !found {
+			// Server or listener closed.
+			return false
+		}
+		s.wg.Add(1)
+		s.conns[c] = struct{}{}
+		return true
+	}
+	s.wg.Done()
+	delete(s.conns, c)
+	return true
+}
+
+// trackSession registers the session with the server. If the server is
+// closing, the session is not registered and should be closed.
+//
+//nolint:revive
+func (s *Server) trackSession(ss ssh.Session, add bool) (ok bool) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if add {
+		if s.closing != nil {
+			// Server closed.
+			return false
+		}
+		s.wg.Add(1)
+		s.sessions[ss] = struct{}{}
+		return true
+	}
+	s.wg.Done()
+	delete(s.sessions, ss)
+	return true
+}
+
+// Close the server and all active connections. Server can be re-used
+// after Close is done.
+func (s *Server) Close() error {
+	s.mu.Lock()
+
+	// Guard against multiple calls to Close and
+	// accepting new connections during close.
+	if s.closing != nil {
+		s.mu.Unlock()
+		return xerrors.New("server is closing")
+	}
+	s.closing = make(chan struct{})
+
+	// Close all active sessions to gracefully
+	// terminate client connections.
+	for ss := range s.sessions {
+		// We call Close on the underlying channel here because we don't
+		// want to send an exit status to the client (via Exit()).
+		// Typically OpenSSH clients will return 255 as the exit status.
+		_ = ss.Close()
+	}
+
+	// Close all active listeners and connections.
+	for l := range s.listeners {
+		_ = l.Close()
+	}
+	for c := range s.conns {
+		_ = c.Close()
+	}
+
+	// Close the underlying SSH server.
+	err := s.srv.Close()
+
+	s.mu.Unlock()
+	s.wg.Wait() // Wait for all goroutines to exit.
+
+	s.mu.Lock()
+	close(s.closing)
+	s.closing = nil
+	s.mu.Unlock()
+
+	return err
+}
+
+// Shutdown gracefully closes all active SSH connections and stops
+// accepting new connections.
+//
+// Shutdown is not implemented.
+func (*Server) Shutdown(_ context.Context) error {
+	// TODO(mafredri): Implement shutdown, SIGHUP running commands, etc.
+	return nil
+}
+
+func isLoginShell(rawCommand string) bool {
+	return len(rawCommand) == 0
+}
+
+// isQuietLogin checks if the SSH server should perform a quiet login or not.
+//
+// https://github.com/openssh/openssh-portable/blob/25bd659cc72268f2858c5415740c442ee950049f/session.c#L816
+func isQuietLogin(fs afero.Fs, rawCommand string) bool {
+	// We are always quiet unless this is a login shell.
+	if !isLoginShell(rawCommand) {
+		return true
+	}
+
+	// Best effort, if we can't get the home directory,
+	// we can't lookup .hushlogin.
+	homedir, err := userHomeDir()
+	if err != nil {
+		return false
+	}
+
+	_, err = fs.Stat(filepath.Join(homedir, ".hushlogin"))
+	return err == nil
+}
+
+// showServiceBanner will write the service banner if enabled and not blank
+// along with a blank line for spacing.
+func showServiceBanner(session io.Writer, banner *codersdk.ServiceBannerConfig) error {
+	if banner.Enabled && banner.Message != "" {
+		// The banner supports Markdown so we might want to parse it but Markdown is
+		// still fairly readable in its raw form.
+		message := strings.TrimSpace(banner.Message) + "\n\n"
+		return writeWithCarriageReturn(strings.NewReader(message), session)
+	}
+	return nil
+}
+
+// showMOTD will output the message of the day from
+// the given filename to dest, if the file exists.
+//
+// https://github.com/openssh/openssh-portable/blob/25bd659cc72268f2858c5415740c442ee950049f/session.c#L784
+func showMOTD(fs afero.Fs, dest io.Writer, filename string) error {
+	if filename == "" {
+		return nil
+	}
+
+	f, err := fs.Open(filename)
+	if err != nil {
+		if xerrors.Is(err, os.ErrNotExist) {
+			// This is not an error, there simply isn't a MOTD to show.
+			return nil
+		}
+		return xerrors.Errorf("open MOTD: %w", err)
+	}
+	defer f.Close()
+
+	return writeWithCarriageReturn(f, dest)
+}
+
+// writeWithCarriageReturn writes each line with a carriage return to ensure
+// that each line starts at the beginning of the terminal.
+func writeWithCarriageReturn(src io.Reader, dest io.Writer) error {
+	s := bufio.NewScanner(src)
+	for s.Scan() {
+		_, err := fmt.Fprint(dest, s.Text()+"\r\n")
+		if err != nil {
+			return xerrors.Errorf("write line: %w", err)
+		}
+	}
+	if err := s.Err(); err != nil {
+		return xerrors.Errorf("read line: %w", err)
+	}
+	return nil
+}
+
+// userHomeDir returns the home directory of the current user, giving
+// priority to the $HOME environment variable.
+func userHomeDir() (string, error) {
+	// First we check the environment.
+	homedir, err := os.UserHomeDir()
+	if err == nil {
+		return homedir, nil
+	}
+
+	// As a fallback, we try the user information.
+	u, err := user.Current()
+	if err != nil {
+		return "", xerrors.Errorf("current user: %w", err)
+	}
+	return u.HomeDir, nil
+}

agent/agentssh/agentssh_internal_test.go

@@ -0,0 +1,197 @@
+//go:build !windows
+
+package agentssh
+
+import (
+	"bufio"
+	"context"
+	"io"
+	"net"
+	"testing"
+
+	gliderssh "github.com/gliderlabs/ssh"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/spf13/afero"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/coder/coder/v2/pty"
+	"github.com/coder/coder/v2/testutil"
+
+	"cdr.dev/slog/sloggers/slogtest"
+)
+
+const longScript = `
+echo "started"
+sleep 30
+echo "done"
+`
+
+// Test_sessionStart_orphan tests running a command that takes a long time to
+// exit normally, and terminate the SSH session context early to verify that we
+// return quickly and don't leave the command running as an "orphan" with no
+// active SSH session.
+func Test_sessionStart_orphan(t *testing.T) {
+	t.Parallel()
+
+	ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitMedium)
+	defer cancel()
+	logger := slogtest.Make(t, nil)
+	s, err := NewServer(ctx, logger, prometheus.NewRegistry(), afero.NewMemMapFs(), 0, "")
+	require.NoError(t, err)
+	defer s.Close()
+
+	// Here we're going to call the handler directly with a faked SSH session
+	// that just uses io.Pipes instead of a network socket.  There is a large
+	// variation in the time between closing the socket from the client side and
+	// the SSH server canceling the session Context, which would lead to a flaky
+	// test if we did it that way.  So instead, we directly cancel the context
+	// in this test.
+	sessionCtx, sessionCancel := context.WithCancel(ctx)
+	toClient, fromClient, sess := newTestSession(sessionCtx)
+	ptyInfo := gliderssh.Pty{}
+	windowSize := make(chan gliderssh.Window)
+	close(windowSize)
+	// the command gets the session context so that Go will terminate it when
+	// the session expires.
+	cmd := pty.CommandContext(sessionCtx, "sh", "-c", longScript)
+
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+
+		// we don't really care what the error is here.  In the larger scenario,
+		// the client has disconnected, so we can't return any error information
+		// to them.
+		_ = s.startPTYSession(sess, "ssh", cmd, ptyInfo, windowSize)
+	}()
+
+	readDone := make(chan struct{})
+	go func() {
+		defer close(readDone)
+		s := bufio.NewScanner(toClient)
+		assert.True(t, s.Scan())
+		txt := s.Text()
+		assert.Equal(t, "started", txt, "output corrupted")
+	}()
+
+	waitForChan(ctx, t, readDone, "read timeout")
+	// process is started, and should be sleeping for ~30 seconds
+
+	sessionCancel()
+
+	// now, we wait for the handler to complete.  If it does so before the
+	// main test timeout, we consider this a pass.  If not, it indicates
+	// that the server isn't properly shutting down sessions when they are
+	// disconnected client side, which could lead to processes hanging around
+	// indefinitely.
+	waitForChan(ctx, t, done, "handler timeout")
+
+	err = fromClient.Close()
+	require.NoError(t, err)
+}
+
+func waitForChan(ctx context.Context, t *testing.T, c <-chan struct{}, msg string) {
+	t.Helper()
+	select {
+	case <-c:
+		// OK!
+	case <-ctx.Done():
+		t.Fatal(msg)
+	}
+}
+
+type testSession struct {
+	ctx testSSHContext
+
+	// c2p is the client -> pty buffer
+	toPty *io.PipeReader
+	// p2c is the pty -> client buffer
+	fromPty *io.PipeWriter
+}
+
+type testSSHContext struct {
+	context.Context
+}
+
+func newTestSession(ctx context.Context) (toClient *io.PipeReader, fromClient *io.PipeWriter, s ptySession) {
+	toClient, fromPty := io.Pipe()
+	toPty, fromClient := io.Pipe()
+
+	return toClient, fromClient, &testSession{
+		ctx:     testSSHContext{ctx},
+		toPty:   toPty,
+		fromPty: fromPty,
+	}
+}
+
+func (s *testSession) Context() gliderssh.Context {
+	return s.ctx
+}
+
+func (*testSession) DisablePTYEmulation() {}
+
+// RawCommand returns "quiet logon" so that the PTY handler doesn't attempt to
+// write the message of the day, which will interfere with our tests.  It writes
+// the message of the day if it's a shell login (zero length RawCommand()).
+func (*testSession) RawCommand() string { return "quiet logon" }
+
+func (s *testSession) Read(p []byte) (n int, err error) {
+	return s.toPty.Read(p)
+}
+
+func (s *testSession) Write(p []byte) (n int, err error) {
+	return s.fromPty.Write(p)
+}
+
+func (testSSHContext) Lock() {
+	panic("not implemented")
+}
+
+func (testSSHContext) Unlock() {
+	panic("not implemented")
+}
+
+// User returns the username used when establishing the SSH connection.
+func (testSSHContext) User() string {
+	panic("not implemented")
+}
+
+// SessionID returns the session hash.
+func (testSSHContext) SessionID() string {
+	panic("not implemented")
+}
+
+// ClientVersion returns the version reported by the client.
+func (testSSHContext) ClientVersion() string {
+	panic("not implemented")
+}
+
+// ServerVersion returns the version reported by the server.
+func (testSSHContext) ServerVersion() string {
+	panic("not implemented")
+}
+
+// RemoteAddr returns the remote address for this connection.
+func (testSSHContext) RemoteAddr() net.Addr {
+	panic("not implemented")
+}
+
+// LocalAddr returns the local address for this connection.
+func (testSSHContext) LocalAddr() net.Addr {
+	panic("not implemented")
+}
+
+// Permissions returns the Permissions object used for this connection.
+func (testSSHContext) Permissions() *gliderssh.Permissions {
+	panic("not implemented")
+}
+
+// SetValue allows you to easily write new values into the underlying context.
+func (testSSHContext) SetValue(_, _ interface{}) {
+	panic("not implemented")
+}
+
+func (testSSHContext) KeepAlive() *gliderssh.SessionKeepAlive {
+	panic("not implemented")
+}

agent/agentssh/agentssh_test.go

@@ -0,0 +1,181 @@
+// Package agentssh_test provides tests for basic functinoality of the agentssh
+// package, more test coverage can be found in the `agent` and `cli` package(s).
+package agentssh_test
+
+import (
+	"bytes"
+	"context"
+	"net"
+	"runtime"
+	"strings"
+	"sync"
+	"testing"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/spf13/afero"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.uber.org/atomic"
+	"go.uber.org/goleak"
+	"golang.org/x/crypto/ssh"
+
+	"cdr.dev/slog/sloggers/slogtest"
+
+	"github.com/coder/coder/v2/agent/agentssh"
+	"github.com/coder/coder/v2/codersdk/agentsdk"
+	"github.com/coder/coder/v2/pty/ptytest"
+)
+
+func TestMain(m *testing.M) {
+	goleak.VerifyTestMain(m)
+}
+
+func TestNewServer_ServeClient(t *testing.T) {
+	t.Parallel()
+
+	ctx := context.Background()
+	logger := slogtest.Make(t, nil)
+	s, err := agentssh.NewServer(ctx, logger, prometheus.NewRegistry(), afero.NewMemMapFs(), 0, "")
+	require.NoError(t, err)
+	defer s.Close()
+
+	// The assumption is that these are set before serving SSH connections.
+	s.AgentToken = func() string { return "" }
+	s.Manifest = atomic.NewPointer(&agentsdk.Manifest{})
+
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	require.NoError(t, err)
+
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		err := s.Serve(ln)
+		assert.Error(t, err) // Server is closed.
+	}()
+
+	c := sshClient(t, ln.Addr().String())
+
+	var b bytes.Buffer
+	sess, err := c.NewSession()
+	sess.Stdout = &b
+	require.NoError(t, err)
+	err = sess.Start("echo hello")
+	require.NoError(t, err)
+
+	err = sess.Wait()
+	require.NoError(t, err)
+
+	require.Equal(t, "hello", strings.TrimSpace(b.String()))
+
+	err = s.Close()
+	require.NoError(t, err)
+	<-done
+}
+
+func TestNewServer_ExecuteShebang(t *testing.T) {
+	t.Parallel()
+	if runtime.GOOS == "windows" {
+		t.Skip("bash doesn't exist on Windows")
+	}
+
+	ctx := context.Background()
+	logger := slogtest.Make(t, nil)
+	s, err := agentssh.NewServer(ctx, logger, prometheus.NewRegistry(), afero.NewMemMapFs(), 0, "")
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		_ = s.Close()
+	})
+	s.AgentToken = func() string { return "" }
+	s.Manifest = atomic.NewPointer(&agentsdk.Manifest{})
+
+	t.Run("Basic", func(t *testing.T) {
+		t.Parallel()
+		cmd, err := s.CreateCommand(ctx, `#!/bin/bash
+		echo test`, nil)
+		require.NoError(t, err)
+		output, err := cmd.AsExec().CombinedOutput()
+		require.NoError(t, err)
+		require.Equal(t, "test\n", string(output))
+	})
+	t.Run("Args", func(t *testing.T) {
+		t.Parallel()
+		cmd, err := s.CreateCommand(ctx, `#!/usr/bin/env bash
+		echo test`, nil)
+		require.NoError(t, err)
+		output, err := cmd.AsExec().CombinedOutput()
+		require.NoError(t, err)
+		require.Equal(t, "test\n", string(output))
+	})
+}
+
+func TestNewServer_CloseActiveConnections(t *testing.T) {
+	t.Parallel()
+
+	ctx := context.Background()
+	logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true})
+	s, err := agentssh.NewServer(ctx, logger, prometheus.NewRegistry(), afero.NewMemMapFs(), 0, "")
+	require.NoError(t, err)
+	defer s.Close()
+
+	// The assumption is that these are set before serving SSH connections.
+	s.AgentToken = func() string { return "" }
+	s.Manifest = atomic.NewPointer(&agentsdk.Manifest{})
+
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	require.NoError(t, err)
+
+	var wg sync.WaitGroup
+	wg.Add(2)
+	go func() {
+		defer wg.Done()
+		err := s.Serve(ln)
+		assert.Error(t, err) // Server is closed.
+	}()
+
+	pty := ptytest.New(t)
+
+	doClose := make(chan struct{})
+	go func() {
+		defer wg.Done()
+		c := sshClient(t, ln.Addr().String())
+		sess, err := c.NewSession()
+		sess.Stdin = pty.Input()
+		sess.Stdout = pty.Output()
+		sess.Stderr = pty.Output()
+
+		assert.NoError(t, err)
+		err = sess.Start("")
+		assert.NoError(t, err)
+
+		close(doClose)
+		err = sess.Wait()
+		assert.Error(t, err)
+	}()
+
+	<-doClose
+	err = s.Close()
+	require.NoError(t, err)
+
+	wg.Wait()
+}
+
+func sshClient(t *testing.T, addr string) *ssh.Client {
+	conn, err := net.Dial("tcp", addr)
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		_ = conn.Close()
+	})
+
+	sshConn, channels, requests, err := ssh.NewClientConn(conn, "localhost:22", &ssh.ClientConfig{
+		HostKeyCallback: ssh.InsecureIgnoreHostKey(), //nolint:gosec // This is a test.
+	})
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		_ = sshConn.Close()
+	})
+	c := ssh.NewClient(sshConn, channels, requests)
+	t.Cleanup(func() {
+		_ = c.Close()
+	})
+	return c
+}

agent/agentssh/bicopy.go

@@ -0,0 +1,47 @@
+package agentssh
+
+import (
+	"context"
+	"io"
+	"sync"
+)
+
+// Bicopy copies all of the data between the two connections and will close them
+// after one or both of them are done writing. If the context is canceled, both
+// of the connections will be closed.
+func Bicopy(ctx context.Context, c1, c2 io.ReadWriteCloser) {
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	defer func() {
+		_ = c1.Close()
+		_ = c2.Close()
+	}()
+
+	var wg sync.WaitGroup
+	copyFunc := func(dst io.WriteCloser, src io.Reader) {
+		defer func() {
+			wg.Done()
+			// If one side of the copy fails, ensure the other one exits as
+			// well.
+			cancel()
+		}()
+		_, _ = io.Copy(dst, src)
+	}
+
+	wg.Add(2)
+	go copyFunc(c1, c2)
+	go copyFunc(c2, c1)
+
+	// Convert waitgroup to a channel so we can also wait on the context.
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		wg.Wait()
+	}()
+
+	select {
+	case <-ctx.Done():
+	case <-done:
+	}
+}

agent/agentssh/forward.go

@@ -1,4 +1,4 @@
-package agent
+package agentssh
 
 import (
 	"context"

agent/agentssh/metrics.go

@@ -0,0 +1,85 @@
+package agentssh
+
+import (
+	"strings"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+type sshServerMetrics struct {
+	failedConnectionsTotal prometheus.Counter
+	sftpConnectionsTotal   prometheus.Counter
+	sftpServerErrors       prometheus.Counter
+	x11HandlerErrors       *prometheus.CounterVec
+	sessionsTotal          *prometheus.CounterVec
+	sessionErrors          *prometheus.CounterVec
+}
+
+func newSSHServerMetrics(registerer prometheus.Registerer) *sshServerMetrics {
+	failedConnectionsTotal := prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace: "agent", Subsystem: "ssh_server", Name: "failed_connections_total",
+	})
+	registerer.MustRegister(failedConnectionsTotal)
+
+	sftpConnectionsTotal := prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace: "agent", Subsystem: "ssh_server", Name: "sftp_connections_total",
+	})
+	registerer.MustRegister(sftpConnectionsTotal)
+
+	sftpServerErrors := prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace: "agent", Subsystem: "ssh_server", Name: "sftp_server_errors_total",
+	})
+	registerer.MustRegister(sftpServerErrors)
+
+	x11HandlerErrors := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: "agent",
+			Subsystem: "x11_handler",
+			Name:      "errors_total",
+		},
+		[]string{"error_type"},
+	)
+	registerer.MustRegister(x11HandlerErrors)
+
+	sessionsTotal := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: "agent",
+			Subsystem: "sessions",
+			Name:      "total",
+		},
+		[]string{"magic_type", "pty"},
+	)
+	registerer.MustRegister(sessionsTotal)
+
+	sessionErrors := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: "agent",
+			Subsystem: "sessions",
+			Name:      "errors_total",
+		},
+		[]string{"magic_type", "pty", "error_type"},
+	)
+	registerer.MustRegister(sessionErrors)
+
+	return &sshServerMetrics{
+		failedConnectionsTotal: failedConnectionsTotal,
+		sftpConnectionsTotal:   sftpConnectionsTotal,
+		sftpServerErrors:       sftpServerErrors,
+		x11HandlerErrors:       x11HandlerErrors,
+		sessionsTotal:          sessionsTotal,
+		sessionErrors:          sessionErrors,
+	}
+}
+
+func magicTypeMetricLabel(magicType string) string {
+	switch magicType {
+	case MagicSessionTypeVSCode:
+	case MagicSessionTypeJetBrains:
+	case "":
+		magicType = "ssh"
+	default:
+		magicType = "unknown"
+	}
+	// Always be case insensitive
+	return strings.ToLower(magicType)
+}

agent/agentssh/x11.go

@@ -0,0 +1,200 @@
+package agentssh
+
+import (
+	"context"
+	"encoding/binary"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"net"
+	"os"
+	"path/filepath"
+	"strconv"
+	"time"
+
+	"github.com/gliderlabs/ssh"
+	"github.com/gofrs/flock"
+	"github.com/spf13/afero"
+	gossh "golang.org/x/crypto/ssh"
+	"golang.org/x/xerrors"
+
+	"cdr.dev/slog"
+)
+
+// x11Callback is called when the client requests X11 forwarding.
+// It adds an Xauthority entry to the Xauthority file.
+func (s *Server) x11Callback(ctx ssh.Context, x11 ssh.X11) bool {
+	hostname, err := os.Hostname()
+	if err != nil {
+		s.logger.Warn(ctx, "failed to get hostname", slog.Error(err))
+		s.metrics.x11HandlerErrors.WithLabelValues("hostname").Add(1)
+		return false
+	}
+
+	err = s.fs.MkdirAll(s.x11SocketDir, 0o700)
+	if err != nil {
+		s.logger.Warn(ctx, "failed to make the x11 socket dir", slog.F("dir", s.x11SocketDir), slog.Error(err))
+		s.metrics.x11HandlerErrors.WithLabelValues("socker_dir").Add(1)
+		return false
+	}
+
+	err = addXauthEntry(ctx, s.fs, hostname, strconv.Itoa(int(x11.ScreenNumber)), x11.AuthProtocol, x11.AuthCookie)
+	if err != nil {
+		s.logger.Warn(ctx, "failed to add Xauthority entry", slog.Error(err))
+		s.metrics.x11HandlerErrors.WithLabelValues("xauthority").Add(1)
+		return false
+	}
+	return true
+}
+
+// x11Handler is called when a session has requested X11 forwarding.
+// It listens for X11 connections and forwards them to the client.
+func (s *Server) x11Handler(ctx ssh.Context, x11 ssh.X11) bool {
+	serverConn, valid := ctx.Value(ssh.ContextKeyConn).(*gossh.ServerConn)
+	if !valid {
+		s.logger.Warn(ctx, "failed to get server connection")
+		return false
+	}
+	// We want to overwrite the socket so that subsequent connections will succeed.
+	socketPath := filepath.Join(s.x11SocketDir, fmt.Sprintf("X%d", x11.ScreenNumber))
+	err := os.Remove(socketPath)
+	if err != nil && !errors.Is(err, os.ErrNotExist) {
+		s.logger.Warn(ctx, "failed to remove existing X11 socket", slog.Error(err))
+		return false
+	}
+	listener, err := net.Listen("unix", socketPath)
+	if err != nil {
+		s.logger.Warn(ctx, "failed to listen for X11", slog.Error(err))
+		return false
+	}
+	s.trackListener(listener, true)
+
+	go func() {
+		defer listener.Close()
+		defer s.trackListener(listener, false)
+		handledFirstConnection := false
+
+		for {
+			conn, err := listener.Accept()
+			if err != nil {
+				if errors.Is(err, net.ErrClosed) {
+					return
+				}
+				s.logger.Warn(ctx, "failed to accept X11 connection", slog.Error(err))
+				return
+			}
+			if x11.SingleConnection && handledFirstConnection {
+				s.logger.Warn(ctx, "X11 connection rejected because single connection is enabled")
+				_ = conn.Close()
+				continue
+			}
+			handledFirstConnection = true
+
+			unixConn, ok := conn.(*net.UnixConn)
+			if !ok {
+				s.logger.Warn(ctx, fmt.Sprintf("failed to cast connection to UnixConn. got: %T", conn))
+				return
+			}
+			unixAddr, ok := unixConn.LocalAddr().(*net.UnixAddr)
+			if !ok {
+				s.logger.Warn(ctx, fmt.Sprintf("failed to cast local address to UnixAddr. got: %T", unixConn.LocalAddr()))
+				return
+			}
+
+			channel, reqs, err := serverConn.OpenChannel("x11", gossh.Marshal(struct {
+				OriginatorAddress string
+				OriginatorPort    uint32
+			}{
+				OriginatorAddress: unixAddr.Name,
+				OriginatorPort:    0,
+			}))
+			if err != nil {
+				s.logger.Warn(ctx, "failed to open X11 channel", slog.Error(err))
+				return
+			}
+			go gossh.DiscardRequests(reqs)
+			go Bicopy(ctx, conn, channel)
+		}
+	}()
+	return true
+}
+
+// addXauthEntry adds an Xauthority entry to the Xauthority file.
+// The Xauthority file is located at ~/.Xauthority.
+func addXauthEntry(ctx context.Context, fs afero.Fs, host string, display string, authProtocol string, authCookie string) error {
+	// Get the Xauthority file path
+	homeDir, err := os.UserHomeDir()
+	if err != nil {
+		return xerrors.Errorf("failed to get user home directory: %w", err)
+	}
+
+	xauthPath := filepath.Join(homeDir, ".Xauthority")
+
+	lock := flock.New(xauthPath)
+	defer lock.Close()
+	ok, err := lock.TryLockContext(ctx, 100*time.Millisecond)
+	if !ok {
+		return xerrors.Errorf("failed to lock Xauthority file: %w", err)
+	}
+	if err != nil {
+		return xerrors.Errorf("failed to lock Xauthority file: %w", err)
+	}
+
+	// Open or create the Xauthority file
+	file, err := fs.OpenFile(xauthPath, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0o600)
+	if err != nil {
+		return xerrors.Errorf("failed to open Xauthority file: %w", err)
+	}
+	defer file.Close()
+
+	// Convert the authCookie from hex string to byte slice
+	authCookieBytes, err := hex.DecodeString(authCookie)
+	if err != nil {
+		return xerrors.Errorf("failed to decode auth cookie: %w", err)
+	}
+
+	// Write Xauthority entry
+	family := uint16(0x0100) // FamilyLocal
+	err = binary.Write(file, binary.BigEndian, family)
+	if err != nil {
+		return xerrors.Errorf("failed to write family: %w", err)
+	}
+
+	err = binary.Write(file, binary.BigEndian, uint16(len(host)))
+	if err != nil {
+		return xerrors.Errorf("failed to write host length: %w", err)
+	}
+	_, err = file.WriteString(host)
+	if err != nil {
+		return xerrors.Errorf("failed to write host: %w", err)
+	}
+
+	err = binary.Write(file, binary.BigEndian, uint16(len(display)))
+	if err != nil {
+		return xerrors.Errorf("failed to write display length: %w", err)
+	}
+	_, err = file.WriteString(display)
+	if err != nil {
+		return xerrors.Errorf("failed to write display: %w", err)
+	}
+
+	err = binary.Write(file, binary.BigEndian, uint16(len(authProtocol)))
+	if err != nil {
+		return xerrors.Errorf("failed to write auth protocol length: %w", err)
+	}
+	_, err = file.WriteString(authProtocol)
+	if err != nil {
+		return xerrors.Errorf("failed to write auth protocol: %w", err)
+	}
+
+	err = binary.Write(file, binary.BigEndian, uint16(len(authCookieBytes)))
+	if err != nil {
+		return xerrors.Errorf("failed to write auth cookie length: %w", err)
+	}
+	_, err = file.Write(authCookieBytes)
+	if err != nil {
+		return xerrors.Errorf("failed to write auth cookie: %w", err)
+	}
+
+	return nil
+}

agent/agentssh/x11_test.go

@@ -0,0 +1,100 @@
+package agentssh_test
+
+import (
+	"context"
+	"encoding/hex"
+	"net"
+	"os"
+	"path/filepath"
+	"runtime"
+	"testing"
+
+	"github.com/gliderlabs/ssh"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/spf13/afero"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.uber.org/atomic"
+	gossh "golang.org/x/crypto/ssh"
+
+	"cdr.dev/slog"
+	"cdr.dev/slog/sloggers/slogtest"
+	"github.com/coder/coder/v2/agent/agentssh"
+	"github.com/coder/coder/v2/codersdk/agentsdk"
+	"github.com/coder/coder/v2/testutil"
+)
+
+func TestServer_X11(t *testing.T) {
+	t.Parallel()
+	if runtime.GOOS != "linux" {
+		t.Skip("X11 forwarding is only supported on Linux")
+	}
+
+	ctx := context.Background()
+	logger := slogtest.Make(t, nil).Leveled(slog.LevelDebug)
+	fs := afero.NewOsFs()
+	dir := t.TempDir()
+	s, err := agentssh.NewServer(ctx, logger, prometheus.NewRegistry(), fs, 0, dir)
+	require.NoError(t, err)
+	defer s.Close()
+
+	// The assumption is that these are set before serving SSH connections.
+	s.AgentToken = func() string { return "" }
+	s.Manifest = atomic.NewPointer(&agentsdk.Manifest{})
+
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	require.NoError(t, err)
+
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		err := s.Serve(ln)
+		assert.Error(t, err) // Server is closed.
+	}()
+
+	c := sshClient(t, ln.Addr().String())
+
+	sess, err := c.NewSession()
+	require.NoError(t, err)
+
+	reply, err := sess.SendRequest("x11-req", true, gossh.Marshal(ssh.X11{
+		AuthProtocol: "MIT-MAGIC-COOKIE-1",
+		AuthCookie:   hex.EncodeToString([]byte("cookie")),
+		ScreenNumber: 0,
+	}))
+	require.NoError(t, err)
+	assert.True(t, reply)
+
+	err = sess.Shell()
+	require.NoError(t, err)
+
+	x11Chans := c.HandleChannelOpen("x11")
+	payload := "hello world"
+	require.Eventually(t, func() bool {
+		conn, err := net.Dial("unix", filepath.Join(dir, "X0"))
+		if err == nil {
+			_, err = conn.Write([]byte(payload))
+			assert.NoError(t, err)
+			_ = conn.Close()
+		}
+		return err == nil
+	}, testutil.WaitShort, testutil.IntervalFast)
+
+	x11 := <-x11Chans
+	ch, reqs, err := x11.Accept()
+	require.NoError(t, err)
+	go gossh.DiscardRequests(reqs)
+	got := make([]byte, len(payload))
+	_, err = ch.Read(got)
+	require.NoError(t, err)
+	assert.Equal(t, payload, string(got))
+	_ = ch.Close()
+	_ = s.Close()
+	<-done
+
+	// Ensure the Xauthority file was written!
+	home, err := os.UserHomeDir()
+	require.NoError(t, err)
+	_, err = fs.Stat(filepath.Join(home, ".Xauthority"))
+	require.NoError(t, err)
+}

agent/agenttest/agent.go

@@ -0,0 +1,57 @@
+package agenttest
+
+import (
+	"context"
+	"net/url"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"cdr.dev/slog"
+	"cdr.dev/slog/sloggers/slogtest"
+	"github.com/coder/coder/v2/agent"
+	"github.com/coder/coder/v2/codersdk/agentsdk"
+)
+
+// New starts a new agent for use in tests.
+// The agent will use the provided coder URL and session token.
+// The options passed to agent.New() can be modified by passing an optional
+// variadic func(*agent.Options).
+// Returns the agent. Closing the agent is handled by the test cleanup.
+// It is the responsibility of the caller to call coderdtest.AwaitWorkspaceAgents
+// to ensure agent is connected.
+func New(t testing.TB, coderURL *url.URL, agentToken string, opts ...func(*agent.Options)) agent.Agent {
+	t.Helper()
+
+	var o agent.Options
+	log := slogtest.Make(t, nil).Leveled(slog.LevelDebug).Named("agent")
+	o.Logger = log
+
+	for _, opt := range opts {
+		opt(&o)
+	}
+
+	if o.Client == nil {
+		agentClient := agentsdk.New(coderURL)
+		agentClient.SetSessionToken(agentToken)
+		agentClient.SDK.SetLogger(log)
+		o.Client = agentClient
+	}
+
+	if o.ExchangeToken == nil {
+		o.ExchangeToken = func(_ context.Context) (string, error) {
+			return agentToken, nil
+		}
+	}
+
+	if o.LogDir == "" {
+		o.LogDir = t.TempDir()
+	}
+
+	agt := agent.New(o)
+	t.Cleanup(func() {
+		assert.NoError(t, agt.Close(), "failed to close agent during cleanup")
+	})
+
+	return agt
+}

agent/agenttest/client.go

@@ -0,0 +1,224 @@
+package agenttest
+
+import (
+	"context"
+	"io"
+	"net"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/google/uuid"
+	"golang.org/x/exp/maps"
+	"golang.org/x/xerrors"
+
+	"cdr.dev/slog"
+	"github.com/coder/coder/v2/codersdk"
+	"github.com/coder/coder/v2/codersdk/agentsdk"
+	"github.com/coder/coder/v2/tailnet"
+	"github.com/coder/coder/v2/testutil"
+)
+
+func NewClient(t testing.TB,
+	logger slog.Logger,
+	agentID uuid.UUID,
+	manifest agentsdk.Manifest,
+	statsChan chan *agentsdk.Stats,
+	coordinator tailnet.Coordinator,
+) *Client {
+	if manifest.AgentID == uuid.Nil {
+		manifest.AgentID = agentID
+	}
+	return &Client{
+		t:              t,
+		logger:         logger.Named("client"),
+		agentID:        agentID,
+		manifest:       manifest,
+		statsChan:      statsChan,
+		coordinator:    coordinator,
+		derpMapUpdates: make(chan agentsdk.DERPMapUpdate),
+	}
+}
+
+type Client struct {
+	t                    testing.TB
+	logger               slog.Logger
+	agentID              uuid.UUID
+	manifest             agentsdk.Manifest
+	metadata             map[string]agentsdk.Metadata
+	statsChan            chan *agentsdk.Stats
+	coordinator          tailnet.Coordinator
+	LastWorkspaceAgent   func()
+	PatchWorkspaceLogs   func() error
+	GetServiceBannerFunc func() (codersdk.ServiceBannerConfig, error)
+
+	mu              sync.Mutex // Protects following.
+	lifecycleStates []codersdk.WorkspaceAgentLifecycle
+	startup         agentsdk.PostStartupRequest
+	logs            []agentsdk.Log
+	derpMapUpdates  chan agentsdk.DERPMapUpdate
+}
+
+func (c *Client) Manifest(_ context.Context) (agentsdk.Manifest, error) {
+	return c.manifest, nil
+}
+
+func (c *Client) Listen(_ context.Context) (net.Conn, error) {
+	clientConn, serverConn := net.Pipe()
+	closed := make(chan struct{})
+	c.LastWorkspaceAgent = func() {
+		_ = serverConn.Close()
+		_ = clientConn.Close()
+		<-closed
+	}
+	c.t.Cleanup(c.LastWorkspaceAgent)
+	go func() {
+		_ = c.coordinator.ServeAgent(serverConn, c.agentID, "")
+		close(closed)
+	}()
+	return clientConn, nil
+}
+
+func (c *Client) ReportStats(ctx context.Context, _ slog.Logger, statsChan <-chan *agentsdk.Stats, setInterval func(time.Duration)) (io.Closer, error) {
+	doneCh := make(chan struct{})
+	ctx, cancel := context.WithCancel(ctx)
+
+	go func() {
+		defer close(doneCh)
+
+		setInterval(500 * time.Millisecond)
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case stat := <-statsChan:
+				select {
+				case c.statsChan <- stat:
+				case <-ctx.Done():
+					return
+				default:
+					// We don't want to send old stats.
+					continue
+				}
+			}
+		}
+	}()
+	return closeFunc(func() error {
+		cancel()
+		<-doneCh
+		close(c.statsChan)
+		return nil
+	}), nil
+}
+
+func (c *Client) GetLifecycleStates() []codersdk.WorkspaceAgentLifecycle {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return c.lifecycleStates
+}
+
+func (c *Client) PostLifecycle(ctx context.Context, req agentsdk.PostLifecycleRequest) error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.lifecycleStates = append(c.lifecycleStates, req.State)
+	c.logger.Debug(ctx, "post lifecycle", slog.F("req", req))
+	return nil
+}
+
+func (c *Client) PostAppHealth(ctx context.Context, req agentsdk.PostAppHealthsRequest) error {
+	c.logger.Debug(ctx, "post app health", slog.F("req", req))
+	return nil
+}
+
+func (c *Client) GetStartup() agentsdk.PostStartupRequest {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return c.startup
+}
+
+func (c *Client) GetMetadata() map[string]agentsdk.Metadata {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return maps.Clone(c.metadata)
+}
+
+func (c *Client) PostMetadata(ctx context.Context, req agentsdk.PostMetadataRequest) error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if c.metadata == nil {
+		c.metadata = make(map[string]agentsdk.Metadata)
+	}
+	for _, md := range req.Metadata {
+		c.metadata[md.Key] = md
+		c.logger.Debug(ctx, "post metadata", slog.F("key", md.Key), slog.F("md", md))
+	}
+	return nil
+}
+
+func (c *Client) PostStartup(ctx context.Context, startup agentsdk.PostStartupRequest) error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.startup = startup
+	c.logger.Debug(ctx, "post startup", slog.F("req", startup))
+	return nil
+}
+
+func (c *Client) GetStartupLogs() []agentsdk.Log {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return c.logs
+}
+
+func (c *Client) PatchLogs(ctx context.Context, logs agentsdk.PatchLogs) error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if c.PatchWorkspaceLogs != nil {
+		return c.PatchWorkspaceLogs()
+	}
+	c.logs = append(c.logs, logs.Logs...)
+	c.logger.Debug(ctx, "patch startup logs", slog.F("req", logs))
+	return nil
+}
+
+func (c *Client) SetServiceBannerFunc(f func() (codersdk.ServiceBannerConfig, error)) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	c.GetServiceBannerFunc = f
+}
+
+func (c *Client) GetServiceBanner(ctx context.Context) (codersdk.ServiceBannerConfig, error) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.logger.Debug(ctx, "get service banner")
+	if c.GetServiceBannerFunc != nil {
+		return c.GetServiceBannerFunc()
+	}
+	return codersdk.ServiceBannerConfig{}, nil
+}
+
+func (c *Client) PushDERPMapUpdate(update agentsdk.DERPMapUpdate) error {
+	timer := time.NewTimer(testutil.WaitShort)
+	defer timer.Stop()
+	select {
+	case c.derpMapUpdates <- update:
+	case <-timer.C:
+		return xerrors.New("timeout waiting to push derp map update")
+	}
+
+	return nil
+}
+
+func (c *Client) DERPMapUpdates(_ context.Context) (<-chan agentsdk.DERPMapUpdate, io.Closer, error) {
+	closed := make(chan struct{})
+	return c.derpMapUpdates, closeFunc(func() error {
+		close(closed)
+		return nil
+	}), nil
+}
+
+type closeFunc func() error
+
+func (c closeFunc) Close() error {
+	return c()
+}

agent/api.go

@@ -5,13 +5,13 @@ import (
 	"sync"
 	"time"
 
-	"github.com/go-chi/chi"
+	"github.com/go-chi/chi/v5"
 
-	"github.com/coder/coder/coderd/httpapi"
-	"github.com/coder/coder/codersdk"
+	"github.com/coder/coder/v2/coderd/httpapi"
+	"github.com/coder/coder/v2/codersdk"
 )
 
-func (*agent) apiHandler() http.Handler {
+func (a *agent) apiHandler() http.Handler {
 	r := chi.NewRouter()
 	r.Get("/", func(rw http.ResponseWriter, r *http.Request) {
 		httpapi.Write(r.Context(), rw, http.StatusOK, codersdk.Response{
@@ -19,16 +19,24 @@ func (*agent) apiHandler() http.Handler {
 		})
 	})
 
-	lp := &listeningPortsHandler{}
+	// Make a copy to ensure the map is not modified after the handler is
+	// created.
+	cpy := make(map[int]string)
+	for k, b := range a.ignorePorts {
+		cpy[k] = b
+	}
+
+	lp := &listeningPortsHandler{ignorePorts: cpy}
 	r.Get("/api/v0/listening-ports", lp.handler)
 
 	return r
 }
 
 type listeningPortsHandler struct {
-	mut   sync.Mutex
-	ports []codersdk.WorkspaceAgentListeningPort
-	mtime time.Time
+	mut         sync.Mutex
+	ports       []codersdk.WorkspaceAgentListeningPort
+	mtime       time.Time
+	ignorePorts map[int]string
 }
 
 // handler returns a list of listening ports. This is tested by coderd's

agent/apphealth.go

@@ -10,8 +10,8 @@ import (
 	"golang.org/x/xerrors"
 
 	"cdr.dev/slog"
-	"github.com/coder/coder/codersdk"
-	"github.com/coder/coder/codersdk/agentsdk"
+	"github.com/coder/coder/v2/codersdk"
+	"github.com/coder/coder/v2/codersdk/agentsdk"
 	"github.com/coder/retry"
 )
 

agent/apphealth_test.go

@@ -13,11 +13,11 @@ import (
 
 	"cdr.dev/slog"
 	"cdr.dev/slog/sloggers/slogtest"
-	"github.com/coder/coder/agent"
-	"github.com/coder/coder/coderd/httpapi"
-	"github.com/coder/coder/codersdk"
-	"github.com/coder/coder/codersdk/agentsdk"
-	"github.com/coder/coder/testutil"
+	"github.com/coder/coder/v2/agent"
+	"github.com/coder/coder/v2/coderd/httpapi"
+	"github.com/coder/coder/v2/codersdk"
+	"github.com/coder/coder/v2/codersdk/agentsdk"
+	"github.com/coder/coder/v2/testutil"
 )
 
 func TestAppHealth_Healthy(t *testing.T) {

agent/metrics.go

@@ -0,0 +1,130 @@
+package agent
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/prometheus/client_golang/prometheus"
+	prompb "github.com/prometheus/client_model/go"
+	"tailscale.com/util/clientmetric"
+
+	"cdr.dev/slog"
+
+	"github.com/coder/coder/v2/codersdk/agentsdk"
+)
+
+type agentMetrics struct {
+	connectionsTotal      prometheus.Counter
+	reconnectingPTYErrors *prometheus.CounterVec
+}
+
+func newAgentMetrics(registerer prometheus.Registerer) *agentMetrics {
+	connectionsTotal := prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace: "agent", Subsystem: "reconnecting_pty", Name: "connections_total",
+	})
+	registerer.MustRegister(connectionsTotal)
+
+	reconnectingPTYErrors := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: "agent",
+			Subsystem: "reconnecting_pty",
+			Name:      "errors_total",
+		},
+		[]string{"error_type"},
+	)
+	registerer.MustRegister(reconnectingPTYErrors)
+
+	return &agentMetrics{
+		connectionsTotal:      connectionsTotal,
+		reconnectingPTYErrors: reconnectingPTYErrors,
+	}
+}
+
+func (a *agent) collectMetrics(ctx context.Context) []agentsdk.AgentMetric {
+	var collected []agentsdk.AgentMetric
+
+	// Tailscale internal metrics
+	metrics := clientmetric.Metrics()
+	for _, m := range metrics {
+		if isIgnoredMetric(m.Name()) {
+			continue
+		}
+
+		collected = append(collected, agentsdk.AgentMetric{
+			Name:  m.Name(),
+			Type:  asMetricType(m.Type()),
+			Value: float64(m.Value()),
+		})
+	}
+
+	metricFamilies, err := a.prometheusRegistry.Gather()
+	if err != nil {
+		a.logger.Error(ctx, "can't gather agent metrics", slog.Error(err))
+		return collected
+	}
+
+	for _, metricFamily := range metricFamilies {
+		for _, metric := range metricFamily.GetMetric() {
+			labels := toAgentMetricLabels(metric.Label)
+
+			if metric.Counter != nil {
+				collected = append(collected, agentsdk.AgentMetric{
+					Name:   metricFamily.GetName(),
+					Type:   agentsdk.AgentMetricTypeCounter,
+					Value:  metric.Counter.GetValue(),
+					Labels: labels,
+				})
+			} else if metric.Gauge != nil {
+				collected = append(collected, agentsdk.AgentMetric{
+					Name:   metricFamily.GetName(),
+					Type:   agentsdk.AgentMetricTypeGauge,
+					Value:  metric.Gauge.GetValue(),
+					Labels: labels,
+				})
+			} else {
+				a.logger.Error(ctx, "unsupported metric type", slog.F("type", metricFamily.Type.String()))
+			}
+		}
+	}
+	return collected
+}
+
+func toAgentMetricLabels(metricLabels []*prompb.LabelPair) []agentsdk.AgentMetricLabel {
+	if len(metricLabels) == 0 {
+		return nil
+	}
+
+	labels := make([]agentsdk.AgentMetricLabel, 0, len(metricLabels))
+	for _, metricLabel := range metricLabels {
+		labels = append(labels, agentsdk.AgentMetricLabel{
+			Name:  metricLabel.GetName(),
+			Value: metricLabel.GetValue(),
+		})
+	}
+	return labels
+}
+
+// isIgnoredMetric checks if the metric should be ignored, as Coder agent doesn't use related features.
+// Expected metric families: magicsock_*, derp_*, tstun_*, netcheck_*, portmap_*, etc.
+func isIgnoredMetric(metricName string) bool {
+	if strings.HasPrefix(metricName, "dns_") ||
+		strings.HasPrefix(metricName, "controlclient_") ||
+		strings.HasPrefix(metricName, "peerapi_") ||
+		strings.HasPrefix(metricName, "profiles_") ||
+		strings.HasPrefix(metricName, "tstun_") {
+		return true
+	}
+	return false
+}
+
+func asMetricType(typ clientmetric.Type) agentsdk.AgentMetricType {
+	switch typ {
+	case clientmetric.TypeGauge:
+		return agentsdk.AgentMetricTypeGauge
+	case clientmetric.TypeCounter:
+		return agentsdk.AgentMetricTypeCounter
+	default:
+		panic(fmt.Sprintf("unknown metric type: %d", typ))
+	}
+}

agent/ports_supported.go

@@ -8,7 +8,7 @@ import (
 	"github.com/cakturk/go-netstat/netstat"
 	"golang.org/x/xerrors"
 
-	"github.com/coder/coder/codersdk"
+	"github.com/coder/coder/v2/codersdk"
 )
 
 func (lp *listeningPortsHandler) getListeningPorts() ([]codersdk.WorkspaceAgentListeningPort, error) {
@@ -36,6 +36,11 @@ func (lp *listeningPortsHandler) getListeningPorts() ([]codersdk.WorkspaceAgentL
 			continue
 		}
 
+		// Ignore ports that we've been told to ignore.
+		if _, ok := lp.ignorePorts[int(tab.LocalAddr.Port)]; ok {
+			continue
+		}
+
 		// Don't include ports that we've already seen. This can happen on
 		// Windows, and maybe on Linux if you're using a shared listener socket.
 		if _, ok := seen[tab.LocalAddr.Port]; ok {

agent/ports_unsupported.go

@@ -2,7 +2,7 @@
 
 package agent
 
-import "github.com/coder/coder/codersdk"
+import "github.com/coder/coder/v2/codersdk"
 
 func (lp *listeningPortsHandler) getListeningPorts() ([]codersdk.WorkspaceAgentListeningPort, error) {
 	// Can't scan for ports on non-linux or non-windows_amd64 systems at the

agent/reaper/reaper_test.go

@@ -14,58 +14,55 @@ import (
 	"github.com/hashicorp/go-reap"
 	"github.com/stretchr/testify/require"
 
-	"github.com/coder/coder/agent/reaper"
-	"github.com/coder/coder/testutil"
+	"github.com/coder/coder/v2/agent/reaper"
+	"github.com/coder/coder/v2/testutil"
 )
 
-//nolint:paralleltest // Non-parallel subtest.
+// TestReap checks that's the reaper is successfully reaping
+// exited processes and passing the PIDs through the shared
+// channel.
+//
+//nolint:paralleltest
 func TestReap(t *testing.T) {
 	// Don't run the reaper test in CI. It does weird
 	// things like forkexecing which may have unintended
 	// consequences in CI.
-	if _, ok := os.LookupEnv("CI"); ok {
+	if testutil.InCI() {
 		t.Skip("Detected CI, skipping reaper tests")
 	}
 
-	// OK checks that's the reaper is successfully reaping
-	// exited processes and passing the PIDs through the shared
-	// channel.
+	pids := make(reap.PidCh, 1)
+	err := reaper.ForkReap(
+		reaper.WithPIDCallback(pids),
+		// Provide some argument that immediately exits.
+		reaper.WithExecArgs("/bin/sh", "-c", "exit 0"),
+	)
+	require.NoError(t, err)
 
-	//nolint:paralleltest // Signal handling.
-	t.Run("OK", func(t *testing.T) {
-		pids := make(reap.PidCh, 1)
-		err := reaper.ForkReap(
-			reaper.WithPIDCallback(pids),
-			// Provide some argument that immediately exits.
-			reaper.WithExecArgs("/bin/sh", "-c", "exit 0"),
-		)
-		require.NoError(t, err)
+	cmd := exec.Command("tail", "-f", "/dev/null")
+	err = cmd.Start()
+	require.NoError(t, err)
 
-		cmd := exec.Command("tail", "-f", "/dev/null")
-		err = cmd.Start()
-		require.NoError(t, err)
+	cmd2 := exec.Command("tail", "-f", "/dev/null")
+	err = cmd2.Start()
+	require.NoError(t, err)
 
-		cmd2 := exec.Command("tail", "-f", "/dev/null")
-		err = cmd2.Start()
-		require.NoError(t, err)
+	err = cmd.Process.Kill()
+	require.NoError(t, err)
 
-		err = cmd.Process.Kill()
-		require.NoError(t, err)
+	err = cmd2.Process.Kill()
+	require.NoError(t, err)
 
-		err = cmd2.Process.Kill()
-		require.NoError(t, err)
+	expectedPIDs := []int{cmd.Process.Pid, cmd2.Process.Pid}
 
-		expectedPIDs := []int{cmd.Process.Pid, cmd2.Process.Pid}
-
-		for i := 0; i < len(expectedPIDs); i++ {
-			select {
-			case <-time.After(testutil.WaitShort):
-				t.Fatalf("Timed out waiting for process")
-			case pid := <-pids:
-				require.Contains(t, expectedPIDs, pid)
-			}
+	for i := 0; i < len(expectedPIDs); i++ {
+		select {
+		case <-time.After(testutil.WaitShort):
+			t.Fatalf("Timed out waiting for process")
+		case pid := <-pids:
+			require.Contains(t, expectedPIDs, pid)
 		}
-	})
+	}
 }
 
 //nolint:paralleltest // Signal handling.
@@ -73,7 +70,7 @@ func TestReapInterrupt(t *testing.T) {
 	// Don't run the reaper test in CI. It does weird
 	// things like forkexecing which may have unintended
 	// consequences in CI.
-	if _, ok := os.LookupEnv("CI"); ok {
+	if testutil.InCI() {
 		t.Skip("Detected CI, skipping reaper tests")
 	}
 

agent/reconnectingpty/buffered.go

@@ -0,0 +1,241 @@
+package reconnectingpty
+
+import (
+	"context"
+	"errors"
+	"io"
+	"net"
+	"time"
+
+	"github.com/armon/circbuf"
+	"github.com/prometheus/client_golang/prometheus"
+	"golang.org/x/exp/slices"
+	"golang.org/x/xerrors"
+
+	"cdr.dev/slog"
+
+	"github.com/coder/coder/v2/pty"
+)
+
+// bufferedReconnectingPTY provides a reconnectable PTY by using a ring buffer to store
+// scrollback.
+type bufferedReconnectingPTY struct {
+	command *pty.Cmd
+
+	activeConns    map[string]net.Conn
+	circularBuffer *circbuf.Buffer
+
+	ptty    pty.PTYCmd
+	process pty.Process
+
+	metrics *prometheus.CounterVec
+
+	state *ptyState
+	// timer will close the reconnecting pty when it expires.  The timer will be
+	// reset as long as there are active connections.
+	timer   *time.Timer
+	timeout time.Duration
+}
+
+// newBuffered starts the buffered pty.  If the context ends the process will be
+// killed.
+func newBuffered(ctx context.Context, cmd *pty.Cmd, options *Options, logger slog.Logger) *bufferedReconnectingPTY {
+	rpty := &bufferedReconnectingPTY{
+		activeConns: map[string]net.Conn{},
+		command:     cmd,
+		metrics:     options.Metrics,
+		state:       newState(),
+		timeout:     options.Timeout,
+	}
+
+	// Default to buffer 64KiB.
+	circularBuffer, err := circbuf.NewBuffer(64 << 10)
+	if err != nil {
+		rpty.state.setState(StateDone, xerrors.Errorf("create circular buffer: %w", err))
+		return rpty
+	}
+	rpty.circularBuffer = circularBuffer
+
+	// Add TERM then start the command with a pty.  pty.Cmd duplicates Path as the
+	// first argument so remove it.
+	cmdWithEnv := pty.CommandContext(ctx, cmd.Path, cmd.Args[1:]...)
+	cmdWithEnv.Env = append(rpty.command.Env, "TERM=xterm-256color")
+	cmdWithEnv.Dir = rpty.command.Dir
+	ptty, process, err := pty.Start(cmdWithEnv)
+	if err != nil {
+		rpty.state.setState(StateDone, xerrors.Errorf("start pty: %w", err))
+		return rpty
+	}
+	rpty.ptty = ptty
+	rpty.process = process
+
+	go rpty.lifecycle(ctx, logger)
+
+	// Multiplex the output onto the circular buffer and each active connection.
+	// We do not need to separately monitor for the process exiting.  When it
+	// exits, our ptty.OutputReader() will return EOF after reading all process
+	// output.
+	go func() {
+		buffer := make([]byte, 1024)
+		for {
+			read, err := ptty.OutputReader().Read(buffer)
+			if err != nil {
+				// When the PTY is closed, this is triggered.
+				// Error is typically a benign EOF, so only log for debugging.
+				if errors.Is(err, io.EOF) {
+					logger.Debug(ctx, "unable to read pty output, command might have exited", slog.Error(err))
+				} else {
+					logger.Warn(ctx, "unable to read pty output, command might have exited", slog.Error(err))
+					rpty.metrics.WithLabelValues("output_reader").Add(1)
+				}
+				// Could have been killed externally or failed to start at all (command
+				// not found for example).
+				// TODO: Should we check the process's exit code in case the command was
+				//       invalid?
+				rpty.Close(nil)
+				break
+			}
+			part := buffer[:read]
+			rpty.state.cond.L.Lock()
+			_, err = rpty.circularBuffer.Write(part)
+			if err != nil {
+				logger.Error(ctx, "write to circular buffer", slog.Error(err))
+				rpty.metrics.WithLabelValues("write_buffer").Add(1)
+			}
+			// TODO: Instead of ranging over a map, could we send the output to a
+			// channel and have each individual Attach read from that?
+			for cid, conn := range rpty.activeConns {
+				_, err = conn.Write(part)
+				if err != nil {
+					logger.Warn(ctx,
+						"error writing to active connection",
+						slog.F("connection_id", cid),
+						slog.Error(err),
+					)
+					rpty.metrics.WithLabelValues("write").Add(1)
+				}
+			}
+			rpty.state.cond.L.Unlock()
+		}
+	}()
+
+	return rpty
+}
+
+// lifecycle manages the lifecycle of the reconnecting pty.  If the context ends
+// or the reconnecting pty closes the pty will be shut down.
+func (rpty *bufferedReconnectingPTY) lifecycle(ctx context.Context, logger slog.Logger) {
+	rpty.timer = time.AfterFunc(attachTimeout, func() {
+		rpty.Close(xerrors.New("reconnecting pty timeout"))
+	})
+
+	logger.Debug(ctx, "reconnecting pty ready")
+	rpty.state.setState(StateReady, nil)
+
+	state, reasonErr := rpty.state.waitForStateOrContext(ctx, StateClosing)
+	if state < StateClosing {
+		// If we have not closed yet then the context is what unblocked us (which
+		// means the agent is shutting down) so move into the closing phase.
+		rpty.Close(reasonErr)
+	}
+	rpty.timer.Stop()
+
+	rpty.state.cond.L.Lock()
+	// Log these closes only for debugging since the connections or processes
+	// might have already closed on their own.
+	for _, conn := range rpty.activeConns {
+		err := conn.Close()
+		if err != nil {
+			logger.Debug(ctx, "closed conn with error", slog.Error(err))
+		}
+	}
+	// Connections get removed once they close but it is possible there is still
+	// some data that will be written before that happens so clear the map now to
+	// avoid writing to closed connections.
+	rpty.activeConns = map[string]net.Conn{}
+	rpty.state.cond.L.Unlock()
+
+	// Log close/kill only for debugging since the process might have already
+	// closed on its own.
+	err := rpty.ptty.Close()
+	if err != nil {
+		logger.Debug(ctx, "closed ptty with error", slog.Error(err))
+	}
+
+	err = rpty.process.Kill()
+	if err != nil {
+		logger.Debug(ctx, "killed process with error", slog.Error(err))
+	}
+
+	logger.Info(ctx, "closed reconnecting pty")
+	rpty.state.setState(StateDone, reasonErr)
+}
+
+func (rpty *bufferedReconnectingPTY) Attach(ctx context.Context, connID string, conn net.Conn, height, width uint16, logger slog.Logger) error {
+	logger.Info(ctx, "attach to reconnecting pty")
+
+	// This will kill the heartbeat once we hit EOF or an error.
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	err := rpty.doAttach(connID, conn)
+	if err != nil {
+		return err
+	}
+
+	defer func() {
+		rpty.state.cond.L.Lock()
+		defer rpty.state.cond.L.Unlock()
+		delete(rpty.activeConns, connID)
+	}()
+
+	state, err := rpty.state.waitForStateOrContext(ctx, StateReady)
+	if state != StateReady {
+		return err
+	}
+
+	go heartbeat(ctx, rpty.timer, rpty.timeout)
+
+	// Resize the PTY to initial height + width.
+	err = rpty.ptty.Resize(height, width)
+	if err != nil {
+		// We can continue after this, it's not fatal!
+		logger.Warn(ctx, "reconnecting PTY initial resize failed, but will continue", slog.Error(err))
+		rpty.metrics.WithLabelValues("resize").Add(1)
+	}
+
+	// Pipe conn -> pty and block.  pty -> conn is handled in newBuffered().
+	readConnLoop(ctx, conn, rpty.ptty, rpty.metrics, logger)
+	return nil
+}
+
+// doAttach adds the connection to the map and replays the buffer.  It exists
+// separately only for convenience to defer the mutex unlock which is not
+// possible in Attach since it blocks.
+func (rpty *bufferedReconnectingPTY) doAttach(connID string, conn net.Conn) error {
+	rpty.state.cond.L.Lock()
+	defer rpty.state.cond.L.Unlock()
+
+	// Write any previously stored data for the TTY.  Since the command might be
+	// short-lived and have already exited, make sure we always at least output
+	// the buffer before returning, mostly just so tests pass.
+	prevBuf := slices.Clone(rpty.circularBuffer.Bytes())
+	_, err := conn.Write(prevBuf)
+	if err != nil {
+		rpty.metrics.WithLabelValues("write").Add(1)
+		return xerrors.Errorf("write buffer to conn: %w", err)
+	}
+
+	rpty.activeConns[connID] = conn
+
+	return nil
+}
+
+func (rpty *bufferedReconnectingPTY) Wait() {
+	_, _ = rpty.state.waitForState(StateClosing)
+}
+
+func (rpty *bufferedReconnectingPTY) Close(error error) {
+	// The closing state change will be handled by the lifecycle.
+	rpty.state.setState(StateClosing, error)
+}

agent/reconnectingpty/reconnectingpty.go

@@ -0,0 +1,226 @@
+package reconnectingpty
+
+import (
+	"context"
+	"encoding/json"
+	"io"
+	"net"
+	"os/exec"
+	"runtime"
+	"sync"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"golang.org/x/xerrors"
+
+	"cdr.dev/slog"
+
+	"github.com/coder/coder/v2/codersdk"
+	"github.com/coder/coder/v2/pty"
+)
+
+// attachTimeout is the initial timeout for attaching and will probably be far
+// shorter than the reconnect timeout in most cases; in tests it might be
+// longer.  It should be at least long enough for the first screen attach to be
+// able to start up the daemon and for the buffered pty to start.
+const attachTimeout = 30 * time.Second
+
+// Options allows configuring the reconnecting pty.
+type Options struct {
+	// Timeout describes how long to keep the pty alive without any connections.
+	// Once elapsed the pty will be killed.
+	Timeout time.Duration
+	// Metrics tracks various error counters.
+	Metrics *prometheus.CounterVec
+}
+
+// ReconnectingPTY is a pty that can be reconnected within a timeout and to
+// simultaneous connections.  The reconnecting pty can be backed by screen if
+// installed or a (buggy) buffer replay fallback.
+type ReconnectingPTY interface {
+	// Attach pipes the connection and pty, spawning it if necessary, replays
+	// history, then blocks until EOF, an error, or the context's end.  The
+	// connection is expected to send JSON-encoded messages and accept raw output
+	// from the ptty.  If the context ends or the process dies the connection will
+	// be detached.
+	Attach(ctx context.Context, connID string, conn net.Conn, height, width uint16, logger slog.Logger) error
+	// Wait waits for the reconnecting pty to close.  The underlying process might
+	// still be exiting.
+	Wait()
+	// Close kills the reconnecting pty process.
+	Close(err error)
+}
+
+// New sets up a new reconnecting pty that wraps the provided command.  Any
+// errors with starting are returned on Attach().  The reconnecting pty will
+// close itself (and all connections to it) if nothing is attached for the
+// duration of the timeout, if the context ends, or the process exits (buffered
+// backend only).
+func New(ctx context.Context, cmd *pty.Cmd, options *Options, logger slog.Logger) ReconnectingPTY {
+	if options.Timeout == 0 {
+		options.Timeout = 5 * time.Minute
+	}
+	// Screen seems flaky on Darwin.  Locally the tests pass 100% of the time (100
+	// runs) but in CI screen often incorrectly claims the session name does not
+	// exist even though screen -list shows it.  For now, restrict screen to
+	// Linux.
+	backendType := "buffered"
+	if runtime.GOOS == "linux" {
+		_, err := exec.LookPath("screen")
+		if err == nil {
+			backendType = "screen"
+		}
+	}
+
+	logger.Info(ctx, "start reconnecting pty", slog.F("backend_type", backendType))
+
+	switch backendType {
+	case "screen":
+		return newScreen(ctx, cmd, options, logger)
+	default:
+		return newBuffered(ctx, cmd, options, logger)
+	}
+}
+
+// heartbeat resets timer before timeout elapses and blocks until ctx ends.
+func heartbeat(ctx context.Context, timer *time.Timer, timeout time.Duration) {
+	// Reset now in case it is near the end.
+	timer.Reset(timeout)
+
+	// Reset when the context ends to ensure the pty stays up for the full
+	// timeout.
+	defer timer.Reset(timeout)
+
+	heartbeat := time.NewTicker(timeout / 2)
+	defer heartbeat.Stop()
+
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-heartbeat.C:
+			timer.Reset(timeout)
+		}
+	}
+}
+
+// State represents the current state of the reconnecting pty.  States are
+// sequential and will only move forward.
+type State int
+
+const (
+	// StateStarting is the default/start state.  Attaching will block until the
+	// reconnecting pty becomes ready.
+	StateStarting = iota
+	// StateReady means the reconnecting pty is ready to be attached.
+	StateReady
+	// StateClosing means the reconnecting pty has begun closing.  The underlying
+	// process may still be exiting.  Attaching will result in an error.
+	StateClosing
+	// StateDone means the reconnecting pty has completely shut down and the
+	// process has exited.  Attaching will result in an error.
+	StateDone
+)
+
+// ptyState is a helper for tracking the reconnecting PTY's state.
+type ptyState struct {
+	// cond broadcasts state changes and any accompanying errors.
+	cond *sync.Cond
+	// error describes the error that caused the state change, if there was one.
+	// It is not safe to access outside of cond.L.
+	error error
+	// state holds the current reconnecting pty state.  It is not safe to access
+	// this outside of cond.L.
+	state State
+}
+
+func newState() *ptyState {
+	return &ptyState{
+		cond:  sync.NewCond(&sync.Mutex{}),
+		state: StateStarting,
+	}
+}
+
+// setState sets and broadcasts the provided state if it is greater than the
+// current state and the error if one has not already been set.
+func (s *ptyState) setState(state State, err error) {
+	s.cond.L.Lock()
+	defer s.cond.L.Unlock()
+	// Cannot regress states.  For example, trying to close after the process is
+	// done should leave us in the done state and not the closing state.
+	if state <= s.state {
+		return
+	}
+	s.error = err
+	s.state = state
+	s.cond.Broadcast()
+}
+
+// waitForState blocks until the state or a greater one is reached.
+func (s *ptyState) waitForState(state State) (State, error) {
+	s.cond.L.Lock()
+	defer s.cond.L.Unlock()
+	for state > s.state {
+		s.cond.Wait()
+	}
+	return s.state, s.error
+}
+
+// waitForStateOrContext blocks until the state or a greater one is reached or
+// the provided context ends.
+func (s *ptyState) waitForStateOrContext(ctx context.Context, state State) (State, error) {
+	s.cond.L.Lock()
+	defer s.cond.L.Unlock()
+
+	nevermind := make(chan struct{})
+	defer close(nevermind)
+	go func() {
+		select {
+		case <-ctx.Done():
+			// Wake up when the context ends.
+			s.cond.Broadcast()
+		case <-nevermind:
+		}
+	}()
+
+	for ctx.Err() == nil && state > s.state {
+		s.cond.Wait()
+	}
+	if ctx.Err() != nil {
+		return s.state, ctx.Err()
+	}
+	return s.state, s.error
+}
+
+// readConnLoop reads messages from conn and writes to ptty as needed.  Blocks
+// until EOF or an error writing to ptty or reading from conn.
+func readConnLoop(ctx context.Context, conn net.Conn, ptty pty.PTYCmd, metrics *prometheus.CounterVec, logger slog.Logger) {
+	decoder := json.NewDecoder(conn)
+	var req codersdk.ReconnectingPTYRequest
+	for {
+		err := decoder.Decode(&req)
+		if xerrors.Is(err, io.EOF) {
+			return
+		}
+		if err != nil {
+			logger.Warn(ctx, "reconnecting pty failed with read error", slog.Error(err))
+			return
+		}
+		_, err = ptty.InputWriter().Write([]byte(req.Data))
+		if err != nil {
+			logger.Warn(ctx, "reconnecting pty failed with write error", slog.Error(err))
+			metrics.WithLabelValues("input_writer").Add(1)
+			return
+		}
+		// Check if a resize needs to happen!
+		if req.Height == 0 || req.Width == 0 {
+			continue
+		}
+		err = ptty.Resize(req.Height, req.Width)
+		if err != nil {
+			// We can continue after this, it's not fatal!
+			logger.Warn(ctx, "reconnecting pty resize failed, but will continue", slog.Error(err))
+			metrics.WithLabelValues("resize").Add(1)
+		}
+	}
+}

agent/reconnectingpty/screen.go

@@ -0,0 +1,389 @@
+package reconnectingpty
+
+import (
+	"bytes"
+	"context"
+	"crypto/rand"
+	"encoding/hex"
+	"errors"
+	"io"
+	"net"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/gliderlabs/ssh"
+	"github.com/prometheus/client_golang/prometheus"
+	"golang.org/x/xerrors"
+
+	"cdr.dev/slog"
+	"github.com/coder/coder/v2/pty"
+)
+
+// screenReconnectingPTY provides a reconnectable PTY via `screen`.
+type screenReconnectingPTY struct {
+	command *pty.Cmd
+
+	// id holds the id of the session for both creating and attaching.  This will
+	// be generated uniquely for each session because without control of the
+	// screen daemon we do not have its PID and without the PID screen will do
+	// partial matching.  Enforcing a unique ID should guarantee we match on the
+	// right session.
+	id string
+
+	// mutex prevents concurrent attaches to the session.  Screen will happily
+	// spawn two separate sessions with the same name if multiple attaches happen
+	// in a close enough interval.  We are not able to control the screen daemon
+	// ourselves to prevent this because the daemon will spawn with a hardcoded
+	// 24x80 size which results in confusing padding above the prompt once the
+	// attach comes in and resizes.
+	mutex sync.Mutex
+
+	configFile string
+
+	metrics *prometheus.CounterVec
+
+	state *ptyState
+	// timer will close the reconnecting pty when it expires.  The timer will be
+	// reset as long as there are active connections.
+	timer   *time.Timer
+	timeout time.Duration
+}
+
+// newScreen creates a new screen-backed reconnecting PTY.  It writes config
+// settings and creates the socket directory.  If we could, we would want to
+// spawn the daemon here and attach each connection to it but since doing that
+// spawns the daemon with a hardcoded 24x80 size it is not a very good user
+// experience.  Instead we will let the attach command spawn the daemon on its
+// own which causes it to spawn with the specified size.
+func newScreen(ctx context.Context, cmd *pty.Cmd, options *Options, logger slog.Logger) *screenReconnectingPTY {
+	rpty := &screenReconnectingPTY{
+		command: cmd,
+		metrics: options.Metrics,
+		state:   newState(),
+		timeout: options.Timeout,
+	}
+
+	go rpty.lifecycle(ctx, logger)
+
+	// Socket paths are limited to around 100 characters on Linux and macOS which
+	// depending on the temporary directory can be a problem.  To give more leeway
+	// use a short ID.
+	buf := make([]byte, 4)
+	_, err := rand.Read(buf)
+	if err != nil {
+		rpty.state.setState(StateDone, xerrors.Errorf("generate screen id: %w", err))
+		return rpty
+	}
+	rpty.id = hex.EncodeToString(buf)
+
+	settings := []string{
+		// Tell screen not to handle motion for xterm* terminals which allows
+		// scrolling the terminal via the mouse wheel or scroll bar (by default
+		// screen uses it to cycle through the command history).  There does not
+		// seem to be a way to make screen itself scroll on mouse wheel.  tmux can
+		// do it but then there is no scroll bar and it kicks you into copy mode
+		// where keys stop working until you exit copy mode which seems like it
+		// could be confusing.
+		"termcapinfo xterm* ti@:te@",
+		// Enable alternate screen emulation otherwise applications get rendered in
+		// the current window which wipes out visible output resulting in missing
+		// output when scrolling back with the mouse wheel (copy mode still works
+		// since that is screen itself scrolling).
+		"altscreen on",
+		// Remap the control key to C-s since C-a may be used in applications.  C-s
+		// is chosen because it cannot actually be used because by default it will
+		// pause and C-q to resume will just kill the browser window.  We may not
+		// want people using the control key anyway since it will not be obvious
+		// they are in screen and doing things like switching windows makes mouse
+		// wheel scroll wonky due to the terminal doing the scrolling rather than
+		// screen itself (but again copy mode will work just fine).
+		"escape ^Ss",
+	}
+
+	rpty.configFile = filepath.Join(os.TempDir(), "coder-screen", "config")
+	err = os.MkdirAll(filepath.Dir(rpty.configFile), 0o700)
+	if err != nil {
+		rpty.state.setState(StateDone, xerrors.Errorf("make screen config dir: %w", err))
+		return rpty
+	}
+
+	err = os.WriteFile(rpty.configFile, []byte(strings.Join(settings, "\n")), 0o600)
+	if err != nil {
+		rpty.state.setState(StateDone, xerrors.Errorf("create config file: %w", err))
+		return rpty
+	}
+
+	return rpty
+}
+
+// lifecycle manages the lifecycle of the reconnecting pty.  If the context ends
+// the reconnecting pty will be closed.
+func (rpty *screenReconnectingPTY) lifecycle(ctx context.Context, logger slog.Logger) {
+	rpty.timer = time.AfterFunc(attachTimeout, func() {
+		rpty.Close(xerrors.New("reconnecting pty timeout"))
+	})
+
+	logger.Debug(ctx, "reconnecting pty ready")
+	rpty.state.setState(StateReady, nil)
+
+	state, reasonErr := rpty.state.waitForStateOrContext(ctx, StateClosing)
+	if state < StateClosing {
+		// If we have not closed yet then the context is what unblocked us (which
+		// means the agent is shutting down) so move into the closing phase.
+		rpty.Close(reasonErr)
+	}
+	rpty.timer.Stop()
+
+	// If the command errors that the session is already gone that is fine.
+	err := rpty.sendCommand(context.Background(), "quit", []string{"No screen session found"})
+	if err != nil {
+		logger.Error(ctx, "close screen session", slog.Error(err))
+	}
+
+	logger.Info(ctx, "closed reconnecting pty")
+	rpty.state.setState(StateDone, reasonErr)
+}
+
+func (rpty *screenReconnectingPTY) Attach(ctx context.Context, _ string, conn net.Conn, height, width uint16, logger slog.Logger) error {
+	logger.Info(ctx, "attach to reconnecting pty")
+
+	// This will kill the heartbeat once we hit EOF or an error.
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	state, err := rpty.state.waitForStateOrContext(ctx, StateReady)
+	if state != StateReady {
+		return err
+	}
+
+	go heartbeat(ctx, rpty.timer, rpty.timeout)
+
+	ptty, process, err := rpty.doAttach(ctx, conn, height, width, logger)
+	if err != nil {
+		if errors.Is(err, context.Canceled) {
+			// Likely the process was too short-lived and canceled the version command.
+			// TODO: Is it worth distinguishing between that and a cancel from the
+			//       Attach() caller?  Additionally, since this could also happen if
+			//       the command was invalid, should we check the process's exit code?
+			return nil
+		}
+		return err
+	}
+
+	defer func() {
+		// Log only for debugging since the process might have already exited on its
+		// own.
+		err := ptty.Close()
+		if err != nil {
+			logger.Debug(ctx, "closed ptty with error", slog.Error(err))
+		}
+		err = process.Kill()
+		if err != nil {
+			logger.Debug(ctx, "killed process with error", slog.Error(err))
+		}
+	}()
+
+	// Pipe conn -> pty and block.
+	readConnLoop(ctx, conn, ptty, rpty.metrics, logger)
+	return nil
+}
+
+// doAttach spawns the screen client and starts the heartbeat.  It exists
+// separately only so we can defer the mutex unlock which is not possible in
+// Attach since it blocks.
+func (rpty *screenReconnectingPTY) doAttach(ctx context.Context, conn net.Conn, height, width uint16, logger slog.Logger) (pty.PTYCmd, pty.Process, error) {
+	// Ensure another attach does not come in and spawn a duplicate session.
+	rpty.mutex.Lock()
+	defer rpty.mutex.Unlock()
+
+	logger.Debug(ctx, "spawning screen client", slog.F("screen_id", rpty.id))
+
+	// Wrap the command with screen and tie it to the connection's context.
+	cmd := pty.CommandContext(ctx, "screen", append([]string{
+		// -S is for setting the session's name.
+		"-S", rpty.id,
+		// -U tells screen to use UTF-8 encoding.
+		// -x allows attaching to an already attached session.
+		// -RR reattaches to the daemon or creates the session daemon if missing.
+		// -q disables the "New screen..." message that appears for five seconds
+		//    when creating a new session with -RR.
+		// -c is the flag for the config file.
+		"-UxRRqc", rpty.configFile,
+		rpty.command.Path,
+		// pty.Cmd duplicates Path as the first argument so remove it.
+	}, rpty.command.Args[1:]...)...)
+	cmd.Env = append(rpty.command.Env, "TERM=xterm-256color")
+	cmd.Dir = rpty.command.Dir
+	ptty, process, err := pty.Start(cmd, pty.WithPTYOption(
+		pty.WithSSHRequest(ssh.Pty{
+			Window: ssh.Window{
+				// Make sure to spawn at the right size because if we resize afterward it
+				// leaves confusing padding (screen will resize such that the screen
+				// contents are aligned to the bottom).
+				Height: int(height),
+				Width:  int(width),
+			},
+		}),
+	))
+	if err != nil {
+		rpty.metrics.WithLabelValues("screen_spawn").Add(1)
+		return nil, nil, err
+	}
+
+	// This context lets us abort the version command if the process dies.
+	versionCtx, versionCancel := context.WithCancel(ctx)
+	defer versionCancel()
+
+	// Pipe pty -> conn and close the connection when the process exits.
+	// We do not need to separately monitor for the process exiting.  When it
+	// exits, our ptty.OutputReader() will return EOF after reading all process
+	// output.
+	go func() {
+		defer versionCancel()
+		defer func() {
+			err := conn.Close()
+			if err != nil {
+				// Log only for debugging since the connection might have already closed
+				// on its own.
+				logger.Debug(ctx, "closed connection with error", slog.Error(err))
+			}
+		}()
+		buffer := make([]byte, 1024)
+		for {
+			read, err := ptty.OutputReader().Read(buffer)
+			if err != nil {
+				// When the PTY is closed, this is triggered.
+				// Error is typically a benign EOF, so only log for debugging.
+				if errors.Is(err, io.EOF) {
+					logger.Debug(ctx, "unable to read pty output; screen might have exited", slog.Error(err))
+				} else {
+					logger.Warn(ctx, "unable to read pty output; screen might have exited", slog.Error(err))
+					rpty.metrics.WithLabelValues("screen_output_reader").Add(1)
+				}
+				// The process might have died because the session itself died or it
+				// might have been separately killed and the session is still up (for
+				// example `exit` or we killed it when the connection closed).  If the
+				// session is still up we might leave the reconnecting pty in memory
+				// around longer than it needs to be but it will eventually clean up
+				// with the timer or context, or the next attach will respawn the screen
+				// daemon which is fine too.
+				break
+			}
+			part := buffer[:read]
+			_, err = conn.Write(part)
+			if err != nil {
+				// Connection might have been closed.
+				if errors.Unwrap(err).Error() != "endpoint is closed for send" {
+					logger.Warn(ctx, "error writing to active conn", slog.Error(err))
+					rpty.metrics.WithLabelValues("screen_write").Add(1)
+				}
+				break
+			}
+		}
+	}()
+
+	// Version seems to be the only command without a side effect (other than
+	// making the version pop up briefly) so use it to wait for the session to
+	// come up.  If we do not wait we could end up spawning multiple sessions with
+	// the same name.
+	err = rpty.sendCommand(versionCtx, "version", nil)
+	if err != nil {
+		// Log only for debugging since the process might already have closed.
+		closeErr := ptty.Close()
+		if closeErr != nil {
+			logger.Debug(ctx, "closed ptty with error", slog.Error(closeErr))
+		}
+		closeErr = process.Kill()
+		if closeErr != nil {
+			logger.Debug(ctx, "killed process with error", slog.Error(closeErr))
+		}
+		rpty.metrics.WithLabelValues("screen_wait").Add(1)
+		return nil, nil, err
+	}
+
+	return ptty, process, nil
+}
+
+// sendCommand runs a screen command against a running screen session.  If the
+// command fails with an error matching anything in successErrors it will be
+// considered a success state (for example "no session" when quitting and the
+// session is already dead).  The command will be retried until successful, the
+// timeout is reached, or the context ends.  A canceled context will return the
+// canceled context's error as-is while a timed-out context returns together
+// with the last error from the command.
+func (rpty *screenReconnectingPTY) sendCommand(ctx context.Context, command string, successErrors []string) error {
+	ctx, cancel := context.WithTimeout(ctx, attachTimeout)
+	defer cancel()
+
+	var lastErr error
+	run := func() bool {
+		var stdout bytes.Buffer
+		//nolint:gosec
+		cmd := exec.CommandContext(ctx, "screen",
+			// -x targets an attached session.
+			"-x", rpty.id,
+			// -c is the flag for the config file.
+			"-c", rpty.configFile,
+			// -X runs a command in the matching session.
+			"-X", command,
+		)
+		cmd.Env = append(rpty.command.Env, "TERM=xterm-256color")
+		cmd.Dir = rpty.command.Dir
+		cmd.Stdout = &stdout
+		err := cmd.Run()
+		if err == nil {
+			return true
+		}
+
+		stdoutStr := stdout.String()
+		for _, se := range successErrors {
+			if strings.Contains(stdoutStr, se) {
+				return true
+			}
+		}
+
+		// Things like "exit status 1" are imprecise so include stdout as it may
+		// contain more information ("no screen session found" for example).
+		if !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded) {
+			lastErr = xerrors.Errorf("`screen -x %s -X %s`: %w: %s", rpty.id, command, err, stdoutStr)
+		}
+
+		return false
+	}
+
+	// Run immediately.
+	if done := run(); done {
+		return nil
+	}
+
+	// Then run on an interval.
+	ticker := time.NewTicker(250 * time.Millisecond)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-ctx.Done():
+			if errors.Is(ctx.Err(), context.Canceled) {
+				return ctx.Err()
+			}
+			return errors.Join(ctx.Err(), lastErr)
+		case <-ticker.C:
+			if done := run(); done {
+				return nil
+			}
+		}
+	}
+}
+
+func (rpty *screenReconnectingPTY) Wait() {
+	_, _ = rpty.state.waitForState(StateClosing)
+}
+
+func (rpty *screenReconnectingPTY) Close(err error) {
+	// The closing state change will be handled by the lifecycle.
+	rpty.state.setState(StateClosing, err)
+}

agent/usershell/usershell_darwin.go

@@ -1,8 +1,25 @@
 package usershell
 
-import "os"
+import (
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+
+	"golang.org/x/xerrors"
+)
 
 // Get returns the $SHELL environment variable.
-func Get(_ string) (string, error) {
-	return os.Getenv("SHELL"), nil
+func Get(username string) (string, error) {
+	// This command will output "UserShell: /bin/zsh" if successful, we
+	// can ignore the error since we have fallback behavior.
+	out, _ := exec.Command("dscl", ".", "-read", filepath.Join("/Users", username), "UserShell").Output()
+	s, ok := strings.CutPrefix(string(out), "UserShell: ")
+	if ok {
+		return strings.TrimSpace(s), nil
+	}
+	if s = os.Getenv("SHELL"); s != "" {
+		return s, nil
+	}
+	return "", xerrors.Errorf("shell for user %q not found via dscl or in $SHELL", username)
 }

agent/usershell/usershell_other.go

@@ -27,5 +27,8 @@ func Get(username string) (string, error) {
 		}
 		return parts[6], nil
 	}
-	return "", xerrors.Errorf("user %q not found in /etc/passwd", username)
+	if s := os.Getenv("SHELL"); s != "" {
+		return s, nil
+	}
+	return "", xerrors.Errorf("shell for user %q not found in /etc/passwd or $SHELL", username)
 }

agent/usershell/usershell_other_test.go

@@ -1,27 +0,0 @@
-//go:build !windows && !darwin
-// +build !windows,!darwin
-
-package usershell_test
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/require"
-
-	"github.com/coder/coder/agent/usershell"
-)
-
-func TestGet(t *testing.T) {
-	t.Parallel()
-	t.Run("Has", func(t *testing.T) {
-		t.Parallel()
-		shell, err := usershell.Get("root")
-		require.NoError(t, err)
-		require.NotEmpty(t, shell)
-	})
-	t.Run("NotFound", func(t *testing.T) {
-		t.Parallel()
-		_, err := usershell.Get("notauser")
-		require.Error(t, err)
-	})
-}

agent/usershell/usershell_test.go

@@ -0,0 +1,46 @@
+package usershell_test
+
+import (
+	"os/user"
+	"runtime"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/coder/coder/v2/agent/usershell"
+)
+
+//nolint:paralleltest,tparallel // This test sets an environment variable.
+func TestGet(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.SkipNow()
+	}
+
+	t.Run("Fallback", func(t *testing.T) {
+		t.Setenv("SHELL", "/bin/sh")
+
+		t.Run("NonExistentUser", func(t *testing.T) {
+			shell, err := usershell.Get("notauser")
+			require.NoError(t, err)
+			require.Equal(t, "/bin/sh", shell)
+		})
+	})
+
+	t.Run("NoFallback", func(t *testing.T) {
+		// Disable env fallback for these tests.
+		t.Setenv("SHELL", "")
+
+		t.Run("NotFound", func(t *testing.T) {
+			_, err := usershell.Get("notauser")
+			require.Error(t, err)
+		})
+
+		t.Run("User", func(t *testing.T) {
+			u, err := user.Current()
+			require.NoError(t, err)
+			shell, err := usershell.Get(u.Username)
+			require.NoError(t, err)
+			require.NotEmpty(t, shell)
+		})
+	})
+}

buildinfo/boring.go

@@ -0,0 +1,7 @@
+//go:build boringcrypto
+
+package buildinfo
+
+import "crypto/boring"
+
+var boringcrypto = boring.Enabled()

buildinfo/buildinfo.go

@@ -30,8 +30,15 @@ var (
 )
 
 const (
-	// develPrefix is prefixed to developer versions of the application.
-	develPrefix = "v0.0.0-devel"
+	// noVersion is the reported version when the version cannot be determined.
+	// Usually because `go build` is run instead of `make build`.
+	noVersion = "v0.0.0"
+
+	// develPreRelease is the pre-release tag for developer versions of the
+	// application. This includes CI builds. The pre-release tag should be appended
+	// to the version with a "-".
+	// Example: v0.0.0-devel
+	develPreRelease = "devel"
 )
 
 // Version returns the semantic version of the build.
@@ -45,7 +52,8 @@ func Version() string {
 		if tag == "" {
 			// This occurs when the tag hasn't been injected,
 			// like when using "go run".
-			version = develPrefix + revision
+			// <version>-<pre-release>+<revision>
+			version = fmt.Sprintf("%s-%s%s", noVersion, develPreRelease, revision)
 			return
 		}
 		version = "v" + tag
@@ -63,18 +71,23 @@ func Version() string {
 // disregarded. If it detects that either version is a developer build it
 // returns true.
 func VersionsMatch(v1, v2 string) bool {
-	// Developer versions are disregarded...hopefully they know what they are
-	// doing.
-	if strings.HasPrefix(v1, develPrefix) || strings.HasPrefix(v2, develPrefix) {
+	// If no version is attached, then it is a dev build outside of CI. The version
+	// will be disregarded... hopefully they know what they are doing.
+	if strings.Contains(v1, noVersion) || strings.Contains(v2, noVersion) {
 		return true
 	}
 
 	return semver.MajorMinor(v1) == semver.MajorMinor(v2)
 }
 
+func IsDevVersion(v string) bool {
+	return strings.Contains(v, "-"+develPreRelease)
+}
+
 // IsDev returns true if this is a development build.
+// CI builds are also considered development builds.
 func IsDev() bool {
-	return strings.HasPrefix(Version(), develPrefix)
+	return IsDevVersion(Version())
 }
 
 // IsSlim returns true if this is a slim build.
@@ -87,6 +100,10 @@ func IsAGPL() bool {
 	return strings.Contains(agpl, "t")
 }
 
+func IsBoringCrypto() bool {
+	return boringcrypto
+}
+
 // ExternalURL returns a URL referencing the current Coder version.
 // For production builds, this will link directly to a release.
 // For development builds, this will link to a commit.

buildinfo/buildinfo_test.go

@@ -7,7 +7,7 @@ import (
 	"github.com/stretchr/testify/require"
 	"golang.org/x/mod/semver"
 
-	"github.com/coder/coder/buildinfo"
+	"github.com/coder/coder/v2/buildinfo"
 )
 
 func TestBuildInfo(t *testing.T) {
@@ -57,13 +57,19 @@ func TestBuildInfo(t *testing.T) {
 				expectMatch: true,
 			},
 			// Our CI instance uses a "-devel" prerelease
-			// flag. This is not the same as a developer WIP build.
+			// flag.
 			{
-				name:        "DevelPreleaseNotIgnored",
+				name:        "DevelPreleaseMajor",
 				v1:          "v1.1.1-devel+123abac",
 				v2:          "v1.2.3",
 				expectMatch: false,
 			},
+			{
+				name:        "DevelPreleaseSame",
+				v1:          "v1.1.1-devel+123abac",
+				v2:          "v1.1.9",
+				expectMatch: true,
+			},
 			{
 				name:        "MajorMismatch",
 				v1:          "v1.2.3",

buildinfo/notboring.go

@@ -0,0 +1,5 @@
+//go:build !boringcrypto
+
+package buildinfo
+
+var boringcrypto = false

cli/agent.go

@@ -11,58 +11,113 @@ import (
 	"os/signal"
 	"path/filepath"
 	"runtime"
+	"strconv"
+	"strings"
 	"sync"
 	"time"
 
 	"cloud.google.com/go/compute/metadata"
-	"github.com/spf13/cobra"
 	"golang.org/x/xerrors"
 	"gopkg.in/natefinch/lumberjack.v2"
+	"tailscale.com/util/clientmetric"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/common/expfmt"
 
 	"cdr.dev/slog"
 	"cdr.dev/slog/sloggers/sloghuman"
-	"github.com/coder/coder/agent"
-	"github.com/coder/coder/agent/reaper"
-	"github.com/coder/coder/buildinfo"
-	"github.com/coder/coder/cli/cliflag"
-	"github.com/coder/coder/codersdk/agentsdk"
+	"cdr.dev/slog/sloggers/slogjson"
+	"cdr.dev/slog/sloggers/slogstackdriver"
+	"github.com/coder/coder/v2/agent"
+	"github.com/coder/coder/v2/agent/agentproc"
+	"github.com/coder/coder/v2/agent/reaper"
+	"github.com/coder/coder/v2/buildinfo"
+	"github.com/coder/coder/v2/cli/clibase"
+	"github.com/coder/coder/v2/codersdk"
+	"github.com/coder/coder/v2/codersdk/agentsdk"
 )
 
-func workspaceAgent() *cobra.Command {
+func (r *RootCmd) workspaceAgent() *clibase.Cmd {
 	var (
-		auth         string
-		logDir       string
-		pprofAddress string
-		noReap       bool
+		auth                string
+		logDir              string
+		pprofAddress        string
+		noReap              bool
+		sshMaxTimeout       time.Duration
+		tailnetListenPort   int64
+		prometheusAddress   string
+		debugAddress        string
+		slogHumanPath       string
+		slogJSONPath        string
+		slogStackdriverPath string
 	)
-	cmd := &cobra.Command{
-		Use: "agent",
+	cmd := &clibase.Cmd{
+		Use:   "agent",
+		Short: `Starts the Coder workspace agent.`,
 		// This command isn't useful to manually execute.
 		Hidden: true,
-		RunE: func(cmd *cobra.Command, _ []string) error {
-			ctx, cancel := context.WithCancel(cmd.Context())
+		Handler: func(inv *clibase.Invocation) error {
+			ctx, cancel := context.WithCancel(inv.Context())
 			defer cancel()
 
-			rawURL, err := cmd.Flags().GetString(varAgentURL)
-			if err != nil {
-				return xerrors.Errorf("CODER_AGENT_URL must be set: %w", err)
-			}
-			coderURL, err := url.Parse(rawURL)
-			if err != nil {
-				return xerrors.Errorf("parse %q: %w", rawURL, err)
+			var (
+				ignorePorts = map[int]string{}
+				isLinux     = runtime.GOOS == "linux"
+
+				sinks      = []slog.Sink{}
+				logClosers = []func() error{}
+			)
+			defer func() {
+				for _, closer := range logClosers {
+					_ = closer()
+				}
+			}()
+
+			addSinkIfProvided := func(sinkFn func(io.Writer) slog.Sink, loc string) error {
+				switch loc {
+				case "":
+					// Do nothing.
+
+				case "/dev/stderr":
+					sinks = append(sinks, sinkFn(inv.Stderr))
+
+				case "/dev/stdout":
+					sinks = append(sinks, sinkFn(inv.Stdout))
+
+				default:
+					fi, err := os.OpenFile(loc, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0o644)
+					if err != nil {
+						return xerrors.Errorf("open log file %q: %w", loc, err)
+					}
+					sinks = append(sinks, sinkFn(fi))
+					logClosers = append(logClosers, fi.Close)
+				}
+				return nil
 			}
 
-			isLinux := runtime.GOOS == "linux"
+			if err := addSinkIfProvided(sloghuman.Sink, slogHumanPath); err != nil {
+				return xerrors.Errorf("add human sink: %w", err)
+			}
+			if err := addSinkIfProvided(slogjson.Sink, slogJSONPath); err != nil {
+				return xerrors.Errorf("add json sink: %w", err)
+			}
+			if err := addSinkIfProvided(slogstackdriver.Sink, slogStackdriverPath); err != nil {
+				return xerrors.Errorf("add stackdriver sink: %w", err)
+			}
 
 			// Spawn a reaper so that we don't accumulate a ton
 			// of zombie processes.
 			if reaper.IsInitProcess() && !noReap && isLinux {
-				logWriter := &lumberjack.Logger{
+				logWriter := &lumberjackWriteCloseFixer{w: &lumberjack.Logger{
 					Filename: filepath.Join(logDir, "coder-agent-init.log"),
 					MaxSize:  5, // MB
-				}
+					// Without this, rotated logs will never be deleted.
+					MaxBackups: 1,
+				}}
 				defer logWriter.Close()
-				logger := slog.Make(sloghuman.Sink(cmd.ErrOrStderr()), sloghuman.Sink(logWriter)).Leveled(slog.LevelDebug)
+
+				sinks = append(sinks, sloghuman.Sink(logWriter))
+				logger := slog.Make(sinks...).Leveled(slog.LevelDebug)
 
 				logger.Info(ctx, "spawning reaper process")
 				// Do not start a reaper on the child process. It's important
@@ -73,7 +128,7 @@ func workspaceAgent() *cobra.Command {
 					reaper.WithCatchSignals(InterruptSignals...),
 				)
 				if err != nil {
-					logger.Error(ctx, "failed to reap", slog.Error(err))
+					logger.Error(ctx, "agent process reaper unable to fork", slog.Error(err))
 					return xerrors.Errorf("fork reap: %w", err)
 				}
 
@@ -92,36 +147,51 @@ func workspaceAgent() *cobra.Command {
 			ctx, stopNotify := signal.NotifyContext(ctx, InterruptSignals...)
 			defer stopNotify()
 
-			// dumpHandler does signal handling, so we call it after the
+			// DumpHandler does signal handling, so we call it after the
 			// reaper.
-			go dumpHandler(ctx)
+			go DumpHandler(ctx)
 
-			ljLogger := &lumberjack.Logger{
+			logWriter := &lumberjackWriteCloseFixer{w: &lumberjack.Logger{
 				Filename: filepath.Join(logDir, "coder-agent.log"),
 				MaxSize:  5, // MB
-			}
-			defer ljLogger.Close()
-			logWriter := &closeWriter{w: ljLogger}
+				// Without this, rotated logs will never be deleted.
+				MaxBackups: 1,
+			}}
 			defer logWriter.Close()
 
-			logger := slog.Make(sloghuman.Sink(cmd.ErrOrStderr()), sloghuman.Sink(logWriter)).Leveled(slog.LevelDebug)
+			sinks = append(sinks, sloghuman.Sink(logWriter))
+			logger := slog.Make(sinks...).Leveled(slog.LevelDebug)
 
 			version := buildinfo.Version()
-			logger.Info(ctx, "starting agent",
-				slog.F("url", coderURL),
+			logger.Info(ctx, "agent is starting now",
+				slog.F("url", r.agentURL),
 				slog.F("auth", auth),
 				slog.F("version", version),
 			)
-			client := agentsdk.New(coderURL)
-			client.SDK.Logger = logger
+			client := agentsdk.New(r.agentURL)
+			client.SDK.SetLogger(logger)
 			// Set a reasonable timeout so requests can't hang forever!
-			client.SDK.HTTPClient.Timeout = 10 * time.Second
+			// The timeout needs to be reasonably long, because requests
+			// with large payloads can take a bit. e.g. startup scripts
+			// may take a while to insert.
+			client.SDK.HTTPClient.Timeout = 30 * time.Second
 
 			// Enable pprof handler
 			// This prevents the pprof import from being accidentally deleted.
 			_ = pprof.Handler
-			pprofSrvClose := serveHandler(ctx, logger, nil, pprofAddress, "pprof")
+			pprofSrvClose := ServeHandler(ctx, logger, nil, pprofAddress, "pprof")
 			defer pprofSrvClose()
+			if port, err := extractPort(pprofAddress); err == nil {
+				ignorePorts[port] = "pprof"
+			}
+
+			if port, err := extractPort(prometheusAddress); err == nil {
+				ignorePorts[port] = "prometheus"
+			}
+
+			if port, err := extractPort(debugAddress); err == nil {
+				ignorePorts[port] = "debug"
+			}
 
 			// exchangeToken returns a session token.
 			// This is abstracted to allow for the same looping condition
@@ -129,9 +199,19 @@ func workspaceAgent() *cobra.Command {
 			var exchangeToken func(context.Context) (agentsdk.AuthenticateResponse, error)
 			switch auth {
 			case "token":
-				token, err := cmd.Flags().GetString(varAgentToken)
-				if err != nil {
-					return xerrors.Errorf("CODER_AGENT_TOKEN must be set for token auth: %w", err)
+				token, _ := inv.ParsedFlags().GetString(varAgentToken)
+				if token == "" {
+					tokenFile, _ := inv.ParsedFlags().GetString(varAgentTokenFile)
+					if tokenFile != "" {
+						tokenBytes, err := os.ReadFile(tokenFile)
+						if err != nil {
+							return xerrors.Errorf("read token file %q: %w", tokenFile, err)
+						}
+						token = strings.TrimSpace(string(tokenBytes))
+					}
+				}
+				if token == "" {
+					return xerrors.Errorf("CODER_AGENT_TOKEN or CODER_AGENT_TOKEN_FILE must be set for token auth")
 				}
 				client.SetSessionToken(token)
 			case "google-instance-identity":
@@ -184,10 +264,27 @@ func workspaceAgent() *cobra.Command {
 				return xerrors.Errorf("add executable to $PATH: %w", err)
 			}
 
-			closer := agent.New(agent.Options{
-				Client: client,
-				Logger: logger,
-				LogDir: logDir,
+			prometheusRegistry := prometheus.NewRegistry()
+			subsystemsRaw := inv.Environ.Get(agent.EnvAgentSubsystem)
+			subsystems := []codersdk.AgentSubsystem{}
+			for _, s := range strings.Split(subsystemsRaw, ",") {
+				subsystem := codersdk.AgentSubsystem(strings.TrimSpace(s))
+				if subsystem == "" {
+					continue
+				}
+				if !subsystem.Valid() {
+					return xerrors.Errorf("invalid subsystem %q", subsystem)
+				}
+				subsystems = append(subsystems, subsystem)
+			}
+
+			procTicker := time.NewTicker(time.Second)
+			defer procTicker.Stop()
+			agnt := agent.New(agent.Options{
+				Client:            client,
+				Logger:            logger,
+				LogDir:            logDir,
+				TailnetListenPort: uint16(tailnetListenPort),
 				ExchangeToken: func(ctx context.Context) (string, error) {
 					if exchangeToken == nil {
 						return client.SDK.SessionToken(), nil
@@ -200,22 +297,119 @@ func workspaceAgent() *cobra.Command {
 					return resp.SessionToken, nil
 				},
 				EnvironmentVariables: map[string]string{
-					"GIT_ASKPASS": executablePath,
+					"GIT_ASKPASS":         executablePath,
+					agent.EnvProcPrioMgmt: os.Getenv(agent.EnvProcPrioMgmt),
 				},
+				IgnorePorts:   ignorePorts,
+				SSHMaxTimeout: sshMaxTimeout,
+				Subsystems:    subsystems,
+
+				PrometheusRegistry: prometheusRegistry,
+				Syscaller:          agentproc.NewSyscaller(),
+				// Intentionally set this to nil. It's mainly used
+				// for testing.
+				ModifiedProcesses: nil,
 			})
+
+			prometheusSrvClose := ServeHandler(ctx, logger, prometheusMetricsHandler(prometheusRegistry, logger), prometheusAddress, "prometheus")
+			defer prometheusSrvClose()
+
+			debugSrvClose := ServeHandler(ctx, logger, agnt.HTTPDebug(), debugAddress, "debug")
+			defer debugSrvClose()
+
 			<-ctx.Done()
-			return closer.Close()
+			return agnt.Close()
+		},
+	}
+
+	cmd.Options = clibase.OptionSet{
+		{
+			Flag:        "auth",
+			Default:     "token",
+			Description: "Specify the authentication type to use for the agent.",
+			Env:         "CODER_AGENT_AUTH",
+			Value:       clibase.StringOf(&auth),
+		},
+		{
+			Flag:        "log-dir",
+			Default:     os.TempDir(),
+			Description: "Specify the location for the agent log files.",
+			Env:         "CODER_AGENT_LOG_DIR",
+			Value:       clibase.StringOf(&logDir),
+		},
+		{
+			Flag:        "pprof-address",
+			Default:     "127.0.0.1:6060",
+			Env:         "CODER_AGENT_PPROF_ADDRESS",
+			Value:       clibase.StringOf(&pprofAddress),
+			Description: "The address to serve pprof.",
+		},
+		{
+			Flag: "no-reap",
+
+			Env:         "",
+			Description: "Do not start a process reaper.",
+			Value:       clibase.BoolOf(&noReap),
+		},
+		{
+			Flag: "ssh-max-timeout",
+			// tcpip.KeepaliveIdleOption = 72h + 1min (forwardTCPSockOpts() in tailnet/conn.go)
+			Default:     "72h",
+			Env:         "CODER_AGENT_SSH_MAX_TIMEOUT",
+			Description: "Specify the max timeout for a SSH connection, it is advisable to set it to a minimum of 60s, but no more than 72h.",
+			Value:       clibase.DurationOf(&sshMaxTimeout),
+		},
+		{
+			Flag:        "tailnet-listen-port",
+			Default:     "0",
+			Env:         "CODER_AGENT_TAILNET_LISTEN_PORT",
+			Description: "Specify a static port for Tailscale to use for listening.",
+			Value:       clibase.Int64Of(&tailnetListenPort),
+		},
+		{
+			Flag:        "prometheus-address",
+			Default:     "127.0.0.1:2112",
+			Env:         "CODER_AGENT_PROMETHEUS_ADDRESS",
+			Value:       clibase.StringOf(&prometheusAddress),
+			Description: "The bind address to serve Prometheus metrics.",
+		},
+		{
+			Flag:        "debug-address",
+			Default:     "127.0.0.1:2113",
+			Env:         "CODER_AGENT_DEBUG_ADDRESS",
+			Value:       clibase.StringOf(&debugAddress),
+			Description: "The bind address to serve a debug HTTP server.",
+		},
+		{
+			Name:        "Human Log Location",
+			Description: "Output human-readable logs to a given file.",
+			Flag:        "log-human",
+			Env:         "CODER_AGENT_LOGGING_HUMAN",
+			Default:     "/dev/stderr",
+			Value:       clibase.StringOf(&slogHumanPath),
+		},
+		{
+			Name:        "JSON Log Location",
+			Description: "Output JSON logs to a given file.",
+			Flag:        "log-json",
+			Env:         "CODER_AGENT_LOGGING_JSON",
+			Default:     "",
+			Value:       clibase.StringOf(&slogJSONPath),
+		},
+		{
+			Name:        "Stackdriver Log Location",
+			Description: "Output Stackdriver compatible logs to a given file.",
+			Flag:        "log-stackdriver",
+			Env:         "CODER_AGENT_LOGGING_STACKDRIVER",
+			Default:     "",
+			Value:       clibase.StringOf(&slogStackdriverPath),
 		},
 	}
 
-	cliflag.StringVarP(cmd.Flags(), &auth, "auth", "", "CODER_AGENT_AUTH", "token", "Specify the authentication type to use for the agent")
-	cliflag.StringVarP(cmd.Flags(), &logDir, "log-dir", "", "CODER_AGENT_LOG_DIR", os.TempDir(), "Specify the location for the agent log files")
-	cliflag.StringVarP(cmd.Flags(), &pprofAddress, "pprof-address", "", "CODER_AGENT_PPROF_ADDRESS", "127.0.0.1:6060", "The address to serve pprof.")
-	cliflag.BoolVarP(cmd.Flags(), &noReap, "no-reap", "", "", false, "Do not start a process reaper.")
 	return cmd
 }
 
-func serveHandler(ctx context.Context, logger slog.Logger, handler http.Handler, addr, name string) (closeFunc func()) {
+func ServeHandler(ctx context.Context, logger slog.Logger, handler http.Handler, addr, name string) (closeFunc func()) {
 	logger.Debug(ctx, "http server listening", slog.F("addr", addr), slog.F("name", name))
 
 	// ReadHeaderTimeout is purposefully not enabled. It caused some issues with
@@ -238,16 +432,16 @@ func serveHandler(ctx context.Context, logger slog.Logger, handler http.Handler,
 	}
 }
 
-// closeWriter is a wrapper around an io.WriteCloser that prevents
-// writes after Close. This is necessary because lumberjack will
-// re-open the file on write.
-type closeWriter struct {
+// lumberjackWriteCloseFixer is a wrapper around an io.WriteCloser that
+// prevents writes after Close. This is necessary because lumberjack
+// re-opens the file on Write.
+type lumberjackWriteCloseFixer struct {
 	w      io.WriteCloser
 	mu     sync.Mutex // Protects following.
 	closed bool
 }
 
-func (c *closeWriter) Close() error {
+func (c *lumberjackWriteCloseFixer) Close() error {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 
@@ -255,7 +449,7 @@ func (c *closeWriter) Close() error {
 	return c.w.Close()
 }
 
-func (c *closeWriter) Write(p []byte) (int, error) {
+func (c *lumberjackWriteCloseFixer) Write(p []byte) (int, error) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 
@@ -264,3 +458,58 @@ func (c *closeWriter) Write(p []byte) (int, error) {
 	}
 	return c.w.Write(p)
 }
+
+// extractPort handles different url strings.
+// - localhost:6060
+// - http://localhost:6060
+func extractPort(u string) (int, error) {
+	port, firstError := urlPort(u)
+	if firstError == nil {
+		return port, nil
+	}
+
+	// Try with a scheme
+	port, err := urlPort("http://" + u)
+	if err == nil {
+		return port, nil
+	}
+	return -1, xerrors.Errorf("invalid url %q: %w", u, firstError)
+}
+
+// urlPort extracts the port from a valid URL.
+func urlPort(u string) (int, error) {
+	parsed, err := url.Parse(u)
+	if err != nil {
+		return -1, xerrors.Errorf("invalid url %q: %w", u, err)
+	}
+	if parsed.Port() != "" {
+		port, err := strconv.ParseUint(parsed.Port(), 10, 16)
+		if err == nil && port > 0 && port < 1<<16 {
+			return int(port), nil
+		}
+	}
+	return -1, xerrors.Errorf("invalid port: %s", u)
+}
+
+func prometheusMetricsHandler(prometheusRegistry *prometheus.Registry, logger slog.Logger) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "text/plain")
+
+		// Based on: https://github.com/tailscale/tailscale/blob/280255acae604796a1113861f5a84e6fa2dc6121/ipn/localapi/localapi.go#L489
+		clientmetric.WritePrometheusExpositionFormat(w)
+
+		metricFamilies, err := prometheusRegistry.Gather()
+		if err != nil {
+			logger.Error(context.Background(), "Prometheus handler can't gather metric families", slog.Error(err))
+			return
+		}
+
+		for _, metricFamily := range metricFamilies {
+			_, err = expfmt.MetricFamilyToText(w, metricFamily)
+			if err != nil {
+				logger.Error(context.Background(), "expfmt.MetricFamilyToText failed", slog.Error(err))
+				return
+			}
+		}
+	})
+}

cli/agent_internal_test.go

@@ -0,0 +1,69 @@
+package cli
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func Test_extractPort(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name      string
+		urlString string
+		want      int
+		wantErr   bool
+	}{
+		{
+			name:      "Empty",
+			urlString: "",
+			wantErr:   true,
+		},
+		{
+			name:      "NoScheme",
+			urlString: "localhost:6060",
+			want:      6060,
+		},
+		{
+			name:      "WithScheme",
+			urlString: "http://localhost:6060",
+			want:      6060,
+		},
+		{
+			name:      "NoPort",
+			urlString: "http://localhost",
+			wantErr:   true,
+		},
+		{
+			name:      "NoPortNoScheme",
+			urlString: "localhost",
+			wantErr:   true,
+		},
+		{
+			name:      "OnlyPort",
+			urlString: "6060",
+			wantErr:   true,
+		},
+		{
+			name:      "127.0.0.1",
+			urlString: "127.0.0.1:2113",
+			want:      2113,
+			wantErr:   false,
+		},
+	}
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+			got, err := extractPort(tt.urlString)
+			if tt.wantErr {
+				require.Error(t, err, fmt.Sprintf("extractPort(%v)", tt.urlString))
+			} else {
+				require.NoError(t, err, fmt.Sprintf("extractPort(%v)", tt.urlString))
+				require.Equal(t, tt.want, got, fmt.Sprintf("extractPort(%v)", tt.urlString))
+			}
+		})
+	}
+}

cli/agent_test.go

@@ -2,6 +2,7 @@ package cli_test
 
 import (
 	"context"
+	"fmt"
 	"os"
 	"path/filepath"
 	"runtime"
@@ -12,11 +13,13 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
-	"github.com/coder/coder/cli/clitest"
-	"github.com/coder/coder/coderd/coderdtest"
-	"github.com/coder/coder/provisioner/echo"
-	"github.com/coder/coder/provisionersdk/proto"
-	"github.com/coder/coder/testutil"
+	"github.com/coder/coder/v2/agent"
+	"github.com/coder/coder/v2/cli/clitest"
+	"github.com/coder/coder/v2/coderd/coderdtest"
+	"github.com/coder/coder/v2/codersdk"
+	"github.com/coder/coder/v2/provisioner/echo"
+	"github.com/coder/coder/v2/provisionersdk/proto"
+	"github.com/coder/coder/v2/pty/ptytest"
 )
 
 func TestWorkspaceAgent(t *testing.T) {
@@ -31,49 +34,30 @@ func TestWorkspaceAgent(t *testing.T) {
 		})
 		user := coderdtest.CreateFirstUser(t, client)
 		version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, &echo.Responses{
-			Parse: echo.ParseComplete,
-			ProvisionApply: []*proto.Provision_Response{{
-				Type: &proto.Provision_Response_Complete{
-					Complete: &proto.Provision_Complete{
-						Resources: []*proto.Resource{{
-							Name: "somename",
-							Type: "someinstance",
-							Agents: []*proto.Agent{{
-								Id:   uuid.NewString(),
-								Name: "someagent",
-								Auth: &proto.Agent_Token{
-									Token: authToken,
-								},
-							}},
-						}},
-					},
-				},
-			}},
+			Parse:          echo.ParseComplete,
+			ProvisionApply: echo.ProvisionApplyWithAgent(authToken),
 		})
 		template := coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID)
-		coderdtest.AwaitTemplateVersionJob(t, client, version.ID)
+		coderdtest.AwaitTemplateVersionJobCompleted(t, client, version.ID)
 		workspace := coderdtest.CreateWorkspace(t, client, user.OrganizationID, template.ID)
-		coderdtest.AwaitWorkspaceBuildJob(t, client, workspace.LatestBuild.ID)
+		coderdtest.AwaitWorkspaceBuildJobCompleted(t, client, workspace.LatestBuild.ID)
 
 		logDir := t.TempDir()
-		cmd, _ := clitest.New(t,
+		inv, _ := clitest.New(t,
 			"agent",
 			"--auth", "token",
 			"--agent-token", authToken,
 			"--agent-url", client.URL.String(),
 			"--log-dir", logDir,
 		)
-		ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitMedium)
-		defer cancel()
-		errC := make(chan error, 1)
-		go func() {
-			errC <- cmd.ExecuteContext(ctx)
-		}()
-		coderdtest.AwaitWorkspaceAgents(t, client, workspace.ID)
 
-		cancel()
-		err := <-errC
-		require.NoError(t, err)
+		pty := ptytest.New(t).Attach(inv)
+
+		clitest.Start(t, inv)
+		ctx := inv.Context()
+		pty.ExpectMatchContext(ctx, "agent is starting now")
+
+		coderdtest.AwaitWorkspaceAgents(t, client, workspace.ID)
 
 		info, err := os.Stat(filepath.Join(logDir, "coder-agent.log"))
 		require.NoError(t, err)
@@ -91,9 +75,9 @@ func TestWorkspaceAgent(t *testing.T) {
 		user := coderdtest.CreateFirstUser(t, client)
 		version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, &echo.Responses{
 			Parse: echo.ParseComplete,
-			ProvisionApply: []*proto.Provision_Response{{
-				Type: &proto.Provision_Response_Complete{
-					Complete: &proto.Provision_Complete{
+			ProvisionApply: []*proto.Response{{
+				Type: &proto.Response_Apply{
+					Apply: &proto.ApplyComplete{
 						Resources: []*proto.Resource{{
 							Name: "somename",
 							Type: "someinstance",
@@ -108,20 +92,17 @@ func TestWorkspaceAgent(t *testing.T) {
 			}},
 		})
 		template := coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID)
-		coderdtest.AwaitTemplateVersionJob(t, client, version.ID)
+		coderdtest.AwaitTemplateVersionJobCompleted(t, client, version.ID)
 		workspace := coderdtest.CreateWorkspace(t, client, user.OrganizationID, template.ID)
-		coderdtest.AwaitWorkspaceBuildJob(t, client, workspace.LatestBuild.ID)
+		coderdtest.AwaitWorkspaceBuildJobCompleted(t, client, workspace.LatestBuild.ID)
 
-		cmd, _ := clitest.New(t, "agent", "--auth", "azure-instance-identity", "--agent-url", client.URL.String())
-		ctx, cancelFunc := context.WithCancel(context.Background())
-		defer cancelFunc()
-		errC := make(chan error)
-		go func() {
-			// A linting error occurs for weakly typing the context value here.
-			//nolint // The above seems reasonable for a one-off test.
-			ctx := context.WithValue(ctx, "azure-client", metadataClient)
-			errC <- cmd.ExecuteContext(ctx)
-		}()
+		inv, _ := clitest.New(t, "agent", "--auth", "azure-instance-identity", "--agent-url", client.URL.String())
+		inv = inv.WithContext(
+			//nolint:revive,staticcheck
+			context.WithValue(inv.Context(), "azure-client", metadataClient),
+		)
+		ctx := inv.Context()
+		clitest.Start(t, inv)
 		coderdtest.AwaitWorkspaceAgents(t, client, workspace.ID)
 		workspace, err := client.Workspace(ctx, workspace.ID)
 		require.NoError(t, err)
@@ -132,10 +113,7 @@ func TestWorkspaceAgent(t *testing.T) {
 		dialer, err := client.DialWorkspaceAgent(ctx, resources[0].Agents[0].ID, nil)
 		require.NoError(t, err)
 		defer dialer.Close()
-		require.True(t, dialer.AwaitReachable(context.Background()))
-		cancelFunc()
-		err = <-errC
-		require.NoError(t, err)
+		require.True(t, dialer.AwaitReachable(ctx))
 	})
 
 	t.Run("AWS", func(t *testing.T) {
@@ -149,9 +127,9 @@ func TestWorkspaceAgent(t *testing.T) {
 		user := coderdtest.CreateFirstUser(t, client)
 		version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, &echo.Responses{
 			Parse: echo.ParseComplete,
-			ProvisionApply: []*proto.Provision_Response{{
-				Type: &proto.Provision_Response_Complete{
-					Complete: &proto.Provision_Complete{
+			ProvisionApply: []*proto.Response{{
+				Type: &proto.Response_Apply{
+					Apply: &proto.ApplyComplete{
 						Resources: []*proto.Resource{{
 							Name: "somename",
 							Type: "someinstance",
@@ -166,20 +144,17 @@ func TestWorkspaceAgent(t *testing.T) {
 			}},
 		})
 		template := coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID)
-		coderdtest.AwaitTemplateVersionJob(t, client, version.ID)
+		coderdtest.AwaitTemplateVersionJobCompleted(t, client, version.ID)
 		workspace := coderdtest.CreateWorkspace(t, client, user.OrganizationID, template.ID)
-		coderdtest.AwaitWorkspaceBuildJob(t, client, workspace.LatestBuild.ID)
+		coderdtest.AwaitWorkspaceBuildJobCompleted(t, client, workspace.LatestBuild.ID)
 
-		cmd, _ := clitest.New(t, "agent", "--auth", "aws-instance-identity", "--agent-url", client.URL.String())
-		ctx, cancelFunc := context.WithCancel(context.Background())
-		defer cancelFunc()
-		errC := make(chan error)
-		go func() {
-			// A linting error occurs for weakly typing the context value here.
-			//nolint // The above seems reasonable for a one-off test.
-			ctx := context.WithValue(ctx, "aws-client", metadataClient)
-			errC <- cmd.ExecuteContext(ctx)
-		}()
+		inv, _ := clitest.New(t, "agent", "--auth", "aws-instance-identity", "--agent-url", client.URL.String())
+		inv = inv.WithContext(
+			//nolint:revive,staticcheck
+			context.WithValue(inv.Context(), "aws-client", metadataClient),
+		)
+		clitest.Start(t, inv)
+		ctx := inv.Context()
 		coderdtest.AwaitWorkspaceAgents(t, client, workspace.ID)
 		workspace, err := client.Workspace(ctx, workspace.ID)
 		require.NoError(t, err)
@@ -190,26 +165,24 @@ func TestWorkspaceAgent(t *testing.T) {
 		dialer, err := client.DialWorkspaceAgent(ctx, resources[0].Agents[0].ID, nil)
 		require.NoError(t, err)
 		defer dialer.Close()
-		require.True(t, dialer.AwaitReachable(context.Background()))
-		cancelFunc()
-		err = <-errC
-		require.NoError(t, err)
+		require.True(t, dialer.AwaitReachable(ctx))
 	})
 
 	t.Run("GoogleCloud", func(t *testing.T) {
 		t.Parallel()
 		instanceID := "instanceidentifier"
-		validator, metadata := coderdtest.NewGoogleInstanceIdentity(t, instanceID, false)
+		validator, metadataClient := coderdtest.NewGoogleInstanceIdentity(t, instanceID, false)
 		client := coderdtest.New(t, &coderdtest.Options{
 			GoogleTokenValidator:     validator,
 			IncludeProvisionerDaemon: true,
 		})
-		user := coderdtest.CreateFirstUser(t, client)
-		version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, &echo.Responses{
+		owner := coderdtest.CreateFirstUser(t, client)
+		member, _ := coderdtest.CreateAnotherUser(t, client, owner.OrganizationID)
+		version := coderdtest.CreateTemplateVersion(t, client, owner.OrganizationID, &echo.Responses{
 			Parse: echo.ParseComplete,
-			ProvisionApply: []*proto.Provision_Response{{
-				Type: &proto.Provision_Response_Complete{
-					Complete: &proto.Provision_Complete{
+			ProvisionApply: []*proto.Response{{
+				Type: &proto.Response_Apply{
+					Apply: &proto.ApplyComplete{
 						Resources: []*proto.Resource{{
 							Name: "somename",
 							Type: "someinstance",
@@ -223,21 +196,23 @@ func TestWorkspaceAgent(t *testing.T) {
 				},
 			}},
 		})
-		template := coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID)
-		coderdtest.AwaitTemplateVersionJob(t, client, version.ID)
-		workspace := coderdtest.CreateWorkspace(t, client, user.OrganizationID, template.ID)
-		coderdtest.AwaitWorkspaceBuildJob(t, client, workspace.LatestBuild.ID)
+		template := coderdtest.CreateTemplate(t, client, owner.OrganizationID, version.ID)
+		coderdtest.AwaitTemplateVersionJobCompleted(t, client, version.ID)
+		workspace := coderdtest.CreateWorkspace(t, member, owner.OrganizationID, template.ID)
+		coderdtest.AwaitWorkspaceBuildJobCompleted(t, client, workspace.LatestBuild.ID)
+
+		inv, cfg := clitest.New(t, "agent", "--auth", "google-instance-identity", "--agent-url", client.URL.String())
+		ptytest.New(t).Attach(inv)
+		clitest.SetupConfig(t, member, cfg)
+		clitest.Start(t,
+			inv.WithContext(
+				//nolint:revive,staticcheck
+				context.WithValue(inv.Context(), "gcp-client", metadataClient),
+			),
+		)
+
+		ctx := inv.Context()
 
-		cmd, _ := clitest.New(t, "agent", "--auth", "google-instance-identity", "--agent-url", client.URL.String())
-		ctx, cancelFunc := context.WithCancel(context.Background())
-		defer cancelFunc()
-		errC := make(chan error)
-		go func() {
-			// A linting error occurs for weakly typing the context value here.
-			//nolint // The above seems reasonable for a one-off test.
-			ctx := context.WithValue(ctx, "gcp-client", metadata)
-			errC <- cmd.ExecuteContext(ctx)
-		}()
 		coderdtest.AwaitWorkspaceAgents(t, client, workspace.ID)
 		workspace, err := client.Workspace(ctx, workspace.ID)
 		require.NoError(t, err)
@@ -248,7 +223,7 @@ func TestWorkspaceAgent(t *testing.T) {
 		dialer, err := client.DialWorkspaceAgent(ctx, resources[0].Agents[0].ID, nil)
 		require.NoError(t, err)
 		defer dialer.Close()
-		require.True(t, dialer.AwaitReachable(context.Background()))
+		require.True(t, dialer.AwaitReachable(ctx))
 		sshClient, err := dialer.SSHClient(ctx)
 		require.NoError(t, err)
 		defer sshClient.Close()
@@ -264,9 +239,47 @@ func TestWorkspaceAgent(t *testing.T) {
 		require.NoError(t, err)
 		_, err = uuid.Parse(strings.TrimSpace(string(token)))
 		require.NoError(t, err)
+	})
 
-		cancelFunc()
-		err = <-errC
-		require.NoError(t, err)
+	t.Run("PostStartup", func(t *testing.T) {
+		t.Parallel()
+
+		authToken := uuid.NewString()
+		client := coderdtest.New(t, &coderdtest.Options{
+			IncludeProvisionerDaemon: true,
+		})
+		user := coderdtest.CreateFirstUser(t, client)
+		version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, &echo.Responses{
+			Parse:          echo.ParseComplete,
+			ProvisionApply: echo.ProvisionApplyWithAgent(authToken),
+		})
+		template := coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID)
+		coderdtest.AwaitTemplateVersionJobCompleted(t, client, version.ID)
+		workspace := coderdtest.CreateWorkspace(t, client, user.OrganizationID, template.ID)
+		coderdtest.AwaitWorkspaceBuildJobCompleted(t, client, workspace.LatestBuild.ID)
+
+		logDir := t.TempDir()
+		inv, _ := clitest.New(t,
+			"agent",
+			"--auth", "token",
+			"--agent-token", authToken,
+			"--agent-url", client.URL.String(),
+			"--log-dir", logDir,
+		)
+		// Set the subsystems for the agent.
+		inv.Environ.Set(agent.EnvAgentSubsystem, fmt.Sprintf("%s,%s", codersdk.AgentSubsystemExectrace, codersdk.AgentSubsystemEnvbox))
+
+		pty := ptytest.New(t).Attach(inv)
+
+		clitest.Start(t, inv)
+		pty.ExpectMatchContext(inv.Context(), "agent is starting now")
+
+		resources := coderdtest.AwaitWorkspaceAgents(t, client, workspace.ID)
+		require.Len(t, resources, 1)
+		require.Len(t, resources[0].Agents, 1)
+		require.Len(t, resources[0].Agents[0].Subsystems, 2)
+		// Sorted
+		require.Equal(t, codersdk.AgentSubsystemEnvbox, resources[0].Agents[0].Subsystems[0])
+		require.Equal(t, codersdk.AgentSubsystemExectrace, resources[0].Agents[0].Subsystems[1])
 	})
 }

cli/clibase/clibase.go

@@ -0,0 +1,80 @@
+// Package clibase offers an all-in-one solution for a highly configurable CLI
+// application. Within Coder, we use it for all of our subcommands, which
+// demands more functionality than cobra/viber offers.
+//
+// The Command interface is loosely based on the chi middleware pattern and
+// http.Handler/HandlerFunc.
+package clibase
+
+import (
+	"strings"
+
+	"golang.org/x/exp/maps"
+)
+
+// Group describes a hierarchy of groups that an option or command belongs to.
+type Group struct {
+	Parent      *Group `json:"parent,omitempty"`
+	Name        string `json:"name,omitempty"`
+	YAML        string `json:"yaml,omitempty"`
+	Description string `json:"description,omitempty"`
+}
+
+// Ancestry returns the group and all of its parents, in order.
+func (g *Group) Ancestry() []Group {
+	if g == nil {
+		return nil
+	}
+
+	groups := []Group{*g}
+	for p := g.Parent; p != nil; p = p.Parent {
+		// Prepend to the slice so that the order is correct.
+		groups = append([]Group{*p}, groups...)
+	}
+	return groups
+}
+
+func (g *Group) FullName() string {
+	var names []string
+	for _, g := range g.Ancestry() {
+		names = append(names, g.Name)
+	}
+	return strings.Join(names, " / ")
+}
+
+// Annotations is an arbitrary key-mapping used to extend the Option and Command types.
+// Its methods won't panic if the map is nil.
+type Annotations map[string]string
+
+// Mark sets a value on the annotations map, creating one
+// if it doesn't exist. Mark does not mutate the original and
+// returns a copy. It is suitable for chaining.
+func (a Annotations) Mark(key string, value string) Annotations {
+	var aa Annotations
+	if a != nil {
+		aa = maps.Clone(a)
+	} else {
+		aa = make(Annotations)
+	}
+	aa[key] = value
+	return aa
+}
+
+// IsSet returns true if the key is set in the annotations map.
+func (a Annotations) IsSet(key string) bool {
+	if a == nil {
+		return false
+	}
+	_, ok := a[key]
+	return ok
+}
+
+// Get retrieves a key from the map, returning false if the key is not found
+// or the map is nil.
+func (a Annotations) Get(key string) (string, bool) {
+	if a == nil {
+		return "", false
+	}
+	v, ok := a[key]
+	return v, ok
+}

cli/clibase/cmd.go

@@ -0,0 +1,581 @@
+package clibase
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"strings"
+	"unicode"
+
+	"github.com/spf13/pflag"
+	"golang.org/x/exp/slices"
+	"golang.org/x/xerrors"
+	"gopkg.in/yaml.v3"
+
+	"github.com/coder/coder/v2/coderd/util/slice"
+)
+
+// Cmd describes an executable command.
+type Cmd struct {
+	// Parent is the direct parent of the command.
+	Parent *Cmd
+	// Children is a list of direct descendants.
+	Children []*Cmd
+	// Use is provided in form "command [flags] [args...]".
+	Use string
+
+	// Aliases is a list of alternative names for the command.
+	Aliases []string
+
+	// Short is a one-line description of the command.
+	Short string
+
+	// Hidden determines whether the command should be hidden from help.
+	Hidden bool
+
+	// RawArgs determines whether the command should receive unparsed arguments.
+	// No flags are parsed when set, and the command is responsible for parsing
+	// its own flags.
+	RawArgs bool
+
+	// Long is a detailed description of the command,
+	// presented on its help page. It may contain examples.
+	Long        string
+	Options     OptionSet
+	Annotations Annotations
+
+	// Middleware is called before the Handler.
+	// Use Chain() to combine multiple middlewares.
+	Middleware  MiddlewareFunc
+	Handler     HandlerFunc
+	HelpHandler HandlerFunc
+}
+
+// AddSubcommands adds the given subcommands, setting their
+// Parent field automatically.
+func (c *Cmd) AddSubcommands(cmds ...*Cmd) {
+	for _, cmd := range cmds {
+		cmd.Parent = c
+		c.Children = append(c.Children, cmd)
+	}
+}
+
+// Walk calls fn for the command and all its children.
+func (c *Cmd) Walk(fn func(*Cmd)) {
+	fn(c)
+	for _, child := range c.Children {
+		child.Parent = c
+		child.Walk(fn)
+	}
+}
+
+// PrepareAll performs initialization and linting on the command and all its children.
+func (c *Cmd) PrepareAll() error {
+	if c.Use == "" {
+		return xerrors.New("command must have a Use field so that it has a name")
+	}
+	var merr error
+
+	for i := range c.Options {
+		opt := &c.Options[i]
+		if opt.Name == "" {
+			switch {
+			case opt.Flag != "":
+				opt.Name = opt.Flag
+			case opt.Env != "":
+				opt.Name = opt.Env
+			case opt.YAML != "":
+				opt.Name = opt.YAML
+			default:
+				merr = errors.Join(merr, xerrors.Errorf("option must have a Name, Flag, Env or YAML field"))
+			}
+		}
+		if opt.Description != "" {
+			// Enforce that description uses sentence form.
+			if unicode.IsLower(rune(opt.Description[0])) {
+				merr = errors.Join(merr, xerrors.Errorf("option %q description should start with a capital letter", opt.Name))
+			}
+			if !strings.HasSuffix(opt.Description, ".") {
+				merr = errors.Join(merr, xerrors.Errorf("option %q description should end with a period", opt.Name))
+			}
+		}
+	}
+
+	slices.SortFunc(c.Options, func(a, b Option) int {
+		return slice.Ascending(a.Name, b.Name)
+	})
+	slices.SortFunc(c.Children, func(a, b *Cmd) int {
+		return slice.Ascending(a.Name(), b.Name())
+	})
+	for _, child := range c.Children {
+		child.Parent = c
+		err := child.PrepareAll()
+		if err != nil {
+			merr = errors.Join(merr, xerrors.Errorf("command %v: %w", child.Name(), err))
+		}
+	}
+	return merr
+}
+
+// Name returns the first word in the Use string.
+func (c *Cmd) Name() string {
+	return strings.Split(c.Use, " ")[0]
+}
+
+// FullName returns the full invocation name of the command,
+// as seen on the command line.
+func (c *Cmd) FullName() string {
+	var names []string
+	if c.Parent != nil {
+		names = append(names, c.Parent.FullName())
+	}
+	names = append(names, c.Name())
+	return strings.Join(names, " ")
+}
+
+// FullName returns usage of the command, preceded
+// by the usage of its parents.
+func (c *Cmd) FullUsage() string {
+	var uses []string
+	if c.Parent != nil {
+		uses = append(uses, c.Parent.FullName())
+	}
+	uses = append(uses, c.Use)
+	return strings.Join(uses, " ")
+}
+
+// FullOptions returns the options of the command and its parents.
+func (c *Cmd) FullOptions() OptionSet {
+	var opts OptionSet
+	if c.Parent != nil {
+		opts = append(opts, c.Parent.FullOptions()...)
+	}
+	opts = append(opts, c.Options...)
+	return opts
+}
+
+// Invoke creates a new invocation of the command, with
+// stdio discarded.
+//
+// The returned invocation is not live until Run() is called.
+func (c *Cmd) Invoke(args ...string) *Invocation {
+	return &Invocation{
+		Command: c,
+		Args:    args,
+		Stdout:  io.Discard,
+		Stderr:  io.Discard,
+		Stdin:   strings.NewReader(""),
+	}
+}
+
+// Invocation represents an instance of a command being executed.
+type Invocation struct {
+	ctx         context.Context
+	Command     *Cmd
+	parsedFlags *pflag.FlagSet
+	Args        []string
+	// Environ is a list of environment variables. Use EnvsWithPrefix to parse
+	// os.Environ.
+	Environ Environ
+	Stdout  io.Writer
+	Stderr  io.Writer
+	Stdin   io.Reader
+}
+
+// WithOS returns the invocation as a main package, filling in the invocation's unset
+// fields with OS defaults.
+func (inv *Invocation) WithOS() *Invocation {
+	return inv.with(func(i *Invocation) {
+		i.Stdout = os.Stdout
+		i.Stderr = os.Stderr
+		i.Stdin = os.Stdin
+		i.Args = os.Args[1:]
+		i.Environ = ParseEnviron(os.Environ(), "")
+	})
+}
+
+func (inv *Invocation) Context() context.Context {
+	if inv.ctx == nil {
+		return context.Background()
+	}
+	return inv.ctx
+}
+
+func (inv *Invocation) ParsedFlags() *pflag.FlagSet {
+	if inv.parsedFlags == nil {
+		panic("flags not parsed, has Run() been called?")
+	}
+	return inv.parsedFlags
+}
+
+type runState struct {
+	allArgs      []string
+	commandDepth int
+
+	flagParseErr error
+}
+
+func copyFlagSetWithout(fs *pflag.FlagSet, without string) *pflag.FlagSet {
+	fs2 := pflag.NewFlagSet("", pflag.ContinueOnError)
+	fs2.Usage = func() {}
+	fs.VisitAll(func(f *pflag.Flag) {
+		if f.Name == without {
+			return
+		}
+		fs2.AddFlag(f)
+	})
+	return fs2
+}
+
+// run recursively executes the command and its children.
+// allArgs is wired through the stack so that global flags can be accepted
+// anywhere in the command invocation.
+func (inv *Invocation) run(state *runState) error {
+	err := inv.Command.Options.ParseEnv(inv.Environ)
+	if err != nil {
+		return xerrors.Errorf("parsing env: %w", err)
+	}
+
+	// Now the fun part, argument parsing!
+
+	children := make(map[string]*Cmd)
+	for _, child := range inv.Command.Children {
+		child.Parent = inv.Command
+		for _, name := range append(child.Aliases, child.Name()) {
+			if _, ok := children[name]; ok {
+				return xerrors.Errorf("duplicate command name: %s", name)
+			}
+			children[name] = child
+		}
+	}
+
+	if inv.parsedFlags == nil {
+		inv.parsedFlags = pflag.NewFlagSet(inv.Command.Name(), pflag.ContinueOnError)
+		// We handle Usage ourselves.
+		inv.parsedFlags.Usage = func() {}
+	}
+
+	// If we find a duplicate flag, we want the deeper command's flag to override
+	// the shallow one. Unfortunately, pflag has no way to remove a flag, so we
+	// have to create a copy of the flagset without a value.
+	inv.Command.Options.FlagSet().VisitAll(func(f *pflag.Flag) {
+		if inv.parsedFlags.Lookup(f.Name) != nil {
+			inv.parsedFlags = copyFlagSetWithout(inv.parsedFlags, f.Name)
+		}
+		inv.parsedFlags.AddFlag(f)
+	})
+
+	var parsedArgs []string
+
+	if !inv.Command.RawArgs {
+		// Flag parsing will fail on intermediate commands in the command tree,
+		// so we check the error after looking for a child command.
+		state.flagParseErr = inv.parsedFlags.Parse(state.allArgs)
+		parsedArgs = inv.parsedFlags.Args()
+	}
+
+	// Set value sources for flags.
+	for i, opt := range inv.Command.Options {
+		if fl := inv.parsedFlags.Lookup(opt.Flag); fl != nil && fl.Changed {
+			inv.Command.Options[i].ValueSource = ValueSourceFlag
+		}
+	}
+
+	// Read YAML configs, if any.
+	for _, opt := range inv.Command.Options {
+		path, ok := opt.Value.(*YAMLConfigPath)
+		if !ok || path.String() == "" {
+			continue
+		}
+
+		byt, err := os.ReadFile(path.String())
+		if err != nil {
+			return xerrors.Errorf("reading yaml: %w", err)
+		}
+
+		var n yaml.Node
+		err = yaml.Unmarshal(byt, &n)
+		if err != nil {
+			return xerrors.Errorf("decoding yaml: %w", err)
+		}
+
+		err = inv.Command.Options.UnmarshalYAML(&n)
+		if err != nil {
+			return xerrors.Errorf("applying yaml: %w", err)
+		}
+	}
+
+	err = inv.Command.Options.SetDefaults()
+	if err != nil {
+		return xerrors.Errorf("setting defaults: %w", err)
+	}
+
+	// Run child command if found (next child only)
+	// We must do subcommand detection after flag parsing so we don't mistake flag
+	// values for subcommand names.
+	if len(parsedArgs) > state.commandDepth {
+		nextArg := parsedArgs[state.commandDepth]
+		if child, ok := children[nextArg]; ok {
+			child.Parent = inv.Command
+			inv.Command = child
+			state.commandDepth++
+			return inv.run(state)
+		}
+	}
+
+	// Flag parse errors are irrelevant for raw args commands.
+	if !inv.Command.RawArgs && state.flagParseErr != nil && !errors.Is(state.flagParseErr, pflag.ErrHelp) {
+		return xerrors.Errorf(
+			"parsing flags (%v) for %q: %w",
+			state.allArgs,
+			inv.Command.FullName(), state.flagParseErr,
+		)
+	}
+
+	// All options should be set. Check all required options have sources,
+	// meaning they were set by the user in some way (env, flag, etc).
+	var missing []string
+	for _, opt := range inv.Command.Options {
+		if opt.Required && opt.ValueSource == ValueSourceNone {
+			missing = append(missing, opt.Flag)
+		}
+	}
+	if len(missing) > 0 {
+		return xerrors.Errorf("Missing values for the required flags: %s", strings.Join(missing, ", "))
+	}
+
+	if inv.Command.RawArgs {
+		// If we're at the root command, then the name is omitted
+		// from the arguments, so we can just use the entire slice.
+		if state.commandDepth == 0 {
+			inv.Args = state.allArgs
+		} else {
+			argPos, err := findArg(inv.Command.Name(), state.allArgs, inv.parsedFlags)
+			if err != nil {
+				panic(err)
+			}
+			inv.Args = state.allArgs[argPos+1:]
+		}
+	} else {
+		// In non-raw-arg mode, we want to skip over flags.
+		inv.Args = parsedArgs[state.commandDepth:]
+	}
+
+	mw := inv.Command.Middleware
+	if mw == nil {
+		mw = Chain()
+	}
+
+	ctx := inv.ctx
+	if ctx == nil {
+		ctx = context.Background()
+	}
+
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+	inv = inv.WithContext(ctx)
+
+	if inv.Command.Handler == nil || errors.Is(state.flagParseErr, pflag.ErrHelp) {
+		if inv.Command.HelpHandler == nil {
+			return xerrors.Errorf("no handler or help for command %s", inv.Command.FullName())
+		}
+		return inv.Command.HelpHandler(inv)
+	}
+
+	err = mw(inv.Command.Handler)(inv)
+	if err != nil {
+		return &RunCommandError{
+			Cmd: inv.Command,
+			Err: err,
+		}
+	}
+	return nil
+}
+
+type RunCommandError struct {
+	Cmd *Cmd
+	Err error
+}
+
+func (e *RunCommandError) Unwrap() error {
+	return e.Err
+}
+
+func (e *RunCommandError) Error() string {
+	return fmt.Sprintf("running command %q: %+v", e.Cmd.FullName(), e.Err)
+}
+
+// findArg returns the index of the first occurrence of arg in args, skipping
+// over all flags.
+func findArg(want string, args []string, fs *pflag.FlagSet) (int, error) {
+	for i := 0; i < len(args); i++ {
+		arg := args[i]
+		if !strings.HasPrefix(arg, "-") {
+			if arg == want {
+				return i, nil
+			}
+			continue
+		}
+
+		// This is a flag!
+		if strings.Contains(arg, "=") {
+			// The flag contains the value in the same arg, just skip.
+			continue
+		}
+
+		// We need to check if NoOptValue is set, then we should not wait
+		// for the next arg to be the value.
+		f := fs.Lookup(strings.TrimLeft(arg, "-"))
+		if f == nil {
+			return -1, xerrors.Errorf("unknown flag: %s", arg)
+		}
+		if f.NoOptDefVal != "" {
+			continue
+		}
+
+		if i == len(args)-1 {
+			return -1, xerrors.Errorf("flag %s requires a value", arg)
+		}
+
+		// Skip the value.
+		i++
+	}
+
+	return -1, xerrors.Errorf("arg %s not found", want)
+}
+
+// Run executes the command.
+// If two command share a flag name, the first command wins.
+//
+//nolint:revive
+func (inv *Invocation) Run() (err error) {
+	defer func() {
+		// Pflag is panicky, so additional context is helpful in tests.
+		if flag.Lookup("test.v") == nil {
+			return
+		}
+		if r := recover(); r != nil {
+			err = xerrors.Errorf("panic recovered for %s: %v", inv.Command.FullName(), r)
+			panic(err)
+		}
+	}()
+	// We close Stdin to prevent deadlocks, e.g. when the command
+	// has ended but an io.Copy is still reading from Stdin.
+	defer func() {
+		if inv.Stdin == nil {
+			return
+		}
+		rc, ok := inv.Stdin.(io.ReadCloser)
+		if !ok {
+			return
+		}
+		e := rc.Close()
+		err = errors.Join(err, e)
+	}()
+	err = inv.run(&runState{
+		allArgs: inv.Args,
+	})
+	return err
+}
+
+// WithContext returns a copy of the Invocation with the given context.
+func (inv *Invocation) WithContext(ctx context.Context) *Invocation {
+	return inv.with(func(i *Invocation) {
+		i.ctx = ctx
+	})
+}
+
+// with returns a copy of the Invocation with the given function applied.
+func (inv *Invocation) with(fn func(*Invocation)) *Invocation {
+	i2 := *inv
+	fn(&i2)
+	return &i2
+}
+
+// MiddlewareFunc returns the next handler in the chain,
+// or nil if there are no more.
+type MiddlewareFunc func(next HandlerFunc) HandlerFunc
+
+func chain(ms ...MiddlewareFunc) MiddlewareFunc {
+	return MiddlewareFunc(func(next HandlerFunc) HandlerFunc {
+		if len(ms) > 0 {
+			return chain(ms[1:]...)(ms[0](next))
+		}
+		return next
+	})
+}
+
+// Chain returns a Handler that first calls middleware in order.
+//
+//nolint:revive
+func Chain(ms ...MiddlewareFunc) MiddlewareFunc {
+	// We need to reverse the array to provide top-to-bottom execution
+	// order when defining a command.
+	reversed := make([]MiddlewareFunc, len(ms))
+	for i := range ms {
+		reversed[len(ms)-1-i] = ms[i]
+	}
+	return chain(reversed...)
+}
+
+func RequireNArgs(want int) MiddlewareFunc {
+	return RequireRangeArgs(want, want)
+}
+
+// RequireRangeArgs returns a Middleware that requires the number of arguments
+// to be between start and end (inclusive). If end is -1, then the number of
+// arguments must be at least start.
+func RequireRangeArgs(start, end int) MiddlewareFunc {
+	if start < 0 {
+		panic("start must be >= 0")
+	}
+	return func(next HandlerFunc) HandlerFunc {
+		return func(i *Invocation) error {
+			got := len(i.Args)
+			switch {
+			case start == end && got != start:
+				switch start {
+				case 0:
+					if len(i.Command.Children) > 0 {
+						return xerrors.Errorf("unrecognized subcommand %q", i.Args[0])
+					}
+					return xerrors.Errorf("wanted no args but got %v %v", got, i.Args)
+				default:
+					return xerrors.Errorf(
+						"wanted %v args but got %v %v",
+						start,
+						got,
+						i.Args,
+					)
+				}
+			case start > 0 && end == -1:
+				switch {
+				case got < start:
+					return xerrors.Errorf(
+						"wanted at least %v args but got %v",
+						start,
+						got,
+					)
+				default:
+					return next(i)
+				}
+			case start > end:
+				panic("start must be <= end")
+			case got < start || got > end:
+				return xerrors.Errorf(
+					"wanted between %v and %v args but got %v",
+					start, end,
+					got,
+				)
+			default:
+				return next(i)
+			}
+		}
+	}
+}
+
+// HandlerFunc handles an Invocation of a command.
+type HandlerFunc func(i *Invocation) error

cli/clibase/cmd_test.go

@@ -0,0 +1,719 @@
+package clibase_test
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"golang.org/x/xerrors"
+
+	"github.com/coder/coder/v2/cli/clibase"
+)
+
+// ioBufs is the standard input, output, and error for a command.
+type ioBufs struct {
+	Stdin  bytes.Buffer
+	Stdout bytes.Buffer
+	Stderr bytes.Buffer
+}
+
+// fakeIO sets Stdin, Stdout, and Stderr to buffers.
+func fakeIO(i *clibase.Invocation) *ioBufs {
+	var b ioBufs
+	i.Stdout = &b.Stdout
+	i.Stderr = &b.Stderr
+	i.Stdin = &b.Stdin
+	return &b
+}
+
+func TestCommand(t *testing.T) {
+	t.Parallel()
+
+	cmd := func() *clibase.Cmd {
+		var (
+			verbose bool
+			lower   bool
+			prefix  string
+			reqBool bool
+			reqStr  string
+		)
+		return &clibase.Cmd{
+			Use: "root [subcommand]",
+			Options: clibase.OptionSet{
+				clibase.Option{
+					Name:  "verbose",
+					Flag:  "verbose",
+					Value: clibase.BoolOf(&verbose),
+				},
+				clibase.Option{
+					Name:  "prefix",
+					Flag:  "prefix",
+					Value: clibase.StringOf(&prefix),
+				},
+			},
+			Children: []*clibase.Cmd{
+				{
+					Use:   "required-flag --req-bool=true --req-string=foo",
+					Short: "Example with required flags",
+					Options: clibase.OptionSet{
+						clibase.Option{
+							Name:     "req-bool",
+							Flag:     "req-bool",
+							Value:    clibase.BoolOf(&reqBool),
+							Required: true,
+						},
+						clibase.Option{
+							Name: "req-string",
+							Flag: "req-string",
+							Value: clibase.Validate(clibase.StringOf(&reqStr), func(value *clibase.String) error {
+								ok := strings.Contains(value.String(), " ")
+								if !ok {
+									return xerrors.Errorf("string must contain a space")
+								}
+								return nil
+							}),
+							Required: true,
+						},
+					},
+					Handler: func(i *clibase.Invocation) error {
+						_, _ = i.Stdout.Write([]byte(fmt.Sprintf("%s-%t", reqStr, reqBool)))
+						return nil
+					},
+				},
+				{
+					Use:   "toupper [word]",
+					Short: "Converts a word to upper case",
+					Middleware: clibase.Chain(
+						clibase.RequireNArgs(1),
+					),
+					Aliases: []string{"up"},
+					Options: clibase.OptionSet{
+						clibase.Option{
+							Name:  "lower",
+							Flag:  "lower",
+							Value: clibase.BoolOf(&lower),
+						},
+					},
+					Handler: func(i *clibase.Invocation) error {
+						_, _ = i.Stdout.Write([]byte(prefix))
+						w := i.Args[0]
+						if lower {
+							w = strings.ToLower(w)
+						} else {
+							w = strings.ToUpper(w)
+						}
+						_, _ = i.Stdout.Write(
+							[]byte(
+								w,
+							),
+						)
+						if verbose {
+							i.Stdout.Write([]byte("!!!"))
+						}
+						return nil
+					},
+				},
+			},
+		}
+	}
+
+	t.Run("SimpleOK", func(t *testing.T) {
+		t.Parallel()
+		i := cmd().Invoke("toupper", "hello")
+		io := fakeIO(i)
+		i.Run()
+		require.Equal(t, "HELLO", io.Stdout.String())
+	})
+
+	t.Run("Alias", func(t *testing.T) {
+		t.Parallel()
+		i := cmd().Invoke(
+			"up", "hello",
+		)
+		io := fakeIO(i)
+		i.Run()
+		require.Equal(t, "HELLO", io.Stdout.String())
+	})
+
+	t.Run("NoSubcommand", func(t *testing.T) {
+		t.Parallel()
+		i := cmd().Invoke(
+			"na",
+		)
+		io := fakeIO(i)
+		err := i.Run()
+		require.Empty(t, io.Stdout.String())
+		require.Error(t, err)
+	})
+
+	t.Run("BadArgs", func(t *testing.T) {
+		t.Parallel()
+		i := cmd().Invoke(
+			"toupper",
+		)
+		io := fakeIO(i)
+		err := i.Run()
+		require.Empty(t, io.Stdout.String())
+		require.Error(t, err)
+	})
+
+	t.Run("UnknownFlags", func(t *testing.T) {
+		t.Parallel()
+		i := cmd().Invoke(
+			"toupper", "--unknown",
+		)
+		io := fakeIO(i)
+		err := i.Run()
+		require.Empty(t, io.Stdout.String())
+		require.Error(t, err)
+	})
+
+	t.Run("Verbose", func(t *testing.T) {
+		t.Parallel()
+		i := cmd().Invoke(
+			"--verbose", "toupper", "hello",
+		)
+		io := fakeIO(i)
+		require.NoError(t, i.Run())
+		require.Equal(t, "HELLO!!!", io.Stdout.String())
+	})
+
+	t.Run("Verbose=", func(t *testing.T) {
+		t.Parallel()
+		i := cmd().Invoke(
+			"--verbose=true", "toupper", "hello",
+		)
+		io := fakeIO(i)
+		require.NoError(t, i.Run())
+		require.Equal(t, "HELLO!!!", io.Stdout.String())
+	})
+
+	t.Run("PrefixSpace", func(t *testing.T) {
+		t.Parallel()
+		i := cmd().Invoke(
+			"--prefix", "conv: ", "toupper", "hello",
+		)
+		io := fakeIO(i)
+		require.NoError(t, i.Run())
+		require.Equal(t, "conv: HELLO", io.Stdout.String())
+	})
+
+	t.Run("GlobalFlagsAnywhere", func(t *testing.T) {
+		t.Parallel()
+		i := cmd().Invoke(
+			"toupper", "--prefix", "conv: ", "hello", "--verbose",
+		)
+		io := fakeIO(i)
+		require.NoError(t, i.Run())
+		require.Equal(t, "conv: HELLO!!!", io.Stdout.String())
+	})
+
+	t.Run("LowerVerbose", func(t *testing.T) {
+		t.Parallel()
+		i := cmd().Invoke(
+			"toupper", "--verbose", "hello", "--lower",
+		)
+		io := fakeIO(i)
+		require.NoError(t, i.Run())
+		require.Equal(t, "hello!!!", io.Stdout.String())
+	})
+
+	t.Run("ParsedFlags", func(t *testing.T) {
+		t.Parallel()
+		i := cmd().Invoke(
+			"toupper", "--verbose", "hello", "--lower",
+		)
+		_ = fakeIO(i)
+		require.NoError(t, i.Run())
+		require.Equal(t,
+			"true",
+			i.ParsedFlags().Lookup("verbose").Value.String(),
+		)
+	})
+
+	t.Run("NoDeepChild", func(t *testing.T) {
+		t.Parallel()
+		i := cmd().Invoke(
+			"root", "level", "level", "toupper", "--verbose", "hello", "--lower",
+		)
+		fio := fakeIO(i)
+		require.Error(t, i.Run(), fio.Stdout.String())
+	})
+
+	t.Run("RequiredFlagsMissing", func(t *testing.T) {
+		t.Parallel()
+		i := cmd().Invoke(
+			"required-flag",
+		)
+		fio := fakeIO(i)
+		err := i.Run()
+		require.Error(t, err, fio.Stdout.String())
+		require.ErrorContains(t, err, "Missing values")
+	})
+
+	t.Run("RequiredFlagsMissingBool", func(t *testing.T) {
+		t.Parallel()
+		i := cmd().Invoke(
+			"required-flag", "--req-string", "foo bar",
+		)
+		fio := fakeIO(i)
+		err := i.Run()
+		require.Error(t, err, fio.Stdout.String())
+		require.ErrorContains(t, err, "Missing values for the required flags: req-bool")
+	})
+
+	t.Run("RequiredFlagsMissingString", func(t *testing.T) {
+		t.Parallel()
+		i := cmd().Invoke(
+			"required-flag", "--req-bool", "true",
+		)
+		fio := fakeIO(i)
+		err := i.Run()
+		require.Error(t, err, fio.Stdout.String())
+		require.ErrorContains(t, err, "Missing values for the required flags: req-string")
+	})
+
+	t.Run("RequiredFlagsInvalid", func(t *testing.T) {
+		t.Parallel()
+		i := cmd().Invoke(
+			"required-flag", "--req-string", "nospace",
+		)
+		fio := fakeIO(i)
+		err := i.Run()
+		require.Error(t, err, fio.Stdout.String())
+		require.ErrorContains(t, err, "string must contain a space")
+	})
+
+	t.Run("RequiredFlagsOK", func(t *testing.T) {
+		t.Parallel()
+		i := cmd().Invoke(
+			"required-flag", "--req-bool", "true", "--req-string", "foo bar",
+		)
+		fio := fakeIO(i)
+		err := i.Run()
+		require.NoError(t, err, fio.Stdout.String())
+	})
+}
+
+func TestCommand_DeepNest(t *testing.T) {
+	t.Parallel()
+	cmd := &clibase.Cmd{
+		Use: "1",
+		Children: []*clibase.Cmd{
+			{
+				Use: "2",
+				Children: []*clibase.Cmd{
+					{
+						Use: "3",
+						Handler: func(i *clibase.Invocation) error {
+							i.Stdout.Write([]byte("3"))
+							return nil
+						},
+					},
+				},
+			},
+		},
+	}
+	inv := cmd.Invoke("2", "3")
+	stdio := fakeIO(inv)
+	err := inv.Run()
+	require.NoError(t, err)
+	require.Equal(t, "3", stdio.Stdout.String())
+}
+
+func TestCommand_FlagOverride(t *testing.T) {
+	t.Parallel()
+	var flag string
+
+	cmd := &clibase.Cmd{
+		Use: "1",
+		Options: clibase.OptionSet{
+			{
+				Name:  "flag",
+				Flag:  "f",
+				Value: clibase.DiscardValue,
+			},
+		},
+		Children: []*clibase.Cmd{
+			{
+				Use: "2",
+				Options: clibase.OptionSet{
+					{
+						Name:  "flag",
+						Flag:  "f",
+						Value: clibase.StringOf(&flag),
+					},
+				},
+				Handler: func(i *clibase.Invocation) error {
+					return nil
+				},
+			},
+		},
+	}
+
+	err := cmd.Invoke("2", "--f", "mhmm").Run()
+	require.NoError(t, err)
+
+	require.Equal(t, "mhmm", flag)
+}
+
+func TestCommand_MiddlewareOrder(t *testing.T) {
+	t.Parallel()
+
+	mw := func(letter string) clibase.MiddlewareFunc {
+		return func(next clibase.HandlerFunc) clibase.HandlerFunc {
+			return (func(i *clibase.Invocation) error {
+				_, _ = i.Stdout.Write([]byte(letter))
+				return next(i)
+			})
+		}
+	}
+
+	cmd := &clibase.Cmd{
+		Use:   "toupper [word]",
+		Short: "Converts a word to upper case",
+		Middleware: clibase.Chain(
+			mw("A"),
+			mw("B"),
+			mw("C"),
+		),
+		Handler: (func(i *clibase.Invocation) error {
+			return nil
+		}),
+	}
+
+	i := cmd.Invoke(
+		"hello", "world",
+	)
+	io := fakeIO(i)
+	require.NoError(t, i.Run())
+	require.Equal(t, "ABC", io.Stdout.String())
+}
+
+func TestCommand_RawArgs(t *testing.T) {
+	t.Parallel()
+
+	cmd := func() *clibase.Cmd {
+		return &clibase.Cmd{
+			Use: "root",
+			Options: clibase.OptionSet{
+				{
+					Name:  "password",
+					Flag:  "password",
+					Value: clibase.StringOf(new(string)),
+				},
+			},
+			Children: []*clibase.Cmd{
+				{
+					Use:     "sushi <args...>",
+					Short:   "Throws back raw output",
+					RawArgs: true,
+					Handler: (func(i *clibase.Invocation) error {
+						if v := i.ParsedFlags().Lookup("password").Value.String(); v != "codershack" {
+							return xerrors.Errorf("password %q is wrong!", v)
+						}
+						i.Stdout.Write([]byte(strings.Join(i.Args, " ")))
+						return nil
+					}),
+				},
+			},
+		}
+	}
+
+	t.Run("OK", func(t *testing.T) {
+		// Flag parsed before the raw arg command should still work.
+		t.Parallel()
+
+		i := cmd().Invoke(
+			"--password", "codershack", "sushi", "hello", "--verbose", "world",
+		)
+		io := fakeIO(i)
+		require.NoError(t, i.Run())
+		require.Equal(t, "hello --verbose world", io.Stdout.String())
+	})
+
+	t.Run("BadFlag", func(t *testing.T) {
+		// Verbose before the raw arg command should fail.
+		t.Parallel()
+
+		i := cmd().Invoke(
+			"--password", "codershack", "--verbose", "sushi", "hello", "world",
+		)
+		io := fakeIO(i)
+		require.Error(t, i.Run())
+		require.Empty(t, io.Stdout.String())
+	})
+
+	t.Run("NoPassword", func(t *testing.T) {
+		// Flag parsed before the raw arg command should still work.
+		t.Parallel()
+		i := cmd().Invoke(
+			"sushi", "hello", "--verbose", "world",
+		)
+		_ = fakeIO(i)
+		require.Error(t, i.Run())
+	})
+}
+
+func TestCommand_RootRaw(t *testing.T) {
+	t.Parallel()
+	cmd := &clibase.Cmd{
+		RawArgs: true,
+		Handler: func(i *clibase.Invocation) error {
+			i.Stdout.Write([]byte(strings.Join(i.Args, " ")))
+			return nil
+		},
+	}
+
+	inv := cmd.Invoke("hello", "--verbose", "--friendly")
+	stdio := fakeIO(inv)
+	err := inv.Run()
+	require.NoError(t, err)
+
+	require.Equal(t, "hello --verbose --friendly", stdio.Stdout.String())
+}
+
+func TestCommand_HyphenHyphen(t *testing.T) {
+	t.Parallel()
+	cmd := &clibase.Cmd{
+		Handler: (func(i *clibase.Invocation) error {
+			i.Stdout.Write([]byte(strings.Join(i.Args, " ")))
+			return nil
+		}),
+	}
+
+	inv := cmd.Invoke("--", "--verbose", "--friendly")
+	stdio := fakeIO(inv)
+	err := inv.Run()
+	require.NoError(t, err)
+
+	require.Equal(t, "--verbose --friendly", stdio.Stdout.String())
+}
+
+func TestCommand_ContextCancels(t *testing.T) {
+	t.Parallel()
+
+	var gotCtx context.Context
+
+	cmd := &clibase.Cmd{
+		Handler: (func(i *clibase.Invocation) error {
+			gotCtx = i.Context()
+			if err := gotCtx.Err(); err != nil {
+				return xerrors.Errorf("unexpected context error: %w", i.Context().Err())
+			}
+			return nil
+		}),
+	}
+
+	err := cmd.Invoke().Run()
+	require.NoError(t, err)
+
+	require.Error(t, gotCtx.Err())
+}
+
+func TestCommand_Help(t *testing.T) {
+	t.Parallel()
+
+	cmd := func() *clibase.Cmd {
+		return &clibase.Cmd{
+			Use: "root",
+			HelpHandler: (func(i *clibase.Invocation) error {
+				i.Stdout.Write([]byte("abdracadabra"))
+				return nil
+			}),
+			Handler: (func(i *clibase.Invocation) error {
+				return xerrors.New("should not be called")
+			}),
+		}
+	}
+
+	t.Run("NoHandler", func(t *testing.T) {
+		t.Parallel()
+
+		c := cmd()
+		c.HelpHandler = nil
+		err := c.Invoke("--help").Run()
+		require.Error(t, err)
+	})
+
+	t.Run("Long", func(t *testing.T) {
+		t.Parallel()
+
+		inv := cmd().Invoke("--help")
+		stdio := fakeIO(inv)
+		err := inv.Run()
+		require.NoError(t, err)
+
+		require.Contains(t, stdio.Stdout.String(), "abdracadabra")
+	})
+
+	t.Run("Short", func(t *testing.T) {
+		t.Parallel()
+
+		inv := cmd().Invoke("-h")
+		stdio := fakeIO(inv)
+		err := inv.Run()
+		require.NoError(t, err)
+
+		require.Contains(t, stdio.Stdout.String(), "abdracadabra")
+	})
+}
+
+func TestCommand_SliceFlags(t *testing.T) {
+	t.Parallel()
+
+	cmd := func(want ...string) *clibase.Cmd {
+		var got []string
+		return &clibase.Cmd{
+			Use: "root",
+			Options: clibase.OptionSet{
+				{
+					Name:    "arr",
+					Flag:    "arr",
+					Default: "bad,bad,bad",
+					Value:   clibase.StringArrayOf(&got),
+				},
+			},
+			Handler: (func(i *clibase.Invocation) error {
+				require.Equal(t, want, got)
+				return nil
+			}),
+		}
+	}
+
+	err := cmd("good", "good", "good").Invoke("--arr", "good", "--arr", "good", "--arr", "good").Run()
+	require.NoError(t, err)
+
+	err = cmd("bad", "bad", "bad").Invoke().Run()
+	require.NoError(t, err)
+}
+
+func TestCommand_EmptySlice(t *testing.T) {
+	t.Parallel()
+
+	cmd := func(want ...string) *clibase.Cmd {
+		var got []string
+		return &clibase.Cmd{
+			Use: "root",
+			Options: clibase.OptionSet{
+				{
+					Name:    "arr",
+					Flag:    "arr",
+					Default: "def,def,def",
+					Env:     "ARR",
+					Value:   clibase.StringArrayOf(&got),
+				},
+			},
+			Handler: (func(i *clibase.Invocation) error {
+				require.Equal(t, want, got)
+				return nil
+			}),
+		}
+	}
+
+	// Base-case, uses default.
+	err := cmd("def", "def", "def").Invoke().Run()
+	require.NoError(t, err)
+
+	// Empty-env uses default, too.
+	inv := cmd("def", "def", "def").Invoke()
+	inv.Environ.Set("ARR", "")
+	require.NoError(t, err)
+
+	// Reset to nothing at all via flag.
+	inv = cmd().Invoke("--arr", "")
+	inv.Environ.Set("ARR", "cant see")
+	err = inv.Run()
+	require.NoError(t, err)
+
+	// Reset to a specific value with flag.
+	inv = cmd("great").Invoke("--arr", "great")
+	inv.Environ.Set("ARR", "")
+	err = inv.Run()
+	require.NoError(t, err)
+}
+
+func TestCommand_DefaultsOverride(t *testing.T) {
+	t.Parallel()
+
+	test := func(name string, want string, fn func(t *testing.T, inv *clibase.Invocation)) {
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+
+			var (
+				got    string
+				config clibase.YAMLConfigPath
+			)
+			cmd := &clibase.Cmd{
+				Options: clibase.OptionSet{
+					{
+						Name:    "url",
+						Flag:    "url",
+						Default: "def.com",
+						Env:     "URL",
+						Value:   clibase.StringOf(&got),
+						YAML:    "url",
+					},
+					{
+						Name:    "config",
+						Flag:    "config",
+						Default: "",
+						Value:   &config,
+					},
+				},
+				Handler: (func(i *clibase.Invocation) error {
+					_, _ = fmt.Fprintf(i.Stdout, "%s", got)
+					return nil
+				}),
+			}
+
+			inv := cmd.Invoke()
+			stdio := fakeIO(inv)
+			fn(t, inv)
+			err := inv.Run()
+			require.NoError(t, err)
+			require.Equal(t, want, stdio.Stdout.String())
+		})
+	}
+
+	test("DefaultOverNothing", "def.com", func(t *testing.T, inv *clibase.Invocation) {})
+
+	test("FlagOverDefault", "good.com", func(t *testing.T, inv *clibase.Invocation) {
+		inv.Args = []string{"--url", "good.com"}
+	})
+
+	test("EnvOverDefault", "good.com", func(t *testing.T, inv *clibase.Invocation) {
+		inv.Environ.Set("URL", "good.com")
+	})
+
+	test("FlagOverEnv", "good.com", func(t *testing.T, inv *clibase.Invocation) {
+		inv.Environ.Set("URL", "bad.com")
+		inv.Args = []string{"--url", "good.com"}
+	})
+
+	test("FlagOverYAML", "good.com", func(t *testing.T, inv *clibase.Invocation) {
+		fi, err := os.CreateTemp(t.TempDir(), "config.yaml")
+		require.NoError(t, err)
+		defer fi.Close()
+
+		_, err = fi.WriteString("url: bad.com")
+		require.NoError(t, err)
+
+		inv.Args = []string{"--config", fi.Name(), "--url", "good.com"}
+	})
+
+	test("YAMLOverDefault", "good.com", func(t *testing.T, inv *clibase.Invocation) {
+		fi, err := os.CreateTemp(t.TempDir(), "config.yaml")
+		require.NoError(t, err)
+		defer fi.Close()
+
+		_, err = fi.WriteString("url: good.com")
+		require.NoError(t, err)
+
+		inv.Args = []string{"--config", fi.Name()}
+	})
+}

cli/clibase/env.go

@@ -0,0 +1,76 @@
+package clibase
+
+import "strings"
+
+// name returns the name of the environment variable.
+func envName(line string) string {
+	return strings.ToUpper(
+		strings.SplitN(line, "=", 2)[0],
+	)
+}
+
+// value returns the value of the environment variable.
+func envValue(line string) string {
+	tokens := strings.SplitN(line, "=", 2)
+	if len(tokens) < 2 {
+		return ""
+	}
+	return tokens[1]
+}
+
+// Var represents a single environment variable of form
+// NAME=VALUE.
+type EnvVar struct {
+	Name  string
+	Value string
+}
+
+type Environ []EnvVar
+
+func (e Environ) ToOS() []string {
+	var env []string
+	for _, v := range e {
+		env = append(env, v.Name+"="+v.Value)
+	}
+	return env
+}
+
+func (e Environ) Lookup(name string) (string, bool) {
+	for _, v := range e {
+		if v.Name == name {
+			return v.Value, true
+		}
+	}
+	return "", false
+}
+
+func (e Environ) Get(name string) string {
+	v, _ := e.Lookup(name)
+	return v
+}
+
+func (e *Environ) Set(name, value string) {
+	for i, v := range *e {
+		if v.Name == name {
+			(*e)[i].Value = value
+			return
+		}
+	}
+	*e = append(*e, EnvVar{Name: name, Value: value})
+}
+
+// ParseEnviron returns all environment variables starting with
+// prefix without said prefix.
+func ParseEnviron(environ []string, prefix string) Environ {
+	var filtered []EnvVar
+	for _, line := range environ {
+		name := envName(line)
+		if strings.HasPrefix(name, prefix) {
+			filtered = append(filtered, EnvVar{
+				Name:  strings.TrimPrefix(name, prefix),
+				Value: envValue(line),
+			})
+		}
+	}
+	return filtered
+}

cli/clibase/env_test.go

@@ -0,0 +1,44 @@
+package clibase_test
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/coder/coder/v2/cli/clibase"
+)
+
+func TestFilterNamePrefix(t *testing.T) {
+	t.Parallel()
+	type args struct {
+		environ []string
+		prefix  string
+	}
+	tests := []struct {
+		name string
+		args args
+		want clibase.Environ
+	}{
+		{"empty", args{[]string{}, "SHIRE"}, nil},
+		{
+			"ONE",
+			args{
+				[]string{
+					"SHIRE_BRANDYBUCK=hmm",
+				},
+				"SHIRE_",
+			},
+			[]clibase.EnvVar{
+				{Name: "BRANDYBUCK", Value: "hmm"},
+			},
+		},
+	}
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+			if got := clibase.ParseEnviron(tt.args.environ, tt.args.prefix); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("FilterNamePrefix() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

cli/clibase/option.go

@@ -0,0 +1,346 @@
+package clibase
+
+import (
+	"bytes"
+	"encoding/json"
+	"os"
+	"strings"
+
+	"github.com/hashicorp/go-multierror"
+	"github.com/spf13/pflag"
+	"golang.org/x/xerrors"
+)
+
+type ValueSource string
+
+const (
+	ValueSourceNone    ValueSource = ""
+	ValueSourceFlag    ValueSource = "flag"
+	ValueSourceEnv     ValueSource = "env"
+	ValueSourceYAML    ValueSource = "yaml"
+	ValueSourceDefault ValueSource = "default"
+)
+
+// Option is a configuration option for a CLI application.
+type Option struct {
+	Name        string `json:"name,omitempty"`
+	Description string `json:"description,omitempty"`
+	// Required means this value must be set by some means. It requires
+	// `ValueSource != ValueSourceNone`
+	// If `Default` is set, then `Required` is ignored.
+	Required bool `json:"required,omitempty"`
+
+	// Flag is the long name of the flag used to configure this option. If unset,
+	// flag configuring is disabled.
+	Flag string `json:"flag,omitempty"`
+	// FlagShorthand is the one-character shorthand for the flag. If unset, no
+	// shorthand is used.
+	FlagShorthand string `json:"flag_shorthand,omitempty"`
+
+	// Env is the environment variable used to configure this option. If unset,
+	// environment configuring is disabled.
+	Env string `json:"env,omitempty"`
+
+	// YAML is the YAML key used to configure this option. If unset, YAML
+	// configuring is disabled.
+	YAML string `json:"yaml,omitempty"`
+
+	// Default is parsed into Value if set.
+	Default string `json:"default,omitempty"`
+	// Value includes the types listed in values.go.
+	Value pflag.Value `json:"value,omitempty"`
+
+	// Annotations enable extensions to clibase higher up in the stack. It's useful for
+	// help formatting and documentation generation.
+	Annotations Annotations `json:"annotations,omitempty"`
+
+	// Group is a group hierarchy that helps organize this option in help, configs
+	// and other documentation.
+	Group *Group `json:"group,omitempty"`
+
+	// UseInstead is a list of options that should be used instead of this one.
+	// The field is used to generate a deprecation warning.
+	UseInstead []Option `json:"use_instead,omitempty"`
+
+	Hidden bool `json:"hidden,omitempty"`
+
+	ValueSource ValueSource `json:"value_source,omitempty"`
+}
+
+// optionNoMethods is just a wrapper around Option so we can defer to the
+// default json.Unmarshaler behavior.
+type optionNoMethods Option
+
+func (o *Option) UnmarshalJSON(data []byte) error {
+	// If an option has no values, we have no idea how to unmarshal it.
+	// So just discard the json data.
+	if o.Value == nil {
+		o.Value = &DiscardValue
+	}
+
+	return json.Unmarshal(data, (*optionNoMethods)(o))
+}
+
+func (o Option) YAMLPath() string {
+	if o.YAML == "" {
+		return ""
+	}
+	var gs []string
+	for _, g := range o.Group.Ancestry() {
+		gs = append(gs, g.YAML)
+	}
+	return strings.Join(append(gs, o.YAML), ".")
+}
+
+// OptionSet is a group of options that can be applied to a command.
+type OptionSet []Option
+
+// UnmarshalJSON implements json.Unmarshaler for OptionSets. Options have an
+// interface Value type that cannot handle unmarshalling because the types cannot
+// be inferred. Since it is a slice, instantiating the Options first does not
+// help.
+//
+// However, we typically do instantiate the slice to have the correct types.
+// So this unmarshaller will attempt to find the named option in the existing
+// set, if it cannot, the value is discarded. If the option exists, the value
+// is unmarshalled into the existing option, and replaces the existing option.
+//
+// The value is discarded if it's type cannot be inferred. This behavior just
+// feels "safer", although it should never happen if the correct option set
+// is passed in. The situation where this could occur is if a client and server
+// are on different versions with different options.
+func (optSet *OptionSet) UnmarshalJSON(data []byte) error {
+	dec := json.NewDecoder(bytes.NewBuffer(data))
+	// Should be a json array, so consume the starting open bracket.
+	t, err := dec.Token()
+	if err != nil {
+		return xerrors.Errorf("read array open bracket: %w", err)
+	}
+	if t != json.Delim('[') {
+		return xerrors.Errorf("expected array open bracket, got %q", t)
+	}
+
+	// As long as json elements exist, consume them. The counter is used for
+	// better errors.
+	var i int
+OptionSetDecodeLoop:
+	for dec.More() {
+		var opt Option
+		// jValue is a placeholder value that allows us to capture the
+		// raw json for the value to attempt to unmarshal later.
+		var jValue jsonValue
+		opt.Value = &jValue
+		err := dec.Decode(&opt)
+		if err != nil {
+			return xerrors.Errorf("decode %d option: %w", i, err)
+		}
+		// This counter is used to contextualize errors to show which element of
+		// the array we failed to decode. It is only used in the error above, as
+		// if the above works, we can instead use the Option.Name which is more
+		// descriptive and useful. So increment here for the next decode.
+		i++
+
+		// Try to see if the option already exists in the option set.
+		// If it does, just update the existing option.
+		for optIndex, have := range *optSet {
+			if have.Name == opt.Name {
+				if jValue != nil {
+					err := json.Unmarshal(jValue, &(*optSet)[optIndex].Value)
+					if err != nil {
+						return xerrors.Errorf("decode option %q value: %w", have.Name, err)
+					}
+					// Set the opt's value
+					opt.Value = (*optSet)[optIndex].Value
+				} else {
+					// Hopefully the user passed empty values in the option set. There is no easy way
+					// to tell, and if we do not do this, it breaks json.Marshal if we do it again on
+					// this new option set.
+					opt.Value = (*optSet)[optIndex].Value
+				}
+				// Override the existing.
+				(*optSet)[optIndex] = opt
+				// Go to the next option to decode.
+				continue OptionSetDecodeLoop
+			}
+		}
+
+		// If the option doesn't exist, the value will be discarded.
+		// We do this because we cannot infer the type of the value.
+		opt.Value = DiscardValue
+		*optSet = append(*optSet, opt)
+	}
+
+	t, err = dec.Token()
+	if err != nil {
+		return xerrors.Errorf("read array close bracket: %w", err)
+	}
+	if t != json.Delim(']') {
+		return xerrors.Errorf("expected array close bracket, got %q", t)
+	}
+
+	return nil
+}
+
+// Add adds the given Options to the OptionSet.
+func (optSet *OptionSet) Add(opts ...Option) {
+	*optSet = append(*optSet, opts...)
+}
+
+// Filter will only return options that match the given filter. (return true)
+func (optSet OptionSet) Filter(filter func(opt Option) bool) OptionSet {
+	cpy := make(OptionSet, 0)
+	for _, opt := range optSet {
+		if filter(opt) {
+			cpy = append(cpy, opt)
+		}
+	}
+	return cpy
+}
+
+// FlagSet returns a pflag.FlagSet for the OptionSet.
+func (optSet *OptionSet) FlagSet() *pflag.FlagSet {
+	if optSet == nil {
+		return &pflag.FlagSet{}
+	}
+
+	fs := pflag.NewFlagSet("", pflag.ContinueOnError)
+	for _, opt := range *optSet {
+		if opt.Flag == "" {
+			continue
+		}
+		var noOptDefValue string
+		{
+			no, ok := opt.Value.(NoOptDefValuer)
+			if ok {
+				noOptDefValue = no.NoOptDefValue()
+			}
+		}
+
+		val := opt.Value
+		if val == nil {
+			val = DiscardValue
+		}
+
+		fs.AddFlag(&pflag.Flag{
+			Name:        opt.Flag,
+			Shorthand:   opt.FlagShorthand,
+			Usage:       opt.Description,
+			Value:       val,
+			DefValue:    "",
+			Changed:     false,
+			Deprecated:  "",
+			NoOptDefVal: noOptDefValue,
+			Hidden:      opt.Hidden,
+		})
+	}
+	fs.Usage = func() {
+		_, _ = os.Stderr.WriteString("Override (*FlagSet).Usage() to print help text.\n")
+	}
+	return fs
+}
+
+// ParseEnv parses the given environment variables into the OptionSet.
+// Use EnvsWithPrefix to filter out prefixes.
+func (optSet *OptionSet) ParseEnv(vs []EnvVar) error {
+	if optSet == nil {
+		return nil
+	}
+
+	var merr *multierror.Error
+
+	// We parse environment variables first instead of using a nested loop to
+	// avoid N*M complexity when there are a lot of options and environment
+	// variables.
+	envs := make(map[string]string)
+	for _, v := range vs {
+		envs[v.Name] = v.Value
+	}
+
+	for i, opt := range *optSet {
+		if opt.Env == "" {
+			continue
+		}
+
+		envVal, ok := envs[opt.Env]
+		if !ok {
+			// Homebrew strips all environment variables that do not start with `HOMEBREW_`.
+			// This prevented using brew to invoke the Coder agent, because the environment
+			// variables to not get passed down.
+			//
+			// A customer wanted to use their custom tap inside a workspace, which was failing
+			// because the agent lacked the environment variables to authenticate with Git.
+			envVal, ok = envs[`HOMEBREW_`+opt.Env]
+		}
+		// Currently, empty values are treated as if the environment variable is
+		// unset. This behavior is technically not correct as there is now no
+		// way for a user to change a Default value to an empty string from
+		// the environment. Unfortunately, we have old configuration files
+		// that rely on the faulty behavior.
+		//
+		// TODO: We should remove this hack in May 2023, when deployments
+		// have had months to migrate to the new behavior.
+		if !ok || envVal == "" {
+			continue
+		}
+
+		(*optSet)[i].ValueSource = ValueSourceEnv
+		if err := opt.Value.Set(envVal); err != nil {
+			merr = multierror.Append(
+				merr, xerrors.Errorf("parse %q: %w", opt.Name, err),
+			)
+		}
+	}
+
+	return merr.ErrorOrNil()
+}
+
+// SetDefaults sets the default values for each Option, skipping values
+// that already have a value source.
+func (optSet *OptionSet) SetDefaults() error {
+	if optSet == nil {
+		return nil
+	}
+
+	var merr *multierror.Error
+
+	for i, opt := range *optSet {
+		// Skip values that may have already been set by the user.
+		if opt.ValueSource != ValueSourceNone {
+			continue
+		}
+
+		if opt.Default == "" {
+			continue
+		}
+
+		if opt.Value == nil {
+			merr = multierror.Append(
+				merr,
+				xerrors.Errorf(
+					"parse %q: no Value field set\nFull opt: %+v",
+					opt.Name, opt,
+				),
+			)
+			continue
+		}
+		(*optSet)[i].ValueSource = ValueSourceDefault
+		if err := opt.Value.Set(opt.Default); err != nil {
+			merr = multierror.Append(
+				merr, xerrors.Errorf("parse %q: %w", opt.Name, err),
+			)
+		}
+	}
+	return merr.ErrorOrNil()
+}
+
+// ByName returns the Option with the given name, or nil if no such option
+// exists.
+func (optSet *OptionSet) ByName(name string) *Option {
+	for i := range *optSet {
+		opt := &(*optSet)[i]
+		if opt.Name == name {
+			return opt
+		}
+	}
+	return nil
+}

cli/clibase/option_test.go

@@ -0,0 +1,391 @@
+package clibase_test
+
+import (
+	"bytes"
+	"encoding/json"
+	"regexp"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/coder/coder/v2/cli/clibase"
+	"github.com/coder/coder/v2/coderd/coderdtest"
+	"github.com/coder/coder/v2/codersdk"
+)
+
+func TestOptionSet_ParseFlags(t *testing.T) {
+	t.Parallel()
+
+	t.Run("SimpleString", func(t *testing.T) {
+		t.Parallel()
+
+		var workspaceName clibase.String
+
+		os := clibase.OptionSet{
+			clibase.Option{
+				Name:          "Workspace Name",
+				Value:         &workspaceName,
+				Flag:          "workspace-name",
+				FlagShorthand: "n",
+			},
+		}
+
+		var err error
+		err = os.FlagSet().Parse([]string{"--workspace-name", "foo"})
+		require.NoError(t, err)
+		require.EqualValues(t, "foo", workspaceName)
+
+		err = os.FlagSet().Parse([]string{"-n", "f"})
+		require.NoError(t, err)
+		require.EqualValues(t, "f", workspaceName)
+	})
+
+	t.Run("StringArray", func(t *testing.T) {
+		t.Parallel()
+
+		var names clibase.StringArray
+
+		os := clibase.OptionSet{
+			clibase.Option{
+				Name:          "name",
+				Value:         &names,
+				Flag:          "name",
+				FlagShorthand: "n",
+			},
+		}
+
+		err := os.SetDefaults()
+		require.NoError(t, err)
+
+		err = os.FlagSet().Parse([]string{"--name", "foo", "--name", "bar"})
+		require.NoError(t, err)
+		require.EqualValues(t, []string{"foo", "bar"}, names)
+	})
+
+	t.Run("ExtraFlags", func(t *testing.T) {
+		t.Parallel()
+
+		var workspaceName clibase.String
+
+		os := clibase.OptionSet{
+			clibase.Option{
+				Name:  "Workspace Name",
+				Value: &workspaceName,
+			},
+		}
+
+		err := os.FlagSet().Parse([]string{"--some-unknown", "foo"})
+		require.Error(t, err)
+	})
+
+	t.Run("RegexValid", func(t *testing.T) {
+		t.Parallel()
+
+		var regexpString clibase.Regexp
+
+		os := clibase.OptionSet{
+			clibase.Option{
+				Name:  "RegexpString",
+				Value: &regexpString,
+				Flag:  "regexp-string",
+			},
+		}
+
+		err := os.FlagSet().Parse([]string{"--regexp-string", "$test^"})
+		require.NoError(t, err)
+	})
+
+	t.Run("RegexInvalid", func(t *testing.T) {
+		t.Parallel()
+
+		var regexpString clibase.Regexp
+
+		os := clibase.OptionSet{
+			clibase.Option{
+				Name:  "RegexpString",
+				Value: &regexpString,
+				Flag:  "regexp-string",
+			},
+		}
+
+		err := os.FlagSet().Parse([]string{"--regexp-string", "(("})
+		require.Error(t, err)
+	})
+}
+
+func TestOptionSet_ParseEnv(t *testing.T) {
+	t.Parallel()
+
+	t.Run("SimpleString", func(t *testing.T) {
+		t.Parallel()
+
+		var workspaceName clibase.String
+
+		os := clibase.OptionSet{
+			clibase.Option{
+				Name:  "Workspace Name",
+				Value: &workspaceName,
+				Env:   "WORKSPACE_NAME",
+			},
+		}
+
+		err := os.ParseEnv([]clibase.EnvVar{
+			{Name: "WORKSPACE_NAME", Value: "foo"},
+		})
+		require.NoError(t, err)
+		require.EqualValues(t, "foo", workspaceName)
+	})
+
+	t.Run("EmptyValue", func(t *testing.T) {
+		t.Parallel()
+
+		var workspaceName clibase.String
+
+		os := clibase.OptionSet{
+			clibase.Option{
+				Name:    "Workspace Name",
+				Value:   &workspaceName,
+				Default: "defname",
+				Env:     "WORKSPACE_NAME",
+			},
+		}
+
+		err := os.SetDefaults()
+		require.NoError(t, err)
+
+		err = os.ParseEnv(clibase.ParseEnviron([]string{"CODER_WORKSPACE_NAME="}, "CODER_"))
+		require.NoError(t, err)
+		require.EqualValues(t, "defname", workspaceName)
+	})
+
+	t.Run("StringSlice", func(t *testing.T) {
+		t.Parallel()
+
+		var actual clibase.StringArray
+		expected := []string{"foo", "bar", "baz"}
+
+		os := clibase.OptionSet{
+			clibase.Option{
+				Name:  "name",
+				Value: &actual,
+				Env:   "NAMES",
+			},
+		}
+
+		err := os.SetDefaults()
+		require.NoError(t, err)
+
+		err = os.ParseEnv([]clibase.EnvVar{
+			{Name: "NAMES", Value: "foo,bar,baz"},
+		})
+		require.NoError(t, err)
+		require.EqualValues(t, expected, actual)
+	})
+
+	t.Run("StructMapStringString", func(t *testing.T) {
+		t.Parallel()
+
+		var actual clibase.Struct[map[string]string]
+		expected := map[string]string{"foo": "bar", "baz": "zap"}
+
+		os := clibase.OptionSet{
+			clibase.Option{
+				Name:  "labels",
+				Value: &actual,
+				Env:   "LABELS",
+			},
+		}
+
+		err := os.SetDefaults()
+		require.NoError(t, err)
+
+		err = os.ParseEnv([]clibase.EnvVar{
+			{Name: "LABELS", Value: `{"foo":"bar","baz":"zap"}`},
+		})
+		require.NoError(t, err)
+		require.EqualValues(t, expected, actual.Value)
+	})
+
+	t.Run("Homebrew", func(t *testing.T) {
+		t.Parallel()
+
+		var agentToken clibase.String
+
+		os := clibase.OptionSet{
+			clibase.Option{
+				Name:  "Agent Token",
+				Value: &agentToken,
+				Env:   "AGENT_TOKEN",
+			},
+		}
+
+		err := os.ParseEnv([]clibase.EnvVar{
+			{Name: "HOMEBREW_AGENT_TOKEN", Value: "foo"},
+		})
+		require.NoError(t, err)
+		require.EqualValues(t, "foo", agentToken)
+	})
+}
+
+func TestOptionSet_JsonMarshal(t *testing.T) {
+	t.Parallel()
+
+	// This unit test ensures if the source optionset is missing the option
+	// and cannot determine the type, it will not panic. The unmarshal will
+	// succeed with a best effort.
+	t.Run("MissingSrcOption", func(t *testing.T) {
+		t.Parallel()
+
+		var str clibase.String = "something"
+		var arr clibase.StringArray = []string{"foo", "bar"}
+		opts := clibase.OptionSet{
+			clibase.Option{
+				Name:  "StringOpt",
+				Value: &str,
+			},
+			clibase.Option{
+				Name:  "ArrayOpt",
+				Value: &arr,
+			},
+		}
+		data, err := json.Marshal(opts)
+		require.NoError(t, err, "marshal option set")
+
+		tgt := clibase.OptionSet{}
+		err = json.Unmarshal(data, &tgt)
+		require.NoError(t, err, "unmarshal option set")
+		for i := range opts {
+			compareOptionsExceptValues(t, opts[i], tgt[i])
+			require.Empty(t, tgt[i].Value.String(), "unknown value types are empty")
+		}
+	})
+
+	t.Run("RegexCase", func(t *testing.T) {
+		t.Parallel()
+
+		val := clibase.Regexp(*regexp.MustCompile(".*"))
+		opts := clibase.OptionSet{
+			clibase.Option{
+				Name:    "Regex",
+				Value:   &val,
+				Default: ".*",
+			},
+		}
+		data, err := json.Marshal(opts)
+		require.NoError(t, err, "marshal option set")
+
+		var foundVal clibase.Regexp
+		newOpts := clibase.OptionSet{
+			clibase.Option{
+				Name:  "Regex",
+				Value: &foundVal,
+			},
+		}
+		err = json.Unmarshal(data, &newOpts)
+		require.NoError(t, err, "unmarshal option set")
+
+		require.EqualValues(t, opts[0].Value.String(), newOpts[0].Value.String())
+	})
+
+	t.Run("AllValues", func(t *testing.T) {
+		t.Parallel()
+
+		vals := coderdtest.DeploymentValues(t)
+		opts := vals.Options()
+		sources := []clibase.ValueSource{
+			clibase.ValueSourceNone,
+			clibase.ValueSourceFlag,
+			clibase.ValueSourceEnv,
+			clibase.ValueSourceYAML,
+			clibase.ValueSourceDefault,
+		}
+		for i := range opts {
+			opts[i].ValueSource = sources[i%len(sources)]
+		}
+
+		data, err := json.Marshal(opts)
+		require.NoError(t, err, "marshal option set")
+
+		newOpts := (&codersdk.DeploymentValues{}).Options()
+		err = json.Unmarshal(data, &newOpts)
+		require.NoError(t, err, "unmarshal option set")
+
+		for i := range opts {
+			exp := opts[i]
+			found := newOpts[i]
+
+			compareOptionsExceptValues(t, exp, found)
+			compareValues(t, exp, found)
+		}
+
+		thirdOpts := (&codersdk.DeploymentValues{}).Options()
+		data, err = json.Marshal(newOpts)
+		require.NoError(t, err, "marshal option set")
+
+		err = json.Unmarshal(data, &thirdOpts)
+		require.NoError(t, err, "unmarshal option set")
+		// Compare to the original opts again
+		for i := range opts {
+			exp := opts[i]
+			found := thirdOpts[i]
+
+			compareOptionsExceptValues(t, exp, found)
+			compareValues(t, exp, found)
+		}
+	})
+}
+
+func compareOptionsExceptValues(t *testing.T, exp, found clibase.Option) {
+	t.Helper()
+
+	require.Equalf(t, exp.Name, found.Name, "option name %q", exp.Name)
+	require.Equalf(t, exp.Description, found.Description, "option description %q", exp.Name)
+	require.Equalf(t, exp.Required, found.Required, "option required %q", exp.Name)
+	require.Equalf(t, exp.Flag, found.Flag, "option flag %q", exp.Name)
+	require.Equalf(t, exp.FlagShorthand, found.FlagShorthand, "option flag shorthand %q", exp.Name)
+	require.Equalf(t, exp.Env, found.Env, "option env %q", exp.Name)
+	require.Equalf(t, exp.YAML, found.YAML, "option yaml %q", exp.Name)
+	require.Equalf(t, exp.Default, found.Default, "option default %q", exp.Name)
+	require.Equalf(t, exp.ValueSource, found.ValueSource, "option value source %q", exp.Name)
+	require.Equalf(t, exp.Hidden, found.Hidden, "option hidden %q", exp.Name)
+	require.Equalf(t, exp.Annotations, found.Annotations, "option annotations %q", exp.Name)
+	require.Equalf(t, exp.Group, found.Group, "option group %q", exp.Name)
+	// UseInstead is the same comparison problem, just check the length
+	require.Equalf(t, len(exp.UseInstead), len(found.UseInstead), "option use instead %q", exp.Name)
+}
+
+func compareValues(t *testing.T, exp, found clibase.Option) {
+	t.Helper()
+
+	if (exp.Value == nil || found.Value == nil) || (exp.Value.String() != found.Value.String() && found.Value.String() == "") {
+		// If the string values are different, this can be a "nil" issue.
+		// So only run this case if the found string is the empty string.
+		// We use MarshalYAML for struct strings, and it will return an
+		// empty string '""' for nil slices/maps/etc.
+		// So use json to compare.
+
+		expJSON, err := json.Marshal(exp.Value)
+		require.NoError(t, err, "marshal")
+		foundJSON, err := json.Marshal(found.Value)
+		require.NoError(t, err, "marshal")
+
+		expJSON = normalizeJSON(expJSON)
+		foundJSON = normalizeJSON(foundJSON)
+		assert.Equalf(t, string(expJSON), string(foundJSON), "option value %q", exp.Name)
+	} else {
+		assert.Equal(t,
+			exp.Value.String(),
+			found.Value.String(),
+			"option value %q", exp.Name)
+	}
+}
+
+// normalizeJSON handles the fact that an empty map/slice is not the same
+// as a nil empty/slice. For our purposes, they are the same.
+func normalizeJSON(data []byte) []byte {
+	if bytes.Equal(data, []byte("[]")) || bytes.Equal(data, []byte("{}")) {
+		return []byte("null")
+	}
+	return data
+}

cli/clibase/values.go

@@ -0,0 +1,567 @@
+package clibase
+
+import (
+	"encoding/csv"
+	"encoding/json"
+	"fmt"
+	"net"
+	"net/url"
+	"reflect"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/spf13/pflag"
+	"golang.org/x/xerrors"
+	"gopkg.in/yaml.v3"
+)
+
+// NoOptDefValuer describes behavior when no
+// option is passed into the flag.
+//
+// This is useful for boolean or otherwise binary flags.
+type NoOptDefValuer interface {
+	NoOptDefValue() string
+}
+
+// Validator is a wrapper around a pflag.Value that allows for validation
+// of the value after or before it has been set.
+type Validator[T pflag.Value] struct {
+	Value T
+	// validate is called after the value is set.
+	validate func(T) error
+}
+
+func Validate[T pflag.Value](opt T, validate func(value T) error) *Validator[T] {
+	return &Validator[T]{Value: opt, validate: validate}
+}
+
+func (i *Validator[T]) String() string {
+	return i.Value.String()
+}
+
+func (i *Validator[T]) Set(input string) error {
+	err := i.Value.Set(input)
+	if err != nil {
+		return err
+	}
+	if i.validate != nil {
+		err = i.validate(i.Value)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (i *Validator[T]) Type() string {
+	return i.Value.Type()
+}
+
+// values.go contains a standard set of value types that can be used as
+// Option Values.
+
+type Int64 int64
+
+func Int64Of(i *int64) *Int64 {
+	return (*Int64)(i)
+}
+
+func (i *Int64) Set(s string) error {
+	ii, err := strconv.ParseInt(s, 10, 64)
+	*i = Int64(ii)
+	return err
+}
+
+func (i Int64) Value() int64 {
+	return int64(i)
+}
+
+func (i Int64) String() string {
+	return strconv.Itoa(int(i))
+}
+
+func (Int64) Type() string {
+	return "int"
+}
+
+type Bool bool
+
+func BoolOf(b *bool) *Bool {
+	return (*Bool)(b)
+}
+
+func (b *Bool) Set(s string) error {
+	if s == "" {
+		*b = Bool(false)
+		return nil
+	}
+	bb, err := strconv.ParseBool(s)
+	*b = Bool(bb)
+	return err
+}
+
+func (*Bool) NoOptDefValue() string {
+	return "true"
+}
+
+func (b Bool) String() string {
+	return strconv.FormatBool(bool(b))
+}
+
+func (b Bool) Value() bool {
+	return bool(b)
+}
+
+func (Bool) Type() string {
+	return "bool"
+}
+
+type String string
+
+func StringOf(s *string) *String {
+	return (*String)(s)
+}
+
+func (*String) NoOptDefValue() string {
+	return ""
+}
+
+func (s *String) Set(v string) error {
+	*s = String(v)
+	return nil
+}
+
+func (s String) String() string {
+	return string(s)
+}
+
+func (s String) Value() string {
+	return string(s)
+}
+
+func (String) Type() string {
+	return "string"
+}
+
+var _ pflag.SliceValue = &StringArray{}
+
+// StringArray is a slice of strings that implements pflag.Value and pflag.SliceValue.
+type StringArray []string
+
+func StringArrayOf(ss *[]string) *StringArray {
+	return (*StringArray)(ss)
+}
+
+func (s *StringArray) Append(v string) error {
+	*s = append(*s, v)
+	return nil
+}
+
+func (s *StringArray) Replace(vals []string) error {
+	*s = vals
+	return nil
+}
+
+func (s *StringArray) GetSlice() []string {
+	return *s
+}
+
+func readAsCSV(v string) ([]string, error) {
+	return csv.NewReader(strings.NewReader(v)).Read()
+}
+
+func writeAsCSV(vals []string) string {
+	var sb strings.Builder
+	err := csv.NewWriter(&sb).Write(vals)
+	if err != nil {
+		return fmt.Sprintf("error: %s", err)
+	}
+	return sb.String()
+}
+
+func (s *StringArray) Set(v string) error {
+	if v == "" {
+		*s = nil
+		return nil
+	}
+	ss, err := readAsCSV(v)
+	if err != nil {
+		return err
+	}
+	*s = append(*s, ss...)
+	return nil
+}
+
+func (s StringArray) String() string {
+	return writeAsCSV([]string(s))
+}
+
+func (s StringArray) Value() []string {
+	return []string(s)
+}
+
+func (StringArray) Type() string {
+	return "string-array"
+}
+
+type Duration time.Duration
+
+func DurationOf(d *time.Duration) *Duration {
+	return (*Duration)(d)
+}
+
+func (d *Duration) Set(v string) error {
+	dd, err := time.ParseDuration(v)
+	*d = Duration(dd)
+	return err
+}
+
+func (d *Duration) Value() time.Duration {
+	return time.Duration(*d)
+}
+
+func (d *Duration) String() string {
+	return time.Duration(*d).String()
+}
+
+func (Duration) Type() string {
+	return "duration"
+}
+
+func (d *Duration) MarshalYAML() (interface{}, error) {
+	return yaml.Node{
+		Kind:  yaml.ScalarNode,
+		Value: d.String(),
+	}, nil
+}
+
+func (d *Duration) UnmarshalYAML(n *yaml.Node) error {
+	return d.Set(n.Value)
+}
+
+type URL url.URL
+
+func URLOf(u *url.URL) *URL {
+	return (*URL)(u)
+}
+
+func (u *URL) Set(v string) error {
+	uu, err := url.Parse(v)
+	if err != nil {
+		return err
+	}
+	*u = URL(*uu)
+	return nil
+}
+
+func (u *URL) String() string {
+	uu := url.URL(*u)
+	return uu.String()
+}
+
+func (u *URL) MarshalYAML() (interface{}, error) {
+	return yaml.Node{
+		Kind:  yaml.ScalarNode,
+		Value: u.String(),
+	}, nil
+}
+
+func (u *URL) UnmarshalYAML(n *yaml.Node) error {
+	return u.Set(n.Value)
+}
+
+func (u *URL) MarshalJSON() ([]byte, error) {
+	return json.Marshal(u.String())
+}
+
+func (u *URL) UnmarshalJSON(b []byte) error {
+	var s string
+	err := json.Unmarshal(b, &s)
+	if err != nil {
+		return err
+	}
+	return u.Set(s)
+}
+
+func (*URL) Type() string {
+	return "url"
+}
+
+func (u *URL) Value() *url.URL {
+	return (*url.URL)(u)
+}
+
+// HostPort is a host:port pair.
+type HostPort struct {
+	Host string
+	Port string
+}
+
+func (hp *HostPort) Set(v string) error {
+	if v == "" {
+		return xerrors.Errorf("must not be empty")
+	}
+	var err error
+	hp.Host, hp.Port, err = net.SplitHostPort(v)
+	return err
+}
+
+func (hp *HostPort) String() string {
+	if hp.Host == "" && hp.Port == "" {
+		return ""
+	}
+	// Warning: net.JoinHostPort must be used over concatenation to support
+	// IPv6 addresses.
+	return net.JoinHostPort(hp.Host, hp.Port)
+}
+
+func (hp *HostPort) MarshalJSON() ([]byte, error) {
+	return json.Marshal(hp.String())
+}
+
+func (hp *HostPort) UnmarshalJSON(b []byte) error {
+	var s string
+	err := json.Unmarshal(b, &s)
+	if err != nil {
+		return err
+	}
+	if s == "" {
+		hp.Host = ""
+		hp.Port = ""
+		return nil
+	}
+	return hp.Set(s)
+}
+
+func (hp *HostPort) MarshalYAML() (interface{}, error) {
+	return yaml.Node{
+		Kind:  yaml.ScalarNode,
+		Value: hp.String(),
+	}, nil
+}
+
+func (hp *HostPort) UnmarshalYAML(n *yaml.Node) error {
+	return hp.Set(n.Value)
+}
+
+func (*HostPort) Type() string {
+	return "host:port"
+}
+
+var (
+	_ yaml.Marshaler   = new(Struct[struct{}])
+	_ yaml.Unmarshaler = new(Struct[struct{}])
+)
+
+// Struct is a special value type that encodes an arbitrary struct.
+// It implements the flag.Value interface, but in general these values should
+// only be accepted via config for ergonomics.
+//
+// The string encoding type is YAML.
+type Struct[T any] struct {
+	Value T
+}
+
+//nolint:revive
+func (s *Struct[T]) Set(v string) error {
+	return yaml.Unmarshal([]byte(v), &s.Value)
+}
+
+//nolint:revive
+func (s *Struct[T]) String() string {
+	byt, err := yaml.Marshal(s.Value)
+	if err != nil {
+		return "decode failed: " + err.Error()
+	}
+	return string(byt)
+}
+
+func (s *Struct[T]) MarshalYAML() (interface{}, error) {
+	var n yaml.Node
+	err := n.Encode(s.Value)
+	if err != nil {
+		return nil, err
+	}
+	return n, nil
+}
+
+func (s *Struct[T]) UnmarshalYAML(n *yaml.Node) error {
+	// HACK: for compatibility with flags, we use nil slices instead of empty
+	// slices. In most cases, nil slices and empty slices are treated
+	// the same, so this behavior may be removed at some point.
+	if typ := reflect.TypeOf(s.Value); typ.Kind() == reflect.Slice && len(n.Content) == 0 {
+		reflect.ValueOf(&s.Value).Elem().Set(reflect.Zero(typ))
+		return nil
+	}
+	return n.Decode(&s.Value)
+}
+
+//nolint:revive
+func (s *Struct[T]) Type() string {
+	return fmt.Sprintf("struct[%T]", s.Value)
+}
+
+func (s *Struct[T]) MarshalJSON() ([]byte, error) {
+	return json.Marshal(s.Value)
+}
+
+func (s *Struct[T]) UnmarshalJSON(b []byte) error {
+	return json.Unmarshal(b, &s.Value)
+}
+
+// DiscardValue does nothing but implements the pflag.Value interface.
+// It's useful in cases where you want to accept an option, but access the
+// underlying value directly instead of through the Option methods.
+var DiscardValue discardValue
+
+type discardValue struct{}
+
+func (discardValue) Set(string) error {
+	return nil
+}
+
+func (discardValue) String() string {
+	return ""
+}
+
+func (discardValue) Type() string {
+	return "discard"
+}
+
+func (discardValue) UnmarshalJSON([]byte) error {
+	return nil
+}
+
+// jsonValue is intentionally not exported. It is just used to store the raw JSON
+// data for a value to defer it's unmarshal. It implements the pflag.Value to be
+// usable in an Option.
+type jsonValue json.RawMessage
+
+func (jsonValue) Set(string) error {
+	return xerrors.Errorf("json value is read-only")
+}
+
+func (jsonValue) String() string {
+	return ""
+}
+
+func (jsonValue) Type() string {
+	return "json"
+}
+
+func (j *jsonValue) UnmarshalJSON(data []byte) error {
+	if j == nil {
+		return xerrors.New("json.RawMessage: UnmarshalJSON on nil pointer")
+	}
+	*j = append((*j)[0:0], data...)
+	return nil
+}
+
+var _ pflag.Value = (*Enum)(nil)
+
+type Enum struct {
+	Choices []string
+	Value   *string
+}
+
+func EnumOf(v *string, choices ...string) *Enum {
+	return &Enum{
+		Choices: choices,
+		Value:   v,
+	}
+}
+
+func (e *Enum) Set(v string) error {
+	for _, c := range e.Choices {
+		if v == c {
+			*e.Value = v
+			return nil
+		}
+	}
+	return xerrors.Errorf("invalid choice: %s, should be one of %v", v, e.Choices)
+}
+
+func (e *Enum) Type() string {
+	return fmt.Sprintf("enum[%v]", strings.Join(e.Choices, "|"))
+}
+
+func (e *Enum) String() string {
+	return *e.Value
+}
+
+type Regexp regexp.Regexp
+
+func (r *Regexp) MarshalJSON() ([]byte, error) {
+	return json.Marshal(r.String())
+}
+
+func (r *Regexp) UnmarshalJSON(data []byte) error {
+	var source string
+	err := json.Unmarshal(data, &source)
+	if err != nil {
+		return err
+	}
+
+	exp, err := regexp.Compile(source)
+	if err != nil {
+		return xerrors.Errorf("invalid regex expression: %w", err)
+	}
+	*r = Regexp(*exp)
+	return nil
+}
+
+func (r *Regexp) MarshalYAML() (interface{}, error) {
+	return yaml.Node{
+		Kind:  yaml.ScalarNode,
+		Value: r.String(),
+	}, nil
+}
+
+func (r *Regexp) UnmarshalYAML(n *yaml.Node) error {
+	return r.Set(n.Value)
+}
+
+func (r *Regexp) Set(v string) error {
+	exp, err := regexp.Compile(v)
+	if err != nil {
+		return xerrors.Errorf("invalid regex expression: %w", err)
+	}
+	*r = Regexp(*exp)
+	return nil
+}
+
+func (r Regexp) String() string {
+	return r.Value().String()
+}
+
+func (r *Regexp) Value() *regexp.Regexp {
+	if r == nil {
+		return nil
+	}
+	return (*regexp.Regexp)(r)
+}
+
+func (Regexp) Type() string {
+	return "regexp"
+}
+
+var _ pflag.Value = (*YAMLConfigPath)(nil)
+
+// YAMLConfigPath is a special value type that encodes a path to a YAML
+// configuration file where options are read from.
+type YAMLConfigPath string
+
+func (p *YAMLConfigPath) Set(v string) error {
+	*p = YAMLConfigPath(v)
+	return nil
+}
+
+func (p *YAMLConfigPath) String() string {
+	return string(*p)
+}
+
+func (*YAMLConfigPath) Type() string {
+	return "yaml-config-path"
+}

cli/clibase/yaml.go

@@ -0,0 +1,295 @@
+package clibase
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+
+	"github.com/mitchellh/go-wordwrap"
+	"golang.org/x/xerrors"
+	"gopkg.in/yaml.v3"
+)
+
+var (
+	_ yaml.Marshaler   = new(OptionSet)
+	_ yaml.Unmarshaler = new(OptionSet)
+)
+
+// deepMapNode returns the mapping node at the given path,
+// creating it if it doesn't exist.
+func deepMapNode(n *yaml.Node, path []string, headComment string) *yaml.Node {
+	if len(path) == 0 {
+		return n
+	}
+
+	// Name is every two nodes.
+	for i := 0; i < len(n.Content)-1; i += 2 {
+		if n.Content[i].Value == path[0] {
+			// Found matching name, recurse.
+			return deepMapNode(n.Content[i+1], path[1:], headComment)
+		}
+	}
+
+	// Not found, create it.
+	nameNode := yaml.Node{
+		Kind:        yaml.ScalarNode,
+		Value:       path[0],
+		HeadComment: headComment,
+	}
+	valueNode := yaml.Node{
+		Kind: yaml.MappingNode,
+	}
+	n.Content = append(n.Content, &nameNode)
+	n.Content = append(n.Content, &valueNode)
+	return deepMapNode(&valueNode, path[1:], headComment)
+}
+
+// MarshalYAML converts the option set to a YAML node, that can be
+// converted into bytes via yaml.Marshal.
+//
+// The node is returned to enable post-processing higher up in
+// the stack.
+//
+// It is isomorphic with FromYAML.
+func (optSet *OptionSet) MarshalYAML() (any, error) {
+	root := yaml.Node{
+		Kind: yaml.MappingNode,
+	}
+
+	for _, opt := range *optSet {
+		if opt.YAML == "" {
+			continue
+		}
+
+		defValue := opt.Default
+		if defValue == "" {
+			defValue = "<unset>"
+		}
+		comment := wordwrap.WrapString(
+			fmt.Sprintf("%s\n(default: %s, type: %s)", opt.Description, defValue, opt.Value.Type()),
+			80,
+		)
+		nameNode := yaml.Node{
+			Kind:        yaml.ScalarNode,
+			Value:       opt.YAML,
+			HeadComment: comment,
+		}
+		var valueNode yaml.Node
+		if opt.Value == nil {
+			valueNode = yaml.Node{
+				Kind:  yaml.ScalarNode,
+				Value: "null",
+			}
+		} else if m, ok := opt.Value.(yaml.Marshaler); ok {
+			v, err := m.MarshalYAML()
+			if err != nil {
+				return nil, xerrors.Errorf(
+					"marshal %q: %w", opt.Name, err,
+				)
+			}
+			valueNode, ok = v.(yaml.Node)
+			if !ok {
+				return nil, xerrors.Errorf(
+					"marshal %q: unexpected underlying type %T",
+					opt.Name, v,
+				)
+			}
+		} else {
+			// The all-other types case.
+			//
+			// A bit of a hack, we marshal and then unmarshal to get
+			// the underlying node.
+			byt, err := yaml.Marshal(opt.Value)
+			if err != nil {
+				return nil, xerrors.Errorf(
+					"marshal %q: %w", opt.Name, err,
+				)
+			}
+
+			var docNode yaml.Node
+			err = yaml.Unmarshal(byt, &docNode)
+			if err != nil {
+				return nil, xerrors.Errorf(
+					"unmarshal %q: %w", opt.Name, err,
+				)
+			}
+			if len(docNode.Content) != 1 {
+				return nil, xerrors.Errorf(
+					"unmarshal %q: expected one node, got %d",
+					opt.Name, len(docNode.Content),
+				)
+			}
+
+			valueNode = *docNode.Content[0]
+		}
+		var group []string
+		for _, g := range opt.Group.Ancestry() {
+			if g.YAML == "" {
+				return nil, xerrors.Errorf(
+					"group yaml name is empty for %q, groups: %+v",
+					opt.Name,
+					opt.Group,
+				)
+			}
+			group = append(group, g.YAML)
+		}
+		var groupDesc string
+		if opt.Group != nil {
+			groupDesc = wordwrap.WrapString(opt.Group.Description, 80)
+		}
+		parentValueNode := deepMapNode(
+			&root, group,
+			groupDesc,
+		)
+		parentValueNode.Content = append(
+			parentValueNode.Content,
+			&nameNode,
+			&valueNode,
+		)
+	}
+	return &root, nil
+}
+
+// mapYAMLNodes converts parent into a map with keys of form "group.subgroup.option"
+// and values as the corresponding YAML nodes.
+func mapYAMLNodes(parent *yaml.Node) (map[string]*yaml.Node, error) {
+	if parent.Kind != yaml.MappingNode {
+		return nil, xerrors.Errorf("expected mapping node, got type %v", parent.Kind)
+	}
+	if len(parent.Content)%2 != 0 {
+		return nil, xerrors.Errorf("expected an even number of k/v pairs, got %d", len(parent.Content))
+	}
+	var (
+		key  string
+		m    = make(map[string]*yaml.Node, len(parent.Content)/2)
+		merr error
+	)
+	for i, child := range parent.Content {
+		if i%2 == 0 {
+			if child.Kind != yaml.ScalarNode {
+				// We immediately because the rest of the code is bound to fail
+				// if we don't know to expect a key or a value.
+				return nil, xerrors.Errorf("expected scalar node for key, got type %v", child.Kind)
+			}
+			key = child.Value
+			continue
+		}
+
+		// We don't know if this is a grouped simple option or complex option,
+		// so we store both "key" and "group.key". Since we're storing pointers,
+		// the additional memory is of little concern.
+		m[key] = child
+		if child.Kind != yaml.MappingNode {
+			continue
+		}
+
+		sub, err := mapYAMLNodes(child)
+		if err != nil {
+			merr = errors.Join(merr, xerrors.Errorf("mapping node %q: %w", key, err))
+			continue
+		}
+		for k, v := range sub {
+			m[key+"."+k] = v
+		}
+	}
+
+	return m, nil
+}
+
+func (o *Option) setFromYAMLNode(n *yaml.Node) error {
+	o.ValueSource = ValueSourceYAML
+	if um, ok := o.Value.(yaml.Unmarshaler); ok {
+		return um.UnmarshalYAML(n)
+	}
+
+	switch n.Kind {
+	case yaml.ScalarNode:
+		return o.Value.Set(n.Value)
+	case yaml.SequenceNode:
+		// We treat empty values as nil for consistency with other option
+		// mechanisms.
+		if len(n.Content) == 0 {
+			o.Value = nil
+			return nil
+		}
+		return n.Decode(o.Value)
+	case yaml.MappingNode:
+		return xerrors.Errorf("mapping nodes must implement yaml.Unmarshaler")
+	default:
+		return xerrors.Errorf("unexpected node kind %v", n.Kind)
+	}
+}
+
+// UnmarshalYAML converts the given YAML node into the option set.
+// It is isomorphic with ToYAML.
+func (optSet *OptionSet) UnmarshalYAML(rootNode *yaml.Node) error {
+	// The rootNode will be a DocumentNode if it's read from a file. We do
+	// not support multiple documents in a single file.
+	if rootNode.Kind == yaml.DocumentNode {
+		if len(rootNode.Content) != 1 {
+			return xerrors.Errorf("expected one node in document, got %d", len(rootNode.Content))
+		}
+		rootNode = rootNode.Content[0]
+	}
+
+	yamlNodes, err := mapYAMLNodes(rootNode)
+	if err != nil {
+		return xerrors.Errorf("mapping nodes: %w", err)
+	}
+
+	matchedNodes := make(map[string]*yaml.Node, len(yamlNodes))
+
+	var merr error
+	for i := range *optSet {
+		opt := &(*optSet)[i]
+		if opt.YAML == "" {
+			continue
+		}
+		var group []string
+		for _, g := range opt.Group.Ancestry() {
+			if g.YAML == "" {
+				return xerrors.Errorf(
+					"group yaml name is empty for %q, groups: %+v",
+					opt.Name,
+					opt.Group,
+				)
+			}
+			group = append(group, g.YAML)
+			delete(yamlNodes, strings.Join(group, "."))
+		}
+
+		key := strings.Join(append(group, opt.YAML), ".")
+		node, ok := yamlNodes[key]
+		if !ok {
+			continue
+		}
+
+		matchedNodes[key] = node
+		if opt.ValueSource != ValueSourceNone {
+			continue
+		}
+		if err := opt.setFromYAMLNode(node); err != nil {
+			merr = errors.Join(merr, xerrors.Errorf("setting %q: %w", opt.YAML, err))
+		}
+	}
+
+	// Remove all matched nodes and their descendants from yamlNodes so we
+	// can accurately report unknown options.
+	for k := range yamlNodes {
+		var key string
+		for _, part := range strings.Split(k, ".") {
+			if key != "" {
+				key += "."
+			}
+			key += part
+			if _, ok := matchedNodes[key]; ok {
+				delete(yamlNodes, k)
+			}
+		}
+	}
+	for k := range yamlNodes {
+		merr = errors.Join(merr, xerrors.Errorf("unknown option %q", k))
+	}
+
+	return merr
+}

cli/clibase/yaml_test.go

@@ -0,0 +1,202 @@
+package clibase_test
+
+import (
+	"testing"
+
+	"github.com/spf13/pflag"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/exp/slices"
+	"gopkg.in/yaml.v3"
+
+	"github.com/coder/coder/v2/cli/clibase"
+)
+
+func TestOptionSet_YAML(t *testing.T) {
+	t.Parallel()
+
+	t.Run("RequireKey", func(t *testing.T) {
+		t.Parallel()
+		var workspaceName clibase.String
+		os := clibase.OptionSet{
+			clibase.Option{
+				Name:    "Workspace Name",
+				Value:   &workspaceName,
+				Default: "billie",
+			},
+		}
+
+		node, err := os.MarshalYAML()
+		require.NoError(t, err)
+		require.Len(t, node.(*yaml.Node).Content, 0)
+	})
+
+	t.Run("SimpleString", func(t *testing.T) {
+		t.Parallel()
+
+		var workspaceName clibase.String
+
+		os := clibase.OptionSet{
+			clibase.Option{
+				Name:        "Workspace Name",
+				Value:       &workspaceName,
+				Default:     "billie",
+				Description: "The workspace's name.",
+				Group:       &clibase.Group{YAML: "names"},
+				YAML:        "workspaceName",
+			},
+		}
+
+		err := os.SetDefaults()
+		require.NoError(t, err)
+
+		n, err := os.MarshalYAML()
+		require.NoError(t, err)
+		// Visually inspect for now.
+		byt, err := yaml.Marshal(n)
+		require.NoError(t, err)
+		t.Logf("Raw YAML:\n%s", string(byt))
+	})
+}
+
+func TestOptionSet_YAMLUnknownOptions(t *testing.T) {
+	t.Parallel()
+	os := clibase.OptionSet{
+		{
+			Name:        "Workspace Name",
+			Default:     "billie",
+			Description: "The workspace's name.",
+			YAML:        "workspaceName",
+			Value:       new(clibase.String),
+		},
+	}
+
+	const yamlDoc = `something: else`
+	err := yaml.Unmarshal([]byte(yamlDoc), &os)
+	require.Error(t, err)
+	require.Empty(t, os[0].Value.String())
+
+	os[0].YAML = "something"
+
+	err = yaml.Unmarshal([]byte(yamlDoc), &os)
+	require.NoError(t, err)
+
+	require.Equal(t, "else", os[0].Value.String())
+}
+
+// TestOptionSet_YAMLIsomorphism tests that the YAML representations of an
+// OptionSet converts to the same OptionSet when read back in.
+func TestOptionSet_YAMLIsomorphism(t *testing.T) {
+	t.Parallel()
+	// This is used to form a generic.
+	//nolint:unused
+	type kid struct {
+		Name string `yaml:"name"`
+		Age  int    `yaml:"age"`
+	}
+
+	for _, tc := range []struct {
+		name      string
+		os        clibase.OptionSet
+		zeroValue func() pflag.Value
+	}{
+		{
+			name: "SimpleString",
+			os: clibase.OptionSet{
+				{
+					Name:        "Workspace Name",
+					Default:     "billie",
+					Description: "The workspace's name.",
+					Group:       &clibase.Group{YAML: "names"},
+					YAML:        "workspaceName",
+				},
+			},
+			zeroValue: func() pflag.Value {
+				return clibase.StringOf(new(string))
+			},
+		},
+		{
+			name: "Array",
+			os: clibase.OptionSet{
+				{
+					YAML:    "names",
+					Default: "jill,jack,joan",
+				},
+			},
+			zeroValue: func() pflag.Value {
+				return clibase.StringArrayOf(&[]string{})
+			},
+		},
+		{
+			name: "ComplexObject",
+			os: clibase.OptionSet{
+				{
+					YAML: "kids",
+					Default: `- name: jill
+  age: 12
+- name: jack
+  age: 13`,
+				},
+			},
+			zeroValue: func() pflag.Value {
+				return &clibase.Struct[[]kid]{}
+			},
+		},
+		{
+			name: "DeepGroup",
+			os: clibase.OptionSet{
+				{
+					YAML:    "names",
+					Default: "jill,jack,joan",
+					Group:   &clibase.Group{YAML: "kids", Parent: &clibase.Group{YAML: "family"}},
+				},
+			},
+			zeroValue: func() pflag.Value {
+				return clibase.StringArrayOf(&[]string{})
+			},
+		},
+	} {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+
+			// Set initial values.
+			for i := range tc.os {
+				tc.os[i].Value = tc.zeroValue()
+			}
+			err := tc.os.SetDefaults()
+			require.NoError(t, err)
+
+			y, err := tc.os.MarshalYAML()
+			require.NoError(t, err)
+
+			toByt, err := yaml.Marshal(y)
+			require.NoError(t, err)
+
+			t.Logf("Raw YAML:\n%s", string(toByt))
+
+			var y2 yaml.Node
+			err = yaml.Unmarshal(toByt, &y2)
+			require.NoError(t, err)
+
+			os2 := slices.Clone(tc.os)
+			for i := range os2 {
+				os2[i].Value = tc.zeroValue()
+				os2[i].ValueSource = clibase.ValueSourceNone
+			}
+
+			// os2 values should be zeroed whereas tc.os should be
+			// set to defaults.
+			// This check makes sure we aren't mixing pointers.
+			require.NotEqual(t, tc.os, os2)
+			err = os2.UnmarshalYAML(&y2)
+			require.NoError(t, err)
+
+			want := tc.os
+			for i := range want {
+				want[i].ValueSource = clibase.ValueSourceYAML
+			}
+
+			require.Equal(t, tc.os, os2)
+		})
+	}
+}

cli/cliflag/cliflag.go

@@ -1,185 +0,0 @@
-// Package cliflag extends flagset with environment variable defaults.
-//
-// Usage:
-//
-// cliflag.String(root.Flags(), &address, "address", "a", "CODER_ADDRESS", "127.0.0.1:3000", "The address to serve the API and dashboard")
-//
-// Will produce the following usage docs:
-//
-//	-a, --address string              The address to serve the API and dashboard (uses $CODER_ADDRESS). (default "127.0.0.1:3000")
-package cliflag
-
-import (
-	"fmt"
-	"os"
-	"strconv"
-	"strings"
-	"time"
-
-	"github.com/spf13/cobra"
-	"github.com/spf13/pflag"
-
-	"github.com/coder/coder/cli/cliui"
-)
-
-// IsSetBool returns the value of the boolean flag if it is set.
-// It returns false if the flag isn't set or if any error occurs attempting
-// to parse the value of the flag.
-func IsSetBool(cmd *cobra.Command, name string) bool {
-	val, ok := IsSet(cmd, name)
-	if !ok {
-		return false
-	}
-
-	b, err := strconv.ParseBool(val)
-	return err == nil && b
-}
-
-// IsSet returns the string value of the flag and whether it was set.
-func IsSet(cmd *cobra.Command, name string) (string, bool) {
-	flag := cmd.Flag(name)
-	if flag == nil {
-		return "", false
-	}
-
-	return flag.Value.String(), flag.Changed
-}
-
-// String sets a string flag on the given flag set.
-func String(flagset *pflag.FlagSet, name, shorthand, env, def, usage string) {
-	v, ok := os.LookupEnv(env)
-	if !ok || v == "" {
-		v = def
-	}
-	flagset.StringP(name, shorthand, v, fmtUsage(usage, env))
-}
-
-// StringVarP sets a string flag on the given flag set.
-func StringVarP(flagset *pflag.FlagSet, p *string, name string, shorthand string, env string, def string, usage string) {
-	v, ok := os.LookupEnv(env)
-	if !ok || v == "" {
-		v = def
-	}
-	flagset.StringVarP(p, name, shorthand, v, fmtUsage(usage, env))
-}
-
-func StringArray(flagset *pflag.FlagSet, name, shorthand, env string, def []string, usage string) {
-	v, ok := os.LookupEnv(env)
-	if !ok || v == "" {
-		if v == "" {
-			def = []string{}
-		} else {
-			def = strings.Split(v, ",")
-		}
-	}
-	flagset.StringArrayP(name, shorthand, def, fmtUsage(usage, env))
-}
-
-func StringArrayVarP(flagset *pflag.FlagSet, ptr *[]string, name string, shorthand string, env string, def []string, usage string) {
-	val, ok := os.LookupEnv(env)
-	if ok {
-		if val == "" {
-			def = []string{}
-		} else {
-			def = strings.Split(val, ",")
-		}
-	}
-	flagset.StringArrayVarP(ptr, name, shorthand, def, fmtUsage(usage, env))
-}
-
-// Uint8VarP sets a uint8 flag on the given flag set.
-func Uint8VarP(flagset *pflag.FlagSet, ptr *uint8, name string, shorthand string, env string, def uint8, usage string) {
-	val, ok := os.LookupEnv(env)
-	if !ok || val == "" {
-		flagset.Uint8VarP(ptr, name, shorthand, def, fmtUsage(usage, env))
-		return
-	}
-
-	vi64, err := strconv.ParseUint(val, 10, 8)
-	if err != nil {
-		flagset.Uint8VarP(ptr, name, shorthand, def, fmtUsage(usage, env))
-		return
-	}
-
-	flagset.Uint8VarP(ptr, name, shorthand, uint8(vi64), fmtUsage(usage, env))
-}
-
-// IntVarP sets a uint8 flag on the given flag set.
-func IntVarP(flagset *pflag.FlagSet, ptr *int, name string, shorthand string, env string, def int, usage string) {
-	val, ok := os.LookupEnv(env)
-	if !ok || val == "" {
-		flagset.IntVarP(ptr, name, shorthand, def, fmtUsage(usage, env))
-		return
-	}
-
-	vi64, err := strconv.ParseUint(val, 10, 8)
-	if err != nil {
-		flagset.IntVarP(ptr, name, shorthand, def, fmtUsage(usage, env))
-		return
-	}
-
-	flagset.IntVarP(ptr, name, shorthand, int(vi64), fmtUsage(usage, env))
-}
-
-func Bool(flagset *pflag.FlagSet, name, shorthand, env string, def bool, usage string) {
-	val, ok := os.LookupEnv(env)
-	if !ok || val == "" {
-		flagset.BoolP(name, shorthand, def, fmtUsage(usage, env))
-		return
-	}
-
-	valb, err := strconv.ParseBool(val)
-	if err != nil {
-		flagset.BoolP(name, shorthand, def, fmtUsage(usage, env))
-		return
-	}
-
-	flagset.BoolP(name, shorthand, valb, fmtUsage(usage, env))
-}
-
-// BoolVarP sets a bool flag on the given flag set.
-func BoolVarP(flagset *pflag.FlagSet, ptr *bool, name string, shorthand string, env string, def bool, usage string) {
-	val, ok := os.LookupEnv(env)
-	if !ok || val == "" {
-		flagset.BoolVarP(ptr, name, shorthand, def, fmtUsage(usage, env))
-		return
-	}
-
-	valb, err := strconv.ParseBool(val)
-	if err != nil {
-		flagset.BoolVarP(ptr, name, shorthand, def, fmtUsage(usage, env))
-		return
-	}
-
-	flagset.BoolVarP(ptr, name, shorthand, valb, fmtUsage(usage, env))
-}
-
-// DurationVarP sets a time.Duration flag on the given flag set.
-func DurationVarP(flagset *pflag.FlagSet, ptr *time.Duration, name string, shorthand string, env string, def time.Duration, usage string) {
-	val, ok := os.LookupEnv(env)
-	if !ok || val == "" {
-		flagset.DurationVarP(ptr, name, shorthand, def, fmtUsage(usage, env))
-		return
-	}
-
-	valb, err := time.ParseDuration(val)
-	if err != nil {
-		flagset.DurationVarP(ptr, name, shorthand, def, fmtUsage(usage, env))
-		return
-	}
-
-	flagset.DurationVarP(ptr, name, shorthand, valb, fmtUsage(usage, env))
-}
-
-func fmtUsage(u string, env string) string {
-	if env != "" {
-		// Avoid double dotting.
-		dot := "."
-		if strings.HasSuffix(u, ".") {
-			dot = ""
-		}
-		u = fmt.Sprintf("%s%s\n"+cliui.Styles.Placeholder.Render("Consumes $%s"), u, dot, env)
-	}
-
-	return u
-}

cli/cliflag/cliflag_test.go

@@ -1,277 +0,0 @@
-package cliflag_test
-
-import (
-	"fmt"
-	"strconv"
-	"testing"
-	"time"
-
-	"github.com/spf13/pflag"
-	"github.com/stretchr/testify/require"
-
-	"github.com/coder/coder/cli/cliflag"
-	"github.com/coder/coder/cryptorand"
-)
-
-// Testcliflag cannot run in parallel because it uses t.Setenv.
-//
-//nolint:paralleltest
-func TestCliflag(t *testing.T) {
-	t.Run("StringDefault", func(t *testing.T) {
-		flagset, name, shorthand, env, usage := randomFlag()
-		def, _ := cryptorand.String(10)
-		cliflag.String(flagset, name, shorthand, env, def, usage)
-		got, err := flagset.GetString(name)
-		require.NoError(t, err)
-		require.Equal(t, def, got)
-		require.Contains(t, flagset.FlagUsages(), usage)
-		require.Contains(t, flagset.FlagUsages(), fmt.Sprintf("Consumes $%s", env))
-	})
-
-	t.Run("StringEnvVar", func(t *testing.T) {
-		flagset, name, shorthand, env, usage := randomFlag()
-		envValue, _ := cryptorand.String(10)
-		t.Setenv(env, envValue)
-		def, _ := cryptorand.String(10)
-		cliflag.String(flagset, name, shorthand, env, def, usage)
-		got, err := flagset.GetString(name)
-		require.NoError(t, err)
-		require.Equal(t, envValue, got)
-	})
-
-	t.Run("StringVarPDefault", func(t *testing.T) {
-		var ptr string
-		flagset, name, shorthand, env, usage := randomFlag()
-		def, _ := cryptorand.String(10)
-
-		cliflag.StringVarP(flagset, &ptr, name, shorthand, env, def, usage)
-		got, err := flagset.GetString(name)
-		require.NoError(t, err)
-		require.Equal(t, def, got)
-		require.Contains(t, flagset.FlagUsages(), usage)
-		require.Contains(t, flagset.FlagUsages(), fmt.Sprintf("Consumes $%s", env))
-	})
-
-	t.Run("StringVarPEnvVar", func(t *testing.T) {
-		var ptr string
-		flagset, name, shorthand, env, usage := randomFlag()
-		envValue, _ := cryptorand.String(10)
-		t.Setenv(env, envValue)
-		def, _ := cryptorand.String(10)
-
-		cliflag.StringVarP(flagset, &ptr, name, shorthand, env, def, usage)
-		got, err := flagset.GetString(name)
-		require.NoError(t, err)
-		require.Equal(t, envValue, got)
-	})
-
-	t.Run("EmptyEnvVar", func(t *testing.T) {
-		var ptr string
-		flagset, name, shorthand, _, usage := randomFlag()
-		def, _ := cryptorand.String(10)
-
-		cliflag.StringVarP(flagset, &ptr, name, shorthand, "", def, usage)
-		got, err := flagset.GetString(name)
-		require.NoError(t, err)
-		require.Equal(t, def, got)
-		require.Contains(t, flagset.FlagUsages(), usage)
-		require.NotContains(t, flagset.FlagUsages(), "Consumes")
-	})
-
-	t.Run("StringArrayDefault", func(t *testing.T) {
-		var ptr []string
-		flagset, name, shorthand, env, usage := randomFlag()
-		def := []string{"hello"}
-		cliflag.StringArrayVarP(flagset, &ptr, name, shorthand, env, def, usage)
-		got, err := flagset.GetStringArray(name)
-		require.NoError(t, err)
-		require.Equal(t, def, got)
-	})
-
-	t.Run("StringArrayEnvVar", func(t *testing.T) {
-		var ptr []string
-		flagset, name, shorthand, env, usage := randomFlag()
-		t.Setenv(env, "wow,test")
-		cliflag.StringArrayVarP(flagset, &ptr, name, shorthand, env, nil, usage)
-		got, err := flagset.GetStringArray(name)
-		require.NoError(t, err)
-		require.Equal(t, []string{"wow", "test"}, got)
-	})
-
-	t.Run("StringArrayEnvVarEmpty", func(t *testing.T) {
-		var ptr []string
-		flagset, name, shorthand, env, usage := randomFlag()
-		t.Setenv(env, "")
-		cliflag.StringArrayVarP(flagset, &ptr, name, shorthand, env, nil, usage)
-		got, err := flagset.GetStringArray(name)
-		require.NoError(t, err)
-		require.Equal(t, []string{}, got)
-	})
-
-	t.Run("UInt8Default", func(t *testing.T) {
-		var ptr uint8
-		flagset, name, shorthand, env, usage := randomFlag()
-		def, _ := cryptorand.Int63n(10)
-
-		cliflag.Uint8VarP(flagset, &ptr, name, shorthand, env, uint8(def), usage)
-		got, err := flagset.GetUint8(name)
-		require.NoError(t, err)
-		require.Equal(t, uint8(def), got)
-		require.Contains(t, flagset.FlagUsages(), usage)
-		require.Contains(t, flagset.FlagUsages(), fmt.Sprintf("Consumes $%s", env))
-	})
-
-	t.Run("UInt8EnvVar", func(t *testing.T) {
-		var ptr uint8
-		flagset, name, shorthand, env, usage := randomFlag()
-		envValue, _ := cryptorand.Int63n(10)
-		t.Setenv(env, strconv.FormatUint(uint64(envValue), 10))
-		def, _ := cryptorand.Int()
-
-		cliflag.Uint8VarP(flagset, &ptr, name, shorthand, env, uint8(def), usage)
-		got, err := flagset.GetUint8(name)
-		require.NoError(t, err)
-		require.Equal(t, uint8(envValue), got)
-	})
-
-	t.Run("UInt8FailParse", func(t *testing.T) {
-		var ptr uint8
-		flagset, name, shorthand, env, usage := randomFlag()
-		envValue, _ := cryptorand.String(10)
-		t.Setenv(env, envValue)
-		def, _ := cryptorand.Int63n(10)
-
-		cliflag.Uint8VarP(flagset, &ptr, name, shorthand, env, uint8(def), usage)
-		got, err := flagset.GetUint8(name)
-		require.NoError(t, err)
-		require.Equal(t, uint8(def), got)
-	})
-
-	t.Run("IntDefault", func(t *testing.T) {
-		var ptr int
-		flagset, name, shorthand, env, usage := randomFlag()
-		def, _ := cryptorand.Int63n(10)
-
-		cliflag.IntVarP(flagset, &ptr, name, shorthand, env, int(def), usage)
-		got, err := flagset.GetInt(name)
-		require.NoError(t, err)
-		require.Equal(t, int(def), got)
-		require.Contains(t, flagset.FlagUsages(), usage)
-		require.Contains(t, flagset.FlagUsages(), fmt.Sprintf("Consumes $%s", env))
-	})
-
-	t.Run("IntEnvVar", func(t *testing.T) {
-		var ptr int
-		flagset, name, shorthand, env, usage := randomFlag()
-		envValue, _ := cryptorand.Int63n(10)
-		t.Setenv(env, strconv.FormatUint(uint64(envValue), 10))
-		def, _ := cryptorand.Int()
-
-		cliflag.IntVarP(flagset, &ptr, name, shorthand, env, def, usage)
-		got, err := flagset.GetInt(name)
-		require.NoError(t, err)
-		require.Equal(t, int(envValue), got)
-	})
-
-	t.Run("IntFailParse", func(t *testing.T) {
-		var ptr int
-		flagset, name, shorthand, env, usage := randomFlag()
-		envValue, _ := cryptorand.String(10)
-		t.Setenv(env, envValue)
-		def, _ := cryptorand.Int63n(10)
-
-		cliflag.IntVarP(flagset, &ptr, name, shorthand, env, int(def), usage)
-		got, err := flagset.GetInt(name)
-		require.NoError(t, err)
-		require.Equal(t, int(def), got)
-	})
-
-	t.Run("BoolDefault", func(t *testing.T) {
-		var ptr bool
-		flagset, name, shorthand, env, usage := randomFlag()
-		def, _ := cryptorand.Bool()
-
-		cliflag.BoolVarP(flagset, &ptr, name, shorthand, env, def, usage)
-		got, err := flagset.GetBool(name)
-		require.NoError(t, err)
-		require.Equal(t, def, got)
-		require.Contains(t, flagset.FlagUsages(), usage)
-		require.Contains(t, flagset.FlagUsages(), fmt.Sprintf("Consumes $%s", env))
-	})
-
-	t.Run("BoolEnvVar", func(t *testing.T) {
-		var ptr bool
-		flagset, name, shorthand, env, usage := randomFlag()
-		envValue, _ := cryptorand.Bool()
-		t.Setenv(env, strconv.FormatBool(envValue))
-		def, _ := cryptorand.Bool()
-
-		cliflag.BoolVarP(flagset, &ptr, name, shorthand, env, def, usage)
-		got, err := flagset.GetBool(name)
-		require.NoError(t, err)
-		require.Equal(t, envValue, got)
-	})
-
-	t.Run("BoolFailParse", func(t *testing.T) {
-		var ptr bool
-		flagset, name, shorthand, env, usage := randomFlag()
-		envValue, _ := cryptorand.String(10)
-		t.Setenv(env, envValue)
-		def, _ := cryptorand.Bool()
-
-		cliflag.BoolVarP(flagset, &ptr, name, shorthand, env, def, usage)
-		got, err := flagset.GetBool(name)
-		require.NoError(t, err)
-		require.Equal(t, def, got)
-	})
-
-	t.Run("DurationDefault", func(t *testing.T) {
-		var ptr time.Duration
-		flagset, name, shorthand, env, usage := randomFlag()
-		def, _ := cryptorand.Duration()
-
-		cliflag.DurationVarP(flagset, &ptr, name, shorthand, env, def, usage)
-		got, err := flagset.GetDuration(name)
-		require.NoError(t, err)
-		require.Equal(t, def, got)
-		require.Contains(t, flagset.FlagUsages(), usage)
-		require.Contains(t, flagset.FlagUsages(), fmt.Sprintf("Consumes $%s", env))
-	})
-
-	t.Run("DurationEnvVar", func(t *testing.T) {
-		var ptr time.Duration
-		flagset, name, shorthand, env, usage := randomFlag()
-		envValue, _ := cryptorand.Duration()
-		t.Setenv(env, envValue.String())
-		def, _ := cryptorand.Duration()
-
-		cliflag.DurationVarP(flagset, &ptr, name, shorthand, env, def, usage)
-		got, err := flagset.GetDuration(name)
-		require.NoError(t, err)
-		require.Equal(t, envValue, got)
-	})
-
-	t.Run("DurationFailParse", func(t *testing.T) {
-		var ptr time.Duration
-		flagset, name, shorthand, env, usage := randomFlag()
-		envValue, _ := cryptorand.String(10)
-		t.Setenv(env, envValue)
-		def, _ := cryptorand.Duration()
-
-		cliflag.DurationVarP(flagset, &ptr, name, shorthand, env, def, usage)
-		got, err := flagset.GetDuration(name)
-		require.NoError(t, err)
-		require.Equal(t, def, got)
-	})
-}
-
-func randomFlag() (*pflag.FlagSet, string, string, string, string) {
-	fsname, _ := cryptorand.String(10)
-	flagset := pflag.NewFlagSet(fsname, pflag.PanicOnError)
-	name, _ := cryptorand.String(10)
-	shorthand, _ := cryptorand.String(1)
-	env, _ := cryptorand.String(10)
-	usage, _ := cryptorand.String(10)
-
-	return flagset, name, shorthand, env, usage
-}

cli/clistat/cgroup.go

@@ -0,0 +1,360 @@
+package clistat
+
+import (
+	"bufio"
+	"bytes"
+	"strconv"
+	"strings"
+
+	"github.com/hashicorp/go-multierror"
+	"github.com/spf13/afero"
+	"golang.org/x/xerrors"
+	"tailscale.com/types/ptr"
+)
+
+// Paths for CGroupV1.
+// Ref: https://www.kernel.org/doc/Documentation/cgroup-v1/cpuacct.txt
+const (
+	// CPU usage of all tasks in cgroup in nanoseconds.
+	cgroupV1CPUAcctUsage = "/sys/fs/cgroup/cpu,cpuacct/cpuacct.usage"
+	// CFS quota and period for cgroup in MICROseconds
+	cgroupV1CFSQuotaUs = "/sys/fs/cgroup/cpu,cpuacct/cpu.cfs_quota_us"
+	// CFS period for cgroup in MICROseconds
+	cgroupV1CFSPeriodUs = "/sys/fs/cgroup/cpu,cpuacct/cpu.cfs_period_us"
+	// Maximum memory usable by cgroup in bytes
+	cgroupV1MemoryMaxUsageBytes = "/sys/fs/cgroup/memory/memory.limit_in_bytes"
+	// Current memory usage of cgroup in bytes
+	cgroupV1MemoryUsageBytes = "/sys/fs/cgroup/memory/memory.usage_in_bytes"
+	// Other memory stats - we are interested in total_inactive_file
+	cgroupV1MemoryStat = "/sys/fs/cgroup/memory/memory.stat"
+)
+
+// Paths for CGroupV2.
+// Ref: https://docs.kernel.org/admin-guide/cgroup-v2.html
+const (
+	// Contains quota and period in microseconds separated by a space.
+	cgroupV2CPUMax = "/sys/fs/cgroup/cpu.max"
+	// Contains current CPU usage under usage_usec
+	cgroupV2CPUStat = "/sys/fs/cgroup/cpu.stat"
+	// Contains current cgroup memory usage in bytes.
+	cgroupV2MemoryUsageBytes = "/sys/fs/cgroup/memory.current"
+	// Contains max cgroup memory usage in bytes.
+	cgroupV2MemoryMaxBytes = "/sys/fs/cgroup/memory.max"
+	// Other memory stats - we are interested in total_inactive_file
+	cgroupV2MemoryStat = "/sys/fs/cgroup/memory.stat"
+)
+
+// ContainerCPU returns the CPU usage of the container cgroup.
+// This is calculated as difference of two samples of the
+// CPU usage of the container cgroup.
+// The total is read from the relevant path in /sys/fs/cgroup.
+// If there is no limit set, the total is assumed to be the
+// number of host cores multiplied by the CFS period.
+// If the system is not containerized, this always returns nil.
+func (s *Statter) ContainerCPU() (*Result, error) {
+	// Firstly, check if we are containerized.
+	if ok, err := IsContainerized(s.fs); err != nil || !ok {
+		return nil, nil //nolint: nilnil
+	}
+
+	total, err := s.cGroupCPUTotal()
+	if err != nil {
+		return nil, xerrors.Errorf("get total cpu: %w", err)
+	}
+	used1, err := s.cGroupCPUUsed()
+	if err != nil {
+		return nil, xerrors.Errorf("get cgroup CPU usage: %w", err)
+	}
+
+	// The measurements in /sys/fs/cgroup are counters.
+	// We need to wait for a bit to get a difference.
+	// Note that someone could reset the counter in the meantime.
+	// We can't do anything about that.
+	s.wait(s.sampleInterval)
+
+	used2, err := s.cGroupCPUUsed()
+	if err != nil {
+		return nil, xerrors.Errorf("get cgroup CPU usage: %w", err)
+	}
+
+	if used2 < used1 {
+		// Someone reset the counter. Best we can do is count from zero.
+		used1 = 0
+	}
+
+	r := &Result{
+		Unit:   "cores",
+		Used:   used2 - used1,
+		Prefix: PrefixDefault,
+	}
+
+	if total > 0 {
+		r.Total = ptr.To(total)
+	}
+	return r, nil
+}
+
+func (s *Statter) cGroupCPUTotal() (used float64, err error) {
+	if s.isCGroupV2() {
+		return s.cGroupV2CPUTotal()
+	}
+
+	// Fall back to CGroupv1
+	return s.cGroupV1CPUTotal()
+}
+
+func (s *Statter) cGroupCPUUsed() (used float64, err error) {
+	if s.isCGroupV2() {
+		return s.cGroupV2CPUUsed()
+	}
+
+	return s.cGroupV1CPUUsed()
+}
+
+func (s *Statter) isCGroupV2() bool {
+	// Check for the presence of /sys/fs/cgroup/cpu.max
+	_, err := s.fs.Stat(cgroupV2CPUMax)
+	return err == nil
+}
+
+func (s *Statter) cGroupV2CPUUsed() (used float64, err error) {
+	usageUs, err := readInt64Prefix(s.fs, cgroupV2CPUStat, "usage_usec")
+	if err != nil {
+		return 0, xerrors.Errorf("get cgroupv2 cpu used: %w", err)
+	}
+	periodUs, err := readInt64SepIdx(s.fs, cgroupV2CPUMax, " ", 1)
+	if err != nil {
+		return 0, xerrors.Errorf("get cpu period: %w", err)
+	}
+
+	return float64(usageUs) / float64(periodUs), nil
+}
+
+func (s *Statter) cGroupV2CPUTotal() (total float64, err error) {
+	var quotaUs, periodUs int64
+	periodUs, err = readInt64SepIdx(s.fs, cgroupV2CPUMax, " ", 1)
+	if err != nil {
+		return 0, xerrors.Errorf("get cpu period: %w", err)
+	}
+
+	quotaUs, err = readInt64SepIdx(s.fs, cgroupV2CPUMax, " ", 0)
+	if err != nil {
+		if xerrors.Is(err, strconv.ErrSyntax) {
+			// If the value is not a valid integer, assume it is the string
+			// 'max' and that there is no limit set.
+			return -1, nil
+		}
+		return 0, xerrors.Errorf("get cpu quota: %w", err)
+	}
+
+	return float64(quotaUs) / float64(periodUs), nil
+}
+
+func (s *Statter) cGroupV1CPUTotal() (float64, error) {
+	periodUs, err := readInt64(s.fs, cgroupV1CFSPeriodUs)
+	if err != nil {
+		// Try alternate path under /sys/fs/cpu
+		var merr error
+		merr = multierror.Append(merr, xerrors.Errorf("get cpu period: %w", err))
+		periodUs, err = readInt64(s.fs, strings.Replace(cgroupV1CFSPeriodUs, "cpu,cpuacct", "cpu", 1))
+		if err != nil {
+			merr = multierror.Append(merr, xerrors.Errorf("get cpu period: %w", err))
+			return 0, merr
+		}
+	}
+
+	quotaUs, err := readInt64(s.fs, cgroupV1CFSQuotaUs)
+	if err != nil {
+		// Try alternate path under /sys/fs/cpu
+		var merr error
+		merr = multierror.Append(merr, xerrors.Errorf("get cpu quota: %w", err))
+		quotaUs, err = readInt64(s.fs, strings.Replace(cgroupV1CFSQuotaUs, "cpu,cpuacct", "cpu", 1))
+		if err != nil {
+			merr = multierror.Append(merr, xerrors.Errorf("get cpu quota: %w", err))
+			return 0, merr
+		}
+	}
+
+	if quotaUs < 0 {
+		return -1, nil
+	}
+
+	return float64(quotaUs) / float64(periodUs), nil
+}
+
+func (s *Statter) cGroupV1CPUUsed() (float64, error) {
+	usageNs, err := readInt64(s.fs, cgroupV1CPUAcctUsage)
+	if err != nil {
+		// Try alternate path under /sys/fs/cgroup/cpuacct
+		var merr error
+		merr = multierror.Append(merr, xerrors.Errorf("read cpu used: %w", err))
+		usageNs, err = readInt64(s.fs, strings.Replace(cgroupV1CPUAcctUsage, "cpu,cpuacct", "cpuacct", 1))
+		if err != nil {
+			merr = multierror.Append(merr, xerrors.Errorf("read cpu used: %w", err))
+			return 0, merr
+		}
+	}
+
+	// usage is in ns, convert to us
+	usageNs /= 1000
+	periodUs, err := readInt64(s.fs, cgroupV1CFSPeriodUs)
+	if err != nil {
+		// Try alternate path under /sys/fs/cpu
+		var merr error
+		merr = multierror.Append(merr, xerrors.Errorf("get cpu period: %w", err))
+		periodUs, err = readInt64(s.fs, strings.Replace(cgroupV1CFSPeriodUs, "cpu,cpuacct", "cpu", 1))
+		if err != nil {
+			merr = multierror.Append(merr, xerrors.Errorf("get cpu period: %w", err))
+			return 0, merr
+		}
+	}
+
+	return float64(usageNs) / float64(periodUs), nil
+}
+
+// ContainerMemory returns the memory usage of the container cgroup.
+// If the system is not containerized, this always returns nil.
+func (s *Statter) ContainerMemory(p Prefix) (*Result, error) {
+	if ok, err := IsContainerized(s.fs); err != nil || !ok {
+		return nil, nil //nolint:nilnil
+	}
+
+	if s.isCGroupV2() {
+		return s.cGroupV2Memory(p)
+	}
+
+	// Fall back to CGroupv1
+	return s.cGroupV1Memory(p)
+}
+
+func (s *Statter) cGroupV2Memory(p Prefix) (*Result, error) {
+	r := &Result{
+		Unit:   "B",
+		Prefix: p,
+	}
+	maxUsageBytes, err := readInt64(s.fs, cgroupV2MemoryMaxBytes)
+	if err != nil {
+		if !xerrors.Is(err, strconv.ErrSyntax) {
+			return nil, xerrors.Errorf("read memory total: %w", err)
+		}
+		// If the value is not a valid integer, assume it is the string
+		// 'max' and that there is no limit set.
+	} else {
+		r.Total = ptr.To(float64(maxUsageBytes))
+	}
+
+	currUsageBytes, err := readInt64(s.fs, cgroupV2MemoryUsageBytes)
+	if err != nil {
+		return nil, xerrors.Errorf("read memory usage: %w", err)
+	}
+
+	inactiveFileBytes, err := readInt64Prefix(s.fs, cgroupV2MemoryStat, "inactive_file")
+	if err != nil {
+		return nil, xerrors.Errorf("read memory stats: %w", err)
+	}
+
+	r.Used = float64(currUsageBytes - inactiveFileBytes)
+	return r, nil
+}
+
+func (s *Statter) cGroupV1Memory(p Prefix) (*Result, error) {
+	r := &Result{
+		Unit:   "B",
+		Prefix: p,
+	}
+	maxUsageBytes, err := readInt64(s.fs, cgroupV1MemoryMaxUsageBytes)
+	if err != nil {
+		if !xerrors.Is(err, strconv.ErrSyntax) {
+			return nil, xerrors.Errorf("read memory total: %w", err)
+		}
+		// I haven't found an instance where this isn't a valid integer.
+		// Nonetheless, if it is not, assume there is no limit set.
+		maxUsageBytes = -1
+	}
+
+	// need a space after total_rss so we don't hit something else
+	usageBytes, err := readInt64(s.fs, cgroupV1MemoryUsageBytes)
+	if err != nil {
+		return nil, xerrors.Errorf("read memory usage: %w", err)
+	}
+
+	totalInactiveFileBytes, err := readInt64Prefix(s.fs, cgroupV1MemoryStat, "total_inactive_file")
+	if err != nil {
+		return nil, xerrors.Errorf("read memory stats: %w", err)
+	}
+
+	// If max usage bytes is -1, there is no memory limit set.
+	if maxUsageBytes > 0 {
+		r.Total = ptr.To(float64(maxUsageBytes))
+	}
+
+	// Total memory used is usage - total_inactive_file
+	r.Used = float64(usageBytes - totalInactiveFileBytes)
+
+	return r, nil
+}
+
+// read an int64 value from path
+func readInt64(fs afero.Fs, path string) (int64, error) {
+	data, err := afero.ReadFile(fs, path)
+	if err != nil {
+		return 0, xerrors.Errorf("read %s: %w", path, err)
+	}
+
+	val, err := strconv.ParseInt(string(bytes.TrimSpace(data)), 10, 64)
+	if err != nil {
+		return 0, xerrors.Errorf("parse %s: %w", path, err)
+	}
+
+	return val, nil
+}
+
+// read an int64 value from path at field idx separated by sep
+func readInt64SepIdx(fs afero.Fs, path, sep string, idx int) (int64, error) {
+	data, err := afero.ReadFile(fs, path)
+	if err != nil {
+		return 0, xerrors.Errorf("read %s: %w", path, err)
+	}
+
+	parts := strings.Split(string(data), sep)
+	if len(parts) < idx {
+		return 0, xerrors.Errorf("expected line %q to have at least %d parts", string(data), idx+1)
+	}
+
+	val, err := strconv.ParseInt(strings.TrimSpace(parts[idx]), 10, 64)
+	if err != nil {
+		return 0, xerrors.Errorf("parse %s: %w", path, err)
+	}
+
+	return val, nil
+}
+
+// read the first int64 value from path prefixed with prefix
+func readInt64Prefix(fs afero.Fs, path, prefix string) (int64, error) {
+	data, err := afero.ReadFile(fs, path)
+	if err != nil {
+		return 0, xerrors.Errorf("read %s: %w", path, err)
+	}
+
+	scn := bufio.NewScanner(bytes.NewReader(data))
+	for scn.Scan() {
+		line := strings.TrimSpace(scn.Text())
+		if !strings.HasPrefix(line, prefix) {
+			continue
+		}
+
+		parts := strings.Fields(line)
+		if len(parts) != 2 {
+			return 0, xerrors.Errorf("parse %s: expected two fields but got %s", path, line)
+		}
+
+		val, err := strconv.ParseInt(strings.TrimSpace(parts[1]), 10, 64)
+		if err != nil {
+			return 0, xerrors.Errorf("parse %s: %w", path, err)
+		}
+
+		return val, nil
+	}
+
+	return 0, xerrors.Errorf("parse %s: did not find line with prefix %s", path, prefix)
+}