chore: add v2.9.1 changelog

* fix: separate signals for passive, active, and forced shutdown

`SIGTERM`: Passive shutdown stopping provisioner daemons from accepting new
jobs but waiting for existing jobs to successfully complete.

`SIGINT` (old existing behavior): Notify provisioner daemons to cancel in-flight jobs, wait 5s for jobs to be exited, then force quit.

`SIGKILL`: Untouched from before, will force-quit.

* Revert dramatic signal changes

* Rename

* Fix shutdown behavior for provisioner daemons

* Add test for graceful shutdown

(cherry picked from commit 895df54051)

.devcontainer/devcontainer.json

@@ -4,9 +4,10 @@
 
   "features": {
     // See all possible options here https://github.com/devcontainers/features/tree/main/src/docker-in-docker
-    "ghcr.io/devcontainers/features/docker-in-docker:2": {}
+    "ghcr.io/devcontainers/features/docker-in-docker:2": {
+      "moby": "false"
+    }
   },
   // SYS_PTRACE to enable go debugging
-  // without --priviliged the Github Codespace build fails (not required otherwise)
-  "runArgs": ["--cap-add=SYS_PTRACE", "--privileged"]
+  "runArgs": ["--cap-add=SYS_PTRACE"]
 }

.dockerignore

@@ -0,0 +1,6 @@
+# Ignore all files and folders
+**
+
+# Include flake.nix and flake.lock
+!flake.nix
+!flake.lock

.git-blame-ignore-revs

@@ -0,0 +1,5 @@
+# If you would like `git blame` to ignore commits from this file, run...
+#   git config blame.ignoreRevsFile .git-blame-ignore-revs
+
+# chore: format code with semicolons when using prettier (#9555)
+988c9af0153561397686c119da9d1336d2433fdd

.gitattributes

@@ -6,9 +6,12 @@ coderd/apidoc/swagger.json linguist-generated=true
 coderd/database/dump.sql linguist-generated=true
 peerbroker/proto/*.go linguist-generated=true
 provisionerd/proto/*.go linguist-generated=true
+provisionerd/proto/version.go linguist-generated=false
 provisionersdk/proto/*.go linguist-generated=true
 *.tfplan.json linguist-generated=true
 *.tfstate.json linguist-generated=true
 *.tfstate.dot linguist-generated=true
 *.tfplan.dot linguist-generated=true
+site/e2e/provisionerGenerated.ts linguist-generated=true
 site/src/api/typesGenerated.ts linguist-generated=true
+site/src/pages/SetupPage/countries.tsx linguist-generated=true

.github/actions/setup-go/action.yaml

@@ -4,61 +4,15 @@ description: |
 inputs:
   version:
     description: "The Go version to use."
-    default: "1.20.6"
+    default: "1.21.5"
 runs:
   using: "composite"
   steps:
-    - name: Cache go toolchain
-      uses: buildjet/cache@v3
-      with:
-        path: |
-          ${{ runner.tool_cache }}/go/${{ inputs.version }}
-        key: gotoolchain-${{ runner.os }}-${{ inputs.version }}
-        restore-keys: |
-          gotoolchain-${{ runner.os }}-
-
     - name: Setup Go
       uses: buildjet/setup-go@v4
       with:
-        # We do our own caching for implementation clarity.
-        cache: false
         go-version: ${{ inputs.version }}
 
-    - name: Get cache dirs
-      shell: bash
-      run: |
-        set -x
-        echo "GOMODCACHE=$(go env GOMODCACHE)" >> $GITHUB_ENV
-        echo "GOCACHE=$(go env GOCACHE)" >> $GITHUB_ENV
-
-    # We split up GOMODCACHE from GOCACHE because the latter must be invalidated
-    # on code change, but the former can be kept.
-    - name: Cache $GOMODCACHE
-      uses: buildjet/cache@v3
-      with:
-        path: |
-          ${{ env.GOMODCACHE }}
-        key: gomodcache-${{ runner.os }}-${{ hashFiles('**/go.sum') }}-${{ github.job }}
-        # restore-keys aren't used because it causes the cache to grow
-        # infinitely. go.sum changes very infrequently, so rebuilding from
-        # scratch every now and then isn't terrible.
-
-    - name: Cache $GOCACHE
-      uses: buildjet/cache@v3
-      with:
-        path: |
-          ${{ env.GOCACHE }}
-        # Job name must be included in the key for effective test cache reuse.
-        # The key format is intentionally different than GOMODCACHE, because any
-        # time a Go file changes we invalidate this cache, whereas GOMODCACHE is
-        # only invalidated when go.sum changes.
-        # The number in the key is incremented when the cache gets too large,
-        # since this technically grows without bound.
-        key: gocache2-${{ runner.os }}-${{ github.job }}-${{ hashFiles('**/*.go', 'go.**') }}
-        restore-keys: |
-          gocache2-${{ runner.os }}-${{ github.job }}-
-          gocache2-${{ runner.os }}-
-
     - name: Install gotestsum
       shell: bash
       run: go install gotest.tools/gotestsum@latest

.github/actions/setup-node/action.yaml

@@ -17,7 +17,7 @@ runs:
     - name: Setup Node
       uses: buildjet/setup-node@v3
       with:
-        node-version: 18.17.0
+        node-version: 18.19.0
         # See https://github.com/actions/setup-node#caching-global-packages-data
         cache: "pnpm"
         cache-dependency-path: ${{ inputs.directory }}/pnpm-lock.yaml

.github/actions/setup-sqlc/action.yaml

@@ -5,6 +5,6 @@ runs:
   using: "composite"
   steps:
     - name: Setup sqlc
-      uses: sqlc-dev/setup-sqlc@v3
+      uses: sqlc-dev/setup-sqlc@v4
       with:
-        sqlc-version: "1.19.1"
+        sqlc-version: "1.25.0"

.github/actions/setup-tf/action.yaml

@@ -5,7 +5,7 @@ runs:
   using: "composite"
   steps:
     - name: Install Terraform
-      uses: hashicorp/setup-terraform@v2
+      uses: hashicorp/setup-terraform@v3
       with:
-        terraform_version: ~1.5
+        terraform_version: 1.5.7
         terraform_wrapper: false

.github/actions/upload-datadog/action.yaml

@@ -20,7 +20,7 @@ runs:
           echo "No API key provided, skipping..."
           exit 0
         fi
-        npm install -g @datadog/datadog-ci@2.10.0
+        npm install -g @datadog/datadog-ci@2.21.0
         datadog-ci junit upload --service coder ./gotests.xml \
           --tags os:${{runner.os}} --tags runner_name:${{runner.name}}
       env:

.github/dependabot.yaml

@@ -8,7 +8,7 @@ updates:
       timezone: "America/Chicago"
     labels: []
     commit-message:
-      prefix: "chore"
+      prefix: "ci"
     ignore:
       # These actions deliver the latest versions by updating the major
       # release tag, so ignore minor and patch versions
@@ -38,19 +38,12 @@ updates:
     commit-message:
       prefix: "chore"
     labels: []
+    open-pull-requests-limit: 15
     ignore:
       # Ignore patch updates for all dependencies
       - dependency-name: "*"
         update-types:
           - version-update:semver-patch
-    groups:
-      otel:
-        patterns:
-          - "go.nhat.io/otelsql"
-          - "go.opentelemetry.io/otel*"
-      golang-x:
-        patterns:
-          - "golang.org/x/*"
 
   # Update our Dockerfile.
   - package-ecosystem: "docker"
@@ -66,10 +59,6 @@ updates:
       # We need to coordinate terraform updates with the version hardcoded in
       # our Go code.
       - dependency-name: "terraform"
-    groups:
-      scripts-docker:
-        patterns:
-          - "*"
 
   - package-ecosystem: "npm"
     directory: "/site/"
@@ -92,36 +81,11 @@ updates:
       - dependency-name: "@types/node"
         update-types:
           - version-update:semver-major
+    open-pull-requests-limit: 15
     groups:
-      react:
+      site:
         patterns:
-          - "react*"
-          - "@types/react*"
-      xterm:
-        patterns:
-          - "xterm*"
-      xstate:
-        patterns:
-          - "xstate"
-          - "@xstate*"
-      mui:
-        patterns:
-          - "@mui*"
-      storybook:
-        patterns:
-          - "@storybook*"
-          - "storybook*"
-      eslint:
-        patterns:
-          - "eslint*"
-          - "@eslint*"
-          - "@typescript-eslint/eslint-plugin"
-          - "@typescript-eslint/parser"
-      jest:
-        patterns:
-          - "jest*"
-          - "@swc/jest"
-          - "@types/jest"
+          - "*"
 
   - package-ecosystem: "npm"
     directory: "/offlinedocs/"
@@ -144,22 +108,12 @@ updates:
       - dependency-name: "@types/node"
         update-types:
           - version-update:semver-major
-
-  # Update dogfood.
-  - package-ecosystem: "docker"
-    directory: "/dogfood/"
-    schedule:
-      interval: "weekly"
-      time: "06:00"
-      timezone: "America/Chicago"
-    commit-message:
-      prefix: "chore"
-    labels: []
     groups:
-      dogfood-docker:
+      offlinedocs:
         patterns:
           - "*"
 
+  # Update dogfood.
   - package-ecosystem: "terraform"
     directory: "/dogfood/"
     schedule:

.github/fly-wsproxies/jnb-coder.toml

@@ -0,0 +1,34 @@
+app = "jnb-coder"
+primary_region = "jnb"
+
+[experimental]
+  entrypoint = ["/bin/sh", "-c", "CODER_DERP_SERVER_RELAY_URL=\"http://[${FLY_PRIVATE_IP}]:3000\" /opt/coder wsproxy server"]
+  auto_rollback = true
+
+[build]
+  image = "ghcr.io/coder/coder-preview:main"
+
+[env]
+  CODER_ACCESS_URL = "https://jnb.fly.dev.coder.com"
+  CODER_HTTP_ADDRESS = "0.0.0.0:3000"
+  CODER_PRIMARY_ACCESS_URL = "https://dev.coder.com"
+  CODER_WILDCARD_ACCESS_URL = "*--apps.jnb.fly.dev.coder.com"
+  CODER_VERBOSE = "true"
+
+[http_service]
+  internal_port = 3000
+  force_https = true
+  auto_stop_machines = true
+  auto_start_machines = true
+  min_machines_running = 0
+
+# Ref: https://fly.io/docs/reference/configuration/#http_service-concurrency
+[http_service.concurrency]
+  type = "requests"
+  soft_limit = 50
+  hard_limit = 100
+
+[[vm]]
+  cpu_kind = "shared"
+  cpus = 2
+  memory_mb = 512

.github/fly-wsproxies/paris-coder.toml

@@ -0,0 +1,34 @@
+app = "paris-coder"
+primary_region = "cdg"
+
+[experimental]
+  entrypoint = ["/bin/sh", "-c", "CODER_DERP_SERVER_RELAY_URL=\"http://[${FLY_PRIVATE_IP}]:3000\" /opt/coder wsproxy server"]
+  auto_rollback = true
+
+[build]
+  image = "ghcr.io/coder/coder-preview:main"
+
+[env]
+  CODER_ACCESS_URL = "https://paris.fly.dev.coder.com"
+  CODER_HTTP_ADDRESS = "0.0.0.0:3000"
+  CODER_PRIMARY_ACCESS_URL = "https://dev.coder.com"
+  CODER_WILDCARD_ACCESS_URL = "*--apps.paris.fly.dev.coder.com"
+  CODER_VERBOSE = "true"
+
+[http_service]
+  internal_port = 3000
+  force_https = true
+  auto_stop_machines = true
+  auto_start_machines = true
+  min_machines_running = 0
+
+# Ref: https://fly.io/docs/reference/configuration/#http_service-concurrency
+[http_service.concurrency]
+  type = "requests"
+  soft_limit = 50
+  hard_limit = 100
+
+[[vm]]
+  cpu_kind = "shared"
+  cpus = 2
+  memory_mb = 512

.github/fly-wsproxies/sao-paulo-coder.toml

@@ -0,0 +1,34 @@
+app = "sao-paulo-coder"
+primary_region = "gru"
+
+[experimental]
+  entrypoint = ["/bin/sh", "-c", "CODER_DERP_SERVER_RELAY_URL=\"http://[${FLY_PRIVATE_IP}]:3000\" /opt/coder wsproxy server"]
+  auto_rollback = true
+
+[build]
+  image = "ghcr.io/coder/coder-preview:main"
+
+[env]
+  CODER_ACCESS_URL = "https://sao-paulo.fly.dev.coder.com"
+  CODER_HTTP_ADDRESS = "0.0.0.0:3000"
+  CODER_PRIMARY_ACCESS_URL = "https://dev.coder.com"
+  CODER_WILDCARD_ACCESS_URL = "*--apps.sao-paulo.fly.dev.coder.com"
+  CODER_VERBOSE = "true"
+
+[http_service]
+  internal_port = 3000
+  force_https = true
+  auto_stop_machines = true
+  auto_start_machines = true
+  min_machines_running = 0
+
+# Ref: https://fly.io/docs/reference/configuration/#http_service-concurrency
+[http_service.concurrency]
+  type = "requests"
+  soft_limit = 50
+  hard_limit = 100
+
+[[vm]]
+  cpu_kind = "shared"
+  cpus = 2
+  memory_mb = 512

.github/fly-wsproxies/sydney-coder.toml

@@ -0,0 +1,34 @@
+app = "sydney-coder"
+primary_region = "syd"
+
+[experimental]
+  entrypoint = ["/bin/sh", "-c", "CODER_DERP_SERVER_RELAY_URL=\"http://[${FLY_PRIVATE_IP}]:3000\" /opt/coder wsproxy server"]
+  auto_rollback = true
+
+[build]
+  image = "ghcr.io/coder/coder-preview:main"
+
+[env]
+  CODER_ACCESS_URL = "https://sydney.fly.dev.coder.com"
+  CODER_HTTP_ADDRESS = "0.0.0.0:3000"
+  CODER_PRIMARY_ACCESS_URL = "https://dev.coder.com"
+  CODER_WILDCARD_ACCESS_URL = "*--apps.sydney.fly.dev.coder.com"
+  CODER_VERBOSE = "true"
+
+[http_service]
+  internal_port = 3000
+  force_https = true
+  auto_stop_machines = true
+  auto_start_machines = true
+  min_machines_running = 0
+
+# Ref: https://fly.io/docs/reference/configuration/#http_service-concurrency
+[http_service.concurrency]
+  type = "requests"
+  soft_limit = 50
+  hard_limit = 100
+
+[[vm]]
+  cpu_kind = "shared"
+  cpus = 2
+  memory_mb = 512

.github/pr-deployments/certificate.yaml

@@ -0,0 +1,13 @@
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: pr${PR_NUMBER}-tls
+  namespace: pr-deployment-certs
+spec:
+  secretName: pr${PR_NUMBER}-tls
+  issuerRef:
+    name: letsencrypt
+    kind: ClusterIssuer
+  dnsNames:
+    - "${PR_HOSTNAME}"
+    - "*.${PR_HOSTNAME}"

.github/pr-deployments/rbac.yaml

@@ -0,0 +1,31 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: coder-workspace-pr${PR_NUMBER}
+  namespace: pr${PR_NUMBER}
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: coder-workspace-pr${PR_NUMBER}
+  namespace: pr${PR_NUMBER}
+rules:
+  - apiGroups: ["*"]
+    resources: ["*"]
+    verbs: ["*"]
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: coder-workspace-pr${PR_NUMBER}
+  namespace: pr${PR_NUMBER}
+subjects:
+  - kind: ServiceAccount
+    name: coder-workspace-pr${PR_NUMBER}
+    namespace: pr${PR_NUMBER}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: coder-workspace-pr${PR_NUMBER}

.github/pr-deployments/template/main.tf

@@ -0,0 +1,313 @@
+terraform {
+  required_providers {
+    coder = {
+      source = "coder/coder"
+    }
+    kubernetes = {
+      source = "hashicorp/kubernetes"
+    }
+  }
+}
+
+provider "coder" {
+}
+
+variable "namespace" {
+  type        = string
+  description = "The Kubernetes namespace to create workspaces in (must exist prior to creating workspaces)"
+}
+
+data "coder_parameter" "cpu" {
+  name         = "cpu"
+  display_name = "CPU"
+  description  = "The number of CPU cores"
+  default      = "2"
+  icon         = "/icon/memory.svg"
+  mutable      = true
+  option {
+    name  = "2 Cores"
+    value = "2"
+  }
+  option {
+    name  = "4 Cores"
+    value = "4"
+  }
+  option {
+    name  = "6 Cores"
+    value = "6"
+  }
+  option {
+    name  = "8 Cores"
+    value = "8"
+  }
+}
+
+data "coder_parameter" "memory" {
+  name         = "memory"
+  display_name = "Memory"
+  description  = "The amount of memory in GB"
+  default      = "2"
+  icon         = "/icon/memory.svg"
+  mutable      = true
+  option {
+    name  = "2 GB"
+    value = "2"
+  }
+  option {
+    name  = "4 GB"
+    value = "4"
+  }
+  option {
+    name  = "6 GB"
+    value = "6"
+  }
+  option {
+    name  = "8 GB"
+    value = "8"
+  }
+}
+
+data "coder_parameter" "home_disk_size" {
+  name         = "home_disk_size"
+  display_name = "Home disk size"
+  description  = "The size of the home disk in GB"
+  default      = "10"
+  type         = "number"
+  icon         = "/emojis/1f4be.png"
+  mutable      = false
+  validation {
+    min = 1
+    max = 99999
+  }
+}
+
+provider "kubernetes" {
+  config_path = null
+}
+
+data "coder_workspace" "me" {}
+
+resource "coder_agent" "main" {
+  os             = "linux"
+  arch           = "amd64"
+  startup_script = <<-EOT
+    set -e
+
+    # install and start code-server
+    curl -fsSL https://code-server.dev/install.sh | sh -s -- --method=standalone --prefix=/tmp/code-server
+    /tmp/code-server/bin/code-server --auth none --port 13337 >/tmp/code-server.log 2>&1 &
+
+  EOT
+
+  # The following metadata blocks are optional. They are used to display
+  # information about your workspace in the dashboard. You can remove them
+  # if you don't want to display any information.
+  # For basic resources, you can use the `coder stat` command.
+  # If you need more control, you can write your own script.
+  metadata {
+    display_name = "CPU Usage"
+    key          = "0_cpu_usage"
+    script       = "coder stat cpu"
+    interval     = 10
+    timeout      = 1
+  }
+
+  metadata {
+    display_name = "RAM Usage"
+    key          = "1_ram_usage"
+    script       = "coder stat mem"
+    interval     = 10
+    timeout      = 1
+  }
+
+  metadata {
+    display_name = "Home Disk"
+    key          = "3_home_disk"
+    script       = "coder stat disk --path $${HOME}"
+    interval     = 60
+    timeout      = 1
+  }
+
+  metadata {
+    display_name = "CPU Usage (Host)"
+    key          = "4_cpu_usage_host"
+    script       = "coder stat cpu --host"
+    interval     = 10
+    timeout      = 1
+  }
+
+  metadata {
+    display_name = "Memory Usage (Host)"
+    key          = "5_mem_usage_host"
+    script       = "coder stat mem --host"
+    interval     = 10
+    timeout      = 1
+  }
+
+  metadata {
+    display_name = "Load Average (Host)"
+    key          = "6_load_host"
+    # get load avg scaled by number of cores
+    script   = <<EOT
+      echo "`cat /proc/loadavg | awk '{ print $1 }'` `nproc`" | awk '{ printf "%0.2f", $1/$2 }'
+    EOT
+    interval = 60
+    timeout  = 1
+  }
+}
+
+# code-server
+resource "coder_app" "code-server" {
+  agent_id     = coder_agent.main.id
+  slug         = "code-server"
+  display_name = "code-server"
+  icon         = "/icon/code.svg"
+  url          = "http://localhost:13337?folder=/home/coder"
+  subdomain    = false
+  share        = "owner"
+
+  healthcheck {
+    url       = "http://localhost:13337/healthz"
+    interval  = 3
+    threshold = 10
+  }
+}
+
+resource "kubernetes_persistent_volume_claim" "home" {
+  metadata {
+    name      = "coder-${lower(data.coder_workspace.me.owner)}-${lower(data.coder_workspace.me.name)}-home"
+    namespace = var.namespace
+    labels = {
+      "app.kubernetes.io/name"     = "coder-pvc"
+      "app.kubernetes.io/instance" = "coder-pvc-${lower(data.coder_workspace.me.owner)}-${lower(data.coder_workspace.me.name)}"
+      "app.kubernetes.io/part-of"  = "coder"
+      //Coder-specific labels.
+      "com.coder.resource"       = "true"
+      "com.coder.workspace.id"   = data.coder_workspace.me.id
+      "com.coder.workspace.name" = data.coder_workspace.me.name
+      "com.coder.user.id"        = data.coder_workspace.me.owner_id
+      "com.coder.user.username"  = data.coder_workspace.me.owner
+    }
+    annotations = {
+      "com.coder.user.email" = data.coder_workspace.me.owner_email
+    }
+  }
+  wait_until_bound = false
+  spec {
+    access_modes = ["ReadWriteOnce"]
+    resources {
+      requests = {
+        storage = "${data.coder_parameter.home_disk_size.value}Gi"
+      }
+    }
+  }
+}
+
+resource "kubernetes_deployment" "main" {
+  count = data.coder_workspace.me.start_count
+  depends_on = [
+    kubernetes_persistent_volume_claim.home
+  ]
+  wait_for_rollout = false
+  metadata {
+    name      = "coder-${lower(data.coder_workspace.me.owner)}-${lower(data.coder_workspace.me.name)}"
+    namespace = var.namespace
+    labels = {
+      "app.kubernetes.io/name"     = "coder-workspace"
+      "app.kubernetes.io/instance" = "coder-workspace-${lower(data.coder_workspace.me.owner)}-${lower(data.coder_workspace.me.name)}"
+      "app.kubernetes.io/part-of"  = "coder"
+      "com.coder.resource"         = "true"
+      "com.coder.workspace.id"     = data.coder_workspace.me.id
+      "com.coder.workspace.name"   = data.coder_workspace.me.name
+      "com.coder.user.id"          = data.coder_workspace.me.owner_id
+      "com.coder.user.username"    = data.coder_workspace.me.owner
+    }
+    annotations = {
+      "com.coder.user.email" = data.coder_workspace.me.owner_email
+    }
+  }
+
+  spec {
+    replicas = 1
+    selector {
+      match_labels = {
+        "app.kubernetes.io/name" = "coder-workspace"
+      }
+    }
+    strategy {
+      type = "Recreate"
+    }
+
+    template {
+      metadata {
+        labels = {
+          "app.kubernetes.io/name" = "coder-workspace"
+        }
+      }
+      spec {
+        security_context {
+          run_as_user = 1000
+          fs_group    = 1000
+        }
+
+        service_account_name = "coder-workspace-${var.namespace}"
+        container {
+          name              = "dev"
+          image             = "bencdr/devops-tools"
+          image_pull_policy = "Always"
+          command           = ["sh", "-c", coder_agent.main.init_script]
+          security_context {
+            run_as_user = "1000"
+          }
+          env {
+            name  = "CODER_AGENT_TOKEN"
+            value = coder_agent.main.token
+          }
+          resources {
+            requests = {
+              "cpu"    = "250m"
+              "memory" = "512Mi"
+            }
+            limits = {
+              "cpu"    = "${data.coder_parameter.cpu.value}"
+              "memory" = "${data.coder_parameter.memory.value}Gi"
+            }
+          }
+          volume_mount {
+            mount_path = "/home/coder"
+            name       = "home"
+            read_only  = false
+          }
+        }
+
+        volume {
+          name = "home"
+          persistent_volume_claim {
+            claim_name = kubernetes_persistent_volume_claim.home.metadata.0.name
+            read_only  = false
+          }
+        }
+
+        affinity {
+          // This affinity attempts to spread out all workspace pods evenly across
+          // nodes.
+          pod_anti_affinity {
+            preferred_during_scheduling_ignored_during_execution {
+              weight = 1
+              pod_affinity_term {
+                topology_key = "kubernetes.io/hostname"
+                label_selector {
+                  match_expressions {
+                    key      = "app.kubernetes.io/name"
+                    operator = "In"
+                    values   = ["coder-workspace"]
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}

.github/pr-deployments/values.yaml

@@ -0,0 +1,38 @@
+coder:
+  image:
+    repo: "${REPO}"
+    tag: "pr${PR_NUMBER}"
+    pullPolicy: Always
+  service:
+    type: ClusterIP
+  ingress:
+    enable: true
+    className: traefik
+    host: "${PR_HOSTNAME}"
+    wildcardHost: "*.${PR_HOSTNAME}"
+    tls:
+      enable: true
+      secretName: "pr${PR_NUMBER}-tls"
+      wildcardSecretName: "pr${PR_NUMBER}-tls"
+  env:
+    - name: "CODER_ACCESS_URL"
+      value: "https://${PR_HOSTNAME}"
+    - name: "CODER_WILDCARD_ACCESS_URL"
+      value: "*.${PR_HOSTNAME}"
+    - name: "CODER_EXPERIMENTS"
+      value: "${EXPERIMENTS}"
+    - name: CODER_PG_CONNECTION_URL
+      valueFrom:
+        secretKeyRef:
+          name: coder-db-url
+          key: url
+    - name: "CODER_OAUTH2_GITHUB_ALLOW_SIGNUPS"
+      value: "true"
+    - name: "CODER_OAUTH2_GITHUB_CLIENT_ID"
+      value: "${PR_DEPLOYMENTS_GITHUB_OAUTH_CLIENT_ID}"
+    - name: "CODER_OAUTH2_GITHUB_CLIENT_SECRET"
+      value: "${PR_DEPLOYMENTS_GITHUB_OAUTH_CLIENT_SECRET}"
+    - name: "CODER_OAUTH2_GITHUB_ALLOWED_ORGS"
+      value: "coder"
+    - name: "CODER_DERP_CONFIG_URL"
+      value: "https://controlplane.tailscale.com/derpmap/default"

.github/workflows/ci.yaml

@@ -24,27 +24,29 @@ permissions:
 # additional changes
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
 
 jobs:
   changes:
     runs-on: ubuntu-latest
     outputs:
       docs-only: ${{ steps.filter.outputs.docs_count == steps.filter.outputs.all_count }}
+      docs: ${{ steps.filter.outputs.docs }}
       go: ${{ steps.filter.outputs.go }}
       ts: ${{ steps.filter.outputs.ts }}
       k8s: ${{ steps.filter.outputs.k8s }}
       ci: ${{ steps.filter.outputs.ci }}
+      db: ${{ steps.filter.outputs.db }}
       offlinedocs-only: ${{ steps.filter.outputs.offlinedocs_count == steps.filter.outputs.all_count }}
       offlinedocs: ${{ steps.filter.outputs.offlinedocs }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 1
       # For pull requests it's not necessary to checkout the code
       - name: check changed files
-        uses: dorny/paths-filter@v2
+        uses: dorny/paths-filter@v3
         id: filter
         with:
           filters: |
@@ -53,10 +55,12 @@ jobs:
             docs:
               - "docs/**"
               - "README.md"
-              - "examples/templates/**"
               - "examples/web-server/**"
               - "examples/monitoring/**"
               - "examples/lima/**"
+            db:
+              - "**.sql"
+              - "coderd/database/**"
             go:
               - "**.sql"
               - "**.go"
@@ -110,7 +114,7 @@ jobs:
     runs-on: ${{ github.repository_owner == 'coder' && 'buildjet-8vcpu-ubuntu-2204' || 'ubuntu-latest' }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 1
 
@@ -122,12 +126,13 @@ jobs:
 
       - name: Get golangci-lint cache dir
         run: |
-          go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.53.2
+          linter_ver=$(egrep -o 'GOLANGCI_LINT_VERSION=\S+' dogfood/Dockerfile | cut -d '=' -f 2)
+          go install github.com/golangci/golangci-lint/cmd/golangci-lint@v$linter_ver
           dir=$(golangci-lint cache status | awk '/Dir/ { print $2 }')
           echo "LINT_CACHE_DIR=$dir" >> $GITHUB_ENV
 
       - name: golangci-lint cache
-        uses: buildjet/cache@v3
+        uses: buildjet/cache@v4
         with:
           path: |
             ${{ env.LINT_CACHE_DIR }}
@@ -137,7 +142,7 @@ jobs:
 
       # Check for any typos
       - name: Check for typos
-        uses: crate-ci/typos@v1.16.1
+        uses: crate-ci/typos@v1.18.2
         with:
           config: .github/workflows/typos.toml
 
@@ -165,7 +170,7 @@ jobs:
     if: needs.changes.outputs.docs-only == 'false' || needs.changes.outputs.ci == 'true' || github.ref == 'refs/heads/main'
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 1
 
@@ -184,20 +189,17 @@ jobs:
           go install storj.io/drpc/cmd/protoc-gen-go-drpc@v0.0.33
           go install golang.org/x/tools/cmd/goimports@latest
           go install github.com/mikefarah/yq/v4@v4.30.6
-          go install github.com/golang/mock/mockgen@v1.6.0
+          go install go.uber.org/mock/mockgen@v0.4.0
 
       - name: Install Protoc
         run: |
-          # protoc must be in lockstep with our dogfood Dockerfile or the
-          # version in the comments will differ. This is also defined in
-          # security.yaml
-          set -x
-          cd dogfood
-          DOCKER_BUILDKIT=1 docker build . --target proto -t protoc
-          protoc_path=/usr/local/bin/protoc
-          docker run --rm --entrypoint cat protoc /tmp/bin/protoc > $protoc_path
-          chmod +x $protoc_path
-          protoc --version
+          mkdir -p /tmp/proto
+          pushd /tmp/proto
+          curl -L -o protoc.zip https://github.com/protocolbuffers/protobuf/releases/download/v23.3/protoc-23.3-linux-x86_64.zip
+          unzip protoc.zip
+          cp -r ./bin/* /usr/local/bin
+          cp -r ./include /usr/local/bin/include
+          popd
 
       - name: make gen
         run: "make --output-sync -j -B gen"
@@ -212,7 +214,7 @@ jobs:
     timeout-minutes: 7
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 1
 
@@ -220,14 +222,14 @@ jobs:
         uses: ./.github/actions/setup-node
 
       - name: Setup Go
-        uses: buildjet/setup-go@v4
+        uses: buildjet/setup-go@v5
         with:
           # This doesn't need caching. It's super fast anyways!
           cache: false
-          go-version: 1.20.6
+          go-version: 1.21.5
 
       - name: Install shfmt
-        run: go install mvdan.cc/sh/v3/cmd/shfmt@v3.5.0
+        run: go install mvdan.cc/sh/v3/cmd/shfmt@v3.7.0
 
       - name: make fmt
         run: |
@@ -238,7 +240,7 @@ jobs:
         run: ./scripts/check_unstaged.sh
 
   test-go:
-    runs-on: ${{ matrix.os == 'ubuntu-latest' && github.repository_owner == 'coder' && 'buildjet-4vcpu-ubuntu-2204' || matrix.os == 'macos-latest' && github.repository_owner == 'coder' && 'macos-latest-xl' || matrix.os == 'windows-2019' && github.repository_owner == 'coder' && 'windows-latest-8-cores' || matrix.os }}
+    runs-on: ${{ matrix.os == 'ubuntu-latest' && github.repository_owner == 'coder' && 'buildjet-4vcpu-ubuntu-2204' || matrix.os == 'macos-latest' && github.repository_owner == 'coder' && 'macos-latest-xlarge' || matrix.os == 'windows-2022' && github.repository_owner == 'coder' && 'windows-latest-16-cores' || matrix.os }}
     needs: changes
     if: needs.changes.outputs.go == 'true' || needs.changes.outputs.ci == 'true' || github.ref == 'refs/heads/main'
     timeout-minutes: 20
@@ -248,10 +250,10 @@ jobs:
         os:
           - ubuntu-latest
           - macos-latest
-          - windows-2019
+          - windows-2022
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 1
 
@@ -275,29 +277,36 @@ jobs:
             echo "cover=false" >> $GITHUB_OUTPUT
           fi
 
+          # if macOS, install google-chrome for scaletests. As another concern,
+          # should we really have this kind of external dependency requirement
+          # on standard CI?
+          if [ "${{ matrix.os }}" == "macos-latest" ]; then
+            brew install google-chrome
+          fi
+
           # By default Go will use the number of logical CPUs, which
           # is a fine default.
           PARALLEL_FLAG=""
 
+          # macOS will output "The default interactive shell is now zsh"
+          # intermittently in CI...
+          if [ "${{ matrix.os }}" == "macos-latest" ]; then
+            touch ~/.bash_profile && echo "export BASH_SILENCE_DEPRECATION_WARNING=1" >> ~/.bash_profile
+          fi
           export TS_DEBUG_DISCO=true
           gotestsum --junitfile="gotests.xml" --jsonfile="gotests.json" \
             --packages="./..." -- $PARALLEL_FLAG -short -failfast $COVERAGE_FLAGS
 
-      - name: Print test stats
-        if: success() || failure()
-        run: |
-          # Artifacts are not available after rerunning a job,
-          # so we need to print the test stats to the log.
-          go run ./scripts/ci-report/main.go gotests.json | tee gotests_stats.json
-
       - name: Upload test stats to Datadog
+        timeout-minutes: 1
+        continue-on-error: true
         uses: ./.github/actions/upload-datadog
         if: success() || failure()
         with:
           api-key: ${{ secrets.DATADOG_API_KEY }}
 
       - name: Check code coverage
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
         # This action has a tendency to error out unexpectedly, it has
         # the `fail_ci_if_error` option that defaults to `false`, but
         # that is no guarantee, see:
@@ -311,7 +320,8 @@ jobs:
 
   test-go-pg:
     runs-on: ${{ github.repository_owner == 'coder' && 'buildjet-8vcpu-ubuntu-2204' || 'ubuntu-latest' }}
-    needs: changes
+    needs:
+      - changes
     if: needs.changes.outputs.go == 'true' || needs.changes.outputs.ci == 'true' || github.ref == 'refs/heads/main'
     # This timeout must be greater than the timeout set by `go test` in
     # `make test-postgres` to ensure we receive a trace of running
@@ -320,7 +330,7 @@ jobs:
     timeout-minutes: 25
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 1
 
@@ -335,21 +345,16 @@ jobs:
           export TS_DEBUG_DISCO=true
           make test-postgres
 
-      - name: Print test stats
-        if: success() || failure()
-        run: |
-          # Artifacts are not available after rerunning a job,
-          # so we need to print the test stats to the log.
-          go run ./scripts/ci-report/main.go gotests.json | tee gotests_stats.json
-
       - name: Upload test stats to Datadog
+        timeout-minutes: 1
+        continue-on-error: true
         uses: ./.github/actions/upload-datadog
         if: success() || failure()
         with:
           api-key: ${{ secrets.DATADOG_API_KEY }}
 
       - name: Check code coverage
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
         # This action has a tendency to error out unexpectedly, it has
         # the `fail_ci_if_error` option that defaults to `false`, but
         # that is no guarantee, see:
@@ -368,7 +373,7 @@ jobs:
     timeout-minutes: 25
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 1
 
@@ -383,105 +388,13 @@ jobs:
           gotestsum --junitfile="gotests.xml" -- -race ./...
 
       - name: Upload test stats to Datadog
+        timeout-minutes: 1
+        continue-on-error: true
         uses: ./.github/actions/upload-datadog
         if: always()
         with:
           api-key: ${{ secrets.DATADOG_API_KEY }}
 
-  deploy:
-    name: "deploy"
-    runs-on: ${{ github.repository_owner == 'coder' && 'buildjet-8vcpu-ubuntu-2204' || 'ubuntu-latest' }}
-    timeout-minutes: 30
-    needs: changes
-    if: |
-      github.ref == 'refs/heads/main' && !github.event.pull_request.head.repo.fork
-      && needs.changes.outputs.docs-only == 'false'
-    permissions:
-      contents: read
-      id-token: write
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-
-      - name: Authenticate to Google Cloud
-        uses: google-github-actions/auth@v1
-        with:
-          workload_identity_provider: projects/573722524737/locations/global/workloadIdentityPools/github/providers/github
-          service_account: coder-ci@coder-dogfood.iam.gserviceaccount.com
-
-      - name: Set up Google Cloud SDK
-        uses: google-github-actions/setup-gcloud@v1
-
-      - name: Setup Node
-        uses: ./.github/actions/setup-node
-
-      - name: Setup Go
-        uses: ./.github/actions/setup-go
-
-      - name: Install goimports
-        run: go install golang.org/x/tools/cmd/goimports@latest
-      - name: Install nfpm
-        run: go install github.com/goreleaser/nfpm/v2/cmd/nfpm@v2.16.0
-
-      - name: Install zstd
-        run: sudo apt-get install -y zstd
-
-      - name: Build Release
-        run: |
-          set -euo pipefail
-          go mod download
-
-          version="$(./scripts/version.sh)"
-          make gen/mark-fresh
-          make -j \
-            build/coder_"$version"_windows_amd64.zip \
-            build/coder_"$version"_linux_amd64.{tar.gz,deb}
-
-      - name: Install Release
-        run: |
-          set -euo pipefail
-
-          regions=(
-            # gcp-region-id instance-name systemd-service-name
-            "us-central1-a coder coder"
-            "australia-southeast1-b coder-sydney coder-workspace-proxy"
-            "europe-west3-c coder-europe coder-workspace-proxy"
-            "southamerica-east1-b coder-brazil coder-workspace-proxy"
-          )
-
-          deb_pkg="./build/coder_$(./scripts/version.sh)_linux_amd64.deb"
-          if [ ! -f "$deb_pkg" ]; then
-            echo "deb package not found: $deb_pkg"
-            ls -l ./build
-            exit 1
-          fi
-
-          gcloud config set project coder-dogfood
-          for region in "${regions[@]}"; do
-            echo "::group::$region"
-            set -- $region
-
-            set -x
-            gcloud config set compute/zone "$1"
-            gcloud compute scp "$deb_pkg" "${2}:/tmp/coder.deb"
-            gcloud compute ssh "$2" -- /bin/sh -c "set -eux; sudo dpkg -i --force-confdef /tmp/coder.deb; sudo systemctl daemon-reload; sudo service '$3' restart"
-            set +x
-
-            echo "::endgroup::"
-          done
-
-      - name: Upload build artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: coder
-          path: |
-            ./build/*.zip
-            ./build/*.tar.gz
-            ./build/*.deb
-          retention-days: 7
-
   test-js:
     runs-on: ${{ github.repository_owner == 'coder' && 'buildjet-8vcpu-ubuntu-2204' || 'ubuntu-latest' }}
     needs: changes
@@ -489,7 +402,7 @@ jobs:
     timeout-minutes: 20
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 1
 
@@ -500,7 +413,7 @@ jobs:
         working-directory: site
 
       - name: Check code coverage
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
         # This action has a tendency to error out unexpectedly, it has
         # the `fail_ci_if_error` option that defaults to `false`, but
         # that is no guarantee, see:
@@ -513,13 +426,13 @@ jobs:
           flags: unittest-js
 
   test-e2e:
-    runs-on: ${{ github.repository_owner == 'coder' && 'buildjet-8vcpu-ubuntu-2204' || 'ubuntu-latest' }}
+    runs-on: ${{ github.repository_owner == 'coder' && 'buildjet-16vcpu-ubuntu-2204' || 'ubuntu-latest' }}
     needs: changes
     if: needs.changes.outputs.go == 'true' || needs.changes.outputs.ts == 'true' || needs.changes.outputs.ci == 'true' || github.ref == 'refs/heads/main'
     timeout-minutes: 20
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 1
 
@@ -532,6 +445,24 @@ jobs:
       - name: Setup Terraform
         uses: ./.github/actions/setup-tf
 
+      - name: go install tools
+        run: |
+          go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.30
+          go install storj.io/drpc/cmd/protoc-gen-go-drpc@v0.0.33
+          go install golang.org/x/tools/cmd/goimports@latest
+          go install github.com/mikefarah/yq/v4@v4.30.6
+          go install go.uber.org/mock/mockgen@v0.4.0
+
+      - name: Install Protoc
+        run: |
+          mkdir -p /tmp/proto
+          pushd /tmp/proto
+          curl -L -o protoc.zip https://github.com/protocolbuffers/protobuf/releases/download/v23.3/protoc-23.3-linux-x86_64.zip
+          unzip protoc.zip
+          cp -r ./bin/* /usr/local/bin
+          cp -r ./include /usr/local/bin/include
+          popd
+
       - name: Build
         run: |
           make -B site/out/index.html
@@ -539,19 +470,27 @@ jobs:
       - run: pnpm playwright:install
         working-directory: site
 
-      - run: pnpm playwright:test
+      - run: pnpm playwright:test --workers 1
         env:
           DEBUG: pw:api
         working-directory: site
 
       - name: Upload Playwright Failed Tests
         if: always() && github.actor != 'dependabot[bot]' && runner.os == 'Linux' && !github.event.pull_request.head.repo.fork
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: failed-test-videos
           path: ./site/test-results/**/*.webm
           retention-days: 7
 
+      - name: Upload pprof dumps
+        if: always() && github.actor != 'dependabot[bot]' && runner.os == 'Linux' && !github.event.pull_request.head.repo.fork
+        uses: actions/upload-artifact@v4
+        with:
+          name: debug-pprof-dumps
+          path: ./site/test-results/**/debug-pprof-*.txt
+          retention-days: 7
+
   chromatic:
     # REMARK: this is only used to build storybook and deploy it to Chromatic.
     runs-on: ubuntu-latest
@@ -559,7 +498,7 @@ jobs:
     if: needs.changes.outputs.ts == 'true' || needs.changes.outputs.ci == 'true'
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           # Required by Chromatic for build-over-build history, otherwise we
           # only get 1 commit on shallow checkout.
@@ -573,12 +512,13 @@ jobs:
       # the check to pass. This is desired in PRs, but not in mainline.
       - name: Publish to Chromatic (non-mainline)
         if: github.ref != 'refs/heads/main' && github.repository_owner == 'coder'
-        uses: chromaui/action@v1
+        uses: chromaui/action@v10
         env:
           NODE_OPTIONS: "--max_old_space_size=4096"
           STORYBOOK: true
         with:
-          buildScriptName: "storybook:build"
+          # Do a fast, testing build for change previews
+          buildScriptName: "storybook:ci"
           exitOnceUploaded: true
           # This will prevent CI from failing when Chromatic detects visual changes
           exitZeroOnChanges: true
@@ -586,11 +526,14 @@ jobs:
           # https://www.chromatic.com/docs/github-actions#forked-repositories
           projectToken: 695c25b6cb65
           workingDir: "./site"
+          storybookBaseDir: "./site"
           # Prevent excessive build runs on minor version changes
           skip: "@(renovate/**|dependabot/**)"
           # Run TurboSnap to trace file dependencies to related stories
           # and tell chromatic to only take snapshots of relevent stories
           onlyChanged: true
+          # Avoid uploading single files, because that's very slow
+          zip: true
 
       # This is a separate step for mainline only that auto accepts and changes
       # instead of holding CI up. Since we squash/merge, this is defensive to
@@ -600,7 +543,7 @@ jobs:
       # infinitely "in progress" in mainline unless we re-review each build.
       - name: Publish to Chromatic (mainline)
         if: github.ref == 'refs/heads/main' && github.repository_owner == 'coder'
-        uses: chromaui/action@v1
+        uses: chromaui/action@v10
         env:
           NODE_OPTIONS: "--max_old_space_size=4096"
           STORYBOOK: true
@@ -608,21 +551,26 @@ jobs:
           autoAcceptChanges: true
           # This will prevent CI from failing when Chromatic detects visual changes
           exitZeroOnChanges: true
+          # Do a full build with documentation for mainline builds
           buildScriptName: "storybook:build"
           projectToken: 695c25b6cb65
           workingDir: "./site"
+          storybookBaseDir: "./site"
           # Run TurboSnap to trace file dependencies to related stories
           # and tell chromatic to only take snapshots of relevent stories
           onlyChanged: true
+          # Avoid uploading single files, because that's very slow
+          zip: true
 
   offlinedocs:
     name: offlinedocs
     needs: changes
     runs-on: ${{ github.repository_owner == 'coder' && 'buildjet-8vcpu-ubuntu-2204' || 'ubuntu-latest' }}
-    if: needs.changes.outputs.offlinedocs == 'true' || needs.changes.outputs.ci == 'true'
+    if: needs.changes.outputs.offlinedocs == 'true' || needs.changes.outputs.ci == 'true' || needs.changes.outputs.docs == 'true'
+
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           # 0 is required here for version.sh to work.
           fetch-depth: 0
@@ -632,17 +580,29 @@ jobs:
         with:
           directory: offlinedocs
 
+      - name: Install Protoc
+        run: |
+          mkdir -p /tmp/proto
+          pushd /tmp/proto
+          curl -L -o protoc.zip https://github.com/protocolbuffers/protobuf/releases/download/v23.3/protoc-23.3-linux-x86_64.zip
+          unzip protoc.zip
+          cp -r ./bin/* /usr/local/bin
+          cp -r ./include /usr/local/bin/include
+          popd
+
       - name: Setup Go
         uses: ./.github/actions/setup-go
 
       - name: Install go tools
         run: |
-          go install github.com/golang/mock/mockgen@v1.6.0
+          go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.30
+          go install storj.io/drpc/cmd/protoc-gen-go-drpc@v0.0.33
+          go install golang.org/x/tools/cmd/goimports@latest
+          go install github.com/mikefarah/yq/v4@v4.30.6
+          go install go.uber.org/mock/mockgen@v0.4.0
 
       - name: Setup sqlc
-        uses: sqlc-dev/setup-sqlc@v3
-        with:
-          sqlc-version: "1.19.1"
+        uses: ./.github/actions/setup-sqlc
 
       - name: Format
         run: |
@@ -668,7 +628,9 @@ jobs:
       - test-go-pg
       - test-go-race
       - test-js
+      - test-e2e
       - offlinedocs
+      - sqlc-vet
     # Allow this job to run even if the needed jobs fail, are skipped or
     # cancelled.
     if: always()
@@ -683,6 +645,8 @@ jobs:
           echo "- test-go-pg: ${{ needs.test-go-pg.result }}"
           echo "- test-go-race: ${{ needs.test-go-race.result }}"
           echo "- test-js: ${{ needs.test-js.result }}"
+          echo "- test-e2e: ${{ needs.test-e2e.result }}"
+          echo "- offlinedocs: ${{ needs.offlinedocs.result }}"
           echo
 
           # We allow skipped jobs to pass, but not failed or cancelled jobs.
@@ -693,48 +657,233 @@ jobs:
 
           echo "Required checks have passed"
 
-  build-main-image:
-    # This build and publihes ghcr.io/coder/coder-preview:main for each merge commit to main branch.
-    # We are only building this for amd64 plateform. (>95% pulls are for amd64)
+  build:
+    # This builds and publishes ghcr.io/coder/coder-preview:main for each commit
+    # to main branch. We are only building this for amd64 platform. (>95% pulls
+    # are for amd64)
     needs: changes
-    if: github.ref == 'refs/heads/main' && needs.changes.outputs.docs-only == 'false'
+    if: needs.changes.outputs.docs-only == 'false' && !github.event.pull_request.head.repo.fork
     runs-on: ${{ github.repository_owner == 'coder' && 'buildjet-8vcpu-ubuntu-2204' || 'ubuntu-latest' }}
     env:
       DOCKER_CLI_EXPERIMENTAL: "enabled"
+    outputs:
+      IMAGE: ghcr.io/coder/coder-preview:${{ steps.build-docker.outputs.tag }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
+      - name: GHCR Login
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
       - name: Setup Node
         uses: ./.github/actions/setup-node
 
       - name: Setup Go
         uses: ./.github/actions/setup-go
 
-      - name: Setup sqlc
-        uses: ./.github/actions/setup-sqlc
+      - name: Install nfpm
+        run: go install github.com/goreleaser/nfpm/v2/cmd/nfpm@v2.35.1
 
-      - name: GHCR Login
-        uses: docker/login-action@v2
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Install zstd
+        run: sudo apt-get install -y zstd
 
-      - name: Build and push Linux amd64 Docker image
+      - name: Build
         run: |
           set -euxo pipefail
           go mod download
-          make gen/mark-fresh
-          export DOCKER_IMAGE_NO_PREREQUISITES=true
+
           version="$(./scripts/version.sh)"
-          export CODER_IMAGE_BUILD_BASE_TAG="$(CODER_IMAGE_BASE=coder-base ./scripts/image_tag.sh --version "$version")"
-          make -j build/coder_linux_amd64
-          ./scripts/build_docker.sh \
-            --arch amd64 \
-            --target ghcr.io/coder/coder-preview:main \
-            --version $version \
-            --push \
-            build/coder_linux_amd64
+          tag="main-$(echo "$version" | sed 's/+/-/g')"
+          echo "tag=$tag" >> $GITHUB_OUTPUT
+
+          make gen/mark-fresh
+          make -j \
+            build/coder_linux_{amd64,arm64,armv7} \
+            build/coder_"$version"_windows_amd64.zip \
+            build/coder_"$version"_linux_amd64.{tar.gz,deb}
+
+      - name: Build Linux Docker images
+        id: build-docker
+        env:
+          CODER_IMAGE_BASE: ghcr.io/coder/coder-preview
+          CODER_IMAGE_TAG_PREFIX: main
+          DOCKER_CLI_EXPERIMENTAL: "enabled"
+        run: |
+          set -euxo pipefail
+
+          # build Docker images for each architecture
+          version="$(./scripts/version.sh)"
+          tag="main-$(echo "$version" | sed 's/+/-/g')"
+          echo "tag=$tag" >> $GITHUB_OUTPUT
+
+          # build images for each architecture
+          make -j build/coder_"$version"_linux_{amd64,arm64,armv7}.tag
+
+          # only push if we are on main branch
+          if [ "${{ github.ref }}" == "refs/heads/main" ]; then
+            # build and push multi-arch manifest, this depends on the other images
+            # being pushed so will automatically push them
+            make -j push/build/coder_"$version"_linux_{amd64,arm64,armv7}.tag
+
+            # Define specific tags
+            tags=("$tag" "main" "latest")
+
+            # Create and push a multi-arch manifest for each tag
+            # we are adding `latest` tag and keeping `main` for backward
+            # compatibality
+            for t in "${tags[@]}"; do
+                ./scripts/build_docker_multiarch.sh \
+                    --push \
+                    --target "ghcr.io/coder/coder-preview:$t" \
+                    --version $version \
+                    $(cat build/coder_"$version"_linux_{amd64,arm64,armv7}.tag)
+            done
+          fi
+
+      - name: Prune old images
+        if: github.ref == 'refs/heads/main'
+        uses: vlaurin/action-ghcr-prune@v0.6.0
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          organization: coder
+          container: coder-preview
+          keep-younger-than: 7 # days
+          keep-tags: latest
+          keep-tags-regexes: ^pr
+          prune-tags-regexes: |
+            ^main-
+            ^v
+          prune-untagged: true
+
+      - name: Upload build artifacts
+        if: github.ref == 'refs/heads/main'
+        uses: actions/upload-artifact@v4
+        with:
+          name: coder
+          path: |
+            ./build/*.zip
+            ./build/*.tar.gz
+            ./build/*.deb
+          retention-days: 7
+
+  deploy:
+    name: "deploy"
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    needs:
+      - changes
+      - build
+    if: |
+      github.ref == 'refs/heads/main' && !github.event.pull_request.head.repo.fork
+      && needs.changes.outputs.docs-only == 'false'
+    permissions:
+      contents: read
+      id-token: write
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Authenticate to Google Cloud
+        uses: google-github-actions/auth@v2
+        with:
+          workload_identity_provider: projects/573722524737/locations/global/workloadIdentityPools/github/providers/github
+          service_account: coder-ci@coder-dogfood.iam.gserviceaccount.com
+
+      - name: Set up Google Cloud SDK
+        uses: google-github-actions/setup-gcloud@v2
+
+      - name: Set up Flux CLI
+        uses: fluxcd/flux2/action@main
+        with:
+          # Keep this up to date with the version of flux installed in dogfood cluster
+          version: "2.2.1"
+
+      - name: Get Cluster Credentials
+        uses: "google-github-actions/get-gke-credentials@v2"
+        with:
+          cluster_name: dogfood-v2
+          location: us-central1-a
+          project_id: coder-dogfood-v2
+
+      - name: Reconcile Flux
+        run: |
+          set -euxo pipefail
+          flux --namespace flux-system reconcile source git flux-system
+          flux --namespace flux-system reconcile source git coder-main
+          flux --namespace flux-system reconcile kustomization flux-system
+          flux --namespace flux-system reconcile kustomization coder
+          flux --namespace flux-system reconcile source chart coder-coder
+          flux --namespace flux-system reconcile source chart coder-coder-provisioner
+          flux --namespace coder reconcile helmrelease coder
+          flux --namespace coder reconcile helmrelease coder-provisioner
+
+      # Just updating Flux is usually not enough. The Helm release may get
+      # redeployed, but unless something causes the Deployment to update the
+      # pods won't be recreated. It's important that the pods get recreated,
+      # since we use `imagePullPolicy: Always` to ensure we're running the
+      # latest image.
+      - name: Rollout Deployment
+        run: |
+          set -euxo pipefail
+          kubectl --namespace coder rollout restart deployment/coder
+          kubectl --namespace coder rollout status deployment/coder
+          kubectl --namespace coder rollout restart deployment/coder-provisioner
+          kubectl --namespace coder rollout status deployment/coder-provisioner
+
+  deploy-wsproxies:
+    runs-on: ubuntu-latest
+    needs: build
+    if: github.ref == 'refs/heads/main' && !github.event.pull_request.head.repo.fork
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Setup flyctl
+        uses: superfly/flyctl-actions/setup-flyctl@master
+
+      - name: Deploy workspace proxies
+        run: |
+          flyctl deploy --image "$IMAGE" --app paris-coder --config ./.github/fly-wsproxies/paris-coder.toml --env "CODER_PROXY_SESSION_TOKEN=$TOKEN_PARIS" --yes
+          flyctl deploy --image "$IMAGE" --app sydney-coder --config ./.github/fly-wsproxies/sydney-coder.toml --env "CODER_PROXY_SESSION_TOKEN=$TOKEN_SYDNEY" --yes
+          flyctl deploy --image "$IMAGE" --app sao-paulo-coder --config ./.github/fly-wsproxies/sao-paulo-coder.toml --env "CODER_PROXY_SESSION_TOKEN=$TOKEN_SAO_PAULO" --yes
+          flyctl deploy --image "$IMAGE" --app jnb-coder --config ./.github/fly-wsproxies/jnb-coder.toml --env "CODER_PROXY_SESSION_TOKEN=$TOKEN_JNB" --yes
+        env:
+          FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
+          IMAGE: ${{ needs.build.outputs.IMAGE }}
+          TOKEN_PARIS: ${{ secrets.FLY_PARIS_CODER_PROXY_SESSION_TOKEN }}
+          TOKEN_SYDNEY: ${{ secrets.FLY_SYDNEY_CODER_PROXY_SESSION_TOKEN }}
+          TOKEN_SAO_PAULO: ${{ secrets.FLY_SAO_PAULO_CODER_PROXY_SESSION_TOKEN }}
+          TOKEN_JNB: ${{ secrets.FLY_JNB_CODER_PROXY_SESSION_TOKEN }}
+
+  # sqlc-vet runs a postgres docker container, runs Coder migrations, and then
+  # runs sqlc-vet to ensure all queries are valid. This catches any mistakes
+  # in migrations or sqlc queries that makes a query unable to be prepared.
+  sqlc-vet:
+    runs-on: ${{ github.repository_owner == 'coder' && 'buildjet-8vcpu-ubuntu-2204' || 'ubuntu-latest' }}
+    needs: changes
+    if: needs.changes.outputs.db == 'true' || needs.changes.outputs.ci == 'true' || github.ref == 'refs/heads/main'
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+      # We need golang to run the migration main.go
+      - name: Setup Go
+        uses: ./.github/actions/setup-go
+
+      - name: Setup sqlc
+        uses: ./.github/actions/setup-sqlc
+
+      - name: Setup and run sqlc vet
+        run: |
+          make sqlc-vet

.github/workflows/contrib.yaml

@@ -26,7 +26,7 @@ jobs:
       pull-requests: write
     steps:
       - name: auto-approve dependabot
-        uses: hmarr/auto-approve-action@v3
+        uses: hmarr/auto-approve-action@v4
         if: github.actor == 'dependabot[bot]'
 
   cla:
@@ -34,7 +34,7 @@ jobs:
     steps:
       - name: cla
         if: (github.event.comment.body == 'recheck' || github.event.comment.body == 'I have read the CLA Document and I hereby sign the CLA') || github.event_name == 'pull_request_target'
-        uses: contributor-assistant/github-action@v2.3.0
+        uses: contributor-assistant/github-action@v2.3.1
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           # the below token should have repo scope and must be manually added by you in the repository's secret
@@ -46,7 +46,8 @@ jobs:
           path-to-document: "https://github.com/coder/cla/blob/main/README.md"
           # branch should not be protected
           branch: "main"
-          allowlist: dependabot*
+          # Some users have signed a corporate CLA with Coder so are exempt from signing our community one.
+          allowlist: "coryb,aaronlehmann,dependabot*"
 
   release-labels:
     runs-on: ubuntu-latest
@@ -54,7 +55,7 @@ jobs:
     if: ${{ github.event_name == 'pull_request_target' && success() && !github.event.pull_request.draft }}
     steps:
       - name: release-labels
-        uses: actions/github-script@v6
+        uses: actions/github-script@v7
         with:
           # This script ensures PR title and labels are in sync:
           #

.github/workflows/docker-base.yaml

@@ -32,10 +32,10 @@ jobs:
     if: github.repository_owner == 'coder'
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Docker login
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.actor }}

.github/workflows/dogfood.yaml

@@ -7,20 +7,26 @@ on:
     paths:
       - "dogfood/**"
       - ".github/workflows/dogfood.yaml"
-  # Uncomment these lines when testing with CI.
-  # pull_request:
-  #   paths:
-  #     - "dogfood/**"
-  #     - ".github/workflows/dogfood.yaml"
+      - "flake.lock"
+      - "flake.nix"
+  pull_request:
+    paths:
+      - "dogfood/**"
+      - ".github/workflows/dogfood.yaml"
+      - "flake.lock"
+      - "flake.nix"
   workflow_dispatch:
 
 jobs:
-  deploy_image:
-    runs-on: buildjet-4vcpu-ubuntu-2204
+  build_image:
+    runs-on: ubuntu-latest
     steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
       - name: Get branch name
         id: branch-name
-        uses: tj-actions/branch-names@v6.5
+        uses: tj-actions/branch-names@v8
 
       - name: "Branch name to Docker tag name"
         id: docker-tag-name
@@ -30,49 +36,78 @@ jobs:
           tag=${tag//\//--}
           echo "tag=${tag}" >> $GITHUB_OUTPUT
 
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
+      - name: Set up Depot CLI
+        uses: depot/setup-action@v1
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
 
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        if: github.ref == 'refs/heads/main'
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_PASSWORD }}
 
-      - name: Build and push
-        uses: docker/build-push-action@v4
+      - name: Build and push Non-Nix image
+        uses: depot/build-push-action@v1
         with:
+          project: b4q6ltmpzh
+          token: ${{ secrets.DEPOT_TOKEN }}
+          buildx-fallback: true
           context: "{{defaultContext}}:dogfood"
           pull: true
-          push: true
+          save: true
+          push: ${{ github.ref == 'refs/heads/main' }}
           tags: "codercom/oss-dogfood:${{ steps.docker-tag-name.outputs.tag }},codercom/oss-dogfood:latest"
-          cache-from: type=registry,ref=codercom/oss-dogfood:latest
-          cache-to: type=inline
+
+      - name: Build and push Nix image
+        uses: depot/build-push-action@v1
+        with:
+          project: b4q6ltmpzh
+          token: ${{ secrets.DEPOT_TOKEN }}
+          buildx-fallback: true
+          context: "."
+          file: "dogfood/Dockerfile.nix"
+          pull: true
+          save: true
+          push: ${{ github.ref == 'refs/heads/main' }}
+          tags: "codercom/oss-dogfood-nix:${{ steps.docker-tag-name.outputs.tag }},codercom/oss-dogfood-nix:latest"
 
   deploy_template:
-    needs: deploy_image
+    needs: build_image
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
+
+      - name: Setup Terraform
+        uses: ./.github/actions/setup-tf
+
+      - name: Terraform init and validate
+        run: |
+          cd dogfood
+          terraform init -upgrade
+          terraform validate
 
       - name: Get short commit SHA
+        if: github.ref == 'refs/heads/main'
         id: vars
         run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
 
       - name: Get latest commit title
+        if: github.ref == 'refs/heads/main'
         id: message
         run: echo "pr_title=$(git log --format=%s -n 1 ${{ github.sha }})" >> $GITHUB_OUTPUT
 
       - name: "Get latest Coder binary from the server"
+        if: github.ref == 'refs/heads/main'
         run: |
           curl -fsSL "https://dev.coder.com/bin/coder-linux-amd64" -o "./coder"
           chmod +x "./coder"
 
       - name: "Push template"
+        if: github.ref == 'refs/heads/main'
         run: |
           ./coder templates push $CODER_TEMPLATE_NAME --directory $CODER_TEMPLATE_DIR --yes --name=$CODER_TEMPLATE_VERSION --message="$CODER_TEMPLATE_MESSAGE"
         env:

.github/workflows/nightly-gauntlet.yaml

@@ -17,7 +17,7 @@ jobs:
     timeout-minutes: 240
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Setup Go
         uses: ./.github/actions/setup-go
@@ -44,7 +44,7 @@ jobs:
     timeout-minutes: 10
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Setup Go
         uses: ./.github/actions/setup-go

.github/workflows/pr-auto-assign.yaml

@@ -14,4 +14,4 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Assign author
-        uses: toshimaru/auto-author-assign@v1.6.2
+        uses: toshimaru/auto-author-assign@v2.1.0

.github/workflows/pr-cleanup.yaml

@@ -1,4 +1,4 @@
-name: Cleanup PR deployment and image
+name: pr-cleanup
 on:
   pull_request:
     types: closed
@@ -35,14 +35,14 @@ jobs:
 
       - name: Set up kubeconfig
         run: |
-          set -euxo pipefail
+          set -euo pipefail
           mkdir -p ~/.kube
           echo "${{ secrets.PR_DEPLOYMENTS_KUBECONFIG }}" > ~/.kube/config
           export KUBECONFIG=~/.kube/config
 
       - name: Delete helm release
         run: |
-          set -euxo pipefail
+          set -euo pipefail
           helm delete --namespace "pr${{ steps.pr_number.outputs.PR_NUMBER }}" "pr${{ steps.pr_number.outputs.PR_NUMBER }}" || echo "helm release not found"
 
       - name: "Remove PR namespace"
@@ -51,7 +51,7 @@ jobs:
 
       - name: "Remove DNS records"
         run: |
-          set -euxo pipefail
+          set -euo pipefail
           # Get identifier for the record
           record_id=$(curl -X GET "https://api.cloudflare.com/client/v4/zones/${{ secrets.PR_DEPLOYMENTS_ZONE_ID }}/dns_records?name=%2A.pr${{ steps.pr_number.outputs.PR_NUMBER }}.${{ secrets.PR_DEPLOYMENTS_DOMAIN }}" \
           -H "Authorization: Bearer ${{ secrets.PR_DEPLOYMENTS_CLOUDFLARE_API_TOKEN }}" \

.github/workflows/pr-deploy.yaml

@@ -4,24 +4,26 @@
 # 3. when a PR is updated
 name: Deploy PR
 on:
-  pull_request:
-    types: synchronize
+  push:
+    branches-ignore:
+      - main
   workflow_dispatch:
     inputs:
-      pr_number:
-        description: "PR number"
-        type: number
-        required: true
-      skip_build:
-        description: "Skip build job"
-        required: false
-        type: boolean
-        default: false
       experiments:
         description: "Experiments to enable"
         required: false
         type: string
         default: "*"
+      build:
+        description: "Force new build"
+        required: false
+        type: boolean
+        default: false
+      deploy:
+        description: "Force new deployment"
+        required: false
+        type: boolean
+        default: false
 
 env:
   REPO: ghcr.io/coder/coder-preview
@@ -29,43 +31,66 @@ env:
 permissions:
   contents: read
   packages: write
-  pull-requests: write
-
-concurrency:
-  group: ${{ github.workflow }}-PR-${{ github.event.pull_request.number || github.event.inputs.pr_number }}
-  cancel-in-progress: true
+  pull-requests: write # needed for commenting on PRs
 
 jobs:
+  check_pr:
+    runs-on: ubuntu-latest
+    outputs:
+      PR_OPEN: ${{ steps.check_pr.outputs.pr_open }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Check if PR is open
+        id: check_pr
+        run: |
+          set -euo pipefail
+          pr_open=true
+          if [[ "$(gh pr view --json state | jq -r '.state')" != "OPEN" ]]; then
+            echo "PR doesn't exist or is closed."
+            pr_open=false
+          fi
+          echo "pr_open=$pr_open" >> $GITHUB_OUTPUT
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
   get_info:
-    if: github.event_name == 'workflow_dispatch' || github.event_name == 'pull_request'
+    needs: check_pr
+    if: ${{ needs.check_pr.outputs.PR_OPEN == 'true' }}
     outputs:
       PR_NUMBER: ${{ steps.pr_info.outputs.PR_NUMBER }}
       PR_TITLE: ${{ steps.pr_info.outputs.PR_TITLE }}
       PR_URL: ${{ steps.pr_info.outputs.PR_URL }}
-      PR_BRANCH: ${{ steps.pr_info.outputs.PR_BRANCH }}
       CODER_BASE_IMAGE_TAG: ${{ steps.set_tags.outputs.CODER_BASE_IMAGE_TAG }}
       CODER_IMAGE_TAG: ${{ steps.set_tags.outputs.CODER_IMAGE_TAG }}
-      NEW: ${{ steps.check_deployment.outputs.new }}
-      BUILD: ${{ steps.filter.outputs.all_count > steps.filter.outputs.ignored_count || steps.check_deployment.outputs.new }}
+      NEW: ${{ steps.check_deployment.outputs.NEW }}
+      BUILD: ${{ steps.build_conditionals.outputs.first_or_force_build == 'true' || steps.build_conditionals.outputs.automatic_rebuild == 'true' }}
 
     runs-on: "ubuntu-latest"
     steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
       - name: Get PR number, title, and branch name
         id: pr_info
         run: |
-          set -euxo pipefail
-          PR_NUMBER=${{ github.event.inputs.pr_number || github.event.pull_request.number }}
-          PR_TITLE=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" https://api.github.com/repos/coder/coder/pulls/$PR_NUMBER | jq -r '.title')
-          PR_BRANCH=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" https://api.github.com/repos/coder/coder/pulls/$PR_NUMBER | jq -r '.head.ref')
-          echo "PR_URL=https://github.com/coder/coder/pull/$PR_NUMBER" >> $GITHUB_OUTPUT
+          set -euo pipefail
+          PR_NUMBER=$(gh pr view --json number | jq -r '.number')
+          PR_TITLE=$(gh pr view --json title | jq -r '.title')
+          PR_URL=$(gh pr view --json url | jq -r '.url')
+          echo "PR_URL=$PR_URL" >> $GITHUB_OUTPUT
           echo "PR_NUMBER=$PR_NUMBER" >> $GITHUB_OUTPUT
           echo "PR_TITLE=$PR_TITLE" >> $GITHUB_OUTPUT
-          echo "PR_BRANCH=$PR_BRANCH" >> $GITHUB_OUTPUT
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Set required tags
         id: set_tags
         run: |
-          set -euxo pipefail
+          set -euo pipefail
           echo "CODER_BASE_IMAGE_TAG=$CODER_BASE_IMAGE_TAG" >> $GITHUB_OUTPUT
           echo "CODER_IMAGE_TAG=$CODER_IMAGE_TAG" >> $GITHUB_OUTPUT
         env:
@@ -74,61 +99,30 @@ jobs:
 
       - name: Set up kubeconfig
         run: |
-          set -euxo pipefail
+          set -euo pipefail
           mkdir -p ~/.kube
           echo "${{ secrets.PR_DEPLOYMENTS_KUBECONFIG }}" > ~/.kube/config
+          chmod 644 ~/.kube/config
           export KUBECONFIG=~/.kube/config
 
       - name: Check if the helm deployment already exists
         id: check_deployment
         run: |
-          set -euxo pipefail
+          set -euo pipefail
           if helm status "pr${{ steps.pr_info.outputs.PR_NUMBER }}" --namespace "pr${{ steps.pr_info.outputs.PR_NUMBER }}" > /dev/null 2>&1; then
             echo "Deployment already exists. Skipping deployment."
-            new=false
+            NEW=false
           else
             echo "Deployment doesn't exist."
-            new=true
+            NEW=true
           fi
-          echo "new=$new" >> $GITHUB_OUTPUT
-
-      - name: Find Comment
-        uses: peter-evans/find-comment@v2
-        if: github.event_name == 'workflow_dispatch' || steps.check_deployment.outputs.NEW == 'false'
-        id: fc
-        with:
-          issue-number: ${{ steps.pr_info.outputs.PR_NUMBER }}
-          comment-author: "github-actions[bot]"
-          body-includes: ":rocket:"
-          direction: last
-
-      - name: Comment on PR
-        id: comment_id
-        if: github.event_name == 'workflow_dispatch' || steps.check_deployment.outputs.NEW == 'false'
-        uses: peter-evans/create-or-update-comment@v3
-        with:
-          comment-id: ${{ steps.fc.outputs.comment-id }}
-          issue-number: ${{ steps.pr_info.outputs.PR_NUMBER }}
-          edit-mode: replace
-          body: |
-            ---
-            :rocket: Deploying PR ${{ steps.pr_info.outputs.PR_NUMBER }} ...
-            ---
-          reactions: eyes
-          reactions-edit-mode: replace
-
-      - name: Checkout
-        if: github.event_name == 'workflow_dispatch' || steps.check_deployment.outputs.NEW == 'false'
-        uses: actions/checkout@v3
-        with:
-          ref: ${{ steps.pr_info.outputs.PR_BRANCH }}
-          fetch-depth: 0
+          echo "NEW=$NEW" >> $GITHUB_OUTPUT
 
       - name: Check changed files
-        if: github.event_name == 'workflow_dispatch' || steps.check_deployment.outputs.NEW == 'false'
-        uses: dorny/paths-filter@v2
+        uses: dorny/paths-filter@v3
         id: filter
         with:
+          base: ${{ github.ref }}
           filters: |
             all:
               - "**"
@@ -149,57 +143,85 @@ jobs:
               - "scripts/**/*[^D][^o][^c][^k][^e][^r][^f][^i][^l][^e][.][b][^a][^s][^e]*"
 
       - name: Print number of changed files
-        if: github.event_name == 'workflow_dispatch' || steps.check_deployment.outputs.NEW == 'false'
         run: |
-          set -euxo pipefail
+          set -euo pipefail
           echo "Total number of changed files: ${{ steps.filter.outputs.all_count }}"
           echo "Number of ignored files: ${{ steps.filter.outputs.ignored_count }}"
 
+      - name: Build conditionals
+        id: build_conditionals
+        run: |
+          set -euo pipefail
+          # build if the workflow is manually triggered and the deployment doesn't exist (first build or force rebuild)
+          echo "first_or_force_build=${{ (github.event_name == 'workflow_dispatch' && steps.check_deployment.outputs.NEW == 'true') || github.event.inputs.build == 'true' }}" >> $GITHUB_OUTPUT
+          # build if the deployment alreday exist and there are changes in the files that we care about (automatic updates)
+          echo "automatic_rebuild=${{ steps.check_deployment.outputs.NEW == 'false' && steps.filter.outputs.all_count > steps.filter.outputs.ignored_count }}" >> $GITHUB_OUTPUT
+
+  comment-pr:
+    needs: get_info
+    if: needs.get_info.outputs.BUILD == 'true' || github.event.inputs.deploy == 'true'
+    runs-on: "ubuntu-latest"
+    steps:
+      - name: Find Comment
+        uses: peter-evans/find-comment@v3
+        id: fc
+        with:
+          issue-number: ${{ needs.get_info.outputs.PR_NUMBER }}
+          comment-author: "github-actions[bot]"
+          body-includes: ":rocket:"
+          direction: last
+
+      - name: Comment on PR
+        id: comment_id
+        uses: peter-evans/create-or-update-comment@v4
+        with:
+          comment-id: ${{ steps.fc.outputs.comment-id }}
+          issue-number: ${{ needs.get_info.outputs.PR_NUMBER }}
+          edit-mode: replace
+          body: |
+            ---
+            :rocket: Deploying PR ${{ needs.get_info.outputs.PR_NUMBER }} ...
+            ---
+          reactions: eyes
+          reactions-edit-mode: replace
+
   build:
     needs: get_info
-    # Skips the build job if the workflow was triggered by a workflow_dispatch event and the skip_build input is set to true
-    # or if the workflow was triggered by an issue_comment event and the comment body contains --skip-build
-    # always run the build job if a pull_request event triggered the workflow
-    if: |
-      (github.event_name == 'workflow_dispatch' && github.event.inputs.skip_build == 'false') ||
-      (github.event_name == 'pull_request' && needs.get_info.result == 'success' && needs.get_info.outputs.NEW == 'false')
+    # Run build job only if there are changes in the files that we care about or if the workflow is manually triggered with --build flag
+    if: needs.get_info.outputs.BUILD == 'true'
     runs-on: ${{ github.repository_owner == 'coder' && 'buildjet-8vcpu-ubuntu-2204' || 'ubuntu-latest' }}
+    # This concurrency only cancels build jobs if a new build is triggred. It will avoid cancelling the current deployemtn in case of docs chnages.
+    concurrency:
+      group: build-${{ github.workflow }}-${{ github.ref }}-${{ needs.get_info.outputs.BUILD }}
+      cancel-in-progress: true
     env:
       DOCKER_CLI_EXPERIMENTAL: "enabled"
       CODER_IMAGE_TAG: ${{ needs.get_info.outputs.CODER_IMAGE_TAG }}
-      PR_NUMBER: ${{ needs.get_info.outputs.PR_NUMBER }}
-      PR_BRANCH: ${{ needs.get_info.outputs.PR_BRANCH }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
-          ref: ${{ env.PR_BRANCH }}
           fetch-depth: 0
 
       - name: Setup Node
-        if: needs.get_info.outputs.BUILD == 'true'
         uses: ./.github/actions/setup-node
 
       - name: Setup Go
-        if: needs.get_info.outputs.BUILD == 'true'
         uses: ./.github/actions/setup-go
 
       - name: Setup sqlc
-        if: needs.get_info.outputs.BUILD == 'true'
         uses: ./.github/actions/setup-sqlc
 
       - name: GHCR Login
-        if: needs.get_info.outputs.BUILD == 'true'
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Build and push Linux amd64 Docker image
-        if: needs.get_info.outputs.BUILD == 'true'
         run: |
-          set -euxo pipefail
+          set -euo pipefail
           go mod download
           make gen/mark-fresh
           export DOCKER_IMAGE_NO_PREREQUISITES=true
@@ -217,35 +239,43 @@ jobs:
     needs: [build, get_info]
     # Run deploy job only if build job was successful or skipped
     if: |
-      always() && (needs.build.result == 'success' || needs.build.result == 'skipped') && 
-      (github.event_name == 'workflow_dispatch' || needs.get_info.outputs.NEW == 'false')
+      always() && (needs.build.result == 'success' || needs.build.result == 'skipped') &&
+      (needs.get_info.outputs.BUILD == 'true' || github.event.inputs.deploy == 'true')
     runs-on: "ubuntu-latest"
     env:
       CODER_IMAGE_TAG: ${{ needs.get_info.outputs.CODER_IMAGE_TAG }}
       PR_NUMBER: ${{ needs.get_info.outputs.PR_NUMBER }}
       PR_TITLE: ${{ needs.get_info.outputs.PR_TITLE }}
       PR_URL: ${{ needs.get_info.outputs.PR_URL }}
-      PR_BRANCH: ${{ needs.get_info.outputs.PR_BRANCH }}
-      PR_DEPLOYMENT_ACCESS_URL: "pr${{ needs.get_info.outputs.PR_NUMBER }}.${{ secrets.PR_DEPLOYMENTS_DOMAIN }}"
+      PR_HOSTNAME: "pr${{ needs.get_info.outputs.PR_NUMBER }}.${{ secrets.PR_DEPLOYMENTS_DOMAIN }}"
     steps:
       - name: Set up kubeconfig
         run: |
-          set -euxo pipefail
+          set -euo pipefail
           mkdir -p ~/.kube
           echo "${{ secrets.PR_DEPLOYMENTS_KUBECONFIG }}" > ~/.kube/config
+          chmod 644 ~/.kube/config
           export KUBECONFIG=~/.kube/config
 
       - name: Check if image exists
-        if: needs.get_info.outputs.NEW == 'true'
         run: |
-          set -euxo pipefail
-          foundTag=$(curl -fsSL https://github.com/coder/coder/pkgs/container/coder-preview | grep -o ${{ env.CODER_IMAGE_TAG }} | head -n 1)
+          set -euo pipefail
+          foundTag=$(
+            gh api /orgs/coder/packages/container/coder-preview/versions |
+            jq -r --arg tag "pr${{ env.PR_NUMBER }}" '.[] |
+            select(.metadata.container.tags == [$tag]) |
+            .metadata.container.tags[0]'
+          )
           if [ -z "$foundTag" ]; then
             echo "Image not found"
             echo "${{ env.CODER_IMAGE_TAG }} not found in ghcr.io/coder/coder-preview"
-            echo "Please remove --skip-build from the comment and try again"
             exit 1
+          else
+            echo "Image found"
+            echo "$foundTag tag found in ghcr.io/coder/coder-preview"
           fi
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Add DNS record to Cloudflare
         if: needs.get_info.outputs.NEW == 'true'
@@ -253,43 +283,27 @@ jobs:
           curl -X POST "https://api.cloudflare.com/client/v4/zones/${{ secrets.PR_DEPLOYMENTS_ZONE_ID }}/dns_records" \
             -H "Authorization: Bearer ${{ secrets.PR_DEPLOYMENTS_CLOUDFLARE_API_TOKEN }}" \
             -H "Content-Type:application/json" \
-            --data '{"type":"CNAME","name":"*.${{ env.PR_DEPLOYMENT_ACCESS_URL }}","content":"${{ env.PR_DEPLOYMENT_ACCESS_URL }}","ttl":1,"proxied":false}'
-
-      - name: Checkout
-        uses: actions/checkout@v3
-        with:
-          ref: ${{ env.PR_BRANCH }}
+            --data '{"type":"CNAME","name":"*.${{ env.PR_HOSTNAME }}","content":"${{ env.PR_HOSTNAME }}","ttl":1,"proxied":false}'
 
       - name: Create PR namespace
-        if: needs.get_info.outputs.NEW == 'true'
+        if: needs.get_info.outputs.NEW == 'true' || github.event.inputs.deploy == 'true'
         run: |
-          set -euxo pipefail
+          set -euo pipefail
           # try to delete the namespace, but don't fail if it doesn't exist
           kubectl delete namespace "pr${{ env.PR_NUMBER }}" || true
           kubectl create namespace "pr${{ env.PR_NUMBER }}"
 
+      - name: Checkout
+        uses: actions/checkout@v4
+
       - name: Check and Create Certificate
-        if: needs.get_info.outputs.NEW == 'true'
+        if: needs.get_info.outputs.NEW == 'true' || github.event.inputs.deploy == 'true'
         run: |
           # Using kubectl to check if a Certificate resource already exists
           # we are doing this to avoid letsenrypt rate limits
           if ! kubectl get certificate pr${{ env.PR_NUMBER }}-tls -n pr-deployment-certs > /dev/null 2>&1; then
             echo "Certificate doesn't exist. Creating a new one."
-            cat <<EOF | kubectl apply -f -
-            apiVersion: cert-manager.io/v1
-            kind: Certificate
-            metadata:
-              name: pr${{ env.PR_NUMBER }}-tls
-              namespace: pr-deployment-certs
-            spec:
-              secretName: pr${{ env.PR_NUMBER }}-tls
-              issuerRef:
-                name: letsencrypt
-                kind: ClusterIssuer
-              dnsNames:
-              - "${{ env.PR_DEPLOYMENT_ACCESS_URL }}"
-              - "*.${{ env.PR_DEPLOYMENT_ACCESS_URL }}"
-          EOF
+            envsubst < ./.github/pr-deployments/certificate.yaml | kubectl apply -f -
           else
             echo "Certificate exists. Skipping certificate creation."
           fi
@@ -306,7 +320,7 @@ jobs:
           )
 
       - name: Set up PostgreSQL database
-        if: needs.get_info.outputs.NEW == 'true'
+        if: needs.get_info.outputs.NEW == 'true' || github.event.inputs.deploy == 'true'
         run: |
           helm repo add bitnami https://charts.bitnami.com/bitnami
           helm install coder-db bitnami/postgresql \
@@ -318,82 +332,46 @@ jobs:
           kubectl create secret generic coder-db-url -n pr${{ env.PR_NUMBER }} \
             --from-literal=url="postgres://coder:coder@coder-db-postgresql.pr${{ env.PR_NUMBER }}.svc.cluster.local:5432/coder?sslmode=disable"
 
-      - name: Create values.yaml
-        if: github.event_name == 'workflow_dispatch'
+      - name: Create a service account, role, and rolebinding for the PR namespace
+        if: needs.get_info.outputs.NEW == 'true' || github.event.inputs.deploy == 'true'
         run: |
-          cat <<EOF > pr-deploy-values.yaml
-          coder:
-            image:
-              repo: ${{ env.REPO }}
-              tag: pr${{ env.PR_NUMBER }}
-              pullPolicy: Always
-            service:
-              type: ClusterIP
-            ingress:
-              enable: true
-              className: traefik
-              host: ${{ env.PR_DEPLOYMENT_ACCESS_URL }}
-              wildcardHost: "*.${{ env.PR_DEPLOYMENT_ACCESS_URL }}"
-              tls:
-                enable: true
-                secretName: pr${{ env.PR_NUMBER }}-tls
-                wildcardSecretName: pr${{ env.PR_NUMBER }}-tls
-            env:
-              - name: "CODER_ACCESS_URL"
-                value: "https://${{ env.PR_DEPLOYMENT_ACCESS_URL }}"
-              - name: "CODER_WILDCARD_ACCESS_URL"
-                value: "*.${{ env.PR_DEPLOYMENT_ACCESS_URL }}"
-              - name: "CODER_EXPERIMENTS"
-                value: "${{ github.event.inputs.experiments }}"
-              - name: CODER_PG_CONNECTION_URL
-                valueFrom:
-                  secretKeyRef:
-                    name: coder-db-url
-                    key: url
-              - name: "CODER_OAUTH2_GITHUB_ALLOW_SIGNUPS"
-                value: "true"
-              - name: "CODER_OAUTH2_GITHUB_CLIENT_ID"
-                value: "${{ secrets.PR_DEPLOYMENTS_GITHUB_OAUTH_CLIENT_ID }}"
-              - name: "CODER_OAUTH2_GITHUB_CLIENT_SECRET"
-                value: "${{ secrets.PR_DEPLOYMENTS_GITHUB_OAUTH_CLIENT_SECRET }}"
-              - name: "CODER_OAUTH2_GITHUB_ALLOWED_ORGS"
-                value: "coder"
-          EOF
+          set -euo pipefail
+          # Create service account, role, rolebinding
+          envsubst < ./.github/pr-deployments/rbac.yaml | kubectl apply -f -
+
+      - name: Create values.yaml
+        env:
+          EXPERIMENTS: ${{ github.event.inputs.experiments }}
+          PR_DEPLOYMENTS_GITHUB_OAUTH_CLIENT_ID: ${{ secrets.PR_DEPLOYMENTS_GITHUB_OAUTH_CLIENT_ID }}
+          PR_DEPLOYMENTS_GITHUB_OAUTH_CLIENT_SECRET: ${{ secrets.PR_DEPLOYMENTS_GITHUB_OAUTH_CLIENT_SECRET }}
+        run: |
+          set -euo pipefail
+          envsubst < ./.github/pr-deployments/values.yaml > ./pr-deploy-values.yaml
 
       - name: Install/Upgrade Helm chart
         run: |
-          set -euxo pipefail
-          if [[ ${{ github.event_name }} == "workflow_dispatch" ]]; then
-            helm upgrade --install "pr${{ env.PR_NUMBER }}" ./helm \
-            --namespace "pr${{ env.PR_NUMBER }}" \
-            --values ./pr-deploy-values.yaml \
-            --force
-          else
-            if [[ ${{ needs.get_info.outputs.BUILD }} == "true" ]]; then
-              helm upgrade --install "pr${{ env.PR_NUMBER }}" ./helm \
-              --namespace "pr${{ env.PR_NUMBER }}" \
-              --reuse-values \
-              --force
-            else
-              echo "Skipping helm upgrade, as there is no new image to deploy"
-            fi
-          fi
+          set -euo pipefail
+          helm dependency update --skip-refresh ./helm/coder
+          helm upgrade --install "pr${{ env.PR_NUMBER }}" ./helm/coder \
+          --namespace "pr${{ env.PR_NUMBER }}" \
+          --values ./pr-deploy-values.yaml \
+          --force
 
       - name: Install coder-logstream-kube
-        if: needs.get_info.outputs.NEW == 'true'
+        if: needs.get_info.outputs.NEW == 'true' || github.event.inputs.deploy == 'true'
         run: |
           helm repo add coder-logstream-kube https://helm.coder.com/logstream-kube
           helm upgrade --install coder-logstream-kube coder-logstream-kube/coder-logstream-kube \
             --namespace "pr${{ env.PR_NUMBER }}" \
-            --set url="https://pr${{ env.PR_NUMBER }}.${{ secrets.PR_DEPLOYMENTS_DOMAIN }}"
+            --set url="https://${{ env.PR_HOSTNAME }}"
 
       - name: Get Coder binary
-        if: needs.get_info.outputs.NEW == 'true'
+        if: needs.get_info.outputs.NEW == 'true' || github.event.inputs.deploy == 'true'
         run: |
-          set -euxo pipefail
+          set -euo pipefail
 
           DEST="${HOME}/coder"
-          URL="https://${{ env.PR_DEPLOYMENT_ACCESS_URL }}/bin/coder-linux-amd64"
+          URL="https://${{ env.PR_HOSTNAME }}/bin/coder-linux-amd64"
 
           mkdir -p "$(dirname ${DEST})"
 
@@ -414,10 +392,10 @@ jobs:
           mv "${DEST}" /usr/local/bin/coder
 
       - name: Create first user, template and workspace
-        if: needs.get_info.outputs.NEW == 'true'
+        if: needs.get_info.outputs.NEW == 'true' || github.event.inputs.deploy == 'true'
         id: setup_deployment
         run: |
-          set -euxo pipefail
+          set -euo pipefail
 
           # Create first user
 
@@ -429,28 +407,23 @@ jobs:
           echo "password=$password" >> $GITHUB_OUTPUT
 
           coder login \
-            --first-user-username test \
+            --first-user-username coder \
             --first-user-email pr${{ env.PR_NUMBER }}@coder.com \
             --first-user-password $password \
             --first-user-trial \
             --use-token-as-session \
-            https://${{ env.PR_DEPLOYMENT_ACCESS_URL }}
+            https://${{ env.PR_HOSTNAME }}
 
           # Create template
-          coder templates init --id kubernetes && cd ./kubernetes/ && coder templates create -y --variable namespace=pr${{ env.PR_NUMBER }}
+          cd ./.github/pr-deployments/template
+          coder templates push -y --variable namespace=pr${{ env.PR_NUMBER }} kubernetes
 
           # Create workspace
-          cat <<EOF > workspace.yaml
-          cpu: "2"
-          memory: "4"
-          home_disk_size: "2"
-          EOF
-
-          coder create --template="kubernetes" test --rich-parameter-file ./workspace.yaml -y
-          coder stop test -y
+          coder create --template="kubernetes" kube --parameter cpu=2 --parameter memory=4 --parameter home_disk_size=2 -y
+          coder stop kube -y
 
       - name: Send Slack notification
-        if: needs.get_info.outputs.NEW == 'true'
+        if: needs.get_info.outputs.NEW == 'true' || github.event.inputs.deploy == 'true'
         run: |
           curl -s -o /dev/null -X POST -H 'Content-type: application/json' \
             -d \
@@ -458,7 +431,7 @@ jobs:
               "pr_number": "'"${{ env.PR_NUMBER }}"'",
               "pr_url": "'"${{ env.PR_URL }}"'",
               "pr_title": "'"${{ env.PR_TITLE }}"'",
-              "pr_access_url": "'"https://${{ env.PR_DEPLOYMENT_ACCESS_URL }}"'",
+              "pr_access_url": "'"https://${{ env.PR_HOSTNAME }}"'",
               "pr_username": "'"test"'",
               "pr_email": "'"pr${{ env.PR_NUMBER }}@coder.com"'",
               "pr_password": "'"${{ steps.setup_deployment.outputs.password }}"'",
@@ -468,7 +441,7 @@ jobs:
           echo "Slack notification sent"
 
       - name: Find Comment
-        uses: peter-evans/find-comment@v2
+        uses: peter-evans/find-comment@v3
         id: fc
         with:
           issue-number: ${{ env.PR_NUMBER }}
@@ -477,7 +450,7 @@ jobs:
           direction: last
 
       - name: Comment on PR
-        uses: peter-evans/create-or-update-comment@v3
+        uses: peter-evans/create-or-update-comment@v4
         env:
           STATUS: ${{ needs.get_info.outputs.NEW == 'true' && 'Created' || 'Updated' }}
         with:

.github/workflows/release.yaml

@@ -28,10 +28,6 @@ env:
   # https://github.blog/changelog/2022-06-10-github-actions-inputs-unified-across-manual-and-reusable-workflows/
   CODER_RELEASE: ${{ !inputs.dry_run }}
   CODER_DRY_RUN: ${{ inputs.dry_run }}
-  # For some reason, setup-go won't actually pick up a new patch version if
-  # it has an old one cached. We need to manually specify the versions so we
-  # can get the latest release. Never use "~1.xx" here!
-  CODER_GO_VERSION: "1.20.6"
 
 jobs:
   release:
@@ -44,7 +40,7 @@ jobs:
       version: ${{ steps.version.outputs.version }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
@@ -89,7 +85,7 @@ jobs:
           cat "$CODER_RELEASE_NOTES_FILE"
 
       - name: Docker Login
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
@@ -98,16 +94,8 @@ jobs:
       - name: Setup Go
         uses: ./.github/actions/setup-go
 
-      - name: Cache Node
-        id: cache-node
-        uses: buildjet/cache@v3
-        with:
-          path: |
-            **/node_modules
-            .eslintcache
-          key: js-${{ runner.os }}-test-${{ hashFiles('**/yarn.lock') }}
-          restore-keys: |
-            js-${{ runner.os }}-
+      - name: Setup Node
+        uses: ./.github/actions/setup-node
 
       - name: Install nsis and zstd
         run: sudo apt-get install -y nsis zstd
@@ -115,7 +103,7 @@ jobs:
       - name: Install nfpm
         run: |
           set -euo pipefail
-          wget -O /tmp/nfpm.deb https://github.com/goreleaser/nfpm/releases/download/v2.18.1/nfpm_amd64.deb
+          wget -O /tmp/nfpm.deb https://github.com/goreleaser/nfpm/releases/download/v2.35.1/nfpm_2.35.1_amd64.deb
           sudo dpkg -i /tmp/nfpm.deb
           rm /tmp/nfpm.deb
 
@@ -153,7 +141,8 @@ jobs:
             build/coder_"$version"_linux_{amd64,armv7,arm64}.{tar.gz,apk,deb,rpm} \
             build/coder_"$version"_{darwin,windows}_{amd64,arm64}.zip \
             build/coder_"$version"_windows_amd64_installer.exe \
-            build/coder_helm_"$version".tgz
+            build/coder_helm_"$version".tgz \
+            build/provisioner_helm_"$version".tgz
         env:
           CODER_SIGN_DARWIN: "1"
           AC_CERTIFICATE_FILE: /tmp/apple_cert.p12
@@ -292,13 +281,13 @@ jobs:
           CODER_GPG_RELEASE_KEY_BASE64: ${{ secrets.GPG_RELEASE_KEY_BASE64 }}
 
       - name: Authenticate to Google Cloud
-        uses: google-github-actions/auth@v1
+        uses: google-github-actions/auth@v2
         with:
           workload_identity_provider: ${{ secrets.GCP_WORKLOAD_ID_PROVIDER }}
           service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }}
 
       - name: Setup GCloud SDK
-        uses: "google-github-actions/setup-gcloud@v1"
+        uses: "google-github-actions/setup-gcloud@v2"
 
       - name: Publish Helm Chart
         if: ${{ !inputs.dry_run }}
@@ -307,15 +296,17 @@ jobs:
           version="$(./scripts/version.sh)"
           mkdir -p build/helm
           cp "build/coder_helm_${version}.tgz" build/helm
+          cp "build/provisioner_helm_${version}.tgz" build/helm
           gsutil cp gs://helm.coder.com/v2/index.yaml build/helm/index.yaml
           helm repo index build/helm --url https://helm.coder.com/v2 --merge build/helm/index.yaml
           gsutil -h "Cache-Control:no-cache,max-age=0" cp build/helm/coder_helm_${version}.tgz gs://helm.coder.com/v2
+          gsutil -h "Cache-Control:no-cache,max-age=0" cp build/helm/provisioner_helm_${version}.tgz gs://helm.coder.com/v2
           gsutil -h "Cache-Control:no-cache,max-age=0" cp build/helm/index.yaml gs://helm.coder.com/v2
           gsutil -h "Cache-Control:no-cache,max-age=0" cp helm/artifacthub-repo.yml gs://helm.coder.com/v2
 
       - name: Upload artifacts to actions (if dry-run)
         if: ${{ inputs.dry_run }}
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: release-artifacts
           path: |
@@ -330,20 +321,100 @@ jobs:
 
       - name: Start Packer builds
         if: ${{ !inputs.dry_run }}
-        uses: peter-evans/repository-dispatch@v2
+        uses: peter-evans/repository-dispatch@v3
         with:
           token: ${{ secrets.CDRCI_GITHUB_TOKEN }}
           repository: coder/packages
           event-type: coder-release
           client-payload: '{"coder_version": "${{ steps.version.outputs.version }}"}'
 
+  publish-homebrew:
+    name: Publish to Homebrew tap
+    runs-on: ubuntu-latest
+    needs: release
+    if: ${{ !inputs.dry_run }}
+
+    steps:
+      # TODO: skip this if it's not a new release (i.e. a backport). This is
+      #       fine right now because it just makes a PR that we can close.
+      - name: Update homebrew
+        env:
+          # Variables used by the `gh` command
+          GH_REPO: coder/homebrew-coder
+          GH_TOKEN: ${{ secrets.CDRCI_GITHUB_TOKEN }}
+        run: |
+          # Keep version number around for reference, removing any potential leading v
+          coder_version="$(echo "${{ needs.release.outputs.version }}" | tr -d v)"
+
+          set -euxo pipefail
+
+          # Setup Git
+          git config --global user.email "ci@coder.com"
+          git config --global user.name "Coder CI"
+          git config --global credential.helper "store"
+
+          temp_dir="$(mktemp -d)"
+          cd "$temp_dir"
+
+          # Download checksums
+          checksums_url="$(gh release view --repo coder/coder "v$coder_version" --json assets \
+            | jq -r ".assets | map(.url) | .[]" \
+            | grep -e ".checksums.txt\$")"
+          wget "$checksums_url" -O checksums.txt
+
+          # Get the SHAs
+          darwin_arm_sha="$(cat checksums.txt | grep "darwin_arm64.zip" | awk '{ print $1 }')"
+          darwin_intel_sha="$(cat checksums.txt | grep "darwin_amd64.zip" | awk '{ print $1 }')"
+          linux_sha="$(cat checksums.txt | grep "linux_amd64.tar.gz" | awk '{ print $1 }')"
+
+          echo "macOS arm64: $darwin_arm_sha"
+          echo "macOS amd64: $darwin_intel_sha"
+          echo "Linux amd64: $linux_sha"
+
+          # Check out the homebrew repo
+          git clone "https://github.com/$GH_REPO" homebrew-coder
+          brew_branch="auto-release/$coder_version"
+          cd homebrew-coder
+
+          # Check if a PR already exists.
+          pr_count="$(gh pr list --search "head:$brew_branch" --json id,closed | jq -r ".[] | select(.closed == false) | .id" | wc -l)"
+          if [[ "$pr_count" > 0 ]]; then
+            echo "Bailing out as PR already exists" 2>&1
+            exit 0
+          fi
+
+          # Set up cdrci credentials for pushing to homebrew-coder
+          echo "https://x-access-token:$GH_TOKEN@github.com" >> ~/.git-credentials
+          # Update the formulae and push
+          git checkout -b "$brew_branch"
+          ./scripts/update-v2.sh "$coder_version" "$darwin_arm_sha" "$darwin_intel_sha" "$linux_sha"
+          git add .
+          git commit -m "coder $coder_version"
+          git push -u origin -f "$brew_branch"
+
+          # Create PR
+          gh pr create \
+            -B master -H "$brew_branch" \
+            -t "coder $coder_version" \
+            -b "" \
+            -r "${{ github.actor }}" \
+            -a "${{ github.actor }}" \
+            -b "This automatic PR was triggered by the release of Coder v$coder_version"
+
   publish-winget:
     name: Publish to winget-pkgs
     runs-on: windows-latest
     needs: release
+    if: ${{ !inputs.dry_run }}
+
     steps:
+      - name: Sync fork
+        run: gh repo sync cdrci/winget-pkgs -b master
+        env:
+          GH_TOKEN: ${{ secrets.CDRCI_GITHUB_TOKEN }}
+
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
@@ -368,33 +439,26 @@ jobs:
 
           $release_assets = gh release view --repo coder/coder "v${version}" --json assets | `
             ConvertFrom-Json
-          # Get the installer URL from the release assets.
-          $installer_url = $release_assets.assets | `
+          # Get the installer URLs from the release assets.
+          $amd64_installer_url = $release_assets.assets | `
             Where-Object name -Match ".*_windows_amd64_installer.exe$" | `
             Select -ExpandProperty url
+          $amd64_zip_url = $release_assets.assets | `
+            Where-Object name -Match ".*_windows_amd64.zip$" | `
+            Select -ExpandProperty url
+          $arm64_zip_url = $release_assets.assets | `
+            Where-Object name -Match ".*_windows_arm64.zip$" | `
+            Select -ExpandProperty url
 
-          echo "Installer URL: ${installer_url}"
+          echo "amd64 Installer URL: ${amd64_installer_url}"
+          echo "amd64 zip URL: ${amd64_zip_url}"
+          echo "arm64 zip URL: ${arm64_zip_url}"
           echo "Package version: ${version}"
 
-          # Bail if dry-run.
-          if ($env:CODER_DRY_RUN -match "t") {
-            echo "Skipping submission due to dry-run."
-            exit 0
-          }
-
-          # The URL "|X64" suffix forces the architecture as it cannot be
-          # sniffed properly from the URL. wingetcreate checks both the URL and
-          # binary magic bytes for the architecture and they need to both match,
-          # but they only check for `x64`, `win64` and `_64` in the URL. Our URL
-          # contains `amd64` which doesn't match sadly.
-          #
-          # wingetcreate will still do the binary magic bytes check, so if we
-          # accidentally change the architecture of the installer, it will fail
-          # submission.
           .\wingetcreate.exe update Coder.Coder `
             --submit `
             --version "${version}" `
-            --urls "${installer_url}|X64" `
+            --urls "${amd64_installer_url}" "${amd64_zip_url}" "${arm64_zip_url}" `
             --token "$env:WINGET_GH_TOKEN"
 
         env:
@@ -405,7 +469,6 @@ jobs:
           WINGET_GH_TOKEN: ${{ secrets.CDRCI_GITHUB_TOKEN }}
 
       - name: Comment on PR
-        if: ${{ !inputs.dry_run }}
         run: |
           # wait 30 seconds
           Start-Sleep -Seconds 30.0
@@ -421,3 +484,29 @@ jobs:
           # For gh CLI. We need a real token since we're commenting on a PR in a
           # different repo.
           GH_TOKEN: ${{ secrets.CDRCI_GITHUB_TOKEN }}
+
+  # publish-sqlc pushes the latest schema to sqlc cloud.
+  # At present these pushes cannot be tagged, so the last push is always the latest.
+  publish-sqlc:
+    name: "Publish to schema sqlc cloud"
+    runs-on: "ubuntu-latest"
+    needs: release
+    if: ${{ !inputs.dry_run }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+
+      # We need golang to run the migration main.go
+      - name: Setup Go
+        uses: ./.github/actions/setup-go
+
+      - name: Setup sqlc
+        uses: ./.github/actions/setup-sqlc
+
+      - name: Push schema to sqlc cloud
+        # Don't block a release on this
+        continue-on-error: true
+        run: |
+          make sqlc-push

.github/workflows/security.yaml

@@ -21,31 +21,28 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}-security
   cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
-env:
-  CODER_GO_VERSION: "1.20.6"
-
 jobs:
   codeql:
     runs-on: ${{ github.repository_owner == 'coder' && 'buildjet-8vcpu-ubuntu-2204' || 'ubuntu-latest' }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
-
-      - name: Initialize CodeQL
-        uses: github/codeql-action/init@v2
-        with:
-          languages: go, javascript
+        uses: actions/checkout@v4
 
       - name: Setup Go
         uses: ./.github/actions/setup-go
 
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v3
+        with:
+          languages: go, javascript
+
       # Workaround to prevent CodeQL from building the dashboard.
       - name: Remove Makefile
         run: |
           rm Makefile
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v2
+        uses: github/codeql-action/analyze@v3
 
       - name: Send Slack notification on failure
         if: ${{ failure() }}
@@ -62,23 +59,15 @@ jobs:
     runs-on: ${{ github.repository_owner == 'coder' && 'buildjet-8vcpu-ubuntu-2204' || 'ubuntu-latest' }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
       - name: Setup Go
         uses: ./.github/actions/setup-go
 
-      - name: Cache Node
-        id: cache-node
-        uses: buildjet/cache@v3
-        with:
-          path: |
-            **/node_modules
-            .eslintcache
-          key: js-${{ runner.os }}-test-${{ hashFiles('**/yarn.lock') }}
-          restore-keys: |
-            js-${{ runner.os }}-
+      - name: Setup Node
+        uses: ./.github/actions/setup-node
 
       - name: Setup sqlc
         uses: ./.github/actions/setup-sqlc
@@ -86,7 +75,7 @@ jobs:
       - name: Install yq
         run: go run github.com/mikefarah/yq/v4@v4.30.6
       - name: Install mockgen
-        run: go install github.com/golang/mock/mockgen@v1.6.0
+        run: go install go.uber.org/mock/mockgen@v0.4.0
       - name: Install protoc-gen-go
         run: go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.30
       - name: Install protoc-gen-go-drpc
@@ -124,16 +113,8 @@ jobs:
           make -j "$image_job"
           echo "image=$(cat "$image_job")" >> $GITHUB_OUTPUT
 
-      - name: Run Prisma Cloud image scan
-        uses: PaloAltoNetworks/prisma-cloud-scan@v1
-        with:
-          pcc_console_url: ${{ secrets.PRISMA_CLOUD_URL }}
-          pcc_user: ${{ secrets.PRISMA_CLOUD_ACCESS_KEY }}
-          pcc_pass: ${{ secrets.PRISMA_CLOUD_SECRET_KEY }}
-          image_name: ${{ steps.build.outputs.image }}
-
       - name: Run Trivy vulnerability scanner
-        uses: aquasecurity/trivy-action@41f05d9ecffa2ed3f1580af306000f734b733e54
+        uses: aquasecurity/trivy-action@84384bd6e777ef152729993b8145ea352e9dd3ef
         with:
           image-ref: ${{ steps.build.outputs.image }}
           format: sarif
@@ -141,18 +122,28 @@ jobs:
           severity: "CRITICAL,HIGH"
 
       - name: Upload Trivy scan results to GitHub Security tab
-        uses: github/codeql-action/upload-sarif@v2
+        uses: github/codeql-action/upload-sarif@v3
         with:
           sarif_file: trivy-results.sarif
           category: "Trivy"
 
       - name: Upload Trivy scan results as an artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: trivy
           path: trivy-results.sarif
           retention-days: 7
 
+      # Prisma cloud scan runs last because it fails the entire job if it
+      # detects vulnerabilities. :|
+      - name: Run Prisma Cloud image scan
+        uses: PaloAltoNetworks/prisma-cloud-scan@v1
+        with:
+          pcc_console_url: ${{ secrets.PRISMA_CLOUD_URL }}
+          pcc_user: ${{ secrets.PRISMA_CLOUD_ACCESS_KEY }}
+          pcc_pass: ${{ secrets.PRISMA_CLOUD_SECRET_KEY }}
+          image_name: ${{ steps.build.outputs.image }}
+
       - name: Send Slack notification on failure
         if: ${{ failure() }}
         run: |

.github/workflows/stale.yaml

@@ -13,7 +13,7 @@ jobs:
       actions: write
     steps:
       - name: stale
-        uses: actions/stale@v8.0.0
+        uses: actions/stale@v9.0.0
         with:
           stale-issue-label: "stale"
           stale-pr-label: "stale"
@@ -30,11 +30,57 @@ jobs:
           operations-per-run: 60
           # Start with the oldest issues, always.
           ascending: true
+      - name: "Close old issues labeled likely-no"
+        uses: actions/github-script@v7
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const thirtyDaysAgo = new Date(new Date().setDate(new Date().getDate() - 30));
+            console.log(`Looking for issues labeled with 'likely-no' more than 30 days ago, which is after ${thirtyDaysAgo.toISOString()}`);
+
+            const issues = await github.rest.issues.listForRepo({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              labels: 'likely-no',
+              state: 'open',
+            });
+
+            console.log(`Found ${issues.data.length} open issues labeled with 'likely-no'`);
+
+            for (const issue of issues.data) {
+              console.log(`Checking issue #${issue.number} created at ${issue.created_at}`);
+
+              const timeline = await github.rest.issues.listEventsForTimeline({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: issue.number,
+              });
+
+              const labelEvent = timeline.data.find(event => event.event === 'labeled' && event.label.name === 'likely-no');
+              
+              if (labelEvent) {
+                console.log(`Issue #${issue.number} was labeled with 'likely-no' at ${labelEvent.created_at}`);
+
+                if (new Date(labelEvent.created_at) < thirtyDaysAgo) {
+                  console.log(`Issue #${issue.number} is older than 30 days with 'likely-no' label, closing issue.`);
+                  await github.rest.issues.update({
+                    owner: context.repo.owner,
+                    repo: context.repo.repo,
+                    issue_number: issue.number,
+                    state: 'closed',
+                    state_reason: 'not_planned'
+                  });
+                }
+              } else {
+                console.log(`Issue #${issue.number} does not have a 'likely-no' label event in its timeline.`);
+              }
+            }
+
   branches:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Run delete-old-branches-action
         uses: beatlabs/delete-old-branches-action@v0.0.10
         with:
@@ -52,8 +98,8 @@ jobs:
         with:
           token: ${{ github.token }}
           repository: ${{ github.repository }}
-          retain_days: 1
-          keep_minimum_runs: 1
+          retain_days: 30
+          keep_minimum_runs: 30
           delete_workflow_pattern: pr-cleanup.yaml
 
       - name: Delete PR Deploy workflow skipped runs
@@ -61,7 +107,6 @@ jobs:
         with:
           token: ${{ github.token }}
           repository: ${{ github.repository }}
-          retain_days: 0
-          keep_minimum_runs: 0
-          delete_run_by_conclusion_pattern: skipped
+          retain_days: 30
+          keep_minimum_runs: 30
           delete_workflow_pattern: pr-deploy.yaml

.github/workflows/typos.toml

@@ -14,6 +14,7 @@ darcula = "darcula"
 Hashi = "Hashi"
 trialer = "trialer"
 encrypter = "encrypter"
+hel = "hel"             # as in helsinki
 
 [files]
 extend-exclude = [
@@ -29,4 +30,6 @@ extend-exclude = [
 	"**/*_test.go",
 	"**/*.test.tsx",
 	"**/pnpm-lock.yaml",
+	"tailnet/testdata/**",
+	"site/src/pages/SetupPage/countries.tsx",
 ]

.gitignore

@@ -20,7 +20,6 @@ yarn-error.log
 
 # Front-end ignore patterns.
 .next/
-site/**/*.typegen.ts
 site/build-storybook.log
 site/coverage/
 site/storybook-static/
@@ -30,15 +29,14 @@ site/e2e/states/*.json
 site/e2e/.auth.json
 site/playwright-report/*
 site/.swc
-site/dist/
 
 # Make target for updating golden files (any dir).
 .gen-golden
 
 # Build
-/build/
-/dist/
-site/out/
+build/
+dist/
+out/
 
 # Bundle analysis
 site/stats/
@@ -61,3 +59,12 @@ site/stats/
 ./scaletest/terraform/.terraform.lock.hcl
 scaletest/terraform/secrets.tfvars
 .terraform.tfstate.*
+
+# Nix
+result
+
+# Data dumps from unit tests
+**/*.test.sql
+
+# Filebrowser.db
+**/filebrowser.db

.golangci.yaml

@@ -2,12 +2,19 @@
 # Over time we should try tightening some of these.
 
 linters-settings:
+  dupl:
+    # goal: 100
+    threshold: 412
+
   exhaustruct:
     include:
       # Gradually extend to cover more of the codebase.
       - 'httpmw\.\w+'
+      # We want to enforce all values are specified when inserting or updating
+      # a database row. Ref: #9936
+      - 'github.com/coder/coder/v2/coderd/database\.[^G][^e][^t]\w+Params'
   gocognit:
-    min-complexity: 46 # Min code complexity (def 30).
+    min-complexity: 300
 
   goconst:
     min-len: 4 # Min length of string consts (def 3).
@@ -118,10 +125,6 @@ linters-settings:
   goimports:
     local-prefixes: coder.com,cdr.dev,go.coder.com,github.com/cdr,github.com/coder
 
-  gocyclo:
-    # goal: 30
-    min-complexity: 47
-
   importas:
     no-unaliased: true
 
@@ -131,7 +134,8 @@ linters-settings:
       - trialer
 
   nestif:
-    min-complexity: 4 # Min complexity of if statements (def 5, goal 4)
+    # goal: 10
+    min-complexity: 20
 
   revive:
     # see https://github.com/mgechev/revive#available-rules for details.
@@ -211,6 +215,7 @@ issues:
 run:
   skip-dirs:
     - node_modules
+    - .git
   skip-files:
     - scripts/rules.go
   timeout: 10m
@@ -231,7 +236,12 @@ linters:
     - exportloopref
     - forcetypeassert
     - gocritic
-    - gocyclo
+    # gocyclo is may be useful in the future when we start caring
+    # about testing complexity, but for the time being we should
+    # create a good culture around cognitive complexity.
+    # - gocyclo
+    - gocognit
+    - nestif
     - goimports
     - gomodguard
     - gosec
@@ -267,3 +277,4 @@ linters:
     - typecheck
     - unconvert
     - unused
+    - dupl

.prettierignore

@@ -23,7 +23,6 @@ yarn-error.log
 
 # Front-end ignore patterns.
 .next/
-site/**/*.typegen.ts
 site/build-storybook.log
 site/coverage/
 site/storybook-static/
@@ -33,15 +32,14 @@ site/e2e/states/*.json
 site/e2e/.auth.json
 site/playwright-report/*
 site/.swc
-site/dist/
 
 # Make target for updating golden files (any dir).
 .gen-golden
 
 # Build
-/build/
-/dist/
-site/out/
+build/
+dist/
+out/
 
 # Bundle analysis
 site/stats/
@@ -64,10 +62,19 @@ site/stats/
 ./scaletest/terraform/.terraform.lock.hcl
 scaletest/terraform/secrets.tfvars
 .terraform.tfstate.*
+
+# Nix
+result
+
+# Data dumps from unit tests
+**/*.test.sql
+
+# Filebrowser.db
+**/filebrowser.db
 # .prettierignore.include:
 # Helm templates contain variables that are invalid YAML and can't be formatted
 # by Prettier.
-helm/templates/*.yaml
+helm/**/templates/*.yaml
 
 # Terraform state files used in tests, these are automatically generated.
 # Example: provisioner/terraform/testdata/instance-id/instance-id.tfstate.json
@@ -75,8 +82,13 @@ helm/templates/*.yaml
 
 # Testdata shouldn't be formatted.
 scripts/apitypings/testdata/**/*.ts
+enterprise/tailnet/testdata/*.golden.html
+tailnet/testdata/*.golden.html
 
 # Generated files shouldn't be formatted.
 site/e2e/provisionerGenerated.ts
 
 **/pnpm-lock.yaml
+
+# Ignore generated JSON (e.g. examples/examples.gen.json).
+**/*.gen.json

.prettierignore.include

@@ -1,6 +1,6 @@
 # Helm templates contain variables that are invalid YAML and can't be formatted
 # by Prettier.
-helm/templates/*.yaml
+helm/**/templates/*.yaml
 
 # Terraform state files used in tests, these are automatically generated.
 # Example: provisioner/terraform/testdata/instance-id/instance-id.tfstate.json
@@ -8,8 +8,13 @@ helm/templates/*.yaml
 
 # Testdata shouldn't be formatted.
 scripts/apitypings/testdata/**/*.ts
+enterprise/tailnet/testdata/*.golden.html
+tailnet/testdata/*.golden.html
 
 # Generated files shouldn't be formatted.
 site/e2e/provisionerGenerated.ts
 
 **/pnpm-lock.yaml
+
+# Ignore generated JSON (e.g. examples/examples.gen.json).
+**/*.gen.json

.prettierrc.yaml

@@ -1,18 +1,18 @@
 # This config file is used in conjunction with `.editorconfig` to specify
 # formatting for prettier-supported files. See `.editorconfig` and
-# `site/.editorconfig`for whitespace formatting options.
+# `site/.editorconfig` for whitespace formatting options.
 printWidth: 80
-semi: false
+proseWrap: always
 trailingComma: all
 useTabs: false
 tabWidth: 2
 overrides:
   - files:
       - README.md
+      - docs/api/**/*.md
+      - docs/cli/**/*.md
+      - docs/changelogs/*.md
+      - .github/**/*.{yaml,yml,toml}
+      - scripts/**/*.{yaml,yml,toml}
     options:
       proseWrap: preserve
-  - files:
-      - "site/**/*.yaml"
-      - "site/**/*.yml"
-    options:
-      proseWrap: always

.swaggo

@@ -1,8 +1,8 @@
 // Replace all NullTime with string
-replace github.com/coder/coder/codersdk.NullTime string
+replace github.com/coder/coder/v2/codersdk.NullTime string
 // Prevent swaggo from rendering enums for time.Duration
 replace time.Duration int64
 // Do not expose "echo" provider
-replace github.com/coder/coder/codersdk.ProvisionerType string
+replace github.com/coder/coder/v2/codersdk.ProvisionerType string
 // Do not render netip.Addr
 replace netip.Addr string

.vscode/settings.json

@@ -18,10 +18,11 @@
     "coderdenttest",
     "coderdtest",
     "codersdk",
+    "contravariance",
     "cronstrue",
     "databasefake",
-    "dbfake",
     "dbgen",
+    "dbmem",
     "dbtype",
     "DERP",
     "derphttp",
@@ -39,6 +40,7 @@
     "enterprisemeta",
     "errgroup",
     "eventsourcemock",
+    "externalauth",
     "Failf",
     "fatih",
     "Formik",
@@ -58,6 +60,7 @@
     "idtoken",
     "Iflag",
     "incpatch",
+    "initialisms",
     "ipnstate",
     "isatty",
     "Jobf",
@@ -111,18 +114,19 @@
     "Signup",
     "slogtest",
     "sourcemapped",
+    "spinbutton",
     "Srcs",
     "stdbuf",
     "stretchr",
     "STTY",
     "stuntest",
-    "tanstack",
     "tailbroker",
     "tailcfg",
     "tailexchange",
     "tailnet",
     "tailnettest",
     "Tailscale",
+    "tanstack",
     "tbody",
     "TCGETS",
     "tcpip",
@@ -139,6 +143,7 @@
     "tios",
     "tmpdir",
     "tokenconfig",
+    "Topbar",
     "tparallel",
     "trialer",
     "trimprefix",
@@ -166,10 +171,10 @@
     "workspaceapps",
     "workspacebuilds",
     "workspacename",
-    "wsconncache",
     "wsjson",
     "xerrors",
-    "xstate",
+    "xlarge",
+    "xsmall",
     "yamux"
   ],
   "cSpell.ignorePaths": ["site/package.json", ".vscode/settings.json"],
@@ -186,19 +191,25 @@
     ]
   },
   "eslint.workingDirectories": ["./site"],
-  "files.exclude": {
-    "**/node_modules": true
-  },
   "search.exclude": {
+    "**.pb.go": true,
+    "**/*.gen.json": true,
+    "**/testdata/*": true,
+    "**Generated.ts": true,
+    "coderd/apidoc/**": true,
+    "docs/api/*.md": true,
+    "docs/templates/*.md": true,
+    "LICENSE": true,
     "scripts/metricsdocgen/metrics": true,
-    "docs/api/*.md": true
+    "site/out/**": true,
+    "site/storybook-static/**": true,
+    "**.map": true,
+    "pnpm-lock.yaml": true
   },
   // Ensure files always have a newline.
   "files.insertFinalNewline": true,
   "go.lintTool": "golangci-lint",
   "go.lintFlags": ["--fast"],
-  "go.lintOnSave": "package",
-  "go.coverOnSave": true,
   "go.coverageDecorator": {
     "type": "gutter",
     "coveredGutterStyle": "blockgreen",

Makefile

@@ -50,7 +50,7 @@ endif
 # Note, all find statements should be written with `.` or `./path` as
 # the search path so that these exclusions match.
 FIND_EXCLUSIONS= \
-	-not \( \( -path '*/.git/*' -o -path './build/*' -o -path './vendor/*' -o -path './.coderv2/*' -o -path '*/node_modules/*' -o -path '*/out/*' -o -path './coderd/apidoc/*' -o -path '*/.next/*' \) -prune \)
+	-not \( \( -path '*/.git/*' -o -path './build/*' -o -path './vendor/*' -o -path './.coderv2/*' -o -path '*/node_modules/*' -o -path '*/out/*' -o -path './coderd/apidoc/*' -o -path '*/.next/*' -o -path '*/.terraform/*' \) -prune \)
 # Source files used for make targets, evaluated on use.
 GO_SRC_FILES := $(shell find . $(FIND_EXCLUSIONS) -type f -name '*.go' -not -name '*_test.go')
 # All the shell files in the repo, excluding ignored files.
@@ -107,9 +107,9 @@ endif
 
 
 clean:
-	rm -rf build site/out
-	mkdir -p build site/out/bin
-	git restore site/out
+	rm -rf build/ site/build/ site/out/
+	mkdir -p build/ site/out/bin/
+	git restore site/out/
 .PHONY: clean
 
 build-slim: $(CODER_SLIM_BINARIES)
@@ -344,19 +344,25 @@ push/$(CODER_MAIN_IMAGE): $(CODER_MAIN_IMAGE)
 	docker manifest push "$$image_tag"
 .PHONY: push/$(CODER_MAIN_IMAGE)
 
+# Helm charts that are available
+charts = coder provisioner
+
 # Shortcut for Helm chart package.
-build/coder_helm.tgz: build/coder_helm_$(VERSION).tgz
+$(foreach chart,$(charts),build/$(chart)_helm.tgz): build/%_helm.tgz: build/%_helm_$(VERSION).tgz
 	rm -f "$@"
 	ln "$<" "$@"
 
 # Helm chart package.
-build/coder_helm_$(VERSION).tgz:
+$(foreach chart,$(charts),build/$(chart)_helm_$(VERSION).tgz): build/%_helm_$(VERSION).tgz:
 	./scripts/helm.sh \
 		--version "$(VERSION)" \
+		--chart $* \
 		--output "$@"
 
 site/out/index.html: site/package.json $(shell find ./site $(FIND_EXCLUSIONS) -type f \( -name '*.ts' -o -name '*.tsx' \))
 	cd site
+	# prevents this directory from getting to big, and causing "too much data" errors
+	rm -rf out/assets/
 	../scripts/pnpm_install.sh
 	pnpm build
 
@@ -376,32 +382,44 @@ install: build/coder_$(VERSION)_$(GOOS)_$(GOARCH)$(GOOS_BIN_EXT)
 	cp "$<" "$$output_file"
 .PHONY: install
 
-fmt: fmt/prettier fmt/terraform fmt/shfmt fmt/go
+BOLD := $(shell tput bold)
+GREEN := $(shell tput setaf 2)
+RESET := $(shell tput sgr0)
+
+fmt: fmt/eslint fmt/prettier fmt/terraform fmt/shfmt fmt/go
 .PHONY: fmt
 
 fmt/go:
+	echo "$(GREEN)==>$(RESET) $(BOLD)fmt/go$(RESET)"
 	# VS Code users should check out
 	# https://github.com/mvdan/gofumpt#visual-studio-code
 	go run mvdan.cc/gofumpt@v0.4.0 -w -l .
 .PHONY: fmt/go
 
+fmt/eslint:
+	echo "$(GREEN)==>$(RESET) $(BOLD)fmt/eslint$(RESET)"
+	cd site
+	pnpm run lint:fix
+.PHONY: fmt/eslint
+
 fmt/prettier:
-	echo "--- prettier"
+	echo "$(GREEN)==>$(RESET) $(BOLD)fmt/prettier$(RESET)"
 	cd site
 # Avoid writing files in CI to reduce file write activity
 ifdef CI
 	pnpm run format:check
 else
-	pnpm run format:write
+	pnpm run format
 endif
 .PHONY: fmt/prettier
 
 fmt/terraform: $(wildcard *.tf)
+	echo "$(GREEN)==>$(RESET) $(BOLD)fmt/terraform$(RESET)"
 	terraform fmt -recursive
 .PHONY: fmt/terraform
 
 fmt/shfmt: $(SHELL_SRC_FILES)
-	echo "--- shfmt"
+	echo "$(GREEN)==>$(RESET) $(BOLD)fmt/shfmt$(RESET)"
 # Only do diff check in CI, errors on diff.
 ifdef CI
 	shfmt -d $(SHELL_SRC_FILES)
@@ -415,7 +433,6 @@ lint: lint/shellcheck lint/go lint/ts lint/helm lint/site-icons
 
 lint/site-icons:
 	./scripts/check_site_icons.sh
-
 .PHONY: lint/site-icons
 
 lint/ts:
@@ -425,7 +442,8 @@ lint/ts:
 
 lint/go:
 	./scripts/check_enterprise_imports.sh
-	go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.53.2
+	linter_ver=$(shell egrep -o 'GOLANGCI_LINT_VERSION=\S+' dogfood/Dockerfile | cut -d '=' -f 2)
+	go install github.com/golangci/golangci-lint/cmd/golangci-lint@v$$linter_ver
 	golangci-lint run
 .PHONY: lint/go
 
@@ -445,17 +463,19 @@ lint/helm:
 DB_GEN_FILES := \
 	coderd/database/querier.go \
 	coderd/database/unique_constraint.go \
-	coderd/database/dbfake/dbfake.go \
+	coderd/database/dbmem/dbmem.go \
 	coderd/database/dbmetrics/dbmetrics.go \
 	coderd/database/dbauthz/dbauthz.go \
 	coderd/database/dbmock/dbmock.go
 
 # all gen targets should be added here and to gen/mark-fresh
 gen: \
-	coderd/database/dump.sql \
-	$(DB_GEN_FILES) \
+	tailnet/proto/tailnet.pb.go \
+	agent/proto/agent.pb.go \
 	provisionersdk/proto/provisioner.pb.go \
 	provisionerd/proto/provisionerd.pb.go \
+	coderd/database/dump.sql \
+	$(DB_GEN_FILES) \
 	site/src/api/typesGenerated.ts \
 	coderd/rbac/object_gen.go \
 	docs/admin/prometheus.md \
@@ -466,17 +486,25 @@ gen: \
 	.prettierignore \
 	site/.prettierrc.yaml \
 	site/.prettierignore \
-	site/.eslintignore
+	site/.eslintignore \
+	site/e2e/provisionerGenerated.ts \
+	site/src/theme/icons.json \
+	examples/examples.gen.json \
+	tailnet/tailnettest/coordinatormock.go \
+	tailnet/tailnettest/coordinateemock.go \
+	tailnet/tailnettest/multiagentmock.go
 .PHONY: gen
 
 # Mark all generated files as fresh so make thinks they're up-to-date. This is
 # used during releases so we don't run generation scripts.
 gen/mark-fresh:
 	files="\
-		coderd/database/dump.sql \
-		$(DB_GEN_FILES) \
+		tailnet/proto/tailnet.pb.go \
+		agent/proto/agent.pb.go \
 		provisionersdk/proto/provisioner.pb.go \
 		provisionerd/proto/provisionerd.pb.go \
+		coderd/database/dump.sql \
+		$(DB_GEN_FILES) \
 		site/src/api/typesGenerated.ts \
 		coderd/rbac/object_gen.go \
 		docs/admin/prometheus.md \
@@ -488,6 +516,12 @@ gen/mark-fresh:
 		site/.prettierrc.yaml \
 		site/.prettierignore \
 		site/.eslintignore \
+		site/e2e/provisionerGenerated.ts \
+		site/src/theme/icons.json \
+		examples/examples.gen.json \
+		tailnet/tailnettest/coordinatormock.go \
+		tailnet/tailnettest/coordinateemock.go \
+		tailnet/tailnettest/multiagentmock.go \
 	"
 	for file in $$files; do
 		echo "$$file"
@@ -507,12 +541,33 @@ coderd/database/dump.sql: coderd/database/gen/dump/main.go $(wildcard coderd/dat
 	go run ./coderd/database/gen/dump/main.go
 
 # Generates Go code for querying the database.
+# coderd/database/queries.sql.go
+# coderd/database/models.go
 coderd/database/querier.go: coderd/database/sqlc.yaml coderd/database/dump.sql $(wildcard coderd/database/queries/*.sql)
 	./coderd/database/generate.sh
 
 coderd/database/dbmock/dbmock.go: coderd/database/db.go coderd/database/querier.go
 	go generate ./coderd/database/dbmock/
 
+tailnet/tailnettest/coordinatormock.go tailnet/tailnettest/multiagentmock.go tailnet/tailnettest/coordinateemock.go: tailnet/coordinator.go tailnet/multiagent.go
+	go generate ./tailnet/tailnettest/
+
+tailnet/proto/tailnet.pb.go: tailnet/proto/tailnet.proto
+	protoc \
+		--go_out=. \
+		--go_opt=paths=source_relative \
+		--go-drpc_out=. \
+		--go-drpc_opt=paths=source_relative \
+		./tailnet/proto/tailnet.proto
+
+agent/proto/agent.pb.go: agent/proto/agent.proto
+	protoc \
+		--go_out=. \
+		--go_opt=paths=source_relative \
+		--go-drpc_out=. \
+		--go-drpc_opt=paths=source_relative \
+		./agent/proto/agent.proto
+
 provisionersdk/proto/provisioner.pb.go: provisionersdk/proto/provisioner.proto
 	protoc \
 		--go_out=. \
@@ -529,31 +584,57 @@ provisionerd/proto/provisionerd.pb.go: provisionerd/proto/provisionerd.proto
 		--go-drpc_opt=paths=source_relative \
 		./provisionerd/proto/provisionerd.proto
 
-site/src/api/typesGenerated.ts: scripts/apitypings/main.go $(shell find ./codersdk $(FIND_EXCLUSIONS) -type f -name '*.go')
-	go run scripts/apitypings/main.go > site/src/api/typesGenerated.ts
+site/src/api/typesGenerated.ts: $(wildcard scripts/apitypings/*) $(shell find ./codersdk $(FIND_EXCLUSIONS) -type f -name '*.go')
+	go run ./scripts/apitypings/ > $@
+	./scripts/pnpm_install.sh
+	pnpm exec prettier --write "$@"
+
+site/e2e/provisionerGenerated.ts: provisionerd/proto/provisionerd.pb.go provisionersdk/proto/provisioner.pb.go
 	cd site
-	pnpm run format:types
+	../scripts/pnpm_install.sh
+	pnpm run gen:provisioner
+
+site/src/theme/icons.json: $(wildcard scripts/gensite/*) $(wildcard site/static/icon/*)
+	go run ./scripts/gensite/ -icons "$@"
+	./scripts/pnpm_install.sh
+	pnpm exec prettier --write "$@"
+
+examples/examples.gen.json: scripts/examplegen/main.go examples/examples.go $(shell find ./examples/templates)
+	go run ./scripts/examplegen/main.go > examples/examples.gen.json
 
 coderd/rbac/object_gen.go: scripts/rbacgen/main.go coderd/rbac/object.go
 	go run scripts/rbacgen/main.go ./coderd/rbac > coderd/rbac/object_gen.go
 
 docs/admin/prometheus.md: scripts/metricsdocgen/main.go scripts/metricsdocgen/metrics
 	go run scripts/metricsdocgen/main.go
-	pnpm run format:write:only ./docs/admin/prometheus.md
+	./scripts/pnpm_install.sh
+	pnpm exec prettier --write ./docs/admin/prometheus.md
 
-docs/cli.md: scripts/clidocgen/main.go $(GO_SRC_FILES)
-	BASE_PATH="." go run ./scripts/clidocgen
-	pnpm run format:write:only ./docs/cli.md ./docs/cli/*.md ./docs/manifest.json
+docs/cli.md: scripts/clidocgen/main.go examples/examples.gen.json $(GO_SRC_FILES)
+	CI=true BASE_PATH="." go run ./scripts/clidocgen
+	./scripts/pnpm_install.sh
+	pnpm exec prettier --write ./docs/cli.md ./docs/cli/*.md ./docs/manifest.json
 
-docs/admin/audit-logs.md: scripts/auditdocgen/main.go enterprise/audit/table.go coderd/rbac/object_gen.go
+docs/admin/audit-logs.md: coderd/database/querier.go scripts/auditdocgen/main.go enterprise/audit/table.go coderd/rbac/object_gen.go
 	go run scripts/auditdocgen/main.go
-	pnpm run format:write:only ./docs/admin/audit-logs.md
+	./scripts/pnpm_install.sh
+	pnpm exec prettier --write ./docs/admin/audit-logs.md
 
 coderd/apidoc/swagger.json: $(shell find ./scripts/apidocgen $(FIND_EXCLUSIONS) -type f) $(wildcard coderd/*.go) $(wildcard enterprise/coderd/*.go) $(wildcard codersdk/*.go) $(wildcard enterprise/wsproxy/wsproxysdk/*.go) $(DB_GEN_FILES) .swaggo docs/manifest.json coderd/rbac/object_gen.go
 	./scripts/apidocgen/generate.sh
-	pnpm run format:write:only ./docs/api ./docs/manifest.json ./coderd/apidoc/swagger.json
+	./scripts/pnpm_install.sh
+	pnpm exec prettier --write ./docs/api ./docs/manifest.json ./coderd/apidoc/swagger.json
 
-update-golden-files: cli/testdata/.gen-golden helm/tests/testdata/.gen-golden scripts/ci-report/testdata/.gen-golden enterprise/cli/testdata/.gen-golden
+update-golden-files: \
+	cli/testdata/.gen-golden \
+	helm/coder/tests/testdata/.gen-golden \
+	helm/provisioner/tests/testdata/.gen-golden \
+	scripts/ci-report/testdata/.gen-golden \
+	enterprise/cli/testdata/.gen-golden \
+	enterprise/tailnet/testdata/.gen-golden \
+	tailnet/testdata/.gen-golden \
+	coderd/.gen-golden \
+	provisioner/terraform/testdata/.gen-golden
 .PHONY: update-golden-files
 
 cli/testdata/.gen-golden: $(wildcard cli/testdata/*.golden) $(wildcard cli/*.tpl) $(GO_SRC_FILES) $(wildcard cli/*_test.go)
@@ -564,8 +645,28 @@ enterprise/cli/testdata/.gen-golden: $(wildcard enterprise/cli/testdata/*.golden
 	go test ./enterprise/cli -run="TestEnterpriseCommandHelp" -update
 	touch "$@"
 
-helm/tests/testdata/.gen-golden: $(wildcard helm/tests/testdata/*.yaml) $(wildcard helm/tests/testdata/*.golden) $(GO_SRC_FILES) $(wildcard helm/tests/*_test.go)
-	go test ./helm/tests -run=TestUpdateGoldenFiles -update
+tailnet/testdata/.gen-golden: $(wildcard tailnet/testdata/*.golden.html) $(GO_SRC_FILES) $(wildcard tailnet/*_test.go)
+	go test ./tailnet -run="TestDebugTemplate" -update
+	touch "$@"
+
+enterprise/tailnet/testdata/.gen-golden: $(wildcard enterprise/tailnet/testdata/*.golden.html) $(GO_SRC_FILES) $(wildcard enterprise/tailnet/*_test.go)
+	go test ./enterprise/tailnet -run="TestDebugTemplate" -update
+	touch "$@"
+
+helm/coder/tests/testdata/.gen-golden: $(wildcard helm/coder/tests/testdata/*.yaml) $(wildcard helm/coder/tests/testdata/*.golden) $(GO_SRC_FILES) $(wildcard helm/coder/tests/*_test.go)
+	go test ./helm/coder/tests -run=TestUpdateGoldenFiles -update
+	touch "$@"
+
+helm/provisioner/tests/testdata/.gen-golden: $(wildcard helm/provisioner/tests/testdata/*.yaml) $(wildcard helm/provisioner/tests/testdata/*.golden) $(GO_SRC_FILES) $(wildcard helm/provisioner/tests/*_test.go)
+	go test ./helm/provisioner/tests -run=TestUpdateGoldenFiles -update
+	touch "$@"
+
+coderd/.gen-golden: $(wildcard coderd/testdata/*/*.golden) $(GO_SRC_FILES) $(wildcard coderd/*_test.go)
+	go test ./coderd -run="Test.*Golden$$" -update
+	touch "$@"
+
+provisioner/terraform/testdata/.gen-golden: $(wildcard provisioner/terraform/testdata/*/*.golden) $(GO_SRC_FILES) $(wildcard provisioner/terraform/*_test.go)
+	go test ./provisioner/terraform -run="Test.*Golden$$" -update
 	touch "$@"
 
 scripts/ci-report/testdata/.gen-golden: $(wildcard scripts/ci-report/testdata/*) $(wildcard scripts/ci-report/*.go)
@@ -586,7 +687,7 @@ site/.prettierrc.yaml: .prettierrc.yaml
 	# - ./ -> ../
 	# - ./site -> ./
 	yq \
-		'.overrides[].files |= map(. | sub("^./"; "") | sub("^"; "../") | sub("../site/"; "./"))' \
+		'.overrides[].files |= map(. | sub("^./"; "") | sub("^"; "../") | sub("../site/"; "./") | sub("../!"; "!../"))' \
 		"$<" >> "$@"
 
 # Combine .gitignore with .prettierignore.include to generate .prettierignore.
@@ -636,6 +737,33 @@ test:
 	gotestsum --format standard-quiet -- -v -short -count=1 ./...
 .PHONY: test
 
+# sqlc-cloud-is-setup will fail if no SQLc auth token is set. Use this as a
+# dependency for any sqlc-cloud related targets.
+sqlc-cloud-is-setup:
+	if [[ "$(SQLC_AUTH_TOKEN)" == "" ]]; then
+		echo "ERROR: 'SQLC_AUTH_TOKEN' must be set to auth with sqlc cloud before running verify." 1>&2
+		exit 1
+	fi
+.PHONY: sqlc-cloud-is-setup
+
+sqlc-push: sqlc-cloud-is-setup test-postgres-docker
+	echo "--- sqlc push"
+	SQLC_DATABASE_URL="postgresql://postgres:postgres@localhost:5432/$(shell go run scripts/migrate-ci/main.go)" \
+	sqlc push -f coderd/database/sqlc.yaml && echo "Passed sqlc push"
+.PHONY: sqlc-push
+
+sqlc-verify: sqlc-cloud-is-setup test-postgres-docker
+	echo "--- sqlc verify"
+	SQLC_DATABASE_URL="postgresql://postgres:postgres@localhost:5432/$(shell go run scripts/migrate-ci/main.go)" \
+	sqlc verify -f coderd/database/sqlc.yaml && echo "Passed sqlc verify"
+.PHONY: sqlc-verify
+
+sqlc-vet: test-postgres-docker
+	echo "--- sqlc vet"
+	SQLC_DATABASE_URL="postgresql://postgres:postgres@localhost:5432/$(shell go run scripts/migrate-ci/main.go)" \
+	sqlc vet -f coderd/database/sqlc.yaml && echo "Passed sqlc vet"
+.PHONY: sqlc-vet
+
 # When updating -timeout for this test, keep in sync with
 # test-go-postgres (.github/workflows/coder.yaml).
 # Do add coverage flags so that test caching works.

README.md

@@ -7,7 +7,7 @@
   </a>
 
   <h1>
-  Self-Hosted Remote Development Environments
+  Self-Hosted Cloud Development Environments
   </h1>
 
   <a href="https://coder.com#gh-light-mode-only">
@@ -31,9 +31,9 @@
 
 </div>
 
-[Coder](https://coder.com) enables organizations to set up development environments in the cloud. Environments are defined with Terraform, connected through a secure high-speed Wireguard® tunnel, and are automatically shut down when not in use to save on costs. Coder gives engineering teams the flexibility to use the cloud for workloads that are most beneficial to them.
+[Coder](https://coder.com) enables organizations to set up development environments in their public or private cloud infrastructure. Cloud development environments are defined with Terraform, connected through a secure high-speed Wireguard® tunnel, and are automatically shut down when not in use to save on costs. Coder gives engineering teams the flexibility to use the cloud for workloads that are most beneficial to them.
 
-- Define development environments in Terraform
+- Define cloud development environments in Terraform
   - EC2 VMs, Kubernetes Pods, Docker Containers, etc.
 - Automatically shutdown idle resources to save on costs
 - Onboard developers in seconds instead of days
@@ -44,7 +44,7 @@
 
 ## Quickstart
 
-The most convenient way to try Coder is to install it on your local machine and experiment with provisioning development environments using Docker (works on Linux, macOS, and Windows).
+The most convenient way to try Coder is to install it on your local machine and experiment with provisioning cloud development environments using Docker (works on Linux, macOS, and Windows).
 
 ```
 # First, install Coder
@@ -70,11 +70,11 @@ curl -L https://coder.com/install.sh | sh
 
 You can run the install script with `--dry-run` to see the commands that will be used to install without executing them. You can modify the installation process by including flags. Run the install script with `--help` for reference.
 
-> See [install](docs/install) for additional methods.
+> See [install](https://coder.com/docs/v2/latest/install) for additional methods.
 
 Once installed, you can start a production deployment<sup>1</sup> with a single command:
 
-```console
+```shell
 # Automatically sets up an external access URL on *.try.coder.app
 coder server
 
@@ -100,7 +100,7 @@ Browse our docs [here](https://coder.com/docs/v2) or visit a specific section be
 
 Feel free to [open an issue](https://github.com/coder/coder/issues/new) if you have questions, run into bugs, or have a feature request.
 
-[Join our Discord](https://discord.gg/coder) to provide feedback on in-progress features, and chat with the community using Coder!
+[Join our Discord](https://discord.gg/coder) or [Slack](https://cdr.co/join-community) to provide feedback on in-progress features, and chat with the community using Coder!
 
 ## Contributing
 

SECURITY.md

@@ -1,7 +1,7 @@
 # Coder Security
 
-Coder welcomes feedback from security researchers and the general public
-to help improve our security. If you believe you have discovered a vulnerability,
+Coder welcomes feedback from security researchers and the general public to help
+improve our security. If you believe you have discovered a vulnerability,
 privacy issue, exposed data, or other security issues in any of our assets, we
 want to hear from you. This policy outlines steps for reporting vulnerabilities
 to us, what we expect, what you can expect from us.
@@ -10,64 +10,72 @@ You can see the pretty version [here](https://coder.com/security/policy)
 
 # Why Coder's security matters
 
-If an attacker could fully compromise a Coder installation, they could spin
-up expensive workstations, steal valuable credentials, or steal proprietary
-source code. We take this risk very seriously and employ routine pen testing,
-vulnerability scanning, and code reviews. We also welcome the contributions
-from the community that helped make this product possible.
+If an attacker could fully compromise a Coder installation, they could spin up
+expensive workstations, steal valuable credentials, or steal proprietary source
+code. We take this risk very seriously and employ routine pen testing,
+vulnerability scanning, and code reviews. We also welcome the contributions from
+the community that helped make this product possible.
 
 # Where should I report security issues?
 
-Please report security issues to security@coder.com, providing
-all relevant information. The more details you provide, the easier it will be
-for us to triage and fix the issue.
+Please report security issues to security@coder.com, providing all relevant
+information. The more details you provide, the easier it will be for us to
+triage and fix the issue.
 
 # Out of Scope
 
-Our primary concern is around an abuse of the Coder application that allows
-an attacker to gain access to another users workspace, or spin up unwanted
+Our primary concern is around an abuse of the Coder application that allows an
+attacker to gain access to another users workspace, or spin up unwanted
 workspaces.
 
 - DOS/DDOS attacks affecting availability --> While we do support rate limiting
-  of requests, we primarily leave this to the owner of the Coder installation. Our
-  rationale is that a DOS attack only affecting availability is not a valuable
-  target for attackers.
+  of requests, we primarily leave this to the owner of the Coder installation.
+  Our rationale is that a DOS attack only affecting availability is not a
+  valuable target for attackers.
 - Abuse of a compromised user credential --> If a user credential is compromised
-  outside of the Coder ecosystem, then we consider it beyond the scope of our application.
-  However, if an unprivileged user could escalate their permissions or gain access
-  to another workspace, that is a cause for concern.
+  outside of the Coder ecosystem, then we consider it beyond the scope of our
+  application. However, if an unprivileged user could escalate their permissions
+  or gain access to another workspace, that is a cause for concern.
 - Vulnerabilities in third party systems --> Vulnerabilities discovered in
-  out-of-scope systems should be reported to the appropriate vendor or applicable authority.
+  out-of-scope systems should be reported to the appropriate vendor or
+  applicable authority.
 
 # Our Commitments
 
 When working with us, according to this policy, you can expect us to:
 
-- Respond to your report promptly, and work with you to understand and validate your report;
-- Strive to keep you informed about the progress of a vulnerability as it is processed;
-- Work to remediate discovered vulnerabilities in a timely manner, within our operational constraints; and
-- Extend Safe Harbor for your vulnerability research that is related to this policy.
+- Respond to your report promptly, and work with you to understand and validate
+  your report;
+- Strive to keep you informed about the progress of a vulnerability as it is
+  processed;
+- Work to remediate discovered vulnerabilities in a timely manner, within our
+  operational constraints; and
+- Extend Safe Harbor for your vulnerability research that is related to this
+  policy.
 
 # Our Expectations
 
-In participating in our vulnerability disclosure program in good faith, we ask that you:
+In participating in our vulnerability disclosure program in good faith, we ask
+that you:
 
-- Play by the rules, including following this policy and any other relevant agreements.
-  If there is any inconsistency between this policy and any other applicable terms, the
-  terms of this policy will prevail;
+- Play by the rules, including following this policy and any other relevant
+  agreements. If there is any inconsistency between this policy and any other
+  applicable terms, the terms of this policy will prevail;
 - Report any vulnerability you’ve discovered promptly;
-- Avoid violating the privacy of others, disrupting our systems, destroying data, and/or
-  harming user experience;
+- Avoid violating the privacy of others, disrupting our systems, destroying
+  data, and/or harming user experience;
 - Use only the Official Channels to discuss vulnerability information with us;
-- Provide us a reasonable amount of time (at least 90 days from the initial report) to
-  resolve the issue before you disclose it publicly;
-- Perform testing only on in-scope systems, and respect systems and activities which
-  are out-of-scope;
-- If a vulnerability provides unintended access to data: Limit the amount of data you
-  access to the minimum required for effectively demonstrating a Proof of Concept; and
-  cease testing and submit a report immediately if you encounter any user data during testing,
-  such as Personally Identifiable Information (PII), Personal Healthcare Information (PHI),
-  credit card data, or proprietary information;
-- You should only interact with test accounts you own or with explicit permission from
+- Provide us a reasonable amount of time (at least 90 days from the initial
+  report) to resolve the issue before you disclose it publicly;
+- Perform testing only on in-scope systems, and respect systems and activities
+  which are out-of-scope;
+- If a vulnerability provides unintended access to data: Limit the amount of
+  data you access to the minimum required for effectively demonstrating a Proof
+  of Concept; and cease testing and submit a report immediately if you encounter
+  any user data during testing, such as Personally Identifiable Information
+  (PII), Personal Healthcare Information (PHI), credit card data, or proprietary
+  information;
+- You should only interact with test accounts you own or with explicit
+  permission from
 - the account holder; and
 - Do not engage in extortion.

agent/agentproc/agentproctest/doc.go

@@ -0,0 +1,5 @@
+// Package agentproctest contains utility functions
+// for testing process management in the agent.
+package agentproctest
+
+//go:generate mockgen -destination ./syscallermock.go -package agentproctest github.com/coder/coder/v2/agent/agentproc Syscaller

agent/agentproc/agentproctest/proc.go

@@ -0,0 +1,49 @@
+package agentproctest
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/spf13/afero"
+	"github.com/stretchr/testify/require"
+
+	"github.com/coder/coder/v2/agent/agentproc"
+	"github.com/coder/coder/v2/cryptorand"
+)
+
+func GenerateProcess(t *testing.T, fs afero.Fs, muts ...func(*agentproc.Process)) agentproc.Process {
+	t.Helper()
+
+	pid, err := cryptorand.Intn(1<<31 - 1)
+	require.NoError(t, err)
+
+	arg1, err := cryptorand.String(5)
+	require.NoError(t, err)
+
+	arg2, err := cryptorand.String(5)
+	require.NoError(t, err)
+
+	arg3, err := cryptorand.String(5)
+	require.NoError(t, err)
+
+	cmdline := fmt.Sprintf("%s\x00%s\x00%s", arg1, arg2, arg3)
+
+	process := agentproc.Process{
+		CmdLine: cmdline,
+		PID:     int32(pid),
+	}
+
+	for _, mut := range muts {
+		mut(&process)
+	}
+
+	process.Dir = fmt.Sprintf("%s/%d", "/proc", process.PID)
+
+	err = fs.MkdirAll(process.Dir, 0o555)
+	require.NoError(t, err)
+
+	err = afero.WriteFile(fs, fmt.Sprintf("%s/cmdline", process.Dir), []byte(process.CmdLine), 0o444)
+	require.NoError(t, err)
+
+	return process
+}

agent/agentproc/agentproctest/syscallermock.go

@@ -0,0 +1,83 @@
+// Code generated by MockGen. DO NOT EDIT.
+// Source: github.com/coder/coder/v2/agent/agentproc (interfaces: Syscaller)
+//
+// Generated by this command:
+//
+//	mockgen -destination ./syscallermock.go -package agentproctest github.com/coder/coder/v2/agent/agentproc Syscaller
+//
+
+// Package agentproctest is a generated GoMock package.
+package agentproctest
+
+import (
+	reflect "reflect"
+	syscall "syscall"
+
+	gomock "go.uber.org/mock/gomock"
+)
+
+// MockSyscaller is a mock of Syscaller interface.
+type MockSyscaller struct {
+	ctrl     *gomock.Controller
+	recorder *MockSyscallerMockRecorder
+}
+
+// MockSyscallerMockRecorder is the mock recorder for MockSyscaller.
+type MockSyscallerMockRecorder struct {
+	mock *MockSyscaller
+}
+
+// NewMockSyscaller creates a new mock instance.
+func NewMockSyscaller(ctrl *gomock.Controller) *MockSyscaller {
+	mock := &MockSyscaller{ctrl: ctrl}
+	mock.recorder = &MockSyscallerMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use.
+func (m *MockSyscaller) EXPECT() *MockSyscallerMockRecorder {
+	return m.recorder
+}
+
+// GetPriority mocks base method.
+func (m *MockSyscaller) GetPriority(arg0 int32) (int, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetPriority", arg0)
+	ret0, _ := ret[0].(int)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// GetPriority indicates an expected call of GetPriority.
+func (mr *MockSyscallerMockRecorder) GetPriority(arg0 any) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPriority", reflect.TypeOf((*MockSyscaller)(nil).GetPriority), arg0)
+}
+
+// Kill mocks base method.
+func (m *MockSyscaller) Kill(arg0 int32, arg1 syscall.Signal) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Kill", arg0, arg1)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// Kill indicates an expected call of Kill.
+func (mr *MockSyscallerMockRecorder) Kill(arg0, arg1 any) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Kill", reflect.TypeOf((*MockSyscaller)(nil).Kill), arg0, arg1)
+}
+
+// SetPriority mocks base method.
+func (m *MockSyscaller) SetPriority(arg0 int32, arg1 int) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "SetPriority", arg0, arg1)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// SetPriority indicates an expected call of SetPriority.
+func (mr *MockSyscallerMockRecorder) SetPriority(arg0, arg1 any) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetPriority", reflect.TypeOf((*MockSyscaller)(nil).SetPriority), arg0, arg1)
+}

agent/agentproc/doc.go

@@ -0,0 +1,3 @@
+// Package agentproc contains logic for interfacing with local
+// processes running in the same context as the agent.
+package agentproc

agent/agentproc/proc_other.go

@@ -0,0 +1,24 @@
+//go:build !linux
+// +build !linux
+
+package agentproc
+
+import (
+	"github.com/spf13/afero"
+)
+
+func (*Process) Niceness(Syscaller) (int, error) {
+	return 0, errUnimplemented
+}
+
+func (*Process) SetNiceness(Syscaller, int) error {
+	return errUnimplemented
+}
+
+func (*Process) Cmd() string {
+	return ""
+}
+
+func List(afero.Fs, Syscaller) ([]*Process, error) {
+	return nil, errUnimplemented
+}

agent/agentproc/proc_test.go

@@ -0,0 +1,166 @@
+package agentproc_test
+
+import (
+	"runtime"
+	"syscall"
+	"testing"
+
+	"github.com/spf13/afero"
+	"github.com/stretchr/testify/require"
+	"go.uber.org/mock/gomock"
+	"golang.org/x/xerrors"
+
+	"github.com/coder/coder/v2/agent/agentproc"
+	"github.com/coder/coder/v2/agent/agentproc/agentproctest"
+)
+
+func TestList(t *testing.T) {
+	t.Parallel()
+
+	if runtime.GOOS != "linux" {
+		t.Skipf("skipping non-linux environment")
+	}
+
+	t.Run("OK", func(t *testing.T) {
+		t.Parallel()
+
+		var (
+			fs            = afero.NewMemMapFs()
+			sc            = agentproctest.NewMockSyscaller(gomock.NewController(t))
+			expectedProcs = make(map[int32]agentproc.Process)
+		)
+
+		for i := 0; i < 4; i++ {
+			proc := agentproctest.GenerateProcess(t, fs)
+			expectedProcs[proc.PID] = proc
+
+			sc.EXPECT().
+				Kill(proc.PID, syscall.Signal(0)).
+				Return(nil)
+		}
+
+		actualProcs, err := agentproc.List(fs, sc)
+		require.NoError(t, err)
+		require.Len(t, actualProcs, len(expectedProcs))
+		for _, proc := range actualProcs {
+			expected, ok := expectedProcs[proc.PID]
+			require.True(t, ok)
+			require.Equal(t, expected.PID, proc.PID)
+			require.Equal(t, expected.CmdLine, proc.CmdLine)
+			require.Equal(t, expected.Dir, proc.Dir)
+		}
+	})
+
+	t.Run("FinishedProcess", func(t *testing.T) {
+		t.Parallel()
+
+		var (
+			fs            = afero.NewMemMapFs()
+			sc            = agentproctest.NewMockSyscaller(gomock.NewController(t))
+			expectedProcs = make(map[int32]agentproc.Process)
+		)
+
+		for i := 0; i < 3; i++ {
+			proc := agentproctest.GenerateProcess(t, fs)
+			expectedProcs[proc.PID] = proc
+
+			sc.EXPECT().
+				Kill(proc.PID, syscall.Signal(0)).
+				Return(nil)
+		}
+
+		// Create a process that's already finished. We're not adding
+		// it to the map because it should be skipped over.
+		proc := agentproctest.GenerateProcess(t, fs)
+		sc.EXPECT().
+			Kill(proc.PID, syscall.Signal(0)).
+			Return(xerrors.New("os: process already finished"))
+
+		actualProcs, err := agentproc.List(fs, sc)
+		require.NoError(t, err)
+		require.Len(t, actualProcs, len(expectedProcs))
+		for _, proc := range actualProcs {
+			expected, ok := expectedProcs[proc.PID]
+			require.True(t, ok)
+			require.Equal(t, expected.PID, proc.PID)
+			require.Equal(t, expected.CmdLine, proc.CmdLine)
+			require.Equal(t, expected.Dir, proc.Dir)
+		}
+	})
+
+	t.Run("NoSuchProcess", func(t *testing.T) {
+		t.Parallel()
+
+		var (
+			fs            = afero.NewMemMapFs()
+			sc            = agentproctest.NewMockSyscaller(gomock.NewController(t))
+			expectedProcs = make(map[int32]agentproc.Process)
+		)
+
+		for i := 0; i < 3; i++ {
+			proc := agentproctest.GenerateProcess(t, fs)
+			expectedProcs[proc.PID] = proc
+
+			sc.EXPECT().
+				Kill(proc.PID, syscall.Signal(0)).
+				Return(nil)
+		}
+
+		// Create a process that doesn't exist. We're not adding
+		// it to the map because it should be skipped over.
+		proc := agentproctest.GenerateProcess(t, fs)
+		sc.EXPECT().
+			Kill(proc.PID, syscall.Signal(0)).
+			Return(syscall.ESRCH)
+
+		actualProcs, err := agentproc.List(fs, sc)
+		require.NoError(t, err)
+		require.Len(t, actualProcs, len(expectedProcs))
+		for _, proc := range actualProcs {
+			expected, ok := expectedProcs[proc.PID]
+			require.True(t, ok)
+			require.Equal(t, expected.PID, proc.PID)
+			require.Equal(t, expected.CmdLine, proc.CmdLine)
+			require.Equal(t, expected.Dir, proc.Dir)
+		}
+	})
+}
+
+// These tests are not very interesting but they provide some modicum of
+// confidence.
+func TestProcess(t *testing.T) {
+	t.Parallel()
+
+	if runtime.GOOS != "linux" {
+		t.Skipf("skipping non-linux environment")
+	}
+
+	t.Run("SetNiceness", func(t *testing.T) {
+		t.Parallel()
+
+		var (
+			sc   = agentproctest.NewMockSyscaller(gomock.NewController(t))
+			proc = &agentproc.Process{
+				PID: 32,
+			}
+			score = 20
+		)
+
+		sc.EXPECT().SetPriority(proc.PID, score).Return(nil)
+		err := proc.SetNiceness(sc, score)
+		require.NoError(t, err)
+	})
+
+	t.Run("Cmd", func(t *testing.T) {
+		t.Parallel()
+
+		var (
+			proc = &agentproc.Process{
+				CmdLine: "helloworld\x00--arg1\x00--arg2",
+			}
+			expectedName = "helloworld --arg1 --arg2"
+		)
+
+		require.Equal(t, expectedName, proc.Cmd())
+	})
+}

agent/agentproc/proc_unix.go

@@ -0,0 +1,109 @@
+//go:build linux
+// +build linux
+
+package agentproc
+
+import (
+	"errors"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"syscall"
+
+	"github.com/spf13/afero"
+	"golang.org/x/xerrors"
+)
+
+func List(fs afero.Fs, syscaller Syscaller) ([]*Process, error) {
+	d, err := fs.Open(defaultProcDir)
+	if err != nil {
+		return nil, xerrors.Errorf("open dir %q: %w", defaultProcDir, err)
+	}
+	defer d.Close()
+
+	entries, err := d.Readdirnames(0)
+	if err != nil {
+		return nil, xerrors.Errorf("readdirnames: %w", err)
+	}
+
+	processes := make([]*Process, 0, len(entries))
+	for _, entry := range entries {
+		pid, err := strconv.ParseInt(entry, 10, 32)
+		if err != nil {
+			continue
+		}
+
+		// Check that the process still exists.
+		exists, err := isProcessExist(syscaller, int32(pid))
+		if err != nil {
+			return nil, xerrors.Errorf("check process exists: %w", err)
+		}
+		if !exists {
+			continue
+		}
+
+		cmdline, err := afero.ReadFile(fs, filepath.Join(defaultProcDir, entry, "cmdline"))
+		if err != nil {
+			var errNo syscall.Errno
+			if xerrors.As(err, &errNo) && errNo == syscall.EPERM {
+				continue
+			}
+			return nil, xerrors.Errorf("read cmdline: %w", err)
+		}
+		processes = append(processes, &Process{
+			PID:     int32(pid),
+			CmdLine: string(cmdline),
+			Dir:     filepath.Join(defaultProcDir, entry),
+		})
+	}
+
+	return processes, nil
+}
+
+func isProcessExist(syscaller Syscaller, pid int32) (bool, error) {
+	err := syscaller.Kill(pid, syscall.Signal(0))
+	if err == nil {
+		return true, nil
+	}
+	if err.Error() == "os: process already finished" {
+		return false, nil
+	}
+
+	var errno syscall.Errno
+	if !errors.As(err, &errno) {
+		return false, err
+	}
+
+	switch errno {
+	case syscall.ESRCH:
+		return false, nil
+	case syscall.EPERM:
+		return true, nil
+	}
+
+	return false, xerrors.Errorf("kill: %w", err)
+}
+
+func (p *Process) Niceness(sc Syscaller) (int, error) {
+	nice, err := sc.GetPriority(p.PID)
+	if err != nil {
+		return 0, xerrors.Errorf("get priority for %q: %w", p.CmdLine, err)
+	}
+	return nice, nil
+}
+
+func (p *Process) SetNiceness(sc Syscaller, score int) error {
+	err := sc.SetPriority(p.PID, score)
+	if err != nil {
+		return xerrors.Errorf("set priority for %q: %w", p.CmdLine, err)
+	}
+	return nil
+}
+
+func (p *Process) Cmd() string {
+	return strings.Join(p.cmdLine(), " ")
+}
+
+func (p *Process) cmdLine() []string {
+	return strings.Split(p.CmdLine, "\x00")
+}

agent/agentproc/syscaller.go

@@ -0,0 +1,20 @@
+package agentproc
+
+import (
+	"syscall"
+)
+
+type Syscaller interface {
+	SetPriority(pid int32, priority int) error
+	GetPriority(pid int32) (int, error)
+	Kill(pid int32, sig syscall.Signal) error
+}
+
+// nolint: unused // used on some but no all platforms
+const defaultProcDir = "/proc"
+
+type Process struct {
+	Dir     string
+	CmdLine string
+	PID     int32
+}

agent/agentproc/syscaller_other.go

@@ -0,0 +1,30 @@
+//go:build !linux
+// +build !linux
+
+package agentproc
+
+import (
+	"syscall"
+
+	"golang.org/x/xerrors"
+)
+
+func NewSyscaller() Syscaller {
+	return nopSyscaller{}
+}
+
+var errUnimplemented = xerrors.New("unimplemented")
+
+type nopSyscaller struct{}
+
+func (nopSyscaller) SetPriority(int32, int) error {
+	return errUnimplemented
+}
+
+func (nopSyscaller) GetPriority(int32) (int, error) {
+	return 0, errUnimplemented
+}
+
+func (nopSyscaller) Kill(int32, syscall.Signal) error {
+	return errUnimplemented
+}

agent/agentproc/syscaller_unix.go

@@ -0,0 +1,42 @@
+//go:build linux
+// +build linux
+
+package agentproc
+
+import (
+	"syscall"
+
+	"golang.org/x/sys/unix"
+	"golang.org/x/xerrors"
+)
+
+func NewSyscaller() Syscaller {
+	return UnixSyscaller{}
+}
+
+type UnixSyscaller struct{}
+
+func (UnixSyscaller) SetPriority(pid int32, nice int) error {
+	err := unix.Setpriority(unix.PRIO_PROCESS, int(pid), nice)
+	if err != nil {
+		return xerrors.Errorf("set priority: %w", err)
+	}
+	return nil
+}
+
+func (UnixSyscaller) GetPriority(pid int32) (int, error) {
+	nice, err := unix.Getpriority(0, int(pid))
+	if err != nil {
+		return 0, xerrors.Errorf("get priority: %w", err)
+	}
+	return nice, nil
+}
+
+func (UnixSyscaller) Kill(pid int32, sig syscall.Signal) error {
+	err := syscall.Kill(int(pid), sig)
+	if err != nil {
+		return xerrors.Errorf("kill: %w", err)
+	}
+
+	return nil
+}

agent/agentscripts/agentscripts.go

@@ -0,0 +1,400 @@
+package agentscripts
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"os/user"
+	"path/filepath"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/robfig/cron/v3"
+	"github.com/spf13/afero"
+	"golang.org/x/sync/errgroup"
+	"golang.org/x/xerrors"
+
+	"cdr.dev/slog"
+
+	"github.com/coder/coder/v2/agent/agentssh"
+	"github.com/coder/coder/v2/codersdk"
+	"github.com/coder/coder/v2/codersdk/agentsdk"
+)
+
+var (
+	// ErrTimeout is returned when a script times out.
+	ErrTimeout = xerrors.New("script timed out")
+	// ErrOutputPipesOpen is returned when a script exits leaving the output
+	// pipe(s) (stdout, stderr) open. This happens because we set WaitDelay on
+	// the command, which gives us two things:
+	//
+	// 1. The ability to ensure that a script exits (this is important for e.g.
+	//    blocking login, and avoiding doing so indefinitely)
+	// 2. Improved command cancellation on timeout
+	ErrOutputPipesOpen = xerrors.New("script exited without closing output pipes")
+
+	parser = cron.NewParser(cron.Second | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.DowOptional)
+)
+
+type ScriptLogger interface {
+	Send(ctx context.Context, log ...agentsdk.Log) error
+	Flush(context.Context) error
+}
+
+// Options are a set of options for the runner.
+type Options struct {
+	DataDirBase     string
+	LogDir          string
+	Logger          slog.Logger
+	SSHServer       *agentssh.Server
+	Filesystem      afero.Fs
+	GetScriptLogger func(logSourceID uuid.UUID) ScriptLogger
+}
+
+// New creates a runner for the provided scripts.
+func New(opts Options) *Runner {
+	cronCtx, cronCtxCancel := context.WithCancel(context.Background())
+	return &Runner{
+		Options:       opts,
+		cronCtx:       cronCtx,
+		cronCtxCancel: cronCtxCancel,
+		cron:          cron.New(cron.WithParser(parser)),
+		closed:        make(chan struct{}),
+		dataDir:       filepath.Join(opts.DataDirBase, "coder-script-data"),
+		scriptsExecuted: prometheus.NewCounterVec(prometheus.CounterOpts{
+			Namespace: "agent",
+			Subsystem: "scripts",
+			Name:      "executed_total",
+		}, []string{"success"}),
+	}
+}
+
+type Runner struct {
+	Options
+
+	cronCtx       context.Context
+	cronCtxCancel context.CancelFunc
+	cmdCloseWait  sync.WaitGroup
+	closed        chan struct{}
+	closeMutex    sync.Mutex
+	cron          *cron.Cron
+	initialized   atomic.Bool
+	scripts       []codersdk.WorkspaceAgentScript
+	dataDir       string
+
+	// scriptsExecuted includes all scripts executed by the workspace agent. Agents
+	// execute startup scripts, and scripts on a cron schedule. Both will increment
+	// this counter.
+	scriptsExecuted *prometheus.CounterVec
+}
+
+// DataDir returns the directory where scripts data is stored.
+func (r *Runner) DataDir() string {
+	return r.dataDir
+}
+
+// ScriptBinDir returns the directory where scripts can store executable
+// binaries.
+func (r *Runner) ScriptBinDir() string {
+	return filepath.Join(r.dataDir, "bin")
+}
+
+func (r *Runner) RegisterMetrics(reg prometheus.Registerer) {
+	if reg == nil {
+		// If no registry, do nothing.
+		return
+	}
+	reg.MustRegister(r.scriptsExecuted)
+}
+
+// Init initializes the runner with the provided scripts.
+// It also schedules any scripts that have a schedule.
+// This function must be called before Execute.
+func (r *Runner) Init(scripts []codersdk.WorkspaceAgentScript) error {
+	if r.initialized.Load() {
+		return xerrors.New("init: already initialized")
+	}
+	r.initialized.Store(true)
+	r.scripts = scripts
+	r.Logger.Info(r.cronCtx, "initializing agent scripts", slog.F("script_count", len(scripts)), slog.F("log_dir", r.LogDir))
+
+	err := r.Filesystem.MkdirAll(r.ScriptBinDir(), 0o700)
+	if err != nil {
+		return xerrors.Errorf("create script bin dir: %w", err)
+	}
+
+	for _, script := range scripts {
+		if script.Cron == "" {
+			continue
+		}
+		script := script
+		_, err := r.cron.AddFunc(script.Cron, func() {
+			err := r.trackRun(r.cronCtx, script)
+			if err != nil {
+				r.Logger.Warn(context.Background(), "run agent script on schedule", slog.Error(err))
+			}
+		})
+		if err != nil {
+			return xerrors.Errorf("add schedule: %w", err)
+		}
+	}
+	return nil
+}
+
+// StartCron starts the cron scheduler.
+// This is done async to allow for the caller to execute scripts prior.
+func (r *Runner) StartCron() {
+	// cron.Start() and cron.Stop() does not guarantee that the cron goroutine
+	// has exited by the time the `cron.Stop()` context returns, so we need to
+	// track it manually.
+	err := r.trackCommandGoroutine(func() {
+		// Since this is run async, in quick unit tests, it is possible the
+		// Close() function gets called before we even start the cron.
+		// In these cases, the Run() will never end.
+		// So if we are closed, we just return, and skip the Run() entirely.
+		select {
+		case <-r.cronCtx.Done():
+			// The cronCtx is canceled before cron.Close() happens. So if the ctx is
+			// canceled, then Close() will be called, or it is about to be called.
+			// So do nothing!
+		default:
+			r.cron.Run()
+		}
+	})
+	if err != nil {
+		r.Logger.Warn(context.Background(), "start cron failed", slog.Error(err))
+	}
+}
+
+// Execute runs a set of scripts according to a filter.
+func (r *Runner) Execute(ctx context.Context, filter func(script codersdk.WorkspaceAgentScript) bool) error {
+	if filter == nil {
+		// Execute em' all!
+		filter = func(script codersdk.WorkspaceAgentScript) bool {
+			return true
+		}
+	}
+	var eg errgroup.Group
+	for _, script := range r.scripts {
+		if !filter(script) {
+			continue
+		}
+		script := script
+		eg.Go(func() error {
+			err := r.trackRun(ctx, script)
+			if err != nil {
+				return xerrors.Errorf("run agent script %q: %w", script.LogSourceID, err)
+			}
+			return nil
+		})
+	}
+	return eg.Wait()
+}
+
+// trackRun wraps "run" with metrics.
+func (r *Runner) trackRun(ctx context.Context, script codersdk.WorkspaceAgentScript) error {
+	err := r.run(ctx, script)
+	if err != nil {
+		r.scriptsExecuted.WithLabelValues("false").Add(1)
+	} else {
+		r.scriptsExecuted.WithLabelValues("true").Add(1)
+	}
+	return err
+}
+
+// run executes the provided script with the timeout.
+// If the timeout is exceeded, the process is sent an interrupt signal.
+// If the process does not exit after a few seconds, it is forcefully killed.
+// This function immediately returns after a timeout, and does not wait for the process to exit.
+func (r *Runner) run(ctx context.Context, script codersdk.WorkspaceAgentScript) error {
+	logPath := script.LogPath
+	if logPath == "" {
+		logPath = fmt.Sprintf("coder-script-%s.log", script.LogSourceID)
+	}
+	if logPath[0] == '~' {
+		// First we check the environment.
+		homeDir, err := os.UserHomeDir()
+		if err != nil {
+			u, err := user.Current()
+			if err != nil {
+				return xerrors.Errorf("current user: %w", err)
+			}
+			homeDir = u.HomeDir
+		}
+		logPath = filepath.Join(homeDir, logPath[1:])
+	}
+	logPath = os.ExpandEnv(logPath)
+	if !filepath.IsAbs(logPath) {
+		logPath = filepath.Join(r.LogDir, logPath)
+	}
+
+	scriptDataDir := filepath.Join(r.DataDir(), script.LogSourceID.String())
+	err := r.Filesystem.MkdirAll(scriptDataDir, 0o700)
+	if err != nil {
+		return xerrors.Errorf("%s script: create script temp dir: %w", scriptDataDir, err)
+	}
+
+	logger := r.Logger.With(
+		slog.F("log_source_id", script.LogSourceID),
+		slog.F("log_path", logPath),
+		slog.F("script_data_dir", scriptDataDir),
+	)
+	logger.Info(ctx, "running agent script", slog.F("script", script.Script))
+
+	fileWriter, err := r.Filesystem.OpenFile(logPath, os.O_CREATE|os.O_RDWR, 0o600)
+	if err != nil {
+		return xerrors.Errorf("open %s script log file: %w", logPath, err)
+	}
+	defer func() {
+		err := fileWriter.Close()
+		if err != nil {
+			logger.Warn(ctx, fmt.Sprintf("close %s script log file", logPath), slog.Error(err))
+		}
+	}()
+
+	var cmd *exec.Cmd
+	cmdCtx := ctx
+	if script.Timeout > 0 {
+		var ctxCancel context.CancelFunc
+		cmdCtx, ctxCancel = context.WithTimeout(ctx, script.Timeout)
+		defer ctxCancel()
+	}
+	cmdPty, err := r.SSHServer.CreateCommand(cmdCtx, script.Script, nil)
+	if err != nil {
+		return xerrors.Errorf("%s script: create command: %w", logPath, err)
+	}
+	cmd = cmdPty.AsExec()
+	cmd.SysProcAttr = cmdSysProcAttr()
+	cmd.WaitDelay = 10 * time.Second
+	cmd.Cancel = cmdCancel(cmd)
+
+	// Expose env vars that can be used in the script for storing data
+	// and binaries. In the future, we may want to expose more env vars
+	// for the script to use, like CODER_SCRIPT_DATA_DIR for persistent
+	// storage.
+	cmd.Env = append(cmd.Env, "CODER_SCRIPT_DATA_DIR="+scriptDataDir)
+	cmd.Env = append(cmd.Env, "CODER_SCRIPT_BIN_DIR="+r.ScriptBinDir())
+
+	scriptLogger := r.GetScriptLogger(script.LogSourceID)
+	// If ctx is canceled here (or in a writer below), we may be
+	// discarding logs, but that's okay because we're shutting down
+	// anyway. We could consider creating a new context here if we
+	// want better control over flush during shutdown.
+	defer func() {
+		if err := scriptLogger.Flush(ctx); err != nil {
+			logger.Warn(ctx, "flush startup logs failed", slog.Error(err))
+		}
+	}()
+
+	infoW := agentsdk.LogsWriter(ctx, scriptLogger.Send, script.LogSourceID, codersdk.LogLevelInfo)
+	defer infoW.Close()
+	errW := agentsdk.LogsWriter(ctx, scriptLogger.Send, script.LogSourceID, codersdk.LogLevelError)
+	defer errW.Close()
+	cmd.Stdout = io.MultiWriter(fileWriter, infoW)
+	cmd.Stderr = io.MultiWriter(fileWriter, errW)
+
+	start := time.Now()
+	defer func() {
+		end := time.Now()
+		execTime := end.Sub(start)
+		exitCode := 0
+		if err != nil {
+			exitCode = 255 // Unknown status.
+			var exitError *exec.ExitError
+			if xerrors.As(err, &exitError) {
+				exitCode = exitError.ExitCode()
+			}
+			logger.Warn(ctx, fmt.Sprintf("%s script failed", logPath), slog.F("execution_time", execTime), slog.F("exit_code", exitCode), slog.Error(err))
+		} else {
+			logger.Info(ctx, fmt.Sprintf("%s script completed", logPath), slog.F("execution_time", execTime), slog.F("exit_code", exitCode))
+		}
+	}()
+
+	err = cmd.Start()
+	if err != nil {
+		if errors.Is(err, context.DeadlineExceeded) {
+			return ErrTimeout
+		}
+		return xerrors.Errorf("%s script: start command: %w", logPath, err)
+	}
+
+	cmdDone := make(chan error, 1)
+	err = r.trackCommandGoroutine(func() {
+		cmdDone <- cmd.Wait()
+	})
+	if err != nil {
+		return xerrors.Errorf("%s script: track command goroutine: %w", logPath, err)
+	}
+	select {
+	case <-cmdCtx.Done():
+		// Wait for the command to drain!
+		select {
+		case <-cmdDone:
+		case <-time.After(10 * time.Second):
+		}
+		err = cmdCtx.Err()
+	case err = <-cmdDone:
+	}
+	switch {
+	case errors.Is(err, exec.ErrWaitDelay):
+		err = ErrOutputPipesOpen
+		message := fmt.Sprintf("script exited successfully, but output pipes were not closed after %s", cmd.WaitDelay)
+		details := fmt.Sprint(
+			"This usually means a child process was started with references to stdout or stderr. As a result, this " +
+				"process may now have been terminated. Consider redirecting the output or using a separate " +
+				"\"coder_script\" for the process, see " +
+				"https://coder.com/docs/v2/latest/templates/troubleshooting#startup-script-issues for more information.",
+		)
+		// Inform the user by propagating the message via log writers.
+		_, _ = fmt.Fprintf(cmd.Stderr, "WARNING: %s. %s\n", message, details)
+		// Also log to agent logs for ease of debugging.
+		r.Logger.Warn(ctx, message, slog.F("details", details), slog.Error(err))
+
+	case errors.Is(err, context.DeadlineExceeded):
+		err = ErrTimeout
+	}
+	return err
+}
+
+func (r *Runner) Close() error {
+	r.closeMutex.Lock()
+	defer r.closeMutex.Unlock()
+	if r.isClosed() {
+		return nil
+	}
+	close(r.closed)
+	// Must cancel the cron ctx BEFORE stopping the cron.
+	r.cronCtxCancel()
+	<-r.cron.Stop().Done()
+	r.cmdCloseWait.Wait()
+	return nil
+}
+
+func (r *Runner) trackCommandGoroutine(fn func()) error {
+	r.closeMutex.Lock()
+	defer r.closeMutex.Unlock()
+	if r.isClosed() {
+		return xerrors.New("track command goroutine: closed")
+	}
+	r.cmdCloseWait.Add(1)
+	go func() {
+		defer r.cmdCloseWait.Done()
+		fn()
+	}()
+	return nil
+}
+
+func (r *Runner) isClosed() bool {
+	select {
+	case <-r.closed:
+		return true
+	default:
+		return false
+	}
+}

agent/agentscripts/agentscripts_other.go

@@ -0,0 +1,20 @@
+//go:build !windows
+
+package agentscripts
+
+import (
+	"os/exec"
+	"syscall"
+)
+
+func cmdSysProcAttr() *syscall.SysProcAttr {
+	return &syscall.SysProcAttr{
+		Setsid: true,
+	}
+}
+
+func cmdCancel(cmd *exec.Cmd) func() error {
+	return func() error {
+		return syscall.Kill(-cmd.Process.Pid, syscall.SIGHUP)
+	}
+}

agent/agentscripts/agentscripts_test.go

@@ -0,0 +1,181 @@
+package agentscripts_test
+
+import (
+	"context"
+	"path/filepath"
+	"runtime"
+	"testing"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/spf13/afero"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.uber.org/goleak"
+
+	"cdr.dev/slog/sloggers/slogtest"
+	"github.com/coder/coder/v2/agent/agentscripts"
+	"github.com/coder/coder/v2/agent/agentssh"
+	"github.com/coder/coder/v2/codersdk"
+	"github.com/coder/coder/v2/codersdk/agentsdk"
+	"github.com/coder/coder/v2/testutil"
+)
+
+func TestMain(m *testing.M) {
+	goleak.VerifyTestMain(m)
+}
+
+func TestExecuteBasic(t *testing.T) {
+	t.Parallel()
+	ctx := testutil.Context(t, testutil.WaitShort)
+	fLogger := newFakeScriptLogger()
+	runner := setup(t, func(uuid2 uuid.UUID) agentscripts.ScriptLogger {
+		return fLogger
+	})
+	defer runner.Close()
+	err := runner.Init([]codersdk.WorkspaceAgentScript{{
+		LogSourceID: uuid.New(),
+		Script:      "echo hello",
+	}})
+	require.NoError(t, err)
+	require.NoError(t, runner.Execute(context.Background(), func(script codersdk.WorkspaceAgentScript) bool {
+		return true
+	}))
+	log := testutil.RequireRecvCtx(ctx, t, fLogger.logs)
+	require.Equal(t, "hello", log.Output)
+}
+
+func TestEnv(t *testing.T) {
+	t.Parallel()
+	fLogger := newFakeScriptLogger()
+	runner := setup(t, func(uuid2 uuid.UUID) agentscripts.ScriptLogger {
+		return fLogger
+	})
+	defer runner.Close()
+	id := uuid.New()
+	script := "echo $CODER_SCRIPT_DATA_DIR\necho $CODER_SCRIPT_BIN_DIR\n"
+	if runtime.GOOS == "windows" {
+		script = `
+			cmd.exe /c echo %CODER_SCRIPT_DATA_DIR%
+			cmd.exe /c echo %CODER_SCRIPT_BIN_DIR%
+		`
+	}
+	err := runner.Init([]codersdk.WorkspaceAgentScript{{
+		LogSourceID: id,
+		Script:      script,
+	}})
+	require.NoError(t, err)
+
+	ctx := testutil.Context(t, testutil.WaitLong)
+
+	done := testutil.Go(t, func() {
+		err := runner.Execute(ctx, func(script codersdk.WorkspaceAgentScript) bool {
+			return true
+		})
+		assert.NoError(t, err)
+	})
+	defer func() {
+		select {
+		case <-ctx.Done():
+		case <-done:
+		}
+	}()
+
+	var log []agentsdk.Log
+	for {
+		select {
+		case <-ctx.Done():
+			require.Fail(t, "timed out waiting for logs")
+		case l := <-fLogger.logs:
+			t.Logf("log: %s", l.Output)
+			log = append(log, l)
+		}
+		if len(log) >= 2 {
+			break
+		}
+	}
+	require.Contains(t, log[0].Output, filepath.Join(runner.DataDir(), id.String()))
+	require.Contains(t, log[1].Output, runner.ScriptBinDir())
+}
+
+func TestTimeout(t *testing.T) {
+	t.Parallel()
+	runner := setup(t, nil)
+	defer runner.Close()
+	err := runner.Init([]codersdk.WorkspaceAgentScript{{
+		LogSourceID: uuid.New(),
+		Script:      "sleep infinity",
+		Timeout:     time.Millisecond,
+	}})
+	require.NoError(t, err)
+	require.ErrorIs(t, runner.Execute(context.Background(), nil), agentscripts.ErrTimeout)
+}
+
+// TestCronClose exists because cron.Run() can happen after cron.Close().
+// If this happens, there used to be a deadlock.
+func TestCronClose(t *testing.T) {
+	t.Parallel()
+	runner := agentscripts.New(agentscripts.Options{})
+	runner.StartCron()
+	require.NoError(t, runner.Close(), "close runner")
+}
+
+func setup(t *testing.T, getScriptLogger func(logSourceID uuid.UUID) agentscripts.ScriptLogger) *agentscripts.Runner {
+	t.Helper()
+	if getScriptLogger == nil {
+		// noop
+		getScriptLogger = func(uuid uuid.UUID) agentscripts.ScriptLogger {
+			return noopScriptLogger{}
+		}
+	}
+	fs := afero.NewMemMapFs()
+	logger := slogtest.Make(t, nil)
+	s, err := agentssh.NewServer(context.Background(), logger, prometheus.NewRegistry(), fs, nil)
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		_ = s.Close()
+	})
+	return agentscripts.New(agentscripts.Options{
+		LogDir:          t.TempDir(),
+		DataDirBase:     t.TempDir(),
+		Logger:          logger,
+		SSHServer:       s,
+		Filesystem:      fs,
+		GetScriptLogger: getScriptLogger,
+	})
+}
+
+type noopScriptLogger struct{}
+
+func (noopScriptLogger) Send(context.Context, ...agentsdk.Log) error {
+	return nil
+}
+
+func (noopScriptLogger) Flush(context.Context) error {
+	return nil
+}
+
+type fakeScriptLogger struct {
+	logs chan agentsdk.Log
+}
+
+func (f *fakeScriptLogger) Send(ctx context.Context, logs ...agentsdk.Log) error {
+	for _, log := range logs {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case f.logs <- log:
+			// OK!
+		}
+	}
+	return nil
+}
+
+func (*fakeScriptLogger) Flush(context.Context) error {
+	return nil
+}
+
+func newFakeScriptLogger() *fakeScriptLogger {
+	return &fakeScriptLogger{make(chan agentsdk.Log, 100)}
+}

agent/agentscripts/agentscripts_windows.go

@@ -0,0 +1,17 @@
+package agentscripts
+
+import (
+	"os"
+	"os/exec"
+	"syscall"
+)
+
+func cmdSysProcAttr() *syscall.SysProcAttr {
+	return &syscall.SysProcAttr{}
+}
+
+func cmdCancel(cmd *exec.Cmd) func() error {
+	return func() error {
+		return cmd.Process.Signal(os.Interrupt)
+	}
+}

agent/agentssh/agentssh.go

@@ -19,6 +19,8 @@ import (
 	"time"
 
 	"github.com/gliderlabs/ssh"
+	"github.com/google/uuid"
+	"github.com/kballard/go-shellquote"
 	"github.com/pkg/sftp"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/spf13/afero"
@@ -28,10 +30,9 @@ import (
 
 	"cdr.dev/slog"
 
-	"github.com/coder/coder/agent/usershell"
-	"github.com/coder/coder/codersdk"
-	"github.com/coder/coder/codersdk/agentsdk"
-	"github.com/coder/coder/pty"
+	"github.com/coder/coder/v2/agent/usershell"
+	"github.com/coder/coder/v2/codersdk"
+	"github.com/coder/coder/v2/pty"
 )
 
 const (
@@ -45,10 +46,36 @@ const (
 	MagicSessionTypeEnvironmentVariable = "CODER_SSH_SESSION_TYPE"
 	// MagicSessionTypeVSCode is set in the SSH config by the VS Code extension to identify itself.
 	MagicSessionTypeVSCode = "vscode"
-	// MagicSessionTypeJetBrains is set in the SSH config by the JetBrains extension to identify itself.
+	// MagicSessionTypeJetBrains is set in the SSH config by the JetBrains
+	// extension to identify itself.
 	MagicSessionTypeJetBrains = "jetbrains"
+	// MagicProcessCmdlineJetBrains is a string in a process's command line that
+	// uniquely identifies it as JetBrains software.
+	MagicProcessCmdlineJetBrains = "idea.vendor.name=JetBrains"
 )
 
+// Config sets configuration parameters for the agent SSH server.
+type Config struct {
+	// MaxTimeout sets the absolute connection timeout, none if empty. If set to
+	// 3 seconds or more, keep alive will be used instead.
+	MaxTimeout time.Duration
+	// MOTDFile returns the path to the message of the day file. If set, the
+	// file will be displayed to the user upon login.
+	MOTDFile func() string
+	// ServiceBanner returns the configuration for the Coder service banner.
+	ServiceBanner func() *codersdk.ServiceBannerConfig
+	// UpdateEnv updates the environment variables for the command to be
+	// executed. It can be used to add, modify or replace environment variables.
+	UpdateEnv func(current []string) (updated []string, err error)
+	// WorkingDirectory sets the working directory for commands and defines
+	// where users will land when they connect via SSH. Default is the home
+	// directory of the user.
+	WorkingDirectory func() string
+	// X11SocketDir is the directory where X11 sockets are created. Default is
+	// /tmp/.X11-unix.
+	X11SocketDir string
+}
+
 type Server struct {
 	mu        sync.RWMutex // Protects following.
 	fs        afero.Fs
@@ -60,14 +87,10 @@ type Server struct {
 	// a lock on mu but protected by closing.
 	wg sync.WaitGroup
 
-	logger       slog.Logger
-	srv          *ssh.Server
-	x11SocketDir string
+	logger slog.Logger
+	srv    *ssh.Server
 
-	Env           map[string]string
-	AgentToken    func() string
-	Manifest      *atomic.Pointer[agentsdk.Manifest]
-	ServiceBanner *atomic.Pointer[codersdk.ServiceBannerConfig]
+	config *Config
 
 	connCountVSCode     atomic.Int64
 	connCountJetBrains  atomic.Int64
@@ -76,7 +99,7 @@ type Server struct {
 	metrics *sshServerMetrics
 }
 
-func NewServer(ctx context.Context, logger slog.Logger, prometheusRegistry *prometheus.Registry, fs afero.Fs, maxTimeout time.Duration, x11SocketDir string) (*Server, error) {
+func NewServer(ctx context.Context, logger slog.Logger, prometheusRegistry *prometheus.Registry, fs afero.Fs, config *Config) (*Server, error) {
 	// Clients' should ignore the host key when connecting.
 	// The agent needs to authenticate with coderd to SSH,
 	// so SSH authentication doesn't improve security.
@@ -88,28 +111,54 @@ func NewServer(ctx context.Context, logger slog.Logger, prometheusRegistry *prom
 	if err != nil {
 		return nil, err
 	}
-	if x11SocketDir == "" {
-		x11SocketDir = filepath.Join(os.TempDir(), ".X11-unix")
+	if config == nil {
+		config = &Config{}
+	}
+	if config.X11SocketDir == "" {
+		config.X11SocketDir = filepath.Join(os.TempDir(), ".X11-unix")
+	}
+	if config.UpdateEnv == nil {
+		config.UpdateEnv = func(current []string) ([]string, error) { return current, nil }
+	}
+	if config.MOTDFile == nil {
+		config.MOTDFile = func() string { return "" }
+	}
+	if config.ServiceBanner == nil {
+		config.ServiceBanner = func() *codersdk.ServiceBannerConfig { return &codersdk.ServiceBannerConfig{} }
+	}
+	if config.WorkingDirectory == nil {
+		config.WorkingDirectory = func() string {
+			home, err := userHomeDir()
+			if err != nil {
+				return ""
+			}
+			return home
+		}
 	}
 
 	forwardHandler := &ssh.ForwardedTCPHandler{}
-	unixForwardHandler := &forwardedUnixHandler{log: logger}
+	unixForwardHandler := newForwardedUnixHandler(logger)
 
 	metrics := newSSHServerMetrics(prometheusRegistry)
 	s := &Server{
-		listeners:    make(map[net.Listener]struct{}),
-		fs:           fs,
-		conns:        make(map[net.Conn]struct{}),
-		sessions:     make(map[ssh.Session]struct{}),
-		logger:       logger,
-		x11SocketDir: x11SocketDir,
+		listeners: make(map[net.Listener]struct{}),
+		fs:        fs,
+		conns:     make(map[net.Conn]struct{}),
+		sessions:  make(map[ssh.Session]struct{}),
+		logger:    logger,
+
+		config: config,
 
 		metrics: metrics,
 	}
 
 	srv := &ssh.Server{
 		ChannelHandlers: map[string]ssh.ChannelHandler{
-			"direct-tcpip":                   ssh.DirectTCPIPHandler,
+			"direct-tcpip": func(srv *ssh.Server, conn *gossh.ServerConn, newChan gossh.NewChannel, ctx ssh.Context) {
+				// Wrapper is designed to find and track JetBrains Gateway connections.
+				wrapped := NewJetbrainsChannelWatcher(ctx, s.logger, newChan, &s.connCountJetBrains)
+				ssh.DirectTCPIPHandler(srv, conn, wrapped, ctx)
+			},
 			"direct-streamlocal@openssh.com": directStreamLocalHandler,
 			"session":                        ssh.DefaultSessionHandler,
 		},
@@ -140,7 +189,7 @@ func NewServer(ctx context.Context, logger slog.Logger, prometheusRegistry *prom
 		},
 		ReversePortForwardingCallback: func(ctx ssh.Context, bindHost string, bindPort uint32) bool {
 			// Allow reverse port forwarding all!
-			s.logger.Debug(ctx, "local port forward",
+			s.logger.Debug(ctx, "reverse port forward",
 				slog.F("bind_host", bindHost),
 				slog.F("bind_port", bindPort))
 			return true
@@ -162,14 +211,16 @@ func NewServer(ctx context.Context, logger slog.Logger, prometheusRegistry *prom
 		},
 	}
 
-	// The MaxTimeout functionality has been substituted with the introduction of the KeepAlive feature.
-	// In cases where very short timeouts are set, the SSH server will automatically switch to the connection timeout for both read and write operations.
-	if maxTimeout >= 3*time.Second {
+	// The MaxTimeout functionality has been substituted with the introduction
+	// of the KeepAlive feature. In cases where very short timeouts are set, the
+	// SSH server will automatically switch to the connection timeout for both
+	// read and write operations.
+	if config.MaxTimeout >= 3*time.Second {
 		srv.ClientAliveCountMax = 3
-		srv.ClientAliveInterval = maxTimeout / time.Duration(srv.ClientAliveCountMax)
+		srv.ClientAliveInterval = config.MaxTimeout / time.Duration(srv.ClientAliveCountMax)
 		srv.MaxTimeout = 0
 	} else {
-		srv.MaxTimeout = maxTimeout
+		srv.MaxTimeout = config.MaxTimeout
 	}
 
 	s.srv = srv
@@ -191,9 +242,16 @@ func (s *Server) ConnStats() ConnStats {
 }
 
 func (s *Server) sessionHandler(session ssh.Session) {
-	logger := s.logger.With(slog.F("remote_addr", session.RemoteAddr()), slog.F("local_addr", session.LocalAddr()))
-	logger.Info(session.Context(), "handling ssh session")
 	ctx := session.Context()
+	logger := s.logger.With(
+		slog.F("remote_addr", session.RemoteAddr()),
+		slog.F("local_addr", session.LocalAddr()),
+		// Assigning a random uuid for each session is useful for tracking
+		// logs for the same ssh session.
+		slog.F("id", uuid.NewString()),
+	)
+	logger.Info(ctx, "handling ssh session")
+
 	if !s.trackSession(session, true) {
 		// See (*Server).Close() for why we call Close instead of Exit.
 		_ = session.Close()
@@ -217,7 +275,7 @@ func (s *Server) sessionHandler(session ssh.Session) {
 	switch ss := session.Subsystem(); ss {
 	case "":
 	case "sftp":
-		s.sftpHandler(session)
+		s.sftpHandler(logger, session)
 		return
 	default:
 		logger.Warn(ctx, "unsupported subsystem", slog.F("subsystem", ss))
@@ -225,11 +283,32 @@ func (s *Server) sessionHandler(session ssh.Session) {
 		return
 	}
 
-	err := s.sessionStart(session, extraEnv)
+	err := s.sessionStart(logger, session, extraEnv)
 	var exitError *exec.ExitError
 	if xerrors.As(err, &exitError) {
-		logger.Info(ctx, "ssh session returned", slog.Error(exitError))
-		_ = session.Exit(exitError.ExitCode())
+		code := exitError.ExitCode()
+		if code == -1 {
+			// If we return -1 here, it will be transmitted as an
+			// uint32(4294967295). This exit code is nonsense, so
+			// instead we return 255 (same as OpenSSH). This is
+			// also the same exit code that the shell returns for
+			// -1.
+			//
+			// For signals, we could consider sending 128+signal
+			// instead (however, OpenSSH doesn't seem to do this).
+			code = 255
+		}
+		logger.Info(ctx, "ssh session returned",
+			slog.Error(exitError),
+			slog.F("process_exit_code", exitError.ExitCode()),
+			slog.F("exit_code", code),
+		)
+
+		// TODO(mafredri): For signal exit, there's also an "exit-signal"
+		// request (session.Exit sends "exit-status"), however, since it's
+		// not implemented on the session interface and not used by
+		// OpenSSH, we'll leave it for now.
+		_ = session.Exit(code)
 		return
 	}
 	if err != nil {
@@ -243,7 +322,7 @@ func (s *Server) sessionHandler(session ssh.Session) {
 	_ = session.Exit(0)
 }
 
-func (s *Server) sessionStart(session ssh.Session, extraEnv []string) (retErr error) {
+func (s *Server) sessionStart(logger slog.Logger, session ssh.Session, extraEnv []string) (retErr error) {
 	ctx := session.Context()
 	env := append(session.Environ(), extraEnv...)
 	var magicType string
@@ -251,21 +330,23 @@ func (s *Server) sessionStart(session ssh.Session, extraEnv []string) (retErr er
 		if !strings.HasPrefix(kv, MagicSessionTypeEnvironmentVariable) {
 			continue
 		}
-		magicType = strings.TrimPrefix(kv, MagicSessionTypeEnvironmentVariable+"=")
+		magicType = strings.ToLower(strings.TrimPrefix(kv, MagicSessionTypeEnvironmentVariable+"="))
 		env = append(env[:index], env[index+1:]...)
 	}
+
+	// Always force lowercase checking to be case-insensitive.
 	switch magicType {
 	case MagicSessionTypeVSCode:
 		s.connCountVSCode.Add(1)
 		defer s.connCountVSCode.Add(-1)
 	case MagicSessionTypeJetBrains:
-		s.connCountJetBrains.Add(1)
-		defer s.connCountJetBrains.Add(-1)
+		// Do nothing here because JetBrains launches hundreds of ssh sessions.
+		// We instead track JetBrains in the single persistent tcp forwarding channel.
 	case "":
 		s.connCountSSHSession.Add(1)
 		defer s.connCountSSHSession.Add(-1)
 	default:
-		s.logger.Warn(ctx, "invalid magic ssh session type specified", slog.F("type", magicType))
+		logger.Warn(ctx, "invalid magic ssh session type specified", slog.F("type", magicType))
 	}
 
 	magicTypeLabel := magicTypeMetricLabel(magicType)
@@ -298,12 +379,12 @@ func (s *Server) sessionStart(session ssh.Session, extraEnv []string) (retErr er
 	}
 
 	if isPty {
-		return s.startPTYSession(session, magicTypeLabel, cmd, sshPty, windowSize)
+		return s.startPTYSession(logger, session, magicTypeLabel, cmd, sshPty, windowSize)
 	}
-	return s.startNonPTYSession(session, magicTypeLabel, cmd.AsExec())
+	return s.startNonPTYSession(logger, session, magicTypeLabel, cmd.AsExec())
 }
 
-func (s *Server) startNonPTYSession(session ssh.Session, magicTypeLabel string, cmd *exec.Cmd) error {
+func (s *Server) startNonPTYSession(logger slog.Logger, session ssh.Session, magicTypeLabel string, cmd *exec.Cmd) error {
 	s.metrics.sessionsTotal.WithLabelValues(magicTypeLabel, "no").Add(1)
 
 	cmd.Stdout = session
@@ -327,6 +408,17 @@ func (s *Server) startNonPTYSession(session ssh.Session, magicTypeLabel string,
 		s.metrics.sessionErrors.WithLabelValues(magicTypeLabel, "no", "start_command").Add(1)
 		return xerrors.Errorf("start: %w", err)
 	}
+	sigs := make(chan ssh.Signal, 1)
+	session.Signals(sigs)
+	defer func() {
+		session.Signals(nil)
+		close(sigs)
+	}()
+	go func() {
+		for sig := range sigs {
+			s.handleSignal(logger, sig, cmd.Process, magicTypeLabel)
+		}
+	}()
 	return cmd.Wait()
 }
 
@@ -337,9 +429,10 @@ type ptySession interface {
 	Context() ssh.Context
 	DisablePTYEmulation()
 	RawCommand() string
+	Signals(chan<- ssh.Signal)
 }
 
-func (s *Server) startPTYSession(session ptySession, magicTypeLabel string, cmd *pty.Cmd, sshPty ssh.Pty, windowSize <-chan ssh.Window) (retErr error) {
+func (s *Server) startPTYSession(logger slog.Logger, session ptySession, magicTypeLabel string, cmd *pty.Cmd, sshPty ssh.Pty, windowSize <-chan ssh.Window) (retErr error) {
 	s.metrics.sessionsTotal.WithLabelValues(magicTypeLabel, "yes").Add(1)
 
 	ctx := session.Context()
@@ -348,26 +441,21 @@ func (s *Server) startPTYSession(session ptySession, magicTypeLabel string, cmd
 	session.DisablePTYEmulation()
 
 	if isLoginShell(session.RawCommand()) {
-		serviceBanner := s.ServiceBanner.Load()
+		serviceBanner := s.config.ServiceBanner()
 		if serviceBanner != nil {
 			err := showServiceBanner(session, serviceBanner)
 			if err != nil {
-				s.logger.Error(ctx, "agent failed to show service banner", slog.Error(err))
+				logger.Error(ctx, "agent failed to show service banner", slog.Error(err))
 				s.metrics.sessionErrors.WithLabelValues(magicTypeLabel, "yes", "service_banner").Add(1)
 			}
 		}
 	}
 
 	if !isQuietLogin(s.fs, session.RawCommand()) {
-		manifest := s.Manifest.Load()
-		if manifest != nil {
-			err := showMOTD(s.fs, session, manifest.MOTDFile)
-			if err != nil {
-				s.logger.Error(ctx, "agent failed to show MOTD", slog.Error(err))
-				s.metrics.sessionErrors.WithLabelValues(magicTypeLabel, "yes", "motd").Add(1)
-			}
-		} else {
-			s.logger.Warn(ctx, "metadata lookup failed, unable to show MOTD")
+		err := showMOTD(s.fs, session, s.config.MOTDFile())
+		if err != nil {
+			logger.Error(ctx, "agent failed to show MOTD", slog.Error(err))
+			s.metrics.sessionErrors.WithLabelValues(magicTypeLabel, "yes", "motd").Add(1)
 		}
 	}
 
@@ -376,7 +464,7 @@ func (s *Server) startPTYSession(session ptySession, magicTypeLabel string, cmd
 	// The pty package sets `SSH_TTY` on supported platforms.
 	ptty, process, err := pty.Start(cmd, pty.WithPTYOption(
 		pty.WithSSHRequest(sshPty),
-		pty.WithLogger(slog.Stdlib(ctx, s.logger, slog.LevelInfo)),
+		pty.WithLogger(slog.Stdlib(ctx, logger, slog.LevelInfo)),
 	))
 	if err != nil {
 		s.metrics.sessionErrors.WithLabelValues(magicTypeLabel, "yes", "start_command").Add(1)
@@ -385,20 +473,43 @@ func (s *Server) startPTYSession(session ptySession, magicTypeLabel string, cmd
 	defer func() {
 		closeErr := ptty.Close()
 		if closeErr != nil {
-			s.logger.Warn(ctx, "failed to close tty", slog.Error(closeErr))
+			logger.Warn(ctx, "failed to close tty", slog.Error(closeErr))
 			s.metrics.sessionErrors.WithLabelValues(magicTypeLabel, "yes", "close").Add(1)
 			if retErr == nil {
 				retErr = closeErr
 			}
 		}
 	}()
+	sigs := make(chan ssh.Signal, 1)
+	session.Signals(sigs)
+	defer func() {
+		session.Signals(nil)
+		close(sigs)
+	}()
 	go func() {
-		for win := range windowSize {
-			resizeErr := ptty.Resize(uint16(win.Height), uint16(win.Width))
-			// If the pty is closed, then command has exited, no need to log.
-			if resizeErr != nil && !errors.Is(resizeErr, pty.ErrClosed) {
-				s.logger.Warn(ctx, "failed to resize tty", slog.Error(resizeErr))
-				s.metrics.sessionErrors.WithLabelValues(magicTypeLabel, "yes", "resize").Add(1)
+		for {
+			if sigs == nil && windowSize == nil {
+				return
+			}
+
+			select {
+			case sig, ok := <-sigs:
+				if !ok {
+					sigs = nil
+					continue
+				}
+				s.handleSignal(logger, sig, process, magicTypeLabel)
+			case win, ok := <-windowSize:
+				if !ok {
+					windowSize = nil
+					continue
+				}
+				resizeErr := ptty.Resize(uint16(win.Height), uint16(win.Width))
+				// If the pty is closed, then command has exited, no need to log.
+				if resizeErr != nil && !errors.Is(resizeErr, pty.ErrClosed) {
+					logger.Warn(ctx, "failed to resize tty", slog.Error(resizeErr))
+					s.metrics.sessionErrors.WithLabelValues(magicTypeLabel, "yes", "resize").Add(1)
+				}
 			}
 		}
 	}()
@@ -419,7 +530,7 @@ func (s *Server) startPTYSession(session ptySession, magicTypeLabel string, cmd
 	// 2. The client hangs up, which cancels the command's Context, and go will
 	//    kill the command's process.  This then has the same effect as (1).
 	n, err := io.Copy(session, ptty.OutputReader())
-	s.logger.Debug(ctx, "copy output done", slog.F("bytes", n), slog.Error(err))
+	logger.Debug(ctx, "copy output done", slog.F("bytes", n), slog.Error(err))
 	if err != nil {
 		s.metrics.sessionErrors.WithLabelValues(magicTypeLabel, "yes", "output_io_copy").Add(1)
 		return xerrors.Errorf("copy error: %w", err)
@@ -432,7 +543,7 @@ func (s *Server) startPTYSession(session ptySession, magicTypeLabel string, cmd
 	// ExitErrors just mean the command we run returned a non-zero exit code, which is normal
 	// and not something to be concerned about.  But, if it's something else, we should log it.
 	if err != nil && !xerrors.As(err, &exitErr) {
-		s.logger.Warn(ctx, "process wait exited with error", slog.Error(err))
+		logger.Warn(ctx, "process wait exited with error", slog.Error(err))
 		s.metrics.sessionErrors.WithLabelValues(magicTypeLabel, "yes", "wait").Add(1)
 	}
 	if err != nil {
@@ -441,7 +552,19 @@ func (s *Server) startPTYSession(session ptySession, magicTypeLabel string, cmd
 	return nil
 }
 
-func (s *Server) sftpHandler(session ssh.Session) {
+func (s *Server) handleSignal(logger slog.Logger, ssig ssh.Signal, signaler interface{ Signal(os.Signal) error }, magicTypeLabel string) {
+	ctx := context.Background()
+	sig := osSignalFrom(ssig)
+	logger = logger.With(slog.F("ssh_signal", ssig), slog.F("signal", sig.String()))
+	logger.Info(ctx, "received signal from client")
+	err := signaler.Signal(sig)
+	if err != nil {
+		logger.Warn(ctx, "signaling the process failed", slog.Error(err))
+		s.metrics.sessionErrors.WithLabelValues(magicTypeLabel, "yes", "signal").Add(1)
+	}
+}
+
+func (s *Server) sftpHandler(logger slog.Logger, session ssh.Session) {
 	s.metrics.sftpConnectionsTotal.Add(1)
 
 	ctx := session.Context()
@@ -457,20 +580,20 @@ func (s *Server) sftpHandler(session ssh.Session) {
 	// directory so that SFTP connections land there.
 	homedir, err := userHomeDir()
 	if err != nil {
-		s.logger.Warn(ctx, "get sftp working directory failed, unable to get home dir", slog.Error(err))
+		logger.Warn(ctx, "get sftp working directory failed, unable to get home dir", slog.Error(err))
 	} else {
 		opts = append(opts, sftp.WithServerWorkingDirectory(homedir))
 	}
 
 	server, err := sftp.NewServer(session, opts...)
 	if err != nil {
-		s.logger.Debug(ctx, "initialize sftp server", slog.Error(err))
+		logger.Debug(ctx, "initialize sftp server", slog.Error(err))
 		return
 	}
 	defer server.Close()
 
 	err = server.Serve()
-	if errors.Is(err, io.EOF) {
+	if err == nil || errors.Is(err, io.EOF) {
 		// Unless we call `session.Exit(0)` here, the client won't
 		// receive `exit-status` because `(*sftp.Server).Close()`
 		// calls `Close()` on the underlying connection (session),
@@ -482,7 +605,7 @@ func (s *Server) sftpHandler(session ssh.Session) {
 		_ = session.Exit(0)
 		return
 	}
-	s.logger.Warn(ctx, "sftp server closed with error", slog.Error(err))
+	logger.Warn(ctx, "sftp server closed with error", slog.Error(err))
 	s.metrics.sftpServerErrors.Add(1)
 	_ = session.Exit(1)
 }
@@ -502,19 +625,38 @@ func (s *Server) CreateCommand(ctx context.Context, script string, env []string)
 		return nil, xerrors.Errorf("get user shell: %w", err)
 	}
 
-	manifest := s.Manifest.Load()
-	if manifest == nil {
-		return nil, xerrors.Errorf("no metadata was provided")
-	}
-
 	// OpenSSH executes all commands with the users current shell.
 	// We replicate that behavior for IDE support.
 	caller := "-c"
 	if runtime.GOOS == "windows" {
 		caller = "/c"
 	}
+	name := shell
 	args := []string{caller, script}
 
+	// A preceding space is generally not idiomatic for a shebang,
+	// but in Terraform it's quite standard to use <<EOF for a multi-line
+	// string which would indent with spaces, so we accept it for user-ease.
+	if strings.HasPrefix(strings.TrimSpace(script), "#!") {
+		// If the script starts with a shebang, we should
+		// execute it directly. This is useful for running
+		// scripts that aren't executable.
+		shebang := strings.SplitN(strings.TrimSpace(script), "\n", 2)[0]
+		shebang = strings.TrimSpace(shebang)
+		shebang = strings.TrimPrefix(shebang, "#!")
+		words, err := shellquote.Split(shebang)
+		if err != nil {
+			return nil, xerrors.Errorf("split shebang: %w", err)
+		}
+		name = words[0]
+		if len(words) > 1 {
+			args = words[1:]
+		} else {
+			args = []string{}
+		}
+		args = append(args, caller, script)
+	}
+
 	// gliderlabs/ssh returns a command slice of zero
 	// when a shell is requested.
 	if len(script) == 0 {
@@ -526,8 +668,8 @@ func (s *Server) CreateCommand(ctx context.Context, script string, env []string)
 		}
 	}
 
-	cmd := pty.CommandContext(ctx, shell, args...)
-	cmd.Dir = manifest.Directory
+	cmd := pty.CommandContext(ctx, name, args...)
+	cmd.Dir = s.config.WorkingDirectory()
 
 	// If the metadata directory doesn't exist, we run the command
 	// in the users home directory.
@@ -541,21 +683,7 @@ func (s *Server) CreateCommand(ctx context.Context, script string, env []string)
 		cmd.Dir = homedir
 	}
 	cmd.Env = append(os.Environ(), env...)
-	executablePath, err := os.Executable()
-	if err != nil {
-		return nil, xerrors.Errorf("getting os executable: %w", err)
-	}
-	// Set environment variables reliable detection of being inside a
-	// Coder workspace.
-	cmd.Env = append(cmd.Env, "CODER=true")
 	cmd.Env = append(cmd.Env, fmt.Sprintf("USER=%s", username))
-	// Git on Windows resolves with UNIX-style paths.
-	// If using backslashes, it's unable to find the executable.
-	unixExecutablePath := strings.ReplaceAll(executablePath, "\\", "/")
-	cmd.Env = append(cmd.Env, fmt.Sprintf(`GIT_SSH_COMMAND=%s gitssh --`, unixExecutablePath))
-
-	// Specific Coder subcommands require the agent token exposed!
-	cmd.Env = append(cmd.Env, fmt.Sprintf("CODER_AGENT_TOKEN=%s", s.AgentToken()))
 
 	// Set SSH connection environment variables (these are also set by OpenSSH
 	// and thus expected to be present by SSH clients). Since the agent does
@@ -566,26 +694,9 @@ func (s *Server) CreateCommand(ctx context.Context, script string, env []string)
 	cmd.Env = append(cmd.Env, fmt.Sprintf("SSH_CLIENT=%s %s %s", srcAddr, srcPort, dstPort))
 	cmd.Env = append(cmd.Env, fmt.Sprintf("SSH_CONNECTION=%s %s %s %s", srcAddr, srcPort, dstAddr, dstPort))
 
-	// This adds the ports dialog to code-server that enables
-	// proxying a port dynamically.
-	cmd.Env = append(cmd.Env, fmt.Sprintf("VSCODE_PROXY_URI=%s", manifest.VSCodePortProxyURI))
-
-	// Hide Coder message on code-server's "Getting Started" page
-	cmd.Env = append(cmd.Env, "CS_DISABLE_GETTING_STARTED_OVERRIDE=true")
-
-	// Load environment variables passed via the agent.
-	// These should override all variables we manually specify.
-	for envKey, value := range manifest.EnvironmentVariables {
-		// Expanding environment variables allows for customization
-		// of the $PATH, among other variables. Customers can prepend
-		// or append to the $PATH, so allowing expand is required!
-		cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", envKey, os.ExpandEnv(value)))
-	}
-
-	// Agent-level environment variables should take over all!
-	// This is used for setting agent-specific variables like "CODER_AGENT_TOKEN".
-	for envKey, value := range s.Env {
-		cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", envKey, value))
+	cmd.Env, err = s.config.UpdateEnv(cmd.Env)
+	if err != nil {
+		return nil, xerrors.Errorf("apply env: %w", err)
 	}
 
 	return cmd, nil

agent/agentssh/agentssh_internal_test.go

@@ -15,8 +15,8 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
-	"github.com/coder/coder/pty"
-	"github.com/coder/coder/testutil"
+	"github.com/coder/coder/v2/pty"
+	"github.com/coder/coder/v2/testutil"
 
 	"cdr.dev/slog/sloggers/slogtest"
 )
@@ -37,7 +37,7 @@ func Test_sessionStart_orphan(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitMedium)
 	defer cancel()
 	logger := slogtest.Make(t, nil)
-	s, err := NewServer(ctx, logger, prometheus.NewRegistry(), afero.NewMemMapFs(), 0, "")
+	s, err := NewServer(ctx, logger, prometheus.NewRegistry(), afero.NewMemMapFs(), nil)
 	require.NoError(t, err)
 	defer s.Close()
 
@@ -63,7 +63,7 @@ func Test_sessionStart_orphan(t *testing.T) {
 		// we don't really care what the error is here.  In the larger scenario,
 		// the client has disconnected, so we can't return any error information
 		// to them.
-		_ = s.startPTYSession(sess, "ssh", cmd, ptyInfo, windowSize)
+		_ = s.startPTYSession(logger, sess, "ssh", cmd, ptyInfo, windowSize)
 	}()
 
 	readDone := make(chan struct{})
@@ -114,6 +114,11 @@ type testSSHContext struct {
 	context.Context
 }
 
+var (
+	_ gliderssh.Context = testSSHContext{}
+	_ ptySession        = &testSession{}
+)
+
 func newTestSession(ctx context.Context) (toClient *io.PipeReader, fromClient *io.PipeWriter, s ptySession) {
 	toClient, fromPty := io.Pipe()
 	toPty, fromClient := io.Pipe()
@@ -144,6 +149,10 @@ func (s *testSession) Write(p []byte) (n int, err error) {
 	return s.fromPty.Write(p)
 }
 
+func (*testSession) Signals(_ chan<- gliderssh.Signal) {
+	// Not implemented, but will be called.
+}
+
 func (testSSHContext) Lock() {
 	panic("not implemented")
 }

agent/agentssh/agentssh_test.go

@@ -3,9 +3,12 @@
 package agentssh_test
 
 import (
+	"bufio"
 	"bytes"
 	"context"
+	"fmt"
 	"net"
+	"runtime"
 	"strings"
 	"sync"
 	"testing"
@@ -14,15 +17,14 @@ import (
 	"github.com/spf13/afero"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-	"go.uber.org/atomic"
 	"go.uber.org/goleak"
 	"golang.org/x/crypto/ssh"
 
 	"cdr.dev/slog/sloggers/slogtest"
 
-	"github.com/coder/coder/agent/agentssh"
-	"github.com/coder/coder/codersdk/agentsdk"
-	"github.com/coder/coder/pty/ptytest"
+	"github.com/coder/coder/v2/agent/agentssh"
+	"github.com/coder/coder/v2/pty/ptytest"
+	"github.com/coder/coder/v2/testutil"
 )
 
 func TestMain(m *testing.M) {
@@ -34,14 +36,10 @@ func TestNewServer_ServeClient(t *testing.T) {
 
 	ctx := context.Background()
 	logger := slogtest.Make(t, nil)
-	s, err := agentssh.NewServer(ctx, logger, prometheus.NewRegistry(), afero.NewMemMapFs(), 0, "")
+	s, err := agentssh.NewServer(ctx, logger, prometheus.NewRegistry(), afero.NewMemMapFs(), nil)
 	require.NoError(t, err)
 	defer s.Close()
 
-	// The assumption is that these are set before serving SSH connections.
-	s.AgentToken = func() string { return "" }
-	s.Manifest = atomic.NewPointer(&agentsdk.Manifest{})
-
 	ln, err := net.Listen("tcp", "127.0.0.1:0")
 	require.NoError(t, err)
 
@@ -56,8 +54,8 @@ func TestNewServer_ServeClient(t *testing.T) {
 
 	var b bytes.Buffer
 	sess, err := c.NewSession()
-	sess.Stdout = &b
 	require.NoError(t, err)
+	sess.Stdout = &b
 	err = sess.Start("echo hello")
 	require.NoError(t, err)
 
@@ -71,19 +69,49 @@ func TestNewServer_ServeClient(t *testing.T) {
 	<-done
 }
 
+func TestNewServer_ExecuteShebang(t *testing.T) {
+	t.Parallel()
+	if runtime.GOOS == "windows" {
+		t.Skip("bash doesn't exist on Windows")
+	}
+
+	ctx := context.Background()
+	logger := slogtest.Make(t, nil)
+	s, err := agentssh.NewServer(ctx, logger, prometheus.NewRegistry(), afero.NewMemMapFs(), nil)
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		_ = s.Close()
+	})
+
+	t.Run("Basic", func(t *testing.T) {
+		t.Parallel()
+		cmd, err := s.CreateCommand(ctx, `#!/bin/bash
+		echo test`, nil)
+		require.NoError(t, err)
+		output, err := cmd.AsExec().CombinedOutput()
+		require.NoError(t, err)
+		require.Equal(t, "test\n", string(output))
+	})
+	t.Run("Args", func(t *testing.T) {
+		t.Parallel()
+		cmd, err := s.CreateCommand(ctx, `#!/usr/bin/env bash
+		echo test`, nil)
+		require.NoError(t, err)
+		output, err := cmd.AsExec().CombinedOutput()
+		require.NoError(t, err)
+		require.Equal(t, "test\n", string(output))
+	})
+}
+
 func TestNewServer_CloseActiveConnections(t *testing.T) {
 	t.Parallel()
 
 	ctx := context.Background()
 	logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true})
-	s, err := agentssh.NewServer(ctx, logger, prometheus.NewRegistry(), afero.NewMemMapFs(), 0, "")
+	s, err := agentssh.NewServer(ctx, logger, prometheus.NewRegistry(), afero.NewMemMapFs(), nil)
 	require.NoError(t, err)
 	defer s.Close()
 
-	// The assumption is that these are set before serving SSH connections.
-	s.AgentToken = func() string { return "" }
-	s.Manifest = atomic.NewPointer(&agentsdk.Manifest{})
-
 	ln, err := net.Listen("tcp", "127.0.0.1:0")
 	require.NoError(t, err)
 
@@ -102,6 +130,7 @@ func TestNewServer_CloseActiveConnections(t *testing.T) {
 		defer wg.Done()
 		c := sshClient(t, ln.Addr().String())
 		sess, err := c.NewSession()
+		assert.NoError(t, err)
 		sess.Stdin = pty.Input()
 		sess.Stdout = pty.Output()
 		sess.Stderr = pty.Output()
@@ -122,6 +151,151 @@ func TestNewServer_CloseActiveConnections(t *testing.T) {
 	wg.Wait()
 }
 
+func TestNewServer_Signal(t *testing.T) {
+	t.Parallel()
+
+	t.Run("Stdout", func(t *testing.T) {
+		t.Parallel()
+
+		ctx := context.Background()
+		logger := slogtest.Make(t, nil)
+		s, err := agentssh.NewServer(ctx, logger, prometheus.NewRegistry(), afero.NewMemMapFs(), nil)
+		require.NoError(t, err)
+		defer s.Close()
+
+		ln, err := net.Listen("tcp", "127.0.0.1:0")
+		require.NoError(t, err)
+
+		done := make(chan struct{})
+		go func() {
+			defer close(done)
+			err := s.Serve(ln)
+			assert.Error(t, err) // Server is closed.
+		}()
+		defer func() {
+			err := s.Close()
+			require.NoError(t, err)
+			<-done
+		}()
+
+		c := sshClient(t, ln.Addr().String())
+
+		sess, err := c.NewSession()
+		require.NoError(t, err)
+		r, err := sess.StdoutPipe()
+		require.NoError(t, err)
+
+		// Perform multiple sleeps since the interrupt signal doesn't propagate to
+		// the process group, this lets us exit early.
+		sleeps := strings.Repeat("sleep 1 && ", int(testutil.WaitMedium.Seconds()))
+		err = sess.Start(fmt.Sprintf("echo hello && %s echo bye", sleeps))
+		require.NoError(t, err)
+
+		sc := bufio.NewScanner(r)
+		for sc.Scan() {
+			t.Log(sc.Text())
+			if strings.Contains(sc.Text(), "hello") {
+				break
+			}
+		}
+		require.NoError(t, sc.Err())
+
+		err = sess.Signal(ssh.SIGKILL)
+		require.NoError(t, err)
+
+		// Assumption, signal propagates and the command exists, closing stdout.
+		for sc.Scan() {
+			t.Log(sc.Text())
+			require.NotContains(t, sc.Text(), "bye")
+		}
+		require.NoError(t, sc.Err())
+
+		err = sess.Wait()
+		exitErr := &ssh.ExitError{}
+		require.ErrorAs(t, err, &exitErr)
+		wantCode := 255
+		if runtime.GOOS == "windows" {
+			wantCode = 1
+		}
+		require.Equal(t, wantCode, exitErr.ExitStatus())
+	})
+	t.Run("PTY", func(t *testing.T) {
+		t.Parallel()
+
+		ctx := context.Background()
+		logger := slogtest.Make(t, nil)
+		s, err := agentssh.NewServer(ctx, logger, prometheus.NewRegistry(), afero.NewMemMapFs(), nil)
+		require.NoError(t, err)
+		defer s.Close()
+
+		ln, err := net.Listen("tcp", "127.0.0.1:0")
+		require.NoError(t, err)
+
+		done := make(chan struct{})
+		go func() {
+			defer close(done)
+			err := s.Serve(ln)
+			assert.Error(t, err) // Server is closed.
+		}()
+		defer func() {
+			err := s.Close()
+			require.NoError(t, err)
+			<-done
+		}()
+
+		c := sshClient(t, ln.Addr().String())
+
+		pty := ptytest.New(t)
+
+		sess, err := c.NewSession()
+		require.NoError(t, err)
+		r, err := sess.StdoutPipe()
+		require.NoError(t, err)
+
+		// Note, we request pty but don't use ptytest here because we can't
+		// easily test for no text before EOF.
+		sess.Stdin = pty.Input()
+		sess.Stderr = pty.Output()
+
+		err = sess.RequestPty("xterm", 80, 80, nil)
+		require.NoError(t, err)
+
+		// Perform multiple sleeps since the interrupt signal doesn't propagate to
+		// the process group, this lets us exit early.
+		sleeps := strings.Repeat("sleep 1 && ", int(testutil.WaitMedium.Seconds()))
+		err = sess.Start(fmt.Sprintf("echo hello && %s echo bye", sleeps))
+		require.NoError(t, err)
+
+		sc := bufio.NewScanner(r)
+		for sc.Scan() {
+			t.Log(sc.Text())
+			if strings.Contains(sc.Text(), "hello") {
+				break
+			}
+		}
+		require.NoError(t, sc.Err())
+
+		err = sess.Signal(ssh.SIGKILL)
+		require.NoError(t, err)
+
+		// Assumption, signal propagates and the command exists, closing stdout.
+		for sc.Scan() {
+			t.Log(sc.Text())
+			require.NotContains(t, sc.Text(), "bye")
+		}
+		require.NoError(t, sc.Err())
+
+		err = sess.Wait()
+		exitErr := &ssh.ExitError{}
+		require.ErrorAs(t, err, &exitErr)
+		wantCode := 255
+		if runtime.GOOS == "windows" {
+			wantCode = 1
+		}
+		require.Equal(t, wantCode, exitErr.ExitStatus())
+	})
+}
+
 func sshClient(t *testing.T, addr string) *ssh.Client {
 	conn, err := net.Dial("tcp", addr)
 	require.NoError(t, err)

agent/agentssh/forward.go

@@ -2,11 +2,14 @@ package agentssh
 
 import (
 	"context"
+	"errors"
 	"fmt"
+	"io/fs"
 	"net"
 	"os"
 	"path/filepath"
 	"sync"
+	"syscall"
 
 	"github.com/gliderlabs/ssh"
 	gossh "golang.org/x/crypto/ssh"
@@ -33,70 +36,98 @@ type forwardedStreamLocalPayload struct {
 type forwardedUnixHandler struct {
 	sync.Mutex
 	log      slog.Logger
-	forwards map[string]net.Listener
+	forwards map[forwardKey]net.Listener
+}
+
+type forwardKey struct {
+	sessionID string
+	addr      string
+}
+
+func newForwardedUnixHandler(log slog.Logger) *forwardedUnixHandler {
+	return &forwardedUnixHandler{
+		log:      log,
+		forwards: make(map[forwardKey]net.Listener),
+	}
 }
 
 func (h *forwardedUnixHandler) HandleSSHRequest(ctx ssh.Context, _ *ssh.Server, req *gossh.Request) (bool, []byte) {
-	h.Lock()
-	if h.forwards == nil {
-		h.forwards = make(map[string]net.Listener)
-	}
-	h.Unlock()
+	h.log.Debug(ctx, "handling SSH unix forward")
 	conn, ok := ctx.Value(ssh.ContextKeyConn).(*gossh.ServerConn)
 	if !ok {
 		h.log.Warn(ctx, "SSH unix forward request from client with no gossh connection")
 		return false, nil
 	}
+	log := h.log.With(slog.F("session_id", ctx.SessionID()), slog.F("remote_addr", conn.RemoteAddr()))
 
 	switch req.Type {
 	case "streamlocal-forward@openssh.com":
 		var reqPayload streamLocalForwardPayload
 		err := gossh.Unmarshal(req.Payload, &reqPayload)
 		if err != nil {
-			h.log.Warn(ctx, "parse streamlocal-forward@openssh.com request payload from client", slog.Error(err))
+			h.log.Warn(ctx, "parse streamlocal-forward@openssh.com request (SSH unix forward) payload from client", slog.Error(err))
 			return false, nil
 		}
 
 		addr := reqPayload.SocketPath
+		log = log.With(slog.F("socket_path", addr))
+		log.Debug(ctx, "request begin SSH unix forward")
+
+		key := forwardKey{
+			sessionID: ctx.SessionID(),
+			addr:      addr,
+		}
+
 		h.Lock()
-		_, ok := h.forwards[addr]
+		_, ok := h.forwards[key]
 		h.Unlock()
 		if ok {
-			h.log.Warn(ctx, "SSH unix forward request for socket path that is already being forwarded (maybe to another client?)",
-				slog.F("socket_path", addr),
-			)
-			return false, nil
+			// In cases where `ExitOnForwardFailure=yes` is set, returning false
+			// here will cause the connection to be closed. To avoid this, and
+			// to match OpenSSH behavior, we silently ignore the second forward
+			// request.
+			log.Warn(ctx, "SSH unix forward request for socket path that is already being forwarded on this session, ignoring")
+			return true, nil
 		}
 
 		// Create socket parent dir if not exists.
 		parentDir := filepath.Dir(addr)
 		err = os.MkdirAll(parentDir, 0o700)
 		if err != nil {
-			h.log.Warn(ctx, "create parent dir for SSH unix forward request",
+			log.Warn(ctx, "create parent dir for SSH unix forward request",
 				slog.F("parent_dir", parentDir),
-				slog.F("socket_path", addr),
 				slog.Error(err),
 			)
 			return false, nil
 		}
 
-		ln, err := net.Listen("unix", addr)
-		if err != nil {
-			h.log.Warn(ctx, "listen on Unix socket for SSH unix forward request",
-				slog.F("socket_path", addr),
-				slog.Error(err),
-			)
+		// Remove existing socket if it exists. We do not use os.Remove() here
+		// so that directories are kept. Note that it's possible that we will
+		// overwrite a regular file here. Both of these behaviors match OpenSSH,
+		// however, which is why we unlink.
+		err = unlink(addr)
+		if err != nil && !errors.Is(err, fs.ErrNotExist) {
+			log.Warn(ctx, "remove existing socket for SSH unix forward request", slog.Error(err))
 			return false, nil
 		}
 
+		lc := &net.ListenConfig{}
+		ln, err := lc.Listen(ctx, "unix", addr)
+		if err != nil {
+			log.Warn(ctx, "listen on Unix socket for SSH unix forward request", slog.Error(err))
+			return false, nil
+		}
+		log.Debug(ctx, "SSH unix forward listening on socket")
+
 		// The listener needs to successfully start before it can be added to
 		// the map, so we don't have to worry about checking for an existing
 		// listener.
 		//
 		// This is also what the upstream TCP version of this code does.
 		h.Lock()
-		h.forwards[addr] = ln
+		h.forwards[key] = ln
 		h.Unlock()
+		log.Debug(ctx, "SSH unix forward added to cache")
 
 		ctx, cancel := context.WithCancel(ctx)
 		go func() {
@@ -110,14 +141,13 @@ func (h *forwardedUnixHandler) HandleSSHRequest(ctx ssh.Context, _ *ssh.Server,
 				c, err := ln.Accept()
 				if err != nil {
 					if !xerrors.Is(err, net.ErrClosed) {
-						h.log.Warn(ctx, "accept on local Unix socket for SSH unix forward request",
-							slog.F("socket_path", addr),
-							slog.Error(err),
-						)
+						log.Warn(ctx, "accept on local Unix socket for SSH unix forward request", slog.Error(err))
 					}
 					// closed below
+					log.Debug(ctx, "SSH unix forward listener closed")
 					break
 				}
+				log.Debug(ctx, "accepted SSH unix forward connection")
 				payload := gossh.Marshal(&forwardedStreamLocalPayload{
 					SocketPath: addr,
 				})
@@ -125,10 +155,7 @@ func (h *forwardedUnixHandler) HandleSSHRequest(ctx ssh.Context, _ *ssh.Server,
 				go func() {
 					ch, reqs, err := conn.OpenChannel("forwarded-streamlocal@openssh.com", payload)
 					if err != nil {
-						h.log.Warn(ctx, "open SSH channel to forward Unix connection to client",
-							slog.F("socket_path", addr),
-							slog.Error(err),
-						)
+						h.log.Warn(ctx, "open SSH unix forward channel to client", slog.Error(err))
 						_ = c.Close()
 						return
 					}
@@ -138,11 +165,11 @@ func (h *forwardedUnixHandler) HandleSSHRequest(ctx ssh.Context, _ *ssh.Server,
 			}
 
 			h.Lock()
-			ln2, ok := h.forwards[addr]
-			if ok && ln2 == ln {
-				delete(h.forwards, addr)
+			if ln2, ok := h.forwards[key]; ok && ln2 == ln {
+				delete(h.forwards, key)
 			}
 			h.Unlock()
+			log.Debug(ctx, "SSH unix forward listener removed from cache")
 			_ = ln.Close()
 		}()
 
@@ -152,15 +179,25 @@ func (h *forwardedUnixHandler) HandleSSHRequest(ctx ssh.Context, _ *ssh.Server,
 		var reqPayload streamLocalForwardPayload
 		err := gossh.Unmarshal(req.Payload, &reqPayload)
 		if err != nil {
-			h.log.Warn(ctx, "parse cancel-streamlocal-forward@openssh.com request payload from client", slog.Error(err))
+			h.log.Warn(ctx, "parse cancel-streamlocal-forward@openssh.com (SSH unix forward) request payload from client", slog.Error(err))
 			return false, nil
 		}
-		h.Lock()
-		ln, ok := h.forwards[reqPayload.SocketPath]
-		h.Unlock()
-		if ok {
-			_ = ln.Close()
+		log.Debug(ctx, "request to cancel SSH unix forward", slog.F("socket_path", reqPayload.SocketPath))
+
+		key := forwardKey{
+			sessionID: ctx.SessionID(),
+			addr:      reqPayload.SocketPath,
 		}
+
+		h.Lock()
+		ln, ok := h.forwards[key]
+		delete(h.forwards, key)
+		h.Unlock()
+		if !ok {
+			log.Warn(ctx, "SSH unix forward not found in cache")
+			return true, nil
+		}
+		_ = ln.Close()
 		return true, nil
 
 	default:
@@ -201,3 +238,15 @@ func directStreamLocalHandler(_ *ssh.Server, _ *gossh.ServerConn, newChan gossh.
 
 	Bicopy(ctx, ch, dconn)
 }
+
+// unlink removes files and unlike os.Remove, directories are kept.
+func unlink(path string) error {
+	// Ignore EINTR like os.Remove, see ignoringEINTR in os/file_posix.go
+	// for more details.
+	for {
+		err := syscall.Unlink(path)
+		if !errors.Is(err, syscall.EINTR) {
+			return err
+		}
+	}
+}

agent/agentssh/jetbrainstrack.go

@@ -0,0 +1,97 @@
+package agentssh
+
+import (
+	"context"
+	"strings"
+	"sync"
+
+	"github.com/gliderlabs/ssh"
+	"go.uber.org/atomic"
+	gossh "golang.org/x/crypto/ssh"
+
+	"cdr.dev/slog"
+)
+
+// localForwardChannelData is copied from the ssh package.
+type localForwardChannelData struct {
+	DestAddr string
+	DestPort uint32
+
+	OriginAddr string
+	OriginPort uint32
+}
+
+// JetbrainsChannelWatcher is used to track JetBrains port forwarded (Gateway)
+// channels. If the port forward is something other than JetBrains, this struct
+// is a noop.
+type JetbrainsChannelWatcher struct {
+	gossh.NewChannel
+	jetbrainsCounter *atomic.Int64
+	logger           slog.Logger
+}
+
+func NewJetbrainsChannelWatcher(ctx ssh.Context, logger slog.Logger, newChannel gossh.NewChannel, counter *atomic.Int64) gossh.NewChannel {
+	d := localForwardChannelData{}
+	if err := gossh.Unmarshal(newChannel.ExtraData(), &d); err != nil {
+		// If the data fails to unmarshal, do nothing.
+		logger.Warn(ctx, "failed to unmarshal port forward data", slog.Error(err))
+		return newChannel
+	}
+
+	// If we do get a port, we should be able to get the matching PID and from
+	// there look up the invocation.
+	cmdline, err := getListeningPortProcessCmdline(d.DestPort)
+	if err != nil {
+		logger.Warn(ctx, "failed to inspect port",
+			slog.F("destination_port", d.DestPort),
+			slog.Error(err))
+		return newChannel
+	}
+
+	// If this is not JetBrains, then we do not need to do anything special.  We
+	// attempt to match on something that appears unique to JetBrains software.
+	if !strings.Contains(strings.ToLower(cmdline), strings.ToLower(MagicProcessCmdlineJetBrains)) {
+		return newChannel
+	}
+
+	logger.Debug(ctx, "discovered forwarded JetBrains process",
+		slog.F("destination_port", d.DestPort))
+
+	return &JetbrainsChannelWatcher{
+		NewChannel:       newChannel,
+		jetbrainsCounter: counter,
+		logger:           logger.With(slog.F("destination_port", d.DestPort)),
+	}
+}
+
+func (w *JetbrainsChannelWatcher) Accept() (gossh.Channel, <-chan *gossh.Request, error) {
+	c, r, err := w.NewChannel.Accept()
+	if err != nil {
+		return c, r, err
+	}
+	w.jetbrainsCounter.Add(1)
+	// nolint: gocritic // JetBrains is a proper noun and should be capitalized
+	w.logger.Debug(context.Background(), "JetBrains watcher accepted channel")
+
+	return &ChannelOnClose{
+		Channel: c,
+		done: func() {
+			w.jetbrainsCounter.Add(-1)
+			// nolint: gocritic // JetBrains is a proper noun and should be capitalized
+			w.logger.Debug(context.Background(), "JetBrains watcher channel closed")
+		},
+	}, r, err
+}
+
+type ChannelOnClose struct {
+	gossh.Channel
+	// once ensures close only decrements the counter once.
+	// Because close can be called multiple times.
+	once sync.Once
+	done func()
+}
+
+func (c *ChannelOnClose) Close() error {
+	c.once.Do(c.done)
+	return c.Channel.Close()
+}

agent/agentssh/metrics.go

@@ -1,6 +1,8 @@
 package agentssh
 
 import (
+	"strings"
+
 	"github.com/prometheus/client_golang/prometheus"
 )
 
@@ -78,5 +80,6 @@ func magicTypeMetricLabel(magicType string) string {
 	default:
 		magicType = "unknown"
 	}
-	return magicType
+	// Always be case insensitive
+	return strings.ToLower(magicType)
 }

agent/agentssh/portinspection_supported.go

@@ -0,0 +1,51 @@
+//go:build linux
+
+package agentssh
+
+import (
+	"errors"
+	"fmt"
+	"os"
+
+	"github.com/cakturk/go-netstat/netstat"
+	"golang.org/x/xerrors"
+)
+
+func getListeningPortProcessCmdline(port uint32) (string, error) {
+	acceptFn := func(s *netstat.SockTabEntry) bool {
+		return s.LocalAddr != nil && uint32(s.LocalAddr.Port) == port
+	}
+	tabs4, err4 := netstat.TCPSocks(acceptFn)
+	tabs6, err6 := netstat.TCP6Socks(acceptFn)
+
+	// In the common case, we want to check ipv4 listening addresses.  If this
+	// fails, we should return an error.  We also need to check ipv6.  The
+	// assumption is, if we have an err4, and 0 ipv6 addresses listed, then we are
+	// interested in the err4 (and vice versa).  So return both errors (at least 1
+	// is non-nil) if the other list is empty.
+	if (err4 != nil && len(tabs6) == 0) || (err6 != nil && len(tabs4) == 0) {
+		return "", xerrors.Errorf("inspect port %d: %w", port, errors.Join(err4, err6))
+	}
+
+	var proc *netstat.Process
+	if len(tabs4) > 0 {
+		proc = tabs4[0].Process
+	} else if len(tabs6) > 0 {
+		proc = tabs6[0].Process
+	}
+	if proc == nil {
+		// Either nothing is listening on this port or we were unable to read the
+		// process details (permission issues reading /proc/$pid/* potentially).
+		// Or, perhaps /proc/net/tcp{,6} is not listing the port for some reason.
+		return "", nil
+	}
+
+	// The process name provided by go-netstat does not include the full command
+	// line so grab that instead.
+	pid := proc.Pid
+	data, err := os.ReadFile(fmt.Sprintf("/proc/%d/cmdline", pid))
+	if err != nil {
+		return "", xerrors.Errorf("read /proc/%d/cmdline: %w", pid, err)
+	}
+	return string(data), nil
+}

agent/agentssh/portinspection_unsupported.go

@@ -0,0 +1,9 @@
+//go:build !linux
+
+package agentssh
+
+func getListeningPortProcessCmdline(uint32) (string, error) {
+	// We are not worrying about other platforms at the moment because Gateway
+	// only supports Linux anyway.
+	return "", nil
+}

agent/agentssh/signal_other.go

@@ -0,0 +1,45 @@
+//go:build !windows
+
+package agentssh
+
+import (
+	"os"
+
+	"github.com/gliderlabs/ssh"
+	"golang.org/x/sys/unix"
+)
+
+func osSignalFrom(sig ssh.Signal) os.Signal {
+	switch sig {
+	case ssh.SIGABRT:
+		return unix.SIGABRT
+	case ssh.SIGALRM:
+		return unix.SIGALRM
+	case ssh.SIGFPE:
+		return unix.SIGFPE
+	case ssh.SIGHUP:
+		return unix.SIGHUP
+	case ssh.SIGILL:
+		return unix.SIGILL
+	case ssh.SIGINT:
+		return unix.SIGINT
+	case ssh.SIGKILL:
+		return unix.SIGKILL
+	case ssh.SIGPIPE:
+		return unix.SIGPIPE
+	case ssh.SIGQUIT:
+		return unix.SIGQUIT
+	case ssh.SIGSEGV:
+		return unix.SIGSEGV
+	case ssh.SIGTERM:
+		return unix.SIGTERM
+	case ssh.SIGUSR1:
+		return unix.SIGUSR1
+	case ssh.SIGUSR2:
+		return unix.SIGUSR2
+
+	// Unhandled, use sane fallback.
+	default:
+		return unix.SIGKILL
+	}
+}

agent/agentssh/signal_windows.go

@@ -0,0 +1,15 @@
+package agentssh
+
+import (
+	"os"
+
+	"github.com/gliderlabs/ssh"
+)
+
+func osSignalFrom(sig ssh.Signal) os.Signal {
+	switch sig {
+	// Signals are not supported on Windows.
+	default:
+		return os.Kill
+	}
+}

agent/agentssh/x11.go

@@ -6,6 +6,7 @@ import (
 	"encoding/hex"
 	"errors"
 	"fmt"
+	"io"
 	"net"
 	"os"
 	"path/filepath"
@@ -31,9 +32,9 @@ func (s *Server) x11Callback(ctx ssh.Context, x11 ssh.X11) bool {
 		return false
 	}
 
-	err = s.fs.MkdirAll(s.x11SocketDir, 0o700)
+	err = s.fs.MkdirAll(s.config.X11SocketDir, 0o700)
 	if err != nil {
-		s.logger.Warn(ctx, "failed to make the x11 socket dir", slog.F("dir", s.x11SocketDir), slog.Error(err))
+		s.logger.Warn(ctx, "failed to make the x11 socket dir", slog.F("dir", s.config.X11SocketDir), slog.Error(err))
 		s.metrics.x11HandlerErrors.WithLabelValues("socker_dir").Add(1)
 		return false
 	}
@@ -56,7 +57,7 @@ func (s *Server) x11Handler(ctx ssh.Context, x11 ssh.X11) bool {
 		return false
 	}
 	// We want to overwrite the socket so that subsequent connections will succeed.
-	socketPath := filepath.Join(s.x11SocketDir, fmt.Sprintf("X%d", x11.ScreenNumber))
+	socketPath := filepath.Join(s.config.X11SocketDir, fmt.Sprintf("X%d", x11.ScreenNumber))
 	err := os.Remove(socketPath)
 	if err != nil && !errors.Is(err, os.ErrNotExist) {
 		s.logger.Warn(ctx, "failed to remove existing X11 socket", slog.Error(err))
@@ -141,7 +142,7 @@ func addXauthEntry(ctx context.Context, fs afero.Fs, host string, display string
 	}
 
 	// Open or create the Xauthority file
-	file, err := fs.OpenFile(xauthPath, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0o600)
+	file, err := fs.OpenFile(xauthPath, os.O_RDWR|os.O_CREATE, 0o600)
 	if err != nil {
 		return xerrors.Errorf("failed to open Xauthority file: %w", err)
 	}
@@ -153,7 +154,105 @@ func addXauthEntry(ctx context.Context, fs afero.Fs, host string, display string
 		return xerrors.Errorf("failed to decode auth cookie: %w", err)
 	}
 
-	// Write Xauthority entry
+	// Read the Xauthority file and look for an existing entry for the host,
+	// display, and auth protocol. If an entry is found, overwrite the auth
+	// cookie (if it fits). Otherwise, mark the entry for deletion.
+	type deleteEntry struct {
+		start, end int
+	}
+	var deleteEntries []deleteEntry
+	pos := 0
+	updated := false
+	for {
+		entry, err := readXauthEntry(file)
+		if err != nil {
+			if errors.Is(err, io.EOF) {
+				break
+			}
+			return xerrors.Errorf("failed to read Xauthority entry: %w", err)
+		}
+
+		nextPos := pos + entry.Len()
+		cookieStartPos := nextPos - len(entry.authCookie)
+
+		if entry.family == 0x0100 && entry.address == host && entry.display == display && entry.authProtocol == authProtocol {
+			if !updated && len(entry.authCookie) == len(authCookieBytes) {
+				// Overwrite the auth cookie
+				_, err := file.WriteAt(authCookieBytes, int64(cookieStartPos))
+				if err != nil {
+					return xerrors.Errorf("failed to write auth cookie: %w", err)
+				}
+				updated = true
+			} else {
+				// Mark entry for deletion.
+				if len(deleteEntries) > 0 && deleteEntries[len(deleteEntries)-1].end == pos {
+					deleteEntries[len(deleteEntries)-1].end = nextPos
+				} else {
+					deleteEntries = append(deleteEntries, deleteEntry{
+						start: pos,
+						end:   nextPos,
+					})
+				}
+			}
+		}
+
+		pos = nextPos
+	}
+
+	// In case the magic cookie changed, or we've previously bloated the
+	// Xauthority file, we may have to delete entries.
+	if len(deleteEntries) > 0 {
+		// Read the entire file into memory. This is not ideal, but it's the
+		// simplest way to delete entries from the middle of the file. The
+		// Xauthority file is small, so this should be fine.
+		_, err = file.Seek(0, io.SeekStart)
+		if err != nil {
+			return xerrors.Errorf("failed to seek Xauthority file: %w", err)
+		}
+		data, err := io.ReadAll(file)
+		if err != nil {
+			return xerrors.Errorf("failed to read Xauthority file: %w", err)
+		}
+
+		// Delete the entries in reverse order.
+		for i := len(deleteEntries) - 1; i >= 0; i-- {
+			entry := deleteEntries[i]
+			// Safety check: ensure the entry is still there.
+			if entry.start > len(data) || entry.end > len(data) {
+				continue
+			}
+			data = append(data[:entry.start], data[entry.end:]...)
+		}
+
+		// Write the data back to the file.
+		_, err = file.Seek(0, io.SeekStart)
+		if err != nil {
+			return xerrors.Errorf("failed to seek Xauthority file: %w", err)
+		}
+		_, err = file.Write(data)
+		if err != nil {
+			return xerrors.Errorf("failed to write Xauthority file: %w", err)
+		}
+
+		// Truncate the file.
+		err = file.Truncate(int64(len(data)))
+		if err != nil {
+			return xerrors.Errorf("failed to truncate Xauthority file: %w", err)
+		}
+	}
+
+	// Return if we've already updated the entry.
+	if updated {
+		return nil
+	}
+
+	// Ensure we're at the end (append).
+	_, err = file.Seek(0, io.SeekEnd)
+	if err != nil {
+		return xerrors.Errorf("failed to seek Xauthority file: %w", err)
+	}
+
+	// Append Xauthority entry.
 	family := uint16(0x0100) // FamilyLocal
 	err = binary.Write(file, binary.BigEndian, family)
 	if err != nil {
@@ -198,3 +297,96 @@ func addXauthEntry(ctx context.Context, fs afero.Fs, host string, display string
 
 	return nil
 }
+
+// xauthEntry is an representation of an Xauthority entry.
+//
+// The Xauthority file format is as follows:
+//
+// - 16-bit family
+// - 16-bit address length
+// - address
+// - 16-bit display length
+// - display
+// - 16-bit auth protocol length
+// - auth protocol
+// - 16-bit auth cookie length
+// - auth cookie
+type xauthEntry struct {
+	family       uint16
+	address      string
+	display      string
+	authProtocol string
+	authCookie   []byte
+}
+
+func (e xauthEntry) Len() int {
+	// 5 * uint16 = 10 bytes for the family/length fields.
+	return 2*5 + len(e.address) + len(e.display) + len(e.authProtocol) + len(e.authCookie)
+}
+
+func readXauthEntry(r io.Reader) (xauthEntry, error) {
+	var entry xauthEntry
+
+	// Read family
+	err := binary.Read(r, binary.BigEndian, &entry.family)
+	if err != nil {
+		return xauthEntry{}, xerrors.Errorf("failed to read family: %w", err)
+	}
+
+	// Read address
+	var addressLength uint16
+	err = binary.Read(r, binary.BigEndian, &addressLength)
+	if err != nil {
+		return xauthEntry{}, xerrors.Errorf("failed to read address length: %w", err)
+	}
+
+	addressBytes := make([]byte, addressLength)
+	_, err = r.Read(addressBytes)
+	if err != nil {
+		return xauthEntry{}, xerrors.Errorf("failed to read address: %w", err)
+	}
+	entry.address = string(addressBytes)
+
+	// Read display
+	var displayLength uint16
+	err = binary.Read(r, binary.BigEndian, &displayLength)
+	if err != nil {
+		return xauthEntry{}, xerrors.Errorf("failed to read display length: %w", err)
+	}
+
+	displayBytes := make([]byte, displayLength)
+	_, err = r.Read(displayBytes)
+	if err != nil {
+		return xauthEntry{}, xerrors.Errorf("failed to read display: %w", err)
+	}
+	entry.display = string(displayBytes)
+
+	// Read auth protocol
+	var authProtocolLength uint16
+	err = binary.Read(r, binary.BigEndian, &authProtocolLength)
+	if err != nil {
+		return xauthEntry{}, xerrors.Errorf("failed to read auth protocol length: %w", err)
+	}
+
+	authProtocolBytes := make([]byte, authProtocolLength)
+	_, err = r.Read(authProtocolBytes)
+	if err != nil {
+		return xauthEntry{}, xerrors.Errorf("failed to read auth protocol: %w", err)
+	}
+	entry.authProtocol = string(authProtocolBytes)
+
+	// Read auth cookie
+	var authCookieLength uint16
+	err = binary.Read(r, binary.BigEndian, &authCookieLength)
+	if err != nil {
+		return xauthEntry{}, xerrors.Errorf("failed to read auth cookie length: %w", err)
+	}
+
+	entry.authCookie = make([]byte, authCookieLength)
+	_, err = r.Read(entry.authCookie)
+	if err != nil {
+		return xauthEntry{}, xerrors.Errorf("failed to read auth cookie: %w", err)
+	}
+
+	return entry, nil
+}

agent/agentssh/x11_internal_test.go

@@ -0,0 +1,254 @@
+package agentssh
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/spf13/afero"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func Test_addXauthEntry(t *testing.T) {
+	t.Parallel()
+
+	type testEntry struct {
+		address      string
+		display      string
+		authProtocol string
+		authCookie   string
+	}
+	tests := []struct {
+		name         string
+		authFile     []byte
+		wantAuthFile []byte
+		entries      []testEntry
+	}{
+		{
+			name:     "add entry",
+			authFile: nil,
+			wantAuthFile: []byte{
+				// w/unix:0  MIT-MAGIC-COOKIE-1  00
+				//
+				// 00000000: 0100 0001 7700 0130 0012 4d49 542d 4d41  ....w..0..MIT-MA
+				// 00000010: 4749 432d 434f 4f4b 4945 2d31 0001 00    GIC-COOKIE-1...
+				0x01, 0x00, 0x00, 0x01, 0x77, 0x00, 0x01, 0x30,
+				0x00, 0x12, 0x4d, 0x49, 0x54, 0x2d, 0x4d, 0x41,
+				0x47, 0x49, 0x43, 0x2d, 0x43, 0x4f, 0x4f, 0x4b,
+				0x49, 0x45, 0x2d, 0x31, 0x00, 0x01, 0x00,
+			},
+			entries: []testEntry{
+				{
+					address:      "w",
+					display:      "0",
+					authProtocol: "MIT-MAGIC-COOKIE-1",
+					authCookie:   "00",
+				},
+			},
+		},
+		{
+			name:     "add two entries",
+			authFile: []byte{},
+			wantAuthFile: []byte{
+				// w/unix:0  MIT-MAGIC-COOKIE-1  00
+				// w/unix:1  MIT-MAGIC-COOKIE-1  11
+				//
+				// 00000000: 0100 0001 7700 0130 0012 4d49 542d 4d41  ....w..0..MIT-MA
+				// 00000010: 4749 432d 434f 4f4b 4945 2d31 0001 0001  GIC-COOKIE-1....
+				// 00000020: 0000 0177 0001 3100 124d 4954 2d4d 4147  ...w..1..MIT-MAG
+				// 00000030: 4943 2d43 4f4f 4b49 452d 3100 0111       IC-COOKIE-1...
+				0x01, 0x00, 0x00, 0x01, 0x77, 0x00, 0x01, 0x30,
+				0x00, 0x12, 0x4d, 0x49, 0x54, 0x2d, 0x4d, 0x41,
+				0x47, 0x49, 0x43, 0x2d, 0x43, 0x4f, 0x4f, 0x4b,
+				0x49, 0x45, 0x2d, 0x31, 0x00, 0x01, 0x00,
+				0x01, 0x00, 0x00, 0x01, 0x77, 0x00, 0x01, 0x31,
+				0x00, 0x12, 0x4d, 0x49, 0x54, 0x2d, 0x4d, 0x41,
+				0x47, 0x49, 0x43, 0x2d, 0x43, 0x4f, 0x4f, 0x4b,
+				0x49, 0x45, 0x2d, 0x31, 0x00, 0x01, 0x11,
+			},
+			entries: []testEntry{
+				{
+					address:      "w",
+					display:      "0",
+					authProtocol: "MIT-MAGIC-COOKIE-1",
+					authCookie:   "00",
+				},
+				{
+					address:      "w",
+					display:      "1",
+					authProtocol: "MIT-MAGIC-COOKIE-1",
+					authCookie:   "11",
+				},
+			},
+		},
+		{
+			name: "update entry with new auth cookie length",
+			authFile: []byte{
+				// w/unix:0  MIT-MAGIC-COOKIE-1  00
+				// w/unix:1  MIT-MAGIC-COOKIE-1  11
+				//
+				// 00000000: 0100 0001 7700 0130 0012 4d49 542d 4d41  ....w..0..MIT-MA
+				// 00000010: 4749 432d 434f 4f4b 4945 2d31 0001 0001  GIC-COOKIE-1....
+				// 00000020: 0000 0177 0001 3100 124d 4954 2d4d 4147  ...w..1..MIT-MAG
+				// 00000030: 4943 2d43 4f4f 4b49 452d 3100 0111       IC-COOKIE-1...
+				0x01, 0x00, 0x00, 0x01, 0x77, 0x00, 0x01, 0x30,
+				0x00, 0x12, 0x4d, 0x49, 0x54, 0x2d, 0x4d, 0x41,
+				0x47, 0x49, 0x43, 0x2d, 0x43, 0x4f, 0x4f, 0x4b,
+				0x49, 0x45, 0x2d, 0x31, 0x00, 0x01, 0x00,
+				0x01, 0x00, 0x00, 0x01, 0x77, 0x00, 0x01, 0x31,
+				0x00, 0x12, 0x4d, 0x49, 0x54, 0x2d, 0x4d, 0x41,
+				0x47, 0x49, 0x43, 0x2d, 0x43, 0x4f, 0x4f, 0x4b,
+				0x49, 0x45, 0x2d, 0x31, 0x00, 0x01, 0x11,
+			},
+			wantAuthFile: []byte{
+				// The order changed, due to new length of auth cookie resulting
+				// in remove + append, we verify that the implementation is
+				// behaving as expected (changing the order is not a requirement,
+				// simply an implementation detail).
+				0x01, 0x00, 0x00, 0x01, 0x77, 0x00, 0x01, 0x31,
+				0x00, 0x12, 0x4d, 0x49, 0x54, 0x2d, 0x4d, 0x41,
+				0x47, 0x49, 0x43, 0x2d, 0x43, 0x4f, 0x4f, 0x4b,
+				0x49, 0x45, 0x2d, 0x31, 0x00, 0x01, 0x11,
+				0x01, 0x00, 0x00, 0x01, 0x77, 0x00, 0x01, 0x30,
+				0x00, 0x12, 0x4d, 0x49, 0x54, 0x2d, 0x4d, 0x41,
+				0x47, 0x49, 0x43, 0x2d, 0x43, 0x4f, 0x4f, 0x4b,
+				0x49, 0x45, 0x2d, 0x31, 0x00, 0x02, 0xff, 0xff,
+			},
+			entries: []testEntry{
+				{
+					address:      "w",
+					display:      "0",
+					authProtocol: "MIT-MAGIC-COOKIE-1",
+					authCookie:   "ffff",
+				},
+			},
+		},
+		{
+			name: "update entry",
+			authFile: []byte{
+				// 00000000: 0100 0001 7700 0130 0012 4d49 542d 4d41  ....w..0..MIT-MA
+				// 00000010: 4749 432d 434f 4f4b 4945 2d31 0001 0001  GIC-COOKIE-1....
+				// 00000020: 0000 0177 0001 3100 124d 4954 2d4d 4147  ...w..1..MIT-MAG
+				// 00000030: 4943 2d43 4f4f 4b49 452d 3100 0111       IC-COOKIE-1...
+				0x01, 0x00, 0x00, 0x01, 0x77, 0x00, 0x01, 0x30,
+				0x00, 0x12, 0x4d, 0x49, 0x54, 0x2d, 0x4d, 0x41,
+				0x47, 0x49, 0x43, 0x2d, 0x43, 0x4f, 0x4f, 0x4b,
+				0x49, 0x45, 0x2d, 0x31, 0x00, 0x01, 0x00,
+				0x01, 0x00, 0x00, 0x01, 0x77, 0x00, 0x01, 0x31,
+				0x00, 0x12, 0x4d, 0x49, 0x54, 0x2d, 0x4d, 0x41,
+				0x47, 0x49, 0x43, 0x2d, 0x43, 0x4f, 0x4f, 0x4b,
+				0x49, 0x45, 0x2d, 0x31, 0x00, 0x01, 0x11,
+			},
+			wantAuthFile: []byte{
+				// 00000000: 0100 0001 7700 0130 0012 4d49 542d 4d41  ....w..0..MIT-MA
+				// 00000010: 4749 432d 434f 4f4b 4945 2d31 0001 0001  GIC-COOKIE-1....
+				// 00000020: 0000 0177 0001 3100 124d 4954 2d4d 4147  ...w..1..MIT-MAG
+				// 00000030: 4943 2d43 4f4f 4b49 452d 3100 0111       IC-COOKIE-1...
+				0x01, 0x00, 0x00, 0x01, 0x77, 0x00, 0x01, 0x30,
+				0x00, 0x12, 0x4d, 0x49, 0x54, 0x2d, 0x4d, 0x41,
+				0x47, 0x49, 0x43, 0x2d, 0x43, 0x4f, 0x4f, 0x4b,
+				0x49, 0x45, 0x2d, 0x31, 0x00, 0x01, 0xff,
+				0x01, 0x00, 0x00, 0x01, 0x77, 0x00, 0x01, 0x31,
+				0x00, 0x12, 0x4d, 0x49, 0x54, 0x2d, 0x4d, 0x41,
+				0x47, 0x49, 0x43, 0x2d, 0x43, 0x4f, 0x4f, 0x4b,
+				0x49, 0x45, 0x2d, 0x31, 0x00, 0x01, 0x11,
+			},
+			entries: []testEntry{
+				{
+					address:      "w",
+					display:      "0",
+					authProtocol: "MIT-MAGIC-COOKIE-1",
+					authCookie:   "ff",
+				},
+			},
+		},
+		{
+			name: "clean up old entries",
+			authFile: []byte{
+				// w/unix:0  MIT-MAGIC-COOKIE-1  80507df050756cdefa504b65adb3bcfb
+				// w/unix:0  MIT-MAGIC-COOKIE-1  267b37f6cbc11b97beb826bb1aab8570
+				// w/unix:0  MIT-MAGIC-COOKIE-1  516e22e2b11d1bd0115dff09c028ca5c
+				//
+				// 00000000: 0100 0001 7700 0130 0012 4d49 542d 4d41  ....w..0..MIT-MA
+				// 00000010: 4749 432d 434f 4f4b 4945 2d31 0010 8050  GIC-COOKIE-1...P
+				// 00000020: 7df0 5075 6cde fa50 4b65 adb3 bcfb 0100  }.Pul..PKe......
+				// 00000030: 0001 7700 0130 0012 4d49 542d 4d41 4749  ..w..0..MIT-MAGI
+				// 00000040: 432d 434f 4f4b 4945 2d31 0010 267b 37f6  C-COOKIE-1..&{7.
+				// 00000050: cbc1 1b97 beb8 26bb 1aab 8570 0100 0001  ......&....p....
+				// 00000060: 7700 0130 0012 4d49 542d 4d41 4749 432d  w..0..MIT-MAGIC-
+				// 00000070: 434f 4f4b 4945 2d31 0010 516e 22e2 b11d  COOKIE-1..Qn"...
+				// 00000080: 1bd0 115d ff09 c028 ca5c                 ...]...(.\
+				0x01, 0x00, 0x00, 0x01, 0x77, 0x00, 0x01, 0x30,
+				0x00, 0x12, 0x4d, 0x49, 0x54, 0x2d, 0x4d, 0x41,
+				0x47, 0x49, 0x43, 0x2d, 0x43, 0x4f, 0x4f, 0x4b,
+				0x49, 0x45, 0x2d, 0x31, 0x00, 0x10, 0x80, 0x50,
+				0x7d, 0xf0, 0x50, 0x75, 0x6c, 0xde, 0xfa, 0x50,
+				0x4b, 0x65, 0xad, 0xb3, 0xbc, 0xfb, 0x01, 0x00,
+				0x00, 0x01, 0x77, 0x00, 0x01, 0x30, 0x00, 0x12,
+				0x4d, 0x49, 0x54, 0x2d, 0x4d, 0x41, 0x47, 0x49,
+				0x43, 0x2d, 0x43, 0x4f, 0x4f, 0x4b, 0x49, 0x45,
+				0x2d, 0x31, 0x00, 0x10, 0x26, 0x7b, 0x37, 0xf6,
+				0xcb, 0xc1, 0x1b, 0x97, 0xbe, 0xb8, 0x26, 0xbb,
+				0x1a, 0xab, 0x85, 0x70, 0x01, 0x00, 0x00, 0x01,
+				0x77, 0x00, 0x01, 0x30, 0x00, 0x12, 0x4d, 0x49,
+				0x54, 0x2d, 0x4d, 0x41, 0x47, 0x49, 0x43, 0x2d,
+				0x43, 0x4f, 0x4f, 0x4b, 0x49, 0x45, 0x2d, 0x31,
+				0x00, 0x10, 0x51, 0x6e, 0x22, 0xe2, 0xb1, 0x1d,
+				0x1b, 0xd0, 0x11, 0x5d, 0xff, 0x09, 0xc0, 0x28,
+				0xca, 0x5c,
+			},
+			wantAuthFile: []byte{
+				// w/unix:0  MIT-MAGIC-COOKIE-1  516e5bc892b7162b844abd1fc1a7c16e
+				//
+				// 00000000: 0100 0001 7700 0130 0012 4d49 542d 4d41  ....w..0..MIT-MA
+				// 00000010: 4749 432d 434f 4f4b 4945 2d31 0010 516e  GIC-COOKIE-1..Qn
+				// 00000020: 5bc8 92b7 162b 844a bd1f c1a7 c16e       [....+.J.....n
+				0x01, 0x00, 0x00, 0x01, 0x77, 0x00, 0x01, 0x30,
+				0x00, 0x12, 0x4d, 0x49, 0x54, 0x2d, 0x4d, 0x41,
+				0x47, 0x49, 0x43, 0x2d, 0x43, 0x4f, 0x4f, 0x4b,
+				0x49, 0x45, 0x2d, 0x31, 0x00, 0x10, 0x51, 0x6e,
+				0x5b, 0xc8, 0x92, 0xb7, 0x16, 0x2b, 0x84, 0x4a,
+				0xbd, 0x1f, 0xc1, 0xa7, 0xc1, 0x6e,
+			},
+			entries: []testEntry{
+				{
+					address:      "w",
+					display:      "0",
+					authProtocol: "MIT-MAGIC-COOKIE-1",
+					authCookie:   "516e5bc892b7162b844abd1fc1a7c16e",
+				},
+			},
+		},
+	}
+
+	homedir, err := os.UserHomeDir()
+	require.NoError(t, err)
+
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			fs := afero.NewMemMapFs()
+			if tt.authFile != nil {
+				err := afero.WriteFile(fs, filepath.Join(homedir, ".Xauthority"), tt.authFile, 0o600)
+				require.NoError(t, err)
+			}
+
+			for _, entry := range tt.entries {
+				err := addXauthEntry(context.Background(), fs, entry.address, entry.display, entry.authProtocol, entry.authCookie)
+				require.NoError(t, err)
+			}
+
+			gotAuthFile, err := afero.ReadFile(fs, filepath.Join(homedir, ".Xauthority"))
+			require.NoError(t, err)
+
+			if diff := cmp.Diff(tt.wantAuthFile, gotAuthFile); diff != "" {
+				assert.Failf(t, "addXauthEntry() mismatch", "(-want +got):\n%s", diff)
+			}
+		})
+	}
+}

agent/agentssh/x11_test.go

@@ -14,14 +14,12 @@ import (
 	"github.com/spf13/afero"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-	"go.uber.org/atomic"
 	gossh "golang.org/x/crypto/ssh"
 
 	"cdr.dev/slog"
 	"cdr.dev/slog/sloggers/slogtest"
-	"github.com/coder/coder/agent/agentssh"
-	"github.com/coder/coder/codersdk/agentsdk"
-	"github.com/coder/coder/testutil"
+	"github.com/coder/coder/v2/agent/agentssh"
+	"github.com/coder/coder/v2/testutil"
 )
 
 func TestServer_X11(t *testing.T) {
@@ -34,14 +32,12 @@ func TestServer_X11(t *testing.T) {
 	logger := slogtest.Make(t, nil).Leveled(slog.LevelDebug)
 	fs := afero.NewOsFs()
 	dir := t.TempDir()
-	s, err := agentssh.NewServer(ctx, logger, prometheus.NewRegistry(), fs, 0, dir)
+	s, err := agentssh.NewServer(ctx, logger, prometheus.NewRegistry(), fs, &agentssh.Config{
+		X11SocketDir: dir,
+	})
 	require.NoError(t, err)
 	defer s.Close()
 
-	// The assumption is that these are set before serving SSH connections.
-	s.AgentToken = func() string { return "" }
-	s.Manifest = atomic.NewPointer(&agentsdk.Manifest{})
-
 	ln, err := net.Listen("tcp", "127.0.0.1:0")
 	require.NoError(t, err)
 

agent/agenttest/agent.go

@@ -0,0 +1,57 @@
+package agenttest
+
+import (
+	"context"
+	"net/url"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"cdr.dev/slog"
+	"cdr.dev/slog/sloggers/slogtest"
+	"github.com/coder/coder/v2/agent"
+	"github.com/coder/coder/v2/codersdk/agentsdk"
+)
+
+// New starts a new agent for use in tests.
+// The agent will use the provided coder URL and session token.
+// The options passed to agent.New() can be modified by passing an optional
+// variadic func(*agent.Options).
+// Returns the agent. Closing the agent is handled by the test cleanup.
+// It is the responsibility of the caller to call coderdtest.AwaitWorkspaceAgents
+// to ensure agent is connected.
+func New(t testing.TB, coderURL *url.URL, agentToken string, opts ...func(*agent.Options)) agent.Agent {
+	t.Helper()
+
+	var o agent.Options
+	log := slogtest.Make(t, nil).Leveled(slog.LevelDebug).Named("agent")
+	o.Logger = log
+
+	for _, opt := range opts {
+		opt(&o)
+	}
+
+	if o.Client == nil {
+		agentClient := agentsdk.New(coderURL)
+		agentClient.SetSessionToken(agentToken)
+		agentClient.SDK.SetLogger(log)
+		o.Client = agentClient
+	}
+
+	if o.ExchangeToken == nil {
+		o.ExchangeToken = func(_ context.Context) (string, error) {
+			return agentToken, nil
+		}
+	}
+
+	if o.LogDir == "" {
+		o.LogDir = t.TempDir()
+	}
+
+	agt := agent.New(o)
+	t.Cleanup(func() {
+		assert.NoError(t, agt.Close(), "failed to close agent during cleanup")
+	})
+
+	return agt
+}

agent/agenttest/client.go

@@ -3,162 +3,133 @@ package agenttest
 import (
 	"context"
 	"io"
-	"net"
 	"sync"
+	"sync/atomic"
 	"testing"
 	"time"
 
 	"github.com/google/uuid"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 	"golang.org/x/exp/maps"
+	"golang.org/x/exp/slices"
 	"golang.org/x/xerrors"
+	"google.golang.org/protobuf/types/known/durationpb"
+	"storj.io/drpc"
+	"storj.io/drpc/drpcmux"
+	"storj.io/drpc/drpcserver"
+	"tailscale.com/tailcfg"
 
 	"cdr.dev/slog"
-	"github.com/coder/coder/codersdk"
-	"github.com/coder/coder/codersdk/agentsdk"
-	"github.com/coder/coder/tailnet"
-	"github.com/coder/coder/testutil"
+	agentproto "github.com/coder/coder/v2/agent/proto"
+	"github.com/coder/coder/v2/codersdk"
+	"github.com/coder/coder/v2/codersdk/agentsdk"
+	drpcsdk "github.com/coder/coder/v2/codersdk/drpc"
+	"github.com/coder/coder/v2/tailnet"
+	"github.com/coder/coder/v2/tailnet/proto"
+	"github.com/coder/coder/v2/testutil"
 )
 
+const statsInterval = 500 * time.Millisecond
+
 func NewClient(t testing.TB,
 	logger slog.Logger,
 	agentID uuid.UUID,
 	manifest agentsdk.Manifest,
-	statsChan chan *agentsdk.Stats,
+	statsChan chan *agentproto.Stats,
 	coordinator tailnet.Coordinator,
 ) *Client {
 	if manifest.AgentID == uuid.Nil {
 		manifest.AgentID = agentID
 	}
+	coordPtr := atomic.Pointer[tailnet.Coordinator]{}
+	coordPtr.Store(&coordinator)
+	mux := drpcmux.New()
+	derpMapUpdates := make(chan *tailcfg.DERPMap)
+	drpcService := &tailnet.DRPCService{
+		CoordPtr:               &coordPtr,
+		Logger:                 logger.Named("tailnetsvc"),
+		DerpMapUpdateFrequency: time.Microsecond,
+		DerpMapFn:              func() *tailcfg.DERPMap { return <-derpMapUpdates },
+	}
+	err := proto.DRPCRegisterTailnet(mux, drpcService)
+	require.NoError(t, err)
+	mp, err := agentsdk.ProtoFromManifest(manifest)
+	require.NoError(t, err)
+	fakeAAPI := NewFakeAgentAPI(t, logger, mp, statsChan)
+	err = agentproto.DRPCRegisterAgent(mux, fakeAAPI)
+	require.NoError(t, err)
+	server := drpcserver.NewWithOptions(mux, drpcserver.Options{
+		Log: func(err error) {
+			if xerrors.Is(err, io.EOF) {
+				return
+			}
+			logger.Debug(context.Background(), "drpc server error", slog.Error(err))
+		},
+	})
 	return &Client{
 		t:              t,
 		logger:         logger.Named("client"),
 		agentID:        agentID,
-		manifest:       manifest,
-		statsChan:      statsChan,
 		coordinator:    coordinator,
-		derpMapUpdates: make(chan agentsdk.DERPMapUpdate),
+		server:         server,
+		fakeAgentAPI:   fakeAAPI,
+		derpMapUpdates: derpMapUpdates,
 	}
 }
 
 type Client struct {
-	t                    testing.TB
-	logger               slog.Logger
-	agentID              uuid.UUID
-	manifest             agentsdk.Manifest
-	metadata             map[string]agentsdk.PostMetadataRequest
-	statsChan            chan *agentsdk.Stats
-	coordinator          tailnet.Coordinator
-	LastWorkspaceAgent   func()
-	PatchWorkspaceLogs   func() error
-	GetServiceBannerFunc func() (codersdk.ServiceBannerConfig, error)
+	t                  testing.TB
+	logger             slog.Logger
+	agentID            uuid.UUID
+	coordinator        tailnet.Coordinator
+	server             *drpcserver.Server
+	fakeAgentAPI       *FakeAgentAPI
+	LastWorkspaceAgent func()
 
-	mu              sync.Mutex // Protects following.
-	lifecycleStates []codersdk.WorkspaceAgentLifecycle
-	startup         agentsdk.PostStartupRequest
-	logs            []agentsdk.Log
-	derpMapUpdates  chan agentsdk.DERPMapUpdate
+	mu             sync.Mutex // Protects following.
+	logs           []agentsdk.Log
+	derpMapUpdates chan *tailcfg.DERPMap
+	derpMapOnce    sync.Once
 }
 
-func (c *Client) Manifest(_ context.Context) (agentsdk.Manifest, error) {
-	return c.manifest, nil
+func (*Client) RewriteDERPMap(*tailcfg.DERPMap) {}
+
+func (c *Client) Close() {
+	c.derpMapOnce.Do(func() { close(c.derpMapUpdates) })
 }
 
-func (c *Client) Listen(_ context.Context) (net.Conn, error) {
-	clientConn, serverConn := net.Pipe()
-	closed := make(chan struct{})
+func (c *Client) ConnectRPC(ctx context.Context) (drpc.Conn, error) {
+	conn, lis := drpcsdk.MemTransportPipe()
 	c.LastWorkspaceAgent = func() {
-		_ = serverConn.Close()
-		_ = clientConn.Close()
-		<-closed
+		_ = conn.Close()
+		_ = lis.Close()
 	}
 	c.t.Cleanup(c.LastWorkspaceAgent)
+	serveCtx, cancel := context.WithCancel(ctx)
+	c.t.Cleanup(cancel)
+	streamID := tailnet.StreamID{
+		Name: "agenttest",
+		ID:   c.agentID,
+		Auth: tailnet.AgentCoordinateeAuth{ID: c.agentID},
+	}
+	serveCtx = tailnet.WithStreamID(serveCtx, streamID)
 	go func() {
-		_ = c.coordinator.ServeAgent(serverConn, c.agentID, "")
-		close(closed)
+		_ = c.server.Serve(serveCtx, lis)
 	}()
-	return clientConn, nil
-}
-
-func (c *Client) ReportStats(ctx context.Context, _ slog.Logger, statsChan <-chan *agentsdk.Stats, setInterval func(time.Duration)) (io.Closer, error) {
-	doneCh := make(chan struct{})
-	ctx, cancel := context.WithCancel(ctx)
-
-	go func() {
-		defer close(doneCh)
-
-		setInterval(500 * time.Millisecond)
-		for {
-			select {
-			case <-ctx.Done():
-				return
-			case stat := <-statsChan:
-				select {
-				case c.statsChan <- stat:
-				case <-ctx.Done():
-					return
-				default:
-					// We don't want to send old stats.
-					continue
-				}
-			}
-		}
-	}()
-	return closeFunc(func() error {
-		cancel()
-		<-doneCh
-		close(c.statsChan)
-		return nil
-	}), nil
+	return conn, nil
 }
 
 func (c *Client) GetLifecycleStates() []codersdk.WorkspaceAgentLifecycle {
-	c.mu.Lock()
-	defer c.mu.Unlock()
-	return c.lifecycleStates
+	return c.fakeAgentAPI.GetLifecycleStates()
 }
 
-func (c *Client) PostLifecycle(ctx context.Context, req agentsdk.PostLifecycleRequest) error {
-	c.mu.Lock()
-	defer c.mu.Unlock()
-	c.lifecycleStates = append(c.lifecycleStates, req.State)
-	c.logger.Debug(ctx, "post lifecycle", slog.F("req", req))
-	return nil
+func (c *Client) GetStartup() <-chan *agentproto.Startup {
+	return c.fakeAgentAPI.startupCh
 }
 
-func (c *Client) PostAppHealth(ctx context.Context, req agentsdk.PostAppHealthsRequest) error {
-	c.logger.Debug(ctx, "post app health", slog.F("req", req))
-	return nil
-}
-
-func (c *Client) GetStartup() agentsdk.PostStartupRequest {
-	c.mu.Lock()
-	defer c.mu.Unlock()
-	return c.startup
-}
-
-func (c *Client) GetMetadata() map[string]agentsdk.PostMetadataRequest {
-	c.mu.Lock()
-	defer c.mu.Unlock()
-	return maps.Clone(c.metadata)
-}
-
-func (c *Client) PostMetadata(ctx context.Context, key string, req agentsdk.PostMetadataRequest) error {
-	c.mu.Lock()
-	defer c.mu.Unlock()
-	if c.metadata == nil {
-		c.metadata = make(map[string]agentsdk.PostMetadataRequest)
-	}
-	c.metadata[key] = req
-	c.logger.Debug(ctx, "post metadata", slog.F("key", key), slog.F("req", req))
-	return nil
-}
-
-func (c *Client) PostStartup(ctx context.Context, startup agentsdk.PostStartupRequest) error {
-	c.mu.Lock()
-	defer c.mu.Unlock()
-	c.startup = startup
-	c.logger.Debug(ctx, "post startup", slog.F("req", startup))
-	return nil
+func (c *Client) GetMetadata() map[string]agentsdk.Metadata {
+	return c.fakeAgentAPI.GetMetadata()
 }
 
 func (c *Client) GetStartupLogs() []agentsdk.Log {
@@ -167,35 +138,11 @@ func (c *Client) GetStartupLogs() []agentsdk.Log {
 	return c.logs
 }
 
-func (c *Client) PatchLogs(ctx context.Context, logs agentsdk.PatchLogs) error {
-	c.mu.Lock()
-	defer c.mu.Unlock()
-	if c.PatchWorkspaceLogs != nil {
-		return c.PatchWorkspaceLogs()
-	}
-	c.logs = append(c.logs, logs.Logs...)
-	c.logger.Debug(ctx, "patch startup logs", slog.F("req", logs))
-	return nil
-}
-
 func (c *Client) SetServiceBannerFunc(f func() (codersdk.ServiceBannerConfig, error)) {
-	c.mu.Lock()
-	defer c.mu.Unlock()
-
-	c.GetServiceBannerFunc = f
+	c.fakeAgentAPI.SetServiceBannerFunc(f)
 }
 
-func (c *Client) GetServiceBanner(ctx context.Context) (codersdk.ServiceBannerConfig, error) {
-	c.mu.Lock()
-	defer c.mu.Unlock()
-	c.logger.Debug(ctx, "get service banner")
-	if c.GetServiceBannerFunc != nil {
-		return c.GetServiceBannerFunc()
-	}
-	return codersdk.ServiceBannerConfig{}, nil
-}
-
-func (c *Client) PushDERPMapUpdate(update agentsdk.DERPMapUpdate) error {
+func (c *Client) PushDERPMapUpdate(update *tailcfg.DERPMap) error {
 	timer := time.NewTimer(testutil.WaitShort)
 	defer timer.Stop()
 	select {
@@ -207,16 +154,139 @@ func (c *Client) PushDERPMapUpdate(update agentsdk.DERPMapUpdate) error {
 	return nil
 }
 
-func (c *Client) DERPMapUpdates(_ context.Context) (<-chan agentsdk.DERPMapUpdate, io.Closer, error) {
-	closed := make(chan struct{})
-	return c.derpMapUpdates, closeFunc(func() error {
-		close(closed)
-		return nil
-	}), nil
+func (c *Client) SetLogsChannel(ch chan<- *agentproto.BatchCreateLogsRequest) {
+	c.fakeAgentAPI.SetLogsChannel(ch)
 }
 
-type closeFunc func() error
+type FakeAgentAPI struct {
+	sync.Mutex
+	t      testing.TB
+	logger slog.Logger
 
-func (c closeFunc) Close() error {
-	return c()
+	manifest        *agentproto.Manifest
+	startupCh       chan *agentproto.Startup
+	statsCh         chan *agentproto.Stats
+	appHealthCh     chan *agentproto.BatchUpdateAppHealthRequest
+	logsCh          chan<- *agentproto.BatchCreateLogsRequest
+	lifecycleStates []codersdk.WorkspaceAgentLifecycle
+	metadata        map[string]agentsdk.Metadata
+
+	getServiceBannerFunc func() (codersdk.ServiceBannerConfig, error)
+}
+
+func (f *FakeAgentAPI) GetManifest(context.Context, *agentproto.GetManifestRequest) (*agentproto.Manifest, error) {
+	return f.manifest, nil
+}
+
+func (f *FakeAgentAPI) SetServiceBannerFunc(fn func() (codersdk.ServiceBannerConfig, error)) {
+	f.Lock()
+	defer f.Unlock()
+	f.getServiceBannerFunc = fn
+	f.logger.Info(context.Background(), "updated ServiceBannerFunc")
+}
+
+func (f *FakeAgentAPI) GetServiceBanner(context.Context, *agentproto.GetServiceBannerRequest) (*agentproto.ServiceBanner, error) {
+	f.Lock()
+	defer f.Unlock()
+	if f.getServiceBannerFunc == nil {
+		return &agentproto.ServiceBanner{}, nil
+	}
+	sb, err := f.getServiceBannerFunc()
+	if err != nil {
+		return nil, err
+	}
+	return agentsdk.ProtoFromServiceBanner(sb), nil
+}
+
+func (f *FakeAgentAPI) UpdateStats(ctx context.Context, req *agentproto.UpdateStatsRequest) (*agentproto.UpdateStatsResponse, error) {
+	f.logger.Debug(ctx, "update stats called", slog.F("req", req))
+	// empty request is sent to get the interval; but our tests don't want empty stats requests
+	if req.Stats != nil {
+		f.statsCh <- req.Stats
+	}
+	return &agentproto.UpdateStatsResponse{ReportInterval: durationpb.New(statsInterval)}, nil
+}
+
+func (f *FakeAgentAPI) GetLifecycleStates() []codersdk.WorkspaceAgentLifecycle {
+	f.Lock()
+	defer f.Unlock()
+	return slices.Clone(f.lifecycleStates)
+}
+
+func (f *FakeAgentAPI) UpdateLifecycle(_ context.Context, req *agentproto.UpdateLifecycleRequest) (*agentproto.Lifecycle, error) {
+	f.Lock()
+	defer f.Unlock()
+	s, err := agentsdk.LifecycleStateFromProto(req.GetLifecycle().GetState())
+	if assert.NoError(f.t, err) {
+		f.lifecycleStates = append(f.lifecycleStates, s)
+	}
+	return req.GetLifecycle(), nil
+}
+
+func (f *FakeAgentAPI) BatchUpdateAppHealths(ctx context.Context, req *agentproto.BatchUpdateAppHealthRequest) (*agentproto.BatchUpdateAppHealthResponse, error) {
+	f.logger.Debug(ctx, "batch update app health", slog.F("req", req))
+	f.appHealthCh <- req
+	return &agentproto.BatchUpdateAppHealthResponse{}, nil
+}
+
+func (f *FakeAgentAPI) AppHealthCh() <-chan *agentproto.BatchUpdateAppHealthRequest {
+	return f.appHealthCh
+}
+
+func (f *FakeAgentAPI) UpdateStartup(_ context.Context, req *agentproto.UpdateStartupRequest) (*agentproto.Startup, error) {
+	f.startupCh <- req.GetStartup()
+	return req.GetStartup(), nil
+}
+
+func (f *FakeAgentAPI) GetMetadata() map[string]agentsdk.Metadata {
+	f.Lock()
+	defer f.Unlock()
+	return maps.Clone(f.metadata)
+}
+
+func (f *FakeAgentAPI) BatchUpdateMetadata(ctx context.Context, req *agentproto.BatchUpdateMetadataRequest) (*agentproto.BatchUpdateMetadataResponse, error) {
+	f.Lock()
+	defer f.Unlock()
+	if f.metadata == nil {
+		f.metadata = make(map[string]agentsdk.Metadata)
+	}
+	for _, md := range req.Metadata {
+		smd := agentsdk.MetadataFromProto(md)
+		f.metadata[md.Key] = smd
+		f.logger.Debug(ctx, "post metadata", slog.F("key", md.Key), slog.F("md", md))
+	}
+	return &agentproto.BatchUpdateMetadataResponse{}, nil
+}
+
+func (f *FakeAgentAPI) SetLogsChannel(ch chan<- *agentproto.BatchCreateLogsRequest) {
+	f.Lock()
+	defer f.Unlock()
+	f.logsCh = ch
+}
+
+func (f *FakeAgentAPI) BatchCreateLogs(ctx context.Context, req *agentproto.BatchCreateLogsRequest) (*agentproto.BatchCreateLogsResponse, error) {
+	f.logger.Info(ctx, "batch create logs called", slog.F("req", req))
+	f.Lock()
+	ch := f.logsCh
+	f.Unlock()
+	if ch != nil {
+		select {
+		case <-ctx.Done():
+			return nil, ctx.Err()
+		case ch <- req:
+			// ok
+		}
+	}
+	return &agentproto.BatchCreateLogsResponse{}, nil
+}
+
+func NewFakeAgentAPI(t testing.TB, logger slog.Logger, manifest *agentproto.Manifest, statsCh chan *agentproto.Stats) *FakeAgentAPI {
+	return &FakeAgentAPI{
+		t:           t,
+		logger:      logger.Named("FakeAgentAPI"),
+		manifest:    manifest,
+		statsCh:     statsCh,
+		startupCh:   make(chan *agentproto.Startup, 100),
+		appHealthCh: make(chan *agentproto.BatchUpdateAppHealthRequest, 100),
+	}
 }

agent/api.go

@@ -5,10 +5,10 @@ import (
 	"sync"
 	"time"
 
-	"github.com/go-chi/chi"
+	"github.com/go-chi/chi/v5"
 
-	"github.com/coder/coder/coderd/httpapi"
-	"github.com/coder/coder/codersdk"
+	"github.com/coder/coder/v2/coderd/httpapi"
+	"github.com/coder/coder/v2/codersdk"
 )
 
 func (a *agent) apiHandler() http.Handler {
@@ -26,17 +26,30 @@ func (a *agent) apiHandler() http.Handler {
 		cpy[k] = b
 	}
 
-	lp := &listeningPortsHandler{ignorePorts: cpy}
+	cacheDuration := 1 * time.Second
+	if a.portCacheDuration > 0 {
+		cacheDuration = a.portCacheDuration
+	}
+
+	lp := &listeningPortsHandler{
+		ignorePorts:   cpy,
+		cacheDuration: cacheDuration,
+	}
 	r.Get("/api/v0/listening-ports", lp.handler)
 
 	return r
 }
 
 type listeningPortsHandler struct {
-	mut         sync.Mutex
-	ports       []codersdk.WorkspaceAgentListeningPort
-	mtime       time.Time
-	ignorePorts map[int]string
+	ignorePorts   map[int]string
+	cacheDuration time.Duration
+
+	//nolint: unused  // used on some but not all platforms
+	mut sync.Mutex
+	//nolint: unused  // used on some but not all platforms
+	ports []codersdk.WorkspaceAgentListeningPort
+	//nolint: unused  // used on some but not all platforms
+	mtime time.Time
 }
 
 // handler returns a list of listening ports. This is tested by coderd's

agent/apphealth.go

@@ -10,8 +10,8 @@ import (
 	"golang.org/x/xerrors"
 
 	"cdr.dev/slog"
-	"github.com/coder/coder/codersdk"
-	"github.com/coder/coder/codersdk/agentsdk"
+	"github.com/coder/coder/v2/codersdk"
+	"github.com/coder/coder/v2/codersdk/agentsdk"
 	"github.com/coder/retry"
 )
 
@@ -26,7 +26,12 @@ type WorkspaceAppHealthReporter func(ctx context.Context)
 
 // NewWorkspaceAppHealthReporter creates a WorkspaceAppHealthReporter that reports app health to coderd.
 func NewWorkspaceAppHealthReporter(logger slog.Logger, apps []codersdk.WorkspaceApp, postWorkspaceAgentAppHealth PostWorkspaceAgentAppHealth) WorkspaceAppHealthReporter {
+	logger = logger.Named("apphealth")
+
 	runHealthcheckLoop := func(ctx context.Context) error {
+		ctx, cancel := context.WithCancel(ctx)
+		defer cancel()
+
 		// no need to run this loop if no apps for this workspace.
 		if len(apps) == 0 {
 			return nil
@@ -87,6 +92,7 @@ func NewWorkspaceAppHealthReporter(logger slog.Logger, apps []codersdk.Workspace
 						return nil
 					}()
 					if err != nil {
+						nowUnhealthy := false
 						mu.Lock()
 						if failures[app.ID] < int(app.Healthcheck.Threshold) {
 							// increment the failure count and keep status the same.
@@ -96,14 +102,21 @@ func NewWorkspaceAppHealthReporter(logger slog.Logger, apps []codersdk.Workspace
 							// set to unhealthy if we hit the failure threshold.
 							// we stop incrementing at the threshold to prevent the failure value from increasing forever.
 							health[app.ID] = codersdk.WorkspaceAppHealthUnhealthy
+							nowUnhealthy = true
 						}
 						mu.Unlock()
+						logger.Debug(ctx, "error checking app health",
+							slog.F("id", app.ID.String()),
+							slog.F("slug", app.Slug),
+							slog.F("now_unhealthy", nowUnhealthy), slog.Error(err),
+						)
 					} else {
 						mu.Lock()
 						// we only need one successful health check to be considered healthy.
 						health[app.ID] = codersdk.WorkspaceAppHealthHealthy
 						failures[app.ID] = 0
 						mu.Unlock()
+						logger.Debug(ctx, "workspace app healthy", slog.F("id", app.ID.String()), slog.F("slug", app.Slug))
 					}
 
 					t.Reset(time.Duration(app.Healthcheck.Interval) * time.Second)
@@ -137,7 +150,9 @@ func NewWorkspaceAppHealthReporter(logger slog.Logger, apps []codersdk.Workspace
 					Healths: lastHealth,
 				})
 				if err != nil {
-					logger.Error(ctx, "failed to report workspace app stat", slog.Error(err))
+					logger.Error(ctx, "failed to report workspace app health", slog.Error(err))
+				} else {
+					logger.Debug(ctx, "sent workspace app health", slog.F("health", lastHealth))
 				}
 			}
 		}

agent/apphealth_test.go

@@ -4,20 +4,25 @@ import (
 	"context"
 	"net/http"
 	"net/http/httptest"
+	"strings"
 	"sync"
 	"sync/atomic"
 	"testing"
 	"time"
 
+	"github.com/google/uuid"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
 	"cdr.dev/slog"
 	"cdr.dev/slog/sloggers/slogtest"
-	"github.com/coder/coder/agent"
-	"github.com/coder/coder/coderd/httpapi"
-	"github.com/coder/coder/codersdk"
-	"github.com/coder/coder/codersdk/agentsdk"
-	"github.com/coder/coder/testutil"
+	"github.com/coder/coder/v2/agent"
+	"github.com/coder/coder/v2/agent/agenttest"
+	"github.com/coder/coder/v2/agent/proto"
+	"github.com/coder/coder/v2/coderd/httpapi"
+	"github.com/coder/coder/v2/codersdk"
+	"github.com/coder/coder/v2/codersdk/agentsdk"
+	"github.com/coder/coder/v2/testutil"
 )
 
 func TestAppHealth_Healthy(t *testing.T) {
@@ -40,12 +45,23 @@ func TestAppHealth_Healthy(t *testing.T) {
 			},
 			Health: codersdk.WorkspaceAppHealthInitializing,
 		},
+		{
+			Slug: "app3",
+			Healthcheck: codersdk.Healthcheck{
+				Interval:  2,
+				Threshold: 1,
+			},
+			Health: codersdk.WorkspaceAppHealthInitializing,
+		},
 	}
 	handlers := []http.Handler{
 		nil,
 		http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 			httpapi.Write(r.Context(), w, http.StatusOK, nil)
 		}),
+		http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			httpapi.Write(r.Context(), w, http.StatusOK, nil)
+		}),
 	}
 	getApps, closeFn := setupAppReporter(ctx, t, apps, handlers)
 	defer closeFn()
@@ -58,7 +74,7 @@ func TestAppHealth_Healthy(t *testing.T) {
 			return false
 		}
 
-		return apps[1].Health == codersdk.WorkspaceAppHealthHealthy
+		return apps[1].Health == codersdk.WorkspaceAppHealthHealthy && apps[2].Health == codersdk.WorkspaceAppHealthHealthy
 	}, testutil.WaitLong, testutil.IntervalSlow)
 }
 
@@ -163,6 +179,12 @@ func TestAppHealth_NotSpamming(t *testing.T) {
 
 func setupAppReporter(ctx context.Context, t *testing.T, apps []codersdk.WorkspaceApp, handlers []http.Handler) (agent.WorkspaceAgentApps, func()) {
 	closers := []func(){}
+	for i, app := range apps {
+		if app.ID == uuid.Nil {
+			app.ID = uuid.New()
+			apps[i] = app
+		}
+	}
 	for i, handler := range handlers {
 		if handler == nil {
 			continue
@@ -181,23 +203,43 @@ func setupAppReporter(ctx context.Context, t *testing.T, apps []codersdk.Workspa
 		var newApps []codersdk.WorkspaceApp
 		return append(newApps, apps...), nil
 	}
-	postWorkspaceAgentAppHealth := func(_ context.Context, req agentsdk.PostAppHealthsRequest) error {
-		mu.Lock()
-		for id, health := range req.Healths {
-			for i, app := range apps {
-				if app.ID != id {
-					continue
+
+	// We don't care about manifest or stats in this test since it's not using
+	// a full agent and these RPCs won't get called.
+	//
+	// We use a proper fake agent API so we can test the conversion code and the
+	// request code as well. Before we were bypassing these by using a custom
+	// post function.
+	fakeAAPI := agenttest.NewFakeAgentAPI(t, slogtest.Make(t, nil), nil, nil)
+
+	// Process events from the channel and update the health of the apps.
+	go func() {
+		appHealthCh := fakeAAPI.AppHealthCh()
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case req := <-appHealthCh:
+				mu.Lock()
+				for _, update := range req.Updates {
+					updateID, err := uuid.FromBytes(update.Id)
+					assert.NoError(t, err)
+					updateHealth := codersdk.WorkspaceAppHealth(strings.ToLower(proto.AppHealth_name[int32(update.Health)]))
+
+					for i, app := range apps {
+						if app.ID != updateID {
+							continue
+						}
+						app.Health = updateHealth
+						apps[i] = app
+					}
 				}
-				app.Health = health
-				apps[i] = app
+				mu.Unlock()
 			}
 		}
-		mu.Unlock()
+	}()
 
-		return nil
-	}
-
-	go agent.NewWorkspaceAppHealthReporter(slogtest.Make(t, nil).Leveled(slog.LevelDebug), apps, postWorkspaceAgentAppHealth)(ctx)
+	go agent.NewWorkspaceAppHealthReporter(slogtest.Make(t, nil).Leveled(slog.LevelDebug), apps, agentsdk.AppHealthPoster(fakeAAPI))(ctx)
 
 	return workspaceAgentApps, func() {
 		for _, closeFn := range closers {

agent/metrics.go

@@ -10,13 +10,15 @@ import (
 	"tailscale.com/util/clientmetric"
 
 	"cdr.dev/slog"
-
-	"github.com/coder/coder/codersdk/agentsdk"
+	"github.com/coder/coder/v2/agent/proto"
 )
 
 type agentMetrics struct {
 	connectionsTotal      prometheus.Counter
 	reconnectingPTYErrors *prometheus.CounterVec
+	// startupScriptSeconds is the time in seconds that the start script(s)
+	// took to run. This is reported once per agent.
+	startupScriptSeconds *prometheus.GaugeVec
 }
 
 func newAgentMetrics(registerer prometheus.Registerer) *agentMetrics {
@@ -35,14 +37,23 @@ func newAgentMetrics(registerer prometheus.Registerer) *agentMetrics {
 	)
 	registerer.MustRegister(reconnectingPTYErrors)
 
+	startupScriptSeconds := prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: "coderd",
+		Subsystem: "agentstats",
+		Name:      "startup_script_seconds",
+		Help:      "Amount of time taken to run the startup script in seconds.",
+	}, []string{"success"})
+	registerer.MustRegister(startupScriptSeconds)
+
 	return &agentMetrics{
 		connectionsTotal:      connectionsTotal,
 		reconnectingPTYErrors: reconnectingPTYErrors,
+		startupScriptSeconds:  startupScriptSeconds,
 	}
 }
 
-func (a *agent) collectMetrics(ctx context.Context) []agentsdk.AgentMetric {
-	var collected []agentsdk.AgentMetric
+func (a *agent) collectMetrics(ctx context.Context) []*proto.Stats_Metric {
+	var collected []*proto.Stats_Metric
 
 	// Tailscale internal metrics
 	metrics := clientmetric.Metrics()
@@ -51,7 +62,7 @@ func (a *agent) collectMetrics(ctx context.Context) []agentsdk.AgentMetric {
 			continue
 		}
 
-		collected = append(collected, agentsdk.AgentMetric{
+		collected = append(collected, &proto.Stats_Metric{
 			Name:  m.Name(),
 			Type:  asMetricType(m.Type()),
 			Value: float64(m.Value()),
@@ -69,16 +80,16 @@ func (a *agent) collectMetrics(ctx context.Context) []agentsdk.AgentMetric {
 			labels := toAgentMetricLabels(metric.Label)
 
 			if metric.Counter != nil {
-				collected = append(collected, agentsdk.AgentMetric{
+				collected = append(collected, &proto.Stats_Metric{
 					Name:   metricFamily.GetName(),
-					Type:   agentsdk.AgentMetricTypeCounter,
+					Type:   proto.Stats_Metric_COUNTER,
 					Value:  metric.Counter.GetValue(),
 					Labels: labels,
 				})
 			} else if metric.Gauge != nil {
-				collected = append(collected, agentsdk.AgentMetric{
+				collected = append(collected, &proto.Stats_Metric{
 					Name:   metricFamily.GetName(),
-					Type:   agentsdk.AgentMetricTypeGauge,
+					Type:   proto.Stats_Metric_GAUGE,
 					Value:  metric.Gauge.GetValue(),
 					Labels: labels,
 				})
@@ -90,14 +101,14 @@ func (a *agent) collectMetrics(ctx context.Context) []agentsdk.AgentMetric {
 	return collected
 }
 
-func toAgentMetricLabels(metricLabels []*prompb.LabelPair) []agentsdk.AgentMetricLabel {
+func toAgentMetricLabels(metricLabels []*prompb.LabelPair) []*proto.Stats_Metric_Label {
 	if len(metricLabels) == 0 {
 		return nil
 	}
 
-	labels := make([]agentsdk.AgentMetricLabel, 0, len(metricLabels))
+	labels := make([]*proto.Stats_Metric_Label, 0, len(metricLabels))
 	for _, metricLabel := range metricLabels {
-		labels = append(labels, agentsdk.AgentMetricLabel{
+		labels = append(labels, &proto.Stats_Metric_Label{
 			Name:  metricLabel.GetName(),
 			Value: metricLabel.GetValue(),
 		})
@@ -118,12 +129,12 @@ func isIgnoredMetric(metricName string) bool {
 	return false
 }
 
-func asMetricType(typ clientmetric.Type) agentsdk.AgentMetricType {
+func asMetricType(typ clientmetric.Type) proto.Stats_Metric_Type {
 	switch typ {
 	case clientmetric.TypeGauge:
-		return agentsdk.AgentMetricTypeGauge
+		return proto.Stats_Metric_GAUGE
 	case clientmetric.TypeCounter:
-		return agentsdk.AgentMetricTypeCounter
+		return proto.Stats_Metric_COUNTER
 	default:
 		panic(fmt.Sprintf("unknown metric type: %d", typ))
 	}

agent/ports_supported.go

@@ -8,14 +8,14 @@ import (
 	"github.com/cakturk/go-netstat/netstat"
 	"golang.org/x/xerrors"
 
-	"github.com/coder/coder/codersdk"
+	"github.com/coder/coder/v2/codersdk"
 )
 
 func (lp *listeningPortsHandler) getListeningPorts() ([]codersdk.WorkspaceAgentListeningPort, error) {
 	lp.mut.Lock()
 	defer lp.mut.Unlock()
 
-	if time.Since(lp.mtime) < time.Second {
+	if time.Since(lp.mtime) < lp.cacheDuration {
 		// copy
 		ports := make([]codersdk.WorkspaceAgentListeningPort, len(lp.ports))
 		copy(ports, lp.ports)

agent/ports_unsupported.go

@@ -2,9 +2,9 @@
 
 package agent
 
-import "github.com/coder/coder/codersdk"
+import "github.com/coder/coder/v2/codersdk"
 
-func (lp *listeningPortsHandler) getListeningPorts() ([]codersdk.WorkspaceAgentListeningPort, error) {
+func (*listeningPortsHandler) getListeningPorts() ([]codersdk.WorkspaceAgentListeningPort, error) {
 	// Can't scan for ports on non-linux or non-windows_amd64 systems at the
 	// moment. The UI will not show any "no ports found" message to the user, so
 	// the user won't suspect a thing.

agent/proto/agent.proto

@@ -0,0 +1,263 @@
+syntax = "proto3";
+option go_package = "github.com/coder/coder/v2/agent/proto";
+
+package coder.agent.v2;
+
+import "tailnet/proto/tailnet.proto";
+import "google/protobuf/timestamp.proto";
+import "google/protobuf/duration.proto";
+
+message WorkspaceApp {
+	bytes id = 1;
+	string url = 2;
+	bool external = 3;
+	string slug = 4;
+	string display_name = 5;
+	string command = 6;
+	string icon = 7;
+	bool subdomain = 8;
+	string subdomain_name = 9;
+
+	enum SharingLevel {
+		SHARING_LEVEL_UNSPECIFIED = 0;
+		OWNER = 1;
+		AUTHENTICATED = 2;
+		PUBLIC = 3;
+	}
+	SharingLevel sharing_level = 10;
+
+	message Healthcheck {
+		string url = 1;
+		google.protobuf.Duration interval = 2;
+		int32 threshold = 3;
+	}
+	Healthcheck healthcheck = 11;
+
+	enum Health {
+		HEALTH_UNSPECIFIED = 0;
+		DISABLED = 1;
+		INITIALIZING = 2;
+		HEALTHY = 3;
+		UNHEALTHY = 4;
+	}
+	Health health = 12;
+}
+
+message WorkspaceAgentScript {
+	bytes log_source_id = 1;
+	string log_path = 2;
+	string script = 3;
+	string cron = 4;
+	bool run_on_start = 5;
+	bool run_on_stop = 6;
+	bool start_blocks_login = 7;
+	google.protobuf.Duration timeout = 8;
+}
+
+message WorkspaceAgentMetadata {
+	message Result {
+		google.protobuf.Timestamp collected_at = 1;
+		int64 age = 2;
+		string value = 3;
+		string error = 4;
+	}
+	Result result = 1;
+
+	message Description {
+		string display_name = 1;
+		string key = 2;
+		string script = 3;
+		google.protobuf.Duration interval = 4;
+		google.protobuf.Duration timeout = 5;
+	}
+	Description description = 2;
+}
+
+message Manifest {
+	bytes agent_id = 1;
+	string agent_name = 15;
+	string owner_username = 13;
+	bytes workspace_id = 14;
+	string workspace_name = 16;
+	uint32 git_auth_configs = 2;
+	map<string, string> environment_variables = 3;
+	string directory = 4;
+	string vs_code_port_proxy_uri = 5;
+	string motd_path = 6;
+	bool disable_direct_connections = 7;
+	bool derp_force_websockets = 8;
+
+	coder.tailnet.v2.DERPMap derp_map = 9;
+	repeated WorkspaceAgentScript scripts = 10;
+	repeated WorkspaceApp apps = 11;
+	repeated WorkspaceAgentMetadata.Description metadata = 12;
+}
+
+message GetManifestRequest {}
+
+message ServiceBanner {
+	bool enabled = 1;
+	string message = 2;
+	string background_color = 3;
+}
+
+message GetServiceBannerRequest {}
+
+message Stats {
+	// ConnectionsByProto is a count of connections by protocol.
+	map<string, int64> connections_by_proto = 1;
+	// ConnectionCount is the number of connections received by an agent.
+	int64 connection_count = 2;
+	// ConnectionMedianLatencyMS is the median latency of all connections in milliseconds.
+	double connection_median_latency_ms = 3;
+	// RxPackets is the number of received packets.
+	int64 rx_packets = 4;
+	// RxBytes is the number of received bytes.
+	int64 rx_bytes = 5;
+	// TxPackets is the number of transmitted bytes.
+	int64 tx_packets = 6;
+	// TxBytes is the number of transmitted bytes.
+	int64 tx_bytes = 7;
+
+	// SessionCountVSCode is the number of connections received by an agent
+	// that are from our VS Code extension.
+	int64 session_count_vscode = 8;
+	// SessionCountJetBrains is the number of connections received by an agent
+	// that are from our JetBrains extension.
+	int64 session_count_jetbrains = 9;
+	// SessionCountReconnectingPTY is the number of connections received by an agent
+	// that are from the reconnecting web terminal.
+	int64 session_count_reconnecting_pty = 10;
+	// SessionCountSSH is the number of connections received by an agent
+	// that are normal, non-tagged SSH sessions.
+	int64 session_count_ssh = 11;
+
+	message Metric {
+		string name = 1;
+
+		enum Type {
+			TYPE_UNSPECIFIED = 0;
+			COUNTER = 1;
+			GAUGE = 2;
+		}
+		Type type = 2;
+
+		double value = 3;
+
+		message Label {
+			string name = 1;
+			string value = 2;
+		}
+		repeated Label labels = 4;
+	}
+	repeated Metric metrics = 12;
+}
+
+message UpdateStatsRequest{
+	Stats stats = 1;
+}
+
+message UpdateStatsResponse {
+	google.protobuf.Duration report_interval = 1;
+}
+
+message Lifecycle {
+	enum State {
+		STATE_UNSPECIFIED = 0;
+		CREATED = 1;
+		STARTING = 2;
+		START_TIMEOUT = 3;
+		START_ERROR = 4;
+		READY = 5;
+		SHUTTING_DOWN = 6;
+		SHUTDOWN_TIMEOUT = 7;
+		SHUTDOWN_ERROR = 8;
+		OFF = 9;
+	}
+	State state = 1;
+	google.protobuf.Timestamp changed_at = 2;
+}
+
+message UpdateLifecycleRequest {
+	Lifecycle lifecycle = 1;
+}
+
+enum AppHealth {
+	APP_HEALTH_UNSPECIFIED = 0;
+	DISABLED = 1;
+	INITIALIZING = 2;
+	HEALTHY = 3;
+	UNHEALTHY = 4;
+}
+
+message BatchUpdateAppHealthRequest {
+	message HealthUpdate {
+		bytes id = 1;
+		AppHealth health = 2;
+	}
+	repeated HealthUpdate updates = 1;
+}
+
+message BatchUpdateAppHealthResponse {}
+
+message Startup {
+	string version = 1;
+	string expanded_directory = 2;
+	enum Subsystem {
+		SUBSYSTEM_UNSPECIFIED = 0;
+		ENVBOX = 1;
+		ENVBUILDER = 2;
+		EXECTRACE = 3;
+	}
+	repeated Subsystem subsystems = 3;
+}
+
+message UpdateStartupRequest{
+	Startup startup = 1;
+}
+
+message Metadata {
+	string key = 1;
+	WorkspaceAgentMetadata.Result result = 2;
+}
+
+message BatchUpdateMetadataRequest {
+	repeated Metadata metadata = 2;
+}
+
+message BatchUpdateMetadataResponse {}
+
+message Log {
+	google.protobuf.Timestamp created_at = 1;
+	string output = 2;
+
+	enum Level {
+		LEVEL_UNSPECIFIED = 0;
+		TRACE = 1;
+		DEBUG = 2;
+		INFO = 3;
+		WARN = 4;
+		ERROR = 5;
+	}
+	Level level = 3;
+}
+
+message BatchCreateLogsRequest {
+	bytes log_source_id = 1;
+	repeated Log logs = 2;
+}
+
+message BatchCreateLogsResponse {
+	bool log_limit_exceeded = 1;
+}
+
+service Agent {
+	rpc GetManifest(GetManifestRequest) returns (Manifest);
+	rpc GetServiceBanner(GetServiceBannerRequest) returns (ServiceBanner);
+	rpc UpdateStats(UpdateStatsRequest) returns (UpdateStatsResponse);
+	rpc UpdateLifecycle(UpdateLifecycleRequest) returns (Lifecycle);
+	rpc BatchUpdateAppHealths(BatchUpdateAppHealthRequest) returns (BatchUpdateAppHealthResponse);
+	rpc UpdateStartup(UpdateStartupRequest) returns (Startup);
+	rpc BatchUpdateMetadata(BatchUpdateMetadataRequest) returns (BatchUpdateMetadataResponse);
+	rpc BatchCreateLogs(BatchCreateLogsRequest) returns (BatchCreateLogsResponse);
+}

agent/proto/agent_drpc.pb.go

@@ -0,0 +1,391 @@
+// Code generated by protoc-gen-go-drpc. DO NOT EDIT.
+// protoc-gen-go-drpc version: v0.0.33
+// source: agent/proto/agent.proto
+
+package proto
+
+import (
+	context "context"
+	errors "errors"
+	protojson "google.golang.org/protobuf/encoding/protojson"
+	proto "google.golang.org/protobuf/proto"
+	drpc "storj.io/drpc"
+	drpcerr "storj.io/drpc/drpcerr"
+)
+
+type drpcEncoding_File_agent_proto_agent_proto struct{}
+
+func (drpcEncoding_File_agent_proto_agent_proto) Marshal(msg drpc.Message) ([]byte, error) {
+	return proto.Marshal(msg.(proto.Message))
+}
+
+func (drpcEncoding_File_agent_proto_agent_proto) MarshalAppend(buf []byte, msg drpc.Message) ([]byte, error) {
+	return proto.MarshalOptions{}.MarshalAppend(buf, msg.(proto.Message))
+}
+
+func (drpcEncoding_File_agent_proto_agent_proto) Unmarshal(buf []byte, msg drpc.Message) error {
+	return proto.Unmarshal(buf, msg.(proto.Message))
+}
+
+func (drpcEncoding_File_agent_proto_agent_proto) JSONMarshal(msg drpc.Message) ([]byte, error) {
+	return protojson.Marshal(msg.(proto.Message))
+}
+
+func (drpcEncoding_File_agent_proto_agent_proto) JSONUnmarshal(buf []byte, msg drpc.Message) error {
+	return protojson.Unmarshal(buf, msg.(proto.Message))
+}
+
+type DRPCAgentClient interface {
+	DRPCConn() drpc.Conn
+
+	GetManifest(ctx context.Context, in *GetManifestRequest) (*Manifest, error)
+	GetServiceBanner(ctx context.Context, in *GetServiceBannerRequest) (*ServiceBanner, error)
+	UpdateStats(ctx context.Context, in *UpdateStatsRequest) (*UpdateStatsResponse, error)
+	UpdateLifecycle(ctx context.Context, in *UpdateLifecycleRequest) (*Lifecycle, error)
+	BatchUpdateAppHealths(ctx context.Context, in *BatchUpdateAppHealthRequest) (*BatchUpdateAppHealthResponse, error)
+	UpdateStartup(ctx context.Context, in *UpdateStartupRequest) (*Startup, error)
+	BatchUpdateMetadata(ctx context.Context, in *BatchUpdateMetadataRequest) (*BatchUpdateMetadataResponse, error)
+	BatchCreateLogs(ctx context.Context, in *BatchCreateLogsRequest) (*BatchCreateLogsResponse, error)
+}
+
+type drpcAgentClient struct {
+	cc drpc.Conn
+}
+
+func NewDRPCAgentClient(cc drpc.Conn) DRPCAgentClient {
+	return &drpcAgentClient{cc}
+}
+
+func (c *drpcAgentClient) DRPCConn() drpc.Conn { return c.cc }
+
+func (c *drpcAgentClient) GetManifest(ctx context.Context, in *GetManifestRequest) (*Manifest, error) {
+	out := new(Manifest)
+	err := c.cc.Invoke(ctx, "/coder.agent.v2.Agent/GetManifest", drpcEncoding_File_agent_proto_agent_proto{}, in, out)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *drpcAgentClient) GetServiceBanner(ctx context.Context, in *GetServiceBannerRequest) (*ServiceBanner, error) {
+	out := new(ServiceBanner)
+	err := c.cc.Invoke(ctx, "/coder.agent.v2.Agent/GetServiceBanner", drpcEncoding_File_agent_proto_agent_proto{}, in, out)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *drpcAgentClient) UpdateStats(ctx context.Context, in *UpdateStatsRequest) (*UpdateStatsResponse, error) {
+	out := new(UpdateStatsResponse)
+	err := c.cc.Invoke(ctx, "/coder.agent.v2.Agent/UpdateStats", drpcEncoding_File_agent_proto_agent_proto{}, in, out)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *drpcAgentClient) UpdateLifecycle(ctx context.Context, in *UpdateLifecycleRequest) (*Lifecycle, error) {
+	out := new(Lifecycle)
+	err := c.cc.Invoke(ctx, "/coder.agent.v2.Agent/UpdateLifecycle", drpcEncoding_File_agent_proto_agent_proto{}, in, out)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *drpcAgentClient) BatchUpdateAppHealths(ctx context.Context, in *BatchUpdateAppHealthRequest) (*BatchUpdateAppHealthResponse, error) {
+	out := new(BatchUpdateAppHealthResponse)
+	err := c.cc.Invoke(ctx, "/coder.agent.v2.Agent/BatchUpdateAppHealths", drpcEncoding_File_agent_proto_agent_proto{}, in, out)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *drpcAgentClient) UpdateStartup(ctx context.Context, in *UpdateStartupRequest) (*Startup, error) {
+	out := new(Startup)
+	err := c.cc.Invoke(ctx, "/coder.agent.v2.Agent/UpdateStartup", drpcEncoding_File_agent_proto_agent_proto{}, in, out)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *drpcAgentClient) BatchUpdateMetadata(ctx context.Context, in *BatchUpdateMetadataRequest) (*BatchUpdateMetadataResponse, error) {
+	out := new(BatchUpdateMetadataResponse)
+	err := c.cc.Invoke(ctx, "/coder.agent.v2.Agent/BatchUpdateMetadata", drpcEncoding_File_agent_proto_agent_proto{}, in, out)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *drpcAgentClient) BatchCreateLogs(ctx context.Context, in *BatchCreateLogsRequest) (*BatchCreateLogsResponse, error) {
+	out := new(BatchCreateLogsResponse)
+	err := c.cc.Invoke(ctx, "/coder.agent.v2.Agent/BatchCreateLogs", drpcEncoding_File_agent_proto_agent_proto{}, in, out)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+type DRPCAgentServer interface {
+	GetManifest(context.Context, *GetManifestRequest) (*Manifest, error)
+	GetServiceBanner(context.Context, *GetServiceBannerRequest) (*ServiceBanner, error)
+	UpdateStats(context.Context, *UpdateStatsRequest) (*UpdateStatsResponse, error)
+	UpdateLifecycle(context.Context, *UpdateLifecycleRequest) (*Lifecycle, error)
+	BatchUpdateAppHealths(context.Context, *BatchUpdateAppHealthRequest) (*BatchUpdateAppHealthResponse, error)
+	UpdateStartup(context.Context, *UpdateStartupRequest) (*Startup, error)
+	BatchUpdateMetadata(context.Context, *BatchUpdateMetadataRequest) (*BatchUpdateMetadataResponse, error)
+	BatchCreateLogs(context.Context, *BatchCreateLogsRequest) (*BatchCreateLogsResponse, error)
+}
+
+type DRPCAgentUnimplementedServer struct{}
+
+func (s *DRPCAgentUnimplementedServer) GetManifest(context.Context, *GetManifestRequest) (*Manifest, error) {
+	return nil, drpcerr.WithCode(errors.New("Unimplemented"), drpcerr.Unimplemented)
+}
+
+func (s *DRPCAgentUnimplementedServer) GetServiceBanner(context.Context, *GetServiceBannerRequest) (*ServiceBanner, error) {
+	return nil, drpcerr.WithCode(errors.New("Unimplemented"), drpcerr.Unimplemented)
+}
+
+func (s *DRPCAgentUnimplementedServer) UpdateStats(context.Context, *UpdateStatsRequest) (*UpdateStatsResponse, error) {
+	return nil, drpcerr.WithCode(errors.New("Unimplemented"), drpcerr.Unimplemented)
+}
+
+func (s *DRPCAgentUnimplementedServer) UpdateLifecycle(context.Context, *UpdateLifecycleRequest) (*Lifecycle, error) {
+	return nil, drpcerr.WithCode(errors.New("Unimplemented"), drpcerr.Unimplemented)
+}
+
+func (s *DRPCAgentUnimplementedServer) BatchUpdateAppHealths(context.Context, *BatchUpdateAppHealthRequest) (*BatchUpdateAppHealthResponse, error) {
+	return nil, drpcerr.WithCode(errors.New("Unimplemented"), drpcerr.Unimplemented)
+}
+
+func (s *DRPCAgentUnimplementedServer) UpdateStartup(context.Context, *UpdateStartupRequest) (*Startup, error) {
+	return nil, drpcerr.WithCode(errors.New("Unimplemented"), drpcerr.Unimplemented)
+}
+
+func (s *DRPCAgentUnimplementedServer) BatchUpdateMetadata(context.Context, *BatchUpdateMetadataRequest) (*BatchUpdateMetadataResponse, error) {
+	return nil, drpcerr.WithCode(errors.New("Unimplemented"), drpcerr.Unimplemented)
+}
+
+func (s *DRPCAgentUnimplementedServer) BatchCreateLogs(context.Context, *BatchCreateLogsRequest) (*BatchCreateLogsResponse, error) {
+	return nil, drpcerr.WithCode(errors.New("Unimplemented"), drpcerr.Unimplemented)
+}
+
+type DRPCAgentDescription struct{}
+
+func (DRPCAgentDescription) NumMethods() int { return 8 }
+
+func (DRPCAgentDescription) Method(n int) (string, drpc.Encoding, drpc.Receiver, interface{}, bool) {
+	switch n {
+	case 0:
+		return "/coder.agent.v2.Agent/GetManifest", drpcEncoding_File_agent_proto_agent_proto{},
+			func(srv interface{}, ctx context.Context, in1, in2 interface{}) (drpc.Message, error) {
+				return srv.(DRPCAgentServer).
+					GetManifest(
+						ctx,
+						in1.(*GetManifestRequest),
+					)
+			}, DRPCAgentServer.GetManifest, true
+	case 1:
+		return "/coder.agent.v2.Agent/GetServiceBanner", drpcEncoding_File_agent_proto_agent_proto{},
+			func(srv interface{}, ctx context.Context, in1, in2 interface{}) (drpc.Message, error) {
+				return srv.(DRPCAgentServer).
+					GetServiceBanner(
+						ctx,
+						in1.(*GetServiceBannerRequest),
+					)
+			}, DRPCAgentServer.GetServiceBanner, true
+	case 2:
+		return "/coder.agent.v2.Agent/UpdateStats", drpcEncoding_File_agent_proto_agent_proto{},
+			func(srv interface{}, ctx context.Context, in1, in2 interface{}) (drpc.Message, error) {
+				return srv.(DRPCAgentServer).
+					UpdateStats(
+						ctx,
+						in1.(*UpdateStatsRequest),
+					)
+			}, DRPCAgentServer.UpdateStats, true
+	case 3:
+		return "/coder.agent.v2.Agent/UpdateLifecycle", drpcEncoding_File_agent_proto_agent_proto{},
+			func(srv interface{}, ctx context.Context, in1, in2 interface{}) (drpc.Message, error) {
+				return srv.(DRPCAgentServer).
+					UpdateLifecycle(
+						ctx,
+						in1.(*UpdateLifecycleRequest),
+					)
+			}, DRPCAgentServer.UpdateLifecycle, true
+	case 4:
+		return "/coder.agent.v2.Agent/BatchUpdateAppHealths", drpcEncoding_File_agent_proto_agent_proto{},
+			func(srv interface{}, ctx context.Context, in1, in2 interface{}) (drpc.Message, error) {
+				return srv.(DRPCAgentServer).
+					BatchUpdateAppHealths(
+						ctx,
+						in1.(*BatchUpdateAppHealthRequest),
+					)
+			}, DRPCAgentServer.BatchUpdateAppHealths, true
+	case 5:
+		return "/coder.agent.v2.Agent/UpdateStartup", drpcEncoding_File_agent_proto_agent_proto{},
+			func(srv interface{}, ctx context.Context, in1, in2 interface{}) (drpc.Message, error) {
+				return srv.(DRPCAgentServer).
+					UpdateStartup(
+						ctx,
+						in1.(*UpdateStartupRequest),
+					)
+			}, DRPCAgentServer.UpdateStartup, true
+	case 6:
+		return "/coder.agent.v2.Agent/BatchUpdateMetadata", drpcEncoding_File_agent_proto_agent_proto{},
+			func(srv interface{}, ctx context.Context, in1, in2 interface{}) (drpc.Message, error) {
+				return srv.(DRPCAgentServer).
+					BatchUpdateMetadata(
+						ctx,
+						in1.(*BatchUpdateMetadataRequest),
+					)
+			}, DRPCAgentServer.BatchUpdateMetadata, true
+	case 7:
+		return "/coder.agent.v2.Agent/BatchCreateLogs", drpcEncoding_File_agent_proto_agent_proto{},
+			func(srv interface{}, ctx context.Context, in1, in2 interface{}) (drpc.Message, error) {
+				return srv.(DRPCAgentServer).
+					BatchCreateLogs(
+						ctx,
+						in1.(*BatchCreateLogsRequest),
+					)
+			}, DRPCAgentServer.BatchCreateLogs, true
+	default:
+		return "", nil, nil, nil, false
+	}
+}
+
+func DRPCRegisterAgent(mux drpc.Mux, impl DRPCAgentServer) error {
+	return mux.Register(impl, DRPCAgentDescription{})
+}
+
+type DRPCAgent_GetManifestStream interface {
+	drpc.Stream
+	SendAndClose(*Manifest) error
+}
+
+type drpcAgent_GetManifestStream struct {
+	drpc.Stream
+}
+
+func (x *drpcAgent_GetManifestStream) SendAndClose(m *Manifest) error {
+	if err := x.MsgSend(m, drpcEncoding_File_agent_proto_agent_proto{}); err != nil {
+		return err
+	}
+	return x.CloseSend()
+}
+
+type DRPCAgent_GetServiceBannerStream interface {
+	drpc.Stream
+	SendAndClose(*ServiceBanner) error
+}
+
+type drpcAgent_GetServiceBannerStream struct {
+	drpc.Stream
+}
+
+func (x *drpcAgent_GetServiceBannerStream) SendAndClose(m *ServiceBanner) error {
+	if err := x.MsgSend(m, drpcEncoding_File_agent_proto_agent_proto{}); err != nil {
+		return err
+	}
+	return x.CloseSend()
+}
+
+type DRPCAgent_UpdateStatsStream interface {
+	drpc.Stream
+	SendAndClose(*UpdateStatsResponse) error
+}
+
+type drpcAgent_UpdateStatsStream struct {
+	drpc.Stream
+}
+
+func (x *drpcAgent_UpdateStatsStream) SendAndClose(m *UpdateStatsResponse) error {
+	if err := x.MsgSend(m, drpcEncoding_File_agent_proto_agent_proto{}); err != nil {
+		return err
+	}
+	return x.CloseSend()
+}
+
+type DRPCAgent_UpdateLifecycleStream interface {
+	drpc.Stream
+	SendAndClose(*Lifecycle) error
+}
+
+type drpcAgent_UpdateLifecycleStream struct {
+	drpc.Stream
+}
+
+func (x *drpcAgent_UpdateLifecycleStream) SendAndClose(m *Lifecycle) error {
+	if err := x.MsgSend(m, drpcEncoding_File_agent_proto_agent_proto{}); err != nil {
+		return err
+	}
+	return x.CloseSend()
+}
+
+type DRPCAgent_BatchUpdateAppHealthsStream interface {
+	drpc.Stream
+	SendAndClose(*BatchUpdateAppHealthResponse) error
+}
+
+type drpcAgent_BatchUpdateAppHealthsStream struct {
+	drpc.Stream
+}
+
+func (x *drpcAgent_BatchUpdateAppHealthsStream) SendAndClose(m *BatchUpdateAppHealthResponse) error {
+	if err := x.MsgSend(m, drpcEncoding_File_agent_proto_agent_proto{}); err != nil {
+		return err
+	}
+	return x.CloseSend()
+}
+
+type DRPCAgent_UpdateStartupStream interface {
+	drpc.Stream
+	SendAndClose(*Startup) error
+}
+
+type drpcAgent_UpdateStartupStream struct {
+	drpc.Stream
+}
+
+func (x *drpcAgent_UpdateStartupStream) SendAndClose(m *Startup) error {
+	if err := x.MsgSend(m, drpcEncoding_File_agent_proto_agent_proto{}); err != nil {
+		return err
+	}
+	return x.CloseSend()
+}
+
+type DRPCAgent_BatchUpdateMetadataStream interface {
+	drpc.Stream
+	SendAndClose(*BatchUpdateMetadataResponse) error
+}
+
+type drpcAgent_BatchUpdateMetadataStream struct {
+	drpc.Stream
+}
+
+func (x *drpcAgent_BatchUpdateMetadataStream) SendAndClose(m *BatchUpdateMetadataResponse) error {
+	if err := x.MsgSend(m, drpcEncoding_File_agent_proto_agent_proto{}); err != nil {
+		return err
+	}
+	return x.CloseSend()
+}
+
+type DRPCAgent_BatchCreateLogsStream interface {
+	drpc.Stream
+	SendAndClose(*BatchCreateLogsResponse) error
+}
+
+type drpcAgent_BatchCreateLogsStream struct {
+	drpc.Stream
+}
+
+func (x *drpcAgent_BatchCreateLogsStream) SendAndClose(m *BatchCreateLogsResponse) error {
+	if err := x.MsgSend(m, drpcEncoding_File_agent_proto_agent_proto{}); err != nil {
+		return err
+	}
+	return x.CloseSend()
+}

agent/proto/compare.go

@@ -0,0 +1,26 @@
+package proto
+
+func LabelsEqual(a, b []*Stats_Metric_Label) bool {
+	am := make(map[string]string, len(a))
+	for _, lbl := range a {
+		v := lbl.GetValue()
+		if v == "" {
+			// Prometheus considers empty labels as equivalent to being absent
+			continue
+		}
+		am[lbl.GetName()] = lbl.GetValue()
+	}
+	lenB := 0
+	for _, lbl := range b {
+		v := lbl.GetValue()
+		if v == "" {
+			// Prometheus considers empty labels as equivalent to being absent
+			continue
+		}
+		lenB++
+		if am[lbl.GetName()] != v {
+			return false
+		}
+	}
+	return len(am) == lenB
+}

agent/proto/compare_test.go

@@ -0,0 +1,77 @@
+package proto_test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/coder/coder/v2/agent/proto"
+)
+
+func TestLabelsEqual(t *testing.T) {
+	t.Parallel()
+	for _, tc := range []struct {
+		name string
+		a    []*proto.Stats_Metric_Label
+		b    []*proto.Stats_Metric_Label
+		eq   bool
+	}{
+		{
+			name: "mainlineEq",
+			a: []*proto.Stats_Metric_Label{
+				{Name: "credulity", Value: "sus"},
+				{Name: "color", Value: "aquamarine"},
+			},
+			b: []*proto.Stats_Metric_Label{
+				{Name: "credulity", Value: "sus"},
+				{Name: "color", Value: "aquamarine"},
+			},
+			eq: true,
+		},
+		{
+			name: "emptyValue",
+			a: []*proto.Stats_Metric_Label{
+				{Name: "credulity", Value: "sus"},
+				{Name: "color", Value: "aquamarine"},
+				{Name: "singularity", Value: ""},
+			},
+			b: []*proto.Stats_Metric_Label{
+				{Name: "credulity", Value: "sus"},
+				{Name: "color", Value: "aquamarine"},
+			},
+			eq: true,
+		},
+		{
+			name: "extra",
+			a: []*proto.Stats_Metric_Label{
+				{Name: "credulity", Value: "sus"},
+				{Name: "color", Value: "aquamarine"},
+				{Name: "opacity", Value: "seyshells"},
+			},
+			b: []*proto.Stats_Metric_Label{
+				{Name: "credulity", Value: "sus"},
+				{Name: "color", Value: "aquamarine"},
+			},
+			eq: false,
+		},
+		{
+			name: "different",
+			a: []*proto.Stats_Metric_Label{
+				{Name: "credulity", Value: "sus"},
+				{Name: "color", Value: "aquamarine"},
+			},
+			b: []*proto.Stats_Metric_Label{
+				{Name: "credulity", Value: "legit"},
+				{Name: "color", Value: "aquamarine"},
+			},
+			eq: false,
+		},
+	} {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			require.Equal(t, tc.eq, proto.LabelsEqual(tc.a, tc.b))
+			require.Equal(t, tc.eq, proto.LabelsEqual(tc.b, tc.a))
+		})
+	}
+}

agent/proto/version.go

@@ -0,0 +1,10 @@
+package proto
+
+import (
+	"github.com/coder/coder/v2/tailnet/proto"
+)
+
+// CurrentVersion is the current version of the agent API.  It is tied to the
+// tailnet API version to avoid confusion, since agents connect to the tailnet
+// API over the same websocket.
+var CurrentVersion = proto.CurrentVersion

agent/reaper/reaper_test.go

@@ -14,8 +14,8 @@ import (
 	"github.com/hashicorp/go-reap"
 	"github.com/stretchr/testify/require"
 
-	"github.com/coder/coder/agent/reaper"
-	"github.com/coder/coder/testutil"
+	"github.com/coder/coder/v2/agent/reaper"
+	"github.com/coder/coder/v2/testutil"
 )
 
 // TestReap checks that's the reaper is successfully reaping

agent/reconnectingpty/buffered.go

@@ -0,0 +1,241 @@
+package reconnectingpty
+
+import (
+	"context"
+	"errors"
+	"io"
+	"net"
+	"time"
+
+	"github.com/armon/circbuf"
+	"github.com/prometheus/client_golang/prometheus"
+	"golang.org/x/exp/slices"
+	"golang.org/x/xerrors"
+
+	"cdr.dev/slog"
+
+	"github.com/coder/coder/v2/pty"
+)
+
+// bufferedReconnectingPTY provides a reconnectable PTY by using a ring buffer to store
+// scrollback.
+type bufferedReconnectingPTY struct {
+	command *pty.Cmd
+
+	activeConns    map[string]net.Conn
+	circularBuffer *circbuf.Buffer
+
+	ptty    pty.PTYCmd
+	process pty.Process
+
+	metrics *prometheus.CounterVec
+
+	state *ptyState
+	// timer will close the reconnecting pty when it expires.  The timer will be
+	// reset as long as there are active connections.
+	timer   *time.Timer
+	timeout time.Duration
+}
+
+// newBuffered starts the buffered pty.  If the context ends the process will be
+// killed.
+func newBuffered(ctx context.Context, cmd *pty.Cmd, options *Options, logger slog.Logger) *bufferedReconnectingPTY {
+	rpty := &bufferedReconnectingPTY{
+		activeConns: map[string]net.Conn{},
+		command:     cmd,
+		metrics:     options.Metrics,
+		state:       newState(),
+		timeout:     options.Timeout,
+	}
+
+	// Default to buffer 64KiB.
+	circularBuffer, err := circbuf.NewBuffer(64 << 10)
+	if err != nil {
+		rpty.state.setState(StateDone, xerrors.Errorf("create circular buffer: %w", err))
+		return rpty
+	}
+	rpty.circularBuffer = circularBuffer
+
+	// Add TERM then start the command with a pty.  pty.Cmd duplicates Path as the
+	// first argument so remove it.
+	cmdWithEnv := pty.CommandContext(ctx, cmd.Path, cmd.Args[1:]...)
+	cmdWithEnv.Env = append(rpty.command.Env, "TERM=xterm-256color")
+	cmdWithEnv.Dir = rpty.command.Dir
+	ptty, process, err := pty.Start(cmdWithEnv)
+	if err != nil {
+		rpty.state.setState(StateDone, xerrors.Errorf("start pty: %w", err))
+		return rpty
+	}
+	rpty.ptty = ptty
+	rpty.process = process
+
+	go rpty.lifecycle(ctx, logger)
+
+	// Multiplex the output onto the circular buffer and each active connection.
+	// We do not need to separately monitor for the process exiting.  When it
+	// exits, our ptty.OutputReader() will return EOF after reading all process
+	// output.
+	go func() {
+		buffer := make([]byte, 1024)
+		for {
+			read, err := ptty.OutputReader().Read(buffer)
+			if err != nil {
+				// When the PTY is closed, this is triggered.
+				// Error is typically a benign EOF, so only log for debugging.
+				if errors.Is(err, io.EOF) {
+					logger.Debug(ctx, "unable to read pty output, command might have exited", slog.Error(err))
+				} else {
+					logger.Warn(ctx, "unable to read pty output, command might have exited", slog.Error(err))
+					rpty.metrics.WithLabelValues("output_reader").Add(1)
+				}
+				// Could have been killed externally or failed to start at all (command
+				// not found for example).
+				// TODO: Should we check the process's exit code in case the command was
+				//       invalid?
+				rpty.Close(nil)
+				break
+			}
+			part := buffer[:read]
+			rpty.state.cond.L.Lock()
+			_, err = rpty.circularBuffer.Write(part)
+			if err != nil {
+				logger.Error(ctx, "write to circular buffer", slog.Error(err))
+				rpty.metrics.WithLabelValues("write_buffer").Add(1)
+			}
+			// TODO: Instead of ranging over a map, could we send the output to a
+			// channel and have each individual Attach read from that?
+			for cid, conn := range rpty.activeConns {
+				_, err = conn.Write(part)
+				if err != nil {
+					logger.Warn(ctx,
+						"error writing to active connection",
+						slog.F("connection_id", cid),
+						slog.Error(err),
+					)
+					rpty.metrics.WithLabelValues("write").Add(1)
+				}
+			}
+			rpty.state.cond.L.Unlock()
+		}
+	}()
+
+	return rpty
+}
+
+// lifecycle manages the lifecycle of the reconnecting pty.  If the context ends
+// or the reconnecting pty closes the pty will be shut down.
+func (rpty *bufferedReconnectingPTY) lifecycle(ctx context.Context, logger slog.Logger) {
+	rpty.timer = time.AfterFunc(attachTimeout, func() {
+		rpty.Close(xerrors.New("reconnecting pty timeout"))
+	})
+
+	logger.Debug(ctx, "reconnecting pty ready")
+	rpty.state.setState(StateReady, nil)
+
+	state, reasonErr := rpty.state.waitForStateOrContext(ctx, StateClosing)
+	if state < StateClosing {
+		// If we have not closed yet then the context is what unblocked us (which
+		// means the agent is shutting down) so move into the closing phase.
+		rpty.Close(reasonErr)
+	}
+	rpty.timer.Stop()
+
+	rpty.state.cond.L.Lock()
+	// Log these closes only for debugging since the connections or processes
+	// might have already closed on their own.
+	for _, conn := range rpty.activeConns {
+		err := conn.Close()
+		if err != nil {
+			logger.Debug(ctx, "closed conn with error", slog.Error(err))
+		}
+	}
+	// Connections get removed once they close but it is possible there is still
+	// some data that will be written before that happens so clear the map now to
+	// avoid writing to closed connections.
+	rpty.activeConns = map[string]net.Conn{}
+	rpty.state.cond.L.Unlock()
+
+	// Log close/kill only for debugging since the process might have already
+	// closed on its own.
+	err := rpty.ptty.Close()
+	if err != nil {
+		logger.Debug(ctx, "closed ptty with error", slog.Error(err))
+	}
+
+	err = rpty.process.Kill()
+	if err != nil {
+		logger.Debug(ctx, "killed process with error", slog.Error(err))
+	}
+
+	logger.Info(ctx, "closed reconnecting pty")
+	rpty.state.setState(StateDone, reasonErr)
+}
+
+func (rpty *bufferedReconnectingPTY) Attach(ctx context.Context, connID string, conn net.Conn, height, width uint16, logger slog.Logger) error {
+	logger.Info(ctx, "attach to reconnecting pty")
+
+	// This will kill the heartbeat once we hit EOF or an error.
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	err := rpty.doAttach(connID, conn)
+	if err != nil {
+		return err
+	}
+
+	defer func() {
+		rpty.state.cond.L.Lock()
+		defer rpty.state.cond.L.Unlock()
+		delete(rpty.activeConns, connID)
+	}()
+
+	state, err := rpty.state.waitForStateOrContext(ctx, StateReady)
+	if state != StateReady {
+		return err
+	}
+
+	go heartbeat(ctx, rpty.timer, rpty.timeout)
+
+	// Resize the PTY to initial height + width.
+	err = rpty.ptty.Resize(height, width)
+	if err != nil {
+		// We can continue after this, it's not fatal!
+		logger.Warn(ctx, "reconnecting PTY initial resize failed, but will continue", slog.Error(err))
+		rpty.metrics.WithLabelValues("resize").Add(1)
+	}
+
+	// Pipe conn -> pty and block.  pty -> conn is handled in newBuffered().
+	readConnLoop(ctx, conn, rpty.ptty, rpty.metrics, logger)
+	return nil
+}
+
+// doAttach adds the connection to the map and replays the buffer.  It exists
+// separately only for convenience to defer the mutex unlock which is not
+// possible in Attach since it blocks.
+func (rpty *bufferedReconnectingPTY) doAttach(connID string, conn net.Conn) error {
+	rpty.state.cond.L.Lock()
+	defer rpty.state.cond.L.Unlock()
+
+	// Write any previously stored data for the TTY.  Since the command might be
+	// short-lived and have already exited, make sure we always at least output
+	// the buffer before returning, mostly just so tests pass.
+	prevBuf := slices.Clone(rpty.circularBuffer.Bytes())
+	_, err := conn.Write(prevBuf)
+	if err != nil {
+		rpty.metrics.WithLabelValues("write").Add(1)
+		return xerrors.Errorf("write buffer to conn: %w", err)
+	}
+
+	rpty.activeConns[connID] = conn
+
+	return nil
+}
+
+func (rpty *bufferedReconnectingPTY) Wait() {
+	_, _ = rpty.state.waitForState(StateClosing)
+}
+
+func (rpty *bufferedReconnectingPTY) Close(error error) {
+	// The closing state change will be handled by the lifecycle.
+	rpty.state.setState(StateClosing, error)
+}

agent/reconnectingpty/reconnectingpty.go

@@ -0,0 +1,226 @@
+package reconnectingpty
+
+import (
+	"context"
+	"encoding/json"
+	"io"
+	"net"
+	"os/exec"
+	"runtime"
+	"sync"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"golang.org/x/xerrors"
+
+	"cdr.dev/slog"
+
+	"github.com/coder/coder/v2/codersdk"
+	"github.com/coder/coder/v2/pty"
+)
+
+// attachTimeout is the initial timeout for attaching and will probably be far
+// shorter than the reconnect timeout in most cases; in tests it might be
+// longer.  It should be at least long enough for the first screen attach to be
+// able to start up the daemon and for the buffered pty to start.
+const attachTimeout = 30 * time.Second
+
+// Options allows configuring the reconnecting pty.
+type Options struct {
+	// Timeout describes how long to keep the pty alive without any connections.
+	// Once elapsed the pty will be killed.
+	Timeout time.Duration
+	// Metrics tracks various error counters.
+	Metrics *prometheus.CounterVec
+}
+
+// ReconnectingPTY is a pty that can be reconnected within a timeout and to
+// simultaneous connections.  The reconnecting pty can be backed by screen if
+// installed or a (buggy) buffer replay fallback.
+type ReconnectingPTY interface {
+	// Attach pipes the connection and pty, spawning it if necessary, replays
+	// history, then blocks until EOF, an error, or the context's end.  The
+	// connection is expected to send JSON-encoded messages and accept raw output
+	// from the ptty.  If the context ends or the process dies the connection will
+	// be detached.
+	Attach(ctx context.Context, connID string, conn net.Conn, height, width uint16, logger slog.Logger) error
+	// Wait waits for the reconnecting pty to close.  The underlying process might
+	// still be exiting.
+	Wait()
+	// Close kills the reconnecting pty process.
+	Close(err error)
+}
+
+// New sets up a new reconnecting pty that wraps the provided command.  Any
+// errors with starting are returned on Attach().  The reconnecting pty will
+// close itself (and all connections to it) if nothing is attached for the
+// duration of the timeout, if the context ends, or the process exits (buffered
+// backend only).
+func New(ctx context.Context, cmd *pty.Cmd, options *Options, logger slog.Logger) ReconnectingPTY {
+	if options.Timeout == 0 {
+		options.Timeout = 5 * time.Minute
+	}
+	// Screen seems flaky on Darwin.  Locally the tests pass 100% of the time (100
+	// runs) but in CI screen often incorrectly claims the session name does not
+	// exist even though screen -list shows it.  For now, restrict screen to
+	// Linux.
+	backendType := "buffered"
+	if runtime.GOOS == "linux" {
+		_, err := exec.LookPath("screen")
+		if err == nil {
+			backendType = "screen"
+		}
+	}
+
+	logger.Info(ctx, "start reconnecting pty", slog.F("backend_type", backendType))
+
+	switch backendType {
+	case "screen":
+		return newScreen(ctx, cmd, options, logger)
+	default:
+		return newBuffered(ctx, cmd, options, logger)
+	}
+}
+
+// heartbeat resets timer before timeout elapses and blocks until ctx ends.
+func heartbeat(ctx context.Context, timer *time.Timer, timeout time.Duration) {
+	// Reset now in case it is near the end.
+	timer.Reset(timeout)
+
+	// Reset when the context ends to ensure the pty stays up for the full
+	// timeout.
+	defer timer.Reset(timeout)
+
+	heartbeat := time.NewTicker(timeout / 2)
+	defer heartbeat.Stop()
+
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-heartbeat.C:
+			timer.Reset(timeout)
+		}
+	}
+}
+
+// State represents the current state of the reconnecting pty.  States are
+// sequential and will only move forward.
+type State int
+
+const (
+	// StateStarting is the default/start state.  Attaching will block until the
+	// reconnecting pty becomes ready.
+	StateStarting = iota
+	// StateReady means the reconnecting pty is ready to be attached.
+	StateReady
+	// StateClosing means the reconnecting pty has begun closing.  The underlying
+	// process may still be exiting.  Attaching will result in an error.
+	StateClosing
+	// StateDone means the reconnecting pty has completely shut down and the
+	// process has exited.  Attaching will result in an error.
+	StateDone
+)
+
+// ptyState is a helper for tracking the reconnecting PTY's state.
+type ptyState struct {
+	// cond broadcasts state changes and any accompanying errors.
+	cond *sync.Cond
+	// error describes the error that caused the state change, if there was one.
+	// It is not safe to access outside of cond.L.
+	error error
+	// state holds the current reconnecting pty state.  It is not safe to access
+	// this outside of cond.L.
+	state State
+}
+
+func newState() *ptyState {
+	return &ptyState{
+		cond:  sync.NewCond(&sync.Mutex{}),
+		state: StateStarting,
+	}
+}
+
+// setState sets and broadcasts the provided state if it is greater than the
+// current state and the error if one has not already been set.
+func (s *ptyState) setState(state State, err error) {
+	s.cond.L.Lock()
+	defer s.cond.L.Unlock()
+	// Cannot regress states.  For example, trying to close after the process is
+	// done should leave us in the done state and not the closing state.
+	if state <= s.state {
+		return
+	}
+	s.error = err
+	s.state = state
+	s.cond.Broadcast()
+}
+
+// waitForState blocks until the state or a greater one is reached.
+func (s *ptyState) waitForState(state State) (State, error) {
+	s.cond.L.Lock()
+	defer s.cond.L.Unlock()
+	for state > s.state {
+		s.cond.Wait()
+	}
+	return s.state, s.error
+}
+
+// waitForStateOrContext blocks until the state or a greater one is reached or
+// the provided context ends.
+func (s *ptyState) waitForStateOrContext(ctx context.Context, state State) (State, error) {
+	s.cond.L.Lock()
+	defer s.cond.L.Unlock()
+
+	nevermind := make(chan struct{})
+	defer close(nevermind)
+	go func() {
+		select {
+		case <-ctx.Done():
+			// Wake up when the context ends.
+			s.cond.Broadcast()
+		case <-nevermind:
+		}
+	}()
+
+	for ctx.Err() == nil && state > s.state {
+		s.cond.Wait()
+	}
+	if ctx.Err() != nil {
+		return s.state, ctx.Err()
+	}
+	return s.state, s.error
+}
+
+// readConnLoop reads messages from conn and writes to ptty as needed.  Blocks
+// until EOF or an error writing to ptty or reading from conn.
+func readConnLoop(ctx context.Context, conn net.Conn, ptty pty.PTYCmd, metrics *prometheus.CounterVec, logger slog.Logger) {
+	decoder := json.NewDecoder(conn)
+	for {
+		var req codersdk.ReconnectingPTYRequest
+		err := decoder.Decode(&req)
+		if xerrors.Is(err, io.EOF) {
+			return
+		}
+		if err != nil {
+			logger.Warn(ctx, "reconnecting pty failed with read error", slog.Error(err))
+			return
+		}
+		_, err = ptty.InputWriter().Write([]byte(req.Data))
+		if err != nil {
+			logger.Warn(ctx, "reconnecting pty failed with write error", slog.Error(err))
+			metrics.WithLabelValues("input_writer").Add(1)
+			return
+		}
+		// Check if a resize needs to happen!
+		if req.Height == 0 || req.Width == 0 {
+			continue
+		}
+		err = ptty.Resize(req.Height, req.Width)
+		if err != nil {
+			// We can continue after this, it's not fatal!
+			logger.Warn(ctx, "reconnecting pty resize failed, but will continue", slog.Error(err))
+			metrics.WithLabelValues("resize").Add(1)
+		}
+	}
+}

agent/reconnectingpty/screen.go

@@ -0,0 +1,396 @@
+package reconnectingpty
+
+import (
+	"bytes"
+	"context"
+	"crypto/rand"
+	"encoding/hex"
+	"errors"
+	"io"
+	"net"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/gliderlabs/ssh"
+	"github.com/prometheus/client_golang/prometheus"
+	"golang.org/x/xerrors"
+
+	"cdr.dev/slog"
+	"github.com/coder/coder/v2/pty"
+)
+
+// screenReconnectingPTY provides a reconnectable PTY via `screen`.
+type screenReconnectingPTY struct {
+	command *pty.Cmd
+
+	// id holds the id of the session for both creating and attaching.  This will
+	// be generated uniquely for each session because without control of the
+	// screen daemon we do not have its PID and without the PID screen will do
+	// partial matching.  Enforcing a unique ID should guarantee we match on the
+	// right session.
+	id string
+
+	// mutex prevents concurrent attaches to the session.  Screen will happily
+	// spawn two separate sessions with the same name if multiple attaches happen
+	// in a close enough interval.  We are not able to control the screen daemon
+	// ourselves to prevent this because the daemon will spawn with a hardcoded
+	// 24x80 size which results in confusing padding above the prompt once the
+	// attach comes in and resizes.
+	mutex sync.Mutex
+
+	configFile string
+
+	metrics *prometheus.CounterVec
+
+	state *ptyState
+	// timer will close the reconnecting pty when it expires.  The timer will be
+	// reset as long as there are active connections.
+	timer   *time.Timer
+	timeout time.Duration
+}
+
+// newScreen creates a new screen-backed reconnecting PTY.  It writes config
+// settings and creates the socket directory.  If we could, we would want to
+// spawn the daemon here and attach each connection to it but since doing that
+// spawns the daemon with a hardcoded 24x80 size it is not a very good user
+// experience.  Instead we will let the attach command spawn the daemon on its
+// own which causes it to spawn with the specified size.
+func newScreen(ctx context.Context, cmd *pty.Cmd, options *Options, logger slog.Logger) *screenReconnectingPTY {
+	rpty := &screenReconnectingPTY{
+		command: cmd,
+		metrics: options.Metrics,
+		state:   newState(),
+		timeout: options.Timeout,
+	}
+
+	go rpty.lifecycle(ctx, logger)
+
+	// Socket paths are limited to around 100 characters on Linux and macOS which
+	// depending on the temporary directory can be a problem.  To give more leeway
+	// use a short ID.
+	buf := make([]byte, 4)
+	_, err := rand.Read(buf)
+	if err != nil {
+		rpty.state.setState(StateDone, xerrors.Errorf("generate screen id: %w", err))
+		return rpty
+	}
+	rpty.id = hex.EncodeToString(buf)
+
+	settings := []string{
+		// Disable the startup message that appears for five seconds.
+		"startup_message off",
+		// Some message are hard-coded, the best we can do is set msgwait to 0
+		// which seems to hide them. This can happen for example if screen shows
+		// the version message when starting up.
+		"msgminwait 0",
+		"msgwait 0",
+		// Tell screen not to handle motion for xterm* terminals which allows
+		// scrolling the terminal via the mouse wheel or scroll bar (by default
+		// screen uses it to cycle through the command history).  There does not
+		// seem to be a way to make screen itself scroll on mouse wheel.  tmux can
+		// do it but then there is no scroll bar and it kicks you into copy mode
+		// where keys stop working until you exit copy mode which seems like it
+		// could be confusing.
+		"termcapinfo xterm* ti@:te@",
+		// Enable alternate screen emulation otherwise applications get rendered in
+		// the current window which wipes out visible output resulting in missing
+		// output when scrolling back with the mouse wheel (copy mode still works
+		// since that is screen itself scrolling).
+		"altscreen on",
+		// Remap the control key to C-s since C-a may be used in applications.  C-s
+		// is chosen because it cannot actually be used because by default it will
+		// pause and C-q to resume will just kill the browser window.  We may not
+		// want people using the control key anyway since it will not be obvious
+		// they are in screen and doing things like switching windows makes mouse
+		// wheel scroll wonky due to the terminal doing the scrolling rather than
+		// screen itself (but again copy mode will work just fine).
+		"escape ^Ss",
+	}
+
+	rpty.configFile = filepath.Join(os.TempDir(), "coder-screen", "config")
+	err = os.MkdirAll(filepath.Dir(rpty.configFile), 0o700)
+	if err != nil {
+		rpty.state.setState(StateDone, xerrors.Errorf("make screen config dir: %w", err))
+		return rpty
+	}
+
+	err = os.WriteFile(rpty.configFile, []byte(strings.Join(settings, "\n")), 0o600)
+	if err != nil {
+		rpty.state.setState(StateDone, xerrors.Errorf("create config file: %w", err))
+		return rpty
+	}
+
+	return rpty
+}
+
+// lifecycle manages the lifecycle of the reconnecting pty.  If the context ends
+// the reconnecting pty will be closed.
+func (rpty *screenReconnectingPTY) lifecycle(ctx context.Context, logger slog.Logger) {
+	rpty.timer = time.AfterFunc(attachTimeout, func() {
+		rpty.Close(xerrors.New("reconnecting pty timeout"))
+	})
+
+	logger.Debug(ctx, "reconnecting pty ready")
+	rpty.state.setState(StateReady, nil)
+
+	state, reasonErr := rpty.state.waitForStateOrContext(ctx, StateClosing)
+	if state < StateClosing {
+		// If we have not closed yet then the context is what unblocked us (which
+		// means the agent is shutting down) so move into the closing phase.
+		rpty.Close(reasonErr)
+	}
+	rpty.timer.Stop()
+
+	// If the command errors that the session is already gone that is fine.
+	err := rpty.sendCommand(context.Background(), "quit", []string{"No screen session found"})
+	if err != nil {
+		logger.Error(ctx, "close screen session", slog.Error(err))
+	}
+
+	logger.Info(ctx, "closed reconnecting pty")
+	rpty.state.setState(StateDone, reasonErr)
+}
+
+func (rpty *screenReconnectingPTY) Attach(ctx context.Context, _ string, conn net.Conn, height, width uint16, logger slog.Logger) error {
+	logger.Info(ctx, "attach to reconnecting pty")
+
+	// This will kill the heartbeat once we hit EOF or an error.
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	state, err := rpty.state.waitForStateOrContext(ctx, StateReady)
+	if state != StateReady {
+		return err
+	}
+
+	go heartbeat(ctx, rpty.timer, rpty.timeout)
+
+	ptty, process, err := rpty.doAttach(ctx, conn, height, width, logger)
+	if err != nil {
+		if errors.Is(err, context.Canceled) {
+			// Likely the process was too short-lived and canceled the version command.
+			// TODO: Is it worth distinguishing between that and a cancel from the
+			//       Attach() caller?  Additionally, since this could also happen if
+			//       the command was invalid, should we check the process's exit code?
+			return nil
+		}
+		return err
+	}
+
+	defer func() {
+		// Log only for debugging since the process might have already exited on its
+		// own.
+		err := ptty.Close()
+		if err != nil {
+			logger.Debug(ctx, "closed ptty with error", slog.Error(err))
+		}
+		err = process.Kill()
+		if err != nil {
+			logger.Debug(ctx, "killed process with error", slog.Error(err))
+		}
+	}()
+
+	// Pipe conn -> pty and block.
+	readConnLoop(ctx, conn, ptty, rpty.metrics, logger)
+	return nil
+}
+
+// doAttach spawns the screen client and starts the heartbeat.  It exists
+// separately only so we can defer the mutex unlock which is not possible in
+// Attach since it blocks.
+func (rpty *screenReconnectingPTY) doAttach(ctx context.Context, conn net.Conn, height, width uint16, logger slog.Logger) (pty.PTYCmd, pty.Process, error) {
+	// Ensure another attach does not come in and spawn a duplicate session.
+	rpty.mutex.Lock()
+	defer rpty.mutex.Unlock()
+
+	logger.Debug(ctx, "spawning screen client", slog.F("screen_id", rpty.id))
+
+	// Wrap the command with screen and tie it to the connection's context.
+	cmd := pty.CommandContext(ctx, "screen", append([]string{
+		// -S is for setting the session's name.
+		"-S", rpty.id,
+		// -U tells screen to use UTF-8 encoding.
+		// -x allows attaching to an already attached session.
+		// -RR reattaches to the daemon or creates the session daemon if missing.
+		// -q disables the "New screen..." message that appears for five seconds
+		//    when creating a new session with -RR.
+		// -c is the flag for the config file.
+		"-UxRRqc", rpty.configFile,
+		rpty.command.Path,
+		// pty.Cmd duplicates Path as the first argument so remove it.
+	}, rpty.command.Args[1:]...)...)
+	cmd.Env = append(rpty.command.Env, "TERM=xterm-256color")
+	cmd.Dir = rpty.command.Dir
+	ptty, process, err := pty.Start(cmd, pty.WithPTYOption(
+		pty.WithSSHRequest(ssh.Pty{
+			Window: ssh.Window{
+				// Make sure to spawn at the right size because if we resize afterward it
+				// leaves confusing padding (screen will resize such that the screen
+				// contents are aligned to the bottom).
+				Height: int(height),
+				Width:  int(width),
+			},
+		}),
+	))
+	if err != nil {
+		rpty.metrics.WithLabelValues("screen_spawn").Add(1)
+		return nil, nil, err
+	}
+
+	// This context lets us abort the version command if the process dies.
+	versionCtx, versionCancel := context.WithCancel(ctx)
+	defer versionCancel()
+
+	// Pipe pty -> conn and close the connection when the process exits.
+	// We do not need to separately monitor for the process exiting.  When it
+	// exits, our ptty.OutputReader() will return EOF after reading all process
+	// output.
+	go func() {
+		defer versionCancel()
+		defer func() {
+			err := conn.Close()
+			if err != nil {
+				// Log only for debugging since the connection might have already closed
+				// on its own.
+				logger.Debug(ctx, "closed connection with error", slog.Error(err))
+			}
+		}()
+		buffer := make([]byte, 1024)
+		for {
+			read, err := ptty.OutputReader().Read(buffer)
+			if err != nil {
+				// When the PTY is closed, this is triggered.
+				// Error is typically a benign EOF, so only log for debugging.
+				if errors.Is(err, io.EOF) {
+					logger.Debug(ctx, "unable to read pty output; screen might have exited", slog.Error(err))
+				} else {
+					logger.Warn(ctx, "unable to read pty output; screen might have exited", slog.Error(err))
+					rpty.metrics.WithLabelValues("screen_output_reader").Add(1)
+				}
+				// The process might have died because the session itself died or it
+				// might have been separately killed and the session is still up (for
+				// example `exit` or we killed it when the connection closed).  If the
+				// session is still up we might leave the reconnecting pty in memory
+				// around longer than it needs to be but it will eventually clean up
+				// with the timer or context, or the next attach will respawn the screen
+				// daemon which is fine too.
+				break
+			}
+			part := buffer[:read]
+			_, err = conn.Write(part)
+			if err != nil {
+				// Connection might have been closed.
+				if errors.Unwrap(err).Error() != "endpoint is closed for send" {
+					logger.Warn(ctx, "error writing to active conn", slog.Error(err))
+					rpty.metrics.WithLabelValues("screen_write").Add(1)
+				}
+				break
+			}
+		}
+	}()
+
+	// Version seems to be the only command without a side effect (other than
+	// making the version pop up briefly) so use it to wait for the session to
+	// come up.  If we do not wait we could end up spawning multiple sessions with
+	// the same name.
+	err = rpty.sendCommand(versionCtx, "version", nil)
+	if err != nil {
+		// Log only for debugging since the process might already have closed.
+		closeErr := ptty.Close()
+		if closeErr != nil {
+			logger.Debug(ctx, "closed ptty with error", slog.Error(closeErr))
+		}
+		closeErr = process.Kill()
+		if closeErr != nil {
+			logger.Debug(ctx, "killed process with error", slog.Error(closeErr))
+		}
+		rpty.metrics.WithLabelValues("screen_wait").Add(1)
+		return nil, nil, err
+	}
+
+	return ptty, process, nil
+}
+
+// sendCommand runs a screen command against a running screen session.  If the
+// command fails with an error matching anything in successErrors it will be
+// considered a success state (for example "no session" when quitting and the
+// session is already dead).  The command will be retried until successful, the
+// timeout is reached, or the context ends.  A canceled context will return the
+// canceled context's error as-is while a timed-out context returns together
+// with the last error from the command.
+func (rpty *screenReconnectingPTY) sendCommand(ctx context.Context, command string, successErrors []string) error {
+	ctx, cancel := context.WithTimeout(ctx, attachTimeout)
+	defer cancel()
+
+	var lastErr error
+	run := func() bool {
+		var stdout bytes.Buffer
+		//nolint:gosec
+		cmd := exec.CommandContext(ctx, "screen",
+			// -x targets an attached session.
+			"-x", rpty.id,
+			// -c is the flag for the config file.
+			"-c", rpty.configFile,
+			// -X runs a command in the matching session.
+			"-X", command,
+		)
+		cmd.Env = append(rpty.command.Env, "TERM=xterm-256color")
+		cmd.Dir = rpty.command.Dir
+		cmd.Stdout = &stdout
+		err := cmd.Run()
+		if err == nil {
+			return true
+		}
+
+		stdoutStr := stdout.String()
+		for _, se := range successErrors {
+			if strings.Contains(stdoutStr, se) {
+				return true
+			}
+		}
+
+		// Things like "exit status 1" are imprecise so include stdout as it may
+		// contain more information ("no screen session found" for example).
+		if !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded) {
+			lastErr = xerrors.Errorf("`screen -x %s -X %s`: %w: %s", rpty.id, command, err, stdoutStr)
+		}
+
+		return false
+	}
+
+	// Run immediately.
+	if done := run(); done {
+		return nil
+	}
+
+	// Then run on an interval.
+	ticker := time.NewTicker(250 * time.Millisecond)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-ctx.Done():
+			if errors.Is(ctx.Err(), context.Canceled) {
+				return ctx.Err()
+			}
+			return errors.Join(ctx.Err(), lastErr)
+		case <-ticker.C:
+			if done := run(); done {
+				return nil
+			}
+		}
+	}
+}
+
+func (rpty *screenReconnectingPTY) Wait() {
+	_, _ = rpty.state.waitForState(StateClosing)
+}
+
+func (rpty *screenReconnectingPTY) Close(err error) {
+	// The closing state change will be handled by the lifecycle.
+	rpty.state.setState(StateClosing, err)
+}

agent/stats.go

@@ -0,0 +1,126 @@
+package agent
+
+import (
+	"context"
+	"sync"
+	"time"
+
+	"golang.org/x/xerrors"
+	"tailscale.com/types/netlogtype"
+
+	"cdr.dev/slog"
+	"github.com/coder/coder/v2/agent/proto"
+)
+
+const maxConns = 2048
+
+type networkStatsSource interface {
+	SetConnStatsCallback(maxPeriod time.Duration, maxConns int, dump func(start, end time.Time, virtual, physical map[netlogtype.Connection]netlogtype.Counts))
+}
+
+type statsCollector interface {
+	Collect(ctx context.Context, networkStats map[netlogtype.Connection]netlogtype.Counts) *proto.Stats
+}
+
+type statsDest interface {
+	UpdateStats(ctx context.Context, req *proto.UpdateStatsRequest) (*proto.UpdateStatsResponse, error)
+}
+
+// statsReporter is a subcomponent of the agent that handles registering the stats callback on the
+// networkStatsSource (tailnet.Conn in prod), handling the callback, calling back to the
+// statsCollector (agent in prod) to collect additional stats, then sending the update to the
+// statsDest (agent API in prod)
+type statsReporter struct {
+	*sync.Cond
+	networkStats *map[netlogtype.Connection]netlogtype.Counts
+	unreported   bool
+	lastInterval time.Duration
+
+	source    networkStatsSource
+	collector statsCollector
+	logger    slog.Logger
+}
+
+func newStatsReporter(logger slog.Logger, source networkStatsSource, collector statsCollector) *statsReporter {
+	return &statsReporter{
+		Cond:      sync.NewCond(&sync.Mutex{}),
+		logger:    logger,
+		source:    source,
+		collector: collector,
+	}
+}
+
+func (s *statsReporter) callback(_, _ time.Time, virtual, _ map[netlogtype.Connection]netlogtype.Counts) {
+	s.L.Lock()
+	defer s.L.Unlock()
+	s.logger.Debug(context.Background(), "got stats callback")
+	s.networkStats = &virtual
+	s.unreported = true
+	s.Broadcast()
+}
+
+// reportLoop programs the source (tailnet.Conn) to send it stats via the
+// callback, then reports them to the dest.
+//
+// It's intended to be called within the larger retry loop that establishes a
+// connection to the agent API, then passes that connection to go routines like
+// this that use it.  There is no retry and we fail on the first error since
+// this will be inside a larger retry loop.
+func (s *statsReporter) reportLoop(ctx context.Context, dest statsDest) error {
+	// send an initial, blank report to get the interval
+	resp, err := dest.UpdateStats(ctx, &proto.UpdateStatsRequest{})
+	if err != nil {
+		return xerrors.Errorf("initial update: %w", err)
+	}
+	s.lastInterval = resp.ReportInterval.AsDuration()
+	s.source.SetConnStatsCallback(s.lastInterval, maxConns, s.callback)
+
+	// use a separate goroutine to monitor the context so that we notice immediately, rather than
+	// waiting for the next callback (which might never come if we are closing!)
+	ctxDone := false
+	go func() {
+		<-ctx.Done()
+		s.L.Lock()
+		defer s.L.Unlock()
+		ctxDone = true
+		s.Broadcast()
+	}()
+	defer s.logger.Debug(ctx, "reportLoop exiting")
+
+	s.L.Lock()
+	defer s.L.Unlock()
+	for {
+		for !s.unreported && !ctxDone {
+			s.Wait()
+		}
+		if ctxDone {
+			return nil
+		}
+		networkStats := *s.networkStats
+		s.unreported = false
+		if err = s.reportLocked(ctx, dest, networkStats); err != nil {
+			return xerrors.Errorf("report stats: %w", err)
+		}
+	}
+}
+
+func (s *statsReporter) reportLocked(
+	ctx context.Context, dest statsDest, networkStats map[netlogtype.Connection]netlogtype.Counts,
+) error {
+	// here we want to do our collecting/reporting while it is unlocked, but then relock
+	// when we return to reportLoop.
+	s.L.Unlock()
+	defer s.L.Lock()
+	stats := s.collector.Collect(ctx, networkStats)
+	resp, err := dest.UpdateStats(ctx, &proto.UpdateStatsRequest{Stats: stats})
+	if err != nil {
+		return err
+	}
+	interval := resp.GetReportInterval().AsDuration()
+	if interval != s.lastInterval {
+		s.logger.Info(ctx, "new stats report interval", slog.F("interval", interval))
+		s.lastInterval = interval
+		s.source.SetConnStatsCallback(s.lastInterval, maxConns, s.callback)
+	}
+	return nil
+}

agent/stats_internal_test.go

@@ -0,0 +1,212 @@
+package agent
+
+import (
+	"context"
+	"net/netip"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+	"google.golang.org/protobuf/types/known/durationpb"
+	"tailscale.com/types/ipproto"
+
+	"tailscale.com/types/netlogtype"
+
+	"cdr.dev/slog"
+	"cdr.dev/slog/sloggers/slogtest"
+	"github.com/coder/coder/v2/agent/proto"
+	"github.com/coder/coder/v2/testutil"
+)
+
+func TestStatsReporter(t *testing.T) {
+	t.Parallel()
+	ctx := testutil.Context(t, testutil.WaitShort)
+	logger := slogtest.Make(t, nil).Leveled(slog.LevelDebug)
+	fSource := newFakeNetworkStatsSource(ctx, t)
+	fCollector := newFakeCollector(t)
+	fDest := newFakeStatsDest()
+	uut := newStatsReporter(logger, fSource, fCollector)
+
+	loopErr := make(chan error, 1)
+	loopCtx, loopCancel := context.WithCancel(ctx)
+	go func() {
+		err := uut.reportLoop(loopCtx, fDest)
+		loopErr <- err
+	}()
+
+	// initial request to get duration
+	req := testutil.RequireRecvCtx(ctx, t, fDest.reqs)
+	require.NotNil(t, req)
+	require.Nil(t, req.Stats)
+	interval := time.Second * 34
+	testutil.RequireSendCtx(ctx, t, fDest.resps, &proto.UpdateStatsResponse{ReportInterval: durationpb.New(interval)})
+
+	// call to source to set the callback and interval
+	gotInterval := testutil.RequireRecvCtx(ctx, t, fSource.period)
+	require.Equal(t, interval, gotInterval)
+
+	// callback returning netstats
+	netStats := map[netlogtype.Connection]netlogtype.Counts{
+		{
+			Proto: ipproto.TCP,
+			Src:   netip.MustParseAddrPort("192.168.1.33:4887"),
+			Dst:   netip.MustParseAddrPort("192.168.2.99:9999"),
+		}: {
+			TxPackets: 22,
+			TxBytes:   23,
+			RxPackets: 24,
+			RxBytes:   25,
+		},
+	}
+	fSource.callback(time.Now(), time.Now(), netStats, nil)
+
+	// collector called to complete the stats
+	gotNetStats := testutil.RequireRecvCtx(ctx, t, fCollector.calls)
+	require.Equal(t, netStats, gotNetStats)
+
+	// while we are collecting the stats, send in two new netStats to simulate
+	// what happens if we don't keep up.  Only the latest should be kept.
+	netStats0 := map[netlogtype.Connection]netlogtype.Counts{
+		{
+			Proto: ipproto.TCP,
+			Src:   netip.MustParseAddrPort("192.168.1.33:4887"),
+			Dst:   netip.MustParseAddrPort("192.168.2.99:9999"),
+		}: {
+			TxPackets: 10,
+			TxBytes:   10,
+			RxPackets: 10,
+			RxBytes:   10,
+		},
+	}
+	fSource.callback(time.Now(), time.Now(), netStats0, nil)
+	netStats1 := map[netlogtype.Connection]netlogtype.Counts{
+		{
+			Proto: ipproto.TCP,
+			Src:   netip.MustParseAddrPort("192.168.1.33:4887"),
+			Dst:   netip.MustParseAddrPort("192.168.2.99:9999"),
+		}: {
+			TxPackets: 11,
+			TxBytes:   11,
+			RxPackets: 11,
+			RxBytes:   11,
+		},
+	}
+	fSource.callback(time.Now(), time.Now(), netStats1, nil)
+
+	// complete first collection
+	stats := &proto.Stats{SessionCountJetbrains: 55}
+	testutil.RequireSendCtx(ctx, t, fCollector.stats, stats)
+
+	// destination called to report the first stats
+	update := testutil.RequireRecvCtx(ctx, t, fDest.reqs)
+	require.NotNil(t, update)
+	require.Equal(t, stats, update.Stats)
+	testutil.RequireSendCtx(ctx, t, fDest.resps, &proto.UpdateStatsResponse{ReportInterval: durationpb.New(interval)})
+
+	// second update -- only netStats1 is reported
+	gotNetStats = testutil.RequireRecvCtx(ctx, t, fCollector.calls)
+	require.Equal(t, netStats1, gotNetStats)
+	stats = &proto.Stats{SessionCountJetbrains: 66}
+	testutil.RequireSendCtx(ctx, t, fCollector.stats, stats)
+	update = testutil.RequireRecvCtx(ctx, t, fDest.reqs)
+	require.NotNil(t, update)
+	require.Equal(t, stats, update.Stats)
+	interval2 := 27 * time.Second
+	testutil.RequireSendCtx(ctx, t, fDest.resps, &proto.UpdateStatsResponse{ReportInterval: durationpb.New(interval2)})
+
+	// set the new interval
+	gotInterval = testutil.RequireRecvCtx(ctx, t, fSource.period)
+	require.Equal(t, interval2, gotInterval)
+
+	loopCancel()
+	err := testutil.RequireRecvCtx(ctx, t, loopErr)
+	require.NoError(t, err)
+}
+
+type fakeNetworkStatsSource struct {
+	sync.Mutex
+	ctx      context.Context
+	t        testing.TB
+	callback func(start, end time.Time, virtual, physical map[netlogtype.Connection]netlogtype.Counts)
+	period   chan time.Duration
+}
+
+func (f *fakeNetworkStatsSource) SetConnStatsCallback(maxPeriod time.Duration, _ int, dump func(start time.Time, end time.Time, virtual map[netlogtype.Connection]netlogtype.Counts, physical map[netlogtype.Connection]netlogtype.Counts)) {
+	f.Lock()
+	defer f.Unlock()
+	f.callback = dump
+	select {
+	case <-f.ctx.Done():
+		f.t.Error("timeout")
+	case f.period <- maxPeriod:
+		// OK
+	}
+}
+
+func newFakeNetworkStatsSource(ctx context.Context, t testing.TB) *fakeNetworkStatsSource {
+	f := &fakeNetworkStatsSource{
+		ctx:    ctx,
+		t:      t,
+		period: make(chan time.Duration),
+	}
+	return f
+}
+
+type fakeCollector struct {
+	t     testing.TB
+	calls chan map[netlogtype.Connection]netlogtype.Counts
+	stats chan *proto.Stats
+}
+
+func (f *fakeCollector) Collect(ctx context.Context, networkStats map[netlogtype.Connection]netlogtype.Counts) *proto.Stats {
+	select {
+	case <-ctx.Done():
+		f.t.Error("timeout on collect")
+		return nil
+	case f.calls <- networkStats:
+		// ok
+	}
+	select {
+	case <-ctx.Done():
+		f.t.Error("timeout on collect")
+		return nil
+	case s := <-f.stats:
+		return s
+	}
+}
+
+func newFakeCollector(t testing.TB) *fakeCollector {
+	return &fakeCollector{
+		t:     t,
+		calls: make(chan map[netlogtype.Connection]netlogtype.Counts),
+		stats: make(chan *proto.Stats),
+	}
+}
+
+type fakeStatsDest struct {
+	reqs  chan *proto.UpdateStatsRequest
+	resps chan *proto.UpdateStatsResponse
+}
+
+func (f *fakeStatsDest) UpdateStats(ctx context.Context, req *proto.UpdateStatsRequest) (*proto.UpdateStatsResponse, error) {
+	select {
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	case f.reqs <- req:
+		// OK
+	}
+	select {
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	case resp := <-f.resps:
+		return resp, nil
+	}
+}
+
+func newFakeStatsDest() *fakeStatsDest {
+	return &fakeStatsDest{
+		reqs:  make(chan *proto.UpdateStatsRequest),
+		resps: make(chan *proto.UpdateStatsResponse),
+	}
+}

agent/usershell/usershell_darwin.go

@@ -13,6 +13,10 @@ import (
 func Get(username string) (string, error) {
 	// This command will output "UserShell: /bin/zsh" if successful, we
 	// can ignore the error since we have fallback behavior.
+	if !filepath.IsLocal(username) {
+		return "", xerrors.Errorf("username is nonlocal path: %s", username)
+	}
+	//nolint: gosec // input checked above
 	out, _ := exec.Command("dscl", ".", "-read", filepath.Join("/Users", username), "UserShell").Output()
 	s, ok := strings.CutPrefix(string(out), "UserShell: ")
 	if ok {

agent/usershell/usershell_test.go

@@ -7,7 +7,7 @@ import (
 
 	"github.com/stretchr/testify/require"
 
-	"github.com/coder/coder/agent/usershell"
+	"github.com/coder/coder/v2/agent/usershell"
 )
 
 //nolint:paralleltest,tparallel // This test sets an environment variable.

apiversion/apiversion.go

@@ -0,0 +1,89 @@
+package apiversion
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+
+	"golang.org/x/xerrors"
+)
+
+// New returns an *APIVersion with the given major.minor and
+// additional supported major versions.
+func New(maj, min int) *APIVersion {
+	v := &APIVersion{
+		supportedMajor:   maj,
+		supportedMinor:   min,
+		additionalMajors: make([]int, 0),
+	}
+	return v
+}
+
+type APIVersion struct {
+	supportedMajor   int
+	supportedMinor   int
+	additionalMajors []int
+}
+
+func (v *APIVersion) WithBackwardCompat(majs ...int) *APIVersion {
+	v.additionalMajors = append(v.additionalMajors, majs[:]...)
+	return v
+}
+
+func (v *APIVersion) String() string {
+	return fmt.Sprintf("%d.%d", v.supportedMajor, v.supportedMinor)
+}
+
+// Validate validates the given version against the given constraints:
+// A given major.minor version is valid iff:
+//  1. The requested major version is contained within v.supportedMajors
+//  2. If the requested major version is the 'current major', then
+//     the requested minor version must be less than or equal to the supported
+//     minor version.
+//
+// For example, given majors {1, 2} and minor 2, then:
+// - 0.x is not supported,
+// - 1.x is supported,
+// - 2.0, 2.1, and 2.2 are supported,
+// - 2.3+ is not supported.
+func (v *APIVersion) Validate(version string) error {
+	major, minor, err := Parse(version)
+	if err != nil {
+		return err
+	}
+	if major > v.supportedMajor {
+		return xerrors.Errorf("server is at version %d.%d, behind requested major version %s",
+			v.supportedMajor, v.supportedMinor, version)
+	}
+	if major == v.supportedMajor {
+		if minor > v.supportedMinor {
+			return xerrors.Errorf("server is at version %d.%d, behind requested minor version %s",
+				v.supportedMajor, v.supportedMinor, version)
+		}
+		return nil
+	}
+	for _, mjr := range v.additionalMajors {
+		if major == mjr {
+			return nil
+		}
+	}
+	return xerrors.Errorf("version %s is no longer supported", version)
+}
+
+// Parse parses a valid major.minor version string into (major, minor).
+// Both major and minor must be valid integers separated by a period '.'.
+func Parse(version string) (major int, minor int, err error) {
+	parts := strings.Split(version, ".")
+	if len(parts) != 2 {
+		return 0, 0, xerrors.Errorf("invalid version string: %s", version)
+	}
+	major, err = strconv.Atoi(parts[0])
+	if err != nil {
+		return 0, 0, xerrors.Errorf("invalid major version: %s", version)
+	}
+	minor, err = strconv.Atoi(parts[1])
+	if err != nil {
+		return 0, 0, xerrors.Errorf("invalid minor version: %s", version)
+	}
+	return major, minor, nil
+}

apiversion/apiversion_test.go

@@ -0,0 +1,90 @@
+package apiversion_test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/coder/coder/v2/apiversion"
+)
+
+func TestAPIVersionValidate(t *testing.T) {
+	t.Parallel()
+
+	// Given
+	v := apiversion.New(2, 1).WithBackwardCompat(1)
+
+	for _, tc := range []struct {
+		name          string
+		version       string
+		expectedError string
+	}{
+		{
+			name:    "OK",
+			version: "2.1",
+		},
+		{
+			name:    "MinorOK",
+			version: "2.0",
+		},
+		{
+			name:    "MajorOK",
+			version: "1.0",
+		},
+		{
+			name:          "TooNewMinor",
+			version:       "2.2",
+			expectedError: "behind requested minor version",
+		},
+		{
+			name:          "TooNewMajor",
+			version:       "3.1",
+			expectedError: "behind requested major version",
+		},
+		{
+			name:          "Malformed0",
+			version:       "cats",
+			expectedError: "invalid version string",
+		},
+		{
+			name:          "Malformed1",
+			version:       "cats.dogs",
+			expectedError: "invalid major version",
+		},
+		{
+			name:          "Malformed2",
+			version:       "1.dogs",
+			expectedError: "invalid minor version",
+		},
+		{
+			name:          "Malformed3",
+			version:       "1.0.1",
+			expectedError: "invalid version string",
+		},
+		{
+			name:          "Malformed4",
+			version:       "11",
+			expectedError: "invalid version string",
+		},
+		{
+			name:          "TooOld",
+			version:       "0.8",
+			expectedError: "no longer supported",
+		},
+	} {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+
+			// When
+			err := v.Validate(tc.version)
+
+			// Then
+			if tc.expectedError == "" {
+				require.NoError(t, err)
+			} else {
+				require.ErrorContains(t, err, tc.expectedError)
+			}
+		})
+	}
+}

buildinfo/boring.go

@@ -0,0 +1,7 @@
+//go:build boringcrypto
+
+package buildinfo
+
+import "crypto/boring"
+
+var boringcrypto = boring.Enabled()

buildinfo/buildinfo.go

@@ -30,8 +30,15 @@ var (
 )
 
 const (
-	// develPrefix is prefixed to developer versions of the application.
-	develPrefix = "v0.0.0-devel"
+	// noVersion is the reported version when the version cannot be determined.
+	// Usually because `go build` is run instead of `make build`.
+	noVersion = "v0.0.0"
+
+	// develPreRelease is the pre-release tag for developer versions of the
+	// application. This includes CI builds. The pre-release tag should be appended
+	// to the version with a "-".
+	// Example: v0.0.0-devel
+	develPreRelease = "devel"
 )
 
 // Version returns the semantic version of the build.
@@ -45,7 +52,8 @@ func Version() string {
 		if tag == "" {
 			// This occurs when the tag hasn't been injected,
 			// like when using "go run".
-			version = develPrefix + revision
+			// <version>-<pre-release>+<revision>
+			version = fmt.Sprintf("%s-%s%s", noVersion, develPreRelease, revision)
 			return
 		}
 		version = "v" + tag
@@ -63,18 +71,23 @@ func Version() string {
 // disregarded. If it detects that either version is a developer build it
 // returns true.
 func VersionsMatch(v1, v2 string) bool {
-	// Developer versions are disregarded...hopefully they know what they are
-	// doing.
-	if strings.HasPrefix(v1, develPrefix) || strings.HasPrefix(v2, develPrefix) {
+	// If no version is attached, then it is a dev build outside of CI. The version
+	// will be disregarded... hopefully they know what they are doing.
+	if strings.Contains(v1, noVersion) || strings.Contains(v2, noVersion) {
 		return true
 	}
 
 	return semver.MajorMinor(v1) == semver.MajorMinor(v2)
 }
 
+func IsDevVersion(v string) bool {
+	return strings.Contains(v, "-"+develPreRelease)
+}
+
 // IsDev returns true if this is a development build.
+// CI builds are also considered development builds.
 func IsDev() bool {
-	return strings.HasPrefix(Version(), develPrefix)
+	return IsDevVersion(Version())
 }
 
 // IsSlim returns true if this is a slim build.
@@ -87,6 +100,10 @@ func IsAGPL() bool {
 	return strings.Contains(agpl, "t")
 }
 
+func IsBoringCrypto() bool {
+	return boringcrypto
+}
+
 // ExternalURL returns a URL referencing the current Coder version.
 // For production builds, this will link directly to a release.
 // For development builds, this will link to a commit.

buildinfo/buildinfo_test.go

@@ -7,7 +7,7 @@ import (
 	"github.com/stretchr/testify/require"
 	"golang.org/x/mod/semver"
 
-	"github.com/coder/coder/buildinfo"
+	"github.com/coder/coder/v2/buildinfo"
 )
 
 func TestBuildInfo(t *testing.T) {
@@ -57,13 +57,19 @@ func TestBuildInfo(t *testing.T) {
 				expectMatch: true,
 			},
 			// Our CI instance uses a "-devel" prerelease
-			// flag. This is not the same as a developer WIP build.
+			// flag.
 			{
-				name:        "DevelPreleaseNotIgnored",
+				name:        "DevelPreleaseMajor",
 				v1:          "v1.1.1-devel+123abac",
 				v2:          "v1.2.3",
 				expectMatch: false,
 			},
+			{
+				name:        "DevelPreleaseSame",
+				v1:          "v1.1.1-devel+123abac",
+				v2:          "v1.1.9",
+				expectMatch: true,
+			},
 			{
 				name:        "MajorMismatch",
 				v1:          "v1.2.3",

buildinfo/notboring.go

@@ -0,0 +1,5 @@
+//go:build !boringcrypto
+
+package buildinfo
+
+var boringcrypto = false

cli/agent.go

@@ -8,10 +8,10 @@ import (
 	"net/http/pprof"
 	"net/url"
 	"os"
-	"os/signal"
 	"path/filepath"
 	"runtime"
 	"strconv"
+	"strings"
 	"sync"
 	"time"
 
@@ -27,18 +27,20 @@ import (
 	"cdr.dev/slog/sloggers/sloghuman"
 	"cdr.dev/slog/sloggers/slogjson"
 	"cdr.dev/slog/sloggers/slogstackdriver"
-	"github.com/coder/coder/agent"
-	"github.com/coder/coder/agent/reaper"
-	"github.com/coder/coder/buildinfo"
-	"github.com/coder/coder/cli/clibase"
-	"github.com/coder/coder/codersdk"
-	"github.com/coder/coder/codersdk/agentsdk"
+	"github.com/coder/coder/v2/agent"
+	"github.com/coder/coder/v2/agent/agentproc"
+	"github.com/coder/coder/v2/agent/reaper"
+	"github.com/coder/coder/v2/buildinfo"
+	"github.com/coder/coder/v2/cli/clibase"
+	"github.com/coder/coder/v2/codersdk"
+	"github.com/coder/coder/v2/codersdk/agentsdk"
 )
 
 func (r *RootCmd) workspaceAgent() *clibase.Cmd {
 	var (
 		auth                string
 		logDir              string
+		scriptDataDir       string
 		pprofAddress        string
 		noReap              bool
 		sshMaxTimeout       time.Duration
@@ -115,7 +117,7 @@ func (r *RootCmd) workspaceAgent() *clibase.Cmd {
 				defer logWriter.Close()
 
 				sinks = append(sinks, sloghuman.Sink(logWriter))
-				logger := slog.Make(sinks...).Leveled(slog.LevelDebug)
+				logger := inv.Logger.AppendSinks(sinks...).Leveled(slog.LevelDebug)
 
 				logger.Info(ctx, "spawning reaper process")
 				// Do not start a reaper on the child process. It's important
@@ -123,7 +125,7 @@ func (r *RootCmd) workspaceAgent() *clibase.Cmd {
 				args := append(os.Args, "--no-reap")
 				err := reaper.ForkReap(
 					reaper.WithExecArgs(args...),
-					reaper.WithCatchSignals(InterruptSignals...),
+					reaper.WithCatchSignals(StopSignals...),
 				)
 				if err != nil {
 					logger.Error(ctx, "agent process reaper unable to fork", slog.Error(err))
@@ -142,23 +144,24 @@ func (r *RootCmd) workspaceAgent() *clibase.Cmd {
 			// Note that we don't want to handle these signals in the
 			// process that runs as PID 1, that's why we do this after
 			// the reaper forked.
-			ctx, stopNotify := signal.NotifyContext(ctx, InterruptSignals...)
+			ctx, stopNotify := inv.SignalNotifyContext(ctx, StopSignals...)
 			defer stopNotify()
 
 			// DumpHandler does signal handling, so we call it after the
 			// reaper.
-			go DumpHandler(ctx)
+			go DumpHandler(ctx, "agent")
 
 			logWriter := &lumberjackWriteCloseFixer{w: &lumberjack.Logger{
 				Filename: filepath.Join(logDir, "coder-agent.log"),
 				MaxSize:  5, // MB
-				// Without this, rotated logs will never be deleted.
-				MaxBackups: 1,
+				// Per customer incident on November 17th, 2023, its helpful
+				// to have the log of the last few restarts to debug a failing agent.
+				MaxBackups: 10,
 			}}
 			defer logWriter.Close()
 
 			sinks = append(sinks, sloghuman.Sink(logWriter))
-			logger := slog.Make(sinks...).Leveled(slog.LevelDebug)
+			logger := inv.Logger.AppendSinks(sinks...).Leveled(slog.LevelDebug)
 
 			version := buildinfo.Version()
 			logger.Info(ctx, "agent is starting now",
@@ -197,9 +200,19 @@ func (r *RootCmd) workspaceAgent() *clibase.Cmd {
 			var exchangeToken func(context.Context) (agentsdk.AuthenticateResponse, error)
 			switch auth {
 			case "token":
-				token, err := inv.ParsedFlags().GetString(varAgentToken)
-				if err != nil {
-					return xerrors.Errorf("CODER_AGENT_TOKEN must be set for token auth: %w", err)
+				token, _ := inv.ParsedFlags().GetString(varAgentToken)
+				if token == "" {
+					tokenFile, _ := inv.ParsedFlags().GetString(varAgentTokenFile)
+					if tokenFile != "" {
+						tokenBytes, err := os.ReadFile(tokenFile)
+						if err != nil {
+							return xerrors.Errorf("read token file %q: %w", tokenFile, err)
+						}
+						token = strings.TrimSpace(string(tokenBytes))
+					}
+				}
+				if token == "" {
+					return xerrors.Errorf("CODER_AGENT_TOKEN or CODER_AGENT_TOKEN_FILE must be set for token auth")
 				}
 				client.SetSessionToken(token)
 			case "google-instance-identity":
@@ -253,11 +266,31 @@ func (r *RootCmd) workspaceAgent() *clibase.Cmd {
 			}
 
 			prometheusRegistry := prometheus.NewRegistry()
-			subsystem := inv.Environ.Get(agent.EnvAgentSubsystem)
+			subsystemsRaw := inv.Environ.Get(agent.EnvAgentSubsystem)
+			subsystems := []codersdk.AgentSubsystem{}
+			for _, s := range strings.Split(subsystemsRaw, ",") {
+				subsystem := codersdk.AgentSubsystem(strings.TrimSpace(s))
+				if subsystem == "" {
+					continue
+				}
+				if !subsystem.Valid() {
+					return xerrors.Errorf("invalid subsystem %q", subsystem)
+				}
+				subsystems = append(subsystems, subsystem)
+			}
+
+			environmentVariables := map[string]string{
+				"GIT_ASKPASS": executablePath,
+			}
+			if v, ok := os.LookupEnv(agent.EnvProcPrioMgmt); ok {
+				environmentVariables[agent.EnvProcPrioMgmt] = v
+			}
+
 			agnt := agent.New(agent.Options{
 				Client:            client,
 				Logger:            logger,
 				LogDir:            logDir,
+				ScriptDataDir:     scriptDataDir,
 				TailnetListenPort: uint16(tailnetListenPort),
 				ExchangeToken: func(ctx context.Context) (string, error) {
 					if exchangeToken == nil {
@@ -270,14 +303,16 @@ func (r *RootCmd) workspaceAgent() *clibase.Cmd {
 					client.SetSessionToken(resp.SessionToken)
 					return resp.SessionToken, nil
 				},
-				EnvironmentVariables: map[string]string{
-					"GIT_ASKPASS": executablePath,
-				},
-				IgnorePorts:   ignorePorts,
-				SSHMaxTimeout: sshMaxTimeout,
-				Subsystem:     codersdk.AgentSubsystem(subsystem),
+				EnvironmentVariables: environmentVariables,
+				IgnorePorts:          ignorePorts,
+				SSHMaxTimeout:        sshMaxTimeout,
+				Subsystems:           subsystems,
 
 				PrometheusRegistry: prometheusRegistry,
+				Syscaller:          agentproc.NewSyscaller(),
+				// Intentionally set this to nil. It's mainly used
+				// for testing.
+				ModifiedProcesses: nil,
 			})
 
 			prometheusSrvClose := ServeHandler(ctx, logger, prometheusMetricsHandler(prometheusRegistry, logger), prometheusAddress, "prometheus")
@@ -306,6 +341,13 @@ func (r *RootCmd) workspaceAgent() *clibase.Cmd {
 			Env:         "CODER_AGENT_LOG_DIR",
 			Value:       clibase.StringOf(&logDir),
 		},
+		{
+			Flag:        "script-data-dir",
+			Default:     os.TempDir(),
+			Description: "Specify the location for storing script data.",
+			Env:         "CODER_AGENT_SCRIPT_DATA_DIR",
+			Value:       clibase.StringOf(&scriptDataDir),
+		},
 		{
 			Flag:        "pprof-address",
 			Default:     "127.0.0.1:6060",

cli/agent_test.go

@@ -2,6 +2,7 @@ package cli_test
 
 import (
 	"context"
+	"fmt"
 	"os"
 	"path/filepath"
 	"runtime"
@@ -12,13 +13,14 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
-	"github.com/coder/coder/agent"
-	"github.com/coder/coder/cli/clitest"
-	"github.com/coder/coder/coderd/coderdtest"
-	"github.com/coder/coder/codersdk"
-	"github.com/coder/coder/provisioner/echo"
-	"github.com/coder/coder/provisionersdk/proto"
-	"github.com/coder/coder/pty/ptytest"
+	"github.com/coder/coder/v2/agent"
+	"github.com/coder/coder/v2/cli/clitest"
+	"github.com/coder/coder/v2/coderd/coderdtest"
+	"github.com/coder/coder/v2/coderd/database"
+	"github.com/coder/coder/v2/coderd/database/dbfake"
+	"github.com/coder/coder/v2/codersdk"
+	"github.com/coder/coder/v2/provisionersdk/proto"
+	"github.com/coder/coder/v2/testutil"
 )
 
 func TestWorkspaceAgent(t *testing.T) {
@@ -27,83 +29,63 @@ func TestWorkspaceAgent(t *testing.T) {
 	t.Run("LogDirectory", func(t *testing.T) {
 		t.Parallel()
 
-		authToken := uuid.NewString()
-		client := coderdtest.New(t, &coderdtest.Options{
-			IncludeProvisionerDaemon: true,
-		})
+		client, db := coderdtest.NewWithDatabase(t, nil)
 		user := coderdtest.CreateFirstUser(t, client)
-		version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, &echo.Responses{
-			Parse:          echo.ParseComplete,
-			ProvisionApply: echo.ProvisionApplyWithAgent(authToken),
-		})
-		template := coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID)
-		coderdtest.AwaitTemplateVersionJob(t, client, version.ID)
-		workspace := coderdtest.CreateWorkspace(t, client, user.OrganizationID, template.ID)
-		coderdtest.AwaitWorkspaceBuildJob(t, client, workspace.LatestBuild.ID)
-
+		r := dbfake.WorkspaceBuild(t, db, database.Workspace{
+			OrganizationID: user.OrganizationID,
+			OwnerID:        user.UserID,
+		}).
+			WithAgent().
+			Do()
 		logDir := t.TempDir()
 		inv, _ := clitest.New(t,
 			"agent",
 			"--auth", "token",
-			"--agent-token", authToken,
+			"--agent-token", r.AgentToken,
 			"--agent-url", client.URL.String(),
 			"--log-dir", logDir,
 		)
 
-		pty := ptytest.New(t).Attach(inv)
-
 		clitest.Start(t, inv)
-		ctx := inv.Context()
-		pty.ExpectMatchContext(ctx, "agent is starting now")
 
-		coderdtest.AwaitWorkspaceAgents(t, client, workspace.ID)
+		coderdtest.AwaitWorkspaceAgents(t, client, r.Workspace.ID)
 
-		info, err := os.Stat(filepath.Join(logDir, "coder-agent.log"))
-		require.NoError(t, err)
-		require.Greater(t, info.Size(), int64(0))
+		require.Eventually(t, func() bool {
+			info, err := os.Stat(filepath.Join(logDir, "coder-agent.log"))
+			if err != nil {
+				return false
+			}
+			return info.Size() > 0
+		}, testutil.WaitLong, testutil.IntervalMedium)
 	})
 
 	t.Run("Azure", func(t *testing.T) {
 		t.Parallel()
 		instanceID := "instanceidentifier"
 		certificates, metadataClient := coderdtest.NewAzureInstanceIdentity(t, instanceID)
-		client := coderdtest.New(t, &coderdtest.Options{
-			AzureCertificates:        certificates,
-			IncludeProvisionerDaemon: true,
+		client, db := coderdtest.NewWithDatabase(t, &coderdtest.Options{
+			AzureCertificates: certificates,
 		})
 		user := coderdtest.CreateFirstUser(t, client)
-		version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, &echo.Responses{
-			Parse: echo.ParseComplete,
-			ProvisionApply: []*proto.Provision_Response{{
-				Type: &proto.Provision_Response_Complete{
-					Complete: &proto.Provision_Complete{
-						Resources: []*proto.Resource{{
-							Name: "somename",
-							Type: "someinstance",
-							Agents: []*proto.Agent{{
-								Auth: &proto.Agent_InstanceId{
-									InstanceId: instanceID,
-								},
-							}},
-						}},
-					},
-				},
-			}},
-		})
-		template := coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID)
-		coderdtest.AwaitTemplateVersionJob(t, client, version.ID)
-		workspace := coderdtest.CreateWorkspace(t, client, user.OrganizationID, template.ID)
-		coderdtest.AwaitWorkspaceBuildJob(t, client, workspace.LatestBuild.ID)
+		r := dbfake.WorkspaceBuild(t, db, database.Workspace{
+			OrganizationID: user.OrganizationID,
+			OwnerID:        user.UserID,
+		}).WithAgent(func(agents []*proto.Agent) []*proto.Agent {
+			agents[0].Auth = &proto.Agent_InstanceId{InstanceId: instanceID}
+			return agents
+		}).Do()
 
 		inv, _ := clitest.New(t, "agent", "--auth", "azure-instance-identity", "--agent-url", client.URL.String())
 		inv = inv.WithContext(
 			//nolint:revive,staticcheck
 			context.WithValue(inv.Context(), "azure-client", metadataClient),
 		)
+
 		ctx := inv.Context()
 		clitest.Start(t, inv)
-		coderdtest.AwaitWorkspaceAgents(t, client, workspace.ID)
-		workspace, err := client.Workspace(ctx, workspace.ID)
+		coderdtest.NewWorkspaceAgentWaiter(t, client, r.Workspace.ID).
+			MatchResources(matchAgentWithVersion).Wait()
+		workspace, err := client.Workspace(ctx, r.Workspace.ID)
 		require.NoError(t, err)
 		resources := workspace.LatestBuild.Resources
 		if assert.NotEmpty(t, workspace.LatestBuild.Resources) && assert.NotEmpty(t, resources[0].Agents) {
@@ -119,43 +101,30 @@ func TestWorkspaceAgent(t *testing.T) {
 		t.Parallel()
 		instanceID := "instanceidentifier"
 		certificates, metadataClient := coderdtest.NewAWSInstanceIdentity(t, instanceID)
-		client := coderdtest.New(t, &coderdtest.Options{
-			AWSCertificates:          certificates,
-			IncludeProvisionerDaemon: true,
+		client, db := coderdtest.NewWithDatabase(t, &coderdtest.Options{
+			AWSCertificates: certificates,
 		})
 		user := coderdtest.CreateFirstUser(t, client)
-		version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, &echo.Responses{
-			Parse: echo.ParseComplete,
-			ProvisionApply: []*proto.Provision_Response{{
-				Type: &proto.Provision_Response_Complete{
-					Complete: &proto.Provision_Complete{
-						Resources: []*proto.Resource{{
-							Name: "somename",
-							Type: "someinstance",
-							Agents: []*proto.Agent{{
-								Auth: &proto.Agent_InstanceId{
-									InstanceId: instanceID,
-								},
-							}},
-						}},
-					},
-				},
-			}},
-		})
-		template := coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID)
-		coderdtest.AwaitTemplateVersionJob(t, client, version.ID)
-		workspace := coderdtest.CreateWorkspace(t, client, user.OrganizationID, template.ID)
-		coderdtest.AwaitWorkspaceBuildJob(t, client, workspace.LatestBuild.ID)
+		r := dbfake.WorkspaceBuild(t, db, database.Workspace{
+			OrganizationID: user.OrganizationID,
+			OwnerID:        user.UserID,
+		}).WithAgent(func(agents []*proto.Agent) []*proto.Agent {
+			agents[0].Auth = &proto.Agent_InstanceId{InstanceId: instanceID}
+			return agents
+		}).Do()
 
 		inv, _ := clitest.New(t, "agent", "--auth", "aws-instance-identity", "--agent-url", client.URL.String())
 		inv = inv.WithContext(
 			//nolint:revive,staticcheck
 			context.WithValue(inv.Context(), "aws-client", metadataClient),
 		)
+
 		clitest.Start(t, inv)
 		ctx := inv.Context()
-		coderdtest.AwaitWorkspaceAgents(t, client, workspace.ID)
-		workspace, err := client.Workspace(ctx, workspace.ID)
+		coderdtest.NewWorkspaceAgentWaiter(t, client, r.Workspace.ID).
+			MatchResources(matchAgentWithVersion).
+			Wait()
+		workspace, err := client.Workspace(ctx, r.Workspace.ID)
 		require.NoError(t, err)
 		resources := workspace.LatestBuild.Resources
 		if assert.NotEmpty(t, resources) && assert.NotEmpty(t, resources[0].Agents) {
@@ -171,37 +140,22 @@ func TestWorkspaceAgent(t *testing.T) {
 		t.Parallel()
 		instanceID := "instanceidentifier"
 		validator, metadataClient := coderdtest.NewGoogleInstanceIdentity(t, instanceID, false)
-		client := coderdtest.New(t, &coderdtest.Options{
-			GoogleTokenValidator:     validator,
-			IncludeProvisionerDaemon: true,
+		client, db := coderdtest.NewWithDatabase(t, &coderdtest.Options{
+			GoogleTokenValidator: validator,
 		})
-		user := coderdtest.CreateFirstUser(t, client)
-		version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, &echo.Responses{
-			Parse: echo.ParseComplete,
-			ProvisionApply: []*proto.Provision_Response{{
-				Type: &proto.Provision_Response_Complete{
-					Complete: &proto.Provision_Complete{
-						Resources: []*proto.Resource{{
-							Name: "somename",
-							Type: "someinstance",
-							Agents: []*proto.Agent{{
-								Auth: &proto.Agent_InstanceId{
-									InstanceId: instanceID,
-								},
-							}},
-						}},
-					},
-				},
-			}},
-		})
-		template := coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID)
-		coderdtest.AwaitTemplateVersionJob(t, client, version.ID)
-		workspace := coderdtest.CreateWorkspace(t, client, user.OrganizationID, template.ID)
-		coderdtest.AwaitWorkspaceBuildJob(t, client, workspace.LatestBuild.ID)
+		owner := coderdtest.CreateFirstUser(t, client)
+		member, memberUser := coderdtest.CreateAnotherUser(t, client, owner.OrganizationID)
+		r := dbfake.WorkspaceBuild(t, db, database.Workspace{
+			OrganizationID: owner.OrganizationID,
+			OwnerID:        memberUser.ID,
+		}).WithAgent(func(agents []*proto.Agent) []*proto.Agent {
+			agents[0].Auth = &proto.Agent_InstanceId{InstanceId: instanceID}
+			return agents
+		}).Do()
 
 		inv, cfg := clitest.New(t, "agent", "--auth", "google-instance-identity", "--agent-url", client.URL.String())
-		ptytest.New(t).Attach(inv)
-		clitest.SetupConfig(t, client, cfg)
+		clitest.SetupConfig(t, member, cfg)
+
 		clitest.Start(t,
 			inv.WithContext(
 				//nolint:revive,staticcheck
@@ -210,9 +164,10 @@ func TestWorkspaceAgent(t *testing.T) {
 		)
 
 		ctx := inv.Context()
-
-		coderdtest.AwaitWorkspaceAgents(t, client, workspace.ID)
-		workspace, err := client.Workspace(ctx, workspace.ID)
+		coderdtest.NewWorkspaceAgentWaiter(t, client, r.Workspace.ID).
+			MatchResources(matchAgentWithVersion).
+			Wait()
+		workspace, err := client.Workspace(ctx, r.Workspace.ID)
 		require.NoError(t, err)
 		resources := workspace.LatestBuild.Resources
 		if assert.NotEmpty(t, resources) && assert.NotEmpty(t, resources[0].Agents) {
@@ -242,39 +197,59 @@ func TestWorkspaceAgent(t *testing.T) {
 	t.Run("PostStartup", func(t *testing.T) {
 		t.Parallel()
 
-		authToken := uuid.NewString()
-		client := coderdtest.New(t, &coderdtest.Options{
-			IncludeProvisionerDaemon: true,
-		})
+		client, db := coderdtest.NewWithDatabase(t, nil)
 		user := coderdtest.CreateFirstUser(t, client)
-		version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, &echo.Responses{
-			Parse:          echo.ParseComplete,
-			ProvisionApply: echo.ProvisionApplyWithAgent(authToken),
-		})
-		template := coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID)
-		coderdtest.AwaitTemplateVersionJob(t, client, version.ID)
-		workspace := coderdtest.CreateWorkspace(t, client, user.OrganizationID, template.ID)
-		coderdtest.AwaitWorkspaceBuildJob(t, client, workspace.LatestBuild.ID)
+		r := dbfake.WorkspaceBuild(t, db, database.Workspace{
+			OrganizationID: user.OrganizationID,
+			OwnerID:        user.UserID,
+		}).WithAgent().Do()
 
 		logDir := t.TempDir()
 		inv, _ := clitest.New(t,
 			"agent",
 			"--auth", "token",
-			"--agent-token", authToken,
+			"--agent-token", r.AgentToken,
 			"--agent-url", client.URL.String(),
 			"--log-dir", logDir,
 		)
-		// Set the subsystem for the agent.
-		inv.Environ.Set(agent.EnvAgentSubsystem, string(codersdk.AgentSubsystemEnvbox))
-
-		pty := ptytest.New(t).Attach(inv)
+		// Set the subsystems for the agent.
+		inv.Environ.Set(agent.EnvAgentSubsystem, fmt.Sprintf("%s,%s", codersdk.AgentSubsystemExectrace, codersdk.AgentSubsystemEnvbox))
 
 		clitest.Start(t, inv)
-		pty.ExpectMatchContext(inv.Context(), "agent is starting now")
 
-		resources := coderdtest.AwaitWorkspaceAgents(t, client, workspace.ID)
+		resources := coderdtest.NewWorkspaceAgentWaiter(t, client, r.Workspace.ID).
+			MatchResources(matchAgentWithSubsystems).Wait()
 		require.Len(t, resources, 1)
 		require.Len(t, resources[0].Agents, 1)
-		require.Equal(t, codersdk.AgentSubsystemEnvbox, resources[0].Agents[0].Subsystem)
+		require.Len(t, resources[0].Agents[0].Subsystems, 2)
+		// Sorted
+		require.Equal(t, codersdk.AgentSubsystemEnvbox, resources[0].Agents[0].Subsystems[0])
+		require.Equal(t, codersdk.AgentSubsystemExectrace, resources[0].Agents[0].Subsystems[1])
 	})
 }
+
+func matchAgentWithVersion(rs []codersdk.WorkspaceResource) bool {
+	if len(rs) < 1 {
+		return false
+	}
+	if len(rs[0].Agents) < 1 {
+		return false
+	}
+	if rs[0].Agents[0].Version == "" {
+		return false
+	}
+	return true
+}
+
+func matchAgentWithSubsystems(rs []codersdk.WorkspaceResource) bool {
+	if len(rs) < 1 {
+		return false
+	}
+	if len(rs[0].Agents) < 1 {
+		return false
+	}
+	if len(rs[0].Agents[0].Subsystems) < 1 {
+		return false
+	}
+	return true
+}

cli/autoupdate.go

@@ -0,0 +1,58 @@
+package cli
+
+import (
+	"fmt"
+	"strings"
+
+	"golang.org/x/xerrors"
+
+	"github.com/coder/coder/v2/cli/clibase"
+	"github.com/coder/coder/v2/cli/cliui"
+	"github.com/coder/coder/v2/codersdk"
+)
+
+func (r *RootCmd) autoupdate() *clibase.Cmd {
+	client := new(codersdk.Client)
+	cmd := &clibase.Cmd{
+		Annotations: workspaceCommand,
+		Use:         "autoupdate <workspace> <always|never>",
+		Short:       "Toggle auto-update policy for a workspace",
+		Middleware: clibase.Chain(
+			clibase.RequireNArgs(2),
+			r.InitClient(client),
+		),
+		Handler: func(inv *clibase.Invocation) error {
+			policy := strings.ToLower(inv.Args[1])
+			err := validateAutoUpdatePolicy(policy)
+			if err != nil {
+				return xerrors.Errorf("validate policy: %w", err)
+			}
+
+			workspace, err := namedWorkspace(inv.Context(), client, inv.Args[0])
+			if err != nil {
+				return xerrors.Errorf("get workspace: %w", err)
+			}
+
+			err = client.UpdateWorkspaceAutomaticUpdates(inv.Context(), workspace.ID, codersdk.UpdateWorkspaceAutomaticUpdatesRequest{
+				AutomaticUpdates: codersdk.AutomaticUpdates(policy),
+			})
+			if err != nil {
+				return xerrors.Errorf("update workspace automatic updates policy: %w", err)
+			}
+			_, _ = fmt.Fprintf(inv.Stdout, "Updated workspace %q auto-update policy to %q\n", workspace.Name, policy)
+			return nil
+		},
+	}
+
+	cmd.Options = append(cmd.Options, cliui.SkipPromptOption())
+	return cmd
+}
+
+func validateAutoUpdatePolicy(arg string) error {
+	switch codersdk.AutomaticUpdates(arg) {
+	case codersdk.AutomaticUpdatesAlways, codersdk.AutomaticUpdatesNever:
+		return nil
+	default:
+		return xerrors.Errorf("invalid option %q must be either of %q or %q", arg, codersdk.AutomaticUpdatesAlways, codersdk.AutomaticUpdatesNever)
+	}
+}