2.9.4 changelog

chore: add generate script for azure instance identity (#13028 )
* chore: add generate script for azure instance identity This also adds new issuing certificates from: https://learn.microsoft.com/en-us/azure/security/fundamentals/azure-ca-details?tabs=certificate-authority-chains * Fix shell lint * Fix shell fmt * Fix RSA issuing certificate
2024-04-22 20:10:30 +00:00 · 2024-04-22 19:42:51 +00:00 · 2024-04-17 22:21:07 +00:00 · 2024-04-17 21:27:46 +00:00 · 2024-04-17 21:05:38 +00:00 · 2024-04-17 21:02:58 +00:00
38 changed files with 818 additions and 87 deletions
@@ -125,7 +125,7 @@ func (r *RootCmd) workspaceAgent() *clibase.Cmd {
 				args := append(os.Args, "--no-reap")
 				err := reaper.ForkReap(
 					reaper.WithExecArgs(args...),
-					reaper.WithCatchSignals(InterruptSignals...),
+					reaper.WithCatchSignals(StopSignals...),
 				)
 				if err != nil {
 					logger.Error(ctx, "agent process reaper unable to fork", slog.Error(err))
@@ -144,7 +144,7 @@ func (r *RootCmd) workspaceAgent() *clibase.Cmd {
 			// Note that we don't want to handle these signals in the
 			// process that runs as PID 1, that's why we do this after
 			// the reaper forked.
-			ctx, stopNotify := inv.SignalNotifyContext(ctx, InterruptSignals...)
+			ctx, stopNotify := inv.SignalNotifyContext(ctx, StopSignals...)
 			defer stopNotify()

 			// DumpHandler does signal handling, so we call it after the
@@ -890,7 +890,7 @@ func (r *RootCmd) scaletestWorkspaceTraffic() *clibase.Cmd {
 		Handler: func(inv *clibase.Invocation) (err error) {
 			ctx := inv.Context()

-			notifyCtx, stop := signal.NotifyContext(ctx, InterruptSignals...) // Checked later.
+			notifyCtx, stop := signal.NotifyContext(ctx, StopSignals...) // Checked later.
 			defer stop()
 			ctx = notifyCtx

@@ -65,7 +65,7 @@ fi
 		Handler: func(inv *clibase.Invocation) error {
 			ctx := inv.Context()

-			ctx, stop := inv.SignalNotifyContext(ctx, InterruptSignals...)
+			ctx, stop := inv.SignalNotifyContext(ctx, StopSignals...)
 			defer stop()

 			client, err := r.createAgentClient()
@@ -25,7 +25,7 @@ func (r *RootCmd) gitAskpass() *clibase.Cmd {
 		Handler: func(inv *clibase.Invocation) error {
 			ctx := inv.Context()

-			ctx, stop := inv.SignalNotifyContext(ctx, InterruptSignals...)
+			ctx, stop := inv.SignalNotifyContext(ctx, StopSignals...)
 			defer stop()

 			user, host, err := gitauth.ParseAskpass(inv.Args[0])
@@ -29,7 +29,7 @@ func (r *RootCmd) gitssh() *clibase.Cmd {

 			// Catch interrupt signals to ensure the temporary private
 			// key file is cleaned up on most cases.
-			ctx, stop := inv.SignalNotifyContext(ctx, InterruptSignals...)
+			ctx, stop := inv.SignalNotifyContext(ctx, StopSignals...)
 			defer stop()

 			// Early check so errors are reported immediately.
@@ -337,7 +337,7 @@ func (r *RootCmd) Server(newAPI func(context.Context, *coderd.Options) (*coderd.

 			// Register signals early on so that graceful shutdown can't
 			// be interrupted by additional signals. Note that we avoid
-			// shadowing cancel() (from above) here because notifyStop()
+			// shadowing cancel() (from above) here because stopCancel()
 			// restores default behavior for the signals. This protects
 			// the shutdown sequence from abruptly terminating things
 			// like: database migrations, provisioner work, workspace
@@ -345,8 +345,10 @@ func (r *RootCmd) Server(newAPI func(context.Context, *coderd.Options) (*coderd.
 			//
 			// To get out of a graceful shutdown, the user can send
 			// SIGQUIT with ctrl+\ or SIGKILL with `kill -9`.
-			notifyCtx, notifyStop := inv.SignalNotifyContext(ctx, InterruptSignals...)
-			defer notifyStop()
+			stopCtx, stopCancel := signalNotifyContext(ctx, inv, StopSignalsNoInterrupt...)
+			defer stopCancel()
+			interruptCtx, interruptCancel := signalNotifyContext(ctx, inv, InterruptSignals...)
+			defer interruptCancel()

 			cacheDir := vals.CacheDir.String()
 			err = os.MkdirAll(cacheDir, 0o700)
@@ -1028,13 +1030,18 @@ func (r *RootCmd) Server(newAPI func(context.Context, *coderd.Options) (*coderd.
 			hangDetector.Start()
 			defer hangDetector.Close()

+			waitForProvisionerJobs := false
 			// Currently there is no way to ask the server to shut
 			// itself down, so any exit signal will result in a non-zero
 			// exit of the server.
 			var exitErr error
 			select {
-			case <-notifyCtx.Done():
-				exitErr = notifyCtx.Err()
+			case <-stopCtx.Done():
+				exitErr = stopCtx.Err()
+				waitForProvisionerJobs = true
+				_, _ = io.WriteString(inv.Stdout, cliui.Bold("Stop caught, waiting for provisioner jobs to complete and gracefully exiting. Use ctrl+\\ to force quit"))
+			case <-interruptCtx.Done():
+				exitErr = interruptCtx.Err()
 				_, _ = io.WriteString(inv.Stdout, cliui.Bold("Interrupt caught, gracefully exiting. Use ctrl+\\ to force quit"))
 			case <-tunnelDone:
 				exitErr = xerrors.New("dev tunnel closed unexpectedly")
@@ -1082,7 +1089,16 @@ func (r *RootCmd) Server(newAPI func(context.Context, *coderd.Options) (*coderd.
 					defer wg.Done()

 					r.Verbosef(inv, "Shutting down provisioner daemon %d...", id)
-					err := shutdownWithTimeout(provisionerDaemon.Shutdown, 5*time.Second)
+					timeout := 5 * time.Second
+					if waitForProvisionerJobs {
+						// It can last for a long time...
+						timeout = 30 * time.Minute
+					}
+
+					err := shutdownWithTimeout(func(ctx context.Context) error {
+						// We only want to cancel active jobs if we aren't exiting gracefully.
+						return provisionerDaemon.Shutdown(ctx, !waitForProvisionerJobs)
+					}, timeout)
 					if err != nil {
 						cliui.Errorf(inv.Stderr, "Failed to shut down provisioner daemon %d: %s\n", id, err)
 						return
@@ -2512,3 +2528,12 @@ func escapePostgresURLUserInfo(v string) (string, error) {

 	return v, nil
 }
+
+func signalNotifyContext(ctx context.Context, inv *clibase.Invocation, sig ...os.Signal) (context.Context, context.CancelFunc) {
+	// On Windows, some of our signal functions lack support.
+	// If we pass in no signals, we should just return the context as-is.
+	if len(sig) == 0 {
+		return context.WithCancel(ctx)
+	}
+	return inv.SignalNotifyContext(ctx, sig...)
+}
@@ -47,7 +47,7 @@ func (r *RootCmd) newCreateAdminUserCommand() *clibase.Cmd {
 				logger = logger.Leveled(slog.LevelDebug)
 			}

-			ctx, cancel := inv.SignalNotifyContext(ctx, InterruptSignals...)
+			ctx, cancel := inv.SignalNotifyContext(ctx, StopSignals...)
 			defer cancel()

 			if newUserDBURL == "" {
@@ -21,6 +21,7 @@ import (
 	"net/url"
 	"os"
 	"path/filepath"
+	"reflect"
 	"runtime"
 	"strconv"
 	"strings"
@@ -1605,7 +1606,7 @@ func TestServer_Production(t *testing.T) {
 }

 //nolint:tparallel,paralleltest // This test cannot be run in parallel due to signal handling.
-func TestServer_Shutdown(t *testing.T) {
+func TestServer_InterruptShutdown(t *testing.T) {
 	t.Skip("This test issues an interrupt signal which will propagate to the test runner.")

 	if runtime.GOOS == "windows" {
@@ -1638,6 +1639,46 @@ func TestServer_Shutdown(t *testing.T) {
 	require.NoError(t, err)
 }

+func TestServer_GracefulShutdown(t *testing.T) {
+	t.Parallel()
+	if runtime.GOOS == "windows" {
+		// Sending interrupt signal isn't supported on Windows!
+		t.SkipNow()
+	}
+	ctx, cancelFunc := context.WithCancel(context.Background())
+	defer cancelFunc()
+
+	root, cfg := clitest.New(t,
+		"server",
+		"--in-memory",
+		"--http-address", ":0",
+		"--access-url", "http://example.com",
+		"--provisioner-daemons", "1",
+		"--cache-dir", t.TempDir(),
+	)
+	var stopFunc context.CancelFunc
+	root = root.WithTestSignalNotifyContext(t, func(parent context.Context, signals ...os.Signal) (context.Context, context.CancelFunc) {
+		if !reflect.DeepEqual(cli.StopSignalsNoInterrupt, signals) {
+			return context.WithCancel(ctx)
+		}
+		var ctx context.Context
+		ctx, stopFunc = context.WithCancel(parent)
+		return ctx, stopFunc
+	})
+	serverErr := make(chan error, 1)
+	pty := ptytest.New(t).Attach(root)
+	go func() {
+		serverErr <- root.WithContext(ctx).Run()
+	}()
+	_ = waitAccessURL(t, cfg)
+	// It's fair to assume `stopFunc` isn't nil here, because the server
+	// has started and access URL is propagated.
+	stopFunc()
+	pty.ExpectMatch("waiting for provisioner jobs to complete")
+	err := <-serverErr
+	require.NoError(t, err)
+}
+
 func BenchmarkServerHelp(b *testing.B) {
 	// server --help is a good proxy for measuring the
 	// constant overhead of each command.
@@ -7,8 +7,23 @@ import (
 	"syscall"
 )

-var InterruptSignals = []os.Signal{
+// StopSignals is the list of signals that are used for handling
+// shutdown behavior.
+var StopSignals = []os.Signal{
 	os.Interrupt,
 	syscall.SIGTERM,
 	syscall.SIGHUP,
 }
+
+// StopSignals is the list of signals that are used for handling
+// graceful shutdown behavior.
+var StopSignalsNoInterrupt = []os.Signal{
+	syscall.SIGTERM,
+	syscall.SIGHUP,
+}
+
+// InterruptSignals is the list of signals that are used for handling
+// immediate shutdown behavior.
+var InterruptSignals = []os.Signal{
+	os.Interrupt,
+}
@@ -6,4 +6,12 @@ import (
 	"os"
 )

-var InterruptSignals = []os.Signal{os.Interrupt}
+var StopSignals = []os.Signal{
+	os.Interrupt,
+}
+
+var StopSignalsNoInterrupt = []os.Signal{}
+
+var InterruptSignals = []os.Signal{
+	os.Interrupt,
+}
@@ -25,11 +25,8 @@ import (
 	"golang.org/x/xerrors"
 	"gvisor.dev/gvisor/pkg/tcpip/adapters/gonet"

-	"github.com/coder/retry"
-
 	"cdr.dev/slog"
 	"cdr.dev/slog/sloggers/sloghuman"
-
 	"github.com/coder/coder/v2/cli/clibase"
 	"github.com/coder/coder/v2/cli/cliui"
 	"github.com/coder/coder/v2/cli/cliutil"
@@ -37,6 +34,8 @@ import (
 	"github.com/coder/coder/v2/coderd/util/ptr"
 	"github.com/coder/coder/v2/codersdk"
 	"github.com/coder/coder/v2/cryptorand"
+	"github.com/coder/coder/v2/pty"
+	"github.com/coder/retry"
 )

 var (
@@ -73,7 +72,7 @@ func (r *RootCmd) ssh() *clibase.Cmd {
 			// session can persist for up to 72 hours, since we set a long
 			// timeout on the Agent side of the connection.  In particular,
 			// OpenSSH sends SIGHUP to terminate a proxy command.
-			ctx, stop := inv.SignalNotifyContext(inv.Context(), InterruptSignals...)
+			ctx, stop := inv.SignalNotifyContext(inv.Context(), StopSignals...)
 			defer stop()
 			ctx, cancel := context.WithCancel(ctx)
 			defer cancel()
@@ -339,15 +338,22 @@ func (r *RootCmd) ssh() *clibase.Cmd {
 				}
 			}

-			stdoutFile, validOut := inv.Stdout.(*os.File)
 			stdinFile, validIn := inv.Stdin.(*os.File)
-			if validOut && validIn && isatty.IsTerminal(stdoutFile.Fd()) {
-				state, err := term.MakeRaw(int(stdinFile.Fd()))
+			stdoutFile, validOut := inv.Stdout.(*os.File)
+			if validIn && validOut && isatty.IsTerminal(stdinFile.Fd()) && isatty.IsTerminal(stdoutFile.Fd()) {
+				inState, err := pty.MakeInputRaw(stdinFile.Fd())
 				if err != nil {
 					return err
 				}
 				defer func() {
-					_ = term.Restore(int(stdinFile.Fd()), state)
+					_ = pty.RestoreTerminal(stdinFile.Fd(), inState)
+				}()
+				outState, err := pty.MakeOutputRaw(stdoutFile.Fd())
+				if err != nil {
+					return err
+				}
+				defer func() {
+					_ = pty.RestoreTerminal(stdoutFile.Fd(), outState)
 				}()

 				windowChange := listenWindowSize(ctx)
@@ -185,6 +185,142 @@ QYLbNYkedkNuhRmEBesPqj4aDz68ZDI6fJ92sj2q18QvJUJ5Qz728AvtFOat+Ajg
 K0PFqPYEAviUKr162NB1XZJxf6uyIjUlnG4UEdHfUqdhl0R84mMtrYINksTzQ2sH
 YM8fEhqICtTlcRLr/FErUaPUe9648nziSnA0qKH7rUZqP/Ifmbo+WNZSZG1BbgOh
 lk+521W+Ncih3HRbvRBE0LWYT8vWKnfjgZKxwHwJ
+-----END CERTIFICATE-----`,
+	// Microsoft Azure RSA TLS Issuing CA 03
+	`-----BEGIN CERTIFICATE-----
+MIIFrDCCBJSgAwIBAgIQBRllJkSaXj0aOHSPXc/rzDANBgkqhkiG9w0BAQwFADBh
+MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3
+d3cuZGlnaWNlcnQuY29tMSAwHgYDVQQDExdEaWdpQ2VydCBHbG9iYWwgUm9vdCBH
+MjAeFw0yMzA2MDgwMDAwMDBaFw0yNjA4MjUyMzU5NTlaMF0xCzAJBgNVBAYTAlVT
+MR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xLjAsBgNVBAMTJU1pY3Jv
+c29mdCBBenVyZSBSU0EgVExTIElzc3VpbmcgQ0EgMDMwggIiMA0GCSqGSIb3DQEB
+AQUAA4ICDwAwggIKAoICAQCUaitvevlZirydcTjMIt2fr5ei7LvQx7bdIVobgEZ1
+Qlqf3BH6etKdmZChydkN0XXAb8Ysew8aCixKtrVeDCe5xRRCnKaFcEvqg2cSfbpX
+FevXDvfbTK2ed7YASOJ/pv31stqHd9m0xWZLCmsXZ8x6yIxgEGVHjIAOCyTAgcQy
+8ItIjmxn3Vu2FFVBemtP38Nzur/8id85uY7QPspI8Er8qVBBBHp6PhxTIKxAZpZb
+XtBf2VxIKbvUGEvCxWCrKNfv+j0oEqDpXOqGFpVBK28Q48u/0F+YBUY8FKP4rfgF
+I4lG9mnzMmCL76k+HjyBtU5zikDGqgm4mlPXgSRqEh0CvQS7zyrBRWiJCfK0g67f
+69CVGa7fji8pz99J59s8bYW7jgyro93LCGb4N3QfJLurB//ehDp33XdIhizJtopj
+UoFUGLnomVnMRTUNtMSAy7J4r1yjJDLufgnrPZ0yjYo6nyMiFswCaMmFfclUKtGz
+zbPDpIBuf0hmvJAt0LyWlYUst5geusPxbkM5XOhLn7px+/y+R0wMT3zNZYQxlsLD
+bXGYsRdE9jxcIts+IQwWZGnmHhhC1kvKC/nAYcqBZctMQB5q/qsPH652dc73zOx6
+Bp2gTZqokGCv5PGxiXcrwouOUIlYgizBDYGBDU02S4BRDM3oW9motVUonBnF8JHV
+RwIDAQABo4IBYjCCAV4wEgYDVR0TAQH/BAgwBgEB/wIBADAdBgNVHQ4EFgQU/glx
+QFUFEETYpIF1uJ4a6UoGiMgwHwYDVR0jBBgwFoAUTiJUIBiV5uNu5g/6+rkS7QYX
+jzkwDgYDVR0PAQH/BAQDAgGGMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcD
+AjB2BggrBgEFBQcBAQRqMGgwJAYIKwYBBQUHMAGGGGh0dHA6Ly9vY3NwLmRpZ2lj
+ZXJ0LmNvbTBABggrBgEFBQcwAoY0aHR0cDovL2NhY2VydHMuZGlnaWNlcnQuY29t
+L0RpZ2lDZXJ0R2xvYmFsUm9vdEcyLmNydDBCBgNVHR8EOzA5MDegNaAzhjFodHRw
+Oi8vY3JsMy5kaWdpY2VydC5jb20vRGlnaUNlcnRHbG9iYWxSb290RzIuY3JsMB0G
+A1UdIAQWMBQwCAYGZ4EMAQIBMAgGBmeBDAECAjANBgkqhkiG9w0BAQwFAAOCAQEA
+AQkxu6RRPlD3yrYhxg9jIlVZKjAnC9H+D0SSq4j1I8dNImZ4QjexTEv+224CSvy4
+zfp9gmeRfC8rnrr4FN4UFppYIgqR4H7jIUVMG9ECUcQj2Ef11RXqKOg5LK3fkoFz
+/Nb9CYvg4Ws9zv8xmE1Mr2N6WDgLuTBIwul2/7oakjj8MA5EeijIjHgB1/0r5mPm
+eFYVx8xCuX/j7+q4tH4PiHzzBcfqb3k0iR4DlhiZfDmy4FuNWXGM8ZoMM43EnRN/
+meqAcMkABZhY4gqeWZbOgxber297PnGOCcIplOwpPfLu1A1K9frVwDzAG096a8L0
+ItQCmz7TjRH4ptX5Zh9pw==
+-----END CERTIFICATE-----`,
+	// Microsoft Azure RSA TLS Issuing CA 04
+	`-----BEGIN CERTIFICATE-----
+MIIFrDCCBJSgAwIBAgIQCfluwpVVXyR0nq8eXc7UnTANBgkqhkiG9w0BAQwFADBh
+MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3
+d3cuZGlnaWNlcnQuY29tMSAwHgYDVQQDExdEaWdpQ2VydCBHbG9iYWwgUm9vdCBH
+MjAeFw0yMzA2MDgwMDAwMDBaFw0yNjA4MjUyMzU5NTlaMF0xCzAJBgNVBAYTAlVT
+MR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xLjAsBgNVBAMTJU1pY3Jv
+c29mdCBBenVyZSBSU0EgVExTIElzc3VpbmcgQ0EgMDQwggIiMA0GCSqGSIb3DQEB
+AQUAA4ICDwAwggIKAoICAQDBeUy13eRZ/QC5bN7/IOGxodny7Xm2BFc88d3cca3y
+HyyVx1Y60+afY6DAo/2Ls1uzAfbDfMzAVWJazPH4tckaItDv//htEbbNJnAGvZPB
+4VqNviwDEmlAWT/MTAmzXfTgWXuUNgRlzZbjoFaPm+t6iJ6HdvDpWQAJbsBUZCga
+t257tM28JnAHUTWdiDBn+2z6EGh2DA6BCx04zHDKVSegLY8+5P80Lqze0d6i3T2J
+J7rfxCmxUXfCGOv9iQIUZfhv4vCb8hsm/JdNUMiomJhSPa0bi3rda/swuJHCH//d
+wz2AGzZRRGdj7Kna4t6ToxK17lAF3Q6Qp368C9cE6JLMj+3UbY3umWCPRA5/Dms4
+/wl3GvDEw7HpyKsvRNPpjDZyiFzZGC2HZmGMsrZMT3hxmyQwmz1O3eGYdO5EIq1S
+W/vT1yShZTSusqmICQo5gWWRZTwCENekSbVX9qRr77o0pjKtuBMZTGQTixwpT/rg
+Ul7Mr4M2nqK55Kovy/kUN1znfPdW/Fj9iCuvPKwKFdyt2RVgxJDvgIF/bNoRkRxh
+wVB6qRgs4EiTrNbRoZAHEFF5wRBf9gWn9HeoI66VtdMZvJRH+0/FDWB4/zwxS16n
+nADJaVPXh6JHJFYs9p0wZmvct3GNdWrOLRAG2yzbfFZS8fJcX1PYxXXo4By16yGW
+hQIDAQABo4IBYjCCAV4wEgYDVR0TAQH/BAgwBgEB/wIBADAdBgNVHQ4EFgQUO3DR
+U+l2JZ1gqMpmD8abrm9UFmowHwYDVR0jBBgwFoAUTiJUIBiV5uNu5g/6+rkS7QYX
+jzkwDgYDVR0PAQH/BAQDAgGGMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcD
+AjB2BggrBgEFBQcBAQRqMGgwJAYIKwYBBQUHMAGGGGh0dHA6Ly9vY3NwLmRpZ2lj
+ZXJ0LmNvbTBABggrBgEFBQcwAoY0aHR0cDovL2NhY2VydHMuZGlnaWNlcnQuY29t
+L0RpZ2lDZXJ0R2xvYmFsUm9vdEcyLmNydDBCBgNVHR8EOzA5MDegNaAzhjFodHRw
+Oi8vY3JsMy5kaWdpY2VydC5jb20vRGlnaUNlcnRHbG9iYWxSb290RzIuY3JsMB0G
+A1UdIAQWMBQwCAYGZ4EMAQIBMAgGBmeBDAECAjANBgkqhkiG9w0BAQwFAAOCAQEA
+o9sJvBNLQSJ1e7VaG3cSZHBz6zjS70A1gVO1pqsmX34BWDPz1TAlOyJiLlA+eUF4
+B2OWHd3F//dJJ/3TaCFunjBhZudv3busl7flz42K/BG/eOdlg0kiUf07PCYY5/FK
+YTIch51j1moFlBqbglwkdNIVae2tOu0OdX2JiA+bprYcGxa7eayLetvPiA77ynTc
+UNMKOqYB41FZHOXe5IXDI5t2RsDM9dMEZv4+cOb9G9qXcgDar1AzPHEt/39335zC
+HofQ0QuItCDCDzahWZci9Nn9hb/SvAtPWHZLkLBG6I0iwGxvMwcTTc9Jnb4Flysr
+mQlwKsS2MphOoI23Qq3cSA==
+-----END CERTIFICATE-----`,
+	// Microsoft Azure RSA TLS Issuing CA 07
+	`-----BEGIN CERTIFICATE-----
+MIIFrDCCBJSgAwIBAgIQCkOpUJsBNS+JlXnscgi6UDANBgkqhkiG9w0BAQwFADBh
+MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3
+d3cuZGlnaWNlcnQuY29tMSAwHgYDVQQDExdEaWdpQ2VydCBHbG9iYWwgUm9vdCBH
+MjAeFw0yMzA2MDgwMDAwMDBaFw0yNjA4MjUyMzU5NTlaMF0xCzAJBgNVBAYTAlVT
+MR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xLjAsBgNVBAMTJU1pY3Jv
+c29mdCBBenVyZSBSU0EgVExTIElzc3VpbmcgQ0EgMDcwggIiMA0GCSqGSIb3DQEB
+AQUAA4ICDwAwggIKAoICAQC1ZF7KYus5OO3GWqJoR4xznLDNCjocogqeCIVdi4eE
+BmF3zIYeuXXNoJAUF+mn86NBt3yMM0559JZDkiSDi9MpA2By4yqQlTHzfbOrvs7I
+4LWsOYTEClVFQgzXqa2ps2g855HPQW1hZXVh/yfmbtrCNVa//G7FPDqSdrAQ+M8w
+0364kyZApds/RPcqGORjZNokrNzYcGub27vqE6BGP6XeQO5YDFobi9BvvTOO+ZA9
+HGIU7FbdLhRm6YP+FO8NRpvterfqZrRt3bTn8GT5LsOTzIQgJMt4/RWLF4EKNc97
+CXOSCZFn7mFNx4SzTvy23B46z9dQPfWBfTFaxU5pIa0uVWv+jFjG7l1odu0WZqBd
+j0xnvXggu564CXmLz8F3draOH6XS7Ys9sTVM3Ow20MJyHtuA3hBDv+tgRhrGvNRD
+MbSzTO6axNWvL46HWVEChHYlxVBCTfSQmpbcAdZOQtUfs9E4sCFrqKcRPdg7ryhY
+fGbj3q0SLh55559ITttdyYE+wE4RhODgILQ3MaYZoyiL1E/4jqCOoRaFhF5R++vb
+YpemcpWx7unptfOpPRRnnN4U3pqZDj4yXexcyS52Rd8BthFY/cBg8XIR42BPeVRl
+OckZ+ttduvKVbvmGf+rFCSUoy1tyRwQNXzqeZTLrX+REqgFDOMVe0I49Frc2/Avw
+3wIDAQABo4IBYjCCAV4wEgYDVR0TAQH/BAgwBgEB/wIBADAdBgNVHQ4EFgQUzhUW
+O+oCo6Zr2tkr/eWMUr56UKgwHwYDVR0jBBgwFoAUTiJUIBiV5uNu5g/6+rkS7QYX
+jzkwDgYDVR0PAQH/BAQDAgGGMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcD
+AjB2BggrBgEFBQcBAQRqMGgwJAYIKwYBBQUHMAGGGGh0dHA6Ly9vY3NwLmRpZ2lj
+ZXJ0LmNvbTBABggrBgEFBQcwAoY0aHR0cDovL2NhY2VydHMuZGlnaWNlcnQuY29t
+L0RpZ2lDZXJ0R2xvYmFsUm9vdEcyLmNydDBCBgNVHR8EOzA5MDegNaAzhjFodHRw
+Oi8vY3JsMy5kaWdpY2VydC5jb20vRGlnaUNlcnRHbG9iYWxSb290RzIuY3JsMB0G
+A1UdIAQWMBQwCAYGZ4EMAQIBMAgGBmeBDAECAjANBgkqhkiG9w0BAQwFAAOCAQEA
+bbV8m4/LCSvb0nBF9jb7MVLH/9JjHGbn0QjB4R4bMlGHbDXDWtW9pFqMPrRh2Q76
+Bqm+yrrgX83jPZAcvOd7F7+lzDxZnYoFEWhxW9WnuM8Te5x6HBPCPRbIuzf9pSUT
+/ozvbKFCDxxgC2xKmgp6NwxRuGcy5KQQh4xkq/hJrnnF3RLakrkUBYFPUneip+wS
+BzAfK3jHXnkNCPNvKeLIXfLMsffEzP/j8hFkjWL3oh5yaj1HmlW8RE4Tl/GdUVzQ
+D1x42VSusQuRGtuSxLhzBNBeJtyD//2u7wY2uLYpgK0o3X0iIJmwpt7Ovp6Bs4tI
+E/peia+Qcdk9Qsr+1VgCGA==
+-----END CERTIFICATE-----`,
+	// Microsoft Azure RSA TLS Issuing CA 08
+	`-----BEGIN CERTIFICATE-----
+MIIFrDCCBJSgAwIBAgIQDvt+VH7fD/EGmu5XaW17oDANBgkqhkiG9w0BAQwFADBh
+MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3
+d3cuZGlnaWNlcnQuY29tMSAwHgYDVQQDExdEaWdpQ2VydCBHbG9iYWwgUm9vdCBH
+MjAeFw0yMzA2MDgwMDAwMDBaFw0yNjA4MjUyMzU5NTlaMF0xCzAJBgNVBAYTAlVT
+MR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xLjAsBgNVBAMTJU1pY3Jv
+c29mdCBBenVyZSBSU0EgVExTIElzc3VpbmcgQ0EgMDgwggIiMA0GCSqGSIb3DQEB
+AQUAA4ICDwAwggIKAoICAQCy7oIFzcDVZVbomWZtSwrAX8LiKXsbCcwuFL7FHkD5
+m67olmOdTueOKhNER5ykFs/meKG1fwzd35/+Q1+KTxcV89IIXmErtSsj8EWu7rdE
+AVYnYMFbstqwkIVNEoz4OIM82hn+N5p57zkHGPogzF6TOPRUOK8yYyCPeqnHvoVp
+E5b0kZL4QT8bdyhSRQbUsUiSaOuF5y3eZ9Vc92baDkhY7CFZE2ThLLv5PQ0WxzLo
+t3t18d2vQP5x29I0n6NFsj37J2d/EH/Z6a/lhAVzKjfYloGcQ1IPyDEIGh9gYJnM
+LFZiUbm/GBmlpKVr8M03OWKCR0thRbfnU6UoskrwGrECAnnojFEUw+j8i6gFLBNW
+XtBOtYvgl8SHCCVKUUUl4YOfR5zF4OkKirJuUbOmB2AOmLjYJIcabDvxMcmryhQi
+nog+/+jgHJnY62opgStkdaImMPzyLB7ZaWVnxpRdtFKO1ZvGkZeRNvbPAUKR2kNe
+knuh3NtFvz2dY3xP7AfhyLE/t8vW72nAzlRKz++L70CgCvj/yeObPwaAPDd2sZ0o
+j2u/N+k6egGq04e+GBW+QYCSoJ5eAY36il0fu7dYSHYDo7RB5aPTLqnybp8wMeAa
+tcagc8U9OM42ghELTaWFARuyoCmgqR7y8fAU9Njhcqrm6+0Xzv/vzMfhL4Ulpf1G
+7wIDAQABo4IBYjCCAV4wEgYDVR0TAQH/BAgwBgEB/wIBADAdBgNVHQ4EFgQU9n4v
+vYCjSrJwW+vfmh/Y7cphgAcwHwYDVR0jBBgwFoAUTiJUIBiV5uNu5g/6+rkS7QYX
+jzkwDgYDVR0PAQH/BAQDAgGGMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcD
+AjB2BggrBgEFBQcBAQRqMGgwJAYIKwYBBQUHMAGGGGh0dHA6Ly9vY3NwLmRpZ2lj
+ZXJ0LmNvbTBABggrBgEFBQcwAoY0aHR0cDovL2NhY2VydHMuZGlnaWNlcnQuY29t
+L0RpZ2lDZXJ0R2xvYmFsUm9vdEcyLmNydDBCBgNVHR8EOzA5MDegNaAzhjFodHRw
+Oi8vY3JsMy5kaWdpY2VydC5jb20vRGlnaUNlcnRHbG9iYWxSb290RzIuY3JsMB0G
+A1UdIAQWMBQwCAYGZ4EMAQIBMAgGBmeBDAECAjANBgkqhkiG9w0BAQwFAAOCAQEA
+loABcB94CeH6DWKwa4550BTzLxlTHVNseQJ5SetnPpBuPNLPgOLe9Y7ZMn4ZK6mh
+feK7RiMzan4UF9CD5rF3TcCevo3IxrdV+YfBwvlbGYv+6JmX3mAMlaUb23Y2pONo
+ixFJEOcAMKKR55mSC5W4nQ6jDfp7Qy/504MQpdjJflk90RHsIZGXVPw/JdbBp0w6
+pDb4o5CqydmZqZMrEvbGk1p8kegFkBekp/5WVfd86BdH2xs+GKO3hyiA8iBrBCGJ
+fqrijbRnZm7q5+ydXF3jhJDJWfxW5EBYZBJrUz/a+8K/78BjwI8z2VYJpG4t6r4o
+tOGB5sEyDPDwqx00Rouu8g==
 -----END CERTIFICATE-----`,
 	// Microsoft Azure TLS Issuing CA 01
 	`-----BEGIN CERTIFICATE-----
@@ -35,6 +35,11 @@ func TestValidate(t *testing.T) {
 		payload: "MIILiQYJKoZIhvcNAQcCoIILejCCC3YCAQExDzANBgkqhkiG9w0BAQsFADCCAUAGCSqGSIb3DQEHAaCCATEEggEteyJsaWNlbnNlVHlwZSI6IiIsIm5vbmNlIjoiMjAyMzAzMDgtMjMwOTMzIiwicGxhbiI6eyJuYW1lIjoiIiwicHJvZHVjdCI6IiIsInB1Ymxpc2hlciI6IiJ9LCJza3UiOiIxOC4wNC1MVFMiLCJzdWJzY3JpcHRpb25JZCI6IjBhZmJmZmZhLTVkZjktNGEzYi05ODdlLWZlNzU3NzYyNDI3MiIsInRpbWVTdGFtcCI6eyJjcmVhdGVkT24iOiIwMy8wOC8yMyAxNzowOTozMyAtMDAwMCIsImV4cGlyZXNPbiI6IjAzLzA4LzIzIDIzOjA5OjMzIC0wMDAwIn0sInZtSWQiOiI5OTA4NzhkNC0wNjhhLTRhYzQtOWVlOS0xMjMxZDIyMThlZjIifaCCCHswggh3MIIGX6ADAgECAhMzAIXQK9n2YdJHP1paAAAAhdArMA0GCSqGSIb3DQEBDAUAMFkxCzAJBgNVBAYTAlVTMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xKjAoBgNVBAMTIU1pY3Jvc29mdCBBenVyZSBUTFMgSXNzdWluZyBDQSAwNTAeFw0yMzAyMDMxOTAxMThaFw0yNDAxMjkxOTAxMThaMGgxCzAJBgNVBAYTAlVTMQswCQYDVQQIEwJXQTEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMRowGAYDVQQDExFtZXRhZGF0YS5henVyZS51czCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMrbkY7Z8ffglHPokuGfRDOBjFt6n68OuReoq2CbnhyEdosDsfJBsoCr5vV3mVcpil1+y0HeabKr+PdJ6GWCXiymxxgMtNMIuz/kt4OVOJSkV3wJyMNYRjGUAB53jw2cJnhIgLy6QmxOm2cnDb+IBFGn7WAw/XqT8taDd6RPDHR6P+XqpWuMN/MheCOdJRagmr8BUNt95eOhRAGZeUWHKcCssBa9xZNmTzgd26NuBRpeGVrjuPCaQXiGWXvJ7zujWOiMopgw7UWXMiJp6J+Nn75Dx+MbPjlLYYBhFEEBaXj0iKuj/3/lm3nkkMLcYPxEJE0lPuX1yQQLUx3l1bBYyykCAwEAAaOCBCcwggQjMIIBfQYKKwYBBAHWeQIEAgSCAW0EggFpAWcAdgDuzdBk1dsazsVct520zROiModGfLzs3sNRSFlGcR+1mwAAAYYYsLzVAAAEAwBHMEUCIQD+BaiDS1uFyVGdeMc5vBUpJOmBhxgRyTkH3kQG+KD6RwIgWIMxqyGtmM9rH5CrWoruToiz7NNfDmp11LLHZNaKpq4AdgBz2Z6JG0yWeKAgfUed5rLGHNBRXnEZKoxrgBB6wXdytQAAAYYYsL0bAAAEAwBHMEUCIQDNxRWECEZmEk9zRmRPNv3QP0lDsUzaKhYvFPmah/wkKwIgXyCv+fvWga+XB2bcKQqom10nvTDBExIZeoOWBSfKVLgAdQB2/4g/Crb7lVHCYcz1h7o0tKTNuyncaEIKn+ZnTFo6dAAAAYYYsL0bAAAEAwBGMEQCICCTSeyEisZwmi49g941B6exndOFwF4JqtoXbWmFcxRcAiBCDaVJJN0e0ZVSPkx9NVMGWvBjQbIYtSG4LEkCdDsMejAnBgkrBgEEAYI3FQoEGjAYMAoGCCsGAQUFBwMCMAoGCCsGAQUFBwMBMDwGCSsGAQQBgjcVBwQvMC0GJSsGAQQBgjcVCIe91xuB5+tGgoGdLo7QDIfw2h1dgoTlaYLzpz4CAWQCASUwga4GCCsGAQUFBwEBBIGhMIGeMG0GCCsGAQUFBzAChmFodHRwOi8vd3d3Lm1pY3Jvc29mdC5jb20vcGtpb3BzL2NlcnRzL01pY3Jvc29mdCUyMEF6dXJlJTIwVExTJTIwSXNzdWluZyUyMENBJTIwMDUlMjAtJTIweHNpZ24uY3J0MC0GCCsGAQUFBzABhiFodHRwOi8vb25lb2NzcC5taWNyb3NvZnQuY29tL29jc3AwHQYDVR0OBBYEFBcZK26vkjWcbAk7XwJHTP/lxgeXMA4GA1UdDwEB/wQEAwIEsDA9BgNVHREENjA0gh91c2dvdnZpcmdpbmlhLm1ldGFkYXRhLmF6dXJlLnVzghFtZXRhZGF0YS5henVyZS51czAMBgNVHRMBAf8EAjAAMGQGA1UdHwRdMFswWaBXoFWGU2h0dHA6Ly93d3cubWljcm9zb2Z0LmNvbS9wa2lvcHMvY3JsL01pY3Jvc29mdCUyMEF6dXJlJTIwVExTJTIwSXNzdWluZyUyMENBJTIwMDUuY3JsMGYGA1UdIARfMF0wUQYMKwYBBAGCN0yDfQEBMEEwPwYIKwYBBQUHAgEWM2h0dHA6Ly93d3cubWljcm9zb2Z0LmNvbS9wa2lvcHMvRG9jcy9SZXBvc2l0b3J5Lmh0bTAIBgZngQwBAgIwHwYDVR0jBBgwFoAUx7KcfxzjuFrv6WgaqF2UwSZSamgwHQYDVR0lBBYwFAYIKwYBBQUHAwIGCCsGAQUFBwMBMA0GCSqGSIb3DQEBDAUAA4ICAQCUExuLe7D71C5kek65sqKXUodQJXVVpFG0Y4l9ZacBFql8BgHvu2Qvt8zfWsyCHy4A2KcMeHLwi2DdspyTjxSnwkuPcQ4ndhgAqrLkfoTc435NnnsiyzCUNDeGIQ+g+QSRPV86u6LmvFr0ZaOqxp6eJDPYewHhKyGLQuUyBjUNkhS+tGzuvsHaeCUYclmbZFN75IQSvBmL0XOsOD7wXPZB1a68D26wyCIbIC8MuFwxreTrvdRKt/5zIfBnku6S6xRgkzH64gfBLbU5e2VCdaKzElWEKRLJgl3R6raNRqFot+XNfa26H5sMZpZkuHrvkPZcvd5zOfL7fnVZoMLo4A3kFpet7tr1ls0ifqodzlOBMNrUdf+o3kJ1seCjzx2WdFP+2liO80d0oHKiv8djuttlPfQkV8WATmyLoZVoPcNovayrVUjTWFMXqIShhhTbIJ3ZRSZrz6rZLok0Xin3+4d28iMsi7tjxnBW/A/eiPrqs7f2v2rLXuf5/XHuzHIYQpiZpnvA90mE1HBB9fv4sETsw9TuL2nXai/c06HGGM06i4o+lRuyvymrlt/QPR7SCPXl5fZFVAavLtu1UtafrK/qcKQTHnVJeZ20+JdDIJDP2qcxQvdw7XA88aa/Y/olM+yHIjpaPpsRFa2o8UB0ct+x1cTAhLhj3vNwhZHoFlVcFzGCAZswggGXAgEBMHAwWTELMAkGA1UEBhMCVVMxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEqMCgGA1UEAxMhTWljcm9zb2Z0IEF6dXJlIFRMUyBJc3N1aW5nIENBIDA1AhMzAIXQK9n2YdJHP1paAAAAhdArMA0GCSqGSIb3DQEBCwUAMA0GCSqGSIb3DQEBAQUABIIBAFuEf//loqaib860Ys5yZkrRj1QiSDSzkU+Vxx9fYXzWzNT4KgMhkEhRRvoE6TR/tIUzbKFQxIVRrlW2lbGSj8JEeLoEVlp2Pc4gNRJeX2N9qVDPvy9lmYuBm1XjypLPwvYjvfPjsLRKkNdQ5MWzrC3F2q2OOQP4sviy/DCcoDitEmqmqiCuog/DiS5xETivde3pTZGiFwKlgzptj4/KYN/iZTzU25fFSCD5Mq2IxHRj39gFkqpFekdSRihSH0W3oyPfic/E3H0rVtSkiFm2SL6nPjILjhaJcV7az+X7Qu4AXYZ/TrabX+OW5dJ69SoJ01DfnqGD0sll0+P3QSUHEvA=",
 		vmID:    "990878d4-068a-4ac4-9ee9-1231d2218ef2",
 		date:    mustTime(time.RFC3339, "2023-04-01T00:00:00Z"),
+	}, {
+		name:    "rsa",
+		payload: "MIILnwYJKoZIhvcNAQcCoIILkDCCC4wCAQExDzANBgkqhkiG9w0BAQsFADCCAUUGCSqGSIb3DQEHAaCCATYEggEyeyJsaWNlbnNlVHlwZSI6IiIsIm5vbmNlIjoiMjAyNDA0MjItMjMzMjQ1IiwicGxhbiI6eyJuYW1lIjoiIiwicHJvZHVjdCI6IiIsInB1Ymxpc2hlciI6IiJ9LCJza3UiOiIyMF8wNC1sdHMtZ2VuMiIsInN1YnNjcmlwdGlvbklkIjoiMDVlOGIyODUtNGNlMS00NmEzLWI0YzktZjUxYmE2N2Q2YWNjIiwidGltZVN0YW1wIjp7ImNyZWF0ZWRPbiI6IjA0LzIyLzI0IDE3OjMyOjQ1IC0wMDAwIiwiZXhwaXJlc09uIjoiMDQvMjIvMjQgMjM6MzI6NDUgLTAwMDAifSwidm1JZCI6Ijk2MGE0YjRhLWRhYjItNDRlZi05YjczLTc3NTMwNDNiNGYxNiJ9oIIIiDCCCIQwggZsoAMCAQICEzMAJtj/yBIW1kk+vsIAAAAm2P8wDQYJKoZIhvcNAQEMBQAwXTELMAkGA1UEBhMCVVMxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEuMCwGA1UEAxMlTWljcm9zb2Z0IEF6dXJlIFJTQSBUTFMgSXNzdWluZyBDQSAwODAeFw0yNDA0MTgwODM1MzdaFw0yNTA0MTMwODM1MzdaMGkxCzAJBgNVBAYTAlVTMQswCQYDVQQIEwJXQTEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMRswGQYDVQQDExJtZXRhZGF0YS5henVyZS5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQD0T031XgxaebNQjKFQZ4BudeN+wOEHQoFq/x+cKSXM8HJrC2pF8y/ngSsuCLGt72M+30KxdbPHl56kd52uwDw1ZBrQO6Xw+GorRbtM4YQi+gLr8t9x+GUfuOX7E+5juidXax7la5ZhpVVLb3f+8NyxbphvEdFadXcgyQga1pl4v1U8elkbX3PPtEQXzwYotU+RU/ZTwXMYqfvJuaKwc4T2s083kaL3DwAfVxL0f6ey/MXuNQb4+ho15y9/f9gwMyzMDLlYChmY6cGSS4tsyrG5SrybE3jl8LZ1ZLVJ2fAIxbmJzBn1q+Eu4G6TZlnMDEsjznf7gqnP+n/o7N6l0sY1AgMBAAGjggQvMIIEKzCCAX4GCisGAQQB1nkCBAIEggFuBIIBagFoAHYAzxFW7tUufK/zh1vZaS6b6RpxZ0qwF+ysAdJbd87MOwgAAAGO8GIJ/QAABAMARzBFAiEAvJQ2mDRow9TMvLddWpYqNXLiehSFsj2+xUqh8yP/B8YCIBJjVoELj3kdVr3ceAuZFte9FH6sBsgeMsIgfndho6hRAHUAfVkeEuF4KnscYWd8Xv340IdcFKBOlZ65Ay/ZDowuebgAAAGO8GIK2AAABAMARjBEAiAxXD1R9yLASrpMh4ie0wn3AjCoSPniZ8virEVz8tKnkwIgWxGU9DjjQk7gPWYVBsiXP9t1WPJ6mNJ1UkmAw8iDdFoAdwBVgdTCFpA2AUrqC5tXPFPwwOQ4eHAlCBcvo6odBxPTDAAAAY7wYgrtAAAEAwBIMEYCIQCaSjdXbUhrDyPNsRqewp5UdVYABGQAIgNwfKsq/JpbmAIhAPy5qQ6H2enXwuKsorEZTwIkKIoMgLsWs4anx9lXTJMeMCcGCSsGAQQBgjcVCgQaMBgwCgYIKwYBBQUHAwIwCgYIKwYBBQUHAwEwPAYJKwYBBAGCNxUHBC8wLQYlKwYBBAGCNxUIh73XG4Hn60aCgZ0ujtAMh/DaHV2ChOVpgvOnPgIBZAIBJjCBtAYIKwYBBQUHAQEEgacwgaQwcwYIKwYBBQUHMAKGZ2h0dHA6Ly93d3cubWljcm9zb2Z0LmNvbS9wa2lvcHMvY2VydHMvTWljcm9zb2Z0JTIwQXp1cmUlMjBSU0ElMjBUTFMlMjBJc3N1aW5nJTIwQ0ElMjAwOCUyMC0lMjB4c2lnbi5jcnQwLQYIKwYBBQUHMAGGIWh0dHA6Ly9vbmVvY3NwLm1pY3Jvc29mdC5jb20vb2NzcDAdBgNVHQ4EFgQUnqRq3WHOZDoNmLD/arJg9RscxLowDgYDVR0PAQH/BAQDAgWgMDgGA1UdEQQxMC+CGWVhc3R1cy5tZXRhZGF0YS5henVyZS5jb22CEm1ldGFkYXRhLmF6dXJlLmNvbTAMBgNVHRMBAf8EAjAAMGoGA1UdHwRjMGEwX6BdoFuGWWh0dHA6Ly93d3cubWljcm9zb2Z0LmNvbS9wa2lvcHMvY3JsL01pY3Jvc29mdCUyMEF6dXJlJTIwUlNBJTIwVExTJTIwSXNzdWluZyUyMENBJTIwMDguY3JsMGYGA1UdIARfMF0wUQYMKwYBBAGCN0yDfQEBMEEwPwYIKwYBBQUHAgEWM2h0dHA6Ly93d3cubWljcm9zb2Z0LmNvbS9wa2lvcHMvRG9jcy9SZXBvc2l0b3J5Lmh0bTAIBgZngQwBAgIwHwYDVR0jBBgwFoAU9n4vvYCjSrJwW+vfmh/Y7cphgAcwHQYDVR0lBBYwFAYIKwYBBQUHAwIGCCsGAQUFBwMBMA0GCSqGSIb3DQEBDAUAA4ICAQB4FwyqZFVdmB9Hu+YUJOJrGUYRlXbnCmdXlLi5w2QRCf9RKIykGdv28dH1ezhXJUCj3jCVZMav4GaSl0dPUcTetfnc/UrwsmbGRIMubbGjCz75FcNz/kXy7E/jPeyJrxsuO/ijyZNUSy0EQF3NuhTJw/SfAQtXv48NmVFDM2QMMhMRLDfOV4CPcialAFACFQTt6LMdG2hlB972Bffl+BVPkUKDLj89xQRd/cyWYweYfPCsNLYLDml98rY3v4yVKAvv+l7IOuKOzhlOe9U1oPJK7AP7GZzojKrisPQt4HlP4zEmeUzJtL6RqGdHac7/lUMVPOniE/L+5gBDBsN3nOGJ/QE+bBsmfdn4ewuLj6/LCd/JhCZFDeyTvtuX43JWIr9e0UOtENCG3Ub4SuUftf58+NuedCaNMZW2jqrFvQl+sCX+v1kkxxmRphU7B8TZP0SHaBDqeIqHPNWD7eyn/7+VTY54wrwF1v5S6b5zpL1tjZ55c9wpVBT6m77mNuR/2l7/VSh/qL2LgKVVo06q+Qz2c0pIjOI+7FobLRNtb7C8SqkdwuT1b0vnZslA8ZUEtwUm5RHcGu66sg/hb4lGNZbAklxGeAR3uQju0OQN/Lj4kXiii737dci0lIpIKA92hUKybLrYCyZDhp5I6is0gTdm4+rxVEY1K39R3cF3U5thuzGCAZ8wggGbAgEBMHQwXTELMAkGA1UEBhMCVVMxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEuMCwGA1UEAxMlTWljcm9zb2Z0IEF6dXJlIFJTQSBUTFMgSXNzdWluZyBDQSAwOAITMwAm2P/IEhbWST6+wgAAACbY/zANBgkqhkiG9w0BAQsFADANBgkqhkiG9w0BAQEFAASCAQDRukRXI01EvAoF0J+C1aYCmjwAtMlnQr5fBKod8T75FhM+mTJ2GApCyc5H8hn7IDl8ki8DdKfLjipnuEvjknZcVkfrzE72R9Pu+C2ffKfrSsJmsBHPMEKBPtlzhexCYiPamMGdVg8HqX6mhQkjjavk1SY+ewZvyEeuq+RSQIBVL1lw0UOWv+txDKlu9v69skb1DQ2HSet0sejEb48vqGeN4TMSoQFNeBOzHDkEeoqXxtZqsUhMtQzbwrpAFcUREB8DaCOXcv1DOminJB3Q19bpuMQ/2+Fc3HJtTTWRV3+3b7VnQl/sUDzTjcWXvwjrLGKk3MSTcQ+1rJRlBzkOJ+aK",
+		vmID:    "960a4b4a-dab2-44ef-9b73-7753043b4f16",
+		date:    mustTime(time.RFC3339, "2024-04-22T17:32:44Z"),
 	}} {
 		tc := tc
 		t.Run(tc.name, func(t *testing.T) {
@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+# See: https://learn.microsoft.com/en-us/azure/security/fundamentals/azure-ca-details?tabs=certificate-authority-chains
+declare -a CERTIFICATES=(
+	"Microsoft RSA TLS CA 01=https://crt.sh/?d=3124375355"
+	"Microsoft RSA TLS CA 02=https://crt.sh/?d=3124375356"
+	"Microsoft Azure RSA TLS Issuing CA 03=https://www.microsoft.com/pkiops/certs/Microsoft%20Azure%20RSA%20TLS%20Issuing%20CA%2003%20-%20xsign.crt"
+	"Microsoft Azure RSA TLS Issuing CA 04=https://www.microsoft.com/pkiops/certs/Microsoft%20Azure%20RSA%20TLS%20Issuing%20CA%2004%20-%20xsign.crt"
+	"Microsoft Azure RSA TLS Issuing CA 07=https://www.microsoft.com/pkiops/certs/Microsoft%20Azure%20RSA%20TLS%20Issuing%20CA%2007%20-%20xsign.crt"
+	"Microsoft Azure RSA TLS Issuing CA 08=https://www.microsoft.com/pkiops/certs/Microsoft%20Azure%20RSA%20TLS%20Issuing%20CA%2008%20-%20xsign.crt"
+	"Microsoft Azure TLS Issuing CA 01=https://www.microsoft.com/pki/certs/Microsoft%20Azure%20TLS%20Issuing%20CA%2001.cer"
+	"Microsoft Azure TLS Issuing CA 02=https://www.microsoft.com/pki/certs/Microsoft%20Azure%20TLS%20Issuing%20CA%2002.cer"
+	"Microsoft Azure TLS Issuing CA 05=https://www.microsoft.com/pki/certs/Microsoft%20Azure%20TLS%20Issuing%20CA%2005.cer"
+	"Microsoft Azure TLS Issuing CA 06=https://www.microsoft.com/pki/certs/Microsoft%20Azure%20TLS%20Issuing%20CA%2006.cer"
+)
+
+CONTENT="var Certificates = []string{"
+
+for CERT in "${CERTIFICATES[@]}"; do
+	IFS="=" read -r NAME URL <<<"$CERT"
+	echo "Downloading certificate: $NAME"
+	PEM=$(curl -sSL "$URL" | openssl x509 -outform PEM)
+	echo "$PEM"
+
+	CONTENT+="\n// $NAME\n\`$PEM\`,"
+done
+
+CONTENT+="\n}"
+
+sed -i '/var Certificates = /,$d' azureidentity.go
+# shellcheck disable=SC2059
+printf "$CONTENT" >>azureidentity.go
+gofmt -w azureidentity.go
@@ -498,7 +498,7 @@ func (c *provisionerdCloser) Close() error {
 	c.closed = true
 	ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitShort)
 	defer cancel()
-	shutdownErr := c.d.Shutdown(ctx)
+	shutdownErr := c.d.Shutdown(ctx, true)
 	closeErr := c.d.Close()
 	if shutdownErr != nil {
 		return shutdownErr
@@ -51,7 +51,7 @@ SELECT
  template_versions.archived,
  COALESCE(visible_users.avatar_url, ''::text) AS created_by_avatar_url,
  COALESCE(visible_users.username, ''::text) AS created_by_username
-FROM (public.template_versions
+FROM (template_versions
  LEFT JOIN visible_users ON (template_versions.created_by = visible_users.id));

 COMMENT ON VIEW template_version_with_user IS 'Joins in the username + avatar url of the created by user.';
@@ -53,7 +53,7 @@ SELECT
  template_versions.archived,
  COALESCE(visible_users.avatar_url, ''::text) AS created_by_avatar_url,
  COALESCE(visible_users.username, ''::text) AS created_by_username
-FROM (public.template_versions
+FROM (template_versions
  LEFT JOIN visible_users ON (template_versions.created_by = visible_users.id));

 COMMENT ON VIEW template_version_with_user IS 'Joins in the username + avatar url of the created by user.';
@@ -0,0 +1,29 @@
+## Changelog
+
+### Features
+
+- Add separate signals for shutdown handling on Coder server and provisionerd.
+  (#12358) (@kylecarbs)
+
+  > `SIGTERM` will stop accepting new provisioner jobs and wait running jobs to
+  > finish before shutting down.
+  >
+  > `SIGINT` (existing behavior) will cancel in-flight jobs then shut down.
+
+### Bug fixes
+
+- Fixed an issue where single-replica workspace proxy deployments may enter an
+  unhealthy state due to replica management errors. (#12641) (@deansheather)
+
+- Fixed an issue preventing upgrade to `v2.9.0` due to a migration that hard a
+  hardcoded schema name. (#12620) (@95gabor)
+
+Compare: [`v2.9.0...v2.9.1`](https://github.com/coder/coder/compare/v2.9.0...v2.9.1)
+
+## Container image
+
+- `docker pull ghcr.io/coder/coder:v2.9.1`
+
+## Install/upgrade
+
+Refer to our docs to [install](https://coder.com/docs/v2/latest/install) or [upgrade](https://coder.com/docs/v2/latest/admin/upgrade) Coder, or use a release asset below.
@@ -0,0 +1,20 @@
+## Changelog
+
+### Bug fixes
+
+- Fix issue causing Coder's external auth to open duplicate windows
+  (#12729) (@aslilac)
+
+- Fix: remove unnecessary href from external 
+  auth flow (#12586) (@emryk)
+
+
+Compare: [`v2.9.1...v2.9.2`](https://github.com/coder/coder/compare/v2.9.1...v2.9.2)
+
+## Container image
+
+- `docker pull ghcr.io/coder/coder:v2.9.2`
+
+## Install/upgrade
+
+Refer to our docs to [install](https://coder.com/docs/v2/latest/install) or [upgrade](https://coder.com/docs/v2/latest/admin/upgrade) Coder, or use a release asset below.
@@ -0,0 +1,20 @@
+## Changelog
+
+### Bug fixes
+
+- Fix: make terminal raw in `ssh`` command on Windows
+  (#12990) (@deansheather)
+
+- Fix: stop sending DeleteTailnetPeer when coordinator is unhealthy, causing outages
+  (#12923) (@spikecurtis)
+
+
+Compare: [`v2.9.2...v2.9.3`](https://github.com/coder/coder/compare/v2.9.2...v2.9.3)
+
+## Container image
+
+- `docker pull ghcr.io/coder/coder:v2.9.3`
+
+## Install/upgrade
+
+Refer to our docs to [install](https://coder.com/docs/v2/latest/install) or [upgrade](https://coder.com/docs/v2/latest/admin/upgrade) Coder, or use a release asset below.
@@ -0,0 +1,16 @@
+## Changelog
+
+### Chores
+
+- Added generate script for Azure Instance Identity (#13028, 57a4f5074) (@kylecarbs)
+
+Compare: [`v2.9.3...v2.9.4`](https://github.com/coder/coder/compare/v2.9.3...v2.9.4)
+
+## Container image
+
+- `docker pull ghcr.io/coder/coder:v2.9.4`
+
+## Install/upgrade
+
+Refer to our docs to [install](https://coder.com/docs/v2/latest/install) or [upgrade](https://coder.com/docs/v2/latest/admin/upgrade) Coder, or use a release asset below.
+
@@ -88,8 +88,10 @@ func (r *RootCmd) provisionerDaemonStart() *clibase.Cmd {
 			ctx, cancel := context.WithCancel(inv.Context())
 			defer cancel()

-			notifyCtx, notifyStop := inv.SignalNotifyContext(ctx, agpl.InterruptSignals...)
-			defer notifyStop()
+			stopCtx, stopCancel := inv.SignalNotifyContext(ctx, agpl.StopSignalsNoInterrupt...)
+			defer stopCancel()
+			interruptCtx, interruptCancel := inv.SignalNotifyContext(ctx, agpl.InterruptSignals...)
+			defer interruptCancel()

 			tags, err := agpl.ParseProvisionerTags(rawTags)
 			if err != nil {
@@ -212,10 +214,17 @@ func (r *RootCmd) provisionerDaemonStart() *clibase.Cmd {
 				Metrics:        metrics,
 			})

+			waitForProvisionerJobs := false
 			var exitErr error
 			select {
-			case <-notifyCtx.Done():
-				exitErr = notifyCtx.Err()
+			case <-stopCtx.Done():
+				exitErr = stopCtx.Err()
+				_, _ = fmt.Fprintln(inv.Stdout, cliui.Bold(
+					"Stop caught, waiting for provisioner jobs to complete and gracefully exiting. Use ctrl+\\ to force quit",
+				))
+				waitForProvisionerJobs = true
+			case <-interruptCtx.Done():
+				exitErr = interruptCtx.Err()
 				_, _ = fmt.Fprintln(inv.Stdout, cliui.Bold(
 					"Interrupt caught, gracefully exiting. Use ctrl+\\ to force quit",
 				))
@@ -225,7 +234,7 @@ func (r *RootCmd) provisionerDaemonStart() *clibase.Cmd {
 				cliui.Errorf(inv.Stderr, "Unexpected error, shutting down server: %s\n", exitErr)
 			}

-			err = srv.Shutdown(ctx)
+			err = srv.Shutdown(ctx, waitForProvisionerJobs)
 			if err != nil {
 				return xerrors.Errorf("shutdown: %w", err)
 			}
@@ -142,7 +142,7 @@ func (r *RootCmd) proxyServer() *clibase.Cmd {
 			//
 			// To get out of a graceful shutdown, the user can send
 			// SIGQUIT with ctrl+\ or SIGKILL with `kill -9`.
-			notifyCtx, notifyStop := inv.SignalNotifyContext(ctx, cli.InterruptSignals...)
+			notifyCtx, notifyStop := inv.SignalNotifyContext(ctx, cli.StopSignals...)
 			defer notifyStop()

 			// Clean up idle connections at the end, e.g.
@@ -441,7 +441,7 @@ func TestProvisionerDaemonServe(t *testing.T) {
 		build := coderdtest.AwaitWorkspaceBuildJobCompleted(t, client, workspace.LatestBuild.ID)
 		require.Equal(t, codersdk.WorkspaceStatusRunning, build.Status)

-		err = pd.Shutdown(ctx)
+		err = pd.Shutdown(ctx, false)
 		require.NoError(t, err)
 		err = terraformServer.Close()
 		require.NoError(t, err)
@@ -231,6 +231,17 @@ func (c *pgCoord) Coordinate(
 	logger := c.logger.With(slog.F("peer_id", id))
 	reqs := make(chan *proto.CoordinateRequest, agpl.RequestBufferSize)
 	resps := make(chan *proto.CoordinateResponse, agpl.ResponseBufferSize)
+	if !c.querier.isHealthy() {
+		// If the coordinator is unhealthy, we don't want to hook this Coordinate call up to the
+		// binder, as that can cause an unnecessary call to DeleteTailnetPeer when the connIO is
+		// closed.  Instead, we just close the response channel and bail out.
+		// c.f. https://github.com/coder/coder/issues/12923
+		c.logger.Info(ctx, "closed incoming coordinate call while unhealthy",
+			slog.F("peer_id", id),
+		)
+		close(resps)
+		return reqs, resps
+	}
 	cIO := newConnIO(c.ctx, ctx, logger, c.bindings, c.tunnelerCh, reqs, resps, id, name, a)
 	err := agpl.SendCtx(c.ctx, c.newConnections, cIO)
 	if err != nil {
@@ -842,7 +853,12 @@ func (q *querier) newConn(c *connIO) {
 	defer q.mu.Unlock()
 	if !q.healthy {
 		err := c.Close()
-		q.logger.Info(q.ctx, "closed incoming connection while unhealthy",
+		// This can only happen during a narrow window where we were healthy
+		// when pgCoord checked before accepting the connection, but now are
+		// unhealthy now that we get around to processing it. Seeing a small
+		// number of these logs is not worrying, but a large number probably
+		// indicates something is amiss.
+		q.logger.Warn(q.ctx, "closed incoming connection while unhealthy",
 			slog.Error(err),
 			slog.F("peer_id", c.UniqueID()),
 		)
@@ -865,6 +881,12 @@ func (q *querier) newConn(c *connIO) {
 	})
 }

+func (q *querier) isHealthy() bool {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+	return q.healthy
+}
+
 func (q *querier) cleanupConn(c *connIO) {
 	logger := q.logger.With(slog.F("peer_id", c.UniqueID()))
 	q.mu.Lock()
@@ -13,6 +13,7 @@ import (
 	"github.com/google/uuid"
 	"github.com/stretchr/testify/require"
 	"go.uber.org/mock/gomock"
+	"golang.org/x/xerrors"
 	gProto "google.golang.org/protobuf/proto"

 	"cdr.dev/slog"
@@ -21,6 +22,8 @@ import (
 	"github.com/coder/coder/v2/coderd/database"
 	"github.com/coder/coder/v2/coderd/database/dbmock"
 	"github.com/coder/coder/v2/coderd/database/dbtestutil"
+	"github.com/coder/coder/v2/coderd/database/pubsub"
+	agpl "github.com/coder/coder/v2/tailnet"
 	"github.com/coder/coder/v2/tailnet/proto"
 	"github.com/coder/coder/v2/testutil"
 )
@@ -291,3 +294,51 @@ func TestGetDebug(t *testing.T) {
 	require.Equal(t, peerID, debug.Tunnels[0].SrcID)
 	require.Equal(t, dstID, debug.Tunnels[0].DstID)
 }
+
+// TestPGCoordinatorUnhealthy tests that when the coordinator fails to send heartbeats and is
+// unhealthy it disconnects any peers and does not send any extraneous database queries.
+func TestPGCoordinatorUnhealthy(t *testing.T) {
+	t.Parallel()
+	ctx := testutil.Context(t, testutil.WaitShort)
+	logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}).Leveled(slog.LevelDebug)
+
+	ctrl := gomock.NewController(t)
+	mStore := dbmock.NewMockStore(ctrl)
+	ps := pubsub.NewInMemory()
+
+	// after 3 failed heartbeats, the coordinator is unhealthy
+	mStore.EXPECT().
+		UpsertTailnetCoordinator(gomock.Any(), gomock.Any()).
+		MinTimes(3).
+		Return(database.TailnetCoordinator{}, xerrors.New("badness"))
+	mStore.EXPECT().
+		DeleteCoordinator(gomock.Any(), gomock.Any()).
+		Times(1).
+		Return(nil)
+	// But, in particular we DO NOT want the coordinator to call DeleteTailnetPeer, as this is
+	// unnecessary and can spam the database. c.f. https://github.com/coder/coder/issues/12923
+
+	// these cleanup queries run, but we don't care for this test
+	mStore.EXPECT().CleanTailnetCoordinators(gomock.Any()).AnyTimes().Return(nil)
+	mStore.EXPECT().CleanTailnetLostPeers(gomock.Any()).AnyTimes().Return(nil)
+	mStore.EXPECT().CleanTailnetTunnels(gomock.Any()).AnyTimes().Return(nil)
+
+	coordinator, err := newPGCoordInternal(ctx, logger, ps, mStore)
+	require.NoError(t, err)
+
+	require.Eventually(t, func() bool {
+		return !coordinator.querier.isHealthy()
+	}, testutil.WaitShort, testutil.IntervalFast)
+
+	pID := uuid.UUID{5}
+	_, resps := coordinator.Coordinate(ctx, pID, "test", agpl.AgentCoordinateeAuth{ID: pID})
+	resp := testutil.RequireRecvCtx(ctx, t, resps)
+	require.Nil(t, resp, "channel should be closed")
+
+	// give the coordinator some time to process any pending work.  We are
+	// testing here that a database call is absent, so we don't want to race to
+	// shut down the test.
+	time.Sleep(testutil.IntervalMedium)
+	_ = coordinator.Close()
+	require.Eventually(t, ctrl.Satisfied, testutil.WaitShort, testutil.IntervalFast)
+}
@@ -449,8 +449,21 @@ func (s *Server) handleRegister(res wsproxysdk.RegisterWorkspaceProxyResponse) e
 }

 func (s *Server) pingSiblingReplicas(replicas []codersdk.Replica) {
+	ctx, cancel := context.WithTimeout(s.ctx, 10*time.Second)
+	defer cancel()
+
+	errStr := pingSiblingReplicas(ctx, s.Logger, &s.replicaPingSingleflight, s.derpMeshTLSConfig, replicas)
+	s.replicaErrMut.Lock()
+	s.replicaErr = errStr
+	s.replicaErrMut.Unlock()
+	if s.Options.ReplicaErrCallback != nil {
+		s.Options.ReplicaErrCallback(replicas, s.replicaErr)
+	}
+}
+
+func pingSiblingReplicas(ctx context.Context, logger slog.Logger, sf *singleflight.Group, derpMeshTLSConfig *tls.Config, replicas []codersdk.Replica) string {
 	if len(replicas) == 0 {
-		return
+		return ""
 	}

 	// Avoid pinging multiple times at once if the list hasn't changed.
@@ -462,18 +475,11 @@ func (s *Server) pingSiblingReplicas(replicas []codersdk.Replica) {
 	singleflightStr := strings.Join(relayURLs, " ") // URLs can't contain spaces.

 	//nolint:dogsled
-	_, _, _ = s.replicaPingSingleflight.Do(singleflightStr, func() (any, error) {
-		const (
-			perReplicaTimeout = 3 * time.Second
-			fullTimeout       = 10 * time.Second
-		)
-		ctx, cancel := context.WithTimeout(s.ctx, fullTimeout)
-		defer cancel()
-
+	errStrInterface, _, _ := sf.Do(singleflightStr, func() (any, error) {
 		client := http.Client{
-			Timeout: perReplicaTimeout,
+			Timeout: 3 * time.Second,
 			Transport: &http.Transport{
-				TLSClientConfig:   s.derpMeshTLSConfig,
+				TLSClientConfig:   derpMeshTLSConfig,
 				DisableKeepAlives: true,
 			},
 		}
@@ -485,7 +491,7 @@ func (s *Server) pingSiblingReplicas(replicas []codersdk.Replica) {
 				err := replicasync.PingPeerReplica(ctx, client, peer.RelayAddress)
 				if err != nil {
 					errs <- xerrors.Errorf("ping sibling replica %s (%s): %w", peer.Hostname, peer.RelayAddress, err)
-					s.Logger.Warn(ctx, "failed to ping sibling replica, this could happen if the replica has shutdown",
+					logger.Warn(ctx, "failed to ping sibling replica, this could happen if the replica has shutdown",
 						slog.F("replica_hostname", peer.Hostname),
 						slog.F("replica_relay_address", peer.RelayAddress),
 						slog.Error(err),
@@ -504,20 +510,14 @@ func (s *Server) pingSiblingReplicas(replicas []codersdk.Replica) {
 			}
 		}

-		s.replicaErrMut.Lock()
-		defer s.replicaErrMut.Unlock()
-		s.replicaErr = ""
-		if len(replicaErrs) > 0 {
-			s.replicaErr = fmt.Sprintf("Failed to dial peers: %s", strings.Join(replicaErrs, ", "))
+		if len(replicaErrs) == 0 {
+			return "", nil
 		}
-		if s.Options.ReplicaErrCallback != nil {
-			s.Options.ReplicaErrCallback(replicas, s.replicaErr)
-		}
-
-		//nolint:nilnil // we don't actually use the return value of the
-		// singleflight here
-		return nil, nil
+		return fmt.Sprintf("Failed to dial peers: %s", strings.Join(replicaErrs, ", ")), nil
 	})
+
+	//nolint:forcetypeassert
+	return errStrInterface.(string)
 }

 func (s *Server) handleRegisterFailure(err error) {
@@ -590,7 +590,8 @@ func (s *Server) healthReport(rw http.ResponseWriter, r *http.Request) {

 	s.replicaErrMut.Lock()
 	if s.replicaErr != "" {
-		report.Errors = append(report.Errors, "High availability networking: it appears you are running more than one replica of the proxy, but the replicas are unable to establish a mesh for networking: "+s.replicaErr)
+		report.Warnings = append(report.Warnings,
+			"High availability networking: it appears you are running more than one replica of the proxy, but the replicas are unable to establish a mesh for networking: "+s.replicaErr)
 	}
 	s.replicaErrMut.Unlock()

@@ -563,7 +563,7 @@ func TestWorkspaceProxyDERPMeshProbe(t *testing.T) {
 		return proxyRes
 	}

-	registerBrokenProxy := func(ctx context.Context, t *testing.T, primaryAccessURL *url.URL, accessURL, token string) {
+	registerBrokenProxy := func(ctx context.Context, t *testing.T, primaryAccessURL *url.URL, accessURL, token string) uuid.UUID {
 		t.Helper()
 		// Create a HTTP server that always replies with 500.
 		srv := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
@@ -574,21 +574,23 @@ func TestWorkspaceProxyDERPMeshProbe(t *testing.T) {
 		// Register a proxy.
 		wsproxyClient := wsproxysdk.New(primaryAccessURL)
 		wsproxyClient.SetSessionToken(token)
-
 		hostname, err := cryptorand.String(6)
 		require.NoError(t, err)
+		replicaID := uuid.New()
 		_, err = wsproxyClient.RegisterWorkspaceProxy(ctx, wsproxysdk.RegisterWorkspaceProxyRequest{
 			AccessURL:           accessURL,
 			WildcardHostname:    "",
 			DerpEnabled:         true,
 			DerpOnly:            false,
-			ReplicaID:           uuid.New(),
+			ReplicaID:           replicaID,
 			ReplicaHostname:     hostname,
 			ReplicaError:        "",
 			ReplicaRelayAddress: srv.URL,
 			Version:             buildinfo.Version(),
 		})
 		require.NoError(t, err)
+
+		return replicaID
 	}

 	t.Run("ProbeOK", func(t *testing.T) {
@@ -781,8 +783,120 @@ func TestWorkspaceProxyDERPMeshProbe(t *testing.T) {
 		resp.Body.Close()
 		require.NoError(t, err)

-		require.Len(t, respJSON.Errors, 1, "proxy is healthy")
-		require.Contains(t, respJSON.Errors[0], "High availability networking")
+		require.Len(t, respJSON.Warnings, 1, "proxy is healthy")
+		require.Contains(t, respJSON.Warnings[0], "High availability networking")
+	})
+
+	// This test catches a regression we detected on dogfood which caused
+	// proxies to remain unhealthy after a mesh failure if they dropped to zero
+	// siblings after the failure.
+	t.Run("HealthyZero", func(t *testing.T) {
+		t.Parallel()
+
+		deploymentValues := coderdtest.DeploymentValues(t)
+		deploymentValues.Experiments = []string{
+			"*",
+		}
+
+		client, closer, api, _ := coderdenttest.NewWithAPI(t, &coderdenttest.Options{
+			Options: &coderdtest.Options{
+				DeploymentValues:         deploymentValues,
+				AppHostname:              "*.primary.test.coder.com",
+				IncludeProvisionerDaemon: true,
+				RealIPConfig: &httpmw.RealIPConfig{
+					TrustedOrigins: []*net.IPNet{{
+						IP:   net.ParseIP("127.0.0.1"),
+						Mask: net.CIDRMask(8, 32),
+					}},
+					TrustedHeaders: []string{
+						"CF-Connecting-IP",
+					},
+				},
+			},
+			LicenseOptions: &coderdenttest.LicenseOptions{
+				Features: license.Features{
+					codersdk.FeatureWorkspaceProxy: 1,
+				},
+			},
+		})
+		t.Cleanup(func() {
+			_ = closer.Close()
+		})
+
+		proxyURL, err := url.Parse("https://proxy2.test.coder.com")
+		require.NoError(t, err)
+
+		// Create 1 real proxy replica.
+		replicaPingErr := make(chan string, 4)
+		proxy := coderdenttest.NewWorkspaceProxyReplica(t, api, client, &coderdenttest.ProxyOptions{
+			Name:     "proxy-2",
+			ProxyURL: proxyURL,
+			ReplicaPingCallback: func(replicas []codersdk.Replica, err string) {
+				replicaPingErr <- err
+			},
+		})
+
+		ctx := testutil.Context(t, testutil.WaitLong)
+		otherReplicaID := registerBrokenProxy(ctx, t, api.AccessURL, proxyURL.String(), proxy.Options.ProxySessionToken)
+
+		// Force the proxy to re-register immediately.
+		err = proxy.RegisterNow()
+		require.NoError(t, err, "failed to force proxy to re-register")
+
+		// Wait for the ping to fail.
+		for {
+			replicaErr := testutil.RequireRecvCtx(ctx, t, replicaPingErr)
+			t.Log("replica ping error:", replicaErr)
+			if replicaErr != "" {
+				break
+			}
+		}
+
+		// GET /healthz-report
+		u := proxy.ServerURL.ResolveReference(&url.URL{Path: "/healthz-report"})
+		req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
+		require.NoError(t, err)
+		resp, err := http.DefaultClient.Do(req)
+		require.NoError(t, err)
+		var respJSON codersdk.ProxyHealthReport
+		err = json.NewDecoder(resp.Body).Decode(&respJSON)
+		resp.Body.Close()
+		require.NoError(t, err)
+		require.Len(t, respJSON.Warnings, 1, "proxy is healthy")
+		require.Contains(t, respJSON.Warnings[0], "High availability networking")
+
+		// Deregister the other replica.
+		wsproxyClient := wsproxysdk.New(api.AccessURL)
+		wsproxyClient.SetSessionToken(proxy.Options.ProxySessionToken)
+		err = wsproxyClient.DeregisterWorkspaceProxy(ctx, wsproxysdk.DeregisterWorkspaceProxyRequest{
+			ReplicaID: otherReplicaID,
+		})
+		require.NoError(t, err)
+
+		// Force the proxy to re-register immediately.
+		err = proxy.RegisterNow()
+		require.NoError(t, err, "failed to force proxy to re-register")
+
+		// Wait for the ping to be skipped.
+		for {
+			replicaErr := testutil.RequireRecvCtx(ctx, t, replicaPingErr)
+			t.Log("replica ping error:", replicaErr)
+			// Should be empty because there are no more peers. This was where
+			// the regression was.
+			if replicaErr == "" {
+				break
+			}
+		}
+
+		// GET /healthz-report
+		req, err = http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
+		require.NoError(t, err)
+		resp, err = http.DefaultClient.Do(req)
+		require.NoError(t, err)
+		err = json.NewDecoder(resp.Body).Decode(&respJSON)
+		resp.Body.Close()
+		require.NoError(t, err)
+		require.Len(t, respJSON.Warnings, 0, "proxy is unhealthy")
 	})
 }

@@ -474,15 +474,18 @@ func (p *Server) isClosed() bool {
 	}
 }

-// Shutdown triggers a graceful exit of each registered provisioner.
-func (p *Server) Shutdown(ctx context.Context) error {
+// Shutdown gracefully exists with the option to cancel the active job.
+// If false, it will wait for the job to complete.
+//
+//nolint:revive
+func (p *Server) Shutdown(ctx context.Context, cancelActiveJob bool) error {
 	p.mutex.Lock()
 	p.opts.Logger.Info(ctx, "attempting graceful shutdown")
 	if !p.shuttingDownB {
 		close(p.shuttingDownCh)
 		p.shuttingDownB = true
 	}
-	if p.activeJob != nil {
+	if cancelActiveJob && p.activeJob != nil {
 		p.activeJob.Cancel()
 	}
 	p.mutex.Unlock()
@@ -671,7 +671,7 @@ func TestProvisionerd(t *testing.T) {
 			}),
 		})
 		require.Condition(t, closedWithin(updateChan, testutil.WaitShort))
-		err := server.Shutdown(context.Background())
+		err := server.Shutdown(context.Background(), true)
 		require.NoError(t, err)
 		require.Condition(t, closedWithin(completeChan, testutil.WaitShort))
 		require.NoError(t, server.Close())
@@ -762,7 +762,7 @@ func TestProvisionerd(t *testing.T) {
 		require.Condition(t, closedWithin(completeChan, testutil.WaitShort))
 		ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitShort)
 		defer cancel()
-		require.NoError(t, server.Shutdown(ctx))
+		require.NoError(t, server.Shutdown(ctx, true))
 		require.NoError(t, server.Close())
 	})

@@ -853,7 +853,7 @@ func TestProvisionerd(t *testing.T) {
 		require.Condition(t, closedWithin(completeChan, testutil.WaitShort))
 		ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitShort)
 		defer cancel()
-		require.NoError(t, server.Shutdown(ctx))
+		require.NoError(t, server.Shutdown(ctx, true))
 		require.NoError(t, server.Close())
 	})

@@ -944,7 +944,7 @@ func TestProvisionerd(t *testing.T) {
 		t.Log("completeChan closed")
 		ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitShort)
 		defer cancel()
-		require.NoError(t, server.Shutdown(ctx))
+		require.NoError(t, server.Shutdown(ctx, true))
 		require.NoError(t, server.Close())
 	})

@@ -1039,7 +1039,7 @@ func TestProvisionerd(t *testing.T) {
 		require.Condition(t, closedWithin(completeChan, testutil.WaitShort))
 		ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitShort)
 		defer cancel()
-		require.NoError(t, server.Shutdown(ctx))
+		require.NoError(t, server.Shutdown(ctx, true))
 		require.NoError(t, server.Close())
 		assert.Equal(t, ops[len(ops)-1], "CompleteJob")
 		assert.Contains(t, ops[0:len(ops)-1], "Log: Cleaning Up | ")
@@ -1076,7 +1076,7 @@ func createProvisionerd(t *testing.T, dialer provisionerd.Dialer, connector prov
 	t.Cleanup(func() {
 		ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitShort)
 		defer cancel()
-		_ = server.Shutdown(ctx)
+		_ = server.Shutdown(ctx, true)
 		_ = server.Close()
 	})
 	return server
@@ -0,0 +1,31 @@
+package pty
+
+// TerminalState differs per-platform.
+type TerminalState struct {
+	state terminalState
+}
+
+// MakeInputRaw calls term.MakeRaw on non-Windows platforms. On Windows it sets
+// special terminal modes that enable VT100 emulation as well as setting the
+// same modes that term.MakeRaw sets.
+//
+//nolint:revive
+func MakeInputRaw(fd uintptr) (*TerminalState, error) {
+	return makeInputRaw(fd)
+}
+
+// MakeOutputRaw does nothing on non-Windows platforms. On Windows it sets
+// special terminal modes that enable VT100 emulation as well as setting the
+// same modes that term.MakeRaw sets.
+//
+//nolint:revive
+func MakeOutputRaw(fd uintptr) (*TerminalState, error) {
+	return makeOutputRaw(fd)
+}
+
+// RestoreTerminal restores the terminal back to its original state.
+//
+//nolint:revive
+func RestoreTerminal(fd uintptr, state *TerminalState) error {
+	return restoreTerminal(fd, state)
+}
@@ -0,0 +1,36 @@
+//go:build !windows
+// +build !windows
+
+package pty
+
+import "golang.org/x/term"
+
+type terminalState *term.State
+
+//nolint:revive
+func makeInputRaw(fd uintptr) (*TerminalState, error) {
+	s, err := term.MakeRaw(int(fd))
+	if err != nil {
+		return nil, err
+	}
+	return &TerminalState{
+		state: s,
+	}, nil
+}
+
+//nolint:revive
+func makeOutputRaw(_ uintptr) (*TerminalState, error) {
+	// Does nothing. makeInputRaw does enough for both input and output.
+	return &TerminalState{
+		state: nil,
+	}, nil
+}
+
+//nolint:revive
+func restoreTerminal(fd uintptr, state *TerminalState) error {
+	if state == nil || state.state == nil {
+		return nil
+	}
+
+	return term.Restore(int(fd), state.state)
+}
@@ -0,0 +1,65 @@
+//go:build windows
+// +build windows
+
+package pty
+
+import "golang.org/x/sys/windows"
+
+type terminalState uint32
+
+// This is adapted from term.MakeRaw, but adds
+// ENABLE_VIRTUAL_TERMINAL_PROCESSING to the output mode and
+// ENABLE_VIRTUAL_TERMINAL_INPUT to the input mode.
+//
+// See: https://github.com/golang/term/blob/5b15d269ba1f54e8da86c8aa5574253aea0c2198/term_windows.go#L23
+//
+// Copyright 2019 The Go Authors. BSD-3-Clause license. See:
+// https://github.com/golang/term/blob/master/LICENSE
+func makeRaw(handle windows.Handle, input bool) (uint32, error) {
+	var prevState uint32
+	if err := windows.GetConsoleMode(handle, &prevState); err != nil {
+		return 0, err
+	}
+
+	var raw uint32
+	if input {
+		raw = prevState &^ (windows.ENABLE_ECHO_INPUT | windows.ENABLE_PROCESSED_INPUT | windows.ENABLE_LINE_INPUT | windows.ENABLE_PROCESSED_OUTPUT)
+		raw |= windows.ENABLE_VIRTUAL_TERMINAL_INPUT
+	} else {
+		raw = prevState | windows.ENABLE_VIRTUAL_TERMINAL_PROCESSING
+	}
+
+	if err := windows.SetConsoleMode(handle, raw); err != nil {
+		return 0, err
+	}
+	return prevState, nil
+}
+
+//nolint:revive
+func makeInputRaw(handle uintptr) (*TerminalState, error) {
+	prevState, err := makeRaw(windows.Handle(handle), true)
+	if err != nil {
+		return nil, err
+	}
+
+	return &TerminalState{
+		state: terminalState(prevState),
+	}, nil
+}
+
+//nolint:revive
+func makeOutputRaw(handle uintptr) (*TerminalState, error) {
+	prevState, err := makeRaw(windows.Handle(handle), false)
+	if err != nil {
+		return nil, err
+	}
+
+	return &TerminalState{
+		state: terminalState(prevState),
+	}, nil
+}
+
+//nolint:revive
+func restoreTerminal(handle uintptr, state *TerminalState) error {
+	return windows.SetConsoleMode(windows.Handle(handle), uint32(state.state))
+}
@@ -233,6 +233,46 @@ describe("CreateWorkspacePage", () => {
    );
  });

+  it("optional external auth is optional", async () => {
+    jest
+      .spyOn(API, "getWorkspaceQuota")
+      .mockResolvedValueOnce(MockWorkspaceQuota);
+    jest
+      .spyOn(API, "getUsers")
+      .mockResolvedValueOnce({ users: [MockUser], count: 1 });
+    jest.spyOn(API, "createWorkspace").mockResolvedValueOnce(MockWorkspace);
+    jest
+      .spyOn(API, "getTemplateVersionExternalAuth")
+      .mockResolvedValue([
+        { ...MockTemplateVersionExternalAuthGithub, optional: true },
+      ]);
+
+    renderCreateWorkspacePage();
+    await waitForLoaderToBeRemoved();
+
+    const nameField = await screen.findByLabelText(nameLabelText);
+    // have to use fireEvent b/c userEvent isn't cleaning up properly between tests
+    fireEvent.change(nameField, {
+      target: { value: "test" },
+    });
+
+    // Ensure we're not logged in
+    await screen.findByText("Login with GitHub");
+
+    const submitButton = screen.getByText(createWorkspaceText);
+    await userEvent.click(submitButton);
+
+    await waitFor(() =>
+      expect(API.createWorkspace).toBeCalledWith(
+        MockUser.organization_ids[0],
+        MockUser.id,
+        expect.objectContaining({
+          ...MockWorkspaceRequest,
+        }),
+      ),
+    );
+  });
+
  it("auto create a workspace if uses mode=auto", async () => {
    const param = "first_parameter";
    const paramValue = "It works!";
@@ -105,6 +105,11 @@ const CreateWorkspacePage: FC = () => {
    userParametersQuery.data ? userParametersQuery.data : [],
  );

+  const hasAllRequiredExternalAuth = Boolean(
+    !isLoadingExternalAuth &&
+    externalAuth?.every((auth) => auth.optional || auth.authenticated),
+  );
+
  const autoCreationStartedRef = useRef(false);
  const automateWorkspaceCreation = useEffectEvent(async () => {
    if (autoCreationStartedRef.current) {
@@ -143,14 +148,15 @@ const CreateWorkspacePage: FC = () => {
      </Helmet>
      {loadFormDataError && <ErrorAlert error={loadFormDataError} />}
      {isLoadingFormData ||
-      isLoadingExternalAuth ||
-      autoCreateWorkspaceMutation.isLoading ? (
+        isLoadingExternalAuth ||
+        autoCreateWorkspaceMutation.isLoading ? (
        <Loader />
      ) : (
        <CreateWorkspacePageView
          mode={mode}
          defaultName={defaultName}
          defaultOwner={me}
+          hasAllRequiredExternalAuth={hasAllRequiredExternalAuth}
          autofillParameters={autofillParameters}
          error={createWorkspaceMutation.error}
          resetMutation={createWorkspaceMutation.reset}
@@ -198,10 +204,10 @@ const useExternalAuth = (versionId: string | undefined) => {
  const { data: externalAuth, isLoading: isLoadingExternalAuth } = useQuery(
    versionId
      ? {
-          ...templateVersionExternalAuth(versionId),
-          refetchInterval:
-            externalAuthPollingState === "polling" ? 1000 : false,
-        }
+        ...templateVersionExternalAuth(versionId),
+        refetchInterval:
+          externalAuthPollingState === "polling" ? 1000 : false,
+      }
      : { enabled: false },
  );

@@ -21,6 +21,7 @@ const meta: Meta<typeof CreateWorkspacePageView> = {
    template: MockTemplate,
    parameters: [],
    externalAuth: [],
+    hasAllRequiredExternalAuth: true,
    mode: "form",
    permissions: {
      createWorkspaceForUser: true,
@@ -134,6 +135,7 @@ export const ExternalAuth: Story = {
        optional: true,
      },
    ],
+    hasAllRequiredExternalAuth: false,
  },
 };

@@ -159,6 +161,7 @@ export const ExternalAuthError: Story = {
        optional: true,
      },
    ],
+    hasAllRequiredExternalAuth: false,
  },
 };

@@ -60,6 +60,7 @@ export interface CreateWorkspacePageViewProps {
  externalAuth: TypesGen.TemplateVersionExternalAuth[];
  externalAuthPollingState: ExternalAuthPollingState;
  startPollingExternalAuth: () => void;
+  hasAllRequiredExternalAuth: boolean;
  parameters: TypesGen.TemplateVersionParameter[];
  autofillParameters: AutofillBuildParameter[];
  permissions: CreateWSPermissions;
@@ -82,6 +83,7 @@ export const CreateWorkspacePageView: FC<CreateWorkspacePageViewProps> = ({
  externalAuth,
  externalAuthPollingState,
  startPollingExternalAuth,
+  hasAllRequiredExternalAuth,
  parameters,
  autofillParameters,
  permissions,
@@ -92,7 +94,6 @@ export const CreateWorkspacePageView: FC<CreateWorkspacePageViewProps> = ({
  const [owner, setOwner] = useState(defaultOwner);
  const [searchParams] = useSearchParams();
  const disabledParamsList = searchParams?.get("disable_params")?.split(",");
-  const requiresExternalAuth = externalAuth.some((auth) => !auth.authenticated);
  const [suggestedName, setSuggestedName] = useState(() =>
    generateWorkspaceName(),
  );
@@ -117,7 +118,7 @@ export const CreateWorkspacePageView: FC<CreateWorkspacePageViewProps> = ({
      }),
      enableReinitialize: true,
      onSubmit: (request) => {
-        if (requiresExternalAuth) {
+        if (!hasAllRequiredExternalAuth) {
          return;
        }

@@ -144,10 +145,6 @@ export const CreateWorkspacePageView: FC<CreateWorkspacePageViewProps> = ({
    [autofillParameters],
  );

-  const hasAllRequiredExternalAuth = externalAuth.every(
-    (auth) => auth.optional || auth.authenticated,
-  );
-
  return (
    <Margins size="medium">
      <PageHeader actions={<Button onClick={onCancel}>Cancel</Button>}>
@@ -29,7 +29,6 @@ export const ExternalAuthButton: FC<ExternalAuthButtonProps> = ({
        <LoadingButton
          fullWidth
          loading={isLoading}
-          href={auth.authenticate_url}
          variant="contained"
          size="xlarge"
          startIcon={
Author	SHA1	Message	Date
Stephen Kirby	864ba21a32	2.9.4 changelog	2024-04-22 20:10:30 +00:00
Kyle Carberry	57a4f50747	chore: add generate script for azure instance identity (#13028 ) * chore: add generate script for azure instance identity This also adds new issuing certificates from: https://learn.microsoft.com/en-us/azure/security/fundamentals/azure-ca-details?tabs=certificate-authority-chains * Fix shell lint * Fix shell fmt * Fix RSA issuing certificate	2024-04-22 19:42:51 +00:00
Stephen Kirby	a0cf836c8a	changelog for 2.9.3	2024-04-17 22:21:07 +00:00
Colin Adler	cfa5be52f7	Revert "feat: add src_id and dst_id indexes to tailnet_tunnels (#12911 )" Can't backport non-contiguous migrations This reverts commit `b2e8b93759`.	2024-04-17 21:27:46 +00:00
Dean Sheather	bf296b2f19	fix: make terminal raw in ssh command on windows (#12990 ) (cherry picked from commit `d426569d4a`)	2024-04-17 21:05:38 +00:00
Spike Curtis	b2e8b93759	feat: add src_id and dst_id indexes to tailnet_tunnels (#12911 ) Fixes #12780 Adds indexes to the `tailnet_tunnels` table to speed up `GetTailnetTunnelPeerIDs` and `GetTailnetTunnelPeerBindings` queries, which match on `src_id` and `dst_id`. (cherry picked from commit `a231b5aef5`)	2024-04-17 21:02:58 +00:00
Spike Curtis	31ce3f5aa4	fix: stop sending DeleteTailnetPeer when coordinator is unhealthy (#12925 ) fixes #12923 Prevents Coordinate peer connections from generating spurious database queries like DeleteTailnetPeer when the coordinator is unhealthy. It does this by checking the health of the querier before accepting a connection, rather than unconditionally accepting it only for it to get swatted down later. (cherry picked from commit `06eae954c9`)	2024-04-17 21:02:52 +00:00
Stephen Kirby	b4b8d095a4	mini changelog	2024-04-05 18:41:42 +00:00
Steven Masley	9310973f36	chore: external auth flow opens new window, it does not need an href (#12586 )	2024-04-05 15:53:05 +00:00
Stephen Kirby	9cdd580dcf	merged	2024-04-05 15:52:31 +00:00
Kayla Washburn-Love	df9990a42e	fix: create workspace with optional auth providers (#12729 )	2024-04-05 15:52:19 +00:00
Dean Sheather	2a2fd706b5	chore: add v2.9.1 changelog	2024-03-19 12:37:21 +00:00
Kyle Carberry	b49c01546f	fix: separate signals for passive, active, and forced shutdown (#12358 ) * fix: separate signals for passive, active, and forced shutdown `SIGTERM`: Passive shutdown stopping provisioner daemons from accepting new jobs but waiting for existing jobs to successfully complete. `SIGINT` (old existing behavior): Notify provisioner daemons to cancel in-flight jobs, wait 5s for jobs to be exited, then force quit. `SIGKILL`: Untouched from before, will force-quit. * Revert dramatic signal changes * Rename * Fix shutdown behavior for provisioner daemons * Add test for graceful shutdown (cherry picked from commit `895df54051`)	2024-03-19 12:25:04 +00:00
Dean Sheather	374127eba5	fix: prevent single replica proxies from staying unhealthy (#12641 ) In the peer healthcheck code, when an error pinging peers is detected we write a "replicaErr" string with the error reason. However, if there are no peer replicas to ping we returned early without setting the string to empty. This would cause replicas that had peers (which were failing) and then the peers left to permanently show an error until a new peer appeared. Also demotes DERP replica checking to a "warning" rather than an "error" which should prevent the primary from removing the proxy from the region map if DERP meshing is non-functional. This can happen without causing problems if the peer is shutting down so we don't want to disrupt everything if there isn't an issue. (cherry picked from commit `cf50461ab4`)	2024-03-19 12:23:40 +00:00
Gábor	793df2edbd	fix(migration): removed hardcoded public (#12620 ) (cherry picked from commit `9c69672382`)	2024-03-19 12:22:48 +00:00