Use tee

codersdk/toolsdk/bash.go

@@ -50,8 +50,13 @@ The workspace parameter supports various formats:
 The timeout_ms parameter specifies the command timeout in milliseconds (defaults to 60000ms, maximum of 300000ms).
 If the command times out, all output captured up to that point is returned with a cancellation message.
 
-For background commands (background: true), output is captured until the timeout is reached, then the command
-continues running in the background. The captured output is returned as the result.
+For background commands (background: true), output is captured until the timeout
+is reached, then the command continues running in the background. The initially
+captured output is returned in the result. Make sure to examine the output along
+with the exit code to determine if the command started successfully. An exit
+code of 124 means the timeout was reached and is expected for background
+commands. Consider a lower timeout of a few seconds, enough to get some output
+to validate that the command started successfully.
 
 For file operations (list, write, edit), always prefer the dedicated file tools.
 Do not use bash commands (ls, cat, echo, heredoc, etc.) to list, write, or read
@@ -129,10 +134,18 @@ Examples:
 		}
 		command := args.Command
 		if args.Background {
-			// For background commands, use nohup directly to ensure they survive SSH session
-			// termination. This captures output normally but allows the process to continue
-			// running even after the SSH connection closes.
-			command = fmt.Sprintf("nohup %s </dev/null 2>&1", args.Command)
+			// For background commands, use nohup to ensure they survive SSH session
+			// termination.  To capture output, write to a file instead of stdout
+			// because commands may crash when the pipes close.  Lastly, wrap with
+			// `$SHELL -c` because the command may include shellisms like cd, &&,
+			// etc. that nohup cannot handle.
+			programName := strings.SplitN(args.Command, " ", 2)[0]
+			escapedCommand := strings.ReplaceAll(args.Command, "'", "'\"'\"'")
+			command = fmt.Sprintf(
+				`tmp="$(mktemp --suffix=%q)" ; echo "$tmp" ; nohup </dev/null >>"$tmp" 2>&1 "${SHELL:-sh}" -c '%s'`,
+				"-"+programName,
+				escapedCommand,
+			)
 		}
 
 		// Create context with command timeout (replace the broader MCP timeout)
@@ -143,15 +156,35 @@ Examples:
 		output, err := executeCommandWithTimeout(commandCtx, session, command)
 		outputStr := strings.TrimSpace(string(output))
 
+		// For background commands, get the real output from the temp file.
+		if args.Background {
+			tmpfile := outputStr
+			reader, _, rerr := conn.ReadFile(ctx, tmpfile, 0, maxFileLimit)
+			if rerr != nil {
+				outputStr = fmt.Sprintf("Failed to read command output: %s", rerr)
+			} else {
+				defer reader.Close()
+				bs, rerr := io.ReadAll(reader)
+				if rerr != nil {
+					outputStr = fmt.Sprintf("Failed to read command output: %s", rerr)
+				} else {
+					outputStr = strings.TrimSpace(string(bs))
+				}
+			}
+		}
+
 		// Handle command execution results
 		if err != nil {
 			// Check if the command timed out
 			if errors.Is(context.Cause(commandCtx), context.DeadlineExceeded) {
+				msg := "Command canceled due to timeout"
 				if args.Background {
-					outputStr += "\nCommand continues running in background"
-				} else {
-					outputStr += "\nCommand canceled due to timeout"
+					msg = "Command continues running in background"
 				}
+				if outputStr != "" {
+					outputStr += "\n"
+				}
+				outputStr += msg
 				return WorkspaceBashResult{
 					Output:   outputStr,
 					ExitCode: 124,
@@ -363,9 +396,10 @@ func executeCommandWithTimeout(ctx context.Context, session *gossh.Session, comm
 		// Command completed normally
 		return safeWriter.Bytes(), err
 	case <-ctx.Done():
-		// Context was canceled (timeout or other cancellation)
-		// Close the session to stop the command, but handle errors gracefully
-		closeErr := session.Close()
+		// Context was canceled (timeout or other cancellation).  The SSH server
+		// will *not* terminate commands spawned without a tty, so send a signal
+		// then close the SSH session.
+		closeErr := errors.Join(session.Signal(gossh.SIGTERM), session.Close())
 
 		// Give a brief moment to collect any remaining output and for goroutines to finish
 		timer := time.NewTimer(100 * time.Millisecond)

codersdk/toolsdk/bash_test.go

@@ -2,7 +2,11 @@ package toolsdk_test
 
 import (
 	"context"
+	"fmt"
+	"os"
+	"path/filepath"
 	"runtime"
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -229,10 +233,11 @@ func TestWorkspaceBashTimeoutIntegration(t *testing.T) {
 		deps, err := toolsdk.NewDeps(client)
 		require.NoError(t, err)
 
+		sentinelFilePath := filepath.Join(os.TempDir(), "fg-test-sentinel")
 		args := toolsdk.WorkspaceBashArgs{
 			Workspace: workspace.Name,
-			Command:   `echo "123" && sleep 60 && echo "456"`, // This command would take 60+ seconds
-			TimeoutMs: 2000,                                   // 2 seconds timeout - should timeout after first echo
+			Command:   sentinelCommand(60, sentinelFilePath), // This command would take 60+ seconds.
+			TimeoutMs: 2000,                                  // 2 seconds timeout - should timeout after first echo
 		}
 
 		result, err := testTool(t, toolsdk.WorkspaceBash, deps, args)
@@ -247,11 +252,27 @@ func TestWorkspaceBashTimeoutIntegration(t *testing.T) {
 
 		t.Logf("result.Output: %s", result.Output)
 
-		// Should contain the first echo output
-		require.Contains(t, result.Output, "123")
+		// Should contain the started output.
+		require.Contains(t, result.Output, "started")
 
-		// Should NOT contain the second echo (it never executed due to timeout)
-		require.NotContains(t, result.Output, "456", "Should not contain output after sleep")
+		// Should NOT contain the done echo
+		require.NotContains(t, result.Output, "done", "Should not contain output after sleep")
+
+		// Should contain the started text.
+		b, err := os.ReadFile(sentinelFilePath)
+		require.NoError(t, err)
+		require.Contains(t, string(b), "started")
+
+		// The command should have caught the exit and logged that.
+		require.Eventually(t, func() bool {
+			b, err := os.ReadFile(sentinelFilePath)
+			return err == nil && strings.Contains(string(b), "exited")
+		}, testutil.WaitMedium, testutil.IntervalMedium)
+
+		// But it should never have made it to the done echo.
+		b, err = os.ReadFile(sentinelFilePath)
+		require.NoError(t, err)
+		require.NotContains(t, string(b), "done")
 	})
 
 	t.Run("NormalCommandExecution", func(t *testing.T) {
@@ -393,7 +414,6 @@ func TestWorkspaceBashBackgroundIntegration(t *testing.T) {
 
 		client, workspace, agentToken := setupWorkspaceForAgent(t, nil)
 
-		// Start the agent and wait for it to be fully ready
 		_ = agenttest.New(t, client.URL, agentToken)
 
 		// Wait for workspace agents to be ready
@@ -402,11 +422,13 @@ func TestWorkspaceBashBackgroundIntegration(t *testing.T) {
 		deps, err := toolsdk.NewDeps(client)
 		require.NoError(t, err)
 
+		sentinelFilePath := filepath.Join(os.TempDir(), "bg-test-sentinel")
 		args := toolsdk.WorkspaceBashArgs{
-			Workspace:  workspace.Name,
-			Command:    `echo "started" && sleep 4 && echo "done" > /tmp/bg-test-done`, // Command that will timeout but continue
-			TimeoutMs:  2000,                                                           // 2000ms timeout (shorter than command duration)
-			Background: true,                                                           // Run in background
+			Workspace: workspace.Name,
+			// Command that will run for at least 4 seconds.
+			Command:    sentinelCommand(4, sentinelFilePath),
+			TimeoutMs:  2000, // 2000ms timeout (shorter than command duration)
+			Background: true, // Run in background
 		}
 
 		result, err := testTool(t, toolsdk.WorkspaceBash, deps, args)
@@ -422,17 +444,35 @@ func TestWorkspaceBashBackgroundIntegration(t *testing.T) {
 		// Should capture output before timeout
 		require.Contains(t, result.Output, "started", "Should contain output captured before timeout")
 
+		// Should contain the started text.
+		b, err := os.ReadFile(sentinelFilePath)
+		require.NoError(t, err)
+		require.Contains(t, string(b), "started")
+
 		// Should contain background continuation message
 		require.Contains(t, result.Output, "Command continues running in background")
 
 		// Wait for the background command to complete (even though SSH session timed out)
 		require.Eventually(t, func() bool {
-			checkArgs := toolsdk.WorkspaceBashArgs{
-				Workspace: workspace.Name,
-				Command:   `cat /tmp/bg-test-done 2>/dev/null || echo "not found"`,
-			}
-			checkResult, err := toolsdk.WorkspaceBash.Handler(t.Context(), deps, checkArgs)
-			return err == nil && checkResult.Output == "done"
+			b, err := os.ReadFile(sentinelFilePath)
+			return err == nil && strings.Contains(string(b), "done")
 		}, testutil.WaitMedium, testutil.IntervalMedium, "Background command should continue running and complete after timeout")
 	})
 }
+
+// sentinelCommand takes a timeout and sentinel file and creates a command that:
+//
+//  1. Writes "started" to stdout and the file (overwriting it)
+//  2. Waits `time` seconds
+//  3. Writes "slept" to stdout and waits for that to flush
+//  4. Writes "done" to stdout and the file (appending)
+//  5. Writes "exit" to stdout and the file (appending) on exit (assuming the
+//     command is not violently killed).
+func sentinelCommand(time int64, filePath string) string {
+	// The extra sleep 1 is so the echo has a chance to flush (to test closed
+	// pipes), otherwise the command can exit before it crashes.
+	return fmt.Sprintf(
+		`trap "echo exited | tee -a %s" EXIT ; echo started | tee %s && sleep %d && echo slept && sleep 1 && echo done | tee -a %s`,
+		filePath, filePath, time, filePath,
+	)
+}