wip

coderd/provisionerdserver/provisionerdserver.go

@@ -510,6 +510,10 @@ func (s *server) acquireProtoJob(ctx context.Context, job database.ProvisionerJo
 		if err != nil {
 			return nil, failJob(fmt.Sprintf("get owner: %s", err))
 		}
+		terraformValues, err := s.Database.GetTemplateVersionTerraformValues(ctx, templateVersion.ID)
+		if err != nil && !xerrors.Is(err, sql.ErrNoRows) { // Templates can exist without terraform values
+			return nil, failJob(fmt.Sprintf("get template version terraform values: %s", err))
+		}
 		var ownerSSHPublicKey, ownerSSHPrivateKey string
 		if ownerSSHKey, err := s.Database.GetGitSSHKey(ctx, owner.ID); err != nil {
 			if !xerrors.Is(err, sql.ErrNoRows) {
@@ -699,6 +703,11 @@ func (s *server) acquireProtoJob(ctx context.Context, job database.ProvisionerJo
 			}
 		}
 
+		var moduleFilesUUID *string
+		if terraformValues.CachedModuleFiles.Valid {
+			moduleFilesUUID = ptr.Ref(terraformValues.CachedModuleFiles.UUID.String())
+		}
+
 		activeVersion := template.ActiveVersionID == templateVersion.ID
 		protoJob.Type = &proto.AcquiredJob_WorkspaceBuild_{
 			WorkspaceBuild: &proto.AcquiredJob_WorkspaceBuild{
@@ -709,6 +718,7 @@ func (s *server) acquireProtoJob(ctx context.Context, job database.ProvisionerJo
 				PreviousParameterValues: convertRichParameterValues(lastWorkspaceBuildParameters),
 				VariableValues:          asVariableValues(templateVariables),
 				ExternalAuthProviders:   externalAuthProviders,
+				ModulesFilesUuid:        moduleFilesUUID,
 				// If active and experiment is enabled, allow workspace reuse existing TF
 				// workspaces (directories) for a faster startup.
 				ExpReuseTerraformWorkspace: ptr.Ref(s.Experiments.Enabled(codersdk.ExperimentTerraformWorkspace) && // Experiment required
@@ -1526,6 +1536,16 @@ UploadFileStream:
 	return nil
 }
 
+func (s *server) DownloadFile(request *proto.DownloadFileRequest, stream proto.DRPCProvisionerDaemon_DownloadFileStream) error {
+	// Always terminate the stream with an empty response.
+	if request == nil || request.FileUuid == "" {
+		_ = stream.Close()
+		return xerrors.New("file request uuid is required")
+	}
+
+	return fmt.Errorf("not implemented")
+}
+
 // CompleteJob is triggered by a provision daemon to mark a provisioner job as completed.
 func (s *server) CompleteJob(ctx context.Context, completed *proto.CompletedJob) (*proto.Empty, error) {
 	ctx, span := s.startTrace(ctx, tracing.FuncName())

provisionerd/proto/provisionerd.proto

@@ -27,6 +27,9 @@ message AcquiredJob {
     // for the first time.
     repeated provisioner.RichParameterValue previous_parameter_values = 10;
     optional bool exp_reuse_terraform_workspace = 11;
+    // Templates before dynamic parameters and templates with module
+    // directories that are too large will not have module files.
+    optional string modules_files_uuid = 12;
   }
   message TemplateImport {
     provisioner.Metadata metadata = 1;
@@ -158,6 +161,7 @@ message CommitQuotaResponse {
 
 message CancelAcquire {}
 
+// TODO: Can we safely rename this to just `UploadFile`?
 message UploadFileRequest {
   oneof type {
     provisioner.DataUpload data_upload = 1;
@@ -165,6 +169,10 @@ message UploadFileRequest {
   }
 }
 
+message DownloadFileRequest {
+  string file_uuid = 1;
+}
+
 service ProvisionerDaemon {
   // AcquireJob requests a job. Implementations should
   // hold a lock on the job until CompleteJob() is
@@ -196,4 +204,8 @@ service ProvisionerDaemon {
   // UploadFile streams files to be inserted into the database.
   // The file upload_type should be used to determine how to handle the file.
   rpc UploadFile(stream UploadFileRequest) returns (Empty);
+
+  // DownloadFile can download files from the database.
+  // Useful for retrieving large files such as terraform modules.
+  rpc DownloadFile(DownloadFileRequest) returns (stream UploadFileRequest);
 }

provisionerd/proto/provisionerd_drpc.pb.go

@@ -1,5 +1,5 @@
 // Code generated by protoc-gen-go-drpc. DO NOT EDIT.
-// protoc-gen-go-drpc version: v0.0.34
+// protoc-gen-go-drpc version: (devel)
 // source: provisionerd/proto/provisionerd.proto
 
 package proto
@@ -45,6 +45,7 @@ type DRPCProvisionerDaemonClient interface {
 	FailJob(ctx context.Context, in *FailedJob) (*Empty, error)
 	CompleteJob(ctx context.Context, in *CompletedJob) (*Empty, error)
 	UploadFile(ctx context.Context) (DRPCProvisionerDaemon_UploadFileClient, error)
+	DownloadFile(ctx context.Context, in *DownloadFileRequest) (DRPCProvisionerDaemon_DownloadFileClient, error)
 }
 
 type drpcProvisionerDaemonClient struct {
@@ -186,6 +187,46 @@ func (x *drpcProvisionerDaemon_UploadFileClient) CloseAndRecvMsg(m *Empty) error
 	return x.MsgRecv(m, drpcEncoding_File_provisionerd_proto_provisionerd_proto{})
 }
 
+func (c *drpcProvisionerDaemonClient) DownloadFile(ctx context.Context, in *DownloadFileRequest) (DRPCProvisionerDaemon_DownloadFileClient, error) {
+	stream, err := c.cc.NewStream(ctx, "/provisionerd.ProvisionerDaemon/DownloadFile", drpcEncoding_File_provisionerd_proto_provisionerd_proto{})
+	if err != nil {
+		return nil, err
+	}
+	x := &drpcProvisionerDaemon_DownloadFileClient{stream}
+	if err := x.MsgSend(in, drpcEncoding_File_provisionerd_proto_provisionerd_proto{}); err != nil {
+		return nil, err
+	}
+	if err := x.CloseSend(); err != nil {
+		return nil, err
+	}
+	return x, nil
+}
+
+type DRPCProvisionerDaemon_DownloadFileClient interface {
+	drpc.Stream
+	Recv() (*UploadFileRequest, error)
+}
+
+type drpcProvisionerDaemon_DownloadFileClient struct {
+	drpc.Stream
+}
+
+func (x *drpcProvisionerDaemon_DownloadFileClient) GetStream() drpc.Stream {
+	return x.Stream
+}
+
+func (x *drpcProvisionerDaemon_DownloadFileClient) Recv() (*UploadFileRequest, error) {
+	m := new(UploadFileRequest)
+	if err := x.MsgRecv(m, drpcEncoding_File_provisionerd_proto_provisionerd_proto{}); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+func (x *drpcProvisionerDaemon_DownloadFileClient) RecvMsg(m *UploadFileRequest) error {
+	return x.MsgRecv(m, drpcEncoding_File_provisionerd_proto_provisionerd_proto{})
+}
+
 type DRPCProvisionerDaemonServer interface {
 	AcquireJob(context.Context, *Empty) (*AcquiredJob, error)
 	AcquireJobWithCancel(DRPCProvisionerDaemon_AcquireJobWithCancelStream) error
@@ -194,6 +235,7 @@ type DRPCProvisionerDaemonServer interface {
 	FailJob(context.Context, *FailedJob) (*Empty, error)
 	CompleteJob(context.Context, *CompletedJob) (*Empty, error)
 	UploadFile(DRPCProvisionerDaemon_UploadFileStream) error
+	DownloadFile(*DownloadFileRequest, DRPCProvisionerDaemon_DownloadFileStream) error
 }
 
 type DRPCProvisionerDaemonUnimplementedServer struct{}
@@ -226,9 +268,13 @@ func (s *DRPCProvisionerDaemonUnimplementedServer) UploadFile(DRPCProvisionerDae
 	return drpcerr.WithCode(errors.New("Unimplemented"), drpcerr.Unimplemented)
 }
 
+func (s *DRPCProvisionerDaemonUnimplementedServer) DownloadFile(*DownloadFileRequest, DRPCProvisionerDaemon_DownloadFileStream) error {
+	return drpcerr.WithCode(errors.New("Unimplemented"), drpcerr.Unimplemented)
+}
+
 type DRPCProvisionerDaemonDescription struct{}
 
-func (DRPCProvisionerDaemonDescription) NumMethods() int { return 7 }
+func (DRPCProvisionerDaemonDescription) NumMethods() int { return 8 }
 
 func (DRPCProvisionerDaemonDescription) Method(n int) (string, drpc.Encoding, drpc.Receiver, interface{}, bool) {
 	switch n {
@@ -293,6 +339,15 @@ func (DRPCProvisionerDaemonDescription) Method(n int) (string, drpc.Encoding, dr
 						&drpcProvisionerDaemon_UploadFileStream{in1.(drpc.Stream)},
 					)
 			}, DRPCProvisionerDaemonServer.UploadFile, true
+	case 7:
+		return "/provisionerd.ProvisionerDaemon/DownloadFile", drpcEncoding_File_provisionerd_proto_provisionerd_proto{},
+			func(srv interface{}, ctx context.Context, in1, in2 interface{}) (drpc.Message, error) {
+				return nil, srv.(DRPCProvisionerDaemonServer).
+					DownloadFile(
+						in1.(*DownloadFileRequest),
+						&drpcProvisionerDaemon_DownloadFileStream{in2.(drpc.Stream)},
+					)
+			}, DRPCProvisionerDaemonServer.DownloadFile, true
 	default:
 		return "", nil, nil, nil, false
 	}
@@ -436,3 +491,16 @@ func (x *drpcProvisionerDaemon_UploadFileStream) Recv() (*UploadFileRequest, err
 func (x *drpcProvisionerDaemon_UploadFileStream) RecvMsg(m *UploadFileRequest) error {
 	return x.MsgRecv(m, drpcEncoding_File_provisionerd_proto_provisionerd_proto{})
 }
+
+type DRPCProvisionerDaemon_DownloadFileStream interface {
+	drpc.Stream
+	Send(*UploadFileRequest) error
+}
+
+type drpcProvisionerDaemon_DownloadFileStream struct {
+	drpc.Stream
+}
+
+func (x *drpcProvisionerDaemon_DownloadFileStream) Send(m *UploadFileRequest) error {
+	return x.MsgSend(m, drpcEncoding_File_provisionerd_proto_provisionerd_proto{})
+}

provisionerd/runner/runner.go

@@ -50,6 +50,7 @@ type Runner struct {
 	job                 *proto.AcquiredJob
 	sender              JobUpdater
 	quotaCommitter      QuotaCommitter
+	fileDownloader      FileDownloader
 	logger              slog.Logger
 	provisioner         sdkproto.DRPCProvisionerClient
 	lastUpdate          atomic.Pointer[time.Time]
@@ -102,6 +103,10 @@ type QuotaCommitter interface {
 	CommitQuota(ctx context.Context, in *proto.CommitQuotaRequest) (*proto.CommitQuotaResponse, error)
 }
 
+type FileDownloader interface {
+	DownloadFile(*proto.DownloadFileRequest, proto.DRPCProvisionerDaemon_DownloadFileStream) error
+}
+
 type Options struct {
 	Updater             JobUpdater
 	QuotaCommitter      QuotaCommitter
@@ -1025,6 +1030,18 @@ func (r *Runner) runWorkspaceBuild(ctx context.Context) (*proto.CompletedJob, *p
 		return nil, failedJob
 	}
 
+	if r.job.GetWorkspaceBuild().ModulesFilesUuid != nil &&
+		*r.job.GetWorkspaceBuild().ModulesFilesUuid != "" &&
+		*r.job.GetWorkspaceBuild().ModulesFilesUuid != uuid.Nil.String() {
+		// TODO: Download module files
+		//r.fileDownloader.DownloadFile(&proto.DownloadFileRequest{
+		//	FileUuid: *r.job.GetWorkspaceBuild().ModulesFilesUuid,
+		//})
+		//
+		//stream := NewStream
+		//r.provisioner
+	}
+
 	resp, failed := r.buildWorkspace(ctx, "Planning infrastructure", &sdkproto.Request{
 		Type: &sdkproto.Request_Plan{
 			Plan: &sdkproto.PlanRequest{

provisionersdk/proto/provisioner_drpc.pb.go

@@ -1,5 +1,5 @@
 // Code generated by protoc-gen-go-drpc. DO NOT EDIT.
-// protoc-gen-go-drpc version: v0.0.34
+// protoc-gen-go-drpc version: (devel)
 // source: provisionersdk/proto/provisioner.proto
 
 package proto