feat: transformations support in run (#1066)

Adds support for `transformations` during run.

go.mod

@@ -26,7 +26,7 @@ require (
 	github.com/speakeasy-api/huh v1.1.2
 	github.com/speakeasy-api/openapi-generation/v2 v2.452.0
 	github.com/speakeasy-api/openapi-overlay v0.9.0
-	github.com/speakeasy-api/sdk-gen-config v1.26.0
+	github.com/speakeasy-api/sdk-gen-config v1.27.2
 	github.com/speakeasy-api/speakeasy-client-sdk-go/v3 v3.14.11
 	github.com/speakeasy-api/speakeasy-core v0.16.0
 	github.com/speakeasy-api/speakeasy-proxy v0.0.2

go.sum

@@ -508,8 +508,8 @@ github.com/speakeasy-api/openapi-generation/v2 v2.452.0 h1:NTUddl7zBsAElDcvLdxnl
 github.com/speakeasy-api/openapi-generation/v2 v2.452.0/go.mod h1:pBDHLrszZpze4DmS/zE2tWqrZw4Dv/mjbuVZ/xULE40=
 github.com/speakeasy-api/openapi-overlay v0.9.0 h1:Wrz6NO02cNlLzx1fB093lBlYxSI54VRhy1aSutx0PQg=
 github.com/speakeasy-api/openapi-overlay v0.9.0/go.mod h1:f5FloQrHA7MsxYg9djzMD5h6dxrHjVVByWKh7an8TRc=
-github.com/speakeasy-api/sdk-gen-config v1.26.0 h1:ry8fvak8+aRo7DNVfV9gISOnjlRb2A3tRxQ5aixRxWQ=
-github.com/speakeasy-api/sdk-gen-config v1.26.0/go.mod h1:e9PjnCRHGa4K4EFKVU+kKmihOZjJ2V4utcU+274+bnQ=
+github.com/speakeasy-api/sdk-gen-config v1.27.2 h1:UU/yPyoxhN6gGYJ/MPQZtOvCFl/5hnyKsq9P053cXY8=
+github.com/speakeasy-api/sdk-gen-config v1.27.2/go.mod h1:e9PjnCRHGa4K4EFKVU+kKmihOZjJ2V4utcU+274+bnQ=
 github.com/speakeasy-api/speakeasy-client-sdk-go/v3 v3.14.11 h1:3lVkbqhH5Q8NBXyPH2yt7ilEQ7Mmy3WLdd+A5OB7Km8=
 github.com/speakeasy-api/speakeasy-client-sdk-go/v3 v3.14.11/go.mod h1:b4fiZ1Wid0JHwwiYqhaPifDwjmC15uiN7A8Cmid+9kw=
 github.com/speakeasy-api/speakeasy-core v0.16.0 h1:UpMEUFQ76rQyTFQ9JTv8EsX9V5DEr1ePJbdvw9+nJ68=

integration/resources/renameOperationOverlay.yaml

@@ -0,0 +1,14 @@
+overlay: 1.0.0
+info:
+  title: Change operationId for findByTags
+  version: 0.0.0
+actions:
+  - target: $["paths"]["/pet/findByTags"]["get"]["operationId"]
+    update: findByTagsNew
+  - target: $["paths"]["/pet/findByTags"]["get"]
+    update:
+      x-codeSamples:
+        - lang: go
+          label: updatePet
+          source: |-
+            package main

integration/workflow_test.go

@@ -233,6 +233,7 @@ func execute(t *testing.T, wd string, args ...string) Runnable {
 	execCmd := exec.Command("go", append([]string{"run", mainGo}, args...)...)
 	execCmd.Env = os.Environ()
 	execCmd.Dir = wd
+
 	// store stdout and stderr in a buffer and output it all in one go if there's a failure
 	out := bytes.Buffer{}
 	execCmd.Stdout = &out
@@ -277,11 +278,13 @@ func (c *cmdRunner) Run() error {
 
 func TestSpecWorkflows(t *testing.T) {
 	tests := []struct {
-		name          string
-		inputDocs     []string
-		overlays      []string
-		out           string
-		expectedPaths []string
+		name            string
+		inputDocs       []string
+		overlays        []string
+		transformations []workflow.Transformation
+		out             string
+		expectedPaths   []string
+		unexpectedPaths []string
 	}{
 		{
 			name: "overlay with local document",
@@ -323,6 +326,80 @@ func TestSpecWorkflows(t *testing.T) {
 			},
 			out: "output.yaml",
 		},
+		{
+			name:      "test simple transformation",
+			inputDocs: []string{"spec.yaml"},
+			transformations: []workflow.Transformation{
+				{
+					FilterOperations: &workflow.FilterOperationsOptions{
+						Operations: "findPetsByTags",
+					},
+				},
+			},
+			out: "output.yaml",
+			expectedPaths: []string{
+				"/pet/findByTags",
+			},
+			unexpectedPaths: []string{
+				"/pet/findByStatus",
+			},
+		},
+		{
+			name:      "test merge with transformation",
+			inputDocs: []string{"part1.yaml", "part2.yaml"},
+			transformations: []workflow.Transformation{
+				{
+					FilterOperations: &workflow.FilterOperationsOptions{
+						Operations: "getInventory",
+					},
+				},
+			},
+			out: "output.yaml",
+			expectedPaths: []string{
+				"/store/inventory",
+			},
+			unexpectedPaths: []string{
+				"/store/order",
+			},
+		},
+		{
+			name:      "test overlay with transformation",
+			inputDocs: []string{"spec.yaml"},
+			overlays:  []string{"renameOperationOverlay.yaml"},
+			transformations: []workflow.Transformation{
+				{
+					FilterOperations: &workflow.FilterOperationsOptions{
+						Operations: "findByTagsNew",
+					},
+				},
+			},
+			out: "output.yaml",
+			expectedPaths: []string{
+				"/pet/findByTags",
+			},
+			unexpectedPaths: []string{
+				"/pet/findByStatus",
+			},
+		},
+		{
+			name:      "test merge, overlay, and transformation",
+			inputDocs: []string{"part1.yaml", "part2.yaml"},
+			overlays:  []string{"renameOperationOverlay.yaml"},
+			transformations: []workflow.Transformation{
+				{
+					FilterOperations: &workflow.FilterOperationsOptions{
+						Operations: "findByTagsNew",
+					},
+				},
+			},
+			out: "output.yaml",
+			expectedPaths: []string{
+				"/pet/findByTags",
+			},
+			unexpectedPaths: []string{
+				"/pet/findByStatus",
+			},
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -340,6 +417,7 @@ func TestSpecWorkflows(t *testing.T) {
 				if isLocalFileReference(inputDoc) {
 					err := copyFile(fmt.Sprintf("resources/%s", inputDoc), fmt.Sprintf("%s/%s", temp, inputDoc))
 					require.NoError(t, err)
+					inputDoc = filepath.Join(temp, inputDoc)
 				}
 				inputs = append(inputs, workflow.Document{
 					Location: workflow.LocationString(inputDoc),
@@ -350,17 +428,21 @@ func TestSpecWorkflows(t *testing.T) {
 				if isLocalFileReference(overlay) {
 					err := copyFile(fmt.Sprintf("resources/%s", overlay), fmt.Sprintf("%s/%s", temp, overlay))
 					require.NoError(t, err)
+					overlay = filepath.Join(temp, overlay)
 				}
 				overlays = append(overlays, workflow.Overlay{
 					Document: &workflow.Document{
 						Location: workflow.LocationString(overlay),
 					},
 				})
 			}
+
+			outputFull := filepath.Join(temp, tt.out)
 			workflowFile.Sources["first-source"] = workflow.Source{
-				Inputs:   inputs,
-				Overlays: overlays,
-				Output:   &tt.out,
+				Inputs:          inputs,
+				Overlays:        overlays,
+				Transformations: tt.transformations,
+				Output:          &outputFull,
 			}
 
 			err := os.MkdirAll(filepath.Join(temp, ".speakeasy"), 0o755)
@@ -371,11 +453,12 @@ func TestSpecWorkflows(t *testing.T) {
 			err = workflow.Save(temp, workflowFile)
 			require.NoError(t, err)
 			args := []string{"run", "-s", "all", "--pinned", "--skip-compile"}
+
 			cmdErr := execute(t, temp, args...).Run()
 			require.NoError(t, cmdErr)
 
 			content, err := os.ReadFile(filepath.Join(temp, tt.out))
-			require.NoError(t, err, "No readable file %s exists", tt.out)
+			require.NoError(t, err, "No readable file %s exists", filepath.Join(temp, tt.out))
 
 			if len(tt.overlays) > 0 {
 				if !strings.Contains(string(content), "x-codeSamples") {
@@ -390,6 +473,14 @@ func TestSpecWorkflows(t *testing.T) {
 					}
 				}
 			}
+
+			if len(tt.unexpectedPaths) > 0 {
+				for _, path := range tt.unexpectedPaths {
+					if strings.Contains(string(content), path) {
+						t.Errorf("Unexpected path %s found in output document", path)
+					}
+				}
+			}
 		})
 	}
 }

internal/run/frozenSource.go

@@ -24,7 +24,7 @@ func NewFrozenSource(w *Workflow, parentStep *workflowTracking.WorkflowStep, sou
 	}
 }
 
-func (f FrozenSource) Do(ctx context.Context, _, _ string) (string, error) {
+func (f FrozenSource) Do(ctx context.Context, _ string) (string, error) {
 	mergeStep := f.parentStep.NewSubstep("Download OAS from lockfile")
 
 	// Check lockfile exists, produce an error if not

internal/run/merge.go

@@ -30,13 +30,10 @@ func NewMerge(w *Workflow, parentStep *workflowTracking.WorkflowStep, source wor
 	}
 }
 
-func (m Merge) Do(ctx context.Context, _, outputLocation string) (string, error) {
+func (m Merge) Do(ctx context.Context, _ string) (string, error) {
 	mergeStep := m.parentStep.NewSubstep("Merge Documents")
 
 	mergeLocation := m.source.GetTempMergeLocation()
-	if len(m.source.Overlays) == 0 {
-		mergeLocation = outputLocation
-	}
 
 	log.From(ctx).Infof("Merging %d schemas into %s...", len(m.source.Inputs), mergeLocation)
 

internal/run/minimumViableSpec.go

@@ -3,89 +3,39 @@ package run
 import (
 	"context"
 	"fmt"
+	"github.com/AlekSi/pointer"
 	"github.com/speakeasy-api/openapi-generation/v2/pkg/errors"
 	"github.com/speakeasy-api/sdk-gen-config/workflow"
-	"github.com/speakeasy-api/speakeasy-core/openapi"
 	"github.com/speakeasy-api/speakeasy/internal/charm/styles"
-	"github.com/speakeasy-api/speakeasy/internal/download"
 	"github.com/speakeasy-api/speakeasy/internal/studio/modifications"
-	"github.com/speakeasy-api/speakeasy/internal/transform"
 	"github.com/speakeasy-api/speakeasy/internal/workflowTracking"
-	"os"
-	"path/filepath"
+	"strings"
 )
 
 func (w *Workflow) retryWithMinimumViableSpec(ctx context.Context, parentStep *workflowTracking.WorkflowStep, sourceID, targetID string, vErrs []error) (string, *SourceResult, error) {
-	invalidOperationToErr := make(map[string]error)
+	var invalidOperations []string
 	for _, err := range vErrs {
 		vErr := errors.GetValidationErr(err)
-		for _, op := range vErr.AffectedOperationIDs {
-			invalidOperationToErr[op] = err // TODO: support multiple errors per operation?
+		if vErr.Severity == errors.SeverityError {
+			for _, op := range vErr.AffectedOperationIDs {
+				invalidOperations = append(invalidOperations, op)
+			}
 		}
 	}
 
 	substep := parentStep.NewSubstep("Retrying with minimum viable document")
 	source := w.workflow.Sources[sourceID]
-	baseLocation := source.Inputs[0].Location.Resolve()
-	workingDir := workflow.GetTempDir()
-
-	// This is intended to only be used from quickstart, we must assume a singular input document
-	if len(source.Inputs)+len(source.Overlays) > 1 {
-		return "", nil, fmt.Errorf("multiple inputs are not supported for minimum viable spec")
-	}
-
-	tempBase := fmt.Sprintf("downloaded_%s%s", randStringBytes(10), filepath.Ext(baseLocation))
-
-	if source.Inputs[0].IsRemote() {
-		outResolved, err := download.ResolveRemoteDocument(ctx, source.Inputs[0], tempBase)
-		if err != nil {
-			return "", nil, fmt.Errorf("failed to download remote document: %w", err)
-		}
-
-		baseLocation = outResolved
-	}
-
-	overlayOut := filepath.Join(workingDir, fmt.Sprintf("mvs_overlay_%s.yaml", randStringBytes(10)))
-	if err := os.MkdirAll(workingDir, os.ModePerm); err != nil {
-		return "", nil, err
-	}
-	overlayFile, err := os.Create(overlayOut)
-	if err != nil {
-		return "", nil, fmt.Errorf("failed to create overlay file: %w", err)
-	}
-	defer overlayFile.Close()
-
-	failedRetry := false
-	defer func() {
-		os.Remove(overlayOut)
-		os.Remove(filepath.Join(workingDir, tempBase))
-		if failedRetry {
-			source.Overlays = []workflow.Overlay{}
-			w.workflow.Sources[sourceID] = source
-		}
-	}()
-
-	_, _, model, err := openapi.LoadDocument(ctx, source.Inputs[0].Location.Resolve())
-	if err != nil {
-		return "", nil, fmt.Errorf("failed to load document: %w", err)
-	}
-
-	overlay := transform.BuildRemoveInvalidOperationsOverlay(model, invalidOperationToErr)
-
-	overlayPath, err := modifications.GetOverlayPath(workingDir)
-	if err != nil {
-		return "", nil, fmt.Errorf("failed to get overlay path: %w", err)
-	}
-
-	if _, err = modifications.UpsertOverlay(overlayPath, &source, overlay); err != nil {
-		return "", nil, fmt.Errorf("failed to upsert overlay: %w", err)
-	}
 
+	source.Transformations = append(source.Transformations, workflow.Transformation{
+		FilterOperations: &workflow.FilterOperationsOptions{
+			Operations: strings.Join(invalidOperations, ","),
+			Exclude:    pointer.ToBool(true),
+		},
+	})
 	w.workflow.Sources[sourceID] = source
 
 	sourcePath, sourceRes, err := w.RunSource(ctx, substep, sourceID, targetID)
 	if err != nil {
-		failedRetry = true
 		return "", nil, fmt.Errorf("failed to re-run source: %w", err)
 	}
 

internal/run/overlay.go

@@ -17,29 +17,23 @@ import (
 )
 
 type Overlay struct {
-	workflow       *Workflow
-	parentStep     *workflowTracking.WorkflowStep
-	source         workflow.Source
-	outputLocation string
-	ruleset        string
+	parentStep *workflowTracking.WorkflowStep
+	source     workflow.Source
 }
 
 var _ SourceStep = Overlay{}
 
-func NewOverlay(w *Workflow, parentStep *workflowTracking.WorkflowStep, source workflow.Source, outputLocation, ruleset string) Overlay {
+func NewOverlay(parentStep *workflowTracking.WorkflowStep, source workflow.Source) Overlay {
 	return Overlay{
-		workflow:       w,
-		parentStep:     parentStep,
-		source:         source,
-		outputLocation: outputLocation,
-		ruleset:        ruleset,
+		parentStep: parentStep,
+		source:     source,
 	}
 }
 
-func (o Overlay) Do(ctx context.Context, inputPath, outputPath string) (string, error) {
+func (o Overlay) Do(ctx context.Context, inputPath string) (string, error) {
 	overlayStep := o.parentStep.NewSubstep("Applying Overlays")
 
-	overlayLocation := outputPath
+	overlayLocation := o.source.GetTempOverlayLocation()
 
 	log.From(ctx).Infof("Applying %d overlays into %s...", len(o.source.Overlays), overlayLocation)