From 2bfdcfbcdb6665a8f4eafc961270d2edf859f7de Mon Sep 17 00:00:00 2001 From: Tristan Cartledge Date: Tue, 14 Nov 2023 10:21:37 +0000 Subject: [PATCH 1/2] feat: add support for overlays --- .github/workflows/sdk-generation.yaml | 17 ++++++ action.yml | 13 +++++ internal/cli/cli.go | 28 ++++++++++ internal/document/document.go | 79 ++++++++++++++++++++------- internal/environment/environment.go | 4 ++ 5 files changed, 120 insertions(+), 21 deletions(-) diff --git a/.github/workflows/sdk-generation.yaml b/.github/workflows/sdk-generation.yaml index aa76aab3..1f584731 100644 --- a/.github/workflows/sdk-generation.yaml +++ b/.github/workflows/sdk-generation.yaml @@ -33,6 +33,21 @@ on: - https://example.com/openapi2.json required: false type: string + overlay_docs: + description: |- + A yaml string containing a list of overlay documents to use, if multiple documents are provided they will be applied to the OpenAPI document in the order provided. + + If the document lives within the repo a relative path can be provided, if the document is hosted publicly a URL can be provided. + + If the documents are hosted privately a URL can be provided along with the `openapi_doc_auth_header` and `openapi_doc_auth_token` inputs. + Each document will be fetched using the provided auth header and token, so they need to be valid for all documents. + + For example: + overlay_docs: | + - https://example.com/overlay1.json + - https://example.com/overlay2.json + required: false + type: string languages: description: |- A yaml string containing a list of languages to generate SDKs for example: @@ -187,6 +202,7 @@ jobs: openapi_doc_auth_header: ${{ inputs.openapi_doc_auth_header }} openapi_doc_auth_token: ${{ secrets.openapi_doc_auth_token }} openapi_docs: ${{ inputs.openapi_docs }} + overlay_docs: ${{ inputs.overlay_docs }} github_access_token: ${{ secrets.github_access_token }} action: validate speakeasy_api_key: ${{ secrets.speakeasy_api_key }} @@ -245,6 +261,7 @@ jobs: openapi_doc_auth_header: ${{ inputs.openapi_doc_auth_header }} openapi_doc_auth_token: ${{ secrets.openapi_doc_auth_token }} openapi_docs: ${{ inputs.openapi_docs }} + overlay_docs: ${{ inputs.overlay_docs }} github_access_token: ${{ secrets.github_access_token }} languages: ${{ inputs.languages }} create_release: ${{ inputs.create_release }} diff --git a/action.yml b/action.yml index 087bb12b..efcea448 100644 --- a/action.yml +++ b/action.yml @@ -32,6 +32,19 @@ inputs: - https://example.com/openapi1.json - https://example.com/openapi2.json required: false + overlay_docs: + description: |- + A yaml string containing a list of overlay documents to use, if multiple documents are provided they will be applied to the OpenAPI document in the order provided. + + If the document lives within the repo a relative path can be provided, if the document is hosted publicly a URL can be provided. + + If the documents are hosted privately a URL can be provided along with the `openapi_doc_auth_header` and `openapi_doc_auth_token` inputs. + Each document will be fetched using the provided auth header and token, so they need to be valid for all documents. + + For example: + overlay_docs: | + - https://example.com/overlay1.json + - https://example.com/overlay2.json openapi_doc_output: description: "The path to output the modified OpenAPI spec" required: false diff --git a/internal/cli/cli.go b/internal/cli/cli.go index 98a7eb25..46cb1a0d 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -2,6 +2,7 @@ package cli import ( "fmt" + "os" "regexp" "strconv" "strings" @@ -18,6 +19,7 @@ var ( OutputTestsVersion = version.Must(version.NewVersion("1.33.2")) LLMSuggestionVersion = version.Must(version.NewVersion("1.47.1")) GranularChangeLogVersion = version.Must(version.NewVersion("1.70.2")) + OverlayVersion = version.Must(version.NewVersion("1.112.1")) ) func IsAtLeastVersion(version *version.Version) bool { @@ -281,3 +283,29 @@ func MergeDocuments(files []string, output string) error { fmt.Println(out) return nil } + +func ApplyOverlay(overlayPath, inPath, outPath string) error { + if !IsAtLeastVersion(OverlayVersion) { + return fmt.Errorf("speakeasy version %s does not support applying overlays", OverlayVersion) + } + + args := []string{ + "overlay", + "apply", + "-o", + "overlayPath", + "-s", + inPath, + } + + out, err := runSpeakeasyCommand(args...) + if err != nil { + return fmt.Errorf("error applying overlay: %w - %s", err, out) + } + + if err := os.WriteFile(outPath, []byte(out), os.ModePerm); err != nil { + return fmt.Errorf("error writing overlay output: %w", err) + } + + return nil +} diff --git a/internal/document/document.go b/internal/document/document.go index 40176903..50c9cf2f 100644 --- a/internal/document/document.go +++ b/internal/document/document.go @@ -24,16 +24,17 @@ type file struct { } func GetOpenAPIFileInfo() (string, string, string, error) { - files, err := getFiles() + // TODO OPENAPI_DOC_LOCATION is deprecated and should be removed in the future + openapiFiles, err := getFiles(environment.GetOpenAPIDocs(), environment.GetOpenAPIDocLocation()) if err != nil { return "", "", "", err } - if len(files) > 1 && !cli.IsAtLeastVersion(cli.MergeVersion) { + if len(openapiFiles) > 1 && !cli.IsAtLeastVersion(cli.MergeVersion) { return "", "", "", fmt.Errorf("multiple openapi files are only supported in speakeasy version %s or higher", cli.MergeVersion.String()) } - filePaths, err := resolveFiles(files) + resolvedOpenAPIFiles, err := resolveFiles(openapiFiles, "openapi") if err != nil { return "", "", "", err } @@ -41,12 +42,33 @@ func GetOpenAPIFileInfo() (string, string, string, error) { basePath := "" filePath := "" - if len(filePaths) == 1 { - filePath = filePaths[0] + if len(resolvedOpenAPIFiles) == 1 { + filePath = resolvedOpenAPIFiles[0] basePath = filepath.Dir(filePath) } else { - basePath = filepath.Dir(filePaths[0]) - filePath, err = mergeFiles(filePaths) + basePath = filepath.Dir(resolvedOpenAPIFiles[0]) + filePath, err = mergeFiles(resolvedOpenAPIFiles) + if err != nil { + return "", "", "", err + } + } + + overlayFiles, err := getFiles(environment.GetOverlayDocs(), "") + if err != nil { + return "", "", "", err + } + + if len(overlayFiles) > 1 && !cli.IsAtLeastVersion(cli.OverlayVersion) { + return "", "", "", fmt.Errorf("overlay files are only supported in speakeasy version %s or higher", cli.OverlayVersion.String()) + } + + resolvedOverlayFiles, err := resolveFiles(overlayFiles, "overlay") + if err != nil { + return "", "", "", err + } + + if len(resolvedOverlayFiles) > 0 { + filePath, err = applyOverlay(filePath, resolvedOverlayFiles) if err != nil { return "", "", "", err } @@ -100,7 +122,25 @@ func mergeFiles(files []string) (string, error) { return outPath, nil } -func resolveFiles(files []file) ([]string, error) { +func applyOverlay(filePath string, overlayFiles []string) (string, error) { + outPath := filepath.Join(environment.GetWorkspace(), "repo", "openapi", "openapi_overlay") + + if err := os.MkdirAll(filepath.Dir(outPath), os.ModePerm); err != nil { + return "", fmt.Errorf("failed to create openapi directory: %w", err) + } + + for _, overlayFile := range overlayFiles { + if err := cli.ApplyOverlay(overlayFile, filePath, outPath); err != nil { + return "", fmt.Errorf("failed to apply overlay: %w", err) + } + + filePath = outPath + } + + return outPath, nil +} + +func resolveFiles(files []file, typ string) ([]string, error) { workspace := environment.GetWorkspace() outFiles := []string{} @@ -109,18 +149,18 @@ func resolveFiles(files []file) ([]string, error) { localPath := filepath.Join(workspace, "repo", file.Location) if _, err := os.Stat(localPath); err == nil { - fmt.Println("Found local OpenAPI file: ", localPath) + fmt.Printf("Found local %s file: %s\n", typ, localPath) outFiles = append(outFiles, localPath) } else { u, err := url.Parse(file.Location) if err != nil { - return nil, fmt.Errorf("failed to parse openapi url: %w", err) + return nil, fmt.Errorf("failed to parse %s url: %w", typ, err) } - fmt.Println("Downloading openapi file from: ", u.String()) + fmt.Printf("Downloading %s file from: %s\n", typ, u.String()) - filePath := filepath.Join(environment.GetWorkspace(), "openapi", fmt.Sprintf("openapi_%d", i)) + filePath := filepath.Join(environment.GetWorkspace(), typ, fmt.Sprintf("%s_%d", typ, i)) if environment.GetAction() == environment.ActionValidate { if extension := path.Ext(u.Path); extension != "" { @@ -129,11 +169,11 @@ func resolveFiles(files []file) ([]string, error) { } if err := os.MkdirAll(filepath.Dir(filePath), os.ModePerm); err != nil { - return nil, fmt.Errorf("failed to create openapi directory: %w", err) + return nil, fmt.Errorf("failed to create %s directory: %w", typ, err) } if err := download.DownloadFile(u.String(), filePath, file.Header, file.Token); err != nil { - return nil, fmt.Errorf("failed to download openapi file: %w", err) + return nil, fmt.Errorf("failed to download %s file: %w", typ, err) } outFiles = append(outFiles, filePath) @@ -143,17 +183,14 @@ func resolveFiles(files []file) ([]string, error) { return outFiles, nil } -func getFiles() ([]file, error) { - docsYaml := environment.GetOpenAPIDocs() - +func getFiles(filesYaml string, defaultFile string) ([]file, error) { var fileLocations []string - if err := yaml.Unmarshal([]byte(docsYaml), &fileLocations); err != nil { + if err := yaml.Unmarshal([]byte(filesYaml), &fileLocations); err != nil { return nil, fmt.Errorf("failed to parse openapi_docs input: %w", err) } - // TODO OPENAPI_DOC_LOCATION is deprecated and should be removed in the future - if len(fileLocations) == 0 { - fileLocations = append(fileLocations, environment.GetOpenAPIDocLocation()) + if len(fileLocations) == 0 && defaultFile != "" { + fileLocations = append(fileLocations, defaultFile) } files := []file{} diff --git a/internal/environment/environment.go b/internal/environment/environment.go index 79ac082f..23f6fee4 100644 --- a/internal/environment/environment.go +++ b/internal/environment/environment.go @@ -117,6 +117,10 @@ func GetOpenAPIDocs() string { return os.Getenv("INPUT_OPENAPI_DOCS") } +func GetOverlayDocs() string { + return os.Getenv("INPUT_OVERLAY_DOCS") +} + func GetOpenAPIDocOutput() string { return os.Getenv("INPUT_OPENAPI_DOC_OUTPUT") } From 0aab5710634fcee1422a650195f6eed0a7906084 Mon Sep 17 00:00:00 2001 From: Tristan Cartledge Date: Tue, 14 Nov 2023 10:41:08 +0000 Subject: [PATCH 2/2] fixes --- Makefile | 3 +++ internal/cli/cli.go | 2 +- internal/document/document.go | 4 ++-- testing/overlay-test.env | 8 ++++++++ 4 files changed, 14 insertions(+), 3 deletions(-) create mode 100644 testing/overlay-test.env diff --git a/Makefile b/Makefile index 8c6af055..da16f9bc 100644 --- a/Makefile +++ b/Makefile @@ -17,3 +17,6 @@ test-release-mode-multi-sdk: test-validate-action: ./testing/test.sh ./testing/validate-action.env + +test-overlay: + ./testing/test.sh ./testing/overlay-test.env \ No newline at end of file diff --git a/internal/cli/cli.go b/internal/cli/cli.go index 46cb1a0d..869ce813 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -293,7 +293,7 @@ func ApplyOverlay(overlayPath, inPath, outPath string) error { "overlay", "apply", "-o", - "overlayPath", + overlayPath, "-s", inPath, } diff --git a/internal/document/document.go b/internal/document/document.go index 50c9cf2f..82ee4890 100644 --- a/internal/document/document.go +++ b/internal/document/document.go @@ -109,7 +109,7 @@ func GetOpenAPIFileInfo() (string, string, string, error) { } func mergeFiles(files []string) (string, error) { - outPath := filepath.Join(environment.GetWorkspace(), "repo", "openapi", "openapi_merged") + outPath := filepath.Join(environment.GetWorkspace(), "repo", ".openapi", "openapi_merged") if err := os.MkdirAll(filepath.Dir(outPath), os.ModePerm); err != nil { return "", fmt.Errorf("failed to create openapi directory: %w", err) @@ -123,7 +123,7 @@ func mergeFiles(files []string) (string, error) { } func applyOverlay(filePath string, overlayFiles []string) (string, error) { - outPath := filepath.Join(environment.GetWorkspace(), "repo", "openapi", "openapi_overlay") + outPath := filepath.Join(environment.GetWorkspace(), "repo", ".openapi", "openapi_overlay") if err := os.MkdirAll(filepath.Dir(outPath), os.ModePerm); err != nil { return "", fmt.Errorf("failed to create openapi directory: %w", err) diff --git a/testing/overlay-test.env b/testing/overlay-test.env new file mode 100644 index 00000000..b6f5709d --- /dev/null +++ b/testing/overlay-test.env @@ -0,0 +1,8 @@ +INPUT_MODE="pr" +INPUT_ACTION="generate" +INPUT_LANGUAGES="- go" +INPUT_OPENAPI_DOCS="[\"base_oas.yaml\"]" +INPUT_OVERLAY_DOCS="[\"terraform_overlay.yaml\"]" +GITHUB_REPOSITORY="speakeasy-api/sdk-generation-action-overlay-test" +INPUT_FORCE=true +RUN_FINALIZE=true