From 990c77ab699c164b97021301742e3c87c942db7e Mon Sep 17 00:00:00 2001 From: Alex Simenduev Date: Tue, 7 Nov 2023 12:46:53 +0200 Subject: [PATCH 1/4] Fix: excluding files from unneededGoFiles was not working --- pkg/codegen/golang.go | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/pkg/codegen/golang.go b/pkg/codegen/golang.go index 8bdc380..91347af 100644 --- a/pkg/codegen/golang.go +++ b/pkg/codegen/golang.go @@ -17,11 +17,14 @@ package codegen import ( "bytes" "fmt" + "path" "path/filepath" + "strings" ijson "github.com/pulumi/crd2pulumi/internal/json" "github.com/pulumi/pulumi/pkg/v3/codegen" goGen "github.com/pulumi/pulumi/pkg/v3/codegen/go" + "github.com/pulumi/pulumi/pkg/v3/codegen/schema" ) var unneededGoFiles = codegen.NewStringSet( @@ -63,21 +66,47 @@ func GenerateGo(pg *PackageGenerator, name string) (buffers map[string]*bytes.Bu pkg.Language[langName] = jsonData files, err := goGen.GeneratePackage("crd2pulumi", pkg) - if err != nil { return nil, fmt.Errorf("could not generate Go package: %w", err) } + packageRoot, err := getPackageRoot(pkg.Reference()) + if err != nil { + return nil, fmt.Errorf("could not get package root: %w", err) + } + pkg.Name = oldName delete(pkg.Language, langName) buffers = map[string]*bytes.Buffer{} for path, code := range files { newPath, _ := filepath.Rel(name, path) - if !unneededGoFiles.Has(newPath) { + pkgRelPath := strings.TrimPrefix(path, packageRoot+"/") + + if !unneededGoFiles.Has(pkgRelPath) { buffers[newPath] = bytes.NewBuffer(code) } } return buffers, err } + +// Similar to "packageRoot" method from pulumi/pkg/codegen/go/gen.go +func getPackageRoot(pkg schema.PackageReference) (string, error) { + def, err := pkg.Definition() + if err != nil { + return "", err + } + var info goGen.GoPackageInfo + if goInfo, ok := def.Language["go"].(goGen.GoPackageInfo); ok { + info = goInfo + } + if info.RootPackageName != "" { + // package structure is flat + return "", nil + } + if info.ImportBasePath != "" { + return path.Base(info.ImportBasePath), nil + } + return strings.ReplaceAll(pkg.Name(), "-", ""), nil +} From 5d6127942bf23f82e2b97b50ed7d27bd245666f9 Mon Sep 17 00:00:00 2001 From: Alex Simenduev Date: Thu, 9 Nov 2023 09:55:26 +0200 Subject: [PATCH 2/4] add changelog entry, and apply markdown fromatting --- CHANGELOG.md | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f4beaf..a06d96d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,23 +1,30 @@ -CHANGELOG -========= +# CHANGELOG ## Unreleased +- Fix: excluding files from unneededGoFiles was not working () + ## 1.2.5 (2023-05-31) -- Remove underscores in generated nested types (https://github.com/pulumi/crd2pulumi/pull/114) + +- Remove underscores in generated nested types () ## 1.2.4 (2023-03-23) + - Requires Go 1.19 or higher now to build - Fix issue [#108](https://github.com/pulumi/crd2pulumi/issues/108) - crd2pulumi generator splits types apart into duplicate entires in pulumiTypes.go and pulumiTypes1.go ## 1.2.3 (2022-10-18) -- Fix issue [#43: crd properties with - in name](https://github.com/pulumi/crd2pulumi/issues/43) (https://github.com/pulumi/crd2pulumi/pull/99) + +- Fix issue [#43: crd properties with - in name](https://github.com/pulumi/crd2pulumi/issues/43) () ## 1.2.2 (2022-07-20) + - Fix regression that caused code in all languages to be generated regardless of selection. ## 1.2.1 (2022-07-19) + This release is a refactor with no user-affecting changes. + - Create public interface for codegen in the `pkg/codegen` namespace while placing internal utilities under `internal/` - Simplify cobra usage, simplify program config substantially @@ -30,13 +37,14 @@ This release is a refactor with no user-affecting changes. more appropriately named files or packages. - Update to latest Pulumi SDK as well as update all other dependencies. - Update to Go 1.18 -- Upgrade to go 1.17 (https://github.com/pulumi/crd2pulumi/pull/75) +- Upgrade to go 1.17 () ## 1.2.0 (2022-02-07) -- [python] Do not overwrite _utilities.py (https://github.com/pulumi/crd2pulumi/pull/73/) + +- [python] Do not overwrite _utilities.py () ## 1.1.0 (2022-01-04) -- Update to Pulumi v3.21.0 (https://github.com/pulumi/crd2pulumi/pull/63) -- Fix x-kubernetes-int-or-string precedence (https://github.com/pulumi/crd2pulumi/pull/60) -- Add generating CRD from URL (https://github.com/pulumi/crd2pulumi/pull/62) +- Update to Pulumi v3.21.0 () +- Fix x-kubernetes-int-or-string precedence () +- Add generating CRD from URL () From fcd55207a1f2fe83316b81944673ca141e54ec92 Mon Sep 17 00:00:00 2001 From: Alex Simenduev Date: Tue, 12 Dec 2023 19:17:02 +0200 Subject: [PATCH 3/4] add test for UneededGoFiles --- pkg/codegen/golang.go | 8 ++--- tests/unneeded_go_files_test.go | 55 +++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 4 deletions(-) create mode 100644 tests/unneeded_go_files_test.go diff --git a/pkg/codegen/golang.go b/pkg/codegen/golang.go index af7730b..a934cbc 100644 --- a/pkg/codegen/golang.go +++ b/pkg/codegen/golang.go @@ -27,7 +27,7 @@ import ( "github.com/pulumi/pulumi/pkg/v3/codegen/schema" ) -var unneededGoFiles = codegen.NewStringSet( +var UnneededGoFiles = codegen.NewStringSet( // The root directory doesn't define any resources: "doc.go", "init.go", @@ -76,7 +76,7 @@ func GenerateGo(pg *PackageGenerator, name string) (buffers map[string]*bytes.Bu return nil, fmt.Errorf("could not generate Go package: %w", err) } - packageRoot, err := getPackageRoot(pkg.Reference()) + packageRoot, err := goPackageRoot(pkg.Reference()) if err != nil { return nil, fmt.Errorf("could not get package root: %w", err) } @@ -89,7 +89,7 @@ func GenerateGo(pg *PackageGenerator, name string) (buffers map[string]*bytes.Bu newPath, _ := filepath.Rel(name, path) pkgRelPath := strings.TrimPrefix(path, packageRoot+"/") - if !unneededGoFiles.Has(pkgRelPath) { + if !UnneededGoFiles.Has(pkgRelPath) { buffers[newPath] = bytes.NewBuffer(code) } } @@ -98,7 +98,7 @@ func GenerateGo(pg *PackageGenerator, name string) (buffers map[string]*bytes.Bu } // Similar to "packageRoot" method from pulumi/pkg/codegen/go/gen.go -func getPackageRoot(pkg schema.PackageReference) (string, error) { +func goPackageRoot(pkg schema.PackageReference) (string, error) { def, err := pkg.Definition() if err != nil { return "", err diff --git a/tests/unneeded_go_files_test.go b/tests/unneeded_go_files_test.go new file mode 100644 index 0000000..5457c19 --- /dev/null +++ b/tests/unneeded_go_files_test.go @@ -0,0 +1,55 @@ +package tests + +import ( + "io" + "strings" + "testing" + + "github.com/pulumi/crd2pulumi/pkg/codegen" +) + +func TestUnneededGoFiles(t *testing.T) { + mockCRDYaml := `--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +spec: + group: uneeded-go-files-test.pulumi.com + names: + plural: testresources + singular: testresource + kind: TestResource + scope: Namespaced + versions: + - test: + name: test + schema: + openAPIV3Schema: + properties: + testProperty: + type: string` + + yamlSources := []io.ReadCloser{ + io.NopCloser(strings.NewReader(mockCRDYaml)), + } + + // Invoke ReadPackagesFromSource + pg, err := codegen.ReadPackagesFromSource("", yamlSources) + if err != nil { + t.Fatalf("ReadPackagesFromSource failed: %v", err) + } + + // Pick a generated file we want to exclude + uneededGoFile := "uneededgofilestest/test/testResource.go" + codegen.UnneededGoFiles.Add(uneededGoFile) + + // Generate the code from the mocked CRD + buffers, err := codegen.GenerateGo(pg, "crds") + if err != nil { + t.Fatalf("GenerateGo failed: %v", err) + } + + // Assert that buffers do not contain unneeded file + if _, exists := buffers["../kubernetes/"+uneededGoFile]; exists { + t.Errorf("Uneeded GO file was not excluded by GoGenerate, %s", uneededGoFile) + } +} From 4684144f7e356a44a590d34a1081d214420773b9 Mon Sep 17 00:00:00 2001 From: Alex Simenduev Date: Tue, 12 Dec 2023 19:17:09 +0200 Subject: [PATCH 4/4] replace deprecated ioutil methods --- cmd/root.go | 5 ++--- tests/crds_test.go | 7 +++---- tests/schema_test.go | 3 +-- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index 14b3e2b..4d7f1a5 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -19,7 +19,6 @@ import ( "errors" "fmt" "io" - "io/ioutil" "os" "github.com/pulumi/crd2pulumi/pkg/codegen" @@ -29,7 +28,7 @@ import ( // Version specifies the crd2pulumi version. It should be set by the linker via LDFLAGS. This defaults to dev var Version = "dev" -const long = `crd2pulumi is a CLI tool that generates typed Kubernetes +const long = `crd2pulumi is a CLI tool that generates typed Kubernetes CustomResources to use in Pulumi programs, based on a CustomResourceDefinition YAML schema.` @@ -92,7 +91,7 @@ func Execute() error { } var err error if shouldUseStdin { - err = codegen.Generate(cs, []io.ReadCloser{ioutil.NopCloser(bytes.NewBuffer(stdinData))}) + err = codegen.Generate(cs, []io.ReadCloser{io.NopCloser(bytes.NewBuffer(stdinData))}) } else { err = codegen.GenerateFromFiles(cs, args) } diff --git a/tests/crds_test.go b/tests/crds_test.go index 6e10b43..fad7119 100644 --- a/tests/crds_test.go +++ b/tests/crds_test.go @@ -17,7 +17,6 @@ package tests import ( "fmt" "io/fs" - "io/ioutil" "os" "os/exec" "path/filepath" @@ -32,7 +31,7 @@ const gkeManagedCertsUrl = "https://raw.githubusercontent.com/GoogleCloudPlatfor // execCrd2Pulumi runs the crd2pulumi binary in a temporary directory func execCrd2Pulumi(t *testing.T, lang, path string, additionalValidation func(t *testing.T, path string)) { - tmpdir, err := ioutil.TempDir("", "crd2pulumi_test") + tmpdir, err := os.MkdirTemp("", "crd2pulumi_test") assert.Nil(t, err, "expected to create a temp dir for the CRD output") t.Cleanup(func() { t.Logf("removing temp dir %q for %s test", tmpdir, lang) @@ -98,7 +97,7 @@ func TestCRDsWithUnderscore(t *testing.T) { // Ensure inputs are camelCased. filename := filepath.Join(path, "pulumi_crds", "juice", "v1alpha1", "_inputs.py") t.Logf("validating underscored field names in %s", filename) - pythonInputs, err := ioutil.ReadFile(filename) + pythonInputs, err := os.ReadFile(filename) if err != nil { t.Fatalf("expected to read generated Python code: %s", err) } @@ -108,7 +107,7 @@ func TestCRDsWithUnderscore(t *testing.T) { // Ensure outputs are camelCased. filename = filepath.Join(path, "pulumi_crds", "juice", "v1alpha1", "outputs.py") t.Logf("validating underscored field names in %s", filename) - pythonInputs, err = ioutil.ReadFile(filename) + pythonInputs, err = os.ReadFile(filename) if err != nil { t.Fatalf("expected to read generated Python code: %s", err) } diff --git a/tests/schema_test.go b/tests/schema_test.go index ff6e8c7..b1589a5 100644 --- a/tests/schema_test.go +++ b/tests/schema_test.go @@ -18,7 +18,6 @@ import ( "bytes" "encoding/json" "fmt" - "io/ioutil" "os" "testing" @@ -51,7 +50,7 @@ func UnmarshalSchemas(yamlPath string) (map[string]any, error) { } func UnmarshalTypeSpecJSON(jsonPath string) (map[string]pschema.TypeSpec, error) { - jsonFile, err := ioutil.ReadFile(jsonPath) + jsonFile, err := os.ReadFile(jsonPath) if err != nil { return nil, fmt.Errorf("could not read file %s: %w", jsonPath, err) }