Fix: excluding files from unneededGoFiles was not working (#120)

## Bug

files in `unneededGoFiles` were not excluded.

## Bug Description

`unneededGoFiles` string set contain files which are relative to the
import path (`kubernetes/*`), while the paths of generated files are
relative to `pkg.Name` (`crds/*`). This PR makes sure to use a proper
path when comparing files in `unneededGoFiles`.

---------

Co-authored-by: Matthew (Matt) Jeffryes <github@mjeffryes.net>

Author: shamil

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## Unreleased
 
+- Fix: excluding files from unneededGoFiles was not working (<https://github.com/pulumi/crd2pulumi/pull/120>)
 - Support kubernetes provider v4 (<https://github.com/pulumi/crd2pulumi/pull/119>)
 
 ## 1.2.5 (2023-05-31)

cmd/root.go

@@ -19,7 +19,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 
 	"github.com/pulumi/crd2pulumi/pkg/codegen"
@@ -29,7 +28,7 @@ import (
 // Version specifies the crd2pulumi version. It should be set by the linker via LDFLAGS. This defaults to dev
 var Version = "dev"
 
-const long = `crd2pulumi is a CLI tool that generates typed Kubernetes 
+const long = `crd2pulumi is a CLI tool that generates typed Kubernetes
 CustomResources to use in Pulumi programs, based on a
 CustomResourceDefinition YAML schema.`
 
@@ -92,7 +91,7 @@ func Execute() error {
 				}
 				var err error
 				if shouldUseStdin {
-					err = codegen.Generate(cs, []io.ReadCloser{ioutil.NopCloser(bytes.NewBuffer(stdinData))})
+					err = codegen.Generate(cs, []io.ReadCloser{io.NopCloser(bytes.NewBuffer(stdinData))})
 				} else {
 					err = codegen.GenerateFromFiles(cs, args)
 				}

pkg/codegen/golang.go

@@ -17,14 +17,17 @@ package codegen
 import (
 	"bytes"
 	"fmt"
+	"path"
 	"path/filepath"
+	"strings"
 
 	ijson "github.com/pulumi/crd2pulumi/internal/json"
 	"github.com/pulumi/pulumi/pkg/v3/codegen"
 	goGen "github.com/pulumi/pulumi/pkg/v3/codegen/go"
+	"github.com/pulumi/pulumi/pkg/v3/codegen/schema"
 )
 
-var unneededGoFiles = codegen.NewStringSet(
+var UnneededGoFiles = codegen.NewStringSet(
 	// The root directory doesn't define any resources:
 	"doc.go",
 	"init.go",
@@ -69,21 +72,47 @@ func GenerateGo(pg *PackageGenerator, name string) (buffers map[string]*bytes.Bu
 	pkg.Language[langName] = jsonData
 
 	files, err := goGen.GeneratePackage("crd2pulumi", pkg)
-
 	if err != nil {
 		return nil, fmt.Errorf("could not generate Go package: %w", err)
 	}
 
+	packageRoot, err := goPackageRoot(pkg.Reference())
+	if err != nil {
+		return nil, fmt.Errorf("could not get package root: %w", err)
+	}
+
 	pkg.Name = oldName
 	delete(pkg.Language, langName)
 
 	buffers = map[string]*bytes.Buffer{}
 	for path, code := range files {
 		newPath, _ := filepath.Rel(name, path)
-		if !unneededGoFiles.Has(newPath) {
+		pkgRelPath := strings.TrimPrefix(path, packageRoot+"/")
+
+		if !UnneededGoFiles.Has(pkgRelPath) {
 			buffers[newPath] = bytes.NewBuffer(code)
 		}
 	}
 
 	return buffers, err
 }
+
+// Similar to "packageRoot" method from pulumi/pkg/codegen/go/gen.go
+func goPackageRoot(pkg schema.PackageReference) (string, error) {
+	def, err := pkg.Definition()
+	if err != nil {
+		return "", err
+	}
+	var info goGen.GoPackageInfo
+	if goInfo, ok := def.Language["go"].(goGen.GoPackageInfo); ok {
+		info = goInfo
+	}
+	if info.RootPackageName != "" {
+		// package structure is flat
+		return "", nil
+	}
+	if info.ImportBasePath != "" {
+		return path.Base(info.ImportBasePath), nil
+	}
+	return strings.ReplaceAll(pkg.Name(), "-", ""), nil
+}

tests/crds_test.go

@@ -17,7 +17,6 @@ package tests
 import (
 	"fmt"
 	"io/fs"
-	"io/ioutil"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -32,7 +31,7 @@ const gkeManagedCertsUrl = "https://raw.githubusercontent.com/GoogleCloudPlatfor
 
 // execCrd2Pulumi runs the crd2pulumi binary in a temporary directory
 func execCrd2Pulumi(t *testing.T, lang, path string, additionalValidation func(t *testing.T, path string)) {
-	tmpdir, err := ioutil.TempDir("", "crd2pulumi_test")
+	tmpdir, err := os.MkdirTemp("", "crd2pulumi_test")
 	assert.Nil(t, err, "expected to create a temp dir for the CRD output")
 	t.Cleanup(func() {
 		t.Logf("removing temp dir %q for %s test", tmpdir, lang)
@@ -98,7 +97,7 @@ func TestCRDsWithUnderscore(t *testing.T) {
 		// Ensure inputs are camelCased.
 		filename := filepath.Join(path, "pulumi_crds", "juice", "v1alpha1", "_inputs.py")
 		t.Logf("validating underscored field names in %s", filename)
-		pythonInputs, err := ioutil.ReadFile(filename)
+		pythonInputs, err := os.ReadFile(filename)
 		if err != nil {
 			t.Fatalf("expected to read generated Python code: %s", err)
 		}
@@ -108,7 +107,7 @@ func TestCRDsWithUnderscore(t *testing.T) {
 		// Ensure outputs are camelCased.
 		filename = filepath.Join(path, "pulumi_crds", "juice", "v1alpha1", "outputs.py")
 		t.Logf("validating underscored field names in %s", filename)
-		pythonInputs, err = ioutil.ReadFile(filename)
+		pythonInputs, err = os.ReadFile(filename)
 		if err != nil {
 			t.Fatalf("expected to read generated Python code: %s", err)
 		}

tests/schema_test.go

@@ -18,7 +18,6 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"testing"
 
@@ -51,7 +50,7 @@ func UnmarshalSchemas(yamlPath string) (map[string]any, error) {
 }
 
 func UnmarshalTypeSpecJSON(jsonPath string) (map[string]pschema.TypeSpec, error) {
-	jsonFile, err := ioutil.ReadFile(jsonPath)
+	jsonFile, err := os.ReadFile(jsonPath)
 	if err != nil {
 		return nil, fmt.Errorf("could not read file %s: %w", jsonPath, err)
 	}

tests/unneeded_go_files_test.go

@@ -0,0 +1,55 @@
+package tests
+
+import (
+	"io"
+	"strings"
+	"testing"
+
+	"github.com/pulumi/crd2pulumi/pkg/codegen"
+)
+
+func TestUnneededGoFiles(t *testing.T) {
+	mockCRDYaml := `---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+spec:
+  group: uneeded-go-files-test.pulumi.com
+  names:
+    plural: testresources
+    singular: testresource
+    kind: TestResource
+  scope: Namespaced
+  versions:
+    - test:
+      name: test
+      schema:
+        openAPIV3Schema:
+          properties:
+            testProperty:
+              type: string`
+
+	yamlSources := []io.ReadCloser{
+		io.NopCloser(strings.NewReader(mockCRDYaml)),
+	}
+
+	// Invoke ReadPackagesFromSource
+	pg, err := codegen.ReadPackagesFromSource("", yamlSources)
+	if err != nil {
+		t.Fatalf("ReadPackagesFromSource failed: %v", err)
+	}
+
+	// Pick a generated file we want to exclude
+	uneededGoFile := "uneededgofilestest/test/testResource.go"
+	codegen.UnneededGoFiles.Add(uneededGoFile)
+
+	// Generate the code from the mocked CRD
+	buffers, err := codegen.GenerateGo(pg, "crds")
+	if err != nil {
+		t.Fatalf("GenerateGo failed: %v", err)
+	}
+
+	// Assert that buffers do not contain unneeded file
+	if _, exists := buffers["../kubernetes/"+uneededGoFile]; exists {
+		t.Errorf("Uneeded GO file was not excluded by GoGenerate, %s", uneededGoFile)
+	}
+}