Add generating CRD from URL (#62)

Author: akrantz01

cmd/root.go

@@ -55,6 +55,7 @@ CustomResourceDefinition YAML schema.`
 const example = `crd2pulumi --nodejs crontabs.yaml
 crd2pulumi -dgnp crd-certificates.yaml crd-issuers.yaml crd-challenges.yaml
 crd2pulumi --pythonPath=crds/python/istio --nodejsPath=crds/nodejs/istio crd-all.gen.yaml crd-mixer.yaml crd-operator.yaml
+crd2pulumi --pythonPath=crds/python/gke https://raw.githubusercontent.com/GoogleCloudPlatform/gke-managed-certs/master/deploy/managedcertificates-crd.yaml
 
 Notice that by just setting a language-specific output path (--pythonPath, --nodejsPath, etc) the code will
 still get generated, so setting -p, -n, etc becomes unnecessary.
@@ -117,7 +118,7 @@ func NewLanguageSettings(flags *pflag.FlagSet) (gen.LanguageSettings, []string)
 		if golang {
 			notices = append(notices, "-g is not necessary if --goPath is already set")
 		}
-	} else if golang || goName != gen.DefaultName{
+	} else if golang || goName != gen.DefaultName {
 		path := filepath.Join(defaultOutputPath, Go)
 		ls.GoPath = &path
 	}

gen/generate.go

@@ -18,8 +18,11 @@ import (
 	"bytes"
 	"fmt"
 	"io/ioutil"
+	"net/http"
+	"net/url"
 	"os"
 	"path/filepath"
+	"regexp"
 	"strings"
 	"unicode"
 
@@ -41,6 +44,10 @@ const (
 	v1      string = "apiextensions.k8s.io/v1"
 )
 
+// fetchUrlRe is a regex to determine whether the requested file should
+// be fetched from a remote or read from the filesystem
+var fetchUrlRe = regexp.MustCompile(`^\w+://`)
+
 // Version specifies the crd2pulumi version. It should be set by the linker via LDFLAGS. This defaults to dev
 var Version string = "dev"
 
@@ -123,6 +130,27 @@ type PackageGenerator struct {
 	schemaPackageWithObjectMetaType *pschema.Package
 }
 
+func FetchFile(u *url.URL) ([]byte, error) {
+	req, err := http.NewRequest("GET", u.String(), nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Add("Accept", "application/x-yaml")
+	req.Header.Add("Accept", "text/yaml")
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("failed to connect to HTTP server: %s", err)
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("error getting CRD. Status=%d", resp.StatusCode)
+	}
+
+	defer resp.Body.Close()
+	return ioutil.ReadAll(resp.Body)
+}
+
 // Read contents of file, with special case for stdin '-'
 func ReadFileOrStdin(path string) ([]byte, error) {
 	if path == "-" {
@@ -132,11 +160,30 @@ func ReadFileOrStdin(path string) ([]byte, error) {
 	}
 }
 
+func LoadCRD(pathOrUrl string) ([]byte, error) {
+	if fetchUrlRe.MatchString(pathOrUrl) {
+		u, err := url.Parse(pathOrUrl)
+		if err != nil {
+			return nil, err
+		}
+
+		switch u.Scheme {
+		case "https", "http":
+			return FetchFile(u)
+		default:
+			return nil, fmt.Errorf("scheme %q is not supported", u.Scheme)
+		}
+	}
+
+	// If it isn't a http/s scheme, try it as a file
+	return ReadFileOrStdin(pathOrUrl)
+}
+
 func NewPackageGenerator(yamlPaths []string) (PackageGenerator, error) {
 	yamlFiles := make([][]byte, 0, len(yamlPaths))
 
 	for _, yamlPath := range yamlPaths {
-		yamlFile, err := ReadFileOrStdin(yamlPath)
+		yamlFile, err := LoadCRD(yamlPath)
 		if err != nil {
 			return PackageGenerator{}, errors.Wrapf(err, "could not read file %s", yamlPath)
 		}

tests/crds_test.go

@@ -25,22 +25,38 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-// TestCRDs enumerates all CRD YAML files, and generates them in each language.
-func TestCRDs(t *testing.T) {
+var languages = []string{"dotnet", "go", "nodejs", "python"}
+
+const gkeManagedCertsUrl = "https://raw.githubusercontent.com/GoogleCloudPlatform/gke-managed-certs/master/deploy/managedcertificates-crd.yaml"
+
+// execCrd2Pulumi runs the crd2pulumi binary in a temporary directory
+func execCrd2Pulumi(t *testing.T, lang, path string) {
+	tmpdir, err := ioutil.TempDir("", "")
+	assert.Nil(t, err, "expected to create a temp dir for the CRD output")
+	defer os.RemoveAll(tmpdir)
+	langFlag := "--" + lang + "Path"
+	t.Logf("crd2pulumi %s=%s %s: running", langFlag, tmpdir, path)
+	crdCmd := exec.Command("crd2pulumi", langFlag, tmpdir, "--force", path)
+	crdOut, err := crdCmd.CombinedOutput()
+	t.Logf("crd2pulumi %s=%s %s: output=\n%s", langFlag, tmpdir, path, crdOut)
+	assert.Nil(t, err, "expected crd2pulumi for '%s=%s %s' to succeed", langFlag, tmpdir, path)
+}
+
+// TestCRDsFromFile enumerates all CRD YAML files, and generates them in each language.
+func TestCRDsFromFile(t *testing.T) {
 	filepath.WalkDir("crds", func(path string, d fs.DirEntry, err error) error {
 		if !d.IsDir() && (filepath.Ext(path) == ".yml" || filepath.Ext(path) == ".yaml") {
-			for _, lang := range []string{"dotnet", "go", "nodejs", "python"} {
-				tmpdir, err := ioutil.TempDir("", "")
-				assert.Nil(t, err, "expected to create a temp dir for the CRD output")
-				defer os.RemoveAll(tmpdir)
-				langFlag := "--" + lang + "Path"
-				t.Logf("crd2pulumi %s=%s %s: running", langFlag, tmpdir, path)
-				crdCmd := exec.Command("crd2pulumi", langFlag, tmpdir, "--force", path)
-				crdOut, err := crdCmd.CombinedOutput()
-				t.Logf("crd2pulumi %s=%s %s: output=\n%s", langFlag, tmpdir, path, crdOut)
-				assert.Nil(t, err, "expected crd2pulumi for '%s=%s %s' to succeed", langFlag, tmpdir, path)
+			for _, lang := range languages {
+				execCrd2Pulumi(t, lang, path)
 			}
 		}
 		return nil
 	})
 }
+
+// TestCRDsFromUrl pulls the CRD YAML file from a URL and generates it in each language
+func TestCRDsFromUrl(t *testing.T) {
+	for _, lang := range languages {
+		execCrd2Pulumi(t, lang, gkeManagedCertsUrl)
+	}
+}