Add Kube YAML parser/reader

README.md

@@ -145,6 +145,14 @@ The function documentation can be accessed via [pkg.go.dev](https://pkg.go.dev/g
    // write to the devfile on disk
    err = devfile.WriteYamlDevfile()
    ```
+6. To parse the outerloop Kubernetes/OpenShift component's uri or inline content, call the read and parse functions
+   ```go
+   // Read the YAML content
+   values, err := ReadKubernetesYaml(src, fs)
+
+   // Get the Kubernetes resources
+   resources, err := ParseKubernetesYaml(values)
+   ```
 
 ## Projects using devfile/library
 

go.mod

@@ -20,6 +20,7 @@ require (
 	github.com/spf13/afero v1.2.2
 	github.com/stretchr/testify v1.7.0
 	github.com/xeipuuv/gojsonschema v1.2.0
+	gopkg.in/yaml.v3 v3.0.1
 	k8s.io/api v0.21.3
 	k8s.io/apimachinery v0.21.3
 	k8s.io/client-go v0.21.3

go.sum

@@ -748,8 +748,9 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
 gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

pkg/devfile/generator/generators.go

@@ -2,6 +2,7 @@ package generator
 
 import (
 	"fmt"
+
 	v1 "github.com/devfile/api/v2/pkg/apis/workspaces/v1alpha2"
 	"github.com/devfile/library/pkg/devfile/parser"
 	"github.com/devfile/library/pkg/devfile/parser/data/v2/common"

pkg/devfile/parser/reader.go

@@ -0,0 +1,129 @@
+package parser
+
+import (
+	"bytes"
+	"io"
+
+	"github.com/devfile/library/pkg/testingutil/filesystem"
+	"github.com/devfile/library/pkg/util"
+	"github.com/pkg/errors"
+	"gopkg.in/yaml.v3"
+	k8yaml "sigs.k8s.io/yaml"
+
+	routev1 "github.com/openshift/api/route/v1"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	extensionsv1 "k8s.io/api/extensions/v1beta1"
+)
+
+// YamlSrc specifies the src of the yaml in either Path, URL or Data format
+type YamlSrc struct {
+	// Path to the yaml file
+	Path string
+	// URL of the yaml file
+	URL string
+	// Data is the yaml content in []byte format
+	Data []byte
+}
+
+// KubernetesResources struct contains the Deployments, Services,
+// Routes and Ingresses resources
+type KubernetesResources struct {
+	Deployments []appsv1.Deployment
+	Services    []corev1.Service
+	Routes      []routev1.Route
+	Ingresses   []extensionsv1.Ingress
+}
+
+// ReadKubernetesYaml reads a yaml Kubernetes file from either the Path, URL or Data provided.
+// It returns all the parsed Kubernetes objects as an array of interface.
+// Consumers interested in the Kubernetes resources are expected to Unmarshal
+// it to the struct of the respective Kubernetes resource.
+func ReadKubernetesYaml(src YamlSrc, fs filesystem.Filesystem) ([]interface{}, error) {
+
+	var data []byte
+	var err error
+
+	if src.URL != "" {
+		data, err = util.DownloadFileInMemory(src.URL)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to download file %q", src.URL)
+		}
+	} else if src.Path != "" {
+		data, err = fs.ReadFile(src.Path)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to read yaml from path %q", src.Path)
+		}
+	} else if len(src.Data) > 0 {
+		data = src.Data
+	}
+
+	var values []interface{}
+	dec := yaml.NewDecoder(bytes.NewReader(data))
+	for {
+		var value interface{}
+		err = dec.Decode(&value)
+		if err != nil {
+			if err == io.EOF {
+				break
+			}
+			return nil, err
+		}
+		values = append(values, value)
+	}
+
+	return values, nil
+}
+
+// ParseKubernetesYaml Unmarshals the interface array of the Kubernetes resources
+// and returns it as a KubernetesResources struct. Only Deployment, Service, Route
+// and Ingress are processed. Consumers interested in other Kubernetes resources
+// are expected to parse the values interface array an Unmarshal it to their
+// desired Kuberenetes struct
+func ParseKubernetesYaml(values []interface{}) (KubernetesResources, error) {
+	var deployments []appsv1.Deployment
+	var services []corev1.Service
+	var routes []routev1.Route
+	var ingresses []extensionsv1.Ingress
+
+	for _, value := range values {
+		var deployment appsv1.Deployment
+		var service corev1.Service
+		var route routev1.Route
+		var ingress extensionsv1.Ingress
+
+		byteData, err := k8yaml.Marshal(value)
+		if err != nil {
+			return KubernetesResources{}, err
+		}
+
+		kubernetesMap := value.(map[string]interface{})
+		kind := kubernetesMap["kind"]
+
+		switch kind {
+		case "Deployment":
+			err = k8yaml.Unmarshal(byteData, &deployment)
+			deployments = append(deployments, deployment)
+		case "Service":
+			err = k8yaml.Unmarshal(byteData, &service)
+			services = append(services, service)
+		case "Route":
+			err = k8yaml.Unmarshal(byteData, &route)
+			routes = append(routes, route)
+		case "Ingress":
+			err = k8yaml.Unmarshal(byteData, &ingress)
+			ingresses = append(ingresses, ingress)
+		}
+
+		if err != nil {
+			return KubernetesResources{}, err
+		}
+	}
+
+	return KubernetesResources{
+		Deployments: deployments,
+		Services:    services,
+		Routes:      routes,
+		Ingresses:   ingresses,
+	}, nil
+}

pkg/devfile/parser/reader_test.go

@@ -0,0 +1,189 @@
+package parser
+
+import (
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"reflect"
+	"testing"
+
+	"github.com/devfile/library/pkg/testingutil/filesystem"
+	"github.com/devfile/library/pkg/util"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestReadAndParseKubernetesYaml(t *testing.T) {
+	const serverIP = "127.0.0.1:9080"
+	var data []byte
+
+	fs := filesystem.DefaultFs{}
+	absPath, err := util.GetAbsPath("../../../tests/yamls/resources.yaml")
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	data, err = fs.ReadFile(absPath)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	// Mocking the YAML file endpoint on a very basic level
+	testServer := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		_, err = w.Write(data)
+		if err != nil {
+			t.Errorf("Unexpected error while writing data: %v", err)
+		}
+	}))
+	// create a listener with the desired port.
+	l, err := net.Listen("tcp", serverIP)
+	if err != nil {
+		t.Errorf("Unexpected error while creating listener: %v", err)
+		return
+	}
+
+	// NewUnstartedServer creates a listener. Close that listener and replace
+	// with the one we created.
+	testServer.Listener.Close()
+	testServer.Listener = l
+
+	testServer.Start()
+	defer testServer.Close()
+
+	badData := append(data, 59)
+
+	tests := []struct {
+		name                string
+		src                 YamlSrc
+		fs                  filesystem.Filesystem
+		wantErr             bool
+		wantDeploymentNames []string
+		wantServiceNames    []string
+		wantRouteNames      []string
+		wantIngressNames    []string
+		wantOtherNames      []string
+	}{
+		{
+			name: "Read the YAML from the URL",
+			src: YamlSrc{
+				URL: "http://" + serverIP,
+			},
+			fs:                  filesystem.DefaultFs{},
+			wantDeploymentNames: []string{"deploy-sample", "deploy-sample-2"},
+			wantServiceNames:    []string{"service-sample", "service-sample-2"},
+			wantRouteNames:      []string{"route-sample", "route-sample-2"},
+			wantIngressNames:    []string{"ingress-sample", "ingress-sample-2"},
+			wantOtherNames:      []string{"pvc-sample", "pvc-sample-2"},
+		},
+		{
+			name: "Read the YAML from the Path",
+			src: YamlSrc{
+				Path: "../../../tests/yamls/resources.yaml",
+			},
+			fs:                  filesystem.DefaultFs{},
+			wantDeploymentNames: []string{"deploy-sample", "deploy-sample-2"},
+			wantServiceNames:    []string{"service-sample", "service-sample-2"},
+			wantRouteNames:      []string{"route-sample", "route-sample-2"},
+			wantIngressNames:    []string{"ingress-sample", "ingress-sample-2"},
+			wantOtherNames:      []string{"pvc-sample", "pvc-sample-2"},
+		},
+		{
+			name: "Read the YAML from the Data",
+			src: YamlSrc{
+				Data: data,
+			},
+			fs:                  filesystem.DefaultFs{},
+			wantDeploymentNames: []string{"deploy-sample", "deploy-sample-2"},
+			wantServiceNames:    []string{"service-sample", "service-sample-2"},
+			wantRouteNames:      []string{"route-sample", "route-sample-2"},
+			wantIngressNames:    []string{"ingress-sample", "ingress-sample-2"},
+			wantOtherNames:      []string{"pvc-sample", "pvc-sample-2"},
+		},
+		{
+			name: "Bad URL",
+			src: YamlSrc{
+				URL: "http://badurl",
+			},
+			fs:      filesystem.DefaultFs{},
+			wantErr: true,
+		},
+		{
+			name: "Bad Path",
+			src: YamlSrc{
+				Path: "$%^&",
+			},
+			fs:      filesystem.DefaultFs{},
+			wantErr: true,
+		},
+		{
+			name: "Bad Data",
+			src: YamlSrc{
+				Data: badData,
+			},
+			fs:      filesystem.DefaultFs{},
+			wantErr: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			values, err := ReadKubernetesYaml(tt.src, tt.fs)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("unexpected error: %v", err)
+				return
+			}
+
+			for _, value := range values {
+				kubernetesMap := value.(map[string]interface{})
+
+				kind := kubernetesMap["kind"]
+				metadataMap := kubernetesMap["metadata"].(map[string]interface{})
+				name := metadataMap["name"]
+
+				switch kind {
+				case "Deployment":
+					assert.Contains(t, tt.wantDeploymentNames, name)
+				case "Service":
+					assert.Contains(t, tt.wantServiceNames, name)
+				case "Route":
+					assert.Contains(t, tt.wantRouteNames, name)
+				case "Ingress":
+					assert.Contains(t, tt.wantIngressNames, name)
+				default:
+					assert.Contains(t, tt.wantOtherNames, name)
+				}
+			}
+
+			if len(values) > 0 {
+				resources, err := ParseKubernetesYaml(values)
+				if err != nil {
+					t.Error(err)
+					return
+				}
+
+				if reflect.DeepEqual(resources, KubernetesResources{}) {
+					t.Error("Kubernetes resources is empty, expected to contain some resources")
+				} else {
+					deployments := resources.Deployments
+					services := resources.Services
+					routes := resources.Routes
+					ingresses := resources.Ingresses
+
+					for _, deploy := range deployments {
+						assert.Contains(t, tt.wantDeploymentNames, deploy.Name)
+					}
+					for _, svc := range services {
+						assert.Contains(t, tt.wantServiceNames, svc.Name)
+					}
+					for _, route := range routes {
+						assert.Contains(t, tt.wantRouteNames, route.Name)
+					}
+					for _, ingress := range ingresses {
+						assert.Contains(t, tt.wantIngressNames, ingress.Name)
+					}
+				}
+			}
+		})
+	}
+}