Merge pull request #82 from maysunfaisal/useGlobalVar-1

Consume top level variables & attributes

Author: maysunfaisal

README.md

@@ -12,9 +12,10 @@ The Devfile Parser library is a Golang module that:
 
 The function documentation can be accessed via [pkg.go.dev](https://pkg.go.dev/github.com/devfile/library). 
 1. To parse a devfile, visit pkg/devfile/parse.go 
-   ```
+   ```go
    // Parses the devfile and validates the devfile data
-   devfile, err := devfilePkg.ParseAndValidate(devfileLocation)
+   // if top-level variables are not substituted successfully, the warnings can be logged by parsing variableWarning
+   devfile, variableWarning, err := devfilePkg.ParseDevfileAndValidate(devfileLocation)
 
    // To get all the components from the devfile
    components, err := devfile.Data.GetComponents(DevfileOptions{})
@@ -46,7 +47,7 @@ The function documentation can be accessed via [pkg.go.dev](https://pkg.go.dev/g
    })
    ```
 2. To get the Kubernetes objects from the devfile, visit pkg/devfile/generator/generators.go
-   ```
+   ```go
     // To get a slice of Kubernetes containers of type corev1.Container from the devfile component containers
     containers, err := generator.GetContainers(devfile)
 

devfile.yaml

@@ -4,6 +4,8 @@ metadata:
   version: 1.0.0
   attributes:
     alpha.build-dockerfile: /relative/path/to/Dockerfile
+variables:
+  test: testValue
 parent:
   # uri: https://raw.githubusercontent.com/odo-devfiles/registry/master/devfiles/nodejs/devfile.yaml
   id: nodejs
@@ -58,7 +60,7 @@ components:
     endpoints:
     - name: http-9090
       targetPort: 9090
-    image: registry.access.redhat.com/ubi8/nodejs-12:1-45
+    image: "{{invalid-var}}"
     memoryLimit: 1024Mi
     mountSources: true
     sourceMapping: /project
@@ -69,7 +71,7 @@ commands:
     group:
       isDefault: false
       kind: build
-    workingDir: /project
+    workingDir: "{{test}}"
   id: install2
   attributes:
     tool: odo

go.mod

@@ -3,7 +3,7 @@ module github.com/devfile/library
 go 1.13
 
 require (
-	github.com/devfile/api/v2 v2.0.0-20210408144711-a313872749ed
+	github.com/devfile/api/v2 v2.0.0-20210420202853-ff3c01bf8292
 	github.com/fatih/color v1.7.0
 	github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32
 	github.com/gobwas/glob v0.2.3

go.sum

@@ -40,10 +40,13 @@ func parserTest() {
 		}
 		fmt.Println("parsing devfile from ./devfile.yaml")
 	}
-	devfile, err := devfilepkg.ParseDevfileAndValidate(args)
+	devfile, warning, err := devfilepkg.ParseDevfileAndValidate(args)
 	if err != nil {
 		fmt.Println(err)
 	} else {
+		if len(warning.Commands) > 0 || len(warning.Components) > 0 || len(warning.Projects) > 0 || len(warning.StarterProjects) > 0 {
+			fmt.Printf("top-level variables were not substituted successfully %+v\n", warning)
+		}
 		devdata := devfile.Data
 		if (reflect.TypeOf(devdata) == reflect.TypeOf(&v2.DevfileV2{})) {
 			d := devdata.(*v2.DevfileV2)
@@ -66,7 +69,7 @@ func parserTest() {
 		}
 		for _, command := range commands {
 			if command.Exec != nil {
-				fmt.Printf("command %s is with kind: %s", command.Id, command.Exec.Group.Kind)
+				fmt.Printf("command %s is with kind: %s\n", command.Id, command.Exec.Group.Kind)
 				fmt.Printf("workingDir is: %s\n", command.Exec.WorkingDir)
 			}
 		}

main.go

@@ -1,6 +1,7 @@
 package devfile
 
 import (
+	"github.com/devfile/api/v2/pkg/validation/variables"
 	"github.com/devfile/library/pkg/devfile/parser"
 	"github.com/devfile/library/pkg/devfile/validate"
 )
@@ -69,21 +70,25 @@ func ParseAndValidate(path string) (d parser.DevfileObj, err error) {
 	return d, err
 }
 
-// ParseDevfileAndValidate func parses the devfile data
-// and validates the devfile integrity with the schema
-// and validates the devfile data.
-// Creates devfile context and runtime objects.
-func ParseDevfileAndValidate(args parser.ParserArgs) (d parser.DevfileObj, err error) {
+// ParseDevfileAndValidate func parses the devfile data, validates the devfile integrity with the schema
+// replaces the top-level variable keys if present and validates the devfile data.
+// It returns devfile context and runtime objects, variable substitution warning if any and an error.
+func ParseDevfileAndValidate(args parser.ParserArgs) (d parser.DevfileObj, varWarning variables.VariableWarning, err error) {
 	d, err = parser.ParseDevfile(args)
 	if err != nil {
-		return d, err
+		return d, varWarning, err
+	}
+
+	if d.Data.GetSchemaVersion() != "2.0.0" {
+		// replace the top level variable keys with their values in the devfile
+		varWarning = variables.ValidateAndReplaceGlobalVariable(d.Data.GetDevfileWorkspaceSpec())
 	}
 
 	// generic validation on devfile content
 	err = validate.ValidateDevfileData(d.Data)
 	if err != nil {
-		return d, err
+		return d, varWarning, err
 	}
 
-	return d, err
+	return d, varWarning, err
 }

pkg/devfile/parse.go

@@ -2,61 +2,81 @@ package data
 
 import (
 	v1 "github.com/devfile/api/v2/pkg/apis/workspaces/v1alpha2"
+	"github.com/devfile/api/v2/pkg/attributes"
 	devfilepkg "github.com/devfile/api/v2/pkg/devfile"
 	"github.com/devfile/library/pkg/devfile/parser/data/v2/common"
 )
 
 // DevfileData is an interface that defines functions for Devfile data operations
 type DevfileData interface {
+
+	// header related methods
+
 	GetSchemaVersion() string
 	SetSchemaVersion(version string)
 	GetMetadata() devfilepkg.DevfileMetadata
 	SetMetadata(metadata devfilepkg.DevfileMetadata)
 
+	// top-level attributes related method
+
+	GetAttributes() (attributes.Attributes, error)
+	AddAttributes(key string, value interface{}) error
+	UpdateAttributes(key string, value interface{}) error
+
 	// parent related methods
+
 	GetParent() *v1.Parent
 	SetParent(parent *v1.Parent)
 
 	// event related methods
+
 	GetEvents() v1.Events
 	AddEvents(events v1.Events) error
 	UpdateEvents(postStart, postStop, preStart, preStop []string)
 
 	// component related methods
+
 	GetComponents(common.DevfileOptions) ([]v1.Component, error)
 	AddComponents(components []v1.Component) error
 	UpdateComponent(component v1.Component)
 	DeleteComponent(name string) error
 
 	// project related methods
+
 	GetProjects(common.DevfileOptions) ([]v1.Project, error)
 	AddProjects(projects []v1.Project) error
 	UpdateProject(project v1.Project)
 	DeleteProject(name string) error
 
 	// starter projects related commands
+
 	GetStarterProjects(common.DevfileOptions) ([]v1.StarterProject, error)
 	AddStarterProjects(projects []v1.StarterProject) error
 	UpdateStarterProject(project v1.StarterProject)
 	DeleteStarterProject(name string) error
 
 	// command related methods
+
 	GetCommands(common.DevfileOptions) ([]v1.Command, error)
 	AddCommands(commands []v1.Command) error
 	UpdateCommand(command v1.Command)
 	DeleteCommand(id string) error
 
 	// volume mount related methods
+
 	AddVolumeMounts(containerName string, volumeMounts []v1.VolumeMount) error
 	DeleteVolumeMount(name string) error
 	GetVolumeMountPaths(mountName, containerName string) ([]string, error)
 
 	// workspace related methods
+
 	GetDevfileWorkspaceSpecContent() *v1.DevWorkspaceTemplateSpecContent
 	SetDevfileWorkspaceSpecContent(content v1.DevWorkspaceTemplateSpecContent)
+	GetDevfileWorkspaceSpec() *v1.DevWorkspaceTemplateSpec
 	SetDevfileWorkspaceSpec(spec v1.DevWorkspaceTemplateSpec)
 
 	// utils
+
 	GetDevfileContainerComponents(common.DevfileOptions) ([]v1.Component, error)
 	GetDevfileVolumeComponents(common.DevfileOptions) ([]v1.Component, error)
 }

pkg/devfile/parser/data/interface.go

@@ -9,6 +9,11 @@ const JsonSchema210 = `{
     "schemaVersion"
   ],
   "properties": {
+    "attributes": {
+      "description": "Map of implementation-dependant free-form YAML attributes.",
+      "type": "object",
+      "additionalProperties": true
+    },
     "commands": {
       "description": "Predefined, ready-to-use, devworkspace-related commands",
       "type": "array",
@@ -635,7 +640,7 @@ const JsonSchema210 = `{
       "type": "object",
       "properties": {
         "attributes": {
-          "description": "Map of implementation-dependant free-form YAML attributes.",
+          "description": "Map of implementation-dependant free-form YAML attributes. Deprecated, use the top-level attributes field instead.",
           "type": "object",
           "additionalProperties": true
         },
@@ -707,6 +712,11 @@ const JsonSchema210 = `{
         }
       ],
       "properties": {
+        "attributes": {
+          "description": "Overrides of attributes encapsulated in a parent devfile. Overriding is done according to K8S strategic merge patch standard rules.",
+          "type": "object",
+          "additionalProperties": true
+        },
         "commands": {
           "description": "Overrides of commands encapsulated in a parent devfile or a plugin. Overriding is done according to K8S strategic merge patch standard rules.",
           "type": "array",
@@ -1527,6 +1537,13 @@ const JsonSchema210 = `{
         "uri": {
           "description": "Uri of a Devfile yaml file",
           "type": "string"
+        },
+        "variables": {
+          "description": "Overrides of variables encapsulated in a parent devfile. Overriding is done according to K8S strategic merge patch standard rules.",
+          "type": "object",
+          "additionalProperties": {
+            "type": "string"
+          }
         }
       },
       "additionalProperties": false
@@ -1786,6 +1803,13 @@ const JsonSchema210 = `{
         },
         "additionalProperties": false
       }
+    },
+    "variables": {
+      "description": "Map of key-value variables used for string replacement in the devfile. Values can can be referenced via {{variable-key}} to replace the corresponding value in string fields in the devfile. Replacement cannot be used for\n\n - schemaVersion, metadata, parent source  - element identifiers, e.g. command id, component name, endpoint name, project name  - references to identifiers, e.g. in events, a command's component, container's volume mount name  - string enums, e.g. command group kind, endpoint exposure",
+      "type": "object",
+      "additionalProperties": {
+        "type": "string"
+      }
     }
   },
   "additionalProperties": false

pkg/devfile/parser/data/v2/2.1.0/devfileJsonSchema210.go

@@ -0,0 +1,52 @@
+package v2
+
+import (
+	"fmt"
+
+	"github.com/devfile/api/v2/pkg/attributes"
+)
+
+// GetAttributes gets the devfile top level attributes
+func (d *DevfileV2) GetAttributes() (attributes.Attributes, error) {
+	// This feature was introduced in 2.1.0; so any version 2.1.0 and up should use the 2.1.0 implementation
+	switch d.SchemaVersion {
+	case "2.0.0":
+		return attributes.Attributes{}, fmt.Errorf("top-level attributes is not supported in devfile schema version 2.0.0")
+	default:
+		return d.Attributes, nil
+	}
+}
+
+// UpdateAttributes updates the devfile top level attribute for the specific key, err out if key is absent
+func (d *DevfileV2) UpdateAttributes(key string, value interface{}) error {
+	var err error
+
+	// This feature was introduced in 2.1.0; so any version 2.1.0 and up should use the 2.1.0 implementation
+	switch d.SchemaVersion {
+	case "2.0.0":
+		return fmt.Errorf("top-level attributes is not supported in devfile schema version 2.0.0")
+	default:
+		if d.Attributes.Exists(key) {
+			d.Attributes.Put(key, value, &err)
+		} else {
+			return fmt.Errorf("cannot update top-level attribute, key %s is not present", key)
+		}
+	}
+
+	return err
+}
+
+// AddAttributes adds to the devfile top level attributes, value will be overwritten if key is already present
+func (d *DevfileV2) AddAttributes(key string, value interface{}) error {
+	var err error
+
+	// This feature was introduced in 2.1.0; so any version 2.1.0 and up should use the 2.1.0 implementation
+	switch d.SchemaVersion {
+	case "2.0.0":
+		return fmt.Errorf("top-level attributes is not supported in devfile schema version 2.0.0")
+	default:
+		d.Attributes.Put(key, value, &err)
+	}
+
+	return err
+}