Merge pull request #66 from yangcao77/359-concurrentMapWrites

concurrent URImap read and writes

Author: yangcao77

pkg/devfile/parser/context/context.go

@@ -9,8 +9,6 @@ import (
 	"k8s.io/klog"
 )
 
-var URIMap = make(map[string]bool)
-
 // DevfileCtx stores context info regarding devfile
 type DevfileCtx struct {
 
@@ -34,6 +32,9 @@ type DevfileCtx struct {
 
 	// filesystem for devfile
 	fs filesystem.Filesystem
+
+	// trace of all url referenced
+	uriMap map[string]bool
 }
 
 // NewDevfileCtx returns a new DevfileCtx type object
@@ -70,10 +71,13 @@ func (d *DevfileCtx) Populate() (err error) {
 		return err
 	}
 	klog.V(4).Infof("absolute devfile path: '%s'", d.absPath)
-	if URIMap[d.absPath] {
+	if d.uriMap == nil {
+		d.uriMap = make(map[string]bool)
+	}
+	if d.uriMap[d.absPath] {
 		return fmt.Errorf("URI %v is recursively referenced", d.absPath)
 	}
-	URIMap[d.absPath] = true
+	d.uriMap[d.absPath] = true
 	// Read and save devfile content
 	if err := d.SetDevfileContent(); err != nil {
 		return err
@@ -88,10 +92,13 @@ func (d *DevfileCtx) PopulateFromURL() (err error) {
 	if err != nil {
 		return err
 	}
-	if URIMap[d.url] {
+	if d.uriMap == nil {
+		d.uriMap = make(map[string]bool)
+	}
+	if d.uriMap[d.url] {
 		return fmt.Errorf("URI %v is recursively referenced", d.url)
 	}
-	URIMap[d.url] = true
+	d.uriMap[d.url] = true
 	// Read and save devfile content
 	if err := d.SetDevfileContent(); err != nil {
 		return err
@@ -132,3 +139,13 @@ func (d *DevfileCtx) SetAbsPath() (err error) {
 	return nil
 
 }
+
+// GetURIMap func returns current devfile uri map
+func (d *DevfileCtx) GetURIMap() map[string]bool {
+	return d.uriMap
+}
+
+// SetURIMap set uri map in the devfile ctx
+func (d *DevfileCtx) SetURIMap(uriMap map[string]bool) {
+	d.uriMap = uriMap
+}

pkg/devfile/parser/parse.go

@@ -48,9 +48,7 @@ func parseDevfile(d DevfileObj, flattenedDevfile bool) (DevfileObj, error) {
 			return DevfileObj{}, err
 		}
 	}
-	for uri := range devfileCtx.URIMap {
-		delete(devfileCtx.URIMap, uri)
-	}
+
 	// Successful
 	return d, nil
 }
@@ -185,27 +183,40 @@ func parseFromURI(uri string, curDevfileCtx devfileCtx.DevfileCtx) (DevfileObj,
 	if err != nil {
 		return DevfileObj{}, err
 	}
-
-	// absolute URL address
-	if strings.HasPrefix(uri, "http://") || strings.HasPrefix(uri, "https://") {
-		return ParseFromURL(uri)
-	}
+	// NewDevfileCtx
+	var d DevfileObj
+	absoluteURL := strings.HasPrefix(uri, "http://") || strings.HasPrefix(uri, "https://")
 
 	// relative path on disk
-	if curDevfileCtx.GetAbsPath() != "" {
-		return Parse(path.Join(path.Dir(curDevfileCtx.GetAbsPath()), uri))
+	if !absoluteURL && curDevfileCtx.GetAbsPath() != "" {
+		d.Ctx = devfileCtx.NewDevfileCtx(path.Join(path.Dir(curDevfileCtx.GetAbsPath()), uri))
+		d.Ctx.SetURIMap(curDevfileCtx.GetURIMap())
+
+		// Fill the fields of DevfileCtx struct
+		err = d.Ctx.Populate()
+		if err != nil {
+			return DevfileObj{}, err
+		}
+		return parseDevfile(d, true)
 	}
 
-	if curDevfileCtx.GetURL() != "" {
+	// absolute URL address
+	if absoluteURL {
+		d.Ctx = devfileCtx.NewURLDevfileCtx(uri)
+	} else if curDevfileCtx.GetURL() != "" {
 		u, err := url.Parse(curDevfileCtx.GetURL())
 		if err != nil {
 			return DevfileObj{}, err
 		}
-
 		u.Path = path.Join(path.Dir(u.Path), uri)
-		// u.String() is the joint absolute URL path
-		return ParseFromURL(u.String())
+		d.Ctx = devfileCtx.NewURLDevfileCtx(u.String())
 	}
+	d.Ctx.SetURIMap(curDevfileCtx.GetURIMap())
+	// Fill the fields of DevfileCtx struct
+	err = d.Ctx.PopulateFromURL()
+	if err != nil {
+		return DevfileObj{}, err
+	}
+	return parseDevfile(d, true)
 
-	return DevfileObj{}, fmt.Errorf("fail to parse from uri: %s", uri)
 }

pkg/devfile/parser/parse_test.go

@@ -2221,6 +2221,7 @@ func Test_parseParentAndPlugin_RecursivelyReference_withMultipleURI(t *testing.T
 			},
 		},
 	}
+
 	parentDevfile1 := DevfileObj{
 		Ctx: devfileCtx.NewDevfileCtx(OutputDevfileYamlPath),
 		Data: &v2.DevfileV2{