fix: indexing of outside variables must be evaluated. (#386)

generate/genhcl/genhcl_test.go

@@ -2092,6 +2092,130 @@ func TestPartialEval(t *testing.T) {
 				str("string", "b"),
 			),
 		},
+		{
+			name: "indexing of outside variables",
+			globals: hcldoc(
+				globals(
+					number("depth", 1),
+				),
+			),
+			config: hcldoc(
+				expr("folder_id", `data.google_active_folder[global.depth].0.id`),
+			),
+			want: hcldoc(
+				expr("folder_id", `data.google_active_folder[1].0.id`),
+			),
+		},
+		{
+			name: "indexing of outside variables with interpolation of single var",
+			globals: hcldoc(
+				globals(
+					number("depth", 1),
+				),
+			),
+			config: hcldoc(
+				expr("folder_id", `data.google_active_folder["${global.depth}"].0.id`),
+			),
+			want: hcldoc(
+				expr("folder_id", `data.google_active_folder[1].0.id`),
+			),
+		},
+		{
+			name: "indexing of outside variables with interpolation",
+			globals: hcldoc(
+				globals(
+					number("depth", 1),
+				),
+			),
+			config: hcldoc(
+				expr("folder_id", `data.google_active_folder["l${global.depth}"].0.id`),
+			),
+			want: hcldoc(
+				expr("folder_id", `data.google_active_folder["l1"].0.id`),
+			),
+		},
+		{
+			name: "outside variable with splat operator",
+			config: hcldoc(
+				expr("folder_id", `data.test[*].0.id`),
+			),
+			want: hcldoc(
+				expr("folder_id", `data.test[*].0.id`),
+			),
+		},
+		{
+			name: "outside variable with splat getattr operator",
+			config: hcldoc(
+				expr("folder_id", `data.test.*.0.id`),
+			),
+			want: hcldoc(
+				expr("folder_id", `data.test.*.0.id`),
+			),
+		},
+		{
+			name: "multiple indexing",
+			config: hcldoc(
+				expr("a", `data.test[0][0][0]`),
+			),
+			want: hcldoc(
+				expr("a", `data.test[0][0][0]`),
+			),
+		},
+		{
+			name: "multiple indexing with evaluation",
+			globals: hcldoc(
+				globals(
+					number("val", 1),
+				),
+			),
+			config: hcldoc(
+				expr("a", `data.test[global.val][0][0]`),
+			),
+			want: hcldoc(
+				expr("a", `data.test[1][0][0]`),
+			),
+		},
+		{
+			name: "multiple indexing with evaluation 2",
+			globals: hcldoc(
+				globals(
+					number("val", 1),
+				),
+			),
+			config: hcldoc(
+				expr("a", `data.test[0][global.val][global.val+1]`),
+			),
+			want: hcldoc(
+				expr("a", `data.test[0][1][1+1]`),
+			),
+		},
+		{
+			name: "nested indexing",
+			globals: hcldoc(
+				globals(
+					expr("obj", `{
+						key = {
+							key2 = {
+								val = "hello"
+							}
+						}
+					}`),
+					expr("obj2", `{
+						keyname = "key"
+					}`),
+					expr("key", `{
+						key2 = "keyname"
+					}`),
+					str("key2", "key2"),
+				),
+			),
+			config: hcldoc(
+				expr("hello", `global.obj[global.obj2[global.key[global.key2]]][global.key2]["val"]`),
+			),
+			want: hcldoc(
+				str("hello", "hello"),
+			),
+		},
 		{
 			name: "obj for loop without eval references",
 			config: hcldoc(

hcl/eval/partial.go

@@ -216,13 +216,36 @@ func (e *engine) emitn(n int) {
 	}
 }
 
-func (e *engine) emitVariable(v variable) {
+func (e *engine) emitVariable(v variable) error {
 	tos := e.evalstack.peek()
+	for i, original := range v.index {
+		// Prepare a subengine to evaluate the indexing tokens.
+		// The partial eval engine expects the token stream to be EOF terminated.
+		index := copytokens(original)
+		index = append(index, tokenEOF())
+		subengine := newPartialEvalEngine(index, e.ctx)
+		subengine.multiline++
+
+		// this will only fail in the case of `<ident>[]` but this would be a
+		// syntax error (hopeful) caught by hcl lib.
+		tok := subengine.peekn(subengine.skip(0))
+		if tok.Type != hclsyntax.TokenStar {
+			evaluatedIndex, err := subengine.Eval()
+			if err != nil {
+				return err
+			}
+
+			// remove EOF
+			v.index[i] = evaluatedIndex[0 : len(evaluatedIndex)-1]
+		}
+	}
 	tos.pushEvaluated(v.alltokens()...)
-	for i := 0; i < v.size(); i++ {
-		tos.pushOriginal(e.peek())
+	for _, tok := range v.original {
+		tos.pushOriginal(tok)
 		e.pos++
 	}
+
+	return nil
 }
 
 func (e *engine) emitTokens(original hclwrite.Tokens, evaluated hclwrite.Tokens) {
@@ -252,7 +275,7 @@ func (e *engine) emitnlparens() {
 
 func (e *engine) skipTokens(from int, tokens ...hclsyntax.TokenType) int {
 	i := from
-	for e.hasTokens() {
+	for i < len(e.tokens) {
 		found := false
 		for _, t := range tokens {
 			if e.peekn(i).Type == t {
@@ -397,13 +420,13 @@ loop:
 			e.peek().Type, e.tokens[e.pos:].Bytes()))
 	}
 
+	e.emitnlparens()
+	e.emitComments()
+
 	if !e.hasTokens() {
 		return nil
 	}
 
-	e.emitnlparens()
-	e.emitComments()
-
 	// exprTerm INDEX,GETATTR,SPLAT (expression acessors)
 	tok = e.peek()
 	switch tok.Type {
@@ -418,6 +441,10 @@ loop:
 	e.emitnlparens()
 	e.emitComments()
 
+	if !e.hasTokens() {
+		return nil
+	}
+
 	// operation && conditional
 
 	tok = e.peek()
@@ -486,7 +513,7 @@ func (e *engine) evalAcessors() error {
 				parsed = true
 			}
 
-			if e.peek().Type == hclsyntax.TokenDot {
+			if e.hasTokens() && e.peek().Type == hclsyntax.TokenDot {
 				err := e.evalGetAttr()
 				if err != nil {
 					return err
@@ -511,6 +538,8 @@ func (e *engine) evalIndex() error {
 	e.newnode()
 	e.multiline++
 
+	defer func() { e.multiline-- }()
+
 	e.assert(hclsyntax.TokenOBrack)
 	e.emit()
 
@@ -535,6 +564,11 @@ func (e *engine) evalIndex() error {
 	e.emit()
 	e.emitnlparens()
 	e.emitComments()
+
+	if !e.hasTokens() {
+		return nil
+	}
+
 	tok := e.peek()
 	switch tok.Type {
 	case hclsyntax.TokenOBrack, hclsyntax.TokenDot:
@@ -545,8 +579,6 @@ func (e *engine) evalIndex() error {
 		e.commit()
 	}
 
-	e.multiline--
-
 	return nil
 }
 
@@ -721,7 +753,10 @@ func (e *engine) evalForExpr(matchOpenType, matchCloseType hclsyntax.TokenType)
 				)
 			}
 
-			e.emitVariable(v)
+			err := e.emitVariable(v)
+			if err != nil {
+				return err
+			}
 		} else {
 			e.emit()
 		}
@@ -841,8 +876,7 @@ func (e *engine) evalVar() error {
 	}
 
 	if !v.isTerramate {
-		e.emitVariable(v)
-		return nil
+		return e.emitVariable(v)
 	}
 
 	var expr []byte
@@ -1138,14 +1172,17 @@ func (e *engine) parseVariable(tokens hclwrite.Tokens) (v variable, found bool)
 	nsvar := string(v.name[0].Bytes)
 	v.isTerramate = nsvar == "global" || nsvar == "terramate"
 
-	if pos < len(tokens) && tokens[pos].Type == hclsyntax.TokenOBrack {
-		v.index = parseIndexing(tokens[pos:])
+	for pos < len(tokens) && tokens[pos].Type == hclsyntax.TokenOBrack {
+		index, skip := parseIndexing(tokens[pos:])
+		v.index = append(v.index, index)
+		pos += skip
 	}
 
+	v.original = tokens[0:pos]
 	return v, true
 }
 
-func parseIndexing(tokens hclwrite.Tokens) hclwrite.Tokens {
+func parseIndexing(tokens hclwrite.Tokens) (hclwrite.Tokens, int) {
 	assertToken(tokens[0], hclsyntax.TokenOBrack)
 
 	pos := 1
@@ -1165,22 +1202,14 @@ func parseIndexing(tokens hclwrite.Tokens) hclwrite.Tokens {
 		}
 
 		if matchingBracks == 0 {
-			if tokens[pos+1].Type == hclsyntax.TokenOBrack {
-				// beginning of next '[' sequence.
-				// this is for the parsing of a.b[<expr][<expr2]...
-				matchingBracks++
-				pos += 2
-				continue
-			}
-
 			break
 		}
 
 		pos++
 	}
 
 	assertToken(tokens[pos], hclsyntax.TokenCBrack)
-	return tokens[1:pos]
+	return tokens[1:pos], pos + 1
 }
 
 func (e *engine) canEvaluateIdent() bool {
@@ -1367,26 +1396,29 @@ func copytoken(tok *hclwrite.Token) *hclwrite.Token {
 
 // variable is a low-level representation of a variable in terms of tokens.
 type variable struct {
-	name  hclwrite.Tokens
-	index hclwrite.Tokens
+	name hclwrite.Tokens
+
+	// a variable can have nested indexing. eg.: global.a[0][1][global.b][0]
+	index []hclwrite.Tokens
 
+	original    hclwrite.Tokens
 	isTerramate bool
 }
 
 func (v variable) alltokens() hclwrite.Tokens {
 	tokens := v.name
-	if len(v.index) > 0 {
+	for _, index := range v.index {
 		tokens = append(tokens, tokenOBrack())
-		tokens = append(tokens, v.index...)
+		tokens = append(tokens, index...)
 		tokens = append(tokens, tokenCBrack())
 	}
 	return tokens
 }
 
 func (v variable) size() int {
 	sz := len(v.name)
-	if len(v.index) > 0 {
-		sz += len(v.index) + 2 // `[` <tokens> `]`
+	for _, index := range v.index {
+		sz += len(index) + 2 // `[` <tokens> `]`
 	}
 	return sz
 }

hcl/eval/testdata/fuzz/FuzzPartialEval/27686927c7dc0f8c0ff73b62d86fc5694f47a584961548219da9b6c94d3795d7

@@ -0,0 +1,2 @@
+go test fuzz v1
+string("A0.0[*]")

hcl/eval/testdata/fuzz/FuzzPartialEval/373b91ace300794a0f36f4e2569d1673fea0a77c43bbd49eb6c780af070c60e4

@@ -0,0 +1,2 @@
+go test fuzz v1
+string("A.0[0.*]")

hcl/eval/testdata/fuzz/FuzzPartialEval/5a94c7dbdff6650d02e9192dce3edd12d1806f97f179f4aaf79807d3b161f7aa

@@ -0,0 +1,2 @@
+go test fuzz v1
+string("A.0[0\n]")

hcl/eval/testdata/fuzz/FuzzPartialEval/85a518a4f8e2a5fe09d702b5fd98ac14ff8500c8bb7427e3c57d527774883602

@@ -0,0 +1,2 @@
+go test fuzz v1
+string("A.0[0[0]]")

hcl/eval/testdata/fuzz/FuzzPartialEval/93cbb0905135e8197e3886fff96488ff203b7d8864f240a4c770731ea3c49b82

@@ -0,0 +1,2 @@
+go test fuzz v1
+string("\"${(\"${(A.0[\n*])}\")}\"")

hcl/eval/testdata/fuzz/FuzzPartialEval/94c8f22dea083f8c72e23cecdf51cd065c84976d84d38f9b0e7d7a97f1152332

@@ -0,0 +1,2 @@
+go test fuzz v1
+string("A.0[A0\n]")

hcl/eval/testdata/fuzz/FuzzPartialEval/c832672669ea3aa88a312b9521927abb062921923c382b96072120ece09bfd88

@@ -0,0 +1,2 @@
+go test fuzz v1
+string("A00.0[0][0]")

hcl/eval/testdata/fuzz/FuzzPartialEval/de40bb3543f566ed458f6279839aec6538c52adc1371bc3be2bb2d2a669d66df

@@ -0,0 +1,2 @@
+go test fuzz v1
+string("A.0[0.A]")

hcl/eval/tokens.go

@@ -46,3 +46,9 @@ func tokenCBrack() *hclwrite.Token {
 		Bytes: []byte("]"),
 	}
 }
+
+func tokenEOF() *hclwrite.Token {
+	return &hclwrite.Token{
+		Type: hclsyntax.TokenEOF,
+	}
+}