Added templatestring function similar to templatefile (#1223)

Signed-off-by: sanskruti-shahu <sanskruti.shahu@harness.io>
Signed-off-by: Sanskruti Shahu <76054960+sanskruti-shahu@users.noreply.github.com>
Co-authored-by: James Humphries <James@james-humphries.co.uk>
This commit is contained in:
Sanskruti Shahu 2024-02-28 20:26:05 +05:30 committed by GitHub
parent 659242f977
commit 835dcb8121
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
12 changed files with 517 additions and 60 deletions

View File

@ -7,6 +7,7 @@ UPGRADE NOTES:
NEW FEATURES:
ENHANCEMENTS:
* Added `templatestring` function that takes a string and renders it as a template using a supplied set of template variables. ([#1223](https://github.com/opentofu/opentofu/pull/1223))
* Added `-concise` flag to omit the refreshing state logs when tofu plan is run. ([#1225](https://github.com/opentofu/opentofu/pull/1225))
* `nonsensitive` function no longer returns error when applied to values that are not sensitive ([#369](https://github.com/opentofu/opentofu/pull/369))
* Managing large local terraform.tfstate files is now much faster. ([#579](https://github.com/opentofu/opentofu/pull/579))

View File

@ -427,6 +427,10 @@ var DescriptionList = map[string]descriptionEntry{
Description: "`templatefile` reads the file at the given path and renders its content as a template using a supplied set of template variables.",
ParamDescription: []string{"", ""},
},
"templatestring": {
Description: "`templatestring` processes the provided string as a template using a supplied set of template variables.",
ParamDescription: []string{"", ""},
},
"textdecodebase64": {
Description: "`textdecodebase64` function decodes a string that was previously Base64-encoded, and then interprets the result as characters in a specified character encoding.",
ParamDescription: []string{"", ""},

View File

@ -101,63 +101,6 @@ func MakeTemplateFileFunc(baseDir string, funcsCb func() map[string]function.Fun
return expr, nil
}
renderTmpl := func(expr hcl.Expression, varsVal cty.Value) (cty.Value, error) {
if varsTy := varsVal.Type(); !(varsTy.IsMapType() || varsTy.IsObjectType()) {
return cty.DynamicVal, function.NewArgErrorf(1, "invalid vars value: must be a map") // or an object, but we don't strongly distinguish these most of the time
}
ctx := &hcl.EvalContext{
Variables: varsVal.AsValueMap(),
}
// We require all of the variables to be valid HCL identifiers, because
// otherwise there would be no way to refer to them in the template
// anyway. Rejecting this here gives better feedback to the user
// than a syntax error somewhere in the template itself.
for n := range ctx.Variables {
if !hclsyntax.ValidIdentifier(n) {
// This error message intentionally doesn't describe _all_ of
// the different permutations that are technically valid as an
// HCL identifier, but rather focuses on what we might
// consider to be an "idiomatic" variable name.
return cty.DynamicVal, function.NewArgErrorf(1, "invalid template variable name %q: must start with a letter, followed by zero or more letters, digits, and underscores", n)
}
}
// We'll pre-check references in the template here so we can give a
// more specialized error message than HCL would by default, so it's
// clearer that this problem is coming from a templatefile call.
for _, traversal := range expr.Variables() {
root := traversal.RootName()
if _, ok := ctx.Variables[root]; !ok {
return cty.DynamicVal, function.NewArgErrorf(1, "vars map does not contain key %q, referenced at %s", root, traversal[0].SourceRange())
}
}
givenFuncs := funcsCb() // this callback indirection is to avoid chicken/egg problems
funcs := make(map[string]function.Function, len(givenFuncs))
for name, fn := range givenFuncs {
if name == "templatefile" {
// We stub this one out to prevent recursive calls.
funcs[name] = function.New(&function.Spec{
Params: params,
Type: func(args []cty.Value) (cty.Type, error) {
return cty.NilType, fmt.Errorf("cannot recursively call templatefile from inside templatefile call")
},
})
continue
}
funcs[name] = fn
}
ctx.Functions = funcs
val, diags := expr.Value(ctx)
if diags.HasErrors() {
return cty.DynamicVal, diags
}
return val, nil
}
return function.New(&function.Spec{
Params: params,
Type: func(args []cty.Value) (cty.Type, error) {
@ -177,7 +120,7 @@ func MakeTemplateFileFunc(baseDir string, funcsCb func() map[string]function.Fun
// This is safe even if args[1] contains unknowns because the HCL
// template renderer itself knows how to short-circuit those.
val, err := renderTmpl(expr, args[1])
val, err := renderTemplate(expr, args[1], funcsCb)
return val.Type(), err
},
Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) {
@ -186,7 +129,7 @@ func MakeTemplateFileFunc(baseDir string, funcsCb func() map[string]function.Fun
if err != nil {
return cty.DynamicVal, err
}
result, err := renderTmpl(expr, args[1])
result, err := renderTemplate(expr, args[1], funcsCb)
return result.WithMarks(pathMarks), err
},
})

View File

@ -151,7 +151,7 @@ func TestTemplateFile(t *testing.T) {
cty.StringVal("testdata/recursive.tmpl"),
cty.MapValEmpty(cty.String),
cty.NilVal,
`testdata/recursive.tmpl:1,3-16: Error in function call; Call to function "templatefile" failed: cannot recursively call templatefile from inside templatefile call.`,
`testdata/recursive.tmpl:1,3-16: Error in function call; Call to function "templatefile" failed: cannot recursively call templatefile from inside templatefile or templatestring.`,
},
{
cty.StringVal("testdata/list.tmpl"),

View File

@ -0,0 +1,89 @@
// Copyright (c) The OpenTofu Authors
// SPDX-License-Identifier: MPL-2.0
// Copyright (c) 2023 HashiCorp, Inc.
// SPDX-License-Identifier: MPL-2.0
package funcs
import (
"fmt"
"github.com/hashicorp/hcl/v2"
"github.com/hashicorp/hcl/v2/hclsyntax"
"github.com/zclconf/go-cty/cty"
"github.com/zclconf/go-cty/cty/function"
)
func renderTemplate(expr hcl.Expression, varsVal cty.Value, funcsCb func() map[string]function.Function) (cty.Value, error) {
if varsTy := varsVal.Type(); !(varsTy.IsMapType() || varsTy.IsObjectType()) {
return cty.DynamicVal, function.NewArgErrorf(1, "invalid vars value: must be a map") // or an object, but we don't strongly distinguish these most of the time
}
ctx := &hcl.EvalContext{
Variables: varsVal.AsValueMap(),
}
// We require all of the variables to be valid HCL identifiers, because
// otherwise there would be no way to refer to them in the template
// anyway. Rejecting this here gives better feedback to the user
// than a syntax error somewhere in the template itself.
for n := range ctx.Variables {
if !hclsyntax.ValidIdentifier(n) {
// This error message intentionally doesn't describe _all_ of
// the different permutations that are technically valid as an
// HCL identifier, but rather focuses on what we might
// consider to be an "idiomatic" variable name.
return cty.DynamicVal, function.NewArgErrorf(1, "invalid template variable name %q: must start with a letter, followed by zero or more letters, digits, and underscores", n)
}
}
// currFilename stores the filename of the template file, if any.
currFilename := expr.Range().Filename
// We'll pre-check references in the template here so we can give a
// more specialized error message than HCL would by default, so it's
// clearer that this problem is coming from a templatefile/templatestring call.
for _, traversal := range expr.Variables() {
root := traversal.RootName()
referencedPos := fmt.Sprintf("%q", root)
if currFilename != templateStringFilename {
referencedPos = fmt.Sprintf("%q, referenced at %s", root, traversal[0].SourceRange())
}
if _, ok := ctx.Variables[root]; !ok {
return cty.DynamicVal, function.NewArgErrorf(1, "vars map does not contain key %s", referencedPos)
}
}
givenFuncs := funcsCb() // this callback indirection is to avoid chicken/egg problems
funcs := make(map[string]function.Function, len(givenFuncs))
for name, fn := range givenFuncs {
if name == "templatefile" {
// We stub this one out to prevent recursive calls.
funcs[name] = function.New(&function.Spec{
Params: []function.Parameter{
{
Name: "path",
Type: cty.String,
AllowMarked: true,
},
{
Name: "vars",
Type: cty.DynamicPseudoType,
},
},
Type: func(args []cty.Value) (cty.Type, error) {
return cty.NilType, fmt.Errorf("cannot recursively call templatefile from inside templatefile or templatestring")
},
})
continue
}
funcs[name] = fn
}
ctx.Functions = funcs
val, diags := expr.Value(ctx)
if diags.HasErrors() {
return cty.DynamicVal, diags
}
return val, nil
}

View File

@ -0,0 +1,112 @@
// Copyright (c) The OpenTofu Authors
// SPDX-License-Identifier: MPL-2.0
// Copyright (c) 2023 HashiCorp, Inc.
// SPDX-License-Identifier: MPL-2.0
package funcs
import (
"testing"
"github.com/hashicorp/hcl/v2"
"github.com/opentofu/opentofu/internal/lang/marks"
"github.com/zclconf/go-cty/cty"
"github.com/zclconf/go-cty/cty/function"
)
func TestRenderTemplate(t *testing.T) {
tests := map[string]struct {
Expr hcl.Expression
Vars cty.Value
Want cty.Value
Err string
}{
"String interpolation with variable": {
hcl.StaticExpr(cty.StringVal("Hello, ${name}!"), hcl.Range{}),
cty.MapVal(map[string]cty.Value{
"name": cty.StringVal("Jodie"),
}),
cty.StringVal("Hello, ${name}!"),
``,
},
"Looping through list": {
hcl.StaticExpr(cty.StringVal("Items: %{ for x in list ~} ${x} %{ endfor ~}"), hcl.Range{}),
cty.ObjectVal(map[string]cty.Value{
"list": cty.ListVal([]cty.Value{
cty.StringVal("a"),
cty.StringVal("b"),
cty.StringVal("c"),
}),
}),
cty.StringVal("Items: %{ for x in list ~} ${x} %{ endfor ~}"),
``,
},
"Looping through map": {
hcl.StaticExpr(cty.StringVal("%{ for key, value in list ~} ${key}:${value} %{ endfor ~}"), hcl.Range{}),
cty.ObjectVal(map[string]cty.Value{
"list": cty.ObjectVal(map[string]cty.Value{
"item1": cty.StringVal("a"),
"item2": cty.StringVal("b"),
"item3": cty.StringVal("c"),
}),
}),
cty.StringVal("%{ for key, value in list ~} ${key}:${value} %{ endfor ~}"),
``,
},
"Invalid template variable name": {
hcl.StaticExpr(cty.StringVal("Hello, ${1}!"), hcl.Range{}),
cty.MapVal(map[string]cty.Value{
"1": cty.StringVal("Jodie"),
}),
cty.NilVal,
`invalid template variable name "1": must start with a letter, followed by zero or more letters, digits, and underscores`,
},
"Interpolation of a boolean value": {
hcl.StaticExpr(cty.StringVal("${val}"), hcl.Range{}),
cty.ObjectVal(map[string]cty.Value{
"val": cty.True,
}),
cty.StringVal("${val}"),
``,
},
"Sensitive string template": {
hcl.StaticExpr(cty.StringVal("My password is 1234").Mark(marks.Sensitive), hcl.Range{}),
cty.EmptyObjectVal,
cty.StringVal("My password is 1234").Mark(marks.Sensitive),
``,
},
"Sensitive template variable": {
hcl.StaticExpr(cty.StringVal("My password is ${pass}"), hcl.Range{}),
cty.ObjectVal(map[string]cty.Value{
"pass": cty.StringVal("secret").Mark(marks.Sensitive),
}),
cty.StringVal("My password is ${pass}"),
``,
},
}
for name, test := range tests {
t.Run(name, func(t *testing.T) {
got, err := renderTemplate(test.Expr, test.Vars, func() map[string]function.Function {
return map[string]function.Function{}
})
if err != nil {
if test.Err == "" {
t.Fatalf("unexpected error: %s", err)
} else {
if got, want := err.Error(), test.Err; got != want {
t.Errorf("wrong error\ngot: %s\nwant: %s", got, want)
}
}
} else if test.Err != "" {
t.Fatal("succeeded; want error")
} else {
if !got.RawEquals(test.Want) {
t.Errorf("wrong result\ngot: %#v\nwant: %#v", got, test.Want)
}
}
})
}
}

View File

@ -9,6 +9,8 @@ import (
"regexp"
"strings"
"github.com/hashicorp/hcl/v2"
"github.com/hashicorp/hcl/v2/hclsyntax"
"github.com/zclconf/go-cty/cty"
"github.com/zclconf/go-cty/cty/function"
)
@ -164,3 +166,69 @@ func Replace(str, substr, replace cty.Value) (cty.Value, error) {
func StrContains(str, substr cty.Value) (cty.Value, error) {
return StrContainsFunc.Call([]cty.Value{str, substr})
}
// This constant provides a placeholder value for filename indicating
// that no file is needed for templatestring.
const (
templateStringFilename = "NoFileNeeded"
)
// MakeTemplateStringFunc constructs a function that takes a string and
// an arbitrary object of named values and attempts to render that string
// as a template using HCL template syntax.
func MakeTemplateStringFunc(content string, funcsCb func() map[string]function.Function) function.Function {
params := []function.Parameter{
{
Name: "data",
Type: cty.String,
AllowMarked: true,
},
{
Name: "vars",
Type: cty.DynamicPseudoType,
AllowMarked: true,
},
}
loadTmpl := func(content string, marks cty.ValueMarks) (hcl.Expression, error) {
expr, diags := hclsyntax.ParseTemplate([]byte(content), templateStringFilename, hcl.Pos{Line: 1, Column: 1})
if diags.HasErrors() {
return nil, diags
}
return expr, nil
}
return function.New(&function.Spec{
Params: params,
Type: func(args []cty.Value) (cty.Type, error) {
if !(args[0].IsKnown() && args[1].IsKnown()) {
return cty.DynamicPseudoType, nil
}
// We'll render our template now to see what result type it produces.
// A template consisting only of a single interpolation can potentially
// return any type.
dataArg, dataMarks := args[0].Unmark()
expr, err := loadTmpl(dataArg.AsString(), dataMarks)
if err != nil {
return cty.DynamicPseudoType, err
}
// This is safe even if args[1] contains unknowns because the HCL
// template renderer itself knows how to short-circuit those.
val, err := renderTemplate(expr, args[1], funcsCb)
return val.Type(), err
},
Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) {
dataArg, dataMarks := args[0].Unmark()
expr, err := loadTmpl(dataArg.AsString(), dataMarks)
if err != nil {
return cty.DynamicVal, err
}
result, err := renderTemplate(expr, args[1], funcsCb)
return result.WithMarks(dataMarks), err
},
})
}

View File

@ -9,7 +9,9 @@ import (
"fmt"
"testing"
"github.com/opentofu/opentofu/internal/lang/marks"
"github.com/zclconf/go-cty/cty"
"github.com/zclconf/go-cty/cty/function"
)
func TestReplace(t *testing.T) {
@ -255,3 +257,119 @@ func TestStartsWith(t *testing.T) {
})
}
}
func TestTemplateString(t *testing.T) {
tests := map[string]struct {
Content cty.Value
Vars cty.Value
Want cty.Value
Err string
}{
"Simple string template": {
cty.StringVal("Hello, Jodie!"),
cty.EmptyObjectVal,
cty.StringVal("Hello, Jodie!"),
``,
},
"String interpolation with variable": {
cty.StringVal("Hello, ${name}!"),
cty.MapVal(map[string]cty.Value{
"name": cty.StringVal("Jodie"),
}),
cty.StringVal("Hello, Jodie!"),
``,
},
"Looping through list": {
cty.StringVal("Items: %{ for x in list ~} ${x} %{ endfor ~}"),
cty.ObjectVal(map[string]cty.Value{
"list": cty.ListVal([]cty.Value{
cty.StringVal("a"),
cty.StringVal("b"),
cty.StringVal("c"),
}),
}),
cty.StringVal("Items: a b c "),
``,
},
"Looping through map": {
cty.StringVal("%{ for key, value in list ~} ${key}:${value} %{ endfor ~}"),
cty.ObjectVal(map[string]cty.Value{
"list": cty.ObjectVal(map[string]cty.Value{
"item1": cty.StringVal("a"),
"item2": cty.StringVal("b"),
"item3": cty.StringVal("c"),
}),
}),
cty.StringVal("item1:a item2:b item3:c "),
``,
},
"Invalid template variable name": {
cty.StringVal("Hello, ${1}!"),
cty.MapVal(map[string]cty.Value{
"1": cty.StringVal("Jodie"),
}),
cty.NilVal,
`invalid template variable name "1": must start with a letter, followed by zero or more letters, digits, and underscores`,
},
"Variable not present in vars map": {
cty.StringVal("Hello, ${name}!"),
cty.EmptyObjectVal,
cty.NilVal,
`vars map does not contain key "name"`,
},
"Interpolation of a boolean value": {
cty.StringVal("${val}"),
cty.ObjectVal(map[string]cty.Value{
"val": cty.True,
}),
cty.True,
``,
},
"Sensitive string template": {
cty.StringVal("My password is 1234").Mark(marks.Sensitive),
cty.EmptyObjectVal,
cty.StringVal("My password is 1234").Mark(marks.Sensitive),
``,
},
"Sensitive template variable": {
cty.StringVal("My password is ${pass}"),
cty.ObjectVal(map[string]cty.Value{
"pass": cty.StringVal("secret").Mark(marks.Sensitive),
}),
cty.StringVal("My password is secret").Mark(marks.Sensitive),
``,
},
}
templateStringFn := MakeTemplateStringFunc(".", func() map[string]function.Function {
return map[string]function.Function{}
})
for _, test := range tests {
t.Run(fmt.Sprintf("TemplateString(%#v, %#v)", test.Content, test.Vars), func(t *testing.T) {
got, err := templateStringFn.Call([]cty.Value{test.Content, test.Vars})
if argErr, ok := err.(function.ArgError); ok {
if argErr.Index < 0 || argErr.Index > 1 {
t.Errorf("ArgError index %d is out of range for templatestring (must be 0 or 1)", argErr.Index)
}
}
if err != nil {
if test.Err == "" {
t.Fatalf("unexpected error: %s", err)
} else {
if got, want := err.Error(), test.Err; got != want {
t.Errorf("wrong error\ngot: %s\nwant: %s", got, want)
}
}
} else if test.Err != "" {
t.Fatal("succeeded; want error")
} else {
if !got.RawEquals(test.Want) {
t.Errorf("wrong result\ngot: %#v\nwant: %#v", got, test.Want)
}
}
})
}
}

View File

@ -162,6 +162,12 @@ func (s *Scope) Functions() map[string]function.Function {
return s.funcs
})
// Registers "templatestring" function in function map.
s.funcs["templatestring"] = funcs.MakeTemplateStringFunc(s.BaseDir, func() map[string]function.Function {
// This anonymous function returns the existing map of functions for initialization.
return s.funcs
})
if s.ConsoleMode {
// The type function is only available in OpenTofu console.
s.funcs["type"] = funcs.TypeFunc

View File

@ -979,6 +979,13 @@ func TestFunctions(t *testing.T) {
},
},
"templatestring": {
{
`templatestring("Hello, $${name}!", {name = "Jodie"})`,
cty.StringVal("Hello, Jodie!"),
},
},
"timeadd": {
{
`timeadd("2017-11-22T00:00:00Z", "1s")`,

View File

@ -1099,6 +1099,11 @@
"path": "language/functions/templatefile",
"hidden": true
},
{
"title": "templatestring",
"path": "language/functions/templatestring",
"hidden": true
},
{
"title": "textdecodebase64",
"path": "language/functions/textdecodebase64",

View File

@ -0,0 +1,104 @@
---
sidebar_label: templatestring
description: |-
The `templatestring` function takes a string and renders it as a template using a supplied set of template variables.
---
# `templatestring` Function
`templatestring` function enables string rendering as template by substituting placeholders with values from a provided set of template variables.
```hcl
templatestring(str, vars)
```
The template syntax follows the rules for [string templates](/docs/language/expressions/strings#string-templates) in the main OpenTofu language, employing interpolation sequences delimited with `${ ... }`. This function offers the flexibility to factor out longer template sequences into a separate string for enhanced readability and manageability.
When using the `templatestring` function, it's important to keep in mind the usage of escape sequences to prevent premature interpolation. To ensure that placeholders are treated as literals until they are converted to template variables, use the escape sequences `$${...}` and `%%{...}` to represent `${...}` and `%{...}` literals, respectively. This approach prevents the template from interpreting the placeholders as interpolation or template directive sequences before they are processed. By following this convention, you can accurately represent template strings without encountering unexpected syntax errors. Remember to apply these escape sequences when defining your template strings to ensure proper handling by the `templatestring` function.
The "vars" argument must be an object. Within the template string, each key in the map serves as a variable for interpolation. Additionally, the template can utilize any other function available in the OpenTofu language. Variable names must adhere to OpenTofu naming conventions, starting with a letter followed by zero or more letters, digits, or underscores.
Since strings in OpenTofu represent sequences of Unicode characters, templatestring interprets the template string as UTF-8 encoded text, ensuring proper handling of Unicode characters. Any invalid UTF-8 sequences within the template string will result in an error.
## Examples
### Simple String Template
A basic example showcasing the use of templatestring for a static string.
```hcl
output "result" {
value = templatestring("Hello, Jodie!", {})
}
```
```
result = Hello, Jodie!
```
### String interpolation with variable
This example illustrates string interpolation by inserting a variable value into the template.
```hcl
output "result" {
value = templatestring("Hello, $${name}!", { name = "Alice" })
}
```
```
result = Hello, Alice!
```
### Lists
This example demonstrates the usage of the templatestring function with a list variable.
```hcl
output "result" {
value = templatestring("List Items: $${join(\", \", list)}", { list=["value1","value2","value3"] })
}
```
```
result = "List Items : value1, value2, value3"
```
### Maps
This example demonstrates the usage of the templatestring function with a map variable. It iterates over the key-value pairs in the map.
```hcl
output "result" {
value = templatestring("%%{ for key, value in list ~} $${key}:$${value} %%{ endfor ~}", { list={key1="value1", key2="value2", key3="value3"} })
}
```
```
result = "key1:value1 key2:value2 key3:value3 "
```
### Generating JSON or YAML
When generating JSON or YAML syntax strings, writing a template with numerous interpolation sequences and directives can be cumbersome. Instead, simplify the process by using a template consisting of a single interpolated call to either [`jsonencode`](/docs/language/functions/jsonencode) or
[`yamlencode`](/docs/language/functions/yamlencode), specifying the value to encode using [standard OpenTofu expression syntax](/docs/language/expressions) as in the following examples:
```hcl
locals {
list = ["Value1", "Value2", "Value3"]
formatted_list = "%{ for value in local.list ~}${value} %{ endfor ~}"
}
output "result" {
value = templatestring(yamlencode(local.formatted_list), {})
}
```
```
result = <<-EOT
"Value1 Value2 Value3 "
EOT
```
For more information, see the main documentation for
[`jsonencode`](/docs/language/functions/jsonencode) and [`yamlencode`](/docs/language/functions/yamlencode).