Functions: decode_tfvars, encode_tfvars, encode_expr on bult-in provider for compatibility (#2306)

Signed-off-by: Ilia Gogotchuri <ilia.gogotchuri0@gmail.com>
Co-authored-by: Oleksandr Levchenkov <ollevche@gmail.com>
This commit is contained in:
Ilia Gogotchuri 2024-12-25 13:21:59 +04:00 committed by GitHub
parent c5b43b9f1a
commit 2d9cef1f55
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 651 additions and 25 deletions

View File

@ -4,6 +4,11 @@ UPGRADE NOTES:
NEW FEATURES:
- New builtin provider functions added ([#2306](https://github.com/opentofu/opentofu/pull/2306)) :
- `provider::terraform::decode_tfvars` - Decode a TFVars file content into an object.
- `provider::terraform::encode_tfvars` - Encode an object into a string with the same format as a TFVars file.
- `provider::terraform::encode_expr` - Encode an arbitrary expression into a string with valid OpenTofu syntax.
ENHANCEMENTS:
BUG FIXES:

View File

@ -13,14 +13,27 @@ import (
"github.com/opentofu/opentofu/internal/addrs"
"github.com/opentofu/opentofu/internal/encryption"
"github.com/opentofu/opentofu/internal/providers"
"github.com/zclconf/go-cty/cty"
)
// Provider is an implementation of providers.Interface
type Provider struct{}
type Provider struct {
funcs map[string]providerFunc
}
// NewProvider returns a new tofu provider
func NewProvider() providers.Interface {
return &Provider{}
return &Provider{
funcs: getProviderFuncs(),
}
}
func (p *Provider) getFunctionSpecs() map[string]providers.FunctionSpec {
funcSpecs := make(map[string]providers.FunctionSpec)
for name, fn := range p.funcs {
funcSpecs[name] = fn.GetFunctionSpec()
}
return funcSpecs
}
// GetSchema returns the complete schema for the provider.
@ -32,6 +45,7 @@ func (p *Provider) GetProviderSchema() providers.GetProviderSchemaResponse {
ResourceTypes: map[string]providers.Schema{
"terraform_data": dataStoreResourceSchema(),
},
Functions: p.getFunctionSpecs(),
}
}
@ -161,14 +175,48 @@ func (p *Provider) ValidateResourceConfig(req providers.ValidateResourceConfigRe
}
func (p *Provider) GetFunctions() providers.GetFunctionsResponse {
panic("unimplemented - terraform provider has no functions")
return providers.GetFunctionsResponse{
Functions: p.getFunctionSpecs(),
}
}
func (p *Provider) CallFunction(r providers.CallFunctionRequest) providers.CallFunctionResponse {
panic("unimplemented - terraform provider has no functions")
fn, ok := p.funcs[r.Name]
if !ok {
return providers.CallFunctionResponse{
Error: fmt.Errorf("provider function %q not found", r.Name),
}
}
v, err := fn.Call(r.Arguments)
return providers.CallFunctionResponse{
Result: v,
Error: err,
}
}
// Close is a noop for this provider, since it's run in-process.
func (p *Provider) Close() error {
return nil
}
// providerFunc is an interface representing a built-in provider function
type providerFunc interface {
// Name returns the name of the function which is used to call it
Name() string
// GetFunctionSpec returns the provider function specification
GetFunctionSpec() providers.FunctionSpec
// Call is used to invoke the function
Call(args []cty.Value) (cty.Value, error)
}
// getProviderFuncs returns a map of functions that are registered in the provider
func getProviderFuncs() map[string]providerFunc {
decodeTFVars := &decodeTFVarsFunc{}
encodeTFVars := &encodeTFVarsFunc{}
encodeExpr := &encodeExprFunc{}
return map[string]providerFunc{
decodeTFVars.Name(): decodeTFVars,
encodeTFVars.Name(): encodeTFVars,
encodeExpr.Name(): encodeExpr,
}
}

View File

@ -0,0 +1,175 @@
// Copyright (c) The OpenTofu Authors
// SPDX-License-Identifier: MPL-2.0
// Copyright (c) 2023 HashiCorp, Inc.
// SPDX-License-Identifier: MPL-2.0
package tf
import (
"errors"
"fmt"
"github.com/hashicorp/hcl/v2/hclwrite"
"github.com/hashicorp/hcl/v2"
"github.com/hashicorp/hcl/v2/hclsyntax"
"github.com/opentofu/opentofu/internal/providers"
"github.com/zclconf/go-cty/cty"
)
// "decode_tfvars"
// "encode_tfvars"
// "encode_expr"
// decodeTFVarsFunc decodes a TFVars file content into a cty object
type decodeTFVarsFunc struct{}
func (f *decodeTFVarsFunc) Name() string {
return "decode_tfvars"
}
func (f *decodeTFVarsFunc) GetFunctionSpec() providers.FunctionSpec {
params := []providers.FunctionParameterSpec{
{
Name: "content",
Type: cty.String,
Description: "TFVars file content to decode",
DescriptionFormat: providers.TextFormattingPlain,
},
}
return providers.FunctionSpec{
Parameters: params,
Return: cty.DynamicPseudoType,
Summary: "Decode a TFVars file content into an object",
Description: "provider::terraform::decode_tfvars decodes a TFVars file content into an object",
DescriptionFormat: providers.TextFormattingPlain,
}
}
var FailedToDecodeError = errors.New("failed to decode tfvars content")
func wrapDiagErrors(m error, diag hcl.Diagnostics) error {
//Prepend the main error
errs := append([]error{m}, diag.Errs()...)
return errors.Join(errs...)
}
func (f *decodeTFVarsFunc) Call(args []cty.Value) (cty.Value, error) {
varsFileContent := args[0].AsString()
schema, diag := hclsyntax.ParseConfig([]byte(varsFileContent), "", hcl.Pos{Line: 0, Column: 0})
if schema == nil || diag.HasErrors() {
return cty.NullVal(cty.DynamicPseudoType), wrapDiagErrors(FailedToDecodeError, diag)
}
attrs, diag := schema.Body.JustAttributes()
// Check if there are any errors.
// attrs == nil does not mean that there are no attributes, attrs - is still initialized as an empty map
if attrs == nil || diag.HasErrors() {
return cty.NullVal(cty.DynamicPseudoType), wrapDiagErrors(FailedToDecodeError, diag)
}
vals := make(map[string]cty.Value)
for name, attr := range attrs {
val, diag := attr.Expr.Value(nil)
if diag.HasErrors() {
return cty.NullVal(cty.DynamicPseudoType), wrapDiagErrors(FailedToDecodeError, diag)
}
vals[name] = val
}
return cty.ObjectVal(vals), nil
}
// encodeTFVarsFunc encodes an object into a string with the same format as a TFVars file
type encodeTFVarsFunc struct{}
func (f *encodeTFVarsFunc) Name() string {
return "encode_tfvars"
}
func (f *encodeTFVarsFunc) GetFunctionSpec() providers.FunctionSpec {
params := []providers.FunctionParameterSpec{
{
Name: "input",
// The input type is determined at runtime
Type: cty.DynamicPseudoType,
Description: "Input to encode for TFVars file. Must be an object with key that are valid identifiers",
DescriptionFormat: providers.TextFormattingPlain,
},
}
return providers.FunctionSpec{
Parameters: params,
Return: cty.String,
Summary: "Encode an object into a string with the same format as a TFVars file",
Description: "provider::terraform::encode_tfvars encodes an object into a string with the same format as a TFVars file",
DescriptionFormat: providers.TextFormattingPlain,
}
}
var InvalidInputError = errors.New("invalid input")
func (f *encodeTFVarsFunc) Call(args []cty.Value) (cty.Value, error) {
toEncode := args[0]
// null is invalid input
if toEncode.IsNull() {
return cty.NullVal(cty.String), fmt.Errorf("%w: must not be null", InvalidInputError)
}
if !toEncode.Type().IsObjectType() {
return cty.NullVal(cty.String), fmt.Errorf("%w: must be an object", InvalidInputError)
}
ef := hclwrite.NewEmptyFile()
body := ef.Body()
// Iterate over the elements of the input value
it := toEncode.ElementIterator()
for it.Next() {
key, val := it.Element()
// Check if the key is a string, known and not null, otherwise AsString method panics
if !key.Type().Equals(cty.String) || !key.IsKnown() || key.IsNull() {
return cty.NullVal(cty.String), fmt.Errorf("%w: object key must be a string: %v", InvalidInputError, key)
}
name := key.AsString()
if valid := hclsyntax.ValidIdentifier(name); !valid {
return cty.NullVal(cty.String), fmt.Errorf("%w: object key: %s - must be a valid identifier", InvalidInputError, name)
}
body.SetAttributeValue(key.AsString(), val)
}
b := ef.Bytes()
return cty.StringVal(string(b)), nil
}
// encodeExprFunc encodes an expression into a string
type encodeExprFunc struct{}
func (f *encodeExprFunc) Name() string {
return "encode_expr"
}
func (f *encodeExprFunc) GetFunctionSpec() providers.FunctionSpec {
params := []providers.FunctionParameterSpec{
{
Name: "expr",
Type: cty.DynamicPseudoType,
Description: "expression to encode",
DescriptionFormat: providers.TextFormattingPlain,
},
}
return providers.FunctionSpec{
Parameters: params,
Return: cty.String,
Summary: "Takes an arbitrary expression and converts it into a string with valid OpenTofu syntax",
Description: "provider::terraform::encode_expr takes an arbitrary expression and converts it into a string with valid OpenTofu syntax",
DescriptionFormat: providers.TextFormattingPlain,
}
}
var UnknownInputError = errors.New("input is not wholly known")
func (f *encodeExprFunc) Call(args []cty.Value) (cty.Value, error) {
toEncode := args[0]
nf := hclwrite.NewEmptyFile()
if !toEncode.IsWhollyKnown() {
return cty.NullVal(cty.String), UnknownInputError
}
tokens := hclwrite.TokensForValue(toEncode)
body := nf.Body()
body.AppendUnstructuredTokens(tokens)
return cty.StringVal(string(nf.Bytes())), nil
}

View File

@ -0,0 +1,332 @@
// Copyright (c) The OpenTofu Authors
// SPDX-License-Identifier: MPL-2.0
// Copyright (c) 2023 HashiCorp, Inc.
// SPDX-License-Identifier: MPL-2.0
package tf
import (
"errors"
"strings"
"testing"
"github.com/hashicorp/hcl/v2/hclwrite"
"github.com/zclconf/go-cty/cty"
)
type test struct {
name string
arg cty.Value
want cty.Value
expectedError error
}
// formatHCLWithoutWhitespaces removes all whitespaces from the HCL string
// This will not result in a valid HCL string, but it will allow us to compare the result without worrying about whitespaces
func formatHCLWithoutWhitespaces(val cty.Value) string {
if val.IsNull() || !val.Type().Equals(cty.String) {
panic("formatHCLWithoutWhitespaces only works with string values")
}
f := string(hclwrite.Format([]byte(val.AsString())))
f = strings.ReplaceAll(f, " ", "")
f = strings.ReplaceAll(f, "\n", "")
f = strings.ReplaceAll(f, "\t", "")
return f
}
func TestDecodeTFVarsFunc(t *testing.T) {
tests := []test{
{
name: "basic test",
arg: cty.StringVal(`
test = 2
`),
want: cty.ObjectVal(map[string]cty.Value{
"test": cty.NumberIntVal(2),
}),
expectedError: nil,
},
{
name: "object basic test",
arg: cty.StringVal(`
test = {
k = "v"
}
`),
want: cty.ObjectVal(map[string]cty.Value{
"test": cty.ObjectVal(map[string]cty.Value{
"k": cty.StringVal("v"),
}),
}),
expectedError: nil,
},
{
name: "list basic test",
arg: cty.StringVal(`
test = [
"i1",
"i2",
3
]
`),
want: cty.ObjectVal(map[string]cty.Value{
"test": cty.TupleVal([]cty.Value{
cty.StringVal("i1"),
cty.StringVal("i2"),
cty.NumberIntVal(3),
}),
}),
},
{
name: "list of objects",
arg: cty.StringVal(`
test = [
{
o1k1 = "o1v1"
},
{
o2k1 = "o2v1"
o2k2 = {
o3k1 = "o3v1"
}
}
]
`),
want: cty.ObjectVal(map[string]cty.Value{
"test": cty.TupleVal([]cty.Value{
cty.ObjectVal(map[string]cty.Value{
"o1k1": cty.StringVal("o1v1"),
}),
cty.ObjectVal(map[string]cty.Value{
"o2k1": cty.StringVal("o2v1"),
"o2k2": cty.ObjectVal(map[string]cty.Value{
"o3k1": cty.StringVal("o3v1"),
}),
}),
}),
}),
},
{
name: "empty object",
arg: cty.StringVal(""),
want: cty.ObjectVal(map[string]cty.Value{}),
},
{
name: "invalid content",
arg: cty.StringVal("test"), // not a valid HCL
want: cty.NullVal(cty.DynamicPseudoType),
expectedError: FailedToDecodeError,
},
{
name: "invalid content 2",
arg: cty.StringVal("{}"), // not a valid HCL
want: cty.NullVal(cty.DynamicPseudoType),
expectedError: FailedToDecodeError,
},
{
name: "invalid content 3",
arg: cty.StringVal("\"5*5\": 3"), // not a valid HCL
want: cty.NullVal(cty.DynamicPseudoType),
expectedError: FailedToDecodeError,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
decodeTFVars := &decodeTFVarsFunc{}
got, err := decodeTFVars.Call([]cty.Value{tt.arg})
if !errors.Is(err, tt.expectedError) {
t.Errorf("Call() error = %v, expected %v", err, tt.expectedError)
}
if got.NotEqual(tt.want).True() {
t.Errorf("Call() got = %v, want %v", got, tt.want)
}
})
}
}
func TestEncodeTFVarsFunc(t *testing.T) {
tests := []test{
{
name: "empty object",
arg: cty.ObjectVal(map[string]cty.Value{}),
want: cty.StringVal(""),
},
{
name: "basic test",
arg: cty.ObjectVal(map[string]cty.Value{
"test": cty.NumberIntVal(2),
}),
want: cty.StringVal(`
test = 2
`),
expectedError: nil,
},
{
name: "object basic test",
arg: cty.ObjectVal(map[string]cty.Value{
"test": cty.ObjectVal(map[string]cty.Value{
"k": cty.StringVal("v"),
}),
}),
want: cty.StringVal(`
test = {
k = "v"
}
`),
expectedError: nil,
},
{
name: "list basic test",
arg: cty.ObjectVal(map[string]cty.Value{
"test": cty.TupleVal([]cty.Value{
cty.StringVal("i1"),
cty.StringVal("i2"),
cty.NumberIntVal(3),
}),
}),
want: cty.StringVal(`
test = ["i1", "i2", 3]
`),
},
{
name: "list of objects",
arg: cty.ObjectVal(map[string]cty.Value{
"test": cty.TupleVal([]cty.Value{
cty.ObjectVal(map[string]cty.Value{
"o1k1": cty.StringVal("o1v1"),
}),
cty.ObjectVal(map[string]cty.Value{
"o2k1": cty.StringVal("o2v1"),
"o2k2": cty.ObjectVal(map[string]cty.Value{
"o3k1": cty.StringVal("o3v1"),
}),
}),
}),
}),
want: cty.StringVal(`
test = [
{
o1k1 = "o1v1"
},
{
o2k1 = "o2v1"
o2k2 = {
o3k1 = "o3v1"
}
}
]
`),
},
{
name: "null input",
arg: cty.NullVal(cty.DynamicPseudoType),
want: cty.StringVal(""),
expectedError: InvalidInputError,
},
{
name: "invalid input: not an object",
arg: cty.StringVal("test"), // not an object
want: cty.StringVal(""),
expectedError: InvalidInputError,
},
{
name: "invalid input: Object with invalid key",
arg: cty.ObjectVal(map[string]cty.Value{"7*7": cty.StringVal("test")}), // invalid key
want: cty.StringVal(""),
expectedError: InvalidInputError,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
encodeTFVars := &encodeTFVarsFunc{}
got, err := encodeTFVars.Call([]cty.Value{tt.arg})
if err != nil {
if tt.expectedError == nil {
t.Fatalf("Call() unexpected error: %v", err)
}
if !errors.Is(err, tt.expectedError) {
t.Fatalf("Call() error = %v, expected %v", err, tt.expectedError)
}
return
}
formattedRequirement := formatHCLWithoutWhitespaces(tt.want)
formattedGot := formatHCLWithoutWhitespaces(got)
if formattedGot != formattedRequirement {
t.Errorf("Call() got: %v, want: %v", formattedGot, formattedRequirement)
}
})
}
}
func TestEncodeExprFunc(t *testing.T) {
tests := []test{
{
name: "string",
arg: cty.StringVal("test"),
want: cty.StringVal(`"test"`),
expectedError: nil,
},
{
name: "number",
arg: cty.NumberIntVal(2),
want: cty.StringVal("2"),
expectedError: nil,
},
{
name: "bool",
arg: cty.True,
want: cty.StringVal("true"),
expectedError: nil,
},
{
name: "null",
arg: cty.NullVal(cty.String),
want: cty.StringVal("null"),
},
{
name: "tuple",
arg: cty.TupleVal([]cty.Value{cty.StringVal("test"), cty.StringVal("test2")}),
want: cty.StringVal(`["test", "test2"]`),
expectedError: nil,
},
{
name: "tuple with objects",
arg: cty.TupleVal([]cty.Value{cty.ObjectVal(map[string]cty.Value{"test": cty.StringVal("test")}), cty.ObjectVal(map[string]cty.Value{"test2": cty.StringVal("test2")})}),
want: cty.StringVal(`[{test = "test"}, {test2 = "test2"}]`),
expectedError: nil,
},
{
name: "object",
arg: cty.ObjectVal(map[string]cty.Value{"test": cty.StringVal("test")}),
want: cty.StringVal(`{test = "test"}`),
expectedError: nil,
},
{
name: "nested object",
arg: cty.ObjectVal(map[string]cty.Value{"test": cty.ObjectVal(map[string]cty.Value{"test": cty.StringVal("test")})}),
want: cty.StringVal(`{test = {test = "test"}}`),
expectedError: nil,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
encodeExpr := &encodeExprFunc{}
got, err := encodeExpr.Call([]cty.Value{tt.arg})
if !errors.Is(err, tt.expectedError) {
t.Errorf("Call() error = %v, expected %v", err, tt.expectedError)
}
formattedRequirement := formatHCLWithoutWhitespaces(tt.want)
formattedGot := formatHCLWithoutWhitespaces(got)
if formattedGot != formattedRequirement {
t.Errorf("Call() got: %v, want: %v", formattedGot, formattedRequirement)
}
})
}
}

View File

@ -145,6 +145,10 @@
"title": "Provider Requirements",
"path": "language/providers/requirements"
},
{
"title": "Built-in Provider",
"path": "language/providers/builtin"
},
{
"title": "Dependency Lock File",
"path": "language/files/dependency-lock"
@ -717,7 +721,7 @@
},
{
"title": "<code>issensitive</code>",
"path": "language/functions/issensitive",
"path": "language/functions/issensitive"
},
{
"title": "<code>tobool</code>",
@ -1193,7 +1197,7 @@
"title": "urlencode",
"path": "language/functions/urlencode",
"hidden": true
},
},
{
"title": "urldecode",
"path": "language/functions/urldecode",

View File

@ -61,6 +61,9 @@ locals {
}
```
### Built-in Provider Functions:
OpenTofu has a built-in provider `terraform.io/builtin/terraform` which provides [additional functions](../providers/builtin.mdx#functions) that can be used in OpenTofu configurations.
### Notes for Provider Authors:
* Support for functions was added in protocol version 5.5 and 6.5.
* OpenTofu's provider protocol is compatible with Terraform's provider protocol.

View File

@ -0,0 +1,74 @@
---
sidebar_position: 3
sidebar_label: Built-in Provider
---
# Built-in Provider
Most providers are distributed separately as plugins, but there
is one provider that is built into OpenTofu itself. This provider enables the
[the `terraform_remote_state` data source](../state/remote-state-data.mdx).
Because this provider is built in to OpenTofu, you don't need to declare it
in the `required_providers` block in order to use its features (except provider functions).
It has a special provider source address, which is
`terraform.io/builtin/terraform`. This address may sometimes appear in
OpenTofu's error messages and other output in order to unambiguously refer
to the built-in provider, as opposed to a hypothetical third-party provider
with the type name "terraform".
There is also an existing provider with the source address
`hashicorp/terraform`, which is an older version of the now-built-in provider.
`hashicorp/terraform` is not compatible with OpenTofu and should never be declared in a
`required_providers` block.
## Functions
The built-in provider has additional functions, which can be called after declaring the provider in the `required_providers` block.
```hcl
terraform {
required_providers {
terraform = {
source = "terraform.io/builtin/terraform"
}
}
}
```
### decode_tfvars
`decode_tfvars` takes the content of the .tfvars file as a string input and returns a decoded object.
```hcl
locals {
content = file("./example_file.tfvars")
decoded = provider::terraform::decode_tfvars(local.content) # Returns object
}
```
### encode_tfvars
`encode_tfvars` takes an object and returns the string representation of the object that can be used as the content of the .tfvars file.
```hcl
locals {
object = {
key1 = "value1"
key2 = "value2"
}
encoded = provider::terraform::encode_tfvars(local.object) # Returns string
}
```
The keys in the object need to be [valid identifiers](../syntax/configuration.mdx#identifiers).
### encode_expr
`encode_expr` takes an arbitrary [expression](../expressions/index.mdx) and converts it into a string with valid OpenTofu syntax.
```hcl
locals {
expression = {
key1 = "value1"
key2 = "value2"
}
encoded = provider::terraform::encode_expr(local.expression) # Returns string
}
```

View File

@ -1,4 +1,6 @@
---
sidebar_position: 1
sidebar_label: Provider Configuration
description: >-
Learn how to set up providers, including how to use the alias meta-argument to
specify multiple configurations for a single provider.

View File

@ -1,4 +1,6 @@
---
sidebar_position: 2
sidebar_label: Provider Requirements
description: >-
Providers are plugins that allow OpenTofu to interact with services, cloud
providers, and other APIs. Learn how to declare providers in a configuration.
@ -248,25 +250,6 @@ often it forces users of the module to update many modules simultaneously when
performing routine upgrades. Specify a minimum version, document any known
incompatibilities, and let the root module manage the maximum version.
## Built-in Providers
Most providers are distributed separately as plugins, but there
is one provider that is built into OpenTofu itself. This provider enables the
[the `terraform_remote_state` data source](../../language/state/remote-state-data.mdx).
Because this provider is built in to OpenTofu, you don't need to declare it
in the `required_providers` block in order to use its features. However, for
consistency it _does_ have a special provider source address, which is
`terraform.io/builtin/terraform`. This address may sometimes appear in
OpenTofu's error messages and other output in order to unambiguously refer
to the built-in provider, as opposed to a hypothetical third-party provider
with the type name "tofu".
There is also an existing provider with the source address
`hashicorp/terraform`, which is an older version of the now-built-in provider.
`hashicorp/terraform` is not compatible with OpenTofu and should never be declared in a
`required_providers` block.
## In-house Providers
Anyone can develop and distribute their own providers.