Add provider functions to provider.Interface with GRPC implementation (#1437)

Signed-off-by: Christian Mesh <christianmesh1@gmail.com>

internal/builtin/providers/tf/provider.go

@@ -160,6 +160,10 @@ func (p *Provider) ValidateResourceConfig(req providers.ValidateResourceConfigRe
 	return validateDataStoreResourceConfig(req)
 }
 
+func (p *Provider) CallFunction(r providers.CallFunctionRequest) providers.CallFunctionResponse {
+	panic("unimplemented - terraform provider has no functions")
+}
+
 // Close is a noop for this provider, since it's run in-process.
 func (p *Provider) Close() error {
 	return nil

internal/legacy/tofu/provider_mock.go

@@ -362,6 +362,10 @@ func (p *MockProvider) ReadDataSource(r providers.ReadDataSourceRequest) provide
 	return p.ReadDataSourceResponse
 }
 
+func (p *MockProvider) CallFunction(r providers.CallFunctionRequest) providers.CallFunctionResponse {
+	panic("Not Implemented")
+}
+
 func (p *MockProvider) Close() error {
 	p.CloseCalled = true
 	return p.CloseError

internal/plugin/convert/function.go

@@ -0,0 +1,63 @@
+package convert
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/opentofu/opentofu/internal/providers"
+	"github.com/opentofu/opentofu/internal/tfplugin5"
+	"github.com/zclconf/go-cty/cty"
+)
+
+func ProtoToCtyType(in []byte) cty.Type {
+	var out cty.Type
+	if err := json.Unmarshal(in, &out); err != nil {
+		panic(err)
+	}
+	return out
+}
+
+func ProtoToTextFormatting(proto tfplugin5.StringKind) providers.TextFormatting {
+	switch proto {
+	case tfplugin5.StringKind_PLAIN:
+		return providers.TextFormattingPlain
+	case tfplugin5.StringKind_MARKDOWN:
+		return providers.TextFormattingMarkdown
+	default:
+		panic(fmt.Sprintf("Invalid text tfplugin5.StringKind %v", proto))
+	}
+}
+
+func ProtoToFunctionParameterSpec(proto *tfplugin5.Function_Parameter) providers.FunctionParameterSpec {
+	return providers.FunctionParameterSpec{
+		Name:               proto.Name,
+		Type:               ProtoToCtyType(proto.Type),
+		AllowNullValue:     proto.AllowNullValue,
+		AllowUnknownValues: proto.AllowUnknownValues,
+		Description:        proto.Description,
+		DescriptionFormat:  ProtoToTextFormatting(proto.DescriptionKind),
+	}
+}
+
+func ProtoToFunctionSpec(proto *tfplugin5.Function) providers.FunctionSpec {
+	params := make([]providers.FunctionParameterSpec, len(proto.Parameters))
+	for i, param := range proto.Parameters {
+		params[i] = ProtoToFunctionParameterSpec(param)
+	}
+
+	var varParam *providers.FunctionParameterSpec
+	if proto.VariadicParameter != nil {
+		param := ProtoToFunctionParameterSpec(proto.VariadicParameter)
+		varParam = &param
+	}
+
+	return providers.FunctionSpec{
+		Parameters:         params,
+		VariadicParameter:  varParam,
+		Return:             ProtoToCtyType(proto.Return.Type),
+		Summary:            proto.Summary,
+		Description:        proto.Description,
+		DescriptionFormat:  ProtoToTextFormatting(proto.DescriptionKind),
+		DeprecationMessage: proto.DeprecationMessage,
+	}
+}

internal/plugin/grpc_provider.go

@@ -76,6 +76,8 @@ type GRPCProvider struct {
 	schema providers.GetProviderSchemaResponse
 }
 
+var _ providers.Interface = new(GRPCProvider)
+
 func (p *GRPCProvider) GetProviderSchema() (resp providers.GetProviderSchemaResponse) {
 	logger.Trace("GRPCProvider: GetProviderSchema")
 	p.mu.Lock()
@@ -102,6 +104,7 @@ func (p *GRPCProvider) GetProviderSchema() (resp providers.GetProviderSchemaResp
 
 	resp.ResourceTypes = make(map[string]providers.Schema)
 	resp.DataSources = make(map[string]providers.Schema)
+	resp.Functions = make(map[string]providers.FunctionSpec)
 
 	// Some providers may generate quite large schemas, and the internal default
 	// grpc response size limit is 4MB. 64MB should cover most any use case, and
@@ -144,6 +147,10 @@ func (p *GRPCProvider) GetProviderSchema() (resp providers.GetProviderSchemaResp
 		resp.DataSources[name] = convert.ProtoToProviderSchema(data)
 	}
 
+	for name, fn := range protoResp.Functions {
+		resp.Functions[name] = convert.ProtoToFunctionSpec(fn)
+	}
+
 	if protoResp.ServerCapabilities != nil {
 		resp.ServerCapabilities.PlanDestroy = protoResp.ServerCapabilities.PlanDestroy
 		resp.ServerCapabilities.GetProviderSchemaOptional = protoResp.ServerCapabilities.GetProviderSchemaOptional
@@ -686,6 +693,96 @@ func (p *GRPCProvider) ReadDataSource(r providers.ReadDataSourceRequest) (resp p
 	return resp
 }
 
+func (p *GRPCProvider) CallFunction(r providers.CallFunctionRequest) (resp providers.CallFunctionResponse) {
+	logger.Trace("GRPCProvider: CallFunction")
+
+	schema := p.GetProviderSchema()
+	if schema.Diagnostics.HasErrors() {
+		// This should be unreachable
+		resp.Error = schema.Diagnostics.Err()
+		return resp
+	}
+
+	spec, ok := schema.Functions[r.Name]
+	if !ok {
+		// This should be unreachable
+		resp.Error = fmt.Errorf("invalid CallFunctionRequest: function %s not defined in provider schema", r.Name)
+		return resp
+	}
+
+	protoReq := &proto.CallFunction_Request{
+		Name:      r.Name,
+		Arguments: make([]*proto.DynamicValue, len(r.Arguments)),
+	}
+
+	// Translate the arguments
+	// As this is functionality is always sitting behind cty/function.Function, we skip some validation
+	// checks of from the function and param spec.  We still include basic validation to prevent panics,
+	// just in case there are bugs in cty
+	if len(r.Arguments) < len(spec.Parameters) {
+		// This should be unreachable
+		resp.Error = fmt.Errorf("invalid CallFunctionRequest: function %s expected %d parameters and got %d instead", r.Name, len(spec.Parameters), len(r.Arguments))
+		return resp
+	}
+
+	for i, arg := range r.Arguments {
+		var paramSpec providers.FunctionParameterSpec
+		if i < len(spec.Parameters) {
+			paramSpec = spec.Parameters[i]
+		} else {
+			// We are past the end of spec.Parameters, this is either variadic or an error
+			if spec.VariadicParameter != nil {
+				paramSpec = *spec.VariadicParameter
+			} else {
+				// This should be unreachable
+				resp.Error = fmt.Errorf("invalid CallFunctionRequest: too many arguments passed to non-variadic function %s", r.Name)
+			}
+		}
+
+		if arg.IsNull() {
+			if paramSpec.AllowNullValue {
+				continue
+			} else {
+				resp.Error = &providers.CallFunctionArgumentError{
+					Text:             fmt.Sprintf("parameter %s is null, which is not allowed for function %s", paramSpec.Name, r.Name),
+					FunctionArgument: i,
+				}
+			}
+
+		}
+
+		encodedArg, err := msgpack.Marshal(arg, paramSpec.Type)
+		if err != nil {
+			resp.Error = err
+			return
+		}
+
+		protoReq.Arguments[i] = &proto.DynamicValue{
+			Msgpack: encodedArg,
+		}
+	}
+
+	protoResp, err := p.client.CallFunction(p.ctx, protoReq)
+	if err != nil {
+		resp.Error = err
+		return
+	}
+
+	if protoResp.Error != nil {
+		err := &providers.CallFunctionArgumentError{
+			Text: protoResp.Error.Text,
+		}
+		if protoResp.Error.FunctionArgument != nil {
+			err.FunctionArgument = int(*protoResp.Error.FunctionArgument)
+		}
+		resp.Error = err
+		return
+	}
+
+	resp.Result, resp.Error = decodeDynamicValue(protoResp.Result, spec.Return)
+	return
+}
+
 // closing the grpc connection is final, and tofu will call it at the end of every phase.
 func (p *GRPCProvider) Close() error {
 	logger.Trace("GRPCProvider: Close")

internal/plugin/grpc_provider_test.go

@@ -96,6 +96,25 @@ func providerProtoSchema() *proto.GetProviderSchema_Response {
 				},
 			},
 		},
+		Functions: map[string]*proto.Function{
+			"fn": &proto.Function{
+				Parameters: []*proto.Function_Parameter{{
+					Name:               "par_a",
+					Type:               []byte(`"string"`),
+					AllowNullValue:     false,
+					AllowUnknownValues: false,
+				}},
+				VariadicParameter: &proto.Function_Parameter{
+					Name:               "par_var",
+					Type:               []byte(`"string"`),
+					AllowNullValue:     true,
+					AllowUnknownValues: false,
+				},
+				Return: &proto.Function_Return{
+					Type: []byte(`"string"`),
+				},
+			},
+		},
 	}
 }
 
@@ -876,3 +895,29 @@ func TestGRPCProvider_ReadDataSourceJSON(t *testing.T) {
 		t.Fatal(cmp.Diff(expected, resp.State, typeComparer, valueComparer, equateEmpty))
 	}
 }
+
+func TestGRPCProvider_CallFunction(t *testing.T) {
+	client := mockProviderClient(t)
+	p := &GRPCProvider{
+		client: client,
+	}
+
+	client.EXPECT().CallFunction(
+		gomock.Any(),
+		gomock.Any(),
+	).Return(&proto.CallFunction_Response{
+		Result: &proto.DynamicValue{Json: []byte(`"foo"`)},
+	}, nil)
+
+	resp := p.CallFunction(providers.CallFunctionRequest{
+		Name:      "fn",
+		Arguments: []cty.Value{cty.StringVal("bar"), cty.NilVal},
+	})
+
+	if resp.Error != nil {
+		t.Fatal(resp.Error)
+	}
+	if resp.Result != cty.StringVal("foo") {
+		t.Fatalf("%v", resp.Result)
+	}
+}

internal/plugin6/convert/function.go

@@ -0,0 +1,63 @@
+package convert
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/opentofu/opentofu/internal/providers"
+	"github.com/opentofu/opentofu/internal/tfplugin6"
+	"github.com/zclconf/go-cty/cty"
+)
+
+func ProtoToCtyType(in []byte) cty.Type {
+	var out cty.Type
+	if err := json.Unmarshal(in, &out); err != nil {
+		panic(err)
+	}
+	return out
+}
+
+func ProtoToTextFormatting(proto tfplugin6.StringKind) providers.TextFormatting {
+	switch proto {
+	case tfplugin6.StringKind_PLAIN:
+		return providers.TextFormattingPlain
+	case tfplugin6.StringKind_MARKDOWN:
+		return providers.TextFormattingMarkdown
+	default:
+		panic(fmt.Sprintf("Invalid text tfplugin6.StringKind %v", proto))
+	}
+}
+
+func ProtoToFunctionParameterSpec(proto *tfplugin6.Function_Parameter) providers.FunctionParameterSpec {
+	return providers.FunctionParameterSpec{
+		Name:               proto.Name,
+		Type:               ProtoToCtyType(proto.Type),
+		AllowNullValue:     proto.AllowNullValue,
+		AllowUnknownValues: proto.AllowUnknownValues,
+		Description:        proto.Description,
+		DescriptionFormat:  ProtoToTextFormatting(proto.DescriptionKind),
+	}
+}
+
+func ProtoToFunctionSpec(proto *tfplugin6.Function) providers.FunctionSpec {
+	params := make([]providers.FunctionParameterSpec, len(proto.Parameters))
+	for i, param := range proto.Parameters {
+		params[i] = ProtoToFunctionParameterSpec(param)
+	}
+
+	var varParam *providers.FunctionParameterSpec
+	if proto.VariadicParameter != nil {
+		param := ProtoToFunctionParameterSpec(proto.VariadicParameter)
+		varParam = &param
+	}
+
+	return providers.FunctionSpec{
+		Parameters:         params,
+		VariadicParameter:  varParam,
+		Return:             ProtoToCtyType(proto.Return.Type),
+		Summary:            proto.Summary,
+		Description:        proto.Description,
+		DescriptionFormat:  ProtoToTextFormatting(proto.DescriptionKind),
+		DeprecationMessage: proto.DeprecationMessage,
+	}
+}

internal/plugin6/grpc_provider.go

@@ -76,6 +76,8 @@ type GRPCProvider struct {
 	schema providers.GetProviderSchemaResponse
 }
 
+var _ providers.Interface = new(GRPCProvider)
+
 func (p *GRPCProvider) GetProviderSchema() (resp providers.GetProviderSchemaResponse) {
 	logger.Trace("GRPCProvider.v6: GetProviderSchema")
 	p.mu.Lock()
@@ -102,6 +104,7 @@ func (p *GRPCProvider) GetProviderSchema() (resp providers.GetProviderSchemaResp
 
 	resp.ResourceTypes = make(map[string]providers.Schema)
 	resp.DataSources = make(map[string]providers.Schema)
+	resp.Functions = make(map[string]providers.FunctionSpec)
 
 	// Some providers may generate quite large schemas, and the internal default
 	// grpc response size limit is 4MB. 64MB should cover most any use case, and
@@ -144,6 +147,10 @@ func (p *GRPCProvider) GetProviderSchema() (resp providers.GetProviderSchemaResp
 		resp.DataSources[name] = convert.ProtoToProviderSchema(data)
 	}
 
+	for name, fn := range protoResp.Functions {
+		resp.Functions[name] = convert.ProtoToFunctionSpec(fn)
+	}
+
 	if protoResp.ServerCapabilities != nil {
 		resp.ServerCapabilities.PlanDestroy = protoResp.ServerCapabilities.PlanDestroy
 		resp.ServerCapabilities.GetProviderSchemaOptional = protoResp.ServerCapabilities.GetProviderSchemaOptional
@@ -675,6 +682,96 @@ func (p *GRPCProvider) ReadDataSource(r providers.ReadDataSourceRequest) (resp p
 	return resp
 }
 
+func (p *GRPCProvider) CallFunction(r providers.CallFunctionRequest) (resp providers.CallFunctionResponse) {
+	logger.Trace("GRPCProvider6: CallFunction")
+
+	schema := p.GetProviderSchema()
+	if schema.Diagnostics.HasErrors() {
+		// This should be unreachable
+		resp.Error = schema.Diagnostics.Err()
+		return resp
+	}
+
+	spec, ok := schema.Functions[r.Name]
+	if !ok {
+		// This should be unreachable
+		resp.Error = fmt.Errorf("invalid CallFunctionRequest: function %s not defined in provider schema", r.Name)
+		return resp
+	}
+
+	protoReq := &proto6.CallFunction_Request{
+		Name:      r.Name,
+		Arguments: make([]*proto6.DynamicValue, len(r.Arguments)),
+	}
+
+	// Translate the arguments
+	// As this is functionality is always sitting behind cty/function.Function, we skip some validation
+	// checks of from the function and param spec.  We still include basic validation to prevent panics,
+	// just in case there are bugs in cty
+	if len(r.Arguments) < len(spec.Parameters) {
+		// This should be unreachable
+		resp.Error = fmt.Errorf("invalid CallFunctionRequest: function %s expected %d parameters and got %d instead", r.Name, len(spec.Parameters), len(r.Arguments))
+		return resp
+	}
+
+	for i, arg := range r.Arguments {
+		var paramSpec providers.FunctionParameterSpec
+		if i < len(spec.Parameters) {
+			paramSpec = spec.Parameters[i]
+		} else {
+			// We are past the end of spec.Parameters, this is either variadic or an error
+			if spec.VariadicParameter != nil {
+				paramSpec = *spec.VariadicParameter
+			} else {
+				// This should be unreachable
+				resp.Error = fmt.Errorf("invalid CallFunctionRequest: too many arguments passed to non-variadic function %s", r.Name)
+			}
+		}
+
+		if arg.IsNull() {
+			if paramSpec.AllowNullValue {
+				continue
+			} else {
+				resp.Error = &providers.CallFunctionArgumentError{
+					Text:             fmt.Sprintf("parameter %s is null, which is not allowed for function %s", paramSpec.Name, r.Name),
+					FunctionArgument: i,
+				}
+			}
+
+		}
+
+		encodedArg, err := msgpack.Marshal(arg, paramSpec.Type)
+		if err != nil {
+			resp.Error = err
+			return
+		}
+
+		protoReq.Arguments[i] = &proto6.DynamicValue{
+			Msgpack: encodedArg,
+		}
+	}
+
+	protoResp, err := p.client.CallFunction(p.ctx, protoReq)
+	if err != nil {
+		resp.Error = err
+		return
+	}
+
+	if protoResp.Error != nil {
+		err := &providers.CallFunctionArgumentError{
+			Text: protoResp.Error.Text,
+		}
+		if protoResp.Error.FunctionArgument != nil {
+			err.FunctionArgument = int(*protoResp.Error.FunctionArgument)
+		}
+		resp.Error = err
+		return
+	}
+
+	resp.Result, resp.Error = decodeDynamicValue(protoResp.Result, spec.Return)
+	return
+}
+
 // closing the grpc connection is final, and tofu will call it at the end of every phase.
 func (p *GRPCProvider) Close() error {
 	logger.Trace("GRPCProvider.v6: Close")

internal/plugin6/grpc_provider_test.go

@@ -103,6 +103,25 @@ func providerProtoSchema() *proto.GetProviderSchema_Response {
 				},
 			},
 		},
+		Functions: map[string]*proto.Function{
+			"fn": &proto.Function{
+				Parameters: []*proto.Function_Parameter{{
+					Name:               "par_a",
+					Type:               []byte(`"string"`),
+					AllowNullValue:     false,
+					AllowUnknownValues: false,
+				}},
+				VariadicParameter: &proto.Function_Parameter{
+					Name:               "par_var",
+					Type:               []byte(`"string"`),
+					AllowNullValue:     true,
+					AllowUnknownValues: false,
+				},
+				Return: &proto.Function_Return{
+					Type: []byte(`"string"`),
+				},
+			},
+		},
 	}
 }
 
@@ -883,3 +902,29 @@ func TestGRPCProvider_ReadDataSourceJSON(t *testing.T) {
 		t.Fatal(cmp.Diff(expected, resp.State, typeComparer, valueComparer, equateEmpty))
 	}
 }
+
+func TestGRPCProvider_CallFunction(t *testing.T) {
+	client := mockProviderClient(t)
+	p := &GRPCProvider{
+		client: client,
+	}
+
+	client.EXPECT().CallFunction(
+		gomock.Any(),
+		gomock.Any(),
+	).Return(&proto.CallFunction_Response{
+		Result: &proto.DynamicValue{Json: []byte(`"foo"`)},
+	}, nil)
+
+	resp := p.CallFunction(providers.CallFunctionRequest{
+		Name:      "fn",
+		Arguments: []cty.Value{cty.StringVal("bar"), cty.NilVal},
+	})
+
+	if resp.Error != nil {
+		t.Fatal(resp.Error)
+	}
+	if resp.Result != cty.StringVal("foo") {
+		t.Fatalf("%v", resp.Result)
+	}
+}

internal/provider-simple-v6/provider.go

@@ -147,6 +147,10 @@ func (s simple) ReadDataSource(req providers.ReadDataSourceRequest) (resp provid
 	return resp
 }
 
+func (s simple) CallFunction(r providers.CallFunctionRequest) providers.CallFunctionResponse {
+	panic("Not Implemented")
+}
+
 func (s simple) Close() error {
 	return nil
 }

internal/provider-simple/provider.go

@@ -138,6 +138,10 @@ func (s simple) ReadDataSource(req providers.ReadDataSourceRequest) (resp provid
 	return resp
 }
 
+func (s simple) CallFunction(r providers.CallFunctionRequest) providers.CallFunctionResponse {
+	panic("Not Implemented")
+}
+
 func (s simple) Close() error {
 	return nil
 }

internal/providers/provider.go

@@ -16,6 +16,11 @@ import (
 // Interface represents the set of methods required for a complete resource
 // provider plugin.
 type Interface interface {
+	// GetMetadata is not yet implemented or used at this time.  It may
+	// be used in the future to avoid loading a provider's full schema
+	// for initial validation.  This could result in some potential
+	// memory savings.
+
 	// GetSchema returns the complete schema for the provider.
 	GetProviderSchema() GetProviderSchemaResponse
 
@@ -72,6 +77,13 @@ type Interface interface {
 	// ReadDataSource returns the data source's current state.
 	ReadDataSource(ReadDataSourceRequest) ReadDataSourceResponse
 
+	// GetFunctions not yet implemented or used at this stage as it is not required.
+	// tofu queries a full set of provider schemas early on in the process which contain
+	// the required information.
+
+	// CallFunction requests that the given function is called and response returned.
+	CallFunction(CallFunctionRequest) CallFunctionResponse
+
 	// Close shuts down the plugin process if applicable.
 	Close() error
 }
@@ -99,6 +111,9 @@ type GetProviderSchemaResponse struct {
 
 	// ServerCapabilities lists optional features supported by the provider.
 	ServerCapabilities ServerCapabilities
+
+	// Functions lists all functions supported by this provider.
+	Functions map[string]FunctionSpec
 }
 
 // Schema pairs a provider or resource schema with that schema's version.
@@ -130,6 +145,44 @@ type ServerCapabilities struct {
 	GetProviderSchemaOptional bool
 }
 
+type FunctionSpec struct {
+	// List of parameters required to call the function
+	Parameters []FunctionParameterSpec
+	// Optional Spec for variadic parameters
+	VariadicParameter *FunctionParameterSpec
+	// Type which the function will return
+	Return cty.Type
+	// Human-readable shortened documentation for the function
+	Summary string
+	// Human-readable documentation for the function
+	Description string
+	// Formatting type of the Description field
+	DescriptionFormat TextFormatting
+	// Human-readable message present if the function is deprecated
+	DeprecationMessage string
+}
+
+type FunctionParameterSpec struct {
+	// Human-readable display name for the parameter
+	Name string
+	// Type constraint for the parameter
+	Type cty.Type
+	// Null values alowed for the parameter
+	AllowNullValue bool
+	// Unknown Values allowd for the parameter
+	// Implies the Return type of the function is also Unknown
+	AllowUnknownValues bool
+	// Human-readable documentation for the parameter
+	Description string
+	// Formatting type of the Description field
+	DescriptionFormat TextFormatting
+}
+
+type TextFormatting string
+
+const TextFormattingPlain = TextFormatting("Plain")
+const TextFormattingMarkdown = TextFormatting("Markdown")
+
 type ValidateProviderConfigRequest struct {
 	// Config is the raw configuration value for the provider.
 	Config cty.Value
@@ -421,3 +474,22 @@ type ReadDataSourceResponse struct {
 	// Diagnostics contains any warnings or errors from the method call.
 	Diagnostics tfdiags.Diagnostics
 }
+
+type CallFunctionRequest struct {
+	Name      string
+	Arguments []cty.Value
+}
+
+type CallFunctionResponse struct {
+	Result cty.Value
+	Error  error
+}
+
+type CallFunctionArgumentError struct {
+	Text             string
+	FunctionArgument int
+}
+
+func (err *CallFunctionArgumentError) Error() string {
+	return err.Text
+}

internal/tofu/provider_mock.go

@@ -516,6 +516,10 @@ func (p *MockProvider) ReadDataSource(r providers.ReadDataSourceRequest) (resp p
 	return resp
 }
 
+func (p *MockProvider) CallFunction(r providers.CallFunctionRequest) providers.CallFunctionResponse {
+	panic("Not Implemented")
+}
+
 func (p *MockProvider) Close() error {
 	p.Lock()
 	defer p.Unlock()