IntenseWebs/opentofu
3
0
Fork 0
mirror of https://github.com/opentofu/opentofu.git synced 2025-02-25 18:45:20 -06:00
Code Issues Packages Projects Releases Wiki Activity

Add basic global schema cache

Browse Source 
Add a single global schema cache for providers. This allows multiple
provider instances to share a single copy of the schema, and prevents
loading the schema multiple times for a given provider type during a
single command.

This does not currently work with some provider releases, which are
using GetProviderSchema to trigger certain initializations. A new server
capability will be introduced to trigger reloading their schemas, but
not store duplicate results.
This commit is contained in:
James Bardin 2023-07-03 12:37:50 -04:00
parent d199d427a1
commit 53901a7e62
7 changed files with 157 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
internal/backend/local/backend_apply.go
View File

@ -99,13 +99,7 @@ func (b *Local) opApply(
// plan.Errored will be true in this case, which our plan
// renderer can rely on to tailor its messaging.
if plan != nil && (len(plan.Changes.Resources) != 0 || len(plan.Changes.Outputs) != 0) {
schemas, moreDiags := lr.Core.Schemas(lr.Config, lr.InputState)
// If schema loading returns errors then we'll just give up and
// ignore them to avoid distracting from the plan-time errors we're
// mainly trying to report here.
if !moreDiags.HasErrors() {
op.View.Plan(plan, schemas)
}
op.View.Plan(plan, schemas)
}
op.ReportResult(runningOp, diags)
return

2
internal/command/meta_providers.go
View File

@ -382,10 +382,12 @@ func providerFactory(meta *providercache.CachedProvider) providers.Factory {
case 5:
p := raw.(*tfplugin.GRPCProvider)
p.PluginClient = client
p.Addr = meta.Provider
return p, nil
case 6:
p := raw.(*tfplugin6.GRPCProvider)
p.PluginClient = client
p.Addr = meta.Provider
return p, nil
default:
panic("unsupported protocol version")

54
internal/plugin/grpc_provider.go
View File

@ -12,6 +12,7 @@ import (
"github.com/zclconf/go-cty/cty"
plugin "github.com/hashicorp/go-plugin"
"github.com/hashicorp/terraform/internal/addrs"
"github.com/hashicorp/terraform/internal/logging"
"github.com/hashicorp/terraform/internal/plugin/convert"
"github.com/hashicorp/terraform/internal/providers"
@ -54,6 +55,11 @@ type GRPCProvider struct {
// used in an end to end test of a provider.
TestServer *grpc.Server
// Addr uniquely identifies the type of provider.
// Normally executed providers will have this set during initialization,
// but it may not always be available for alternative execute modes.
Addr addrs.Provider
// Proto client use to make the grpc service calls.
client proto.ProviderClient
@ -67,24 +73,18 @@ type GRPCProvider struct {
schemas providers.GetProviderSchemaResponse
}
// getSchema is used internally to get the cached provider schema
func (p *GRPCProvider) getSchema() providers.GetProviderSchemaResponse {
p.mu.Lock()
// unlock inline in case GetSchema needs to be called
if p.schemas.Provider.Block != nil {
p.mu.Unlock()
return p.schemas
}
p.mu.Unlock()
return p.GetProviderSchema()
}
func (p *GRPCProvider) GetProviderSchema() (resp providers.GetProviderSchemaResponse) {
logger.Trace("GRPCProvider: GetProviderSchema")
p.mu.Lock()
defer p.mu.Unlock()
// check the global cache if we can
if !p.Addr.IsZero() {
if resp, ok := providers.SchemaCache.Get(p.Addr); ok {
return resp
}
}
if p.schemas.Provider.Block != nil {
return p.schemas
}
@ -137,7 +137,13 @@ func (p *GRPCProvider) GetProviderSchema() (resp providers.GetProviderSchemaResp
resp.ServerCapabilities.PlanDestroy = protoResp.ServerCapabilities.PlanDestroy
}
p.schemas = resp
// set the global cache if we can
if !p.Addr.IsZero() {
providers.SchemaCache.Set(p.Addr, resp)
} else {
// otherwise store it in the local cache
p.schemas = resp
}
return resp
}
@ -145,7 +151,7 @@ func (p *GRPCProvider) GetProviderSchema() (resp providers.GetProviderSchemaResp
func (p *GRPCProvider) ValidateProviderConfig(r providers.ValidateProviderConfigRequest) (resp providers.ValidateProviderConfigResponse) {
logger.Trace("GRPCProvider: ValidateProviderConfig")
schema := p.getSchema()
schema := p.GetProviderSchema()
if schema.Diagnostics.HasErrors() {
resp.Diagnostics = schema.Diagnostics
return resp
@ -183,7 +189,7 @@ func (p *GRPCProvider) ValidateProviderConfig(r providers.ValidateProviderConfig
func (p *GRPCProvider) ValidateResourceConfig(r providers.ValidateResourceConfigRequest) (resp providers.ValidateResourceConfigResponse) {
logger.Trace("GRPCProvider: ValidateResourceConfig")
schema := p.getSchema()
schema := p.GetProviderSchema()
if schema.Diagnostics.HasErrors() {
resp.Diagnostics = schema.Diagnostics
return resp
@ -219,7 +225,7 @@ func (p *GRPCProvider) ValidateResourceConfig(r providers.ValidateResourceConfig
func (p *GRPCProvider) ValidateDataResourceConfig(r providers.ValidateDataResourceConfigRequest) (resp providers.ValidateDataResourceConfigResponse) {
logger.Trace("GRPCProvider: ValidateDataResourceConfig")
schema := p.getSchema()
schema := p.GetProviderSchema()
if schema.Diagnostics.HasErrors() {
resp.Diagnostics = schema.Diagnostics
return resp
@ -254,7 +260,7 @@ func (p *GRPCProvider) ValidateDataResourceConfig(r providers.ValidateDataResour
func (p *GRPCProvider) UpgradeResourceState(r providers.UpgradeResourceStateRequest) (resp providers.UpgradeResourceStateResponse) {
logger.Trace("GRPCProvider: UpgradeResourceState")
schema := p.getSchema()
schema := p.GetProviderSchema()
if schema.Diagnostics.HasErrors() {
resp.Diagnostics = schema.Diagnostics
return resp
@ -301,7 +307,7 @@ func (p *GRPCProvider) UpgradeResourceState(r providers.UpgradeResourceStateRequ
func (p *GRPCProvider) ConfigureProvider(r providers.ConfigureProviderRequest) (resp providers.ConfigureProviderResponse) {
logger.Trace("GRPCProvider: ConfigureProvider")
schema := p.getSchema()
schema := p.GetProviderSchema()
if schema.Diagnostics.HasErrors() {
resp.Diagnostics = schema.Diagnostics
return resp
@ -349,7 +355,7 @@ func (p *GRPCProvider) Stop() error {
func (p *GRPCProvider) ReadResource(r providers.ReadResourceRequest) (resp providers.ReadResourceResponse) {
logger.Trace("GRPCProvider: ReadResource")
schema := p.getSchema()
schema := p.GetProviderSchema()
if schema.Diagnostics.HasErrors() {
resp.Diagnostics = schema.Diagnostics
return resp
@ -405,7 +411,7 @@ func (p *GRPCProvider) ReadResource(r providers.ReadResourceRequest) (resp provi
func (p *GRPCProvider) PlanResourceChange(r providers.PlanResourceChangeRequest) (resp providers.PlanResourceChangeResponse) {
logger.Trace("GRPCProvider: PlanResourceChange")
schema := p.getSchema()
schema := p.GetProviderSchema()
if schema.Diagnostics.HasErrors() {
resp.Diagnostics = schema.Diagnostics
return resp
@ -491,7 +497,7 @@ func (p *GRPCProvider) PlanResourceChange(r providers.PlanResourceChangeRequest)
func (p *GRPCProvider) ApplyResourceChange(r providers.ApplyResourceChangeRequest) (resp providers.ApplyResourceChangeResponse) {
logger.Trace("GRPCProvider: ApplyResourceChange")
schema := p.getSchema()
schema := p.GetProviderSchema()
if schema.Diagnostics.HasErrors() {
resp.Diagnostics = schema.Diagnostics
return resp
@ -562,7 +568,7 @@ func (p *GRPCProvider) ApplyResourceChange(r providers.ApplyResourceChangeReques
func (p *GRPCProvider) ImportResourceState(r providers.ImportResourceStateRequest) (resp providers.ImportResourceStateResponse) {
logger.Trace("GRPCProvider: ImportResourceState")
schema := p.getSchema()
schema := p.GetProviderSchema()
if schema.Diagnostics.HasErrors() {
resp.Diagnostics = schema.Diagnostics
return resp
@ -607,7 +613,7 @@ func (p *GRPCProvider) ImportResourceState(r providers.ImportResourceStateReques
func (p *GRPCProvider) ReadDataSource(r providers.ReadDataSourceRequest) (resp providers.ReadDataSourceResponse) {
logger.Trace("GRPCProvider: ReadDataSource")
schema := p.getSchema()
schema := p.GetProviderSchema()
if schema.Diagnostics.HasErrors() {
resp.Diagnostics = schema.Diagnostics
return resp

61
internal/plugin6/grpc_provider.go
View File

@ -12,6 +12,7 @@ import (
"github.com/zclconf/go-cty/cty"
plugin "github.com/hashicorp/go-plugin"
"github.com/hashicorp/terraform/internal/addrs"
"github.com/hashicorp/terraform/internal/logging"
"github.com/hashicorp/terraform/internal/plugin6/convert"
"github.com/hashicorp/terraform/internal/providers"
@ -54,6 +55,11 @@ type GRPCProvider struct {
// used in an end to end test of a provider.
TestServer *grpc.Server
// Addr uniquely identifies the type of provider.
// Normally executed providers will have this set during initialization,
// but it may not always be available for alternative execute modes.
Addr addrs.Provider
// Proto client use to make the grpc service calls.
client proto6.ProviderClient
@ -67,31 +73,18 @@ type GRPCProvider struct {
schemas providers.GetProviderSchemaResponse
}
func New(client proto6.ProviderClient, ctx context.Context) GRPCProvider {
return GRPCProvider{
client: client,
ctx: ctx,
}
}
// getSchema is used internally to get the cached provider schema.
func (p *GRPCProvider) getSchema() providers.GetProviderSchemaResponse {
p.mu.Lock()
// unlock inline in case GetProviderSchema needs to be called
if p.schemas.Provider.Block != nil {
p.mu.Unlock()
return p.schemas
}
p.mu.Unlock()
return p.GetProviderSchema()
}
func (p *GRPCProvider) GetProviderSchema() (resp providers.GetProviderSchemaResponse) {
logger.Trace("GRPCProvider.v6: GetProviderSchema")
p.mu.Lock()
defer p.mu.Unlock()
// check the global cache if we can
if !p.Addr.IsZero() {
if resp, ok := providers.SchemaCache.Get(p.Addr); ok {
return resp
}
}
if p.schemas.Provider.Block != nil {
return p.schemas
}
@ -144,7 +137,13 @@ func (p *GRPCProvider) GetProviderSchema() (resp providers.GetProviderSchemaResp
resp.ServerCapabilities.PlanDestroy = protoResp.ServerCapabilities.PlanDestroy
}
p.schemas = resp
// set the global cache if we can
if !p.Addr.IsZero() {
providers.SchemaCache.Set(p.Addr, resp)
} else {
// otherwise store it in the local cache
p.schemas = resp
}
return resp
}
@ -152,7 +151,7 @@ func (p *GRPCProvider) GetProviderSchema() (resp providers.GetProviderSchemaResp
func (p *GRPCProvider) ValidateProviderConfig(r providers.ValidateProviderConfigRequest) (resp providers.ValidateProviderConfigResponse) {
logger.Trace("GRPCProvider.v6: ValidateProviderConfig")
schema := p.getSchema()
schema := p.GetProviderSchema()
if schema.Diagnostics.HasErrors() {
resp.Diagnostics = schema.Diagnostics
return resp
@ -183,7 +182,7 @@ func (p *GRPCProvider) ValidateProviderConfig(r providers.ValidateProviderConfig
func (p *GRPCProvider) ValidateResourceConfig(r providers.ValidateResourceConfigRequest) (resp providers.ValidateResourceConfigResponse) {
logger.Trace("GRPCProvider.v6: ValidateResourceConfig")
schema := p.getSchema()
schema := p.GetProviderSchema()
if schema.Diagnostics.HasErrors() {
resp.Diagnostics = schema.Diagnostics
return resp
@ -219,7 +218,7 @@ func (p *GRPCProvider) ValidateResourceConfig(r providers.ValidateResourceConfig
func (p *GRPCProvider) ValidateDataResourceConfig(r providers.ValidateDataResourceConfigRequest) (resp providers.ValidateDataResourceConfigResponse) {
logger.Trace("GRPCProvider.v6: ValidateDataResourceConfig")
schema := p.getSchema()
schema := p.GetProviderSchema()
if schema.Diagnostics.HasErrors() {
resp.Diagnostics = schema.Diagnostics
return resp
@ -254,7 +253,7 @@ func (p *GRPCProvider) ValidateDataResourceConfig(r providers.ValidateDataResour
func (p *GRPCProvider) UpgradeResourceState(r providers.UpgradeResourceStateRequest) (resp providers.UpgradeResourceStateResponse) {
logger.Trace("GRPCProvider.v6: UpgradeResourceState")
schema := p.getSchema()
schema := p.GetProviderSchema()
if schema.Diagnostics.HasErrors() {
resp.Diagnostics = schema.Diagnostics
return resp
@ -301,7 +300,7 @@ func (p *GRPCProvider) UpgradeResourceState(r providers.UpgradeResourceStateRequ
func (p *GRPCProvider) ConfigureProvider(r providers.ConfigureProviderRequest) (resp providers.ConfigureProviderResponse) {
logger.Trace("GRPCProvider.v6: ConfigureProvider")
schema := p.getSchema()
schema := p.GetProviderSchema()
var mp []byte
@ -345,7 +344,7 @@ func (p *GRPCProvider) Stop() error {
func (p *GRPCProvider) ReadResource(r providers.ReadResourceRequest) (resp providers.ReadResourceResponse) {
logger.Trace("GRPCProvider.v6: ReadResource")
schema := p.getSchema()
schema := p.GetProviderSchema()
if schema.Diagnostics.HasErrors() {
resp.Diagnostics = schema.Diagnostics
return resp
@ -401,7 +400,7 @@ func (p *GRPCProvider) ReadResource(r providers.ReadResourceRequest) (resp provi
func (p *GRPCProvider) PlanResourceChange(r providers.PlanResourceChangeRequest) (resp providers.PlanResourceChangeResponse) {
logger.Trace("GRPCProvider.v6: PlanResourceChange")
schema := p.getSchema()
schema := p.GetProviderSchema()
if schema.Diagnostics.HasErrors() {
resp.Diagnostics = schema.Diagnostics
return resp
@ -487,7 +486,7 @@ func (p *GRPCProvider) PlanResourceChange(r providers.PlanResourceChangeRequest)
func (p *GRPCProvider) ApplyResourceChange(r providers.ApplyResourceChangeRequest) (resp providers.ApplyResourceChangeResponse) {
logger.Trace("GRPCProvider.v6: ApplyResourceChange")
schema := p.getSchema()
schema := p.GetProviderSchema()
if schema.Diagnostics.HasErrors() {
resp.Diagnostics = schema.Diagnostics
return resp
@ -558,7 +557,7 @@ func (p *GRPCProvider) ApplyResourceChange(r providers.ApplyResourceChangeReques
func (p *GRPCProvider) ImportResourceState(r providers.ImportResourceStateRequest) (resp providers.ImportResourceStateResponse) {
logger.Trace("GRPCProvider.v6: ImportResourceState")
schema := p.getSchema()
schema := p.GetProviderSchema()
if schema.Diagnostics.HasErrors() {
resp.Diagnostics = schema.Diagnostics
return resp
@ -603,7 +602,7 @@ func (p *GRPCProvider) ImportResourceState(r providers.ImportResourceStateReques
func (p *GRPCProvider) ReadDataSource(r providers.ReadDataSourceRequest) (resp providers.ReadDataSourceResponse) {
logger.Trace("GRPCProvider.v6: ReadDataSource")
schema := p.getSchema()
schema := p.GetProviderSchema()
if schema.Diagnostics.HasErrors() {
resp.Diagnostics = schema.Diagnostics
return resp

7
internal/providers/provider.go
View File

@ -73,6 +73,13 @@ type Interface interface {
Close() error
}
// All access to a provider's schema internally should bt through the Schemas
// type.
// TODO: Acccess to schemas was split over two types with slightly different
// structures in different parts of the code. These types need to be unified
// for easier and more consistent access, but there is however a large amount
// of code which directly uses this structure, so it is not easily refactored
// out.
type GetProviderSchemaResponse struct {
// Provider is the schema for the provider itself.
Provider Schema

41
internal/providers/schema_cache.go Normal file
View File

@ -0,0 +1,41 @@
// Copyright (c) HashiCorp, Inc.
// SPDX-License-Identifier: MPL-2.0
package providers
import (
"sync"
"github.com/hashicorp/terraform/internal/addrs"
)
// SchemaCache is a global cache of GetProviderSchemaResponses.
// This will be accessed by both core and the provider clients to ensure that
// large schemas are stored in a single location.
var SchemaCache = &schemaCache{
m: make(map[addrs.Provider]GetProviderSchemaResponse),
}
// Global cache for provider schemas
// Cache the entire response to ensure we capture any new fields, like
// ServerCapabilities. This also serves to capture errors so that multiple
// concurrent calls resulting in an error can be handled in the same manner.
type schemaCache struct {
mu sync.Mutex
m map[addrs.Provider]GetProviderSchemaResponse
}
func (c *schemaCache) Set(p addrs.Provider, s GetProviderSchemaResponse) {
c.mu.Lock()
defer c.mu.Unlock()
c.m[p] = s
}
func (c *schemaCache) Get(p addrs.Provider) (GetProviderSchemaResponse, bool) {
c.mu.Lock()
defer c.mu.Unlock()
s, ok := c.m[p]
return s, ok
}

60
internal/providers/schemas.go
View File

@ -6,6 +6,7 @@ package providers
import (
"github.com/hashicorp/terraform/internal/addrs"
"github.com/hashicorp/terraform/internal/configs/configschema"
"github.com/hashicorp/terraform/internal/tfdiags"
)
// Schemas is an overall container for all of the schemas for all configurable
@ -13,23 +14,30 @@ import (
//
// The schema for each individual configurable object is represented by nested
// instances of type Schema (singular) within this data structure.
//
// This type used to be known as terraform.ProviderSchema, but moved out here
// as part of our ongoing efforts to shrink down the "terraform" package.
// There's still a type alias at the old name, but we should prefer using
// providers.Schema in new code. However, a consequence of this transitional
// situation is that the "terraform" package still has the responsibility for
// constructing a providers.Schemas object based on responses from the provider
// API; hopefully we'll continue this refactor later so that functions in this
// package totally encapsulate the unmarshalling and include this as part of
// providers.GetProviderSchemaResponse.
type Schemas struct {
Provider *configschema.Block
ProviderMeta *configschema.Block
ResourceTypes map[string]*configschema.Block
DataSources map[string]*configschema.Block
// Provider is the schema for the provider itself.
Provider *configschema.Block
// ProviderMeta is the schema for the provider's meta info in a module
ProviderMeta *configschema.Block
// ResourceTypes map the resource type name to that type's schema.
ResourceTypes map[string]*configschema.Block
ResourceTypeSchemaVersions map[string]uint64
// DataSources maps the data source name to that data source's schema.
DataSources map[string]*configschema.Block
// ServerCapabilities lists optional features supported by the provider.
ServerCapabilities ServerCapabilities
// Diagnostics contains any warnings or errors from the method call.
// While diagnostics are only relevant to the initial call, we add these to
// the cached structure so that concurrent calls can handle failures
// gracefully when the original call did not succeed.
// TODO: can we be sure the original failure get handled correctly, and
// ignore this entirely?
Diagnostics tfdiags.Diagnostics
}
// SchemaForResourceType attempts to find a schema for the given mode and type.
@ -53,6 +61,30 @@ func (ss *Schemas) SchemaForResourceAddr(addr addrs.Resource) (schema *configsch
return ss.SchemaForResourceType(addr.Mode, addr.Type)
}
func SchemaResponseToSchemas(resp GetProviderSchemaResponse) *Schemas {
var schemas = &Schemas{
ResourceTypes: make(map[string]*configschema.Block),
ResourceTypeSchemaVersions: make(map[string]uint64),
DataSources: make(map[string]*configschema.Block),
ServerCapabilities: resp.ServerCapabilities,
Diagnostics: resp.Diagnostics,
}
schemas.Provider = resp.Provider.Block
schemas.ProviderMeta = resp.ProviderMeta.Block
for name, res := range resp.ResourceTypes {
schemas.ResourceTypes[name] = res.Block
schemas.ResourceTypeSchemaVersions[name] = uint64(res.Version)
}
for name, dat := range resp.DataSources {
schemas.DataSources[name] = dat.Block
}
return schemas
}
// Schema pairs a provider or resource schema with that schema's version.
// This is used to be able to upgrade the schema in UpgradeResourceState.
//